commit ddc03123beabb8cb6a06e90c1196d1f170cd309c from: Omar Polo date: Sat Mar 28 17:36:03 2020 UTC initial commit commit - /dev/null commit + ddc03123beabb8cb6a06e90c1196d1f170cd309c blob - /dev/null blob + 4c1bcdb20ce64f9c54585b5c08a1e25808f2b25e (mode 644) --- /dev/null +++ .gitignore @@ -0,0 +1 @@ +resources/out/ blob - /dev/null blob + c20dbcd971d7ccfbba6def2611e03501ffaec7c4 (mode 644) --- /dev/null +++ build.boot @@ -0,0 +1,11 @@ +(set-env! + :resource-paths #{"src" "resources"} + :dependencies '[[hiccup "1.0.5"] + [ring "1.8.0"] + [commonmark-hiccup "0.1.0"]]) + +(task-options! + pom {:project 'blog + :version "0.1.0"}) + +(require '[blog.core :refer :all]) blob - /dev/null blob + d3db89a884777aff35a97c9f8351689076b2fe99 (mode 644) --- /dev/null +++ resources/css/style.css @@ -0,0 +1,318 @@ +@charset "utf-8"; + +html, body, div, span, applet, object, iframe, +h1, h2, h3, h4, h5, h6, p, blockquote, pre, +a, abbr, acronym, address, big, cite, code, +del, dfn, em, img, ins, kbd, q, s, samp, +small, strike, strong, sub, sup, tt, var, +b, u, i, center, +dl, dt, dd, ol, ul, li, +fieldset, form, label, legend, +table, caption, tbody, tfoot, thead, tr, th, td, +article, aside, canvas, details, embed, +figure, figcaption, footer, header, hgroup, +menu, nav, output, ruby, section, summary, +time, mark, audio, video { + margin: 0; + padding: 0; + border: 0; + font-size: 100%; + font: inherit; + vertical-align: baseline; + box-sizing: border-box; +} + +html, body { + line-height: 1; + font-size: 16px; +} + +body { + background-color: #ffffea; + color: #383838; + font-family: monospace; + padding: 1rem; +} + +#wrapper>header>nav>ul { + padding: 0; + list-style: none; +} + +#wrapper>header>nav>ul li { + display: inline-block; + margin: 3px 20px; +} + +#wrapper>header>nav>ul li:first-of-type { + margin-left: 0; +} + +#wrapper>header>nav>ul li a { + text-decoration: none; +} + +/* HTML5 display-role reset for older browsers */ +article, aside, details, figcaption, figure, +footer, header, hgroup, menu, nav, section { + display: block; +} + +#wrapper { + max-width: 900px; + margin: 0 auto; +} + +body > header { + max-width: 960px; + margin: 0 auto; +} + +body > header > nav ul { + list-style: none; + padding: 0; + margin: 0; +} + +body > header > nav li { + display: inline; + margin-right: 20px; +} + +[data-title~=":"] { + display: none; +} + +.heading { + margin-bottom: 30px; +} + +h1.main-title::after { + content: '-----------------------------------------------------------------'; +} + +p.subtitle { + font-style: italic; + font-size: 120%; + margin: 0; +} + + +body main { + margin: 0 auto; +} + +body:not(.article) main { + max-width: 550px; +} + +body.article main { + max-width: 960px; +} + +article { + margin-bottom: 30px; +} + +h1 { + position: relative; + display: table-cell; + padding: 20px 0 30px; + margin: 0; + overflow: hidden; + + font-size: 200%; +} + +/* a cool hack stolen from blog.soykaf.com */ +h1::after { + /* should be long enough */ + content: '================================================================='; + position: absolute; + bottom: 0px; + left: 0; + white-space: nowrap; +} + +h1::after, h2::before, h3::before, h4::before, h5::before, h6::before { + /* color: hsla(0, 0%, 36%, 1); */ + color: #242424; +} + +h1>a { + text-decoration: none; +} + +h2, h3, h4, h5, h6 { + margin: 2rem 0 1rem; +} + +h2 { + font-size: 180%; +} + +h3 { + font-size: 160%; +} + +h4 { + font-size: 140%; +} + +h5 { + font-size: 120%; +} + +h6 { + font-size: 100%; +} + +h2::before { + content: "## "; +} + +h3::before { + content: "### "; +} + +h4::before { + content: "#### "; +} + +h5::before { + content: "##### "; +} + +h6::before { + content: "###### "; +} + +ul.tags { + padding: 0; + margin: 0; + list-style: none; +} + +ul.tags::before { + content: 'tagged with: '; +} + +ul.tags li { + display: inline-block; + margin: 0 5px; +} + +header > p { + margin: 0; +} + +p { + margin: 1rem 0; +} + +p, li { + line-height: 1.5rem; +} + +a { + text-decoration: none; + color: #4b4baf; +} + +a:hover { + text-decoration: underline; +} + +/* other bit stolen from blog.soykaf.com */ +a[href*="://"]::after, a[rel*="external"]::after { + content: " "url("data:image/svg+xml,%3Csvg%20xmlns='http://www.w3.org/2000/svg'%20class='i-external'%20viewBox='0%200%2032%2032'%20width='14'%20height='14'%20fill='none'%20stroke='%23ff9800'%20stroke-linecap='round'%20stroke-linejoin='round'%20stroke-width='9.38%'%3E%3Cpath%20d='M14%209%20L3%209%203%2029%2023%2029%2023%2018%20M18%204%20L28%204%2028%2014%20M28%204%20L14%2018'/%3E%3C/svg%3E"); +} + +ul, ol { + padding-left: 30px; + list-style: disc; +} + +ol { + list-style: decimal; +} + +pre, code { + color: #971174; +} + +pre { + padding: 20px; + overflow-x: auto; + border-top: 1px; + border-bottom: 1px; + border-color: #999999; + border-style: solid; +} + +em { + font-style: italic; +} + +strong { + font-weight: bold; +} + +hr { + border: 0; + height: 1px; + background-color: hsla(0, 0%, 25%, 1); +} + +blockquote, q { + quotes: none; +} + +blockquote { + border-left: 3px solid hsla(113, 80%, 45%, 1); + padding-left: 10px; + font-style: italic; +} + +blockquote em { + font-style: normal; +} + +blockquote:before, blockquote:after, +q:before, q:after { + content: ''; + content: none; +} + +img { + max-width: 100%; + height: auto; +} + +figure figcaption { + padding: 5px 0; + text-align: center; +} + +table { + border-collapse: collapse; + border-spacing: 0; +} + +nav.post-navigation { + margin-top: 50px; + display: flex; + flex-direction: row; +} + +nav.post-navigation a.next { + margin-left: auto; +} + +footer { + border-top: 1px solid #e6e0e0; + margin-top: 100px; + padding-top: 10px; + font-style: italic; + text-align: center; +} blob - /dev/null blob + 650afbb15a17a0d819900b8cb7a7e53b69eb43c7 (mode 644) Binary files /dev/null and resources/favicon.ico differ blob - /dev/null blob + d0ee5bbef857e58bc173363dc6f80a689241b732 (mode 644) Binary files /dev/null and resources/img/unbound-dashboard.png differ blob - /dev/null blob + 9ff575e190d7b94f42b35c3ab037f8b19f0eb291 (mode 644) --- /dev/null +++ resources/posts/abbreviations-vi.md @@ -0,0 +1,15 @@ +Sometimes you use a tool for years and still have something new to +learn. I'm talking about `vi(1)`, not vim or nvim, but nvi! (the +OpenBSD ones at least.) + +While I was reading [NetBSD desktop pt.6 "vi(1) editor, tmux and +unicode +`$TERM`"](https://www.unitedbsd.com/d/12-netbsd-desktop-pt-6-vi1-editor-tmux-and-unicode-term) + +I knew vim end emacs had abbreviations, but I never thought that vi +has them too. + +Now my `~/.nexrc` is more rich: + + ab #d #define + ab #i #include blob - /dev/null blob + 4b6bd864ce2597598ba6952c77c100302a717e31 (mode 644) --- /dev/null +++ resources/posts/core-naming-on-linux.md @@ -0,0 +1,38 @@ +Let's say you're debugging something on linux. Let's say that thing +crashed. What do you do? I will open gdb, load the executable and the +core file and try to figure it out why the thing crashed. + +Well, it's exactly what happened. But there was no core file for +me. Poor me. + +The defaults on OpenBSD, if I recall correctly, are to generate a core +file named `$prgname.core` in the directory where the program was +running before the crash. Also, but mind that I might be wrong, the +default ulimit settings is `unlimited` for the core files. + +So, here's a quick description on *how to get that cores* on linux, +primarly aimed at myself: + +#### ulimit + +A `ulimit -c unlimited` is needed since the default may be 0. This can +cause big core dumps in the `$HOME` if something like firefox or chrome +chrashes, but I prefer to have it in `.profile` than to have to remember +to execute the command. + +#### pattern + +On systems with systemd there should be a systemd-*somethingsomething* +daemon and *somethingsomething*-ctl to manage 'em. I prefer have those +cores in the directory I'm in. Bonus points if they are called like +I'm used. + +```echo %e.core | sudo tee /proc/sys/kernel/core_pattern``` + +`%e` is replaced with the program name. + +However, on my machine, the core files are called `$prgname.core.$pid`. To disable the `.pid` at the end + +```echo 0 | sudo tee /proc/sys/kernel/core_uses_pid``` + +And that's all! \ No newline at end of file blob - /dev/null blob + 130779350554c45707c8b1f4b9daccf96c6b9069 (mode 644) --- /dev/null +++ resources/posts/cpp-css.md @@ -0,0 +1,60 @@ +The first version of this website had a theme switcher. It was +implemented with CSS variables (and a bit of javascript). Then the +javascript switcher was eventually removed, and the theme forced to be +dark, but I kept the CSS variables *just in case* (read: I'm lazy.) + +*Edit*: this is no longer the case. The current version of the website +is *yet another one*, with a 100% rewritten (and pure) CSS. + +The real reason I left the CSS variables was that I didn't wanted to +use a CSS preprocessor (such as `less` or `sass`) to manage such a +simple file (306 line, with blanks and comments). But, at the same +time, I didn't want to copy-paste the colors everywhere. + +## Introducing the C preprocessor + +The C preprocessor is a simple and well-known beast (sort of, at +least), and it's included in the base system installation of most +(pratically all, I presume) OSes. + +If you have never used it, here's a quick howto. + +You can define constants with +``` +#define PI 3.141592653589793238462643383279502884197169 +``` +and use them whenever you like, for instance + +```css +double p = PI / 4; +``` + +The preprocessor is more powerful, it supports `#include`s and +function-like macro (even variadic). But `#define`s are enough to +manage a couple of CSS variables. + +Now, let's see how this applies to CSS. Given a file with the +following content + + #define BASE1 #221635 + + body { + background-color: BASE1; + } + +we can *compile* it with + + $ cpp -P file.css > a.css + +and obtain a valid CSS file `a.css`. + +## Conclusions + +It's weird. It's weird to invoke `cpp` to *build* a CSS files. + +But it's also *satisfying*, in some sense. + +As a conclusion, I would like to note that another option is to use +m4, a general purpose macro language that should be present on every +POSIX system. Unfortunately, I don't know the language very well, so I +opted to `cpp`. blob - /dev/null blob + 491dad90e822db97e65d069277237e88142f8fc0 (mode 644) --- /dev/null +++ resources/posts/css-sprites-awk-imagemagick.md @@ -0,0 +1,69 @@ +CSS sprites are a nice idea, stolen from the game developers I think, +that consist to merge all the images you need in a single one and +download only that. I will not prove that it's best than downloading +*n* images, but it should clear that downloading a file, albeit a bit +bigger, requires less TCP segments going back and forth. + +The other day I needed a CSS sprite. Being the lazy person I am, right +after The GIMP was open I thought that I didn't want to do the math. + +After a bit of searching, I found a post where the author wrote a +script to generate the sprites (and the relative css file) with image +magick. I have lost that link, and the history of my browser doesn't +help. I remember that the domain name contains "php", but nothing +more. However, I rewritten the script, so it better fits my needs. + +The idea is to join all the image using `convert(1)` and then generate +a CSS file with the offsets using `identify(1)`. Both programs should +come along with imagemagick. + +Here's the script + + #!/bin/sh + + convert sprites/* -append img/sprites.png + + for i in sprites/*; do + identify -format '%W %H %f\n' $i + done | awk ' + function fname(path) { + sub (".png", "", path) + return path + } + + BEGIN { + y = 0 + + print "%sprite-base {" + print " background-image: url(/img/sprites.png);" + print " background-repeat: no-repeat;" + print "}" + print "" + } + + { + width = $1 + height = $2 + class = fname($3) + + print "%sprite-" class " {" + print " @extend %sprite-base;" + if (y == 0) + print " background-position: 0 0;" + else + print " background-position: 0 -" y "px;" + print "}" + print "" + + y += height; + } + ' > scss/_sprites.scss + +Assuming that the images are within `sprites/` and all of them are png +files, this script will generate the sprite in `img/sprites.png` and a +SASS file in `sass/_sprits.scss` with one placeholder for every image +(the naming scheme is `%sprite-$NAMEFILE` with the `$NAMEFILE` being +the file name without extension). + +It should be trivial to edit to handle pure CSS output, and eventually +other image formats. blob - /dev/null blob + 9ff575e190d7b94f42b35c3ab037f8b19f0eb291 (mode 644) --- /dev/null +++ resources/posts/freebsd-repository-with-synth.md @@ -0,0 +1,15 @@ +Sometimes you use a tool for years and still have something new to +learn. I'm talking about `vi(1)`, not vim or nvim, but nvi! (the +OpenBSD ones at least.) + +While I was reading [NetBSD desktop pt.6 "vi(1) editor, tmux and +unicode +`$TERM`"](https://www.unitedbsd.com/d/12-netbsd-desktop-pt-6-vi1-editor-tmux-and-unicode-term) + +I knew vim end emacs had abbreviations, but I never thought that vi +has them too. + +Now my `~/.nexrc` is more rich: + + ab #d #define + ab #i #include blob - /dev/null blob + 1d266c2d9083fe473982d238f40759cd8b078e15 (mode 644) --- /dev/null +++ resources/posts/fstar-openbsd.md @@ -0,0 +1,73 @@ +[F*](https://fstar-lang.org) is a *general purpose functional +programming language with effects aimed at program verification.* +Recently I've been playing a bit with it, it's nice, and here's a +quick guide on how to compile it on OpenBSD. + +----- + +*Edit*: This "guide" is partially incomplete. Doing some re-install +after this guide was published, I noticed that not everything is as I +wrote it here. Things change, I suppose. Some problems have been +fixed, but not everything. In particular `ocamlfind` will complain +(probably multiple times) during the build that it cannot find the +`XXX` package, and a simple `opam install XXX` will amend. + +----- + +We'll need both `git` and GNU `make` from ports, and also ocaml (to +build F* and run F* programs), opam (the ocaml package manager), +ocaml-camlp4 and python 3 (to build `z3`, a theorem prover). + + $ pkg_add git gmake ocaml ocaml-camlp4 opam python + +*Note* I've installed camlp4 from the ports instead of through opam +because I was getting an error. I don't have much experience with +ocaml, but I've read *somewhere* that installing through the package +manager solved those problems. + +## Building z3 + +z3 is the engine that powers the verification system of F* +(AFAIK). It's not available through ports, so we'll need to build from +source. Even though is a project that came from Microsoft, it seems to +run on OpenBSD just fine. Building it is also straightforward: + + $ git clone https://github.com/Z3Prover/z3 + $ cd z3 + $ CXX=clang++ python3 script/mk_make.py + $ cd build + $ make + $ cp z3 $SOMEWHERE_IN_PATH # (maybe?) + +*Note* by default `script/mk_make.py` will try to use base g++ and the +build will fail. The version of g++ is too ancient and doesn't support +C++11. + +## Building F* + +First of all, we need some ocaml dependencies, so make sure to +initialize opam (see `opam init`) and add the suggested stuff to your +shell init file. + + $ opam install ocamlfind batteries stdint zarith ppx_deriving ppx_deriving_yojson ocaml-migrate-parsetree process + +We can now build the F* compiler from the ocaml output present in the +repo. You can also build the ocaml output by yourself, but I've skip +this step. + + $ gmake -j9 -C src/ocaml-output + +The last step is to build the library. While the docs describes this +as an optional step, I wasn't able to compile F* programs without it. + + $ gmake -j9 -C ulib/ml + +This was all. Let's try the hello world now! + + $ gmake -C examples/hello hello + +It should take a decent amount of time to compile, output a *lot* of +text and, finally, at the end, a beautiful "Hello World". + +Congrats, now you have a working F* installation! + blob - /dev/null blob + b411b93dc2b288fea39ed9fafcda829024062239 (mode 644) --- /dev/null +++ resources/posts/maildir-mutt-time.md @@ -0,0 +1,22 @@ +Mutt is, by far, my favourite mail client. It's small, configurable, +fast and it does not get in your way. It's a great tool. + +My setup involves various maildir: I have a decent amount of filters +that I use to order the mail in different directories. + +One thing that I've found annoying is switching maildir: opening ones +with ~3k mail takes too much time, at least on my machine. Now I think +that using the `threads` sorting was the culprit, but it's only a +guess. + +Anyway, while I was searching the manpage for a totally different +issue, I've found the `header_cache` settings. Quoting the manpage: + +> This variable points to the header cache database. [...] By default +> it is *unset* so no header caching will be used. +> +> Header caching can greatly improves speed when opening POP, IMAP HM +> or Maildir folders, see "caching" for details. + +There are also other relating settings, like `header_cache_compress` +that may be interestig. blob - /dev/null blob + 9aa2763075e4865de58066a111fe5ed03c8d677d (mode 644) --- /dev/null +++ resources/posts/poor-mans-unbound-dashboard.md @@ -0,0 +1,251 @@ +I have a tiny i686 at home with OpenBSD where I run, amongst some other +things, an instance of unbound. + +Last night I decided that I wanted a dashboard to collect some statistics +about it. + +My first thought was the ELK stack. The only problem is the ram. +The little i686 has 1GB of ram, I don't know if it's enough to run +logstash, let alone the whole ELK. + +A simple solution would be to collect the logs elsewhere, but I'm not +going to do this for various reason (lazyness being the first, followed +by the fact that having statistics about my dns queries isn't that +useful in my opinion, even if it's nice-to-have.) + +Instead, my solution involves a bit of bash (don't hate me on this), +some fifos, tmux and ttyplot. + +The primarly source of inspiration is [this +post](https://dataswamp.org/~solene/2019-07-29-ttyplot.html) that I red +some time ago: it's about plotting various system statistics with ttyplot. + +The result is this + +![unbound dashboard screenshot](/img/unbound-dashboard.png) + +(note that I usually disable colors in xterm) + +## The flow + + .- + / various -------> multiple + unbound stats ------- fifos -------> ttyplot + \ -------> per tmux pane + `- + +The idea is to run `unbound-control stats` every once in a while, +multiplexing its output and draw each (interesting) stats with ttyplot +in a tmux pane. + +Why the fifos? Well, if I'm not wrong, every time you call +`unbound-control stats` it will clear the statistics, so you can't run +it *n* times to collect *n* different stats. And since the whole setup +requires only a couple of script, the easiest way was to use some fifos. + +The whole setup requires three script: + + - `gen-dashboard.sh` + - `dashboard.sh` + - `mystatd.sh` + +### `gen-dashboard.sh` + +This is the startup script. I run it on my crontab as `@reboot +/path/to/gen-dashboard.sh`. It will create the required fifos, then +spawn a tmux session and create two windows and some panes. + +```sh +#!/bin/sh + +# create the fifos +for f in netstat queries hit miss time; do + mkfifo /tmp/my-$f +done + +session=dashboard + +tmux new-session -d -s $session + +# start mystatd.sh +tmux new-window -t $session:1 -n 'logs' +tmux send-keys "/path/to/mystatd.sh" C-m + +# create the dashboard +tmux select-window -t $session:0 + +# setup the layout of the panes +tmux split-window -h +tmux select-pane -L +tmux split-window -v +tmux select-pane -R +tmux split-window -v -p 66 +tmux split-window -v -p 50 + +# load the correct ttyplot in the panes +tmux select-pane -t 0 +tmux send-keys "/path/to/dashboard.sh netstat" C-m + +tmux select-pane -t 1 +tmux send-keys "/path/to/dashboard.sh queries" C-m + +tmux select-pane -t 2 +tmux send-keys "/path/to/dashboard.sh hit" C-m + +tmux select-pane -t 3 +tmux send-keys "/path/to/dashboard.sh miss" C-m + +tmux select-pane -t 4 +tmux send-keys "/path/to/dashboard.sh time" C-m +``` + +(A possible improvement may be to tell tmux which command to run when +creating a pane instead of sending the keys to the shell, but it works +neverthless.) + +There's nothing special about this script, so let's move to the next. + +### `dashboard.sh` + +This script also isn't interesting, all it does is pull the data out of +the correct fifo and start ttyplot with the correct labels and units. + +```sh +#!/bin/sh + +if [ -z "$1" ]; then + echo "missing dashboard type" + echo "usage: $0 " + exit 0 +fi + +case "$1" in + netstat) + (while :; do + cat /tmp/my-netstat + done) | ttyplot -t "IN Bandwidth in KB/s" \ + -u "KB/s" -c "#" + ;; + + queries) + (while :; do + cat /tmp/my-queries + done) | ttyplot -t "DNS Queries/5s" \ + -u "q/5s" -c "#" + ;; + + hit) + (while :; do + cat /tmp/my-hit + done) | ttyplot -t "DNS cache hit/5s" \ + -u "ch/5s" -c "#" + ;; + + miss) + (while :; do + cat /tmp/my-miss + done) | ttyplot -t "DNS cache miss/5s" \ + -u "cm/5s" -c "#" + ;; + + time) + (while :; do + cat /tmp/my-time + done) | ttyplot -t "DNS query time avg/5s" \ + -c "#" + ;; + + *) + printf "%s\n" "$1 is not a valid dashboard" + exit 1 + ;; +esac +``` + +### `mystatd.sh` + +This is the (only?) interesting script. It's also the only one that +requires bash, because I'm lazy, it was already installed as dependecy of +something, and because of the `>(cmd)` construct. Rewriting the script +using only pure sh(1) constructs is left as an exercise to the reader +(hint: you need some extra fifo.) + +```sh +#!/usr/bin/env bash + +filter() { + grep "$1" | awk -F= '{print $2}' > /tmp/my-$2 +} + +# unbound stats +( while :; do + unbound-control stats \ + | grep -v thread0 \ + | tee >(filter queries= queries) \ + | tee >(filter hit hit) \ + | tee >(filter miss miss) \ + | filter time.avg time + + sleep 5 +done ) & + +# netstat - ty solene@ for the awk +( + (while :; do + netstat -ibn + sleep 1 + done) | awk ' + BEGIN { + old=-1 + } + /^em0/ { + if(!index($4,":") && old>=0) { + print ($5-old)/1024 + fflush + old = $5 + } + if(old==-1) { + old = $5 + } + }' | tee -a /tmp/my-netstat +) & + +wait +``` + +The first piece collects the stat from unbound. Let's break it in pieces. + + - `unbound-control stats` outputs the stats. Keep in mind that this + requires some privileges. I've solved this by creating a script + in /usr/local/bin that executes the command and allowed my user to + launch that script via `doas(1)`. Or you can run `mystatd.sh` as root. + Do as you please. + - `grep -v thread0` removes the per-thread stats (since my unbound + uses only one thread). A more solid approach like `egrep -v ^thread` + is probably better. + - `tee >(filter queries= queries) |` duplicates the stream: one copy + goes to the subshell with `filter` and another copy goes on the pipes. + - `filter` is just a small function to grep the desired entry and send + it to `/tmp/my-$something` + +The netstat bit filters the output of netstat (the awk is copied-pasted +from the previously linked post by solene@). You may want to change the +`^em0` to match your network device. + +And that's all! + +## Possible improvements + + - if you `SIGINT` `mystatd.sh` some of its subprocess still run. Maybe a + `trap` is needed. Since it is the only bash running on that system, + `pkill bash` is, albeit a bit aggressive, a working solution. + - replace bash. It's not difficult, but requires more fifos. + - ... + +## Final considerations + +This was fun. Now I have a tmux session I remotely attach with cool +graphs. This doesn't cover the archiviation of the statistics tho. +I think it should be trivial to add (just one more `|tee -a` to a local +file, maybe a cronjob to do rotation, ...) but for the moment I'm happy +with this result. blob - /dev/null blob + 386074e5398d9811831020524d519f2bd8a52554 (mode 644) --- /dev/null +++ resources/posts/sh-multiplexing-data.md @@ -0,0 +1,113 @@ +In my [previous article](/2019-10-16-poor-mans-unbound-dashboard.html) +I used a bash(1) construct to send the same data to two different places. + +The fact that I was using a bash-specific (more or less) construct really +bothered me this time. So I thought if there is a way in sh(1) to take +a piece of data from a pipeline and send it to two different places. + +The canonical way to do this is `tee(1)`, and it's what I've used it the linked article: + + something | tee >(cmd) | something-else + +(the `>(cmd)` construct is bash, and zsh I think, specific: it executes +`cmd` in a subshell and execute `tee` with something like `/dev/fd/63` +-- at least on linux and OpenBSD.) + +Now I'll try to describe three different approaches to emulate the same +behavior of `>(...)` in pure sh(1), trying to outline both the advantages +and the cons. + +## temporary files + +That is: + + something | tee a-file | something-else + cmd < a-file + + # and then, probably + rm a-file + +(or even better with `mktemp(1)`) + +This is the most simple approach I can think of. The obvious disadvantage +is the need to create and delete files quickly. This shouldn't be a huge +problem in most cases, but for script do this constantly and (possibly) +quickly it can be. Plus, at least for me, it doesn't seem very elegant, +but it's subjective. + +## fifo to the rescue + +An almost similar approach is to use fifo. A fifo, aka a named pipe, +is a special file that you can write to and read from, in a FIFO +fashion. + + mkfifo a-fifo + something | tee a-fifo | something-else + while read line < a-fifo; do echo $line; done | cmd + +This is especially useful if you need to have a long-running script that +needs to duplicate a stream from one source. You create the fifos at the +startup and then you're done, instead of constantly creating temporary +files over and over again. An annoying thing is that you need to wrap +the read from the fifo in some sort of loop: AFAIK to read the next item +in a fifo you need to close and re-open it. + +## An unexpected sed + +Let's add another requisite: other than dumbly copy the stream, suppose +that you want also to apply some sort of transformation to one branch +of the duplication, like filter out something. + +Well, you can adapt the last two examples by adding a `| sed '...'`. +Or you can use the `w` command of sed. + +I'll be a bit more specific: let's say that you need to *dispatch* +something taken from a single stream and feed *n* subprocess. It's +exactly what I've tried to achieve in my last post: I've used `>(cmd)` +to dispatch the unbound statistics to different fifo, and then I had +different processes pulling out those stats from the fifo to draw graphs. + +The code was like + + unbound-control stats \ + | tee >(grep something > a-fifo) \ + | tee >(grep something-else > b-fifo) \ + | ... + +Well, that can be rewritten as + + unbound-control stats \ + | sed 's/something/&/w a-fifo' \ + | sed 's/something-else/&/w b-fifo \ + | ... + +by leveraging some sed behaviours: + +1. every line read is printed to stdout (eventually transformed) +2. after the `s` command you can use `w` to write the transformed lines + to a local file + +I've got this *illumination* by reading [Sed - An introduction and +Tutorial by Bruce Barnett](http://www.grymoire.com/Unix/Sed.html). + +I tried to use `sed 'g/something/w a-fifo'` but `sed` complained +about "extra characters after command", so I assume that the no-op +`s/something/&/` is needed. + +----- + +Edit: I tried with `g/...` because I thought that `sed` had a `g` command +that behaves like the `vi(1)` or `sam(1)` `g` command. `sed` has a `g` +command, but does something different. The correct form would be `sed +'/something/w a-fifo'` because `sed`, like `awk(1)`, has the ability to +perform commands only on lines that match a regular expression. + +----- + + +I find this last technique pretty, even prettier than the `>(grep ... >)` +idiom I used previously since it avoids the subshell to do the filtering +and because it does not make assumption on the shell used -- it only +requires a POSIX shell. + +I hope you found this interesting. At least to me it really is. blob - /dev/null blob + 5563d222ae0d6a900b7d46a4a30e39f45ce3b35a (mode 644) --- /dev/null +++ resources/posts/spell-checking-vi.md @@ -0,0 +1,34 @@ +UNIX is all about files and programs that do one thing and to it well, +right? `vi(1)` is one of my favorite text editors. However it lacks +some feature: but here's where the *composition* shines. + +It's stupid and dead-simple actually, but I haven't thought +about it until some weeks ago. With a simple + + map « :w^M:!aspell -c %^M:e!^M^M + +in your `~/.nexrc` it's simple to do spell checking in `vi`. + +**Friendly remainder**: the `^M` is literally the *enter* key inserted +with `C-v ENTER` or `C-v C-m`. + +I've also the following binding in my `~/.nexrc` to spell check +Italian text: + + map » :w^M:!aspell --lang=it -c %^M:e!^M^M + + +### What's that gibberish? + +OK, it may be non-obvious what that that mapping does, so let's split +it into pieces: + + - `map «` starts a mapping on the `«` key + - `:w^M` writes the current file (the return is necessary to *enter* the command) + - `:!aspell -c %^M` run aspell over the file (`%` is replaced with the current file name) + - `:e!^M` force vi to re-read the file + - `^M` Tell vi to render the editor. After a command execution vi doesn't render its interface. Rather, it wait (a bit like `ex`) for a command. + +### Why the `«` and `»` characters? + +Those keys aren't bind to anything and are simple to type with my keyboard layout. blob - /dev/null blob + 4c3247e50b2a55910349f5e53c0c5db2d59f8d3a (mode 644) --- /dev/null +++ src/blog/core.clj @@ -0,0 +1,148 @@ +(ns blog.core + (:require [blog.time :as time] + [blog.templates :as templates] + [boot.core :as core] + [boot.task.built-in :as task] + [ring.adapter.jetty :as jetty] + [ring.middleware.resource :refer [wrap-resource]] + [ring.middleware.content-type :refer [wrap-content-type]] + [clojure.java.io :as io] + [clojure.java.shell :refer [sh]]) + (:import (java.io File))) + +(defn copy-file [src dst] + (with-open [in (io/input-stream (io/file src)) + out (io/output-stream (io/file dst))] + (io/copy in out))) + +(defn post [{:keys [slug title short date tags]}] + (let [f (io/resource (str "posts/" slug ".md"))] + {:slug slug + :title title + :short short + :date (time/parse date) + :body (slurp f) + :tags tags})) + +(def per-tag (atom {})) +(def posts (atom [])) + +(defn add-post! [m] + (let [p (post m)] + (swap! posts conj p) + (doseq [t (:tags m)] + (swap! per-tag update t conj p)))) + +(load "posts") + +(defn create-dirs! [] + (doseq [d ["resources/out" + "resources/out/css" + "resources/out/post" + "resources/out/tag" + "resources/out/img"]] + (.. (File. d) mkdirs))) + +(defn post-pages [] + (let [tags (keys @per-tag)] + (map-indexed (fn [i posts] + {:filename (if (= i 0) + "index.html" + (str (inc i) ".html")) + :tags tags + :nth (inc i) + :posts posts + :has-next true + :has-prev true}) + (partition-all 6 @posts)))) + +(defn fix-next-last + "Fix the :has-prev/:has-next for the post pages. This assumes + that `(not (empty? post-pages))`" + [post-pages] + (-> post-pages + (->> (into [])) + (update 0 assoc :has-prev false) + (update (dec (count post-pages)) assoc :has-next false))) + +(defn render-post-list [] + (doseq [p (fix-next-last (post-pages)) + :let [{:keys [filename]} p]] + (spit (str "resources/out/" filename) + (templates/home-page p)))) + +(defn render-post [{s :slug, :as post}] + (spit (str "resources/out/post/" s ".html") + (templates/post-page post))) + +(defn render-tags [tags] + (spit (str "resources/out/tags.html") + (templates/tags-page tags))) + +(defn render-tag [tag posts] + (spit (str "resources/out/tag/" tag ".html") + (templates/tag-page tag posts))) + +(defn copy-dir + "Copy the content of resources/`dir` to resources/out/`dir`, assuming + these two directories exists. It does not copy recursively." + [dir] + (let [in (io/file (str "resources/" dir "/")) + out (str "resources/out/" dir "/")] + (doseq [f (->> in file-seq (filter #(.isFile %)))] + (println "copying" (.getName f) "to" (str out (.getName f))) + (io/copy f (io/file (str out (.getName f))))))) + +(comment + (copy-dir "img") + (io/copy (io/file "resources/img/unbound-dashboard.png") + (io/file "resources/out/img/unbound-dashboard.png")) +) + +(defn copy-assets + "Copy css and images to their places" + [] + (copy-dir "img") + (copy-file "resources/favicon.ico" "resources/out/favicon.ico") + (copy-file "resources/css/style.css" "resources/out/css/style.css")) + +(core/deftask build + "Build the blog" + [] + (create-dirs!) + (copy-assets) + (render-post-list) + (doseq [p @posts] + (render-post p)) + (render-tags (keys @per-tag)) + (doseq [t @per-tag + :let [[tag posts] t]] + (render-tag (name tag) posts))) + +(def j (atom nil)) + +(core/deftask serve + "Serve a preview" + [] + (reset! + j + (jetty/run-jetty (-> (fn [_] {:status 404, :body "not found"}) + (wrap-resource "out") + (wrap-content-type)) + {:port 3000 + :join? false}))) + +(core/deftask deploy + "Copy the files to the server" + [] + (sh "openrsync" "-r" "--delete" "resources/out/" "op:sites/www.omarpolo.com/")) + +(defn stop-jetty [] + (.stop @j) + (reset! j nil)) + +(comment + (build) + (serve) + (stop-jetty) +) blob - /dev/null blob + 32b6f52a0584e165bf432318cfab24456fe98d3e (mode 644) --- /dev/null +++ src/blog/posts.clj @@ -0,0 +1,74 @@ +(comment + (add-post! {:title "Blogging with clojure" + :slug "blogging-with-clojure" + :date "2020/03/27" + :tags #{:clojure} + :short "Build static blogs with clojure & boot"})) + +(comment + (add-post! {:title "Build a personal FreeBSD repository" + :slug "freebsd-repository-with-synth" + :date "2020/03/24" + :tags #{:FreeBSD} + :short "Build with ease. Scripts included!"})) + +(comment + (add-post! {:title "A tale of three editors" + :slug "tale-of-three-editors" + :date "2019/10/04" + :tags #{:emacs :vi :acme} + :short "The gigantic, the fake minimal and the glorious. Or escaping the emacs-verse if you prefer."})) + +(add-post! {:title "Multiplexing data in sh(1)" + :slug "sh-multiplexing-data" + :date "2019/10/17" + :tags #{:sh} + :short "please put something clever here"}) + +(add-post! {:title "Poor man's unbound dashboard" + :slug "poor-mans-unbound-dashboard" + :date "2019/10/16" + :tags #{:OpenBSD :tmux} + :short "A little bit of bash, fifos, tmux & ttyplot"}) + +(add-post! {:title "Core naming in linux" + :slug "core-naming-on-linux" + :date "2019/10/15" + :tags #{:OpenBSD :linux} + :short "Porting BSD behaviours to linux"}) + +(add-post! {:title "CSS splites with awk and ImageMagick" + :slug "css-sprites-awk-imagemagick" + :date "2019/08/08" + :tags #{:css :awk} + :short "I like to automatize things"}) + +(add-post! {:title "Using the C preprocessors for CSS generation" + :slug "cpp-css" + :date "2019/03/30" + :tags #{:css} + :short "old tools for new websites"}) + +(add-post! {:title "F* on OpenBSD" + :slug "fstar-openbsd" + :date "2019/03/25" + :tags #{:fstar :OpenBSD} + :short "It just works"}) + +(add-post! {:title "Abbreviations in vi(1)" + :slug "abbreviations-vi" + :date "2018/09/14" + :tags #{:vi} + :short "type even less"}) + +(add-post! {:title "Improving maildir opening time in mutt" + :slug "maildir-mutt-time" + :date "2018/09/14" + :tags #{:mutt} + :short "header_cache is just awesome"}) + +(add-post! {:title "Spell checking in vi(1)" + :slug "spell-checking-vi" + :date "2018/09/08" + :tags #{:vi} + :short "Aspell plays nice with vi."}) blob - /dev/null blob + e08fa1f4cd7306a5db7b149446d64b3fe1e54f5b (mode 644) --- /dev/null +++ src/blog/templates.clj @@ -0,0 +1,95 @@ +(ns blog.templates + (:require [blog.time :as time] + [hiccup.page :refer [html5 include-css]] + [commonmark-hiccup.core :refer [markdown->hiccup default-config]])) + +(defn link-item [{:keys [url text]}] + [:li [:a {:href url} text]]) + +(defn header [{:keys [tags]}] + (list + [:header + [:nav + [:ul + (link-item {:url "/", :text "Home"}) + (link-item {:url "/tags.html", :text "All Tags"})]] + [:div + [:h1 [:a {:href "/"} "yumh"]] + [:p "writing about things, sometimes."]]])) + +(defn with-page + [{:keys [title class], :as d} & body] + (html5 {:lang "en"} + [:head + [:meta {:charset "utf8"}] + [:meta {:name "viewport", :content "width=device-width, initial-scale=1"}] + [:link {:rel "shortcut icon", :href "/favicon.ico"}] + [:title title] + (include-css "/css/style.css")] + [:body {:class (or class "")} + (header d) + [:main body] + [:footer + [:p "Blog powered by " + [:code "(clojure)"]]]])) + +(defn post-fragment + [{:keys [full? title-with-link?]} + {:keys [title date slug tags short body], :as post}] + [:article + [:header + [:h1 (if title-with-link? + [:a {:href (str "post/" slug ".html")} title] + title)] + [:p.author "Written by " [:em "Omar Polo"] " on " (time/fmt-loc date)] + [:ul.tags (map #(vector :li [:a {:href (str "/tag/" (name %) ".html")} + (str "#" (name %))]) + tags)]] + [:section + (if full? + (markdown->hiccup default-config body) + [:p short])]]) + +(defn home-page + [{:keys [posts has-next has-prev nth]}] + (with-page {:title "Home"} + (map (partial post-fragment {:title-with-link? true}) + posts) + [:nav.post-navigation + (if has-prev + [:a.prev {:href (str "/" (if (= (dec nth) 1) + "index" + (dec nth)) ".html")} + "« Newer Posts"]) + (if has-next + [:a.next {:href (str "/" (inc nth) ".html")} + "Older Posts »"])])) + +(defn post-page + [{:keys [title], :as post}] + (with-page {:title title + :class "article"} + (post-fragment {:full? true} + post))) + +(defn tags-page + [tags] + (with-page {:title "All tags" + :class "tags"} + [:h2 "All tags"] + [:nav + [:ul + (map #(vector :li [:a {:href (str "/tag/" (name %) ".html")} (str "#" (name %))]) + (sort (fn [a b] + (compare (.toLowerCase (name a)) + (.toLowerCase (name b)))) tags))]])) + +(defn tag-page + [tag posts] + (with-page {:title (str "Posts tagged with #" tag) + :class "tag"} + [:h2 "Post tagged with " [:code "#" tag]] + (map (partial post-fragment {:title-with-link? true}) + (->> posts + (sort-by :date) + (reverse))))) blob - /dev/null blob + e64701e8994a7f3c198b9b5ae5066c373b80786f (mode 644) --- /dev/null +++ src/blog/time.clj @@ -0,0 +1,27 @@ +(ns blog.time + (:import (java.time.format DateTimeFormatter FormatStyle) + (java.time LocalDate) + (java.util Locale))) + +(def pattern + (DateTimeFormatter/ofPattern "yyyy/MM/dd")) + +(def loc (Locale/forLanguageTag "en")) + +(def pattern-loc + (DateTimeFormatter/ofPattern "dd MMMM yyy" loc)) + +(defn fmt [d] + (.format d pattern)) + +(defn fmt-loc [d] + (.format d pattern-loc)) + +(defn parse [s] + (LocalDate/parse s pattern)) + +(comment + (parse "2020/03/24") + (fmt (LocalDate/now)) + (fmt-loc (LocalDate/now)) + )