commit - f4262cd049b12d1ddebb100c2ab8aca3983e4b45
commit + 38232a0aceee26098c65f025b64dbdad5ebeee9b
blob - 6d5e7ffb83940f3518c98bd667a2a4da4db3012f
blob + e385e15b393c9867b61ccf028d9c76a567e7172f
--- .mblaze/Makefile
+++ .mblaze/Makefile
${INSTALL} -m 0644 ${DISTFILES} ${DESTDIR}/
install:
- mkdir -p ${DESTDIR}${SHAREDIR}/gotmarc/mblaze
- ${INSTALL_DATA} filter seq ${DESTDIR}${SHAREDIR}/gotmarc/mblaze
+ mkdir -p ${DESTDIR}${SHAREDIR}/smarc/mblaze
+ ${INSTALL_DATA} filter seq ${DESTDIR}${SHAREDIR}/smarc/mblaze
uninstall:
- rm -f ${DESTDIR}${SHAREDIR}/gotmarc/mblaze/filter
- rm -f ${DESTDIR}${SHAREDIR}/gotmarc/mblaze/seq
+ rm -f ${DESTDIR}${SHAREDIR}/smarc/mblaze/filter
+ rm -f ${DESTDIR}${SHAREDIR}/smarc/mblaze/seq
.PHONY: all dist install uninstall
include ../config.mk
blob - 9289bc39e41cda915f55fe1ec281b329cfc60a2c (mode 644)
blob + /dev/null
--- GotMArc.pm
+++ /dev/null
-# GotMArc was written by Omar Polo <op@openbsd.org> and is placed in
-# the public domain. The author hereby disclaims copyright to this
-# source code.
-
-package GotMArc;
-use strict;
-use warnings;
-use v5.32;
-use Exporter;
-use File::Basename;
-
-our @ISA = qw(Exporter);
-our @EXPORT_OK = qw(san urlencode parse initpage endpage index_header
- search thread_header threntry thrslice thrnav);
-
-sub san {
- my $str = shift;
- $str =~ s/&/\&/g;
- $str =~ s/</\</g;
- $str =~ s/>/\>/g;
- return $str;
-}
-
-sub urlencode {
- my $str = shift;
- unless (defined($str)) {
- my ($pkg, $file, $line) = caller 1;
- die "bad $pkg / $file:$line";
- }
- $str =~ s/([^-_~.A-Za-z0-9])/sprintf("%%%2X", ord($1))/ge;
- return $str;
-}
-
-sub ssan {
- my $str = shift;
- $str =~ s/\s+/ /g;
- $str =~ s/\s+$//;
- return san($str);
-}
-
-sub parse {
- my $line = shift;
- my ($indent, $fname, $date, $from, $subj) = $line =~ m{
- ^([^-]*)- # the indent level
- ([^ ]+)\s # filename
- (\d{4}-\d\d-\d\d[ ]\d\d:\d\d) # date
- <([^>]+)> # from
- (.*) # subject
- }x or die "can't parse: $line";
-
- my $level = length($indent);
- $level = 10 if $indent =~ m/\.\.\d+\.\./;
-
- $from = ssan($from);
- $subj = ssan($subj);
-
- my ($time, $id) = split /\./, basename($fname);
- my $mid = "$time.$id";
-
- return {level => $level, fname => $fname,
- mid => $mid, date => $date, from => $from, subj => $subj};
-}
-
-sub readall {
- my $path = shift;
- local $/ = undef;
- open my $fh, "<", $path or die "can't open $path: $!";
- <$fh>;
-}
-
-my $templates = $ENV{TMPLDIR};
-die 'undefined $TMPLDIR' unless defined $templates;
-
-my $small_logo = readall "$templates/logo-small.html";
-my $hdr = readall "$templates/head.html";
-my $foot = readall "$templates/foot.html";
-my $idxhdr = readall "$templates/index-header.html";
-my $search = readall "$templates/search.html";
-
-sub initpage {
- my ($fh, $title) = @_;
- say $fh $hdr =~ s/TITLE/$title/r;
-}
-
-sub endpage {
- my $fh = shift;
- say $fh $foot;
-}
-
-sub index_header {
- my ($fh, $page, $subtitle) = @_;
- my $html = $idxhdr =~ s/PAGE/$page/r;
- $html =~ s/SUBTITLE/$subtitle/;
- print $fh $html;
-}
-
-sub search {
- my $fh = shift;
- my $html = $search =~ s/QUERY//r;
- print $fh $html;
-}
-
-sub thread_header {
- my ($fh, $e, $mail, $p, $n) = @_;
-
- my @entries = @$e;
-
- my $enctid = urlencode $mail->{tid} if defined $mail;
- my $encmid = urlencode $mail->{mid} if defined $mail;
-
- print $fh "<header class='mail-header'>\n";
-
- print $fh "<p>";
- print $fh $small_logo;
- print $fh "<a href='/'>Index</a>";
- print $fh " | <a href='/thread/$enctid.html#$encmid'>Thread</a>"
- if defined $enctid;
- print $fh " | <a href='/search'>Search</a>";
- print $fh "</p>\n";
-
- say $fh "<dl>";
- foreach my $entry (@entries) {
- my ($k, $v) = split /: /, $entry, 2;
- chomp $v;
- say $fh "<dt>$k:</dt><dd>$v</dd>";
- }
- say $fh "</dl>";
-
- say $fh "<p>Download raw <a href='/text/$encmid.txt'>body</a>.</p>"
- if defined $encmid;
-
- if (defined($p) and defined($n)) {
- say $fh "<details>";
- say $fh "<summary>Thread</summary>";
- thrslice($fh, $mail, $p, $n);
- say $fh "</details>";
- thrnav($fh, $p, $n, $mail->{mid}, $mail->{tid});
- }
-
- say $fh "</header>\n";
-}
-
-sub threntry {
- my ($fh, $type, $base, $last_level, $mail, $cur) = @_;
- my $level = $mail->{level} - $base;
-
- say $fh "</ul></li>" x ($last_level - $level) if $last_level > $level;
- say $fh "<li><ul>" if $last_level < $level;
-
- my $encmid = urlencode $mail->{mid};
-
- print $fh "<li id='$encmid' class='mail'>";
- print $fh "<p class='mail-meta'>";
- print $fh "<time>$mail->{date}</time> ";
- print $fh "<span class='from'>$mail->{from}</span>";
- print $fh "<span class='colon'>:</span>";
- print $fh "</p>";
- print $fh "<p class='subject'>";
-
- my $subj = $mail->{subj};
- if (!defined($cur) || $mail->{mid} ne $cur->{mid}) {
- print $fh "<a href='/$type/$encmid.html'>$subj</a>";
- } else {
- print $fh "<span>$subj</span>";
- }
-
- print $fh "</p>";
- print $fh "</li>";
-
- return $level;
-}
-
-sub min_level {
- my $l = 999;
- return 0 unless @_;
- for (@_) {
- $l = $_->{level} if $_->{level} < $l;
- }
- return $l;
-}
-
-sub thrslice {
- my ($fh, $mail, $p, $n) = @_;
- my @prev = @{$p};
- my @next = @{$n};
- my @thread = (@prev, $mail, @next);
- return unless @thread;
- my $base = min_level @thread;
- my $level = 0;
- print $fh "<div class='thread'>";
- print $fh "<ul class='mails'>";
- $level = threntry $fh, "mail", $base, $level, $_, $mail for @thread;
- print $fh "</ul></li>" x $level;
- print $fh "</ul></div>";
-}
-
-sub thrnav {
- my ($fh, $p, $n) = @_;
- my @prev = @{$p};
- my @next = @{$n};
-
- return if !@prev && !@next;
- print $fh "<nav>";
-
- if (@prev) {
- my $mail = $prev[-1];
- my $encmid = $mail->{mid};
- say $fh "<a href='/mail/$encmid.html'>Previous in thread</a>";
- } else {
- say $fh "<span>Previous in thread</span>";
- }
-
- if (@next) {
- my $mail = $next[0];
- my $encmid = $mail->{mid};
- say $fh "<a href='/mail/$encmid.html'>Next in thread</a>";
- } else {
- say $fh "<span>Next in thread</span>";
- }
-
- print $fh "</nav>";
-}
-
-1;
blob - 97faafb39a2cb90cba73863cc62058316deaad49
blob + 30f702b9f760ccca07311ea189206eeefeb3b003
--- Makefile
+++ Makefile
# -- build-related variables --
VERSION = 0.1
-DISTNAME = gotmarc-${VERSION}
+DISTNAME = smarc-${VERSION}
# -- public targets --
install:
mkdir -p ${DESTDIR}${BINDIR}
- ${INSTALL_PROGRAM} gmimport ${DESTDIR}${BINDIR}
- sed -e "/^libexec=/s@=.*@=${LIBEXEC}/gotmarc@" \
- -e "/^mblaze=/s@=.*@=${SHAREDIR}/gotmarc/mblaze@" \
- -e "/^tmpldir=/s@=.*@=${REALSYSCONFDIR}/gotmarc@" \
- gotmarc > ${DESTDIR}${BINDIR}/gotmarc
- chmod 0755 ${DESTDIR}${BINDIR}/gotmarc
- mkdir -p ${DESTDIR}${LIBEXEC}/gotmarc
- ${INSTALL_PROGRAM} filter-ignore ${DESTDIR}${LIBEXEC}/gotmarc
- ${INSTALL_PROGRAM} mexp ${DESTDIR}${LIBEXEC}/gotmarc
- ${INSTALL_PROGRAM} mkindex ${DESTDIR}${LIBEXEC}/gotmarc
- ${INSTALL_PROGRAM} pe ${DESTDIR}${LIBEXEC}/gotmarc
- mkdir -p ${DESTDIR}${SYSCONFDIR}/gotmarc
- ${INSTALL_DATA} style.css ${DESTDIR}${SYSCONFDIR}/gotmarc/
+ ${INSTALL_PROGRAM} smingest ${DESTDIR}${BINDIR}
+ sed -e "/^libexec=/s@=.*@=${LIBEXEC}/smarc@" \
+ -e "/^mblaze=/s@=.*@=${SHAREDIR}/smarc/mblaze@" \
+ -e "/^tmpldir=/s@=.*@=${REALSYSCONFDIR}/smarc@" \
+ smarc > ${DESTDIR}${BINDIR}/smarc
+ chmod 0755 ${DESTDIR}${BINDIR}/smarc
+ mkdir -p ${DESTDIR}${LIBEXEC}/smarc
+ ${INSTALL_PROGRAM} filter-ignore ${DESTDIR}${LIBEXEC}/smarc
+ ${INSTALL_PROGRAM} mexp ${DESTDIR}${LIBEXEC}/smarc
+ ${INSTALL_PROGRAM} mkindex ${DESTDIR}${LIBEXEC}/smarc
+ ${INSTALL_PROGRAM} pe ${DESTDIR}${LIBEXEC}/smarc
+ mkdir -p ${DESTDIR}${SYSCONFDIR}/smarc
+ ${INSTALL_DATA} style.css ${DESTDIR}${SYSCONFDIR}/smarc/
mkdir -p ${DESTDIR}${PERL_LIB}
- ${INSTALL_DATA} GotMArc.pm ${DESTDIR}${PERL_LIB}
+ ${INSTALL_DATA} SMArc.pm ${DESTDIR}${PERL_LIB}
mkdir -p ${DESTDIR}${MANDIR}/man1
- ${INSTALL_MAN} gmimport.1 ${DESTDIR}${MANDIR}/man1/
- ${INSTALL_MAN} gotmarc.1 ${DESTDIR}${MANDIR}/man1/
+ ${INSTALL_MAN} smingest.1 ${DESTDIR}${MANDIR}/man1/
+ ${INSTALL_MAN} smarc.1 ${DESTDIR}${MANDIR}/man1/
mkdir -p ${DESTDIR}${MANDIR}/man7
- ${INSTALL_MAN} gotmarc.7 ${DESTDIR}${MANDIR}/man7/
+ ${INSTALL_MAN} smarc.7 ${DESTDIR}${MANDIR}/man7/
${MAKE} -C .mblaze install
${MAKE} -C msearchd install
${MAKE} -C templates install
uninstall:
- rm -f ${DESTDIR}${BINDIR}/gmimport
- rm -f ${DESTDIR}${BINDIR}/gotmarc
- rm -f ${DESTDIR}${LIBEXEC}/gotmarc/filter-ignore
- rm -f ${DESTDIR}${LIBEXEC}/gotmarc/mexp
- rm -f ${DESTDIR}${LIBEXEC}/gotmarc/mkindex
- rm -f ${DESTDIR}${LIBEXEC}/gotmarc/pe
- rm -f ${DESTDIR}${SYSCONFDIR}/gotmarc/style.css
- rm -f ${DESTDIR}${PERL_LIB}/GotMArc.pm
- rm -f ${DESTDIR}${MANDIR}/man1/gmimport.1
- rm -f ${DESTDIR}${MANDIR}/man1/gotmarc.1
- rm -f ${DESTDIR}${MANDIR}/man7/gotmarc.7
+ rm -f ${DESTDIR}${BINDIR}/smingest
+ rm -f ${DESTDIR}${BINDIR}/smarc
+ rm -f ${DESTDIR}${LIBEXEC}/smarc/filter-ignore
+ rm -f ${DESTDIR}${LIBEXEC}/smarc/mexp
+ rm -f ${DESTDIR}${LIBEXEC}/smarc/mkindex
+ rm -f ${DESTDIR}${LIBEXEC}/smarc/pe
+ rm -f ${DESTDIR}${SYSCONFDIR}/smarc/style.css
+ rm -f ${DESTDIR}${PERL_LIB}/SMArc.pm
+ rm -f ${DESTDIR}${MANDIR}/man1/smingest.1
+ rm -f ${DESTDIR}${MANDIR}/man1/smarc.1
+ rm -f ${DESTDIR}${MANDIR}/man7/smarc.7
${MAKE} -C .mblaze uninstall
${MAKE} -C msearchd uninstall
${MAKE} -C templates uninstall
PRIVKEY = missing-PRIVKEY
PUBKEY = missing-PUBKEY
-DISTFILES = GotMArc.pm Makefile README TODO configure \
- filter-ignore gmimport gmimport.1 gotmarc gotmarc.1 \
- gotmarc.7 mexp mkindex pe style.css
+DISTFILES = SMArc.pm Makefile README TODO configure \
+ filter-ignore smingest smingest.1 smarc smarc.1 \
+ smarc.7 mexp mkindex pe style.css
MANOPTS = man='%N.%S.html;https://man.openbsd.org/%N.%S',style=mandoc.css,toc
MANFLAGS = -Thtml -O${MANOPTS}
man:
touch msearchd.8
- man ${MANFLAGS} -l gmimport.1 > gmimport.1.html
- man ${MANFLAGS} -l gotmarc.1 > gotmarc.1.html
- man ${MANFLAGS} -l gotmarc.7 > gotmarc.7.html
+ man ${MANFLAGS} -l smingest.1 > smingest.1.html
+ man ${MANFLAGS} -l smarc.1 > smarc.1.html
+ man ${MANFLAGS} -l smarc.7 > smarc.7.html
man ${MANFLAGS} -l msearchd/msearchd.8 > msearchd.8.html
rm msearchd.8
mkdir -p .dist/${DISTNAME}
${INSTALL} -m 0644 ${DISTFILES} .dist/${DISTNAME}
cd .dist/${DISTNAME} && chmod 0755 configure filter-ignore \
- gmimport gotmarc mexp mkindex pe
+ smingest smarc mexp mkindex pe
${MAKE} -C .mblaze DESTDIR=${PWD}/.dist/${DISTNAME}/.mblaze dist
${MAKE} -C templates DESTDIR=${PWD}/.dist/${DISTNAME}/templates dist
${MAKE} -C msearchd DESTDIR=${PWD}/.dist/${DISTNAME}/msearchd dist
blob - 389da3009756ace78ee9146f0159713cb7e5c0e2
blob + 9a19ae9aaff787637dc7f2309fb2fada2eb4f070
--- README
+++ README
-GOTMARC -- A static mailing list web archive generator
-======================================================
+SMARC -- static web mail archive generator
+==========================================
A collection of scripts to generate a static mail archive from a
Maildir; originally intended to provide a public web interface for the
How it works
------------
-gotmarc(1) incrementally generate a set of static files from a maildir,
-gmimport(1) and msearchd(8) are used to populate an sqlite3 database to
+smarc(1) incrementally generate a set of static files from a maildir,
+smingest(1) and msearchd(8) are used to populate an sqlite3 database to
provide a web-based full text search on all the mails.
-gotmarc(7) documents the step needed for a first setup, how to operate
+smarc(7) documents the step needed for a first setup, how to operate
it and tips for generating multiple archives.
License
-------
-gotmarc, gmimport and msearchd are released in the public domain. The
+smarc, smingest and msearchd are released in the public domain. The
configure script loosely based on oconfigure and is released under the
terms of ISC license. Some OpenBSD-compats are included to build on
multiple platforms, these are released under a BSD-style license.
blob - /dev/null
blob + 9289bc39e41cda915f55fe1ec281b329cfc60a2c (mode 644)
--- /dev/null
+++ SMArc.pm
+# GotMArc was written by Omar Polo <op@openbsd.org> and is placed in
+# the public domain. The author hereby disclaims copyright to this
+# source code.
+
+package GotMArc;
+use strict;
+use warnings;
+use v5.32;
+use Exporter;
+use File::Basename;
+
+our @ISA = qw(Exporter);
+our @EXPORT_OK = qw(san urlencode parse initpage endpage index_header
+ search thread_header threntry thrslice thrnav);
+
+sub san {
+ my $str = shift;
+ $str =~ s/&/\&/g;
+ $str =~ s/</\</g;
+ $str =~ s/>/\>/g;
+ return $str;
+}
+
+sub urlencode {
+ my $str = shift;
+ unless (defined($str)) {
+ my ($pkg, $file, $line) = caller 1;
+ die "bad $pkg / $file:$line";
+ }
+ $str =~ s/([^-_~.A-Za-z0-9])/sprintf("%%%2X", ord($1))/ge;
+ return $str;
+}
+
+sub ssan {
+ my $str = shift;
+ $str =~ s/\s+/ /g;
+ $str =~ s/\s+$//;
+ return san($str);
+}
+
+sub parse {
+ my $line = shift;
+ my ($indent, $fname, $date, $from, $subj) = $line =~ m{
+ ^([^-]*)- # the indent level
+ ([^ ]+)\s # filename
+ (\d{4}-\d\d-\d\d[ ]\d\d:\d\d) # date
+ <([^>]+)> # from
+ (.*) # subject
+ }x or die "can't parse: $line";
+
+ my $level = length($indent);
+ $level = 10 if $indent =~ m/\.\.\d+\.\./;
+
+ $from = ssan($from);
+ $subj = ssan($subj);
+
+ my ($time, $id) = split /\./, basename($fname);
+ my $mid = "$time.$id";
+
+ return {level => $level, fname => $fname,
+ mid => $mid, date => $date, from => $from, subj => $subj};
+}
+
+sub readall {
+ my $path = shift;
+ local $/ = undef;
+ open my $fh, "<", $path or die "can't open $path: $!";
+ <$fh>;
+}
+
+my $templates = $ENV{TMPLDIR};
+die 'undefined $TMPLDIR' unless defined $templates;
+
+my $small_logo = readall "$templates/logo-small.html";
+my $hdr = readall "$templates/head.html";
+my $foot = readall "$templates/foot.html";
+my $idxhdr = readall "$templates/index-header.html";
+my $search = readall "$templates/search.html";
+
+sub initpage {
+ my ($fh, $title) = @_;
+ say $fh $hdr =~ s/TITLE/$title/r;
+}
+
+sub endpage {
+ my $fh = shift;
+ say $fh $foot;
+}
+
+sub index_header {
+ my ($fh, $page, $subtitle) = @_;
+ my $html = $idxhdr =~ s/PAGE/$page/r;
+ $html =~ s/SUBTITLE/$subtitle/;
+ print $fh $html;
+}
+
+sub search {
+ my $fh = shift;
+ my $html = $search =~ s/QUERY//r;
+ print $fh $html;
+}
+
+sub thread_header {
+ my ($fh, $e, $mail, $p, $n) = @_;
+
+ my @entries = @$e;
+
+ my $enctid = urlencode $mail->{tid} if defined $mail;
+ my $encmid = urlencode $mail->{mid} if defined $mail;
+
+ print $fh "<header class='mail-header'>\n";
+
+ print $fh "<p>";
+ print $fh $small_logo;
+ print $fh "<a href='/'>Index</a>";
+ print $fh " | <a href='/thread/$enctid.html#$encmid'>Thread</a>"
+ if defined $enctid;
+ print $fh " | <a href='/search'>Search</a>";
+ print $fh "</p>\n";
+
+ say $fh "<dl>";
+ foreach my $entry (@entries) {
+ my ($k, $v) = split /: /, $entry, 2;
+ chomp $v;
+ say $fh "<dt>$k:</dt><dd>$v</dd>";
+ }
+ say $fh "</dl>";
+
+ say $fh "<p>Download raw <a href='/text/$encmid.txt'>body</a>.</p>"
+ if defined $encmid;
+
+ if (defined($p) and defined($n)) {
+ say $fh "<details>";
+ say $fh "<summary>Thread</summary>";
+ thrslice($fh, $mail, $p, $n);
+ say $fh "</details>";
+ thrnav($fh, $p, $n, $mail->{mid}, $mail->{tid});
+ }
+
+ say $fh "</header>\n";
+}
+
+sub threntry {
+ my ($fh, $type, $base, $last_level, $mail, $cur) = @_;
+ my $level = $mail->{level} - $base;
+
+ say $fh "</ul></li>" x ($last_level - $level) if $last_level > $level;
+ say $fh "<li><ul>" if $last_level < $level;
+
+ my $encmid = urlencode $mail->{mid};
+
+ print $fh "<li id='$encmid' class='mail'>";
+ print $fh "<p class='mail-meta'>";
+ print $fh "<time>$mail->{date}</time> ";
+ print $fh "<span class='from'>$mail->{from}</span>";
+ print $fh "<span class='colon'>:</span>";
+ print $fh "</p>";
+ print $fh "<p class='subject'>";
+
+ my $subj = $mail->{subj};
+ if (!defined($cur) || $mail->{mid} ne $cur->{mid}) {
+ print $fh "<a href='/$type/$encmid.html'>$subj</a>";
+ } else {
+ print $fh "<span>$subj</span>";
+ }
+
+ print $fh "</p>";
+ print $fh "</li>";
+
+ return $level;
+}
+
+sub min_level {
+ my $l = 999;
+ return 0 unless @_;
+ for (@_) {
+ $l = $_->{level} if $_->{level} < $l;
+ }
+ return $l;
+}
+
+sub thrslice {
+ my ($fh, $mail, $p, $n) = @_;
+ my @prev = @{$p};
+ my @next = @{$n};
+ my @thread = (@prev, $mail, @next);
+ return unless @thread;
+ my $base = min_level @thread;
+ my $level = 0;
+ print $fh "<div class='thread'>";
+ print $fh "<ul class='mails'>";
+ $level = threntry $fh, "mail", $base, $level, $_, $mail for @thread;
+ print $fh "</ul></li>" x $level;
+ print $fh "</ul></div>";
+}
+
+sub thrnav {
+ my ($fh, $p, $n) = @_;
+ my @prev = @{$p};
+ my @next = @{$n};
+
+ return if !@prev && !@next;
+ print $fh "<nav>";
+
+ if (@prev) {
+ my $mail = $prev[-1];
+ my $encmid = $mail->{mid};
+ say $fh "<a href='/mail/$encmid.html'>Previous in thread</a>";
+ } else {
+ say $fh "<span>Previous in thread</span>";
+ }
+
+ if (@next) {
+ my $mail = $next[0];
+ my $encmid = $mail->{mid};
+ say $fh "<a href='/mail/$encmid.html'>Next in thread</a>";
+ } else {
+ say $fh "<span>Next in thread</span>";
+ }
+
+ print $fh "</nav>";
+}
+
+1;
blob - 397e33518ab904ab8aa7c556c08fa3966444f7a3
blob + 18c85084e04090f3f0fc6dc0fec76f804e026dfe
--- configure
+++ configure
echo "running configure for \`msearchd':" >&2
(cd msearchd && ./configure)
echo >&2
-echo "returning to the configure for \`gotmarc':" >&2
+echo "returning to the configure for \`smarc':" >&2
exec > config.mk
echo "config.mk: writing...">&2
blob - 3ea6af4dc11e261dec6f65101a63beb58f2664a7 (mode 755)
blob + /dev/null
--- gmimport
+++ /dev/null
-#!/usr/bin/env perl
-#
-# gmimport was written by Omar Polo <op@openbsd.org> and is placed in the
-# public domain. The author hereby disclaims copyright to this source
-# code.
-
-use strict;
-use warnings;
-use v5.32;
-use utf8;
-
-use Date::Parse;
-use File::Basename;
-
-use OpenBSD::Pledge;
-use OpenBSD::Unveil;
-
-die "usage: $0 dbpath\n" if @ARGV != 1;
-my $dbpath = shift @ARGV;
-
-open(my $sqlite, "|-", "/usr/local/bin/sqlite3", $dbpath)
- or die "can't spawn sqlite3";
-
-unveil("/usr/local/bin/mshow", "rx") or die "unveil mshow: $!";
-pledge("stdio proc exec") or die "pledge: $!";
-
-say $sqlite ".import --csv /dev/stdin email"
- or die "can't speak to sqlite: $!";
-
-while (<>) {
- chomp;
-
- open(my $fh, "-|", "/usr/local/bin/mshow", "-Atext/plain", "-NF", $_)
- or die "can't run mshow $_: $!";
-
- my $f = $_;
- my ($time, $id) = split /\./, basename $_;
- my $mid = "$time.$id";
- $mid =~ s/"/""/g;
-
- my ($from, $subj, $date) = ('', '', undef);
- while (<$fh>) {
- chomp;
- last if /^$/;
- s/"/""/g;
- $from = s/.*?: //r if /^From:/;
- $subj = s/.*?: //r if /^Subject:/;
- $date = str2time(s/.*?: //r) if /^Date:/;
- }
- $date //= time;
- $from =~ s/ +<.*>//;
-
- # leave open for the body
- print $sqlite "\"$mid\",\"$from\",\"$date\",\"$subj\",\"";
-
- while (<$fh>) {
- s/"/""/g;
- print $sqlite $_;
- }
- say $sqlite '"';
-
- close $fh;
-}
-
-close $sqlite;
-die "sqlite3 exited with $?\n" unless $? == 0;
blob - 331662d4de04a2cc038c914e66bf09f0420b1e53 (mode 644)
blob + /dev/null
--- gmimport.1
+++ /dev/null
-.\" gmimport.1 was written by Omar Polo <op@openbsd.org> and is placed in
-.\" the public domain. The author hereby disclaims copyright to this
-.\" source code.
-.Dd May 5, 2023
-.Dt GMIMPORT 1
-.Os
-.Sh NAME
-.Nm gmimport
-.Nd import emails into a sqlite database
-.Sh SYNOPSIS
-.Nm
-.Ar dbpath
-.Sh DESCRIPTION
-.Nm
-reads a sequence of path to emails from standard inputs and inserts them
-into the
-.Xr msearchd 8
-sqlite3 database at
-.Ar dbpath .
-.Sh EXAMPLES
-To index all the messages in the
-.Pa ~/Mail/gotmarc
-maildir, useful to initially populate the database:
-.Pp
-.Dl mlist ~/Mail/gotmarc | gmimport /var/www/msearchd/mails.sqlite3
-.Pp
-Incorporate new messages in the maildir and add them to the database,
-useful after fetching new mails:
-.Pp
-.Dl minc ~/Mail/gotmarc | gmimport /var/www/msearchd/mails.sqlite3
-.Sh SEE ALSO
-.Xr minc 1 ,
-.Xr mlist 1 ,
-.Xr msearchd 8
-.Sh BUGS
-.Nm
-doesn't detect the insert of duplicate messages.
-Should that happen, the database needs to be recreated.
blob - 035828b31af7891b28eeca6413e6f32e0c8d8c8e (mode 755)
blob + /dev/null
--- gotmarc
+++ /dev/null
-#!/bin/sh
-#
-# gotmarc was written by Omar Polo <op@openbsd.org> and is placed in the
-# public domain. The author hereby disclaims copyright to this source
-# code.
-
-progname="$(basename "$0")"
-usage() {
- u="usage: $progname [-c cachedir] [-j n]"
- u="$u [-m maildir] [-o outdir] [-t tmpldir]"
- echo "$u">&2
- exit 1
-}
-
-# changed at install-time
-libexec=.
-mblaze=.mblaze
-tmpldir=templates/
-
-# fix perl include path for in-repo hacking
-if [ "$libexec" = . ]; then
- export PERL5LIB="$PWD${PERL5LIB:+:}$PERL5LIB"
-fi
-
-cachedir=$HOME/.cache/gotmarc
-mdir=$HOME/Mail/gotmarc
-outdir=/var/www/gotmarc
-
-while getopts c:j:m:o:t: flag; do
- case $flag in
- c) cachedir="$OPTARG" ;;
- j) MAKE_JOBS="$OPTARG" ;;
- m) mdir="$OPTARG" ;;
- o) outdir="$OPTARG" ;;
- t) tmpldir="$OPTARG" ;;
- ?) usage ;;
- esac
-done
-
-# set up the env
-export CSUMDIR="$cachedir/threadsum"
-export MAKE_JOBS="${MAKE_JOBS:-1}"
-export MBLAZE="$mblaze"
-export MBLAZE_PAGER=cat
-export MDIR="$mdir"
-export OUTDIR="${outdir:-/var/www/gotmarc}"
-export TMPLDIR="$tmpldir"
-
-# make sure the directories are there
-set -e
-mkdir -p "$CSUMDIR"
-mkdir -p "$OUTDIR/mail"
-mkdir -p "$OUTDIR/parts"
-mkdir -p "$OUTDIR/text"
-mkdir -p "$OUTDIR/thread"
-set +e
-
-export PATH="$libexec:$PATH"
-
-fmt='%i-%R %16D<%64f>%128S'
-mlist "${MDIR}" | mthread -r | mscan -f "$fmt" | pe | mkindex
blob - ff5cece36c866d7184bec2d2bf88229d45621408 (mode 644)
blob + /dev/null
--- gotmarc.1
+++ /dev/null
-.\" gotmarc.1 was written by Omar Polo <op@openbsd.org> and is placed in
-.\" the public domain. The author hereby disclaims copyright to this
-.\" source code.
-.Dd May 5, 2023
-.Dt GOTMARC 1
-.Os
-.Sh NAME
-.Nm gotmarc
-.Nd mailing list static site generator
-.Sh SYNOPSIS
-.Nm
-.Op Fl c Ar cachedir
-.Op Fl j Ar n
-.Op Fl m Ar maildir
-.Op Fl o Ar outdir
-.Op Fl t Ar tmpldir
-.Sh DESCRIPTION
-.Nm
-generates a static web archive from a maildir incrementally.
-.Pp
-The arguments are as follows:
-.Bl -tag -width Ds
-.It Fl c Ar cachedir
-Cache directory used for keeping track of threads during incremental
-updates.
-Defaults to
-.Pa ~/.cache/gotmarc
-.It Fl j Ar n
-Use
-.Ar n
-jobs to export threads.
-With big mailing lists and on a multiprocessor machine this can save
-quite some time.
-.It Fl m Ar maildir
-Source
-.Ar maildir .
-Defaults to
-.Pa ~/Mail/gotmarc .
-.It Fl o Ar outdir
-The destination directory where the HTML files will be written to, it's
-.Pa /var/www/gotmarc
-by default.
-.It Fl t Ar tmpldir
-Path to a directory containing the template files.
-.Pa /etc/gotmarc
-by default.
-.El
-.Sh FILES
-.Bl -tag -width Ds
-.It Pa /etc/gotmarc/foot.html
-Template with the trailing part of the page.
-.It Pa /etc/gotmarc/head.html
-Template with the first part of the page.
-.Dv TITLE
-is substituted with the title.
-.It Pa /etc/gotmarc/index-header.html
-Template with the title for index pages.
-.Dv PAGE
-is replaced with the page number and
-.Dv SUBTITLE
-with the date range.
-.It Pa /etc/gotmarc/logo-small.html
-Small version of the logo, included in the thread header.
-.It Pa /etc/gotmarc/search.html
-Template for the search form.
-.Ev QUERY
-is replaced with the search query.
-.It Pa /var/www/gotmarc
-Default output directory.
-.It Pa ~/.cache/gotmarc
-Default cache directory.
-.It Pa ~/Mail/gotmarc
-Default maildir.
-.El
-.Sh SEE ALSO
-.Xr gotmarc 7
-.Sh HISTORY
-.Nm
-started as a hacky collection of perl scripts to generate a web archive
-for the Game of Trees mailing list.
-Only later it was converted to be a usable generic web mailing list
-archive tool.
-.Sh AUTHORS
-.An -nosplit
-The
-.Nm
-utility was written by
-.An Omar Polo Aq Mt op@openbsd.org .
blob - f7acd2906870c8225c61e6149d82d8f6342b04d0 (mode 644)
blob + /dev/null
--- gotmarc.7
+++ /dev/null
-.\" gotmarc.7 was written by Omar Polo <op@openbsd.org> and is placed in
-.\" the public domain. The author hereby disclaims copyright to this
-.\" source code.
-.Dd May 5, 2023
-.Dt GOTMARC 7
-.Os
-.Sh NAME
-.Nm gotmarc
-.Nd mailing list web archive generation system
-.Sh DESCRIPTION
-.Nm
-is a system to incrementally generate a web archive for a mailing list
-and optionally provide search capabilities.
-The generated archive is a set of HTML files that can be served as-is,
-while searching requires the use of a FastCGI server.
-At a higher level,
-.Nm
-is made of three components:
-.Pp
-.Bl -tag -width msearchd_8_ -compact -offset indent
-.It Xr gotmarc 1
-to generate the static HTML files from a maildir.
-.It Xr gmimport 1
-to import emails into a sqlite3 database.
-.It Xr msearchd 8
-to provide search results.
-.El
-.Sh INITIAL SETUP
-There are several step necessary to initialize the web archive:
-.Pp
-.Bl -enum -compact -offset indent
-.It
-Create and populate the output directory.
-.It
-Customize the templates.
-.It
-Prepare the maildir.
-.It
-Generate the web archive.
-.It
-Set up the database for searching.
-.It
-Configure the web server.
-.El
-.Pp
-It is reccommended to use a dedicate user.
-Commands to be run as a unpriviledged user are preceded by a dollar sign
-.Sq $ ,
-while commands requiring superuser privileges by a hash mark
-.Sq # .
-Hereafter, it will be assumed that the local user is called
-.Sq gotmarc .
-.Ss 1. Create and populate the output directory
-The web archive is made of several static files, mostly HTML, that needs
-to be served by a web server like
-.Xr httpd 8 .
-.Pa /var/www/gotmarc
-is the default location, but a different path can be used.
-To prepare it, issue:
-.Bd -literal -offset indent
-# mkdir -p /var/www/gotmarc
-# chown gotmarc /var/www/gotmarc
-.Ed
-.Pp
-Then copy the CSS file, optionally tweaking it.
-.Pp
-.Dl $ cp /usr/local/share/examples/gotmarc/style.css /var/www/gotmarc
-.Pp
-Other eventual assets
-.Pq e.g.\& logo images
-need to be copied here as well.
-.Ss 2. Customize the templates
-The default templates are installed at
-.Pa /etc/gotmarc .
-Since these are anonymous, they need to be tweaked to include
-information about the mailing list.
-.Pp
-Care should be taken when editing these files after generating the
-archive since existing pages won't be automatically updated.
-The cachedir
-.Pq see Xr gotmarc 1
-needs to be deleted and the web archive generated again.
-.Xr msearchd 8
-has to be stopped and restarted as well.
-.Ss 3. Prepare the maildir
-The maildir with the mailing list entries needs to be prepared.
-It is assumed to be at
-.Pa ~/Mail/gotmarc
-by default, but a different path can be used.
-.Ss 4. Generate the web archive
-.Xr gotmarc 1
-can be finally used to generate the web archive.
-The first run may take a while, depending on the size of the maildir,
-while subsequent runs will be incremental and take less time.
-.Pp
-.Dl $ gotmarc -m path/to/maildir -o path/to/outdir
-.Pp
-On multi-processor machines multiple processes may be used to save some
-time with the
-.Xr gotmarc 1 Fl j No flag.
-.Pp
-The generated files may be compressed to save bandwidth:
-.Pp
-.Dl $ gzip -krq /var/www/gotmarc </dev/null 2>/dev/null
-.Ss 5. Set up the database for searching
-This is an suggested yet optional step.
-.Pp
-.Xr msearchd 8
-offers full text search capabilities using a sqlite3 database that has to
-be populated with
-.Xr gmimport 1 .
-First, create a directory in the
-.Pa /var/www
-.Xr chroot 8
-jail:
-.Bd -literal -offset indent
-# mkdir -p /var/www/msearchd
-# chown gotmarc /var/www/msearchd
-.Ed
-.Pp
-Then, populate the database with all emails in the maildir:
-.Bd -literal -offset indent
-$ sqlite3 /var/www/msearchd/mails.sqlite3 \e
- </usr/local/share/examples/gotmarc/schema.sql
-$ mlist ~/Mail/gotmarc | gmimport /var/www/msearchd/mails.sqlite3
-.Ed
-.Pp
-At this point,
-.Xr msearchd 8
-can be started.
-.Ss 6. Configure the web server
-The web server needs to serve the contents of the outdir as-is and
-handle the requests for
-.Pa /search
-via the
-.Xr msearchd 8
-FastCGI server.
-A sample
-.Xr httpd 8
-configuration is provided here for reference:
-.Bd -literal -offset indent
-server "marc.example.com" {
- listen on * port 80
- root "/gotmarc"
- gzip-static
-
- # leave out when not using msearchd(8)
- location "/search" {
- fastcgi socket "/run/msearchd.sock"
- }
-}
-.Ed
-.Sh HANDLING NEW MESSAGES
-New messages should be fetched periodically using tools like
-.Xr fdm 1
-or
-.Xr mbsync 1 ,
-the database updated with
-.Xr gmimport 1
-and the web archive refreshed using
-.Xr gotmarc 1 .
-.Pp
-It is reccommended to create a script like the following and schedule
-its execution periodically with
-.Xr cron 8 :
-.Bd -literal -offset indent
-#!/bin/sh
-
-set -e
-fdm -l fetch
-minc ~/Mail/gotmarc | gmimport /var/www/msearchd/mails.sqlite3
-gotmarc
-gzip -krq /var/www/gotmarc/ </dev/null 2>/dev/null || true
-.Ed
-.Pp
-If
-.Xr msearchd 8
-is not used,
-new messages still needs to be incorporated
-.Po i.e.\& moved from
-.Pa new/
-to
-.Pa cur/
-.Pc
-but no database has to be updated.
-In that case simplify the
-.Xr minc 1
-invocation as:
-.Pp
-.Dl minc -q ~/Mail/gotmarc
-.Pp
-and don't call
-.Xr gmimport 1
-at all.
-.Sh HANDLING MULTIPLE MAILING LISTS
-If the archive for multiple mailing lists needs to be served from the
-same box, care must be taken to use different directories and database
-files to avoid mixing messages.
-.Pp
-.Xr msearchd 8
-handles only one database at a time, so multiple instances need to be
-run, each pointing at the database for only one mailing list.
-Different FastCGI socket path needs to be used per-instance.
-.Pp
-.Xr gotmarc 1
-outdir, maildir and cachedir must be unique per-mailing list, i.e.\& the
-.Fl c , Fl m No and Fl o
-flag must always be provided.
-.Pp
-Very likely, each mailing list will needs its own set of templates, so
-those needs to be prepared and both
-.Xr gotmarc 1
-and
-.Xr msearchd 8
-have to be pointed at the right template directory.
-.Sh SEE ALSO
-.Xr gmimport 1 ,
-.Xr gotmarc 1 ,
-.Xr minc 1 ,
-.Xr sqlite3 1 ,
-.Xr httpd 8 ,
-.Xr msearchd 8
blob - /dev/null
blob + 5a1f4106e82d9a981c16293f0d4dd285e9bc7701 (mode 755)
--- /dev/null
+++ smarc
+#!/bin/sh
+#
+# smarc was written by Omar Polo <op@openbsd.org> and is placed in the
+# public domain. The author hereby disclaims copyright to this source
+# code.
+
+progname="$(basename "$0")"
+usage() {
+ u="usage: $progname [-c cachedir] [-j n]"
+ u="$u [-m maildir] [-o outdir] [-t tmpldir]"
+ echo "$u">&2
+ exit 1
+}
+
+# changed at install-time
+libexec=.
+mblaze=.mblaze
+tmpldir=templates/
+
+# fix perl include path for in-repo hacking
+if [ "$libexec" = . ]; then
+ export PERL5LIB="$PWD${PERL5LIB:+:}$PERL5LIB"
+fi
+
+cachedir=$HOME/.cache/smarc
+mdir=$HOME/Mail/smarc
+outdir=/var/www/smarc
+
+while getopts c:j:m:o:t: flag; do
+ case $flag in
+ c) cachedir="$OPTARG" ;;
+ j) MAKE_JOBS="$OPTARG" ;;
+ m) mdir="$OPTARG" ;;
+ o) outdir="$OPTARG" ;;
+ t) tmpldir="$OPTARG" ;;
+ ?) usage ;;
+ esac
+done
+
+# set up the env
+export CSUMDIR="$cachedir/threadsum"
+export MAKE_JOBS="${MAKE_JOBS:-1}"
+export MBLAZE="$mblaze"
+export MBLAZE_PAGER=cat
+export MDIR="$mdir"
+export OUTDIR="${outdir:-/var/www/smarc}"
+export TMPLDIR="$tmpldir"
+
+# make sure the directories are there
+set -e
+mkdir -p "$CSUMDIR"
+mkdir -p "$OUTDIR/mail"
+mkdir -p "$OUTDIR/parts"
+mkdir -p "$OUTDIR/text"
+mkdir -p "$OUTDIR/thread"
+set +e
+
+export PATH="$libexec:$PATH"
+
+fmt='%i-%R %16D<%64f>%128S'
+mlist "${MDIR}" | mthread -r | mscan -f "$fmt" | pe | mkindex
blob - 28f05814baa5f9af8a883277bc958b459ee67ba3
blob + 822c400f16c3f7fcc8294aa9405028c48438953c
--- mexp
+++ mexp
use List::Util qw(max min);
-use GotMArc qw(parse san urlencode initpage endpage thread_header
+use SMArc qw(parse san urlencode initpage endpage thread_header
threntry thrslice thrnav);
my $outdir = $ENV{'OUTDIR'};
blob - /dev/null
blob + 4aae3a949ad1d8b9ef98ec06af48af71b0f25ff3 (mode 644)
--- /dev/null
+++ smarc.1
+.\" smarc.1 was written by Omar Polo <op@openbsd.org> and is placed in
+.\" the public domain. The author hereby disclaims copyright to this
+.\" source code.
+.Dd May 5, 2023
+.Dt SMARC 1
+.Os
+.Sh NAME
+.Nm smarc
+.Nd mailing list static site generator
+.Sh SYNOPSIS
+.Nm
+.Op Fl c Ar cachedir
+.Op Fl j Ar n
+.Op Fl m Ar maildir
+.Op Fl o Ar outdir
+.Op Fl t Ar tmpldir
+.Sh DESCRIPTION
+.Nm
+generates a static web archive from a maildir incrementally.
+.Pp
+The arguments are as follows:
+.Bl -tag -width Ds
+.It Fl c Ar cachedir
+Cache directory used for keeping track of threads during incremental
+updates.
+Defaults to
+.Pa ~/.cache/smarc
+.It Fl j Ar n
+Use
+.Ar n
+jobs to export threads.
+With big mailing lists and on a multiprocessor machine this can save
+quite some time.
+.It Fl m Ar maildir
+Source
+.Ar maildir .
+Defaults to
+.Pa ~/Mail/smarc .
+.It Fl o Ar outdir
+The destination directory where the HTML files will be written to, it's
+.Pa /var/www/smarc
+by default.
+.It Fl t Ar tmpldir
+Path to a directory containing the template files.
+.Pa /etc/smarc
+by default.
+.El
+.Sh FILES
+.Bl -tag -width Ds
+.It Pa /etc/smarc/foot.html
+Template with the trailing part of the page.
+.It Pa /etc/smarc/head.html
+Template with the first part of the page.
+.Dv TITLE
+is substituted with the title.
+.It Pa /etc/smarc/index-header.html
+Template with the title for index pages.
+.Dv PAGE
+is replaced with the page number and
+.Dv SUBTITLE
+with the date range.
+.It Pa /etc/smarc/logo-small.html
+Small version of the logo, included in the thread header.
+.It Pa /etc/smarc/search.html
+Template for the search form.
+.Ev QUERY
+is replaced with the search query.
+.It Pa /var/www/smarc
+Default output directory.
+.It Pa ~/.cache/smarc
+Default cache directory.
+.It Pa ~/Mail/smarc
+Default maildir.
+.El
+.Sh SEE ALSO
+.Xr smarc 7
+.Sh HISTORY
+.Nm
+started as a hacky collection of perl scripts under the name of
+.Sq gotmarc
+to generate a web archive for the Game of Trees mailing list.
+Only later, while still being a hacky collection of perl and shell
+scripts, was converted to be a generic web mailing list archive tool and
+the project renamed.
+.Sh AUTHORS
+.An -nosplit
+The
+.Nm
+utility was written by
+.An Omar Polo Aq Mt op@openbsd.org .
blob - 858e62ed2fb4d4c836c282d5edf5cb4b08fb9eb8
blob + 6ab5df9812402e1fb15c313fa441afd83af8e88e
--- mkindex
+++ mkindex
use v5.32;
use File::Temp qw(tempfile);
-use GotMArc qw(parse san urlencode initpage endpage index_header
+use SMArc qw(parse san urlencode initpage endpage index_header
search thread_header threntry);
my $outdir = $ENV{'OUTDIR'};
$page += 1;
my $path;
- ($pfh, $path) = tempfile "/tmp/gotmarc.index.XXXXXXXXXX";
+ ($pfh, $path) = tempfile "/tmp/smarc.index.XXXXXXXXXX";
binmode($pfh, ':utf8');
push @files, $path;
say $pfh "<div class='thread'><ul>";
blob - /dev/null
blob + c7c0a86c093403846ccd4411d99131332e5da4b5 (mode 644)
--- /dev/null
+++ smarc.7
+.\" smarc.7 was written by Omar Polo <op@openbsd.org> and is placed in
+.\" the public domain. The author hereby disclaims copyright to this
+.\" source code.
+.Dd May 5, 2023
+.Dt SMARC 7
+.Os
+.Sh NAME
+.Nm smarc
+.Nd mailing list web archive generation system
+.Sh DESCRIPTION
+.Nm
+is a system to incrementally generate a web archive for a mailing list
+and optionally provide search capabilities.
+The generated archive is a set of HTML files that can be served as-is,
+while searching requires the use of a FastCGI server.
+At a higher level,
+.Nm
+is made of three components:
+.Pp
+.Bl -tag -width msearchd_8_ -compact -offset indent
+.It Xr smarc 1
+to generate the static HTML files from a maildir.
+.It Xr smingest 1
+to import emails into a sqlite3 database.
+.It Xr msearchd 8
+to provide search results.
+.El
+.Sh INITIAL SETUP
+There are several step necessary to initialize the web archive:
+.Pp
+.Bl -enum -compact -offset indent
+.It
+Create and populate the output directory.
+.It
+Customize the templates.
+.It
+Prepare the maildir.
+.It
+Generate the web archive.
+.It
+Set up the database for searching.
+.It
+Configure the web server.
+.El
+.Pp
+It is reccommended to use a dedicate user.
+Commands to be run as a unpriviledged user are preceded by a dollar sign
+.Sq $ ,
+while commands requiring superuser privileges by a hash mark
+.Sq # .
+Hereafter, it will be assumed that the local user is called
+.Sq smarc .
+.Ss 1. Create and populate the output directory
+The web archive is made of several static files, mostly HTML, that needs
+to be served by a web server like
+.Xr httpd 8 .
+.Pa /var/www/smarc
+is the default location, but a different path can be used.
+To prepare it, issue:
+.Bd -literal -offset indent
+# mkdir -p /var/www/smarc
+# chown smarc /var/www/smarc
+.Ed
+.Pp
+Then copy the CSS file, optionally tweaking it.
+.Pp
+.Dl $ cp /usr/local/share/examples/smarc/style.css /var/www/smarc
+.Pp
+Other eventual assets
+.Pq e.g.\& logo images
+need to be copied here as well.
+.Ss 2. Customize the templates
+The default templates are installed at
+.Pa /etc/smarc .
+Since these are anonymous, they need to be tweaked to include
+information about the mailing list.
+.Pp
+Care should be taken when editing these files after generating the
+archive since existing pages won't be automatically updated.
+The cachedir
+.Pq see Xr smarc 1
+needs to be deleted and the web archive generated again.
+.Xr msearchd 8
+has to be stopped and restarted as well.
+.Ss 3. Prepare the maildir
+The maildir with the mailing list entries needs to be prepared.
+It is assumed to be at
+.Pa ~/Mail/smarc
+by default, but a different path can be used.
+.Ss 4. Generate the web archive
+.Xr smarc 1
+can be finally used to generate the web archive.
+The first run may take a while, depending on the size of the maildir,
+while subsequent runs will be incremental and take less time.
+.Pp
+.Dl $ smarc -m path/to/maildir -o path/to/outdir
+.Pp
+On multi-processor machines multiple processes may be used to save some
+time with the
+.Xr smarc 1 Fl j No flag.
+.Pp
+The generated files may be compressed to save bandwidth:
+.Pp
+.Dl $ gzip -krq /var/www/smarc </dev/null 2>/dev/null
+.Ss 5. Set up the database for searching
+This is an suggested yet optional step.
+.Pp
+.Xr msearchd 8
+offers full text search capabilities using a sqlite3 database that has to
+be populated with
+.Xr smingest 1 .
+First, create a directory in the
+.Pa /var/www
+.Xr chroot 8
+jail:
+.Bd -literal -offset indent
+# mkdir -p /var/www/msearchd
+# chown smarc /var/www/msearchd
+.Ed
+.Pp
+Then, populate the database with all emails in the maildir:
+.Bd -literal -offset indent
+$ sqlite3 /var/www/msearchd/mails.sqlite3 \e
+ </usr/local/share/examples/smarc/schema.sql
+$ mlist ~/Mail/smarc | smingest /var/www/msearchd/mails.sqlite3
+.Ed
+.Pp
+At this point,
+.Xr msearchd 8
+can be started.
+.Ss 6. Configure the web server
+The web server needs to serve the contents of the outdir as-is and
+handle the requests for
+.Pa /search
+via the
+.Xr msearchd 8
+FastCGI server.
+A sample
+.Xr httpd 8
+configuration is provided here for reference:
+.Bd -literal -offset indent
+server "marc.example.com" {
+ listen on * port 80
+ root "/smarc"
+ gzip-static
+
+ # leave out when not using msearchd(8)
+ location "/search" {
+ fastcgi socket "/run/msearchd.sock"
+ }
+}
+.Ed
+.Sh HANDLING NEW MESSAGES
+New messages should be fetched periodically using tools like
+.Xr fdm 1
+or
+.Xr mbsync 1 ,
+the database updated with
+.Xr smingest 1
+and the web archive refreshed using
+.Xr smarc 1 .
+.Pp
+It is reccommended to create a script like the following and schedule
+its execution periodically with
+.Xr cron 8 :
+.Bd -literal -offset indent
+#!/bin/sh
+
+set -e
+fdm -l fetch
+minc ~/Mail/smarc | smingest /var/www/msearchd/mails.sqlite3
+smarc
+gzip -krq /var/www/smarc/ </dev/null 2>/dev/null || true
+.Ed
+.Pp
+If
+.Xr msearchd 8
+is not used,
+new messages still needs to be incorporated
+.Po i.e.\& moved from
+.Pa new/
+to
+.Pa cur/
+.Pc
+but no database has to be updated.
+In that case simplify the
+.Xr minc 1
+invocation as:
+.Pp
+.Dl minc -q ~/Mail/smarc
+.Pp
+and don't call
+.Xr smingest 1
+at all.
+.Sh HANDLING MULTIPLE MAILING LISTS
+If the archive for multiple mailing lists needs to be served from the
+same box, care must be taken to use different directories and database
+files to avoid mixing messages.
+.Pp
+.Xr msearchd 8
+handles only one database at a time, so multiple instances need to be
+run, each pointing at the database for only one mailing list.
+Different FastCGI socket path needs to be used per-instance.
+.Pp
+.Xr smarc 1
+outdir, maildir and cachedir must be unique per-mailing list, i.e.\& the
+.Fl c , Fl m No and Fl o
+flag must always be provided.
+.Pp
+Very likely, each mailing list will needs its own set of templates, so
+those needs to be prepared and both
+.Xr smarc 1
+and
+.Xr msearchd 8
+have to be pointed at the right template directory.
+.Sh SEE ALSO
+.Xr minc 1 ,
+.Xr smarc 1 ,
+.Xr smingest 1 ,
+.Xr sqlite3 1 ,
+.Xr httpd 8 ,
+.Xr msearchd 8
blob - ad72cb88fb4d2dffcbaae2c49e06b5c57106888a
blob + 5695726c77a6a9a2fb8ce8a473e32e9d6b5b7ce5
--- msearchd/Makefile
+++ msearchd/Makefile
${INSTALL_MAN} msearchd.8 ${DESTDIR}${MANDIR}/man8
mkdir -p ${DESTDIR}${SBINDIR}
${INSTALL_PROGRAM} ${PROG} ${DESTDIR}${SBINDIR}
- mkdir -p ${DESTDIR}${SYSCONFDIR}/gotmarc
- ${INSTALL_DATA} schema.sql ${DESTDIR}${SYSCONFDIR}/gotmarc
+ mkdir -p ${DESTDIR}${SYSCONFDIR}/smarc
+ ${INSTALL_DATA} schema.sql ${DESTDIR}${SYSCONFDIR}/smarc
uninstall:
rm -f ${DESTDIR}${MANDIR}/man8/msearchd.8
rm -f ${DESTDIR}${SBINDIR}/${PROG}
- rm -f ${DESTDIR}${SYSCONFDIR}/gotmarc/schema.sql
+ rm -f ${DESTDIR}${SYSCONFDIR}/smarc/schema.sql
# -- internal build targets --
blob - 3b89a8f66f92de1b6a2ad17bb0774c2f83e748a3
blob + 5e32b4eff044d7f1e3b5c054e03156380848122e
--- msearchd/msearchd.8
+++ msearchd/msearchd.8
.Ar socket .
.It Fl t Ar tmpldir
Path to a directory containing the template files.
-.Pa /etc/gotmarc
+.Pa /etc/smarc
by default.
.It Fl u Ar user
Drop privileges to
.El
.Sh FILES
.Bl -tag -width Ds
-.It Pa /etc/gotmarc/foot.html
+.It Pa /etc/smarc/foot.html
Template with the trailing part of the page.
-.It Pa /etc/gotmarc/head.html
+.It Pa /etc/smarc/head.html
Template with the first part of the page.
.Dv TITLE
is replaced with
.Dq Search .
-.It Pa /etc/gotmarc/search-header.html
+.It Pa /etc/smarc/search-header.html
Template for the start of the search page.
-.It Pa /etc/gotmarc/search.html
+.It Pa /etc/smarc/search.html
Template for the search form.
.Dv QUERY
is replaced with the search query.
.Bd -literal -offset -indent
server "localhost" {
listen on * port 80
- root "/gotmarc"
+ root "/smarc"
gzip-static
location "/search" {
blob - d94aee1927187a7b25c4952e9db01bc2fcd2fb0c
blob + 9ee7a8f3f1d361a0be1c5776f5ebae5f4d567045
--- msearchd/msearchd.c
+++ msearchd/msearchd.c
#endif
#ifndef MSEARCH_TMPL_DIR
-#define MSEARCH_TMPL_DIR SYSCONFDIR "/gotmarc"
+#define MSEARCH_TMPL_DIR SYSCONFDIR "/smarc"
#endif
#define MAX_CHILDREN 32
blob - /dev/null
blob + ec998e658ae70f4563baf20f9c63779585bf1529 (mode 755)
--- /dev/null
+++ smingest
+#!/usr/bin/env perl
+#
+# smingest was written by Omar Polo <op@openbsd.org> and is placed in the
+# public domain. The author hereby disclaims copyright to this source
+# code.
+
+use strict;
+use warnings;
+use v5.32;
+use utf8;
+
+use Date::Parse;
+use File::Basename;
+
+use OpenBSD::Pledge;
+use OpenBSD::Unveil;
+
+die "usage: $0 dbpath\n" if @ARGV != 1;
+my $dbpath = shift @ARGV;
+
+open(my $sqlite, "|-", "/usr/local/bin/sqlite3", $dbpath)
+ or die "can't spawn sqlite3";
+
+unveil("/usr/local/bin/mshow", "rx") or die "unveil mshow: $!";
+pledge("stdio proc exec") or die "pledge: $!";
+
+say $sqlite ".import --csv /dev/stdin email"
+ or die "can't speak to sqlite: $!";
+
+while (<>) {
+ chomp;
+
+ open(my $fh, "-|", "/usr/local/bin/mshow", "-Atext/plain", "-NF", $_)
+ or die "can't run mshow $_: $!";
+
+ my $f = $_;
+ my ($time, $id) = split /\./, basename $_;
+ my $mid = "$time.$id";
+ $mid =~ s/"/""/g;
+
+ my ($from, $subj, $date) = ('', '', undef);
+ while (<$fh>) {
+ chomp;
+ last if /^$/;
+ s/"/""/g;
+ $from = s/.*?: //r if /^From:/;
+ $subj = s/.*?: //r if /^Subject:/;
+ $date = str2time(s/.*?: //r) if /^Date:/;
+ }
+ $date //= time;
+ $from =~ s/ +<.*>//;
+
+ # leave open for the body
+ print $sqlite "\"$mid\",\"$from\",\"$date\",\"$subj\",\"";
+
+ while (<$fh>) {
+ s/"/""/g;
+ print $sqlite $_;
+ }
+ say $sqlite '"';
+
+ close $fh;
+}
+
+close $sqlite;
+die "sqlite3 exited with $?\n" unless $? == 0;
blob - 9f73857870aa589f38cc4e728d90e5a1069ab776
blob + 01aafeb485501929cc6df81fb550cf399b14265d
--- pe
+++ pe
use Encode qw(encode);
use IO::Poll qw(POLLOUT);
-use GotMArc qw(parse);
+use SMArc qw(parse);
my $jobs = $ENV{'MAKE_JOBS'} // 1;
blob - /dev/null
blob + a2124ad27da92e9fbd048e16a71e5607d3ef2452 (mode 644)
--- /dev/null
+++ smingest.1
+.\" smingest.1 was written by Omar Polo <op@openbsd.org> and is placed in
+.\" the public domain. The author hereby disclaims copyright to this
+.\" source code.
+.Dd May 5, 2023
+.Dt SMINGEST 1
+.Os
+.Sh NAME
+.Nm smingest
+.Nd import emails into a sqlite database
+.Sh SYNOPSIS
+.Nm
+.Ar dbpath
+.Sh DESCRIPTION
+.Nm
+reads a sequence of path to emails from standard inputs and inserts them
+into the
+.Xr msearchd 8
+sqlite3 database at
+.Ar dbpath .
+.Sh EXAMPLES
+To index all the messages in the
+.Pa ~/Mail/smarc
+maildir, useful to initially populate the database:
+.Pp
+.Dl mlist ~/Mail/smarc | smingest /var/www/msearchd/mails.sqlite3
+.Pp
+Incorporate new messages in the maildir and add them to the database,
+useful after fetching new mails:
+.Pp
+.Dl minc ~/Mail/smarc | smingest /var/www/msearchd/mails.sqlite3
+.Sh SEE ALSO
+.Xr minc 1 ,
+.Xr mlist 1 ,
+.Xr msearchd 8
+.Sh BUGS
+.Nm
+doesn't detect the insert of duplicate messages.
+Should that happen, the database needs to be recreated.
blob - dd0133354b29cf2aa0c1067312aa3b0c3563151d
blob + 030d1a875da5e5a062087df49cc93267531045d5
--- templates/Makefile
+++ templates/Makefile
${INSTALL} -m 0644 ${DISTFILES} ${DESTDIR}/
install:
- mkdir -p ${DESTDIR}${SYSCONFDIR}/gotmarc/
- ${INSTALL_DATA} *.html ${DESTDIR}${SYSCONFDIR}/gotmarc/
+ mkdir -p ${DESTDIR}${SYSCONFDIR}/smarc/
+ ${INSTALL_DATA} *.html ${DESTDIR}${SYSCONFDIR}/smarc/
uninstall:
- rm -f ${DESTDIR}${SYSCONFDIR}/gotmarc/${DISTFILES}
+ rm -f ${DESTDIR}${SYSCONFDIR}/smarc/${DISTFILES}
.PHONY: all dist install uninstall
