commit 38232a0aceee26098c65f025b64dbdad5ebeee9b from: Omar Polo date: Sun May 07 07:22:27 2023 UTC rename project: gotmarc -> smarc commit - f4262cd049b12d1ddebb100c2ab8aca3983e4b45 commit + 38232a0aceee26098c65f025b64dbdad5ebeee9b blob - 6d5e7ffb83940f3518c98bd667a2a4da4db3012f blob + e385e15b393c9867b61ccf028d9c76a567e7172f --- .mblaze/Makefile +++ .mblaze/Makefile @@ -8,12 +8,12 @@ dist: ${INSTALL} -m 0644 ${DISTFILES} ${DESTDIR}/ install: - mkdir -p ${DESTDIR}${SHAREDIR}/gotmarc/mblaze - ${INSTALL_DATA} filter seq ${DESTDIR}${SHAREDIR}/gotmarc/mblaze + mkdir -p ${DESTDIR}${SHAREDIR}/smarc/mblaze + ${INSTALL_DATA} filter seq ${DESTDIR}${SHAREDIR}/smarc/mblaze uninstall: - rm -f ${DESTDIR}${SHAREDIR}/gotmarc/mblaze/filter - rm -f ${DESTDIR}${SHAREDIR}/gotmarc/mblaze/seq + rm -f ${DESTDIR}${SHAREDIR}/smarc/mblaze/filter + rm -f ${DESTDIR}${SHAREDIR}/smarc/mblaze/seq .PHONY: all dist install uninstall include ../config.mk blob - 9289bc39e41cda915f55fe1ec281b329cfc60a2c (mode 644) blob + /dev/null --- GotMArc.pm +++ /dev/null @@ -1,224 +0,0 @@ -# GotMArc was written by Omar Polo and is placed in -# the public domain. The author hereby disclaims copyright to this -# source code. - -package GotMArc; -use strict; -use warnings; -use v5.32; -use Exporter; -use File::Basename; - -our @ISA = qw(Exporter); -our @EXPORT_OK = qw(san urlencode parse initpage endpage index_header - search thread_header threntry thrslice thrnav); - -sub san { - my $str = shift; - $str =~ s/&/\&/g; - $str =~ s//\>/g; - return $str; -} - -sub urlencode { - my $str = shift; - unless (defined($str)) { - my ($pkg, $file, $line) = caller 1; - die "bad $pkg / $file:$line"; - } - $str =~ s/([^-_~.A-Za-z0-9])/sprintf("%%%2X", ord($1))/ge; - return $str; -} - -sub ssan { - my $str = shift; - $str =~ s/\s+/ /g; - $str =~ s/\s+$//; - return san($str); -} - -sub parse { - my $line = shift; - my ($indent, $fname, $date, $from, $subj) = $line =~ m{ - ^([^-]*)- # the indent level - ([^ ]+)\s # filename - (\d{4}-\d\d-\d\d[ ]\d\d:\d\d) # date - <([^>]+)> # from - (.*) # subject - }x or die "can't parse: $line"; - - my $level = length($indent); - $level = 10 if $indent =~ m/\.\.\d+\.\./; - - $from = ssan($from); - $subj = ssan($subj); - - my ($time, $id) = split /\./, basename($fname); - my $mid = "$time.$id"; - - return {level => $level, fname => $fname, - mid => $mid, date => $date, from => $from, subj => $subj}; -} - -sub readall { - my $path = shift; - local $/ = undef; - open my $fh, "<", $path or die "can't open $path: $!"; - <$fh>; -} - -my $templates = $ENV{TMPLDIR}; -die 'undefined $TMPLDIR' unless defined $templates; - -my $small_logo = readall "$templates/logo-small.html"; -my $hdr = readall "$templates/head.html"; -my $foot = readall "$templates/foot.html"; -my $idxhdr = readall "$templates/index-header.html"; -my $search = readall "$templates/search.html"; - -sub initpage { - my ($fh, $title) = @_; - say $fh $hdr =~ s/TITLE/$title/r; -} - -sub endpage { - my $fh = shift; - say $fh $foot; -} - -sub index_header { - my ($fh, $page, $subtitle) = @_; - my $html = $idxhdr =~ s/PAGE/$page/r; - $html =~ s/SUBTITLE/$subtitle/; - print $fh $html; -} - -sub search { - my $fh = shift; - my $html = $search =~ s/QUERY//r; - print $fh $html; -} - -sub thread_header { - my ($fh, $e, $mail, $p, $n) = @_; - - my @entries = @$e; - - my $enctid = urlencode $mail->{tid} if defined $mail; - my $encmid = urlencode $mail->{mid} if defined $mail; - - print $fh "
\n"; - - print $fh "

"; - print $fh $small_logo; - print $fh "Index"; - print $fh " | Thread" - if defined $enctid; - print $fh " | Search"; - print $fh "

\n"; - - say $fh "
"; - foreach my $entry (@entries) { - my ($k, $v) = split /: /, $entry, 2; - chomp $v; - say $fh "
$k:
$v
"; - } - say $fh "
"; - - say $fh "

Download raw body.

" - if defined $encmid; - - if (defined($p) and defined($n)) { - say $fh "
"; - say $fh "Thread"; - thrslice($fh, $mail, $p, $n); - say $fh "
"; - thrnav($fh, $p, $n, $mail->{mid}, $mail->{tid}); - } - - say $fh "
\n"; -} - -sub threntry { - my ($fh, $type, $base, $last_level, $mail, $cur) = @_; - my $level = $mail->{level} - $base; - - say $fh "" x ($last_level - $level) if $last_level > $level; - say $fh "
    • " if $last_level < $level; - - my $encmid = urlencode $mail->{mid}; - - print $fh "
    • "; - print $fh "

      "; - print $fh " "; - print $fh "$mail->{from}"; - print $fh ":"; - print $fh "

      "; - print $fh "

      "; - - my $subj = $mail->{subj}; - if (!defined($cur) || $mail->{mid} ne $cur->{mid}) { - print $fh "$subj"; - } else { - print $fh "$subj"; - } - - print $fh "

      "; - print $fh "
      • "; - - return $level; -} - -sub min_level { - my $l = 999; - return 0 unless @_; - for (@_) { - $l = $_->{level} if $_->{level} < $l; - } - return $l; -} - -sub thrslice { - my ($fh, $mail, $p, $n) = @_; - my @prev = @{$p}; - my @next = @{$n}; - my @thread = (@prev, $mail, @next); - return unless @thread; - my $base = min_level @thread; - my $level = 0; - print $fh "
      "; - print $fh "
        "; - $level = threntry $fh, "mail", $base, $level, $_, $mail for @thread; - print $fh "
      " x $level; - print $fh "
    "; -} - -sub thrnav { - my ($fh, $p, $n) = @_; - my @prev = @{$p}; - my @next = @{$n}; - - return if !@prev && !@next; - print $fh ""; -} - -1; blob - 97faafb39a2cb90cba73863cc62058316deaad49 blob + 30f702b9f760ccca07311ea189206eeefeb3b003 --- Makefile +++ Makefile @@ -3,7 +3,7 @@ include config.mk # -- build-related variables -- VERSION = 0.1 -DISTNAME = gotmarc-${VERSION} +DISTNAME = smarc-${VERSION} # -- public targets -- @@ -25,42 +25,42 @@ distclean: clean install: mkdir -p ${DESTDIR}${BINDIR} - ${INSTALL_PROGRAM} gmimport ${DESTDIR}${BINDIR} - sed -e "/^libexec=/s@=.*@=${LIBEXEC}/gotmarc@" \ - -e "/^mblaze=/s@=.*@=${SHAREDIR}/gotmarc/mblaze@" \ - -e "/^tmpldir=/s@=.*@=${REALSYSCONFDIR}/gotmarc@" \ - gotmarc > ${DESTDIR}${BINDIR}/gotmarc - chmod 0755 ${DESTDIR}${BINDIR}/gotmarc - mkdir -p ${DESTDIR}${LIBEXEC}/gotmarc - ${INSTALL_PROGRAM} filter-ignore ${DESTDIR}${LIBEXEC}/gotmarc - ${INSTALL_PROGRAM} mexp ${DESTDIR}${LIBEXEC}/gotmarc - ${INSTALL_PROGRAM} mkindex ${DESTDIR}${LIBEXEC}/gotmarc - ${INSTALL_PROGRAM} pe ${DESTDIR}${LIBEXEC}/gotmarc - mkdir -p ${DESTDIR}${SYSCONFDIR}/gotmarc - ${INSTALL_DATA} style.css ${DESTDIR}${SYSCONFDIR}/gotmarc/ + ${INSTALL_PROGRAM} smingest ${DESTDIR}${BINDIR} + sed -e "/^libexec=/s@=.*@=${LIBEXEC}/smarc@" \ + -e "/^mblaze=/s@=.*@=${SHAREDIR}/smarc/mblaze@" \ + -e "/^tmpldir=/s@=.*@=${REALSYSCONFDIR}/smarc@" \ + smarc > ${DESTDIR}${BINDIR}/smarc + chmod 0755 ${DESTDIR}${BINDIR}/smarc + mkdir -p ${DESTDIR}${LIBEXEC}/smarc + ${INSTALL_PROGRAM} filter-ignore ${DESTDIR}${LIBEXEC}/smarc + ${INSTALL_PROGRAM} mexp ${DESTDIR}${LIBEXEC}/smarc + ${INSTALL_PROGRAM} mkindex ${DESTDIR}${LIBEXEC}/smarc + ${INSTALL_PROGRAM} pe ${DESTDIR}${LIBEXEC}/smarc + mkdir -p ${DESTDIR}${SYSCONFDIR}/smarc + ${INSTALL_DATA} style.css ${DESTDIR}${SYSCONFDIR}/smarc/ mkdir -p ${DESTDIR}${PERL_LIB} - ${INSTALL_DATA} GotMArc.pm ${DESTDIR}${PERL_LIB} + ${INSTALL_DATA} SMArc.pm ${DESTDIR}${PERL_LIB} mkdir -p ${DESTDIR}${MANDIR}/man1 - ${INSTALL_MAN} gmimport.1 ${DESTDIR}${MANDIR}/man1/ - ${INSTALL_MAN} gotmarc.1 ${DESTDIR}${MANDIR}/man1/ + ${INSTALL_MAN} smingest.1 ${DESTDIR}${MANDIR}/man1/ + ${INSTALL_MAN} smarc.1 ${DESTDIR}${MANDIR}/man1/ mkdir -p ${DESTDIR}${MANDIR}/man7 - ${INSTALL_MAN} gotmarc.7 ${DESTDIR}${MANDIR}/man7/ + ${INSTALL_MAN} smarc.7 ${DESTDIR}${MANDIR}/man7/ ${MAKE} -C .mblaze install ${MAKE} -C msearchd install ${MAKE} -C templates install uninstall: - rm -f ${DESTDIR}${BINDIR}/gmimport - rm -f ${DESTDIR}${BINDIR}/gotmarc - rm -f ${DESTDIR}${LIBEXEC}/gotmarc/filter-ignore - rm -f ${DESTDIR}${LIBEXEC}/gotmarc/mexp - rm -f ${DESTDIR}${LIBEXEC}/gotmarc/mkindex - rm -f ${DESTDIR}${LIBEXEC}/gotmarc/pe - rm -f ${DESTDIR}${SYSCONFDIR}/gotmarc/style.css - rm -f ${DESTDIR}${PERL_LIB}/GotMArc.pm - rm -f ${DESTDIR}${MANDIR}/man1/gmimport.1 - rm -f ${DESTDIR}${MANDIR}/man1/gotmarc.1 - rm -f ${DESTDIR}${MANDIR}/man7/gotmarc.7 + rm -f ${DESTDIR}${BINDIR}/smingest + rm -f ${DESTDIR}${BINDIR}/smarc + rm -f ${DESTDIR}${LIBEXEC}/smarc/filter-ignore + rm -f ${DESTDIR}${LIBEXEC}/smarc/mexp + rm -f ${DESTDIR}${LIBEXEC}/smarc/mkindex + rm -f ${DESTDIR}${LIBEXEC}/smarc/pe + rm -f ${DESTDIR}${SYSCONFDIR}/smarc/style.css + rm -f ${DESTDIR}${PERL_LIB}/SMArc.pm + rm -f ${DESTDIR}${MANDIR}/man1/smingest.1 + rm -f ${DESTDIR}${MANDIR}/man1/smarc.1 + rm -f ${DESTDIR}${MANDIR}/man7/smarc.7 ${MAKE} -C .mblaze uninstall ${MAKE} -C msearchd uninstall ${MAKE} -C templates uninstall @@ -69,18 +69,18 @@ uninstall: PRIVKEY = missing-PRIVKEY PUBKEY = missing-PUBKEY -DISTFILES = GotMArc.pm Makefile README TODO configure \ - filter-ignore gmimport gmimport.1 gotmarc gotmarc.1 \ - gotmarc.7 mexp mkindex pe style.css +DISTFILES = SMArc.pm Makefile README TODO configure \ + filter-ignore smingest smingest.1 smarc smarc.1 \ + smarc.7 mexp mkindex pe style.css MANOPTS = man='%N.%S.html;https://man.openbsd.org/%N.%S',style=mandoc.css,toc MANFLAGS = -Thtml -O${MANOPTS} man: touch msearchd.8 - man ${MANFLAGS} -l gmimport.1 > gmimport.1.html - man ${MANFLAGS} -l gotmarc.1 > gotmarc.1.html - man ${MANFLAGS} -l gotmarc.7 > gotmarc.7.html + man ${MANFLAGS} -l smingest.1 > smingest.1.html + man ${MANFLAGS} -l smarc.1 > smarc.1.html + man ${MANFLAGS} -l smarc.7 > smarc.7.html man ${MANFLAGS} -l msearchd/msearchd.8 > msearchd.8.html rm msearchd.8 @@ -101,7 +101,7 @@ ${DISTNAME}.tar.gz: ${DISTFILES} mkdir -p .dist/${DISTNAME} ${INSTALL} -m 0644 ${DISTFILES} .dist/${DISTNAME} cd .dist/${DISTNAME} && chmod 0755 configure filter-ignore \ - gmimport gotmarc mexp mkindex pe + smingest smarc mexp mkindex pe ${MAKE} -C .mblaze DESTDIR=${PWD}/.dist/${DISTNAME}/.mblaze dist ${MAKE} -C templates DESTDIR=${PWD}/.dist/${DISTNAME}/templates dist ${MAKE} -C msearchd DESTDIR=${PWD}/.dist/${DISTNAME}/msearchd dist blob - 389da3009756ace78ee9146f0159713cb7e5c0e2 blob + 9a19ae9aaff787637dc7f2309fb2fada2eb4f070 --- README +++ README @@ -1,5 +1,5 @@ -GOTMARC -- A static mailing list web archive generator -====================================================== +SMARC -- static web mail archive generator +========================================== A collection of scripts to generate a static mail archive from a Maildir; originally intended to provide a public web interface for the @@ -13,18 +13,18 @@ component on libevent and sqlite3. How it works ------------ -gotmarc(1) incrementally generate a set of static files from a maildir, -gmimport(1) and msearchd(8) are used to populate an sqlite3 database to +smarc(1) incrementally generate a set of static files from a maildir, +smingest(1) and msearchd(8) are used to populate an sqlite3 database to provide a web-based full text search on all the mails. -gotmarc(7) documents the step needed for a first setup, how to operate +smarc(7) documents the step needed for a first setup, how to operate it and tips for generating multiple archives. License ------- -gotmarc, gmimport and msearchd are released in the public domain. The +smarc, smingest and msearchd are released in the public domain. The configure script loosely based on oconfigure and is released under the terms of ISC license. Some OpenBSD-compats are included to build on multiple platforms, these are released under a BSD-style license. blob - /dev/null blob + 9289bc39e41cda915f55fe1ec281b329cfc60a2c (mode 644) --- /dev/null +++ SMArc.pm @@ -0,0 +1,224 @@ +# GotMArc was written by Omar Polo and is placed in +# the public domain. The author hereby disclaims copyright to this +# source code. + +package GotMArc; +use strict; +use warnings; +use v5.32; +use Exporter; +use File::Basename; + +our @ISA = qw(Exporter); +our @EXPORT_OK = qw(san urlencode parse initpage endpage index_header + search thread_header threntry thrslice thrnav); + +sub san { + my $str = shift; + $str =~ s/&/\&/g; + $str =~ s//\>/g; + return $str; +} + +sub urlencode { + my $str = shift; + unless (defined($str)) { + my ($pkg, $file, $line) = caller 1; + die "bad $pkg / $file:$line"; + } + $str =~ s/([^-_~.A-Za-z0-9])/sprintf("%%%2X", ord($1))/ge; + return $str; +} + +sub ssan { + my $str = shift; + $str =~ s/\s+/ /g; + $str =~ s/\s+$//; + return san($str); +} + +sub parse { + my $line = shift; + my ($indent, $fname, $date, $from, $subj) = $line =~ m{ + ^([^-]*)- # the indent level + ([^ ]+)\s # filename + (\d{4}-\d\d-\d\d[ ]\d\d:\d\d) # date + <([^>]+)> # from + (.*) # subject + }x or die "can't parse: $line"; + + my $level = length($indent); + $level = 10 if $indent =~ m/\.\.\d+\.\./; + + $from = ssan($from); + $subj = ssan($subj); + + my ($time, $id) = split /\./, basename($fname); + my $mid = "$time.$id"; + + return {level => $level, fname => $fname, + mid => $mid, date => $date, from => $from, subj => $subj}; +} + +sub readall { + my $path = shift; + local $/ = undef; + open my $fh, "<", $path or die "can't open $path: $!"; + <$fh>; +} + +my $templates = $ENV{TMPLDIR}; +die 'undefined $TMPLDIR' unless defined $templates; + +my $small_logo = readall "$templates/logo-small.html"; +my $hdr = readall "$templates/head.html"; +my $foot = readall "$templates/foot.html"; +my $idxhdr = readall "$templates/index-header.html"; +my $search = readall "$templates/search.html"; + +sub initpage { + my ($fh, $title) = @_; + say $fh $hdr =~ s/TITLE/$title/r; +} + +sub endpage { + my $fh = shift; + say $fh $foot; +} + +sub index_header { + my ($fh, $page, $subtitle) = @_; + my $html = $idxhdr =~ s/PAGE/$page/r; + $html =~ s/SUBTITLE/$subtitle/; + print $fh $html; +} + +sub search { + my $fh = shift; + my $html = $search =~ s/QUERY//r; + print $fh $html; +} + +sub thread_header { + my ($fh, $e, $mail, $p, $n) = @_; + + my @entries = @$e; + + my $enctid = urlencode $mail->{tid} if defined $mail; + my $encmid = urlencode $mail->{mid} if defined $mail; + + print $fh "
    \n"; + + print $fh "

    "; + print $fh $small_logo; + print $fh "Index"; + print $fh " | Thread" + if defined $enctid; + print $fh " | Search"; + print $fh "

    \n"; + + say $fh "
    "; + foreach my $entry (@entries) { + my ($k, $v) = split /: /, $entry, 2; + chomp $v; + say $fh "
    $k:
    $v
    "; + } + say $fh "
    "; + + say $fh "

    Download raw body.

    " + if defined $encmid; + + if (defined($p) and defined($n)) { + say $fh "
    "; + say $fh "Thread"; + thrslice($fh, $mail, $p, $n); + say $fh "
    "; + thrnav($fh, $p, $n, $mail->{mid}, $mail->{tid}); + } + + say $fh "
    \n"; +} + +sub threntry { + my ($fh, $type, $base, $last_level, $mail, $cur) = @_; + my $level = $mail->{level} - $base; + + say $fh "
    • " x ($last_level - $level) if $last_level > $level; + say $fh "
    • " if $last_level < $level; + + my $encmid = urlencode $mail->{mid}; + + print $fh "
    • "; + print $fh "

      "; + print $fh " "; + print $fh "$mail->{from}"; + print $fh ":"; + print $fh "

      "; + print $fh "

      "; + + my $subj = $mail->{subj}; + if (!defined($cur) || $mail->{mid} ne $cur->{mid}) { + print $fh "$subj"; + } else { + print $fh "$subj"; + } + + print $fh "

      "; + print $fh "
      • "; + + return $level; +} + +sub min_level { + my $l = 999; + return 0 unless @_; + for (@_) { + $l = $_->{level} if $_->{level} < $l; + } + return $l; +} + +sub thrslice { + my ($fh, $mail, $p, $n) = @_; + my @prev = @{$p}; + my @next = @{$n}; + my @thread = (@prev, $mail, @next); + return unless @thread; + my $base = min_level @thread; + my $level = 0; + print $fh "
      "; + print $fh "
        "; + $level = threntry $fh, "mail", $base, $level, $_, $mail for @thread; + print $fh "
      " x $level; + print $fh "
    "; +} + +sub thrnav { + my ($fh, $p, $n) = @_; + my @prev = @{$p}; + my @next = @{$n}; + + return if !@prev && !@next; + print $fh ""; +} + +1; blob - 397e33518ab904ab8aa7c556c08fa3966444f7a3 blob + 18c85084e04090f3f0fc6dc0fec76f804e026dfe --- configure +++ configure @@ -97,7 +97,7 @@ echo >&2 echo "running configure for \`msearchd':" >&2 (cd msearchd && ./configure) echo >&2 -echo "returning to the configure for \`gotmarc':" >&2 +echo "returning to the configure for \`smarc':" >&2 exec > config.mk echo "config.mk: writing...">&2 blob - 3ea6af4dc11e261dec6f65101a63beb58f2664a7 (mode 755) blob + /dev/null --- gmimport +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env perl -# -# gmimport was written by Omar Polo and is placed in the -# public domain. The author hereby disclaims copyright to this source -# code. - -use strict; -use warnings; -use v5.32; -use utf8; - -use Date::Parse; -use File::Basename; - -use OpenBSD::Pledge; -use OpenBSD::Unveil; - -die "usage: $0 dbpath\n" if @ARGV != 1; -my $dbpath = shift @ARGV; - -open(my $sqlite, "|-", "/usr/local/bin/sqlite3", $dbpath) - or die "can't spawn sqlite3"; - -unveil("/usr/local/bin/mshow", "rx") or die "unveil mshow: $!"; -pledge("stdio proc exec") or die "pledge: $!"; - -say $sqlite ".import --csv /dev/stdin email" - or die "can't speak to sqlite: $!"; - -while (<>) { - chomp; - - open(my $fh, "-|", "/usr/local/bin/mshow", "-Atext/plain", "-NF", $_) - or die "can't run mshow $_: $!"; - - my $f = $_; - my ($time, $id) = split /\./, basename $_; - my $mid = "$time.$id"; - $mid =~ s/"/""/g; - - my ($from, $subj, $date) = ('', '', undef); - while (<$fh>) { - chomp; - last if /^$/; - s/"/""/g; - $from = s/.*?: //r if /^From:/; - $subj = s/.*?: //r if /^Subject:/; - $date = str2time(s/.*?: //r) if /^Date:/; - } - $date //= time; - $from =~ s/ +<.*>//; - - # leave open for the body - print $sqlite "\"$mid\",\"$from\",\"$date\",\"$subj\",\""; - - while (<$fh>) { - s/"/""/g; - print $sqlite $_; - } - say $sqlite '"'; - - close $fh; -} - -close $sqlite; -die "sqlite3 exited with $?\n" unless $? == 0; blob - 331662d4de04a2cc038c914e66bf09f0420b1e53 (mode 644) blob + /dev/null --- gmimport.1 +++ /dev/null @@ -1,38 +0,0 @@ -.\" gmimport.1 was written by Omar Polo and is placed in -.\" the public domain. The author hereby disclaims copyright to this -.\" source code. -.Dd May 5, 2023 -.Dt GMIMPORT 1 -.Os -.Sh NAME -.Nm gmimport -.Nd import emails into a sqlite database -.Sh SYNOPSIS -.Nm -.Ar dbpath -.Sh DESCRIPTION -.Nm -reads a sequence of path to emails from standard inputs and inserts them -into the -.Xr msearchd 8 -sqlite3 database at -.Ar dbpath . -.Sh EXAMPLES -To index all the messages in the -.Pa ~/Mail/gotmarc -maildir, useful to initially populate the database: -.Pp -.Dl mlist ~/Mail/gotmarc | gmimport /var/www/msearchd/mails.sqlite3 -.Pp -Incorporate new messages in the maildir and add them to the database, -useful after fetching new mails: -.Pp -.Dl minc ~/Mail/gotmarc | gmimport /var/www/msearchd/mails.sqlite3 -.Sh SEE ALSO -.Xr minc 1 , -.Xr mlist 1 , -.Xr msearchd 8 -.Sh BUGS -.Nm -doesn't detect the insert of duplicate messages. -Should that happen, the database needs to be recreated. blob - 035828b31af7891b28eeca6413e6f32e0c8d8c8e (mode 755) blob + /dev/null --- gotmarc +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/sh -# -# gotmarc was written by Omar Polo and is placed in the -# public domain. The author hereby disclaims copyright to this source -# code. - -progname="$(basename "$0")" -usage() { - u="usage: $progname [-c cachedir] [-j n]" - u="$u [-m maildir] [-o outdir] [-t tmpldir]" - echo "$u">&2 - exit 1 -} - -# changed at install-time -libexec=. -mblaze=.mblaze -tmpldir=templates/ - -# fix perl include path for in-repo hacking -if [ "$libexec" = . ]; then - export PERL5LIB="$PWD${PERL5LIB:+:}$PERL5LIB" -fi - -cachedir=$HOME/.cache/gotmarc -mdir=$HOME/Mail/gotmarc -outdir=/var/www/gotmarc - -while getopts c:j:m:o:t: flag; do - case $flag in - c) cachedir="$OPTARG" ;; - j) MAKE_JOBS="$OPTARG" ;; - m) mdir="$OPTARG" ;; - o) outdir="$OPTARG" ;; - t) tmpldir="$OPTARG" ;; - ?) usage ;; - esac -done - -# set up the env -export CSUMDIR="$cachedir/threadsum" -export MAKE_JOBS="${MAKE_JOBS:-1}" -export MBLAZE="$mblaze" -export MBLAZE_PAGER=cat -export MDIR="$mdir" -export OUTDIR="${outdir:-/var/www/gotmarc}" -export TMPLDIR="$tmpldir" - -# make sure the directories are there -set -e -mkdir -p "$CSUMDIR" -mkdir -p "$OUTDIR/mail" -mkdir -p "$OUTDIR/parts" -mkdir -p "$OUTDIR/text" -mkdir -p "$OUTDIR/thread" -set +e - -export PATH="$libexec:$PATH" - -fmt='%i-%R %16D<%64f>%128S' -mlist "${MDIR}" | mthread -r | mscan -f "$fmt" | pe | mkindex blob - ff5cece36c866d7184bec2d2bf88229d45621408 (mode 644) blob + /dev/null --- gotmarc.1 +++ /dev/null @@ -1,88 +0,0 @@ -.\" gotmarc.1 was written by Omar Polo and is placed in -.\" the public domain. The author hereby disclaims copyright to this -.\" source code. -.Dd May 5, 2023 -.Dt GOTMARC 1 -.Os -.Sh NAME -.Nm gotmarc -.Nd mailing list static site generator -.Sh SYNOPSIS -.Nm -.Op Fl c Ar cachedir -.Op Fl j Ar n -.Op Fl m Ar maildir -.Op Fl o Ar outdir -.Op Fl t Ar tmpldir -.Sh DESCRIPTION -.Nm -generates a static web archive from a maildir incrementally. -.Pp -The arguments are as follows: -.Bl -tag -width Ds -.It Fl c Ar cachedir -Cache directory used for keeping track of threads during incremental -updates. -Defaults to -.Pa ~/.cache/gotmarc -.It Fl j Ar n -Use -.Ar n -jobs to export threads. -With big mailing lists and on a multiprocessor machine this can save -quite some time. -.It Fl m Ar maildir -Source -.Ar maildir . -Defaults to -.Pa ~/Mail/gotmarc . -.It Fl o Ar outdir -The destination directory where the HTML files will be written to, it's -.Pa /var/www/gotmarc -by default. -.It Fl t Ar tmpldir -Path to a directory containing the template files. -.Pa /etc/gotmarc -by default. -.El -.Sh FILES -.Bl -tag -width Ds -.It Pa /etc/gotmarc/foot.html -Template with the trailing part of the page. -.It Pa /etc/gotmarc/head.html -Template with the first part of the page. -.Dv TITLE -is substituted with the title. -.It Pa /etc/gotmarc/index-header.html -Template with the title for index pages. -.Dv PAGE -is replaced with the page number and -.Dv SUBTITLE -with the date range. -.It Pa /etc/gotmarc/logo-small.html -Small version of the logo, included in the thread header. -.It Pa /etc/gotmarc/search.html -Template for the search form. -.Ev QUERY -is replaced with the search query. -.It Pa /var/www/gotmarc -Default output directory. -.It Pa ~/.cache/gotmarc -Default cache directory. -.It Pa ~/Mail/gotmarc -Default maildir. -.El -.Sh SEE ALSO -.Xr gotmarc 7 -.Sh HISTORY -.Nm -started as a hacky collection of perl scripts to generate a web archive -for the Game of Trees mailing list. -Only later it was converted to be a usable generic web mailing list -archive tool. -.Sh AUTHORS -.An -nosplit -The -.Nm -utility was written by -.An Omar Polo Aq Mt op@openbsd.org . blob - f7acd2906870c8225c61e6149d82d8f6342b04d0 (mode 644) blob + /dev/null --- gotmarc.7 +++ /dev/null @@ -1,222 +0,0 @@ -.\" gotmarc.7 was written by Omar Polo and is placed in -.\" the public domain. The author hereby disclaims copyright to this -.\" source code. -.Dd May 5, 2023 -.Dt GOTMARC 7 -.Os -.Sh NAME -.Nm gotmarc -.Nd mailing list web archive generation system -.Sh DESCRIPTION -.Nm -is a system to incrementally generate a web archive for a mailing list -and optionally provide search capabilities. -The generated archive is a set of HTML files that can be served as-is, -while searching requires the use of a FastCGI server. -At a higher level, -.Nm -is made of three components: -.Pp -.Bl -tag -width msearchd_8_ -compact -offset indent -.It Xr gotmarc 1 -to generate the static HTML files from a maildir. -.It Xr gmimport 1 -to import emails into a sqlite3 database. -.It Xr msearchd 8 -to provide search results. -.El -.Sh INITIAL SETUP -There are several step necessary to initialize the web archive: -.Pp -.Bl -enum -compact -offset indent -.It -Create and populate the output directory. -.It -Customize the templates. -.It -Prepare the maildir. -.It -Generate the web archive. -.It -Set up the database for searching. -.It -Configure the web server. -.El -.Pp -It is reccommended to use a dedicate user. -Commands to be run as a unpriviledged user are preceded by a dollar sign -.Sq $ , -while commands requiring superuser privileges by a hash mark -.Sq # . -Hereafter, it will be assumed that the local user is called -.Sq gotmarc . -.Ss 1. Create and populate the output directory -The web archive is made of several static files, mostly HTML, that needs -to be served by a web server like -.Xr httpd 8 . -.Pa /var/www/gotmarc -is the default location, but a different path can be used. -To prepare it, issue: -.Bd -literal -offset indent -# mkdir -p /var/www/gotmarc -# chown gotmarc /var/www/gotmarc -.Ed -.Pp -Then copy the CSS file, optionally tweaking it. -.Pp -.Dl $ cp /usr/local/share/examples/gotmarc/style.css /var/www/gotmarc -.Pp -Other eventual assets -.Pq e.g.\& logo images -need to be copied here as well. -.Ss 2. Customize the templates -The default templates are installed at -.Pa /etc/gotmarc . -Since these are anonymous, they need to be tweaked to include -information about the mailing list. -.Pp -Care should be taken when editing these files after generating the -archive since existing pages won't be automatically updated. -The cachedir -.Pq see Xr gotmarc 1 -needs to be deleted and the web archive generated again. -.Xr msearchd 8 -has to be stopped and restarted as well. -.Ss 3. Prepare the maildir -The maildir with the mailing list entries needs to be prepared. -It is assumed to be at -.Pa ~/Mail/gotmarc -by default, but a different path can be used. -.Ss 4. Generate the web archive -.Xr gotmarc 1 -can be finally used to generate the web archive. -The first run may take a while, depending on the size of the maildir, -while subsequent runs will be incremental and take less time. -.Pp -.Dl $ gotmarc -m path/to/maildir -o path/to/outdir -.Pp -On multi-processor machines multiple processes may be used to save some -time with the -.Xr gotmarc 1 Fl j No flag. -.Pp -The generated files may be compressed to save bandwidth: -.Pp -.Dl $ gzip -krq /var/www/gotmarc /dev/null -.Ss 5. Set up the database for searching -This is an suggested yet optional step. -.Pp -.Xr msearchd 8 -offers full text search capabilities using a sqlite3 database that has to -be populated with -.Xr gmimport 1 . -First, create a directory in the -.Pa /var/www -.Xr chroot 8 -jail: -.Bd -literal -offset indent -# mkdir -p /var/www/msearchd -# chown gotmarc /var/www/msearchd -.Ed -.Pp -Then, populate the database with all emails in the maildir: -.Bd -literal -offset indent -$ sqlite3 /var/www/msearchd/mails.sqlite3 \e - /dev/null || true -.Ed -.Pp -If -.Xr msearchd 8 -is not used, -new messages still needs to be incorporated -.Po i.e.\& moved from -.Pa new/ -to -.Pa cur/ -.Pc -but no database has to be updated. -In that case simplify the -.Xr minc 1 -invocation as: -.Pp -.Dl minc -q ~/Mail/gotmarc -.Pp -and don't call -.Xr gmimport 1 -at all. -.Sh HANDLING MULTIPLE MAILING LISTS -If the archive for multiple mailing lists needs to be served from the -same box, care must be taken to use different directories and database -files to avoid mixing messages. -.Pp -.Xr msearchd 8 -handles only one database at a time, so multiple instances need to be -run, each pointing at the database for only one mailing list. -Different FastCGI socket path needs to be used per-instance. -.Pp -.Xr gotmarc 1 -outdir, maildir and cachedir must be unique per-mailing list, i.e.\& the -.Fl c , Fl m No and Fl o -flag must always be provided. -.Pp -Very likely, each mailing list will needs its own set of templates, so -those needs to be prepared and both -.Xr gotmarc 1 -and -.Xr msearchd 8 -have to be pointed at the right template directory. -.Sh SEE ALSO -.Xr gmimport 1 , -.Xr gotmarc 1 , -.Xr minc 1 , -.Xr sqlite3 1 , -.Xr httpd 8 , -.Xr msearchd 8 blob - /dev/null blob + 5a1f4106e82d9a981c16293f0d4dd285e9bc7701 (mode 755) --- /dev/null +++ smarc @@ -0,0 +1,61 @@ +#!/bin/sh +# +# smarc was written by Omar Polo and is placed in the +# public domain. The author hereby disclaims copyright to this source +# code. + +progname="$(basename "$0")" +usage() { + u="usage: $progname [-c cachedir] [-j n]" + u="$u [-m maildir] [-o outdir] [-t tmpldir]" + echo "$u">&2 + exit 1 +} + +# changed at install-time +libexec=. +mblaze=.mblaze +tmpldir=templates/ + +# fix perl include path for in-repo hacking +if [ "$libexec" = . ]; then + export PERL5LIB="$PWD${PERL5LIB:+:}$PERL5LIB" +fi + +cachedir=$HOME/.cache/smarc +mdir=$HOME/Mail/smarc +outdir=/var/www/smarc + +while getopts c:j:m:o:t: flag; do + case $flag in + c) cachedir="$OPTARG" ;; + j) MAKE_JOBS="$OPTARG" ;; + m) mdir="$OPTARG" ;; + o) outdir="$OPTARG" ;; + t) tmpldir="$OPTARG" ;; + ?) usage ;; + esac +done + +# set up the env +export CSUMDIR="$cachedir/threadsum" +export MAKE_JOBS="${MAKE_JOBS:-1}" +export MBLAZE="$mblaze" +export MBLAZE_PAGER=cat +export MDIR="$mdir" +export OUTDIR="${outdir:-/var/www/smarc}" +export TMPLDIR="$tmpldir" + +# make sure the directories are there +set -e +mkdir -p "$CSUMDIR" +mkdir -p "$OUTDIR/mail" +mkdir -p "$OUTDIR/parts" +mkdir -p "$OUTDIR/text" +mkdir -p "$OUTDIR/thread" +set +e + +export PATH="$libexec:$PATH" + +fmt='%i-%R %16D<%64f>%128S' +mlist "${MDIR}" | mthread -r | mscan -f "$fmt" | pe | mkindex blob - 28f05814baa5f9af8a883277bc958b459ee67ba3 blob + 822c400f16c3f7fcc8294aa9405028c48438953c --- mexp +++ mexp @@ -12,7 +12,7 @@ use v5.32; use List::Util qw(max min); -use GotMArc qw(parse san urlencode initpage endpage thread_header +use SMArc qw(parse san urlencode initpage endpage thread_header threntry thrslice thrnav); my $outdir = $ENV{'OUTDIR'}; blob - /dev/null blob + 4aae3a949ad1d8b9ef98ec06af48af71b0f25ff3 (mode 644) --- /dev/null +++ smarc.1 @@ -0,0 +1,90 @@ +.\" smarc.1 was written by Omar Polo and is placed in +.\" the public domain. The author hereby disclaims copyright to this +.\" source code. +.Dd May 5, 2023 +.Dt SMARC 1 +.Os +.Sh NAME +.Nm smarc +.Nd mailing list static site generator +.Sh SYNOPSIS +.Nm +.Op Fl c Ar cachedir +.Op Fl j Ar n +.Op Fl m Ar maildir +.Op Fl o Ar outdir +.Op Fl t Ar tmpldir +.Sh DESCRIPTION +.Nm +generates a static web archive from a maildir incrementally. +.Pp +The arguments are as follows: +.Bl -tag -width Ds +.It Fl c Ar cachedir +Cache directory used for keeping track of threads during incremental +updates. +Defaults to +.Pa ~/.cache/smarc +.It Fl j Ar n +Use +.Ar n +jobs to export threads. +With big mailing lists and on a multiprocessor machine this can save +quite some time. +.It Fl m Ar maildir +Source +.Ar maildir . +Defaults to +.Pa ~/Mail/smarc . +.It Fl o Ar outdir +The destination directory where the HTML files will be written to, it's +.Pa /var/www/smarc +by default. +.It Fl t Ar tmpldir +Path to a directory containing the template files. +.Pa /etc/smarc +by default. +.El +.Sh FILES +.Bl -tag -width Ds +.It Pa /etc/smarc/foot.html +Template with the trailing part of the page. +.It Pa /etc/smarc/head.html +Template with the first part of the page. +.Dv TITLE +is substituted with the title. +.It Pa /etc/smarc/index-header.html +Template with the title for index pages. +.Dv PAGE +is replaced with the page number and +.Dv SUBTITLE +with the date range. +.It Pa /etc/smarc/logo-small.html +Small version of the logo, included in the thread header. +.It Pa /etc/smarc/search.html +Template for the search form. +.Ev QUERY +is replaced with the search query. +.It Pa /var/www/smarc +Default output directory. +.It Pa ~/.cache/smarc +Default cache directory. +.It Pa ~/Mail/smarc +Default maildir. +.El +.Sh SEE ALSO +.Xr smarc 7 +.Sh HISTORY +.Nm +started as a hacky collection of perl scripts under the name of +.Sq gotmarc +to generate a web archive for the Game of Trees mailing list. +Only later, while still being a hacky collection of perl and shell +scripts, was converted to be a generic web mailing list archive tool and +the project renamed. +.Sh AUTHORS +.An -nosplit +The +.Nm +utility was written by +.An Omar Polo Aq Mt op@openbsd.org . blob - 858e62ed2fb4d4c836c282d5edf5cb4b08fb9eb8 blob + 6ab5df9812402e1fb15c313fa441afd83af8e88e --- mkindex +++ mkindex @@ -11,7 +11,7 @@ use warnings; use v5.32; use File::Temp qw(tempfile); -use GotMArc qw(parse san urlencode initpage endpage index_header +use SMArc qw(parse san urlencode initpage endpage index_header search thread_header threntry); my $outdir = $ENV{'OUTDIR'}; @@ -63,7 +63,7 @@ sub nextfile { $page += 1; my $path; - ($pfh, $path) = tempfile "/tmp/gotmarc.index.XXXXXXXXXX"; + ($pfh, $path) = tempfile "/tmp/smarc.index.XXXXXXXXXX"; binmode($pfh, ':utf8'); push @files, $path; say $pfh "
      "; blob - /dev/null blob + c7c0a86c093403846ccd4411d99131332e5da4b5 (mode 644) --- /dev/null +++ smarc.7 @@ -0,0 +1,222 @@ +.\" smarc.7 was written by Omar Polo and is placed in +.\" the public domain. The author hereby disclaims copyright to this +.\" source code. +.Dd May 5, 2023 +.Dt SMARC 7 +.Os +.Sh NAME +.Nm smarc +.Nd mailing list web archive generation system +.Sh DESCRIPTION +.Nm +is a system to incrementally generate a web archive for a mailing list +and optionally provide search capabilities. +The generated archive is a set of HTML files that can be served as-is, +while searching requires the use of a FastCGI server. +At a higher level, +.Nm +is made of three components: +.Pp +.Bl -tag -width msearchd_8_ -compact -offset indent +.It Xr smarc 1 +to generate the static HTML files from a maildir. +.It Xr smingest 1 +to import emails into a sqlite3 database. +.It Xr msearchd 8 +to provide search results. +.El +.Sh INITIAL SETUP +There are several step necessary to initialize the web archive: +.Pp +.Bl -enum -compact -offset indent +.It +Create and populate the output directory. +.It +Customize the templates. +.It +Prepare the maildir. +.It +Generate the web archive. +.It +Set up the database for searching. +.It +Configure the web server. +.El +.Pp +It is reccommended to use a dedicate user. +Commands to be run as a unpriviledged user are preceded by a dollar sign +.Sq $ , +while commands requiring superuser privileges by a hash mark +.Sq # . +Hereafter, it will be assumed that the local user is called +.Sq smarc . +.Ss 1. Create and populate the output directory +The web archive is made of several static files, mostly HTML, that needs +to be served by a web server like +.Xr httpd 8 . +.Pa /var/www/smarc +is the default location, but a different path can be used. +To prepare it, issue: +.Bd -literal -offset indent +# mkdir -p /var/www/smarc +# chown smarc /var/www/smarc +.Ed +.Pp +Then copy the CSS file, optionally tweaking it. +.Pp +.Dl $ cp /usr/local/share/examples/smarc/style.css /var/www/smarc +.Pp +Other eventual assets +.Pq e.g.\& logo images +need to be copied here as well. +.Ss 2. Customize the templates +The default templates are installed at +.Pa /etc/smarc . +Since these are anonymous, they need to be tweaked to include +information about the mailing list. +.Pp +Care should be taken when editing these files after generating the +archive since existing pages won't be automatically updated. +The cachedir +.Pq see Xr smarc 1 +needs to be deleted and the web archive generated again. +.Xr msearchd 8 +has to be stopped and restarted as well. +.Ss 3. Prepare the maildir +The maildir with the mailing list entries needs to be prepared. +It is assumed to be at +.Pa ~/Mail/smarc +by default, but a different path can be used. +.Ss 4. Generate the web archive +.Xr smarc 1 +can be finally used to generate the web archive. +The first run may take a while, depending on the size of the maildir, +while subsequent runs will be incremental and take less time. +.Pp +.Dl $ smarc -m path/to/maildir -o path/to/outdir +.Pp +On multi-processor machines multiple processes may be used to save some +time with the +.Xr smarc 1 Fl j No flag. +.Pp +The generated files may be compressed to save bandwidth: +.Pp +.Dl $ gzip -krq /var/www/smarc /dev/null +.Ss 5. Set up the database for searching +This is an suggested yet optional step. +.Pp +.Xr msearchd 8 +offers full text search capabilities using a sqlite3 database that has to +be populated with +.Xr smingest 1 . +First, create a directory in the +.Pa /var/www +.Xr chroot 8 +jail: +.Bd -literal -offset indent +# mkdir -p /var/www/msearchd +# chown smarc /var/www/msearchd +.Ed +.Pp +Then, populate the database with all emails in the maildir: +.Bd -literal -offset indent +$ sqlite3 /var/www/msearchd/mails.sqlite3 \e + /dev/null || true +.Ed +.Pp +If +.Xr msearchd 8 +is not used, +new messages still needs to be incorporated +.Po i.e.\& moved from +.Pa new/ +to +.Pa cur/ +.Pc +but no database has to be updated. +In that case simplify the +.Xr minc 1 +invocation as: +.Pp +.Dl minc -q ~/Mail/smarc +.Pp +and don't call +.Xr smingest 1 +at all. +.Sh HANDLING MULTIPLE MAILING LISTS +If the archive for multiple mailing lists needs to be served from the +same box, care must be taken to use different directories and database +files to avoid mixing messages. +.Pp +.Xr msearchd 8 +handles only one database at a time, so multiple instances need to be +run, each pointing at the database for only one mailing list. +Different FastCGI socket path needs to be used per-instance. +.Pp +.Xr smarc 1 +outdir, maildir and cachedir must be unique per-mailing list, i.e.\& the +.Fl c , Fl m No and Fl o +flag must always be provided. +.Pp +Very likely, each mailing list will needs its own set of templates, so +those needs to be prepared and both +.Xr smarc 1 +and +.Xr msearchd 8 +have to be pointed at the right template directory. +.Sh SEE ALSO +.Xr minc 1 , +.Xr smarc 1 , +.Xr smingest 1 , +.Xr sqlite3 1 , +.Xr httpd 8 , +.Xr msearchd 8 blob - ad72cb88fb4d2dffcbaae2c49e06b5c57106888a blob + 5695726c77a6a9a2fb8ce8a473e32e9d6b5b7ce5 --- msearchd/Makefile +++ msearchd/Makefile @@ -27,13 +27,13 @@ install: ${INSTALL_MAN} msearchd.8 ${DESTDIR}${MANDIR}/man8 mkdir -p ${DESTDIR}${SBINDIR} ${INSTALL_PROGRAM} ${PROG} ${DESTDIR}${SBINDIR} - mkdir -p ${DESTDIR}${SYSCONFDIR}/gotmarc - ${INSTALL_DATA} schema.sql ${DESTDIR}${SYSCONFDIR}/gotmarc + mkdir -p ${DESTDIR}${SYSCONFDIR}/smarc + ${INSTALL_DATA} schema.sql ${DESTDIR}${SYSCONFDIR}/smarc uninstall: rm -f ${DESTDIR}${MANDIR}/man8/msearchd.8 rm -f ${DESTDIR}${SBINDIR}/${PROG} - rm -f ${DESTDIR}${SYSCONFDIR}/gotmarc/schema.sql + rm -f ${DESTDIR}${SYSCONFDIR}/smarc/schema.sql # -- internal build targets -- blob - 3b89a8f66f92de1b6a2ad17bb0774c2f83e748a3 blob + 5e32b4eff044d7f1e3b5c054e03156380848122e --- msearchd/msearchd.8 +++ msearchd/msearchd.8 @@ -62,7 +62,7 @@ Create an bind to the local socket at .Ar socket . .It Fl t Ar tmpldir Path to a directory containing the template files. -.Pa /etc/gotmarc +.Pa /etc/smarc by default. .It Fl u Ar user Drop privileges to @@ -78,16 +78,16 @@ options increase the verbosity. .El .Sh FILES .Bl -tag -width Ds -.It Pa /etc/gotmarc/foot.html +.It Pa /etc/smarc/foot.html Template with the trailing part of the page. -.It Pa /etc/gotmarc/head.html +.It Pa /etc/smarc/head.html Template with the first part of the page. .Dv TITLE is replaced with .Dq Search . -.It Pa /etc/gotmarc/search-header.html +.It Pa /etc/smarc/search-header.html Template for the start of the search page. -.It Pa /etc/gotmarc/search.html +.It Pa /etc/smarc/search.html Template for the search form. .Dv QUERY is replaced with the search query. @@ -102,7 +102,7 @@ Example configuration for .Bd -literal -offset -indent server "localhost" { listen on * port 80 - root "/gotmarc" + root "/smarc" gzip-static location "/search" { blob - d94aee1927187a7b25c4952e9db01bc2fcd2fb0c blob + 9ee7a8f3f1d361a0be1c5776f5ebae5f4d567045 --- msearchd/msearchd.c +++ msearchd/msearchd.c @@ -38,7 +38,7 @@ #endif #ifndef MSEARCH_TMPL_DIR -#define MSEARCH_TMPL_DIR SYSCONFDIR "/gotmarc" +#define MSEARCH_TMPL_DIR SYSCONFDIR "/smarc" #endif #define MAX_CHILDREN 32 blob - /dev/null blob + ec998e658ae70f4563baf20f9c63779585bf1529 (mode 755) --- /dev/null +++ smingest @@ -0,0 +1,66 @@ +#!/usr/bin/env perl +# +# smingest was written by Omar Polo and is placed in the +# public domain. The author hereby disclaims copyright to this source +# code. + +use strict; +use warnings; +use v5.32; +use utf8; + +use Date::Parse; +use File::Basename; + +use OpenBSD::Pledge; +use OpenBSD::Unveil; + +die "usage: $0 dbpath\n" if @ARGV != 1; +my $dbpath = shift @ARGV; + +open(my $sqlite, "|-", "/usr/local/bin/sqlite3", $dbpath) + or die "can't spawn sqlite3"; + +unveil("/usr/local/bin/mshow", "rx") or die "unveil mshow: $!"; +pledge("stdio proc exec") or die "pledge: $!"; + +say $sqlite ".import --csv /dev/stdin email" + or die "can't speak to sqlite: $!"; + +while (<>) { + chomp; + + open(my $fh, "-|", "/usr/local/bin/mshow", "-Atext/plain", "-NF", $_) + or die "can't run mshow $_: $!"; + + my $f = $_; + my ($time, $id) = split /\./, basename $_; + my $mid = "$time.$id"; + $mid =~ s/"/""/g; + + my ($from, $subj, $date) = ('', '', undef); + while (<$fh>) { + chomp; + last if /^$/; + s/"/""/g; + $from = s/.*?: //r if /^From:/; + $subj = s/.*?: //r if /^Subject:/; + $date = str2time(s/.*?: //r) if /^Date:/; + } + $date //= time; + $from =~ s/ +<.*>//; + + # leave open for the body + print $sqlite "\"$mid\",\"$from\",\"$date\",\"$subj\",\""; + + while (<$fh>) { + s/"/""/g; + print $sqlite $_; + } + say $sqlite '"'; + + close $fh; +} + +close $sqlite; +die "sqlite3 exited with $?\n" unless $? == 0; blob - 9f73857870aa589f38cc4e728d90e5a1069ab776 blob + 01aafeb485501929cc6df81fb550cf399b14265d --- pe +++ pe @@ -13,7 +13,7 @@ use Digest::SHA; use Encode qw(encode); use IO::Poll qw(POLLOUT); -use GotMArc qw(parse); +use SMArc qw(parse); my $jobs = $ENV{'MAKE_JOBS'} // 1; blob - /dev/null blob + a2124ad27da92e9fbd048e16a71e5607d3ef2452 (mode 644) --- /dev/null +++ smingest.1 @@ -0,0 +1,38 @@ +.\" smingest.1 was written by Omar Polo and is placed in +.\" the public domain. The author hereby disclaims copyright to this +.\" source code. +.Dd May 5, 2023 +.Dt SMINGEST 1 +.Os +.Sh NAME +.Nm smingest +.Nd import emails into a sqlite database +.Sh SYNOPSIS +.Nm +.Ar dbpath +.Sh DESCRIPTION +.Nm +reads a sequence of path to emails from standard inputs and inserts them +into the +.Xr msearchd 8 +sqlite3 database at +.Ar dbpath . +.Sh EXAMPLES +To index all the messages in the +.Pa ~/Mail/smarc +maildir, useful to initially populate the database: +.Pp +.Dl mlist ~/Mail/smarc | smingest /var/www/msearchd/mails.sqlite3 +.Pp +Incorporate new messages in the maildir and add them to the database, +useful after fetching new mails: +.Pp +.Dl minc ~/Mail/smarc | smingest /var/www/msearchd/mails.sqlite3 +.Sh SEE ALSO +.Xr minc 1 , +.Xr mlist 1 , +.Xr msearchd 8 +.Sh BUGS +.Nm +doesn't detect the insert of duplicate messages. +Should that happen, the database needs to be recreated. blob - dd0133354b29cf2aa0c1067312aa3b0c3563151d blob + 030d1a875da5e5a062087df49cc93267531045d5 --- templates/Makefile +++ templates/Makefile @@ -9,11 +9,11 @@ dist: ${INSTALL} -m 0644 ${DISTFILES} ${DESTDIR}/ install: - mkdir -p ${DESTDIR}${SYSCONFDIR}/gotmarc/ - ${INSTALL_DATA} *.html ${DESTDIR}${SYSCONFDIR}/gotmarc/ + mkdir -p ${DESTDIR}${SYSCONFDIR}/smarc/ + ${INSTALL_DATA} *.html ${DESTDIR}${SYSCONFDIR}/smarc/ uninstall: - rm -f ${DESTDIR}${SYSCONFDIR}/gotmarc/${DISTFILES} + rm -f ${DESTDIR}${SYSCONFDIR}/smarc/${DISTFILES} .PHONY: all dist install uninstall