commit e05b0ff3ebd8086809714527a27b412345ff4d72 from: Russ Cox date: Thu Jul 03 05:34:48 2008 UTC vac: add -a and -x flags Thanks to Michael Kaminsky for the suggestion. commit - d9841dc7adc0ad99e56cf508d5d6b6d2e59afbb5 commit + e05b0ff3ebd8086809714527a27b412345ff4d72 blob - 9243febb6b225ed0deaf86cbc0aced5311a2211b blob + 7de41ee622c593018db745c48c99576f907efa0e --- man/man1/vac.1 +++ man/man1/vac.1 @@ -6,6 +6,9 @@ vac, unvac \- create, extract a vac archive on Venti [ .B -mqsv ] [ +.B -a +.I vacfile +] [ .B -b .I blocksize ] [ @@ -23,6 +26,9 @@ vac, unvac \- create, extract a vac archive on Venti ] [ .B -h .I host +] [ +.B -x +.I excludefile ] .I file ... .PP @@ -66,6 +72,26 @@ vac:64daefaecc4df4b5cb48a368b361ef56012a4f46 .PP The options are: .TP +.BI -a " vacfile +Specifies that vac should create or update a backup archive, inserting +the files under an extra two levels of directory hierarchy named +.I yyyy/mmdd +(year, month, day) +in the style of the dump file system +(see Plan 9's \fIfs\fR(4)). +If +.I vacfile +already exists, an additional backup day is added to the +existing hierarchy, behaving as though the +.B -d +flag was specified giving the most recent backup tree in the archive. +Typically, this option +is used as part of a nightly backup script. +This option cannot be used with +.B -d +or +.BR -f . +.TP .BI -b " blocksize Specifies the block size that data will be broken into. The units for the size can be specified by appending @@ -86,6 +112,12 @@ file tree given by Do not include the file or directory specified by .IR exclude . This option may be repeated multiple times. +.I Exclude +can be a shell pattern as accepted by +.IR rc (1), +with one extension: +.B \&... +matches any sequence of characters including slashes. .TP .BI -f " vacfile The results of @@ -123,8 +155,10 @@ the archive to be unpacked. .TP .B -q Increase the performance of the +.B -a +or .B -d -option by detecting unchanged files based on a match of the files name and other meta data, +options by detecting unchanged files based on a match of the files name and other meta data, rather than examining the contents of the files. .TP .B -s @@ -133,6 +167,27 @@ Print out various statistics on standard error. .B -v Produce more verbose output on standard error, including the name of the files added to the archive and the vac archives that are expanded and merged. +.TP +.BI -x " excfile +Read exclude patterns from the file +.IR excfile . +Blank lines and lines beginning with +.B # +are ignored. +All other lines should be of the form +.B include +.I pattern +or +.B exclude +.I pattern . +When considering whether to include a directory or file +in the vac archive, +the earliest matching pattern in the file +applies. +The patterns are the same syntax accepted by the +.B -e +option. +This option may be repeated multiple times. .PP .I Unvac lists or extracts files stored in the vac archive blob - f5f823d7c1d6c3d01452a0f20ed097b26553277b blob + d7ad2056f95b9b784c16d54e57c377df435ff261 --- src/cmd/vac/file.c +++ src/cmd/vac/file.c @@ -974,6 +974,7 @@ filemetaalloc(VacFile *fp, VacDir *dir, u32int start) vtblockput(b); if((b = vtfileblock(ms, bo, VtORDWR)) == nil) goto Err; + mbunpack(&mb, b->data, ms->dsize); goto Found; } vtblockput(b); @@ -1002,7 +1003,6 @@ Found: me.p = p; me.size = n; vdpack(dir, &me, VacDirVersion); -vdunpack(dir, &me); mbinsert(&mb, i, &me); mbpack(&mb); vtblockput(b); @@ -1166,6 +1166,7 @@ Err: /* * Flush all data associated with f out of the cache and onto venti. * If recursive is set, flush f's children too. + * Vacfiledecref knows how to flush source and msource too. */ int vacfileflush(VacFile *f, int recursive) @@ -1183,25 +1184,12 @@ vacfileflush(VacFile *f, int recursive) ret = -1; filemetaunlock(f); - /* - * Vacfiledecref knows how to flush source and msource too. - */ if(filelock(f) < 0) return -1; - vtfilelock(f->source, -1); - if(vtfileflush(f->source) < 0) - ret = -1; - vtfileunlock(f->source); - if(f->msource){ - vtfilelock(f->msource, -1); - if(vtfileflush(f->msource) < 0) - ret = -1; - vtfileunlock(f->msource); - } - + /* * Lock order prevents us from flushing kids while holding - * lock, so make a list. + * lock, so make a list and then flush without the lock. */ nkids = 0; kids = nil; @@ -1216,14 +1204,32 @@ vacfileflush(VacFile *f, int recursive) p->ref++; } } - fileunlock(f); - - for(i=0; i 0){ + fileunlock(f); + for(i=0; isource, -1); + if(vtfileflush(f->source) < 0) + ret = -1; + vtfileunlock(f->source); + if(f->msource){ + vtfilelock(f->msource, -1); + if(vtfileflush(f->msource) < 0) + ret = -1; + vtfileunlock(f->msource); + } + fileunlock(f); + return ret; } @@ -1332,6 +1338,12 @@ vacfilecreate(VacFile *fp, char *elem, ulong mode) vacfileincref(fp); fileunlock(fp); + + filelock(ff); + vtfilelock(ff->source, -1); + vtfileunlock(ff->source); + fileunlock(ff); + return ff; Err: @@ -2031,7 +2043,7 @@ vacfssync(VacFs *fs) return -1; } vtfileclose(f); - + /* Build a root block. */ memset(&root, 0, sizeof root); strcpy(root.type, "vac"); blob - /dev/null blob + aa21206859e1224f0fbbf4cfcc6e2ccfc807a2a6 (mode 644) --- /dev/null +++ src/cmd/vac/exc @@ -0,0 +1,8 @@ +exclude a/* +exclude b/... +exclude c/[~a]* +exclude d/[a]* +exclude e/[a-z]* +exclude f/?a* +exclude g/*/*/b +exclude h/.../b blob - 3cf8784025865790bde745bd494487cf910cf5c8 blob + 0f691b1e16476ac83d31a204d485bd228c18ff40 --- src/cmd/vac/fns.h +++ src/cmd/vac/fns.h @@ -23,3 +23,8 @@ VacFile *_vacfileroot(VacFs *fs, VtFile *file); int _vacfsnextqid(VacFs *fs, uvlong *qid); void vacfsjumpqid(VacFs*, uvlong step); + +Reprog* glob2regexp(char*); +void loadexcludefile(char*); +int includefile(char*); +void excludepattern(char*); blob - /dev/null blob + fd2a98b34f5004d3eb6a07539220b5fc5c3199e5 (mode 644) --- /dev/null +++ src/cmd/vac/exc.in @@ -0,0 +1,26 @@ +a/abc +a/foo +a/.foo +b/foo +b/.foo +c/abc +c/def +c/zab +d/abc +d/def +d/zab +e/abc +e/.abc +e/ABC +f/a +f/.abc +f/az +f/za +f/zabc +f/zza +g/a/b +g/a/c/b +g/a/c/d/b +h/a/b +h/a/c/b +h/a/c/d/b blob - 98e122b8f95f0d3df3adea289de4743b97821d44 blob + 6ee651cb59f6c504033450409dc66020456ff0d0 --- src/cmd/vac/mkfile +++ src/cmd/vac/mkfile @@ -4,6 +4,7 @@ LIBFILES=\ error\ file\ pack\ + glob\ LIB=${LIBFILES:%=%.$O} $PLAN9/lib/libventi.a @@ -20,3 +21,10 @@ TARG=vac vacfs unvac default:V: all <$PLAN9/src/mkmany + +testglob:V: $O.testinc + $O.testinc exc exc.test + diff exc.out exc.test + ls -l exc.out exc.test + + blob - /dev/null blob + 704b150e6ea48b0b9763bc6b91c630ca3de6e4a3 (mode 644) --- /dev/null +++ src/cmd/vac/exc.out @@ -0,0 +1,26 @@ +0 a/abc +0 a/foo +1 a/.foo +0 b/foo +0 b/.foo +1 c/abc +0 c/def +0 c/zab +0 d/abc +1 d/def +1 d/zab +0 e/abc +1 e/.abc +1 e/ABC +1 f/a +1 f/.abc +1 f/az +0 f/za +0 f/zabc +1 f/zza +1 g/a/b +0 g/a/c/b +1 g/a/c/d/b +0 h/a/b +0 h/a/c/b +0 h/a/c/d/b blob - bf72d3fde9902ff355846e27dbf8b7f6af6d0d75 blob + aa3b5f7c61dd93a8a788439ce09111ca713a093a --- src/cmd/vac/stdinc.h +++ src/cmd/vac/stdinc.h @@ -1,5 +1,7 @@ #include #include +#include #include #include #include +#include blob - /dev/null blob + 863eb848619be881ba23e577da0c7675eb9ed7d0 (mode 644) --- /dev/null +++ src/cmd/vac/glob.c @@ -0,0 +1,180 @@ +#include "stdinc.h" +#include "vac.h" +#include "dat.h" +#include "fns.h" +#include "error.h" + +// Convert globbish pattern to regular expression +// The wildcards are +// +// * any non-slash characters +// ... any characters including / +// ? any single character except / +// [a-z] character class +// [~a-z] negated character class +// + +Reprog* +glob2regexp(char *glob) +{ + char *s, *p, *w; + Reprog *re; + int boe; // beginning of path element + + s = malloc(20*(strlen(glob)+1)); + if(s == nil) + return nil; + w = s; + boe = 1; + *w++ = '^'; + *w++ = '('; + for(p=glob; *p; p++){ + if(p[0] == '.' && p[1] == '.' && p[2] == '.'){ + strcpy(w, ".*"); + w += strlen(w); + p += 3-1; + boe = 0; + continue; + } + if(p[0] == '*'){ + if(boe) + strcpy(w, "([^./][^/]*)?"); + else + strcpy(w, "[^/]*"); + w += strlen(w); + boe = 0; + continue; + } + if(p[0] == '?'){ + if(boe) + strcpy(w, "[^./]"); + else + strcpy(w, "[^/]"); + w += strlen(w); + boe = 0; + continue; + } + if(p[0] == '['){ + *w++ = '['; + if(*++p == '~'){ + *w++ = '^'; + p++; + } + while(*p != ']'){ + if(*p == '/') + goto syntax; + if(*p == '^' || *p == '\\') + *w++ = '\\'; + *w++ = *p++; + } + *w++ = ']'; + boe = 0; + continue; + } + if(strchr("()|^$[]*?+\\.", *p)){ + *w++ = '\\'; + *w++ = *p; + boe = 0; + continue; + } + if(*p == '/'){ + *w++ = '/'; + boe = 1; + continue; + } + *w++ = *p; + boe = 0; + continue; + } + *w++ = ')'; + *w++ = '$'; + *w = 0; + + re = regcomp(s); + if(re == nil){ + syntax: + free(s); + werrstr("glob syntax error"); + return nil; + } + free(s); + return re; +} + +typedef struct Pattern Pattern; +struct Pattern +{ + Reprog *re; + int include; +}; + +Pattern *pattern; +int npattern; + +void +loadexcludefile(char *file) +{ + Biobuf *b; + char *p, *q; + int n, inc; + Reprog *re; + + if((b = Bopen(file, OREAD)) == nil) + sysfatal("open %s: %r", file); + for(n=1; (p=Brdstr(b, '\n', 1)) != nil; free(p), n++){ + q = p+strlen(p); + while(q > p && isspace((uchar)*(q-1))) + *--q = 0; + switch(p[0]){ + case '\0': + case '#': + continue; + } + + inc = 0; + if(strncmp(p, "include ", 8) == 0){ + inc = 1; + }else if(strncmp(p, "exclude ", 8) == 0){ + inc = 0; + }else + sysfatal("%s:%d: line does not begin with include or exclude", file, n); + + if(strchr(p+8, ' ')) + fprint(2, "%s:%d: warning: space in pattern\n", file, n); + + if((re = glob2regexp(p+8)) == nil) + sysfatal("%s:%d: bad glob pattern", file, n); + + pattern = vtrealloc(pattern, (npattern+1)*sizeof pattern[0]); + pattern[npattern].re = re; + pattern[npattern].include = inc; + npattern++; + } + Bterm(b); +} + +void +excludepattern(char *p) +{ + Reprog *re; + + if((re = glob2regexp(p)) == nil) + sysfatal("bad glob pattern %s", p); + + pattern = vtrealloc(pattern, (npattern+1)*sizeof pattern[0]); + pattern[npattern].re = re; + pattern[npattern].include = 0; + npattern++; +} + +int +includefile(char *file) +{ + Pattern *p, *ep; + + for(p=pattern, ep=p+npattern; pre, file, nil, 0)) + return p->include; + return 1; +} + blob - 183fa380fab0121b3bc1a34adfddfce1cace5794 blob + 99651981b13c321e25a4fa5fc52f597a27203882 --- src/cmd/vac/vac.c +++ src/cmd/vac/vac.c @@ -8,14 +8,13 @@ void usage(void) { - fprint(2, "vac [-imqsv] [-b bsize] [-d old.vac] [-e exclude]... [-f new.vac] [-h host] file...\n"); + fprint(2, "vac [-imqsv] [-a archive.vac] [-b bsize] [-d old.vac] [-f new.vac] [-e exclude]... [-h host] file...\n"); threadexitsall("usage"); } enum { BlockSize = 8*1024, - MaxExclude = 1000 }; struct @@ -33,17 +32,16 @@ int verbose; char *host; VtConn *z; VacFs *fs; -char *exclude[MaxExclude]; -int nexclude; +char *archivefile; char *vacfile; int vacmerge(VacFile*, char*); void vac(VacFile*, VacFile*, char*, Dir*); void vacstdin(VacFile*, char*); +VacFile *recentarchive(VacFs*, char*); static u64int unittoull(char*); static void warn(char *fmt, ...); -static int strpcmp(const void*, const void*); static void removevacfile(void); #ifdef PLAN9PORT @@ -81,6 +79,7 @@ threadmain(int argc, char **argv) _p9usepwlibrary = 1; #endif + fmtinstall('F', vtfcallfmt); fmtinstall('H', encodefmt); fmtinstall('V', vtscorefmt); @@ -89,7 +88,14 @@ threadmain(int argc, char **argv) printstats = 0; fsdiff = nil; diffvac = nil; + ARGBEGIN{ + case 'V': + chattyventi++; + break; + case 'a': + archivefile = EARGF(usage()); + break; case 'b': u = unittoull(EARGF(usage())); if(u < 512) @@ -102,12 +108,7 @@ threadmain(int argc, char **argv) diffvac = EARGF(usage()); break; case 'e': - if(nexclude >= MaxExclude) - sysfatal("too many exclusions\n"); - exclude[nexclude] = ARGF(); - if(exclude[nexclude] == nil) - usage(); - nexclude++; + excludepattern(EARGF(usage())); break; case 'f': vacfile = EARGF(usage()); @@ -130,40 +131,101 @@ threadmain(int argc, char **argv) case 'v': verbose++; break; + case 'x': + loadexcludefile(EARGF(usage())); + break; default: usage(); }ARGEND if(argc == 0 && !stdinname) usage(); + + if(archivefile && (vacfile || diffvac)){ + fprint(2, "cannot use -a with -f, -d\n"); + usage(); + } - if(vacfile == nil) - outfd = 1; - else if((outfd = create(vacfile, OWRITE, 0666)) < 0) - sysfatal("create %s: %r", vacfile); - atexit(removevacfile); - - qsort(exclude, nexclude, sizeof(char*), strpcmp); - z = vtdial(host); if(z == nil) sysfatal("could not connect to server: %r"); if(vtconnect(z) < 0) sysfatal("vtconnect: %r"); - - if(diffvac){ - if((fsdiff = vacfsopen(z, diffvac, VtOREAD, 128)) == nil) - warn("vacfsopen %s: %r", diffvac); - } - if((fs = vacfscreate(z, blocksize, 512)) == nil) - sysfatal("vacfscreate: %r"); + // Setup: + // fs is the output vac file system + // f is directory in output vac to write new files + // fdiff is corresponding directory in existing vac + if(archivefile){ + VacFile *fp; + char yyyy[5]; + char mmdd[10]; + char oldpath[40]; + Tm tm; - f = vacfsgetroot(fs); - if(fsdiff) - fdiff = vacfsgetroot(fsdiff); - else fdiff = nil; + if((outfd = open(archivefile, ORDWR)) < 0){ + if(access(archivefile, 0) >= 0) + sysfatal("open %s: %r", archivefile); + if((outfd = create(archivefile, OWRITE, 0666)) < 0) + sysfatal("create %s: %r", archivefile); + atexit(removevacfile); // because it is new + if((fs = vacfscreate(z, blocksize, 512)) == nil) + sysfatal("vacfscreate: %r"); + }else{ + if((fs = vacfsopen(z, archivefile, VtORDWR, 512)) == nil) + sysfatal("vacfsopen %s: %r", archivefile); + if((fdiff = recentarchive(fs, oldpath)) != nil){ + if(verbose) + fprint(2, "diff %s\n", oldpath); + }else + if(verbose) + fprint(2, "no recent archive to diff against\n"); + } + + // Create yyyy/mmdd. + tm = *localtime(time(0)); + snprint(yyyy, sizeof yyyy, "%04d", tm.year+1900); + fp = vacfsgetroot(fs); + if((f = vacfilewalk(fp, yyyy)) == nil + && (f = vacfilecreate(fp, yyyy, ModeDir|0555)) == nil) + sysfatal("vacfscreate %s: %r", yyyy); + vacfiledecref(fp); + fp = f; + + snprint(mmdd, sizeof mmdd, "%02d%02d", tm.mon+1, tm.mday); + n = 0; + while((f = vacfilewalk(fp, mmdd)) != nil){ + vacfiledecref(f); + n++; + snprint(mmdd+4, sizeof mmdd-4, ".%d", n); + } + f = vacfilecreate(fp, mmdd, ModeDir|0555); + if(f == nil) + sysfatal("vacfscreate %s/%s: %r", yyyy, mmdd); + vacfiledecref(fp); + + if(verbose) + fprint(2, "archive %s/%s\n", yyyy, mmdd); + }else{ + if(vacfile == nil) + outfd = 1; + else if((outfd = create(vacfile, OWRITE, 0666)) < 0) + sysfatal("create %s: %r", vacfile); + atexit(removevacfile); + if((fs = vacfscreate(z, blocksize, 512)) == nil) + sysfatal("vacfscreate: %r"); + f = vacfsgetroot(fs); + + fdiff = nil; + if(diffvac){ + if((fsdiff = vacfsopen(z, diffvac, VtOREAD, 128)) == nil) + warn("vacfsopen %s: %r", diffvac); + else + fdiff = vacfsgetroot(fsdiff); + } + } + if(stdinname) vacstdin(f, stdinname); for(i=0; iscore); - vacfsclose(fs); atexitdont(removevacfile); + vacfsclose(fs); vthangup(z); if(printstats){ @@ -243,6 +305,85 @@ threadmain(int argc, char **argv) threadexitsall(0); } +VacFile* +recentarchive(VacFs *fs, char *path) +{ + VacFile *fp, *f; + VacDirEnum *de; + VacDir vd; + char buf[10]; + int year, mmdd, nn, n, n1; + char *p; + + fp = vacfsgetroot(fs); + de = vdeopen(fp); + year = 0; + if(de){ + for(; vderead(de, &vd) > 0; vdcleanup(&vd)){ + if(strlen(vd.elem) != 4) + continue; + if((n = strtol(vd.elem, &p, 10)) < 1900 || *p != 0) + continue; + if(year < n) + year = n; + } + } + vdeclose(de); + if(year == 0){ + vacfiledecref(fp); + return nil; + } + snprint(buf, sizeof buf, "%04d", year); + if((f = vacfilewalk(fp, buf)) == nil){ + fprint(2, "warning: dirread %s but cannot walk", buf); + vacfiledecref(fp); + return nil; + } + fp = f; + + de = vdeopen(fp); + mmdd = 0; + nn = 0; + if(de){ + for(; vderead(de, &vd) > 0; vdcleanup(&vd)){ + if(strlen(vd.elem) < 4) + continue; + if((n = strtol(vd.elem, &p, 10)) < 100 || n > 1231 || p != vd.elem+4) + continue; + if(*p == '.'){ + if(p[1] == '0' || (n1 = strtol(p+1, &p, 10)) == 0 || *p != 0) + continue; + }else{ + if(*p != 0) + continue; + n1 = 0; + } + if(n < mmdd || (n == mmdd && n1 < nn)) + continue; + mmdd = n; + nn = n1; + } + } + vdeclose(de); + if(mmdd == 0){ + vacfiledecref(fp); + return nil; + } + if(nn == 0) + snprint(buf, sizeof buf, "%04d", mmdd); + else + snprint(buf, sizeof buf, "%04d.%d", mmdd, nn); + if((f = vacfilewalk(fp, buf)) == nil){ + fprint(2, "warning: dirread %s but cannot walk", buf); + vacfiledecref(fp); + return nil; + } + vacfiledecref(fp); + + sprint(path, "%04d/%s", year, buf); + return f; +} + static void removevacfile(void) { @@ -250,32 +391,6 @@ removevacfile(void) remove(vacfile); } -static int -strpcmp(const void *p0, const void *p1) -{ - return strcmp(*(char**)p0, *(char**)p1); -} - -static int -isexcluded(char *name) -{ - int bot, top, i, x; - - bot = 0; - top = nexclude; - while(bot < top) { - i = (bot+top)>>1; - x = strcmp(exclude[i], name); - if(x == 0) - return 1; - if(x < 0) - bot = i + 1; - else /* x > 0 */ - top = i; - } - return 0; -} - void plan9tovacdir(VacDir *vd, Dir *dir) { @@ -361,7 +476,7 @@ vac(VacFile *fp, VacFile *diffp, char *name, Dir *d) VacFile *f, *fdiff; VtEntry e; - if(isexcluded(name)){ + if(!includefile(name)){ warn("excluding %s%s", name, (d->mode&DMDIR) ? "/" : ""); return; } blob - /dev/null blob + 0cd431434a80438995760e2410611d4b4d8b163e (mode 644) --- /dev/null +++ src/cmd/vac/testinc.c @@ -0,0 +1,31 @@ +#include "stdinc.h" +#include "vac.h" +#include "dat.h" +#include "fns.h" +#include "error.h" + +void +threadmain(int argc, char **argv) +{ + Biobuf b; + char *p; + + ARGBEGIN{ + default: + goto usage; + }ARGEND + + if(argc != 1){ + usage: + fprint(2, "usage: testinc includefile\n"); + threadexitsall("usage"); + } + + loadexcludefile(argv[0]); + Binit(&b, 0, OREAD); + while((p = Brdline(&b, '\n')) != nil){ + p[Blinelen(&b)-1] = 0; + print("%d %s\n", includefile(p), p); + } + threadexitsall(0); +}