Blob


1 #include "stdinc.h"
2 #include "vac.h"
3 #include "dat.h"
4 #include "fns.h"
6 // TODO: qids
8 void
9 usage(void)
10 {
11 fprint(2, "vac [-imqsv] [-a archive.vac] [-b bsize] [-d old.vac] [-f new.vac] [-e exclude]... [-h host] file...\n");
12 threadexitsall("usage");
13 }
15 enum
16 {
17 BlockSize = 8*1024,
18 CacheSize = 4<<20,
19 };
21 struct
22 {
23 int nfile;
24 int ndir;
25 vlong data;
26 vlong skipdata;
27 int skipfiles;
28 } stats;
30 int qdiff;
31 int merge;
32 int verbose;
33 char *host;
34 VtConn *z;
35 VacFs *fs;
36 char *archivefile;
37 char *vacfile;
39 int vacmerge(VacFile*, char*);
40 void vac(VacFile*, VacFile*, char*, Dir*);
41 void vacstdin(VacFile*, char*);
42 VacFile *recentarchive(VacFs*, char*);
44 static u64int unittoull(char*);
45 static void warn(char *fmt, ...);
46 static void removevacfile(void);
48 #ifdef PLAN9PORT
49 /*
50 * We're between a rock and a hard place here.
51 * The pw library (getpwnam, etc.) reads the
52 * password and group files into an on-stack buffer,
53 * so if you have some huge groups, you overflow
54 * the stack. Because of this, the thread library turns
55 * it off by default, so that dirstat returns "14571" instead of "rsc".
56 * But for vac we want names. So cautiously turn the pwlibrary
57 * back on (see threadmain) and make the main thread stack huge.
58 */
59 extern int _p9usepwlibrary;
60 int mainstacksize = 4*1024*1024;
62 #endif
63 void
64 threadmain(int argc, char **argv)
65 {
66 int i, j, fd, n, printstats;
67 Dir *d;
68 char *s;
69 uvlong u;
70 VacFile *f, *fdiff;
71 VacFs *fsdiff;
72 int blocksize;
73 int outfd;
74 char *stdinname;
75 char *diffvac;
76 uvlong qid;
78 #ifdef PLAN9PORT
79 /* see comment above */
80 _p9usepwlibrary = 1;
81 #endif
83 fmtinstall('F', vtfcallfmt);
84 fmtinstall('H', encodefmt);
85 fmtinstall('V', vtscorefmt);
87 blocksize = BlockSize;
88 stdinname = nil;
89 printstats = 0;
90 fsdiff = nil;
91 diffvac = nil;
93 ARGBEGIN{
94 case 'V':
95 chattyventi++;
96 break;
97 case 'a':
98 archivefile = EARGF(usage());
99 break;
100 case 'b':
101 u = unittoull(EARGF(usage()));
102 if(u < 512)
103 u = 512;
104 blocksize = u;
105 break;
106 case 'd':
107 diffvac = EARGF(usage());
108 break;
109 case 'e':
110 excludepattern(EARGF(usage()));
111 break;
112 case 'f':
113 vacfile = EARGF(usage());
114 break;
115 case 'h':
116 host = EARGF(usage());
117 break;
118 case 'i':
119 stdinname = EARGF(usage());
120 break;
121 case 'm':
122 merge++;
123 break;
124 case 'q':
125 qdiff++;
126 break;
127 case 's':
128 printstats++;
129 break;
130 case 'v':
131 verbose++;
132 break;
133 case 'x':
134 loadexcludefile(EARGF(usage()));
135 break;
136 default:
137 usage();
138 }ARGEND
140 if(argc == 0 && !stdinname)
141 usage();
143 if(archivefile && (vacfile || diffvac)){
144 fprint(2, "cannot use -a with -f, -d\n");
145 usage();
148 z = vtdial(host);
149 if(z == nil)
150 sysfatal("could not connect to server: %r");
151 if(vtconnect(z) < 0)
152 sysfatal("vtconnect: %r");
154 // Setup:
155 // fs is the output vac file system
156 // f is directory in output vac to write new files
157 // fdiff is corresponding directory in existing vac
158 if(archivefile){
159 VacFile *fp;
160 char yyyy[5];
161 char mmdd[10];
162 char oldpath[40];
163 Tm tm;
165 fdiff = nil;
166 if((outfd = open(archivefile, ORDWR)) < 0){
167 if(access(archivefile, 0) >= 0)
168 sysfatal("open %s: %r", archivefile);
169 if((outfd = create(archivefile, OWRITE, 0666)) < 0)
170 sysfatal("create %s: %r", archivefile);
171 atexit(removevacfile); // because it is new
172 if((fs = vacfscreate(z, blocksize, CacheSize)) == nil)
173 sysfatal("vacfscreate: %r");
174 }else{
175 if((fs = vacfsopen(z, archivefile, VtORDWR, CacheSize)) == nil)
176 sysfatal("vacfsopen %s: %r", archivefile);
177 if((fdiff = recentarchive(fs, oldpath)) != nil){
178 if(verbose)
179 fprint(2, "diff %s\n", oldpath);
180 }else
181 if(verbose)
182 fprint(2, "no recent archive to diff against\n");
185 // Create yyyy/mmdd.
186 tm = *localtime(time(0));
187 snprint(yyyy, sizeof yyyy, "%04d", tm.year+1900);
188 fp = vacfsgetroot(fs);
189 if((f = vacfilewalk(fp, yyyy)) == nil
190 && (f = vacfilecreate(fp, yyyy, ModeDir|0555)) == nil)
191 sysfatal("vacfscreate %s: %r", yyyy);
192 vacfiledecref(fp);
193 fp = f;
195 snprint(mmdd, sizeof mmdd, "%02d%02d", tm.mon+1, tm.mday);
196 n = 0;
197 while((f = vacfilewalk(fp, mmdd)) != nil){
198 vacfiledecref(f);
199 n++;
200 snprint(mmdd+4, sizeof mmdd-4, ".%d", n);
202 f = vacfilecreate(fp, mmdd, ModeDir|0555);
203 if(f == nil)
204 sysfatal("vacfscreate %s/%s: %r", yyyy, mmdd);
205 vacfiledecref(fp);
207 if(verbose)
208 fprint(2, "archive %s/%s\n", yyyy, mmdd);
209 }else{
210 if(vacfile == nil)
211 outfd = 1;
212 else if((outfd = create(vacfile, OWRITE, 0666)) < 0)
213 sysfatal("create %s: %r", vacfile);
214 atexit(removevacfile);
215 if((fs = vacfscreate(z, blocksize, CacheSize)) == nil)
216 sysfatal("vacfscreate: %r");
217 f = vacfsgetroot(fs);
219 fdiff = nil;
220 if(diffvac){
221 if((fsdiff = vacfsopen(z, diffvac, VtOREAD, CacheSize)) == nil)
222 warn("vacfsopen %s: %r", diffvac);
223 else
224 fdiff = vacfsgetroot(fsdiff);
228 if(stdinname)
229 vacstdin(f, stdinname);
230 for(i=0; i<argc; i++){
231 // We can't use / and . and .. and ../.. as valid archive
232 // names, so expand to the list of files in the directory.
233 if(argv[i][0] == 0){
234 warn("empty string given as command-line argument");
235 continue;
237 cleanname(argv[i]);
238 if(strcmp(argv[i], "/") == 0
239 || strcmp(argv[i], ".") == 0
240 || strcmp(argv[i], "..") == 0
241 || (strlen(argv[i]) > 3 && strcmp(argv[i]+strlen(argv[i])-3, "/..") == 0)){
242 if((fd = open(argv[i], OREAD)) < 0){
243 warn("open %s: %r", argv[i]);
244 continue;
246 while((n = dirread(fd, &d)) > 0){
247 for(j=0; j<n; j++){
248 s = vtmalloc(strlen(argv[i])+1+strlen(d[j].name)+1);
249 strcpy(s, argv[i]);
250 strcat(s, "/");
251 strcat(s, d[j].name);
252 cleanname(s);
253 vac(f, fdiff, s, &d[j]);
255 free(d);
257 close(fd);
258 continue;
260 if((d = dirstat(argv[i])) == nil){
261 warn("stat %s: %r", argv[i]);
262 continue;
264 vac(f, fdiff, argv[i], d);
265 free(d);
267 if(fdiff)
268 vacfiledecref(fdiff);
270 /*
271 * Record the maximum qid so that vacs can be merged
272 * without introducing overlapping qids. Older versions
273 * of vac arranged that the root would have the largest
274 * qid in the file system, but we can't do that anymore
275 * (the root gets created first!).
276 */
277 if(_vacfsnextqid(fs, &qid) >= 0)
278 vacfilesetqidspace(f, 0, qid);
279 vacfiledecref(f);
281 /*
282 * Copy fsdiff's root block score into fs's slot for that,
283 * so that vacfssync will copy it into root.prev for us.
284 * Just nice documentation, no effect.
285 */
286 if(fsdiff)
287 memmove(fs->score, fsdiff->score, VtScoreSize);
288 if(vacfssync(fs) < 0)
289 fprint(2, "vacfssync: %r\n");
291 fprint(outfd, "vac:%V\n", fs->score);
292 atexitdont(removevacfile);
293 vacfsclose(fs);
294 vthangup(z);
296 if(printstats){
297 fprint(2,
298 "%d files, %d files skipped, %d directories\n"
299 "%lld data bytes written, %lld data bytes skipped\n",
300 stats.nfile, stats.skipfiles, stats.ndir, stats.data, stats.skipdata);
301 dup(2, 1);
302 packetstats();
304 threadexitsall(0);
307 VacFile*
308 recentarchive(VacFs *fs, char *path)
310 VacFile *fp, *f;
311 VacDirEnum *de;
312 VacDir vd;
313 char buf[10];
314 int year, mmdd, nn, n, n1;
315 char *p;
317 fp = vacfsgetroot(fs);
318 de = vdeopen(fp);
319 year = 0;
320 if(de){
321 for(; vderead(de, &vd) > 0; vdcleanup(&vd)){
322 if(strlen(vd.elem) != 4)
323 continue;
324 if((n = strtol(vd.elem, &p, 10)) < 1900 || *p != 0)
325 continue;
326 if(year < n)
327 year = n;
330 vdeclose(de);
331 if(year == 0){
332 vacfiledecref(fp);
333 return nil;
335 snprint(buf, sizeof buf, "%04d", year);
336 if((f = vacfilewalk(fp, buf)) == nil){
337 fprint(2, "warning: dirread %s but cannot walk", buf);
338 vacfiledecref(fp);
339 return nil;
341 fp = f;
343 de = vdeopen(fp);
344 mmdd = 0;
345 nn = 0;
346 if(de){
347 for(; vderead(de, &vd) > 0; vdcleanup(&vd)){
348 if(strlen(vd.elem) < 4)
349 continue;
350 if((n = strtol(vd.elem, &p, 10)) < 100 || n > 1231 || p != vd.elem+4)
351 continue;
352 if(*p == '.'){
353 if(p[1] == '0' || (n1 = strtol(p+1, &p, 10)) == 0 || *p != 0)
354 continue;
355 }else{
356 if(*p != 0)
357 continue;
358 n1 = 0;
360 if(n < mmdd || (n == mmdd && n1 < nn))
361 continue;
362 mmdd = n;
363 nn = n1;
366 vdeclose(de);
367 if(mmdd == 0){
368 vacfiledecref(fp);
369 return nil;
371 if(nn == 0)
372 snprint(buf, sizeof buf, "%04d", mmdd);
373 else
374 snprint(buf, sizeof buf, "%04d.%d", mmdd, nn);
375 if((f = vacfilewalk(fp, buf)) == nil){
376 fprint(2, "warning: dirread %s but cannot walk", buf);
377 vacfiledecref(fp);
378 return nil;
380 vacfiledecref(fp);
382 sprint(path, "%04d/%s", year, buf);
383 return f;
386 static void
387 removevacfile(void)
389 if(vacfile)
390 remove(vacfile);
393 void
394 plan9tovacdir(VacDir *vd, Dir *dir)
396 memset(vd, 0, sizeof *vd);
398 vd->elem = dir->name;
399 vd->uid = dir->uid;
400 vd->gid = dir->gid;
401 vd->mid = dir->muid;
402 if(vd->mid == nil)
403 vd->mid = "";
404 vd->mtime = dir->mtime;
405 vd->mcount = 0;
406 vd->ctime = dir->mtime; /* ctime: not available on plan 9 */
407 vd->atime = dir->atime;
408 vd->size = dir->length;
410 vd->mode = dir->mode & 0777;
411 if(dir->mode & DMDIR)
412 vd->mode |= ModeDir;
413 if(dir->mode & DMAPPEND)
414 vd->mode |= ModeAppend;
415 if(dir->mode & DMEXCL)
416 vd->mode |= ModeExclusive;
417 #ifdef PLAN9PORT
418 if(dir->mode & DMDEVICE)
419 vd->mode |= ModeDevice;
420 if(dir->mode & DMNAMEDPIPE)
421 vd->mode |= ModeNamedPipe;
422 if(dir->mode & DMSYMLINK)
423 vd->mode |= ModeLink;
424 #endif
426 vd->plan9 = 1;
427 vd->p9path = dir->qid.path;
428 vd->p9version = dir->qid.vers;
431 #ifdef PLAN9PORT
432 enum {
433 Special =
434 DMSOCKET |
435 DMSYMLINK |
436 DMNAMEDPIPE |
437 DMDEVICE
438 };
439 #endif
441 /*
442 * Archive the file named name, which has stat info d,
443 * into the vac directory fp (p = parent).
445 * If we're doing a vac -d against another archive, the
446 * equivalent directory to fp in that archive is diffp.
447 */
448 void
449 vac(VacFile *fp, VacFile *diffp, char *name, Dir *d)
451 char *elem, *s;
452 static char *buf;
453 int fd, i, n, bsize;
454 vlong off;
455 Dir *dk; // kids
456 VacDir vd, vddiff;
457 VacFile *f, *fdiff;
458 VtEntry e;
460 if(!includefile(name)){
461 warn("excluding %s%s", name, (d->mode&DMDIR) ? "/" : "");
462 return;
465 if(d->mode&DMDIR)
466 stats.ndir++;
467 else
468 stats.nfile++;
470 if(merge && vacmerge(fp, name) >= 0)
471 return;
473 if(verbose)
474 fprint(2, "%s%s\n", name, (d->mode&DMDIR) ? "/" : "");
476 #ifdef PLAN9PORT
477 if(d->mode&Special)
478 fd = -1;
479 else
480 #endif
481 if((fd = open(name, OREAD)) < 0){
482 warn("open %s: %r", name);
483 return;
486 elem = strrchr(name, '/');
487 if(elem)
488 elem++;
489 else
490 elem = name;
492 plan9tovacdir(&vd, d);
493 if((f = vacfilecreate(fp, elem, vd.mode)) == nil){
494 warn("vacfilecreate %s: %r", name);
495 return;
497 if(diffp)
498 fdiff = vacfilewalk(diffp, elem);
499 else
500 fdiff = nil;
502 if(vacfilesetdir(f, &vd) < 0)
503 warn("vacfilesetdir %s: %r", name);
505 #ifdef PLAN9PORT
506 if(d->mode&(DMSOCKET|DMNAMEDPIPE)){
507 /* don't write anything */
509 else if(d->mode&DMSYMLINK){
510 n = readlink(name, buf, sizeof buf);
511 if(n > 0 && vacfilewrite(f, buf, n, 0) < 0){
512 warn("venti write %s: %r", name);
513 goto Out;
515 stats.data += n;
516 }else if(d->mode&DMDEVICE){
517 snprint(buf, sizeof buf, "%c %d %d",
518 (char)((d->qid.path >> 16) & 0xFF),
519 (int)(d->qid.path & 0xFF),
520 (int)((d->qid.path >> 8) & 0xFF));
521 if(vacfilewrite(f, buf, strlen(buf), 0) < 0){
522 warn("venti write %s: %r", name);
523 goto Out;
525 stats.data += strlen(buf);
526 }else
527 #endif
528 if(d->mode&DMDIR){
529 while((n = dirread(fd, &dk)) > 0){
530 for(i=0; i<n; i++){
531 s = vtmalloc(strlen(name)+1+strlen(dk[i].name)+1);
532 strcpy(s, name);
533 strcat(s, "/");
534 strcat(s, dk[i].name);
535 vac(f, fdiff, s, &dk[i]);
536 free(s);
538 free(dk);
540 }else{
541 off = 0;
542 bsize = fs->bsize;
543 if(buf == nil)
544 buf = vtmallocz(bsize);
545 if(fdiff){
546 /*
547 * Copy fdiff's contents into f by moving the score.
548 * We'll diff and update below.
549 */
550 if(vacfilegetentries(fdiff, &e, nil) >= 0)
551 if(vacfilesetentries(f, &e, nil) >= 0){
552 bsize = e.dsize;
554 /*
555 * Or if -q is set, and the metadata looks the same,
556 * don't even bother reading the file.
557 */
558 if(qdiff && vacfilegetdir(fdiff, &vddiff) >= 0){
559 if(vddiff.mtime == vd.mtime)
560 if(vddiff.size == vd.size)
561 if(!vddiff.plan9 || (/* vddiff.p9path == vd.p9path && */ vddiff.p9version == vd.p9version)){
562 stats.skipfiles++;
563 stats.nfile--;
564 vdcleanup(&vddiff);
565 goto Out;
568 /*
569 * Skip over presumably-unchanged prefix
570 * of an append-only file.
571 */
572 if(vd.mode&ModeAppend)
573 if(vddiff.size < vd.size)
574 if(vddiff.plan9 && vd.plan9)
575 if(vddiff.p9path == vd.p9path){
576 off = vd.size/bsize*bsize;
577 if(seek(fd, off, 0) >= 0)
578 stats.skipdata += off;
579 else{
580 seek(fd, 0, 0); // paranoia
581 off = 0;
585 vdcleanup(&vddiff);
586 // XXX different verbose chatty prints for kaminsky?
590 if(qdiff && verbose)
591 fprint(2, "+%s\n", name);
592 while((n = readn(fd, buf, bsize)) > 0){
593 if(fdiff && sha1matches(f, off/bsize, (uchar*)buf, n)){
594 off += n;
595 stats.skipdata += n;
596 continue;
598 if(vacfilewrite(f, buf, n, off) < 0){
599 warn("venti write %s: %r", name);
600 goto Out;
602 stats.data += n;
603 off += n;
605 /*
606 * Since we started with fdiff's contents,
607 * set the size in case fdiff was bigger.
608 */
609 if(fdiff && vacfilesetsize(f, off) < 0)
610 warn("vtfilesetsize %s: %r", name);
613 Out:
614 vacfileflush(f, 1);
615 vacfiledecref(f);
616 if(fdiff)
617 vacfiledecref(fdiff);
618 close(fd);
621 void
622 vacstdin(VacFile *fp, char *name)
624 vlong off;
625 VacFile *f;
626 static char buf[8192];
627 int n;
629 if((f = vacfilecreate(fp, name, 0666)) == nil){
630 warn("vacfilecreate %s: %r", name);
631 return;
634 off = 0;
635 while((n = read(0, buf, sizeof buf)) > 0){
636 if(vacfilewrite(f, buf, n, off) < 0){
637 warn("venti write %s: %r", name);
638 vacfiledecref(f);
639 return;
641 off += n;
643 vacfileflush(f, 1);
644 vacfiledecref(f);
647 /*
648 * fp is the directory we're writing.
649 * mp is the directory whose contents we're merging in.
650 * d is the directory entry of the file from mp that we want to add to fp.
651 * vacfile is the name of the .vac file, for error messages.
652 * offset is the qid that qid==0 in mp should correspond to.
653 * max is the maximum qid we expect to see (not really needed).
654 */
655 int
656 vacmergefile(VacFile *fp, VacFile *mp, VacDir *d, char *vacfile,
657 vlong offset, vlong max)
659 VtEntry ed, em;
660 VacFile *mf;
661 VacFile *f;
663 mf = vacfilewalk(mp, d->elem);
664 if(mf == nil){
665 warn("could not walk %s in %s", d->elem, vacfile);
666 return -1;
668 if(vacfilegetentries(mf, &ed, &em) < 0){
669 warn("could not get entries for %s in %s", d->elem, vacfile);
670 vacfiledecref(mf);
671 return -1;
674 if((f = vacfilecreate(fp, d->elem, d->mode)) == nil){
675 warn("vacfilecreate %s: %r", d->elem);
676 vacfiledecref(mf);
677 return -1;
679 if(d->qidspace){
680 d->qidoffset += offset;
681 d->qidmax += offset;
682 }else{
683 d->qidspace = 1;
684 d->qidoffset = offset;
685 d->qidmax = max;
687 if(vacfilesetdir(f, d) < 0
688 || vacfilesetentries(f, &ed, &em) < 0
689 || vacfilesetqidspace(f, d->qidoffset, d->qidmax) < 0){
690 warn("vacmergefile %s: %r", d->elem);
691 vacfiledecref(mf);
692 vacfiledecref(f);
693 return -1;
696 vacfiledecref(mf);
697 vacfiledecref(f);
698 return 0;
701 int
702 vacmerge(VacFile *fp, char *name)
704 VacFs *mfs;
705 VacDir vd;
706 VacDirEnum *de;
707 VacFile *mp;
708 uvlong maxqid, offset;
710 if(strlen(name) < 4 || strcmp(name+strlen(name)-4, ".vac") != 0)
711 return -1;
712 if((mfs = vacfsopen(z, name, VtOREAD, CacheSize)) == nil)
713 return -1;
714 if(verbose)
715 fprint(2, "merging %s\n", name);
717 mp = vacfsgetroot(mfs);
718 de = vdeopen(mp);
719 if(de){
720 offset = 0;
721 if(vacfsgetmaxqid(mfs, &maxqid) >= 0){
722 _vacfsnextqid(fs, &offset);
723 vacfsjumpqid(fs, maxqid+1);
725 while(vderead(de, &vd) > 0){
726 if(vd.qid > maxqid){
727 warn("vacmerge %s: maxqid=%lld but %s has %lld",
728 name, maxqid, vd.elem, vd.qid);
729 vacfsjumpqid(fs, vd.qid - maxqid);
730 maxqid = vd.qid;
732 vacmergefile(fp, mp, &vd, name,
733 offset, maxqid);
734 vdcleanup(&vd);
736 vdeclose(de);
738 vacfiledecref(mp);
739 vacfsclose(mfs);
740 return 0;
743 #define TWID64 ((u64int)~(u64int)0)
745 static u64int
746 unittoull(char *s)
748 char *es;
749 u64int n;
751 if(s == nil)
752 return TWID64;
753 n = strtoul(s, &es, 0);
754 if(*es == 'k' || *es == 'K'){
755 n *= 1024;
756 es++;
757 }else if(*es == 'm' || *es == 'M'){
758 n *= 1024*1024;
759 es++;
760 }else if(*es == 'g' || *es == 'G'){
761 n *= 1024*1024*1024;
762 es++;
764 if(*es != '\0')
765 return TWID64;
766 return n;
769 static void
770 warn(char *fmt, ...)
772 va_list arg;
774 va_start(arg, fmt);
775 fprint(2, "vac: ");
776 vfprint(2, fmt, arg);
777 fprint(2, "\n");
778 va_end(arg);