Blob


1 #include "stdinc.h"
2 #include "vac.h"
3 #include "dat.h"
4 #include "fns.h"
6 // TODO: qids
8 void
9 usage(void)
10 {
11 fprint(2, "vac [-imqsv] [-a archive.vac] [-b bsize] [-d old.vac] [-f new.vac] [-e exclude]... [-h host] file...\n");
12 threadexitsall("usage");
13 }
15 enum
16 {
17 BlockSize = 8*1024,
18 };
20 struct
21 {
22 int nfile;
23 int ndir;
24 vlong data;
25 vlong skipdata;
26 int skipfiles;
27 } stats;
29 int qdiff;
30 int merge;
31 int verbose;
32 char *host;
33 VtConn *z;
34 VacFs *fs;
35 char *archivefile;
36 char *vacfile;
38 int vacmerge(VacFile*, char*);
39 void vac(VacFile*, VacFile*, char*, Dir*);
40 void vacstdin(VacFile*, char*);
41 VacFile *recentarchive(VacFs*, char*);
43 static u64int unittoull(char*);
44 static void warn(char *fmt, ...);
45 static void removevacfile(void);
47 #ifdef PLAN9PORT
48 /*
49 * We're between a rock and a hard place here.
50 * The pw library (getpwnam, etc.) reads the
51 * password and group files into an on-stack buffer,
52 * so if you have some huge groups, you overflow
53 * the stack. Because of this, the thread library turns
54 * it off by default, so that dirstat returns "14571" instead of "rsc".
55 * But for vac we want names. So cautiously turn the pwlibrary
56 * back on (see threadmain) and make the main thread stack huge.
57 */
58 extern int _p9usepwlibrary;
59 int mainstacksize = 4*1024*1024;
61 #endif
62 void
63 threadmain(int argc, char **argv)
64 {
65 int i, j, fd, n, printstats;
66 Dir *d;
67 char *s;
68 uvlong u;
69 VacFile *f, *fdiff;
70 VacFs *fsdiff;
71 int blocksize;
72 int outfd;
73 char *stdinname;
74 char *diffvac;
75 uvlong qid;
77 #ifdef PLAN9PORT
78 /* see comment above */
79 _p9usepwlibrary = 1;
80 #endif
82 fmtinstall('F', vtfcallfmt);
83 fmtinstall('H', encodefmt);
84 fmtinstall('V', vtscorefmt);
86 blocksize = BlockSize;
87 stdinname = nil;
88 printstats = 0;
89 fsdiff = nil;
90 diffvac = nil;
92 ARGBEGIN{
93 case 'V':
94 chattyventi++;
95 break;
96 case 'a':
97 archivefile = EARGF(usage());
98 break;
99 case 'b':
100 u = unittoull(EARGF(usage()));
101 if(u < 512)
102 u = 512;
103 if(u > VtMaxLumpSize)
104 u = VtMaxLumpSize;
105 blocksize = u;
106 break;
107 case 'd':
108 diffvac = EARGF(usage());
109 break;
110 case 'e':
111 excludepattern(EARGF(usage()));
112 break;
113 case 'f':
114 vacfile = EARGF(usage());
115 break;
116 case 'h':
117 host = EARGF(usage());
118 break;
119 case 'i':
120 stdinname = EARGF(usage());
121 break;
122 case 'm':
123 merge++;
124 break;
125 case 'q':
126 qdiff++;
127 break;
128 case 's':
129 printstats++;
130 break;
131 case 'v':
132 verbose++;
133 break;
134 case 'x':
135 loadexcludefile(EARGF(usage()));
136 break;
137 default:
138 usage();
139 }ARGEND
141 if(argc == 0 && !stdinname)
142 usage();
144 if(archivefile && (vacfile || diffvac)){
145 fprint(2, "cannot use -a with -f, -d\n");
146 usage();
149 z = vtdial(host);
150 if(z == nil)
151 sysfatal("could not connect to server: %r");
152 if(vtconnect(z) < 0)
153 sysfatal("vtconnect: %r");
155 // Setup:
156 // fs is the output vac file system
157 // f is directory in output vac to write new files
158 // fdiff is corresponding directory in existing vac
159 if(archivefile){
160 VacFile *fp;
161 char yyyy[5];
162 char mmdd[10];
163 char oldpath[40];
164 Tm tm;
166 fdiff = nil;
167 if((outfd = open(archivefile, ORDWR)) < 0){
168 if(access(archivefile, 0) >= 0)
169 sysfatal("open %s: %r", archivefile);
170 if((outfd = create(archivefile, OWRITE, 0666)) < 0)
171 sysfatal("create %s: %r", archivefile);
172 atexit(removevacfile); // because it is new
173 if((fs = vacfscreate(z, blocksize, 512)) == nil)
174 sysfatal("vacfscreate: %r");
175 }else{
176 if((fs = vacfsopen(z, archivefile, VtORDWR, 512)) == nil)
177 sysfatal("vacfsopen %s: %r", archivefile);
178 if((fdiff = recentarchive(fs, oldpath)) != nil){
179 if(verbose)
180 fprint(2, "diff %s\n", oldpath);
181 }else
182 if(verbose)
183 fprint(2, "no recent archive to diff against\n");
186 // Create yyyy/mmdd.
187 tm = *localtime(time(0));
188 snprint(yyyy, sizeof yyyy, "%04d", tm.year+1900);
189 fp = vacfsgetroot(fs);
190 if((f = vacfilewalk(fp, yyyy)) == nil
191 && (f = vacfilecreate(fp, yyyy, ModeDir|0555)) == nil)
192 sysfatal("vacfscreate %s: %r", yyyy);
193 vacfiledecref(fp);
194 fp = f;
196 snprint(mmdd, sizeof mmdd, "%02d%02d", tm.mon+1, tm.mday);
197 n = 0;
198 while((f = vacfilewalk(fp, mmdd)) != nil){
199 vacfiledecref(f);
200 n++;
201 snprint(mmdd+4, sizeof mmdd-4, ".%d", n);
203 f = vacfilecreate(fp, mmdd, ModeDir|0555);
204 if(f == nil)
205 sysfatal("vacfscreate %s/%s: %r", yyyy, mmdd);
206 vacfiledecref(fp);
208 if(verbose)
209 fprint(2, "archive %s/%s\n", yyyy, mmdd);
210 }else{
211 if(vacfile == nil)
212 outfd = 1;
213 else if((outfd = create(vacfile, OWRITE, 0666)) < 0)
214 sysfatal("create %s: %r", vacfile);
215 atexit(removevacfile);
216 if((fs = vacfscreate(z, blocksize, 512)) == nil)
217 sysfatal("vacfscreate: %r");
218 f = vacfsgetroot(fs);
220 fdiff = nil;
221 if(diffvac){
222 if((fsdiff = vacfsopen(z, diffvac, VtOREAD, 128)) == nil)
223 warn("vacfsopen %s: %r", diffvac);
224 else
225 fdiff = vacfsgetroot(fsdiff);
229 if(stdinname)
230 vacstdin(f, stdinname);
231 for(i=0; i<argc; i++){
232 // We can't use / and . and .. and ../.. as valid archive
233 // names, so expand to the list of files in the directory.
234 if(argv[i][0] == 0){
235 warn("empty string given as command-line argument");
236 continue;
238 cleanname(argv[i]);
239 if(strcmp(argv[i], "/") == 0
240 || strcmp(argv[i], ".") == 0
241 || strcmp(argv[i], "..") == 0
242 || (strlen(argv[i]) > 3 && strcmp(argv[i]+strlen(argv[i])-3, "/..") == 0)){
243 if((fd = open(argv[i], OREAD)) < 0){
244 warn("open %s: %r", argv[i]);
245 continue;
247 while((n = dirread(fd, &d)) > 0){
248 for(j=0; j<n; j++){
249 s = vtmalloc(strlen(argv[i])+1+strlen(d[j].name)+1);
250 strcpy(s, argv[i]);
251 strcat(s, "/");
252 strcat(s, d[j].name);
253 cleanname(s);
254 vac(f, fdiff, s, &d[j]);
256 free(d);
258 close(fd);
259 continue;
261 if((d = dirstat(argv[i])) == nil){
262 warn("stat %s: %r", argv[i]);
263 continue;
265 vac(f, fdiff, argv[i], d);
266 free(d);
268 if(fdiff)
269 vacfiledecref(fdiff);
271 /*
272 * Record the maximum qid so that vacs can be merged
273 * without introducing overlapping qids. Older versions
274 * of vac arranged that the root would have the largest
275 * qid in the file system, but we can't do that anymore
276 * (the root gets created first!).
277 */
278 if(_vacfsnextqid(fs, &qid) >= 0)
279 vacfilesetqidspace(f, 0, qid);
280 vacfiledecref(f);
282 /*
283 * Copy fsdiff's root block score into fs's slot for that,
284 * so that vacfssync will copy it into root.prev for us.
285 * Just nice documentation, no effect.
286 */
287 if(fsdiff)
288 memmove(fs->score, fsdiff->score, VtScoreSize);
289 if(vacfssync(fs) < 0)
290 fprint(2, "vacfssync: %r\n");
292 fprint(outfd, "vac:%V\n", fs->score);
293 atexitdont(removevacfile);
294 vacfsclose(fs);
295 vthangup(z);
297 if(printstats){
298 fprint(2,
299 "%d files, %d files skipped, %d directories\n"
300 "%lld data bytes written, %lld data bytes skipped\n",
301 stats.nfile, stats.skipfiles, stats.ndir, stats.data, stats.skipdata);
302 dup(2, 1);
303 packetstats();
305 threadexitsall(0);
308 VacFile*
309 recentarchive(VacFs *fs, char *path)
311 VacFile *fp, *f;
312 VacDirEnum *de;
313 VacDir vd;
314 char buf[10];
315 int year, mmdd, nn, n, n1;
316 char *p;
318 fp = vacfsgetroot(fs);
319 de = vdeopen(fp);
320 year = 0;
321 if(de){
322 for(; vderead(de, &vd) > 0; vdcleanup(&vd)){
323 if(strlen(vd.elem) != 4)
324 continue;
325 if((n = strtol(vd.elem, &p, 10)) < 1900 || *p != 0)
326 continue;
327 if(year < n)
328 year = n;
331 vdeclose(de);
332 if(year == 0){
333 vacfiledecref(fp);
334 return nil;
336 snprint(buf, sizeof buf, "%04d", year);
337 if((f = vacfilewalk(fp, buf)) == nil){
338 fprint(2, "warning: dirread %s but cannot walk", buf);
339 vacfiledecref(fp);
340 return nil;
342 fp = f;
344 de = vdeopen(fp);
345 mmdd = 0;
346 nn = 0;
347 if(de){
348 for(; vderead(de, &vd) > 0; vdcleanup(&vd)){
349 if(strlen(vd.elem) < 4)
350 continue;
351 if((n = strtol(vd.elem, &p, 10)) < 100 || n > 1231 || p != vd.elem+4)
352 continue;
353 if(*p == '.'){
354 if(p[1] == '0' || (n1 = strtol(p+1, &p, 10)) == 0 || *p != 0)
355 continue;
356 }else{
357 if(*p != 0)
358 continue;
359 n1 = 0;
361 if(n < mmdd || (n == mmdd && n1 < nn))
362 continue;
363 mmdd = n;
364 nn = n1;
367 vdeclose(de);
368 if(mmdd == 0){
369 vacfiledecref(fp);
370 return nil;
372 if(nn == 0)
373 snprint(buf, sizeof buf, "%04d", mmdd);
374 else
375 snprint(buf, sizeof buf, "%04d.%d", mmdd, nn);
376 if((f = vacfilewalk(fp, buf)) == nil){
377 fprint(2, "warning: dirread %s but cannot walk", buf);
378 vacfiledecref(fp);
379 return nil;
381 vacfiledecref(fp);
383 sprint(path, "%04d/%s", year, buf);
384 return f;
387 static void
388 removevacfile(void)
390 if(vacfile)
391 remove(vacfile);
394 void
395 plan9tovacdir(VacDir *vd, Dir *dir)
397 memset(vd, 0, sizeof *vd);
399 vd->elem = dir->name;
400 vd->uid = dir->uid;
401 vd->gid = dir->gid;
402 vd->mid = dir->muid;
403 if(vd->mid == nil)
404 vd->mid = "";
405 vd->mtime = dir->mtime;
406 vd->mcount = 0;
407 vd->ctime = dir->mtime; /* ctime: not available on plan 9 */
408 vd->atime = dir->atime;
409 vd->size = dir->length;
411 vd->mode = dir->mode & 0777;
412 if(dir->mode & DMDIR)
413 vd->mode |= ModeDir;
414 if(dir->mode & DMAPPEND)
415 vd->mode |= ModeAppend;
416 if(dir->mode & DMEXCL)
417 vd->mode |= ModeExclusive;
418 #ifdef PLAN9PORT
419 if(dir->mode & DMDEVICE)
420 vd->mode |= ModeDevice;
421 if(dir->mode & DMNAMEDPIPE)
422 vd->mode |= ModeNamedPipe;
423 if(dir->mode & DMSYMLINK)
424 vd->mode |= ModeLink;
425 #endif
427 vd->plan9 = 1;
428 vd->p9path = dir->qid.path;
429 vd->p9version = dir->qid.vers;
432 #ifdef PLAN9PORT
433 enum {
434 Special =
435 DMSOCKET |
436 DMSYMLINK |
437 DMNAMEDPIPE |
438 DMDEVICE
439 };
440 #endif
442 /*
443 * Does block b of f have the same SHA1 hash as the n bytes at buf?
444 */
445 static int
446 sha1matches(VacFile *f, ulong b, uchar *buf, int n)
448 uchar fscore[VtScoreSize];
449 uchar bufscore[VtScoreSize];
451 if(vacfileblockscore(f, b, fscore) < 0)
452 return 0;
453 n = vtzerotruncate(VtDataType, buf, n);
454 sha1(buf, n, bufscore, nil);
455 if(memcmp(bufscore, fscore, VtScoreSize) == 0)
456 return 1;
457 return 0;
460 /*
461 * Archive the file named name, which has stat info d,
462 * into the vac directory fp (p = parent).
464 * If we're doing a vac -d against another archive, the
465 * equivalent directory to fp in that archive is diffp.
466 */
467 void
468 vac(VacFile *fp, VacFile *diffp, char *name, Dir *d)
470 char *elem, *s;
471 static char buf[65536];
472 int fd, i, n, bsize;
473 vlong off;
474 Dir *dk; // kids
475 VacDir vd, vddiff;
476 VacFile *f, *fdiff;
477 VtEntry e;
479 if(!includefile(name)){
480 warn("excluding %s%s", name, (d->mode&DMDIR) ? "/" : "");
481 return;
484 if(d->mode&DMDIR)
485 stats.ndir++;
486 else
487 stats.nfile++;
489 if(merge && vacmerge(fp, name) >= 0)
490 return;
492 if(verbose)
493 fprint(2, "%s%s\n", name, (d->mode&DMDIR) ? "/" : "");
495 #ifdef PLAN9PORT
496 if(d->mode&Special)
497 fd = -1;
498 else
499 #endif
500 if((fd = open(name, OREAD)) < 0){
501 warn("open %s: %r", name);
502 return;
505 elem = strrchr(name, '/');
506 if(elem)
507 elem++;
508 else
509 elem = name;
511 plan9tovacdir(&vd, d);
512 if((f = vacfilecreate(fp, elem, vd.mode)) == nil){
513 warn("vacfilecreate %s: %r", name);
514 return;
516 if(diffp)
517 fdiff = vacfilewalk(diffp, elem);
518 else
519 fdiff = nil;
521 if(vacfilesetdir(f, &vd) < 0)
522 warn("vacfilesetdir %s: %r", name);
524 #ifdef PLAN9PORT
525 if(d->mode&(DMSOCKET|DMNAMEDPIPE)){
526 /* don't write anything */
528 else if(d->mode&DMSYMLINK){
529 n = readlink(name, buf, sizeof buf);
530 if(n > 0 && vacfilewrite(f, buf, n, 0) < 0){
531 warn("venti write %s: %r", name);
532 goto Out;
534 stats.data += n;
535 }else if(d->mode&DMDEVICE){
536 snprint(buf, sizeof buf, "%c %d %d",
537 (char)((d->qid.path >> 16) & 0xFF),
538 (int)(d->qid.path & 0xFF),
539 (int)((d->qid.path >> 8) & 0xFF));
540 if(vacfilewrite(f, buf, strlen(buf), 0) < 0){
541 warn("venti write %s: %r", name);
542 goto Out;
544 stats.data += strlen(buf);
545 }else
546 #endif
547 if(d->mode&DMDIR){
548 while((n = dirread(fd, &dk)) > 0){
549 for(i=0; i<n; i++){
550 s = vtmalloc(strlen(name)+1+strlen(dk[i].name)+1);
551 strcpy(s, name);
552 strcat(s, "/");
553 strcat(s, dk[i].name);
554 vac(f, fdiff, s, &dk[i]);
555 free(s);
557 free(dk);
559 }else{
560 off = 0;
561 bsize = fs->bsize;
562 if(fdiff){
563 /*
564 * Copy fdiff's contents into f by moving the score.
565 * We'll diff and update below.
566 */
567 if(vacfilegetentries(fdiff, &e, nil) >= 0)
568 if(vacfilesetentries(f, &e, nil) >= 0){
569 bsize = e.dsize;
571 /*
572 * Or if -q is set, and the metadata looks the same,
573 * don't even bother reading the file.
574 */
575 if(qdiff && vacfilegetdir(fdiff, &vddiff) >= 0){
576 if(vddiff.mtime == vd.mtime)
577 if(vddiff.size == vd.size)
578 if(!vddiff.plan9 || (/* vddiff.p9path == vd.p9path && */ vddiff.p9version == vd.p9version)){
579 stats.skipfiles++;
580 stats.nfile--;
581 vdcleanup(&vddiff);
582 goto Out;
585 /*
586 * Skip over presumably-unchanged prefix
587 * of an append-only file.
588 */
589 if(vd.mode&ModeAppend)
590 if(vddiff.size < vd.size)
591 if(vddiff.plan9 && vd.plan9)
592 if(vddiff.p9path == vd.p9path){
593 off = vd.size/bsize*bsize;
594 if(seek(fd, off, 0) >= 0)
595 stats.skipdata += off;
596 else{
597 seek(fd, 0, 0); // paranoia
598 off = 0;
602 vdcleanup(&vddiff);
603 // XXX different verbose chatty prints for kaminsky?
607 if(qdiff && verbose)
608 fprint(2, "+%s\n", name);
609 while((n = readn(fd, buf, bsize)) > 0){
610 if(fdiff && sha1matches(f, off/bsize, (uchar*)buf, n)){
611 off += n;
612 stats.skipdata += n;
613 continue;
615 if(vacfilewrite(f, buf, n, off) < 0){
616 warn("venti write %s: %r", name);
617 goto Out;
619 stats.data += n;
620 off += n;
622 /*
623 * Since we started with fdiff's contents,
624 * set the size in case fdiff was bigger.
625 */
626 if(fdiff && vacfilesetsize(f, off) < 0)
627 warn("vtfilesetsize %s: %r", name);
630 Out:
631 vacfileflush(f, 1);
632 vacfiledecref(f);
633 if(fdiff)
634 vacfiledecref(fdiff);
635 close(fd);
638 void
639 vacstdin(VacFile *fp, char *name)
641 vlong off;
642 VacFile *f;
643 static char buf[8192];
644 int n;
646 if((f = vacfilecreate(fp, name, 0666)) == nil){
647 warn("vacfilecreate %s: %r", name);
648 return;
651 off = 0;
652 while((n = read(0, buf, sizeof buf)) > 0){
653 if(vacfilewrite(f, buf, n, off) < 0){
654 warn("venti write %s: %r", name);
655 vacfiledecref(f);
656 return;
658 off += n;
660 vacfileflush(f, 1);
661 vacfiledecref(f);
664 /*
665 * fp is the directory we're writing.
666 * mp is the directory whose contents we're merging in.
667 * d is the directory entry of the file from mp that we want to add to fp.
668 * vacfile is the name of the .vac file, for error messages.
669 * offset is the qid that qid==0 in mp should correspond to.
670 * max is the maximum qid we expect to see (not really needed).
671 */
672 int
673 vacmergefile(VacFile *fp, VacFile *mp, VacDir *d, char *vacfile,
674 vlong offset, vlong max)
676 VtEntry ed, em;
677 VacFile *mf;
678 VacFile *f;
680 mf = vacfilewalk(mp, d->elem);
681 if(mf == nil){
682 warn("could not walk %s in %s", d->elem, vacfile);
683 return -1;
685 if(vacfilegetentries(mf, &ed, &em) < 0){
686 warn("could not get entries for %s in %s", d->elem, vacfile);
687 vacfiledecref(mf);
688 return -1;
691 if((f = vacfilecreate(fp, d->elem, d->mode)) == nil){
692 warn("vacfilecreate %s: %r", d->elem);
693 vacfiledecref(mf);
694 return -1;
696 if(d->qidspace){
697 d->qidoffset += offset;
698 d->qidmax += offset;
699 }else{
700 d->qidspace = 1;
701 d->qidoffset = offset;
702 d->qidmax = max;
704 if(vacfilesetdir(f, d) < 0
705 || vacfilesetentries(f, &ed, &em) < 0
706 || vacfilesetqidspace(f, d->qidoffset, d->qidmax) < 0){
707 warn("vacmergefile %s: %r", d->elem);
708 vacfiledecref(mf);
709 vacfiledecref(f);
710 return -1;
713 vacfiledecref(mf);
714 vacfiledecref(f);
715 return 0;
718 int
719 vacmerge(VacFile *fp, char *name)
721 VacFs *mfs;
722 VacDir vd;
723 VacDirEnum *de;
724 VacFile *mp;
725 uvlong maxqid, offset;
727 if(strlen(name) < 4 || strcmp(name+strlen(name)-4, ".vac") != 0)
728 return -1;
729 if((mfs = vacfsopen(z, name, VtOREAD, 100)) == nil)
730 return -1;
731 if(verbose)
732 fprint(2, "merging %s\n", name);
734 mp = vacfsgetroot(mfs);
735 de = vdeopen(mp);
736 if(de){
737 offset = 0;
738 if(vacfsgetmaxqid(mfs, &maxqid) >= 0){
739 _vacfsnextqid(fs, &offset);
740 vacfsjumpqid(fs, maxqid+1);
742 while(vderead(de, &vd) > 0){
743 if(vd.qid > maxqid){
744 warn("vacmerge %s: maxqid=%lld but %s has %lld",
745 name, maxqid, vd.elem, vd.qid);
746 vacfsjumpqid(fs, vd.qid - maxqid);
747 maxqid = vd.qid;
749 vacmergefile(fp, mp, &vd, name,
750 offset, maxqid);
751 vdcleanup(&vd);
753 vdeclose(de);
755 vacfiledecref(mp);
756 vacfsclose(mfs);
757 return 0;
760 #define TWID64 ((u64int)~(u64int)0)
762 static u64int
763 unittoull(char *s)
765 char *es;
766 u64int n;
768 if(s == nil)
769 return TWID64;
770 n = strtoul(s, &es, 0);
771 if(*es == 'k' || *es == 'K'){
772 n *= 1024;
773 es++;
774 }else if(*es == 'm' || *es == 'M'){
775 n *= 1024*1024;
776 es++;
777 }else if(*es == 'g' || *es == 'G'){
778 n *= 1024*1024*1024;
779 es++;
781 if(*es != '\0')
782 return TWID64;
783 return n;
786 static void
787 warn(char *fmt, ...)
789 va_list arg;
791 va_start(arg, fmt);
792 fprint(2, "vac: ");
793 vfprint(2, fmt, arg);
794 fprint(2, "\n");
795 va_end(arg);