8 int mainstacksize = 128*1024;
10 typedef struct Sink Sink;
11 typedef struct MetaSink MetaSink;
12 typedef struct DirSink DirSink;
18 uchar *pbuf[VtPointerDepth+1];
26 uchar *p; /* current pointer */
27 uchar *ep; /* end pointer */
35 uchar *rp; /* start of current record */
36 uchar *p; /* current pointer */
37 uchar *ep; /* end pointer */
40 static void usage(void);
41 static int strpcmp(const void*, const void*);
42 static void warn(char *fmt, ...);
43 static void cleanup(void);
44 static u64int unittoull(char *s);
45 static void vac(VtConn *z, char *argv[]);
46 static void vacfile(DirSink *dsink, char *lname, char *sname, VacFile*);
47 static void vacstdin(DirSink *dsink, char *name, VacFile *vf);
48 static void vacdata(DirSink *dsink, int fd, char *lname, VacFile*, Dir*);
49 static void vacdir(DirSink *dsink, int fd, char *lname, char *sname, VacFile*);
50 static int vacmerge(DirSink *dsink, char *lname, char *sname);
52 Sink *sinkalloc(VtConn *z, int psize, int dsize);
53 void sinkwrite(Sink *k, uchar *data, int n);
54 void sinkwritescore(Sink *k, uchar *score, int n);
55 void sinkclose(Sink *k);
56 void sinkfree(Sink *k);
58 DirSink *dirsinkalloc(VtConn *z, int psize, int dsize);
59 void dirsinkwrite(DirSink *k, VtEntry*);
60 void dirsinkwritesink(DirSink *k, Sink*);
61 int dirsinkwritefile(DirSink *k, VacFile *vf);
62 void dirsinkclose(DirSink *k);
63 void dirsinkfree(DirSink *k);
65 MetaSink *metasinkalloc(VtConn *z, int psize, int dsize);
66 void metasinkputc(MetaSink *k, int c);
67 void metasinkputstring(MetaSink *k, char *s);
68 void metasinkputuint32(MetaSink *k, ulong x);
69 void metasinkputuint64(MetaSink *k, uvlong x);
70 void metasinkwrite(MetaSink *k, uchar *data, int n);
71 void metasinkwritedir(MetaSink *ms, VacDir *vd);
72 void metasinkeor(MetaSink *k);
73 void metasinkclose(MetaSink *k);
74 void metasinkfree(MetaSink *k);
75 void plan9tovacdir(VacDir*, Dir*, ulong entry, uvlong qid);
92 int bsize = BlockSize;
98 char *exclude[MaxExclude];
107 fprint(2, "usage: %s [-amqsv] [-h host] [-d vacfile] [-b blocksize] [-i name] [-e exclude] [-f vacfile] file ... \n", argv0);
112 threadmain(int argc, char *argv[])
128 bsize = unittoull(p);
138 if(nexclude >= MaxExclude)
139 sysfatal("too many exclusions\n");
140 exclude[nexclude] = ARGF();
141 if(exclude[nexclude] == nil)
182 if(bsize > VtMaxLumpSize)
183 bsize = VtMaxLumpSize;
186 fmtinstall('V', vtscorefmt);
190 sysfatal("could not connect to server: %r");
193 sysfatal("vtconnect: %r");
195 qsort(exclude, nexclude, sizeof(char*), strpcmp);
200 fprint(2, "warning: could not ask server to flush pending writes: %r\n");
203 fprint(2, "files %ld:%ld data %ld:%ld:%ld meta %ld\n", stats.file, stats.sfile,
204 stats.data, stats.skip, stats.sdata, stats.meta);
212 strpcmp(const void *p0, const void *p1)
214 return strcmp(*(char**)p0, *(char**)p1);
218 vacwrite(VtConn *z, uchar score[VtScoreSize], int type, uchar *buf, int n)
222 sha1(buf, n, score, nil);
225 return vtwrite(z, score, type, buf, n);
229 lastelem(char *oname)
235 if((p = strrchr(oname, '/')) == nil)
241 vac(VtConn *z, char *argv[])
246 uchar score[VtScoreSize], buf[VtRootSize];
256 if(getwd(cwd, sizeof(cwd)) == 0)
257 sysfatal("can't find current directory: %r\n");
259 dsink = dirsinkalloc(z, bsize, bsize);
263 fs = vacfsopen(z, dfile, VtOREAD, 1000);
265 fprint(2, "could not open diff: %s: %r\n", dfile);
270 fd = create(oname, OWRITE, 0666);
272 sysfatal("could not create file: %s: %r", oname);
278 sysfatal("dirfstat failed: %r");
280 dir->name = lastelem(oname);
284 for(; *argv; argv++) {
287 for (cp = *argv; *cp; cp++)
299 vff = vacfileopen(fs, cp2);
300 vacfile(dsink, argv[0], cp2, vff);
303 if(cd && chdir(cwd) < 0)
304 sysfatal("can't cd back to %s: %r\n", cwd);
310 vff = vacfileopen(fs, isi);
311 vacstdin(dsink, isi, vff);
318 /* build meta information for the root */
319 ms = metasinkalloc(z, bsize, bsize);
320 /* fake into a directory */
321 dir->mode |= (dir->mode&0444)>>2;
322 dir->qid.type |= QTDIR;
324 plan9tovacdir(&vd, dir, 0, fileid++);
325 if(strcmp(vd.elem, "/") == 0){
327 vd.elem = vtstrdup("root");
329 metasinkwritedir(ms, &vd);
333 ds = dirsinkalloc(z, bsize, bsize);
334 dirsinkwritesink(ds, dsink->sink);
335 dirsinkwritesink(ds, dsink->msink->sink);
336 dirsinkwritesink(ds, ms->sink);
339 memset(&root, 0, sizeof(root));
340 strncpy(root.name, dir->name, sizeof(root.name));
341 root.name[sizeof(root.name)-1] = 0;
343 sprint(root.type, "vac");
344 memmove(root.score, ds->sink->dir.score, VtScoreSize);
345 root.blocksize = maxbsize;
347 vacfsgetscore(fs, root.prev);
355 vtrootpack(&root, buf);
356 if(vacwrite(z, score, VtRootType, buf, VtRootSize) < 0)
357 sysfatal("vacWrite failed: %r");
359 fprint(fd, "vac:%V\n", score);
361 /* avoid remove at cleanup */
366 isexcluded(char *name)
374 x = strcmp(exclude[i], name);
390 if(lstat(name, &st) < 0)
392 if((st.st_mode&S_IFMT) == S_IFLNK)
398 vacfile(DirSink *dsink, char *lname, char *sname, VacFile *vf)
405 if(isexcluded(lname)) {
406 warn("excluding: %s", lname);
410 if(merge && vacmerge(dsink, lname, sname) >= 0)
416 fd = open(sname, OREAD);
418 warn("could not open file: %s: %r", lname);
423 fprint(2, "%s\n", lname);
427 warn("can't stat %s: %r", lname);
431 dir->name = lastelem(sname);
433 entry = dsink->nentry;
435 if(dir->mode & DMDIR)
436 vacdir(dsink, fd, lname, sname, vf);
438 vacdata(dsink, fd, lname, vf, dir);
440 plan9tovacdir(&vd, dir, entry, fileid++);
441 metasinkwritedir(dsink->msink, &vd);
449 vacstdin(DirSink *dsink, char *name, VacFile *vf)
456 fprint(2, "%s\n", "<stdio>");
460 warn("can't stat <stdio>: %r");
465 entry = dsink->nentry;
467 vacdata(dsink, 0, "<stdin>", vf, dir);
469 plan9tovacdir(&vd, dir, entry, fileid++);
470 vd.elem = vtstrdup(name);
471 metasinkwritedir(dsink->msink, &vd);
478 sha1check(u8int *score, uchar *buf, int n)
480 char score2[VtScoreSize];
482 sha1(buf, n, score, nil);
483 if(memcmp(score, score2, VtScoreSize) == 0)
489 vacdataskip(Sink *sink, VacFile *vf, int fd, ulong blocks, uchar *buf, char *lname)
493 uchar score[VtScoreSize];
495 /* skip blocks for append only files */
496 if(seek(fd, (blocks-1)*bsize, 0) != (blocks-1)*bsize) {
497 warn("error seeking: %s", lname);
500 n = readn(fd, buf, bsize);
502 warn("error checking append only file: %s", lname);
505 if(vacfileblockscore(vf, blocks-1, score)<0 || sha1check(score, buf, n)<0) {
506 warn("last block of append file did not match: %s", lname);
510 for(i=0; i<blocks; i++) {
511 if(vacfileblockscore(vf, i, score) < 0) {
512 warn("could not get score: %s: %lud", lname, i);
513 seek(fd, i*bsize, 0);
517 sinkwritescore(sink, score, bsize);
527 vacdata(DirSink *dsink, int fd, char *lname, VacFile *vf, Dir *dir)
532 uchar score[VtScoreSize];
538 if(vf != nil && qdiff) {
539 vacfilegetdir(vf, &vd);
540 if(vd.mtime == dir->mtime)
541 if(vd.size == dir->length)
542 if(!vd.plan9 || /* vd.p9path == dir->qid.path && */ vd.p9version == dir->qid.vers)
543 if(dirsinkwritefile(dsink, vf)) {
549 /* look for an append only file */
550 if((dir->mode&DMAPPEND) != 0)
551 if(vd.size < dir->length)
553 if(vd.p9path == dir->qid.path)
554 vfblocks = vd.size/bsize;
560 buf = vtmalloc(bsize);
561 sink = sinkalloc(dsink->sink->z, bsize, bsize);
563 same = stats.sdata+stats.skip;
566 block += vacdataskip(sink, vf, fd, vfblocks, buf, lname);
568 if(0) fprint(2, "vacData: %s: %ld\n", lname, block);
570 n = readn(fd, buf, bsize);
572 warn("file truncated due to read error: %s: %r", lname);
575 if(vf != nil && vacfileblockscore(vf, block, score) && sha1check(score, buf, n)>=0) {
577 sinkwritescore(sink, score, n);
579 sinkwrite(sink, buf, n);
582 same = stats.sdata+stats.skip - same;
584 if(same && (dir->mode&DMAPPEND) != 0)
585 if(0)fprint(2, "%s: total %lud same %lud:%lud diff %lud\n",
586 lname, block, same, vfblocks, block-same);
589 dirsinkwritesink(dsink, sink);
596 vacdir(DirSink *dsink, int fd, char *lname, char *sname, VacFile *vf)
605 ds = dirsinkalloc(dsink->sink->z, bsize, bsize);
606 while((nd = dirread(fd, &dirs)) > 0){
607 for(i = 0; i < nd; i++){
609 /* check for bad file names */
610 if(name[0] == 0 || strcmp(name, ".") == 0 || strcmp(name, "..") == 0)
612 ln = vtmalloc(strlen(lname) + strlen(name) + 2);
613 sn = vtmalloc(strlen(sname) + strlen(name) + 2);
614 sprint(ln, "%s/%s", lname, name);
615 sprint(sn, "%s/%s", sname, name);
617 vvf = vacfilewalk(vf, name);
620 vacfile(ds, ln, sn, vvf);
629 dirsinkwritesink(dsink, ds->sink);
630 dirsinkwritesink(dsink, ds->msink->sink);
635 vacmergefile(DirSink *dsink, VacFile *vf, VacDir *dir, uvlong offset, uvlong *max)
637 uchar buf[VtEntrySize];
641 if(vacfileread(vf, buf, VtEntrySize, (uvlong)dir->entry*VtEntrySize) != VtEntrySize) {
642 warn("could not read venti dir entry: %s\n", dir->elem);
645 vtentryunpack(&dd, buf, 0);
647 if(dir->mode & ModeDir) {
652 if(vacfileread(vf, buf, VtEntrySize, e*VtEntrySize) != VtEntrySize) {
653 warn("could not read venti dir entry: %s\n", dir->elem);
656 vtentryunpack(&md, buf, 0);
659 /* max might incorrect in some old dumps */
660 if(dir->qid >= *max) {
661 warn("qid out of range: %s", dir->elem);
666 dir->entry = dsink->nentry;
669 dir->qidoffset += offset;
672 dir->qidoffset = offset;
676 dirsinkwrite(dsink, &dd);
677 if(dir->mode & ModeDir)
678 dirsinkwrite(dsink, &md);
679 metasinkwritedir(dsink->msink, dir);
685 vacmerge(DirSink *dsink, char *lname, char *sname)
694 p = strrchr(sname, '.');
695 if(p == 0 || strcmp(p, ".vac"))
699 fs = vacfsopen(dsink->sink->z, sname, VtOREAD, 100);
703 vf = vacfileopen(fs, "/");
706 max = vacfilegetid(vf);
712 fprint(2, "merging: %s\n", lname);
714 if(maxbsize < fs->bsize)
715 maxbsize = fs->bsize;
718 if(vderead(d, &dir) < 1)
720 vacmergefile(dsink, vf, &dir, fileid, &max);
735 sinkalloc(VtConn *z, int psize, int dsize)
740 if(psize < 512 || psize > VtMaxLumpSize)
741 sysfatal("sinkalloc: bad psize");
742 if(dsize < 512 || dsize > VtMaxLumpSize)
743 sysfatal("sinkalloc: bad psize");
745 psize = VtScoreSize*(psize/VtScoreSize);
747 k = vtmallocz(sizeof(Sink));
749 k->dir.flags = VtEntryActive;
750 k->dir.psize = psize;
751 k->dir.dsize = dsize;
752 k->buf = vtmallocz(VtPointerDepth*k->dir.psize + VtScoreSize);
753 for(i=0; i<=VtPointerDepth; i++)
754 k->pbuf[i] = k->buf + i*k->dir.psize;
759 sinkwritescore(Sink *k, uchar score[VtScoreSize], int n)
765 memmove(k->pbuf[0], score, VtScoreSize);
769 for(i=0; i<VtPointerDepth; i++) {
770 k->pbuf[i] += VtScoreSize;
771 if(k->pbuf[i] < k->buf + d->psize*(i+1))
773 if(i == VtPointerDepth-1)
774 sysfatal("file too big");
775 p = k->buf+i*d->psize;
777 if(vacwrite(k->z, k->pbuf[i+1], VtDataType+1+i, p, d->psize) < 0)
778 sysfatal("vacwrite failed: %r");
782 /* round size up to multiple of dsize */
783 d->size = d->dsize * ((d->size + d->dsize-1)/d->dsize);
789 sinkwrite(Sink *k, uchar *p, int n)
792 uchar score[VtScoreSize];
795 sysfatal("sinkWrite: size too big");
797 if((k->dir.type&~VtTypeDepthMask) == VtDirType){
804 if(vacwrite(k->z, score, type, p, n) < 0)
805 sysfatal("vacWrite failed: %r");
807 sinkwritescore(k, score, n);
811 sizetodepth(uvlong s, int psize, int dsize)
816 /* determine pointer depth */
817 np = psize/VtScoreSize;
818 s = (s + dsize - 1)/dsize;
819 for(d = 0; s > 1; d++)
835 memmove(kd->score, vtzeroscore, VtScoreSize);
839 for(n=VtPointerDepth-1; n>0; n--)
840 if(k->pbuf[n] > k->buf + kd->psize*n)
843 base = kd->type&~VtTypeDepthMask;
844 kd->type = base + sizetodepth(kd->size, kd->psize, kd->dsize);
846 /* skip full part of tree */
847 for(i=0; i<n && k->pbuf[i] == k->buf + kd->psize*i; i++)
850 /* is the tree completely full */
851 if(i == n && k->pbuf[n] == k->buf + kd->psize*n + VtScoreSize) {
852 memmove(kd->score, k->pbuf[n] - VtScoreSize, VtScoreSize);
857 /* clean up the edge */
859 p = k->buf+i*kd->psize;
861 if(vacwrite(k->z, k->pbuf[i+1], base+1+i, p, k->pbuf[i]-p) < 0)
862 sysfatal("vacWrite failed: %r");
863 k->pbuf[i+1] += VtScoreSize;
865 memmove(kd->score, k->pbuf[i] - VtScoreSize, VtScoreSize);
876 dirsinkalloc(VtConn *z, int psize, int dsize)
881 ds = VtEntrySize*(dsize/VtEntrySize);
883 k = vtmallocz(sizeof(DirSink));
884 k->sink = sinkalloc(z, psize, ds);
885 k->sink->dir.type = VtDirType;
886 k->msink = metasinkalloc(z, psize, dsize);
887 k->buf = vtmalloc(ds);
894 dirsinkwrite(DirSink *k, VtEntry *dir)
896 if(k->p + VtEntrySize > k->ep) {
897 sinkwrite(k->sink, k->buf, k->p - k->buf);
900 vtentrypack(dir, k->p, 0);
906 dirsinkwritesink(DirSink *k, Sink *sink)
908 dirsinkwrite(k, &sink->dir);
912 dirsinkwritefile(DirSink *k, VacFile *vf)
916 if(vacfilegetvtentry(vf, &dir) < 0)
918 dirsinkwrite(k, &dir);
923 dirsinkclose(DirSink *k)
925 metasinkclose(k->msink);
927 sinkwrite(k->sink, k->buf, k->p - k->buf);
932 dirsinkfree(DirSink *k)
935 metasinkfree(k->msink);
941 metasinkalloc(VtConn *z, int psize, int dsize)
945 k = vtmallocz(sizeof(MetaSink));
946 k->sink = sinkalloc(z, psize, dsize);
947 k->buf = vtmalloc(dsize);
948 k->maxindex = dsize/100; /* 100 byte entries seems reasonable */
951 k->rp = k->p = k->buf + MetaHeaderSize + k->maxindex*MetaIndexSize;
952 k->ep = k->buf + dsize;
956 /* hack to get base to compare routine - not reentrant */
960 dircmp(const void *p0, const void *p1)
965 /* name is first element of entry */
967 q0 = blockbase + (q0[0]<<8) + q0[1];
968 n0 = (q0[6]<<8) + q0[7];
972 q1 = blockbase + (q1[0]<<8) + q1[1];
973 n1 = (q1[6]<<8) + q1[7];
977 return memcmp(q0, q1, n0);
979 r = memcmp(q0, q1, n0);
982 r = memcmp(q0, q1, n1);
988 metasinkflush(MetaSink *k)
996 assert(k->nindex <= k->maxindex);
1003 mb.nindex = k->nindex;
1004 mb.maxindex = k->maxindex;
1008 p += MetaHeaderSize;
1010 /* XXX this is not reentrant! */
1012 qsort(p, k->nindex, MetaIndexSize, dircmp);
1013 p += k->nindex*MetaIndexSize;
1015 memset(p, 0, (k->maxindex-k->nindex)*MetaIndexSize);
1016 p += (k->maxindex-k->nindex)*MetaIndexSize;
1018 sinkwrite(k->sink, k->buf, n);
1020 /* move down partial entry */
1022 memmove(p, k->rp, n);
1029 metasinkputc(MetaSink *k, int c)
1034 sysfatal("directory entry too large");
1040 metasinkputstring(MetaSink *k, char *s)
1043 metasinkputc(k, n>>8);
1045 metasinkwrite(k, (uchar*)s, n);
1049 metasinkputuint32(MetaSink *k, ulong x)
1051 metasinkputc(k, x>>24);
1052 metasinkputc(k, x>>16);
1053 metasinkputc(k, x>>8);
1058 metasinkputuint64(MetaSink *k, uvlong x)
1060 metasinkputuint32(k, x>>32);
1061 metasinkputuint32(k, x);
1065 metasinkwrite(MetaSink *k, uchar *data, int n)
1067 if(k->p + n > k->ep)
1069 if(k->p + n > k->ep)
1070 sysfatal("directory entry too large");
1072 memmove(k->p, data, n);
1077 metasinkwritedir(MetaSink *ms, VacDir *dir)
1079 metasinkputuint32(ms, DirMagic);
1080 metasinkputc(ms, Version>>8);
1081 metasinkputc(ms, Version);
1082 metasinkputstring(ms, dir->elem);
1083 metasinkputuint32(ms, dir->entry);
1084 metasinkputuint64(ms, dir->qid);
1085 metasinkputstring(ms, dir->uid);
1086 metasinkputstring(ms, dir->gid);
1087 metasinkputstring(ms, dir->mid);
1088 metasinkputuint32(ms, dir->mtime);
1089 metasinkputuint32(ms, dir->mcount);
1090 metasinkputuint32(ms, dir->ctime);
1091 metasinkputuint32(ms, dir->atime);
1092 metasinkputuint32(ms, dir->mode);
1095 metasinkputc(ms, DirPlan9Entry); /* plan9 extra info */
1096 metasinkputc(ms, 0); /* plan9 extra size */
1097 metasinkputc(ms, 12); /* plan9 extra size */
1098 metasinkputuint64(ms, dir->p9path);
1099 metasinkputuint32(ms, dir->p9version);
1102 if(dir->qidspace != 0) {
1103 metasinkputc(ms, DirQidSpaceEntry);
1104 metasinkputc(ms, 0);
1105 metasinkputc(ms, 16);
1106 metasinkputuint64(ms, dir->qidoffset);
1107 metasinkputuint64(ms, dir->qidmax);
1111 metasinkputc(ms, DirGenEntry);
1112 metasinkputc(ms, 0);
1113 metasinkputc(ms, 4);
1114 metasinkputuint32(ms, dir->gen);
1122 plan9tovacdir(VacDir *vd, Dir *dir, ulong entry, uvlong qid)
1124 memset(vd, 0, sizeof(VacDir));
1126 vd->elem = vtstrdup(dir->name);
1129 vd->uid = vtstrdup(dir->uid);
1130 vd->gid = vtstrdup(dir->gid);
1131 vd->mid = vtstrdup(dir->muid);
1132 vd->mtime = dir->mtime;
1134 vd->ctime = dir->mtime; /* ctime: not available on plan 9 */
1135 vd->atime = dir->atime;
1137 vd->mode = dir->mode & 0777;
1138 if(dir->mode & DMDIR)
1139 vd->mode |= ModeDir;
1140 if(dir->mode & DMAPPEND)
1141 vd->mode |= ModeAppend;
1142 if(dir->mode & DMEXCL)
1143 vd->mode |= ModeExclusive;
1146 vd->p9path = dir->qid.path;
1147 vd->p9version = dir->qid.vers;
1152 metasinkeor(MetaSink *k)
1157 p = k->buf + MetaHeaderSize;
1158 p += k->nindex * MetaIndexSize;
1159 o = k->rp-k->buf; /* offset from start of block */
1160 n = k->p-k->rp; /* size of entry */
1167 if(k->nindex == k->maxindex)
1172 metasinkclose(MetaSink *k)
1179 metasinkfree(MetaSink *k)
1187 warn(char *fmt, ...)
1192 fprint(2, "%s: ", argv0);
1193 vfprint(2, fmt, arg);
1205 #define TWID64 ((u64int)~(u64int)0)
1215 n = strtoul(s, &es, 0);
1216 if(*es == 'k' || *es == 'K'){
1219 }else if(*es == 'm' || *es == 'M'){
1222 }else if(*es == 'g' || *es == 'G'){
1223 n *= 1024*1024*1024;