6 typedef struct Sink Sink;
7 typedef struct MetaSink MetaSink;
8 typedef struct DirSink DirSink;
14 uchar *pbuf[VtPointerDepth+1];
22 uchar *p; /* current pointer */
23 uchar *ep; /* end pointer */
31 uchar *rp; /* start of current record */
32 uchar *p; /* current pointer */
33 uchar *ep; /* end pointer */
36 static void usage(void);
37 static int strpCmp(void*, void*);
38 static void warn(char *fmt, ...);
39 static void cleanup(void);
40 static u64int unittoull(char *s);
41 static int vac(VtSession *z, char *argv[]);
42 static void vacFile(DirSink *dsink, char *lname, char *sname, VacFile*);
43 static void vacStdin(DirSink *dsink, char *name, VacFile *vf);
44 static void vacData(DirSink *dsink, int fd, char *lname, VacFile*, Dir*);
45 static void vacDir(DirSink *dsink, int fd, char *lname, char *sname, VacFile*);
46 static int vacMerge(DirSink *dsink, char *lname, char *sname);
48 Sink *sinkAlloc(VtSession *z, int psize, int dsize);
49 void sinkWrite(Sink *k, uchar *data, int n);
50 void sinkWriteScore(Sink *k, uchar *score, int n);
51 void sinkClose(Sink *k);
52 void sinkFree(Sink *k);
54 DirSink *dirSinkAlloc(VtSession *z, int psize, int dsize);
55 void dirSinkWrite(DirSink *k, VtEntry*);
56 void dirSinkWriteSink(DirSink *k, Sink*);
57 int dirSinkWriteFile(DirSink *k, VacFile *vf);
58 void dirSinkClose(DirSink *k);
59 void dirSinkFree(DirSink *k);
61 MetaSink *metaSinkAlloc(VtSession *z, int psize, int dsize);
62 void metaSinkPutc(MetaSink *k, int c);
63 void metaSinkPutString(MetaSink *k, char *s);
64 void metaSinkPutUint32(MetaSink *k, ulong x);
65 void metaSinkPutUint64(MetaSink *k, uvlong x);
66 void metaSinkWrite(MetaSink *k, uchar *data, int n);
67 void metaSinkWriteDir(MetaSink *ms, VacDir *vd);
68 void metaSinkEOR(MetaSink *k);
69 void metaSinkClose(MetaSink *k);
70 void metaSinkFree(MetaSink *k);
71 void plan9ToVacDir(VacDir*, Dir*, ulong entry, uvlong qid);
88 int bsize = BlockSize;
94 char *exclude[MaxExclude];
101 main(int argc, char *argv[])
117 bsize = unittoull(p);
127 if(nexclude >= MaxExclude)
128 sysfatal("too many exclusions\n");
129 exclude[nexclude] = ARGF();
130 if(exclude[nexclude] == nil)
168 if(bsize > VtMaxLumpSize)
169 bsize = VtMaxLumpSize;
174 fmtinstall('V', vtScoreFmt);
175 fmtinstall('R', vtErrFmt);
179 vtFatal("could not connect to server: %R");
182 vtFatal("vtConnect: %R");
184 qsort(exclude, nexclude, sizeof(char*), strpCmp);
188 fprint(2, "warning: could not ask server to flush pending writes: %R\n");
191 fprint(2, "files %ld:%ld data %ld:%ld:%ld meta %ld\n", stats.file, stats.sfile,
192 stats.data, stats.skip, stats.sdata, stats.meta);
203 fprint(2, "usage: %s [-amqsv] [-h host] [-d vacfile] [-b blocksize] [-i name] [-e exclude] [-f vacfile] file ... \n", argv0);
208 int strpCmp(void *p0, void *p1)
210 return strcmp(*(char**)p0, *(char**)p1);
215 readBlock(int fd, uchar *buf, int n)
220 m = read(fd, buf+t, n-t);
231 vacWrite(VtSession *z, uchar score[VtScoreSize], int type, uchar *buf, int n)
235 vtSha1(score, buf, n);
238 if(!vtWrite(z, score, type, buf, n))
240 if(!vtSha1Check(score, buf, n)) {
241 uchar score2[VtScoreSize];
243 vtSha1(score2, buf, n);
244 fprint(2, "vtSha1Check: n = %d %V %V\n", n, score, score2);
245 vtSetError("vtSha1Check failed");
253 vac(VtSession *z, char *argv[])
258 uchar score[VtScoreSize], buf[VtRootSize];
268 if(getwd(cwd, sizeof(cwd)) == 0)
269 sysfatal("can't find current directory: %r\n");
271 dsink = dirSinkAlloc(z, bsize, bsize);
275 fs = vfsOpen(z, dfile, 1, 10000);
277 fprint(2, "could not open diff: %s: %s\n", dfile, vtGetError());
282 fd = create(oname, OWRITE, 0666);
284 sysfatal("could not create file: %s: %r", oname);
290 sysfatal("dirfstat failed: %r");
292 for(; *argv; argv++) {
295 for (cp = *argv; *cp; cp++)
307 vff = vfOpen(fs, cp2);
308 vacFile(dsink, argv[0], cp2, vff);
311 if(cd && chdir(cwd) < 0)
312 sysfatal("can't cd back to %s: %r\n", cwd);
318 vff = vfOpen(fs, isi);
319 vacStdin(dsink, isi, vff);
326 /* build meta information for the root */
327 ms = metaSinkAlloc(z, bsize, bsize);
328 /* fake into a directory */
329 dir->mode |= (dir->mode&0444)>>2;
330 dir->qid.type |= QTDIR;
332 plan9ToVacDir(&vd, dir, 0, fileid++);
333 if(strcmp(vd.elem, "/") == 0){
335 vd.elem = vtStrDup("root");
337 metaSinkWriteDir(ms, &vd);
341 ds = dirSinkAlloc(z, bsize, bsize);
342 dirSinkWriteSink(ds, dsink->sink);
343 dirSinkWriteSink(ds, dsink->msink->sink);
344 dirSinkWriteSink(ds, ms->sink);
347 memset(&root, 0, sizeof(root));
348 root.version = VtRootVersion;
349 strncpy(root.name, dir->name, sizeof(root.name));
350 root.name[sizeof(root.name)-1] = 0;
352 sprint(root.type, "vac");
353 memmove(root.score, ds->sink->dir.score, VtScoreSize);
354 root.blockSize = maxbsize;
356 vfsGetScore(fs, root.prev);
364 vtRootPack(&root, buf);
365 if(!vacWrite(z, score, VtRootType, buf, VtRootSize))
366 vtFatal("vacWrite failed: %s", vtGetError());
369 for(i=0; i<VtScoreSize; i++)
370 fprint(fd, "%.2x", score[i]);
373 /* avoid remove at cleanup */
379 isExcluded(char *name)
387 x = strcmp(exclude[i], name);
399 vacFile(DirSink *dsink, char *lname, char *sname, VacFile *vf)
406 if(isExcluded(lname)) {
407 warn("excluding: %s", lname);
411 if(merge && vacMerge(dsink, lname, sname))
414 fd = open(sname, OREAD);
416 warn("could not open file: %s: %s", lname, vtOSError());
421 fprint(2, "%s\n", lname);
425 warn("can't stat %s: %r", lname);
430 entry = dsink->nentry;
432 if(dir->mode & DMDIR)
433 vacDir(dsink, fd, lname, sname, vf);
435 vacData(dsink, fd, lname, vf, dir);
437 plan9ToVacDir(&vd, dir, entry, fileid++);
438 metaSinkWriteDir(dsink->msink, &vd);
446 vacStdin(DirSink *dsink, char *name, VacFile *vf)
453 fprint(2, "%s\n", "<stdio>");
457 warn("can't stat <stdio>: %r");
461 entry = dsink->nentry;
463 vacData(dsink, 0, "<stdin>", vf, dir);
465 plan9ToVacDir(&vd, dir, entry, fileid++);
466 vd.elem = vtStrDup(name);
467 metaSinkWriteDir(dsink->msink, &vd);
474 vacDataSkip(Sink *sink, VacFile *vf, int fd, ulong blocks, uchar *buf, char *lname)
478 uchar score[VtScoreSize];
480 /* skip blocks for append only files */
481 if(seek(fd, (blocks-1)*bsize, 0) != (blocks-1)*bsize) {
482 warn("error seeking: %s", lname);
485 n = readBlock(fd, buf, bsize);
487 warn("error checking append only file: %s", lname);
490 if(!vfGetBlockScore(vf, blocks-1, score) || !vtSha1Check(score, buf, n)) {
491 warn("last block of append file did not match: %s", lname);
495 for(i=0; i<blocks; i++) {
496 if(!vfGetBlockScore(vf, i, score)) {
497 warn("could not get score: %s: %lud", lname, i);
498 seek(fd, i*bsize, 0);
502 sinkWriteScore(sink, score, bsize);
512 vacData(DirSink *dsink, int fd, char *lname, VacFile *vf, Dir *dir)
517 uchar score[VtScoreSize];
523 if(vf != nil && qdiff) {
525 if(vd.mtime == dir->mtime)
526 if(vd.size == dir->length)
527 if(!vd.plan9 || /* vd.p9path == dir->qid.path && */ vd.p9version == dir->qid.vers)
528 if(dirSinkWriteFile(dsink, vf)) {
534 /* look for an append only file */
535 if((dir->mode&DMAPPEND) != 0)
536 if(vd.size < dir->length)
538 if(vd.p9path == dir->qid.path)
539 vfblocks = vd.size/bsize;
545 buf = vtMemAlloc(bsize);
546 sink = sinkAlloc(dsink->sink->z, bsize, bsize);
548 same = stats.sdata+stats.skip;
551 block += vacDataSkip(sink, vf, fd, vfblocks, buf, lname);
553 if(0) fprint(2, "vacData: %s: %ld\n", lname, block);
555 n = readBlock(fd, buf, bsize);
557 warn("file truncated due to read error: %s: %s", lname, vtOSError());
560 if(vf != nil && vfGetBlockScore(vf, block, score) && vtSha1Check(score, buf, n)) {
562 sinkWriteScore(sink, score, n);
564 sinkWrite(sink, buf, n);
567 same = stats.sdata+stats.skip - same;
569 if(same && (dir->mode&DMAPPEND) != 0)
570 if(0)fprint(2, "%s: total %lud same %lud:%lud diff %lud\n",
571 lname, block, same, vfblocks, block-same);
574 dirSinkWriteSink(dsink, sink);
581 vacDir(DirSink *dsink, int fd, char *lname, char *sname, VacFile *vf)
590 ds = dirSinkAlloc(dsink->sink->z, bsize, bsize);
591 while((nd = dirread(fd, &dirs)) > 0){
592 for(i = 0; i < nd; i++){
594 /* check for bad file names */
595 if(name[0] == 0 || strcmp(name, ".") == 0 || strcmp(name, "..") == 0)
597 ln = vtMemAlloc(strlen(lname) + strlen(name) + 2);
598 sn = vtMemAlloc(strlen(sname) + strlen(name) + 2);
599 sprint(ln, "%s/%s", lname, name);
600 sprint(sn, "%s/%s", sname, name);
602 vvf = vfWalk(vf, name);
605 vacFile(ds, ln, sn, vvf);
614 dirSinkWriteSink(dsink, ds->sink);
615 dirSinkWriteSink(dsink, ds->msink->sink);
620 vacMergeFile(DirSink *dsink, VacFile *vf, VacDir *dir, uvlong offset, uvlong *max)
622 uchar buf[VtEntrySize];
626 if(vfRead(vf, buf, VtEntrySize, (uvlong)dir->entry*VtEntrySize) != VtEntrySize) {
627 warn("could not read venti dir entry: %s\n", dir->elem);
630 vtEntryUnpack(&dd, buf, 0);
632 if(dir->mode & ModeDir) {
637 if(vfRead(vf, buf, VtEntrySize, e*VtEntrySize) != VtEntrySize) {
638 warn("could not read venti dir entry: %s\n", dir->elem);
641 vtEntryUnpack(&md, buf, 0);
644 /* max might incorrect in some old dumps */
645 if(dir->qid >= *max) {
646 warn("qid out of range: %s", dir->elem);
651 dir->entry = dsink->nentry;
654 dir->qidOffset += offset;
657 dir->qidOffset = offset;
661 dirSinkWrite(dsink, &dd);
662 if(dir->mode & ModeDir)
663 dirSinkWrite(dsink, &md);
664 metaSinkWriteDir(dsink->msink, dir);
670 vacMerge(DirSink *dsink, char *lname, char *sname)
679 p = strrchr(sname, '.');
680 if(p == 0 || strcmp(p, ".vac"))
684 fs = vfsOpen(dsink->sink->z, sname, 1, 100);
688 vf = vfOpen(fs, "/");
692 d = vdeOpen(fs, "/");
697 fprint(2, "merging: %s\n", lname);
699 if(maxbsize < vfsGetBlockSize(fs))
700 maxbsize = vfsGetBlockSize(fs);
703 if(vdeRead(d, &dir, 1) < 1)
705 vacMergeFile(dsink, vf, &dir, fileid, &max);
720 sinkAlloc(VtSession *z, int psize, int dsize)
725 if(psize < 512 || psize > VtMaxLumpSize)
726 vtFatal("sinkAlloc: bad psize");
727 if(dsize < 512 || dsize > VtMaxLumpSize)
728 vtFatal("sinkAlloc: bad psize");
730 psize = VtScoreSize*(psize/VtScoreSize);
732 k = vtMemAllocZ(sizeof(Sink));
734 k->dir.flags = VtEntryActive;
735 k->dir.psize = psize;
736 k->dir.dsize = dsize;
737 k->buf = vtMemAllocZ(VtPointerDepth*k->dir.psize + VtScoreSize);
738 for(i=0; i<=VtPointerDepth; i++)
739 k->pbuf[i] = k->buf + i*k->dir.psize;
744 sinkWriteScore(Sink *k, uchar score[VtScoreSize], int n)
750 memmove(k->pbuf[0], score, VtScoreSize);
754 for(i=0; i<VtPointerDepth; i++) {
755 k->pbuf[i] += VtScoreSize;
756 if(k->pbuf[i] < k->buf + d->psize*(i+1))
758 if(i == VtPointerDepth-1)
759 vtFatal("file too big");
760 p = k->buf+i*d->psize;
762 if(!vacWrite(k->z, k->pbuf[i+1], VtPointerType0+i, p, d->psize))
763 vtFatal("vacWrite failed: %s", vtGetError());
767 /* round size up to multiple of dsize */
768 d->size = d->dsize * ((d->size + d->dsize-1)/d->dsize);
774 sinkWrite(Sink *k, uchar *p, int n)
777 uchar score[VtScoreSize];
780 vtFatal("sinkWrite: size too big");
782 if(k->dir.flags & VtEntryDir) {
789 if(!vacWrite(k->z, score, type, p, n))
790 vtFatal("vacWrite failed: %s", vtGetError());
792 sinkWriteScore(k, score, n);
796 sizeToDepth(uvlong s, int psize, int dsize)
801 /* determine pointer depth */
802 np = psize/VtScoreSize;
803 s = (s + dsize - 1)/dsize;
804 for(d = 0; s > 1; d++)
820 memmove(kd->score, vtZeroScore, VtScoreSize);
824 for(n=VtPointerDepth-1; n>0; n--)
825 if(k->pbuf[n] > k->buf + kd->psize*n)
828 kd->depth = sizeToDepth(kd->size, kd->psize, kd->dsize);
830 /* skip full part of tree */
831 for(i=0; i<n && k->pbuf[i] == k->buf + kd->psize*i; i++)
834 /* is the tree completely full */
835 if(i == n && k->pbuf[n] == k->buf + kd->psize*n + VtScoreSize) {
836 memmove(kd->score, k->pbuf[n] - VtScoreSize, VtScoreSize);
841 /* clean up the edge */
843 p = k->buf+i*kd->psize;
845 if(!vacWrite(k->z, k->pbuf[i+1], VtPointerType0+i, p, k->pbuf[i]-p))
846 vtFatal("vacWrite failed: %s", vtGetError());
847 k->pbuf[i+1] += VtScoreSize;
849 memmove(kd->score, k->pbuf[i] - VtScoreSize, VtScoreSize);
860 dirSinkAlloc(VtSession *z, int psize, int dsize)
865 ds = VtEntrySize*(dsize/VtEntrySize);
867 k = vtMemAllocZ(sizeof(DirSink));
868 k->sink = sinkAlloc(z, psize, ds);
869 k->sink->dir.flags |= VtEntryDir;
870 k->msink = metaSinkAlloc(z, psize, dsize);
871 k->buf = vtMemAlloc(ds);
878 dirSinkWrite(DirSink *k, VtEntry *dir)
880 if(k->p + VtEntrySize > k->ep) {
881 sinkWrite(k->sink, k->buf, k->p - k->buf);
884 vtEntryPack(dir, k->p, 0);
890 dirSinkWriteSink(DirSink *k, Sink *sink)
892 dirSinkWrite(k, &sink->dir);
896 dirSinkWriteFile(DirSink *k, VacFile *vf)
900 if(!vfGetVtEntry(vf, &dir))
902 dirSinkWrite(k, &dir);
907 dirSinkClose(DirSink *k)
909 metaSinkClose(k->msink);
911 sinkWrite(k->sink, k->buf, k->p - k->buf);
916 dirSinkFree(DirSink *k)
919 metaSinkFree(k->msink);
925 metaSinkAlloc(VtSession *z, int psize, int dsize)
929 k = vtMemAllocZ(sizeof(MetaSink));
930 k->sink = sinkAlloc(z, psize, dsize);
931 k->buf = vtMemAlloc(dsize);
932 k->maxindex = dsize/100; /* 100 byte entries seems reasonable */
935 k->rp = k->p = k->buf + MetaHeaderSize + k->maxindex*MetaIndexSize;
936 k->ep = k->buf + dsize;
940 /* hack to get base to compare routine - not reentrant */
944 dirCmp(void *p0, void *p1)
949 /* name is first element of entry */
951 q0 = blockBase + (q0[0]<<8) + q0[1];
952 n0 = (q0[6]<<8) + q0[7];
956 q1 = blockBase + (q1[0]<<8) + q1[1];
957 n1 = (q1[6]<<8) + q1[7];
961 return memcmp(q0, q1, n0);
963 r = memcmp(q0, q1, n0);
966 r = memcmp(q0, q1, n1);
972 metaSinkFlush(MetaSink *k)
980 assert(k->nindex <= k->maxindex);
987 mb.nindex = k->nindex;
988 mb.maxindex = k->maxindex;
994 /* XXX this is not reentrant! */
996 qsort(p, k->nindex, MetaIndexSize, dirCmp);
997 p += k->nindex*MetaIndexSize;
999 memset(p, 0, (k->maxindex-k->nindex)*MetaIndexSize);
1000 p += (k->maxindex-k->nindex)*MetaIndexSize;
1002 sinkWrite(k->sink, k->buf, n);
1004 /* move down partial entry */
1006 memmove(p, k->rp, n);
1013 metaSinkPutc(MetaSink *k, int c)
1018 vtFatal("directory entry too large");
1024 metaSinkPutString(MetaSink *k, char *s)
1027 metaSinkPutc(k, n>>8);
1029 metaSinkWrite(k, (uchar*)s, n);
1033 metaSinkPutUint32(MetaSink *k, ulong x)
1035 metaSinkPutc(k, x>>24);
1036 metaSinkPutc(k, x>>16);
1037 metaSinkPutc(k, x>>8);
1042 metaSinkPutUint64(MetaSink *k, uvlong x)
1044 metaSinkPutUint32(k, x>>32);
1045 metaSinkPutUint32(k, x);
1049 metaSinkWrite(MetaSink *k, uchar *data, int n)
1051 if(k->p + n > k->ep)
1053 if(k->p + n > k->ep)
1054 vtFatal("directory entry too large");
1056 memmove(k->p, data, n);
1061 metaSinkWriteDir(MetaSink *ms, VacDir *dir)
1063 metaSinkPutUint32(ms, DirMagic);
1064 metaSinkPutc(ms, Version>>8);
1065 metaSinkPutc(ms, Version);
1066 metaSinkPutString(ms, dir->elem);
1067 metaSinkPutUint32(ms, dir->entry);
1068 metaSinkPutUint64(ms, dir->qid);
1069 metaSinkPutString(ms, dir->uid);
1070 metaSinkPutString(ms, dir->gid);
1071 metaSinkPutString(ms, dir->mid);
1072 metaSinkPutUint32(ms, dir->mtime);
1073 metaSinkPutUint32(ms, dir->mcount);
1074 metaSinkPutUint32(ms, dir->ctime);
1075 metaSinkPutUint32(ms, dir->atime);
1076 metaSinkPutUint32(ms, dir->mode);
1079 metaSinkPutc(ms, DirPlan9Entry); /* plan9 extra info */
1080 metaSinkPutc(ms, 0); /* plan9 extra size */
1081 metaSinkPutc(ms, 12); /* plan9 extra size */
1082 metaSinkPutUint64(ms, dir->p9path);
1083 metaSinkPutUint32(ms, dir->p9version);
1086 if(dir->qidSpace != 0) {
1087 metaSinkPutc(ms, DirQidSpaceEntry);
1088 metaSinkPutc(ms, 0);
1089 metaSinkPutc(ms, 16);
1090 metaSinkPutUint64(ms, dir->qidOffset);
1091 metaSinkPutUint64(ms, dir->qidMax);
1095 metaSinkPutc(ms, DirGenEntry);
1096 metaSinkPutc(ms, 0);
1097 metaSinkPutc(ms, 4);
1098 metaSinkPutUint32(ms, dir->gen);
1106 plan9ToVacDir(VacDir *vd, Dir *dir, ulong entry, uvlong qid)
1108 memset(vd, 0, sizeof(VacDir));
1110 vd->elem = vtStrDup(dir->name);
1113 vd->uid = vtStrDup(dir->uid);
1114 vd->gid = vtStrDup(dir->gid);
1115 vd->mid = vtStrDup(dir->muid);
1116 vd->mtime = dir->mtime;
1118 vd->ctime = dir->mtime; /* ctime: not available on plan 9 */
1119 vd->atime = dir->atime;
1121 vd->mode = dir->mode & 0777;
1122 if(dir->mode & DMDIR)
1123 vd->mode |= ModeDir;
1124 if(dir->mode & DMAPPEND)
1125 vd->mode |= ModeAppend;
1126 if(dir->mode & DMEXCL)
1127 vd->mode |= ModeExclusive;
1130 vd->p9path = dir->qid.path;
1131 vd->p9version = dir->qid.vers;
1136 metaSinkEOR(MetaSink *k)
1141 p = k->buf + MetaHeaderSize;
1142 p += k->nindex * MetaIndexSize;
1143 o = k->rp-k->buf; /* offset from start of block */
1144 n = k->p-k->rp; /* size of entry */
1151 if(k->nindex == k->maxindex)
1156 metaSinkClose(MetaSink *k)
1163 metaSinkFree(MetaSink *k)
1171 warn(char *fmt, ...)
1176 fprint(2, "%s: ", argv0);
1177 vfprint(2, fmt, arg);
1189 #define TWID64 ((u64int)~(u64int)0)
1199 n = strtoul(s, &es, 0);
1200 if(*es == 'k' || *es == 'K'){
1203 }else if(*es == 'm' || *es == 'M'){
1206 }else if(*es == 'g' || *es == 'G'){
1207 n *= 1024*1024*1024;