8 int mainstacksize = 128*1024;
10 typedef struct Sink Sink;
11 typedef struct MetaSink MetaSink;
12 typedef struct DirSink DirSink;
18 uchar *pbuf[VtPointerDepth+1];
26 uchar *p; /* current pointer */
27 uchar *ep; /* end pointer */
35 uchar *rp; /* start of current record */
36 uchar *p; /* current pointer */
37 uchar *ep; /* end pointer */
40 static void usage(void);
41 static int strpcmp(const void*, const void*);
42 static void warn(char *fmt, ...);
43 static void cleanup(void);
44 static u64int unittoull(char *s);
45 static void vac(VtConn *z, char *argv[]);
46 static void vacfile(DirSink *dsink, char *lname, char *sname, VacFile*);
47 static void vacstdin(DirSink *dsink, char *name, VacFile *vf);
48 static void vacdata(DirSink *dsink, int fd, char *lname, VacFile*, Dir*);
49 static void vacdir(DirSink *dsink, int fd, char *lname, char *sname, VacFile*);
50 static int vacmerge(DirSink *dsink, char *lname, char *sname);
52 Sink *sinkalloc(VtConn *z, int psize, int dsize);
53 void sinkwrite(Sink *k, uchar *data, int n);
54 void sinkwritescore(Sink *k, uchar *score, int n);
55 void sinkclose(Sink *k);
56 void sinkfree(Sink *k);
58 DirSink *dirsinkalloc(VtConn *z, int psize, int dsize);
59 void dirsinkwrite(DirSink *k, VtEntry*);
60 void dirsinkwritesink(DirSink *k, Sink*);
61 int dirsinkwritefile(DirSink *k, VacFile *vf);
62 void dirsinkclose(DirSink *k);
63 void dirsinkfree(DirSink *k);
65 MetaSink *metasinkalloc(VtConn *z, int psize, int dsize);
66 void metasinkputc(MetaSink *k, int c);
67 void metasinkputstring(MetaSink *k, char *s);
68 void metasinkputuint32(MetaSink *k, ulong x);
69 void metasinkputuint64(MetaSink *k, uvlong x);
70 void metasinkwrite(MetaSink *k, uchar *data, int n);
71 void metasinkwritedir(MetaSink *ms, VacDir *vd);
72 void metasinkeor(MetaSink *k);
73 void metasinkclose(MetaSink *k);
74 void metasinkfree(MetaSink *k);
75 void plan9tovacdir(VacDir*, Dir*, ulong entry, uvlong qid);
92 int bsize = BlockSize;
98 char *exclude[MaxExclude];
107 fprint(2, "usage: %s [-amqsv] [-h host] [-d vacfile] [-b blocksize] [-i name] [-e exclude] [-f vacfile] file ... \n", argv0);
108 threadexitsall("usage");
112 threadmain(int argc, char *argv[])
128 bsize = unittoull(p);
138 if(nexclude >= MaxExclude)
139 sysfatal("too many exclusions\n");
140 exclude[nexclude] = ARGF();
141 if(exclude[nexclude] == nil)
182 if(bsize > VtMaxLumpSize)
183 bsize = VtMaxLumpSize;
186 fmtinstall('V', vtscorefmt);
190 sysfatal("could not connect to server: %r");
193 sysfatal("vtconnect: %r");
195 qsort(exclude, nexclude, sizeof(char*), strpcmp);
200 fprint(2, "warning: could not ask server to flush pending writes: %r\n");
203 fprint(2, "files %ld:%ld data %ld:%ld:%ld meta %ld\n", stats.file, stats.sfile,
204 stats.data, stats.skip, stats.sdata, stats.meta);
212 strpcmp(const void *p0, const void *p1)
214 return strcmp(*(char**)p0, *(char**)p1);
218 vacwrite(VtConn *z, uchar score[VtScoreSize], int type, uchar *buf, int n)
222 sha1(buf, n, score, nil);
225 return vtwrite(z, score, type, buf, n);
229 lastelem(char *oname)
235 if((p = strrchr(oname, '/')) == nil)
241 vac(VtConn *z, char *argv[])
246 uchar score[VtScoreSize], buf[VtRootSize];
256 if(getwd(cwd, sizeof(cwd)) == 0)
257 sysfatal("can't find current directory: %r\n");
259 dsink = dirsinkalloc(z, bsize, bsize);
263 fs = vacfsopen(z, dfile, VtOREAD, 1000);
265 fprint(2, "could not open diff: %s: %r\n", dfile);
270 fd = create(oname, OWRITE, 0666);
272 sysfatal("could not create file: %s: %r", oname);
278 sysfatal("dirfstat failed: %r");
280 dir->name = lastelem(oname);
284 for(; *argv; argv++) {
287 for (cp = *argv; *cp; cp++)
299 vff = vacfileopen(fs, cp2);
300 vacfile(dsink, argv[0], cp2, vff);
303 if(cd && chdir(cwd) < 0)
304 sysfatal("can't cd back to %s: %r\n", cwd);
310 vff = vacfileopen(fs, isi);
311 vacstdin(dsink, isi, vff);
318 /* build meta information for the root */
319 ms = metasinkalloc(z, bsize, bsize);
320 /* fake into a directory */
321 dir->mode |= (dir->mode&0444)>>2;
322 dir->qid.type |= QTDIR;
324 plan9tovacdir(&vd, dir, 0, fileid++);
325 if(strcmp(vd.elem, "/") == 0){
327 vd.elem = vtstrdup("root");
329 metasinkwritedir(ms, &vd);
333 ds = dirsinkalloc(z, bsize, bsize);
334 dirsinkwritesink(ds, dsink->sink);
335 dirsinkwritesink(ds, dsink->msink->sink);
336 dirsinkwritesink(ds, ms->sink);
339 memset(&root, 0, sizeof(root));
340 strncpy(root.name, dir->name, sizeof(root.name));
341 root.name[sizeof(root.name)-1] = 0;
343 sprint(root.type, "vac");
344 memmove(root.score, ds->sink->dir.score, VtScoreSize);
345 root.blocksize = maxbsize;
347 vacfsgetscore(fs, root.prev);
355 vtrootpack(&root, buf);
356 if(vacwrite(z, score, VtRootType, buf, VtRootSize) < 0)
357 sysfatal("vacWrite failed: %r");
359 fprint(fd, "vac:%V\n", score);
361 /* avoid remove at cleanup */
366 isexcluded(char *name)
374 x = strcmp(exclude[i], name);
386 vacfile(DirSink *dsink, char *lname, char *sname, VacFile *vf)
393 if(isexcluded(lname)) {
394 warn("excluding: %s", lname);
398 if(merge && vacmerge(dsink, lname, sname) >= 0)
401 if((dir = dirstat(sname)) == nil){
402 warn("could not stat file %s: %r", lname);
405 if(dir->mode&(DMSYMLINK|DMDEVICE|DMNAMEDPIPE|DMSOCKET)){
411 fd = open(sname, OREAD);
413 warn("could not open file: %s: %r", lname);
418 fprint(2, "%s\n", lname);
422 warn("can't stat %s: %r", lname);
426 dir->name = lastelem(sname);
428 entry = dsink->nentry;
430 if(dir->mode & DMDIR)
431 vacdir(dsink, fd, lname, sname, vf);
433 vacdata(dsink, fd, lname, vf, dir);
435 plan9tovacdir(&vd, dir, entry, fileid++);
436 metasinkwritedir(dsink->msink, &vd);
444 vacstdin(DirSink *dsink, char *name, VacFile *vf)
451 fprint(2, "%s\n", "<stdio>");
455 warn("can't stat <stdio>: %r");
460 entry = dsink->nentry;
462 vacdata(dsink, 0, "<stdin>", vf, dir);
464 plan9tovacdir(&vd, dir, entry, fileid++);
465 vd.elem = vtstrdup(name);
466 metasinkwritedir(dsink->msink, &vd);
473 sha1check(u8int *score, uchar *buf, int n)
475 char score2[VtScoreSize];
477 sha1(buf, n, score, nil);
478 if(memcmp(score, score2, VtScoreSize) == 0)
484 vacdataskip(Sink *sink, VacFile *vf, int fd, ulong blocks, uchar *buf, char *lname)
488 uchar score[VtScoreSize];
490 /* skip blocks for append only files */
491 if(seek(fd, (blocks-1)*bsize, 0) != (blocks-1)*bsize) {
492 warn("error seeking: %s", lname);
495 n = readn(fd, buf, bsize);
497 warn("error checking append only file: %s", lname);
500 if(vacfileblockscore(vf, blocks-1, score)<0 || sha1check(score, buf, n)<0) {
501 warn("last block of append file did not match: %s", lname);
505 for(i=0; i<blocks; i++) {
506 if(vacfileblockscore(vf, i, score) < 0) {
507 warn("could not get score: %s: %lud", lname, i);
508 seek(fd, i*bsize, 0);
512 sinkwritescore(sink, score, bsize);
522 vacdata(DirSink *dsink, int fd, char *lname, VacFile *vf, Dir *dir)
527 uchar score[VtScoreSize];
533 if(vf != nil && qdiff) {
534 vacfilegetdir(vf, &vd);
535 if(vd.mtime == dir->mtime)
536 if(vd.size == dir->length)
537 if(!vd.plan9 || /* vd.p9path == dir->qid.path && */ vd.p9version == dir->qid.vers)
538 if(dirsinkwritefile(dsink, vf)) {
544 /* look for an append only file */
545 if((dir->mode&DMAPPEND) != 0)
546 if(vd.size < dir->length)
548 if(vd.p9path == dir->qid.path)
549 vfblocks = vd.size/bsize;
555 buf = vtmalloc(bsize);
556 sink = sinkalloc(dsink->sink->z, bsize, bsize);
558 same = stats.sdata+stats.skip;
561 block += vacdataskip(sink, vf, fd, vfblocks, buf, lname);
563 if(0) fprint(2, "vacData: %s: %ld\n", lname, block);
565 n = readn(fd, buf, bsize);
567 warn("file truncated due to read error: %s: %r", lname);
570 if(vf != nil && vacfileblockscore(vf, block, score) && sha1check(score, buf, n)>=0) {
572 sinkwritescore(sink, score, n);
574 sinkwrite(sink, buf, n);
577 same = stats.sdata+stats.skip - same;
579 if(same && (dir->mode&DMAPPEND) != 0)
580 if(0)fprint(2, "%s: total %lud same %lud:%lud diff %lud\n",
581 lname, block, same, vfblocks, block-same);
584 dirsinkwritesink(dsink, sink);
591 vacdir(DirSink *dsink, int fd, char *lname, char *sname, VacFile *vf)
600 ds = dirsinkalloc(dsink->sink->z, bsize, bsize);
601 while((nd = dirread(fd, &dirs)) > 0){
602 for(i = 0; i < nd; i++){
604 /* check for bad file names */
605 if(name[0] == 0 || strcmp(name, ".") == 0 || strcmp(name, "..") == 0)
607 ln = vtmalloc(strlen(lname) + strlen(name) + 2);
608 sn = vtmalloc(strlen(sname) + strlen(name) + 2);
609 sprint(ln, "%s/%s", lname, name);
610 sprint(sn, "%s/%s", sname, name);
612 vvf = vacfilewalk(vf, name);
615 vacfile(ds, ln, sn, vvf);
624 dirsinkwritesink(dsink, ds->sink);
625 dirsinkwritesink(dsink, ds->msink->sink);
630 vacmergefile(DirSink *dsink, VacFile *vf, VacDir *dir, uvlong offset, uvlong *max)
632 uchar buf[VtEntrySize];
636 if(vacfileread(vf, buf, VtEntrySize, (uvlong)dir->entry*VtEntrySize) != VtEntrySize) {
637 warn("could not read venti dir entry: %s\n", dir->elem);
640 vtentryunpack(&dd, buf, 0);
642 if(dir->mode & ModeDir) {
647 if(vacfileread(vf, buf, VtEntrySize, e*VtEntrySize) != VtEntrySize) {
648 warn("could not read venti dir entry: %s\n", dir->elem);
651 vtentryunpack(&md, buf, 0);
654 /* max might incorrect in some old dumps */
655 if(dir->qid >= *max) {
656 warn("qid out of range: %s", dir->elem);
661 dir->entry = dsink->nentry;
664 dir->qidoffset += offset;
667 dir->qidoffset = offset;
671 dirsinkwrite(dsink, &dd);
672 if(dir->mode & ModeDir)
673 dirsinkwrite(dsink, &md);
674 metasinkwritedir(dsink->msink, dir);
680 vacmerge(DirSink *dsink, char *lname, char *sname)
689 p = strrchr(sname, '.');
690 if(p == 0 || strcmp(p, ".vac"))
694 fs = vacfsopen(dsink->sink->z, sname, VtOREAD, 100);
698 vf = vacfileopen(fs, "/");
701 max = vacfilegetid(vf);
707 fprint(2, "merging: %s\n", lname);
709 if(maxbsize < fs->bsize)
710 maxbsize = fs->bsize;
713 if(vderead(d, &dir) < 1)
715 vacmergefile(dsink, vf, &dir, fileid, &max);
730 sinkalloc(VtConn *z, int psize, int dsize)
735 if(psize < 512 || psize > VtMaxLumpSize)
736 sysfatal("sinkalloc: bad psize");
737 if(dsize < 512 || dsize > VtMaxLumpSize)
738 sysfatal("sinkalloc: bad psize");
740 psize = VtScoreSize*(psize/VtScoreSize);
742 k = vtmallocz(sizeof(Sink));
744 k->dir.flags = VtEntryActive;
745 k->dir.psize = psize;
746 k->dir.dsize = dsize;
747 k->buf = vtmallocz(VtPointerDepth*k->dir.psize + VtScoreSize);
748 for(i=0; i<=VtPointerDepth; i++)
749 k->pbuf[i] = k->buf + i*k->dir.psize;
754 sinkwritescore(Sink *k, uchar score[VtScoreSize], int n)
760 memmove(k->pbuf[0], score, VtScoreSize);
764 for(i=0; i<VtPointerDepth; i++) {
765 k->pbuf[i] += VtScoreSize;
766 if(k->pbuf[i] < k->buf + d->psize*(i+1))
768 if(i == VtPointerDepth-1)
769 sysfatal("file too big");
770 p = k->buf+i*d->psize;
772 if(vacwrite(k->z, k->pbuf[i+1], VtDataType+1+i, p, d->psize) < 0)
773 sysfatal("vacwrite failed: %r");
777 /* round size up to multiple of dsize */
778 d->size = d->dsize * ((d->size + d->dsize-1)/d->dsize);
784 sinkwrite(Sink *k, uchar *p, int n)
787 uchar score[VtScoreSize];
790 sysfatal("sinkWrite: size too big");
792 if((k->dir.type&~VtTypeDepthMask) == VtDirType){
799 if(vacwrite(k->z, score, type, p, n) < 0)
800 sysfatal("vacWrite failed: %r");
802 sinkwritescore(k, score, n);
806 sizetodepth(uvlong s, int psize, int dsize)
811 /* determine pointer depth */
812 np = psize/VtScoreSize;
813 s = (s + dsize - 1)/dsize;
814 for(d = 0; s > 1; d++)
830 memmove(kd->score, vtzeroscore, VtScoreSize);
834 for(n=VtPointerDepth-1; n>0; n--)
835 if(k->pbuf[n] > k->buf + kd->psize*n)
838 base = kd->type&~VtTypeDepthMask;
839 kd->type = base + sizetodepth(kd->size, kd->psize, kd->dsize);
841 /* skip full part of tree */
842 for(i=0; i<n && k->pbuf[i] == k->buf + kd->psize*i; i++)
845 /* is the tree completely full */
846 if(i == n && k->pbuf[n] == k->buf + kd->psize*n + VtScoreSize) {
847 memmove(kd->score, k->pbuf[n] - VtScoreSize, VtScoreSize);
852 /* clean up the edge */
854 p = k->buf+i*kd->psize;
856 if(vacwrite(k->z, k->pbuf[i+1], base+1+i, p, k->pbuf[i]-p) < 0)
857 sysfatal("vacWrite failed: %r");
858 k->pbuf[i+1] += VtScoreSize;
860 memmove(kd->score, k->pbuf[i] - VtScoreSize, VtScoreSize);
871 dirsinkalloc(VtConn *z, int psize, int dsize)
876 ds = VtEntrySize*(dsize/VtEntrySize);
878 k = vtmallocz(sizeof(DirSink));
879 k->sink = sinkalloc(z, psize, ds);
880 k->sink->dir.type = VtDirType;
881 k->msink = metasinkalloc(z, psize, dsize);
882 k->buf = vtmalloc(ds);
889 dirsinkwrite(DirSink *k, VtEntry *dir)
891 if(k->p + VtEntrySize > k->ep) {
892 sinkwrite(k->sink, k->buf, k->p - k->buf);
895 vtentrypack(dir, k->p, 0);
901 dirsinkwritesink(DirSink *k, Sink *sink)
903 dirsinkwrite(k, &sink->dir);
907 dirsinkwritefile(DirSink *k, VacFile *vf)
911 if(vacfilegetvtentry(vf, &dir) < 0)
913 dirsinkwrite(k, &dir);
918 dirsinkclose(DirSink *k)
920 metasinkclose(k->msink);
922 sinkwrite(k->sink, k->buf, k->p - k->buf);
927 dirsinkfree(DirSink *k)
930 metasinkfree(k->msink);
936 metasinkalloc(VtConn *z, int psize, int dsize)
940 k = vtmallocz(sizeof(MetaSink));
941 k->sink = sinkalloc(z, psize, dsize);
942 k->buf = vtmalloc(dsize);
943 k->maxindex = dsize/100; /* 100 byte entries seems reasonable */
946 k->rp = k->p = k->buf + MetaHeaderSize + k->maxindex*MetaIndexSize;
947 k->ep = k->buf + dsize;
951 /* hack to get base to compare routine - not reentrant */
955 dircmp(const void *p0, const void *p1)
960 /* name is first element of entry */
962 q0 = blockbase + (q0[0]<<8) + q0[1];
963 n0 = (q0[6]<<8) + q0[7];
967 q1 = blockbase + (q1[0]<<8) + q1[1];
968 n1 = (q1[6]<<8) + q1[7];
972 return memcmp(q0, q1, n0);
974 r = memcmp(q0, q1, n0);
977 r = memcmp(q0, q1, n1);
983 metasinkflush(MetaSink *k)
991 assert(k->nindex <= k->maxindex);
998 mb.nindex = k->nindex;
999 mb.maxindex = k->maxindex;
1003 p += MetaHeaderSize;
1005 /* XXX this is not reentrant! */
1007 qsort(p, k->nindex, MetaIndexSize, dircmp);
1008 p += k->nindex*MetaIndexSize;
1010 memset(p, 0, (k->maxindex-k->nindex)*MetaIndexSize);
1011 p += (k->maxindex-k->nindex)*MetaIndexSize;
1013 sinkwrite(k->sink, k->buf, n);
1015 /* move down partial entry */
1017 memmove(p, k->rp, n);
1024 metasinkputc(MetaSink *k, int c)
1029 sysfatal("directory entry too large");
1035 metasinkputstring(MetaSink *k, char *s)
1038 metasinkputc(k, n>>8);
1040 metasinkwrite(k, (uchar*)s, n);
1044 metasinkputuint32(MetaSink *k, ulong x)
1046 metasinkputc(k, x>>24);
1047 metasinkputc(k, x>>16);
1048 metasinkputc(k, x>>8);
1053 metasinkputuint64(MetaSink *k, uvlong x)
1055 metasinkputuint32(k, x>>32);
1056 metasinkputuint32(k, x);
1060 metasinkwrite(MetaSink *k, uchar *data, int n)
1062 if(k->p + n > k->ep)
1064 if(k->p + n > k->ep)
1065 sysfatal("directory entry too large");
1067 memmove(k->p, data, n);
1072 metasinkwritedir(MetaSink *ms, VacDir *dir)
1074 metasinkputuint32(ms, DirMagic);
1075 metasinkputc(ms, Version>>8);
1076 metasinkputc(ms, Version);
1077 metasinkputstring(ms, dir->elem);
1078 metasinkputuint32(ms, dir->entry);
1079 metasinkputuint64(ms, dir->qid);
1080 metasinkputstring(ms, dir->uid);
1081 metasinkputstring(ms, dir->gid);
1082 metasinkputstring(ms, dir->mid);
1083 metasinkputuint32(ms, dir->mtime);
1084 metasinkputuint32(ms, dir->mcount);
1085 metasinkputuint32(ms, dir->ctime);
1086 metasinkputuint32(ms, dir->atime);
1087 metasinkputuint32(ms, dir->mode);
1090 metasinkputc(ms, DirPlan9Entry); /* plan9 extra info */
1091 metasinkputc(ms, 0); /* plan9 extra size */
1092 metasinkputc(ms, 12); /* plan9 extra size */
1093 metasinkputuint64(ms, dir->p9path);
1094 metasinkputuint32(ms, dir->p9version);
1097 if(dir->qidspace != 0) {
1098 metasinkputc(ms, DirQidSpaceEntry);
1099 metasinkputc(ms, 0);
1100 metasinkputc(ms, 16);
1101 metasinkputuint64(ms, dir->qidoffset);
1102 metasinkputuint64(ms, dir->qidmax);
1106 metasinkputc(ms, DirGenEntry);
1107 metasinkputc(ms, 0);
1108 metasinkputc(ms, 4);
1109 metasinkputuint32(ms, dir->gen);
1117 plan9tovacdir(VacDir *vd, Dir *dir, ulong entry, uvlong qid)
1119 memset(vd, 0, sizeof(VacDir));
1121 vd->elem = vtstrdup(dir->name);
1124 vd->uid = vtstrdup(dir->uid);
1125 vd->gid = vtstrdup(dir->gid);
1126 vd->mid = vtstrdup(dir->muid);
1127 vd->mtime = dir->mtime;
1129 vd->ctime = dir->mtime; /* ctime: not available on plan 9 */
1130 vd->atime = dir->atime;
1132 vd->mode = dir->mode & 0777;
1133 if(dir->mode & DMDIR)
1134 vd->mode |= ModeDir;
1135 if(dir->mode & DMAPPEND)
1136 vd->mode |= ModeAppend;
1137 if(dir->mode & DMEXCL)
1138 vd->mode |= ModeExclusive;
1141 vd->p9path = dir->qid.path;
1142 vd->p9version = dir->qid.vers;
1147 metasinkeor(MetaSink *k)
1152 p = k->buf + MetaHeaderSize;
1153 p += k->nindex * MetaIndexSize;
1154 o = k->rp-k->buf; /* offset from start of block */
1155 n = k->p-k->rp; /* size of entry */
1162 if(k->nindex == k->maxindex)
1167 metasinkclose(MetaSink *k)
1174 metasinkfree(MetaSink *k)
1182 warn(char *fmt, ...)
1187 fprint(2, "%s: ", argv0);
1188 vfprint(2, fmt, arg);
1200 #define TWID64 ((u64int)~(u64int)0)
1210 n = strtoul(s, &es, 0);
1211 if(*es == 'k' || *es == 'K'){
1214 }else if(*es == 'm' || *es == 'M'){
1217 }else if(*es == 'g' || *es == 'G'){
1218 n *= 1024*1024*1024;