7 int mainstacksize = 128*1024;
9 typedef struct Sink Sink;
10 typedef struct MetaSink MetaSink;
11 typedef struct DirSink DirSink;
17 uchar *pbuf[VtPointerDepth+1];
25 uchar *p; /* current pointer */
26 uchar *ep; /* end pointer */
34 uchar *rp; /* start of current record */
35 uchar *p; /* current pointer */
36 uchar *ep; /* end pointer */
39 static void usage(void);
40 static int strpcmp(const void*, const void*);
41 static void warn(char *fmt, ...);
42 static void cleanup(void);
43 static u64int unittoull(char *s);
44 static void vac(VtConn *z, char *argv[]);
45 static void vacfile(DirSink *dsink, char *lname, char *sname, VacFile*);
46 static void vacstdin(DirSink *dsink, char *name, VacFile *vf);
47 static void vacdata(DirSink *dsink, int fd, char *lname, VacFile*, Dir*);
48 static void vacdir(DirSink *dsink, int fd, char *lname, char *sname, VacFile*);
49 static int vacmerge(DirSink *dsink, char *lname, char *sname);
51 Sink *sinkalloc(VtConn *z, int psize, int dsize);
52 void sinkwrite(Sink *k, uchar *data, int n);
53 void sinkwritescore(Sink *k, uchar *score, int n);
54 void sinkclose(Sink *k);
55 void sinkfree(Sink *k);
57 DirSink *dirsinkalloc(VtConn *z, int psize, int dsize);
58 void dirsinkwrite(DirSink *k, VtEntry*);
59 void dirsinkwritesink(DirSink *k, Sink*);
60 int dirsinkwritefile(DirSink *k, VacFile *vf);
61 void dirsinkclose(DirSink *k);
62 void dirsinkfree(DirSink *k);
64 MetaSink *metasinkalloc(VtConn *z, int psize, int dsize);
65 void metasinkputc(MetaSink *k, int c);
66 void metasinkputstring(MetaSink *k, char *s);
67 void metasinkputuint32(MetaSink *k, ulong x);
68 void metasinkputuint64(MetaSink *k, uvlong x);
69 void metasinkwrite(MetaSink *k, uchar *data, int n);
70 void metasinkwritedir(MetaSink *ms, VacDir *vd);
71 void metasinkeor(MetaSink *k);
72 void metasinkclose(MetaSink *k);
73 void metasinkfree(MetaSink *k);
74 void plan9tovacdir(VacDir*, Dir*, ulong entry, uvlong qid);
91 int bsize = BlockSize;
97 char *exclude[MaxExclude];
106 fprint(2, "usage: %s [-amqsv] [-h host] [-d vacfile] [-b blocksize] [-i name] [-e exclude] [-f vacfile] file ... \n", argv0);
111 threadmain(int argc, char *argv[])
127 bsize = unittoull(p);
137 if(nexclude >= MaxExclude)
138 sysfatal("too many exclusions\n");
139 exclude[nexclude] = ARGF();
140 if(exclude[nexclude] == nil)
181 if(bsize > VtMaxLumpSize)
182 bsize = VtMaxLumpSize;
185 fmtinstall('V', vtscorefmt);
189 sysfatal("could not connect to server: %r");
192 sysfatal("vtconnect: %r");
194 qsort(exclude, nexclude, sizeof(char*), strpcmp);
199 fprint(2, "warning: could not ask server to flush pending writes: %R\n");
202 fprint(2, "files %ld:%ld data %ld:%ld:%ld meta %ld\n", stats.file, stats.sfile,
203 stats.data, stats.skip, stats.sdata, stats.meta);
211 strpcmp(const void *p0, const void *p1)
213 return strcmp(*(char**)p0, *(char**)p1);
217 vacwrite(VtConn *z, uchar score[VtScoreSize], int type, uchar *buf, int n)
221 sha1(buf, n, score, nil);
224 return vtwrite(z, score, type, buf, n);
228 lastelem(char *oname)
234 if((p = strrchr(oname, '/')) == nil)
240 vac(VtConn *z, char *argv[])
245 uchar score[VtScoreSize], buf[VtRootSize];
255 if(getwd(cwd, sizeof(cwd)) == 0)
256 sysfatal("can't find current directory: %r\n");
258 dsink = dirsinkalloc(z, bsize, bsize);
262 fs = vacfsopen(z, dfile, VtOREAD, 1000);
264 fprint(2, "could not open diff: %s: %r\n", dfile);
269 fd = create(oname, OWRITE, 0666);
271 sysfatal("could not create file: %s: %r", oname);
277 sysfatal("dirfstat failed: %r");
279 dir->name = lastelem(oname);
283 for(; *argv; argv++) {
286 for (cp = *argv; *cp; cp++)
298 vff = vacfileopen(fs, cp2);
299 vacfile(dsink, argv[0], cp2, vff);
302 if(cd && chdir(cwd) < 0)
303 sysfatal("can't cd back to %s: %r\n", cwd);
309 vff = vacfileopen(fs, isi);
310 vacstdin(dsink, isi, vff);
317 /* build meta information for the root */
318 ms = metasinkalloc(z, bsize, bsize);
319 /* fake into a directory */
320 dir->mode |= (dir->mode&0444)>>2;
321 dir->qid.type |= QTDIR;
323 plan9tovacdir(&vd, dir, 0, fileid++);
324 if(strcmp(vd.elem, "/") == 0){
326 vd.elem = vtstrdup("root");
328 metasinkwritedir(ms, &vd);
332 ds = dirsinkalloc(z, bsize, bsize);
333 dirsinkwritesink(ds, dsink->sink);
334 dirsinkwritesink(ds, dsink->msink->sink);
335 dirsinkwritesink(ds, ms->sink);
338 memset(&root, 0, sizeof(root));
339 strncpy(root.name, dir->name, sizeof(root.name));
340 root.name[sizeof(root.name)-1] = 0;
342 sprint(root.type, "vac");
343 memmove(root.score, ds->sink->dir.score, VtScoreSize);
344 root.blocksize = maxbsize;
346 vacfsgetscore(fs, root.prev);
354 vtrootpack(&root, buf);
355 if(vacwrite(z, score, VtRootType, buf, VtRootSize) < 0)
356 sysfatal("vacWrite failed: %r");
358 fprint(fd, "vac:%V\n", score);
360 /* avoid remove at cleanup */
365 isexcluded(char *name)
373 x = strcmp(exclude[i], name);
389 if(lstat(name, &st) < 0)
391 if((st.st_mode&S_IFMT) == S_IFLNK)
397 vacfile(DirSink *dsink, char *lname, char *sname, VacFile *vf)
404 if(isexcluded(lname)) {
405 warn("excluding: %s", lname);
409 if(merge && vacmerge(dsink, lname, sname) >= 0)
415 fd = open(sname, OREAD);
417 warn("could not open file: %s: %r", lname);
422 fprint(2, "%s\n", lname);
426 warn("can't stat %s: %r", lname);
430 dir->name = lastelem(sname);
432 entry = dsink->nentry;
434 if(dir->mode & DMDIR)
435 vacdir(dsink, fd, lname, sname, vf);
437 vacdata(dsink, fd, lname, vf, dir);
439 plan9tovacdir(&vd, dir, entry, fileid++);
440 metasinkwritedir(dsink->msink, &vd);
448 vacstdin(DirSink *dsink, char *name, VacFile *vf)
455 fprint(2, "%s\n", "<stdio>");
459 warn("can't stat <stdio>: %r");
464 entry = dsink->nentry;
466 vacdata(dsink, 0, "<stdin>", vf, dir);
468 plan9tovacdir(&vd, dir, entry, fileid++);
469 vd.elem = vtstrdup(name);
470 metasinkwritedir(dsink->msink, &vd);
477 sha1check(u8int *score, uchar *buf, int n)
479 char score2[VtScoreSize];
481 sha1(buf, n, score, nil);
482 if(memcmp(score, score2, VtScoreSize) == 0)
488 vacdataskip(Sink *sink, VacFile *vf, int fd, ulong blocks, uchar *buf, char *lname)
492 uchar score[VtScoreSize];
494 /* skip blocks for append only files */
495 if(seek(fd, (blocks-1)*bsize, 0) != (blocks-1)*bsize) {
496 warn("error seeking: %s", lname);
499 n = readn(fd, buf, bsize);
501 warn("error checking append only file: %s", lname);
504 if(vacfileblockscore(vf, blocks-1, score)<0 || sha1check(score, buf, n)<0) {
505 warn("last block of append file did not match: %s", lname);
509 for(i=0; i<blocks; i++) {
510 if(vacfileblockscore(vf, i, score) < 0) {
511 warn("could not get score: %s: %lud", lname, i);
512 seek(fd, i*bsize, 0);
516 sinkwritescore(sink, score, bsize);
526 vacdata(DirSink *dsink, int fd, char *lname, VacFile *vf, Dir *dir)
531 uchar score[VtScoreSize];
537 if(vf != nil && qdiff) {
538 vacfilegetdir(vf, &vd);
539 if(vd.mtime == dir->mtime)
540 if(vd.size == dir->length)
541 if(!vd.plan9 || /* vd.p9path == dir->qid.path && */ vd.p9version == dir->qid.vers)
542 if(dirsinkwritefile(dsink, vf)) {
548 /* look for an append only file */
549 if((dir->mode&DMAPPEND) != 0)
550 if(vd.size < dir->length)
552 if(vd.p9path == dir->qid.path)
553 vfblocks = vd.size/bsize;
559 buf = vtmalloc(bsize);
560 sink = sinkalloc(dsink->sink->z, bsize, bsize);
562 same = stats.sdata+stats.skip;
565 block += vacdataskip(sink, vf, fd, vfblocks, buf, lname);
567 if(0) fprint(2, "vacData: %s: %ld\n", lname, block);
569 n = readn(fd, buf, bsize);
571 warn("file truncated due to read error: %s: %r", lname);
574 if(vf != nil && vacfileblockscore(vf, block, score) && sha1check(score, buf, n)>=0) {
576 sinkwritescore(sink, score, n);
578 sinkwrite(sink, buf, n);
581 same = stats.sdata+stats.skip - same;
583 if(same && (dir->mode&DMAPPEND) != 0)
584 if(0)fprint(2, "%s: total %lud same %lud:%lud diff %lud\n",
585 lname, block, same, vfblocks, block-same);
588 dirsinkwritesink(dsink, sink);
595 vacdir(DirSink *dsink, int fd, char *lname, char *sname, VacFile *vf)
604 ds = dirsinkalloc(dsink->sink->z, bsize, bsize);
605 while((nd = dirread(fd, &dirs)) > 0){
606 for(i = 0; i < nd; i++){
608 /* check for bad file names */
609 if(name[0] == 0 || strcmp(name, ".") == 0 || strcmp(name, "..") == 0)
611 ln = vtmalloc(strlen(lname) + strlen(name) + 2);
612 sn = vtmalloc(strlen(sname) + strlen(name) + 2);
613 sprint(ln, "%s/%s", lname, name);
614 sprint(sn, "%s/%s", sname, name);
616 vvf = vacfilewalk(vf, name);
619 vacfile(ds, ln, sn, vvf);
628 dirsinkwritesink(dsink, ds->sink);
629 dirsinkwritesink(dsink, ds->msink->sink);
634 vacmergefile(DirSink *dsink, VacFile *vf, VacDir *dir, uvlong offset, uvlong *max)
636 uchar buf[VtEntrySize];
640 if(vacfileread(vf, buf, VtEntrySize, (uvlong)dir->entry*VtEntrySize) != VtEntrySize) {
641 warn("could not read venti dir entry: %s\n", dir->elem);
644 vtentryunpack(&dd, buf, 0);
646 if(dir->mode & ModeDir) {
651 if(vacfileread(vf, buf, VtEntrySize, e*VtEntrySize) != VtEntrySize) {
652 warn("could not read venti dir entry: %s\n", dir->elem);
655 vtentryunpack(&md, buf, 0);
658 /* max might incorrect in some old dumps */
659 if(dir->qid >= *max) {
660 warn("qid out of range: %s", dir->elem);
665 dir->entry = dsink->nentry;
668 dir->qidoffset += offset;
671 dir->qidoffset = offset;
675 dirsinkwrite(dsink, &dd);
676 if(dir->mode & ModeDir)
677 dirsinkwrite(dsink, &md);
678 metasinkwritedir(dsink->msink, dir);
684 vacmerge(DirSink *dsink, char *lname, char *sname)
693 p = strrchr(sname, '.');
694 if(p == 0 || strcmp(p, ".vac"))
698 fs = vacfsopen(dsink->sink->z, sname, VtOREAD, 100);
702 vf = vacfileopen(fs, "/");
705 max = vacfilegetid(vf);
711 fprint(2, "merging: %s\n", lname);
713 if(maxbsize < fs->bsize)
714 maxbsize = fs->bsize;
717 if(vderead(d, &dir) < 1)
719 vacmergefile(dsink, vf, &dir, fileid, &max);
734 sinkalloc(VtConn *z, int psize, int dsize)
739 if(psize < 512 || psize > VtMaxLumpSize)
740 sysfatal("sinkalloc: bad psize");
741 if(dsize < 512 || dsize > VtMaxLumpSize)
742 sysfatal("sinkalloc: bad psize");
744 psize = VtScoreSize*(psize/VtScoreSize);
746 k = vtmallocz(sizeof(Sink));
748 k->dir.flags = VtEntryActive;
749 k->dir.psize = psize;
750 k->dir.dsize = dsize;
751 k->buf = vtmallocz(VtPointerDepth*k->dir.psize + VtScoreSize);
752 for(i=0; i<=VtPointerDepth; i++)
753 k->pbuf[i] = k->buf + i*k->dir.psize;
758 sinkwritescore(Sink *k, uchar score[VtScoreSize], int n)
764 memmove(k->pbuf[0], score, VtScoreSize);
768 for(i=0; i<VtPointerDepth; i++) {
769 k->pbuf[i] += VtScoreSize;
770 if(k->pbuf[i] < k->buf + d->psize*(i+1))
772 if(i == VtPointerDepth-1)
773 sysfatal("file too big");
774 p = k->buf+i*d->psize;
776 if(vacwrite(k->z, k->pbuf[i+1], VtDataType+1+i, p, d->psize) < 0)
777 sysfatal("vacwrite failed: %r");
781 /* round size up to multiple of dsize */
782 d->size = d->dsize * ((d->size + d->dsize-1)/d->dsize);
788 sinkwrite(Sink *k, uchar *p, int n)
791 uchar score[VtScoreSize];
794 sysfatal("sinkWrite: size too big");
796 if((k->dir.type&~VtTypeDepthMask) == VtDirType){
803 if(vacwrite(k->z, score, type, p, n) < 0)
804 sysfatal("vacWrite failed: %r");
806 sinkwritescore(k, score, n);
810 sizetodepth(uvlong s, int psize, int dsize)
815 /* determine pointer depth */
816 np = psize/VtScoreSize;
817 s = (s + dsize - 1)/dsize;
818 for(d = 0; s > 1; d++)
834 memmove(kd->score, vtzeroscore, VtScoreSize);
838 for(n=VtPointerDepth-1; n>0; n--)
839 if(k->pbuf[n] > k->buf + kd->psize*n)
842 base = kd->type&~VtTypeDepthMask;
843 kd->type = base + sizetodepth(kd->size, kd->psize, kd->dsize);
845 /* skip full part of tree */
846 for(i=0; i<n && k->pbuf[i] == k->buf + kd->psize*i; i++)
849 /* is the tree completely full */
850 if(i == n && k->pbuf[n] == k->buf + kd->psize*n + VtScoreSize) {
851 memmove(kd->score, k->pbuf[n] - VtScoreSize, VtScoreSize);
856 /* clean up the edge */
858 p = k->buf+i*kd->psize;
860 if(vacwrite(k->z, k->pbuf[i+1], base+1+i, p, k->pbuf[i]-p) < 0)
861 sysfatal("vacWrite failed: %r");
862 k->pbuf[i+1] += VtScoreSize;
864 memmove(kd->score, k->pbuf[i] - VtScoreSize, VtScoreSize);
875 dirsinkalloc(VtConn *z, int psize, int dsize)
880 ds = VtEntrySize*(dsize/VtEntrySize);
882 k = vtmallocz(sizeof(DirSink));
883 k->sink = sinkalloc(z, psize, ds);
884 k->sink->dir.type = VtDirType;
885 k->msink = metasinkalloc(z, psize, dsize);
886 k->buf = vtmalloc(ds);
893 dirsinkwrite(DirSink *k, VtEntry *dir)
895 if(k->p + VtEntrySize > k->ep) {
896 sinkwrite(k->sink, k->buf, k->p - k->buf);
899 vtentrypack(dir, k->p, 0);
905 dirsinkwritesink(DirSink *k, Sink *sink)
907 dirsinkwrite(k, &sink->dir);
911 dirsinkwritefile(DirSink *k, VacFile *vf)
915 if(vacfilegetvtentry(vf, &dir) < 0)
917 dirsinkwrite(k, &dir);
922 dirsinkclose(DirSink *k)
924 metasinkclose(k->msink);
926 sinkwrite(k->sink, k->buf, k->p - k->buf);
931 dirsinkfree(DirSink *k)
934 metasinkfree(k->msink);
940 metasinkalloc(VtConn *z, int psize, int dsize)
944 k = vtmallocz(sizeof(MetaSink));
945 k->sink = sinkalloc(z, psize, dsize);
946 k->buf = vtmalloc(dsize);
947 k->maxindex = dsize/100; /* 100 byte entries seems reasonable */
950 k->rp = k->p = k->buf + MetaHeaderSize + k->maxindex*MetaIndexSize;
951 k->ep = k->buf + dsize;
955 /* hack to get base to compare routine - not reentrant */
959 dircmp(const void *p0, const void *p1)
964 /* name is first element of entry */
966 q0 = blockbase + (q0[0]<<8) + q0[1];
967 n0 = (q0[6]<<8) + q0[7];
971 q1 = blockbase + (q1[0]<<8) + q1[1];
972 n1 = (q1[6]<<8) + q1[7];
976 return memcmp(q0, q1, n0);
978 r = memcmp(q0, q1, n0);
981 r = memcmp(q0, q1, n1);
987 metasinkflush(MetaSink *k)
995 assert(k->nindex <= k->maxindex);
1002 mb.nindex = k->nindex;
1003 mb.maxindex = k->maxindex;
1007 p += MetaHeaderSize;
1009 /* XXX this is not reentrant! */
1011 qsort(p, k->nindex, MetaIndexSize, dircmp);
1012 p += k->nindex*MetaIndexSize;
1014 memset(p, 0, (k->maxindex-k->nindex)*MetaIndexSize);
1015 p += (k->maxindex-k->nindex)*MetaIndexSize;
1017 sinkwrite(k->sink, k->buf, n);
1019 /* move down partial entry */
1021 memmove(p, k->rp, n);
1028 metasinkputc(MetaSink *k, int c)
1033 sysfatal("directory entry too large");
1039 metasinkputstring(MetaSink *k, char *s)
1042 metasinkputc(k, n>>8);
1044 metasinkwrite(k, (uchar*)s, n);
1048 metasinkputuint32(MetaSink *k, ulong x)
1050 metasinkputc(k, x>>24);
1051 metasinkputc(k, x>>16);
1052 metasinkputc(k, x>>8);
1057 metasinkputuint64(MetaSink *k, uvlong x)
1059 metasinkputuint32(k, x>>32);
1060 metasinkputuint32(k, x);
1064 metasinkwrite(MetaSink *k, uchar *data, int n)
1066 if(k->p + n > k->ep)
1068 if(k->p + n > k->ep)
1069 sysfatal("directory entry too large");
1071 memmove(k->p, data, n);
1076 metasinkwritedir(MetaSink *ms, VacDir *dir)
1078 metasinkputuint32(ms, DirMagic);
1079 metasinkputc(ms, Version>>8);
1080 metasinkputc(ms, Version);
1081 metasinkputstring(ms, dir->elem);
1082 metasinkputuint32(ms, dir->entry);
1083 metasinkputuint64(ms, dir->qid);
1084 metasinkputstring(ms, dir->uid);
1085 metasinkputstring(ms, dir->gid);
1086 metasinkputstring(ms, dir->mid);
1087 metasinkputuint32(ms, dir->mtime);
1088 metasinkputuint32(ms, dir->mcount);
1089 metasinkputuint32(ms, dir->ctime);
1090 metasinkputuint32(ms, dir->atime);
1091 metasinkputuint32(ms, dir->mode);
1094 metasinkputc(ms, DirPlan9Entry); /* plan9 extra info */
1095 metasinkputc(ms, 0); /* plan9 extra size */
1096 metasinkputc(ms, 12); /* plan9 extra size */
1097 metasinkputuint64(ms, dir->p9path);
1098 metasinkputuint32(ms, dir->p9version);
1101 if(dir->qidspace != 0) {
1102 metasinkputc(ms, DirQidSpaceEntry);
1103 metasinkputc(ms, 0);
1104 metasinkputc(ms, 16);
1105 metasinkputuint64(ms, dir->qidoffset);
1106 metasinkputuint64(ms, dir->qidmax);
1110 metasinkputc(ms, DirGenEntry);
1111 metasinkputc(ms, 0);
1112 metasinkputc(ms, 4);
1113 metasinkputuint32(ms, dir->gen);
1121 plan9tovacdir(VacDir *vd, Dir *dir, ulong entry, uvlong qid)
1123 memset(vd, 0, sizeof(VacDir));
1125 vd->elem = vtstrdup(dir->name);
1128 vd->uid = vtstrdup(dir->uid);
1129 vd->gid = vtstrdup(dir->gid);
1130 vd->mid = vtstrdup(dir->muid);
1131 vd->mtime = dir->mtime;
1133 vd->ctime = dir->mtime; /* ctime: not available on plan 9 */
1134 vd->atime = dir->atime;
1136 vd->mode = dir->mode & 0777;
1137 if(dir->mode & DMDIR)
1138 vd->mode |= ModeDir;
1139 if(dir->mode & DMAPPEND)
1140 vd->mode |= ModeAppend;
1141 if(dir->mode & DMEXCL)
1142 vd->mode |= ModeExclusive;
1145 vd->p9path = dir->qid.path;
1146 vd->p9version = dir->qid.vers;
1151 metasinkeor(MetaSink *k)
1156 p = k->buf + MetaHeaderSize;
1157 p += k->nindex * MetaIndexSize;
1158 o = k->rp-k->buf; /* offset from start of block */
1159 n = k->p-k->rp; /* size of entry */
1166 if(k->nindex == k->maxindex)
1171 metasinkclose(MetaSink *k)
1178 metasinkfree(MetaSink *k)
1186 warn(char *fmt, ...)
1191 fprint(2, "%s: ", argv0);
1192 vfprint(2, fmt, arg);
1204 #define TWID64 ((u64int)~(u64int)0)
1214 n = strtoul(s, &es, 0);
1215 if(*es == 'k' || *es == 'K'){
1218 }else if(*es == 'm' || *es == 'M'){
1221 }else if(*es == 'g' || *es == 'G'){
1222 n *= 1024*1024*1024;