8 int mainstacksize = 128*1024;
10 typedef struct Sink Sink;
11 typedef struct MetaSink MetaSink;
12 typedef struct DirSink DirSink;
18 uchar *pbuf[VtPointerDepth+1];
26 uchar *p; /* current pointer */
27 uchar *ep; /* end pointer */
35 uchar *rp; /* start of current record */
36 uchar *p; /* current pointer */
37 uchar *ep; /* end pointer */
40 static void usage(void);
41 static int strpcmp(const void*, const void*);
42 static void warn(char *fmt, ...);
43 static void cleanup(void);
44 static u64int unittoull(char *s);
45 static void vac(VtConn *z, char *argv[]);
46 static void vacfile(DirSink *dsink, char *lname, char *sname, VacFile*);
47 static void vacstdin(DirSink *dsink, char *name, VacFile *vf);
48 static void vacdata(DirSink *dsink, int fd, char *lname, VacFile*, Dir*);
49 static void vacdir(DirSink *dsink, int fd, char *lname, char *sname, VacFile*);
50 static int vacmerge(DirSink *dsink, char *lname, char *sname);
51 static int vacspecial(DirSink *dsink, Dir *dir, char *lname, char *sname, VacFile *vf);
52 Sink *sinkalloc(VtConn *z, int psize, int dsize);
53 void sinkwrite(Sink *k, uchar *data, int n);
54 void sinkwritescore(Sink *k, uchar *score, int n);
55 void sinkclose(Sink *k);
56 void sinkfree(Sink *k);
58 DirSink *dirsinkalloc(VtConn *z, int psize, int dsize);
59 void dirsinkwrite(DirSink *k, VtEntry*);
60 void dirsinkwritesink(DirSink *k, Sink*);
61 int dirsinkwritefile(DirSink *k, VacFile *vf);
62 void dirsinkclose(DirSink *k);
63 void dirsinkfree(DirSink *k);
65 MetaSink *metasinkalloc(VtConn *z, int psize, int dsize);
66 void metasinkputc(MetaSink *k, int c);
67 void metasinkputstring(MetaSink *k, char *s);
68 void metasinkputuint32(MetaSink *k, ulong x);
69 void metasinkputuint64(MetaSink *k, uvlong x);
70 void metasinkwrite(MetaSink *k, uchar *data, int n);
71 void metasinkwritedir(MetaSink *ms, VacDir *vd);
72 void metasinkeor(MetaSink *k);
73 void metasinkclose(MetaSink *k);
74 void metasinkfree(MetaSink *k);
75 void plan9tovacdir(VacDir*, Dir*, ulong entry, uvlong qid);
92 int bsize = BlockSize;
98 char *exclude[MaxExclude];
107 fprint(2, "usage: %s [-amqsv] [-h host] [-d vacfile] [-b blocksize] [-i name] [-e exclude] [-f vacfile] file ... \n", argv0);
108 threadexitsall("usage");
112 threadmain(int argc, char *argv[])
128 bsize = unittoull(p);
138 if(nexclude >= MaxExclude)
139 sysfatal("too many exclusions\n");
140 exclude[nexclude] = ARGF();
141 if(exclude[nexclude] == nil)
182 if(bsize > VtMaxLumpSize)
183 bsize = VtMaxLumpSize;
186 fmtinstall('V', vtscorefmt);
190 sysfatal("could not connect to server: %r");
193 sysfatal("vtconnect: %r");
195 qsort(exclude, nexclude, sizeof(char*), strpcmp);
200 fprint(2, "warning: could not ask server to flush pending writes: %r\n");
203 fprint(2, "files %ld:%ld data %ld:%ld:%ld meta %ld\n", stats.file, stats.sfile,
204 stats.data, stats.skip, stats.sdata, stats.meta);
212 strpcmp(const void *p0, const void *p1)
214 return strcmp(*(char**)p0, *(char**)p1);
218 vacwrite(VtConn *z, uchar score[VtScoreSize], int type, uchar *buf, int n)
222 sha1(buf, n, score, nil);
225 return vtwrite(z, score, type, buf, n);
229 lastelem(char *oname)
235 if((p = strrchr(oname, '/')) == nil)
241 vac(VtConn *z, char *argv[])
246 uchar score[VtScoreSize], buf[VtRootSize];
256 if(getwd(cwd, sizeof(cwd)) == 0)
257 sysfatal("can't find current directory: %r\n");
259 dsink = dirsinkalloc(z, bsize, bsize);
263 fs = vacfsopen(z, dfile, VtOREAD, 1000);
265 fprint(2, "could not open diff: %s: %r\n", dfile);
270 fd = create(oname, OWRITE, 0666);
272 sysfatal("could not create file: %s: %r", oname);
278 sysfatal("dirfstat failed: %r");
280 dir->name = lastelem(oname);
284 for(; *argv; argv++) {
287 for (cp = *argv; *cp; cp++)
299 vff = vacfileopen(fs, cp2);
300 vacfile(dsink, argv[0], cp2, vff);
303 if(cd && chdir(cwd) < 0)
304 sysfatal("can't cd back to %s: %r\n", cwd);
310 vff = vacfileopen(fs, isi);
311 vacstdin(dsink, isi, vff);
318 /* build meta information for the root */
319 ms = metasinkalloc(z, bsize, bsize);
320 /* fake into a directory */
321 dir->mode = DMDIR|0555;
322 dir->qid.type |= QTDIR;
323 plan9tovacdir(&vd, dir, 0, fileid++);
324 if(strcmp(vd.elem, "/") == 0){
326 vd.elem = vtstrdup("root");
328 metasinkwritedir(ms, &vd);
332 ds = dirsinkalloc(z, bsize, bsize);
333 dirsinkwritesink(ds, dsink->sink);
334 dirsinkwritesink(ds, dsink->msink->sink);
335 dirsinkwritesink(ds, ms->sink);
338 memset(&root, 0, sizeof(root));
339 strncpy(root.name, dir->name, sizeof(root.name));
340 root.name[sizeof(root.name)-1] = 0;
342 sprint(root.type, "vac");
343 memmove(root.score, ds->sink->dir.score, VtScoreSize);
344 root.blocksize = maxbsize;
346 vacfsgetscore(fs, root.prev);
354 vtrootpack(&root, buf);
355 if(vacwrite(z, score, VtRootType, buf, VtRootSize) < 0)
356 sysfatal("vacWrite failed: %r");
358 fprint(fd, "vac:%V\n", score);
360 /* avoid remove at cleanup */
365 isexcluded(char *name)
373 x = strcmp(exclude[i], name);
385 vacfile(DirSink *dsink, char *lname, char *sname, VacFile *vf)
392 if(isexcluded(lname)) {
393 warn("excluding: %s", lname);
397 if(merge && vacmerge(dsink, lname, sname) >= 0)
400 if((dir = dirstat(sname)) == nil){
401 warn("could not stat file %s: %r", lname);
404 if(dir->mode&(DMSYMLINK|DMDEVICE|DMNAMEDPIPE)){
405 vacspecial(dsink, dir, lname, sname, vf);
408 }else if(dir->mode&DMSOCKET){
414 fd = open(sname, OREAD);
416 warn("could not open file: %s: %r", lname);
421 fprint(2, "%s\n", lname);
425 warn("can't stat %s: %r", lname);
429 dir->name = lastelem(sname);
431 entry = dsink->nentry;
433 if(dir->mode & DMDIR)
434 vacdir(dsink, fd, lname, sname, vf);
436 vacdata(dsink, fd, lname, vf, dir);
438 plan9tovacdir(&vd, dir, entry, fileid++);
439 metasinkwritedir(dsink->msink, &vd);
447 vacstdin(DirSink *dsink, char *name, VacFile *vf)
454 fprint(2, "%s\n", "<stdio>");
458 warn("can't stat <stdio>: %r");
463 entry = dsink->nentry;
465 vacdata(dsink, 0, "<stdin>", vf, dir);
467 plan9tovacdir(&vd, dir, entry, fileid++);
468 vd.elem = vtstrdup(name);
469 metasinkwritedir(dsink->msink, &vd);
476 sha1check(u8int *score, uchar *buf, int n)
478 uchar score2[VtScoreSize];
480 sha1(buf, n, score2, nil);
481 if(memcmp(score, score2, VtScoreSize) == 0)
487 vacdataskip(Sink *sink, VacFile *vf, int fd, ulong blocks, uchar *buf, char *lname)
491 uchar score[VtScoreSize];
493 /* skip blocks for append only files */
494 if(seek(fd, (blocks-1)*bsize, 0) != (blocks-1)*bsize) {
495 warn("error seeking: %s", lname);
498 n = readn(fd, buf, bsize);
500 warn("error checking append only file: %s", lname);
503 if(vacfileblockscore(vf, blocks-1, score)<0 || sha1check(score, buf, n)<0) {
504 warn("last block of append file did not match: %s", lname);
508 for(i=0; i<blocks; i++) {
509 if(vacfileblockscore(vf, i, score) < 0) {
510 warn("could not get score: %s: %lud", lname, i);
511 seek(fd, i*bsize, 0);
515 sinkwritescore(sink, score, bsize);
525 vacdata(DirSink *dsink, int fd, char *lname, VacFile *vf, Dir *dir)
530 uchar score[VtScoreSize];
536 if(vf != nil && qdiff) {
537 vacfilegetdir(vf, &vd);
538 if(vd.mtime == dir->mtime)
539 if(vd.size == dir->length)
540 if(!vd.plan9 || /* vd.p9path == dir->qid.path && */ vd.p9version == dir->qid.vers)
541 if(dirsinkwritefile(dsink, vf) >= 0) {
548 fprint(2, "+ %s\n", lname);
550 /* look for an append only file */
551 if((dir->mode&DMAPPEND) != 0)
552 if(vd.size < dir->length)
554 if(vd.p9path == dir->qid.path)
555 vfblocks = vd.size/bsize;
561 buf = vtmalloc(bsize);
562 sink = sinkalloc(dsink->sink->z, bsize, bsize);
564 same = stats.sdata+stats.skip;
567 block += vacdataskip(sink, vf, fd, vfblocks, buf, lname);
569 if(0) fprint(2, "vacData: %s: %ld\n", lname, block);
571 n = readn(fd, buf, bsize);
573 warn("file truncated due to read error: %s: %r", lname);
576 if(vf != nil && vacfileblockscore(vf, block, score)>=0 && sha1check(score, buf, n)>=0) {
578 sinkwritescore(sink, score, n);
580 sinkwrite(sink, buf, n);
583 same = stats.sdata+stats.skip - same;
585 if(same && (dir->mode&DMAPPEND) != 0)
586 if(0)fprint(2, "%s: total %lud same %lud:%lud diff %lud\n",
587 lname, block, same, vfblocks, block-same);
590 dirsinkwritesink(dsink, sink);
597 vacdir(DirSink *dsink, int fd, char *lname, char *sname, VacFile *vf)
606 ds = dirsinkalloc(dsink->sink->z, bsize, bsize);
607 while((nd = dirread(fd, &dirs)) > 0){
608 for(i = 0; i < nd; i++){
610 /* check for bad file names */
611 if(name[0] == 0 || strcmp(name, ".") == 0 || strcmp(name, "..") == 0)
613 ln = vtmalloc(strlen(lname) + strlen(name) + 2);
614 sn = vtmalloc(strlen(sname) + strlen(name) + 2);
622 vvf = vacfilewalk(vf, name);
625 vacfile(ds, ln, sn, vvf);
634 dirsinkwritesink(dsink, ds->sink);
635 dirsinkwritesink(dsink, ds->msink->sink);
640 vacmergefile(DirSink *dsink, VacFile *vf, VacDir *dir, uvlong offset, uvlong *max)
642 uchar buf[VtEntrySize];
646 if(vacfileread(vf, buf, VtEntrySize, (uvlong)dir->entry*VtEntrySize) != VtEntrySize) {
647 warn("could not read venti dir entry: %s\n", dir->elem);
650 vtentryunpack(&dd, buf, 0);
652 if(dir->mode & ModeDir) {
657 if(vacfileread(vf, buf, VtEntrySize, e*VtEntrySize) != VtEntrySize) {
658 warn("could not read venti dir entry: %s\n", dir->elem);
661 vtentryunpack(&md, buf, 0);
664 /* max might incorrect in some old dumps */
665 if(dir->qid >= *max) {
666 warn("qid out of range: %s", dir->elem);
671 dir->entry = dsink->nentry;
674 dir->qidoffset += offset;
677 dir->qidoffset = offset;
681 dirsinkwrite(dsink, &dd);
682 if(dir->mode & ModeDir)
683 dirsinkwrite(dsink, &md);
684 metasinkwritedir(dsink->msink, dir);
690 vacmerge(DirSink *dsink, char *lname, char *sname)
699 if((p=strrchr(sname, '.')) == nil || strcmp(p, ".vac") != 0)
703 fs = vacfsopen(dsink->sink->z, sname, VtOREAD, 100);
707 vf = vacfileopen(fs, "/");
710 max = vacfilegetid(vf);
716 fprint(2, "merging: %s\n", lname);
718 if(maxbsize < fs->bsize)
719 maxbsize = fs->bsize;
722 if(vderead(d, &dir) < 1)
724 vacmergefile(dsink, vf, &dir, fileid, &max);
739 vacspecial(DirSink *dsink, Dir* dir, char *lname, char *sname, VacFile *vf)
742 int buflen, dtype, major, minor, n;
749 buf = malloc(buflen);
754 fprint(2, "%s\n", lname);
756 dir->name = lastelem(sname);
758 if(dir->mode & DMSYMLINK){
759 while((n = readlink(sname, buf, buflen)) == buflen){
761 btmp = vtrealloc(buf, buflen);
769 dir->mode |= DMSYMLINK;
770 }else if(dir->mode & DMDEVICE){
771 dtype = (dir->qid.path >> 16) & 0xFF;
772 minor = dir->qid.path & 0xff;
773 major = (dir->qid.path >> 8) & 0xFF;
774 n = snprint(buf, buflen, "%c %d %d", dtype, major, minor);
777 entry = dsink->nentry;
779 sink = sinkalloc(dsink->sink->z, bsize, bsize);
780 sinkwrite(sink, (uchar*)buf, n);
782 dirsinkwritesink(dsink, sink);
786 dir->name = lastelem(sname);
788 plan9tovacdir(&vd, dir, entry, fileid++);
789 metasinkwritedir(dsink->msink, &vd);
797 sinkalloc(VtConn *z, int psize, int dsize)
802 if(psize < 512 || psize > VtMaxLumpSize)
803 sysfatal("sinkalloc: bad psize");
804 if(dsize < 512 || dsize > VtMaxLumpSize)
805 sysfatal("sinkalloc: bad psize");
807 psize = VtScoreSize*(psize/VtScoreSize);
809 k = vtmallocz(sizeof(Sink));
811 k->dir.flags = VtEntryActive;
812 k->dir.psize = psize;
813 k->dir.dsize = dsize;
814 k->buf = vtmallocz(VtPointerDepth*k->dir.psize + VtScoreSize);
815 for(i=0; i<=VtPointerDepth; i++)
816 k->pbuf[i] = k->buf + i*k->dir.psize;
821 sinkwritescore(Sink *k, uchar score[VtScoreSize], int n)
827 memmove(k->pbuf[0], score, VtScoreSize);
831 for(i=0; i<VtPointerDepth; i++) {
832 k->pbuf[i] += VtScoreSize;
833 if(k->pbuf[i] < k->buf + d->psize*(i+1))
835 if(i == VtPointerDepth-1)
836 sysfatal("file too big");
837 p = k->buf+i*d->psize;
839 if(vacwrite(k->z, k->pbuf[i+1], VtDataType+1+i, p, d->psize) < 0)
840 sysfatal("vacwrite failed: %r");
844 /* round size up to multiple of dsize */
845 d->size = d->dsize * ((d->size + d->dsize-1)/d->dsize);
851 sinkwrite(Sink *k, uchar *p, int n)
854 uchar score[VtScoreSize];
860 sysfatal("sinkWrite: size too big");
862 if((k->dir.type&~VtTypeDepthMask) == VtDirType){
869 if(vacwrite(k->z, score, type, p, n) < 0)
870 sysfatal("vacWrite failed: %r");
872 sinkwritescore(k, score, n);
876 sizetodepth(uvlong s, int psize, int dsize)
881 /* determine pointer depth */
882 np = psize/VtScoreSize;
883 s = (s + dsize - 1)/dsize;
884 for(d = 0; s > 1; d++)
900 memmove(kd->score, vtzeroscore, VtScoreSize);
904 for(n=VtPointerDepth-1; n>0; n--)
905 if(k->pbuf[n] > k->buf + kd->psize*n)
908 base = kd->type&~VtTypeDepthMask;
909 kd->type = base + sizetodepth(kd->size, kd->psize, kd->dsize);
911 /* skip full part of tree */
912 for(i=0; i<n && k->pbuf[i] == k->buf + kd->psize*i; i++)
915 /* is the tree completely full */
916 if(i == n && k->pbuf[n] == k->buf + kd->psize*n + VtScoreSize) {
917 memmove(kd->score, k->pbuf[n] - VtScoreSize, VtScoreSize);
922 /* clean up the edge */
924 p = k->buf+i*kd->psize;
926 if(vacwrite(k->z, k->pbuf[i+1], base+1+i, p, k->pbuf[i]-p) < 0)
927 sysfatal("vacWrite failed: %r");
928 k->pbuf[i+1] += VtScoreSize;
930 memmove(kd->score, k->pbuf[i] - VtScoreSize, VtScoreSize);
941 dirsinkalloc(VtConn *z, int psize, int dsize)
946 ds = VtEntrySize*(dsize/VtEntrySize);
948 k = vtmallocz(sizeof(DirSink));
949 k->sink = sinkalloc(z, psize, ds);
950 k->sink->dir.type = VtDirType;
951 k->msink = metasinkalloc(z, psize, dsize);
952 k->buf = vtmalloc(ds);
959 dirsinkwrite(DirSink *k, VtEntry *dir)
961 if(k->p + VtEntrySize > k->ep) {
962 sinkwrite(k->sink, k->buf, k->p - k->buf);
965 vtentrypack(dir, k->p, 0);
971 dirsinkwritesink(DirSink *k, Sink *sink)
973 dirsinkwrite(k, &sink->dir);
977 dirsinkwritefile(DirSink *k, VacFile *vf)
981 if(vacfilegetvtentry(vf, &dir) < 0)
983 dirsinkwrite(k, &dir);
988 dirsinkclose(DirSink *k)
990 metasinkclose(k->msink);
992 sinkwrite(k->sink, k->buf, k->p - k->buf);
997 dirsinkfree(DirSink *k)
1000 metasinkfree(k->msink);
1006 metasinkalloc(VtConn *z, int psize, int dsize)
1010 k = vtmallocz(sizeof(MetaSink));
1011 k->sink = sinkalloc(z, psize, dsize);
1012 k->buf = vtmalloc(dsize);
1013 k->maxindex = dsize/100; /* 100 byte entries seems reasonable */
1016 k->rp = k->p = k->buf + MetaHeaderSize + k->maxindex*MetaIndexSize;
1017 k->ep = k->buf + dsize;
1021 /* hack to get base to compare routine - not reentrant */
1025 dircmp(const void *p0, const void *p1)
1030 /* name is first element of entry */
1032 q0 = blockbase + (q0[0]<<8) + q0[1];
1033 n0 = (q0[6]<<8) + q0[7];
1037 q1 = blockbase + (q1[0]<<8) + q1[1];
1038 n1 = (q1[6]<<8) + q1[7];
1042 return memcmp(q0, q1, n0);
1044 r = memcmp(q0, q1, n0);
1047 r = memcmp(q0, q1, n1);
1053 metasinkflush(MetaSink *k)
1061 assert(k->nindex <= k->maxindex);
1068 mb.nindex = k->nindex;
1069 mb.maxindex = k->maxindex;
1073 p += MetaHeaderSize;
1075 /* XXX this is not reentrant! */
1077 qsort(p, k->nindex, MetaIndexSize, dircmp);
1078 p += k->nindex*MetaIndexSize;
1080 memset(p, 0, (k->maxindex-k->nindex)*MetaIndexSize);
1081 p += (k->maxindex-k->nindex)*MetaIndexSize;
1083 sinkwrite(k->sink, k->buf, n);
1085 /* move down partial entry */
1087 memmove(p, k->rp, n);
1094 metasinkputc(MetaSink *k, int c)
1099 sysfatal("directory entry too large");
1105 metasinkputstring(MetaSink *k, char *s)
1108 metasinkputc(k, n>>8);
1110 metasinkwrite(k, (uchar*)s, n);
1114 metasinkputuint32(MetaSink *k, ulong x)
1116 metasinkputc(k, x>>24);
1117 metasinkputc(k, x>>16);
1118 metasinkputc(k, x>>8);
1123 metasinkputuint64(MetaSink *k, uvlong x)
1125 metasinkputuint32(k, x>>32);
1126 metasinkputuint32(k, x);
1130 metasinkwrite(MetaSink *k, uchar *data, int n)
1132 if(k->p + n > k->ep)
1134 if(k->p + n > k->ep)
1135 sysfatal("directory entry too large");
1137 memmove(k->p, data, n);
1142 metasinkwritedir(MetaSink *ms, VacDir *dir)
1144 metasinkputuint32(ms, DirMagic);
1145 metasinkputc(ms, Version>>8);
1146 metasinkputc(ms, Version);
1147 metasinkputstring(ms, dir->elem);
1148 metasinkputuint32(ms, dir->entry);
1149 metasinkputuint64(ms, dir->qid);
1150 metasinkputstring(ms, dir->uid);
1151 metasinkputstring(ms, dir->gid);
1152 metasinkputstring(ms, dir->mid);
1153 metasinkputuint32(ms, dir->mtime);
1154 metasinkputuint32(ms, dir->mcount);
1155 metasinkputuint32(ms, dir->ctime);
1156 metasinkputuint32(ms, dir->atime);
1157 metasinkputuint32(ms, dir->mode);
1160 metasinkputc(ms, DirPlan9Entry); /* plan9 extra info */
1161 metasinkputc(ms, 0); /* plan9 extra size */
1162 metasinkputc(ms, 12); /* plan9 extra size */
1163 metasinkputuint64(ms, dir->p9path);
1164 metasinkputuint32(ms, dir->p9version);
1167 if(dir->qidspace != 0) {
1168 metasinkputc(ms, DirQidSpaceEntry);
1169 metasinkputc(ms, 0);
1170 metasinkputc(ms, 16);
1171 metasinkputuint64(ms, dir->qidoffset);
1172 metasinkputuint64(ms, dir->qidmax);
1176 metasinkputc(ms, DirGenEntry);
1177 metasinkputc(ms, 0);
1178 metasinkputc(ms, 4);
1179 metasinkputuint32(ms, dir->gen);
1187 plan9tovacdir(VacDir *vd, Dir *dir, ulong entry, uvlong qid)
1189 memset(vd, 0, sizeof(VacDir));
1191 vd->elem = vtstrdup(dir->name);
1194 vd->uid = vtstrdup(dir->uid);
1195 vd->gid = vtstrdup(dir->gid);
1196 vd->mid = vtstrdup(dir->muid);
1197 vd->mtime = dir->mtime;
1199 vd->ctime = dir->mtime; /* ctime: not available on plan 9 */
1200 vd->atime = dir->atime;
1202 vd->mode = dir->mode & 0777;
1203 if(dir->mode & DMDIR)
1204 vd->mode |= ModeDir;
1205 if(dir->mode & DMAPPEND)
1206 vd->mode |= ModeAppend;
1207 if(dir->mode & DMEXCL)
1208 vd->mode |= ModeExclusive;
1209 if(dir->mode & DMDEVICE)
1210 vd->mode |= ModeDevice;
1211 if(dir->mode & DMNAMEDPIPE)
1212 vd->mode |= ModeNamedPipe;
1213 if(dir->mode & DMSYMLINK)
1214 vd->mode |= ModeLink;
1217 vd->p9path = dir->qid.path;
1218 vd->p9version = dir->qid.vers;
1223 metasinkeor(MetaSink *k)
1228 p = k->buf + MetaHeaderSize;
1229 p += k->nindex * MetaIndexSize;
1230 o = k->rp-k->buf; /* offset from start of block */
1231 n = k->p-k->rp; /* size of entry */
1238 if(k->nindex == k->maxindex)
1243 metasinkclose(MetaSink *k)
1250 metasinkfree(MetaSink *k)
1258 warn(char *fmt, ...)
1263 fprint(2, "%s: ", argv0);
1264 vfprint(2, fmt, arg);
1276 #define TWID64 ((u64int)~(u64int)0)
1286 n = strtoul(s, &es, 0);
1287 if(*es == 'k' || *es == 'K'){
1290 }else if(*es == 'm' || *es == 'M'){
1293 }else if(*es == 'g' || *es == 'G'){
1294 n *= 1024*1024*1024;