1 #include <sys/stat.h>
2 #include "stdinc.h"
3 #include "vac.h"
4 #include "dat.h"
5 #include "fns.h"
6 
7 int mainstacksize = 128*1024;
8 
9 typedef struct Sink Sink;
10 typedef struct MetaSink MetaSink;
11 typedef struct DirSink DirSink;
12 
13 struct Sink {
14 	VtConn *z;
15 	VtEntry dir;
16 	uchar *buf;
17 	uchar *pbuf[VtPointerDepth+1];
18 };
19 
20 struct DirSink {
21 	Sink *sink;
22 	MetaSink *msink;
23 	ulong nentry;
24 	uchar *buf;
25 	uchar *p;	/* current pointer */
26 	uchar *ep;	/* end pointer */
27 };
28 
29 struct MetaSink {
30 	Sink *sink;
31 	uchar *buf;
32 	int maxindex;
33 	int nindex;
34 	uchar *rp;	/* start of current record */
35 	uchar *p;	/* current pointer */
36 	uchar *ep;	/* end pointer */
37 };
38 
39 static void usage(void);
40 static int strpcmp(const void*, const void*);
41 static void warn(char *fmt, ...);
42 static void cleanup(void);
43 static u64int unittoull(char *s);
44 static void vac(VtConn *z, char *argv[]);
45 static void vacfile(DirSink *dsink, char *lname, char *sname, VacFile*);
46 static void vacstdin(DirSink *dsink, char *name, VacFile *vf);
47 static void vacdata(DirSink *dsink, int fd, char *lname, VacFile*, Dir*);
48 static void vacdir(DirSink *dsink, int fd, char *lname, char *sname, VacFile*);
49 static int vacmerge(DirSink *dsink, char *lname, char *sname);
50 
51 Sink *sinkalloc(VtConn *z, int psize, int dsize);
52 void sinkwrite(Sink *k, uchar *data, int n);
53 void sinkwritescore(Sink *k, uchar *score, int n);
54 void sinkclose(Sink *k);
55 void sinkfree(Sink *k);
56 
57 DirSink *dirsinkalloc(VtConn *z, int psize, int dsize);
58 void dirsinkwrite(DirSink *k, VtEntry*);
59 void dirsinkwritesink(DirSink *k, Sink*);
60 int dirsinkwritefile(DirSink *k, VacFile *vf);
61 void dirsinkclose(DirSink *k);
62 void dirsinkfree(DirSink *k);
63 
64 MetaSink *metasinkalloc(VtConn *z, int psize, int dsize);
65 void metasinkputc(MetaSink *k, int c);
66 void metasinkputstring(MetaSink *k, char *s);
67 void metasinkputuint32(MetaSink *k, ulong x);
68 void metasinkputuint64(MetaSink *k, uvlong x);
69 void metasinkwrite(MetaSink *k, uchar *data, int n);
70 void metasinkwritedir(MetaSink *ms, VacDir *vd);
71 void metasinkeor(MetaSink *k);
72 void metasinkclose(MetaSink *k);
73 void metasinkfree(MetaSink *k);
74 void plan9tovacdir(VacDir*, Dir*, ulong entry, uvlong qid);
75 
76 enum {
77 	Version = 8,
78 	BlockSize = 8*1024,
79 	MaxExclude = 1000,
80 };
81 
82 struct {
83 	ulong	file;
84 	ulong	sfile;
85 	ulong	data;
86 	ulong	sdata;
87 	ulong	skip;
88 	ulong	meta;
89 } stats;
90 
91 int bsize = BlockSize;
92 int maxbsize;
93 char *oname, *dfile;
94 int verbose;
95 uvlong fileid = 1;
96 int qdiff;
97 char *exclude[MaxExclude];
98 int nexclude;
99 int nowrite;
100 int merge;
101 char *isi;
102 
103 static void
104 usage(void)
105 {
106 	fprint(2, "usage: %s [-amqsv] [-h host] [-d vacfile] [-b blocksize] [-i name] [-e exclude] [-f vacfile] file ... \n", argv0);
107 	exits("usage");
108 }
109 
110 void
111 threadmain(int argc, char *argv[])
112 {
113 	VtConn *z;
114 	char *p;
115 	char *host = nil;
116 	int statsflag = 0;
117 
118 	atexit(cleanup);
119 
120 	ARGBEGIN{
121 	default:
122 		usage();
123 	case 'b':
124 		p = ARGF();
125 		if(p == 0)
126 			usage();
127 		bsize = unittoull(p);
128 		if(bsize == ~0)
129 			usage();
130 		break;
131 	case 'd':
132 		dfile = ARGF();
133 		if(dfile == nil)
134 			usage();
135 		break;
136 	case 'e':
137 		if(nexclude >= MaxExclude)
138 			sysfatal("too many exclusions\n");
139 		exclude[nexclude] = ARGF();
140 		if(exclude[nexclude] == nil)
141 			usage();
142 		nexclude++;
143 		break;
144 	case 'f':
145 		oname = ARGF();
146 		if(oname == 0)
147 			usage();
148 		break;
149 	case 'h':
150 		host = ARGF();
151 		if(host == nil)
152 			usage();
153 		break;
154 	case 'i':
155 		isi = ARGF();
156 		if(isi == nil)
157 			usage();
158 		break;
159 	case 'n':
160 		nowrite++;
161 		break;
162 	case 'm':
163 		merge++;
164 		break;
165 	case 'q':
166 		qdiff++;
167 		break;
168 	case 's':
169 		statsflag++;
170 		break;
171 	case 'v':
172 		verbose++;
173 		break;
174 	}ARGEND;
175 
176 	if(argc == 0)
177 		usage();
178 
179 	if(bsize < 512)
180 		bsize = 512;
181 	if(bsize > VtMaxLumpSize)
182 		bsize = VtMaxLumpSize;
183 	maxbsize = bsize;
184 
185 	fmtinstall('V', vtscorefmt);
186 
187 	z = vtdial(host);
188 	if(z == nil)
189 		sysfatal("could not connect to server: %r");
190 
191 	if(vtconnect(z) < 0)
192 		sysfatal("vtconnect: %r");
193 
194 	qsort(exclude, nexclude, sizeof(char*), strpcmp);
195 
196 	vac(z, argv);
197 
198 	if(vtsync(z) < 0)
199 		fprint(2, "warning: could not ask server to flush pending writes: %R\n");
200 
201 	if(statsflag)
202 		fprint(2, "files %ld:%ld data %ld:%ld:%ld meta %ld\n", stats.file, stats.sfile,
203 			stats.data, stats.skip, stats.sdata, stats.meta);
204 //packetStats();
205 	vthangup(z);
206 
207 	threadexitsall(0);
208 }
209 
210 static int
211 strpcmp(const void *p0, const void *p1)
212 {
213 	return strcmp(*(char**)p0, *(char**)p1);
214 }
215 
216 int
217 vacwrite(VtConn *z, uchar score[VtScoreSize], int type, uchar *buf, int n)
218 {
219 	assert(n > 0);
220 	if(nowrite){
221 		sha1(buf, n, score, nil);
222 		return 0;
223 	}
224 	return vtwrite(z, score, type, buf, n);
225 }
226 
227 static char*
228 lastelem(char *oname)
229 {
230 	char *p;
231 
232 	if(oname == nil)
233 		abort();
234 	if((p = strrchr(oname, '/')) == nil)
235 		return oname;
236 	return p+1;
237 }
238 
239 static void
240 vac(VtConn *z, char *argv[])
241 {
242 	DirSink *dsink, *ds;
243 	MetaSink *ms;
244 	VtRoot root;
245 	uchar score[VtScoreSize], buf[VtRootSize];
246 	char cwd[2048];
247 	int cd;
248 	char *cp2, *cp;
249 	VacFs *fs;
250 	VacFile *vff;
251 	int fd;
252 	Dir *dir;
253 	VacDir vd;
254 
255 	if(getwd(cwd, sizeof(cwd)) == 0)
256 		sysfatal("can't find current directory: %r\n");
257 
258 	dsink = dirsinkalloc(z, bsize, bsize);
259 
260 	fs = nil;
261 	if(dfile != nil) {
262 		fs = vacfsopen(z, dfile, VtOREAD, 1000);
263 		if(fs == nil)
264 			fprint(2, "could not open diff: %s: %r\n", dfile);
265 	}
266 		
267 
268 	if(oname != nil) {
269 		fd = create(oname, OWRITE, 0666);
270 		if(fd < 0)
271 			sysfatal("could not create file: %s: %r", oname);
272 	} else 
273 		fd = 1;
274 
275 	dir = dirfstat(fd);
276 	if(dir == nil)
277 		sysfatal("dirfstat failed: %r");
278 	if(oname)
279 		dir->name = lastelem(oname);
280 	else
281 		dir->name = "stdin";
282 
283 	for(; *argv; argv++) {
284 		cp2 = *argv;
285 		cd = 0;
286 		for (cp = *argv; *cp; cp++)
287 			if (*cp == '/')
288 				cp2 = cp;
289 		if (cp2 != *argv) {
290 			*cp2 = '\0';
291 			chdir(*argv);
292 			*cp2 = '/';
293 			cp2++;
294 			cd = 1;
295 		}
296 		vff = nil;
297 		if(fs)
298 			vff = vacfileopen(fs, cp2);
299 		vacfile(dsink, argv[0], cp2, vff);
300 		if(vff)
301 			vacfiledecref(vff);
302 		if(cd && chdir(cwd) < 0)
303 			sysfatal("can't cd back to %s: %r\n", cwd);
304 	}
305 	
306 	if(isi) {
307 		vff = nil;
308 		if(fs)
309 			vff = vacfileopen(fs, isi);
310 		vacstdin(dsink, isi, vff);
311 		if(vff)
312 			vacfiledecref(vff);
313 	}
314 
315 	dirsinkclose(dsink);
316 
317 	/* build meta information for the root */
318 	ms = metasinkalloc(z, bsize, bsize);
319 	/* fake into a directory */
320 	dir->mode |= (dir->mode&0444)>>2;
321 	dir->qid.type |= QTDIR;
322 	dir->mode |= DMDIR;
323 	plan9tovacdir(&vd, dir, 0, fileid++);
324 	if(strcmp(vd.elem, "/") == 0){
325 		vtfree(vd.elem);
326 		vd.elem = vtstrdup("root");
327 	}
328 	metasinkwritedir(ms, &vd);
329 	vdcleanup(&vd);
330 	metasinkclose(ms);
331 	
332 	ds = dirsinkalloc(z, bsize, bsize);
333 	dirsinkwritesink(ds, dsink->sink);
334 	dirsinkwritesink(ds, dsink->msink->sink);
335 	dirsinkwritesink(ds, ms->sink);
336 	dirsinkclose(ds);
337 
338 	memset(&root, 0, sizeof(root));		
339 	strncpy(root.name, dir->name, sizeof(root.name));
340 	root.name[sizeof(root.name)-1] = 0;
341 	free(dir);
342 	sprint(root.type, "vac");
343 	memmove(root.score, ds->sink->dir.score, VtScoreSize);
344 	root.blocksize = maxbsize;
345 	if(fs != nil)
346 		vacfsgetscore(fs, root.prev);
347 
348 	metasinkfree(ms);
349 	dirsinkfree(ds);
350 	dirsinkfree(dsink);
351 	if(fs != nil)
352 		vacfsclose(fs);
353 	
354 	vtrootpack(&root, buf);
355 	if(vacwrite(z, score, VtRootType, buf, VtRootSize) < 0)
356 		sysfatal("vacWrite failed: %r");
357 
358 	fprint(fd, "vac:%V\n", score);
359 
360 	/* avoid remove at cleanup */
361 	oname = nil;
362 }
363 
364 static int
365 isexcluded(char *name)
366 {
367 	int bot, top, i, x;
368 
369 	bot = 0;	
370 	top = nexclude;
371 	while(bot < top) {
372 		i = (bot+top)>>1;
373 		x = strcmp(exclude[i], name);
374 		if(x == 0)
375 			return 1;
376 		if(x < 0)
377 			bot = i + 1;
378 		else /* x > 0 */
379 			top = i;
380 	}
381 	return 0;
382 }
383 
384 static int
385 islink(char *name)
386 {
387 	struct stat st;
388 
389 	if(lstat(name, &st) < 0)
390 		return 0;
391 	if((st.st_mode&S_IFMT) == S_IFLNK)
392 		return 1;
393 	return 0;
394 }
395 
396 static void
397 vacfile(DirSink *dsink, char *lname, char *sname, VacFile *vf)
398 {
399 	int fd;
400 	Dir *dir;
401 	VacDir vd;
402 	ulong entry;
403 
404 	if(isexcluded(lname)) {
405 		warn("excluding: %s", lname);
406 		return;
407 	}
408 
409 	if(merge && vacmerge(dsink, lname, sname) >= 0)
410 		return;
411 
412 	if(islink(sname))
413 		return;
414 
415 	fd = open(sname, OREAD);
416 	if(fd < 0) {
417 		warn("could not open file: %s: %r", lname);
418 		return;
419 	}
420 
421 	if(verbose)
422 		fprint(2, "%s\n", lname);
423 
424 	dir = dirfstat(fd);
425 	if(dir == nil) {
426 		warn("can't stat %s: %r", lname);
427 		close(fd);
428 		return;
429 	}
430 	dir->name = lastelem(sname);
431 
432 	entry = dsink->nentry;
433 
434 	if(dir->mode & DMDIR) 
435 		vacdir(dsink, fd, lname, sname, vf);
436 	else
437 		vacdata(dsink, fd, lname, vf, dir);
438 
439 	plan9tovacdir(&vd, dir, entry, fileid++);
440 	metasinkwritedir(dsink->msink, &vd);
441 	vdcleanup(&vd);
442 
443 	free(dir);
444 	close(fd);
445 }
446 
447 static void
448 vacstdin(DirSink *dsink, char *name, VacFile *vf)
449 {
450 	Dir *dir;
451 	VacDir vd;
452 	ulong entry;
453 
454 	if(verbose)
455 		fprint(2, "%s\n", "<stdio>");
456 
457 	dir = dirfstat(0);
458 	if(dir == nil) {
459 		warn("can't stat <stdio>: %r");
460 		return;
461 	}
462 	dir->name = "stdin";
463 
464 	entry = dsink->nentry;
465 
466 	vacdata(dsink, 0, "<stdin>", vf, dir);
467 
468 	plan9tovacdir(&vd, dir, entry, fileid++);
469 	vd.elem = vtstrdup(name);
470 	metasinkwritedir(dsink->msink, &vd);
471 	vdcleanup(&vd);
472 
473 	free(dir);
474 }
475 
476 static int
477 sha1check(u8int *score, uchar *buf, int n)
478 {
479 	char score2[VtScoreSize];
480 
481 	sha1(buf, n, score, nil);
482 	if(memcmp(score, score2, VtScoreSize) == 0)
483 		return 0;
484 	return -1;
485 }
486 
487 static ulong
488 vacdataskip(Sink *sink, VacFile *vf, int fd, ulong blocks, uchar *buf, char *lname)
489 {
490 	int n;
491 	ulong i;
492 	uchar score[VtScoreSize];
493 
494 	/* skip blocks for append only files */
495 	if(seek(fd, (blocks-1)*bsize, 0) != (blocks-1)*bsize) {
496 		warn("error seeking: %s", lname);
497 		goto Err;
498 	}
499 	n = readn(fd, buf, bsize);
500 	if(n < bsize) {
501 		warn("error checking append only file: %s", lname);
502 		goto Err;
503 	}
504 	if(vacfileblockscore(vf, blocks-1, score)<0 || sha1check(score, buf, n)<0) {
505 		warn("last block of append file did not match: %s", lname);
506 		goto Err;
507 	}
508 
509 	for(i=0; i<blocks; i++) {
510 		if(vacfileblockscore(vf, i, score) < 0) {
511 			warn("could not get score: %s: %lud", lname, i);
512 			seek(fd, i*bsize, 0);
513 			return i;
514 		}
515 		stats.skip++;
516 		sinkwritescore(sink, score, bsize);
517 	}
518 
519 	return i;
520 Err:
521 	seek(fd, 0, 0);
522 	return 0;
523 }
524 
525 static void
526 vacdata(DirSink *dsink, int fd, char *lname, VacFile *vf, Dir *dir)
527 {
528 	uchar *buf;
529 	Sink *sink;
530 	int n;
531 	uchar score[VtScoreSize];
532 	ulong block, same;
533 	VacDir vd;
534 	ulong vfblocks;
535 
536 	vfblocks = 0;
537 	if(vf != nil && qdiff) {
538 		vacfilegetdir(vf, &vd);
539 		if(vd.mtime == dir->mtime)
540 		if(vd.size == dir->length)
541 		if(!vd.plan9 || /* vd.p9path == dir->qid.path && */ vd.p9version == dir->qid.vers)
542 		if(dirsinkwritefile(dsink, vf)) {
543 			stats.sfile++;
544 			vdcleanup(&vd);
545 			return;
546 		}
547 
548 		/* look for an append only file */
549 		if((dir->mode&DMAPPEND) != 0)
550 		if(vd.size < dir->length)
551 		if(vd.plan9)
552 		if(vd.p9path == dir->qid.path)
553 			vfblocks = vd.size/bsize;
554 
555 		vdcleanup(&vd);
556 	}
557 	stats.file++;
558 
559 	buf = vtmalloc(bsize);
560 	sink = sinkalloc(dsink->sink->z, bsize, bsize);
561 	block = 0;
562 	same = stats.sdata+stats.skip;
563 
564 	if(vfblocks > 1)
565 		block += vacdataskip(sink, vf, fd, vfblocks, buf, lname);
566 
567 if(0) fprint(2, "vacData: %s: %ld\n", lname, block);
568 	for(;;) {
569 		n = readn(fd, buf, bsize);
570 		if(0 && n < 0)
571 			warn("file truncated due to read error: %s: %r", lname);
572 		if(n <= 0)
573 			break;
574 		if(vf != nil && vacfileblockscore(vf, block, score) && sha1check(score, buf, n)>=0) {
575 			stats.sdata++;
576 			sinkwritescore(sink, score, n);
577 		} else
578 			sinkwrite(sink, buf, n);
579 		block++;
580 	}
581 	same = stats.sdata+stats.skip - same;
582 
583 	if(same && (dir->mode&DMAPPEND) != 0)
584 		if(0)fprint(2, "%s: total %lud same %lud:%lud diff %lud\n",
585 			lname, block, same, vfblocks, block-same);
586 
587 	sinkclose(sink);
588 	dirsinkwritesink(dsink, sink);
589 	sinkfree(sink);
590 	free(buf);
591 }
592 
593 
594 static void
595 vacdir(DirSink *dsink, int fd, char *lname, char *sname, VacFile *vf)
596 {
597 	Dir *dirs;
598 	char *ln, *sn;
599 	int i, nd;
600 	DirSink *ds;
601 	VacFile *vvf;
602 	char *name;
603 
604 	ds = dirsinkalloc(dsink->sink->z, bsize, bsize);
605 	while((nd = dirread(fd, &dirs)) > 0){
606 		for(i = 0; i < nd; i++){
607 			name = dirs[i].name;
608 			/* check for bad file names */
609 			if(name[0] == 0 || strcmp(name, ".") == 0 || strcmp(name, "..") == 0)
610 				continue;
611 			ln = vtmalloc(strlen(lname) + strlen(name) + 2);
612 			sn = vtmalloc(strlen(sname) + strlen(name) + 2);
613 			sprint(ln, "%s/%s", lname, name);
614 			sprint(sn, "%s/%s", sname, name);
615 			if(vf != nil)
616 				vvf = vacfilewalk(vf, name);
617 			else
618 				vvf = nil;
619 			vacfile(ds, ln, sn, vvf);
620 			if(vvf != nil)
621 				vacfiledecref(vvf);
622 			vtfree(ln);
623 			vtfree(sn);
624 		}
625 		free(dirs);
626 	}
627 	dirsinkclose(ds);
628 	dirsinkwritesink(dsink, ds->sink);
629 	dirsinkwritesink(dsink, ds->msink->sink);
630 	dirsinkfree(ds);
631 }
632 
633 static int
634 vacmergefile(DirSink *dsink, VacFile *vf, VacDir *dir, uvlong offset, uvlong *max)
635 {
636 	uchar buf[VtEntrySize];
637 	VtEntry dd, md;
638 	int e;
639 
640 	if(vacfileread(vf, buf, VtEntrySize, (uvlong)dir->entry*VtEntrySize) != VtEntrySize) {
641 		warn("could not read venti dir entry: %s\n", dir->elem);
642 		return -1;
643 	}
644 	vtentryunpack(&dd, buf, 0);
645 
646 	if(dir->mode & ModeDir)	{
647 		e = dir->mentry;
648 		if(e == 0)
649 			e = dir->entry + 1;
650 		
651 		if(vacfileread(vf, buf, VtEntrySize, e*VtEntrySize) != VtEntrySize) {
652 			warn("could not read venti dir entry: %s\n", dir->elem);
653 			return 0;
654 		}
655 		vtentryunpack(&md, buf, 0);
656 	}
657 
658 	/* max might incorrect in some old dumps */
659 	if(dir->qid >= *max) {
660 		warn("qid out of range: %s", dir->elem);
661 		*max = dir->qid;
662 	}
663 
664 	dir->qid += offset;
665 	dir->entry = dsink->nentry;
666 
667 	if(dir->qidspace) {
668 		dir->qidoffset += offset;
669 	} else {
670 		dir->qidspace = 1;
671 		dir->qidoffset = offset;
672 		dir->qidmax = *max;
673 	}
674 
675 	dirsinkwrite(dsink, &dd);
676 	if(dir->mode & ModeDir)	
677 		dirsinkwrite(dsink, &md);
678 	metasinkwritedir(dsink->msink, dir);
679 	
680 	return 0;
681 }
682 
683 static int
684 vacmerge(DirSink *dsink, char *lname, char *sname)
685 {
686 	char *p;
687 	VacFs *fs;
688 	VacFile *vf;
689 	VacDirEnum *d;
690 	VacDir dir;
691 	uvlong max;
692 
693 	p = strrchr(sname, '.');
694 	if(p == 0 || strcmp(p, ".vac"))
695 		return 0;
696 
697 	d = nil;
698 	fs = vacfsopen(dsink->sink->z, sname, VtOREAD, 100);
699 	if(fs == nil)
700 		return -1;
701 
702 	vf = vacfileopen(fs, "/");
703 	if(vf == nil)
704 		goto Done;
705 	max = vacfilegetid(vf);
706 	d = vdeopen(vf);
707 	if(d == nil)
708 		goto Done;
709 
710 	if(verbose)
711 		fprint(2, "merging: %s\n", lname);
712 
713 	if(maxbsize < fs->bsize)
714 		maxbsize = fs->bsize;
715 
716 	for(;;) {
717 		if(vderead(d, &dir) < 1)
718 			break;
719 		vacmergefile(dsink, vf, &dir, fileid, &max);
720 		vdcleanup(&dir);	
721 	}
722 	fileid += max;
723 
724 Done:
725 	if(d != nil)
726 		vdeclose(d);
727 	if(vf != nil)
728 		vacfiledecref(vf);
729 	vacfsclose(fs);
730 	return 0;
731 }
732 
733 Sink *
734 sinkalloc(VtConn *z, int psize, int dsize)
735 {
736 	Sink *k;
737 	int i;
738 
739 	if(psize < 512 || psize > VtMaxLumpSize)
740 		sysfatal("sinkalloc: bad psize");
741 	if(dsize < 512 || dsize > VtMaxLumpSize)
742 		sysfatal("sinkalloc: bad psize");
743 
744 	psize = VtScoreSize*(psize/VtScoreSize);
745 
746 	k = vtmallocz(sizeof(Sink));
747 	k->z = z;
748 	k->dir.flags = VtEntryActive;
749 	k->dir.psize = psize;
750 	k->dir.dsize = dsize;
751 	k->buf = vtmallocz(VtPointerDepth*k->dir.psize + VtScoreSize);
752 	for(i=0; i<=VtPointerDepth; i++)
753 		k->pbuf[i] = k->buf + i*k->dir.psize;
754 	return k;
755 }
756 
757 void
758 sinkwritescore(Sink *k, uchar score[VtScoreSize], int n)
759 {
760 	int i;
761 	uchar *p;
762 	VtEntry *d;
763 
764 	memmove(k->pbuf[0], score, VtScoreSize);
765 
766 	d = &k->dir;
767 
768 	for(i=0; i<VtPointerDepth; i++) {
769 		k->pbuf[i] += VtScoreSize;
770 		if(k->pbuf[i] < k->buf + d->psize*(i+1))
771 			break;
772 		if(i == VtPointerDepth-1)
773 			sysfatal("file too big");
774 		p = k->buf+i*d->psize;
775 		stats.meta++;
776 		if(vacwrite(k->z, k->pbuf[i+1], VtDataType+1+i, p, d->psize) < 0)
777 			sysfatal("vacwrite failed: %r");
778 		k->pbuf[i] = p;
779 	}
780 
781 	/* round size up to multiple of dsize */
782 	d->size = d->dsize * ((d->size + d->dsize-1)/d->dsize);
783 	
784 	d->size += n;
785 }
786 
787 void
788 sinkwrite(Sink *k, uchar *p, int n)
789 {
790 	int type;
791 	uchar score[VtScoreSize];
792 
793 	if(n > k->dir.dsize)
794 		sysfatal("sinkWrite: size too big");
795 
796 	if((k->dir.type&~VtTypeDepthMask) == VtDirType){
797 		type = VtDirType;
798 		stats.meta++;
799 	} else {
800 		type = VtDataType;
801 		stats.data++;
802 	}
803 	if(vacwrite(k->z, score, type, p, n) < 0)
804 		sysfatal("vacWrite failed: %r");
805 
806 	sinkwritescore(k, score, n);
807 }
808 
809 static int
810 sizetodepth(uvlong s, int psize, int dsize)
811 {
812 	int np;
813 	int d;
814 	
815 	/* determine pointer depth */
816 	np = psize/VtScoreSize;
817 	s = (s + dsize - 1)/dsize;
818 	for(d = 0; s > 1; d++)
819 		s = (s + np - 1)/np;
820 	return d;
821 }
822 
823 void
824 sinkclose(Sink *k)
825 {
826 	int i, n, base;
827 	uchar *p;
828 	VtEntry *kd;
829 
830 	kd = &k->dir;
831 
832 	/* empty */
833 	if(kd->size == 0) {
834 		memmove(kd->score, vtzeroscore, VtScoreSize);
835 		return;
836 	}
837 
838 	for(n=VtPointerDepth-1; n>0; n--)
839 		if(k->pbuf[n] > k->buf + kd->psize*n)
840 			break;
841 
842 	base = kd->type&~VtTypeDepthMask;
843 	kd->type = base + sizetodepth(kd->size, kd->psize, kd->dsize);
844 
845 	/* skip full part of tree */
846 	for(i=0; i<n && k->pbuf[i] == k->buf + kd->psize*i; i++)
847 		;
848 
849 	/* is the tree completely full */
850 	if(i == n && k->pbuf[n] == k->buf + kd->psize*n + VtScoreSize) {
851 		memmove(kd->score, k->pbuf[n] - VtScoreSize, VtScoreSize);
852 		return;
853 	}
854 	n++;
855 
856 	/* clean up the edge */
857 	for(; i<n; i++) {
858 		p = k->buf+i*kd->psize;
859 		stats.meta++;
860 		if(vacwrite(k->z, k->pbuf[i+1], base+1+i, p, k->pbuf[i]-p) < 0)
861 			sysfatal("vacWrite failed: %r");
862 		k->pbuf[i+1] += VtScoreSize;
863 	}
864 	memmove(kd->score, k->pbuf[i] - VtScoreSize, VtScoreSize);
865 }
866 
867 void
868 sinkfree(Sink *k)
869 {
870 	vtfree(k->buf);
871 	vtfree(k);
872 }
873 
874 DirSink *
875 dirsinkalloc(VtConn *z, int psize, int dsize)
876 {
877 	DirSink *k;
878 	int ds;
879 
880 	ds = VtEntrySize*(dsize/VtEntrySize);
881 
882 	k = vtmallocz(sizeof(DirSink));
883 	k->sink = sinkalloc(z, psize, ds);
884 	k->sink->dir.type = VtDirType;
885 	k->msink = metasinkalloc(z, psize, dsize);
886 	k->buf = vtmalloc(ds);
887 	k->p = k->buf;
888 	k->ep = k->buf + ds;
889 	return k;
890 }
891 
892 void
893 dirsinkwrite(DirSink *k, VtEntry *dir)
894 {
895 	if(k->p + VtEntrySize > k->ep) {
896 		sinkwrite(k->sink, k->buf, k->p - k->buf);
897 		k->p = k->buf;
898 	}
899 	vtentrypack(dir, k->p, 0);
900 	k->nentry++;
901 	k->p += VtEntrySize;
902 }
903 
904 void
905 dirsinkwritesink(DirSink *k, Sink *sink)
906 {
907 	dirsinkwrite(k, &sink->dir);
908 }
909 
910 int
911 dirsinkwritefile(DirSink *k, VacFile *vf)
912 {
913 	VtEntry dir;
914 
915 	if(vacfilegetvtentry(vf, &dir) < 0)
916 		return -1;
917 	dirsinkwrite(k, &dir);
918 	return 0;
919 }
920 
921 void
922 dirsinkclose(DirSink *k)
923 {
924 	metasinkclose(k->msink);
925 	if(k->p != k->buf)
926 		sinkwrite(k->sink, k->buf, k->p - k->buf);
927 	sinkclose(k->sink);
928 }
929 
930 void
931 dirsinkfree(DirSink *k)
932 {
933 	sinkfree(k->sink);
934 	metasinkfree(k->msink);
935 	vtfree(k->buf);
936 	vtfree(k);
937 }
938 
939 MetaSink*
940 metasinkalloc(VtConn *z, int psize, int dsize)
941 {
942 	MetaSink *k;
943 
944 	k = vtmallocz(sizeof(MetaSink));
945 	k->sink = sinkalloc(z, psize, dsize);
946 	k->buf = vtmalloc(dsize);
947 	k->maxindex = dsize/100;	/* 100 byte entries seems reasonable */
948 	if(k->maxindex < 1)
949 		k->maxindex = 1;
950 	k->rp = k->p = k->buf + MetaHeaderSize + k->maxindex*MetaIndexSize;
951 	k->ep = k->buf + dsize;
952 	return k;
953 }
954 
955 /* hack to get base to compare routine - not reentrant */
956 uchar *blockbase;
957 
958 int
959 dircmp(const void *p0, const void *p1)
960 {
961 	uchar *q0, *q1;
962 	int n0, n1, r;
963 
964 	/* name is first element of entry */
965 	q0 = (uchar*)p0;
966 	q0 = blockbase + (q0[0]<<8) + q0[1];
967 	n0 = (q0[6]<<8) + q0[7];
968 	q0 += 8;
969 
970 	q1 = (uchar*)p1;
971 	q1 = blockbase + (q1[0]<<8) + q1[1];
972 	n1 = (q1[6]<<8) + q1[7];
973 	q1 += 8;
974 
975 	if(n0 == n1)
976 		return memcmp(q0, q1, n0);
977 	else if (n0 < n1) {
978 		r = memcmp(q0, q1, n0);
979 		return (r==0)?1:r;
980 	} else  {
981 		r = memcmp(q0, q1, n1);
982 		return (r==0)?-1:r;
983 	}
984 }
985 
986 void
987 metasinkflush(MetaSink *k)
988 {
989 	uchar *p;
990 	int n;
991 	MetaBlock mb;
992 
993 	if(k->nindex == 0)
994 		return;
995 	assert(k->nindex <= k->maxindex);
996 
997 	p = k->buf;
998 	n = k->rp - p;
999 
1000 	mb.size = n;
1001 	mb.free = 0;
1002 	mb.nindex = k->nindex;
1003 	mb.maxindex = k->maxindex;
1004 	mb.buf = p;
1005 	mbpack(&mb);
1006 	
1007 	p += MetaHeaderSize;
1008 
1009 	/* XXX this is not reentrant! */
1010 	blockbase = k->buf;
1011 	qsort(p, k->nindex, MetaIndexSize, dircmp);
1012 	p += k->nindex*MetaIndexSize;
1013 	
1014 	memset(p, 0, (k->maxindex-k->nindex)*MetaIndexSize);
1015 	p += (k->maxindex-k->nindex)*MetaIndexSize;
1016 
1017 	sinkwrite(k->sink, k->buf, n);
1018 
1019 	/* move down partial entry */
1020 	n = k->p - k->rp;
1021 	memmove(p, k->rp, n);
1022 	k->rp = p;
1023 	k->p = p + n;
1024 	k->nindex = 0;
1025 }
1026 
1027 void
1028 metasinkputc(MetaSink *k, int c)
1029 {
1030 	if(k->p+1 > k->ep)
1031 		metasinkflush(k);
1032 	if(k->p+1 > k->ep)
1033 		sysfatal("directory entry too large");
1034 	k->p[0] = c;
1035 	k->p++;
1036 }
1037 
1038 void
1039 metasinkputstring(MetaSink *k, char *s)
1040 {
1041 	int n = strlen(s);
1042 	metasinkputc(k, n>>8);
1043 	metasinkputc(k, n);
1044 	metasinkwrite(k, (uchar*)s, n);
1045 }
1046 
1047 void
1048 metasinkputuint32(MetaSink *k, ulong x)
1049 {
1050 	metasinkputc(k, x>>24);
1051 	metasinkputc(k, x>>16);
1052 	metasinkputc(k, x>>8);
1053 	metasinkputc(k, x);
1054 }
1055 
1056 void
1057 metasinkputuint64(MetaSink *k, uvlong x)
1058 {
1059 	metasinkputuint32(k, x>>32);
1060 	metasinkputuint32(k, x);
1061 }
1062 
1063 void
1064 metasinkwrite(MetaSink *k, uchar *data, int n)
1065 {
1066 	if(k->p + n > k->ep)
1067 		metasinkflush(k);
1068 	if(k->p + n > k->ep)
1069 		sysfatal("directory entry too large");
1070 	
1071 	memmove(k->p, data, n);
1072 	k->p += n;
1073 }
1074 
1075 void
1076 metasinkwritedir(MetaSink *ms, VacDir *dir)
1077 {
1078 	metasinkputuint32(ms, DirMagic);
1079 	metasinkputc(ms, Version>>8);
1080 	metasinkputc(ms, Version);		
1081 	metasinkputstring(ms, dir->elem);
1082 	metasinkputuint32(ms, dir->entry);
1083 	metasinkputuint64(ms, dir->qid);
1084 	metasinkputstring(ms, dir->uid);
1085 	metasinkputstring(ms, dir->gid);
1086 	metasinkputstring(ms, dir->mid);
1087 	metasinkputuint32(ms, dir->mtime);
1088 	metasinkputuint32(ms, dir->mcount);
1089 	metasinkputuint32(ms, dir->ctime);
1090 	metasinkputuint32(ms, dir->atime);
1091 	metasinkputuint32(ms, dir->mode);
1092 
1093 	if(dir->plan9) {
1094 		metasinkputc(ms, DirPlan9Entry);	/* plan9 extra info */
1095 		metasinkputc(ms, 0);			/* plan9 extra size */
1096 		metasinkputc(ms, 12);			/* plan9 extra size */
1097 		metasinkputuint64(ms, dir->p9path);
1098 		metasinkputuint32(ms, dir->p9version);
1099 	}
1100 
1101 	if(dir->qidspace != 0) {
1102 		metasinkputc(ms, DirQidSpaceEntry);
1103 		metasinkputc(ms, 0);
1104 		metasinkputc(ms, 16);
1105 		metasinkputuint64(ms, dir->qidoffset);
1106 		metasinkputuint64(ms, dir->qidmax);
1107 	}
1108 
1109 	if(dir->gen != 0) {
1110 		metasinkputc(ms, DirGenEntry);
1111 		metasinkputc(ms, 0);
1112 		metasinkputc(ms, 4);
1113 		metasinkputuint32(ms, dir->gen);
1114 	}
1115 
1116 	metasinkeor(ms);
1117 }
1118 
1119 
1120 void
1121 plan9tovacdir(VacDir *vd, Dir *dir, ulong entry, uvlong qid)
1122 {
1123 	memset(vd, 0, sizeof(VacDir));
1124 
1125 	vd->elem = vtstrdup(dir->name);
1126 	vd->entry = entry;
1127 	vd->qid = qid;
1128 	vd->uid = vtstrdup(dir->uid);
1129 	vd->gid = vtstrdup(dir->gid);
1130 	vd->mid = vtstrdup(dir->muid);
1131 	vd->mtime = dir->mtime;
1132 	vd->mcount = 0;
1133 	vd->ctime = dir->mtime;		/* ctime: not available on plan 9 */
1134 	vd->atime = dir->atime;
1135 
1136 	vd->mode = dir->mode & 0777;
1137 	if(dir->mode & DMDIR)
1138 		vd->mode |= ModeDir;
1139 	if(dir->mode & DMAPPEND)
1140 		vd->mode |= ModeAppend;
1141 	if(dir->mode & DMEXCL)
1142 		vd->mode |= ModeExclusive;
1143 
1144 	vd->plan9 = 1;
1145 	vd->p9path = dir->qid.path;
1146 	vd->p9version = dir->qid.vers;
1147 }
1148 
1149 
1150 void
1151 metasinkeor(MetaSink *k)
1152 {
1153 	uchar *p;
1154 	int o, n;
1155 
1156 	p = k->buf + MetaHeaderSize;
1157 	p += k->nindex * MetaIndexSize;
1158 	o = k->rp-k->buf; 	/* offset from start of block */
1159 	n = k->p-k->rp;		/* size of entry */
1160 	p[0] = o >> 8;
1161 	p[1] = o;
1162 	p[2] = n >> 8;
1163 	p[3] = n;
1164 	k->rp = k->p;
1165 	k->nindex++;
1166 	if(k->nindex == k->maxindex)
1167 		metasinkflush(k);
1168 }
1169 
1170 void
1171 metasinkclose(MetaSink *k)
1172 {
1173 	metasinkflush(k);
1174 	sinkclose(k->sink);
1175 }
1176 
1177 void
1178 metasinkfree(MetaSink *k)
1179 {
1180 	sinkfree(k->sink);
1181 	vtfree(k->buf);
1182 	vtfree(k);
1183 }
1184 
1185 static void
1186 warn(char *fmt, ...)
1187 {
1188 	va_list arg;
1189 
1190 	va_start(arg, fmt);
1191 	fprint(2, "%s: ", argv0);
1192 	vfprint(2, fmt, arg);
1193 	fprint(2, "\n");
1194 	va_end(arg);
1195 }
1196 
1197 static void
1198 cleanup(void)
1199 {
1200 	if(oname != nil)
1201 		remove(oname);
1202 }
1203 
1204 #define TWID64	((u64int)~(u64int)0)
1205 
1206 static u64int
1207 unittoull(char *s)
1208 {
1209 	char *es;
1210 	u64int n;
1211 
1212 	if(s == nil)
1213 		return TWID64;
1214 	n = strtoul(s, &es, 0);
1215 	if(*es == 'k' || *es == 'K'){
1216 		n *= 1024;
1217 		es++;
1218 	}else if(*es == 'm' || *es == 'M'){
1219 		n *= 1024*1024;
1220 		es++;
1221 	}else if(*es == 'g' || *es == 'G'){
1222 		n *= 1024*1024*1024;
1223 		es++;
1224 	}
1225 	if(*es != '\0')
1226 		return TWID64;
1227 	return n;
1228 }