op public repos

Blob

Date:: Sun Nov 23 17:55:34 2003 UTC
Message:: start thinking about vac -- doesn't build yet
Actions:: History | Blame | Raw File
1 #include "stdinc.h"
2 #include "vac.h"
3 #include "dat.h"
4 #include "fns.h"
5 
6 typedef struct Sink Sink;
7 typedef struct MetaSink MetaSink;
8 typedef struct DirSink DirSink;
9 
10 struct Sink {
11 	VtSession *z;
12 	VtEntry dir;
13 	uchar *buf;
14 	uchar *pbuf[VtPointerDepth+1];
15 };
16 
17 struct DirSink {
18 	Sink *sink;
19 	MetaSink *msink;
20 	ulong nentry;
21 	uchar *buf;
22 	uchar *p;	/* current pointer */
23 	uchar *ep;	/* end pointer */
24 };
25 
26 struct MetaSink {
27 	Sink *sink;
28 	uchar *buf;
29 	int maxindex;
30 	int nindex;
31 	uchar *rp;	/* start of current record */
32 	uchar *p;	/* current pointer */
33 	uchar *ep;	/* end pointer */
34 };
35 
36 static void usage(void);
37 static int strpCmp(void*, void*);
38 static void warn(char *fmt, ...);
39 static void cleanup(void);
40 static u64int unittoull(char *s);
41 static int vac(VtSession *z, char *argv[]);
42 static void vacFile(DirSink *dsink, char *lname, char *sname, VacFile*);
43 static void vacStdin(DirSink *dsink, char *name, VacFile *vf);
44 static void vacData(DirSink *dsink, int fd, char *lname, VacFile*, Dir*);
45 static void vacDir(DirSink *dsink, int fd, char *lname, char *sname, VacFile*);
46 static int vacMerge(DirSink *dsink, char *lname, char *sname);
47 
48 Sink *sinkAlloc(VtSession *z, int psize, int dsize);
49 void sinkWrite(Sink *k, uchar *data, int n);
50 void sinkWriteScore(Sink *k, uchar *score, int n);
51 void sinkClose(Sink *k);
52 void sinkFree(Sink *k);
53 
54 DirSink *dirSinkAlloc(VtSession *z, int psize, int dsize);
55 void dirSinkWrite(DirSink *k, VtEntry*);
56 void dirSinkWriteSink(DirSink *k, Sink*);
57 int dirSinkWriteFile(DirSink *k, VacFile *vf);
58 void dirSinkClose(DirSink *k);
59 void dirSinkFree(DirSink *k);
60 
61 MetaSink *metaSinkAlloc(VtSession *z, int psize, int dsize);
62 void metaSinkPutc(MetaSink *k, int c);
63 void metaSinkPutString(MetaSink *k, char *s);
64 void metaSinkPutUint32(MetaSink *k, ulong x);
65 void metaSinkPutUint64(MetaSink *k, uvlong x);
66 void metaSinkWrite(MetaSink *k, uchar *data, int n);
67 void metaSinkWriteDir(MetaSink *ms, VacDir *vd);
68 void metaSinkEOR(MetaSink *k);
69 void metaSinkClose(MetaSink *k);
70 void metaSinkFree(MetaSink *k);
71 void plan9ToVacDir(VacDir*, Dir*, ulong entry, uvlong qid);
72 
73 enum {
74 	Version = 8,
75 	BlockSize = 8*1024,
76 	MaxExclude = 1000,
77 };
78 
79 struct {
80 	ulong	file;
81 	ulong	sfile;
82 	ulong	data;
83 	ulong	sdata;
84 	ulong	skip;
85 	ulong	meta;
86 } stats;
87 
88 int bsize = BlockSize;
89 int maxbsize;
90 char *oname, *dfile;
91 int verbose;
92 uvlong fileid = 1;
93 int qdiff;
94 char *exclude[MaxExclude];
95 int nexclude;
96 int nowrite;
97 int merge;
98 char *isi;
99 
100 void
101 main(int argc, char *argv[])
102 {
103 	VtSession *z;
104 	char *p;
105 	char *host = nil;
106 	int statsFlag = 0;
107 
108 	atexit(cleanup);
109 
110 	ARGBEGIN{
111 	default:
112 		usage();
113 	case 'b':
114 		p = ARGF();
115 		if(p == 0)
116 			usage();
117 		bsize = unittoull(p);
118 		if(bsize == ~0)
119 			usage();
120 		break;
121 	case 'd':
122 		dfile = ARGF();
123 		if(dfile == nil)
124 			usage();
125 		break;
126 	case 'e':
127 		if(nexclude >= MaxExclude)
128 			sysfatal("too many exclusions\n");
129 		exclude[nexclude] = ARGF();
130 		if(exclude[nexclude] == nil)
131 			usage();
132 		nexclude++;
133 		break;
134 	case 'f':
135 		oname = ARGF();
136 		if(oname == 0)
137 			usage();
138 		break;
139 	case 'h':
140 		host = ARGF();
141 		if(host == nil)
142 			usage();
143 		break;
144 	case 'i':
145 		isi = ARGF();
146 		if(isi == nil)
147 			usage();
148 		break;
149 	case 'n':
150 		nowrite++;
151 		break;
152 	case 'm':
153 		merge++;
154 		break;
155 	case 'q':
156 		qdiff++;
157 		break;
158 	case 's':
159 		statsFlag++;
160 		break;
161 	case 'v':
162 		verbose++;
163 		break;
164 	}ARGEND;
165 
166 	if(bsize < 512)
167 		bsize = 512;
168 	if(bsize > VtMaxLumpSize)
169 		bsize = VtMaxLumpSize;
170 	maxbsize = bsize;
171 
172 	vtAttach();
173 
174 	fmtinstall('V', vtScoreFmt);
175 	fmtinstall('R', vtErrFmt);
176 
177 	z = vtDial(host, 0);
178 	if(z == nil)
179 		vtFatal("could not connect to server: %R");
180 
181 	if(!vtConnect(z, 0))
182 		vtFatal("vtConnect: %R");
183 
184 	qsort(exclude, nexclude, sizeof(char*), strpCmp);
185 
186 	vac(z, argv);
187 	if(!vtSync(z))
188 		fprint(2, "warning: could not ask server to flush pending writes: %R\n");
189 
190 	if(statsFlag)
191 		fprint(2, "files %ld:%ld data %ld:%ld:%ld meta %ld\n", stats.file, stats.sfile,
192 			stats.data, stats.skip, stats.sdata, stats.meta);
193 //packetStats();
194 	vtClose(z);
195 	vtDetach();
196 
197 	exits(0);
198 }
199 
200 void
201 static usage(void)
202 {
203 	fprint(2, "usage: %s [-amqsv] [-h host] [-d vacfile] [-b blocksize] [-i name] [-e exclude] [-f vacfile] file ... \n", argv0);
204 	exits("usage");
205 }
206 
207 static
208 int strpCmp(void *p0, void *p1)
209 {
210 	return strcmp(*(char**)p0, *(char**)p1);
211 }
212 
213 
214 int
215 readBlock(int fd, uchar *buf, int n)
216 {
217 	int m, t = 0;
218 
219 	while(t < n){
220 		m = read(fd, buf+t, n-t);
221 		if(m < 0)
222 			return -1;
223 		if(m == 0)
224 			break;
225 		t += m;
226 	}
227 	return t;
228 }
229 
230 int
231 vacWrite(VtSession *z, uchar score[VtScoreSize], int type, uchar *buf, int n)
232 {
233 assert(n > 0);
234 	if(nowrite) {
235 		vtSha1(score, buf, n);
236 		return 1;
237 	}
238 	if(!vtWrite(z, score, type, buf, n))
239 		return 0;
240 	if(!vtSha1Check(score, buf, n)) {
241 		uchar score2[VtScoreSize];
242 		
243 		vtSha1(score2, buf, n);
244 fprint(2, "vtSha1Check: n = %d %V %V\n", n, score, score2);
245 		vtSetError("vtSha1Check failed");
246 		return 0;
247 	}
248 	return 1;
249 }
250 
251 
252 static int
253 vac(VtSession *z, char *argv[])
254 {
255 	DirSink *dsink, *ds;
256 	MetaSink *ms;
257 	VtRoot root;
258 	uchar score[VtScoreSize], buf[VtRootSize];
259 	char cwd[2048];
260 	int cd, i;
261 	char *cp2, *cp;
262 	VacFS *fs;
263 	VacFile *vff;
264 	int fd;
265 	Dir *dir;
266 	VacDir vd;
267 
268 	if(getwd(cwd, sizeof(cwd)) == 0)
269 		sysfatal("can't find current directory: %r\n");
270 
271 	dsink = dirSinkAlloc(z, bsize, bsize);
272 
273 	fs = nil;
274 	if(dfile != nil) {
275 		fs = vfsOpen(z, dfile, 1, 10000);
276 		if(fs == nil)
277 			fprint(2, "could not open diff: %s: %s\n", dfile, vtGetError());
278 	}
279 		
280 
281 	if(oname != nil) {
282 		fd = create(oname, OWRITE, 0666);
283 		if(fd < 0)
284 			sysfatal("could not create file: %s: %r", oname);
285 	} else 
286 		fd = 1;
287 
288 	dir = dirfstat(fd);
289 	if(dir == nil)
290 		sysfatal("dirfstat failed: %r");
291 
292 	for(; *argv; argv++) {
293 		cp2 = *argv;
294 		cd = 0;
295 		for (cp = *argv; *cp; cp++)
296 			if (*cp == '/')
297 				cp2 = cp;
298 		if (cp2 != *argv) {
299 			*cp2 = '\0';
300 			chdir(*argv);
301 			*cp2 = '/';
302 			cp2++;
303 			cd = 1;
304 		}
305 		vff = nil;
306 		if(fs)
307 			vff = vfOpen(fs, cp2);
308 		vacFile(dsink, argv[0], cp2, vff);
309 		if(vff)
310 			vfDecRef(vff);
311 		if(cd && chdir(cwd) < 0)
312 			sysfatal("can't cd back to %s: %r\n", cwd);
313 	}
314 	
315 	if(isi) {
316 		vff = nil;
317 		if(fs)
318 			vff = vfOpen(fs, isi);
319 		vacStdin(dsink, isi, vff);
320 		if(vff)
321 			vfDecRef(vff);
322 	}
323 
324 	dirSinkClose(dsink);
325 
326 	/* build meta information for the root */
327 	ms = metaSinkAlloc(z, bsize, bsize);
328 	/* fake into a directory */
329 	dir->mode |= (dir->mode&0444)>>2;
330 	dir->qid.type |= QTDIR;
331 	dir->mode |= DMDIR;
332 	plan9ToVacDir(&vd, dir, 0, fileid++);
333 	if(strcmp(vd.elem, "/") == 0){
334 		vtMemFree(vd.elem);
335 		vd.elem = vtStrDup("root");
336 	}
337 	metaSinkWriteDir(ms, &vd);
338 	vdCleanup(&vd);
339 	metaSinkClose(ms);
340 	
341 	ds = dirSinkAlloc(z, bsize, bsize);
342 	dirSinkWriteSink(ds, dsink->sink);
343 	dirSinkWriteSink(ds, dsink->msink->sink);
344 	dirSinkWriteSink(ds, ms->sink);
345 	dirSinkClose(ds);
346 
347 	memset(&root, 0, sizeof(root));		
348 	root.version = VtRootVersion;
349 	strncpy(root.name, dir->name, sizeof(root.name));
350 	root.name[sizeof(root.name)-1] = 0;
351 	free(dir);
352 	sprint(root.type, "vac");
353 	memmove(root.score, ds->sink->dir.score, VtScoreSize);
354 	root.blockSize = maxbsize;
355 	if(fs != nil)
356 		vfsGetScore(fs, root.prev);
357 
358 	metaSinkFree(ms);
359 	dirSinkFree(ds);
360 	dirSinkFree(dsink);
361 	if(fs != nil)
362 		vfsClose(fs);
363 	
364 	vtRootPack(&root, buf);
365 	if(!vacWrite(z, score, VtRootType, buf, VtRootSize))
366 		vtFatal("vacWrite failed: %s", vtGetError());
367 
368 	fprint(fd, "vac:");
369 	for(i=0; i<VtScoreSize; i++)
370 		fprint(fd, "%.2x", score[i]);
371 	fprint(fd, "\n");
372 	
373 	/* avoid remove at cleanup */
374 	oname = nil;
375 	return 1;
376 }
377 
378 static int
379 isExcluded(char *name)
380 {
381 	int bot, top, i, x;
382 
383 	bot = 0;	
384 	top = nexclude;
385 	while(bot < top) {
386 		i = (bot+top)>>1;
387 		x = strcmp(exclude[i], name);
388 		if(x == 0)
389 			return 1;
390 		if(x < 0)
391 			bot = i + 1;
392 		else /* x > 0 */
393 			top = i;
394 	}
395 	return 0;
396 }
397 
398 static void
399 vacFile(DirSink *dsink, char *lname, char *sname, VacFile *vf)
400 {
401 	int fd;
402 	Dir *dir;
403 	VacDir vd;
404 	ulong entry;
405 
406 	if(isExcluded(lname)) {
407 		warn("excluding: %s", lname);
408 		return;
409 	}
410 
411 	if(merge && vacMerge(dsink, lname, sname))
412 		return;
413 
414 	fd = open(sname, OREAD);
415 	if(fd < 0) {
416 		warn("could not open file: %s: %s", lname, vtOSError());
417 		return;
418 	}
419 
420 	if(verbose)
421 		fprint(2, "%s\n", lname);
422 
423 	dir = dirfstat(fd);
424 	if(dir == nil) {
425 		warn("can't stat %s: %r", lname);
426 		close(fd);
427 		return;
428 	}
429 
430 	entry = dsink->nentry;
431 
432 	if(dir->mode & DMDIR) 
433 		vacDir(dsink, fd, lname, sname, vf);
434 	else
435 		vacData(dsink, fd, lname, vf, dir);
436 
437 	plan9ToVacDir(&vd, dir, entry, fileid++);
438 	metaSinkWriteDir(dsink->msink, &vd);
439 	vdCleanup(&vd);
440 
441 	free(dir);
442 	close(fd);
443 }
444 
445 static void
446 vacStdin(DirSink *dsink, char *name, VacFile *vf)
447 {
448 	Dir *dir;
449 	VacDir vd;
450 	ulong entry;
451 
452 	if(verbose)
453 		fprint(2, "%s\n", "<stdio>");
454 
455 	dir = dirfstat(0);
456 	if(dir == nil) {
457 		warn("can't stat <stdio>: %r");
458 		return;
459 	}
460 
461 	entry = dsink->nentry;
462 
463 	vacData(dsink, 0, "<stdin>", vf, dir);
464 
465 	plan9ToVacDir(&vd, dir, entry, fileid++);
466 	vd.elem = vtStrDup(name);
467 	metaSinkWriteDir(dsink->msink, &vd);
468 	vdCleanup(&vd);
469 
470 	free(dir);
471 }
472 
473 static ulong
474 vacDataSkip(Sink *sink, VacFile *vf, int fd, ulong blocks, uchar *buf, char *lname)
475 {
476 	int n;
477 	ulong i;
478 	uchar score[VtScoreSize];
479 
480 	/* skip blocks for append only files */
481 	if(seek(fd, (blocks-1)*bsize, 0) != (blocks-1)*bsize) {
482 		warn("error seeking: %s", lname);
483 		goto Err;
484 	}
485 	n = readBlock(fd, buf, bsize);
486 	if(n < bsize) {
487 		warn("error checking append only file: %s", lname);
488 		goto Err;
489 	}
490 	if(!vfGetBlockScore(vf, blocks-1, score) || !vtSha1Check(score, buf, n)) {
491 		warn("last block of append file did not match: %s", lname);
492 		goto Err;
493 	}
494 
495 	for(i=0; i<blocks; i++) {
496 		if(!vfGetBlockScore(vf, i, score)) {
497 			warn("could not get score: %s: %lud", lname, i);
498 			seek(fd, i*bsize, 0);
499 			return i;
500 		}
501 		stats.skip++;
502 		sinkWriteScore(sink, score, bsize);
503 	}
504 
505 	return i;
506 Err:
507 	seek(fd, 0, 0);
508 	return 0;
509 }
510 
511 static void
512 vacData(DirSink *dsink, int fd, char *lname, VacFile *vf, Dir *dir)
513 {
514 	uchar *buf;
515 	Sink *sink;
516 	int n;
517 	uchar score[VtScoreSize];
518 	ulong block, same;
519 	VacDir vd;
520 	ulong vfblocks;
521 
522 	vfblocks = 0;
523 	if(vf != nil && qdiff) {
524 		vfGetDir(vf, &vd);
525 		if(vd.mtime == dir->mtime)
526 		if(vd.size == dir->length)
527 		if(!vd.plan9 || /* vd.p9path == dir->qid.path && */ vd.p9version == dir->qid.vers)
528 		if(dirSinkWriteFile(dsink, vf)) {
529 			stats.sfile++;
530 			vdCleanup(&vd);
531 			return;
532 		}
533 
534 		/* look for an append only file */
535 		if((dir->mode&DMAPPEND) != 0)
536 		if(vd.size < dir->length)
537 		if(vd.plan9)
538 		if(vd.p9path == dir->qid.path)
539 			vfblocks = vd.size/bsize;
540 
541 		vdCleanup(&vd);
542 	}
543 	stats.file++;
544 
545 	buf = vtMemAlloc(bsize);
546 	sink = sinkAlloc(dsink->sink->z, bsize, bsize);
547 	block = 0;
548 	same = stats.sdata+stats.skip;
549 
550 	if(vfblocks > 1)
551 		block += vacDataSkip(sink, vf, fd, vfblocks, buf, lname);
552 
553 if(0) fprint(2, "vacData: %s: %ld\n", lname, block);
554 	for(;;) {
555 		n = readBlock(fd, buf, bsize);
556 		if(0 && n < 0)
557 			warn("file truncated due to read error: %s: %s", lname, vtOSError());
558 		if(n <= 0)
559 			break;
560 		if(vf != nil && vfGetBlockScore(vf, block, score) && vtSha1Check(score, buf, n)) {
561 			stats.sdata++;
562 			sinkWriteScore(sink, score, n);
563 		} else
564 			sinkWrite(sink, buf, n);
565 		block++;
566 	}
567 	same = stats.sdata+stats.skip - same;
568 
569 	if(same && (dir->mode&DMAPPEND) != 0)
570 		if(0)fprint(2, "%s: total %lud same %lud:%lud diff %lud\n",
571 			lname, block, same, vfblocks, block-same);
572 
573 	sinkClose(sink);
574 	dirSinkWriteSink(dsink, sink);
575 	sinkFree(sink);
576 	free(buf);
577 }
578 
579 
580 static void
581 vacDir(DirSink *dsink, int fd, char *lname, char *sname, VacFile *vf)
582 {
583 	Dir *dirs;
584 	char *ln, *sn;
585 	int i, nd;
586 	DirSink *ds;
587 	VacFile *vvf;
588 	char *name;
589 
590 	ds = dirSinkAlloc(dsink->sink->z, bsize, bsize);
591 	while((nd = dirread(fd, &dirs)) > 0){
592 		for(i = 0; i < nd; i++){
593 			name = dirs[i].name;
594 			/* check for bad file names */
595 			if(name[0] == 0 || strcmp(name, ".") == 0 || strcmp(name, "..") == 0)
596 				continue;
597 			ln = vtMemAlloc(strlen(lname) + strlen(name) + 2);
598 			sn = vtMemAlloc(strlen(sname) + strlen(name) + 2);
599 			sprint(ln, "%s/%s", lname, name);
600 			sprint(sn, "%s/%s", sname, name);
601 			if(vf != nil)
602 				vvf = vfWalk(vf, name);
603 			else
604 				vvf = nil;
605 			vacFile(ds, ln, sn, vvf);
606 			if(vvf != nil)
607 				vfDecRef(vvf);
608 			vtMemFree(ln);
609 			vtMemFree(sn);
610 		}
611 		free(dirs);
612 	}
613 	dirSinkClose(ds);
614 	dirSinkWriteSink(dsink, ds->sink);
615 	dirSinkWriteSink(dsink, ds->msink->sink);
616 	dirSinkFree(ds);
617 }
618 
619 static int
620 vacMergeFile(DirSink *dsink, VacFile *vf, VacDir *dir, uvlong offset, uvlong *max)
621 {
622 	uchar buf[VtEntrySize];
623 	VtEntry dd, md;
624 	int e;
625 
626 	if(vfRead(vf, buf, VtEntrySize, (uvlong)dir->entry*VtEntrySize) != VtEntrySize) {
627 		warn("could not read venti dir entry: %s\n", dir->elem);
628 		return 0;
629 	}
630 	vtEntryUnpack(&dd, buf, 0);
631 
632 	if(dir->mode & ModeDir)	{
633 		e = dir->mentry;
634 		if(e == 0)
635 			e = dir->entry + 1;
636 		
637 		if(vfRead(vf, buf, VtEntrySize, e*VtEntrySize) != VtEntrySize) {
638 			warn("could not read venti dir entry: %s\n", dir->elem);
639 			return 0;
640 		}
641 		vtEntryUnpack(&md, buf, 0);
642 	}
643 
644 	/* max might incorrect in some old dumps */
645 	if(dir->qid >= *max) {
646 		warn("qid out of range: %s", dir->elem);
647 		*max = dir->qid;
648 	}
649 
650 	dir->qid += offset;
651 	dir->entry = dsink->nentry;
652 
653 	if(dir->qidSpace) {
654 		dir->qidOffset += offset;
655 	} else {
656 		dir->qidSpace = 1;
657 		dir->qidOffset = offset;
658 		dir->qidMax = *max;
659 	}
660 
661 	dirSinkWrite(dsink, &dd);
662 	if(dir->mode & ModeDir)	
663 		dirSinkWrite(dsink, &md);
664 	metaSinkWriteDir(dsink->msink, dir);
665 	
666 	return 1;
667 }
668 
669 static int
670 vacMerge(DirSink *dsink, char *lname, char *sname)
671 {
672 	char *p;
673 	VacFS *fs;
674 	VacFile *vf;
675 	VacDirEnum *d;
676 	VacDir dir;
677 	uvlong max;
678 
679 	p = strrchr(sname, '.');
680 	if(p == 0 || strcmp(p, ".vac"))
681 		return 0;
682 
683 	d = nil;
684 	fs = vfsOpen(dsink->sink->z, sname, 1, 100);
685 	if(fs == nil)
686 		return 0;
687 
688 	vf = vfOpen(fs, "/");
689 	if(vf == nil)
690 		goto Done;
691 	max = vfGetId(vf);
692 	d = vdeOpen(fs, "/");
693 	if(d == nil)
694 		goto Done;
695 
696 	if(verbose)
697 		fprint(2, "merging: %s\n", lname);
698 
699 	if(maxbsize < vfsGetBlockSize(fs))
700 		maxbsize = vfsGetBlockSize(fs);
701 
702 	for(;;) {
703 		if(vdeRead(d, &dir, 1) < 1)
704 			break;
705 		vacMergeFile(dsink, vf, &dir, fileid, &max);
706 		vdCleanup(&dir);	
707 	}
708 	fileid += max;
709 
710 Done:
711 	if(d != nil)
712 		vdeFree(d);
713 	if(vf != nil)
714 		vfDecRef(vf);
715 	vfsClose(fs);
716 	return 1;
717 }
718 
719 Sink *
720 sinkAlloc(VtSession *z, int psize, int dsize)
721 {
722 	Sink *k;
723 	int i;
724 
725 	if(psize < 512 || psize > VtMaxLumpSize)
726 		vtFatal("sinkAlloc: bad psize");
727 	if(dsize < 512 || dsize > VtMaxLumpSize)
728 		vtFatal("sinkAlloc: bad psize");
729 
730 	psize = VtScoreSize*(psize/VtScoreSize);
731 
732 	k = vtMemAllocZ(sizeof(Sink));
733 	k->z = z;
734 	k->dir.flags = VtEntryActive;
735 	k->dir.psize = psize;
736 	k->dir.dsize = dsize;
737 	k->buf = vtMemAllocZ(VtPointerDepth*k->dir.psize + VtScoreSize);
738 	for(i=0; i<=VtPointerDepth; i++)
739 		k->pbuf[i] = k->buf + i*k->dir.psize;
740 	return k;
741 }
742 
743 void
744 sinkWriteScore(Sink *k, uchar score[VtScoreSize], int n)
745 {
746 	int i;
747 	uchar *p;
748 	VtEntry *d;
749 
750 	memmove(k->pbuf[0], score, VtScoreSize);
751 
752 	d = &k->dir;
753 
754 	for(i=0; i<VtPointerDepth; i++) {
755 		k->pbuf[i] += VtScoreSize;
756 		if(k->pbuf[i] < k->buf + d->psize*(i+1))
757 			break;
758 		if(i == VtPointerDepth-1)
759 			vtFatal("file too big");
760 		p = k->buf+i*d->psize;
761 		stats.meta++;
762 		if(!vacWrite(k->z, k->pbuf[i+1], VtPointerType0+i, p, d->psize))
763 			vtFatal("vacWrite failed: %s", vtGetError());
764 		k->pbuf[i] = p;
765 	}
766 
767 	/* round size up to multiple of dsize */
768 	d->size = d->dsize * ((d->size + d->dsize-1)/d->dsize);
769 	
770 	d->size += n;
771 }
772 
773 void
774 sinkWrite(Sink *k, uchar *p, int n)
775 {
776 	int type;
777 	uchar score[VtScoreSize];
778 
779 	if(n > k->dir.dsize)
780 		vtFatal("sinkWrite: size too big");
781 
782 	if(k->dir.flags & VtEntryDir) {
783 		type = VtDirType;
784 		stats.meta++;
785 	} else {
786 		type = VtDataType;
787 		stats.data++;
788 	}
789 	if(!vacWrite(k->z, score, type, p, n))
790 		vtFatal("vacWrite failed: %s", vtGetError());
791 
792 	sinkWriteScore(k, score, n);
793 }
794 
795 static int
796 sizeToDepth(uvlong s, int psize, int dsize)
797 {
798 	int np;
799 	int d;
800 	
801 	/* determine pointer depth */
802 	np = psize/VtScoreSize;
803 	s = (s + dsize - 1)/dsize;
804 	for(d = 0; s > 1; d++)
805 		s = (s + np - 1)/np;
806 	return d;
807 }
808 
809 void
810 sinkClose(Sink *k)
811 {
812 	int i, n;
813 	uchar *p;
814 	VtEntry *kd;
815 
816 	kd = &k->dir;
817 
818 	/* empty */
819 	if(kd->size == 0) {
820 		memmove(kd->score, vtZeroScore, VtScoreSize);
821 		return;
822 	}
823 
824 	for(n=VtPointerDepth-1; n>0; n--)
825 		if(k->pbuf[n] > k->buf + kd->psize*n)
826 			break;
827 
828 	kd->depth = sizeToDepth(kd->size, kd->psize, kd->dsize);
829 
830 	/* skip full part of tree */
831 	for(i=0; i<n && k->pbuf[i] == k->buf + kd->psize*i; i++)
832 		;
833 
834 	/* is the tree completely full */
835 	if(i == n && k->pbuf[n] == k->buf + kd->psize*n + VtScoreSize) {
836 		memmove(kd->score, k->pbuf[n] - VtScoreSize, VtScoreSize);
837 		return;
838 	}
839 	n++;
840 
841 	/* clean up the edge */
842 	for(; i<n; i++) {
843 		p = k->buf+i*kd->psize;
844 		stats.meta++;
845 		if(!vacWrite(k->z, k->pbuf[i+1], VtPointerType0+i, p, k->pbuf[i]-p))
846 			vtFatal("vacWrite failed: %s", vtGetError());
847 		k->pbuf[i+1] += VtScoreSize;
848 	}
849 	memmove(kd->score, k->pbuf[i] - VtScoreSize, VtScoreSize);
850 }
851 
852 void
853 sinkFree(Sink *k)
854 {
855 	vtMemFree(k->buf);
856 	vtMemFree(k);
857 }
858 
859 DirSink *
860 dirSinkAlloc(VtSession *z, int psize, int dsize)
861 {
862 	DirSink *k;
863 	int ds;
864 
865 	ds = VtEntrySize*(dsize/VtEntrySize);
866 
867 	k = vtMemAllocZ(sizeof(DirSink));
868 	k->sink = sinkAlloc(z, psize, ds);
869 	k->sink->dir.flags |= VtEntryDir;
870 	k->msink = metaSinkAlloc(z, psize, dsize);
871 	k->buf = vtMemAlloc(ds);
872 	k->p = k->buf;
873 	k->ep = k->buf + ds;
874 	return k;
875 }
876 
877 void
878 dirSinkWrite(DirSink *k, VtEntry *dir)
879 {
880 	if(k->p + VtEntrySize > k->ep) {
881 		sinkWrite(k->sink, k->buf, k->p - k->buf);
882 		k->p = k->buf;
883 	}
884 	vtEntryPack(dir, k->p, 0);
885 	k->nentry++;
886 	k->p += VtEntrySize;
887 }
888 
889 void
890 dirSinkWriteSink(DirSink *k, Sink *sink)
891 {
892 	dirSinkWrite(k, &sink->dir);
893 }
894 
895 int
896 dirSinkWriteFile(DirSink *k, VacFile *vf)
897 {
898 	VtEntry dir;
899 
900 	if(!vfGetVtEntry(vf, &dir))
901 		return 0;
902 	dirSinkWrite(k, &dir);
903 	return 1;
904 }
905 
906 void
907 dirSinkClose(DirSink *k)
908 {
909 	metaSinkClose(k->msink);
910 	if(k->p != k->buf)
911 		sinkWrite(k->sink, k->buf, k->p - k->buf);
912 	sinkClose(k->sink);
913 }
914 
915 void
916 dirSinkFree(DirSink *k)
917 {
918 	sinkFree(k->sink);
919 	metaSinkFree(k->msink);
920 	vtMemFree(k->buf);
921 	vtMemFree(k);
922 }
923 
924 MetaSink *
925 metaSinkAlloc(VtSession *z, int psize, int dsize)
926 {
927 	MetaSink *k;
928 
929 	k = vtMemAllocZ(sizeof(MetaSink));
930 	k->sink = sinkAlloc(z, psize, dsize);
931 	k->buf = vtMemAlloc(dsize);
932 	k->maxindex = dsize/100;	/* 100 byte entries seems reasonable */
933 	if(k->maxindex < 1)
934 		k->maxindex = 1;
935 	k->rp = k->p = k->buf + MetaHeaderSize + k->maxindex*MetaIndexSize;
936 	k->ep = k->buf + dsize;
937 	return k;
938 }
939 
940 /* hack to get base to compare routine - not reentrant */
941 uchar *blockBase;
942 
943 int
944 dirCmp(void *p0, void *p1)
945 {
946 	uchar *q0, *q1;
947 	int n0, n1, r;
948 
949 	/* name is first element of entry */
950 	q0 = p0;
951 	q0 = blockBase + (q0[0]<<8) + q0[1];
952 	n0 = (q0[6]<<8) + q0[7];
953 	q0 += 8;
954 
955 	q1 = p1;
956 	q1 = blockBase + (q1[0]<<8) + q1[1];
957 	n1 = (q1[6]<<8) + q1[7];
958 	q1 += 8;
959 
960 	if(n0 == n1)
961 		return memcmp(q0, q1, n0);
962 	else if (n0 < n1) {
963 		r = memcmp(q0, q1, n0);
964 		return (r==0)?1:r;
965 	} else  {
966 		r = memcmp(q0, q1, n1);
967 		return (r==0)?-1:r;
968 	}
969 }
970 
971 void
972 metaSinkFlush(MetaSink *k)
973 {
974 	uchar *p;
975 	int n;
976 	MetaBlock mb;
977 
978 	if(k->nindex == 0)
979 		return;
980 	assert(k->nindex <= k->maxindex);
981 
982 	p = k->buf;
983 	n = k->rp - p;
984 
985 	mb.size = n;
986 	mb.free = 0;
987 	mb.nindex = k->nindex;
988 	mb.maxindex = k->maxindex;
989 	mb.buf = p;
990 	mbPack(&mb);
991 	
992 	p += MetaHeaderSize;
993 
994 	/* XXX this is not reentrant! */
995 	blockBase = k->buf;
996 	qsort(p, k->nindex, MetaIndexSize, dirCmp);
997 	p += k->nindex*MetaIndexSize;
998 	
999 	memset(p, 0, (k->maxindex-k->nindex)*MetaIndexSize);
1000 	p += (k->maxindex-k->nindex)*MetaIndexSize;
1001 
1002 	sinkWrite(k->sink, k->buf, n);
1003 
1004 	/* move down partial entry */
1005 	n = k->p - k->rp;
1006 	memmove(p, k->rp, n);
1007 	k->rp = p;
1008 	k->p = p + n;
1009 	k->nindex = 0;
1010 }
1011 
1012 void
1013 metaSinkPutc(MetaSink *k, int c)
1014 {
1015 	if(k->p+1 > k->ep)
1016 		metaSinkFlush(k);
1017 	if(k->p+1 > k->ep)
1018 		vtFatal("directory entry too large");
1019 	k->p[0] = c;
1020 	k->p++;
1021 }
1022 
1023 void
1024 metaSinkPutString(MetaSink *k, char *s)
1025 {
1026 	int n = strlen(s);
1027 	metaSinkPutc(k, n>>8);
1028 	metaSinkPutc(k, n);
1029 	metaSinkWrite(k, (uchar*)s, n);
1030 }
1031 
1032 void
1033 metaSinkPutUint32(MetaSink *k, ulong x)
1034 {
1035 	metaSinkPutc(k, x>>24);
1036 	metaSinkPutc(k, x>>16);
1037 	metaSinkPutc(k, x>>8);
1038 	metaSinkPutc(k, x);
1039 }
1040 
1041 void
1042 metaSinkPutUint64(MetaSink *k, uvlong x)
1043 {
1044 	metaSinkPutUint32(k, x>>32);
1045 	metaSinkPutUint32(k, x);
1046 }
1047 
1048 void
1049 metaSinkWrite(MetaSink *k, uchar *data, int n)
1050 {
1051 	if(k->p + n > k->ep)
1052 		metaSinkFlush(k);
1053 	if(k->p + n > k->ep)
1054 		vtFatal("directory entry too large");
1055 	
1056 	memmove(k->p, data, n);
1057 	k->p += n;
1058 }
1059 
1060 void
1061 metaSinkWriteDir(MetaSink *ms, VacDir *dir)
1062 {
1063 	metaSinkPutUint32(ms, DirMagic);
1064 	metaSinkPutc(ms, Version>>8);
1065 	metaSinkPutc(ms, Version);		
1066 	metaSinkPutString(ms, dir->elem);
1067 	metaSinkPutUint32(ms, dir->entry);
1068 	metaSinkPutUint64(ms, dir->qid);
1069 	metaSinkPutString(ms, dir->uid);
1070 	metaSinkPutString(ms, dir->gid);
1071 	metaSinkPutString(ms, dir->mid);
1072 	metaSinkPutUint32(ms, dir->mtime);
1073 	metaSinkPutUint32(ms, dir->mcount);
1074 	metaSinkPutUint32(ms, dir->ctime);
1075 	metaSinkPutUint32(ms, dir->atime);
1076 	metaSinkPutUint32(ms, dir->mode);
1077 
1078 	if(dir->plan9) {
1079 		metaSinkPutc(ms, DirPlan9Entry);	/* plan9 extra info */
1080 		metaSinkPutc(ms, 0);			/* plan9 extra size */
1081 		metaSinkPutc(ms, 12);			/* plan9 extra size */
1082 		metaSinkPutUint64(ms, dir->p9path);
1083 		metaSinkPutUint32(ms, dir->p9version);
1084 	}
1085 
1086 	if(dir->qidSpace != 0) {
1087 		metaSinkPutc(ms, DirQidSpaceEntry);
1088 		metaSinkPutc(ms, 0);
1089 		metaSinkPutc(ms, 16);
1090 		metaSinkPutUint64(ms, dir->qidOffset);
1091 		metaSinkPutUint64(ms, dir->qidMax);
1092 	}
1093 
1094 	if(dir->gen != 0) {
1095 		metaSinkPutc(ms, DirGenEntry);
1096 		metaSinkPutc(ms, 0);
1097 		metaSinkPutc(ms, 4);
1098 		metaSinkPutUint32(ms, dir->gen);
1099 	}
1100 
1101 	metaSinkEOR(ms);
1102 }
1103 
1104 
1105 void
1106 plan9ToVacDir(VacDir *vd, Dir *dir, ulong entry, uvlong qid)
1107 {
1108 	memset(vd, 0, sizeof(VacDir));
1109 
1110 	vd->elem = vtStrDup(dir->name);
1111 	vd->entry = entry;
1112 	vd->qid = qid;
1113 	vd->uid = vtStrDup(dir->uid);
1114 	vd->gid = vtStrDup(dir->gid);
1115 	vd->mid = vtStrDup(dir->muid);
1116 	vd->mtime = dir->mtime;
1117 	vd->mcount = 0;
1118 	vd->ctime = dir->mtime;		/* ctime: not available on plan 9 */
1119 	vd->atime = dir->atime;
1120 
1121 	vd->mode = dir->mode & 0777;
1122 	if(dir->mode & DMDIR)
1123 		vd->mode |= ModeDir;
1124 	if(dir->mode & DMAPPEND)
1125 		vd->mode |= ModeAppend;
1126 	if(dir->mode & DMEXCL)
1127 		vd->mode |= ModeExclusive;
1128 
1129 	vd->plan9 = 1;
1130 	vd->p9path = dir->qid.path;
1131 	vd->p9version = dir->qid.vers;
1132 }
1133 
1134 
1135 void
1136 metaSinkEOR(MetaSink *k)
1137 {
1138 	uchar *p;
1139 	int o, n;
1140 
1141 	p = k->buf + MetaHeaderSize;
1142 	p += k->nindex * MetaIndexSize;
1143 	o = k->rp-k->buf; 	/* offset from start of block */
1144 	n = k->p-k->rp;		/* size of entry */
1145 	p[0] = o >> 8;
1146 	p[1] = o;
1147 	p[2] = n >> 8;
1148 	p[3] = n;
1149 	k->rp = k->p;
1150 	k->nindex++;
1151 	if(k->nindex == k->maxindex)
1152 		metaSinkFlush(k);
1153 }
1154 
1155 void
1156 metaSinkClose(MetaSink *k)
1157 {
1158 	metaSinkFlush(k);
1159 	sinkClose(k->sink);
1160 }
1161 
1162 void
1163 metaSinkFree(MetaSink *k)
1164 {
1165 	sinkFree(k->sink);
1166 	vtMemFree(k->buf);
1167 	vtMemFree(k);
1168 }
1169 
1170 static void
1171 warn(char *fmt, ...)
1172 {
1173 	va_list arg;
1174 
1175 	va_start(arg, fmt);
1176 	fprint(2, "%s: ", argv0);
1177 	vfprint(2, fmt, arg);
1178 	fprint(2, "\n");
1179 	va_end(arg);
1180 }
1181 
1182 static void
1183 cleanup(void)
1184 {
1185 	if(oname != nil)
1186 		remove(oname);
1187 }
1188 
1189 #define TWID64	((u64int)~(u64int)0)
1190 
1191 static u64int
1192 unittoull(char *s)
1193 {
1194 	char *es;
1195 	u64int n;
1196 
1197 	if(s == nil)
1198 		return TWID64;
1199 	n = strtoul(s, &es, 0);
1200 	if(*es == 'k' || *es == 'K'){
1201 		n *= 1024;
1202 		es++;
1203 	}else if(*es == 'm' || *es == 'M'){
1204 		n *= 1024*1024;
1205 		es++;
1206 	}else if(*es == 'g' || *es == 'G'){
1207 		n *= 1024*1024*1024;
1208 		es++;
1209 	}
1210 	if(*es != '\0')
1211 		return TWID64;
1212 	return n;
1213 }