op public repos

Blob

Date:: Sat Apr 1 19:24:03 2006 UTC
Message:: Use gcc -ansi -pedantic in 9c. Fix many non-C89-isms.
Actions:: History | Blame | Raw File
1 #include <u.h>
2 #include <sys/stat.h>
3 #include "stdinc.h"
4 #include "vac.h"
5 #include "dat.h"
6 #include "fns.h"
7 
8 int mainstacksize = 128*1024;
9 
10 typedef struct Sink Sink;
11 typedef struct MetaSink MetaSink;
12 typedef struct DirSink DirSink;
13 
14 struct Sink {
15 	VtConn *z;
16 	VtEntry dir;
17 	uchar *buf;
18 	uchar *pbuf[VtPointerDepth+1];
19 };
20 
21 struct DirSink {
22 	Sink *sink;
23 	MetaSink *msink;
24 	ulong nentry;
25 	uchar *buf;
26 	uchar *p;	/* current pointer */
27 	uchar *ep;	/* end pointer */
28 };
29 
30 struct MetaSink {
31 	Sink *sink;
32 	uchar *buf;
33 	int maxindex;
34 	int nindex;
35 	uchar *rp;	/* start of current record */
36 	uchar *p;	/* current pointer */
37 	uchar *ep;	/* end pointer */
38 };
39 
40 static void usage(void);
41 static int strpcmp(const void*, const void*);
42 static void warn(char *fmt, ...);
43 static void cleanup(void);
44 static u64int unittoull(char *s);
45 static void vac(VtConn *z, char *argv[]);
46 static void vacfile(DirSink *dsink, char *lname, char *sname, VacFile*);
47 static void vacstdin(DirSink *dsink, char *name, VacFile *vf);
48 static void vacdata(DirSink *dsink, int fd, char *lname, VacFile*, Dir*);
49 static void vacdir(DirSink *dsink, int fd, char *lname, char *sname, VacFile*);
50 static int vacmerge(DirSink *dsink, char *lname, char *sname);
51 static int vacspecial(DirSink *dsink, Dir *dir, char *lname, char *sname, VacFile *vf);
52 Sink *sinkalloc(VtConn *z, int psize, int dsize);
53 void sinkwrite(Sink *k, uchar *data, int n);
54 void sinkwritescore(Sink *k, uchar *score, int n);
55 void sinkclose(Sink *k);
56 void sinkfree(Sink *k);
57 
58 DirSink *dirsinkalloc(VtConn *z, int psize, int dsize);
59 void dirsinkwrite(DirSink *k, VtEntry*);
60 void dirsinkwritesink(DirSink *k, Sink*);
61 int dirsinkwritefile(DirSink *k, VacFile *vf);
62 void dirsinkclose(DirSink *k);
63 void dirsinkfree(DirSink *k);
64 
65 MetaSink *metasinkalloc(VtConn *z, int psize, int dsize);
66 void metasinkputc(MetaSink *k, int c);
67 void metasinkputstring(MetaSink *k, char *s);
68 void metasinkputuint32(MetaSink *k, ulong x);
69 void metasinkputuint64(MetaSink *k, uvlong x);
70 void metasinkwrite(MetaSink *k, uchar *data, int n);
71 void metasinkwritedir(MetaSink *ms, VacDir *vd);
72 void metasinkeor(MetaSink *k);
73 void metasinkclose(MetaSink *k);
74 void metasinkfree(MetaSink *k);
75 void plan9tovacdir(VacDir*, Dir*, ulong entry, uvlong qid);
76 
77 enum {
78 	Version = 8,
79 	BlockSize = 8*1024,
80 	MaxExclude = 1000
81 };
82 
83 struct {
84 	ulong	file;
85 	ulong	sfile;
86 	ulong	data;
87 	ulong	sdata;
88 	ulong	skip;
89 	ulong	meta;
90 } stats;
91 
92 int bsize = BlockSize;
93 int maxbsize;
94 char *oname, *dfile;
95 int verbose;
96 uvlong fileid = 1;
97 int qdiff;
98 char *exclude[MaxExclude];
99 int nexclude;
100 int nowrite;
101 int merge;
102 char *isi;
103 
104 static void
105 usage(void)
106 {
107 	fprint(2, "usage: %s [-amqsv] [-h host] [-d vacfile] [-b blocksize] [-i name] [-e exclude] [-f vacfile] file ... \n", argv0);
108 	threadexitsall("usage");
109 }
110 
111 void
112 threadmain(int argc, char *argv[])
113 {
114 	VtConn *z;
115 	char *p;
116 	char *host = nil;
117 	int statsflag = 0;
118 
119 	atexit(cleanup);
120 
121 	ARGBEGIN{
122 	default:
123 		usage();
124 	case 'b':
125 		p = ARGF();
126 		if(p == 0)
127 			usage();
128 		bsize = unittoull(p);
129 		if(bsize == ~0)
130 			usage();
131 		break;
132 	case 'd':
133 		dfile = ARGF();
134 		if(dfile == nil)
135 			usage();
136 		break;
137 	case 'e':
138 		if(nexclude >= MaxExclude)
139 			sysfatal("too many exclusions\n");
140 		exclude[nexclude] = ARGF();
141 		if(exclude[nexclude] == nil)
142 			usage();
143 		nexclude++;
144 		break;
145 	case 'f':
146 		oname = ARGF();
147 		if(oname == 0)
148 			usage();
149 		break;
150 	case 'h':
151 		host = ARGF();
152 		if(host == nil)
153 			usage();
154 		break;
155 	case 'i':
156 		isi = ARGF();
157 		if(isi == nil)
158 			usage();
159 		break;
160 	case 'n':
161 		nowrite++;
162 		break;
163 	case 'm':
164 		merge++;
165 		break;
166 	case 'q':
167 		qdiff++;
168 		break;
169 	case 's':
170 		statsflag++;
171 		break;
172 	case 'v':
173 		verbose++;
174 		break;
175 	}ARGEND;
176 
177 	if(argc == 0)
178 		usage();
179 
180 	if(bsize < 512)
181 		bsize = 512;
182 	if(bsize > VtMaxLumpSize)
183 		bsize = VtMaxLumpSize;
184 	maxbsize = bsize;
185 
186 	fmtinstall('V', vtscorefmt);
187 
188 	z = vtdial(host);
189 	if(z == nil)
190 		sysfatal("could not connect to server: %r");
191 
192 	if(vtconnect(z) < 0)
193 		sysfatal("vtconnect: %r");
194 
195 	qsort(exclude, nexclude, sizeof(char*), strpcmp);
196 
197 	vac(z, argv);
198 
199 	if(vtsync(z) < 0)
200 		fprint(2, "warning: could not ask server to flush pending writes: %r\n");
201 
202 	if(statsflag)
203 		fprint(2, "files %ld:%ld data %ld:%ld:%ld meta %ld\n", stats.file, stats.sfile,
204 			stats.data, stats.skip, stats.sdata, stats.meta);
205 /*packetStats(); */
206 	vthangup(z);
207 
208 	threadexitsall(0);
209 }
210 
211 static int
212 strpcmp(const void *p0, const void *p1)
213 {
214 	return strcmp(*(char**)p0, *(char**)p1);
215 }
216 
217 int
218 vacwrite(VtConn *z, uchar score[VtScoreSize], int type, uchar *buf, int n)
219 {
220 	assert(n > 0);
221 	if(nowrite){
222 		sha1(buf, n, score, nil);
223 		return 0;
224 	}
225 	return vtwrite(z, score, type, buf, n);
226 }
227 
228 static char*
229 lastelem(char *oname)
230 {
231 	char *p;
232 
233 	if(oname == nil)
234 		abort();
235 	if((p = strrchr(oname, '/')) == nil)
236 		return oname;
237 	return p+1;
238 }
239 
240 static void
241 vac(VtConn *z, char *argv[])
242 {
243 	DirSink *dsink, *ds;
244 	MetaSink *ms;
245 	VtRoot root;
246 	uchar score[VtScoreSize], buf[VtRootSize];
247 	char cwd[2048];
248 	int cd;
249 	char *cp2, *cp;
250 	VacFs *fs;
251 	VacFile *vff;
252 	int fd;
253 	Dir *dir;
254 	VacDir vd;
255 
256 	if(getwd(cwd, sizeof(cwd)) == 0)
257 		sysfatal("can't find current directory: %r\n");
258 
259 	dsink = dirsinkalloc(z, bsize, bsize);
260 
261 	fs = nil;
262 	if(dfile != nil) {
263 		fs = vacfsopen(z, dfile, VtOREAD, 1000);
264 		if(fs == nil)
265 			fprint(2, "could not open diff: %s: %r\n", dfile);
266 	}
267 		
268 
269 	if(oname != nil) {
270 		fd = create(oname, OWRITE, 0666);
271 		if(fd < 0)
272 			sysfatal("could not create file: %s: %r", oname);
273 	} else 
274 		fd = 1;
275 
276 	dir = dirfstat(fd);
277 	if(dir == nil)
278 		sysfatal("dirfstat failed: %r");
279 	if(oname)
280 		dir->name = lastelem(oname);
281 	else
282 		dir->name = "stdin";
283 
284 	for(; *argv; argv++) {
285 		cp2 = *argv;
286 		cd = 0;
287 		for (cp = *argv; *cp; cp++)
288 			if (*cp == '/')
289 				cp2 = cp;
290 		if (cp2 != *argv) {
291 			*cp2 = '\0';
292 			chdir(*argv);
293 			*cp2 = '/';
294 			cp2++;
295 			cd = 1;
296 		}
297 		vff = nil;
298 		if(fs)
299 			vff = vacfileopen(fs, cp2);
300 		vacfile(dsink, argv[0], cp2, vff);
301 		if(vff)
302 			vacfiledecref(vff);
303 		if(cd && chdir(cwd) < 0)
304 			sysfatal("can't cd back to %s: %r\n", cwd);
305 	}
306 	
307 	if(isi) {
308 		vff = nil;
309 		if(fs)
310 			vff = vacfileopen(fs, isi);
311 		vacstdin(dsink, isi, vff);
312 		if(vff)
313 			vacfiledecref(vff);
314 	}
315 
316 	dirsinkclose(dsink);
317 
318 	/* build meta information for the root */
319 	ms = metasinkalloc(z, bsize, bsize);
320 	/* fake into a directory */
321 	dir->mode = DMDIR|0555;
322 	dir->qid.type |= QTDIR;
323 	plan9tovacdir(&vd, dir, 0, fileid++);
324 	if(strcmp(vd.elem, "/") == 0){
325 		vtfree(vd.elem);
326 		vd.elem = vtstrdup("root");
327 	}
328 	metasinkwritedir(ms, &vd);
329 	vdcleanup(&vd);
330 	metasinkclose(ms);
331 	
332 	ds = dirsinkalloc(z, bsize, bsize);
333 	dirsinkwritesink(ds, dsink->sink);
334 	dirsinkwritesink(ds, dsink->msink->sink);
335 	dirsinkwritesink(ds, ms->sink);
336 	dirsinkclose(ds);
337 
338 	memset(&root, 0, sizeof(root));		
339 	strncpy(root.name, dir->name, sizeof(root.name));
340 	root.name[sizeof(root.name)-1] = 0;
341 	free(dir);
342 	sprint(root.type, "vac");
343 	memmove(root.score, ds->sink->dir.score, VtScoreSize);
344 	root.blocksize = maxbsize;
345 	if(fs != nil)
346 		vacfsgetscore(fs, root.prev);
347 
348 	metasinkfree(ms);
349 	dirsinkfree(ds);
350 	dirsinkfree(dsink);
351 	if(fs != nil)
352 		vacfsclose(fs);
353 	
354 	vtrootpack(&root, buf);
355 	if(vacwrite(z, score, VtRootType, buf, VtRootSize) < 0)
356 		sysfatal("vacWrite failed: %r");
357 
358 	fprint(fd, "vac:%V\n", score);
359 
360 	/* avoid remove at cleanup */
361 	oname = nil;
362 }
363 
364 static int
365 isexcluded(char *name)
366 {
367 	int bot, top, i, x;
368 
369 	bot = 0;	
370 	top = nexclude;
371 	while(bot < top) {
372 		i = (bot+top)>>1;
373 		x = strcmp(exclude[i], name);
374 		if(x == 0)
375 			return 1;
376 		if(x < 0)
377 			bot = i + 1;
378 		else /* x > 0 */
379 			top = i;
380 	}
381 	return 0;
382 }
383 
384 static void
385 vacfile(DirSink *dsink, char *lname, char *sname, VacFile *vf)
386 {
387 	int fd;
388 	Dir *dir;
389 	VacDir vd;
390 	ulong entry;
391 
392 	if(isexcluded(lname)) {
393 		warn("excluding: %s", lname);
394 		return;
395 	}
396 
397 	if(merge && vacmerge(dsink, lname, sname) >= 0)
398 		return;
399 
400 	if((dir = dirstat(sname)) == nil){
401 		warn("could not stat file %s: %r", lname);
402 		return;
403 	}
404 	if(dir->mode&(DMSYMLINK|DMDEVICE|DMNAMEDPIPE)){
405 		vacspecial(dsink, dir, lname, sname, vf);
406 		free(dir);
407 		return;
408 	}else if(dir->mode&DMSOCKET){
409 		free(dir);
410 		return;
411 	}
412 	free(dir);
413 	
414 	fd = open(sname, OREAD);
415 	if(fd < 0) {
416 		warn("could not open file: %s: %r", lname);
417 		return;
418 	}
419 
420 	if(verbose)
421 		fprint(2, "%s\n", lname);
422 
423 	dir = dirfstat(fd);
424 	if(dir == nil) {
425 		warn("can't stat %s: %r", lname);
426 		close(fd);
427 		return;
428 	}
429 	dir->name = lastelem(sname);
430 
431 	entry = dsink->nentry;
432 
433 	if(dir->mode & DMDIR) 
434 		vacdir(dsink, fd, lname, sname, vf);
435 	else
436 		vacdata(dsink, fd, lname, vf, dir);
437 
438 	plan9tovacdir(&vd, dir, entry, fileid++);
439 	metasinkwritedir(dsink->msink, &vd);
440 	vdcleanup(&vd);
441 
442 	free(dir);
443 	close(fd);
444 }
445 
446 static void
447 vacstdin(DirSink *dsink, char *name, VacFile *vf)
448 {
449 	Dir *dir;
450 	VacDir vd;
451 	ulong entry;
452 
453 	if(verbose)
454 		fprint(2, "%s\n", "<stdio>");
455 
456 	dir = dirfstat(0);
457 	if(dir == nil) {
458 		warn("can't stat <stdio>: %r");
459 		return;
460 	}
461 	dir->name = "stdin";
462 
463 	entry = dsink->nentry;
464 
465 	vacdata(dsink, 0, "<stdin>", vf, dir);
466 
467 	plan9tovacdir(&vd, dir, entry, fileid++);
468 	vd.elem = vtstrdup(name);
469 	metasinkwritedir(dsink->msink, &vd);
470 	vdcleanup(&vd);
471 
472 	free(dir);
473 }
474 
475 static int
476 sha1check(u8int *score, uchar *buf, int n)
477 {
478 	uchar score2[VtScoreSize];
479 
480 	sha1(buf, n, score2, nil);
481 	if(memcmp(score, score2, VtScoreSize) == 0)
482 		return 0;
483 	return -1;
484 }
485 
486 static ulong
487 vacdataskip(Sink *sink, VacFile *vf, int fd, ulong blocks, uchar *buf, char *lname)
488 {
489 	int n;
490 	ulong i;
491 	uchar score[VtScoreSize];
492 
493 	/* skip blocks for append only files */
494 	if(seek(fd, (blocks-1)*bsize, 0) != (blocks-1)*bsize) {
495 		warn("error seeking: %s", lname);
496 		goto Err;
497 	}
498 	n = readn(fd, buf, bsize);
499 	if(n < bsize) {
500 		warn("error checking append only file: %s", lname);
501 		goto Err;
502 	}
503 	if(vacfileblockscore(vf, blocks-1, score)<0 || sha1check(score, buf, n)<0) {
504 		warn("last block of append file did not match: %s", lname);
505 		goto Err;
506 	}
507 
508 	for(i=0; i<blocks; i++) {
509 		if(vacfileblockscore(vf, i, score) < 0) {
510 			warn("could not get score: %s: %lud", lname, i);
511 			seek(fd, i*bsize, 0);
512 			return i;
513 		}
514 		stats.skip++;
515 		sinkwritescore(sink, score, bsize);
516 	}
517 
518 	return i;
519 Err:
520 	seek(fd, 0, 0);
521 	return 0;
522 }
523 
524 static void
525 vacdata(DirSink *dsink, int fd, char *lname, VacFile *vf, Dir *dir)
526 {
527 	uchar *buf;
528 	Sink *sink;
529 	int n;
530 	uchar score[VtScoreSize];
531 	ulong block, same;
532 	VacDir vd;
533 	ulong vfblocks;
534 
535 	vfblocks = 0;
536 	if(vf != nil && qdiff) {
537 		vacfilegetdir(vf, &vd);
538 		if(vd.mtime == dir->mtime)
539 		if(vd.size == dir->length)
540 		if(!vd.plan9 || /* vd.p9path == dir->qid.path && */ vd.p9version == dir->qid.vers)
541 		if(dirsinkwritefile(dsink, vf)) {
542 			stats.sfile++;
543 			vdcleanup(&vd);
544 			return;
545 		}
546 
547 		/* look for an append only file */
548 		if((dir->mode&DMAPPEND) != 0)
549 		if(vd.size < dir->length)
550 		if(vd.plan9)
551 		if(vd.p9path == dir->qid.path)
552 			vfblocks = vd.size/bsize;
553 
554 		vdcleanup(&vd);
555 	}
556 	stats.file++;
557 
558 	buf = vtmalloc(bsize);
559 	sink = sinkalloc(dsink->sink->z, bsize, bsize);
560 	block = 0;
561 	same = stats.sdata+stats.skip;
562 
563 	if(vfblocks > 1)
564 		block += vacdataskip(sink, vf, fd, vfblocks, buf, lname);
565 
566 if(0) fprint(2, "vacData: %s: %ld\n", lname, block);
567 	for(;;) {
568 		n = readn(fd, buf, bsize);
569 		if(0 && n < 0)
570 			warn("file truncated due to read error: %s: %r", lname);
571 		if(n <= 0)
572 			break;
573 		if(vf != nil && vacfileblockscore(vf, block, score)>=0 && sha1check(score, buf, n)>=0) {
574 			stats.sdata++;
575 			sinkwritescore(sink, score, n);
576 		} else
577 			sinkwrite(sink, buf, n);
578 		block++;
579 	}
580 	same = stats.sdata+stats.skip - same;
581 
582 	if(same && (dir->mode&DMAPPEND) != 0)
583 		if(0)fprint(2, "%s: total %lud same %lud:%lud diff %lud\n",
584 			lname, block, same, vfblocks, block-same);
585 
586 	sinkclose(sink);
587 	dirsinkwritesink(dsink, sink);
588 	sinkfree(sink);
589 	free(buf);
590 }
591 
592 
593 static void
594 vacdir(DirSink *dsink, int fd, char *lname, char *sname, VacFile *vf)
595 {
596 	Dir *dirs;
597 	char *ln, *sn;
598 	int i, nd;
599 	DirSink *ds;
600 	VacFile *vvf;
601 	char *name;
602 
603 	ds = dirsinkalloc(dsink->sink->z, bsize, bsize);
604 	while((nd = dirread(fd, &dirs)) > 0){
605 		for(i = 0; i < nd; i++){
606 			name = dirs[i].name;
607 			/* check for bad file names */
608 			if(name[0] == 0 || strcmp(name, ".") == 0 || strcmp(name, "..") == 0)
609 				continue;
610 			ln = vtmalloc(strlen(lname) + strlen(name) + 2);
611 			sn = vtmalloc(strlen(sname) + strlen(name) + 2);
612 			strcpy(ln, lname);
613 			strcat(ln, "/");
614 			strcat(ln, name);
615 			strcpy(sn, sname);
616 			strcat(sn, "/");
617 			strcat(sn, name);
618 			if(vf != nil)
619 				vvf = vacfilewalk(vf, name);
620 			else
621 				vvf = nil;
622 			vacfile(ds, ln, sn, vvf);
623 			if(vvf != nil)
624 				vacfiledecref(vvf);
625 			vtfree(ln);
626 			vtfree(sn);
627 		}
628 		free(dirs);
629 	}
630 	dirsinkclose(ds);
631 	dirsinkwritesink(dsink, ds->sink);
632 	dirsinkwritesink(dsink, ds->msink->sink);
633 	dirsinkfree(ds);
634 }
635 
636 static int
637 vacmergefile(DirSink *dsink, VacFile *vf, VacDir *dir, uvlong offset, uvlong *max)
638 {
639 	uchar buf[VtEntrySize];
640 	VtEntry dd, md;
641 	int e;
642 
643 	if(vacfileread(vf, buf, VtEntrySize, (uvlong)dir->entry*VtEntrySize) != VtEntrySize) {
644 		warn("could not read venti dir entry: %s\n", dir->elem);
645 		return -1;
646 	}
647 	vtentryunpack(&dd, buf, 0);
648 
649 	if(dir->mode & ModeDir)	{
650 		e = dir->mentry;
651 		if(e == 0)
652 			e = dir->entry + 1;
653 		
654 		if(vacfileread(vf, buf, VtEntrySize, e*VtEntrySize) != VtEntrySize) {
655 			warn("could not read venti dir entry: %s\n", dir->elem);
656 			return 0;
657 		}
658 		vtentryunpack(&md, buf, 0);
659 	}
660 
661 	/* max might incorrect in some old dumps */
662 	if(dir->qid >= *max) {
663 		warn("qid out of range: %s", dir->elem);
664 		*max = dir->qid;
665 	}
666 
667 	dir->qid += offset;
668 	dir->entry = dsink->nentry;
669 
670 	if(dir->qidspace) {
671 		dir->qidoffset += offset;
672 	} else {
673 		dir->qidspace = 1;
674 		dir->qidoffset = offset;
675 		dir->qidmax = *max;
676 	}
677 
678 	dirsinkwrite(dsink, &dd);
679 	if(dir->mode & ModeDir)	
680 		dirsinkwrite(dsink, &md);
681 	metasinkwritedir(dsink->msink, dir);
682 	
683 	return 0;
684 }
685 
686 static int
687 vacmerge(DirSink *dsink, char *lname, char *sname)
688 {
689 	char *p;
690 	VacFs *fs;
691 	VacFile *vf;
692 	VacDirEnum *d;
693 	VacDir dir;
694 	uvlong max;
695 
696 	if((p=strrchr(sname, '.')) == nil || strcmp(p, ".vac") != 0)
697 		return -1;
698 
699 	d = nil;
700 	fs = vacfsopen(dsink->sink->z, sname, VtOREAD, 100);
701 	if(fs == nil)
702 		return -1;
703 
704 	vf = vacfileopen(fs, "/");
705 	if(vf == nil)
706 		goto Done;
707 	max = vacfilegetid(vf);
708 	d = vdeopen(vf);
709 	if(d == nil)
710 		goto Done;
711 
712 	if(verbose)
713 		fprint(2, "merging: %s\n", lname);
714 
715 	if(maxbsize < fs->bsize)
716 		maxbsize = fs->bsize;
717 
718 	for(;;) {
719 		if(vderead(d, &dir) < 1)
720 			break;
721 		vacmergefile(dsink, vf, &dir, fileid, &max);
722 		vdcleanup(&dir);	
723 	}
724 	fileid += max;
725 
726 Done:
727 	if(d != nil)
728 		vdeclose(d);
729 	if(vf != nil)
730 		vacfiledecref(vf);
731 	vacfsclose(fs);
732 	return 0;
733 }
734 
735 static int
736 vacspecial(DirSink *dsink, Dir* dir, char *lname, char *sname, VacFile *vf)
737 {
738 	char *btmp, *buf;
739 	int buflen, dtype, major, minor, n;
740 	ulong entry;
741 	Sink *sink;
742 	VacDir vd;
743 
744 	n = 0;
745 	buflen = 128;
746 	buf = malloc(buflen);
747 	if(buf == nil)
748 		return -1;
749 
750 	if(verbose)
751 		fprint(2, "%s\n", lname);
752 
753 	dir->name = lastelem(sname);
754 
755 	if(dir->mode & DMSYMLINK){
756 		while((n = readlink(sname, buf, buflen)) == buflen){
757 			buflen *= 2;
758 			btmp = vtrealloc(buf, buflen);
759 			if(btmp == nil){
760 				free(buf);
761 				return -1;
762 			}
763 			buf = btmp;
764 		}
765 		dir->mode &= ~DMDIR;
766 		dir->mode |= DMSYMLINK;
767 	}else if(dir->mode & DMDEVICE){
768 		dtype = (dir->qid.path >> 16) & 0xFF;
769 		minor = dir->qid.path & 0xff;
770 		major = (dir->qid.path >> 8) & 0xFF;
771 		n = snprint(buf, buflen, "%c %d %d", dtype, major, minor);
772 	}
773 
774 	entry = dsink->nentry;
775 
776 	sink = sinkalloc(dsink->sink->z, bsize, bsize);
777 	sinkwrite(sink, (uchar*)buf, n);
778 	sinkclose(sink);
779 	dirsinkwritesink(dsink, sink);
780 	sinkfree(sink);
781 	free(buf);
782 	
783 	dir->name = lastelem(sname);
784 	dir->length = n;
785 	plan9tovacdir(&vd, dir, entry, fileid++);
786 	metasinkwritedir(dsink->msink, &vd);
787 	vdcleanup(&vd);
788 
789 	return 0;
790 }
791 
792 
793 Sink *
794 sinkalloc(VtConn *z, int psize, int dsize)
795 {
796 	Sink *k;
797 	int i;
798 
799 	if(psize < 512 || psize > VtMaxLumpSize)
800 		sysfatal("sinkalloc: bad psize");
801 	if(dsize < 512 || dsize > VtMaxLumpSize)
802 		sysfatal("sinkalloc: bad psize");
803 
804 	psize = VtScoreSize*(psize/VtScoreSize);
805 
806 	k = vtmallocz(sizeof(Sink));
807 	k->z = z;
808 	k->dir.flags = VtEntryActive;
809 	k->dir.psize = psize;
810 	k->dir.dsize = dsize;
811 	k->buf = vtmallocz(VtPointerDepth*k->dir.psize + VtScoreSize);
812 	for(i=0; i<=VtPointerDepth; i++)
813 		k->pbuf[i] = k->buf + i*k->dir.psize;
814 	return k;
815 }
816 
817 void
818 sinkwritescore(Sink *k, uchar score[VtScoreSize], int n)
819 {
820 	int i;
821 	uchar *p;
822 	VtEntry *d;
823 
824 	memmove(k->pbuf[0], score, VtScoreSize);
825 
826 	d = &k->dir;
827 
828 	for(i=0; i<VtPointerDepth; i++) {
829 		k->pbuf[i] += VtScoreSize;
830 		if(k->pbuf[i] < k->buf + d->psize*(i+1))
831 			break;
832 		if(i == VtPointerDepth-1)
833 			sysfatal("file too big");
834 		p = k->buf+i*d->psize;
835 		stats.meta++;
836 		if(vacwrite(k->z, k->pbuf[i+1], VtDataType+1+i, p, d->psize) < 0)
837 			sysfatal("vacwrite failed: %r");
838 		k->pbuf[i] = p;
839 	}
840 
841 	/* round size up to multiple of dsize */
842 	d->size = d->dsize * ((d->size + d->dsize-1)/d->dsize);
843 	
844 	d->size += n;
845 }
846 
847 void
848 sinkwrite(Sink *k, uchar *p, int n)
849 {
850 	int type;
851 	uchar score[VtScoreSize];
852 
853 	if(n == 0)
854 		return;
855 
856 	if(n > k->dir.dsize)
857 		sysfatal("sinkWrite: size too big");
858 
859 	if((k->dir.type&~VtTypeDepthMask) == VtDirType){
860 		type = VtDirType;
861 		stats.meta++;
862 	} else {
863 		type = VtDataType;
864 		stats.data++;
865 	}
866 	if(vacwrite(k->z, score, type, p, n) < 0)
867 		sysfatal("vacWrite failed: %r");
868 
869 	sinkwritescore(k, score, n);
870 }
871 
872 static int
873 sizetodepth(uvlong s, int psize, int dsize)
874 {
875 	int np;
876 	int d;
877 	
878 	/* determine pointer depth */
879 	np = psize/VtScoreSize;
880 	s = (s + dsize - 1)/dsize;
881 	for(d = 0; s > 1; d++)
882 		s = (s + np - 1)/np;
883 	return d;
884 }
885 
886 void
887 sinkclose(Sink *k)
888 {
889 	int i, n, base;
890 	uchar *p;
891 	VtEntry *kd;
892 
893 	kd = &k->dir;
894 
895 	/* empty */
896 	if(kd->size == 0) {
897 		memmove(kd->score, vtzeroscore, VtScoreSize);
898 		return;
899 	}
900 
901 	for(n=VtPointerDepth-1; n>0; n--)
902 		if(k->pbuf[n] > k->buf + kd->psize*n)
903 			break;
904 
905 	base = kd->type&~VtTypeDepthMask;
906 	kd->type = base + sizetodepth(kd->size, kd->psize, kd->dsize);
907 
908 	/* skip full part of tree */
909 	for(i=0; i<n && k->pbuf[i] == k->buf + kd->psize*i; i++)
910 		;
911 
912 	/* is the tree completely full */
913 	if(i == n && k->pbuf[n] == k->buf + kd->psize*n + VtScoreSize) {
914 		memmove(kd->score, k->pbuf[n] - VtScoreSize, VtScoreSize);
915 		return;
916 	}
917 	n++;
918 
919 	/* clean up the edge */
920 	for(; i<n; i++) {
921 		p = k->buf+i*kd->psize;
922 		stats.meta++;
923 		if(vacwrite(k->z, k->pbuf[i+1], base+1+i, p, k->pbuf[i]-p) < 0)
924 			sysfatal("vacWrite failed: %r");
925 		k->pbuf[i+1] += VtScoreSize;
926 	}
927 	memmove(kd->score, k->pbuf[i] - VtScoreSize, VtScoreSize);
928 }
929 
930 void
931 sinkfree(Sink *k)
932 {
933 	vtfree(k->buf);
934 	vtfree(k);
935 }
936 
937 DirSink *
938 dirsinkalloc(VtConn *z, int psize, int dsize)
939 {
940 	DirSink *k;
941 	int ds;
942 
943 	ds = VtEntrySize*(dsize/VtEntrySize);
944 
945 	k = vtmallocz(sizeof(DirSink));
946 	k->sink = sinkalloc(z, psize, ds);
947 	k->sink->dir.type = VtDirType;
948 	k->msink = metasinkalloc(z, psize, dsize);
949 	k->buf = vtmalloc(ds);
950 	k->p = k->buf;
951 	k->ep = k->buf + ds;
952 	return k;
953 }
954 
955 void
956 dirsinkwrite(DirSink *k, VtEntry *dir)
957 {
958 	if(k->p + VtEntrySize > k->ep) {
959 		sinkwrite(k->sink, k->buf, k->p - k->buf);
960 		k->p = k->buf;
961 	}
962 	vtentrypack(dir, k->p, 0);
963 	k->nentry++;
964 	k->p += VtEntrySize;
965 }
966 
967 void
968 dirsinkwritesink(DirSink *k, Sink *sink)
969 {
970 	dirsinkwrite(k, &sink->dir);
971 }
972 
973 int
974 dirsinkwritefile(DirSink *k, VacFile *vf)
975 {
976 	VtEntry dir;
977 
978 	if(vacfilegetvtentry(vf, &dir) < 0)
979 		return -1;
980 	dirsinkwrite(k, &dir);
981 	return 0;
982 }
983 
984 void
985 dirsinkclose(DirSink *k)
986 {
987 	metasinkclose(k->msink);
988 	if(k->p != k->buf)
989 		sinkwrite(k->sink, k->buf, k->p - k->buf);
990 	sinkclose(k->sink);
991 }
992 
993 void
994 dirsinkfree(DirSink *k)
995 {
996 	sinkfree(k->sink);
997 	metasinkfree(k->msink);
998 	vtfree(k->buf);
999 	vtfree(k);
1000 }
1001 
1002 MetaSink*
1003 metasinkalloc(VtConn *z, int psize, int dsize)
1004 {
1005 	MetaSink *k;
1006 
1007 	k = vtmallocz(sizeof(MetaSink));
1008 	k->sink = sinkalloc(z, psize, dsize);
1009 	k->buf = vtmalloc(dsize);
1010 	k->maxindex = dsize/100;	/* 100 byte entries seems reasonable */
1011 	if(k->maxindex < 1)
1012 		k->maxindex = 1;
1013 	k->rp = k->p = k->buf + MetaHeaderSize + k->maxindex*MetaIndexSize;
1014 	k->ep = k->buf + dsize;
1015 	return k;
1016 }
1017 
1018 /* hack to get base to compare routine - not reentrant */
1019 uchar *blockbase;
1020 
1021 int
1022 dircmp(const void *p0, const void *p1)
1023 {
1024 	uchar *q0, *q1;
1025 	int n0, n1, r;
1026 
1027 	/* name is first element of entry */
1028 	q0 = (uchar*)p0;
1029 	q0 = blockbase + (q0[0]<<8) + q0[1];
1030 	n0 = (q0[6]<<8) + q0[7];
1031 	q0 += 8;
1032 
1033 	q1 = (uchar*)p1;
1034 	q1 = blockbase + (q1[0]<<8) + q1[1];
1035 	n1 = (q1[6]<<8) + q1[7];
1036 	q1 += 8;
1037 
1038 	if(n0 == n1)
1039 		return memcmp(q0, q1, n0);
1040 	else if (n0 < n1) {
1041 		r = memcmp(q0, q1, n0);
1042 		return (r==0)?1:r;
1043 	} else  {
1044 		r = memcmp(q0, q1, n1);
1045 		return (r==0)?-1:r;
1046 	}
1047 }
1048 
1049 void
1050 metasinkflush(MetaSink *k)
1051 {
1052 	uchar *p;
1053 	int n;
1054 	MetaBlock mb;
1055 
1056 	if(k->nindex == 0)
1057 		return;
1058 	assert(k->nindex <= k->maxindex);
1059 
1060 	p = k->buf;
1061 	n = k->rp - p;
1062 
1063 	mb.size = n;
1064 	mb.free = 0;
1065 	mb.nindex = k->nindex;
1066 	mb.maxindex = k->maxindex;
1067 	mb.buf = p;
1068 	mbpack(&mb);
1069 	
1070 	p += MetaHeaderSize;
1071 
1072 	/* XXX this is not reentrant! */
1073 	blockbase = k->buf;
1074 	qsort(p, k->nindex, MetaIndexSize, dircmp);
1075 	p += k->nindex*MetaIndexSize;
1076 	
1077 	memset(p, 0, (k->maxindex-k->nindex)*MetaIndexSize);
1078 	p += (k->maxindex-k->nindex)*MetaIndexSize;
1079 
1080 	sinkwrite(k->sink, k->buf, n);
1081 
1082 	/* move down partial entry */
1083 	n = k->p - k->rp;
1084 	memmove(p, k->rp, n);
1085 	k->rp = p;
1086 	k->p = p + n;
1087 	k->nindex = 0;
1088 }
1089 
1090 void
1091 metasinkputc(MetaSink *k, int c)
1092 {
1093 	if(k->p+1 > k->ep)
1094 		metasinkflush(k);
1095 	if(k->p+1 > k->ep)
1096 		sysfatal("directory entry too large");
1097 	k->p[0] = c;
1098 	k->p++;
1099 }
1100 
1101 void
1102 metasinkputstring(MetaSink *k, char *s)
1103 {
1104 	int n = strlen(s);
1105 	metasinkputc(k, n>>8);
1106 	metasinkputc(k, n);
1107 	metasinkwrite(k, (uchar*)s, n);
1108 }
1109 
1110 void
1111 metasinkputuint32(MetaSink *k, ulong x)
1112 {
1113 	metasinkputc(k, x>>24);
1114 	metasinkputc(k, x>>16);
1115 	metasinkputc(k, x>>8);
1116 	metasinkputc(k, x);
1117 }
1118 
1119 void
1120 metasinkputuint64(MetaSink *k, uvlong x)
1121 {
1122 	metasinkputuint32(k, x>>32);
1123 	metasinkputuint32(k, x);
1124 }
1125 
1126 void
1127 metasinkwrite(MetaSink *k, uchar *data, int n)
1128 {
1129 	if(k->p + n > k->ep)
1130 		metasinkflush(k);
1131 	if(k->p + n > k->ep)
1132 		sysfatal("directory entry too large");
1133 	
1134 	memmove(k->p, data, n);
1135 	k->p += n;
1136 }
1137 
1138 void
1139 metasinkwritedir(MetaSink *ms, VacDir *dir)
1140 {
1141 	metasinkputuint32(ms, DirMagic);
1142 	metasinkputc(ms, Version>>8);
1143 	metasinkputc(ms, Version);		
1144 	metasinkputstring(ms, dir->elem);
1145 	metasinkputuint32(ms, dir->entry);
1146 	metasinkputuint64(ms, dir->qid);
1147 	metasinkputstring(ms, dir->uid);
1148 	metasinkputstring(ms, dir->gid);
1149 	metasinkputstring(ms, dir->mid);
1150 	metasinkputuint32(ms, dir->mtime);
1151 	metasinkputuint32(ms, dir->mcount);
1152 	metasinkputuint32(ms, dir->ctime);
1153 	metasinkputuint32(ms, dir->atime);
1154 	metasinkputuint32(ms, dir->mode);
1155 
1156 	if(dir->plan9) {
1157 		metasinkputc(ms, DirPlan9Entry);	/* plan9 extra info */
1158 		metasinkputc(ms, 0);			/* plan9 extra size */
1159 		metasinkputc(ms, 12);			/* plan9 extra size */
1160 		metasinkputuint64(ms, dir->p9path);
1161 		metasinkputuint32(ms, dir->p9version);
1162 	}
1163 
1164 	if(dir->qidspace != 0) {
1165 		metasinkputc(ms, DirQidSpaceEntry);
1166 		metasinkputc(ms, 0);
1167 		metasinkputc(ms, 16);
1168 		metasinkputuint64(ms, dir->qidoffset);
1169 		metasinkputuint64(ms, dir->qidmax);
1170 	}
1171 
1172 	if(dir->gen != 0) {
1173 		metasinkputc(ms, DirGenEntry);
1174 		metasinkputc(ms, 0);
1175 		metasinkputc(ms, 4);
1176 		metasinkputuint32(ms, dir->gen);
1177 	}
1178 
1179 	metasinkeor(ms);
1180 }
1181 
1182 
1183 void
1184 plan9tovacdir(VacDir *vd, Dir *dir, ulong entry, uvlong qid)
1185 {
1186 	memset(vd, 0, sizeof(VacDir));
1187 
1188 	vd->elem = vtstrdup(dir->name);
1189 	vd->entry = entry;
1190 	vd->qid = qid;
1191 	vd->uid = vtstrdup(dir->uid);
1192 	vd->gid = vtstrdup(dir->gid);
1193 	vd->mid = vtstrdup(dir->muid);
1194 	vd->mtime = dir->mtime;
1195 	vd->mcount = 0;
1196 	vd->ctime = dir->mtime;		/* ctime: not available on plan 9 */
1197 	vd->atime = dir->atime;
1198 
1199 	vd->mode = dir->mode & 0777;
1200 	if(dir->mode & DMDIR)
1201 		vd->mode |= ModeDir;
1202 	if(dir->mode & DMAPPEND)
1203 		vd->mode |= ModeAppend;
1204 	if(dir->mode & DMEXCL)
1205 		vd->mode |= ModeExclusive;
1206 	if(dir->mode & DMDEVICE)
1207 		vd->mode |= ModeDevice;
1208 	if(dir->mode & DMNAMEDPIPE)
1209 		vd->mode |= ModeNamedPipe;
1210 	if(dir->mode & DMSYMLINK)
1211 		vd->mode |= ModeLink;
1212 
1213 	vd->plan9 = 1;
1214 	vd->p9path = dir->qid.path;
1215 	vd->p9version = dir->qid.vers;
1216 }
1217 
1218 
1219 void
1220 metasinkeor(MetaSink *k)
1221 {
1222 	uchar *p;
1223 	int o, n;
1224 
1225 	p = k->buf + MetaHeaderSize;
1226 	p += k->nindex * MetaIndexSize;
1227 	o = k->rp-k->buf; 	/* offset from start of block */
1228 	n = k->p-k->rp;		/* size of entry */
1229 	p[0] = o >> 8;
1230 	p[1] = o;
1231 	p[2] = n >> 8;
1232 	p[3] = n;
1233 	k->rp = k->p;
1234 	k->nindex++;
1235 	if(k->nindex == k->maxindex)
1236 		metasinkflush(k);
1237 }
1238 
1239 void
1240 metasinkclose(MetaSink *k)
1241 {
1242 	metasinkflush(k);
1243 	sinkclose(k->sink);
1244 }
1245 
1246 void
1247 metasinkfree(MetaSink *k)
1248 {
1249 	sinkfree(k->sink);
1250 	vtfree(k->buf);
1251 	vtfree(k);
1252 }
1253 
1254 static void
1255 warn(char *fmt, ...)
1256 {
1257 	va_list arg;
1258 
1259 	va_start(arg, fmt);
1260 	fprint(2, "%s: ", argv0);
1261 	vfprint(2, fmt, arg);
1262 	fprint(2, "\n");
1263 	va_end(arg);
1264 }
1265 
1266 static void
1267 cleanup(void)
1268 {
1269 	if(oname != nil)
1270 		remove(oname);
1271 }
1272 
1273 #define TWID64	((u64int)~(u64int)0)
1274 
1275 static u64int
1276 unittoull(char *s)
1277 {
1278 	char *es;
1279 	u64int n;
1280 
1281 	if(s == nil)
1282 		return TWID64;
1283 	n = strtoul(s, &es, 0);
1284 	if(*es == 'k' || *es == 'K'){
1285 		n *= 1024;
1286 		es++;
1287 	}else if(*es == 'm' || *es == 'M'){
1288 		n *= 1024*1024;
1289 		es++;
1290 	}else if(*es == 'g' || *es == 'G'){
1291 		n *= 1024*1024*1024;
1292 		es++;
1293 	}
1294 	if(*es != '\0')
1295 		return TWID64;
1296 	return n;
1297 }