1 #include "stdinc.h"
2 #include "dat.h"
3 #include "fns.h"
4 
5 static int
6 writebucket(Index *ix, u32int buck, IBucket *ib, ZBlock *b)
7 {
8 	ISect *is;
9 
10 	is = findibucket(ix, buck, &buck);
11 	if(is == nil)
12 		return -1;
13 	qlock(&stats.lock);
14 	stats.indexwrites++;
15 	qunlock(&stats.lock);
16 	packibucket(ib, b->data);
17 	return writepart(is->part, is->blockbase + ((u64int)buck << is->blocklog), b->data, is->blocksize);
18 }
19 
20 static int
21 buildindex(Index *ix, Part *part, u64int off, u64int clumps, int zero)
22 {
23 	IEStream *ies;
24 	IBucket ib, zib;
25 	ZBlock *z, *b;
26 	u32int next, buck;
27 	int ok;
28 	u64int found = 0;
29 
30 //ZZZ make buffer size configurable
31 	b = alloczblock(ix->blocksize, 0);
32 	z = alloczblock(ix->blocksize, 1);
33 	ies = initiestream(part, off, clumps, 64*1024);
34 	if(b == nil || z == nil || ies == nil){
35 		ok = 0;
36 		goto breakout;
37 		return -1;
38 	}
39 	ok = 0;
40 	next = 0;
41 	ib.data = b->data + IBucketSize;
42 	zib.data = z->data + IBucketSize;
43 	zib.n = 0;
44 	zib.depth = 0;
45 	for(;;){
46 		buck = buildbucket(ix, ies, &ib);
47 		found += ib.n;
48 		if(zero){
49 			for(; next != buck; next++){
50 				if(next == ix->buckets){
51 					if(buck != TWID32){
52 						fprint(2, "bucket out of range\n");
53 						ok = -1;
54 					}
55 					goto breakout;
56 				}
57 				if(writebucket(ix, next, &zib, z) < 0){
58 					fprint(2, "can't write zero bucket to buck=%d: %r", next);
59 					ok = -1;
60 				}
61 			}
62 		}
63 		if(buck >= ix->buckets){
64 			if(buck == TWID32)
65 				break;
66 			fprint(2, "bucket out of range\n");
67 			ok = -1;
68 			goto breakout;
69 		}
70 		if(writebucket(ix, buck, &ib, b) < 0){
71 			fprint(2, "bad bucket found=%lld: %r\n", found);
72 			ok = -1;
73 		}
74 		next = buck + 1;
75 	}
76 breakout:;
77 	fprint(2, "constructed index with %lld entries\n", found);
78 	freeiestream(ies);
79 	freezblock(z);
80 	freezblock(b);
81 	return ok;
82 }
83 
84 void
85 usage(void)
86 {
87 	fprint(2, "usage: buildindex [-Z] [-B blockcachesize] config tmppart\n");
88 	threadexitsall(0);
89 }
90 
91 void
92 threadmain(int argc, char *argv[])
93 {
94 	Part *part;
95 	u64int clumps, base;
96 	u32int bcmem;
97 	int zero;
98 
99 	zero = 1;
100 	bcmem = 0;
101 	ARGBEGIN{
102 	case 'B':
103 		bcmem = unittoull(ARGF());
104 		break;
105 	case 'Z':
106 		zero = 0;
107 		break;
108 	default:
109 		usage();
110 		break;
111 	}ARGEND
112 
113 	if(argc != 2)
114 		usage();
115 
116 	if(initventi(argv[0]) < 0)
117 		sysfatal("can't init venti: %r");
118 
119 	if(bcmem < maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16))
120 		bcmem = maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16);
121 	fprint(2, "initialize %d bytes of disk block cache\n", bcmem);
122 	initdcache(bcmem);
123 
124 	fprint(2, "building a new index %s using %s for temporary storage\n", mainindex->name, argv[1]);
125 
126 	part = initpart(argv[1], 1);
127 	if(part == nil)
128 		sysfatal("can't initialize temporary partition: %r");
129 
130 	clumps = sortrawientries(mainindex, part, &base);
131 	if(clumps == TWID64)
132 		sysfatal("can't build sorted index: %r");
133 	fprint(2, "found and sorted index entries for clumps=%lld at %lld\n", clumps, base);
134 
135 	if(buildindex(mainindex, part, base, clumps, zero) < 0)
136 		sysfatal("can't build new index: %r");
137 	
138 	threadexitsall(0);
139 }