1 /*
2  * Rebuild the Venti index from scratch.
3  */
4 
5 #include "stdinc.h"
6 #include "dat.h"
7 #include "fns.h"
8 
9 /*
10  * Write a single bucket.  Could profit from a big buffer here
11  * so that we can absorb sporadic runs of blocks into one write,
12  * avoiding disk seeks.
13  */
14 static int
15 writebucket(Index *ix, u32int buck, IBucket *ib, ZBlock *b)
16 {
17 	ISect *is;
18 
19 	is = ix->sects[indexsect0(ix, buck)];
20 	if(buck < is->start || buck >= is->stop){
21 		seterr(EAdmin, "cannot find index section for bucket %lud\n", (ulong)buck);
22 		return -1;
23 	}
24 	buck -= is->start;
25 
26 /*
27 	qlock(&stats.lock);
28 	stats.indexwrites++;
29 	qunlock(&stats.lock);
30 */
31 	packibucket(ib, b->data, is->bucketmagic);
32 	return writepart(is->part, is->blockbase + ((u64int)buck << is->blocklog), b->data, is->blocksize);
33 }
34 
35 static int
36 buildindex(Index *ix, Part *part, u64int off, u64int clumps, int zero)
37 {
38 	IEStream *ies;
39 	IBucket ib, zib;
40 	ZBlock *z, *b;
41 	u32int next, buck;
42 	int ok;
43 	uint nbuck;
44 	u64int found = 0;
45 
46 //ZZZ make buffer size configurable
47 	b = alloczblock(ix->blocksize, 0, ix->blocksize);
48 	z = alloczblock(ix->blocksize, 1, ix->blocksize);
49 	ies = initiestream(part, off, clumps, 64*1024);
50 	if(b == nil || z == nil || ies == nil){
51 		ok = 0;
52 		goto breakout;
53 		return -1;
54 	}
55 	ok = 0;
56 	next = 0;
57 	memset(&ib, 0, sizeof ib);
58 	ib.data = b->data + IBucketSize;
59 	zib.data = z->data + IBucketSize;
60 	zib.n = 0;
61 	nbuck = 0;
62 	for(;;){
63 		buck = buildbucket(ix, ies, &ib, ix->blocksize-IBucketSize);
64 		found += ib.n;
65 		if(zero){
66 			for(; next != buck; next++){
67 				if(next == ix->buckets){
68 					if(buck != TWID32){
69 						fprint(2, "bucket out of range\n");
70 						ok = -1;
71 					}
72 					goto breakout;
73 				}
74 				if(writebucket(ix, next, &zib, z) < 0){
75 					fprint(2, "can't write zero bucket to buck=%d: %r", next);
76 					ok = -1;
77 				}
78 			}
79 		}
80 		if(buck >= ix->buckets){
81 			if(buck == TWID32)
82 				break;
83 			fprint(2, "bucket out of range\n");
84 			ok = -1;
85 			goto breakout;
86 		}
87 		if(writebucket(ix, buck, &ib, b) < 0){
88 			fprint(2, "bad bucket found=%lld: %r\n", found);
89 			ok = -1;
90 		}
91 		next = buck + 1;
92 		if(++nbuck%10000 == 0)
93 			fprint(2, "\t%,d buckets written...\n", nbuck);
94 	}
95 breakout:;
96 	fprint(2, "wrote index with %lld entries\n", found);
97 	freeiestream(ies);
98 	freezblock(z);
99 	freezblock(b);
100 	return ok;
101 }
102 
103 void
104 usage(void)
105 {
106 	fprint(2, "usage: buildindex [-Z] [-B blockcachesize] config tmppart\n");
107 	threadexitsall(0);
108 }
109 
110 Config conf;
111 
112 void
113 threadmain(int argc, char *argv[])
114 {
115 	Part *part;
116 	u64int clumps, base;
117 	u32int bcmem;
118 	int zero;
119 
120 	zero = 1;
121 	bcmem = 0;
122 	ventifmtinstall();
123 	ARGBEGIN{
124 	case 'B':
125 		bcmem = unittoull(ARGF());
126 		break;
127 	case 'Z':
128 		zero = 0;
129 		break;
130 	default:
131 		usage();
132 		break;
133 	}ARGEND
134 
135 	if(argc != 2)
136 		usage();
137 
138 	if(initventi(argv[0], &conf) < 0)
139 		sysfatal("can't init venti: %r");
140 
141 	if(bcmem < maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16))
142 		bcmem = maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16);
143 	if(0) fprint(2, "initialize %d bytes of disk block cache\n", bcmem);
144 	initdcache(bcmem);
145 
146 	fprint(2, "building a new index %s using %s for temporary storage\n", mainindex->name, argv[1]);
147 
148 	part = initpart(argv[1], ORDWR|ODIRECT);
149 	if(part == nil)
150 		sysfatal("can't initialize temporary partition: %r");
151 
152 	clumps = sortrawientries(mainindex, part, &base, mainindex->bloom);
153 	if(clumps == TWID64)
154 		sysfatal("can't build sorted index: %r");
155 	fprint(2, "found and sorted index entries for clumps=%lld at %lld\n", clumps, base);
156 
157 	if(buildindex(mainindex, part, base, clumps, zero) < 0)
158 		sysfatal("can't build new index: %r");
159 	
160 	if(mainindex->bloom)
161 		writebloom(mainindex->bloom);
162 
163 	threadexitsall(0);
164 }