2 a0d146ed 2005-07-12 devnull * Rebuild the Venti index from scratch.
5 a0d146ed 2005-07-12 devnull #include "stdinc.h"
6 a0d146ed 2005-07-12 devnull #include "dat.h"
7 a0d146ed 2005-07-12 devnull #include "fns.h"
10 a0d146ed 2005-07-12 devnull * Write a single bucket. Could profit from a big buffer here
11 a0d146ed 2005-07-12 devnull * so that we can absorb sporadic runs of blocks into one write,
12 a0d146ed 2005-07-12 devnull * avoiding disk seeks.
14 a0d146ed 2005-07-12 devnull static int
15 a0d146ed 2005-07-12 devnull writebucket(Index *ix, u32int buck, IBucket *ib, ZBlock *b)
17 a0d146ed 2005-07-12 devnull ISect *is;
19 a0d146ed 2005-07-12 devnull is = ix->sects[indexsect0(ix, buck)];
20 a0d146ed 2005-07-12 devnull if(buck < is->start || buck >= is->stop){
21 a0d146ed 2005-07-12 devnull seterr(EAdmin, "cannot find index section for bucket %lud\n", (ulong)buck);
22 a0d146ed 2005-07-12 devnull return -1;
24 a0d146ed 2005-07-12 devnull buck -= is->start;
27 a0d146ed 2005-07-12 devnull qlock(&stats.lock);
28 a0d146ed 2005-07-12 devnull stats.indexwrites++;
29 a0d146ed 2005-07-12 devnull qunlock(&stats.lock);
31 a0d146ed 2005-07-12 devnull packibucket(ib, b->data, is->bucketmagic);
32 a0d146ed 2005-07-12 devnull return writepart(is->part, is->blockbase + ((u64int)buck << is->blocklog), b->data, is->blocksize);
35 a0d146ed 2005-07-12 devnull static int
36 a0d146ed 2005-07-12 devnull buildindex(Index *ix, Part *part, u64int off, u64int clumps, int zero)
38 a0d146ed 2005-07-12 devnull IEStream *ies;
39 a0d146ed 2005-07-12 devnull IBucket ib, zib;
40 a0d146ed 2005-07-12 devnull ZBlock *z, *b;
41 a0d146ed 2005-07-12 devnull u32int next, buck;
43 a0d146ed 2005-07-12 devnull uint nbuck;
44 a0d146ed 2005-07-12 devnull u64int found = 0;
46 a0d146ed 2005-07-12 devnull //ZZZ make buffer size configurable
47 a0d146ed 2005-07-12 devnull b = alloczblock(ix->blocksize, 0, ix->blocksize);
48 a0d146ed 2005-07-12 devnull z = alloczblock(ix->blocksize, 1, ix->blocksize);
49 a0d146ed 2005-07-12 devnull ies = initiestream(part, off, clumps, 64*1024);
50 a0d146ed 2005-07-12 devnull if(b == nil || z == nil || ies == nil){
52 a0d146ed 2005-07-12 devnull goto breakout;
53 a0d146ed 2005-07-12 devnull return -1;
56 a0d146ed 2005-07-12 devnull next = 0;
57 a0d146ed 2005-07-12 devnull memset(&ib, 0, sizeof ib);
58 a0d146ed 2005-07-12 devnull ib.data = b->data + IBucketSize;
59 a0d146ed 2005-07-12 devnull zib.data = z->data + IBucketSize;
60 a0d146ed 2005-07-12 devnull zib.n = 0;
61 a0d146ed 2005-07-12 devnull nbuck = 0;
63 a0d146ed 2005-07-12 devnull buck = buildbucket(ix, ies, &ib, ix->blocksize-IBucketSize);
64 a0d146ed 2005-07-12 devnull found += ib.n;
65 a0d146ed 2005-07-12 devnull if(zero){
66 a0d146ed 2005-07-12 devnull for(; next != buck; next++){
67 a0d146ed 2005-07-12 devnull if(next == ix->buckets){
68 a0d146ed 2005-07-12 devnull if(buck != TWID32){
69 a0d146ed 2005-07-12 devnull fprint(2, "bucket out of range\n");
72 a0d146ed 2005-07-12 devnull goto breakout;
74 a0d146ed 2005-07-12 devnull if(writebucket(ix, next, &zib, z) < 0){
75 a0d146ed 2005-07-12 devnull fprint(2, "can't write zero bucket to buck=%d: %r", next);
80 a0d146ed 2005-07-12 devnull if(buck >= ix->buckets){
81 a0d146ed 2005-07-12 devnull if(buck == TWID32)
83 a0d146ed 2005-07-12 devnull fprint(2, "bucket out of range\n");
85 a0d146ed 2005-07-12 devnull goto breakout;
87 a0d146ed 2005-07-12 devnull if(writebucket(ix, buck, &ib, b) < 0){
88 a0d146ed 2005-07-12 devnull fprint(2, "bad bucket found=%lld: %r\n", found);
91 a0d146ed 2005-07-12 devnull next = buck + 1;
92 a0d146ed 2005-07-12 devnull if(++nbuck%10000 == 0)
93 a0d146ed 2005-07-12 devnull fprint(2, "\t%,d buckets written...\n", nbuck);
95 a0d146ed 2005-07-12 devnull breakout:;
96 a0d146ed 2005-07-12 devnull fprint(2, "wrote index with %lld entries\n", found);
97 a0d146ed 2005-07-12 devnull freeiestream(ies);
98 a0d146ed 2005-07-12 devnull freezblock(z);
99 a0d146ed 2005-07-12 devnull freezblock(b);
100 a0d146ed 2005-07-12 devnull return ok;
104 a0d146ed 2005-07-12 devnull usage(void)
106 a0d146ed 2005-07-12 devnull fprint(2, "usage: buildindex [-Z] [-B blockcachesize] config tmppart\n");
107 a0d146ed 2005-07-12 devnull threadexitsall(0);
110 a0d146ed 2005-07-12 devnull Config conf;
113 a0d146ed 2005-07-12 devnull threadmain(int argc, char *argv[])
115 a0d146ed 2005-07-12 devnull Part *part;
116 a0d146ed 2005-07-12 devnull u64int clumps, base;
117 a0d146ed 2005-07-12 devnull u32int bcmem;
118 a0d146ed 2005-07-12 devnull int zero;
120 a0d146ed 2005-07-12 devnull zero = 1;
121 a0d146ed 2005-07-12 devnull bcmem = 0;
122 23fb2edb 2005-07-24 devnull ventifmtinstall();
123 a0d146ed 2005-07-12 devnull ARGBEGIN{
124 a0d146ed 2005-07-12 devnull case 'B':
125 a0d146ed 2005-07-12 devnull bcmem = unittoull(ARGF());
127 a0d146ed 2005-07-12 devnull case 'Z':
128 a0d146ed 2005-07-12 devnull zero = 0;
130 a0d146ed 2005-07-12 devnull default:
131 a0d146ed 2005-07-12 devnull usage();
135 a0d146ed 2005-07-12 devnull if(argc != 2)
136 a0d146ed 2005-07-12 devnull usage();
138 a0d146ed 2005-07-12 devnull if(initventi(argv[0], &conf) < 0)
139 a0d146ed 2005-07-12 devnull sysfatal("can't init venti: %r");
141 a0d146ed 2005-07-12 devnull if(bcmem < maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16))
142 a0d146ed 2005-07-12 devnull bcmem = maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16);
143 a0d146ed 2005-07-12 devnull if(0) fprint(2, "initialize %d bytes of disk block cache\n", bcmem);
144 a0d146ed 2005-07-12 devnull initdcache(bcmem);
146 a0d146ed 2005-07-12 devnull fprint(2, "building a new index %s using %s for temporary storage\n", mainindex->name, argv[1]);
148 a0d146ed 2005-07-12 devnull part = initpart(argv[1], ORDWR|ODIRECT);
149 a0d146ed 2005-07-12 devnull if(part == nil)
150 a0d146ed 2005-07-12 devnull sysfatal("can't initialize temporary partition: %r");
152 a0d146ed 2005-07-12 devnull clumps = sortrawientries(mainindex, part, &base, mainindex->bloom);
153 a0d146ed 2005-07-12 devnull if(clumps == TWID64)
154 a0d146ed 2005-07-12 devnull sysfatal("can't build sorted index: %r");
155 a0d146ed 2005-07-12 devnull fprint(2, "found and sorted index entries for clumps=%lld at %lld\n", clumps, base);
157 a0d146ed 2005-07-12 devnull if(buildindex(mainindex, part, base, clumps, zero) < 0)
158 a0d146ed 2005-07-12 devnull sysfatal("can't build new index: %r");
160 d4daacde 2005-07-24 devnull if(mainindex->bloom)
161 d4daacde 2005-07-24 devnull writebloom(mainindex->bloom);
163 a0d146ed 2005-07-12 devnull threadexitsall(0);