1 7a4ee46d 2003-11-23 devnull #include "stdinc.h"
2 7a4ee46d 2003-11-23 devnull #include "dat.h"
3 7a4ee46d 2003-11-23 devnull #include "fns.h"
5 7a4ee46d 2003-11-23 devnull static int
6 7a4ee46d 2003-11-23 devnull writebucket(Index *ix, u32int buck, IBucket *ib, ZBlock *b)
8 7a4ee46d 2003-11-23 devnull ISect *is;
10 7a4ee46d 2003-11-23 devnull is = findisect(ix, buck);
11 7a4ee46d 2003-11-23 devnull if(is == nil){
12 7a4ee46d 2003-11-23 devnull seterr(EAdmin, "bad math in writebucket");
13 7a4ee46d 2003-11-23 devnull return -1;
15 7a4ee46d 2003-11-23 devnull if(buck < is->start || buck >= is->stop)
16 7a4ee46d 2003-11-23 devnull seterr(EAdmin, "index write out of bounds: %d not in [%d,%d)\n",
17 7a4ee46d 2003-11-23 devnull buck, is->start, is->stop);
18 7a4ee46d 2003-11-23 devnull buck -= is->start;
19 7a4ee46d 2003-11-23 devnull qlock(&stats.lock);
20 7a4ee46d 2003-11-23 devnull stats.indexwrites++;
21 7a4ee46d 2003-11-23 devnull qunlock(&stats.lock);
22 7a4ee46d 2003-11-23 devnull packibucket(ib, b->data);
23 7a4ee46d 2003-11-23 devnull return writepart(is->part, is->blockbase + ((u64int)buck << is->blocklog), b->data, is->blocksize);
26 7a4ee46d 2003-11-23 devnull static int
27 7a4ee46d 2003-11-23 devnull buildindex(Index *ix, Part *part, u64int off, u64int clumps, int zero)
29 7a4ee46d 2003-11-23 devnull IEStream *ies;
30 7a4ee46d 2003-11-23 devnull IBucket ib, zib;
31 7a4ee46d 2003-11-23 devnull ZBlock *z, *b;
32 7a4ee46d 2003-11-23 devnull u32int next, buck;
34 7a4ee46d 2003-11-23 devnull u64int found = 0;
36 7a4ee46d 2003-11-23 devnull //ZZZ make buffer size configurable
37 7a4ee46d 2003-11-23 devnull b = alloczblock(ix->blocksize, 0);
38 7a4ee46d 2003-11-23 devnull z = alloczblock(ix->blocksize, 1);
39 7a4ee46d 2003-11-23 devnull ies = initiestream(part, off, clumps, 64*1024);
40 7a4ee46d 2003-11-23 devnull if(b == nil || z == nil || ies == nil){
42 7a4ee46d 2003-11-23 devnull goto breakout;
43 7a4ee46d 2003-11-23 devnull return -1;
46 7a4ee46d 2003-11-23 devnull next = 0;
47 7a4ee46d 2003-11-23 devnull ib.data = b->data + IBucketSize;
48 7a4ee46d 2003-11-23 devnull zib.data = z->data + IBucketSize;
49 7a4ee46d 2003-11-23 devnull zib.n = 0;
50 7a4ee46d 2003-11-23 devnull zib.next = 0;
52 7a4ee46d 2003-11-23 devnull buck = buildbucket(ix, ies, &ib);
53 7a4ee46d 2003-11-23 devnull found += ib.n;
54 7a4ee46d 2003-11-23 devnull if(zero){
55 7a4ee46d 2003-11-23 devnull for(; next != buck; next++){
56 7a4ee46d 2003-11-23 devnull if(next == ix->buckets){
57 7a4ee46d 2003-11-23 devnull if(buck != TWID32){
58 7a4ee46d 2003-11-23 devnull fprint(2, "bucket out of range\n");
61 7a4ee46d 2003-11-23 devnull goto breakout;
63 7a4ee46d 2003-11-23 devnull if(writebucket(ix, next, &zib, z) < 0){
64 7a4ee46d 2003-11-23 devnull fprint(2, "can't write zero bucket to buck=%d: %r", next);
69 7a4ee46d 2003-11-23 devnull if(buck >= ix->buckets){
70 7a4ee46d 2003-11-23 devnull if(buck == TWID32)
72 7a4ee46d 2003-11-23 devnull fprint(2, "bucket out of range\n");
74 7a4ee46d 2003-11-23 devnull goto breakout;
76 7a4ee46d 2003-11-23 devnull if(writebucket(ix, buck, &ib, b) < 0){
77 7a4ee46d 2003-11-23 devnull fprint(2, "bad bucket found=%lld: %r\n", found);
80 7a4ee46d 2003-11-23 devnull next = buck + 1;
82 7a4ee46d 2003-11-23 devnull breakout:;
83 7a4ee46d 2003-11-23 devnull fprint(2, "constructed index with %lld entries\n", found);
84 7a4ee46d 2003-11-23 devnull freeiestream(ies);
85 7a4ee46d 2003-11-23 devnull freezblock(z);
86 7a4ee46d 2003-11-23 devnull freezblock(b);
87 7a4ee46d 2003-11-23 devnull return ok;
91 7a4ee46d 2003-11-23 devnull usage(void)
93 7a4ee46d 2003-11-23 devnull fprint(2, "usage: buildindex [-Z] [-B blockcachesize] config tmppart\n");
94 7a4ee46d 2003-11-23 devnull threadexitsall(0);
98 7a4ee46d 2003-11-23 devnull threadmain(int argc, char *argv[])
100 7a4ee46d 2003-11-23 devnull Part *part;
101 7a4ee46d 2003-11-23 devnull u64int clumps, base;
102 7a4ee46d 2003-11-23 devnull u32int bcmem;
103 7a4ee46d 2003-11-23 devnull int zero;
105 7a4ee46d 2003-11-23 devnull zero = 1;
106 7a4ee46d 2003-11-23 devnull bcmem = 0;
107 7a4ee46d 2003-11-23 devnull ARGBEGIN{
108 7a4ee46d 2003-11-23 devnull case 'B':
109 7a4ee46d 2003-11-23 devnull bcmem = unittoull(ARGF());
111 7a4ee46d 2003-11-23 devnull case 'Z':
112 7a4ee46d 2003-11-23 devnull zero = 0;
114 7a4ee46d 2003-11-23 devnull default:
115 7a4ee46d 2003-11-23 devnull usage();
119 7a4ee46d 2003-11-23 devnull if(argc != 2)
120 7a4ee46d 2003-11-23 devnull usage();
122 7a4ee46d 2003-11-23 devnull if(initventi(argv[0]) < 0)
123 7a4ee46d 2003-11-23 devnull sysfatal("can't init venti: %r");
125 7a4ee46d 2003-11-23 devnull if(bcmem < maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16))
126 7a4ee46d 2003-11-23 devnull bcmem = maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16);
127 7a4ee46d 2003-11-23 devnull fprint(2, "initialize %d bytes of disk block cache\n", bcmem);
128 7a4ee46d 2003-11-23 devnull initdcache(bcmem);
130 7a4ee46d 2003-11-23 devnull fprint(2, "building a new index %s using %s for temporary storage\n", mainindex->name, argv[1]);
132 7a4ee46d 2003-11-23 devnull part = initpart(argv[1], 1);
133 7a4ee46d 2003-11-23 devnull if(part == nil)
134 7a4ee46d 2003-11-23 devnull sysfatal("can't initialize temporary partition: %r");
136 7a4ee46d 2003-11-23 devnull clumps = sortrawientries(mainindex, part, &base);
137 7a4ee46d 2003-11-23 devnull if(clumps == TWID64)
138 7a4ee46d 2003-11-23 devnull sysfatal("can't build sorted index: %r");
139 7a4ee46d 2003-11-23 devnull fprint(2, "found and sorted index entries for clumps=%lld at %lld\n", clumps, base);
141 7a4ee46d 2003-11-23 devnull if(buildindex(mainindex, part, base, clumps, zero) < 0)
142 7a4ee46d 2003-11-23 devnull sysfatal("can't build new index: %r");
144 7a4ee46d 2003-11-23 devnull threadexitsall(0);
