1 a0d146ed 2005-07-12 devnull /*
  2 a0d146ed 2005-07-12 devnull  * Rebuild the Venti index from scratch.
  3 a0d146ed 2005-07-12 devnull  */
  4 a0d146ed 2005-07-12 devnull 
  5 a0d146ed 2005-07-12 devnull #include "stdinc.h"
  6 a0d146ed 2005-07-12 devnull #include "dat.h"
  7 a0d146ed 2005-07-12 devnull #include "fns.h"
  8 a0d146ed 2005-07-12 devnull 
  9 a0d146ed 2005-07-12 devnull /*
 10 a0d146ed 2005-07-12 devnull  * Write a single bucket.  Could profit from a big buffer here
 11 a0d146ed 2005-07-12 devnull  * so that we can absorb sporadic runs of blocks into one write,
 12 a0d146ed 2005-07-12 devnull  * avoiding disk seeks.
 13 a0d146ed 2005-07-12 devnull  */
 14 a0d146ed 2005-07-12 devnull static int
 15 a0d146ed 2005-07-12 devnull writebucket(Index *ix, u32int buck, IBucket *ib, ZBlock *b)
 16 a0d146ed 2005-07-12 devnull {
 17 a0d146ed 2005-07-12 devnull 	ISect *is;
 18 a0d146ed 2005-07-12 devnull 
 19 a0d146ed 2005-07-12 devnull 	is = ix->sects[indexsect0(ix, buck)];
 20 a0d146ed 2005-07-12 devnull 	if(buck < is->start || buck >= is->stop){
 21 a0d146ed 2005-07-12 devnull 		seterr(EAdmin, "cannot find index section for bucket %lud\n", (ulong)buck);
 22 a0d146ed 2005-07-12 devnull 		return -1;
 23 a0d146ed 2005-07-12 devnull 	}
 24 a0d146ed 2005-07-12 devnull 	buck -= is->start;
 25 a0d146ed 2005-07-12 devnull 
 26 a0d146ed 2005-07-12 devnull /*
 27 a0d146ed 2005-07-12 devnull 	qlock(&stats.lock);
 28 a0d146ed 2005-07-12 devnull 	stats.indexwrites++;
 29 a0d146ed 2005-07-12 devnull 	qunlock(&stats.lock);
 30 a0d146ed 2005-07-12 devnull */
 31 a0d146ed 2005-07-12 devnull 	packibucket(ib, b->data, is->bucketmagic);
 32 a0d146ed 2005-07-12 devnull 	return writepart(is->part, is->blockbase + ((u64int)buck << is->blocklog), b->data, is->blocksize);
 33 a0d146ed 2005-07-12 devnull }
 34 a0d146ed 2005-07-12 devnull 
 35 a0d146ed 2005-07-12 devnull static int
 36 a0d146ed 2005-07-12 devnull buildindex(Index *ix, Part *part, u64int off, u64int clumps, int zero)
 37 a0d146ed 2005-07-12 devnull {
 38 a0d146ed 2005-07-12 devnull 	IEStream *ies;
 39 a0d146ed 2005-07-12 devnull 	IBucket ib, zib;
 40 a0d146ed 2005-07-12 devnull 	ZBlock *z, *b;
 41 a0d146ed 2005-07-12 devnull 	u32int next, buck;
 42 a0d146ed 2005-07-12 devnull 	int ok;
 43 a0d146ed 2005-07-12 devnull 	uint nbuck;
 44 a0d146ed 2005-07-12 devnull 	u64int found = 0;
 45 a0d146ed 2005-07-12 devnull 
 46 a0d146ed 2005-07-12 devnull //ZZZ make buffer size configurable
 47 a0d146ed 2005-07-12 devnull 	b = alloczblock(ix->blocksize, 0, ix->blocksize);
 48 a0d146ed 2005-07-12 devnull 	z = alloczblock(ix->blocksize, 1, ix->blocksize);
 49 a0d146ed 2005-07-12 devnull 	ies = initiestream(part, off, clumps, 64*1024);
 50 a0d146ed 2005-07-12 devnull 	if(b == nil || z == nil || ies == nil){
 51 a0d146ed 2005-07-12 devnull 		ok = 0;
 52 a0d146ed 2005-07-12 devnull 		goto breakout;
 53 a0d146ed 2005-07-12 devnull 		return -1;
 54 a0d146ed 2005-07-12 devnull 	}
 55 a0d146ed 2005-07-12 devnull 	ok = 0;
 56 a0d146ed 2005-07-12 devnull 	next = 0;
 57 a0d146ed 2005-07-12 devnull 	memset(&ib, 0, sizeof ib);
 58 a0d146ed 2005-07-12 devnull 	ib.data = b->data + IBucketSize;
 59 a0d146ed 2005-07-12 devnull 	zib.data = z->data + IBucketSize;
 60 a0d146ed 2005-07-12 devnull 	zib.n = 0;
 61 a0d146ed 2005-07-12 devnull 	nbuck = 0;
 62 a0d146ed 2005-07-12 devnull 	for(;;){
 63 a0d146ed 2005-07-12 devnull 		buck = buildbucket(ix, ies, &ib, ix->blocksize-IBucketSize);
 64 a0d146ed 2005-07-12 devnull 		found += ib.n;
 65 a0d146ed 2005-07-12 devnull 		if(zero){
 66 a0d146ed 2005-07-12 devnull 			for(; next != buck; next++){
 67 a0d146ed 2005-07-12 devnull 				if(next == ix->buckets){
 68 a0d146ed 2005-07-12 devnull 					if(buck != TWID32){
 69 a0d146ed 2005-07-12 devnull 						fprint(2, "bucket out of range\n");
 70 a0d146ed 2005-07-12 devnull 						ok = -1;
 71 a0d146ed 2005-07-12 devnull 					}
 72 a0d146ed 2005-07-12 devnull 					goto breakout;
 73 a0d146ed 2005-07-12 devnull 				}
 74 a0d146ed 2005-07-12 devnull 				if(writebucket(ix, next, &zib, z) < 0){
 75 a0d146ed 2005-07-12 devnull 					fprint(2, "can't write zero bucket to buck=%d: %r", next);
 76 a0d146ed 2005-07-12 devnull 					ok = -1;
 77 a0d146ed 2005-07-12 devnull 				}
 78 a0d146ed 2005-07-12 devnull 			}
 79 a0d146ed 2005-07-12 devnull 		}
 80 a0d146ed 2005-07-12 devnull 		if(buck >= ix->buckets){
 81 a0d146ed 2005-07-12 devnull 			if(buck == TWID32)
 82 a0d146ed 2005-07-12 devnull 				break;
 83 a0d146ed 2005-07-12 devnull 			fprint(2, "bucket out of range\n");
 84 a0d146ed 2005-07-12 devnull 			ok = -1;
 85 a0d146ed 2005-07-12 devnull 			goto breakout;
 86 a0d146ed 2005-07-12 devnull 		}
 87 a0d146ed 2005-07-12 devnull 		if(writebucket(ix, buck, &ib, b) < 0){
 88 a0d146ed 2005-07-12 devnull 			fprint(2, "bad bucket found=%lld: %r\n", found);
 89 a0d146ed 2005-07-12 devnull 			ok = -1;
 90 a0d146ed 2005-07-12 devnull 		}
 91 a0d146ed 2005-07-12 devnull 		next = buck + 1;
 92 a0d146ed 2005-07-12 devnull 		if(++nbuck%10000 == 0)
 93 a0d146ed 2005-07-12 devnull 			fprint(2, "\t%,d buckets written...\n", nbuck);
 94 a0d146ed 2005-07-12 devnull 	}
 95 a0d146ed 2005-07-12 devnull breakout:;
 96 a0d146ed 2005-07-12 devnull 	fprint(2, "wrote index with %lld entries\n", found);
 97 a0d146ed 2005-07-12 devnull 	freeiestream(ies);
 98 a0d146ed 2005-07-12 devnull 	freezblock(z);
 99 a0d146ed 2005-07-12 devnull 	freezblock(b);
100 a0d146ed 2005-07-12 devnull 	return ok;
101 a0d146ed 2005-07-12 devnull }
102 a0d146ed 2005-07-12 devnull 
103 a0d146ed 2005-07-12 devnull void
104 a0d146ed 2005-07-12 devnull usage(void)
105 a0d146ed 2005-07-12 devnull {
106 a0d146ed 2005-07-12 devnull 	fprint(2, "usage: buildindex [-Z] [-B blockcachesize] config tmppart\n");
107 a0d146ed 2005-07-12 devnull 	threadexitsall(0);
108 a0d146ed 2005-07-12 devnull }
109 a0d146ed 2005-07-12 devnull 
110 a0d146ed 2005-07-12 devnull Config conf;
111 a0d146ed 2005-07-12 devnull 
112 a0d146ed 2005-07-12 devnull void
113 a0d146ed 2005-07-12 devnull threadmain(int argc, char *argv[])
114 a0d146ed 2005-07-12 devnull {
115 a0d146ed 2005-07-12 devnull 	Part *part;
116 a0d146ed 2005-07-12 devnull 	u64int clumps, base;
117 a0d146ed 2005-07-12 devnull 	u32int bcmem;
118 a0d146ed 2005-07-12 devnull 	int zero;
119 a0d146ed 2005-07-12 devnull 
120 a0d146ed 2005-07-12 devnull 	zero = 1;
121 a0d146ed 2005-07-12 devnull 	bcmem = 0;
122 23fb2edb 2005-07-24 devnull 	ventifmtinstall();
123 a0d146ed 2005-07-12 devnull 	ARGBEGIN{
124 a0d146ed 2005-07-12 devnull 	case 'B':
125 a0d146ed 2005-07-12 devnull 		bcmem = unittoull(ARGF());
126 a0d146ed 2005-07-12 devnull 		break;
127 a0d146ed 2005-07-12 devnull 	case 'Z':
128 a0d146ed 2005-07-12 devnull 		zero = 0;
129 a0d146ed 2005-07-12 devnull 		break;
130 a0d146ed 2005-07-12 devnull 	default:
131 a0d146ed 2005-07-12 devnull 		usage();
132 a0d146ed 2005-07-12 devnull 		break;
133 a0d146ed 2005-07-12 devnull 	}ARGEND
134 a0d146ed 2005-07-12 devnull 
135 a0d146ed 2005-07-12 devnull 	if(argc != 2)
136 a0d146ed 2005-07-12 devnull 		usage();
137 a0d146ed 2005-07-12 devnull 
138 a0d146ed 2005-07-12 devnull 	if(initventi(argv[0], &conf) < 0)
139 a0d146ed 2005-07-12 devnull 		sysfatal("can't init venti: %r");
140 a0d146ed 2005-07-12 devnull 
141 a0d146ed 2005-07-12 devnull 	if(bcmem < maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16))
142 a0d146ed 2005-07-12 devnull 		bcmem = maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16);
143 a0d146ed 2005-07-12 devnull 	if(0) fprint(2, "initialize %d bytes of disk block cache\n", bcmem);
144 a0d146ed 2005-07-12 devnull 	initdcache(bcmem);
145 a0d146ed 2005-07-12 devnull 
146 a0d146ed 2005-07-12 devnull 	fprint(2, "building a new index %s using %s for temporary storage\n", mainindex->name, argv[1]);
147 a0d146ed 2005-07-12 devnull 
148 a0d146ed 2005-07-12 devnull 	part = initpart(argv[1], ORDWR|ODIRECT);
149 a0d146ed 2005-07-12 devnull 	if(part == nil)
150 a0d146ed 2005-07-12 devnull 		sysfatal("can't initialize temporary partition: %r");
151 a0d146ed 2005-07-12 devnull 
152 a0d146ed 2005-07-12 devnull 	clumps = sortrawientries(mainindex, part, &base, mainindex->bloom);
153 a0d146ed 2005-07-12 devnull 	if(clumps == TWID64)
154 a0d146ed 2005-07-12 devnull 		sysfatal("can't build sorted index: %r");
155 a0d146ed 2005-07-12 devnull 	fprint(2, "found and sorted index entries for clumps=%lld at %lld\n", clumps, base);
156 a0d146ed 2005-07-12 devnull 
157 a0d146ed 2005-07-12 devnull 	if(buildindex(mainindex, part, base, clumps, zero) < 0)
158 a0d146ed 2005-07-12 devnull 		sysfatal("can't build new index: %r");
159 a0d146ed 2005-07-12 devnull 	
160 d4daacde 2005-07-24 devnull 	if(mainindex->bloom)
161 d4daacde 2005-07-24 devnull 		writebloom(mainindex->bloom);
162 d4daacde 2005-07-24 devnull 
163 a0d146ed 2005-07-12 devnull 	threadexitsall(0);
164 a0d146ed 2005-07-12 devnull }