1 a0d146ed 2005-07-12 devnull #include "stdinc.h"
  2 a0d146ed 2005-07-12 devnull #include "dat.h"
  3 a0d146ed 2005-07-12 devnull #include "fns.h"
  4 a0d146ed 2005-07-12 devnull 
  5 a0d146ed 2005-07-12 devnull static int extra, missing, wrong;
  6 a0d146ed 2005-07-12 devnull 
  7 a0d146ed 2005-07-12 devnull static void
  8 a0d146ed 2005-07-12 devnull phdr(DBlock *eb)
  9 a0d146ed 2005-07-12 devnull {
 10 a0d146ed 2005-07-12 devnull 	static int did;
 11 7a400ee9 2007-09-25 rsc 
 12 a0d146ed 2005-07-12 devnull 	if(!did){
 13 a0d146ed 2005-07-12 devnull 		did = 1;
 14 a0d146ed 2005-07-12 devnull 		print("# diff actual correct\n");
 15 a0d146ed 2005-07-12 devnull 	}
 16 a0d146ed 2005-07-12 devnull 	print("%s block 0x%llux\n", eb->part->name, eb->addr);
 17 a0d146ed 2005-07-12 devnull }
 18 a0d146ed 2005-07-12 devnull 
 19 a0d146ed 2005-07-12 devnull static void
 20 a0d146ed 2005-07-12 devnull pie(IEntry *ie, char c)
 21 a0d146ed 2005-07-12 devnull {
 22 a0d146ed 2005-07-12 devnull 	print("%c %V %22lld %3d %5d %3d\n",
 23 a0d146ed 2005-07-12 devnull 		c, ie->score, ie->ia.addr, ie->ia.type, ie->ia.size, ie->ia.blocks);
 24 a0d146ed 2005-07-12 devnull }
 25 a0d146ed 2005-07-12 devnull 
 26 a0d146ed 2005-07-12 devnull static int
 27 a0d146ed 2005-07-12 devnull checkbucket(Index *ix, u32int buck, IBucket *ib)
 28 a0d146ed 2005-07-12 devnull {
 29 a0d146ed 2005-07-12 devnull 	ISect *is;
 30 a0d146ed 2005-07-12 devnull 	DBlock *eb;
 31 a0d146ed 2005-07-12 devnull 	IBucket eib;
 32 a0d146ed 2005-07-12 devnull 	IEntry ie, eie;
 33 a0d146ed 2005-07-12 devnull 	int i, ei, ok, c, hdr;
 34 a0d146ed 2005-07-12 devnull 
 35 a0d146ed 2005-07-12 devnull 	is = ix->sects[indexsect0(ix, buck)];
 36 a0d146ed 2005-07-12 devnull 	if(buck < is->start || buck >= is->stop){
 37 a0d146ed 2005-07-12 devnull 		seterr(EAdmin, "cannot find index section for bucket %lud\n", (ulong)buck);
 38 a0d146ed 2005-07-12 devnull 		return -1;
 39 a0d146ed 2005-07-12 devnull 	}
 40 a0d146ed 2005-07-12 devnull 	buck -= is->start;
 41 a0d146ed 2005-07-12 devnull 	eb = getdblock(is->part, is->blockbase + ((u64int)buck << is->blocklog), OREAD);
 42 a0d146ed 2005-07-12 devnull 	if(eb == nil)
 43 a0d146ed 2005-07-12 devnull 		return -1;
 44 a0d146ed 2005-07-12 devnull 	unpackibucket(&eib, eb->data, is->bucketmagic);
 45 a0d146ed 2005-07-12 devnull 
 46 a0d146ed 2005-07-12 devnull 	ok = 0;
 47 a0d146ed 2005-07-12 devnull 	ei = 0;
 48 a0d146ed 2005-07-12 devnull 	hdr = 0;
 49 a0d146ed 2005-07-12 devnull 	for(i = 0; i < ib->n; i++){
 50 a0d146ed 2005-07-12 devnull 		while(ei < eib.n){
 51 a0d146ed 2005-07-12 devnull 			c = ientrycmp(&ib->data[i * IEntrySize], &eib.data[ei * IEntrySize]);
 52 a0d146ed 2005-07-12 devnull 			if(c == 0){
 53 a0d146ed 2005-07-12 devnull 				unpackientry(&ie, &ib->data[i * IEntrySize]);
 54 a0d146ed 2005-07-12 devnull 				unpackientry(&eie, &eib.data[ei * IEntrySize]);
 55 a0d146ed 2005-07-12 devnull 				if(iaddrcmp(&ie.ia, &eie.ia) != 0){
 56 a0d146ed 2005-07-12 devnull 					if(!hdr){
 57 a0d146ed 2005-07-12 devnull 						phdr(eb);
 58 a0d146ed 2005-07-12 devnull 						hdr = 1;
 59 a0d146ed 2005-07-12 devnull 					}
 60 a0d146ed 2005-07-12 devnull 					wrong++;
 61 a0d146ed 2005-07-12 devnull 					pie(&eie, '<');
 62 a0d146ed 2005-07-12 devnull 					pie(&ie, '>');
 63 a0d146ed 2005-07-12 devnull 				}
 64 a0d146ed 2005-07-12 devnull 				ei++;
 65 a0d146ed 2005-07-12 devnull 				goto cont;
 66 a0d146ed 2005-07-12 devnull 			}
 67 a0d146ed 2005-07-12 devnull 			if(c < 0)
 68 a0d146ed 2005-07-12 devnull 				break;
 69 a0d146ed 2005-07-12 devnull 			if(!hdr){
 70 a0d146ed 2005-07-12 devnull 				phdr(eb);
 71 a0d146ed 2005-07-12 devnull 				hdr = 1;
 72 a0d146ed 2005-07-12 devnull 			}
 73 a0d146ed 2005-07-12 devnull 			unpackientry(&eie, &eib.data[ei*IEntrySize]);
 74 a0d146ed 2005-07-12 devnull 			extra++;
 75 a0d146ed 2005-07-12 devnull 			pie(&eie, '<');
 76 a0d146ed 2005-07-12 devnull 			ei++;
 77 a0d146ed 2005-07-12 devnull 			ok = -1;
 78 a0d146ed 2005-07-12 devnull 		}
 79 a0d146ed 2005-07-12 devnull 		if(!hdr){
 80 a0d146ed 2005-07-12 devnull 			phdr(eb);
 81 a0d146ed 2005-07-12 devnull 			hdr = 1;
 82 a0d146ed 2005-07-12 devnull 		}
 83 a0d146ed 2005-07-12 devnull 		unpackientry(&ie, &ib->data[i*IEntrySize]);
 84 a0d146ed 2005-07-12 devnull 		missing++;
 85 a0d146ed 2005-07-12 devnull 		pie(&ie, '>');
 86 a0d146ed 2005-07-12 devnull 		ok = -1;
 87 a0d146ed 2005-07-12 devnull 	cont:;
 88 a0d146ed 2005-07-12 devnull 	}
 89 a0d146ed 2005-07-12 devnull 	for(; ei < eib.n; ei++){
 90 a0d146ed 2005-07-12 devnull 		if(!hdr){
 91 a0d146ed 2005-07-12 devnull 			phdr(eb);
 92 a0d146ed 2005-07-12 devnull 			hdr = 1;
 93 a0d146ed 2005-07-12 devnull 		}
 94 a0d146ed 2005-07-12 devnull 		unpackientry(&eie, &eib.data[ei*IEntrySize]);
 95 a0d146ed 2005-07-12 devnull 		pie(&eie, '<');
 96 a0d146ed 2005-07-12 devnull 		ok = -1;
 97 a0d146ed 2005-07-12 devnull 	}
 98 a0d146ed 2005-07-12 devnull 	putdblock(eb);
 99 a0d146ed 2005-07-12 devnull 	return ok;
100 a0d146ed 2005-07-12 devnull }
101 a0d146ed 2005-07-12 devnull 
102 a0d146ed 2005-07-12 devnull int
103 a0d146ed 2005-07-12 devnull checkindex(Index *ix, Part *part, u64int off, u64int clumps, int zero)
104 a0d146ed 2005-07-12 devnull {
105 a0d146ed 2005-07-12 devnull 	IEStream *ies;
106 a0d146ed 2005-07-12 devnull 	IBucket ib, zib;
107 a0d146ed 2005-07-12 devnull 	ZBlock *z, *b;
108 a0d146ed 2005-07-12 devnull 	u32int next, buck;
109 a0d146ed 2005-07-12 devnull 	int ok, bok;
110 a0d146ed 2005-07-12 devnull u64int found = 0;
111 a0d146ed 2005-07-12 devnull 
112 28b49df3 2006-07-18 devnull /* ZZZ make buffer size configurable */
113 a0d146ed 2005-07-12 devnull 	b = alloczblock(ix->blocksize, 0, ix->blocksize);
114 a0d146ed 2005-07-12 devnull 	z = alloczblock(ix->blocksize, 1, ix->blocksize);
115 a0d146ed 2005-07-12 devnull 	ies = initiestream(part, off, clumps, 64*1024);
116 a0d146ed 2005-07-12 devnull 	if(b == nil || z == nil || ies == nil){
117 a0d146ed 2005-07-12 devnull 		werrstr("allocating: %r");
118 a0d146ed 2005-07-12 devnull 		ok = -1;
119 27d28098 2007-04-21 devnull 		goto out;
120 a0d146ed 2005-07-12 devnull 	}
121 a0d146ed 2005-07-12 devnull 	ok = 0;
122 a0d146ed 2005-07-12 devnull 	next = 0;
123 a0d146ed 2005-07-12 devnull 	memset(&ib, 0, sizeof ib);
124 a0d146ed 2005-07-12 devnull 	ib.data = b->data;
125 a0d146ed 2005-07-12 devnull 	zib.data = z->data;
126 a0d146ed 2005-07-12 devnull 	zib.n = 0;
127 a0d146ed 2005-07-12 devnull 	zib.buck = 0;
128 a0d146ed 2005-07-12 devnull 	for(;;){
129 a0d146ed 2005-07-12 devnull 		buck = buildbucket(ix, ies, &ib, ix->blocksize-IBucketSize);
130 a0d146ed 2005-07-12 devnull 		found += ib.n;
131 a0d146ed 2005-07-12 devnull 		if(zero){
132 a0d146ed 2005-07-12 devnull 			for(; next != buck; next++){
133 a0d146ed 2005-07-12 devnull 				if(next == ix->buckets){
134 a0d146ed 2005-07-12 devnull 					if(buck != TWID32){
135 a0d146ed 2005-07-12 devnull 						ok = -1;
136 a0d146ed 2005-07-12 devnull 						werrstr("internal error: bucket out of range");
137 a0d146ed 2005-07-12 devnull 					}
138 a0d146ed 2005-07-12 devnull 					if(ok < 0)
139 a0d146ed 2005-07-12 devnull 						werrstr("%d spurious entries, %d missing, %d wrong", extra, missing, wrong);
140 27d28098 2007-04-21 devnull 					goto out;
141 a0d146ed 2005-07-12 devnull 				}
142 a0d146ed 2005-07-12 devnull 				bok = checkbucket(ix, next, &zib);
143 a0d146ed 2005-07-12 devnull 				if(bok < 0)
144 a0d146ed 2005-07-12 devnull 					ok = -1;
145 a0d146ed 2005-07-12 devnull 			}
146 a0d146ed 2005-07-12 devnull 		}
147 a0d146ed 2005-07-12 devnull 		if(buck >= ix->buckets){
148 a0d146ed 2005-07-12 devnull 			if(buck == TWID32)
149 a0d146ed 2005-07-12 devnull 				break;
150 a0d146ed 2005-07-12 devnull 			werrstr("internal error: bucket out of range");
151 a0d146ed 2005-07-12 devnull 			ok = -1;
152 27d28098 2007-04-21 devnull 			goto out;
153 a0d146ed 2005-07-12 devnull 		}
154 a0d146ed 2005-07-12 devnull 		bok = checkbucket(ix, buck, &ib);
155 a0d146ed 2005-07-12 devnull 		if(bok < 0)
156 a0d146ed 2005-07-12 devnull 			ok = -1;
157 a0d146ed 2005-07-12 devnull 		next = buck + 1;
158 a0d146ed 2005-07-12 devnull 	}
159 27d28098 2007-04-21 devnull out:
160 a0d146ed 2005-07-12 devnull 	freeiestream(ies);
161 a0d146ed 2005-07-12 devnull 	freezblock(z);
162 a0d146ed 2005-07-12 devnull 	freezblock(b);
163 a0d146ed 2005-07-12 devnull 	return ok;
164 a0d146ed 2005-07-12 devnull }
165 a0d146ed 2005-07-12 devnull 
166 a0d146ed 2005-07-12 devnull int
167 a0d146ed 2005-07-12 devnull checkbloom(Bloom *b1, Bloom *b2, int fix)
168 a0d146ed 2005-07-12 devnull {
169 a0d146ed 2005-07-12 devnull 	u32int *a1, *a2;
170 a0d146ed 2005-07-12 devnull 	int i, n, extra, missing;
171 7a400ee9 2007-09-25 rsc 
172 a0d146ed 2005-07-12 devnull 	if(b1==nil && b2==nil)
173 a0d146ed 2005-07-12 devnull 		return 0;
174 a0d146ed 2005-07-12 devnull 	if(b1==nil || b2==nil){
175 a0d146ed 2005-07-12 devnull 		werrstr("nil/non-nil");
176 a0d146ed 2005-07-12 devnull 		return -1;
177 a0d146ed 2005-07-12 devnull 	}
178 a0d146ed 2005-07-12 devnull 	wbbloomhead(b1);
179 a0d146ed 2005-07-12 devnull 	wbbloomhead(b2);
180 a0d146ed 2005-07-12 devnull 	if(memcmp(b1->data, b2->data, BloomHeadSize) != 0){
181 a0d146ed 2005-07-12 devnull 		werrstr("bloom header mismatch");
182 a0d146ed 2005-07-12 devnull 		return -1;
183 a0d146ed 2005-07-12 devnull 	}
184 a0d146ed 2005-07-12 devnull 	a1 = (u32int*)b1->data;
185 a0d146ed 2005-07-12 devnull 	a2 = (u32int*)b2->data;
186 a0d146ed 2005-07-12 devnull 	n = b1->size/4;
187 a0d146ed 2005-07-12 devnull 	extra = 0;
188 a0d146ed 2005-07-12 devnull 	missing = 0;
189 a0d146ed 2005-07-12 devnull 	for(i=BloomHeadSize/4; i<n; i++){
190 a0d146ed 2005-07-12 devnull 		if(a1[i] != a2[i]){
191 7a400ee9 2007-09-25 rsc // print("%.8ux/%.8ux.", a1[i], a2[i]);
192 7a400ee9 2007-09-25 rsc 			extra   += countbits(a1[i] & ~a2[i]);
193 a0d146ed 2005-07-12 devnull 			missing += countbits(a2[i] & ~a1[i]);
194 a0d146ed 2005-07-12 devnull 		}
195 a0d146ed 2005-07-12 devnull 	}
196 a0d146ed 2005-07-12 devnull 	if(extra || missing)
197 7a400ee9 2007-09-25 rsc 		fprint(2, "bloom filter: %d spurious bits, %d missing bits\n",
198 7a400ee9 2007-09-25 rsc 			extra, missing);
199 a0d146ed 2005-07-12 devnull 	else
200 a0d146ed 2005-07-12 devnull 		fprint(2, "bloom filter: correct\n");
201 a0d146ed 2005-07-12 devnull 	if(!fix && missing){
202 a0d146ed 2005-07-12 devnull 		werrstr("missing bits");
203 a0d146ed 2005-07-12 devnull 		return -1;
204 a0d146ed 2005-07-12 devnull 	}
205 a0d146ed 2005-07-12 devnull 	if(fix && (missing || extra)){
206 a0d146ed 2005-07-12 devnull 		memmove(b1->data, b2->data, b1->size);
207 a0d146ed 2005-07-12 devnull 		return writebloom(b1);
208 a0d146ed 2005-07-12 devnull 	}
209 a0d146ed 2005-07-12 devnull 	return 0;
210 a0d146ed 2005-07-12 devnull }
211 a0d146ed 2005-07-12 devnull 
212 a0d146ed 2005-07-12 devnull 
213 a0d146ed 2005-07-12 devnull void
214 a0d146ed 2005-07-12 devnull usage(void)
215 a0d146ed 2005-07-12 devnull {
216 a0d146ed 2005-07-12 devnull 	fprint(2, "usage: checkindex [-f] [-B blockcachesize] config tmp\n");
217 a0d146ed 2005-07-12 devnull 	threadexitsall(0);
218 a0d146ed 2005-07-12 devnull }
219 a0d146ed 2005-07-12 devnull 
220 a0d146ed 2005-07-12 devnull Config conf;
221 a0d146ed 2005-07-12 devnull 
222 a0d146ed 2005-07-12 devnull void
223 a0d146ed 2005-07-12 devnull threadmain(int argc, char *argv[])
224 a0d146ed 2005-07-12 devnull {
225 a0d146ed 2005-07-12 devnull 	Bloom *oldbloom, *newbloom;
226 a0d146ed 2005-07-12 devnull 	Part *part;
227 a0d146ed 2005-07-12 devnull 	u64int clumps, base;
228 a0d146ed 2005-07-12 devnull 	u32int bcmem;
229 a0d146ed 2005-07-12 devnull 	int fix, skipz, ok;
230 a0d146ed 2005-07-12 devnull 
231 a0d146ed 2005-07-12 devnull 	fix = 0;
232 a0d146ed 2005-07-12 devnull 	bcmem = 0;
233 a0d146ed 2005-07-12 devnull 	skipz = 0;
234 a0d146ed 2005-07-12 devnull 	ARGBEGIN{
235 a0d146ed 2005-07-12 devnull 	case 'B':
236 a0d146ed 2005-07-12 devnull 		bcmem = unittoull(ARGF());
237 a0d146ed 2005-07-12 devnull 		break;
238 a0d146ed 2005-07-12 devnull 	case 'f':
239 a0d146ed 2005-07-12 devnull 		fix++;
240 a0d146ed 2005-07-12 devnull 		break;
241 a0d146ed 2005-07-12 devnull 	case 'Z':
242 a0d146ed 2005-07-12 devnull 		skipz = 1;
243 a0d146ed 2005-07-12 devnull 		break;
244 a0d146ed 2005-07-12 devnull 	default:
245 a0d146ed 2005-07-12 devnull 		usage();
246 a0d146ed 2005-07-12 devnull 		break;
247 a0d146ed 2005-07-12 devnull 	}ARGEND
248 a0d146ed 2005-07-12 devnull 
249 a0d146ed 2005-07-12 devnull 	if(argc != 2)
250 a0d146ed 2005-07-12 devnull 		usage();
251 a0d146ed 2005-07-12 devnull 
252 a0d146ed 2005-07-12 devnull 	ventifmtinstall();
253 a0d146ed 2005-07-12 devnull 
254 a0d146ed 2005-07-12 devnull 	part = initpart(argv[1], ORDWR|ODIRECT);
255 a0d146ed 2005-07-12 devnull 	if(part == nil)
256 a0d146ed 2005-07-12 devnull 		sysfatal("can't initialize temporary partition: %r");
257 a0d146ed 2005-07-12 devnull 
258 a0d146ed 2005-07-12 devnull 	if(!fix)
259 a0d146ed 2005-07-12 devnull 		readonly = 1;
260 a0d146ed 2005-07-12 devnull 
261 a0d146ed 2005-07-12 devnull 	if(initventi(argv[0], &conf) < 0)
262 a0d146ed 2005-07-12 devnull 		sysfatal("can't init venti: %r");
263 28b49df3 2006-07-18 devnull 	if(mainindex->bloom && loadbloom(mainindex->bloom) < 0)
264 28b49df3 2006-07-18 devnull 		sysfatal("can't load bloom filter: %r");
265 a0d146ed 2005-07-12 devnull 	oldbloom = mainindex->bloom;
266 a0d146ed 2005-07-12 devnull 	newbloom = nil;
267 a0d146ed 2005-07-12 devnull 	if(oldbloom){
268 a0d146ed 2005-07-12 devnull 		newbloom = vtmallocz(sizeof *newbloom);
269 a0d146ed 2005-07-12 devnull 		bloominit(newbloom, oldbloom->size, nil);
270 a0d146ed 2005-07-12 devnull 		newbloom->data = vtmallocz(oldbloom->size);
271 a0d146ed 2005-07-12 devnull 	}
272 a0d146ed 2005-07-12 devnull 	if(bcmem < maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16))
273 a0d146ed 2005-07-12 devnull 		bcmem = maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16);
274 a0d146ed 2005-07-12 devnull 	if(0) fprint(2, "initialize %d bytes of disk block cache\n", bcmem);
275 a0d146ed 2005-07-12 devnull 	initdcache(bcmem);
276 a0d146ed 2005-07-12 devnull 
277 a0d146ed 2005-07-12 devnull 	fprint(2, "checkindex: building entry list\n");
278 a0d146ed 2005-07-12 devnull 	clumps = sortrawientries(mainindex, part, &base, newbloom);
279 a0d146ed 2005-07-12 devnull 	if(clumps == TWID64)
280 a0d146ed 2005-07-12 devnull 		sysfatal("can't build sorted index: %r");
281 a0d146ed 2005-07-12 devnull 	fprint(2, "checkindex: checking %lld entries at %lld\n", clumps, base);
282 a0d146ed 2005-07-12 devnull 	ok = 0;
283 a0d146ed 2005-07-12 devnull 	if(checkindex(mainindex, part, base, clumps, !skipz) < 0){
284 a0d146ed 2005-07-12 devnull 		fprint(2, "checkindex: %r\n");
285 a0d146ed 2005-07-12 devnull 		ok = -1;
286 a0d146ed 2005-07-12 devnull 	}
287 a0d146ed 2005-07-12 devnull 	if(checkbloom(oldbloom, newbloom, fix) < 0){
288 a0d146ed 2005-07-12 devnull 		fprint(2, "checkbloom: %r\n");
289 a0d146ed 2005-07-12 devnull 		ok = -1;
290 a0d146ed 2005-07-12 devnull 	}
291 a0d146ed 2005-07-12 devnull 	if(ok < 0)
292 a0d146ed 2005-07-12 devnull 		sysfatal("errors found");
293 a0d146ed 2005-07-12 devnull 	fprint(2, "checkindex: index is correct\n");
294 a0d146ed 2005-07-12 devnull 	threadexitsall(0);
295 a0d146ed 2005-07-12 devnull }