1 #include "stdinc.h"
2 #include "dat.h"
3 #include "fns.h"
4 
5 /*
6  * disk structure conversion routines
7  */
8 #define	U8GET(p)	((p)[0])
9 #define	U16GET(p)	(((p)[0]<<8)|(p)[1])
10 #define	U32GET(p)	((u32int)(((p)[0]<<24)|((p)[1]<<16)|((p)[2]<<8)|(p)[3]))
11 #define	U64GET(p)	(((u64int)U32GET(p)<<32)|(u64int)U32GET((p)+4))
12 
13 #define	U8PUT(p,v)	(p)[0]=(v)&0xFF
14 #define	U16PUT(p,v)	(p)[0]=((v)>>8)&0xFF;(p)[1]=(v)&0xFF
15 #define	U32PUT(p,v)	(p)[0]=((v)>>24)&0xFF;(p)[1]=((v)>>16)&0xFF;(p)[2]=((v)>>8)&0xFF;(p)[3]=(v)&0xFF
16 #define	U64PUT(p,v,t32)	t32=(v)>>32;U32PUT(p,t32);t32=(v);U32PUT((p)+4,t32)
17 
18 int debugarena = -1;		/* hack to improve error reporting */
19 
20 static struct {
21 	u32int m;
22 	char *s;
23 } magics[] = {
24 	ArenaPartMagic, "ArenaPartMagic",
25 	ArenaHeadMagic, "ArenaHeadMagic",
26 	ArenaMagic, "ArenaMagic",
27 	ISectMagic, "ISectMagic",
28 	BloomMagic, "BloomMagic",
29 };
30 
31 static char*
32 fmtmagic(char *s, u32int m)
33 {
34 	int i;
35 
36 	for(i=0; i<nelem(magics); i++)
37 		if(magics[i].m == m)
38 			return magics[i].s;
39 	sprint(s, "%#08ux", m);
40 	return s;
41 }
42 
43 u32int
44 unpackmagic(u8int *buf)
45 {
46 	return U32GET(buf);
47 }
48 
49 void
50 packmagic(u32int magic, u8int *buf)
51 {
52 	U32PUT(buf, magic);
53 }
54 
55 int
56 unpackarenapart(ArenaPart *ap, u8int *buf)
57 {
58 	u8int *p;
59 	u32int m;
60 	char fbuf[20];
61 
62 	p = buf;
63 
64 	m = U32GET(p);
65 	if(m != ArenaPartMagic){
66 		seterr(ECorrupt, "arena set has wrong magic number: %s expected ArenaPartMagic (%#lux)", fmtmagic(fbuf, m), ArenaPartMagic);
67 		return -1;
68 	}
69 	p += U32Size;
70 	ap->version = U32GET(p);
71 	p += U32Size;
72 	ap->blocksize = U32GET(p);
73 	p += U32Size;
74 	ap->arenabase = U32GET(p);
75 	p += U32Size;
76 
77 	if(buf + ArenaPartSize != p)
78 		sysfatal("unpackarenapart unpacked wrong amount");
79 
80 	return 0;
81 }
82 
83 int
84 packarenapart(ArenaPart *ap, u8int *buf)
85 {
86 	u8int *p;
87 
88 	p = buf;
89 
90 	U32PUT(p, ArenaPartMagic);
91 	p += U32Size;
92 	U32PUT(p, ap->version);
93 	p += U32Size;
94 	U32PUT(p, ap->blocksize);
95 	p += U32Size;
96 	U32PUT(p, ap->arenabase);
97 	p += U32Size;
98 
99 	if(buf + ArenaPartSize != p)
100 		sysfatal("packarenapart packed wrong amount");
101 
102 	return 0;
103 }
104 
105 int
106 unpackarena(Arena *arena, u8int *buf)
107 {
108 	int sz;
109 	u8int *p;
110 	u32int m;
111 	char fbuf[20];
112 
113 	p = buf;
114 
115 	m = U32GET(p);
116 	if(m != ArenaMagic){
117 		seterr(ECorrupt, "arena %d has wrong magic number: %s "
118 			"expected ArenaMagic (%#lux)", debugarena,
119 			fmtmagic(fbuf, m), ArenaMagic);
120 		return -1;
121 	}
122 	p += U32Size;
123 	arena->version = U32GET(p);
124 	p += U32Size;
125 	namecp(arena->name, (char*)p);
126 	p += ANameSize;
127 	arena->diskstats.clumps = U32GET(p);
128 	p += U32Size;
129 	arena->diskstats.cclumps = U32GET(p);
130 	p += U32Size;
131 	arena->ctime = U32GET(p);
132 	p += U32Size;
133 	arena->wtime = U32GET(p);
134 	p += U32Size;
135 	if(arena->version == ArenaVersion5){
136 		arena->clumpmagic = U32GET(p);
137 		p += U32Size;
138 	}
139 	arena->diskstats.used = U64GET(p);
140 	p += U64Size;
141 	arena->diskstats.uncsize = U64GET(p);
142 	p += U64Size;
143 	arena->diskstats.sealed = U8GET(p);
144 	p += U8Size;
145 	switch(arena->version){
146 	case ArenaVersion4:
147 		sz = ArenaSize4;
148 		arena->clumpmagic = _ClumpMagic;
149 		break;
150 	case ArenaVersion5:
151 		sz = ArenaSize5;
152 		break;
153 	default:
154 		seterr(ECorrupt, "arena has bad version number %d", arena->version);
155 		return -1;
156 	}
157 	/*
158 	 * Additional fields for the memstats version of the stats.
159 	 * Diskstats reflects what is committed to the index.
160 	 * Memstats reflects what is in the arena.  Originally intended
161 	 * this to be a version 5 extension, but might as well use for
162 	 * all the existing version 4 arenas too.
163 	 *
164 	 * To maintain backwards compatibility with existing venti
165 	 * installations using the older format, we define that if
166 	 * memstats == diskstats, then the extension fields are not
167 	 * included (see packarena below).  That is, only partially
168 	 * indexed arenas have these fields.  Fully indexed arenas
169 	 * (in particular, sealed arenas) do not.
170 	 */
171 	if(U8GET(p) == 1){
172 		sz += ArenaSize5a-ArenaSize5;
173 		p += U8Size;
174 		arena->memstats.clumps = U32GET(p);
175 		p += U32Size;
176 		arena->memstats.cclumps = U32GET(p);
177 		p += U32Size;
178 		arena->memstats.used = U64GET(p);
179 		p += U64Size;
180 		arena->memstats.uncsize = U64GET(p);
181 		p += U64Size;
182 		arena->memstats.sealed = U8GET(p);
183 		p += U8Size;
184 
185 		/*
186 		 * 2008/4/2
187 		 * Packarena (below) used to have a bug in which it would
188 		 * not zero out any existing extension fields when writing
189 		 * the arena metadata.  This would manifest itself as arenas
190 		 * with arena->diskstats.sealed == 1 but arena->memstats.sealed == 0
191 		 * after a server restart.  Because arena->memstats.sealed wouldn't
192 		 * be set, the server might try to fit another block into the arena
193 		 * (and succeed), violating the append-only structure of the log
194 		 * and invalidating any already-computed seal on the arena.
195 		 *
196 		 * It might end up that other fields in arena->memstats end up
197 		 * behind arena->diskstats too, but that would be considerably
198 		 * more rare, and the bug is fixed now.  The case we need to
199 		 * handle is just the sealed mismatch.
200 		 *
201 		 * If we encounter such a bogus arena, fix the sealed field.
202 		 */
203 		if(arena->diskstats.sealed)
204 			arena->memstats.sealed = 1;
205 	}else
206 		arena->memstats = arena->diskstats;
207 	if(buf + sz != p)
208 		sysfatal("unpackarena unpacked wrong amount");
209 
210 	return 0;
211 }
212 
213 int
214 packarena(Arena *arena, u8int *buf)
215 {
216 	return _packarena(arena, buf, 0);
217 }
218 
219 int
220 _packarena(Arena *arena, u8int *buf, int forceext)
221 {
222 	int sz;
223 	u8int *p;
224 	u32int t32;
225 
226 	switch(arena->version){
227 	case ArenaVersion4:
228 		sz = ArenaSize4;
229 		if(arena->clumpmagic != _ClumpMagic)
230 			fprint(2, "warning: writing old arena tail loses clump magic 0x%lux != 0x%lux\n",
231 				(ulong)arena->clumpmagic, (ulong)_ClumpMagic);
232 		break;
233 	case ArenaVersion5:
234 		sz = ArenaSize5;
235 		break;
236 	default:
237 		sysfatal("packarena unknown version %d", arena->version);
238 		return -1;
239 	}
240 
241 	p = buf;
242 
243 	U32PUT(p, ArenaMagic);
244 	p += U32Size;
245 	U32PUT(p, arena->version);
246 	p += U32Size;
247 	namecp((char*)p, arena->name);
248 	p += ANameSize;
249 	U32PUT(p, arena->diskstats.clumps);
250 	p += U32Size;
251 	U32PUT(p, arena->diskstats.cclumps);
252 	p += U32Size;
253 	U32PUT(p, arena->ctime);
254 	p += U32Size;
255 	U32PUT(p, arena->wtime);
256 	p += U32Size;
257 	if(arena->version == ArenaVersion5){
258 		U32PUT(p, arena->clumpmagic);
259 		p += U32Size;
260 	}
261 	U64PUT(p, arena->diskstats.used, t32);
262 	p += U64Size;
263 	U64PUT(p, arena->diskstats.uncsize, t32);
264 	p += U64Size;
265 	U8PUT(p, arena->diskstats.sealed);
266 	p += U8Size;
267 
268 	/*
269 	 * Extension fields; see above.
270 	 */
271 	if(forceext
272 	|| arena->memstats.clumps != arena->diskstats.clumps
273 	|| arena->memstats.cclumps != arena->diskstats.cclumps
274 	|| arena->memstats.used != arena->diskstats.used
275 	|| arena->memstats.uncsize != arena->diskstats.uncsize
276 	|| arena->memstats.sealed != arena->diskstats.sealed){
277 		sz += ArenaSize5a - ArenaSize5;
278 		U8PUT(p, 1);
279 		p += U8Size;
280 		U32PUT(p, arena->memstats.clumps);
281 		p += U32Size;
282 		U32PUT(p, arena->memstats.cclumps);
283 		p += U32Size;
284 		U64PUT(p, arena->memstats.used, t32);
285 		p += U64Size;
286 		U64PUT(p, arena->memstats.uncsize, t32);
287 		p += U64Size;
288 		U8PUT(p, arena->memstats.sealed);
289 		p += U8Size;
290 	}else{
291 		/* Clear any extension fields already on disk. */
292 		memset(p, 0, ArenaSize5a - ArenaSize5);
293 		p += ArenaSize5a - ArenaSize5;
294 		sz += ArenaSize5a - ArenaSize5;
295 	}
296 
297 	if(buf + sz != p)
298 		sysfatal("packarena packed wrong amount");
299 
300 	return 0;
301 }
302 
303 int
304 unpackarenahead(ArenaHead *head, u8int *buf)
305 {
306 	u8int *p;
307 	u32int m;
308 	int sz;
309 	char fbuf[20];
310 
311 	p = buf;
312 
313 	m = U32GET(p);
314 	if(m != ArenaHeadMagic){
315 		seterr(ECorrupt, "arena %d head has wrong magic number: %s "
316 			"expected ArenaHeadMagic (%#lux)", debugarena,
317 			fmtmagic(fbuf, m), ArenaHeadMagic);
318 		return -1;
319 	}
320 
321 	p += U32Size;
322 	head->version = U32GET(p);
323 	p += U32Size;
324 	namecp(head->name, (char*)p);
325 	p += ANameSize;
326 	head->blocksize = U32GET(p);
327 	p += U32Size;
328 	head->size = U64GET(p);
329 	p += U64Size;
330 	if(head->version == ArenaVersion5){
331 		head->clumpmagic = U32GET(p);
332 		p += U32Size;
333 	}
334 
335 	switch(head->version){
336 	case ArenaVersion4:
337 		sz = ArenaHeadSize4;
338 		head->clumpmagic = _ClumpMagic;
339 		break;
340 	case ArenaVersion5:
341 		sz = ArenaHeadSize5;
342 		break;
343 	default:
344 		seterr(ECorrupt, "arena head has unexpected version %d", head->version);
345 		return -1;
346 	}
347 
348 	if(buf + sz != p)
349 		sysfatal("unpackarenahead unpacked wrong amount");
350 
351 	return 0;
352 }
353 
354 int
355 packarenahead(ArenaHead *head, u8int *buf)
356 {
357 	u8int *p;
358 	int sz;
359 	u32int t32;
360 
361 	switch(head->version){
362 	case ArenaVersion4:
363 		sz = ArenaHeadSize4;
364 		if(head->clumpmagic != _ClumpMagic)
365 			fprint(2, "warning: writing old arena header loses clump magic 0x%lux != 0x%lux\n",
366 				(ulong)head->clumpmagic, (ulong)_ClumpMagic);
367 		break;
368 	case ArenaVersion5:
369 		sz = ArenaHeadSize5;
370 		break;
371 	default:
372 		sysfatal("packarenahead unknown version %d", head->version);
373 		return -1;
374 	}
375 
376 	p = buf;
377 
378 	U32PUT(p, ArenaHeadMagic);
379 	p += U32Size;
380 	U32PUT(p, head->version);
381 	p += U32Size;
382 	namecp((char*)p, head->name);
383 	p += ANameSize;
384 	U32PUT(p, head->blocksize);
385 	p += U32Size;
386 	U64PUT(p, head->size, t32);
387 	p += U64Size;
388 	if(head->version == ArenaVersion5){
389 		U32PUT(p, head->clumpmagic);
390 		p += U32Size;
391 	}
392 	if(buf + sz != p)
393 		sysfatal("packarenahead packed wrong amount");
394 
395 	return 0;
396 }
397 
398 static int
399 checkclump(Clump *w)
400 {
401 	if(w->encoding == ClumpENone){
402 		if(w->info.size != w->info.uncsize){
403 			seterr(ECorrupt, "uncompressed wad size mismatch");
404 			return -1;
405 		}
406 	}else if(w->encoding == ClumpECompress){
407 		if(w->info.size >= w->info.uncsize){
408 			seterr(ECorrupt, "compressed lump has inconsistent block sizes %d %d", w->info.size, w->info.uncsize);
409 			return -1;
410 		}
411 	}else{
412 		seterr(ECorrupt, "clump has illegal encoding");
413 		return -1;
414 	}
415 
416 	return 0;
417 }
418 
419 int
420 unpackclump(Clump *c, u8int *buf, u32int cmagic)
421 {
422 	u8int *p;
423 	u32int magic;
424 
425 	p = buf;
426 	magic = U32GET(p);
427 	if(magic != cmagic){
428 		seterr(ECorrupt, "clump has bad magic number=%#8.8ux != %#8.8ux", magic, cmagic);
429 		return -1;
430 	}
431 	p += U32Size;
432 
433 	c->info.type = vtfromdisktype(U8GET(p));
434 	p += U8Size;
435 	c->info.size = U16GET(p);
436 	p += U16Size;
437 	c->info.uncsize = U16GET(p);
438 	p += U16Size;
439 	scorecp(c->info.score, p);
440 	p += VtScoreSize;
441 
442 	c->encoding = U8GET(p);
443 	p += U8Size;
444 	c->creator = U32GET(p);
445 	p += U32Size;
446 	c->time = U32GET(p);
447 	p += U32Size;
448 
449 	if(buf + ClumpSize != p)
450 		sysfatal("unpackclump unpacked wrong amount");
451 
452 	return checkclump(c);
453 }
454 
455 int
456 packclump(Clump *c, u8int *buf, u32int magic)
457 {
458 	u8int *p;
459 
460 	p = buf;
461 	U32PUT(p, magic);
462 	p += U32Size;
463 
464 	U8PUT(p, vttodisktype(c->info.type));
465 	p += U8Size;
466 	U16PUT(p, c->info.size);
467 	p += U16Size;
468 	U16PUT(p, c->info.uncsize);
469 	p += U16Size;
470 	scorecp(p, c->info.score);
471 	p += VtScoreSize;
472 
473 	U8PUT(p, c->encoding);
474 	p += U8Size;
475 	U32PUT(p, c->creator);
476 	p += U32Size;
477 	U32PUT(p, c->time);
478 	p += U32Size;
479 
480 	if(buf + ClumpSize != p)
481 		sysfatal("packclump packed wrong amount");
482 
483 	return checkclump(c);
484 }
485 
486 void
487 unpackclumpinfo(ClumpInfo *ci, u8int *buf)
488 {
489 	u8int *p;
490 
491 	p = buf;
492 	ci->type = vtfromdisktype(U8GET(p));
493 	p += U8Size;
494 	ci->size = U16GET(p);
495 	p += U16Size;
496 	ci->uncsize = U16GET(p);
497 	p += U16Size;
498 	scorecp(ci->score, p);
499 	p += VtScoreSize;
500 
501 	if(buf + ClumpInfoSize != p)
502 		sysfatal("unpackclumpinfo unpacked wrong amount");
503 }
504 
505 void
506 packclumpinfo(ClumpInfo *ci, u8int *buf)
507 {
508 	u8int *p;
509 
510 	p = buf;
511 	U8PUT(p, vttodisktype(ci->type));
512 	p += U8Size;
513 	U16PUT(p, ci->size);
514 	p += U16Size;
515 	U16PUT(p, ci->uncsize);
516 	p += U16Size;
517 	scorecp(p, ci->score);
518 	p += VtScoreSize;
519 
520 	if(buf + ClumpInfoSize != p)
521 		sysfatal("packclumpinfo packed wrong amount");
522 }
523 
524 int
525 unpackisect(ISect *is, u8int *buf)
526 {
527 	u8int *p;
528 	u32int m;
529 	char fbuf[20];
530 
531 	p = buf;
532 
533 
534 	m = U32GET(p);
535 	if(m != ISectMagic){
536 		seterr(ECorrupt, "index section has wrong magic number: %s expected ISectMagic (%#lux)",
537 			fmtmagic(fbuf, m), ISectMagic);
538 		return -1;
539 	}
540 	p += U32Size;
541 	is->version = U32GET(p);
542 	p += U32Size;
543 	namecp(is->name, (char*)p);
544 	p += ANameSize;
545 	namecp(is->index, (char*)p);
546 	p += ANameSize;
547 	is->blocksize = U32GET(p);
548 	p += U32Size;
549 	is->blockbase = U32GET(p);
550 	p += U32Size;
551 	is->blocks = U32GET(p);
552 	p += U32Size;
553 	is->start = U32GET(p);
554 	p += U32Size;
555 	is->stop = U32GET(p);
556 	p += U32Size;
557 	if(buf + ISectSize1 != p)
558 		sysfatal("unpackisect unpacked wrong amount");
559 	is->bucketmagic = 0;
560 	if(is->version == ISectVersion2){
561 		is->bucketmagic = U32GET(p);
562 		p += U32Size;
563 		if(buf + ISectSize2 != p)
564 			sysfatal("unpackisect unpacked wrong amount");
565 	}
566 
567 	return 0;
568 }
569 
570 int
571 packisect(ISect *is, u8int *buf)
572 {
573 	u8int *p;
574 
575 	p = buf;
576 
577 	U32PUT(p, ISectMagic);
578 	p += U32Size;
579 	U32PUT(p, is->version);
580 	p += U32Size;
581 	namecp((char*)p, is->name);
582 	p += ANameSize;
583 	namecp((char*)p, is->index);
584 	p += ANameSize;
585 	U32PUT(p, is->blocksize);
586 	p += U32Size;
587 	U32PUT(p, is->blockbase);
588 	p += U32Size;
589 	U32PUT(p, is->blocks);
590 	p += U32Size;
591 	U32PUT(p, is->start);
592 	p += U32Size;
593 	U32PUT(p, is->stop);
594 	p += U32Size;
595 	if(buf + ISectSize1 != p)
596 		sysfatal("packisect packed wrong amount");
597 	if(is->version == ISectVersion2){
598 		U32PUT(p, is->bucketmagic);
599 		p += U32Size;
600 		if(buf + ISectSize2 != p)
601 			sysfatal("packisect packed wrong amount");
602 	}
603 
604 	return 0;
605 }
606 
607 void
608 unpackientry(IEntry *ie, u8int *buf)
609 {
610 	u8int *p;
611 
612 	p = buf;
613 
614 	scorecp(ie->score, p);
615 	p += VtScoreSize;
616 	/* ie->wtime = U32GET(p); */
617 	p += U32Size;
618 	/* ie->train = U16GET(p); */
619 	p += U16Size;
620 	if(p - buf != IEntryAddrOff)
621 		sysfatal("unpackentry bad IEntryAddrOff amount");
622 	ie->ia.addr = U64GET(p);
623 if(ie->ia.addr>>56) print("%.8H => %llux\n", p, ie->ia.addr);
624 	p += U64Size;
625 	ie->ia.size = U16GET(p);
626 	p += U16Size;
627 	if(p - buf != IEntryTypeOff)
628 		sysfatal("unpackientry bad IEntryTypeOff amount");
629 	ie->ia.type = vtfromdisktype(U8GET(p));
630 	p += U8Size;
631 	ie->ia.blocks = U8GET(p);
632 	p += U8Size;
633 
634 	if(p - buf != IEntrySize)
635 		sysfatal("unpackientry unpacked wrong amount");
636 }
637 
638 void
639 packientry(IEntry *ie, u8int *buf)
640 {
641 	u32int t32;
642 	u8int *p;
643 
644 	p = buf;
645 
646 	scorecp(p, ie->score);
647 	p += VtScoreSize;
648 	U32PUT(p, 0); /* wtime */
649 	p += U32Size;
650 	U16PUT(p, 0); /* train */
651 	p += U16Size;
652 	U64PUT(p, ie->ia.addr, t32);
653 	p += U64Size;
654 	U16PUT(p, ie->ia.size);
655 	p += U16Size;
656 	U8PUT(p, vttodisktype(ie->ia.type));
657 	p += U8Size;
658 	U8PUT(p, ie->ia.blocks);
659 	p += U8Size;
660 
661 	if(p - buf != IEntrySize)
662 		sysfatal("packientry packed wrong amount");
663 }
664 
665 void
666 unpackibucket(IBucket *b, u8int *buf, u32int magic)
667 {
668 	b->n = U16GET(buf);
669 	b->data = buf + IBucketSize;
670 	if(magic && magic != U32GET(buf+U16Size))
671 		b->n = 0;
672 }
673 
674 void
675 packibucket(IBucket *b, u8int *buf, u32int magic)
676 {
677 	U16PUT(buf, b->n);
678 	U32PUT(buf+U16Size, magic);
679 }
680 
681 void
682 packbloomhead(Bloom *b, u8int *buf)
683 {
684 	u8int *p;
685 
686 	p = buf;
687 	U32PUT(p, BloomMagic);
688 	U32PUT(p+4, BloomVersion);
689 	U32PUT(p+8, b->nhash);
690 	U32PUT(p+12, b->size);
691 }
692 
693 int
694 unpackbloomhead(Bloom *b, u8int *buf)
695 {
696 	u8int *p;
697 	u32int m;
698 	char fbuf[20];
699 
700 	p = buf;
701 
702 	m = U32GET(p);
703 	if(m != BloomMagic){
704 		seterr(ECorrupt, "bloom filter has wrong magic number: %s expected BloomMagic (%#lux)", fmtmagic(fbuf, m), (ulong)BloomMagic);
705 		return -1;
706 	}
707 	p += U32Size;
708 
709 	m = U32GET(p);
710 	if(m != BloomVersion){
711 		seterr(ECorrupt, "bloom filter has wrong version %ud expected %ud", (uint)m, (uint)BloomVersion);
712 		return -1;
713 	}
714 	p += U32Size;
715 
716 	b->nhash = U32GET(p);
717 	p += U32Size;
718 
719 	b->size = U32GET(p);
720 	p += U32Size;
721 	if(b->size < BloomHeadSize || b->size > MaxBloomSize || (b->size&(b->size-1))){
722 		seterr(ECorrupt, "bloom filter has invalid size %#lux", b->size);
723 		return -1;
724 	}
725 
726 	if(buf + BloomHeadSize != p)
727 		sysfatal("unpackarena unpacked wrong amount");
728 
729 	return 0;
730 }