1 #include "stdinc.h"
2 #include "dat.h"
3 #include "fns.h"
4 
5 /*
6  * disk structure conversion routines
7  */
8 #define	U8GET(p)	((p)[0])
9 #define	U16GET(p)	(((p)[0]<<8)|(p)[1])
10 #define	U32GET(p)	((u32int)(((p)[0]<<24)|((p)[1]<<16)|((p)[2]<<8)|(p)[3]))
11 #define	U64GET(p)	(((u64int)U32GET(p)<<32)|(u64int)U32GET((p)+4))
12 
13 #define	U8PUT(p,v)	(p)[0]=(v)&0xFF
14 #define	U16PUT(p,v)	(p)[0]=((v)>>8)&0xFF;(p)[1]=(v)&0xFF
15 #define	U32PUT(p,v)	(p)[0]=((v)>>24)&0xFF;(p)[1]=((v)>>16)&0xFF;(p)[2]=((v)>>8)&0xFF;(p)[3]=(v)&0xFF
16 #define	U64PUT(p,v,t32)	t32=(v)>>32;U32PUT(p,t32);t32=(v);U32PUT((p)+4,t32)
17 
18 static struct {
19 	u32int m;
20 	char *s;
21 } magics[] = {
22 	ArenaPartMagic, "ArenaPartMagic",
23 	ArenaHeadMagic, "ArenaHeadMagic",
24 	ArenaMagic, "ArenaMagic",
25 	ISectMagic, "ISectMagic",
26 	BloomMagic, "BloomMagic",
27 };
28 
29 static char*
30 fmtmagic(char *s, u32int m)
31 {
32 	int i;
33 
34 	for(i=0; i<nelem(magics); i++)
35 		if(magics[i].m == m)
36 			return magics[i].s;
37 	sprint(s, "%#08ux", m);
38 	return s;
39 }
40 
41 u32int
42 unpackmagic(u8int *buf)
43 {
44 	return U32GET(buf);
45 }
46 
47 void
48 packmagic(u32int magic, u8int *buf)
49 {
50 	U32PUT(buf, magic);
51 }
52 
53 int
54 unpackarenapart(ArenaPart *ap, u8int *buf)
55 {
56 	u8int *p;
57 	u32int m;
58 	char fbuf[20];
59 
60 	p = buf;
61 
62 	m = U32GET(p);
63 	if(m != ArenaPartMagic){
64 		seterr(ECorrupt, "arena set has wrong magic number: %s expected ArenaPartMagic (%#lux)", fmtmagic(fbuf, m), ArenaPartMagic);
65 		return -1;
66 	}
67 	p += U32Size;
68 	ap->version = U32GET(p);
69 	p += U32Size;
70 	ap->blocksize = U32GET(p);
71 	p += U32Size;
72 	ap->arenabase = U32GET(p);
73 	p += U32Size;
74 
75 	if(buf + ArenaPartSize != p)
76 		sysfatal("unpackarenapart unpacked wrong amount");
77 
78 	return 0;
79 }
80 
81 int
82 packarenapart(ArenaPart *ap, u8int *buf)
83 {
84 	u8int *p;
85 
86 	p = buf;
87 
88 	U32PUT(p, ArenaPartMagic);
89 	p += U32Size;
90 	U32PUT(p, ap->version);
91 	p += U32Size;
92 	U32PUT(p, ap->blocksize);
93 	p += U32Size;
94 	U32PUT(p, ap->arenabase);
95 	p += U32Size;
96 
97 	if(buf + ArenaPartSize != p)
98 		sysfatal("packarenapart packed wrong amount");
99 
100 	return 0;
101 }
102 
103 int
104 unpackarena(Arena *arena, u8int *buf)
105 {
106 	int sz;
107 	u8int *p;
108 	u32int m;
109 	char fbuf[20];
110 
111 	p = buf;
112 
113 	m = U32GET(p);
114 	if(m != ArenaMagic){
115 		seterr(ECorrupt, "arena has wrong magic number: %s expected ArenaMagic (%#lux)", fmtmagic(fbuf, m), ArenaMagic);
116 		return -1;
117 	}
118 	p += U32Size;
119 	arena->version = U32GET(p);
120 	p += U32Size;
121 	namecp(arena->name, (char*)p);
122 	p += ANameSize;
123 	arena->diskstats.clumps = U32GET(p);
124 	p += U32Size;
125 	arena->diskstats.cclumps = U32GET(p);
126 	p += U32Size;
127 	arena->ctime = U32GET(p);
128 	p += U32Size;
129 	arena->wtime = U32GET(p);
130 	p += U32Size;
131 	if(arena->version == ArenaVersion5){
132 		arena->clumpmagic = U32GET(p);
133 		p += U32Size;
134 	}
135 	arena->diskstats.used = U64GET(p);
136 	p += U64Size;
137 	arena->diskstats.uncsize = U64GET(p);
138 	p += U64Size;
139 	arena->diskstats.sealed = U8GET(p);
140 	p += U8Size;
141 	switch(arena->version){
142 	case ArenaVersion4:
143 		sz = ArenaSize4;
144 		arena->clumpmagic = _ClumpMagic;
145 		break;
146 	case ArenaVersion5:
147 		sz = ArenaSize5;
148 		break;
149 	default:
150 		seterr(ECorrupt, "arena has bad version number %d", arena->version);
151 		return -1;
152 	}
153 	/*
154 	 * Additional fields for the memstats version of the stats.
155 	 * Diskstats reflects what is committed to the index.
156 	 * Memstats reflects what is in the arena.  Originally intended
157 	 * this to be a version 5 extension, but might as well use for
158 	 * all the existing version 4 arenas too.
159 	 *
160 	 * To maintain backwards compatibility with existing venti
161 	 * installations using the older format, we define that if 
162 	 * memstats == diskstats, then the extension fields are not
163 	 * included (see packarena below).  That is, only partially
164 	 * indexed arenas have these fields.  Fully indexed arenas
165 	 * (in particular, sealed arenas) do not.
166 	 */
167 	if(U8GET(p) == 1){
168 		sz += ArenaSize5a-ArenaSize5;
169 		p += U8Size;
170 		arena->memstats.clumps = U32GET(p);
171 		p += U32Size;
172 		arena->memstats.cclumps = U32GET(p);
173 		p += U32Size;
174 		arena->memstats.used = U64GET(p);
175 		p += U64Size;
176 		arena->memstats.uncsize = U64GET(p);
177 		p += U64Size;
178 		arena->memstats.sealed = U8GET(p);
179 		p += U8Size;
180 		
181 		/*
182 		 * 2008/4/2
183 		 * Packarena (below) used to have a bug in which it would
184 		 * not zero out any existing extension fields when writing
185 		 * the arena metadata.  This would manifest itself as arenas
186 		 * with arena->diskstats.sealed == 1 but arena->memstats.sealed == 0
187 		 * after a server restart.  Because arena->memstats.sealed wouldn't
188 		 * be set, the server might try to fit another block into the arena
189 		 * (and succeed), violating the append-only structure of the log
190 		 * and invalidating any already-computed seal on the arena.
191 		 *
192 		 * It might end up that other fields in arena->memstats end up
193 		 * behind arena->diskstats too, but that would be considerably
194 		 * more rare, and the bug is fixed now.  The case we need to
195 		 * handle is just the sealed mismatch.
196 		 *
197 		 * If we encounter such a bogus arena, fix the sealed field.
198 		 */
199 		if(arena->diskstats.sealed)
200 			arena->memstats.sealed = 1;
201 	}else
202 		arena->memstats = arena->diskstats;
203 	if(buf + sz != p)
204 		sysfatal("unpackarena unpacked wrong amount");
205 
206 	return 0;
207 }
208 
209 int
210 packarena(Arena *arena, u8int *buf)
211 {
212 	return _packarena(arena, buf, 0);
213 }
214 
215 int
216 _packarena(Arena *arena, u8int *buf, int forceext)
217 {
218 	int sz;
219 	u8int *p;
220 	u32int t32;
221 
222 	switch(arena->version){
223 	case ArenaVersion4:
224 		sz = ArenaSize4;
225 		if(arena->clumpmagic != _ClumpMagic)
226 			fprint(2, "warning: writing old arena tail loses clump magic 0x%lux != 0x%lux\n",
227 				(ulong)arena->clumpmagic, (ulong)_ClumpMagic);
228 		break;
229 	case ArenaVersion5:
230 		sz = ArenaSize5;
231 		break;
232 	default:
233 		sysfatal("packarena unknown version %d", arena->version);
234 		return -1;
235 	}
236 
237 	p = buf;
238 
239 	U32PUT(p, ArenaMagic);
240 	p += U32Size;
241 	U32PUT(p, arena->version);
242 	p += U32Size;
243 	namecp((char*)p, arena->name);
244 	p += ANameSize;
245 	U32PUT(p, arena->diskstats.clumps);
246 	p += U32Size;
247 	U32PUT(p, arena->diskstats.cclumps);
248 	p += U32Size;
249 	U32PUT(p, arena->ctime);
250 	p += U32Size;
251 	U32PUT(p, arena->wtime);
252 	p += U32Size;
253 	if(arena->version == ArenaVersion5){
254 		U32PUT(p, arena->clumpmagic);
255 		p += U32Size;
256 	}
257 	U64PUT(p, arena->diskstats.used, t32);
258 	p += U64Size;
259 	U64PUT(p, arena->diskstats.uncsize, t32);
260 	p += U64Size;
261 	U8PUT(p, arena->diskstats.sealed);
262 	p += U8Size;
263 	
264 	/*
265 	 * Extension fields; see above.
266 	 */
267 	if(forceext
268 	|| arena->memstats.clumps != arena->diskstats.clumps
269 	|| arena->memstats.cclumps != arena->diskstats.cclumps
270 	|| arena->memstats.used != arena->diskstats.used
271 	|| arena->memstats.uncsize != arena->diskstats.uncsize
272 	|| arena->memstats.sealed != arena->diskstats.sealed){
273 		sz += ArenaSize5a - ArenaSize5;
274 		U8PUT(p, 1);
275 		p += U8Size;
276 		U32PUT(p, arena->memstats.clumps);
277 		p += U32Size;
278 		U32PUT(p, arena->memstats.cclumps);
279 		p += U32Size;
280 		U64PUT(p, arena->memstats.used, t32);	
281 		p += U64Size;
282 		U64PUT(p, arena->memstats.uncsize, t32);
283 		p += U64Size;
284 		U8PUT(p, arena->memstats.sealed);
285 		p += U8Size;
286 	}else{
287 		/* Clear any extension fields already on disk. */
288 		memset(p, 0, ArenaSize5a - ArenaSize5);
289 		p += ArenaSize5a - ArenaSize5;
290 		sz += ArenaSize5a - ArenaSize5;
291 	}
292 
293 	if(buf + sz != p)
294 		sysfatal("packarena packed wrong amount");
295 
296 	return 0;
297 }
298 
299 int
300 unpackarenahead(ArenaHead *head, u8int *buf)
301 {
302 	u8int *p;
303 	u32int m;
304 	int sz;
305 	char fbuf[20];
306 
307 	p = buf;
308 
309 	m = U32GET(p);
310 	if(m != ArenaHeadMagic){
311 		seterr(ECorrupt, "arena has wrong magic number: %s expected ArenaHeadMagic (%#lux)", fmtmagic(fbuf, m), ArenaHeadMagic);
312 		return -1;
313 	}
314 
315 	p += U32Size;
316 	head->version = U32GET(p);
317 	p += U32Size;
318 	namecp(head->name, (char*)p);
319 	p += ANameSize;
320 	head->blocksize = U32GET(p);
321 	p += U32Size;
322 	head->size = U64GET(p);
323 	p += U64Size;
324 	if(head->version == ArenaVersion5){
325 		head->clumpmagic = U32GET(p);
326 		p += U32Size;
327 	}
328 
329 	switch(head->version){
330 	case ArenaVersion4:
331 		sz = ArenaHeadSize4;
332 		head->clumpmagic = _ClumpMagic;
333 		break;
334 	case ArenaVersion5:
335 		sz = ArenaHeadSize5;
336 		break;
337 	default:
338 		seterr(ECorrupt, "arena head has unexpected version %d", head->version);
339 		return -1;
340 	}
341 
342 	if(buf + sz != p)
343 		sysfatal("unpackarenahead unpacked wrong amount");
344 
345 	return 0;
346 }
347 
348 int
349 packarenahead(ArenaHead *head, u8int *buf)
350 {
351 	u8int *p;
352 	int sz;
353 	u32int t32;
354 
355 	switch(head->version){
356 	case ArenaVersion4:
357 		sz = ArenaHeadSize4;
358 		if(head->clumpmagic != _ClumpMagic)
359 			fprint(2, "warning: writing old arena header loses clump magic 0x%lux != 0x%lux\n",
360 				(ulong)head->clumpmagic, (ulong)_ClumpMagic);
361 		break;
362 	case ArenaVersion5:
363 		sz = ArenaHeadSize5;
364 		break;
365 	default:
366 		sysfatal("packarenahead unknown version %d", head->version);
367 		return -1;
368 	}
369 
370 	p = buf;
371 
372 	U32PUT(p, ArenaHeadMagic);
373 	p += U32Size;
374 	U32PUT(p, head->version);
375 	p += U32Size;
376 	namecp((char*)p, head->name);
377 	p += ANameSize;
378 	U32PUT(p, head->blocksize);
379 	p += U32Size;
380 	U64PUT(p, head->size, t32);
381 	p += U64Size;
382 	if(head->version == ArenaVersion5){
383 		U32PUT(p, head->clumpmagic);
384 		p += U32Size;
385 	}
386 	if(buf + sz != p)
387 		sysfatal("packarenahead packed wrong amount");
388 
389 	return 0;
390 }
391 
392 static int
393 checkclump(Clump *w)
394 {
395 	if(w->encoding == ClumpENone){
396 		if(w->info.size != w->info.uncsize){
397 			seterr(ECorrupt, "uncompressed wad size mismatch");
398 			return -1;
399 		}
400 	}else if(w->encoding == ClumpECompress){
401 		if(w->info.size >= w->info.uncsize){
402 			seterr(ECorrupt, "compressed lump has inconsistent block sizes %d %d", w->info.size, w->info.uncsize);
403 			return -1;
404 		}
405 	}else{
406 		seterr(ECorrupt, "clump has illegal encoding");
407 		return -1;
408 	}
409 
410 	return 0;
411 }
412 
413 int
414 unpackclump(Clump *c, u8int *buf, u32int cmagic)
415 {
416 	u8int *p;
417 	u32int magic;
418 
419 	p = buf;
420 	magic = U32GET(p);
421 	if(magic != cmagic){
422 		seterr(ECorrupt, "clump has bad magic number=%#8.8ux != %#8.8ux", magic, cmagic);
423 		return -1;
424 	}
425 	p += U32Size;
426 
427 	c->info.type = vtfromdisktype(U8GET(p));
428 	p += U8Size;
429 	c->info.size = U16GET(p);
430 	p += U16Size;
431 	c->info.uncsize = U16GET(p);
432 	p += U16Size;
433 	scorecp(c->info.score, p);
434 	p += VtScoreSize;
435 
436 	c->encoding = U8GET(p);
437 	p += U8Size;
438 	c->creator = U32GET(p);
439 	p += U32Size;
440 	c->time = U32GET(p);
441 	p += U32Size;
442 
443 	if(buf + ClumpSize != p)
444 		sysfatal("unpackclump unpacked wrong amount");
445 
446 	return checkclump(c);
447 }
448 
449 int
450 packclump(Clump *c, u8int *buf, u32int magic)
451 {
452 	u8int *p;
453 
454 	p = buf;
455 	U32PUT(p, magic);
456 	p += U32Size;
457 
458 	U8PUT(p, vttodisktype(c->info.type));
459 	p += U8Size;
460 	U16PUT(p, c->info.size);
461 	p += U16Size;
462 	U16PUT(p, c->info.uncsize);
463 	p += U16Size;
464 	scorecp(p, c->info.score);
465 	p += VtScoreSize;
466 
467 	U8PUT(p, c->encoding);
468 	p += U8Size;
469 	U32PUT(p, c->creator);
470 	p += U32Size;
471 	U32PUT(p, c->time);
472 	p += U32Size;
473 
474 	if(buf + ClumpSize != p)
475 		sysfatal("packclump packed wrong amount");
476 
477 	return checkclump(c);
478 }
479 
480 void
481 unpackclumpinfo(ClumpInfo *ci, u8int *buf)
482 {
483 	u8int *p;
484 
485 	p = buf;
486 	ci->type = vtfromdisktype(U8GET(p));
487 	p += U8Size;
488 	ci->size = U16GET(p);
489 	p += U16Size;
490 	ci->uncsize = U16GET(p);
491 	p += U16Size;
492 	scorecp(ci->score, p);
493 	p += VtScoreSize;
494 
495 	if(buf + ClumpInfoSize != p)
496 		sysfatal("unpackclumpinfo unpacked wrong amount");
497 }
498 
499 void
500 packclumpinfo(ClumpInfo *ci, u8int *buf)
501 {
502 	u8int *p;
503 
504 	p = buf;
505 	U8PUT(p, vttodisktype(ci->type));
506 	p += U8Size;
507 	U16PUT(p, ci->size);
508 	p += U16Size;
509 	U16PUT(p, ci->uncsize);
510 	p += U16Size;
511 	scorecp(p, ci->score);
512 	p += VtScoreSize;
513 
514 	if(buf + ClumpInfoSize != p)
515 		sysfatal("packclumpinfo packed wrong amount");
516 }
517 
518 int
519 unpackisect(ISect *is, u8int *buf)
520 {
521 	u8int *p;
522 	u32int m;
523 	char fbuf[20];
524 
525 	p = buf;
526 
527 
528 	m = U32GET(p);
529 	if(m != ISectMagic){
530 		seterr(ECorrupt, "index section has wrong magic number: %s expected ISectMagic (%#lux)",
531 			fmtmagic(fbuf, m), ISectMagic);
532 		return -1;
533 	}
534 	p += U32Size;
535 	is->version = U32GET(p);
536 	p += U32Size;
537 	namecp(is->name, (char*)p);
538 	p += ANameSize;
539 	namecp(is->index, (char*)p);
540 	p += ANameSize;
541 	is->blocksize = U32GET(p);
542 	p += U32Size;
543 	is->blockbase = U32GET(p);
544 	p += U32Size;
545 	is->blocks = U32GET(p);
546 	p += U32Size;
547 	is->start = U32GET(p);
548 	p += U32Size;
549 	is->stop = U32GET(p);
550 	p += U32Size;
551 	if(buf + ISectSize1 != p)
552 		sysfatal("unpackisect unpacked wrong amount");
553 	is->bucketmagic = 0;
554 	if(is->version == ISectVersion2){
555 		is->bucketmagic = U32GET(p);
556 		p += U32Size;
557 		if(buf + ISectSize2 != p)
558 			sysfatal("unpackisect unpacked wrong amount");
559 	}
560 
561 	return 0;
562 }
563 
564 int
565 packisect(ISect *is, u8int *buf)
566 {
567 	u8int *p;
568 
569 	p = buf;
570 
571 	U32PUT(p, ISectMagic);
572 	p += U32Size;
573 	U32PUT(p, is->version);
574 	p += U32Size;
575 	namecp((char*)p, is->name);
576 	p += ANameSize;
577 	namecp((char*)p, is->index);
578 	p += ANameSize;
579 	U32PUT(p, is->blocksize);
580 	p += U32Size;
581 	U32PUT(p, is->blockbase);
582 	p += U32Size;
583 	U32PUT(p, is->blocks);
584 	p += U32Size;
585 	U32PUT(p, is->start);
586 	p += U32Size;
587 	U32PUT(p, is->stop);
588 	p += U32Size;
589 	if(buf + ISectSize1 != p)
590 		sysfatal("packisect packed wrong amount");
591 	if(is->version == ISectVersion2){
592 		U32PUT(p, is->bucketmagic);
593 		p += U32Size;
594 		if(buf + ISectSize2 != p)
595 			sysfatal("packisect packed wrong amount");
596 	}
597 
598 	return 0;
599 }
600 
601 void
602 unpackientry(IEntry *ie, u8int *buf)
603 {
604 	u8int *p;
605 
606 	p = buf;
607 
608 	scorecp(ie->score, p);
609 	p += VtScoreSize;
610 	/* ie->wtime = U32GET(p); */
611 	p += U32Size;
612 	/* ie->train = U16GET(p); */
613 	p += U16Size;
614 	if(p - buf != IEntryAddrOff)
615 		sysfatal("unpackentry bad IEntryAddrOff amount");
616 	ie->ia.addr = U64GET(p);
617 if(ie->ia.addr>>56) print("%.8H => %llux\n", p, ie->ia.addr);
618 	p += U64Size;
619 	ie->ia.size = U16GET(p);
620 	p += U16Size;
621 	if(p - buf != IEntryTypeOff)
622 		sysfatal("unpackientry bad IEntryTypeOff amount");
623 	ie->ia.type = vtfromdisktype(U8GET(p));
624 	p += U8Size;
625 	ie->ia.blocks = U8GET(p);
626 	p += U8Size;
627 
628 	if(p - buf != IEntrySize)
629 		sysfatal("unpackientry unpacked wrong amount");
630 }
631 
632 void
633 packientry(IEntry *ie, u8int *buf)
634 {
635 	u32int t32;
636 	u8int *p;
637 
638 	p = buf;
639 
640 	scorecp(p, ie->score);
641 	p += VtScoreSize;
642 	U32PUT(p, 0); /* wtime */
643 	p += U32Size;
644 	U16PUT(p, 0); /* train */
645 	p += U16Size;
646 	U64PUT(p, ie->ia.addr, t32);
647 	p += U64Size;
648 	U16PUT(p, ie->ia.size);
649 	p += U16Size;
650 	U8PUT(p, vttodisktype(ie->ia.type));
651 	p += U8Size;
652 	U8PUT(p, ie->ia.blocks);
653 	p += U8Size;
654 
655 	if(p - buf != IEntrySize)
656 		sysfatal("packientry packed wrong amount");
657 }
658 
659 void
660 unpackibucket(IBucket *b, u8int *buf, u32int magic)
661 {
662 	b->n = U16GET(buf);
663 	b->data = buf + IBucketSize;
664 	if(magic && magic != U32GET(buf+U16Size))
665 		b->n = 0;
666 }		
667 
668 void
669 packibucket(IBucket *b, u8int *buf, u32int magic)
670 {
671 	U16PUT(buf, b->n);
672 	U32PUT(buf+U16Size, magic);
673 }
674 
675 void
676 packbloomhead(Bloom *b, u8int *buf)
677 {
678 	u8int *p;
679 
680 	p = buf;
681 	U32PUT(p, BloomMagic);
682 	U32PUT(p+4, BloomVersion);
683 	U32PUT(p+8, b->nhash);
684 	U32PUT(p+12, b->size);
685 }
686 
687 int
688 unpackbloomhead(Bloom *b, u8int *buf)
689 {
690 	u8int *p;
691 	u32int m;
692 	char fbuf[20];
693 
694 	p = buf;
695 
696 	m = U32GET(p);
697 	if(m != BloomMagic){
698 		seterr(ECorrupt, "bloom filter has wrong magic number: %s expected BloomMagic (%#lux)", fmtmagic(fbuf, m), (ulong)BloomMagic);
699 		return -1;
700 	}
701 	p += U32Size;
702 	
703 	m = U32GET(p);
704 	if(m != BloomVersion){
705 		seterr(ECorrupt, "bloom filter has wrong version %ud expected %ud", (uint)m, (uint)BloomVersion);
706 		return -1;
707 	}
708 	p += U32Size;
709 
710 	b->nhash = U32GET(p);
711 	p += U32Size;
712 
713 	b->size = U32GET(p);
714 	p += U32Size;
715 	if(b->size < BloomHeadSize || b->size > MaxBloomSize || (b->size&(b->size-1))){
716 		seterr(ECorrupt, "bloom filter has invalid size %#lux", b->size);
717 		return -1;
718 	}
719 
720 	if(buf + BloomHeadSize != p)
721 		sysfatal("unpackarena unpacked wrong amount");
722 
723 	return 0;
724 }