1 a0d146ed 2005-07-12 devnull #include "stdinc.h"
  2 a0d146ed 2005-07-12 devnull #include "dat.h"
  3 a0d146ed 2005-07-12 devnull #include "fns.h"
  4 a0d146ed 2005-07-12 devnull #include "whack.h"
  5 a0d146ed 2005-07-12 devnull 
  6 a0d146ed 2005-07-12 devnull /*
  7 a0d146ed 2005-07-12 devnull  * Write a lump to disk.  Updates ia with an index address
  8 a0d146ed 2005-07-12 devnull  * for the newly-written lump.  Upon return, the lump will
  9 a0d146ed 2005-07-12 devnull  * have been placed in the disk cache but will likely not be on disk yet.
 10 a0d146ed 2005-07-12 devnull  */
 11 a0d146ed 2005-07-12 devnull int
 12 a0d146ed 2005-07-12 devnull storeclump(Index *ix, ZBlock *zb, u8int *sc, int type, u32int creator, IAddr *ia)
 13 a0d146ed 2005-07-12 devnull {
 14 a0d146ed 2005-07-12 devnull 	ZBlock *cb;
 15 a0d146ed 2005-07-12 devnull 	Clump cl;
 16 a0d146ed 2005-07-12 devnull 	u64int a;
 17 a0d146ed 2005-07-12 devnull 	u8int bh[VtScoreSize];
 18 a0d146ed 2005-07-12 devnull 	int size, dsize;
 19 a0d146ed 2005-07-12 devnull 
 20 a0d146ed 2005-07-12 devnull 	trace(TraceLump, "storeclump enter", sc, type);
 21 a0d146ed 2005-07-12 devnull 	size = zb->len;
 22 a0d146ed 2005-07-12 devnull 	if(size > VtMaxLumpSize){
 23 a0d146ed 2005-07-12 devnull 		seterr(EStrange, "lump too large");
 24 a0d146ed 2005-07-12 devnull 		return -1;
 25 a0d146ed 2005-07-12 devnull 	}
 26 a0d146ed 2005-07-12 devnull 	if(vttypevalid(type) < 0){
 27 a0d146ed 2005-07-12 devnull 		seterr(EStrange, "invalid lump type");
 28 a0d146ed 2005-07-12 devnull 		return -1;
 29 a0d146ed 2005-07-12 devnull 	}
 30 a0d146ed 2005-07-12 devnull 
 31 a0d146ed 2005-07-12 devnull 	if(0){
 32 a0d146ed 2005-07-12 devnull 		scoremem(bh, zb->data, size);
 33 a0d146ed 2005-07-12 devnull 		if(scorecmp(sc, bh) != 0){
 34 a0d146ed 2005-07-12 devnull 			seterr(ECorrupt, "storing clump: corrupted; expected=%V got=%V, size=%d", sc, bh, size);
 35 a0d146ed 2005-07-12 devnull 			return -1;
 36 a0d146ed 2005-07-12 devnull 		}
 37 a0d146ed 2005-07-12 devnull 	}
 38 a0d146ed 2005-07-12 devnull 
 39 a0d146ed 2005-07-12 devnull 	cb = alloczblock(size + ClumpSize + U32Size, 0, 0);
 40 a0d146ed 2005-07-12 devnull 	if(cb == nil)
 41 a0d146ed 2005-07-12 devnull 		return -1;
 42 a0d146ed 2005-07-12 devnull 
 43 a0d146ed 2005-07-12 devnull 	cl.info.type = type;
 44 a0d146ed 2005-07-12 devnull 	cl.info.uncsize = size;
 45 a0d146ed 2005-07-12 devnull 	cl.creator = creator;
 46 a0d146ed 2005-07-12 devnull 	cl.time = now();
 47 a0d146ed 2005-07-12 devnull 	scorecp(cl.info.score, sc);
 48 a0d146ed 2005-07-12 devnull 
 49 a0d146ed 2005-07-12 devnull 	trace(TraceLump, "storeclump whackblock");
 50 a0d146ed 2005-07-12 devnull 	dsize = whackblock(&cb->data[ClumpSize], zb->data, size);
 51 a0d146ed 2005-07-12 devnull 	if(dsize > 0 && dsize < size){
 52 a0d146ed 2005-07-12 devnull 		cl.encoding = ClumpECompress;
 53 a0d146ed 2005-07-12 devnull 	}else{
 54 a0d146ed 2005-07-12 devnull 		if(dsize > size){
 55 a0d146ed 2005-07-12 devnull 			fprint(2, "whack error: dsize=%d size=%d\n", dsize, size);
 56 a0d146ed 2005-07-12 devnull 			abort();
 57 a0d146ed 2005-07-12 devnull 		}
 58 a0d146ed 2005-07-12 devnull 		cl.encoding = ClumpENone;
 59 a0d146ed 2005-07-12 devnull 		dsize = size;
 60 a0d146ed 2005-07-12 devnull 		memmove(&cb->data[ClumpSize], zb->data, size);
 61 a0d146ed 2005-07-12 devnull 	}
 62 a0d146ed 2005-07-12 devnull 	memset(cb->data+ClumpSize+dsize, 0, 4);
 63 a0d146ed 2005-07-12 devnull 	cl.info.size = dsize;
 64 a0d146ed 2005-07-12 devnull 
 65 a0d146ed 2005-07-12 devnull 	ia->addr = 0;
 66 a0d146ed 2005-07-12 devnull 	ia->type = type;
 67 a0d146ed 2005-07-12 devnull 	ia->size = size;
 68 a0d146ed 2005-07-12 devnull 	ia->blocks = (dsize + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog;
 69 a0d146ed 2005-07-12 devnull 
 70 a0d146ed 2005-07-12 devnull 	a = writeiclump(ix, &cl, cb->data, &ia->addr);
 71 a0d146ed 2005-07-12 devnull 
 72 a0d146ed 2005-07-12 devnull 	trace(TraceLump, "storeclump exit %lld", a);
 73 a0d146ed 2005-07-12 devnull 
 74 a0d146ed 2005-07-12 devnull 	freezblock(cb);
 75 a0d146ed 2005-07-12 devnull 	if(a == TWID64)
 76 a0d146ed 2005-07-12 devnull 		return -1;
 77 a0d146ed 2005-07-12 devnull 
 78 a0d146ed 2005-07-12 devnull /*
 79 a0d146ed 2005-07-12 devnull 	qlock(&stats.lock);
 80 a0d146ed 2005-07-12 devnull 	stats.clumpwrites++;
 81 a0d146ed 2005-07-12 devnull 	stats.clumpbwrites += size;
 82 a0d146ed 2005-07-12 devnull 	stats.clumpbcomp += dsize;
 83 a0d146ed 2005-07-12 devnull 	qunlock(&stats.lock);
 84 a0d146ed 2005-07-12 devnull */
 85 a0d146ed 2005-07-12 devnull 
 86 a0d146ed 2005-07-12 devnull 	return 0;
 87 a0d146ed 2005-07-12 devnull }
 88 a0d146ed 2005-07-12 devnull 
 89 a0d146ed 2005-07-12 devnull u32int
 90 a0d146ed 2005-07-12 devnull clumpmagic(Arena *arena, u64int aa)
 91 a0d146ed 2005-07-12 devnull {
 92 a0d146ed 2005-07-12 devnull 	u8int buf[U32Size];
 93 a0d146ed 2005-07-12 devnull 
 94 a0d146ed 2005-07-12 devnull 	if(readarena(arena, aa, buf, U32Size) < 0)
 95 a0d146ed 2005-07-12 devnull 		return TWID32;
 96 a0d146ed 2005-07-12 devnull 	return unpackmagic(buf);
 97 a0d146ed 2005-07-12 devnull }
 98 a0d146ed 2005-07-12 devnull 
 99 a0d146ed 2005-07-12 devnull /*
100 a0d146ed 2005-07-12 devnull  * fetch a block based at addr.
101 a0d146ed 2005-07-12 devnull  * score is filled in with the block's score.
102 a0d146ed 2005-07-12 devnull  * blocks is roughly the length of the clump on disk;
103 a0d146ed 2005-07-12 devnull  * if zero, the length is unknown.
104 a0d146ed 2005-07-12 devnull  */
105 a0d146ed 2005-07-12 devnull ZBlock*
106 a0d146ed 2005-07-12 devnull loadclump(Arena *arena, u64int aa, int blocks, Clump *cl, u8int *score, int verify)
107 a0d146ed 2005-07-12 devnull {
108 a0d146ed 2005-07-12 devnull 	Unwhack uw;
109 a0d146ed 2005-07-12 devnull 	ZBlock *zb, *cb;
110 a0d146ed 2005-07-12 devnull 	u8int bh[VtScoreSize], *buf;
111 a0d146ed 2005-07-12 devnull 	u32int n;
112 a0d146ed 2005-07-12 devnull 	int nunc;
113 a0d146ed 2005-07-12 devnull 
114 a0d146ed 2005-07-12 devnull /*
115 a0d146ed 2005-07-12 devnull 	qlock(&stats.lock);
116 a0d146ed 2005-07-12 devnull 	stats.clumpreads++;
117 a0d146ed 2005-07-12 devnull 	qunlock(&stats.lock);
118 a0d146ed 2005-07-12 devnull */
119 a0d146ed 2005-07-12 devnull 
120 a0d146ed 2005-07-12 devnull 	if(blocks <= 0)
121 a0d146ed 2005-07-12 devnull 		blocks = 1;
122 a0d146ed 2005-07-12 devnull 
123 a0d146ed 2005-07-12 devnull 	trace(TraceLump, "loadclump enter");
124 a0d146ed 2005-07-12 devnull 
125 a0d146ed 2005-07-12 devnull 	cb = alloczblock(blocks << ABlockLog, 0, 0);
126 a0d146ed 2005-07-12 devnull 	if(cb == nil)
127 a0d146ed 2005-07-12 devnull 		return nil;
128 a0d146ed 2005-07-12 devnull 	n = readarena(arena, aa, cb->data, blocks << ABlockLog);
129 a0d146ed 2005-07-12 devnull 	if(n < ClumpSize){
130 a0d146ed 2005-07-12 devnull 		if(n != 0)
131 a0d146ed 2005-07-12 devnull 			seterr(ECorrupt, "loadclump read less than a header");
132 a0d146ed 2005-07-12 devnull 		freezblock(cb);
133 a0d146ed 2005-07-12 devnull 		return nil;
134 a0d146ed 2005-07-12 devnull 	}
135 a0d146ed 2005-07-12 devnull 	trace(TraceLump, "loadclump unpack");
136 a0d146ed 2005-07-12 devnull 	if(unpackclump(cl, cb->data, arena->clumpmagic) < 0){
137 a0d146ed 2005-07-12 devnull 		seterr(ECorrupt, "loadclump %s %llud: %r", arena->name, aa);
138 a0d146ed 2005-07-12 devnull 		freezblock(cb);
139 a0d146ed 2005-07-12 devnull 		return nil;
140 a0d146ed 2005-07-12 devnull 	}
141 a0d146ed 2005-07-12 devnull 	n -= ClumpSize;
142 a0d146ed 2005-07-12 devnull 	if(n < cl->info.size){
143 a0d146ed 2005-07-12 devnull 		freezblock(cb);
144 a0d146ed 2005-07-12 devnull 		n = cl->info.size;
145 a0d146ed 2005-07-12 devnull 		cb = alloczblock(n, 0, 0);
146 a0d146ed 2005-07-12 devnull 		if(cb == nil)
147 a0d146ed 2005-07-12 devnull 			return nil;
148 a0d146ed 2005-07-12 devnull 		if(readarena(arena, aa + ClumpSize, cb->data, n) != n){
149 a0d146ed 2005-07-12 devnull 			seterr(ECorrupt, "loadclump read too little data");
150 a0d146ed 2005-07-12 devnull 			freezblock(cb);
151 a0d146ed 2005-07-12 devnull 			return nil;
152 a0d146ed 2005-07-12 devnull 		}
153 a0d146ed 2005-07-12 devnull 		buf = cb->data;
154 a0d146ed 2005-07-12 devnull 	}else
155 a0d146ed 2005-07-12 devnull 		buf = cb->data + ClumpSize;
156 a0d146ed 2005-07-12 devnull 
157 a0d146ed 2005-07-12 devnull 	scorecp(score, cl->info.score);
158 a0d146ed 2005-07-12 devnull 
159 a0d146ed 2005-07-12 devnull 	zb = alloczblock(cl->info.uncsize, 0, 0);
160 a0d146ed 2005-07-12 devnull 	if(zb == nil){
161 a0d146ed 2005-07-12 devnull 		freezblock(cb);
162 a0d146ed 2005-07-12 devnull 		return nil;
163 a0d146ed 2005-07-12 devnull 	}
164 a0d146ed 2005-07-12 devnull 	switch(cl->encoding){
165 a0d146ed 2005-07-12 devnull 	case ClumpECompress:
166 a0d146ed 2005-07-12 devnull 		trace(TraceLump, "loadclump decompress");
167 a0d146ed 2005-07-12 devnull 		unwhackinit(&uw);
168 a0d146ed 2005-07-12 devnull 		nunc = unwhack(&uw, zb->data, cl->info.uncsize, buf, cl->info.size);
169 a0d146ed 2005-07-12 devnull 		if(nunc != cl->info.uncsize){
170 a0d146ed 2005-07-12 devnull 			if(nunc < 0)
171 a0d146ed 2005-07-12 devnull 				seterr(ECorrupt, "decompression of %llud failed: %s", aa, uw.err);
172 a0d146ed 2005-07-12 devnull 			else
173 a0d146ed 2005-07-12 devnull 				seterr(ECorrupt, "decompression of %llud gave partial block: %d/%d\n", aa, nunc, cl->info.uncsize);
174 a0d146ed 2005-07-12 devnull 			freezblock(cb);
175 a0d146ed 2005-07-12 devnull 			freezblock(zb);
176 a0d146ed 2005-07-12 devnull 			return nil;
177 a0d146ed 2005-07-12 devnull 		}
178 a0d146ed 2005-07-12 devnull 		break;
179 a0d146ed 2005-07-12 devnull 	case ClumpENone:
180 a0d146ed 2005-07-12 devnull 		if(cl->info.size != cl->info.uncsize){
181 a0d146ed 2005-07-12 devnull 			seterr(ECorrupt, "loading clump: bad uncompressed size for uncompressed block %llud", aa);
182 a0d146ed 2005-07-12 devnull 			freezblock(cb);
183 a0d146ed 2005-07-12 devnull 			freezblock(zb);
184 a0d146ed 2005-07-12 devnull 			return nil;
185 a0d146ed 2005-07-12 devnull 		}
186 a0d146ed 2005-07-12 devnull 		scoremem(bh, buf, cl->info.uncsize);
187 a0d146ed 2005-07-12 devnull 		if(scorecmp(cl->info.score, bh) != 0)
188 a0d146ed 2005-07-12 devnull 			seterr(ECorrupt, "pre-copy sha1 wrong at %s %llud: expected=%V got=%V", arena->name, aa, cl->info.score, bh);
189 a0d146ed 2005-07-12 devnull 		memmove(zb->data, buf, cl->info.uncsize);
190 a0d146ed 2005-07-12 devnull 		break;
191 a0d146ed 2005-07-12 devnull 	default:
192 a0d146ed 2005-07-12 devnull 		seterr(ECorrupt, "unknown encoding in loadlump %llud", aa);
193 a0d146ed 2005-07-12 devnull 		freezblock(cb);
194 a0d146ed 2005-07-12 devnull 		freezblock(zb);
195 a0d146ed 2005-07-12 devnull 		return nil;
196 a0d146ed 2005-07-12 devnull 	}
197 a0d146ed 2005-07-12 devnull 	freezblock(cb);
198 a0d146ed 2005-07-12 devnull 
199 a0d146ed 2005-07-12 devnull 	if(verify){
200 a0d146ed 2005-07-12 devnull 		trace(TraceLump, "loadclump verify");
201 a0d146ed 2005-07-12 devnull 		scoremem(bh, zb->data, cl->info.uncsize);
202 a0d146ed 2005-07-12 devnull 		if(scorecmp(cl->info.score, bh) != 0){
203 a0d146ed 2005-07-12 devnull 			seterr(ECorrupt, "loading clump: corrupted at %s %llud; expected=%V got=%V", arena->name, aa, cl->info.score, bh);
204 a0d146ed 2005-07-12 devnull 			freezblock(zb);
205 a0d146ed 2005-07-12 devnull 			return nil;
206 a0d146ed 2005-07-12 devnull 		}
207 a0d146ed 2005-07-12 devnull 		if(vttypevalid(cl->info.type) < 0){
208 a0d146ed 2005-07-12 devnull 			seterr(ECorrupt, "loading lump at %s %llud: invalid lump type %d", arena->name, aa, cl->info.type);
209 a0d146ed 2005-07-12 devnull 			freezblock(zb);
210 a0d146ed 2005-07-12 devnull 			return nil;
211 a0d146ed 2005-07-12 devnull 		}
212 a0d146ed 2005-07-12 devnull 	}
213 a0d146ed 2005-07-12 devnull 
214 a0d146ed 2005-07-12 devnull 	trace(TraceLump, "loadclump exit");
215 a0d146ed 2005-07-12 devnull /*
216 a0d146ed 2005-07-12 devnull 	qlock(&stats.lock);
217 a0d146ed 2005-07-12 devnull 	stats.clumpbreads += cl->info.size;
218 a0d146ed 2005-07-12 devnull 	stats.clumpbuncomp += cl->info.uncsize;
219 a0d146ed 2005-07-12 devnull 	qunlock(&stats.lock);
220 a0d146ed 2005-07-12 devnull */
221 a0d146ed 2005-07-12 devnull 	return zb;
222 a0d146ed 2005-07-12 devnull }