1 a0d146ed 2005-07-12 devnull #include "stdinc.h"
  2 a0d146ed 2005-07-12 devnull #include "dat.h"
  3 a0d146ed 2005-07-12 devnull #include "fns.h"
  4 a0d146ed 2005-07-12 devnull 
  5 a0d146ed 2005-07-12 devnull typedef struct ASum ASum;
  6 a0d146ed 2005-07-12 devnull 
  7 a0d146ed 2005-07-12 devnull struct ASum
  8 a0d146ed 2005-07-12 devnull {
  9 a0d146ed 2005-07-12 devnull 	Arena	*arena;
 10 a0d146ed 2005-07-12 devnull 	ASum	*next;
 11 a0d146ed 2005-07-12 devnull };
 12 a0d146ed 2005-07-12 devnull 
 13 a0d146ed 2005-07-12 devnull static void	sealarena(Arena *arena);
 14 a0d146ed 2005-07-12 devnull static int	okarena(Arena *arena);
 15 a0d146ed 2005-07-12 devnull static int	loadarena(Arena *arena);
 16 a0d146ed 2005-07-12 devnull static CIBlock	*getcib(Arena *arena, int clump, int writing, CIBlock *rock);
 17 a0d146ed 2005-07-12 devnull static void	putcib(Arena *arena, CIBlock *cib);
 18 a0d146ed 2005-07-12 devnull static void	sumproc(void *);
 19 7a400ee9 2007-09-25 rsc static void loadcig(Arena *arena);
 20 a0d146ed 2005-07-12 devnull 
 21 a0d146ed 2005-07-12 devnull static QLock	sumlock;
 22 a0d146ed 2005-07-12 devnull static Rendez	sumwait;
 23 a0d146ed 2005-07-12 devnull static ASum	*sumq;
 24 28b49df3 2006-07-18 devnull static ASum	*sumqtail;
 25 a0d146ed 2005-07-12 devnull static uchar zero[8192];
 26 a0d146ed 2005-07-12 devnull 
 27 a0d146ed 2005-07-12 devnull int	arenasumsleeptime;
 28 a0d146ed 2005-07-12 devnull 
 29 a0d146ed 2005-07-12 devnull int
 30 a0d146ed 2005-07-12 devnull initarenasum(void)
 31 a0d146ed 2005-07-12 devnull {
 32 ef609927 2007-05-03 devnull 	needzeroscore();  /* OS X */
 33 ef609927 2007-05-03 devnull 
 34 01c1b763 2008-07-04 rsc 	qlock(&sumlock);
 35 a0d146ed 2005-07-12 devnull 	sumwait.l = &sumlock;
 36 01c1b763 2008-07-04 rsc 	qunlock(&sumlock);
 37 a0d146ed 2005-07-12 devnull 
 38 a0d146ed 2005-07-12 devnull 	if(vtproc(sumproc, nil) < 0){
 39 a0d146ed 2005-07-12 devnull 		seterr(EOk, "can't start arena checksum slave: %r");
 40 a0d146ed 2005-07-12 devnull 		return -1;
 41 a0d146ed 2005-07-12 devnull 	}
 42 a0d146ed 2005-07-12 devnull 	return 0;
 43 a0d146ed 2005-07-12 devnull }
 44 a0d146ed 2005-07-12 devnull 
 45 a0d146ed 2005-07-12 devnull /*
 46 a0d146ed 2005-07-12 devnull  * make an Arena, and initialize it based upon the disk header and trailer.
 47 a0d146ed 2005-07-12 devnull  */
 48 a0d146ed 2005-07-12 devnull Arena*
 49 a0d146ed 2005-07-12 devnull initarena(Part *part, u64int base, u64int size, u32int blocksize)
 50 a0d146ed 2005-07-12 devnull {
 51 a0d146ed 2005-07-12 devnull 	Arena *arena;
 52 a0d146ed 2005-07-12 devnull 
 53 a0d146ed 2005-07-12 devnull 	arena = MKZ(Arena);
 54 a0d146ed 2005-07-12 devnull 	arena->part = part;
 55 a0d146ed 2005-07-12 devnull 	arena->blocksize = blocksize;
 56 a0d146ed 2005-07-12 devnull 	arena->clumpmax = arena->blocksize / ClumpInfoSize;
 57 a0d146ed 2005-07-12 devnull 	arena->base = base + blocksize;
 58 a0d146ed 2005-07-12 devnull 	arena->size = size - 2 * blocksize;
 59 a0d146ed 2005-07-12 devnull 
 60 a0d146ed 2005-07-12 devnull 	if(loadarena(arena) < 0){
 61 a0d146ed 2005-07-12 devnull 		seterr(ECorrupt, "arena header or trailer corrupted");
 62 a0d146ed 2005-07-12 devnull 		freearena(arena);
 63 a0d146ed 2005-07-12 devnull 		return nil;
 64 a0d146ed 2005-07-12 devnull 	}
 65 a0d146ed 2005-07-12 devnull 	if(okarena(arena) < 0){
 66 a0d146ed 2005-07-12 devnull 		freearena(arena);
 67 a0d146ed 2005-07-12 devnull 		return nil;
 68 a0d146ed 2005-07-12 devnull 	}
 69 a0d146ed 2005-07-12 devnull 
 70 a0d146ed 2005-07-12 devnull 	if(arena->diskstats.sealed && scorecmp(zeroscore, arena->score)==0)
 71 db1cda12 2007-08-23 rsc 		sealarena(arena);
 72 a0d146ed 2005-07-12 devnull 
 73 a0d146ed 2005-07-12 devnull 	return arena;
 74 a0d146ed 2005-07-12 devnull }
 75 a0d146ed 2005-07-12 devnull 
 76 a0d146ed 2005-07-12 devnull void
 77 a0d146ed 2005-07-12 devnull freearena(Arena *arena)
 78 a0d146ed 2005-07-12 devnull {
 79 a0d146ed 2005-07-12 devnull 	if(arena == nil)
 80 a0d146ed 2005-07-12 devnull 		return;
 81 a0d146ed 2005-07-12 devnull 	free(arena);
 82 a0d146ed 2005-07-12 devnull }
 83 a0d146ed 2005-07-12 devnull 
 84 a0d146ed 2005-07-12 devnull Arena*
 85 a0d146ed 2005-07-12 devnull newarena(Part *part, u32int vers, char *name, u64int base, u64int size, u32int blocksize)
 86 a0d146ed 2005-07-12 devnull {
 87 a0d146ed 2005-07-12 devnull 	int bsize;
 88 a0d146ed 2005-07-12 devnull 	Arena *arena;
 89 a0d146ed 2005-07-12 devnull 
 90 a0d146ed 2005-07-12 devnull 	if(nameok(name) < 0){
 91 a0d146ed 2005-07-12 devnull 		seterr(EOk, "illegal arena name", name);
 92 a0d146ed 2005-07-12 devnull 		return nil;
 93 a0d146ed 2005-07-12 devnull 	}
 94 a0d146ed 2005-07-12 devnull 	arena = MKZ(Arena);
 95 a0d146ed 2005-07-12 devnull 	arena->part = part;
 96 a0d146ed 2005-07-12 devnull 	arena->version = vers;
 97 a0d146ed 2005-07-12 devnull 	if(vers == ArenaVersion4)
 98 a0d146ed 2005-07-12 devnull 		arena->clumpmagic = _ClumpMagic;
 99 a0d146ed 2005-07-12 devnull 	else{
100 a0d146ed 2005-07-12 devnull 		do
101 a0d146ed 2005-07-12 devnull 			arena->clumpmagic = fastrand();
102 a0d146ed 2005-07-12 devnull 		while(arena->clumpmagic==_ClumpMagic || arena->clumpmagic==0);
103 a0d146ed 2005-07-12 devnull 	}
104 a0d146ed 2005-07-12 devnull 	arena->blocksize = blocksize;
105 a0d146ed 2005-07-12 devnull 	arena->clumpmax = arena->blocksize / ClumpInfoSize;
106 a0d146ed 2005-07-12 devnull 	arena->base = base + blocksize;
107 a0d146ed 2005-07-12 devnull 	arena->size = size - 2 * blocksize;
108 a0d146ed 2005-07-12 devnull 
109 a0d146ed 2005-07-12 devnull 	namecp(arena->name, name);
110 a0d146ed 2005-07-12 devnull 
111 a0d146ed 2005-07-12 devnull 	bsize = sizeof zero;
112 a0d146ed 2005-07-12 devnull 	if(bsize > arena->blocksize)
113 a0d146ed 2005-07-12 devnull 		bsize = arena->blocksize;
114 a0d146ed 2005-07-12 devnull 
115 a0d146ed 2005-07-12 devnull 	if(wbarena(arena)<0 || wbarenahead(arena)<0
116 a0d146ed 2005-07-12 devnull 	|| writepart(arena->part, arena->base, zero, bsize)<0){
117 a0d146ed 2005-07-12 devnull 		freearena(arena);
118 a0d146ed 2005-07-12 devnull 		return nil;
119 a0d146ed 2005-07-12 devnull 	}
120 a0d146ed 2005-07-12 devnull 
121 a0d146ed 2005-07-12 devnull 	return arena;
122 a0d146ed 2005-07-12 devnull }
123 a0d146ed 2005-07-12 devnull 
124 a0d146ed 2005-07-12 devnull int
125 a0d146ed 2005-07-12 devnull readclumpinfo(Arena *arena, int clump, ClumpInfo *ci)
126 a0d146ed 2005-07-12 devnull {
127 a0d146ed 2005-07-12 devnull 	CIBlock *cib, r;
128 a0d146ed 2005-07-12 devnull 
129 a0d146ed 2005-07-12 devnull 	cib = getcib(arena, clump, 0, &r);
130 a0d146ed 2005-07-12 devnull 	if(cib == nil)
131 a0d146ed 2005-07-12 devnull 		return -1;
132 a0d146ed 2005-07-12 devnull 	unpackclumpinfo(ci, &cib->data->data[cib->offset]);
133 a0d146ed 2005-07-12 devnull 	putcib(arena, cib);
134 a0d146ed 2005-07-12 devnull 	return 0;
135 a0d146ed 2005-07-12 devnull }
136 a0d146ed 2005-07-12 devnull 
137 a0d146ed 2005-07-12 devnull int
138 a0d146ed 2005-07-12 devnull readclumpinfos(Arena *arena, int clump, ClumpInfo *cis, int n)
139 a0d146ed 2005-07-12 devnull {
140 a0d146ed 2005-07-12 devnull 	CIBlock *cib, r;
141 a0d146ed 2005-07-12 devnull 	int i;
142 a0d146ed 2005-07-12 devnull 
143 7a400ee9 2007-09-25 rsc 	/*
144 7a400ee9 2007-09-25 rsc 	 * because the clump blocks are laid out
145 7a400ee9 2007-09-25 rsc 	 * in reverse order at the end of the arena,
146 7a400ee9 2007-09-25 rsc 	 * it can be a few percent faster to read
147 7a400ee9 2007-09-25 rsc 	 * the clumps backwards, which reads the
148 7a400ee9 2007-09-25 rsc 	 * disk blocks forwards.
149 7a400ee9 2007-09-25 rsc 	 */
150 7a400ee9 2007-09-25 rsc 	for(i = n-1; i >= 0; i--){
151 a0d146ed 2005-07-12 devnull 		cib = getcib(arena, clump + i, 0, &r);
152 7a400ee9 2007-09-25 rsc 		if(cib == nil){
153 7a400ee9 2007-09-25 rsc 			n = i;
154 7a400ee9 2007-09-25 rsc 			continue;
155 7a400ee9 2007-09-25 rsc 		}
156 a0d146ed 2005-07-12 devnull 		unpackclumpinfo(&cis[i], &cib->data->data[cib->offset]);
157 a0d146ed 2005-07-12 devnull 		putcib(arena, cib);
158 a0d146ed 2005-07-12 devnull 	}
159 7a400ee9 2007-09-25 rsc 	return n;
160 a0d146ed 2005-07-12 devnull }
161 a0d146ed 2005-07-12 devnull 
162 a0d146ed 2005-07-12 devnull /*
163 a0d146ed 2005-07-12 devnull  * write directory information for one clump
164 a0d146ed 2005-07-12 devnull  * must be called the arena locked
165 a0d146ed 2005-07-12 devnull  */
166 a0d146ed 2005-07-12 devnull int
167 a0d146ed 2005-07-12 devnull writeclumpinfo(Arena *arena, int clump, ClumpInfo *ci)
168 a0d146ed 2005-07-12 devnull {
169 a0d146ed 2005-07-12 devnull 	CIBlock *cib, r;
170 a0d146ed 2005-07-12 devnull 
171 a0d146ed 2005-07-12 devnull 	cib = getcib(arena, clump, 1, &r);
172 a0d146ed 2005-07-12 devnull 	if(cib == nil)
173 a0d146ed 2005-07-12 devnull 		return -1;
174 a0d146ed 2005-07-12 devnull 	dirtydblock(cib->data, DirtyArenaCib);
175 a0d146ed 2005-07-12 devnull 	packclumpinfo(ci, &cib->data->data[cib->offset]);
176 a0d146ed 2005-07-12 devnull 	putcib(arena, cib);
177 a0d146ed 2005-07-12 devnull 	return 0;
178 a0d146ed 2005-07-12 devnull }
179 a0d146ed 2005-07-12 devnull 
180 a0d146ed 2005-07-12 devnull u64int
181 a0d146ed 2005-07-12 devnull arenadirsize(Arena *arena, u32int clumps)
182 a0d146ed 2005-07-12 devnull {
183 a0d146ed 2005-07-12 devnull 	return ((clumps / arena->clumpmax) + 1) * arena->blocksize;
184 a0d146ed 2005-07-12 devnull }
185 a0d146ed 2005-07-12 devnull 
186 a0d146ed 2005-07-12 devnull /*
187 a0d146ed 2005-07-12 devnull  * read a clump of data
188 a0d146ed 2005-07-12 devnull  * n is a hint of the size of the data, not including the header
189 a0d146ed 2005-07-12 devnull  * make sure it won't run off the end, then return the number of bytes actually read
190 a0d146ed 2005-07-12 devnull  */
191 a0d146ed 2005-07-12 devnull u32int
192 a0d146ed 2005-07-12 devnull readarena(Arena *arena, u64int aa, u8int *buf, long n)
193 a0d146ed 2005-07-12 devnull {
194 a0d146ed 2005-07-12 devnull 	DBlock *b;
195 a0d146ed 2005-07-12 devnull 	u64int a;
196 a0d146ed 2005-07-12 devnull 	u32int blocksize, off, m;
197 a0d146ed 2005-07-12 devnull 	long nn;
198 a0d146ed 2005-07-12 devnull 
199 a0d146ed 2005-07-12 devnull 	if(n == 0)
200 a0d146ed 2005-07-12 devnull 		return -1;
201 a0d146ed 2005-07-12 devnull 
202 a0d146ed 2005-07-12 devnull 	qlock(&arena->lock);
203 a0d146ed 2005-07-12 devnull 	a = arena->size - arenadirsize(arena, arena->memstats.clumps);
204 a0d146ed 2005-07-12 devnull 	qunlock(&arena->lock);
205 a0d146ed 2005-07-12 devnull 	if(aa >= a){
206 a0d146ed 2005-07-12 devnull 		seterr(EOk, "reading beyond arena clump storage: clumps=%d aa=%lld a=%lld -1 clumps=%lld\n", arena->memstats.clumps, aa, a, arena->size - arenadirsize(arena, arena->memstats.clumps - 1));
207 a0d146ed 2005-07-12 devnull 		return -1;
208 a0d146ed 2005-07-12 devnull 	}
209 a0d146ed 2005-07-12 devnull 	if(aa + n > a)
210 a0d146ed 2005-07-12 devnull 		n = a - aa;
211 a0d146ed 2005-07-12 devnull 
212 a0d146ed 2005-07-12 devnull 	blocksize = arena->blocksize;
213 a0d146ed 2005-07-12 devnull 	a = arena->base + aa;
214 a0d146ed 2005-07-12 devnull 	off = a & (blocksize - 1);
215 a0d146ed 2005-07-12 devnull 	a -= off;
216 a0d146ed 2005-07-12 devnull 	nn = 0;
217 a0d146ed 2005-07-12 devnull 	for(;;){
218 a0d146ed 2005-07-12 devnull 		b = getdblock(arena->part, a, OREAD);
219 a0d146ed 2005-07-12 devnull 		if(b == nil)
220 a0d146ed 2005-07-12 devnull 			return -1;
221 a0d146ed 2005-07-12 devnull 		m = blocksize - off;
222 a0d146ed 2005-07-12 devnull 		if(m > n - nn)
223 a0d146ed 2005-07-12 devnull 			m = n - nn;
224 a0d146ed 2005-07-12 devnull 		memmove(&buf[nn], &b->data[off], m);
225 a0d146ed 2005-07-12 devnull 		putdblock(b);
226 a0d146ed 2005-07-12 devnull 		nn += m;
227 a0d146ed 2005-07-12 devnull 		if(nn == n)
228 a0d146ed 2005-07-12 devnull 			break;
229 a0d146ed 2005-07-12 devnull 		off = 0;
230 a0d146ed 2005-07-12 devnull 		a += blocksize;
231 a0d146ed 2005-07-12 devnull 	}
232 a0d146ed 2005-07-12 devnull 	return n;
233 a0d146ed 2005-07-12 devnull }
234 a0d146ed 2005-07-12 devnull 
235 a0d146ed 2005-07-12 devnull /*
236 a0d146ed 2005-07-12 devnull  * write some data to the clump section at a given offset
237 a0d146ed 2005-07-12 devnull  * used to fix up corrupted arenas.
238 a0d146ed 2005-07-12 devnull  */
239 a0d146ed 2005-07-12 devnull u32int
240 a0d146ed 2005-07-12 devnull writearena(Arena *arena, u64int aa, u8int *clbuf, u32int n)
241 a0d146ed 2005-07-12 devnull {
242 a0d146ed 2005-07-12 devnull 	DBlock *b;
243 a0d146ed 2005-07-12 devnull 	u64int a;
244 a0d146ed 2005-07-12 devnull 	u32int blocksize, off, m;
245 a0d146ed 2005-07-12 devnull 	long nn;
246 a0d146ed 2005-07-12 devnull 	int ok;
247 a0d146ed 2005-07-12 devnull 
248 a0d146ed 2005-07-12 devnull 	if(n == 0)
249 a0d146ed 2005-07-12 devnull 		return -1;
250 a0d146ed 2005-07-12 devnull 
251 a0d146ed 2005-07-12 devnull 	qlock(&arena->lock);
252 a0d146ed 2005-07-12 devnull 	a = arena->size - arenadirsize(arena, arena->memstats.clumps);
253 a0d146ed 2005-07-12 devnull 	if(aa >= a || aa + n > a){
254 a0d146ed 2005-07-12 devnull 		qunlock(&arena->lock);
255 a0d146ed 2005-07-12 devnull 		seterr(EOk, "writing beyond arena clump storage");
256 a0d146ed 2005-07-12 devnull 		return -1;
257 a0d146ed 2005-07-12 devnull 	}
258 a0d146ed 2005-07-12 devnull 
259 a0d146ed 2005-07-12 devnull 	blocksize = arena->blocksize;
260 a0d146ed 2005-07-12 devnull 	a = arena->base + aa;
261 a0d146ed 2005-07-12 devnull 	off = a & (blocksize - 1);
262 a0d146ed 2005-07-12 devnull 	a -= off;
263 a0d146ed 2005-07-12 devnull 	nn = 0;
264 a0d146ed 2005-07-12 devnull 	for(;;){
265 a0d146ed 2005-07-12 devnull 		b = getdblock(arena->part, a, off != 0 || off + n < blocksize ? ORDWR : OWRITE);
266 a0d146ed 2005-07-12 devnull 		if(b == nil){
267 a0d146ed 2005-07-12 devnull 			qunlock(&arena->lock);
268 a0d146ed 2005-07-12 devnull 			return -1;
269 a0d146ed 2005-07-12 devnull 		}
270 a0d146ed 2005-07-12 devnull 		dirtydblock(b, DirtyArena);
271 a0d146ed 2005-07-12 devnull 		m = blocksize - off;
272 a0d146ed 2005-07-12 devnull 		if(m > n - nn)
273 a0d146ed 2005-07-12 devnull 			m = n - nn;
274 a0d146ed 2005-07-12 devnull 		memmove(&b->data[off], &clbuf[nn], m);
275 a0d146ed 2005-07-12 devnull 		ok = 0;
276 a0d146ed 2005-07-12 devnull 		putdblock(b);
277 a0d146ed 2005-07-12 devnull 		if(ok < 0){
278 a0d146ed 2005-07-12 devnull 			qunlock(&arena->lock);
279 a0d146ed 2005-07-12 devnull 			return -1;
280 a0d146ed 2005-07-12 devnull 		}
281 a0d146ed 2005-07-12 devnull 		nn += m;
282 a0d146ed 2005-07-12 devnull 		if(nn == n)
283 a0d146ed 2005-07-12 devnull 			break;
284 a0d146ed 2005-07-12 devnull 		off = 0;
285 a0d146ed 2005-07-12 devnull 		a += blocksize;
286 a0d146ed 2005-07-12 devnull 	}
287 a0d146ed 2005-07-12 devnull 	qunlock(&arena->lock);
288 a0d146ed 2005-07-12 devnull 	return n;
289 a0d146ed 2005-07-12 devnull }
290 a0d146ed 2005-07-12 devnull 
291 a0d146ed 2005-07-12 devnull /*
292 a0d146ed 2005-07-12 devnull  * allocate space for the clump and write it,
293 a0d146ed 2005-07-12 devnull  * updating the arena directory
294 a0d146ed 2005-07-12 devnull ZZZ question: should this distinguish between an arena
295 a0d146ed 2005-07-12 devnull filling up and real errors writing the clump?
296 a0d146ed 2005-07-12 devnull  */
297 a0d146ed 2005-07-12 devnull u64int
298 45ac814c 2007-10-29 rsc writeaclump(Arena *arena, Clump *c, u8int *clbuf)
299 a0d146ed 2005-07-12 devnull {
300 a0d146ed 2005-07-12 devnull 	DBlock *b;
301 a0d146ed 2005-07-12 devnull 	u64int a, aa;
302 a0d146ed 2005-07-12 devnull 	u32int clump, n, nn, m, off, blocksize;
303 a0d146ed 2005-07-12 devnull 	int ok;
304 a0d146ed 2005-07-12 devnull 
305 a0d146ed 2005-07-12 devnull 	n = c->info.size + ClumpSize + U32Size;
306 a0d146ed 2005-07-12 devnull 	qlock(&arena->lock);
307 a0d146ed 2005-07-12 devnull 	aa = arena->memstats.used;
308 a0d146ed 2005-07-12 devnull 	if(arena->memstats.sealed
309 a0d146ed 2005-07-12 devnull 	|| aa + n + U32Size + arenadirsize(arena, arena->memstats.clumps + 1) > arena->size){
310 a0d146ed 2005-07-12 devnull 		if(!arena->memstats.sealed){
311 7e452401 2007-04-27 devnull 			logerr(EOk, "seal memstats %s", arena->name);
312 a0d146ed 2005-07-12 devnull 			arena->memstats.sealed = 1;
313 9f8a101a 2008-04-16 rsc 			wbarena(arena);
314 a0d146ed 2005-07-12 devnull 		}
315 a0d146ed 2005-07-12 devnull 		qunlock(&arena->lock);
316 a0d146ed 2005-07-12 devnull 		return TWID64;
317 a0d146ed 2005-07-12 devnull 	}
318 a0d146ed 2005-07-12 devnull 	if(packclump(c, &clbuf[0], arena->clumpmagic) < 0){
319 a0d146ed 2005-07-12 devnull 		qunlock(&arena->lock);
320 a0d146ed 2005-07-12 devnull 		return TWID64;
321 a0d146ed 2005-07-12 devnull 	}
322 a0d146ed 2005-07-12 devnull 
323 a0d146ed 2005-07-12 devnull 	/*
324 a0d146ed 2005-07-12 devnull 	 * write the data out one block at a time
325 a0d146ed 2005-07-12 devnull 	 */
326 a0d146ed 2005-07-12 devnull 	blocksize = arena->blocksize;
327 a0d146ed 2005-07-12 devnull 	a = arena->base + aa;
328 a0d146ed 2005-07-12 devnull 	off = a & (blocksize - 1);
329 a0d146ed 2005-07-12 devnull 	a -= off;
330 a0d146ed 2005-07-12 devnull 	nn = 0;
331 a0d146ed 2005-07-12 devnull 	for(;;){
332 a0d146ed 2005-07-12 devnull 		b = getdblock(arena->part, a, off != 0 ? ORDWR : OWRITE);
333 a0d146ed 2005-07-12 devnull 		if(b == nil){
334 a0d146ed 2005-07-12 devnull 			qunlock(&arena->lock);
335 a0d146ed 2005-07-12 devnull 			return TWID64;
336 a0d146ed 2005-07-12 devnull 		}
337 a0d146ed 2005-07-12 devnull 		dirtydblock(b, DirtyArena);
338 a0d146ed 2005-07-12 devnull 		m = blocksize - off;
339 a0d146ed 2005-07-12 devnull 		if(m > n - nn)
340 a0d146ed 2005-07-12 devnull 			m = n - nn;
341 a0d146ed 2005-07-12 devnull 		memmove(&b->data[off], &clbuf[nn], m);
342 a0d146ed 2005-07-12 devnull 		ok = 0;
343 a0d146ed 2005-07-12 devnull 		putdblock(b);
344 a0d146ed 2005-07-12 devnull 		if(ok < 0){
345 a0d146ed 2005-07-12 devnull 			qunlock(&arena->lock);
346 a0d146ed 2005-07-12 devnull 			return TWID64;
347 a0d146ed 2005-07-12 devnull 		}
348 a0d146ed 2005-07-12 devnull 		nn += m;
349 a0d146ed 2005-07-12 devnull 		if(nn == n)
350 a0d146ed 2005-07-12 devnull 			break;
351 a0d146ed 2005-07-12 devnull 		off = 0;
352 a0d146ed 2005-07-12 devnull 		a += blocksize;
353 a0d146ed 2005-07-12 devnull 	}
354 a0d146ed 2005-07-12 devnull 
355 a0d146ed 2005-07-12 devnull 	arena->memstats.used += c->info.size + ClumpSize;
356 a0d146ed 2005-07-12 devnull 	arena->memstats.uncsize += c->info.uncsize;
357 a0d146ed 2005-07-12 devnull 	if(c->info.size < c->info.uncsize)
358 a0d146ed 2005-07-12 devnull 		arena->memstats.cclumps++;
359 a0d146ed 2005-07-12 devnull 
360 7a400ee9 2007-09-25 rsc 	clump = arena->memstats.clumps;
361 7a400ee9 2007-09-25 rsc 	if(clump % ArenaCIGSize == 0){
362 7a400ee9 2007-09-25 rsc 		if(arena->cig == nil){
363 7a400ee9 2007-09-25 rsc 			loadcig(arena);
364 7a400ee9 2007-09-25 rsc 			if(arena->cig == nil)
365 7a400ee9 2007-09-25 rsc 				goto NoCIG;
366 7a400ee9 2007-09-25 rsc 		}
367 7a400ee9 2007-09-25 rsc 		/* add aa as start of next cig */
368 7a400ee9 2007-09-25 rsc 		if(clump/ArenaCIGSize != arena->ncig){
369 7a400ee9 2007-09-25 rsc 			fprint(2, "bad arena cig computation %s: writing clump %d but %d cigs\n",
370 7a400ee9 2007-09-25 rsc 				arena->name, clump, arena->ncig);
371 7a400ee9 2007-09-25 rsc 			arena->ncig = -1;
372 7a400ee9 2007-09-25 rsc 			vtfree(arena->cig);
373 7a400ee9 2007-09-25 rsc 			arena->cig = nil;
374 7a400ee9 2007-09-25 rsc 			goto NoCIG;
375 7a400ee9 2007-09-25 rsc 		}
376 7a400ee9 2007-09-25 rsc 		arena->cig = vtrealloc(arena->cig, (arena->ncig+1)*sizeof arena->cig[0]);
377 7a400ee9 2007-09-25 rsc 		arena->cig[arena->ncig++].offset = aa;
378 7a400ee9 2007-09-25 rsc 	}
379 7a400ee9 2007-09-25 rsc NoCIG:
380 7a400ee9 2007-09-25 rsc 	arena->memstats.clumps++;
381 7a400ee9 2007-09-25 rsc 
382 a0d146ed 2005-07-12 devnull 	if(arena->memstats.clumps == 0)
383 a0d146ed 2005-07-12 devnull 		sysfatal("clumps wrapped");
384 a0d146ed 2005-07-12 devnull 	arena->wtime = now();
385 a0d146ed 2005-07-12 devnull 	if(arena->ctime == 0)
386 a0d146ed 2005-07-12 devnull 		arena->ctime = arena->wtime;
387 a0d146ed 2005-07-12 devnull 
388 a0d146ed 2005-07-12 devnull 	writeclumpinfo(arena, clump, &c->info);
389 28b49df3 2006-07-18 devnull 	wbarena(arena);
390 a0d146ed 2005-07-12 devnull 
391 a0d146ed 2005-07-12 devnull 	qunlock(&arena->lock);
392 a0d146ed 2005-07-12 devnull 
393 a0d146ed 2005-07-12 devnull 	return aa;
394 a0d146ed 2005-07-12 devnull }
395 a0d146ed 2005-07-12 devnull 
396 a0d146ed 2005-07-12 devnull int
397 a0d146ed 2005-07-12 devnull atailcmp(ATailStats *a, ATailStats *b)
398 a0d146ed 2005-07-12 devnull {
399 a0d146ed 2005-07-12 devnull 	/* good test */
400 a0d146ed 2005-07-12 devnull 	if(a->used < b->used)
401 a0d146ed 2005-07-12 devnull 		return -1;
402 a0d146ed 2005-07-12 devnull 	if(a->used > b->used)
403 a0d146ed 2005-07-12 devnull 		return 1;
404 fa325e9b 2020-01-10 cross 
405 a0d146ed 2005-07-12 devnull 	/* suspect tests - why order this way? (no one cares) */
406 a0d146ed 2005-07-12 devnull 	if(a->clumps < b->clumps)
407 a0d146ed 2005-07-12 devnull 		return -1;
408 a0d146ed 2005-07-12 devnull 	if(a->clumps > b->clumps)
409 a0d146ed 2005-07-12 devnull 		return 1;
410 a0d146ed 2005-07-12 devnull 	if(a->cclumps < b->cclumps)
411 a0d146ed 2005-07-12 devnull 		return -1;
412 a0d146ed 2005-07-12 devnull 	if(a->cclumps > b->cclumps)
413 a0d146ed 2005-07-12 devnull 		return 1;
414 a0d146ed 2005-07-12 devnull 	if(a->uncsize < b->uncsize)
415 a0d146ed 2005-07-12 devnull 		return -1;
416 a0d146ed 2005-07-12 devnull 	if(a->uncsize > b->uncsize)
417 a0d146ed 2005-07-12 devnull 		return 1;
418 a0d146ed 2005-07-12 devnull 	if(a->sealed < b->sealed)
419 a0d146ed 2005-07-12 devnull 		return -1;
420 a0d146ed 2005-07-12 devnull 	if(a->sealed > b->sealed)
421 a0d146ed 2005-07-12 devnull 		return 1;
422 fa325e9b 2020-01-10 cross 
423 a0d146ed 2005-07-12 devnull 	/* everything matches */
424 a0d146ed 2005-07-12 devnull 	return 0;
425 a0d146ed 2005-07-12 devnull }
426 a0d146ed 2005-07-12 devnull 
427 a0d146ed 2005-07-12 devnull void
428 a0d146ed 2005-07-12 devnull setatailstate(AState *as)
429 a0d146ed 2005-07-12 devnull {
430 a0d146ed 2005-07-12 devnull 	int i, j, osealed;
431 a0d146ed 2005-07-12 devnull 	Arena *a;
432 a0d146ed 2005-07-12 devnull 	Index *ix;
433 a0d146ed 2005-07-12 devnull 
434 a0d146ed 2005-07-12 devnull 	trace(0, "setatailstate %s 0x%llux clumps %d", as->arena->name, as->aa, as->stats.clumps);
435 a0d146ed 2005-07-12 devnull 
436 28b49df3 2006-07-18 devnull 	/*
437 28b49df3 2006-07-18 devnull 	 * Look up as->arena to find index.
438 28b49df3 2006-07-18 devnull 	 */
439 7c809320 2007-05-04 devnull 	needmainindex();	/* OS X linker */
440 a0d146ed 2005-07-12 devnull 	ix = mainindex;
441 a0d146ed 2005-07-12 devnull 	for(i=0; i<ix->narenas; i++)
442 a0d146ed 2005-07-12 devnull 		if(ix->arenas[i] == as->arena)
443 a0d146ed 2005-07-12 devnull 			break;
444 a0d146ed 2005-07-12 devnull 	if(i==ix->narenas || as->aa < ix->amap[i].start || as->aa >= ix->amap[i].stop || as->arena != ix->arenas[i]){
445 a0d146ed 2005-07-12 devnull 		fprint(2, "funny settailstate 0x%llux\n", as->aa);
446 a0d146ed 2005-07-12 devnull 		return;
447 a0d146ed 2005-07-12 devnull 	}
448 a0d146ed 2005-07-12 devnull 
449 7e452401 2007-04-27 devnull 	for(j=0; j<=i; j++){
450 a0d146ed 2005-07-12 devnull 		a = ix->arenas[j];
451 a0d146ed 2005-07-12 devnull 		if(atailcmp(&a->diskstats, &a->memstats) == 0)
452 7e452401 2007-04-27 devnull 			continue;
453 a0d146ed 2005-07-12 devnull 		qlock(&a->lock);
454 a0d146ed 2005-07-12 devnull 		osealed = a->diskstats.sealed;
455 a0d146ed 2005-07-12 devnull 		if(j == i)
456 a0d146ed 2005-07-12 devnull 			a->diskstats = as->stats;
457 a0d146ed 2005-07-12 devnull 		else
458 a0d146ed 2005-07-12 devnull 			a->diskstats = a->memstats;
459 a0d146ed 2005-07-12 devnull 		wbarena(a);
460 a0d146ed 2005-07-12 devnull 		if(a->diskstats.sealed != osealed && !a->inqueue)
461 a0d146ed 2005-07-12 devnull 			sealarena(a);
462 a0d146ed 2005-07-12 devnull 		qunlock(&a->lock);
463 a0d146ed 2005-07-12 devnull 	}
464 a0d146ed 2005-07-12 devnull }
465 a0d146ed 2005-07-12 devnull 
466 a0d146ed 2005-07-12 devnull /*
467 a0d146ed 2005-07-12 devnull  * once sealed, an arena never has any data added to it.
468 a0d146ed 2005-07-12 devnull  * it should only be changed to fix errors.
469 a0d146ed 2005-07-12 devnull  * this also syncs the clump directory.
470 a0d146ed 2005-07-12 devnull  */
471 a0d146ed 2005-07-12 devnull static void
472 a0d146ed 2005-07-12 devnull sealarena(Arena *arena)
473 a0d146ed 2005-07-12 devnull {
474 a0d146ed 2005-07-12 devnull 	arena->inqueue = 1;
475 a0d146ed 2005-07-12 devnull 	backsumarena(arena);
476 a0d146ed 2005-07-12 devnull }
477 a0d146ed 2005-07-12 devnull 
478 a0d146ed 2005-07-12 devnull void
479 a0d146ed 2005-07-12 devnull backsumarena(Arena *arena)
480 a0d146ed 2005-07-12 devnull {
481 a0d146ed 2005-07-12 devnull 	ASum *as;
482 a0d146ed 2005-07-12 devnull 
483 a0d146ed 2005-07-12 devnull 	as = MK(ASum);
484 a0d146ed 2005-07-12 devnull 	if(as == nil)
485 a0d146ed 2005-07-12 devnull 		return;
486 a0d146ed 2005-07-12 devnull 	qlock(&sumlock);
487 a0d146ed 2005-07-12 devnull 	as->arena = arena;
488 28b49df3 2006-07-18 devnull 	as->next = nil;
489 28b49df3 2006-07-18 devnull 	if(sumq)
490 28b49df3 2006-07-18 devnull 		sumqtail->next = as;
491 28b49df3 2006-07-18 devnull 	else
492 28b49df3 2006-07-18 devnull 		sumq = as;
493 28b49df3 2006-07-18 devnull 	sumqtail = as;
494 01c1b763 2008-07-04 rsc 	/*
495 01c1b763 2008-07-04 rsc 	 * Might get here while initializing arenas,
496 01c1b763 2008-07-04 rsc 	 * before initarenasum has been called.
497 01c1b763 2008-07-04 rsc 	 */
498 01c1b763 2008-07-04 rsc 	if(sumwait.l)
499 01c1b763 2008-07-04 rsc 		rwakeup(&sumwait);
500 a0d146ed 2005-07-12 devnull 	qunlock(&sumlock);
501 a0d146ed 2005-07-12 devnull }
502 a0d146ed 2005-07-12 devnull 
503 a0d146ed 2005-07-12 devnull static void
504 a0d146ed 2005-07-12 devnull sumproc(void *unused)
505 a0d146ed 2005-07-12 devnull {
506 a0d146ed 2005-07-12 devnull 	ASum *as;
507 a0d146ed 2005-07-12 devnull 	Arena *arena;
508 a0d146ed 2005-07-12 devnull 
509 a0d146ed 2005-07-12 devnull 	USED(unused);
510 a0d146ed 2005-07-12 devnull 
511 a0d146ed 2005-07-12 devnull 	for(;;){
512 a0d146ed 2005-07-12 devnull 		qlock(&sumlock);
513 a0d146ed 2005-07-12 devnull 		while(sumq == nil)
514 a0d146ed 2005-07-12 devnull 			rsleep(&sumwait);
515 a0d146ed 2005-07-12 devnull 		as = sumq;
516 a0d146ed 2005-07-12 devnull 		sumq = as->next;
517 a0d146ed 2005-07-12 devnull 		qunlock(&sumlock);
518 a0d146ed 2005-07-12 devnull 		arena = as->arena;
519 a0d146ed 2005-07-12 devnull 		free(as);
520 a0d146ed 2005-07-12 devnull 		sumarena(arena);
521 a0d146ed 2005-07-12 devnull 	}
522 a0d146ed 2005-07-12 devnull }
523 a0d146ed 2005-07-12 devnull 
524 a0d146ed 2005-07-12 devnull void
525 a0d146ed 2005-07-12 devnull sumarena(Arena *arena)
526 a0d146ed 2005-07-12 devnull {
527 a0d146ed 2005-07-12 devnull 	ZBlock *b;
528 a0d146ed 2005-07-12 devnull 	DigestState s;
529 a0d146ed 2005-07-12 devnull 	u64int a, e;
530 a0d146ed 2005-07-12 devnull 	u32int bs;
531 28b49df3 2006-07-18 devnull 	int t;
532 a0d146ed 2005-07-12 devnull 	u8int score[VtScoreSize];
533 a0d146ed 2005-07-12 devnull 
534 a0d146ed 2005-07-12 devnull 	bs = MaxIoSize;
535 a0d146ed 2005-07-12 devnull 	if(bs < arena->blocksize)
536 a0d146ed 2005-07-12 devnull 		bs = arena->blocksize;
537 a0d146ed 2005-07-12 devnull 
538 a0d146ed 2005-07-12 devnull 	/*
539 a0d146ed 2005-07-12 devnull 	 * read & sum all blocks except the last one
540 a0d146ed 2005-07-12 devnull 	 */
541 db1cda12 2007-08-23 rsc 	flushdcache();
542 a0d146ed 2005-07-12 devnull 	memset(&s, 0, sizeof s);
543 a0d146ed 2005-07-12 devnull 	b = alloczblock(bs, 0, arena->part->blocksize);
544 a0d146ed 2005-07-12 devnull 	e = arena->base + arena->size;
545 a0d146ed 2005-07-12 devnull 	for(a = arena->base - arena->blocksize; a + arena->blocksize <= e; a += bs){
546 28b49df3 2006-07-18 devnull 		disksched();
547 28b49df3 2006-07-18 devnull 		while((t=arenasumsleeptime) == SleepForever){
548 28b49df3 2006-07-18 devnull 			sleep(1000);
549 28b49df3 2006-07-18 devnull 			disksched();
550 28b49df3 2006-07-18 devnull 		}
551 28b49df3 2006-07-18 devnull 		sleep(t);
552 a0d146ed 2005-07-12 devnull 		if(a + bs > e)
553 a0d146ed 2005-07-12 devnull 			bs = arena->blocksize;
554 a0d146ed 2005-07-12 devnull 		if(readpart(arena->part, a, b->data, bs) < 0)
555 a0d146ed 2005-07-12 devnull 			goto ReadErr;
556 a0d146ed 2005-07-12 devnull 		addstat(StatSumRead, 1);
557 a0d146ed 2005-07-12 devnull 		addstat(StatSumReadBytes, bs);
558 a0d146ed 2005-07-12 devnull 		sha1(b->data, bs, nil, &s);
559 a0d146ed 2005-07-12 devnull 	}
560 a0d146ed 2005-07-12 devnull 
561 a0d146ed 2005-07-12 devnull 	/*
562 a0d146ed 2005-07-12 devnull 	 * the last one is special, since it may already have the checksum included
563 a0d146ed 2005-07-12 devnull 	 */
564 a0d146ed 2005-07-12 devnull 	bs = arena->blocksize;
565 a0d146ed 2005-07-12 devnull 	if(readpart(arena->part, e, b->data, bs) < 0){
566 a0d146ed 2005-07-12 devnull ReadErr:
567 a0d146ed 2005-07-12 devnull 		logerr(EOk, "sumarena can't sum %s, read at %lld failed: %r", arena->name, a);
568 a0d146ed 2005-07-12 devnull 		freezblock(b);
569 a0d146ed 2005-07-12 devnull 		return;
570 a0d146ed 2005-07-12 devnull 	}
571 a0d146ed 2005-07-12 devnull 	addstat(StatSumRead, 1);
572 a0d146ed 2005-07-12 devnull 	addstat(StatSumReadBytes, bs);
573 a0d146ed 2005-07-12 devnull 
574 a0d146ed 2005-07-12 devnull 	sha1(b->data, bs-VtScoreSize, nil, &s);
575 a0d146ed 2005-07-12 devnull 	sha1(zeroscore, VtScoreSize, nil, &s);
576 a0d146ed 2005-07-12 devnull 	sha1(nil, 0, score, &s);
577 fa325e9b 2020-01-10 cross 
578 a0d146ed 2005-07-12 devnull 	/*
579 a0d146ed 2005-07-12 devnull 	 * check for no checksum or the same
580 a0d146ed 2005-07-12 devnull 	 */
581 db1cda12 2007-08-23 rsc 	if(scorecmp(score, &b->data[bs - VtScoreSize]) != 0
582 db1cda12 2007-08-23 rsc 	&& scorecmp(zeroscore, &b->data[bs - VtScoreSize]) != 0)
583 db1cda12 2007-08-23 rsc 		logerr(EOk, "overwriting mismatched checksums for arena=%s, found=%V calculated=%V",
584 db1cda12 2007-08-23 rsc 			arena->name, &b->data[bs - VtScoreSize], score);
585 a0d146ed 2005-07-12 devnull 	freezblock(b);
586 a0d146ed 2005-07-12 devnull 
587 a0d146ed 2005-07-12 devnull 	qlock(&arena->lock);
588 a0d146ed 2005-07-12 devnull 	scorecp(arena->score, score);
589 db1cda12 2007-08-23 rsc 	wbarena(arena);
590 a0d146ed 2005-07-12 devnull 	qunlock(&arena->lock);
591 a0d146ed 2005-07-12 devnull }
592 a0d146ed 2005-07-12 devnull 
593 a0d146ed 2005-07-12 devnull /*
594 a0d146ed 2005-07-12 devnull  * write the arena trailer block to the partition
595 a0d146ed 2005-07-12 devnull  */
596 a0d146ed 2005-07-12 devnull int
597 a0d146ed 2005-07-12 devnull wbarena(Arena *arena)
598 a0d146ed 2005-07-12 devnull {
599 a0d146ed 2005-07-12 devnull 	DBlock *b;
600 a0d146ed 2005-07-12 devnull 	int bad;
601 a0d146ed 2005-07-12 devnull 
602 a0d146ed 2005-07-12 devnull 	if((b = getdblock(arena->part, arena->base + arena->size, OWRITE)) == nil){
603 a0d146ed 2005-07-12 devnull 		logerr(EAdmin, "can't write arena trailer: %r");
604 a0d146ed 2005-07-12 devnull 		return -1;
605 a0d146ed 2005-07-12 devnull 	}
606 a0d146ed 2005-07-12 devnull 	dirtydblock(b, DirtyArenaTrailer);
607 a0d146ed 2005-07-12 devnull 	bad = okarena(arena)<0 || packarena(arena, b->data)<0;
608 db1cda12 2007-08-23 rsc 	scorecp(b->data + arena->blocksize - VtScoreSize, arena->score);
609 a0d146ed 2005-07-12 devnull 	putdblock(b);
610 a0d146ed 2005-07-12 devnull 	if(bad)
611 a0d146ed 2005-07-12 devnull 		return -1;
612 a0d146ed 2005-07-12 devnull 	return 0;
613 a0d146ed 2005-07-12 devnull }
614 a0d146ed 2005-07-12 devnull 
615 a0d146ed 2005-07-12 devnull int
616 a0d146ed 2005-07-12 devnull wbarenahead(Arena *arena)
617 a0d146ed 2005-07-12 devnull {
618 a0d146ed 2005-07-12 devnull 	ZBlock *b;
619 a0d146ed 2005-07-12 devnull 	ArenaHead head;
620 a0d146ed 2005-07-12 devnull 	int bad;
621 a0d146ed 2005-07-12 devnull 
622 a0d146ed 2005-07-12 devnull 	namecp(head.name, arena->name);
623 a0d146ed 2005-07-12 devnull 	head.version = arena->version;
624 a0d146ed 2005-07-12 devnull 	head.size = arena->size + 2 * arena->blocksize;
625 a0d146ed 2005-07-12 devnull 	head.blocksize = arena->blocksize;
626 a0d146ed 2005-07-12 devnull 	head.clumpmagic = arena->clumpmagic;
627 a0d146ed 2005-07-12 devnull 	b = alloczblock(arena->blocksize, 1, arena->part->blocksize);
628 a0d146ed 2005-07-12 devnull 	if(b == nil){
629 a0d146ed 2005-07-12 devnull 		logerr(EAdmin, "can't write arena header: %r");
630 28b49df3 2006-07-18 devnull /* ZZZ add error message? */
631 a0d146ed 2005-07-12 devnull 		return -1;
632 a0d146ed 2005-07-12 devnull 	}
633 a0d146ed 2005-07-12 devnull 	/*
634 a0d146ed 2005-07-12 devnull 	 * this writepart is okay because it only happens
635 a0d146ed 2005-07-12 devnull 	 * during initialization.
636 a0d146ed 2005-07-12 devnull 	 */
637 a0d146ed 2005-07-12 devnull 	bad = packarenahead(&head, b->data)<0 ||
638 e46cacb0 2007-04-27 devnull 	      writepart(arena->part, arena->base - arena->blocksize, b->data, arena->blocksize)<0 ||
639 e46cacb0 2007-04-27 devnull 	      flushpart(arena->part)<0;
640 a0d146ed 2005-07-12 devnull 	freezblock(b);
641 a0d146ed 2005-07-12 devnull 	if(bad)
642 a0d146ed 2005-07-12 devnull 		return -1;
643 a0d146ed 2005-07-12 devnull 	return 0;
644 a0d146ed 2005-07-12 devnull }
645 a0d146ed 2005-07-12 devnull 
646 a0d146ed 2005-07-12 devnull /*
647 a0d146ed 2005-07-12 devnull  * read the arena header and trailer blocks from disk
648 a0d146ed 2005-07-12 devnull  */
649 a0d146ed 2005-07-12 devnull static int
650 a0d146ed 2005-07-12 devnull loadarena(Arena *arena)
651 a0d146ed 2005-07-12 devnull {
652 a0d146ed 2005-07-12 devnull 	ArenaHead head;
653 a0d146ed 2005-07-12 devnull 	ZBlock *b;
654 a0d146ed 2005-07-12 devnull 
655 a0d146ed 2005-07-12 devnull 	b = alloczblock(arena->blocksize, 0, arena->part->blocksize);
656 a0d146ed 2005-07-12 devnull 	if(b == nil)
657 a0d146ed 2005-07-12 devnull 		return -1;
658 a0d146ed 2005-07-12 devnull 	if(readpart(arena->part, arena->base + arena->size, b->data, arena->blocksize) < 0){
659 a0d146ed 2005-07-12 devnull 		freezblock(b);
660 a0d146ed 2005-07-12 devnull 		return -1;
661 a0d146ed 2005-07-12 devnull 	}
662 a0d146ed 2005-07-12 devnull 	if(unpackarena(arena, b->data) < 0){
663 a0d146ed 2005-07-12 devnull 		freezblock(b);
664 a0d146ed 2005-07-12 devnull 		return -1;
665 a0d146ed 2005-07-12 devnull 	}
666 a0d146ed 2005-07-12 devnull 	if(arena->version != ArenaVersion4 && arena->version != ArenaVersion5){
667 a0d146ed 2005-07-12 devnull 		seterr(EAdmin, "unknown arena version %d", arena->version);
668 a0d146ed 2005-07-12 devnull 		freezblock(b);
669 a0d146ed 2005-07-12 devnull 		return -1;
670 a0d146ed 2005-07-12 devnull 	}
671 a0d146ed 2005-07-12 devnull 	scorecp(arena->score, &b->data[arena->blocksize - VtScoreSize]);
672 a0d146ed 2005-07-12 devnull 
673 a0d146ed 2005-07-12 devnull 	if(readpart(arena->part, arena->base - arena->blocksize, b->data, arena->blocksize) < 0){
674 a0d146ed 2005-07-12 devnull 		logerr(EAdmin, "can't read arena header: %r");
675 a0d146ed 2005-07-12 devnull 		freezblock(b);
676 a0d146ed 2005-07-12 devnull 		return 0;
677 a0d146ed 2005-07-12 devnull 	}
678 a0d146ed 2005-07-12 devnull 	if(unpackarenahead(&head, b->data) < 0)
679 a0d146ed 2005-07-12 devnull 		logerr(ECorrupt, "corrupted arena header: %r");
680 a0d146ed 2005-07-12 devnull 	else if(namecmp(arena->name, head.name)!=0
681 a0d146ed 2005-07-12 devnull 	     || arena->clumpmagic != head.clumpmagic
682 a0d146ed 2005-07-12 devnull 	     || arena->version != head.version
683 a0d146ed 2005-07-12 devnull 	     || arena->blocksize != head.blocksize
684 a0d146ed 2005-07-12 devnull 	     || arena->size + 2 * arena->blocksize != head.size){
685 a0d146ed 2005-07-12 devnull 		if(namecmp(arena->name, head.name)!=0)
686 fa325e9b 2020-01-10 cross 			logerr(ECorrupt, "arena tail name %s head %s",
687 a0d146ed 2005-07-12 devnull 				arena->name, head.name);
688 a0d146ed 2005-07-12 devnull 		else if(arena->clumpmagic != head.clumpmagic)
689 a0d146ed 2005-07-12 devnull 			logerr(ECorrupt, "arena tail clumpmagic 0x%lux head 0x%lux",
690 a0d146ed 2005-07-12 devnull 				(ulong)arena->clumpmagic, (ulong)head.clumpmagic);
691 a0d146ed 2005-07-12 devnull 		else if(arena->version != head.version)
692 a0d146ed 2005-07-12 devnull 			logerr(ECorrupt, "arena tail version %d head version %d",
693 a0d146ed 2005-07-12 devnull 				arena->version, head.version);
694 a0d146ed 2005-07-12 devnull 		else if(arena->blocksize != head.blocksize)
695 a0d146ed 2005-07-12 devnull 			logerr(ECorrupt, "arena tail block size %d head %d",
696 a0d146ed 2005-07-12 devnull 				arena->blocksize, head.blocksize);
697 a0d146ed 2005-07-12 devnull 		else if(arena->size+2*arena->blocksize != head.size)
698 a0d146ed 2005-07-12 devnull 			logerr(ECorrupt, "arena tail size %lud head %lud",
699 a0d146ed 2005-07-12 devnull 				(ulong)arena->size+2*arena->blocksize, head.size);
700 a0d146ed 2005-07-12 devnull 		else
701 a0d146ed 2005-07-12 devnull 			logerr(ECorrupt, "arena header inconsistent with arena data");
702 a0d146ed 2005-07-12 devnull 	}
703 a0d146ed 2005-07-12 devnull 	freezblock(b);
704 a0d146ed 2005-07-12 devnull 
705 a0d146ed 2005-07-12 devnull 	return 0;
706 a0d146ed 2005-07-12 devnull }
707 a0d146ed 2005-07-12 devnull 
708 a0d146ed 2005-07-12 devnull static int
709 a0d146ed 2005-07-12 devnull okarena(Arena *arena)
710 a0d146ed 2005-07-12 devnull {
711 a0d146ed 2005-07-12 devnull 	u64int dsize;
712 a0d146ed 2005-07-12 devnull 	int ok;
713 a0d146ed 2005-07-12 devnull 
714 a0d146ed 2005-07-12 devnull 	ok = 0;
715 a0d146ed 2005-07-12 devnull 	dsize = arenadirsize(arena, arena->diskstats.clumps);
716 a0d146ed 2005-07-12 devnull 	if(arena->diskstats.used + dsize > arena->size){
717 28b49df3 2006-07-18 devnull 		seterr(ECorrupt, "arena %s used > size", arena->name);
718 a0d146ed 2005-07-12 devnull 		ok = -1;
719 a0d146ed 2005-07-12 devnull 	}
720 a0d146ed 2005-07-12 devnull 
721 a0d146ed 2005-07-12 devnull 	if(arena->diskstats.cclumps > arena->diskstats.clumps)
722 28b49df3 2006-07-18 devnull 		logerr(ECorrupt, "arena %s has more compressed clumps than total clumps", arena->name);
723 a0d146ed 2005-07-12 devnull 
724 28b49df3 2006-07-18 devnull 	/*
725 28b49df3 2006-07-18 devnull 	 * This need not be true if some of the disk is corrupted.
726 28b49df3 2006-07-18 devnull 	 *
727 a0d146ed 2005-07-12 devnull 	if(arena->diskstats.uncsize + arena->diskstats.clumps * ClumpSize + arena->blocksize < arena->diskstats.used)
728 28b49df3 2006-07-18 devnull 		logerr(ECorrupt, "arena %s uncompressed size inconsistent with used space %lld %d %lld", arena->name, arena->diskstats.uncsize, arena->diskstats.clumps, arena->diskstats.used);
729 28b49df3 2006-07-18 devnull 	 */
730 a0d146ed 2005-07-12 devnull 
731 f5a8ea6f 2011-06-02 rsc 	/*
732 f5a8ea6f 2011-06-02 rsc 	 * this happens; it's harmless.
733 f5a8ea6f 2011-06-02 rsc 	 *
734 a0d146ed 2005-07-12 devnull 	if(arena->ctime > arena->wtime)
735 28b49df3 2006-07-18 devnull 		logerr(ECorrupt, "arena %s creation time after last write time", arena->name);
736 f5a8ea6f 2011-06-02 rsc 	 */
737 a0d146ed 2005-07-12 devnull 	return ok;
738 a0d146ed 2005-07-12 devnull }
739 a0d146ed 2005-07-12 devnull 
740 a0d146ed 2005-07-12 devnull static CIBlock*
741 a0d146ed 2005-07-12 devnull getcib(Arena *arena, int clump, int writing, CIBlock *rock)
742 a0d146ed 2005-07-12 devnull {
743 a0d146ed 2005-07-12 devnull 	int mode;
744 a0d146ed 2005-07-12 devnull 	CIBlock *cib;
745 a0d146ed 2005-07-12 devnull 	u32int block, off;
746 a0d146ed 2005-07-12 devnull 
747 a0d146ed 2005-07-12 devnull 	if(clump >= arena->memstats.clumps){
748 a0d146ed 2005-07-12 devnull 		seterr(EOk, "clump directory access out of range");
749 a0d146ed 2005-07-12 devnull 		return nil;
750 a0d146ed 2005-07-12 devnull 	}
751 a0d146ed 2005-07-12 devnull 	block = clump / arena->clumpmax;
752 a0d146ed 2005-07-12 devnull 	off = (clump - block * arena->clumpmax) * ClumpInfoSize;
753 a0d146ed 2005-07-12 devnull 	cib = rock;
754 a0d146ed 2005-07-12 devnull 	cib->block = block;
755 a0d146ed 2005-07-12 devnull 	cib->offset = off;
756 a0d146ed 2005-07-12 devnull 
757 a0d146ed 2005-07-12 devnull 	if(writing){
758 a0d146ed 2005-07-12 devnull 		if(off == 0 && clump == arena->memstats.clumps-1)
759 a0d146ed 2005-07-12 devnull 			mode = OWRITE;
760 a0d146ed 2005-07-12 devnull 		else
761 a0d146ed 2005-07-12 devnull 			mode = ORDWR;
762 a0d146ed 2005-07-12 devnull 	}else
763 a0d146ed 2005-07-12 devnull 		mode = OREAD;
764 a0d146ed 2005-07-12 devnull 
765 a0d146ed 2005-07-12 devnull 	cib->data = getdblock(arena->part,
766 a0d146ed 2005-07-12 devnull 		arena->base + arena->size - (block + 1) * arena->blocksize, mode);
767 a0d146ed 2005-07-12 devnull 	if(cib->data == nil)
768 a0d146ed 2005-07-12 devnull 		return nil;
769 a0d146ed 2005-07-12 devnull 	return cib;
770 a0d146ed 2005-07-12 devnull }
771 a0d146ed 2005-07-12 devnull 
772 a0d146ed 2005-07-12 devnull static void
773 a0d146ed 2005-07-12 devnull putcib(Arena *arena, CIBlock *cib)
774 a0d146ed 2005-07-12 devnull {
775 5aec445c 2005-10-29 devnull 	USED(arena);
776 5aec445c 2005-10-29 devnull 
777 a0d146ed 2005-07-12 devnull 	putdblock(cib->data);
778 a0d146ed 2005-07-12 devnull 	cib->data = nil;
779 a0d146ed 2005-07-12 devnull }
780 7a400ee9 2007-09-25 rsc 
781 7a400ee9 2007-09-25 rsc 
782 7a400ee9 2007-09-25 rsc /*
783 fa325e9b 2020-01-10 cross  * For index entry readahead purposes, the arenas are
784 7a400ee9 2007-09-25 rsc  * broken into smaller subpieces, called clump info groups
785 7a400ee9 2007-09-25 rsc  * or cigs.  Each cig has ArenaCIGSize clumps (ArenaCIGSize
786 7a400ee9 2007-09-25 rsc  * is chosen to make the index entries take up about half
787 7a400ee9 2007-09-25 rsc  * a megabyte).  The index entries do not contain enough
788 7a400ee9 2007-09-25 rsc  * information to determine what the clump index is for
789 7a400ee9 2007-09-25 rsc  * a given address in an arena.  That info is needed both for
790 fa325e9b 2020-01-10 cross  * figuring out which clump group an address belongs to
791 7a400ee9 2007-09-25 rsc  * and for prefetching a clump group's index entries from
792 7a400ee9 2007-09-25 rsc  * the arena table of contents.  The first time clump groups
793 7a400ee9 2007-09-25 rsc  * are accessed, we scan the entire arena table of contents
794 fa325e9b 2020-01-10 cross  * (which might be 10s of megabytes), recording the data
795 7a400ee9 2007-09-25 rsc  * offset of each clump group.
796 7a400ee9 2007-09-25 rsc  */
797 7a400ee9 2007-09-25 rsc 
798 fa325e9b 2020-01-10 cross /*
799 7a400ee9 2007-09-25 rsc  * load clump info group information by scanning entire toc.
800 7a400ee9 2007-09-25 rsc  */
801 7a400ee9 2007-09-25 rsc static void
802 7a400ee9 2007-09-25 rsc loadcig(Arena *arena)
803 7a400ee9 2007-09-25 rsc {
804 7a400ee9 2007-09-25 rsc 	u32int i, j, ncig, nci;
805 7a400ee9 2007-09-25 rsc 	ArenaCIG *cig;
806 7a400ee9 2007-09-25 rsc 	ClumpInfo *ci;
807 7a400ee9 2007-09-25 rsc 	u64int offset;
808 7a400ee9 2007-09-25 rsc 	int ms;
809 7a400ee9 2007-09-25 rsc 
810 7a400ee9 2007-09-25 rsc 	if(arena->cig || arena->ncig < 0)
811 7a400ee9 2007-09-25 rsc 		return;
812 7a400ee9 2007-09-25 rsc 
813 7a400ee9 2007-09-25 rsc //	fprint(2, "loadcig %s\n", arena->name);
814 fa325e9b 2020-01-10 cross 
815 7a400ee9 2007-09-25 rsc 	ncig = (arena->memstats.clumps+ArenaCIGSize-1) / ArenaCIGSize;
816 7a400ee9 2007-09-25 rsc 	if(ncig == 0){
817 7a400ee9 2007-09-25 rsc 		arena->cig = vtmalloc(1);
818 7a400ee9 2007-09-25 rsc 		arena->ncig = 0;
819 7a400ee9 2007-09-25 rsc 		return;
820 7a400ee9 2007-09-25 rsc 	}
821 7a400ee9 2007-09-25 rsc 
822 7a400ee9 2007-09-25 rsc 	ms = msec();
823 7a400ee9 2007-09-25 rsc 	cig = vtmalloc(ncig*sizeof cig[0]);
824 7a400ee9 2007-09-25 rsc 	ci = vtmalloc(ArenaCIGSize*sizeof ci[0]);
825 7a400ee9 2007-09-25 rsc 	offset = 0;
826 7a400ee9 2007-09-25 rsc 	for(i=0; i<ncig; i++){
827 7a400ee9 2007-09-25 rsc 		nci = readclumpinfos(arena, i*ArenaCIGSize, ci, ArenaCIGSize);
828 7a400ee9 2007-09-25 rsc 		cig[i].offset = offset;
829 7a400ee9 2007-09-25 rsc 		for(j=0; j<nci; j++)
830 7a400ee9 2007-09-25 rsc 			offset += ClumpSize + ci[j].size;
831 7a400ee9 2007-09-25 rsc 		if(nci < ArenaCIGSize){
832 7a400ee9 2007-09-25 rsc 			if(i != ncig-1){
833 7a400ee9 2007-09-25 rsc 				vtfree(ci);
834 7a400ee9 2007-09-25 rsc 				vtfree(cig);
835 7a400ee9 2007-09-25 rsc 				arena->ncig = -1;
836 7a400ee9 2007-09-25 rsc 				fprint(2, "loadcig %s: got %ud cigs, expected %ud\n", arena->name, i+1, ncig);
837 7a400ee9 2007-09-25 rsc 				goto out;
838 7a400ee9 2007-09-25 rsc 			}
839 7a400ee9 2007-09-25 rsc 		}
840 7a400ee9 2007-09-25 rsc 	}
841 7a400ee9 2007-09-25 rsc 	vtfree(ci);
842 fa325e9b 2020-01-10 cross 
843 7a400ee9 2007-09-25 rsc 	arena->ncig = ncig;
844 7a400ee9 2007-09-25 rsc 	arena->cig = cig;
845 7a400ee9 2007-09-25 rsc 
846 7a400ee9 2007-09-25 rsc out:
847 7a400ee9 2007-09-25 rsc 	ms = msec() - ms;
848 7a400ee9 2007-09-25 rsc 	addstat2(StatCigLoad, 1, StatCigLoadTime, ms);
849 7a400ee9 2007-09-25 rsc }
850 7a400ee9 2007-09-25 rsc 
851 7a400ee9 2007-09-25 rsc /*
852 7a400ee9 2007-09-25 rsc  * convert arena address into arena group + data boundaries.
853 7a400ee9 2007-09-25 rsc  */
854 7a400ee9 2007-09-25 rsc int
855 7a400ee9 2007-09-25 rsc arenatog(Arena *arena, u64int addr, u64int *gstart, u64int *glimit, int *g)
856 7a400ee9 2007-09-25 rsc {
857 7a400ee9 2007-09-25 rsc 	int r, l, m;
858 7a400ee9 2007-09-25 rsc 
859 7a400ee9 2007-09-25 rsc 	qlock(&arena->lock);
860 7a400ee9 2007-09-25 rsc 	if(arena->cig == nil)
861 7a400ee9 2007-09-25 rsc 		loadcig(arena);
862 7a400ee9 2007-09-25 rsc 	if(arena->cig == nil || arena->ncig == 0){
863 7a400ee9 2007-09-25 rsc 		qunlock(&arena->lock);
864 7a400ee9 2007-09-25 rsc 		return -1;
865 7a400ee9 2007-09-25 rsc 	}
866 7a400ee9 2007-09-25 rsc 
867 7a400ee9 2007-09-25 rsc 	l = 1;
868 7a400ee9 2007-09-25 rsc 	r = arena->ncig - 1;
869 7a400ee9 2007-09-25 rsc 	while(l <= r){
870 7a400ee9 2007-09-25 rsc 		m = (r + l) / 2;
871 7a400ee9 2007-09-25 rsc 		if(arena->cig[m].offset <= addr)
872 7a400ee9 2007-09-25 rsc 			l = m + 1;
873 7a400ee9 2007-09-25 rsc 		else
874 7a400ee9 2007-09-25 rsc 			r = m - 1;
875 7a400ee9 2007-09-25 rsc 	}
876 7a400ee9 2007-09-25 rsc 	l--;
877 7a400ee9 2007-09-25 rsc 
878 7a400ee9 2007-09-25 rsc 	*g = l;
879 7a400ee9 2007-09-25 rsc 	*gstart = arena->cig[l].offset;
880 7a400ee9 2007-09-25 rsc 	if(l+1 < arena->ncig)
881 7a400ee9 2007-09-25 rsc 		*glimit = arena->cig[l+1].offset;
882 7a400ee9 2007-09-25 rsc 	else
883 7a400ee9 2007-09-25 rsc 		*glimit = arena->memstats.used;
884 7a400ee9 2007-09-25 rsc 	qunlock(&arena->lock);
885 7a400ee9 2007-09-25 rsc 	return 0;
886 7a400ee9 2007-09-25 rsc }
887 7a400ee9 2007-09-25 rsc 
888 7a400ee9 2007-09-25 rsc /*
889 7a400ee9 2007-09-25 rsc  * load the clump info for group g into the index entries.
890 7a400ee9 2007-09-25 rsc  */
891 7a400ee9 2007-09-25 rsc int
892 7a400ee9 2007-09-25 rsc asumload(Arena *arena, int g, IEntry *entries, int nentries)
893 7a400ee9 2007-09-25 rsc {
894 7a400ee9 2007-09-25 rsc 	int i, base, limit;
895 7a400ee9 2007-09-25 rsc 	u64int addr;
896 7a400ee9 2007-09-25 rsc 	ClumpInfo ci;
897 7a400ee9 2007-09-25 rsc 	IEntry *ie;
898 7a400ee9 2007-09-25 rsc 
899 7a400ee9 2007-09-25 rsc 	if(nentries < ArenaCIGSize){
900 7a400ee9 2007-09-25 rsc 		fprint(2, "asking for too few entries\n");
901 7a400ee9 2007-09-25 rsc 		return -1;
902 7a400ee9 2007-09-25 rsc 	}
903 fa325e9b 2020-01-10 cross 
904 7a400ee9 2007-09-25 rsc 	qlock(&arena->lock);
905 7a400ee9 2007-09-25 rsc 	if(arena->cig == nil)
906 7a400ee9 2007-09-25 rsc 		loadcig(arena);
907 7a400ee9 2007-09-25 rsc 	if(arena->cig == nil || arena->ncig == 0 || g >= arena->ncig){
908 7a400ee9 2007-09-25 rsc 		qunlock(&arena->lock);
909 7a400ee9 2007-09-25 rsc 		return -1;
910 7a400ee9 2007-09-25 rsc 	}
911 fa325e9b 2020-01-10 cross 
912 7a400ee9 2007-09-25 rsc 	addr = 0;
913 7a400ee9 2007-09-25 rsc 	base = g*ArenaCIGSize;
914 7a400ee9 2007-09-25 rsc 	limit = base + ArenaCIGSize;
915 7a400ee9 2007-09-25 rsc 	if(base > arena->memstats.clumps)
916 7a400ee9 2007-09-25 rsc 		base = arena->memstats.clumps;
917 7a400ee9 2007-09-25 rsc 	ie = entries;
918 7a400ee9 2007-09-25 rsc 	for(i=base; i<limit; i++){
919 7a400ee9 2007-09-25 rsc 		if(readclumpinfo(arena, i, &ci) < 0)
920 7a400ee9 2007-09-25 rsc 			break;
921 7a400ee9 2007-09-25 rsc 		if(ci.type != VtCorruptType){
922 7a400ee9 2007-09-25 rsc 			scorecp(ie->score, ci.score);
923 7a400ee9 2007-09-25 rsc 			ie->ia.type = ci.type;
924 7a400ee9 2007-09-25 rsc 			ie->ia.size = ci.uncsize;
925 7a400ee9 2007-09-25 rsc 			ie->ia.blocks = (ci.size + ClumpSize + (1<<ABlockLog) - 1) >> ABlockLog;
926 7a400ee9 2007-09-25 rsc 			ie->ia.addr = addr;
927 7a400ee9 2007-09-25 rsc 			ie++;
928 7a400ee9 2007-09-25 rsc 		}
929 7a400ee9 2007-09-25 rsc 		addr += ClumpSize + ci.size;
930 7a400ee9 2007-09-25 rsc 	}
931 7a400ee9 2007-09-25 rsc 	qunlock(&arena->lock);
932 7a400ee9 2007-09-25 rsc 	return ie - entries;
933 7a400ee9 2007-09-25 rsc }