commit adbb83845c5d3cb02343d7613cfb8385aa909909 from: rsc date: Tue Jan 18 05:25:09 2005 UTC remove venti for now commit - d338028eeb45434dab651466dcd664091517e087 commit + adbb83845c5d3cb02343d7613cfb8385aa909909 blob - 689631701a2d9111ba567a19ad2b5b05567321a1 (mode 644) blob + /dev/null --- src/cmd/venti/README +++ /dev/null @@ -1,3 +0,0 @@ -This is very untested. -It runs on FreeBSD but that's about all I've checked. - blob - 318f9ad24a32ab28101e2ec27f252f9c66625e15 (mode 644) blob + /dev/null --- src/cmd/venti/arena.c +++ /dev/null @@ -1,661 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -typedef struct ASum ASum; - -struct ASum -{ - Arena *arena; - ASum *next; -}; - -static void sealarena(Arena *arena); -static int okarena(Arena *arena); -static int loadarena(Arena *arena); -static CIBlock *getcib(Arena *arena, int clump, int writing, CIBlock *rock); -static void putcib(Arena *arena, CIBlock *cib); -static void sumproc(void *); - -static QLock sumlock; -static Rendez sumwait; -static ASum *sumq; - -int -initarenasum(void) -{ - sumwait.l = &sumlock; - - if(vtproc(sumproc, nil) < 0){ - seterr(EOk, "can't start arena checksum slave: %r"); - return -1; - } - return 0; -} - -/* - * make an Arena, and initialize it based upon the disk header and trailer. - */ -Arena* -initarena(Part *part, u64int base, u64int size, u32int blocksize) -{ - Arena *arena; - - arena = MKZ(Arena); - arena->part = part; - arena->blocksize = blocksize; - arena->clumpmax = arena->blocksize / ClumpInfoSize; - arena->base = base + blocksize; - arena->size = size - 2 * blocksize; - - if(loadarena(arena) < 0){ - seterr(ECorrupt, "arena header or trailer corrupted"); - freearena(arena); - return nil; - } - if(okarena(arena) < 0){ - freearena(arena); - return nil; - } - - if(arena->sealed && scorecmp(zeroscore, arena->score)==0) - backsumarena(arena); - - return arena; -} - -void -freearena(Arena *arena) -{ - if(arena == nil) - return; - if(arena->cib.data != nil){ - putdblock(arena->cib.data); - arena->cib.data = nil; - } - free(arena); -} - -Arena* -newarena(Part *part, char *name, u64int base, u64int size, u32int blocksize) -{ - Arena *arena; - - if(nameok(name) < 0){ - seterr(EOk, "illegal arena name", name); - return nil; - } - arena = MKZ(Arena); - arena->part = part; - arena->version = ArenaVersion; - arena->blocksize = blocksize; - arena->clumpmax = arena->blocksize / ClumpInfoSize; - arena->base = base + blocksize; - arena->size = size - 2 * blocksize; - - namecp(arena->name, name); - - if(wbarena(arena)<0 || wbarenahead(arena)<0){ - freearena(arena); - return nil; - } - - return arena; -} - -int -readclumpinfo(Arena *arena, int clump, ClumpInfo *ci) -{ - CIBlock *cib, r; - - cib = getcib(arena, clump, 0, &r); - if(cib == nil) - return -1; - unpackclumpinfo(ci, &cib->data->data[cib->offset]); - putcib(arena, cib); - return 0; -} - -int -readclumpinfos(Arena *arena, int clump, ClumpInfo *cis, int n) -{ - CIBlock *cib, r; - int i; - - for(i = 0; i < n; i++){ - cib = getcib(arena, clump + i, 0, &r); - if(cib == nil) - break; - unpackclumpinfo(&cis[i], &cib->data->data[cib->offset]); - putcib(arena, cib); - } - return i; -} - -/* - * write directory information for one clump - * must be called the arena locked - */ -int -writeclumpinfo(Arena *arena, int clump, ClumpInfo *ci) -{ - CIBlock *cib, r; - - cib = getcib(arena, clump, 1, &r); - if(cib == nil) - return -1; - dirtydblock(cib->data, DirtyArenaCib); - packclumpinfo(ci, &cib->data->data[cib->offset]); - putcib(arena, cib); - return 0; -} - -u64int -arenadirsize(Arena *arena, u32int clumps) -{ - return ((clumps / arena->clumpmax) + 1) * arena->blocksize; -} - -/* - * read a clump of data - * n is a hint of the size of the data, not including the header - * make sure it won't run off the end, then return the number of bytes actually read - */ -u32int -readarena(Arena *arena, u64int aa, u8int *buf, long n) -{ - DBlock *b; - u64int a; - u32int blocksize, off, m; - long nn; - - if(n == 0) - return -1; - - qlock(&arena->lock); - a = arena->size - arenadirsize(arena, arena->clumps); - qunlock(&arena->lock); - if(aa >= a){ - seterr(EOk, "reading beyond arena clump storage: clumps=%d aa=%lld a=%lld -1 clumps=%lld\n", arena->clumps, aa, a, arena->size - arenadirsize(arena, arena->clumps - 1)); - return -1; - } - if(aa + n > a) - n = a - aa; - - blocksize = arena->blocksize; - a = arena->base + aa; - off = a & (blocksize - 1); - a -= off; - nn = 0; - for(;;){ - b = getdblock(arena->part, a, 1); - if(b == nil) - return -1; - m = blocksize - off; - if(m > n - nn) - m = n - nn; - memmove(&buf[nn], &b->data[off], m); - putdblock(b); - nn += m; - if(nn == n) - break; - off = 0; - a += blocksize; - } - return n; -} - -/* - * write some data to the clump section at a given offset - * used to fix up corrupted arenas. - */ -u32int -writearena(Arena *arena, u64int aa, u8int *clbuf, u32int n) -{ - DBlock *b; - u64int a; - u32int blocksize, off, m; - long nn; - int ok; - - if(n == 0) - return -1; - - qlock(&arena->lock); - a = arena->size - arenadirsize(arena, arena->clumps); - if(aa >= a || aa + n > a){ - qunlock(&arena->lock); - seterr(EOk, "writing beyond arena clump storage"); - return -1; - } - - blocksize = arena->blocksize; - a = arena->base + aa; - off = a & (blocksize - 1); - a -= off; - nn = 0; - for(;;){ - b = getdblock(arena->part, a, off != 0 || off + n < blocksize); - if(b == nil){ - qunlock(&arena->lock); - return -1; - } - dirtydblock(b, DirtyArena); - m = blocksize - off; - if(m > n - nn) - m = n - nn; - memmove(&b->data[off], &clbuf[nn], m); - // ok = writepart(arena->part, a, b->data, blocksize); - ok = 0; - putdblock(b); - if(ok < 0){ - qunlock(&arena->lock); - return -1; - } - nn += m; - if(nn == n) - break; - off = 0; - a += blocksize; - } - qunlock(&arena->lock); - return n; -} - -/* - * allocate space for the clump and write it, - * updating the arena directory -ZZZ question: should this distinguish between an arena -filling up and real errors writing the clump? - */ -u64int -writeaclump(Arena *arena, Clump *c, u8int *clbuf) -{ - DBlock *b; - u64int a, aa; - u32int clump, n, nn, m, off, blocksize; - int ok; - - n = c->info.size + ClumpSize; - qlock(&arena->lock); - aa = arena->used; - if(arena->sealed - || aa + n + U32Size + arenadirsize(arena, arena->clumps + 1) > arena->size){ - if(!arena->sealed) - sealarena(arena); - qunlock(&arena->lock); - return TWID64; - } - if(packclump(c, &clbuf[0]) < 0){ - qunlock(&arena->lock); - return TWID64; - } - - /* - * write the data out one block at a time - */ - blocksize = arena->blocksize; - a = arena->base + aa; - off = a & (blocksize - 1); - a -= off; - nn = 0; - for(;;){ - b = getdblock(arena->part, a, off != 0); - if(b == nil){ - qunlock(&arena->lock); - return TWID64; - } - dirtydblock(b, DirtyArena); - m = blocksize - off; - if(m > n - nn) - m = n - nn; - memmove(&b->data[off], &clbuf[nn], m); - // ok = writepart(arena->part, a, b->data, blocksize); - ok = 0; - putdblock(b); - if(ok < 0){ - qunlock(&arena->lock); - return TWID64; - } - nn += m; - if(nn == n) - break; - off = 0; - a += blocksize; - } - - arena->used += c->info.size + ClumpSize; - arena->uncsize += c->info.uncsize; - if(c->info.size < c->info.uncsize) - arena->cclumps++; - - clump = arena->clumps++; - if(arena->clumps == 0) - sysfatal("clumps wrapped\n"); - arena->wtime = now(); - if(arena->ctime == 0) - arena->ctime = arena->wtime; - - writeclumpinfo(arena, clump, &c->info); -//ZZZ make this an enum param - if((clump & 0x1ff) == 0x1ff){ - flushciblocks(arena); - wbarena(arena); - } - - qunlock(&arena->lock); - return aa; -} - -/* - * once sealed, an arena never has any data added to it. - * it should only be changed to fix errors. - * this also syncs the clump directory. - */ -static void -sealarena(Arena *arena) -{ - flushciblocks(arena); - flushdcache(); - arena->sealed = 1; - wbarena(arena); - backsumarena(arena); -} - -void -backsumarena(Arena *arena) -{ - ASum *as; - - as = MK(ASum); - if(as == nil) - return; - qlock(&sumlock); - as->arena = arena; - as->next = sumq; - sumq = as; - rwakeup(&sumwait); - qunlock(&sumlock); -} - -static void -sumproc(void *unused) -{ - ASum *as; - Arena *arena; - - USED(unused); - - for(;;){ - qlock(&sumlock); - while(sumq == nil) - rsleep(&sumwait); - as = sumq; - sumq = as->next; - qunlock(&sumlock); - arena = as->arena; - free(as); - - sumarena(arena); - } -} - -void -sumarena(Arena *arena) -{ - ZBlock *b; - DigestState s; - u64int a, e; - u32int bs; - u8int score[VtScoreSize]; - - bs = MaxIoSize; - if(bs < arena->blocksize) - bs = arena->blocksize; - - /* - * read & sum all blocks except the last one - */ - memset(&s, 0, sizeof s); - b = alloczblock(bs, 0); - e = arena->base + arena->size; - for(a = arena->base - arena->blocksize; a + arena->blocksize <= e; a += bs){ - if(a + bs > e) - bs = arena->blocksize; - if(readpart(arena->part, a, b->data, bs) < 0) - goto ReadErr; - sha1(b->data, bs, nil, &s); - } - - /* - * the last one is special, since it may already have the checksum included - */ - bs = arena->blocksize; - if(readpart(arena->part, e, b->data, bs) < 0){ -ReadErr: - logerr(EOk, "sumarena can't sum %s, read at %lld failed: %r", arena->name, a); - freezblock(b); - return; - } - - sha1(b->data, bs-VtScoreSize, nil, &s); - sha1(zeroscore, VtScoreSize, nil, &s); - sha1(nil, 0, score, &s); - - /* - * check for no checksum or the same - * - * the writepart is okay because we flushed the dcache in sealarena - */ - if(scorecmp(score, &b->data[bs - VtScoreSize]) != 0){ - if(scorecmp(zeroscore, &b->data[bs - VtScoreSize]) != 0) - logerr(EOk, "overwriting mismatched checksums for arena=%s, found=%V calculated=%V", - arena->name, &b->data[bs - VtScoreSize], score); - scorecp(&b->data[bs - VtScoreSize], score); - if(writepart(arena->part, e, b->data, bs) < 0) - logerr(EOk, "sumarena can't write sum for %s: %r", arena->name); - } - freezblock(b); - - qlock(&arena->lock); - scorecp(arena->score, score); - qunlock(&arena->lock); -} - -/* - * write the arena trailer block to the partition - */ -int -wbarena(Arena *arena) -{ - DBlock *b; - int bad; - - if((b = getdblock(arena->part, arena->base + arena->size, 0)) == nil){ - logerr(EAdmin, "can't write arena trailer: %r"); - return -1; - } - dirtydblock(b, DirtyArenaTrailer); - bad = okarena(arena)<0 || packarena(arena, b->data)<0; - putdblock(b); - if(bad) - return -1; - return 0; -} - -int -wbarenahead(Arena *arena) -{ - ZBlock *b; - ArenaHead head; - int bad; - - namecp(head.name, arena->name); - head.version = arena->version; - head.size = arena->size + 2 * arena->blocksize; - head.blocksize = arena->blocksize; - b = alloczblock(arena->blocksize, 1); - if(b == nil){ - logerr(EAdmin, "can't write arena header: %r"); -///ZZZ add error message? - return -1; - } - /* - * this writepart is okay because it only happens - * during initialization. - */ - bad = packarenahead(&head, b->data)<0 || - writepart(arena->part, arena->base - arena->blocksize, b->data, arena->blocksize)<0; - freezblock(b); - if(bad) - return -1; - return 0; -} - -/* - * read the arena header and trailer blocks from disk - */ -static int -loadarena(Arena *arena) -{ - ArenaHead head; - ZBlock *b; - - b = alloczblock(arena->blocksize, 0); - if(b == nil) - return -1; - if(readpart(arena->part, arena->base + arena->size, b->data, arena->blocksize) < 0){ - freezblock(b); - return -1; - } - if(unpackarena(arena, b->data) < 0){ - freezblock(b); - return -1; - } - if(arena->version != ArenaVersion){ - seterr(EAdmin, "unknown arena version %d", arena->version); - freezblock(b); - return -1; - } - scorecp(arena->score, &b->data[arena->blocksize - VtScoreSize]); - - if(readpart(arena->part, arena->base - arena->blocksize, b->data, arena->blocksize) < 0){ - logerr(EAdmin, "can't read arena header: %r"); - freezblock(b); - return 0; - } - if(unpackarenahead(&head, b->data) < 0) - logerr(ECorrupt, "corrupted arena header: %r"); - else if(namecmp(arena->name, head.name)!=0 - || arena->version != head.version - || arena->blocksize != head.blocksize - || arena->size + 2 * arena->blocksize != head.size) - logerr(ECorrupt, "arena header inconsistent with arena data"); - freezblock(b); - - return 0; -} - -static int -okarena(Arena *arena) -{ - u64int dsize; - int ok; - - ok = 0; - dsize = arenadirsize(arena, arena->clumps); - if(arena->used + dsize > arena->size){ - seterr(ECorrupt, "arena used > size"); - ok = -1; - } - - if(arena->cclumps > arena->clumps) - logerr(ECorrupt, "arena has more compressed clumps than total clumps"); - - if(arena->uncsize + arena->clumps * ClumpSize + arena->blocksize < arena->used) - logerr(ECorrupt, "arena uncompressed size inconsistent with used space %lld %d %lld", arena->uncsize, arena->clumps, arena->used); - - if(arena->ctime > arena->wtime) - logerr(ECorrupt, "arena creation time after last write time"); - - return ok; -} - -static CIBlock* -getcib(Arena *arena, int clump, int writing, CIBlock *rock) -{ - int read; - CIBlock *cib; - u32int block, off; - - if(clump >= arena->clumps){ - seterr(EOk, "clump directory access out of range"); - return nil; - } - block = clump / arena->clumpmax; - off = (clump - block * arena->clumpmax) * ClumpInfoSize; - -/* - if(arena->cib.block == block - && arena->cib.data != nil){ - arena->cib.offset = off; - return &arena->cib; - } - - if(writing){ - flushciblocks(arena); - cib = &arena->cib; - }else - cib = rock; -*/ - cib = rock; - - qlock(&stats.lock); - stats.cireads++; - qunlock(&stats.lock); - - cib->block = block; - cib->offset = off; - - read = 1; - if(writing && off == 0 && clump == arena->clumps-1) - read = 0; - - cib->data = getdblock(arena->part, arena->base + arena->size - (block + 1) * arena->blocksize, read); - if(cib->data == nil) - return nil; - return cib; -} - -static void -putcib(Arena *arena, CIBlock *cib) -{ - if(cib != &arena->cib){ - putdblock(cib->data); - cib->data = nil; - } -} - -/* - * must be called with arena locked - * - * cache turned off now that dcache does write caching too. - */ -int -flushciblocks(Arena *arena) -{ - int ok; - - if(arena->cib.data == nil) - return 0; - qlock(&stats.lock); - stats.ciwrites++; - qunlock(&stats.lock); -// ok = writepart(arena->part, arena->base + arena->size - (arena->cib.block + 1) * arena->blocksize, arena->cib.data->data, arena->blocksize); - ok = 0; - if(ok < 0) - seterr(EAdmin, "failed writing arena directory block"); - putdblock(arena->cib.data); - arena->cib.data = nil; - return ok; -} blob - 5275b938cf76762d7a90f05eb22280d4ea09e1c5 (mode 644) blob + /dev/null --- src/cmd/venti/arenas.c +++ /dev/null @@ -1,404 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -typedef struct AHash AHash; - -/* - * hash table for finding arena's based on their names. - */ -struct AHash -{ - AHash *next; - Arena *arena; -}; - -enum -{ - AHashSize = 512 -}; - -static AHash *ahash[AHashSize]; - -static u32int -hashstr(char *s) -{ - u32int h; - int c; - - h = 0; - for(; c = *s; s++){ - c ^= c << 6; - h += (c << 11) ^ (c >> 1); - c = *s; - h ^= (c << 14) + (c << 7) + (c << 4) + c; - } - return h; -} - -int -addarena(Arena *arena) -{ - AHash *a; - u32int h; - - h = hashstr(arena->name) & (AHashSize - 1); - a = MK(AHash); - if(a == nil) - return -1; - a->arena = arena; - a->next = ahash[h]; - ahash[h] = a; - return 0; -} - -Arena* -findarena(char *name) -{ - AHash *a; - u32int h; - - h = hashstr(name) & (AHashSize - 1); - for(a = ahash[h]; a != nil; a = a->next) - if(strcmp(a->arena->name, name) == 0) - return a->arena; - return nil; -} - -int -delarena(Arena *arena) -{ - AHash *a, *last; - u32int h; - - h = hashstr(arena->name) & (AHashSize - 1); - last = nil; - for(a = ahash[h]; a != nil; a = a->next){ - if(a->arena == arena){ - if(last != nil) - last->next = a->next; - else - ahash[h] = a->next; - free(a); - return 0; - } - last = a; - } - return -1; -} - -ArenaPart* -initarenapart(Part *part) -{ - AMapN amn; - ArenaPart *ap; - ZBlock *b; - u32int i; - int ok; - - b = alloczblock(HeadSize, 0); - if(b == nil || readpart(part, PartBlank, b->data, HeadSize) < 0){ - seterr(EAdmin, "can't read arena partition header: %r"); - return nil; - } - - ap = MKZ(ArenaPart); - if(ap == nil){ - freezblock(b); - return nil; - } - ap->part = part; - ok = unpackarenapart(ap, b->data); - freezblock(b); - if(ok < 0){ - seterr(ECorrupt, "corrupted arena partition header: %r"); - freearenapart(ap, 0); - return nil; - } - - ap->tabbase = (PartBlank + HeadSize + ap->blocksize - 1) & ~(ap->blocksize - 1); - if(ap->version != ArenaPartVersion){ - seterr(ECorrupt, "unknown arena partition version %d", ap->version); - freearenapart(ap, 0); - return nil; - } - if(ap->blocksize & (ap->blocksize - 1)){ - seterr(ECorrupt, "illegal non-power-of-2 block size %d\n", ap->blocksize); - freearenapart(ap, 0); - return nil; - } - if(ap->tabbase >= ap->arenabase){ - seterr(ECorrupt, "arena partition table overlaps with arena storage"); - freearenapart(ap, 0); - return nil; - } - ap->tabsize = ap->arenabase - ap->tabbase; - partblocksize(part, ap->blocksize); - ap->size = ap->part->size & ~(u64int)(ap->blocksize - 1); - - if(readarenamap(&amn, part, ap->tabbase, ap->tabsize) < 0){ - freearenapart(ap, 0); - return nil; - } - ap->narenas = amn.n; - ap->map = amn.map; - if(okamap(ap->map, ap->narenas, ap->arenabase, ap->size, "arena table") < 0){ - freearenapart(ap, 0); - return nil; - } - - ap->arenas = MKNZ(Arena*, ap->narenas); - for(i = 0; i < ap->narenas; i++){ - ap->arenas[i] = initarena(part, ap->map[i].start, ap->map[i].stop - ap->map[i].start, ap->blocksize); - if(ap->arenas[i] == nil){ - freearenapart(ap, 1); - return nil; - } - if(namecmp(ap->map[i].name, ap->arenas[i]->name) != 0){ - seterr(ECorrupt, "arena name mismatches with expected name: %s vs. %s", - ap->map[i].name, ap->arenas[i]->name); - freearenapart(ap, 1); - return nil; - } - if(findarena(ap->arenas[i]->name)){ - seterr(ECorrupt, "duplicate arena name %s in %s", - ap->map[i].name, ap->part->name); - freearenapart(ap, 1); - return nil; - } - } - - for(i = 0; i < ap->narenas; i++) - addarena(ap->arenas[i]); - - return ap; -} - -ArenaPart* -newarenapart(Part *part, u32int blocksize, u32int tabsize) -{ - ArenaPart *ap; - - if(blocksize & (blocksize - 1)){ - seterr(ECorrupt, "illegal non-power-of-2 block size %d\n", blocksize); - return nil; - } - ap = MKZ(ArenaPart); - if(ap == nil) - return nil; - - ap->version = ArenaPartVersion; - ap->part = part; - ap->blocksize = blocksize; - partblocksize(part, blocksize); - ap->size = part->size & ~(u64int)(blocksize - 1); - ap->tabbase = (PartBlank + HeadSize + blocksize - 1) & ~(blocksize - 1); - ap->arenabase = (ap->tabbase + tabsize + blocksize - 1) & ~(blocksize - 1); - ap->tabsize = ap->arenabase - ap->tabbase; - ap->narenas = 0; - - if(wbarenapart(ap) < 0){ - freearenapart(ap, 0); - return nil; - } - - return ap; -} - -int -wbarenapart(ArenaPart *ap) -{ - ZBlock *b; - - if(okamap(ap->map, ap->narenas, ap->arenabase, ap->size, "arena table") < 0) - return -1; - b = alloczblock(HeadSize, 1); - if(b == nil) -//ZZZ set error message? - return -1; - - if(packarenapart(ap, b->data) < 0){ - seterr(ECorrupt, "can't make arena partition header: %r"); - freezblock(b); - return -1; - } - if(writepart(ap->part, PartBlank, b->data, HeadSize) < 0){ - seterr(EAdmin, "can't write arena partition header: %r"); - freezblock(b); - return -1; - } - freezblock(b); - - return wbarenamap(ap->map, ap->narenas, ap->part, ap->tabbase, ap->tabsize); -} - -void -freearenapart(ArenaPart *ap, int freearenas) -{ - int i; - - if(ap == nil) - return; - if(freearenas){ - for(i = 0; i < ap->narenas; i++){ - if(ap->arenas[i] == nil) - continue; - delarena(ap->arenas[i]); - freearena(ap->arenas[i]); - } - } - free(ap->map); - free(ap->arenas); - free(ap); -} - -int -okamap(AMap *am, int n, u64int start, u64int stop, char *what) -{ - u64int last; - u32int i; - - last = start; - for(i = 0; i < n; i++){ - if(am[i].start < last){ - if(i == 0) - seterr(ECorrupt, "invalid start address in %s", what); - else - seterr(ECorrupt, "overlapping ranges in %s", what); - return -1; - } - if(am[i].stop < am[i].start){ - seterr(ECorrupt, "invalid range in %s", what); - return -1; - } - last = am[i].stop; - } - if(last > stop){ - seterr(ECorrupt, "invalid ending address in %s", what); - return -1; - } - return 0; -} - -int -maparenas(AMap *am, Arena **arenas, int n, char *what) -{ - u32int i; - - for(i = 0; i < n; i++){ - arenas[i] = findarena(am[i].name); - if(arenas[i] == nil){ - seterr(EAdmin, "can't find arena '%s' for '%s'\n", am[i].name, what); - return -1; - } - } - return 0; -} - -int -readarenamap(AMapN *amn, Part *part, u64int base, u32int size) -{ - IFile f; - u32int ok; - - if(partifile(&f, part, base, size) < 0) - return -1; - ok = parseamap(&f, amn); - freeifile(&f); - return ok; -} - -int -wbarenamap(AMap *am, int n, Part *part, u64int base, u64int size) -{ - Fmt f; - ZBlock *b; - - b = alloczblock(size, 1); - if(b == nil) - return -1; - - fmtzbinit(&f, b); - - if(outputamap(&f, am, n) < 0){ - seterr(ECorrupt, "arena set size too small"); - freezblock(b); - return -1; - } - if(writepart(part, base, b->data, size) < 0){ - seterr(EAdmin, "can't write arena set: %r"); - freezblock(b); - return -1; - } - freezblock(b); - return 0; -} - -/* - * amap: n '\n' amapelem * n - * n: u32int - * amapelem: name '\t' astart '\t' asize '\n' - * astart, asize: u64int - */ -int -parseamap(IFile *f, AMapN *amn) -{ - AMap *am; - u64int v64; - u32int v; - char *s, *flds[4]; - int i, n; - - /* - * arenas - */ - if(ifileu32int(f, &v) < 0){ - seterr(ECorrupt, "syntax error: bad number of elements in %s", f->name); - return -1; - } - n = v; - if(n > MaxAMap){ - seterr(ECorrupt, "illegal number of elements in %s", f->name); - return -1; - } - am = MKNZ(AMap, n); - if(am == nil) - return -1; - for(i = 0; i < n; i++){ - s = ifileline(f); - if(s == nil || getfields(s, flds, 4, 0, "\t") != 3) - return -1; - if(nameok(flds[0]) < 0) - return -1; - namecp(am[i].name, flds[0]); - if(stru64int(flds[1], &v64) < 0){ - seterr(ECorrupt, "syntax error: bad arena base address in %s", f->name); - free(am); - return -1; - } - am[i].start = v64; - if(stru64int(flds[2], &v64) < 0){ - seterr(ECorrupt, "syntax error: bad arena size in %s", f->name); - free(am); - return -1; - } - am[i].stop = v64; - } - - amn->map = am; - amn->n = n; - return 0; -} - -int -outputamap(Fmt *f, AMap *am, int n) -{ - int i; - - if(fmtprint(f, "%ud\n", n) < 0) - return -1; - for(i = 0; i < n; i++) - if(fmtprint(f, "%s\t%llud\t%llud\n", am[i].name, am[i].start, am[i].stop) < 0) - return -1; - return 0; -} blob - e5aed260f741ed0df517b109d359b47f3445e714 (mode 644) blob + /dev/null --- src/cmd/venti/buildbuck.c +++ /dev/null @@ -1,112 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -struct IEStream -{ - Part *part; - u64int off; /* read position within part */ - u64int n; /* number of valid ientries left to read */ - u32int size; /* allocated space in buffer */ - u8int *buf; - u8int *pos; /* current place in buffer */ - u8int *epos; /* end of valid buffer contents */ -}; - -IEStream* -initiestream(Part *part, u64int off, u64int clumps, u32int size) -{ - IEStream *ies; - -//ZZZ out of memory? - ies = MKZ(IEStream); - ies->buf = MKN(u8int, size); - ies->epos = ies->buf; - ies->pos = ies->epos; - ies->off = off; - ies->n = clumps; - ies->size = size; - ies->part = part; - return ies; -} - -void -freeiestream(IEStream *ies) -{ - if(ies == nil) - return; - free(ies->buf); - free(ies); -} - -static u8int* -peekientry(IEStream *ies) -{ - u32int n, nn; - - n = ies->epos - ies->pos; - if(n < IEntrySize){ - memmove(ies->buf, ies->pos, n); - ies->epos = &ies->buf[n]; - ies->pos = ies->buf; - nn = ies->size; - if(nn > ies->n * IEntrySize) - nn = ies->n * IEntrySize; - nn -= n; - if(nn == 0) - return nil; - if(readpart(ies->part, ies->off, ies->epos, nn) < 0){ - seterr(EOk, "can't read sorted index entries: %r"); - return nil; - } - ies->epos += nn; - ies->off += nn; - } - return ies->pos; -} - -static u32int -iebuck(Index *ix, u8int *b) -{ - return hashbits(b, 32) / ix->div; -} - -u32int -buildbucket(Index *ix, IEStream *ies, IBucket *ib) -{ - IEntry ie1, ie2; - u8int *b; - u32int buck; - - buck = TWID32; - ib->n = 0; - ib->depth = 0; - while(ies->n){ - b = peekientry(ies); - if(b == nil) - return TWID32; -//fprint(2, "b=%p ies->n=%lld ib.n=%d buck=%d score=%V\n", b, ies->n, ib->n, iebuck(ix, b), b); - if(ib->n == 0) - buck = iebuck(ix, b); - else{ - if(buck != iebuck(ix, b)) - break; - if(ientrycmp(&ib->data[(ib->n - 1)* IEntrySize], b) == 0){ - /* - * guess that the larger address is the correct one to use - */ - unpackientry(&ie1, &ib->data[(ib->n - 1)* IEntrySize]); - unpackientry(&ie2, b); - seterr(EOk, "duplicate index entry for score=%V type=%d\n", ie1.score, ie1.ia.type); - ib->n--; - if(ie1.ia.addr > ie2.ia.addr) - memmove(b, &ib->data[ib->n * IEntrySize], IEntrySize); - } - } - memmove(&ib->data[ib->n * IEntrySize], b, IEntrySize); - ib->n++; - ies->n--; - ies->pos += IEntrySize; - } - return buck; -} blob - 8058ba098653cef50a84f99c22b5148590cd2116 (mode 644) blob + /dev/null --- src/cmd/venti/buildindex.c +++ /dev/null @@ -1,139 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -static int -writebucket(Index *ix, u32int buck, IBucket *ib, ZBlock *b) -{ - ISect *is; - - is = findibucket(ix, buck, &buck); - if(is == nil) - return -1; - qlock(&stats.lock); - stats.indexwrites++; - qunlock(&stats.lock); - packibucket(ib, b->data); - return writepart(is->part, is->blockbase + ((u64int)buck << is->blocklog), b->data, is->blocksize); -} - -static int -buildindex(Index *ix, Part *part, u64int off, u64int clumps, int zero) -{ - IEStream *ies; - IBucket ib, zib; - ZBlock *z, *b; - u32int next, buck; - int ok; - u64int found = 0; - -//ZZZ make buffer size configurable - b = alloczblock(ix->blocksize, 0); - z = alloczblock(ix->blocksize, 1); - ies = initiestream(part, off, clumps, 64*1024); - if(b == nil || z == nil || ies == nil){ - ok = 0; - goto breakout; - return -1; - } - ok = 0; - next = 0; - ib.data = b->data + IBucketSize; - zib.data = z->data + IBucketSize; - zib.n = 0; - zib.depth = 0; - for(;;){ - buck = buildbucket(ix, ies, &ib); - found += ib.n; - if(zero){ - for(; next != buck; next++){ - if(next == ix->buckets){ - if(buck != TWID32){ - fprint(2, "bucket out of range\n"); - ok = -1; - } - goto breakout; - } - if(writebucket(ix, next, &zib, z) < 0){ - fprint(2, "can't write zero bucket to buck=%d: %r", next); - ok = -1; - } - } - } - if(buck >= ix->buckets){ - if(buck == TWID32) - break; - fprint(2, "bucket out of range\n"); - ok = -1; - goto breakout; - } - if(writebucket(ix, buck, &ib, b) < 0){ - fprint(2, "bad bucket found=%lld: %r\n", found); - ok = -1; - } - next = buck + 1; - } -breakout:; - fprint(2, "constructed index with %lld entries\n", found); - freeiestream(ies); - freezblock(z); - freezblock(b); - return ok; -} - -void -usage(void) -{ - fprint(2, "usage: buildindex [-Z] [-B blockcachesize] config tmppart\n"); - threadexitsall(0); -} - -void -threadmain(int argc, char *argv[]) -{ - Part *part; - u64int clumps, base; - u32int bcmem; - int zero; - - zero = 1; - bcmem = 0; - ARGBEGIN{ - case 'B': - bcmem = unittoull(ARGF()); - break; - case 'Z': - zero = 0; - break; - default: - usage(); - break; - }ARGEND - - if(argc != 2) - usage(); - - if(initventi(argv[0]) < 0) - sysfatal("can't init venti: %r"); - - if(bcmem < maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16)) - bcmem = maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16); - fprint(2, "initialize %d bytes of disk block cache\n", bcmem); - initdcache(bcmem); - - fprint(2, "building a new index %s using %s for temporary storage\n", mainindex->name, argv[1]); - - part = initpart(argv[1], 1); - if(part == nil) - sysfatal("can't initialize temporary partition: %r"); - - clumps = sortrawientries(mainindex, part, &base); - if(clumps == TWID64) - sysfatal("can't build sorted index: %r"); - fprint(2, "found and sorted index entries for clumps=%lld at %lld\n", clumps, base); - - if(buildindex(mainindex, part, base, clumps, zero) < 0) - sysfatal("can't build new index: %r"); - - threadexitsall(0); -} blob - ae7c641acf9ab772ca8d0363cf78f0e112596dca (mode 644) blob + /dev/null --- src/cmd/venti/checkarenas.c +++ /dev/null @@ -1,120 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -static int verbose; - -static void -checkarena(Arena *arena, int scan, int fix) -{ - Arena old; - int err, e; - - if(verbose && arena->clumps) - printarena(2, arena); - - old = *arena; - - if(scan){ - arena->used = 0; - arena->clumps = 0; - arena->cclumps = 0; - arena->uncsize = 0; - } - - err = 0; - for(;;){ - e = syncarena(arena, 1000, 0, fix); - err |= e; - if(!(e & SyncHeader)) - break; - if(verbose && arena->clumps) - fprint(2, "."); - } - if(verbose && arena->clumps) - fprint(2, "\n"); - - err &= ~SyncHeader; - if(arena->used != old.used - || arena->clumps != old.clumps - || arena->cclumps != old.cclumps - || arena->uncsize != old.uncsize){ - fprint(2, "incorrect arena header fields\n"); - printarena(2, arena); - err |= SyncHeader; - } - - if(!err || !fix) - return; - - fprint(2, "writing fixed arena header fields\n"); - if(wbarena(arena) < 0) - fprint(2, "arena header write failed: %r\n"); -} - -void -usage(void) -{ - fprint(2, "usage: checkarenas [-afv] file\n"); - threadexitsall(0); -} - -void -threadmain(int argc, char *argv[]) -{ - ArenaPart *ap; - Part *part; - char *file; - int i, fix, scan; - - fmtinstall('V', vtscorefmt); - - statsinit(); - - fix = 0; - scan = 0; - ARGBEGIN{ - case 'f': - fix++; - break; - case 'a': - scan = 1; - break; - case 'v': - verbose++; - break; - default: - usage(); - break; - }ARGEND - - if(!fix) - readonly = 1; - - if(argc != 1) - usage(); - - file = argv[0]; - - part = initpart(file, 0); - if(part == nil) - sysfatal("can't open partition %s: %r", file); - - ap = initarenapart(part); - if(ap == nil) - sysfatal("can't initialize arena partition in %s: %r", file); - - if(verbose > 1){ - printarenapart(2, ap); - fprint(2, "\n"); - } - - initdcache(8 * MaxDiskBlock); - - for(i = 0; i < ap->narenas; i++) - checkarena(ap->arenas[i], scan, fix); - - if(verbose > 1) - printstats(); - threadexitsall(0); -} blob - 34edb3708475fc66bab98e394aac46fa84e49c73 (mode 644) blob + /dev/null --- src/cmd/venti/checkindex.c +++ /dev/null @@ -1,183 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -static int -checkbucket(Index *ix, u32int buck, IBucket *ib) -{ - ISect *is; - DBlock *eb; - IBucket eib; - IEntry ie, eie; - int i, ei, ok, c; - - is = findibucket(ix, buck, &buck); - eb = getdblock(is->part, is->blockbase + ((u64int)buck << is->blocklog), 1); - if(eb == nil) - return -1; - unpackibucket(&eib, eb->data); - - ok = 0; - ei = 0; - for(i = 0; i < ib->n; i++){ - while(ei < eib.n){ - c = ientrycmp(&ib->data[i * IEntrySize], &eib.data[ei * IEntrySize]); - if(c == 0){ - unpackientry(&ie, &ib->data[i * IEntrySize]); - unpackientry(&eie, &eib.data[ei * IEntrySize]); - if(iaddrcmp(&ie.ia, &eie.ia) != 0){ - fprint(2, "bad entry in index for score=%V\n", &ib->data[i * IEntrySize]); - fprint(2, "\taddr=%lld type=%d size=%d blocks=%d\n", - ie.ia.addr, ie.ia.type, ie.ia.size, ie.ia.blocks); - fprint(2, "\taddr=%lld type=%d size=%d blocks=%d\n", - eie.ia.addr, eie.ia.type, eie.ia.size, eie.ia.blocks); - } - ei++; - goto cont; - } - if(c < 0) - break; -if(1) - fprint(2, "spurious entry in index for score=%V type=%d\n", - &eib.data[ei * IEntrySize], eib.data[ei * IEntrySize + IEntryTypeOff]); - ei++; - ok = -1; - } - fprint(2, "missing entry in index for score=%V type=%d\n", - &ib->data[i * IEntrySize], ib->data[i * IEntrySize + IEntryTypeOff]); - ok = -1; - cont:; - } - for(; ei < eib.n; ei++){ -if(1) fprint(2, "spurious entry in index for score=%V; found %d entries expected %d\n", - &eib.data[ei * IEntrySize], eib.n, ib->n); - ok = -1; - break; - } - putdblock(eb); - return ok; -} - -int -checkindex(Index *ix, Part *part, u64int off, u64int clumps, int zero) -{ - IEStream *ies; - IBucket ib, zib; - ZBlock *z, *b; - u32int next, buck; - int ok, bok; -u64int found = 0; - -//ZZZ make buffer size configurable - b = alloczblock(ix->blocksize, 0); - z = alloczblock(ix->blocksize, 1); - ies = initiestream(part, off, clumps, 64*1024); - if(b == nil || z == nil || ies == nil){ - ok = -1; - goto breakout; - return -1; - } - ok = 0; - next = 0; - ib.data = b->data; - zib.data = z->data; - zib.n = 0; - zib.depth = 0; - for(;;){ - buck = buildbucket(ix, ies, &ib); - found += ib.n; - if(zero){ - for(; next != buck; next++){ - if(next == ix->buckets){ - if(buck != TWID32) - fprint(2, "bucket out of range\n"); - goto breakout; - } - bok = checkbucket(ix, next, &zib); - if(bok < 0){ - fprint(2, "bad bucket=%d found: %r\n", next); - ok = -1; - } - } - } - if(buck >= ix->buckets){ - if(buck == TWID32) - break; - fprint(2, "bucket out of range\n"); - ok = -1; - goto breakout; - } - bok = checkbucket(ix, buck, &ib); - if(bok < 0){ - fprint(2, "bad bucket found=%lld: %r\n", found); - ok = -1; - } - next = buck + 1; - } -breakout:; -fprint(2, "found %lld entries in sorted list\n", found); - freeiestream(ies); - freezblock(z); - freezblock(b); - return ok; -} - -void -usage(void) -{ - fprint(2, "usage: checkindex [-f] [-B blockcachesize] config tmp\n"); - threadexitsall(0); -} - -void -threadmain(int argc, char *argv[]) -{ - Part *part; - u64int clumps, base; - u32int bcmem; - int fix, skipz; - - fix = 0; - bcmem = 0; - skipz = 0; - ARGBEGIN{ - case 'B': - bcmem = unittoull(ARGF()); - break; - case 'f': - fix++; - break; - case 'Z': - skipz = 1; - break; - default: - usage(); - break; - }ARGEND - - if(!fix) - readonly = 1; - - if(argc != 2) - usage(); - - if(initventi(argv[0]) < 0) - sysfatal("can't init venti: %r"); - - if(bcmem < maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16)) - bcmem = maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16); - fprint(2, "initialize %d bytes of disk block cache\n", bcmem); - initdcache(bcmem); - - part = initpart(argv[1], 1); - if(part == nil) - sysfatal("can't initialize temporary partition: %r"); - - clumps = sortrawientries(mainindex, part, &base); - if(clumps == TWID64) - sysfatal("can't build sorted index: %r"); - fprint(2, "found and sorted index entries for clumps=%lld at %lld\n", clumps, base); - checkindex(mainindex, part, base, clumps, !skipz); - - threadexitsall(0); -} blob - 1aa81bb1b019fce7913841c1c6828e1fca0a6278 (mode 644) blob + /dev/null --- src/cmd/venti/clump.c +++ /dev/null @@ -1,200 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" -#include "whack.h" - -/* - * writes a lump to disk - * returns the address in amap of the clump - */ -int -storeclump(Index *ix, ZBlock *zb, u8int *sc, int type, u32int creator, IAddr *ia) -{ - ZBlock *cb; - Clump cl; - u64int a; - u8int bh[VtScoreSize]; - int size, dsize; - -if(0)print("storeclump %08x %p\n", mainindex->arenas[0], &cl); - size = zb->len; - if(size > VtMaxLumpSize){ - seterr(EStrange, "lump too large"); - return -1; - } - if(vttypevalid(type) < 0){ - seterr(EStrange, "invalid lump type"); - return -1; - } - - if(1){ - scoremem(bh, zb->data, size); - if(scorecmp(sc, bh) != 0){ - seterr(ECorrupt, "storing clump: corrupted; expected=%V got=%V, size=%d", sc, bh, size); - return -1; - } - } - - cb = alloczblock(size + ClumpSize, 0); - if(cb == nil) - return -1; - - cl.info.type = type; - cl.info.uncsize = size; - cl.creator = creator; - cl.time = now(); - scorecp(cl.info.score, sc); - -if(0)print("whackblock %08x %p\n", mainindex->arenas[0], &cl); - dsize=0; // dsize = whackblock(&cb->data[ClumpSize], zb->data, size); -if(0)print("whackedblock %08x %p\n", mainindex->arenas[0], &cl); - if(dsize > 0 && dsize < size){ - cl.encoding = ClumpECompress; - }else{ - cl.encoding = ClumpENone; - dsize = size; - memmove(&cb->data[ClumpSize], zb->data, size); - } - cl.info.size = dsize; - - a = writeiclump(ix, &cl, cb->data); - - freezblock(cb); - if(a == TWID64) - return -1; - - qlock(&stats.lock); - stats.clumpwrites++; - stats.clumpbwrites += size; - stats.clumpbcomp += dsize; - qunlock(&stats.lock); - - ia->addr = a; - ia->type = type; - ia->size = size; - ia->blocks = (dsize + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog; - - return 0; -} - -u32int -clumpmagic(Arena *arena, u64int aa) -{ - u8int buf[U32Size]; - - if(readarena(arena, aa, buf, U32Size) < 0) - return TWID32; - return unpackmagic(buf); -} - -/* - * fetch a block based at addr. - * score is filled in with the block's score. - * blocks is roughly the length of the clump on disk; - * if zero, the length is unknown. - */ -ZBlock* -loadclump(Arena *arena, u64int aa, int blocks, Clump *cl, u8int *score, int verify) -{ - Unwhack uw; - ZBlock *zb, *cb; - u8int bh[VtScoreSize], *buf; - u32int n; - int nunc; - - qlock(&stats.lock); - stats.clumpreads++; - qunlock(&stats.lock); - - if(blocks <= 0) - blocks = 1; - - cb = alloczblock(blocks << ABlockLog, 0); - if(cb == nil) - return nil; - n = readarena(arena, aa, cb->data, blocks << ABlockLog); - if(n < ClumpSize){ - if(n != 0) - seterr(ECorrupt, "loadclump read less than a header"); - freezblock(cb); - return nil; - } - if(unpackclump(cl, cb->data) < 0){ - freezblock(cb); - return nil; - } - n -= ClumpSize; - if(n < cl->info.size){ - freezblock(cb); - n = cl->info.size; - cb = alloczblock(n, 0); - if(cb == nil) - return nil; - if(readarena(arena, aa + ClumpSize, cb->data, n) != n){ - seterr(ECorrupt, "loadclump read too little data"); - freezblock(cb); - return nil; - } - buf = cb->data; - }else - buf = cb->data + ClumpSize; - - scorecp(score, cl->info.score); - - zb = alloczblock(cl->info.uncsize, 0); - if(zb == nil){ - freezblock(cb); - return nil; - } - switch(cl->encoding){ - case ClumpECompress: - unwhackinit(&uw); - nunc = unwhack(&uw, zb->data, cl->info.uncsize, buf, cl->info.size); - if(nunc != cl->info.uncsize){ - if(nunc < 0) - seterr(ECorrupt, "decompression failed: %s", uw.err); - else - seterr(ECorrupt, "decompression gave partial block: %d/%d\n", nunc, cl->info.uncsize); - freezblock(cb); - freezblock(zb); - return nil; - } - break; - case ClumpENone: - if(cl->info.size != cl->info.uncsize){ - seterr(ECorrupt, "loading clump: bad uncompressed size for uncompressed block"); - freezblock(cb); - freezblock(zb); - return nil; - } - memmove(zb->data, buf, cl->info.uncsize); - break; - default: - seterr(ECorrupt, "unknown encoding in loadlump"); - freezblock(cb); - freezblock(zb); - return nil; - } - freezblock(cb); - - if(verify){ - scoremem(bh, zb->data, cl->info.uncsize); - if(scorecmp(cl->info.score, bh) != 0){ - seterr(ECorrupt, "loading clump: corrupted; expected=%V got=%V", cl->info.score, bh); - freezblock(zb); - return nil; - } - if(vttypevalid(cl->info.type) < 0){ - seterr(ECorrupt, "loading lump: invalid lump type %d", cl->info.type); - freezblock(zb); - return nil; - } - } - - qlock(&stats.lock); - stats.clumpbreads += cl->info.size; - stats.clumpbuncomp += cl->info.uncsize; - qunlock(&stats.lock); - - return zb; -} blob - 5a448b677b5f02df21e250386e4868c624c2ffd3 (mode 644) blob + /dev/null --- src/cmd/venti/clumpstats.c +++ /dev/null @@ -1,128 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -int count[VtMaxLumpSize][VtMaxType]; - -enum -{ - ClumpChunks = 32*1024 -}; - -static int -readarenainfo(Arena *arena) -{ - ClumpInfo *ci, *cis; - u32int clump; - int i, n, ok; - - if(arena->clumps) - fprint(2, "reading directory for arena=%s with %d entries\n", arena->name, arena->clumps); - - cis = MKN(ClumpInfo, ClumpChunks); - ok = 0; - for(clump = 0; clump < arena->clumps; clump += n){ - n = ClumpChunks; - - if(n > arena->clumps - clump) - n = arena->clumps - clump; - - if((i=readclumpinfos(arena, clump, cis, n)) != n){ - seterr(EOk, "arena directory read failed %d not %d: %r", i, n); - ok = -1; - break; - } - - for(i = 0; i < n; i++){ - ci = &cis[i]; - if(ci->type >= VtMaxType || ci->uncsize >= VtMaxLumpSize) { - fprint(2, "bad clump: %d: type = %d: size = %d\n", clump+i, ci->type, ci->uncsize); - continue; - } -if(ci->uncsize == 422) -print("%s: %d: %V\n", arena->name, clump+i, ci->score); - count[ci->uncsize][ci->type]++; - } - } - free(cis); - if(ok < 0) - return TWID32; - return clump; -} - -static void -clumpstats(Index *ix) -{ - int ok; - ulong clumps, n; - int i, j, t; - - ok = 0; - clumps = 0; - for(i = 0; i < ix->narenas; i++){ - n = readarenainfo(ix->arenas[i]); - if(n == TWID32){ - ok = -1; - break; - } - clumps += n; - } - - if(ok < 0) - return; - - print("clumps = %ld\n", clumps); - for(i=0; inarenas + mainindex->nsects * 4 + 16)) - bcmem = maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16); - fprint(2, "initialize %d bytes of disk block cache\n", bcmem); - initdcache(bcmem); - - clumpstats(mainindex); - - threadexitsall(0); -} blob - cdcaad10ce97615649da45c5830d0049602f36b4 (mode 644) blob + /dev/null --- src/cmd/venti/config.c +++ /dev/null @@ -1,144 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -Index *mainindex; -int paranoid = 1; /* should verify hashes on disk read */ - -static ArenaPart *configarenas(char *file); -static ISect *configisect(char *file); - -int -initventi(char *file) -{ - Config conf; - - fmtinstall('V', vtscorefmt); - - statsinit(); - - if(file == nil){ - seterr(EOk, "no configuration file"); - return -1; - } - if(runconfig(file, &conf) < 0){ - seterr(EOk, "can't initialize venti: %r"); - return -1; - } - mainindex = initindex(conf.index, conf.sects, conf.nsects); - if(mainindex == nil) - return -1; - return 0; -} - -/* - * configs : - * | configs config - * config : "isect" filename - * | "arenas" filename - * | "index" name - * - * '#' and \n are comments - */ -enum -{ - MaxArgs = 2 -}; -int -runconfig(char *file, Config *config) -{ - ArenaPart **av; - ISect **sv; - IFile f; - char *s, *line, *flds[MaxArgs + 1]; - int i, ok; - - if(readifile(&f, file) < 0) - return -1; - config->index = nil; - config->naparts = 0; - config->aparts = nil; - config->nsects = 0; - config->sects = nil; - ok = -1; - line = nil; - for(;;){ - s = ifileline(&f); - if(s == nil){ - ok = 0; - break; - } - line = estrdup(s); - i = getfields(s, flds, MaxArgs + 1, 1, " \t\r"); - if(i == 2 && strcmp(flds[0], "isect") == 0){ - sv = MKN(ISect*, config->nsects + 1); - for(i = 0; i < config->nsects; i++) - sv[i] = config->sects[i]; - free(config->sects); - config->sects = sv; - config->sects[config->nsects] = configisect(flds[1]); - if(config->sects[config->nsects] == nil) - break; - config->nsects++; - }else if(i == 2 && strcmp(flds[0], "arenas") == 0){ - av = MKN(ArenaPart*, config->naparts + 1); - for(i = 0; i < config->naparts; i++) - av[i] = config->aparts[i]; - free(config->aparts); - config->aparts = av; - config->aparts[config->naparts] = configarenas(flds[1]); - if(config->aparts[config->naparts] == nil) - break; - config->naparts++; - }else if(i == 2 && strcmp(flds[0], "index") == 0){ - if(nameok(flds[1]) < 0){ - seterr(EAdmin, "illegal index name %s in config file %s", flds[1], config); - break; - } - if(config->index != nil){ - seterr(EAdmin, "duplicate indices in config file %s", config); - break; - } - config->index = estrdup(flds[1]); - }else{ - seterr(EAdmin, "illegal line '%s' in configuration file %s", line, config); - break; - } - free(line); - line = nil; - } - free(line); - freeifile(&f); - if(ok < 0){ - free(config->sects); - config->sects = nil; - free(config->aparts); - config->aparts = nil; - } - return ok; -} - -static ISect* -configisect(char *file) -{ - Part *part; - - fprint(2, "configure index section in %s\n", file); - - part = initpart(file, 0); - if(part == nil) - return nil; - return initisect(part); -} - -static ArenaPart* -configarenas(char *file) -{ - Part *part; - - fprint(2, "configure arenas in %s\n", file); - part = initpart(file, 0); - if(part == nil) - return nil; - return initarenapart(part); -} blob - 4688b076424cf03e28c99eaed0ca605b322b80c7 (mode 644) blob + /dev/null --- src/cmd/venti/conv.c +++ /dev/null @@ -1,500 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -/* - * disk structure conversion routines - */ -#define U8GET(p) ((p)[0]) -#define U16GET(p) (((p)[0]<<8)|(p)[1]) -#define U32GET(p) ((u32int)(((p)[0]<<24)|((p)[1]<<16)|((p)[2]<<8)|(p)[3])) -#define U64GET(p) (((u64int)U32GET(p)<<32)|(u64int)U32GET((p)+4)) - -#define U8PUT(p,v) (p)[0]=(v)&0xFF -#define U16PUT(p,v) (p)[0]=((v)>>8)&0xFF;(p)[1]=(v)&0xFF -#define U32PUT(p,v) (p)[0]=((v)>>24)&0xFF;(p)[1]=((v)>>16)&0xFF;(p)[2]=((v)>>8)&0xFF;(p)[3]=(v)&0xFF -#define U64PUT(p,v,t32) t32=(v)>>32;U32PUT(p,t32);t32=(v);U32PUT((p)+4,t32) - -static struct { - u32int m; - char *s; -} magics[] = { - ArenaPartMagic, "ArenaPartMagic", - ArenaHeadMagic, "ArenaHeadMagic", - ArenaMagic, "ArenaMagic", - ISectMagic, "ISectMagic", -}; - -static char* -fmtmagic(char *s, u32int m) -{ - int i; - - for(i=0; iversion = U32GET(p); - p += U32Size; - ap->blocksize = U32GET(p); - p += U32Size; - ap->arenabase = U32GET(p); - p += U32Size; - - if(buf + ArenaPartSize != p) - sysfatal("unpackarenapart unpacked wrong amount"); - - return 0; -} - -int -packarenapart(ArenaPart *ap, u8int *buf) -{ - u8int *p; - - p = buf; - - U32PUT(p, ArenaPartMagic); - p += U32Size; - U32PUT(p, ap->version); - p += U32Size; - U32PUT(p, ap->blocksize); - p += U32Size; - U32PUT(p, ap->arenabase); - p += U32Size; - - if(buf + ArenaPartSize != p) - sysfatal("packarenapart packed wrong amount"); - - return 0; -} - -int -unpackarena(Arena *arena, u8int *buf) -{ - u8int *p; - u32int m; - char fbuf[20]; - - p = buf; - - m = U32GET(p); - if(m != ArenaMagic){ - seterr(ECorrupt, "arena has wrong magic number: %s expected ArenaMagic (%lux)", fmtmagic(fbuf, m), m, ArenaMagic); - return -1; - } - p += U32Size; - arena->version = U32GET(p); - p += U32Size; - namecp(arena->name, (char*)p); - p += ANameSize; - arena->clumps = U32GET(p); - p += U32Size; - arena->cclumps = U32GET(p); - p += U32Size; - arena->ctime = U32GET(p); - p += U32Size; - arena->wtime = U32GET(p); - p += U32Size; - arena->used = U64GET(p); - p += U64Size; - arena->uncsize = U64GET(p); - p += U64Size; - arena->sealed = U8GET(p); - p += U8Size; - - if(buf + ArenaSize != p) - sysfatal("unpackarena unpacked wrong amount"); - - return 0; -} - -int -packarena(Arena *arena, u8int *buf) -{ - u8int *p; - u32int t32; - - p = buf; - - U32PUT(p, ArenaMagic); - p += U32Size; - U32PUT(p, arena->version); - p += U32Size; - namecp((char*)p, arena->name); - p += ANameSize; - U32PUT(p, arena->clumps); - p += U32Size; - U32PUT(p, arena->cclumps); - p += U32Size; - U32PUT(p, arena->ctime); - p += U32Size; - U32PUT(p, arena->wtime); - p += U32Size; - U64PUT(p, arena->used, t32); - p += U64Size; - U64PUT(p, arena->uncsize, t32); - p += U64Size; - U8PUT(p, arena->sealed); - p += U8Size; - - if(buf + ArenaSize != p) - sysfatal("packarena packed wrong amount"); - - return 0; -} - -int -unpackarenahead(ArenaHead *head, u8int *buf) -{ - u8int *p; - u32int m; - char fbuf[20]; - - p = buf; - - m = U32GET(p); - if(m != ArenaHeadMagic){ - seterr(ECorrupt, "arena has wrong magic number: %s expected ArenaHeadMagic (%lux)", - fmtmagic(fbuf, m), ArenaHeadMagic); - return -1; - } - p += U32Size; - head->version = U32GET(p); - p += U32Size; - namecp(head->name, (char*)p); - p += ANameSize; - head->blocksize = U32GET(p); - p += U32Size; - head->size = U64GET(p); - p += U64Size; - - if(buf + ArenaHeadSize != p) - sysfatal("unpackarenahead unpacked wrong amount"); - - return 0; -} - -int -packarenahead(ArenaHead *head, u8int *buf) -{ - u8int *p; - u32int t32; - - p = buf; - - U32PUT(p, ArenaHeadMagic); - p += U32Size; - U32PUT(p, head->version); - p += U32Size; - namecp((char*)p, head->name); - p += ANameSize; - U32PUT(p, head->blocksize); - p += U32Size; - U64PUT(p, head->size, t32); - p += U64Size; - - if(buf + ArenaHeadSize != p) - sysfatal("packarenahead packed wrong amount"); - - return 0; -} - -static int -checkclump(Clump *w) -{ - if(w->encoding == ClumpENone){ - if(w->info.size != w->info.uncsize){ - seterr(ECorrupt, "uncompressed wad size mismatch"); - return -1; - } - }else if(w->encoding == ClumpECompress){ - if(w->info.size >= w->info.uncsize){ - seterr(ECorrupt, "compressed lump has inconsistent block sizes %d %d", w->info.size, w->info.uncsize); - return -1; - } - }else{ - seterr(ECorrupt, "clump has illegal encoding"); - return -1; - } - - return 0; -} - -int -unpackclump(Clump *c, u8int *buf) -{ - u8int *p; - u32int magic; - - p = buf; - magic = U32GET(p); - if(magic != ClumpMagic){ - seterr(ECorrupt, "clump has bad magic number=%#8.8ux", magic); - return -1; - } - p += U32Size; - - c->info.type = vtfromdisktype(U8GET(p)); - p += U8Size; - c->info.size = U16GET(p); - p += U16Size; - c->info.uncsize = U16GET(p); - p += U16Size; - scorecp(c->info.score, p); - p += VtScoreSize; - - c->encoding = U8GET(p); - p += U8Size; - c->creator = U32GET(p); - p += U32Size; - c->time = U32GET(p); - p += U32Size; - - if(buf + ClumpSize != p) - sysfatal("unpackclump unpacked wrong amount"); - - return checkclump(c); -} - -int -packclump(Clump *c, u8int *buf) -{ - u8int *p; - - p = buf; - U32PUT(p, ClumpMagic); - p += U32Size; - - U8PUT(p, vttodisktype(c->info.type)); - p += U8Size; - U16PUT(p, c->info.size); - p += U16Size; - U16PUT(p, c->info.uncsize); - p += U16Size; - scorecp(p, c->info.score); - p += VtScoreSize; - - U8PUT(p, c->encoding); - p += U8Size; - U32PUT(p, c->creator); - p += U32Size; - U32PUT(p, c->time); - p += U32Size; - - if(buf + ClumpSize != p) - sysfatal("packclump packed wrong amount"); - - return checkclump(c); -} - -void -unpackclumpinfo(ClumpInfo *ci, u8int *buf) -{ - u8int *p; - - p = buf; - ci->type = vtfromdisktype(U8GET(p)); - p += U8Size; - ci->size = U16GET(p); - p += U16Size; - ci->uncsize = U16GET(p); - p += U16Size; - scorecp(ci->score, p); - p += VtScoreSize; - - if(buf + ClumpInfoSize != p) - sysfatal("unpackclumpinfo unpacked wrong amount"); -} - -void -packclumpinfo(ClumpInfo *ci, u8int *buf) -{ - u8int *p; - - p = buf; - U8PUT(p, vttodisktype(ci->type)); - p += U8Size; - U16PUT(p, ci->size); - p += U16Size; - U16PUT(p, ci->uncsize); - p += U16Size; - scorecp(p, ci->score); - p += VtScoreSize; - - if(buf + ClumpInfoSize != p) - sysfatal("packclumpinfo packed wrong amount"); -} - -int -unpackisect(ISect *is, u8int *buf) -{ - u8int *p; - u32int m; - char fbuf[20]; - - p = buf; - - - m = U32GET(p); - if(m != ISectMagic){ - seterr(ECorrupt, "index section has wrong magic number: %s expected ISectMagic (%lux)", - fmtmagic(fbuf, m), ISectMagic); - return -1; - } - p += U32Size; - is->version = U32GET(p); - p += U32Size; - namecp(is->name, (char*)p); - p += ANameSize; - namecp(is->index, (char*)p); - p += ANameSize; - is->blocksize = U32GET(p); - p += U32Size; - is->blockbase = U32GET(p); - p += U32Size; - is->blocks = U32GET(p); - p += U32Size; - is->start = U32GET(p); - p += U32Size; - is->stop = U32GET(p); - p += U32Size; - - if(buf + ISectSize != p) - sysfatal("unpackisect unpacked wrong amount"); - - return 0; -} - -int -packisect(ISect *is, u8int *buf) -{ - u8int *p; - - p = buf; - - U32PUT(p, ISectMagic); - p += U32Size; - U32PUT(p, is->version); - p += U32Size; - namecp((char*)p, is->name); - p += ANameSize; - namecp((char*)p, is->index); - p += ANameSize; - U32PUT(p, is->blocksize); - p += U32Size; - U32PUT(p, is->blockbase); - p += U32Size; - U32PUT(p, is->blocks); - p += U32Size; - U32PUT(p, is->start); - p += U32Size; - U32PUT(p, is->stop); - p += U32Size; - - if(buf + ISectSize != p) - sysfatal("packisect packed wrong amount"); - - return 0; -} - -void -unpackientry(IEntry *ie, u8int *buf) -{ - u8int *p; - - p = buf; - - scorecp(ie->score, p); - p += VtScoreSize; - ie->wtime = U32GET(p); - p += U32Size; - ie->train = U16GET(p); - p += U16Size; - ie->ia.addr = U64GET(p); -if(ie->ia.addr>>56) print("%.8H => %llux\n", p, ie->ia.addr); - p += U64Size; - ie->ia.size = U16GET(p); - p += U16Size; - if(p - buf != IEntryTypeOff) - sysfatal("unpackientry bad IEntryTypeOff amount"); - ie->ia.type = vtfromdisktype(U8GET(p)); - p += U8Size; - ie->ia.blocks = U8GET(p); - p += U8Size; - - if(p - buf != IEntrySize) - sysfatal("unpackientry unpacked wrong amount"); -} - -void -packientry(IEntry *ie, u8int *buf) -{ - u32int t32; - u8int *p; - - p = buf; - - scorecp(p, ie->score); - p += VtScoreSize; - U32PUT(p, ie->wtime); - p += U32Size; - U16PUT(p, ie->train); - p += U16Size; - U64PUT(p, ie->ia.addr, t32); - p += U64Size; - U16PUT(p, ie->ia.size); - p += U16Size; - U8PUT(p, vttodisktype(ie->ia.type)); - p += U8Size; - U8PUT(p, ie->ia.blocks); - p += U8Size; - - if(p - buf != IEntrySize) - sysfatal("packientry packed wrong amount"); -} - -void -unpackibucket(IBucket *b, u8int *buf) -{ - b->n = U16GET(buf); - b->depth = U32GET(&buf[U16Size]); - b->data = buf + IBucketSize; -} - -void -packibucket(IBucket *b, u8int *buf) -{ - U16PUT(buf, b->n); - U32PUT(&buf[U16Size], b->depth); -} blob - 8b5bb84d13cb65c00bfa056557b41b26b07d009d (mode 644) blob + /dev/null --- src/cmd/venti/copy.c +++ /dev/null @@ -1,169 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -int fast; - -VtConn *zsrc, *zdst; - -void -usage(void) -{ - fprint(2, "usage: copy src-host dst-host score [type]\n"); - threadexitsall("usage"); -} - -int -parsescore(uchar *score, char *buf, int n) -{ - int i, c; - - memset(score, 0, VtScoreSize); - - if(n < VtScoreSize*2) - return -1; - for(i=0; i= '0' && buf[i] <= '9') - c = buf[i] - '0'; - else if(buf[i] >= 'a' && buf[i] <= 'f') - c = buf[i] - 'a' + 10; - else if(buf[i] >= 'A' && buf[i] <= 'F') - c = buf[i] - 'A' + 10; - else { - return -1; - } - - if((i & 1) == 0) - c <<= 4; - - score[i>>1] |= c; - } - return 0; -} - -void -walk(uchar score[VtScoreSize], uint type, int base) -{ - int i, n, sub; - uchar *buf; - VtEntry e; - VtRoot root; - - if(memcmp(score, vtzeroscore, VtScoreSize) == 0) - return; - - buf = vtmallocz(VtMaxLumpSize); - if(fast && vtread(zdst, score, type, buf, VtMaxLumpSize) >= 0){ - fprint(2, "skip %V\n", score); - free(buf); - return; - } - - n = vtread(zsrc, score, type, buf, VtMaxLumpSize); - if(n < 0){ - fprint(2, "warning: could not read block %V %d: %r", score, type); - return; - } - - switch(type){ - case VtRootType: - if(vtrootunpack(&root, buf) < 0){ - fprint(2, "warning: could not unpack root in %V %d\n", score, type); - break; - } - walk(root.score, VtDirType, 0); - walk(root.prev, VtRootType, 0); - break; - - case VtDirType: - for(i=0; i>VtEntryDepthShift); - walk(e.score, sub, base); - } - break; - - case VtDataType: - break; - - default: /* pointers */ - for(i=0; i= 0) - break; - } - if(type == VtMaxType) - sysfatal("could not find block %V of any type", score); - } - - walk(score, type, VtDirType); - - if(vtsync(zdst) < 0) - sysfatal("could not sync dst server: %r"); - - threadexitsall(0); -} blob - 98c935ed16e8c8f1d572d47aec932d7c9091d60b (mode 644) blob + /dev/null --- src/cmd/venti/dat.h +++ /dev/null @@ -1,528 +0,0 @@ -typedef struct Config Config; -typedef struct AMap AMap; -typedef struct AMapN AMapN; -typedef struct Arena Arena; -typedef struct ArenaHead ArenaHead; -typedef struct ArenaPart ArenaPart; -typedef struct CIBlock CIBlock; -typedef struct Clump Clump; -typedef struct ClumpInfo ClumpInfo; -typedef struct IAddr IAddr; -typedef struct IBucket IBucket; -typedef struct ICache ICache; -typedef struct IEStream IEStream; -typedef struct IEntry IEntry; -typedef struct IFile IFile; -typedef struct ISect ISect; -typedef struct Index Index; -typedef struct Lump Lump; -typedef struct DBlock DBlock; -typedef struct Part Part; -typedef struct Stats Stats; -typedef struct ZBlock ZBlock; - -#define TWID32 ((u32int)~(u32int)0) -#define TWID64 ((u64int)~(u64int)0) -#define TWID8 ((u8int)~(u8int)0) - -enum -{ - ABlockLog = 9, /* log2(512), the quantum for reading arenas */ - ANameSize = 64, - MaxDiskBlock = 64*1024, /* max. allowed size for a disk block */ - MaxIoSize = 64*1024, /* max. allowed size for a disk io operation */ - PartBlank = 256*1024, /* untouched section at beginning of partition */ - HeadSize = 512, /* size of a header after PartBlank */ - MinArenaSize = 1*1024*1024, /* smallest reasonable arena size */ - IndexBase = 1024*1024, /* initial address to use in an index */ - MaxIo = 64*1024, /* max size of a single read or write operation */ - ICacheBits = 16, /* default bits for indexing icache */ - ICacheDepth = 4, /* default depth of an icache hash chain */ - MaxAMap = 2*1024, /* max. allowed arenas in an address mapping; must be < 32*1024 */ - - /* - * return codes from syncarena - */ - SyncDataErr = 1 << 0, /* problem reading the clump data */ - SyncCIErr = 1 << 1, /* found erroneous clump directory entries */ - SyncCIZero = 1 << 2, /* found unwritten clump directory entries */ - SyncFixErr = 1 << 3, /* error writing fixed data */ - SyncHeader = 1 << 4, /* altered header fields */ - - /* - * error severity - */ - EOk = 0, /* error expected in normal operation */ - EStrange, /* strange error that should be logged */ - ECorrupt, /* corrupted data found in arenas */ - EICorrupt, /* corrupted data found in index */ - EAdmin, /* should be brought to administrators' attention */ - ECrash, /* really bad internal error */ - EBug, /* a limitation which should be fixed */ - EInconsist, /* inconsistencies between index and arena */ - EMax, - - /* - * internal disk formats for the venti archival storage system - */ - /* - * magic numbers on disk - */ - ClumpMagic = 0xd15cb10c, /* clump header */ - ClumpFreeMagic = 0, /* free clump; terminates active clump log */ - - ArenaPartMagic = 0xa9e4a5e7, /* arena partition header */ - ArenaMagic = 0xf2a14ead, /* arena trailer */ - ArenaHeadMagic = 0xd15c4ead, /* arena header */ - - ISectMagic = 0xd15c5ec7, /* index header */ - - ArenaPartVersion = 3, - ArenaVersion = 4, - IndexVersion1 = 1, - IndexVersion2 = 2, - ISectVersion = 1, - - /* - * encodings of clumps on disk - */ - ClumpEErr = 0, /* can't happen */ - ClumpENone, /* plain */ - ClumpECompress, /* compressed */ - ClumpEMax, - - /* - * marker for corrupted data on disk - */ - VtTypeCorrupt = VtMaxType, - - /* - * sizes in bytes on disk - */ - U8Size = 1, - U16Size = 2, - U32Size = 4, - U64Size = 8, - - ArenaPartSize = 4 * U32Size, - ArenaSize = 2 * U64Size + 6 * U32Size + ANameSize + U8Size, - ArenaHeadSize = U64Size + 3 * U32Size + ANameSize, - ISectSize = 7 * U32Size + 2 * ANameSize, - ClumpInfoSize = U8Size + 2 * U16Size + VtScoreSize, - ClumpSize = ClumpInfoSize + U8Size + 3 * U32Size, - IBucketSize = U32Size + U16Size, - IEntrySize = U64Size + U32Size + 2*U16Size + 2*U8Size + VtScoreSize, - IEntryTypeOff = VtScoreSize + U64Size + U32Size + 2 * U16Size, - - MaxClumpBlocks = (VtMaxLumpSize + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog, - - /* - * dirty flags - order controls disk write order - */ - DirtyArena = 1, - DirtyIndexSplit, - DirtyIndex, - DirtyIndexBitmap, - DirtyArenaCib, - DirtyArenaTrailer, - DirtyMax, - - VentiZZZZZZZZ -}; - -/* - * results of parsing and initializing a config file - */ -struct Config -{ - char *index; /* name of the index to initialize */ - int naparts; /* arena partitions initialized */ - ArenaPart **aparts; - int nsects; /* index sections initialized */ - ISect **sects; -}; - -/* - * a Part is the low level interface to files or disks. - * there are two main types of partitions - * arena paritions, which some number of arenas, each in a sub-partition. - * index partition, which only have one subpartition. - */ -struct Part -{ - int fd; /* rock for accessing the disk */ - u64int size; /* size of the partiton */ - u32int blocksize; /* block size for reads and writes */ - char *name; - Channel *writechan; /* chan[dcache.nblock](DBlock*) */ -}; - -/* - * a cached block from the partition - * yuck -- most of this is internal structure for the cache - * all other routines should only use data - */ -struct DBlock -{ - u8int *data; - - Part *part; /* partition in which cached */ - u64int addr; /* base address on the partition */ - u16int size; /* amount of data available, not amount allocated; should go away */ - u32int dirty; - u32int dirtying; - DBlock *next; /* doubly linked hash chains */ - DBlock *prev; - u32int heap; /* index in heap table */ - u32int used; /* last reference times */ - u32int used2; - u32int ref; /* reference count */ - QLock lock; /* for access to data only */ - Channel writedonechan; - void* chanbuf[1]; /* buffer for the chan! */ -}; - -/* - * a cached block from the partition - * yuck -- most of this is internal structure for the cache - * all other routines should only use data - * double yuck -- this is mostly the same as a DBlock - */ -struct Lump -{ - Packet *data; - - Part *part; /* partition in which cached */ - u8int score[VtScoreSize]; /* score of packet */ - u8int type; /* type of packet */ - u16int size; /* amount of data allocated to hold packet */ - Lump *next; /* doubly linked hash chains */ - Lump *prev; - u32int heap; /* index in heap table */ - u32int used; /* last reference times */ - u32int used2; - u32int ref; /* reference count */ - QLock lock; /* for access to data only */ -}; - -/* - * mapping between names and address ranges - */ -struct AMap -{ - u64int start; - u64int stop; - char name[ANameSize]; -}; - -/* - * an AMap along with a length - */ -struct AMapN -{ - int n; - AMap *map; -}; - -/* - * an ArenaPart is a partition made up of Arenas - * it exists because most os's don't support many partitions, - * and we want to have many different Arenas - */ -struct ArenaPart -{ - Part *part; - u64int size; /* size of underlying partition, rounded down to blocks */ - Arena **arenas; - u32int tabbase; /* base address of arena table on disk */ - u32int tabsize; /* max. bytes in arena table */ - - /* - * fields stored on disk - */ - u32int version; - u32int blocksize; /* "optimal" block size for reads and writes */ - u32int arenabase; /* base address of first arena */ - - /* - * stored in the arena mapping table on disk - */ - AMap *map; - int narenas; -}; - -/* - * info about one block in the clump info cache - */ -struct CIBlock -{ - u32int block; /* blocks in the directory */ - int offset; /* offsets of one clump in the data */ - DBlock *data; -}; - -/* - * an Arena is a log of Clumps, preceeded by an ArenaHeader, - * and followed by a Arena, each in one disk block. - * struct on disk is not always up to date, but should be self-consistent. - * to sync after reboot, follow clumps starting at used until ClumpFreeMagic if found. - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - */ -struct Arena -{ - QLock lock; /* lock for arena fields, writing to disk */ - Part *part; /* partition in which arena lives */ - int blocksize; /* size of block to read or write */ - u64int base; /* base address on disk */ - u64int size; /* total space in the arena */ - u64int limit; /* storage limit for clumps */ - u8int score[VtScoreSize]; /* score of the entire sealed & summed arena */ - - int clumpmax; /* ClumpInfos per block */ - CIBlock cib; /* dirty clump directory block */ - - /* - * fields stored on disk - */ - u32int version; - char name[ANameSize]; /* text label */ - u32int clumps; /* number of allocated clumps */ - u32int cclumps; /* clumps which are compressed; informational only */ - u32int ctime; /* first time a block was written */ - u32int wtime; /* last time a block was written */ - u64int used; /* number of bytes currently used */ - u64int uncsize; /* total of all clumps's uncsize; informational only */ - u8int sealed; /* arena all filled up? */ -}; - -/* - * redundant storage of some fields at the beginning of each arena - */ -struct ArenaHead -{ - u32int version; - char name[ANameSize]; - u32int blocksize; - u64int size; -}; - -/* - * most interesting meta information for a clump. - * stored in each clump's header and in the Arena's directory, - * stored in reverse order just prior to the arena trailer - */ -struct ClumpInfo -{ - u8int type; - u16int size; /* size of disk data, not including header */ - u16int uncsize; /* size of uncompressed data */ - u8int score[VtScoreSize]; /* score of the uncompressed data only */ -}; - -/* - * header for an immutable clump of data - */ -struct Clump -{ - ClumpInfo info; - u8int encoding; - u32int creator; /* initial client which wrote the block */ - u32int time; /* creation at gmt seconds since 1/1/1970 */ -}; - -/* - * index of all clumps according to their score - * this is just a wrapper to tie together the index sections - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - */ -struct Index -{ - u32int div; /* divisor for mapping score to bucket */ - u32int buckets; /* last bucket used in disk hash table */ - u32int blocksize; - u32int tabsize; /* max. bytes in index config */ - u32int bitblocks; - u32int maxdepth; - u32int bitkeylog; - u32int bitkeymask; - - int mapalloc; /* first arena to check when adding a lump */ - Arena **arenas; /* arenas in the mapping */ - ISect **sects; /* sections which hold the buckets */ - - /* - * fields stored in config file - */ - u32int version; - char name[ANameSize]; /* text label */ - int nsects; - AMap *smap; /* mapping of buckets to index sections */ - int narenas; - AMap *amap; /* mapping from index addesses to arenas */ -}; - -/* - * one part of the bucket storage for an index. - * the index blocks are sequentially allocated - * across all of the sections. - */ -struct ISect -{ - Part *part; - int blocklog; /* log2(blocksize) */ - int buckmax; /* max. entries in a index bucket */ - u32int tabbase; /* base address of index config table on disk */ - u32int tabsize; /* max. bytes in index config */ - - /* - * fields stored on disk - */ - u32int version; - char name[ANameSize]; /* text label */ - char index[ANameSize]; /* index owning the section */ - u32int blocksize; /* size of hash buckets in index */ - u32int blockbase; /* address of start of on disk index table */ - u32int blocks; /* total blocks on disk; some may be unused */ - u32int start; /* first bucket in this section */ - u32int stop; /* limit of buckets in this section */ -}; - -/* - * externally interesting part of an IEntry - */ -struct IAddr -{ - u64int addr; - u16int size; /* uncompressed size */ - u8int type; /* type of block */ - u8int blocks; /* arena io quanta for Clump + data */ -}; - -/* - * entries in the index - * kept in IBuckets in the disk index table, - * cached in the memory ICache. - */ -struct IEntry -{ - u8int score[VtScoreSize]; - IEntry *next; /* next in hash chain */ - u32int wtime; /* last write time */ - u16int train; /* relative train containing the most recent ref; 0 if no ref, 1 if in same car */ - u8int rac; /* read ahead count */ - IAddr ia; -}; - -/* - * buckets in the on disk index table - */ -struct IBucket -{ - u16int n; /* number of active indices */ - u32int depth; /* depth in version 2 (was overflow in v1) */ - u8int *data; -}; - -/* - * temporary buffers used by individual threads - */ -struct ZBlock -{ - u32int len; - u8int *data; -}; - -/* - * simple input buffer for a '\0' terminated text file - */ -struct IFile -{ - char *name; /* name of the file */ - ZBlock *b; /* entire contents of file */ - u32int pos; /* current position in the file */ -}; - -/* - * statistics about the operation of the server - * mainly for performance monitoring and profiling. - */ -struct Stats -{ - QLock lock; - long lumpwrites; /* protocol block writes */ - long lumpreads; /* protocol block reads */ - long lumphit; /* lump cache hit */ - long lumpmiss; /* lump cache miss */ - long clumpwrites; /* clumps to disk */ - vlong clumpbwrites; /* clump data bytes to disk */ - vlong clumpbcomp; /* clump bytes compressed */ - long clumpreads; /* clumps from disk */ - vlong clumpbreads; /* clump data bytes from disk */ - vlong clumpbuncomp; /* clump bytes uncompressed */ - long ciwrites; /* clump directory to disk */ - long cireads; /* clump directory from disk */ - long indexwrites; /* index to disk */ - long indexreads; /* index from disk */ - long indexwreads; /* for writing a new entry */ - long indexareads; /* for allocating an overflow block */ - long indexsplits; /* index block splits */ - long diskwrites; /* total disk writes */ - long diskreads; /* total disk reads */ - vlong diskbwrites; /* total disk bytes written */ - vlong diskbreads; /* total disk bytes read */ - long pchit; /* partition cache hit */ - long pcmiss; /* partition cache miss */ - long pcreads; /* partition cache reads from disk */ - vlong pcbreads; /* partition cache bytes read */ - long icinserts; /* stores into index cache */ - long iclookups; /* index cache lookups */ - long ichits; /* hits in the cache */ - long icfills; /* successful fills from index */ - long absorbedwrites; /* disk writes absorbed by dcache */ - long dirtydblocks; /* blocks dirtied */ - long dcacheflushes; /* times dcache has flushed */ - long dcacheflushwrites; /* blocks written by those flushes */ -}; - -extern Index *mainindex; -extern u32int maxblocksize; /* max. block size used by any partition */ -extern int paranoid; /* should verify hashes on disk read */ -extern int queuewrites; /* put all lump writes on a queue and finish later */ -extern int readonly; /* only allowed to read the disk data */ -extern Stats stats; -extern u8int zeroscore[VtScoreSize]; blob - 7e63323249a624bbbc2cc2aca15111c001bb6f07 (mode 644) blob + /dev/null --- src/cmd/venti/dcache.c +++ /dev/null @@ -1,664 +0,0 @@ -/* - * The locking here is getting a little out of hand. - */ - -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -typedef struct DCache DCache; - -enum -{ - HashLog = 9, - HashSize = 1<dirty */ - u32int flushround; - Rendez anydirty; - Rendez full; - Rendez flush; - Rendez flushdone; - DBlock *free; /* list of available lumps */ - u32int now; /* ticks for usage timestamps */ - int size; /* max. size of any block; allocated to each block */ - DBlock **heads; /* hash table for finding address */ - int nheap; /* number of available victims */ - DBlock **heap; /* heap for locating victims */ - int nblocks; /* number of blocks allocated */ - DBlock *blocks; /* array of block descriptors */ - DBlock **write; /* array of block pointers to be written */ - u8int *mem; /* memory for all block descriptors */ - int ndirty; /* number of dirty blocks */ - int maxdirty; /* max. number of dirty blocks */ -}; - -static DCache dcache; - -static int downheap(int i, DBlock *b); -static int upheap(int i, DBlock *b); -static DBlock *bumpdblock(void); -static void delheap(DBlock *db); -static void fixheap(int i, DBlock *b); -static void _flushdcache(void); -static void flushproc(void*); -static void flushtimerproc(void*); -static void writeproc(void*); - -void -initdcache(u32int mem) -{ - DBlock *b, *last; - u32int nblocks, blocksize; - int i; - - if(mem < maxblocksize * 2) - sysfatal("need at least %d bytes for the disk cache", maxblocksize * 2); - if(maxblocksize == 0) - sysfatal("no max. block size given for disk cache"); - blocksize = maxblocksize; - nblocks = mem / blocksize; - dcache.full.l = &dcache.lock; - dcache.flush.l = &dcache.lock; - dcache.anydirty.l = &dcache.lock; - dcache.flushdone.l = &dcache.lock; - dcache.nblocks = nblocks; - dcache.maxdirty = (nblocks * 3) / 4; - if(1) - fprint(2, "initialize disk cache with %d blocks of %d bytes, maximum %d dirty blocks\n", - nblocks, blocksize, dcache.maxdirty); - dcache.size = blocksize; - dcache.heads = MKNZ(DBlock*, HashSize); - dcache.heap = MKNZ(DBlock*, nblocks); - dcache.blocks = MKNZ(DBlock, nblocks); - dcache.write = MKNZ(DBlock*, nblocks); - dcache.mem = MKNZ(u8int, nblocks * blocksize); - - last = nil; - for(i = 0; i < nblocks; i++){ - b = &dcache.blocks[i]; - b->data = &dcache.mem[i * blocksize]; - b->heap = TWID32; - chaninit(&b->writedonechan, sizeof(void*), 1); - b->next = last; - last = b; - } - dcache.free = last; - dcache.nheap = 0; - - vtproc(flushproc, nil); - vtproc(flushtimerproc, nil); -} - -static u32int -pbhash(u64int addr) -{ - u32int h; - -#define hashit(c) ((((c) * 0x6b43a9b5) >> (32 - HashLog)) & HashMask) - h = (addr >> 32) ^ addr; - return hashit(h); -} - -DBlock* -getdblock(Part *part, u64int addr, int read) -{ - DBlock *b; - u32int h, size; - - size = part->blocksize; - if(size > dcache.size){ - seterr(EAdmin, "block size %d too big for cache", size); - return nil; - } - h = pbhash(addr); - - /* - * look for the block in the cache - */ -//checkdcache(); - qlock(&dcache.lock); -again: - for(b = dcache.heads[h]; b != nil; b = b->next){ - if(b->part == part && b->addr == addr){ - qlock(&stats.lock); - stats.pchit++; - qunlock(&stats.lock); - goto found; - } - } - qlock(&stats.lock); - stats.pcmiss++; - qunlock(&stats.lock); - - /* - * missed: locate the block with the oldest second to last use. - * remove it from the heap, and fix up the heap. - */ - b = bumpdblock(); - if(b == nil){ - logerr(EAdmin, "all disk cache blocks in use"); - rsleep(&dcache.full); - goto again; - } - - /* - * the new block has no last use, so assume it happens sometime in the middle -ZZZ this is not reasonable - */ - b->used = (b->used2 + dcache.now) / 2; - - /* - * rechain the block on the correct hash chain - */ - b->next = dcache.heads[h]; - dcache.heads[h] = b; - if(b->next != nil) - b->next->prev = b; - b->prev = nil; - - b->addr = addr; - b->part = part; - b->size = 0; - -found: - b->ref++; - b->used2 = b->used; - b->used = dcache.now++; - if(b->heap != TWID32) - fixheap(b->heap, b); - - qunlock(&dcache.lock); -//checkdcache(); - - qlock(&b->lock); - if(b->size != size){ - if(b->size < size){ - if(!read) - memset(&b->data[b->size], 0, size - b->size); - else{ - if(readpart(part, addr + b->size, &b->data[b->size], size - b->size) < 0){ - putdblock(b); - return nil; - } - qlock(&stats.lock); - stats.pcreads++; - stats.pcbreads += size - b->size; - qunlock(&stats.lock); - } - } - b->size = size; - } - - return b; -} - -void -putdblock(DBlock *b) -{ - if(b == nil) - return; - - if(b->dirtying){ - b->dirtying = 0; - runlock(&dcache.dirtylock); - } - qunlock(&b->lock); - -//checkdcache(); - qlock(&dcache.lock); - if(b->dirty) - delheap(b); - else if(--b->ref == 0){ - if(b->heap == TWID32) - upheap(dcache.nheap++, b); - rwakeupall(&dcache.full); - } - - qunlock(&dcache.lock); -//checkdcache(); -} - -void -dirtydblock(DBlock *b, int dirty) -{ - int odirty; - Part *p; - - - assert(b->ref != 0); - - /* - * Because we use an rlock to keep track of how - * many blocks are being dirtied (and a wlock to - * stop dirtying), we cannot try to dirty two blocks - * at the same time in the same thread -- if the - * wlock happens between them, we get a deadlock. - * - * The only place in the code where we need to - * dirty multiple blocks at once is in splitiblock, when - * we split an index block. The new block has a dirty - * flag of DirtyIndexSplit already, because it has to get - * written to disk before the old block (DirtyIndex). - * So when we see the DirtyIndexSplit block, we don't - * acquire the rlock (and we don't set dirtying, so putdblock - * won't drop the rlock). This kludginess means that - * the caller will have to be sure to putdblock on the - * new block before putdblock on the old block. - */ - if(dirty == DirtyIndexSplit){ - /* caller should have another dirty block */ - assert(!canwlock(&dcache.dirtylock)); - /* this block should be clean */ - assert(b->dirtying == 0); - assert(b->dirty == 0); - }else{ - rlock(&dcache.dirtylock); - assert(b->dirtying == 0); - b->dirtying = 1; - } - - qlock(&stats.lock); - if(b->dirty) - stats.absorbedwrites++; - stats.dirtydblocks++; - qunlock(&stats.lock); - - /* - * In general, shouldn't mark any block as more than one - * type, except that split index blocks are a subcase of index - * blocks. Only clean blocks ever get marked DirtyIndexSplit, - * though, so we don't need the opposite conjunction here. - */ - odirty = b->dirty; - if(b->dirty) - assert(b->dirty == dirty - || (b->dirty==DirtyIndexSplit && dirty==DirtyIndex)); - else - b->dirty = dirty; - - p = b->part; - if(p->writechan == nil){ -fprint(2, "allocate write proc for part %s\n", p->name); - /* XXX hope this doesn't fail! */ - p->writechan = chancreate(sizeof(DBlock*), dcache.nblocks); - vtproc(writeproc, p); - } - qlock(&dcache.lock); - if(!odirty){ - dcache.ndirty++; - rwakeupall(&dcache.anydirty); - } - qunlock(&dcache.lock); -} - -/* - * remove some block from use and update the free list and counters - */ -static DBlock* -bumpdblock(void) -{ - int flushed; - DBlock *b; - ulong h; - - b = dcache.free; - if(b != nil){ - dcache.free = b->next; - return b; - } - - if(dcache.ndirty >= dcache.maxdirty) - _flushdcache(); - - /* - * remove blocks until we find one that is unused - * referenced blocks are left in the heap even though - * they can't be scavenged; this is simple a speed optimization - */ - flushed = 0; - for(;;){ - if(dcache.nheap == 0){ - _flushdcache(); - return nil; - } - b = dcache.heap[0]; - delheap(b); - if(!b->ref) - break; - } - - fprint(2, "bump %s at %llud\n", b->part->name, b->addr); - - /* - * unchain the block - */ - if(b->prev == nil){ - h = pbhash(b->addr); - if(dcache.heads[h] != b) - sysfatal("bad hash chains in disk cache"); - dcache.heads[h] = b->next; - }else - b->prev->next = b->next; - if(b->next != nil) - b->next->prev = b->prev; - - return b; -} - -/* - * delete an arbitrary block from the heap - */ -static void -delheap(DBlock *db) -{ - if(db->heap == TWID32) - return; - fixheap(db->heap, dcache.heap[--dcache.nheap]); - db->heap = TWID32; -} - -/* - * push an element up or down to it's correct new location - */ -static void -fixheap(int i, DBlock *b) -{ - if(upheap(i, b) == i) - downheap(i, b); -} - -static int -upheap(int i, DBlock *b) -{ - DBlock *bb; - u32int now; - int p; - - now = dcache.now; - for(; i != 0; i = p){ - p = (i - 1) >> 1; - bb = dcache.heap[p]; - if(b->used2 - now >= bb->used2 - now) - break; - dcache.heap[i] = bb; - bb->heap = i; - } - - dcache.heap[i] = b; - b->heap = i; - return i; -} - -static int -downheap(int i, DBlock *b) -{ - DBlock *bb; - u32int now; - int k; - - now = dcache.now; - for(; ; i = k){ - k = (i << 1) + 1; - if(k >= dcache.nheap) - break; - if(k + 1 < dcache.nheap && dcache.heap[k]->used2 - now > dcache.heap[k + 1]->used2 - now) - k++; - bb = dcache.heap[k]; - if(b->used2 - now <= bb->used2 - now) - break; - dcache.heap[i] = bb; - bb->heap = i; - } - - dcache.heap[i] = b; - b->heap = i; - return i; -} - -static void -findblock(DBlock *bb) -{ - DBlock *b, *last; - int h; - - last = nil; - h = pbhash(bb->addr); - for(b = dcache.heads[h]; b != nil; b = b->next){ - if(last != b->prev) - sysfatal("bad prev link"); - if(b == bb) - return; - last = b; - } - sysfatal("block missing from hash table"); -} - -void -checkdcache(void) -{ - DBlock *b; - u32int size, now; - int i, k, refed, nfree; - - qlock(&dcache.lock); - size = dcache.size; - now = dcache.now; - for(i = 0; i < dcache.nheap; i++){ - if(dcache.heap[i]->heap != i) - sysfatal("dc: mis-heaped at %d: %d", i, dcache.heap[i]->heap); - if(i > 0 && dcache.heap[(i - 1) >> 1]->used2 - now > dcache.heap[i]->used2 - now) - sysfatal("dc: bad heap ordering"); - k = (i << 1) + 1; - if(k < dcache.nheap && dcache.heap[i]->used2 - now > dcache.heap[k]->used2 - now) - sysfatal("dc: bad heap ordering"); - k++; - if(k < dcache.nheap && dcache.heap[i]->used2 - now > dcache.heap[k]->used2 - now) - sysfatal("dc: bad heap ordering"); - } - - refed = 0; - for(i = 0; i < dcache.nblocks; i++){ - b = &dcache.blocks[i]; - if(b->data != &dcache.mem[i * size]) - sysfatal("dc: mis-blocked at %d", i); - if(b->ref && b->heap == TWID32) - refed++; - if(b->addr) - findblock(b); - if(b->heap != TWID32 - && dcache.heap[b->heap] != b) - sysfatal("dc: spurious heap value"); - } - - nfree = 0; - for(b = dcache.free; b != nil; b = b->next){ - if(b->addr != 0 || b->heap != TWID32) - sysfatal("dc: bad free list"); - nfree++; - } - - if(dcache.nheap + nfree + refed != dcache.nblocks) - sysfatal("dc: missing blocks: %d %d %d", dcache.nheap, refed, dcache.nblocks); - qunlock(&dcache.lock); -} - -void -flushdcache(void) -{ - u32int flushround; - - qlock(&dcache.lock); - flushround = dcache.flushround; - rwakeupall(&dcache.flush); - while(flushround == dcache.flushround) - rsleep(&dcache.flushdone); - qunlock(&dcache.lock); -} - -static void -_flushdcache(void) -{ - rwakeupall(&dcache.flush); -} - -static int -parallelwrites(DBlock **b, DBlock **eb, int dirty) -{ - DBlock **p; - - for(p=b; pdirty == dirty; p++) - sendp((*p)->part->writechan, *p); - for(p=b; pdirty == dirty; p++) - recvp(&(*p)->writedonechan); - - return p-b; -} - -/* - * Sort first by dirty flag, then by partition, then by address in partition. - */ -static int -writeblockcmp(const void *va, const void *vb) -{ - DBlock *a, *b; - - a = *(DBlock**)va; - b = *(DBlock**)vb; - - if(a->dirty != b->dirty) - return a->dirty - b->dirty; - if(a->part != b->part){ - if(a->part < b->part) - return -1; - if(a->part > b->part) - return 1; - } - if(a->addr < b->addr) - return -1; - return 1; -} - -static void -flushtimerproc(void *v) -{ - u32int round; - - for(;;){ - qlock(&dcache.lock); - while(dcache.ndirty == 0) - rsleep(&dcache.anydirty); - round = dcache.flushround; - qunlock(&dcache.lock); - - sleep(60*1000); - - qlock(&dcache.lock); - if(round == dcache.flushround){ - rwakeupall(&dcache.flush); - while(round == dcache.flushround) - rsleep(&dcache.flushdone); - } - qunlock(&dcache.lock); - } -} - -static void -flushproc(void *v) -{ - int i, j, n; - ulong t0; - DBlock *b, **write; - - USED(v); - for(;;){ - qlock(&dcache.lock); - dcache.flushround++; - rwakeupall(&dcache.flushdone); - rsleep(&dcache.flush); - qunlock(&dcache.lock); - - fprint(2, "flushing dcache: t=0 ms\n"); - t0 = nsec()/1000; - - /* - * Because we don't record any dependencies at all, we must write out - * all blocks currently dirty. Thus we must lock all the blocks that - * are currently dirty. - * - * We grab dirtylock to stop the dirtying of new blocks. - * Then we wait until all the current blocks finish being dirtied. - * Now all the dirty blocks in the system are immutable (clean blocks - * might still get recycled), so we can plan our disk writes. - * - * In a better scheme, dirtiers might lock the block for writing in getdblock, - * so that flushproc could lock all the blocks here and then unlock them as it - * finishes with them. - */ - - fprint(2, "flushproc: wlock at t=%lud\n", (ulong)(nsec()/1000) - t0); - wlock(&dcache.dirtylock); - - fprint(2, "flushproc: build list at t=%lud\n", (ulong)(nsec()/1000) - t0); - write = dcache.write; - n = 0; - for(i=0; idirty) - write[n++] = b; - } - - qsort(write, n, sizeof(write[0]), writeblockcmp); - - /* Write each stage of blocks out. */ - fprint(2, "flushproc: write blocks at t=%lud\n", (ulong)(nsec()/1000) - t0); - i = 0; - for(j=1; jdirty = 0; - --dcache.ndirty; - if(b->ref == 0 && b->heap == TWID32){ - upheap(dcache.nheap++, b); - rwakeupall(&dcache.full); - } - } - qunlock(&dcache.lock); - wunlock(&dcache.dirtylock); - - fprint(2, "flushproc: done at t=%lud\n", (ulong)(nsec()/1000) - t0); - - qlock(&stats.lock); - stats.dcacheflushes++; - stats.dcacheflushwrites += n; - qunlock(&stats.lock); - } -} - -static void -writeproc(void *v) -{ - DBlock *b; - Part *p; - - p = v; - - for(;;){ - b = recvp(p->writechan); - if(writepart(p, b->addr, b->data, b->size) < 0) - fprint(2, "write error: %r\n"); /* XXX details! */ - sendp(&b->writedonechan, b); - } -} blob - 12fcddda3347824d45ae6392559e82f0217978f0 (mode 644) blob + /dev/null --- src/cmd/venti/dump.c +++ /dev/null @@ -1,47 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -void -printindex(int fd, Index *ix) -{ - int i; - - fprint(fd, "index=%s version=%d blocksize=%d tabsize=%d\n", - ix->name, ix->version, ix->blocksize, ix->tabsize); - fprint(fd, "\tbuckets=%d div=%d\n", ix->buckets, ix->div); - for(i = 0; i < ix->nsects; i++) - fprint(fd, "\tsect=%s for buckets [%lld,%lld)\n", ix->smap[i].name, ix->smap[i].start, ix->smap[i].stop); - for(i = 0; i < ix->narenas; i++) - fprint(fd, "\tarena=%s at [%lld,%lld)\n", ix->amap[i].name, ix->amap[i].start, ix->amap[i].stop); -} - -void -printarenapart(int fd, ArenaPart *ap) -{ - int i; - - fprint(fd, "arena partition=%s\n\tversion=%d blocksize=%d arenas=%d\n\tsetbase=%d setsize=%d\n", - ap->part->name, ap->version, ap->blocksize, ap->narenas, ap->tabbase, ap->tabsize); - for(i = 0; i < ap->narenas; i++) - fprint(fd, "\tarena=%s at [%lld,%lld)\n", ap->map[i].name, ap->map[i].start, ap->map[i].stop); -} - -void -printarena(int fd, Arena *arena) -{ - fprint(fd, "arena='%s' [%lld,%lld)\n\tversion=%d created=%d modified=%d", - arena->name, arena->base, arena->base + arena->size + 2 * arena->blocksize, - arena->version, arena->ctime, arena->wtime); - if(arena->sealed) - fprint(2, " sealed\n"); - else - fprint(2, "\n"); - if(scorecmp(zeroscore, arena->score) != 0) - fprint(2, "\tscore=%V\n", arena->score); - - fprint(fd, "\tclumps=%,d compressed clumps=%,d data=%,lld compressed data=%,lld disk storage=%,lld\n", - arena->clumps, arena->cclumps, arena->uncsize, - arena->used - arena->clumps * ClumpSize, - arena->used + arena->clumps * ClumpInfoSize); -} blob - 7718adce1c4b84f00cd5148634539075ddd8ff7b (mode 644) blob + /dev/null --- src/cmd/venti/findscore.c +++ /dev/null @@ -1,122 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -enum -{ - ClumpChunks = 32*1024 -}; - -static int verbose; - -int -clumpinfoeq(ClumpInfo *c, ClumpInfo *d) -{ - return c->type == d->type - && c->size == d->size - && c->uncsize == d->uncsize - && scorecmp(c->score, d->score)==0; -} - -int -findscore(Arena *arena, uchar *score) -{ - IEntry ie; - ClumpInfo *ci, *cis; - u64int a; - u32int clump; - int i, n, found; - -//ZZZ remove fprint? - if(arena->clumps) - fprint(2, "reading directory for arena=%s with %d entries\n", arena->name, arena->clumps); - - cis = MKN(ClumpInfo, ClumpChunks); - found = 0; - a = 0; - memset(&ie, 0, sizeof(IEntry)); - for(clump = 0; clump < arena->clumps; clump += n){ - n = ClumpChunks; - if(n > arena->clumps - clump) - n = arena->clumps - clump; - if(readclumpinfos(arena, clump, cis, n) != n){ - seterr(EOk, "arena directory read failed: %r"); - break; - } - - for(i = 0; i < n; i++){ - ci = &cis[i]; - if(scorecmp(score, ci->score)==0){ - fprint(2, "found at clump=%d with type=%d size=%d csize=%d position=%lld\n", - clump + i, ci->type, ci->uncsize, ci->size, a); - found++; - } - a += ci->size + ClumpSize; - } - } - free(cis); - return found; -} - -void -usage(void) -{ - fprint(2, "usage: findscore [-v] arenafile score\n"); - threadexitsall(0); -} - -void -threadmain(int argc, char *argv[]) -{ - ArenaPart *ap; - Part *part; - char *file; - u8int score[VtScoreSize]; - int i, found; - - fmtinstall('V', vtscorefmt); - statsinit(); - - ARGBEGIN{ - case 'v': - verbose++; - break; - default: - usage(); - break; - }ARGEND - - readonly = 1; - - if(argc != 2) - usage(); - - file = argv[0]; - if(strscore(argv[1], score) < 0) - sysfatal("bad score %s\n", argv[1]); - - part = initpart(file, 0); - if(part == nil) - sysfatal("can't open partition %s: %r", file); - - ap = initarenapart(part); - if(ap == nil) - sysfatal("can't initialize arena partition in %s: %r", file); - - if(verbose > 1){ - printarenapart(2, ap); - fprint(2, "\n"); - } - - initdcache(8 * MaxDiskBlock); - - found = 0; - for(i = 0; i < ap->narenas; i++) - found += findscore(ap->arenas[i], score); - - print("found %d occurances of %V\n", found, score); - - if(verbose > 1) - printstats(); - threadexitsall(0); -} blob - 8e52c56be5951f569d17bb29ce745bd3bea6f32b (mode 644) blob + /dev/null --- src/cmd/venti/fmtarenas.c +++ /dev/null @@ -1,114 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -void -usage(void) -{ - fprint(2, "usage: fmtarenas [-Z] [-b blocksize] [-a arenasize] name file\n"); - threadexitsall(0); -} - -void -threadmain(int argc, char *argv[]) -{ - ArenaPart *ap; - Part *part; - Arena *arena; - u64int addr, limit, asize, apsize; - char *file, *name, aname[ANameSize]; - int i, n, blocksize, tabsize, zero; - - fmtinstall('V', vtscorefmt); - - statsinit(); - - blocksize = 8 * 1024; - asize = 512 * 1024 *1024; - tabsize = 64 * 1024; /* BUG: should be determine from number of arenas */ - zero = 1; - ARGBEGIN{ - case 'a': - asize = unittoull(ARGF()); - if(asize == TWID64) - usage(); - break; - case 'b': - blocksize = unittoull(ARGF()); - if(blocksize == ~0) - usage(); - if(blocksize > MaxDiskBlock){ - fprint(2, "block size too large, max %d\n", MaxDiskBlock); - threadexitsall("usage"); - } - break; - case 'Z': - zero = 0; - break; - default: - usage(); - break; - }ARGEND - - if(argc != 2) - usage(); - - name = argv[0]; - file = argv[1]; - - if(nameok(name) < 0) - sysfatal("illegal name template %s", name); - - part = initpart(file, 1); - if(part == nil) - sysfatal("can't open partition %s: %r", file); - - if(zero) - zeropart(part, blocksize); - - ap = newarenapart(part, blocksize, tabsize); - if(ap == nil) - sysfatal("can't initialize arena: %r"); - - apsize = ap->size - ap->arenabase; - n = apsize / asize; - if(apsize - (n * asize) >= MinArenaSize) - n++; - - fprint(2, "configuring %s with arenas=%d for a total storage of bytes=%lld and directory bytes=%d\n", - file, n, apsize, ap->tabsize); - - ap->narenas = n; - ap->map = MKNZ(AMap, n); - ap->arenas = MKNZ(Arena*, n); - - addr = ap->arenabase; - for(i = 0; i < n; i++){ - limit = addr + asize; - if(limit >= ap->size || ap->size - limit < MinArenaSize){ - limit = ap->size; - if(limit - addr < MinArenaSize) - sysfatal("bad arena set math: runt arena at %lld,%lld %lld\n", addr, limit, ap->size); - } - - snprint(aname, ANameSize, "%s%d", name, i); - - fprint(2, "adding arena %s at [%lld,%lld)\n", aname, addr, limit); - - arena = newarena(part, aname, addr, limit - addr, blocksize); - if(!arena) - fprint(2, "can't make new arena %s: %r", aname); - freearena(arena); - - ap->map[i].start = addr; - ap->map[i].stop = limit; - namecp(ap->map[i].name, aname); - - addr = limit; - } - - if(wbarenapart(ap) < 0) - fprint(2, "can't write back arena partition header for %s: %r\n", file); - - threadexitsall(0); -} blob - 6934e3d5da2d90f2fa24861ae959b5d6b1aa311d (mode 644) blob + /dev/null --- src/cmd/venti/fmtindex.c +++ /dev/null @@ -1,116 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -void -usage(void) -{ - fprint(2, "usage: fmtindex config\n"); - threadexitsall(0); -} - -void -threadmain(int argc, char *argv[]) -{ - Config conf; - Index *ix; - ArenaPart *ap; - Arena **arenas; - AMap *amap; - u64int addr; - char *file; - u32int i, j, n, narenas; - int add; - - fmtinstall('V', vtscorefmt); - statsinit(); - - add = 0; - ARGBEGIN{ - case 'a': - add = 1; - break; - default: - usage(); - break; - }ARGEND - - if(argc != 1) - usage(); - - file = argv[0]; - - if(runconfig(file, &conf) < 0) - sysfatal("can't initialize config %s: %r", file); - if(conf.index == nil) - sysfatal("no index specified in %s", file); - if(nameok(conf.index) < 0) - sysfatal("illegal index name %s", conf.index); - - narenas = 0; - for(i = 0; i < conf.naparts; i++){ - ap = conf.aparts[i]; - narenas += ap->narenas; - } - - if(add){ - ix = initindex(conf.index, conf.sects, conf.nsects); - if(ix == nil) - sysfatal("can't initialize index %s: %r", conf.index); - }else{ - ix = newindex(conf.index, conf.sects, conf.nsects); - if(ix == nil) - sysfatal("can't create new index %s: %r", conf.index); - - n = 0; - for(i = 0; i < ix->nsects; i++) - n += ix->sects[i]->blocks; - - fprint(2, "using %ud buckets of %ud; div=%d\n", ix->buckets, n, ix->div); - } - amap = MKNZ(AMap, narenas); - arenas = MKNZ(Arena*, narenas); - - addr = IndexBase; - n = 0; - for(i = 0; i < conf.naparts; i++){ - ap = conf.aparts[i]; - for(j = 0; j < ap->narenas; j++){ - if(n >= narenas) - sysfatal("too few slots in index's arena set"); - - arenas[n] = ap->arenas[j]; - if(n < ix->narenas){ - if(arenas[n] != ix->arenas[n]) - sysfatal("mismatched arenas %s and %s at slot %d\n", - arenas[n]->name, ix->arenas[n]->name, n); - amap[n] = ix->amap[n]; - if(amap[n].start != addr) - sysfatal("mis-located arena %s in index %s\n", arenas[n]->name, ix->name); - addr = amap[n].stop; - }else{ - amap[n].start = addr; - addr += ap->arenas[j]->size; - amap[n].stop = addr; - namecp(amap[n].name, ap->arenas[j]->name); - fprint(2, "add arena %s at [%lld,%lld)\n", - amap[n].name, amap[n].start, amap[n].stop); - } - - n++; - } - } - fprint(2, "configured index=%s with arenas=%d and storage=%lld\n", - ix->name, n, addr - IndexBase); - fprint(2, "\tbitblocks=%d maxdepth=%d buckets=%d\n", - ix->bitblocks, ix->maxdepth, ix->buckets); - - ix->amap = amap; - ix->arenas = arenas; - ix->narenas = narenas; - - if(wbindex(ix) < 0) - fprint(2, "can't write back arena partition header for %s: %r\n", file); - - threadexitsall(0); -} blob - 77c929677a9b65de321682ca63d9e60e660c2cc5 (mode 644) blob + /dev/null --- src/cmd/venti/fmtisect.c +++ /dev/null @@ -1,69 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -void -usage(void) -{ - fprint(2, "usage: fmtisect [-Z] [-b blocksize] name file\n"); - threadexitsall(0); -} - -void -threadmain(int argc, char *argv[]) -{ - ISect *is; - Part *part; - char *file, *name; - int blocksize, setsize, zero; - - fmtinstall('V', vtscorefmt); - statsinit(); - - blocksize = 8 * 1024; - setsize = 64 * 1024; - zero = 1; - ARGBEGIN{ - case 'b': - blocksize = unittoull(ARGF()); - if(blocksize == ~0) - usage(); - if(blocksize > MaxDiskBlock){ - fprint(2, "block size too large, max %d\n", MaxDiskBlock); - threadexitsall("usage"); - } - break; - case 'Z': - zero = 0; - break; - default: - usage(); - break; - }ARGEND - - if(argc != 2) - usage(); - - name = argv[0]; - file = argv[1]; - - if(nameok(name) < 0) - sysfatal("illegal name %s", name); - - part = initpart(file, 0); - if(part == nil) - sysfatal("can't open partition %s: %r", file); - - if(zero) - zeropart(part, blocksize); - - fprint(2, "configuring index section %s with space for index config bytes=%d\n", name, setsize); - is = newisect(part, name, blocksize, setsize); - if(is == nil) - sysfatal("can't initialize new index: %r"); - - if(wbisect(is) < 0) - fprint(2, "can't write back index section header for %s: %r\n", file); - - threadexitsall(0); -} blob - f8274397704e6ba802cee749f1aff508684c3660 (mode 644) blob + /dev/null --- src/cmd/venti/fns.h +++ /dev/null @@ -1,163 +0,0 @@ -/* - * sorted by 4,/^$/|sort -bd +1 - */ -int addarena(Arena *name); -ZBlock *alloczblock(u32int size, int zeroed); -Arena *amapitoa(Index *index, u64int a, u64int *aa); -u64int arenadirsize(Arena *arena, u32int clumps); -void arenaupdate(Arena *arena, u32int size, u8int *score); -void backsumarena(Arena *arena); -u32int buildbucket(Index *ix, IEStream *ies, IBucket *ib); -void checkdcache(void); -void checklumpcache(void); -int clumpinfoeq(ClumpInfo *c, ClumpInfo *d); -int clumpinfoeq(ClumpInfo *c, ClumpInfo *d); -u32int clumpmagic(Arena *arena, u64int aa); -int delarena(Arena *arena); -void dirtydblock(DBlock*, int); -void *emalloc(ulong); -void *erealloc(void *, ulong); -char *estrdup(char*); -void *ezmalloc(ulong); -Arena *findarena(char *name); -int flushciblocks(Arena *arena); -void flushdcache(void); -void flushqueue(void); -void fmtzbinit(Fmt *f, ZBlock *b); -void freearena(Arena *arena); -void freearenapart(ArenaPart *ap, int freearenas); -void freeiestream(IEStream *ies); -void freeifile(IFile *f); -void freeisect(ISect *is); -void freeindex(Index *index); -void freepart(Part *part); -void freezblock(ZBlock *b); -DBlock *getdblock(Part *part, u64int addr, int read); -u32int hashbits(u8int *score, int nbits); -int httpdinit(char *address); -int iaddrcmp(IAddr *ia1, IAddr *ia2); -int ientrycmp(const void *vie1, const void *vie2); -char *ifileline(IFile *f); -int ifilename(IFile *f, char *dst); -int ifileu32int(IFile *f, u32int *r); -int indexsect(Index *ix, u8int *score); -Arena *initarena(Part *part, u64int base, u64int size, u32int blocksize); -ArenaPart *initarenapart(Part *part); -int initarenasum(void); -void initdcache(u32int mem); -void initicache(int bits, int depth); -IEStream *initiestream(Part *part, u64int off, u64int clumps, u32int size); -ISect *initisect(Part *part); -Index *initindex(char *name, ISect **sects, int n); -void initlumpcache(u32int size, u32int nblocks); -int initlumpqueues(int nq); -Part* initpart(char *name, int writable); -int initventi(char *config); -void insertlump(Lump *lump, Packet *p); -int insertscore(u8int *score, IAddr *ia, int write); -ZBlock *loadclump(Arena *arena, u64int aa, int blocks, Clump *cl, u8int *score, int verify); -DBlock *loadibucket(Index *index, u8int *score, ISect **is, u32int *buck, IBucket *ib); -int loadientry(Index *index, u8int *score, int type, IEntry *ie); -void logerr(int severity, char *fmt, ...); -Lump *lookuplump(u8int *score, int type); -int lookupscore(u8int *score, int type, IAddr *ia, int *rac); -int maparenas(AMap *am, Arena **arenas, int n, char *what); -int namecmp(char *s, char *t); -void namecp(char *dst, char *src); -int nameok(char *name); -Arena *newarena(Part *part, char *name, u64int base, u64int size, u32int blocksize); -ArenaPart *newarenapart(Part *part, u32int blocksize, u32int tabsize); -ISect *newisect(Part *part, char *name, u32int blocksize, u32int tabsize); -Index *newindex(char *name, ISect **sects, int n); -u32int now(void); -int okamap(AMap *am, int n, u64int start, u64int stop, char *what); -int outputamap(Fmt *f, AMap *am, int n); -int outputindex(Fmt *f, Index *ix); -int packarena(Arena *arena, u8int *buf); -int packarenahead(ArenaHead *head, u8int *buf); -int packarenapart(ArenaPart *as, u8int *buf); -int packclump(Clump *c, u8int *buf); -void packclumpinfo(ClumpInfo *ci, u8int *buf); -void packibucket(IBucket *b, u8int *buf); -void packientry(IEntry *i, u8int *buf); -int packisect(ISect *is, u8int *buf); -void packmagic(u32int magic, u8int *buf); -ZBlock *packet2zblock(Packet *p, u32int size); -int parseamap(IFile *f, AMapN *amn); -int parseindex(IFile *f, Index *ix); -void partblocksize(Part *part, u32int blocksize); -int partifile(IFile *f, Part *part, u64int start, u32int size); -void printarenapart(int fd, ArenaPart *ap); -void printarena(int fd, Arena *arena); -void printindex(int fd, Index *ix); -void printstats(void); -void putdblock(DBlock *b); -void putlump(Lump *b); -int queuewrite(Lump *b, Packet *p, int creator); -u32int readarena(Arena *arena, u64int aa, u8int *buf, long n); -int readarenamap(AMapN *amn, Part *part, u64int base, u32int size); -int readclumpinfo(Arena *arena, int clump, ClumpInfo *ci); -int readclumpinfos(Arena *arena, int clump, ClumpInfo *cis, int n); -ZBlock *readfile(char *name); -int readifile(IFile *f, char *name); -Packet *readlump(u8int *score, int type, u32int size); -int readpart(Part *part, u64int addr, u8int *buf, u32int n); -int runconfig(char *config, Config*); -int scorecmp(u8int *, u8int *); -void scoremem(u8int *score, u8int *buf, int size); -void seterr(int severity, char *fmt, ...); -u64int sortrawientries(Index *ix, Part *tmp, u64int *tmpoff); -void statsinit(void); -int storeclump(Index *index, ZBlock *b, u8int *score, int type, u32int creator, IAddr *ia); -int storeientry(Index *index, IEntry *m); -int strscore(char *s, u8int *score); -int stru32int(char *s, u32int *r); -int stru64int(char *s, u64int *r); -void sumarena(Arena *arena); -int syncarena(Arena *arena, u32int n, int zok, int fix); -int syncarenaindex(Index *ix, Arena *arena, u32int clump, u64int a, int fix); -int syncindex(Index *ix, int fix); -int u64log2(u64int v); -u64int unittoull(char *s); -int unpackarena(Arena *arena, u8int *buf); -int unpackarenahead(ArenaHead *head, u8int *buf); -int unpackarenapart(ArenaPart *as, u8int *buf); -int unpackclump(Clump *c, u8int *buf); -void unpackclumpinfo(ClumpInfo *ci, u8int *buf); -void unpackibucket(IBucket *b, u8int *buf); -void unpackientry(IEntry *i, u8int *buf); -int unpackisect(ISect *is, u8int *buf); -u32int unpackmagic(u8int *buf); -int vtproc(void(*)(void*), void*); -int vttypevalid(int type); -int wbarena(Arena *arena); -int wbarenahead(Arena *arena); -int wbarenamap(AMap *am, int n, Part *part, u64int base, u64int size); -int wbarenapart(ArenaPart *ap); -int wbisect(ISect *is); -int wbindex(Index *ix); -int whackblock(u8int *dst, u8int *src, int ssize); -u64int writeaclump(Arena *a, Clump *c, u8int *clbuf); -u32int writearena(Arena *arena, u64int aa, u8int *clbuf, u32int n); -int writeclumpinfo(Arena *arean, int clump, ClumpInfo *ci); -u64int writeiclump(Index *ix, Clump *c, u8int *clbuf); -int writelump(Packet *p, u8int *score, int type, u32int creator); -int writepart(Part *part, u64int addr, u8int *buf, u32int n); -int writeqlump(Lump *u, Packet *p, int creator); -Packet *zblock2packet(ZBlock *zb, u32int size); -void zeropart(Part *part, int blocksize); - -/* -#pragma varargck argpos sysfatal 1 -#pragma varargck argpos logerr 2 -#pragma varargck argpos SetErr 2 -*/ - -#define scorecmp(h1,h2) memcmp((h1),(h2),VtScoreSize) -#define scorecp(h1,h2) memmove((h1),(h2),VtScoreSize) - -#define MK(t) ((t*)emalloc(sizeof(t))) -#define MKZ(t) ((t*)ezmalloc(sizeof(t))) -#define MKN(t,n) ((t*)emalloc((n)*sizeof(t))) -#define MKNZ(t,n) ((t*)ezmalloc((n)*sizeof(t))) -#define MKNA(t,at,n) ((t*)emalloc(sizeof(t) + (n)*sizeof(at))) blob - 6bd637188235a79ee484e771669b747680e6cebf (mode 644) blob + /dev/null --- src/cmd/venti/httpd.c +++ /dev/null @@ -1,453 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" -#include "httpd.h" -#include "xml.h" - -typedef struct HttpObj HttpObj; - -enum -{ - ObjNameSize = 64, - MaxObjs = 16 -}; - -struct HttpObj -{ - char name[ObjNameSize]; - int (*f)(HConnect*); -}; - -static HttpObj objs[MaxObjs]; - -static void listenproc(void*); -static int estats(HConnect *c); -static int dindex(HConnect *c); -static int xindex(HConnect *c); -static int sindex(HConnect *c); -static int notfound(HConnect *c); -static int httpdobj(char *name, int (*f)(HConnect*)); - -int -httpdinit(char *address) -{ - fmtinstall('D', hdatefmt); - fmtinstall('H', httpfmt); - fmtinstall('U', hurlfmt); - - if(address == nil) - address = "tcp!*!http"; - - httpdobj("/stats", estats); - httpdobj("/index", dindex); - httpdobj("/storage", sindex); - httpdobj("/xindex", xindex); - - if(vtproc(listenproc, address) < 0) - return -1; - return 0; -} - -static int -httpdobj(char *name, int (*f)(HConnect*)) -{ - int i; - - if(name == nil || strlen(name) >= ObjNameSize) - return -1; - for(i = 0; i < MaxObjs; i++){ - if(objs[i].name[0] == '\0'){ - strcpy(objs[i].name, name); - objs[i].f = f; - return 0; - } - if(strcmp(objs[i].name, name) == 0) - return -1; - } - return -1; -} - -static HConnect* -mkconnect(void) -{ - HConnect *c; - - c = mallocz(sizeof(HConnect), 1); - if(c == nil) - sysfatal("out of memory"); - c->replog = nil; - c->hpos = c->header; - c->hstop = c->header; - return c; -} - -void httpproc(void*); - -static void -listenproc(void *vaddress) -{ - HConnect *c; - char *address, ndir[NETPATHLEN], dir[NETPATHLEN]; - int ctl, nctl, data; - -//sleep(1000); /* let strace find us */ - - address = vaddress; - ctl = announce(address, dir); - if(ctl < 0){ - fprint(2, "venti: httpd can't announce on %s: %r\n", address); - return; - } - -print("announce ctl %d dir %s\n", ctl, dir); - for(;;){ - /* - * wait for a call (or an error) - */ - nctl = listen(dir, ndir); -print("httpd listen %d %s...\n", nctl, ndir); - if(nctl < 0){ - fprint(2, "venti: httpd can't listen on %s: %r\n", address); - return; - } - - data = accept(ctl, ndir); -print("httpd accept %d...\n", data); - if(data < 0){ - fprint(2, "venti: httpd accept: %r\n"); - close(nctl); - continue; - } -print("httpd close nctl %d\n", nctl); - close(nctl); - c = mkconnect(); - hinit(&c->hin, data, Hread); - hinit(&c->hout, data, Hwrite); - vtproc(httpproc, c); - } -} - -void -httpproc(void *v) -{ - HConnect *c; - int ok, t, i; - -//sleep(1000); /* let strace find us */ - c = v; - - for(t = 15*60*1000; ; t = 15*1000){ - if(hparsereq(c, t) < 0) - break; - - ok = -1; - for(i = 0; i < MaxObjs && objs[i].name[0]; i++){ - if(strcmp(c->req.uri, objs[i].name) == 0){ - ok = (*objs[i].f)(c); - break; - } - } - if(i == MaxObjs) - ok = notfound(c); - if(c->head.closeit) - ok = -1; - hreqcleanup(c); - - if(ok < 0) - break; - } - hreqcleanup(c); - close(c->hin.fd); - free(c); -} - -static int -percent(long v, long total) -{ - if(total == 0) - total = 1; - if(v < 1000*1000) - return (v * 100) / total; - total /= 100; - if(total == 0) - total = 1; - return v / total; -} - -static int -preq(HConnect *c) -{ - if(hparseheaders(c, 15*60*1000) < 0) -{ -fprint(2, "hparseheaders failed\n"); - return -1; -} - if(strcmp(c->req.meth, "GET") != 0 - && strcmp(c->req.meth, "HEAD") != 0) - return hunallowed(c, "GET, HEAD"); - if(c->head.expectother || c->head.expectcont) - return hfail(c, HExpectFail, nil); - return 0; -} - -static int -preqtext(HConnect *c) -{ - Hio *hout; - int r; - - r = preq(c); - if(r < 0) - return r; - - hout = &c->hout; - if(c->req.vermaj){ - hokheaders(c); - hprint(hout, "Content-type: text/plain\r\n"); - if(http11(c)) - hprint(hout, "Transfer-Encoding: chunked\r\n"); - hprint(hout, "\r\n"); - } - - if(http11(c)) - hxferenc(hout, 1); - else - c->head.closeit = 1; - return 0; -} - -static int -notfound(HConnect *c) -{ - int r; - - r = preq(c); - if(r < 0) - return r; - return hfail(c, HNotFound, c->req.uri); -} - -static int -estats(HConnect *c) -{ - Hio *hout; - int r; - - r = preqtext(c); - if(r < 0) - return r; - - - hout = &c->hout; - hprint(hout, "lump writes=%,ld\n", stats.lumpwrites); - hprint(hout, "lump reads=%,ld\n", stats.lumpreads); - hprint(hout, "lump cache read hits=%,ld\n", stats.lumphit); - hprint(hout, "lump cache read misses=%,ld\n", stats.lumpmiss); - - hprint(hout, "clump disk writes=%,ld\n", stats.clumpwrites); - hprint(hout, "clump disk bytes written=%,lld\n", stats.clumpbwrites); - hprint(hout, "clump disk bytes compressed=%,lld\n", stats.clumpbcomp); - hprint(hout, "clump disk reads=%,ld\n", stats.clumpreads); - hprint(hout, "clump disk bytes read=%,lld\n", stats.clumpbreads); - hprint(hout, "clump disk bytes uncompressed=%,lld\n", stats.clumpbuncomp); - - hprint(hout, "clump directory disk writes=%,ld\n", stats.ciwrites); - hprint(hout, "clump directory disk reads=%,ld\n", stats.cireads); - - hprint(hout, "index disk writes=%,ld\n", stats.indexwrites); - hprint(hout, "index disk reads=%,ld\n", stats.indexreads); - hprint(hout, "index disk reads for modify=%,ld\n", stats.indexwreads); - hprint(hout, "index disk reads for allocation=%,ld\n", stats.indexareads); - hprint(hout, "index block splits=%,ld\n", stats.indexsplits); - - hprint(hout, "index cache lookups=%,ld\n", stats.iclookups); - hprint(hout, "index cache hits=%,ld %d%%\n", stats.ichits, - percent(stats.ichits, stats.iclookups)); - hprint(hout, "index cache fills=%,ld %d%%\n", stats.icfills, - percent(stats.icfills, stats.iclookups)); - hprint(hout, "index cache inserts=%,ld\n", stats.icinserts); - - hprint(hout, "disk cache hits=%,ld\n", stats.pchit); - hprint(hout, "disk cache misses=%,ld\n", stats.pcmiss); - hprint(hout, "disk cache reads=%,ld\n", stats.pcreads); - hprint(hout, "disk cache bytes read=%,lld\n", stats.pcbreads); - - hprint(hout, "disk cache writes=%,ld\n", stats.dirtydblocks); - hprint(hout, "disk cache writes absorbed=%,ld %d%%\n", stats.absorbedwrites, - percent(stats.absorbedwrites, stats.dirtydblocks)); - - hprint(hout, "disk cache flushes=%,ld\n", stats.dcacheflushes); - hprint(hout, "disk cache flush writes=%,ld (%,ld per flush)\n", - stats.dcacheflushwrites, - stats.dcacheflushwrites/(stats.dcacheflushes ? stats.dcacheflushes : 1)); - - hprint(hout, "disk writes=%,ld\n", stats.diskwrites); - hprint(hout, "disk bytes written=%,lld\n", stats.diskbwrites); - hprint(hout, "disk reads=%,ld\n", stats.diskreads); - hprint(hout, "disk bytes read=%,lld\n", stats.diskbreads); - - hflush(hout); - return 0; -} - -static int -sindex(HConnect *c) -{ - Hio *hout; - Index *ix; - Arena *arena; - vlong clumps, cclumps, uncsize, used, size; - int i, r, active; - - r = preqtext(c); - if(r < 0) - return r; - hout = &c->hout; - - ix = mainindex; - - hprint(hout, "index=%s\n", ix->name); - - active = 0; - clumps = 0; - cclumps = 0; - uncsize = 0; - used = 0; - size = 0; - for(i = 0; i < ix->narenas; i++){ - arena = ix->arenas[i]; - if(arena != nil && arena->clumps != 0){ - active++; - clumps += arena->clumps; - cclumps += arena->cclumps; - uncsize += arena->uncsize; - used += arena->used; - } - size += arena->size; - } - hprint(hout, "total arenas=%d active=%d\n", ix->narenas, active); - hprint(hout, "total space=%lld used=%lld\n", size, used + clumps * ClumpInfoSize); - hprint(hout, "clumps=%lld compressed clumps=%lld data=%lld compressed data=%lld\n", - clumps, cclumps, uncsize, used - clumps * ClumpSize); - hflush(hout); - return 0; -} - -static void -darena(Hio *hout, Arena *arena) -{ - hprint(hout, "arena='%s' on %s at [%lld,%lld)\n\tversion=%d created=%d modified=%d", - arena->name, arena->part->name, arena->base, arena->base + arena->size + 2 * arena->blocksize, - arena->version, arena->ctime, arena->wtime); - if(arena->sealed) - hprint(hout, " sealed\n"); - else - hprint(hout, "\n"); - if(scorecmp(zeroscore, arena->score) != 0) - hprint(hout, "\tscore=%V\n", arena->score); - - hprint(hout, "\tclumps=%d compressed clumps=%d data=%lld compressed data=%lld disk storage=%lld\n", - arena->clumps, arena->cclumps, arena->uncsize, - arena->used - arena->clumps * ClumpSize, - arena->used + arena->clumps * ClumpInfoSize); -} - -static int -dindex(HConnect *c) -{ - Hio *hout; - Index *ix; - int i, r; - - r = preqtext(c); - if(r < 0) - return r; - hout = &c->hout; - - - ix = mainindex; - hprint(hout, "index=%s version=%d blocksize=%d tabsize=%d\n", - ix->name, ix->version, ix->blocksize, ix->tabsize); - hprint(hout, "\tbuckets=%d div=%d\n", ix->buckets, ix->div); - for(i = 0; i < ix->nsects; i++) - hprint(hout, "\tsect=%s for buckets [%lld,%lld) buckmax=%d\n", ix->smap[i].name, ix->smap[i].start, ix->smap[i].stop, ix->sects[i]->buckmax); - for(i = 0; i < ix->narenas; i++){ - if(ix->arenas[i] != nil && ix->arenas[i]->clumps != 0){ - hprint(hout, "arena=%s at index [%lld,%lld)\n\t", ix->amap[i].name, ix->amap[i].start, ix->amap[i].stop); - darena(hout, ix->arenas[i]); - } - } - hflush(hout); - return 0; -} - -static int -xindex(HConnect *c) -{ - Hio *hout; - int r; - - r = preq(c); - if(r < 0) - return r; - - hout = &c->hout; - if(c->req.vermaj){ - hokheaders(c); - hprint(hout, "Content-type: text/xml\r\n"); - if(http11(c)) - hprint(hout, "Transfer-Encoding: chunked\r\n"); - hprint(hout, "\r\n"); - } - - if(http11(c)) - hxferenc(hout, 1); - else - c->head.closeit = 1; - xmlindex(hout, mainindex, "index", 0); - hflush(hout); - return 0; -} - -void -xmlindent(Hio *hout, int indent) -{ - int i; - - for(i = 0; i < indent; i++) - hputc(hout, '\t'); -} - -void -xmlaname(Hio *hout, char *v, char *tag) -{ - hprint(hout, " %s=\"%s\"", tag, v); -} - -void -xmlscore(Hio *hout, u8int *v, char *tag) -{ - if(scorecmp(zeroscore, v) == 0) - return; - hprint(hout, " %s=\"%V\"", tag, v); -} - -void -xmlsealed(Hio *hout, int v, char *tag) -{ - if(!v) - return; - hprint(hout, " %s=\"yes\"", tag); -} - -void -xmlu32int(Hio *hout, u32int v, char *tag) -{ - hprint(hout, " %s=\"%ud\"", tag, v); -} - -void -xmlu64int(Hio *hout, u64int v, char *tag) -{ - hprint(hout, " %s=\"%llud\"", tag, v); -} blob - fc1d32e7cd7616f23ff47607343f70e988b368ac (mode 644) blob + /dev/null --- src/cmd/venti/icache.c +++ /dev/null @@ -1,198 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -struct ICache -{ - QLock lock; /* locks hash table & all associated data */ - IEntry **heads; /* heads of all the hash chains */ - int bits; /* bits to use for indexing heads */ - u32int size; /* number of heads; == 1 << bits, should be < entries */ - IEntry *base; /* all allocated hash table entries */ - u32int entries; /* elements in base */ - u32int unused; /* index of first unused element in base */ - u32int stolen; /* last head from which an element was stolen */ -}; - -static ICache icache; - -static IEntry *icachealloc(IAddr *ia, u8int *score); - -/* - * bits is the number of bits in the icache hash table - * depth is the average depth - * memory usage is about (1<>= (32 - bits); - return v; -} - -/* -ZZZ need to think about evicting the correct IEntry, -and writing back the wtime. - * look up data score in the index cache - * if this fails, pull it in from the disk index table, if it exists. - * - * must be called with the lump for this score locked - */ -int -lookupscore(u8int *score, int type, IAddr *ia, int *rac) -{ - IEntry d, *ie, *last; - u32int h; - - qlock(&stats.lock); - stats.iclookups++; - qunlock(&stats.lock); - - qlock(&icache.lock); - h = hashbits(score, icache.bits); - last = nil; - for(ie = icache.heads[h]; ie != nil; ie = ie->next){ - if(ie->ia.type == type && scorecmp(ie->score, score)==0){ - if(last != nil) - last->next = ie->next; - else - icache.heads[h] = ie->next; - qlock(&stats.lock); - stats.ichits++; - qunlock(&stats.lock); - ie->rac = 1; - goto found; - } - last = ie; - } - - qunlock(&icache.lock); - - if(loadientry(mainindex, score, type, &d) < 0) - return -1; - - /* - * no one else can load an entry for this score, - * since we have the overall score lock. - */ - qlock(&stats.lock); - stats.icfills++; - qunlock(&stats.lock); - - qlock(&icache.lock); - - ie = icachealloc(&d.ia, score); - -found: - ie->next = icache.heads[h]; - icache.heads[h] = ie; - - *ia = ie->ia; - *rac = ie->rac; - - qunlock(&icache.lock); - - return 0; -} - -/* - * insert a new element in the hash table. - */ -int -insertscore(u8int *score, IAddr *ia, int write) -{ - IEntry *ie, se; - u32int h; - - qlock(&stats.lock); - stats.icinserts++; - qunlock(&stats.lock); - - qlock(&icache.lock); - h = hashbits(score, icache.bits); - - ie = icachealloc(ia, score); - - ie->next = icache.heads[h]; - icache.heads[h] = ie; - - se = *ie; - - qunlock(&icache.lock); - - if(!write) - return 0; - - return storeientry(mainindex, &se); -} - -/* - * allocate a index cache entry which hasn't been used in a while. - * must be called with icache.lock locked - * if the score is already in the table, update the entry. - */ -static IEntry * -icachealloc(IAddr *ia, u8int *score) -{ - IEntry *ie, *last, *next; - u32int h; - - h = hashbits(score, icache.bits); - last = nil; - for(ie = icache.heads[h]; ie != nil; ie = ie->next){ - if(ie->ia.type == ia->type && scorecmp(ie->score, score)==9){ - if(last != nil) - last->next = ie->next; - else - icache.heads[h] = ie->next; - ie->rac = 1; - return ie; - } - last = ie; - } - - h = icache.unused; - if(h < icache.entries){ - ie = &icache.base[h++]; - icache.unused = h; - goto Found; - } - - h = icache.stolen; - for(;;){ - h++; - if(h >= icache.size) - h = 0; - ie = icache.heads[h]; - if(ie != nil){ - last = nil; - for(; next = ie->next; ie = next) - last = ie; - if(last != nil) - last->next = nil; - else - icache.heads[h] = nil; - icache.stolen = h; - goto Found; - } - } -Found: - ie->ia = *ia; - scorecp(ie->score, score); - ie->rac = 0; - return ie; -} blob - 811e487caf43702bcdd9ee201931e450f8a11115 (mode 644) blob + /dev/null --- src/cmd/venti/ifile.c +++ /dev/null @@ -1,93 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -int -readifile(IFile *f, char *name) -{ - ZBlock *b; - - b = readfile(name); - if(b == nil) - return -1; - f->name = name; - f->b = b; - f->pos = 0; - return 0; -} - -void -freeifile(IFile *f) -{ - freezblock(f->b); - f->b = nil; - f->pos = 0; -} - -int -partifile(IFile *f, Part *part, u64int start, u32int size) -{ - ZBlock *b; - - b = alloczblock(size, 0); - if(b == nil) - return -1; - if(readpart(part, start, b->data, size) < 0){ - seterr(EAdmin, "can't read %s: %r", part->name); - freezblock(b); - return -1; - } - f->name = part->name; - f->b = b; - f->pos = 0; - return 0; -} - -/* - * return the next non-blank input line, - * stripped of leading white space and with # comments eliminated - */ -char* -ifileline(IFile *f) -{ - char *s, *e, *t; - int c; - - for(;;){ - s = (char*)&f->b->data[f->pos]; - e = memchr(s, '\n', f->b->len - f->pos); - if(e == nil) - return nil; - *e++ = '\0'; - f->pos = e - (char*)f->b->data; - t = strchr(s, '#'); - if(t != nil) - *t = '\0'; - for(; c = *s; s++) - if(c != ' ' && c != '\t' && c != '\r') - return s; - } -} - -int -ifilename(IFile *f, char *dst) -{ - char *s; - - s = ifileline(f); - if(s == nil || strlen(s) >= ANameSize) - return -1; - namecp(dst, s); - return 0; -} - -int -ifileu32int(IFile *f, u32int *r) -{ - char *s; - - s = ifileline(f); - if(s == nil) - return -1; - return stru32int(s, r); -} blob - c1fc141888576f502526ea8201a4fac56b99cd95 (mode 644) blob + /dev/null --- src/cmd/venti/index.c +++ /dev/null @@ -1,1253 +0,0 @@ -/* - * Index, mapping scores to log positions. - * - * The index is made up of some number of index sections, each of - * which is typically stored on a different disk. The blocks in all the - * index sections are logically numbered, with each index section - * responsible for a range of blocks. Blocks are typically 8kB. - * - * Index Version 1: - * - * The N index blocks are treated as a giant hash table. The top 32 bits - * of score are used as the key for a lookup. Each index block holds - * one hash bucket, which is responsible for ceil(2^32 / N) of the key space. - * - * The index is sized so that a particular bucket is extraordinarily - * unlikely to overflow: assuming compressed data blocks are 4kB - * on disk, and assuming each block has a 40 byte index entry, - * the index data will be 1% of the total data. Since scores are essentially - * random, all buckets should be about the same fullness. - * A factor of 5 gives us a wide comfort boundary to account for - * random variation. So the index disk space should be 5% of the arena disk space. - * - * Problems with Index Version 1: - * - * Because the index size is chosen to handle the worst case load, - * the index is very sparse, especially when the Venti server is mostly empty. - * This has a few bad properties. - * - * Loading an index block (which typically requires a random disk seek) - * is a very expensive operation, yet it yields only a couple index entries. - * We're not making efficient use of the disk arm. - * - * Writing a block requires first checking to see if the block already - * exists on the server, which in turn requires an index lookup. When - * writing fresh data, these lookups will fail. The index entry cache - * cannot serve these, so they must go to disk, which is expensive. - * - * Thus both the reading and the writing of blocks are held back by - * the expense of accessing the index. - * - * Index Version 2: - * - * The index is sized to be exactly 2^M blocks. The index blocks are - * logically arranged in a (not exactly balanced) binary tree of depth at - * most M. The nodes are named by bit strings describing the path from - * the root to the node. The root is . (dot). The left child of the root is .0, - * the right child .1. The node you get to by starting at the root and going - * left right right left is .0110. At the beginning, there is only the root block. - * When a block with name .xxx fills, it splits into .xxx0 and .xxx1. - * All the index data is kept in the leaves of the tree. - * - * Index leaf blocks are laid out on disk by interpreting the bitstring as a - * binary fraction and multiplying by 2^M -- .0 is the first block, .1 is - * the block halfway into the index, .0110 is at position 6/16, and - * .xxx and .xxx0 map to the same block (but only one can be a leaf - * node at any given time, so this is okay!). A cheap implementation of - * this is to append zeros to the bit string to make it M bits long. That's - * the integer index block number. - * - * To find the leaf block that should hold a score, use the bits of the - * score one at a time to walk down the tree to a leaf node. If the tree - * has leaf nodes .0, .10, and .11, then score 0110101... ends up in bucket - * .0 while score 101110101... ends up in bucket .10. There is no leaf node - * .1 because it has split into .10 and .11. - * - * If we know which disk blocks are in use, we can reconstruct the interior - * of the tree: if .xxx1 is in use, then .xxx has been split. We keep an in-use - * bitmap of all index disk blocks to aid in reconstructing the interior of the - * tree. At one bit per index block, the bitmap is small enough to be kept - * in memory even on the largest of Venti servers. - * - * After the root block splits, the index blocks being used will always be - * at least half full (averaged across the entire index). So unlike Version 1, - * Index Version 2 is quite dense, alleviating the two problems above. - * Index block reads now return many index entries. More importantly, - * small servers can keep most of the index in the disk cache, making them - * more likely to handle negative lookups without going to disk. - * - * As the index becomes more full, Index Version 2's performance - * degrades gracefully into Index Version 1. V2 is really an optimization - * for little servers. - * - * Write Ordering for Index Version 2: - * - * Unlike Index Version 1, Version 2 must worry about write ordering - * within the index. What happens if the in-use bitmap is out of sync - * with the actual leaf nodes? What happens if .xxx splits into .xxx0 and - * .xxx1 but only one of the new blocks gets written to disk? - * - * We arrange that when .xxx splits, the .xxx1 block is written first, - * then the .xxx0 block, and finally the in-use bitmap entry for .xxx1. - * The split is committed by the writing of .xxx0. This ordering leaves - * two possible partial disk writes: - * - * (a) If .xxx1 is written but .xxx0 and the bitmap are not, then it's as if - * the split never happened -- we won't think .xxx1 is in use, and we - * won't go looking for it. - * - * (b) If .xxx1 and .xxx0 are written but the bitmap is not, then the first - * time we try to load .xxx, we'll get .xxx0 instead, realize the bitmap is - * out of date, and update the bitmap. - * - * Backwards Compatibility - * - * Because there are large Venti servers in use with Index V1, this code - * will read either index version, but when asked to create a new index, - * will only create V2. - */ - -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -static int bucklook(u8int *score, int type, u8int *data, int n); -static int writebucket(ISect *is, u32int buck, IBucket *ib, DBlock *b); -static int okibucket(IBucket *ib, ISect *is); -static int initindex1(Index*); -static ISect *initisect1(ISect *is); -static int splitiblock(Index *ix, DBlock *b, ISect *is, u32int buck, IBucket *ib); - -#define KEY(k,d) ((d) ? (k)>>(32-(d)) : 0) - -//static QLock indexlock; //ZZZ - -static char IndexMagic[] = "venti index configuration"; - -Index* -initindex(char *name, ISect **sects, int n) -{ - IFile f; - Index *ix; - ISect *is; - u32int last, blocksize, tabsize; - int i; - - if(n <= 0){ - seterr(EOk, "no index sections to initialize index"); - return nil; - } - ix = MKZ(Index); - if(ix == nil){ - seterr(EOk, "can't initialize index: out of memory"); - freeindex(ix); - return nil; - } - - tabsize = sects[0]->tabsize; - if(partifile(&f, sects[0]->part, sects[0]->tabbase, tabsize) < 0) - return nil; - if(parseindex(&f, ix) < 0){ - freeifile(&f); - freeindex(ix); - return nil; - } - freeifile(&f); - if(namecmp(ix->name, name) != 0){ - seterr(ECorrupt, "mismatched index name: found %s expected %s", ix->name, name); - return nil; - } - if(ix->nsects != n){ - seterr(ECorrupt, "mismatched number index sections: found %d expected %d", n, ix->nsects); - freeindex(ix); - return nil; - } - ix->sects = sects; - last = 0; - blocksize = ix->blocksize; - for(i = 0; i < ix->nsects; i++){ - is = sects[i]; - if(namecmp(ix->name, is->index) != 0 - || is->blocksize != blocksize - || is->tabsize != tabsize - || namecmp(is->name, ix->smap[i].name) != 0 - || is->start != ix->smap[i].start - || is->stop != ix->smap[i].stop - || last != is->start - || is->start > is->stop){ - seterr(ECorrupt, "inconsistent index sections in %s", ix->name); - freeindex(ix); - return nil; - } - last = is->stop; - } - ix->tabsize = tabsize; - ix->buckets = last; - - if(initindex1(ix) < 0){ - freeindex(ix); - return nil; - } - - ix->arenas = MKNZ(Arena*, ix->narenas); - if(maparenas(ix->amap, ix->arenas, ix->narenas, ix->name) < 0){ - freeindex(ix); - return nil; - } - - return ix; -} - -static int -initindex1(Index *ix) -{ - u32int buckets; - - switch(ix->version){ - case IndexVersion1: - ix->div = (((u64int)1 << 32) + ix->buckets - 1) / ix->buckets; - buckets = (((u64int)1 << 32) - 1) / ix->div + 1; - if(buckets != ix->buckets){ - seterr(ECorrupt, "inconsistent math for divisor and buckets in %s", ix->name); - return -1; - } - break; - - case IndexVersion2: - buckets = ix->buckets - ix->bitblocks; - if(ix->buckets < ix->bitblocks || (buckets&(buckets-1))) - seterr(ECorrupt, "bucket count not a power of two in %s", ix->name); - ix->maxdepth = u64log2(buckets); - ix->bitkeylog = u64log2(ix->blocksize*8); - ix->bitkeymask = (1<bitkeylog)-1; - break; - } - return 0; -} - -int -wbindex(Index *ix) -{ - Fmt f; - ZBlock *b; - int i; - - if(ix->nsects == 0){ - seterr(EOk, "no sections in index %s", ix->name); - return -1; - } - b = alloczblock(ix->tabsize, 1); - if(b == nil){ - seterr(EOk, "can't write index configuration: out of memory"); - return -1; - } - fmtzbinit(&f, b); - if(outputindex(&f, ix) < 0){ - seterr(EOk, "can't make index configuration: table storage too small %d", ix->tabsize); - freezblock(b); - return -1; - } - for(i = 0; i < ix->nsects; i++){ - if(writepart(ix->sects[i]->part, ix->sects[i]->tabbase, b->data, ix->tabsize) < 0){ - seterr(EOk, "can't write index: %r"); - freezblock(b); - return -1; - } - } - freezblock(b); - - for(i = 0; i < ix->nsects; i++) - if(wbisect(ix->sects[i]) < 0) - return -1; - - return 0; -} - -/* - * index: IndexMagic '\n' version '\n' name '\n' blocksize '\n' [V2: bitblocks '\n'] sections arenas - * version, blocksize: u32int - * name: max. ANameSize string - * sections, arenas: AMap - */ -int -outputindex(Fmt *f, Index *ix) -{ - if(fmtprint(f, "%s\n%ud\n%s\n%ud\n", IndexMagic, ix->version, ix->name, ix->blocksize) < 0 - || (ix->version==IndexVersion2 && fmtprint(f, "%ud\n", ix->bitblocks) < 0) - || outputamap(f, ix->smap, ix->nsects) < 0 - || outputamap(f, ix->amap, ix->narenas) < 0) - return -1; - return 0; -} - -int -parseindex(IFile *f, Index *ix) -{ - AMapN amn; - u32int v; - char *s; - - /* - * magic - */ - s = ifileline(f); - if(s == nil || strcmp(s, IndexMagic) != 0){ - seterr(ECorrupt, "bad index magic for %s", f->name); - return -1; - } - - /* - * version - */ - if(ifileu32int(f, &v) < 0){ - seterr(ECorrupt, "syntax error: bad version number in %s", f->name); - return -1; - } - ix->version = v; - if(ix->version != IndexVersion1 && ix->version != IndexVersion2){ - seterr(ECorrupt, "bad version number in %s", f->name); - return -1; - } - - /* - * name - */ - if(ifilename(f, ix->name) < 0){ - seterr(ECorrupt, "syntax error: bad index name in %s", f->name); - return -1; - } - - /* - * block size - */ - if(ifileu32int(f, &v) < 0){ - seterr(ECorrupt, "syntax error: bad block size number in %s", f->name); - return -1; - } - ix->blocksize = v; - - if(ix->version == IndexVersion2){ - /* - * free bitmap size - */ - if(ifileu32int(f, &v) < 0){ - seterr(ECorrupt, "syntax error: bad bitmap size in %s", f->name); - return -1; - } - ix->bitblocks = v; - } - - if(parseamap(f, &amn) < 0) - return -1; - ix->nsects = amn.n; - ix->smap = amn.map; - - if(parseamap(f, &amn) < 0) - return -1; - ix->narenas = amn.n; - ix->amap = amn.map; - - return 0; -} - -/* - * initialize an entirely new index - */ -Index * -newindex(char *name, ISect **sects, int n) -{ - Index *ix; - AMap *smap; - u64int nb; - u32int div, ub, xb, fb, start, stop, blocksize, tabsize; - int i, j, version; - - version = IndexVersion2; - - if(n < 1){ - seterr(EOk, "creating index with no index sections"); - return nil; - } - - /* - * compute the total buckets available in the index, - * and the total buckets which are used. - */ - nb = 0; - blocksize = sects[0]->blocksize; - tabsize = sects[0]->tabsize; - for(i = 0; i < n; i++){ - if(sects[i]->start != 0 || sects[i]->stop != 0 - || sects[i]->index[0] != '\0'){ - seterr(EOk, "creating new index using non-empty section %s", sects[i]->name); - return nil; - } - if(blocksize != sects[i]->blocksize){ - seterr(EOk, "mismatched block sizes in index sections"); - return nil; - } - if(tabsize != sects[i]->tabsize){ - seterr(EOk, "mismatched config table sizes in index sections"); - return nil; - } - nb += sects[i]->blocks; - } - - /* - * check for duplicate names - */ - for(i = 0; i < n; i++){ - for(j = i + 1; j < n; j++){ - if(namecmp(sects[i]->name, sects[j]->name) == 0){ - seterr(EOk, "duplicate section name %s for index %s", sects[i]->name, name); - return nil; - } - } - } - - if(nb >= ((u64int)1 << 32)){ - seterr(EBug, "index too large"); - return nil; - } - - div = 0; - fb = 0; - if(version == IndexVersion1){ - div = (((u64int)1 << 32) + nb - 1) / nb; - ub = (((u64int)1 << 32) - 1) / div + 1; - if(div < 100){ - seterr(EBug, "index divisor too coarse"); - return nil; - } - }else{ - fb = (nb+blocksize*8-1)/(blocksize*8); - for(ub=1; ub<=((nb-fb)>>1); ub<<=1) - ; - ub += fb; - } - if(ub > nb){ - seterr(EBug, "index initialization math wrong"); - return nil; - } - xb = nb - ub; - - /* - * initialize each of the index sections - * and the section map table - */ - smap = MKNZ(AMap, n); - if(smap == nil){ - seterr(EOk, "can't create new index: out of memory"); - return nil; - } - start = 0; - for(i = 0; i < n; i++){ - stop = start + sects[i]->blocks - xb / n; - if(i == n - 1) - stop = ub; - sects[i]->start = start; - sects[i]->stop = stop; - namecp(sects[i]->index, name); - - smap[i].start = start; - smap[i].stop = stop; - namecp(smap[i].name, sects[i]->name); - start = stop; - } - - /* - * initialize the index itself - */ - ix = MKZ(Index); - if(ix == nil){ - seterr(EOk, "can't create new index: out of memory"); - free(smap); - return nil; - } - ix->version = version; - namecp(ix->name, name); - ix->sects = sects; - ix->smap = smap; - ix->nsects = n; - ix->blocksize = blocksize; - ix->buckets = ub; - ix->tabsize = tabsize; - ix->div = div; - ix->bitblocks = fb; - - if(initindex1(ix) < 0){ - free(smap); - return nil; - } - - return ix; -} - -ISect* -initisect(Part *part) -{ - ISect *is; - ZBlock *b; - int ok; - - b = alloczblock(HeadSize, 0); - if(b == nil || readpart(part, PartBlank, b->data, HeadSize) < 0){ - seterr(EAdmin, "can't read index section header: %r"); - return nil; - } - - is = MKZ(ISect); - if(is == nil){ - freezblock(b); - return nil; - } - is->part = part; - ok = unpackisect(is, b->data); - freezblock(b); - if(ok < 0){ - seterr(ECorrupt, "corrupted index section header: %r"); - freeisect(is); - return nil; - } - - if(is->version != ISectVersion){ - seterr(EAdmin, "unknown index section version %d", is->version); - freeisect(is); - return nil; - } - - return initisect1(is); -} - -ISect* -newisect(Part *part, char *name, u32int blocksize, u32int tabsize) -{ - ISect *is; - u32int tabbase; - - is = MKZ(ISect); - if(is == nil) - return nil; - - namecp(is->name, name); - is->version = ISectVersion; - is->part = part; - is->blocksize = blocksize; - is->start = 0; - is->stop = 0; - tabbase = (PartBlank + HeadSize + blocksize - 1) & ~(blocksize - 1); - is->blockbase = (tabbase + tabsize + blocksize - 1) & ~(blocksize - 1); - is->blocks = is->part->size / blocksize - is->blockbase / blocksize; - - is = initisect1(is); - if(is == nil) - return nil; - - return is; -} - -/* - * initialize the computed parameters for an index - */ -static ISect* -initisect1(ISect *is) -{ - u64int v; - - is->buckmax = (is->blocksize - IBucketSize) / IEntrySize; - is->blocklog = u64log2(is->blocksize); - if(is->blocksize != (1 << is->blocklog)){ - seterr(ECorrupt, "illegal non-power-of-2 bucket size %d\n", is->blocksize); - freeisect(is); - return nil; - } - partblocksize(is->part, is->blocksize); - is->tabbase = (PartBlank + HeadSize + is->blocksize - 1) & ~(is->blocksize - 1); - if(is->tabbase >= is->blockbase){ - seterr(ECorrupt, "index section config table overlaps bucket storage"); - freeisect(is); - return nil; - } - is->tabsize = is->blockbase - is->tabbase; - v = is->part->size & ~(u64int)(is->blocksize - 1); - if(is->blockbase + (u64int)is->blocks * is->blocksize != v){ - seterr(ECorrupt, "invalid blocks in index section %s", is->name); -//ZZZZZZZZZ -// freeisect(is); -// return nil; - } - - if(is->stop - is->start > is->blocks){ - seterr(ECorrupt, "index section overflows available space"); - freeisect(is); - return nil; - } - if(is->start > is->stop){ - seterr(ECorrupt, "invalid index section range"); - freeisect(is); - return nil; - } - - return is; -} - -int -wbisect(ISect *is) -{ - ZBlock *b; - - b = alloczblock(HeadSize, 1); - if(b == nil) -//ZZZ set error? - return -1; - - if(packisect(is, b->data) < 0){ - seterr(ECorrupt, "can't make index section header: %r"); - freezblock(b); - return -1; - } - if(writepart(is->part, PartBlank, b->data, HeadSize) < 0){ - seterr(EAdmin, "can't write index section header: %r"); - freezblock(b); - return -1; - } - freezblock(b); - - return 0; -} - -void -freeisect(ISect *is) -{ - if(is == nil) - return; - free(is); -} - -void -freeindex(Index *ix) -{ - int i; - - if(ix == nil) - return; - free(ix->amap); - free(ix->arenas); - if(ix->sects) - for(i = 0; i < ix->nsects; i++) - freeisect(ix->sects[i]); - free(ix->sects); - free(ix->smap); - free(ix); -} - -/* - * write a clump to an available arena in the index - * and return the address of the clump within the index. -ZZZ question: should this distinguish between an arena -filling up and real errors writing the clump? - */ -u64int -writeiclump(Index *ix, Clump *c, u8int *clbuf) -{ - u64int a; - int i; - - for(i = ix->mapalloc; i < ix->narenas; i++){ - a = writeaclump(ix->arenas[i], c, clbuf); - if(a != TWID64) - return a + ix->amap[i].start; - } - - seterr(EAdmin, "no space left in arenas"); - return TWID64; -} - -/* - * convert an arena index to an relative arena address - */ -Arena* -amapitoa(Index *ix, u64int a, u64int *aa) -{ - int i, r, l, m; - - l = 1; - r = ix->narenas - 1; - while(l <= r){ - m = (r + l) / 2; - if(ix->amap[m].start <= a) - l = m + 1; - else - r = m - 1; - } - l--; - - if(a > ix->amap[l].stop){ -for(i=0; inarenas; i++) - print("arena %d: %llux - %llux\n", i, ix->amap[i].start, ix->amap[i].stop); -print("want arena %d for %llux\n", l, a); - seterr(ECrash, "unmapped address passed to amapitoa"); - return nil; - } - - if(ix->arenas[l] == nil){ - seterr(ECrash, "unmapped arena selected in amapitoa"); - return nil; - } - *aa = a - ix->amap[l].start; - return ix->arenas[l]; -} - -int -iaddrcmp(IAddr *ia1, IAddr *ia2) -{ - return ia1->type != ia2->type - || ia1->size != ia2->size - || ia1->blocks != ia2->blocks - || ia1->addr != ia2->addr; -} - -/* - * lookup the score in the partition - * - * nothing needs to be explicitly locked: - * only static parts of ix are used, and - * the bucket is locked by the DBlock lock. - */ -int -loadientry(Index *ix, u8int *score, int type, IEntry *ie) -{ - ISect *is; - DBlock *b; - IBucket ib; - u32int buck; - int h, ok; - - ok = -1; - - qlock(&stats.lock); - stats.indexreads++; - qunlock(&stats.lock); - b = loadibucket(ix, score, &is, &buck, &ib); - if(b == nil) - return -1; - - if(okibucket(&ib, is) < 0) - goto out; - - h = bucklook(score, type, ib.data, ib.n); - if(h & 1){ - h ^= 1; - unpackientry(ie, &ib.data[h]); - ok = 0; - goto out; - } - -out: - putdblock(b); - return ok; -} - -/* - * insert or update an index entry into the appropriate bucket - */ -int -storeientry(Index *ix, IEntry *ie) -{ - ISect *is; - DBlock *b; - IBucket ib; - u32int buck; - int h, ok; - - ok = 0; - - qlock(&stats.lock); - stats.indexwreads++; - qunlock(&stats.lock); - -top: - b = loadibucket(ix, ie->score, &is, &buck, &ib); - if(b == nil) - return -1; - - if(okibucket(&ib, is) < 0) - goto out; - - h = bucklook(ie->score, ie->ia.type, ib.data, ib.n); - if(h & 1){ - h ^= 1; - dirtydblock(b, DirtyIndex); - packientry(ie, &ib.data[h]); - ok = writebucket(is, buck, &ib, b); - goto out; - } - - if(ib.n < is->buckmax){ - dirtydblock(b, DirtyIndex); - memmove(&ib.data[h + IEntrySize], &ib.data[h], ib.n * IEntrySize - h); - ib.n++; - - packientry(ie, &ib.data[h]); - ok = writebucket(is, buck, &ib, b); - goto out; - } - - /* block is full -- not supposed to happen in V1 */ - if(ix->version == IndexVersion1){ - seterr(EAdmin, "index bucket %ud on %s is full\n", buck, is->part->name); - ok = -1; - goto out; - } - - /* in V2, split the block and try again; splitiblock puts b */ - if(splitiblock(ix, b, is, buck, &ib) < 0) - return -1; - goto top; - -out: - putdblock(b); - return ok; -} - -static int -writebucket(ISect *is, u32int buck, IBucket *ib, DBlock *b) -{ - assert(b->dirty == DirtyIndex || b->dirty == DirtyIndexSplit); - - if(buck >= is->blocks){ - seterr(EAdmin, "index write out of bounds: %d >= %d\n", - buck, is->blocks); - return -1; - } - qlock(&stats.lock); - stats.indexwrites++; - qunlock(&stats.lock); - packibucket(ib, b->data); - // return writepart(is->part, is->blockbase + ((u64int)buck << is->blocklog), b->data, is->blocksize); - return 0; -} - -static int -okibucket(IBucket *ib, ISect *is) -{ - if(ib->n <= is->buckmax) - return 0; - - seterr(EICorrupt, "corrupted disk index bucket: n=%ud max=%ud, depth=%lud range=[%lud,%lud)", - ib->n, is->buckmax, ib->depth, is->start, is->stop); - return -1; -} - -/* - * look for score within data; - * return 1 | byte index of matching index, - * or 0 | index of least element > score - */ -static int -bucklook(u8int *score, int otype, u8int *data, int n) -{ - int i, r, l, m, h, c, cc, type; - - type = vttodisktype(otype); - l = 0; - r = n - 1; - while(l <= r){ - m = (r + l) >> 1; - h = m * IEntrySize; - for(i = 0; i < VtScoreSize; i++){ - c = score[i]; - cc = data[h + i]; - if(c != cc){ - if(c > cc) - l = m + 1; - else - r = m - 1; - goto cont; - } - } - cc = data[h + IEntryTypeOff]; - if(type != cc){ - if(type > cc) - l = m + 1; - else - r = m - 1; - goto cont; - } - return h | 1; - cont:; - } - - return l * IEntrySize; -} - -/* - * compare two IEntries; consistent with bucklook - */ -int -ientrycmp(const void *vie1, const void *vie2) -{ - u8int *ie1, *ie2; - int i, v1, v2; - - ie1 = (u8int*)vie1; - ie2 = (u8int*)vie2; - for(i = 0; i < VtScoreSize; i++){ - v1 = ie1[i]; - v2 = ie2[i]; - if(v1 != v2){ - if(v1 < v2) - return -1; - return 0; - } - } - v1 = ie1[IEntryTypeOff]; - v2 = ie2[IEntryTypeOff]; - if(v1 != v2){ - if(v1 < v2) - return -1; - return 0; - } - return -1; -} - -/* - * find the number of the index section holding bucket #buck - */ -static int -indexsect0(Index *ix, u32int buck) -{ - int r, l, m; - - l = 1; - r = ix->nsects - 1; - while(l <= r){ - m = (r + l) >> 1; - if(ix->sects[m]->start <= buck) - l = m + 1; - else - r = m - 1; - } - return l - 1; -} - -/* - * load the index block at bucket #buck - */ -static DBlock* -loadibucket0(Index *ix, u32int buck, ISect **pis, u32int *pbuck, IBucket *ib, int read) -{ - ISect *is; - DBlock *b; - - is = ix->sects[indexsect0(ix, buck)]; - if(buck < is->start || is->stop <= buck){ - seterr(EAdmin, "index lookup out of range: %ud not found in index\n", buck); - return nil; - } - - buck -= is->start; - if((b = getdblock(is->part, is->blockbase + ((u64int)buck << is->blocklog), read)) == nil) - return nil; - - if(pis) - *pis = is; - if(pbuck) - *pbuck = buck; - if(ib) - unpackibucket(ib, b->data); - return b; -} - -/* - * find the number of the index section holding score - */ -static int -indexsect1(Index *ix, u8int *score) -{ - return indexsect0(ix, hashbits(score, 32) / ix->div); -} - -/* - * load the index block responsible for score. - */ -static DBlock* -loadibucket1(Index *ix, u8int *score, ISect **pis, u32int *pbuck, IBucket *ib) -{ - return loadibucket0(ix, hashbits(score, 32)/ix->div, pis, pbuck, ib, 1); -} - -static u32int -keytobuck(Index *ix, u32int key, int d) -{ - /* clear all but top d bits; can't depend on boundary case shifts */ - if(d == 0) - key = 0; - else if(d != 32) - key &= ~((1<<(32-d))-1); - - /* truncate to maxdepth bits */ - if(ix->maxdepth != 32) - key >>= 32 - ix->maxdepth; - - return ix->bitblocks + key; -} - -/* - * to check whether .xxx has split, check whether .xxx1 is in use. - * it might be worth caching the block for future lookups, but for now - * let's assume the dcache is good enough. - */ -static int -bitmapop(Index *ix, u32int key, int d, int set) -{ - DBlock *b; - int inuse; - u32int key1, buck1; - - if(d >= ix->maxdepth) - return 0; - - - /* construct .xxx1 in bucket number format */ - key1 = key | (1<<(32-d-1)); - buck1 = keytobuck(ix, key1, d+1); - -if(0) fprint(2, "key %d/%0*ub key1 %d/%0*ub buck1 %08ux\n", - d, d, KEY(key, d), d+1, d+1, KEY(key1, d+1), buck1); - - /* check whether buck1 is in use */ - - if((b = loadibucket0(ix, buck1 >> ix->bitkeylog, nil, nil, nil, 1)) == nil){ - seterr(ECorrupt, "cannot load in-use bitmap block"); -fprint(2, "loadibucket: %r\n"); - return -1; - } -if(0) fprint(2, "buck1 %08ux bitkeymask %08ux bitkeylog %d\n", buck1, ix->bitkeymask, ix->bitkeylog); - buck1 &= ix->bitkeymask; - inuse = ((u32int*)b->data)[buck1>>5] & (1<<(buck1&31)); - if(set && !inuse){ - dirtydblock(b, DirtyIndexBitmap); - ((u32int*)b->data)[buck1>>5] |= (1<<(buck1&31)); - } - putdblock(b); - return inuse; -} - -static int -issplit(Index *ix, u32int key, int d) -{ - return bitmapop(ix, key, d, 0); -} - -static int -marksplit(Index *ix, u32int key, int d) -{ - return bitmapop(ix, key, d, 1); -} - -/* - * find the number of the index section holding score. - * it's not terrible to be wrong once in a while, so we just - * do what the bitmap tells us and don't worry about the - * bitmap being out of date. - */ -static int -indexsect2(Index *ix, u8int *score) -{ - u32int key; - int d, is; - - key = hashbits(score, 32); - for(d=0; d<=ix->maxdepth; d++){ - is = issplit(ix, key, d); - if(is == -1) - return 0; /* must return something valid! */ - if(!is) - break; - } - - if(d > ix->maxdepth){ - seterr(EBug, "index bitmap inconsistent with maxdepth"); - return 0; /* must return something valid! */ - } - - return indexsect0(ix, keytobuck(ix, key, d)); -} - -/* - * load the index block responsible for score. - * (unlike indexsect2, must be correct always.) - */ -static DBlock* -loadibucket2(Index *ix, u8int *score, ISect **pis, u32int *pbuck, IBucket *ib) -{ - u32int key; - int d, try, is; - DBlock *b; - - for(try=0; try<32; try++){ - key = hashbits(score, 32); - for(d=0; d<=ix->maxdepth; d++){ - is = issplit(ix, key, d); - if(is == -1) - return nil; - if(!is) - break; - } - if(d > ix->maxdepth){ - seterr(EBug, "index bitmap inconsistent with maxdepth"); - return nil; - } - - if((b = loadibucket0(ix, keytobuck(ix, key, d), pis, pbuck, ib, 1)) == nil) - return nil; - - if(ib->depth == d) - return b; - - if(ib->depth < d){ - fprint(2, "loaded block %ud for %d/%0*ub got %d/%0*ub\n", - *pbuck, d, d, KEY(key,d), ib->depth, ib->depth, KEY(key, ib->depth)); - seterr(EBug, "index block has smaller depth than expected -- cannot happen"); - putdblock(b); - return nil; - } - - /* - * ib->depth > d, meaning the bitmap was out of date. - * fix the bitmap and try again. if we actually updated - * the bitmap in splitiblock, this would only happen if - * venti crashed at an inopportune moment. but this way - * the code gets tested more. - */ -if(0) fprint(2, "update bitmap: %d/%0*ub is split\n", d, d, KEY(key,d)); - putdblock(b); - if(marksplit(ix, key, d) < 0) - return nil; - } - seterr(EBug, "loadibucket2 failed to sync bitmap with disk!"); - return nil; -} - -static int -splitiblock(Index *ix, DBlock *b, ISect *is, u32int buck, IBucket *ib) -{ - int i, d; - u8int *score; - u32int buck0, buck1, key0, key1, key, dmask; - DBlock *b0, *b1; - IBucket ib0, ib1; - - if(ib->depth == ix->maxdepth){ -if(0) fprint(2, "depth=%d == maxdepth\n", ib->depth); - seterr(EAdmin, "index bucket %ud on %s is full\n", buck, is->part->name); - putdblock(b); - return -1; - } - - buck = is->start+buck - ix->bitblocks; - d = ib->depth+1; - buck0 = buck; - buck1 = buck0 | (1<<(ix->maxdepth-d)); - if(ix->maxdepth == 32){ - key0 = buck0; - key1 = buck1; - }else{ - key0 = buck0 << (32-ix->maxdepth); - key1 = buck1 << (32-ix->maxdepth); - } - buck0 += ix->bitblocks; - buck1 += ix->bitblocks; - USED(buck0); - USED(key1); - - if(d == 32) - dmask = TWID32; - else - dmask = TWID32 ^ ((1<<(32-d))-1); - - /* - * Since we hold the lock for b, the bitmap - * thinks buck1 doesn't exist, and the bit - * for buck1 can't be updated without first - * locking and splitting b, it's safe to try to - * acquire the lock on buck1 without dropping b. - * No one else will go after it too. - * - * Also, none of the rest of the code ever locks - * more than one block at a time, so it's okay if - * we do. - */ - if((b1 = loadibucket0(ix, buck1, nil, nil, &ib1, 0)) == nil){ - putdblock(b); - return -1; - } - b0 = b; - ib0 = *ib; - - /* - * Funny locking going on here -- see dirtydblock. - * We must putdblock(b1) before putdblock(b0). - */ - dirtydblock(b0, DirtyIndex); - dirtydblock(b1, DirtyIndexSplit); - - /* - * Split the block contents. - * The block is already sorted, so it's pretty easy: - * the first part stays, the second part goes to b1. - */ - ib0.n = 0; - ib0.depth = d; - ib1.n = 0; - ib1.depth = d; - for(i=0; in; i++){ - score = ib->data+i*IEntrySize; - key = hashbits(score, 32); - if((key&dmask) != key0) - break; - } - ib0.n = i; - ib1.n = ib->n - ib0.n; - memmove(ib1.data, ib0.data+ib0.n*IEntrySize, ib1.n*IEntrySize); -if(0) fprint(2, "splitiblock %d in %d/%0*ub => %d in %d/%0*ub + %d in %d/%0*ub\n", - ib->n, d-1, d-1, key0>>(32-d+1), - ib0.n, d, d, key0>>(32-d), - ib1.n, d, d, key1>>(32-d)); - - packibucket(&ib0, b0->data); - packibucket(&ib1, b1->data); - - /* order matters! see comment above. */ - putdblock(b1); - putdblock(b0); - - /* - * let the recovery code take care of updating the bitmap. - */ - - qlock(&stats.lock); - stats.indexsplits++; - qunlock(&stats.lock); - - return 0; -} - -int -indexsect(Index *ix, u8int *score) -{ - if(ix->version == IndexVersion1) - return indexsect1(ix, score); - return indexsect2(ix, score); -} - -DBlock* -loadibucket(Index *ix, u8int *score, ISect **pis, u32int *pbuck, IBucket *ib) -{ - if(ix->version == IndexVersion1) - return loadibucket1(ix, score, pis, pbuck, ib); - return loadibucket2(ix, score, pis, pbuck, ib); -} - - blob - 746a89cab3216029d45c05f0647c975d3e157de4 (mode 644) blob + /dev/null --- src/cmd/venti/lump.c +++ /dev/null @@ -1,205 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -int queuewrites = 0; - -static Packet *readilump(Lump *u, IAddr *ia, u8int *score, int rac); - -Packet* -readlump(u8int *score, int type, u32int size) -{ - Lump *u; - Packet *p; - IAddr ia; - u32int n; - int rac; - - qlock(&stats.lock); - stats.lumpreads++; - qunlock(&stats.lock); - u = lookuplump(score, type); - if(u->data != nil){ - n = packetsize(u->data); - if(n > size){ - seterr(EOk, "read too small: asked for %d need at least %d", size, n); - putlump(u); - - return nil; - } - p = packetdup(u->data, 0, n); - putlump(u); - - return p; - } - - if(lookupscore(score, type, &ia, &rac) < 0){ - //ZZZ place to check for someone trying to guess scores - seterr(EOk, "no block with that score exists"); - - putlump(u); - return nil; - } - if(ia.size > size){ - seterr(EOk, "read too small 1: asked for %d need at least %d", size, ia.size); - - putlump(u); - return nil; - } - - p = readilump(u, &ia, score, rac); - putlump(u); - - return p; -} - -/* - * save away a lump, and return it's score. - * doesn't store duplicates, but checks that the data is really the same. - */ -int -writelump(Packet *p, u8int *score, int type, u32int creator) -{ - Lump *u; - int ok; - - qlock(&stats.lock); - stats.lumpwrites++; - qunlock(&stats.lock); - - packetsha1(p, score); - - u = lookuplump(score, type); - if(u->data != nil){ - ok = 0; - if(packetcmp(p, u->data) != 0){ - seterr(EStrange, "score collision"); - ok = -1; - } - packetfree(p); - putlump(u); - return ok; - } - - if(queuewrites) - return queuewrite(u, p, creator); - - ok = writeqlump(u, p, creator); - - putlump(u); - return ok; -} - -int -writeqlump(Lump *u, Packet *p, int creator) -{ - ZBlock *flat; - Packet *old; - IAddr ia; - int ok; - int rac; - - if(lookupscore(u->score, u->type, &ia, &rac) == 0){ - /* - * if the read fails, - * assume it was corrupted data and store the block again - */ - old = readilump(u, &ia, u->score, rac); - if(old != nil){ - ok = 0; - if(packetcmp(p, old) != 0){ - seterr(EStrange, "score collision"); - ok = -1; - } - packetfree(p); - packetfree(old); - - return ok; - } - logerr(EAdmin, "writelump: read %V failed, rewriting: %r\n", u->score); - } - - flat = packet2zblock(p, packetsize(p)); - ok = storeclump(mainindex, flat, u->score, u->type, creator, &ia); - freezblock(flat); - if(ok == 0) - ok = insertscore(u->score, &ia, 1); - if(ok == 0) - insertlump(u, p); - else - packetfree(p); - - return ok; -} - -static void -readahead(u64int a, Arena *arena, u64int aa, int n) -{ - u8int buf[ClumpSize]; - Clump cl; - IAddr ia; - - while(n > 0) { - if (aa >= arena->used) - break; - if(readarena(arena, aa, buf, ClumpSize) < ClumpSize) - break; - if(unpackclump(&cl, buf) < 0) - break; - ia.addr = a; - ia.type = cl.info.type; - ia.size = cl.info.uncsize; - ia.blocks = (cl.info.size + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog; - insertscore(cl.info.score, &ia, 0); - a += ClumpSize + cl.info.size; - aa += ClumpSize + cl.info.size; - n--; - } -} - -static Packet* -readilump(Lump *u, IAddr *ia, u8int *score, int rac) -{ - Arena *arena; - ZBlock *zb; - Packet *p, *pp; - Clump cl; - u64int a, aa; - u8int sc[VtScoreSize]; - - arena = amapitoa(mainindex, ia->addr, &aa); - if(arena == nil) - return nil; - - zb = loadclump(arena, aa, ia->blocks, &cl, sc, paranoid); - if(zb == nil) - return nil; - - if(ia->size != cl.info.uncsize){ - seterr(EInconsist, "index and clump size mismatch"); - freezblock(zb); - return nil; - } - if(ia->type != cl.info.type){ - seterr(EInconsist, "index and clump type mismatch"); - freezblock(zb); - return nil; - } - if(scorecmp(score, sc) != 0){ - seterr(ECrash, "score mismatch"); - freezblock(zb); - return nil; - } - - if(rac == 0) { - a = ia->addr + ClumpSize + cl.info.size; - aa += ClumpSize + cl.info.size; - readahead(a, arena, aa, 20); - } - - p = zblock2packet(zb, cl.info.uncsize); - freezblock(zb); - pp = packetdup(p, 0, packetsize(p)); - insertlump(u, pp); - return p; -} blob - 979271af37ec76db1768e707296da7a7ad3ff4ee (mode 644) blob + /dev/null --- src/cmd/venti/lumpcache.c +++ /dev/null @@ -1,380 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -typedef struct LumpCache LumpCache; - -enum -{ - HashLog = 9, - HashSize = 1<type = TWID8; - b->heap = TWID32; - b->next = last; - last = b; - } - lumpcache.free = last; - lumpcache.nheap = 0; -} - -Lump* -lookuplump(u8int *score, int type) -{ - Lump *b; - u32int h; - - h = hashbits(score, HashLog); - - /* - * look for the block in the cache - */ -//checklumpcache(); - qlock(&lumpcache.lock); -again: - for(b = lumpcache.heads[h]; b != nil; b = b->next){ - if(scorecmp(score, b->score)==0 && type == b->type){ - qlock(&stats.lock); - stats.lumphit++; - qunlock(&stats.lock); - goto found; - } - } - - /* - * missed: locate the block with the oldest second to last use. - * remove it from the heap, and fix up the heap. - */ - while(lumpcache.free == nil){ - if(bumplump() == nil){ - logerr(EAdmin, "all lump cache blocks in use"); - rsleep(&lumpcache.full); - goto again; - } - } - qlock(&stats.lock); - stats.lumpmiss++; - qunlock(&stats.lock); - - b = lumpcache.free; - lumpcache.free = b->next; - - /* - * the new block has no last use, so assume it happens sometime in the middle -ZZZ this is not reasonable - */ - b->used = (b->used2 + lumpcache.now) / 2; - - /* - * rechain the block on the correct hash chain - */ - b->next = lumpcache.heads[h]; - lumpcache.heads[h] = b; - if(b->next != nil) - b->next->prev = b; - b->prev = nil; - - scorecp(b->score, score); - b->type = type; - b->size = 0; - b->data = nil; - -found: - b->ref++; - b->used2 = b->used; - b->used = lumpcache.now++; - if(b->heap != TWID32) - fixheap(b->heap, b); - qunlock(&lumpcache.lock); - -//checklumpcache(); - - qlock(&b->lock); - - return b; -} - -void -insertlump(Lump *b, Packet *p) -{ - u32int size; - - /* - * look for the block in the cache - */ -//checklumpcache(); - qlock(&lumpcache.lock); -again: - - /* - * missed: locate the block with the oldest second to last use. - * remove it from the heap, and fix up the heap. - */ - size = packetasize(p); -//ZZZ - while(lumpcache.avail < size){ - if(bumplump() == nil){ - logerr(EAdmin, "all lump cache blocks in use"); - rsleep(&lumpcache.full); - goto again; - } - } - b->data = p; - b->size = size; - lumpcache.avail -= size; - - qunlock(&lumpcache.lock); -//checklumpcache(); -} - -void -putlump(Lump *b) -{ - if(b == nil) - return; - - qunlock(&b->lock); -//checklumpcache(); - qlock(&lumpcache.lock); - if(--b->ref == 0){ - if(b->heap == TWID32) - upheap(lumpcache.nheap++, b); - rwakeup(&lumpcache.full); - } - - qunlock(&lumpcache.lock); -//checklumpcache(); -} - -/* - * remove some lump from use and update the free list and counters - */ -static Lump* -bumplump(void) -{ - Lump *b; - u32int h; - - /* - * remove blocks until we find one that is unused - * referenced blocks are left in the heap even though - * they can't be scavenged; this is simple a speed optimization - */ - for(;;){ - if(lumpcache.nheap == 0) - return nil; - b = lumpcache.heap[0]; - delheap(b); - if(!b->ref){ - rwakeup(&lumpcache.full); - break; - } - } - - /* - * unchain the block - */ - if(b->prev == nil){ - h = hashbits(b->score, HashLog); - if(lumpcache.heads[h] != b) - sysfatal("bad hash chains in lump cache"); - lumpcache.heads[h] = b->next; - }else - b->prev->next = b->next; - if(b->next != nil) - b->next->prev = b->prev; - - if(b->data != nil){ - packetfree(b->data); - b->data = nil; - lumpcache.avail += b->size; - b->size = 0; - } - b->type = TWID8; - - b->next = lumpcache.free; - lumpcache.free = b; - - return b; -} - -/* - * delete an arbitrary block from the heap - */ -static void -delheap(Lump *db) -{ - fixheap(db->heap, lumpcache.heap[--lumpcache.nheap]); - db->heap = TWID32; -} - -/* - * push an element up or down to it's correct new location - */ -static void -fixheap(int i, Lump *b) -{ - if(upheap(i, b) == i) - downheap(i, b); -} - -static int -upheap(int i, Lump *b) -{ - Lump *bb; - u32int now; - int p; - - now = lumpcache.now; - for(; i != 0; i = p){ - p = (i - 1) >> 1; - bb = lumpcache.heap[p]; - if(b->used2 - now >= bb->used2 - now) - break; - lumpcache.heap[i] = bb; - bb->heap = i; - } - - lumpcache.heap[i] = b; - b->heap = i; - return i; -} - -static int -downheap(int i, Lump *b) -{ - Lump *bb; - u32int now; - int k; - - now = lumpcache.now; - for(; ; i = k){ - k = (i << 1) + 1; - if(k >= lumpcache.nheap) - break; - if(k + 1 < lumpcache.nheap && lumpcache.heap[k]->used2 - now > lumpcache.heap[k + 1]->used2 - now) - k++; - bb = lumpcache.heap[k]; - if(b->used2 - now <= bb->used2 - now) - break; - lumpcache.heap[i] = bb; - bb->heap = i; - } - - lumpcache.heap[i] = b; - b->heap = i; - return i; -} - -static void -findblock(Lump *bb) -{ - Lump *b, *last; - int h; - - last = nil; - h = hashbits(bb->score, HashLog); - for(b = lumpcache.heads[h]; b != nil; b = b->next){ - if(last != b->prev) - sysfatal("bad prev link"); - if(b == bb) - return; - last = b; - } - sysfatal("block score=%V type=%#x missing from hash table", bb->score, bb->type); -} - -void -checklumpcache(void) -{ - Lump *b; - u32int size, now, nfree; - int i, k, refed; - - qlock(&lumpcache.lock); - now = lumpcache.now; - for(i = 0; i < lumpcache.nheap; i++){ - if(lumpcache.heap[i]->heap != i) - sysfatal("lc: mis-heaped at %d: %d", i, lumpcache.heap[i]->heap); - if(i > 0 && lumpcache.heap[(i - 1) >> 1]->used2 - now > lumpcache.heap[i]->used2 - now) - sysfatal("lc: bad heap ordering"); - k = (i << 1) + 1; - if(k < lumpcache.nheap && lumpcache.heap[i]->used2 - now > lumpcache.heap[k]->used2 - now) - sysfatal("lc: bad heap ordering"); - k++; - if(k < lumpcache.nheap && lumpcache.heap[i]->used2 - now > lumpcache.heap[k]->used2 - now) - sysfatal("lc: bad heap ordering"); - } - - refed = 0; - size = 0; - for(i = 0; i < lumpcache.nblocks; i++){ - b = &lumpcache.blocks[i]; - if(b->data == nil && b->size != 0) - sysfatal("bad size: %d data=%p", b->size, b->data); - if(b->ref && b->heap == TWID32) - refed++; - if(b->type != TWID8){ - findblock(b); - size += b->size; - } - if(b->heap != TWID32 - && lumpcache.heap[b->heap] != b) - sysfatal("lc: spurious heap value"); - } - if(lumpcache.avail != lumpcache.allowed - size) - sysfatal("mismatched available=%d and allowed=%d - used=%d space", lumpcache.avail, lumpcache.allowed, size); - - nfree = 0; - for(b = lumpcache.free; b != nil; b = b->next){ - if(b->type != TWID8 || b->heap != TWID32) - sysfatal("lc: bad free list"); - nfree++; - } - - if(lumpcache.nheap + nfree + refed != lumpcache.nblocks) - sysfatal("lc: missing blocks: %d %d %d %d", lumpcache.nheap, refed, nfree, lumpcache.nblocks); - qunlock(&lumpcache.lock); -} blob - 1bb8841ee2111ce867bb925d555fa547b74dd49c (mode 644) blob + /dev/null --- src/cmd/venti/lumpqueue.c +++ /dev/null @@ -1,153 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -typedef struct LumpQueue LumpQueue; -typedef struct WLump WLump; - -enum -{ - MaxLumpQ = 1 << 3 /* max. lumps on a single write queue, must be pow 2 */ -}; - -struct WLump -{ - Lump *u; - Packet *p; - int creator; - int gen; -}; - -struct LumpQueue -{ - QLock lock; - Rendez flush; - Rendez full; - Rendez empty; - WLump q[MaxLumpQ]; - int w; - int r; -}; - -static LumpQueue *lumpqs; -static int nqs; - -static QLock glk; -static int gen; - -static void queueproc(void *vq); - -int -initlumpqueues(int nq) -{ - LumpQueue *q; - - int i; - nqs = nq; - - lumpqs = MKNZ(LumpQueue, nq); - - for(i = 0; i < nq; i++){ - q = &lumpqs[i]; - q->full.l = &q->lock; - q->empty.l = &q->lock; - q->flush.l = &q->lock; - - if(vtproc(queueproc, q) < 0){ - seterr(EOk, "can't start write queue slave: %r"); - return -1; - } - } - - return 0; -} - -/* - * queue a lump & it's packet data for writing - */ -int -queuewrite(Lump *u, Packet *p, int creator) -{ - LumpQueue *q; - int i; - - i = indexsect(mainindex, u->score); - if(i < 0 || i >= nqs){ - seterr(EBug, "internal error: illegal index section in queuewrite"); - return -1; - } - - q = &lumpqs[i]; - - qlock(&q->lock); - while(q->r == ((q->w + 1) & (MaxLumpQ - 1))) - rsleep(&q->full); - - q->q[q->w].u = u; - q->q[q->w].p = p; - q->q[q->w].creator = creator; - q->q[q->w].gen = gen; - q->w = (q->w + 1) & (MaxLumpQ - 1); - - rwakeup(&q->empty); - - qunlock(&q->lock); - - return 0; -} - -void -flushqueue(void) -{ - int i; - LumpQueue *q; - - if(!lumpqs) - return; - - qlock(&glk); - gen++; - qunlock(&glk); - - for(i=0; insects; i++){ - q = &lumpqs[i]; - qlock(&q->lock); - while(q->w != q->r && gen - q->q[q->r].gen > 0) - rsleep(&q->flush); - qunlock(&q->lock); - } -} - -static void -queueproc(void *vq) -{ - LumpQueue *q; - Lump *u; - Packet *p; - int creator; - - q = vq; - for(;;){ - qlock(&q->lock); - while(q->w == q->r) - rsleep(&q->empty); - - u = q->q[q->r].u; - p = q->q[q->r].p; - creator = q->q[q->r].creator; - - rwakeup(&q->full); - - qunlock(&q->lock); - - if(writeqlump(u, p, creator) < 0) - fprint(2, "failed to write lump for %V: %r", u->score); - - qlock(&q->lock); - q->r = (q->r + 1) & (MaxLumpQ - 1); - rwakeup(&q->flush); - qunlock(&q->lock); - - putlump(u); - } -} blob - c6f9ef446fb3d0e3ad7276a5814a86d6cfa5c629 blob + 1a1b652da92eb4ef33db5b674f3a5b4826e3eba3 --- src/cmd/venti/mkfile +++ src/cmd/venti/mkfile @@ -1,76 +1,3 @@ -<$PLAN9/src/mkhdr +%:VQ: + echo venti will return once it is debugged. -LIBOFILES=\ - arena.$O\ - arenas.$O\ - buildbuck.$O\ - clump.$O\ - config.$O\ - conv.$O\ - dcache.$O\ - dump.$O\ - httpd.$O\ - icache.$O\ - ifile.$O\ - index.$O\ - lump.$O\ - lumpcache.$O\ - lumpqueue.$O\ - part.$O\ - score.$O\ - sortientry.$O\ - stats.$O\ - syncarena.$O\ - syncindex0.$O\ - unwhack.$O\ - utils.$O\ - unittoull.$O\ - whack.$O\ - xml.$O\ - zeropart.$O\ - -SLIB=libvs.a - -LIB=$SLIB - -HFILES= dat.h\ - fns.h\ - stdinc.h\ - -TARG=\ - venti\ - fmtarenas\ - fmtisect\ - fmtindex\ -# buildindex\ - checkarenas\ -# checkindex\ - clumpstats\ - findscore\ - rdarena\ - wrarena\ - syncindex\ - verifyarena\ - sync\ - read\ - write\ - copy\ - wb\ - -BIN=$BIN/venti - -it:V: all - -<$PLAN9/src/mkmany - -$SLIB: $LIBOFILES - $AR rvc $SLIB $LIBOFILES - -# xml.c:D: mkxml dat.h -# ./mkxml dat.h > xml.c - -ainstall:V: ${TARG:%=%.ainstall} - -%.ainstall:V: $O.% - scp $prereq amsterdam:/usr/local/bin/venti/$stem - blob - 67e5d0e98c65750e415268b54de1244047a2a0dd (mode 644) blob + /dev/null --- src/cmd/venti/mkroot.c +++ /dev/null @@ -1,59 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -char *host; - -void -usage(void) -{ - fprint(2, "usage: mkroot [-h host] name type score blocksize prev\n"); - threadexitsall("usage"); -} - -void -threadmain(int argc, char *argv[]) -{ - uchar score[VtScoreSize]; - uchar buf[VtRootSize]; - VtConn *z; - VtRoot root; - - ARGBEGIN{ - case 'h': - host = EARGF(usage()); - break; - default: - usage(); - break; - }ARGEND - - if(argc != 5) - usage(); - - fmtinstall('V', vtscorefmt); - - strecpy(root.name, root.name+sizeof root.name, argv[0]); - strecpy(root.type, root.type+sizeof root.type, argv[1]); - if(vtparsescore(argv[2], strlen(argv[2]), nil, root.score) < 0) - sysfatal("bad score '%s'", argv[2]); - root.blocksize = atoi(argv[3]); - if(vtparsescore(argv[4], strlen(argv[4]), nil, root.prev) < 0) - sysfatal("bad score '%s'", argv[4]); - vtrootpack(&root, buf); - - z = vtdial(host); - if(z == nil) - sysfatal("could not connect to server: %r"); - - if(vtconnect(z) < 0) - sysfatal("vtconnect: %r"); - - if(vtwrite(z, score, VtRootType, buf, VtRootSize) < 0) - sysfatal("vtwrite: %r"); - if(vtsync(z) < 0) - sysfatal("vtsync: %r"); - vthangup(z); - print("%V\n", score); - threadexitsall(0); -} blob - 66e6c79a3b17b293bdb9258e27c22203426859ba (mode 644) blob + /dev/null --- src/cmd/venti/part.c +++ /dev/null @@ -1,132 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -#define trace 1 - -u32int maxblocksize; -int readonly; - -Part* -initpart(char *name, int writable) -{ - Part *part; - Dir *dir; - int how; - - part = MKZ(Part); - part->name = estrdup(name); - if(!writable && readonly) - how = OREAD; - else - how = ORDWR; - part->fd = open(name, how); - if(part->fd < 0){ - if(how == ORDWR) - part->fd = open(name, OREAD); - if(part->fd < 0){ - freepart(part); - seterr(EOk, "can't open partition='%s': %r", name); - return nil; - } - fprint(2, "warning: %s opened for reading only\n", name); - } - dir = dirfstat(part->fd); - if(dir == nil){ - freepart(part); - seterr(EOk, "can't stat partition='%s': %r", name); - return nil; - } - part->size = dir->length; - part->blocksize = 0; - free(dir); - return part; -} - -void -freepart(Part *part) -{ - if(part == nil) - return; - close(part->fd); - free(part->name); - free(part); -} - -void -partblocksize(Part *part, u32int blocksize) -{ - if(part->blocksize) - sysfatal("resetting partition=%s's block size", part->name); - part->blocksize = blocksize; - if(blocksize > maxblocksize) - maxblocksize = blocksize; -} - -int -writepart(Part *part, u64int addr, u8int *buf, u32int n) -{ - long m, mm, nn; - - qlock(&stats.lock); - stats.diskwrites++; - stats.diskbwrites += n; - qunlock(&stats.lock); - - if(addr > part->size || addr + n > part->size){ - seterr(ECorrupt, "out of bounds write to partition='%s'", part->name); - return -1; - } -if(trace) print("write %s %lud at %llud\n", part->name, n, addr); - for(nn = 0; nn < n; nn += m){ - mm = n - nn; - if(mm > MaxIo) - mm = MaxIo; - m = pwrite(part->fd, &buf[nn], mm, addr + nn); - if(m != mm){ - if(m < 0){ - seterr(EOk, "can't write partition='%s': %r", part->name); - return -1; - } - logerr(EOk, "truncated write to partition='%s' n=%ld wrote=%ld", part->name, mm, m); - } - } - return 0; -} - -int -readpart(Part *part, u64int addr, u8int *buf, u32int n) -{ - long m, mm, nn; - int i; - - qlock(&stats.lock); - stats.diskreads++; - stats.diskbreads += n; - qunlock(&stats.lock); - - if(addr > part->size || addr + n > part->size){ - seterr(ECorrupt, "out of bounds read from partition='%s': addr=%lld n=%d size=%lld", part->name, addr, n, part->size); - return -1; - } -if(trace) print("read %s %lud at %llud\n", part->name, n, addr); - for(nn = 0; nn < n; nn += m){ - mm = n - nn; - if(mm > MaxIo) - mm = MaxIo; - m = -1; - for(i=0; i<4; i++) { - m = pread(part->fd, &buf[nn], mm, addr + nn); - if(m == mm) - break; - } - if(m != mm){ - if(m < 0){ - seterr(EOk, "can't read partition='%s': %r", part->name); - return -1; - } - logerr(EOk, "warning: truncated read from partition='%s' n=%ld read=%ld", part->name, mm, m); - } - } - return 0; -} blob - 95121a9f6ac4b124552a6293cb076a8d48f8543e (mode 644) blob + /dev/null --- src/cmd/venti/printarena.c +++ /dev/null @@ -1,128 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -int readonly = 1; /* for part.c */ - -void -usage(void) -{ - fprint(2, "usage: printarena arenafile [offset]\n"); - threadexitsall("usage"); -} - -static void -rdarena(Arena *arena, u64int offset) -{ - u64int a, aa, e; - u32int magic; - Clump cl; - uchar score[VtScoreSize]; - ZBlock *lump; - - printarena(2, arena); - - a = arena->base; - e = arena->base + arena->size; - if(offset != ~(u64int)0) { - if(offset >= e-a) - sysfatal("bad offset %llud >= %llud\n", - offset, e-a); - aa = offset; - } else - aa = 0; - - for(; aa < e; aa += ClumpSize+cl.info.size) { - magic = clumpmagic(arena, aa); - if(magic == ClumpFreeMagic) - break; - if(magic != ClumpMagic) { - fprint(2, "illegal clump magic number %#8.8ux offset %llud\n", - magic, aa); - break; - } - lump = loadclump(arena, aa, 0, &cl, score, 0); - if(lump == nil) { - fprint(2, "clump %llud failed to read: %r\n", aa); - break; - } - if(cl.info.type != VtTypeCorrupt) { - scoremem(score, lump->data, cl.info.uncsize); - if(scorecmp(cl.info.score, score) != 0) { - fprint(2, "clump %llud has mismatched score\n", aa); - break; - } - if(vttypevalid(cl.info.type) < 0) { - fprint(2, "clump %llud has bad type %d\n", aa, cl.info.type); - break; - } - } - print("%llud %V %d %d\n", aa, score, cl.info.type, cl.info.uncsize); - freezblock(lump); - } - print("end offset %llud\n", aa); -} - -void -threadmain(int argc, char *argv[]) -{ - char *file; - Arena *arena; - u64int offset, aoffset; - Part *part; - Dir *d; - uchar buf[8192]; - ArenaHead head; - - aoffset = 0; - ARGBEGIN{ - case 'o': - aoffset = strtoull(EARGF(usage()), 0, 0); - break; - default: - usage(); - break; - }ARGEND - - offset = ~(u64int)0; - switch(argc) { - default: - usage(); - case 2: - offset = strtoull(argv[1], 0, 0); - /* fall through */ - case 1: - file = argv[0]; - } - - - fmtinstall('V', vtscorefmt); - - statsinit(); - - if((d = dirstat(file)) == nil) - sysfatal("can't stat file %s: %r", file); - - part = initpart(file, 0); - if(part == nil) - sysfatal("can't open file %s: %r", file); - if(readpart(part, aoffset, buf, sizeof buf) < 0) - sysfatal("can't read file %s: %r", file); - - if(unpackarenahead(&head, buf) < 0) - sysfatal("corrupted arena header: %r"); - - if(aoffset+head.size > d->length) - sysfatal("arena is truncated: want %llud bytes have %llud\n", - head.size, d->length); - - partblocksize(part, head.blocksize); - initdcache(8 * MaxDiskBlock); - - arena = initarena(part, aoffset, head.size, head.blocksize); - if(arena == nil) - sysfatal("initarena: %r"); - - rdarena(arena, offset); - threadexitsall(0); -} blob - 8d130def883a24afe8c837dd1a7fe1a128cdc8ce (mode 644) blob + /dev/null --- src/cmd/venti/rdarena.c +++ /dev/null @@ -1,91 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -static int verbose; - -void -usage(void) -{ - fprint(2, "usage: rdarena [-v] arenapart arena\n"); - threadexitsall(0); -} - -static void -rdarena(Arena *arena) -{ - ZBlock *b; - u64int a, e; - u32int bs; - - fprint(2, "copying %s to standard output\n", arena->name); - printarena(2, arena); - - bs = MaxIoSize; - if(bs < arena->blocksize) - bs = arena->blocksize; - - b = alloczblock(bs, 0); - e = arena->base + arena->size + arena->blocksize; - for(a = arena->base - arena->blocksize; a + arena->blocksize <= e; a += bs){ - if(a + bs > e) - bs = arena->blocksize; - if(readpart(arena->part, a, b->data, bs) < 0) - fprint(2, "can't copy %s, read at %lld failed: %r\n", arena->name, a); - if(write(1, b->data, bs) != bs) - sysfatal("can't copy %s, write at %lld failed: %r", arena->name, a); - } - - freezblock(b); -} - -void -threadmain(int argc, char *argv[]) -{ - ArenaPart *ap; - Part *part; - char *file, *aname; - int i; - - fmtinstall('V', vtscorefmt); - statsinit(); - - ARGBEGIN{ - case 'v': - verbose++; - break; - default: - usage(); - break; - }ARGEND - - readonly = 1; - - if(argc != 2) - usage(); - - file = argv[0]; - aname = argv[1]; - - part = initpart(file, 0); - if(part == nil) - sysfatal("can't open partition %s: %r", file); - - ap = initarenapart(part); - if(ap == nil) - sysfatal("can't initialize arena partition in %s: %r", file); - - if(verbose) - printarenapart(2, ap); - - initdcache(8 * MaxDiskBlock); - - for(i = 0; i < ap->narenas; i++){ - if(strcmp(ap->arenas[i]->name, aname) == 0){ - rdarena(ap->arenas[i]); - threadexitsall(0); - } - } - - sysfatal("couldn't find arena %s\n", aname); -} blob - 969a05934b6c47b2f870ca0157d454f542b31bd1 (mode 644) blob + /dev/null --- src/cmd/venti/read.c +++ /dev/null @@ -1,102 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -char *host; - -void -usage(void) -{ - fprint(2, "usage: read [-h host] score [type]\n"); - threadexitsall("usage"); -} - -int -parsescore(uchar *score, char *buf, int n) -{ - int i, c; - - memset(score, 0, VtScoreSize); - - if(n != VtScoreSize*2){ - werrstr("score wrong length %d", n); - return -1; - } - for(i=0; i= '0' && buf[i] <= '9') - c = buf[i] - '0'; - else if(buf[i] >= 'a' && buf[i] <= 'f') - c = buf[i] - 'a' + 10; - else if(buf[i] >= 'A' && buf[i] <= 'F') - c = buf[i] - 'A' + 10; - else { - c = buf[i]; - werrstr("bad score char %d '%c'", c, c); - return -1; - } - - if((i & 1) == 0) - c <<= 4; - - score[i>>1] |= c; - } - return 0; -} - -void -threadmain(int argc, char *argv[]) -{ - int type, n; - uchar score[VtScoreSize]; - uchar *buf; - VtConn *z; - - ARGBEGIN{ - case 'h': - host = EARGF(usage()); - break; - default: - usage(); - break; - }ARGEND - - if(argc != 1 && argc != 2) - usage(); - - - fmtinstall('V', vtscorefmt); - - if(parsescore(score, argv[0], strlen(argv[0])) < 0) - sysfatal("could not parse score '%s': %r", argv[0]); - - buf = vtmallocz(VtMaxLumpSize); - - z = vtdial(host); - if(z == nil) - sysfatal("could not connect to server: %r"); - - if(vtconnect(z) < 0) - sysfatal("vtconnect: %r"); - - if(argc == 1){ - n = -1; - for(type=0; type= 0){ - fprint(2, "venti/read%s%s %V %d\n", host ? " -h" : "", host ? host : "", - score, type); - break; - } - } - }else{ - type = atoi(argv[1]); - n = vtread(z, score, type, buf, VtMaxLumpSize); - } - vthangup(z); - if(n < 0) - sysfatal("could not read block: %r"); - if(write(1, buf, n) != n) - sysfatal("write: %r"); - - threadexitsall(0); -} blob - 0809e84f6468380cf41efd74b55daab069c16ae6 (mode 644) blob + /dev/null --- src/cmd/venti/score.c +++ /dev/null @@ -1,43 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -u8int zeroscore[VtScoreSize]; - -void -scoremem(u8int *score, u8int *buf, int n) -{ - DigestState s; - - memset(&s, 0, sizeof s); - sha1(buf, n, score, &s); -} - -static int -hexv(int c) -{ - if(c >= '0' && c <= '9') - return c - '0'; - if(c >= 'a' && c <= 'f') - return c - 'a' + 10; - if(c >= 'A' && c <= 'F') - return c - 'A' + 10; - return -1; -} - -int -strscore(char *s, u8int *score) -{ - int i, c, d; - - for(i = 0; i < VtScoreSize; i++){ - c = hexv(s[2 * i]); - if(c < 0) - return -1; - d = hexv(s[2 * i + 1]); - if(d < 0) - return -1; - score[i] = (c << 4) + d; - } - return s[2 * i] == '\0'; -} blob - 0e9f4b1e721d5741c76d972388f66559351b5cd1 (mode 644) blob + /dev/null --- src/cmd/venti/sortientry.c +++ /dev/null @@ -1,332 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -typedef struct IEBuck IEBuck; -typedef struct IEBucks IEBucks; - -enum -{ - ClumpChunks = 32*1024 -}; - -struct IEBuck -{ - u32int head; /* head of chain of chunks on the disk */ - u32int used; /* usage of the last chunk */ - u64int total; /* total number of bytes in this bucket */ - u8int *buf; /* chunk of entries for this bucket */ -}; - -struct IEBucks -{ - Part *part; - u64int off; /* offset for writing data in the partition */ - u32int chunks; /* total chunks written to fd */ - u64int max; /* max bytes entered in any one bucket */ - int bits; /* number of bits in initial bucket sort */ - int nbucks; /* 1 << bits, the number of buckets */ - u32int size; /* bytes in each of the buckets chunks */ - u32int usable; /* amount usable for IEntry data */ - u8int *buf; /* buffer for all chunks */ - IEBuck *bucks; -}; - -#define U32GET(p) (((p)[0]<<24)|((p)[1]<<16)|((p)[2]<<8)|(p)[3]) -#define U32PUT(p,v) (p)[0]=(v)>>24;(p)[1]=(v)>>16;(p)[2]=(v)>>8;(p)[3]=(v) - -static IEBucks *initiebucks(Part *part, int bits, u32int size); -static int flushiebuck(IEBucks *ib, int b, int reset); -static int flushiebucks(IEBucks *ib); -static u32int sortiebuck(IEBucks *ib, int b); -static u64int sortiebucks(IEBucks *ib); -static int sprayientry(IEBucks *ib, IEntry *ie); -static u32int readarenainfo(IEBucks *ib, Arena *arena, u64int a); -static u32int readiebuck(IEBucks *ib, int b); -static void freeiebucks(IEBucks *ib); - -/* - * build a sorted file with all Ientries which should be in ix. - * assumes the arenas' directories are up to date. - * reads each, converts the entries to index entries, - * and sorts them. - */ -u64int -sortrawientries(Index *ix, Part *tmp, u64int *base) -{ - IEBucks *ib; - u64int clumps, sorted; - u32int n; - int i, ok; - -//ZZZ should allow configuration of bits, bucket size - ib = initiebucks(tmp, 8, 64*1024); - if(ib == nil){ - seterr(EOk, "can't create sorting buckets: %r"); - return TWID64; - } - ok = 0; - clumps = 0; - for(i = 0; i < ix->narenas; i++){ - n = readarenainfo(ib, ix->arenas[i], ix->amap[i].start); - if(n == TWID32){ - ok = -1; - break; - } - clumps += n; - } -fprint(2, "got %lld clumps - starting sort\n", clumps); - if(ok){ - sorted = sortiebucks(ib); - *base = (u64int)ib->chunks * ib->size; - if(sorted != clumps){ - fprint(2, "sorting messed up: clumps=%lld sorted=%lld\n", clumps, sorted); - ok = -1; - } - } - freeiebucks(ib); - if(ok < 0) - return TWID64; - return clumps; -} - -/* - * read in all of the arena's clump directory, - * convert to IEntry format, and bucket sort based - * on the first few bits. - */ -static u32int -readarenainfo(IEBucks *ib, Arena *arena, u64int a) -{ - IEntry ie; - ClumpInfo *ci, *cis; - u32int clump; - int i, n, ok; - -//ZZZ remove fprint? - if(arena->clumps) - fprint(2, "reading directory for arena=%s with %d entries\n", arena->name, arena->clumps); - - cis = MKN(ClumpInfo, ClumpChunks); - ok = 0; - memset(&ie, 0, sizeof(IEntry)); - for(clump = 0; clump < arena->clumps; clump += n){ - n = ClumpChunks; - if(n > arena->clumps - clump) - n = arena->clumps - clump; - if(readclumpinfos(arena, clump, cis, n) != n){ - seterr(EOk, "arena directory read failed: %r"); - ok = -1; - break; - } - - for(i = 0; i < n; i++){ - ci = &cis[i]; - ie.ia.type = ci->type; - ie.ia.size = ci->uncsize; - ie.ia.addr = a; - a += ci->size + ClumpSize; - ie.ia.blocks = (ci->size + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog; - scorecp(ie.score, ci->score); - sprayientry(ib, &ie); - } - } - free(cis); - if(ok < 0) - return TWID32; - return clump; -} - -/* - * initialize the external bucket sorting data structures - */ -static IEBucks* -initiebucks(Part *part, int bits, u32int size) -{ - IEBucks *ib; - int i; - - ib = MKZ(IEBucks); - if(ib == nil){ - seterr(EOk, "out of memory"); - return nil; - } - ib->bits = bits; - ib->nbucks = 1 << bits; - ib->size = size; - ib->usable = (size - U32Size) / IEntrySize * IEntrySize; - ib->bucks = MKNZ(IEBuck, ib->nbucks); - if(ib->bucks == nil){ - seterr(EOk, "out of memory allocation sorting buckets"); - freeiebucks(ib); - return nil; - } - ib->buf = MKN(u8int, size * (1 << bits)); - if(ib->buf == nil){ - seterr(EOk, "out of memory allocating sorting buckets' buffers"); - freeiebucks(ib); - return nil; - } - for(i = 0; i < ib->nbucks; i++){ - ib->bucks[i].head = TWID32; - ib->bucks[i].buf = &ib->buf[i * size]; - } - ib->part = part; - return ib; -} - -static void -freeiebucks(IEBucks *ib) -{ - if(ib == nil) - return; - free(ib->bucks); - free(ib->buf); - free(ib); -} - -/* - * initial sort: put the entry into the correct bucket - */ -static int -sprayientry(IEBucks *ib, IEntry *ie) -{ - u32int n; - int b; - - b = hashbits(ie->score, ib->bits); - n = ib->bucks[b].used; - packientry(ie, &ib->bucks[b].buf[n]); - n += IEntrySize; - ib->bucks[b].used = n; - if(n + IEntrySize <= ib->usable) - return 0; - return flushiebuck(ib, b, 1); -} - -/* - * finish sorting: - * for each bucket, read it in and sort it - * write out the the final file - */ -static u64int -sortiebucks(IEBucks *ib) -{ - u64int tot; - u32int n; - int i; - - if(flushiebucks(ib) < 0) - return TWID64; - for(i = 0; i < ib->nbucks; i++) - ib->bucks[i].buf = nil; - ib->off = (u64int)ib->chunks * ib->size; - free(ib->buf); -fprint(2, "ib->max = %lld\n", ib->max); -fprint(2, "ib->chunks = %ud\n", ib->chunks); - ib->buf = MKN(u8int, ib->max + U32Size); - if(ib->buf == nil){ - seterr(EOk, "out of memory allocating final sorting buffer; try more buckets"); - return TWID64; - } - tot = 0; - for(i = 0; i < ib->nbucks; i++){ - n = sortiebuck(ib, i); - if(n == TWID32) - return TWID64; - tot += n; - } - return tot; - return 0; -} - -/* - * sort from bucket b of ib into the output file to - */ -static u32int -sortiebuck(IEBucks *ib, int b) -{ - u32int n; - - n = readiebuck(ib, b); - if(n == TWID32) - return TWID32; - qsort(ib->buf, n, IEntrySize, ientrycmp); - if(writepart(ib->part, ib->off, ib->buf, n * IEntrySize) < 0){ - seterr(EOk, "can't write sorted bucket: %r"); - return TWID32; - } - ib->off += n * IEntrySize; - return n; -} - -/* - * write out a single bucket - */ -static int -flushiebuck(IEBucks *ib, int b, int reset) -{ - u32int n; - - if(ib->bucks[b].used == 0) - return 0; - n = ib->bucks[b].used; - U32PUT(&ib->bucks[b].buf[n], ib->bucks[b].head); - n += U32Size; - if(writepart(ib->part, (u64int)ib->chunks * ib->size, ib->bucks[b].buf, n) < 0){ - seterr(EOk, "can't write sorting bucket to file: %r"); - return -1; - } - ib->bucks[b].head = ib->chunks++; - ib->bucks[b].total += ib->bucks[b].used; - if(reset) - ib->bucks[b].used = 0; - return 0; -} - -/* - * write out all of the buckets, and compute - * the maximum size of any bucket - */ -static int -flushiebucks(IEBucks *ib) -{ - int i; - - for(i = 0; i < ib->nbucks; i++){ - if(flushiebuck(ib, i, 0) < 0) - return -1; - if(ib->bucks[i].total > ib->max) - ib->max = ib->bucks[i].total; - } - return 0; -} - -/* - * read in the chained buffers for bucket b, - * and return it's total number of IEntries - */ -static u32int -readiebuck(IEBucks *ib, int b) -{ - u32int head, n, m; - - head = ib->bucks[b].head; - n = 0; - m = ib->bucks[b].used; - if(m == 0) - m = ib->usable; -fprint(2, "%d total = %lld\n", b, ib->bucks[b].total); - while(head != TWID32){ - if(readpart(ib->part, (u64int)head * ib->size, &ib->buf[n], m + U32Size) < 0){ -fprint(2, "n = %ud\n", n); - seterr(EOk, "can't read index sort bucket: %r"); - return TWID32; - } - n += m; - head = U32GET(&ib->buf[n]); - m = ib->usable; - } -fprint(2, "n = %ud\n", n); - return n / IEntrySize; -} blob - 079fd97b624b1749de8afb58d489cefbebe7e8fb (mode 644) blob + /dev/null --- src/cmd/venti/stats.c +++ /dev/null @@ -1,70 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -Stats stats; - -void -statsinit(void) -{ -} - -static int -percent(long v, long total) -{ - if(total == 0) - total = 1; - if(v < 1000*1000) - return (v * 100) / total; - total /= 100; - if(total == 0) - total = 1; - return v / total; -} - -void -printstats(void) -{ - fprint(2, "lump writes=%,ld\n", stats.lumpwrites); - fprint(2, "lump reads=%,ld\n", stats.lumpreads); - fprint(2, "lump cache read hits=%,ld\n", stats.lumphit); - fprint(2, "lump cache read misses=%,ld\n", stats.lumpmiss); - - fprint(2, "clump disk writes=%,ld\n", stats.clumpwrites); - fprint(2, "clump disk bytes written=%,lld\n", stats.clumpbwrites); - fprint(2, "clump disk bytes compressed=%,lld\n", stats.clumpbcomp); - fprint(2, "clump disk reads=%,ld\n", stats.clumpreads); - fprint(2, "clump disk bytes read=%,lld\n", stats.clumpbreads); - fprint(2, "clump disk bytes uncompressed=%,lld\n", stats.clumpbuncomp); - - fprint(2, "clump directory disk writes=%,ld\n", stats.ciwrites); - fprint(2, "clump directory disk reads=%,ld\n", stats.cireads); - - fprint(2, "index disk writes=%,ld\n", stats.indexwrites); - fprint(2, "index disk reads=%,ld\n", stats.indexreads); - fprint(2, "index disk reads for modify=%,ld\n", stats.indexwreads); - fprint(2, "index disk reads for allocation=%,ld\n", stats.indexareads); - fprint(2, "index block splits=%,ld\n", stats.indexsplits); - - fprint(2, "index cache lookups=%,ld\n", stats.iclookups); - fprint(2, "index cache hits=%,ld %d%%\n", stats.ichits, - percent(stats.ichits, stats.iclookups)); - fprint(2, "index cache fills=%,ld %d%%\n", stats.icfills, - percent(stats.icfills, stats.iclookups)); - fprint(2, "index cache inserts=%,ld\n", stats.icinserts); - - fprint(2, "disk cache hits=%,ld\n", stats.pchit); - fprint(2, "disk cache misses=%,ld\n", stats.pcmiss); - fprint(2, "disk cache reads=%,ld\n", stats.pcreads); - fprint(2, "disk cache bytes read=%,lld\n", stats.pcbreads); - - fprint(2, "disk cache writes=%,ld\n", stats.dirtydblocks); - fprint(2, "disk cache writes absorbed=%,ld %d%%\n", stats.absorbedwrites, - percent(stats.absorbedwrites, stats.dirtydblocks)); - - fprint(2, "disk writes=%,ld\n", stats.diskwrites); - fprint(2, "disk bytes written=%,lld\n", stats.diskbwrites); - fprint(2, "disk reads=%,ld\n", stats.diskreads); - fprint(2, "disk bytes read=%,lld\n", stats.diskbreads); - -} blob - 6f8f5ba5cc8642459feb64b6e2330dbeee127b1d (mode 644) blob + /dev/null --- src/cmd/venti/stdinc.h +++ /dev/null @@ -1,6 +0,0 @@ -#include -#include -#include -#include -#include - blob - 7446140062c3b79799d88a65dc80b59a1c6f3b8c (mode 644) blob + /dev/null --- src/cmd/venti/sync.c +++ /dev/null @@ -1,48 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -char *host; - -void -usage(void) -{ - fprint(2, "usage: sync [-h host]\n"); - threadexitsall("usage"); -} - -void -threadmain(int argc, char *argv[]) -{ - VtConn *z; - - ARGBEGIN{ - case 'h': - host = EARGF(usage()); - if(host == nil) - usage(); - break; - default: - usage(); - break; - }ARGEND - - if(argc != 0) - usage(); - - - fmtinstall('V', vtscorefmt); - - z = vtdial(host); - if(z == nil) - sysfatal("could not connect to server: %r"); - - if(vtconnect(z) < 0) - sysfatal("vtconnect: %r"); - - if(vtsync(z) < 0) - sysfatal("vtsync: %r"); - - vthangup(z); - threadexitsall(0); -} blob - 7ba83aa1c880aacbc77510dec28bba2bd462c2a4 (mode 644) blob + /dev/null --- src/cmd/venti/syncarena.c +++ /dev/null @@ -1,165 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -static int writeclumphead(Arena *arena, u64int aa, Clump *cl); -static int writeclumpmagic(Arena *arena, u64int aa, u32int magic); - -int -clumpinfocmp(ClumpInfo *c, ClumpInfo *d) -{ - return c->type != d->type - || c->size != d->size - || c->uncsize != d->uncsize - || scorecmp(c->score, d->score)!=0; -} - -/* - * synchronize the clump info directory with - * with the clumps actually stored in the arena. - * the directory should be at least as up to date - * as the arena's trailer. - * - * checks/updates at most n clumps. - * - * returns 0 if ok, flags if error occurred - */ -int -syncarena(Arena *arena, u32int n, int zok, int fix) -{ - ZBlock *lump; - Clump cl; - ClumpInfo ci; - static ClumpInfo zci = { .type = -1 }; - u8int score[VtScoreSize]; - u64int uncsize, used, aa; - u32int clump, clumps, cclumps, magic; - int err, flush, broken; - - used = arena->used; - clumps = arena->clumps; - cclumps = arena->cclumps; - uncsize = arena->uncsize; - - flush = 0; - err = 0; - for(; n; n--){ - aa = arena->used; - clump = arena->clumps; - magic = clumpmagic(arena, aa); - if(magic == ClumpFreeMagic) - break; - if(magic != ClumpMagic){ - fprint(2, "illegal clump magic number=%#8.8ux at clump=%d\n", magic, clump); - err |= SyncDataErr; -//ZZZ write a zero here? - if(0 && fix && writeclumpmagic(arena, aa, ClumpFreeMagic) < 0){ - fprint(2, "can't write corrected clump free magic: %r"); - err |= SyncFixErr; - } - break; - } - arena->clumps++; - - broken = 0; - lump = loadclump(arena, aa, 0, &cl, score, 0); - if(lump == nil){ - fprint(2, "clump=%d failed to read correctly: %r\n", clump); - err |= SyncDataErr; - }else if(cl.info.type != VtTypeCorrupt){ - scoremem(score, lump->data, cl.info.uncsize); - if(scorecmp(cl.info.score, score) != 0){ - fprint(2, "clump=%d has mismatched score\n", clump); - err |= SyncDataErr; - broken = 1; - }else if(vttypevalid(cl.info.type) < 0){ - fprint(2, "clump=%d has invalid type %d", clump, cl.info.type); - err |= SyncDataErr; - broken = 1; - } - if(broken && fix){ - cl.info.type = VtTypeCorrupt; - if(writeclumphead(arena, aa, &cl) < 0){ - fprint(2, "can't write corrected clump header: %r"); - err |= SyncFixErr; - } - } - } - freezblock(lump); - arena->used += ClumpSize + cl.info.size; - - if(!broken && readclumpinfo(arena, clump, &ci)<0){ - fprint(2, "arena directory read failed\n"); - broken = 1; - }else if(!broken && clumpinfocmp(&ci, &cl.info)!=0){ - if(clumpinfocmp(&ci, &zci) == 0){ - err |= SyncCIZero; - if(!zok) - fprint(2, "unwritten clump info for clump=%d\n", clump); - }else{ - err |= SyncCIErr; - fprint(2, "bad clump info for clump=%d\n", clump); - fprint(2, "\texpected score=%V type=%d size=%d uncsize=%d\n", - cl.info.score, cl.info.type, cl.info.size, cl.info.uncsize); - fprint(2, "\tfound score=%V type=%d size=%d uncsize=%d\n", - ci.score, ci.type, ci.size, ci.uncsize); - } - broken = 1; - } - if(broken && fix){ - flush = 1; - ci = cl.info; - if(writeclumpinfo(arena, clump, &ci) < 0){ - fprint(2, "can't write correct clump directory: %r\n"); - err |= SyncFixErr; - } - } - - arena->uncsize += cl.info.uncsize; - if(cl.info.size < cl.info.uncsize) - arena->cclumps++; - } - - if(flush){ - arena->wtime = now(); - if(arena->ctime == 0 && arena->clumps) - arena->ctime = arena->wtime; - if(flushciblocks(arena) < 0){ - fprint(2, "can't flush arena directory cache: %r"); - err |= SyncFixErr; - } - flushdcache(); - } - - if(used != arena->used - || clumps != arena->clumps - || cclumps != arena->cclumps - || uncsize != arena->uncsize) - err |= SyncHeader; - - return err; -} - -static int -writeclumphead(Arena *arena, u64int aa, Clump *cl) -{ - ZBlock *zb; - int bad; - - zb = alloczblock(ClumpSize, 0); - if(zb == nil) - return -1; - bad = packclump(cl, zb->data)<0 - || writearena(arena, aa, zb->data, ClumpSize) != ClumpSize; - freezblock(zb); - return bad ? -1 : 0; -} - -static int -writeclumpmagic(Arena *arena, u64int aa, u32int magic) -{ - u8int buf[U32Size]; - - packmagic(magic, buf); - return writearena(arena, aa, buf, U32Size) == U32Size; -} blob - 75bb0696360d20743cdbec2b171f4da7178b3d6c (mode 644) blob + /dev/null --- src/cmd/venti/syncindex.c +++ /dev/null @@ -1,56 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -static int verbose; -void -usage(void) -{ - fprint(2, "usage: syncindex [-fv] [-B blockcachesize] config\n"); - threadexitsall("usage"); -} - -void -threadmain(int argc, char *argv[]) -{ - u32int bcmem; - int fix; - - fix = 0; - bcmem = 0; - ARGBEGIN{ - case 'B': - bcmem = unittoull(ARGF()); - break; - case 'f': - fix++; - break; - case 'v': - verbose++; - break; - default: - usage(); - break; - }ARGEND - - if(!fix) - readonly = 1; - - if(argc != 1) - usage(); - - if(initventi(argv[0]) < 0) - sysfatal("can't init venti: %r"); - - if(bcmem < maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16)) - bcmem = maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16); - fprint(2, "initialize %d bytes of disk block cache\n", bcmem); - initdcache(bcmem); - - if(verbose) - printindex(2, mainindex); - if(syncindex(mainindex, fix) < 0) - sysfatal("failed to sync index=%s: %r\n", mainindex->name); - - threadexitsall(0); -} blob - 95db12081af7992db038b125cd0dfb1c0791c74e (mode 644) blob + /dev/null --- src/cmd/venti/syncindex0.c +++ /dev/null @@ -1,144 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -enum -{ - ClumpChunks = 32*1024 -}; - -/* - * shell sort is plenty good enough - * because we're going to do a bunch of disk i/o's - */ -static void -sortclumpinfo(ClumpInfo *ci, int *s, int n) -{ - int i, j, m, t; - - for(m = (n + 3) / 5; m > 0; m = (m + 1) / 3){ - for(i = n - m; i-- > 0;){ - for(j = i + m; j < n; j += m){ - if(memcmp(ci[s[j - m]].score, ci[s[j]].score, VtScoreSize) <= 0) - break; - t = s[j]; - s[j] = s[j - m]; - s[j - m] = t; - } - } - } -} - -int -syncarenaindex(Index *ix, Arena *arena, u32int clump, u64int a, int fix) -{ - Packet *pack; - IEntry ie; - IAddr ia; - ClumpInfo *ci, *cis; - u32int now; - u64int *addrs; - int i, n, ok, *s; - - now = time(nil); - cis = MKN(ClumpInfo, ClumpChunks); - addrs = MKN(u64int, ClumpChunks); - s = MKN(int, ClumpChunks); - ok = 0; - for(; clump < arena->clumps; clump += n){ - n = ClumpChunks; - if(n > arena->clumps - clump) - n = arena->clumps - clump; - n = readclumpinfos(arena, clump, cis, n); - if(n <= 0){ - fprint(2, "arena directory read failed\n"); - ok = -1; - break; - } - - for(i = 0; i < n; i++){ - addrs[i] = a; - a += cis[i].size + ClumpSize; - s[i] = i; - } - - sortclumpinfo(cis, s, n); - - for(i = 0; i < n; i++){ - ci = &cis[s[i]]; - ia.type = ci->type; - ia.size = ci->uncsize; - ia.addr = addrs[s[i]]; - ia.blocks = (ci->size + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog; - - if(loadientry(ix, ci->score, ci->type, &ie) < 0) - fprint(2, "missing block type=%d score=%V\n", ci->type, ci->score); - else if(iaddrcmp(&ia, &ie.ia) != 0){ - fprint(2, "\nmismatched index entry and clump at %d\n", clump + i); - fprint(2, "\tclump: type=%d size=%d blocks=%d addr=%lld\n", ia.type, ia.size, ia.blocks, ia.addr); - fprint(2, "\tindex: type=%d size=%d block=%d addr=%lld\n", ie.ia.type, ie.ia.size, ie.ia.blocks, ie.ia.addr); - pack = readlump(ie.score, ie.ia.type, ie.ia.size); - packetfree(pack); - if(pack != nil){ - fprint(2, "duplicated lump\n"); - continue; - } - }else - continue; - if(!fix){ - ok = -1; - continue; - } - ie.ia = ia; - scorecp(ie.score, ci->score); - ie.train = 0; - ie.wtime = now; - if(storeientry(ix, &ie) < 0){ - fprint(2, "can't fix index: %r"); - ok = -1; - } - } - - if(0 && clump / 1000 != (clump + n) / 1000) - fprint(2, "."); - } - free(cis); - free(addrs); - free(s); - return ok; -} - -int -syncindex(Index *ix, int fix) -{ - Arena *arena; - u64int a; - u32int clump; - int i, e, e1, ok, ok1; - - ok = 0; - for(i = 0; i < ix->narenas; i++){ - arena = ix->arenas[i]; - clump = arena->clumps; - a = arena->used; - e = syncarena(arena, TWID32, fix, fix); - e1 = e; - if(fix) - e1 &= ~(SyncHeader|SyncCIZero); - if(e1 == SyncHeader) - fprint(2, "arena %s: header is out-of-date\n", arena->name); - if(e1) - ok = -1; - else{ - ok1 = syncarenaindex(ix, arena, clump, a + ix->amap[i].start, fix); - if(fix && ok1==0 && (e & SyncHeader) && wbarena(arena) < 0) - fprint(2, "arena=%s header write failed: %r\n", arena->name); - ok |= ok1; - } - } - if(fix && wbindex(ix) < 0){ - fprint(2, "can't write back index header for %s: %r\n", ix->name); - return -1; - } - return ok; -} blob - db35aa0faebec9ae015d8909c7636789e1a13cca (mode 644) blob + /dev/null --- src/cmd/venti/unittoull.c +++ /dev/null @@ -1,27 +0,0 @@ -#include "stdinc.h" - -#define TWID64 ((u64int)~(u64int)0) - -u64int -unittoull(char *s) -{ - char *es; - u64int n; - - if(s == nil) - return TWID64; - n = strtoul(s, &es, 0); - if(*es == 'k' || *es == 'K'){ - n *= 1024; - es++; - }else if(*es == 'm' || *es == 'M'){ - n *= 1024*1024; - es++; - }else if(*es == 'g' || *es == 'G'){ - n *= 1024*1024*1024; - es++; - } - if(*es != '\0') - return TWID64; - return n; -} blob - 5530bd07d003f43c444940d600c9b41cae07a809 (mode 644) blob + /dev/null --- src/cmd/venti/unwhack.c +++ /dev/null @@ -1,179 +0,0 @@ -#include "stdinc.h" -#include "whack.h" - -enum -{ - DMaxFastLen = 7, - DBigLenCode = 0x3c, /* minimum code for large lenth encoding */ - DBigLenBits = 6, - DBigLenBase = 1 /* starting items to encode for big lens */ -}; - -static uchar lenval[1 << (DBigLenBits - 1)] = -{ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 3, 3, 3, 3, 3, 3, 3, 3, - 4, 4, 4, 4, - 5, - 6, - 255, - 255 -}; - -static uchar lenbits[] = -{ - 0, 0, 0, - 2, 3, 5, 5, -}; - -static uchar offbits[16] = -{ - 5, 5, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 12, 13 -}; - -static ushort offbase[16] = -{ - 0, 0x20, - 0x40, 0x60, - 0x80, 0xc0, - 0x100, 0x180, - 0x200, 0x300, - 0x400, 0x600, - 0x800, 0xc00, - 0x1000, - 0x2000 -}; - -void -unwhackinit(Unwhack *uw) -{ - uw->err[0] = '\0'; -} - -int -unwhack(Unwhack *uw, uchar *dst, int ndst, uchar *src, int nsrc) -{ - uchar *s, *d, *dmax, *smax, lit; - ulong uwbits, lithist; - int i, off, len, bits, use, code, uwnbits, overbits; - - d = dst; - dmax = d + ndst; - - smax = src + nsrc; - uwnbits = 0; - uwbits = 0; - overbits = 0; - lithist = ~0; - while(src < smax || uwnbits - overbits >= MinDecode){ - while(uwnbits <= 24){ - uwbits <<= 8; - if(src < smax) - uwbits |= *src++; - else - overbits += 8; - uwnbits += 8; - } - - /* - * literal - */ - len = lenval[(uwbits >> (uwnbits - 5)) & 0x1f]; - if(len == 0){ - if(lithist & 0xf){ - uwnbits -= 9; - lit = (uwbits >> uwnbits) & 0xff; - lit &= 255; - }else{ - uwnbits -= 8; - lit = (uwbits >> uwnbits) & 0x7f; - if(lit < 32){ - if(lit < 24){ - uwnbits -= 2; - lit = (lit << 2) | ((uwbits >> uwnbits) & 3); - }else{ - uwnbits -= 3; - lit = (lit << 3) | ((uwbits >> uwnbits) & 7); - } - lit = (lit - 64) & 0xff; - } - } - if(d >= dmax){ - snprint(uw->err, WhackErrLen, "too much output"); - return -1; - } - *d++ = lit; - lithist = (lithist << 1) | (lit < 32) | (lit > 127); - continue; - } - - /* - * length - */ - if(len < 255) - uwnbits -= lenbits[len]; - else{ - uwnbits -= DBigLenBits; - code = ((uwbits >> uwnbits) & ((1 << DBigLenBits) - 1)) - DBigLenCode; - len = DMaxFastLen; - use = DBigLenBase; - bits = (DBigLenBits & 1) ^ 1; - while(code >= use){ - len += use; - code -= use; - code <<= 1; - uwnbits--; - if(uwnbits < 0){ - snprint(uw->err, WhackErrLen, "len out of range"); - return -1; - } - code |= (uwbits >> uwnbits) & 1; - use <<= bits; - bits ^= 1; - } - len += code; - - while(uwnbits <= 24){ - uwbits <<= 8; - if(src < smax) - uwbits |= *src++; - else - overbits += 8; - uwnbits += 8; - } - } - - /* - * offset - */ - uwnbits -= 4; - bits = (uwbits >> uwnbits) & 0xf; - off = offbase[bits]; - bits = offbits[bits]; - - uwnbits -= bits; - off |= (uwbits >> uwnbits) & ((1 << bits) - 1); - off++; - - if(off > d - dst){ - snprint(uw->err, WhackErrLen, "offset out of range: off=%d d=%ld len=%d nbits=%d", off, d - dst, len, uwnbits); - return -1; - } - if(d + len > dmax){ - snprint(uw->err, WhackErrLen, "len out of range"); - return -1; - } - s = d - off; - for(i = 0; i < len; i++) - d[i] = s[i]; - d += len; - } - if(uwnbits < overbits){ - snprint(uw->err, WhackErrLen, "compressed data overrun"); - return -1; - } - - len = d - dst; - - return len; -} blob - 26a0ef829be3e8e1ab78214d44037418a4cb0b18 (mode 644) blob + /dev/null --- src/cmd/venti/utils.c +++ /dev/null @@ -1,361 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -int -namecmp(char *s, char *t) -{ - return strncmp(s, t, ANameSize); -} - -void -namecp(char *dst, char *src) -{ - strncpy(dst, src, ANameSize - 1); - dst[ANameSize - 1] = '\0'; -} - -int -nameok(char *name) -{ - char *t; - int c; - - if(name == nil) - return -1; - for(t = name; c = *t; t++) - if(t - name >= ANameSize - || c < ' ' || c >= 0x7f) - return -1; - return 0; -} - -int -stru32int(char *s, u32int *r) -{ - char *t; - u32int n, nn, m; - int c; - - m = TWID32 / 10; - n = 0; - for(t = s; ; t++){ - c = *t; - if(c < '0' || c > '9') - break; - if(n > m) - return -1; - nn = n * 10 + c - '0'; - if(nn < n) - return -1; - n = nn; - } - *r = n; - return s != t && *t == '\0'; -} - -int -stru64int(char *s, u64int *r) -{ - char *t; - u64int n, nn, m; - int c; - - m = TWID64 / 10; - n = 0; - for(t = s; ; t++){ - c = *t; - if(c < '0' || c > '9') - break; - if(n > m) - return -1; - nn = n * 10 + c - '0'; - if(nn < n) - return -1; - n = nn; - } - *r = n; - return s != t && *t == '\0'; -} - -int -vttypevalid(int type) -{ - return type < VtMaxType; -} - -void -fmtzbinit(Fmt *f, ZBlock *b) -{ - f->runes = 0; - f->start = b->data; - f->to = f->start; - f->stop = (char*)f->start + b->len; - f->flush = nil; - f->farg = nil; - f->nfmt = 0; - f->args = nil; -} - -static int -sflush(Fmt *f) -{ - char *s; - int n; - - n = (int)f->farg; - n += 256; - f->farg = (void*)n; - s = f->start; - f->start = realloc(s, n); - if(f->start == nil){ - f->start = s; - return 0; - } - f->to = (char*)f->start + ((char*)f->to - s); - f->stop = (char*)f->start + n - 1; - return 1; -} - -static char* -logit(int severity, char *fmt, va_list args) -{ - Fmt f; - int n; - - f.runes = 0; - n = 32; - f.start = malloc(n); - if(f.start == nil) - return nil; - f.to = f.start; - f.stop = (char*)f.start + n - 1; - f.flush = sflush; - f.farg = (void*)n; - f.nfmt = 0; - f.args = args; - n = dofmt(&f, fmt); - if(n < 0) -{ -fprint(2, "dofmt %s failed\n", fmt); - return nil; -} - *(char*)f.to = '\0'; - - if(argv0 == nil) - fprint(2, "%s: err %d: %s\n", argv0, severity, f.start); - else - fprint(2, "err %d: %s\n", severity, f.start); - return f.start; -} - -void -seterr(int severity, char *fmt, ...) -{ - char *s; - va_list args; - - va_start(args, fmt); - s = logit(severity, fmt, args); - va_end(args); - if(s == nil) - werrstr("error setting error"); - else{ - werrstr("%s", s); - free(s); - } -} - -void -logerr(int severity, char *fmt, ...) -{ - char *s; - va_list args; - - va_start(args, fmt); - s = logit(severity, fmt, args); - va_end(args); - free(s); -} - -u32int -now(void) -{ - return time(nil); -} - -void -fatal(char *fmt, ...) -{ - Fmt f; - char buf[256]; - - f.runes = 0; - f.start = buf; - f.to = buf; - f.stop = buf + sizeof(buf); - f.flush = fmtfdflush; - f.farg = (void*)2; - f.nfmt = 0; - fmtprint(&f, "fatal %s error:", argv0); - va_start(f.args, fmt); - dofmt(&f, fmt); - va_end(f.args); - fmtprint(&f, "\n"); - fmtfdflush(&f); - if(0) - abort(); - threadexitsall(buf); -} - -ZBlock * -alloczblock(u32int size, int zeroed) -{ - ZBlock *b; - static ZBlock z; - - b = malloc(sizeof(ZBlock) + size); - if(b == nil){ - seterr(EOk, "out of memory"); - return nil; - } - - *b = z; - b->data = (u8int*)&b[1]; - b->len = size; - if(zeroed) - memset(b->data, 0, size); - return b; -} - -void -freezblock(ZBlock *b) -{ - free(b); -} - -ZBlock* -packet2zblock(Packet *p, u32int size) -{ - ZBlock *b; - - if(p == nil) - return nil; - b = alloczblock(size, 0); - if(b == nil) - return nil; - b->len = size; - if(packetcopy(p, b->data, 0, size) < 0){ - freezblock(b); - return nil; - } - return b; -} - -Packet* -zblock2packet(ZBlock *zb, u32int size) -{ - Packet *p; - - if(zb == nil) - return nil; - p = packetalloc(); - packetappend(p, zb->data, size); - return p; -} - -void * -emalloc(ulong n) -{ - void *p; - - p = malloc(n); - if(p == nil) - sysfatal("out of memory"); - memset(p, 0xa5, n); -if(0)print("emalloc %p-%p by %lux\n", p, (char*)p+n, getcallerpc(&n)); - return p; -} - -void * -ezmalloc(ulong n) -{ - void *p; - - p = malloc(n); - if(p == nil) - sysfatal("out of memory"); - memset(p, 0, n); -if(0)print("ezmalloc %p-%p by %lux\n", p, (char*)p+n, getcallerpc(&n)); - return p; -} - -void * -erealloc(void *p, ulong n) -{ - p = realloc(p, n); - if(p == nil) - sysfatal("out of memory"); -if(0)print("erealloc %p-%p by %lux\n", p, (char*)p+n, getcallerpc(&p)); - return p; -} - -char * -estrdup(char *s) -{ - char *t; - int n; - - n = strlen(s) + 1; - t = emalloc(n); - memmove(t, s, n); -if(0)print("estrdup %p-%p by %lux\n", t, (char*)t+n, getcallerpc(&s)); - return t; -} - -ZBlock* -readfile(char *name) -{ - Part *p; - ZBlock *b; - - p = initpart(name, 1); - if(p == nil) - return nil; - b = alloczblock(p->size, 0); - if(b == nil){ - seterr(EOk, "can't alloc %s: %r", name); - freepart(p); - return nil; - } - if(readpart(p, 0, b->data, p->size) < 0){ - seterr(EOk, "can't read %s: %r", name); - freepart(p); - freezblock(b); - return nil; - } - freepart(p); - return b; -} - -/* - * return floor(log2(v)) - */ -int -u64log2(u64int v) -{ - int i; - - for(i = 0; i < 64; i++) - if((v >> i) <= 1) - break; - return i; -} - -int -vtproc(void (*fn)(void*), void *arg) -{ - proccreate(fn, arg, 256*1024); - return 0; -} - blob - 4c533cc626c782d719e4792aff43b28e218eee7d (mode 644) blob + /dev/null --- src/cmd/venti/venti.c +++ /dev/null @@ -1,178 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -#include "whack.h" - -int debug; -int mainstacksize = 256*1024; - -static void ventiserver(char *vaddr); - -void -threadmain(int argc, char *argv[]) -{ - char *config, *haddr, *vaddr; - u32int mem, icmem, bcmem, minbcmem; - - vaddr = "tcp!*!venti"; - haddr = nil; - config = nil; - mem = 0xffffffffUL; - icmem = 0; - bcmem = 0; - ARGBEGIN{ - case 'a': - vaddr = ARGF(); - if(vaddr == nil) - goto usage; - break; - case 'B': - bcmem = unittoull(ARGF()); - break; - case 'c': - config = ARGF(); - if(config == nil) - goto usage; - break; - case 'C': - mem = unittoull(ARGF()); - break; - case 'd': - debug = 1; - break; - case 'h': - haddr = ARGF(); - if(haddr == nil) - goto usage; - break; - case 'I': - icmem = unittoull(ARGF()); - break; - case 'w': - queuewrites = 1; - break; - default: - goto usage; - }ARGEND - -print("whack %d\n", sizeof(Whack)); - - if(argc){ - usage: - fprint(2, "usage: venti [-dw] [-a ventiaddress] [-h httpaddress] [-c config] [-C cachesize] [-I icachesize] [-B blockcachesize]\n"); - threadexitsall("usage"); - } - - fmtinstall('V', vtscorefmt); - fmtinstall('H', encodefmt); - fmtinstall('F', vtfcallfmt); - - if(config == nil) - config = "venti.conf"; - - - if(initarenasum() < 0) - fprint(2, "warning: can't initialize arena summing process: %r"); - - if(initventi(config) < 0) - sysfatal("can't init server: %r"); - - if(mem == 0xffffffffUL) - mem = 1 * 1024 * 1024; - fprint(2, "initialize %d bytes of lump cache for %d lumps\n", - mem, mem / (8 * 1024)); - initlumpcache(mem, mem / (8 * 1024)); - - icmem = u64log2(icmem / (sizeof(IEntry)+sizeof(IEntry*)) / ICacheDepth); - if(icmem < 4) - icmem = 4; - fprint(2, "initialize %d bytes of index cache for %d index entries\n", - (sizeof(IEntry)+sizeof(IEntry*)) * (1 << icmem) * ICacheDepth, - (1 << icmem) * ICacheDepth); - initicache(icmem, ICacheDepth); - - /* - * need a block for every arena and every process - */ - minbcmem = maxblocksize * - (mainindex->narenas + mainindex->nsects*4 + 16); - if(bcmem < minbcmem) - bcmem = minbcmem; - - fprint(2, "initialize %d bytes of disk block cache\n", bcmem); - initdcache(bcmem); - - fprint(2, "sync arenas and index...\n"); - if(syncindex(mainindex, 1) < 0) - sysfatal("can't sync server: %r"); - - if(queuewrites){ - fprint(2, "initialize write queue...\n"); - if(initlumpqueues(mainindex->nsects) < 0){ - fprint(2, "can't initialize lump queues," - " disabling write queueing: %r"); - queuewrites = 0; - } - } - - if(haddr){ - fprint(2, "starting http server at %s\n", haddr); - if(httpdinit(haddr) < 0) - fprint(2, "warning: can't start http server: %r"); - } - - ventiserver(vaddr); - threadexitsall(0); -} - -static void -vtrerror(VtReq *r, char *error) -{ - r->rx.type = VtRerror; - r->rx.error = estrdup(error); -} - -static void -ventiserver(char *addr) -{ - Packet *p; - VtReq *r; - VtSrv *s; - char err[ERRMAX]; - - s = vtlisten(addr); - if(s == nil) - sysfatal("can't announce %s: %r", addr); - - while((r = vtgetreq(s)) != nil){ - r->rx.type = r->tx.type+1; - // print("req (arenas[0]=%p sects[0]=%p) %F\n", - // mainindex->arenas[0], mainindex->sects[0], &r->tx); - switch(r->tx.type){ - default: - vtrerror(r, "unknown request"); - break; - case VtTread: - if((r->rx.data = readlump(r->tx.score, r->tx.dtype, r->tx.count)) == nil){ - rerrstr(err, sizeof err); - vtrerror(r, err); - } - break; - case VtTwrite: - p = r->tx.data; - r->tx.data = nil; - if(writelump(p, r->rx.score, r->tx.dtype, 0) < 0){ - rerrstr(err, sizeof err); - vtrerror(r, err); - } - break; - case VtTsync: - flushqueue(); - flushdcache(); - break; - } - vtrespond(r); - } -} - blob - 9e7dc31b9b8f11403240eb1c7e84e251dc813e6c (mode 644) blob + /dev/null --- src/cmd/venti/verifyarena.c +++ /dev/null @@ -1,127 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -static int verbose; - -void -usage(void) -{ - fprint(2, "usage: verifyarena [-v]\n"); - threadexitsall(0); -} - -static void -readblock(uchar *buf, int n) -{ - int nr, m; - - for(nr = 0; nr < n; nr += m){ - m = n - nr; - m = read(0, &buf[nr], m); - if(m <= 0) - sysfatal("can't read arena from standard input: %r"); - } -} - -static void -verifyarena(void) -{ - Arena arena; - ArenaHead head; - ZBlock *b; - DigestState s; - u64int n, e; - u32int bs; - u8int score[VtScoreSize]; - - fprint(2, "verify arena from standard input\n"); - - memset(&arena, 0, sizeof arena); - memset(&s, 0, sizeof s); - - /* - * read the little bit, which will included the header - */ - bs = MaxIoSize; - b = alloczblock(bs, 0); - readblock(b->data, HeadSize); - sha1(b->data, HeadSize, nil, &s); - if(unpackarenahead(&head, b->data) < 0) - sysfatal("corrupted arena header: %r"); - if(head.version != ArenaVersion) - fprint(2, "warning: unknown arena version %d\n", head.version); - - /* - * now we know how much to read - * read everything but the last block, which is special - */ - e = head.size - head.blocksize; - for(n = HeadSize; n < e; n += bs){ - if(n + bs > e) - bs = e - n; - readblock(b->data, bs); - sha1(b->data, bs, nil, &s); - } - - /* - * read the last block update the sum. - * the sum is calculated assuming the slot for the sum is zero. - */ - bs = head.blocksize; - readblock(b->data, bs); - sha1(b->data, bs-VtScoreSize, nil, &s); - sha1(zeroscore, VtScoreSize, nil, &s); - sha1(nil, 0, score, &s); - - /* - * validity check on the trailer - */ - arena.blocksize = head.blocksize; - if(unpackarena(&arena, b->data) < 0) - sysfatal("corrupted arena trailer: %r"); - scorecp(arena.score, &b->data[arena.blocksize - VtScoreSize]); - - if(namecmp(arena.name, head.name) != 0) - sysfatal("arena header and trailer names clash: %s vs. %s\n", head.name, arena.name); - if(arena.version != head.version) - sysfatal("arena header and trailer versions clash: %d vs. %d\n", head.version, arena.version); - arena.size = head.size - 2 * head.blocksize; - - /* - * check for no checksum or the same - */ - if(scorecmp(score, arena.score) != 0){ - if(scorecmp(zeroscore, arena.score) != 0) - fprint(2, "warning: mismatched checksums for arena=%s, found=%V calculated=%V", - arena.name, arena.score, score); - scorecp(arena.score, score); - }else - fprint(2, "matched score\n"); - - printarena(2, &arena); -} - -void -threadmain(int argc, char *argv[]) -{ - fmtinstall('V', vtscorefmt); - statsinit(); - - ARGBEGIN{ - case 'v': - verbose++; - break; - default: - usage(); - break; - }ARGEND - - readonly = 1; - - if(argc != 0) - usage(); - - verifyarena(); - threadexitsall(0); -} blob - 9aa6b6d2fbb0a968cc2733684c30f744497e5085 (mode 644) blob + /dev/null --- src/cmd/venti/whack.c +++ /dev/null @@ -1,330 +0,0 @@ -#include "stdinc.h" -#include "whack.h" - -typedef struct Huff Huff; - -enum -{ - MaxFastLen = 9, - BigLenCode = 0x1f4, /* minimum code for large lenth encoding */ - BigLenBits = 9, - BigLenBase = 4, /* starting items to encode for big lens */ - - MinOffBits = 6, - MaxOffBits = MinOffBits + 8, - - MaxLen = 2051 /* max. length encodable in 24 bits */ -}; - -enum -{ - StatBytes, - StatOutBytes, - StatLits, - StatMatches, - StatLitBits, - StatOffBits, - StatLenBits, - - MaxStat -}; - -struct Huff -{ - short bits; /* length of the code */ - ulong encode; /* the code */ -}; - -static Huff lentab[MaxFastLen] = -{ - {2, 0x2}, /* 10 */ - {3, 0x6}, /* 110 */ - {5, 0x1c}, /* 11100 */ - {5, 0x1d}, /* 11101 */ - {6, 0x3c}, /* 111100 */ - {7, 0x7a}, /* 1111010 */ - {7, 0x7b}, /* 1111011 */ - {8, 0xf8}, /* 11111000 */ - {8, 0xf9}, /* 11111001 */ -}; - -static int thwmaxcheck; - -void -whackinit(Whack *tw, int level) -{ - thwmaxcheck = (1 << level); - thwmaxcheck -= thwmaxcheck >> 2; - if(thwmaxcheck < 2) - thwmaxcheck = 2; - else if(thwmaxcheck > 1024) - thwmaxcheck = 1024; - memset(tw, 0, sizeof *tw); - tw->begin = 2 * WhackMaxOff; -} - -/* - * find a string in the dictionary - */ -static int -whackmatch(Whack *b, uchar **ss, uchar *esrc, ulong h, ulong now) -{ - ushort then, off, last; - int bestoff, bestlen, check; - uchar *s, *t; - - s = *ss; - if(esrc < s + MinMatch) - return -1; - if(s + MaxLen < esrc) - esrc = s + MaxLen; - - bestoff = 0; - bestlen = 0; - check = thwmaxcheck; - last = 0; - for(then = b->hash[h]; check-- > 0; then = b->next[then & (WhackMaxOff - 1)]){ - off = now - then; - if(off <= last || off > WhackMaxOff) - break; - - /* - * don't need to check for the end because - * 1) s too close check above - */ - t = s - off; - if(s[0] == t[0] && s[1] == t[1] && s[2] == t[2]){ - if(!bestlen || esrc - s > bestlen && s[bestlen] == t[bestlen]){ - t += 3; - for(s += 3; s < esrc; s++){ - if(*s != *t) - break; - t++; - } - if(s - *ss > bestlen){ - bestlen = s - *ss; - bestoff = off; - if(bestlen > thwmaxcheck) - break; - } - } - } - s = *ss; - last = off; - } - *ss += bestlen; - return bestoff; -} - -/* - * knuth vol. 3 multiplicative hashing - * each byte x chosen according to rules - * 1/4 < x < 3/10, 1/3 x < < 3/7, 4/7 < x < 2/3, 7/10 < x < 3/4 - * with reasonable spread between the bytes & their complements - * - * the 3 byte value appears to be as almost good as the 4 byte value, - * and might be faster on some machines - */ -/* -#define hashit(c) ((((ulong)(c) * 0x6b43a9) >> (24 - HashLog)) & HashMask) -*/ -#define hashit(c) (((((ulong)(c) & 0xffffff) * 0x6b43a9b5) >> (32 - HashLog)) & HashMask) - -/* - * lz77 compression with single lookup in a hash table for each block - */ -int -whack(Whack *w, uchar *dst, uchar *src, int n, ulong stats[WhackStats]) -{ - uchar *s, *ss, *sss, *esrc, *half, *wdst, *wdmax; - ulong cont, code, wbits; - ushort now; - int toff, lithist, h, len, bits, use, wnbits, lits, matches, offbits, lenbits; - - if(n < MinMatch) - return -1; - - wdst = dst; - wdmax = dst + n; - - now = w->begin; - s = src; - w->data = s; - - cont = (s[0] << 16) | (s[1] << 8) | s[2]; - - esrc = s + n; - half = s + (n >> 1); - wnbits = 0; - wbits = 0; - lits = 0; - matches = 0; - offbits = 0; - lenbits = 0; - lithist = ~0; - while(s < esrc){ - h = hashit(cont); - - sss = s; - toff = whackmatch(w, &sss, esrc, h, now); - ss = sss; - - len = ss - s; - for(; wnbits >= 8; wnbits -= 8){ - if(wdst >= wdmax){ - w->begin = now; - return -1; - } - *wdst++ = wbits >> (wnbits - 8); - } - if(len < MinMatch){ - toff = *s; - lithist = (lithist << 1) | toff < 32 | toff > 127; - if(lithist & 0x1e){ - wbits = (wbits << 9) | toff; - wnbits += 9; - }else if(lithist & 1){ - toff = (toff + 64) & 0xff; - if(toff < 96){ - wbits = (wbits << 10) | toff; - wnbits += 10; - }else{ - wbits = (wbits << 11) | toff; - wnbits += 11; - } - }else{ - wbits = (wbits << 8) | toff; - wnbits += 8; - } - lits++; - - /* - * speed hack - * check for compression progress, bail if none achieved - */ - if(s > half){ - if(4 * (s - src) < 5 * lits){ - w->begin = now; - return -1; - } - half = esrc; - } - - if(s + MinMatch <= esrc){ - w->next[now & (WhackMaxOff - 1)] = w->hash[h]; - w->hash[h] = now; - if(s + MinMatch < esrc) - cont = (cont << 8) | s[MinMatch]; - } - now++; - s++; - continue; - } - - matches++; - - /* - * length of match - */ - if(len > MaxLen){ - len = MaxLen; - ss = s + len; - } - len -= MinMatch; - if(len < MaxFastLen){ - bits = lentab[len].bits; - wbits = (wbits << bits) | lentab[len].encode; - wnbits += bits; - lenbits += bits; - }else{ - code = BigLenCode; - bits = BigLenBits; - use = BigLenBase; - len -= MaxFastLen; - while(len >= use){ - len -= use; - code = (code + use) << 1; - use <<= (bits & 1) ^ 1; - bits++; - } - - wbits = (wbits << bits) | (code + len); - wnbits += bits; - lenbits += bits; - - for(; wnbits >= 8; wnbits -= 8){ - if(wdst >= wdmax){ - w->begin = now; - return -1; - } - *wdst++ = wbits >> (wnbits - 8); - } - } - - /* - * offset in history - */ - toff--; - for(bits = MinOffBits; toff >= (1 << bits); bits++) - ; - if(bits < MaxOffBits-1){ - wbits = (wbits << 3) | (bits - MinOffBits); - if(bits != MinOffBits) - bits--; - wnbits += bits + 3; - offbits += bits + 3; - }else{ - wbits = (wbits << 4) | 0xe | (bits - (MaxOffBits-1)); - bits--; - wnbits += bits + 4; - offbits += bits + 4; - } - wbits = (wbits << bits) | toff & ((1 << bits) - 1); - - for(; s != ss; s++){ - if(s + MinMatch <= esrc){ - h = hashit(cont); - w->next[now & (WhackMaxOff - 1)] = w->hash[h]; - w->hash[h] = now; - if(s + MinMatch < esrc) - cont = (cont << 8) | s[MinMatch]; - } - now++; - } - } - - w->begin = now; - - stats[StatBytes] += esrc - src; - stats[StatLits] += lits; - stats[StatMatches] += matches; - stats[StatLitBits] += (wdst - (dst + 2)) * 8 + wnbits - offbits - lenbits; - stats[StatOffBits] += offbits; - stats[StatLenBits] += lenbits; - - if(wnbits & 7){ - wbits <<= 8 - (wnbits & 7); - wnbits += 8 - (wnbits & 7); - } - for(; wnbits >= 8; wnbits -= 8){ - if(wdst >= wdmax) - return -1; - *wdst++ = wbits >> (wnbits - 8); - } - - stats[StatOutBytes] += wdst - dst; - - return wdst - dst; -} - -int -whackblock(uchar *dst, uchar *src, int ssize) -{ - Whack w; - ulong stats[MaxStat]; - int r; - - whackinit(&w, 6); - r = whack(&w, dst, src, ssize, stats); - return r; -} blob - fb966169c073eb5f0e7c69885312e5acc44dfc5c (mode 644) blob + /dev/null --- src/cmd/venti/whack.h +++ /dev/null @@ -1,40 +0,0 @@ -typedef struct Whack Whack; -typedef struct Unwhack Unwhack; - -enum -{ - WhackStats = 8, - WhackErrLen = 64, /* max length of error message from thwack or unthwack */ - WhackMaxOff = 16*1024, /* max allowed offset */ - - HashLog = 14, - HashSize = 1<= 8 */ - - MaxSeqMask = 8, /* number of bits in coding block mask */ - MaxSeqStart = 256 /* max offset of initial coding block */ -}; - -struct Whack -{ - ushort begin; /* time of first byte in hash */ - ushort hash[HashSize]; - ushort next[WhackMaxOff]; - uchar *data; -}; - -struct Unwhack -{ - char err[WhackErrLen]; -}; - -void whackinit(Whack*, int level); -void unwhackinit(Unwhack*); -int whack(Whack*, uchar *dst, uchar *src, int nsrc, ulong stats[WhackStats]); -int unwhack(Unwhack*, uchar *dst, int ndst, uchar *src, int nsrc); - -int whackblock(uchar *dst, uchar *src, int ssize); blob - f232928ae0e9c23dc3cd49c0de565b3def5af17c (mode 644) blob + /dev/null --- src/cmd/venti/wrarena.c +++ /dev/null @@ -1,148 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -char *host; -int readonly = 1; /* for part.c */ - -void -usage(void) -{ - fprint(2, "usage: wrarena [-h host] arenafile [offset]\n"); - threadexitsall("usage"); -} - -static void -rdarena(VtConn *z, Arena *arena, u64int offset) -{ - u64int a, aa, e; - u32int magic; - Clump cl; - uchar score[VtScoreSize]; - ZBlock *lump; - - fprint(2, "copying %s to venti\n", arena->name); - printarena(2, arena); - - a = arena->base; - e = arena->base + arena->size; - if(offset != ~(u64int)0) { - if(offset >= e-a) - sysfatal("bad offset %llud >= %llud\n", - offset, e-a); - aa = offset; - } else - aa = 0; - - for(; aa < e; aa += ClumpSize+cl.info.size) { - magic = clumpmagic(arena, aa); - if(magic == ClumpFreeMagic) - break; - if(magic != ClumpMagic) { - fprint(2, "illegal clump magic number %#8.8ux offset %llud\n", - magic, aa); - break; - } - lump = loadclump(arena, aa, 0, &cl, score, 0); - if(lump == nil) { - fprint(2, "clump %llud failed to read: %r\n", aa); - break; - } - if(cl.info.type != VtTypeCorrupt) { - scoremem(score, lump->data, cl.info.uncsize); - if(scorecmp(cl.info.score, score) != 0) { - fprint(2, "clump %llud has mismatched score\n", aa); - break; - } - if(vttypevalid(cl.info.type) < 0) { - fprint(2, "clump %llud has bad type %d\n", aa, cl.info.type); - break; - } - } - if(z && vtwrite(z, score, cl.info.type, lump->data, cl.info.uncsize) < 0) - sysfatal("failed writing clump %llud: %r", aa); - freezblock(lump); - } - if(z && vtsync(z) < 0) - sysfatal("failed executing sync: %r"); - - print("end offset %llud\n", aa); -} - -void -threadmain(int argc, char *argv[]) -{ - char *file; - VtConn *z; - Arena *arena; - u64int offset, aoffset; - Part *part; - Dir *d; - uchar buf[8192]; - ArenaHead head; - - aoffset = 0; - ARGBEGIN{ - case 'h': - host = EARGF(usage()); - break; - case 'o': - aoffset = strtoull(EARGF(usage()), 0, 0); - break; - default: - usage(); - break; - }ARGEND - - offset = ~(u64int)0; - switch(argc) { - default: - usage(); - case 2: - offset = strtoull(argv[1], 0, 0); - /* fall through */ - case 1: - file = argv[0]; - } - - - fmtinstall('V', vtscorefmt); - - statsinit(); - - if((d = dirstat(file)) == nil) - sysfatal("can't stat file %s: %r", file); - - part = initpart(file, 0); - if(part == nil) - sysfatal("can't open file %s: %r", file); - if(readpart(part, aoffset, buf, sizeof buf) < 0) - sysfatal("can't read file %s: %r", file); - - if(unpackarenahead(&head, buf) < 0) - sysfatal("corrupted arena header: %r"); - - if(aoffset+head.size > d->length) - sysfatal("arena is truncated: want %llud bytes have %llud\n", - head.size, d->length); - - partblocksize(part, head.blocksize); - initdcache(8 * MaxDiskBlock); - - arena = initarena(part, aoffset, head.size, head.blocksize); - if(arena == nil) - sysfatal("initarena: %r"); - - if(host && strcmp(host, "/dev/null") != 0){ - z = vtdial(host); - if(z == nil) - sysfatal("could not connect to server: %r"); - if(vtconnect(z) < 0) - sysfatal("vtconnect: %r"); - }else - z = nil; - - rdarena(z, arena, offset); - vthangup(z); - threadexitsall(0); -} blob - 94b61be44d95a07e8a6e52cf105b7d44671f5350 (mode 644) blob + /dev/null --- src/cmd/venti/write.c +++ /dev/null @@ -1,60 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -char *host; - -void -usage(void) -{ - fprint(2, "usage: write [-z] [-h host] [-t type] VtMaxLumpSize) - sysfatal("data too big"); - z = vtdial(host); - if(z == nil) - sysfatal("could not connect to server: %r"); - if(vtconnect(z) < 0) - sysfatal("vtconnect: %r"); - if(dotrunc) - n = vtzerotruncate(type, p, n); - if(vtwrite(z, score, type, p, n) < 0) - sysfatal("vtwrite: %r"); - vthangup(z); - print("%V\n", score); - threadexitsall(0); -} blob - d50d5bc99555069eda8586d02544612bf65d627b (mode 644) blob + /dev/null --- src/cmd/venti/xml.c +++ /dev/null @@ -1,69 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" -#include "httpd.h" -#include "xml.h" - -void xmlarena(Hio *hout, Arena *s, char *tag, int indent){ - xmlindent(hout, indent); - hprint(hout, "<%s", tag); - xmlaname(hout, s->name, "name"); - xmlu32int(hout, s->version, "version"); - xmlaname(hout, s->part->name, "partition"); - xmlu32int(hout, s->blocksize, "blocksize"); - xmlu64int(hout, s->base, "start"); - xmlu64int(hout, s->base+2*s->blocksize, "stop"); - xmlu32int(hout, s->ctime, "created"); - xmlu32int(hout, s->wtime, "modified"); - xmlsealed(hout, s->sealed, "sealed"); - xmlscore(hout, s->score, "score"); - xmlu32int(hout, s->clumps, "clumps"); - xmlu32int(hout, s->cclumps, "compressedclumps"); - xmlu64int(hout, s->uncsize, "data"); - xmlu64int(hout, s->used - s->clumps * ClumpSize, "compresseddata"); - xmlu64int(hout, s->used + s->clumps * ClumpInfoSize, "storage"); - hprint(hout, "/>\n"); -} - -void xmlindex(Hio *hout, Index *s, char *tag, int indent){ - int i; - xmlindent(hout, indent); - hprint(hout, "<%s", tag); - xmlaname(hout, s->name, "name"); - xmlu32int(hout, s->version, "version"); - xmlu32int(hout, s->blocksize, "blocksize"); - xmlu32int(hout, s->tabsize, "tabsize"); - xmlu32int(hout, s->buckets, "buckets"); - xmlu32int(hout, s->div, "buckdiv"); - hprint(hout, ">\n"); - xmlindent(hout, indent + 1); - hprint(hout, "\n"); - for(i = 0; i < s->nsects; i++) - xmlamap(hout, &s->smap[i], "sect", indent + 2); - xmlindent(hout, indent + 1); - hprint(hout, "\n"); - xmlindent(hout, indent + 1); - hprint(hout, "\n"); - for(i = 0; i < s->narenas; i++) - xmlamap(hout, &s->amap[i], "amap", indent + 2); - xmlindent(hout, indent + 1); - hprint(hout, "\n"); - xmlindent(hout, indent + 1); - hprint(hout, "\n"); - for(i = 0; i < s->narenas; i++) - xmlarena(hout, s->arenas[i], "arena", indent + 2); - xmlindent(hout, indent + 1); - hprint(hout, "\n"); - xmlindent(hout, indent); - hprint(hout, "\n", tag); -} - -void xmlamap(Hio *hout, AMap *s, char *tag, int indent){ - xmlindent(hout, indent); - hprint(hout, "<%s", tag); - xmlaname(hout, s->name, "name"); - xmlu64int(hout, s->start, "start"); - xmlu64int(hout, s->stop, "stop"); - hprint(hout, "/>\n"); -} - blob - c9e52b0bb666b62ae04903313c9e4e1b09ff0d36 (mode 644) blob + /dev/null --- src/cmd/venti/xml.h +++ /dev/null @@ -1,11 +0,0 @@ -void xmlamap(Hio *hout, AMap *v, char *tag, int indent); -void xmlarena(Hio *hout, Arena *v, char *tag, int indent); -void xmlindex(Hio *hout, Index *v, char *tag, int indent); - -void xmlaname(Hio *hout, char *v, char *tag); -void xmlscore(Hio *hout, u8int *v, char *tag); -void xmlsealed(Hio *hout, int v, char *tag); -void xmlu32int(Hio *hout, u32int v, char *tag); -void xmlu64int(Hio *hout, u64int v, char *tag); - -void xmlindent(Hio *hout, int indent); blob - b24c98c631e44ada7cfb114ab0a0e13b63d3ce08 (mode 644) blob + /dev/null --- src/cmd/venti/zeropart.c +++ /dev/null @@ -1,28 +0,0 @@ -#include "stdinc.h" -#include "dat.h" -#include "fns.h" - -void -zeropart(Part *part, int blocksize) -{ - ZBlock *b; - u64int addr; - int w; - - fprint(2, "clearing the partition\n"); - - b = alloczblock(MaxIoSize, 1); - - w = 0; - for(addr = PartBlank; addr + MaxIoSize <= part->size; addr += MaxIoSize){ - if(writepart(part, addr, b->data, MaxIoSize) < 0) - sysfatal("can't initialize %s, writing block %d failed: %r", part->name, w); - w++; - } - - for(; addr + blocksize <= part->size; addr += blocksize) - if(writepart(part, addr, b->data, blocksize) < 0) - sysfatal("can't initialize %s: %r", part->name); - - freezblock(b); -}