commit - d338028eeb45434dab651466dcd664091517e087
commit + adbb83845c5d3cb02343d7613cfb8385aa909909
blob - 689631701a2d9111ba567a19ad2b5b05567321a1 (mode 644)
blob + /dev/null
--- src/cmd/venti/README
+++ /dev/null
-This is very untested.
-It runs on FreeBSD but that's about all I've checked.
-
blob - 318f9ad24a32ab28101e2ec27f252f9c66625e15 (mode 644)
blob + /dev/null
--- src/cmd/venti/arena.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-typedef struct ASum ASum;
-
-struct ASum
-{
- Arena *arena;
- ASum *next;
-};
-
-static void sealarena(Arena *arena);
-static int okarena(Arena *arena);
-static int loadarena(Arena *arena);
-static CIBlock *getcib(Arena *arena, int clump, int writing, CIBlock *rock);
-static void putcib(Arena *arena, CIBlock *cib);
-static void sumproc(void *);
-
-static QLock sumlock;
-static Rendez sumwait;
-static ASum *sumq;
-
-int
-initarenasum(void)
-{
- sumwait.l = &sumlock;
-
- if(vtproc(sumproc, nil) < 0){
- seterr(EOk, "can't start arena checksum slave: %r");
- return -1;
- }
- return 0;
-}
-
-/*
- * make an Arena, and initialize it based upon the disk header and trailer.
- */
-Arena*
-initarena(Part *part, u64int base, u64int size, u32int blocksize)
-{
- Arena *arena;
-
- arena = MKZ(Arena);
- arena->part = part;
- arena->blocksize = blocksize;
- arena->clumpmax = arena->blocksize / ClumpInfoSize;
- arena->base = base + blocksize;
- arena->size = size - 2 * blocksize;
-
- if(loadarena(arena) < 0){
- seterr(ECorrupt, "arena header or trailer corrupted");
- freearena(arena);
- return nil;
- }
- if(okarena(arena) < 0){
- freearena(arena);
- return nil;
- }
-
- if(arena->sealed && scorecmp(zeroscore, arena->score)==0)
- backsumarena(arena);
-
- return arena;
-}
-
-void
-freearena(Arena *arena)
-{
- if(arena == nil)
- return;
- if(arena->cib.data != nil){
- putdblock(arena->cib.data);
- arena->cib.data = nil;
- }
- free(arena);
-}
-
-Arena*
-newarena(Part *part, char *name, u64int base, u64int size, u32int blocksize)
-{
- Arena *arena;
-
- if(nameok(name) < 0){
- seterr(EOk, "illegal arena name", name);
- return nil;
- }
- arena = MKZ(Arena);
- arena->part = part;
- arena->version = ArenaVersion;
- arena->blocksize = blocksize;
- arena->clumpmax = arena->blocksize / ClumpInfoSize;
- arena->base = base + blocksize;
- arena->size = size - 2 * blocksize;
-
- namecp(arena->name, name);
-
- if(wbarena(arena)<0 || wbarenahead(arena)<0){
- freearena(arena);
- return nil;
- }
-
- return arena;
-}
-
-int
-readclumpinfo(Arena *arena, int clump, ClumpInfo *ci)
-{
- CIBlock *cib, r;
-
- cib = getcib(arena, clump, 0, &r);
- if(cib == nil)
- return -1;
- unpackclumpinfo(ci, &cib->data->data[cib->offset]);
- putcib(arena, cib);
- return 0;
-}
-
-int
-readclumpinfos(Arena *arena, int clump, ClumpInfo *cis, int n)
-{
- CIBlock *cib, r;
- int i;
-
- for(i = 0; i < n; i++){
- cib = getcib(arena, clump + i, 0, &r);
- if(cib == nil)
- break;
- unpackclumpinfo(&cis[i], &cib->data->data[cib->offset]);
- putcib(arena, cib);
- }
- return i;
-}
-
-/*
- * write directory information for one clump
- * must be called the arena locked
- */
-int
-writeclumpinfo(Arena *arena, int clump, ClumpInfo *ci)
-{
- CIBlock *cib, r;
-
- cib = getcib(arena, clump, 1, &r);
- if(cib == nil)
- return -1;
- dirtydblock(cib->data, DirtyArenaCib);
- packclumpinfo(ci, &cib->data->data[cib->offset]);
- putcib(arena, cib);
- return 0;
-}
-
-u64int
-arenadirsize(Arena *arena, u32int clumps)
-{
- return ((clumps / arena->clumpmax) + 1) * arena->blocksize;
-}
-
-/*
- * read a clump of data
- * n is a hint of the size of the data, not including the header
- * make sure it won't run off the end, then return the number of bytes actually read
- */
-u32int
-readarena(Arena *arena, u64int aa, u8int *buf, long n)
-{
- DBlock *b;
- u64int a;
- u32int blocksize, off, m;
- long nn;
-
- if(n == 0)
- return -1;
-
- qlock(&arena->lock);
- a = arena->size - arenadirsize(arena, arena->clumps);
- qunlock(&arena->lock);
- if(aa >= a){
- seterr(EOk, "reading beyond arena clump storage: clumps=%d aa=%lld a=%lld -1 clumps=%lld\n", arena->clumps, aa, a, arena->size - arenadirsize(arena, arena->clumps - 1));
- return -1;
- }
- if(aa + n > a)
- n = a - aa;
-
- blocksize = arena->blocksize;
- a = arena->base + aa;
- off = a & (blocksize - 1);
- a -= off;
- nn = 0;
- for(;;){
- b = getdblock(arena->part, a, 1);
- if(b == nil)
- return -1;
- m = blocksize - off;
- if(m > n - nn)
- m = n - nn;
- memmove(&buf[nn], &b->data[off], m);
- putdblock(b);
- nn += m;
- if(nn == n)
- break;
- off = 0;
- a += blocksize;
- }
- return n;
-}
-
-/*
- * write some data to the clump section at a given offset
- * used to fix up corrupted arenas.
- */
-u32int
-writearena(Arena *arena, u64int aa, u8int *clbuf, u32int n)
-{
- DBlock *b;
- u64int a;
- u32int blocksize, off, m;
- long nn;
- int ok;
-
- if(n == 0)
- return -1;
-
- qlock(&arena->lock);
- a = arena->size - arenadirsize(arena, arena->clumps);
- if(aa >= a || aa + n > a){
- qunlock(&arena->lock);
- seterr(EOk, "writing beyond arena clump storage");
- return -1;
- }
-
- blocksize = arena->blocksize;
- a = arena->base + aa;
- off = a & (blocksize - 1);
- a -= off;
- nn = 0;
- for(;;){
- b = getdblock(arena->part, a, off != 0 || off + n < blocksize);
- if(b == nil){
- qunlock(&arena->lock);
- return -1;
- }
- dirtydblock(b, DirtyArena);
- m = blocksize - off;
- if(m > n - nn)
- m = n - nn;
- memmove(&b->data[off], &clbuf[nn], m);
- // ok = writepart(arena->part, a, b->data, blocksize);
- ok = 0;
- putdblock(b);
- if(ok < 0){
- qunlock(&arena->lock);
- return -1;
- }
- nn += m;
- if(nn == n)
- break;
- off = 0;
- a += blocksize;
- }
- qunlock(&arena->lock);
- return n;
-}
-
-/*
- * allocate space for the clump and write it,
- * updating the arena directory
-ZZZ question: should this distinguish between an arena
-filling up and real errors writing the clump?
- */
-u64int
-writeaclump(Arena *arena, Clump *c, u8int *clbuf)
-{
- DBlock *b;
- u64int a, aa;
- u32int clump, n, nn, m, off, blocksize;
- int ok;
-
- n = c->info.size + ClumpSize;
- qlock(&arena->lock);
- aa = arena->used;
- if(arena->sealed
- || aa + n + U32Size + arenadirsize(arena, arena->clumps + 1) > arena->size){
- if(!arena->sealed)
- sealarena(arena);
- qunlock(&arena->lock);
- return TWID64;
- }
- if(packclump(c, &clbuf[0]) < 0){
- qunlock(&arena->lock);
- return TWID64;
- }
-
- /*
- * write the data out one block at a time
- */
- blocksize = arena->blocksize;
- a = arena->base + aa;
- off = a & (blocksize - 1);
- a -= off;
- nn = 0;
- for(;;){
- b = getdblock(arena->part, a, off != 0);
- if(b == nil){
- qunlock(&arena->lock);
- return TWID64;
- }
- dirtydblock(b, DirtyArena);
- m = blocksize - off;
- if(m > n - nn)
- m = n - nn;
- memmove(&b->data[off], &clbuf[nn], m);
- // ok = writepart(arena->part, a, b->data, blocksize);
- ok = 0;
- putdblock(b);
- if(ok < 0){
- qunlock(&arena->lock);
- return TWID64;
- }
- nn += m;
- if(nn == n)
- break;
- off = 0;
- a += blocksize;
- }
-
- arena->used += c->info.size + ClumpSize;
- arena->uncsize += c->info.uncsize;
- if(c->info.size < c->info.uncsize)
- arena->cclumps++;
-
- clump = arena->clumps++;
- if(arena->clumps == 0)
- sysfatal("clumps wrapped\n");
- arena->wtime = now();
- if(arena->ctime == 0)
- arena->ctime = arena->wtime;
-
- writeclumpinfo(arena, clump, &c->info);
-//ZZZ make this an enum param
- if((clump & 0x1ff) == 0x1ff){
- flushciblocks(arena);
- wbarena(arena);
- }
-
- qunlock(&arena->lock);
- return aa;
-}
-
-/*
- * once sealed, an arena never has any data added to it.
- * it should only be changed to fix errors.
- * this also syncs the clump directory.
- */
-static void
-sealarena(Arena *arena)
-{
- flushciblocks(arena);
- flushdcache();
- arena->sealed = 1;
- wbarena(arena);
- backsumarena(arena);
-}
-
-void
-backsumarena(Arena *arena)
-{
- ASum *as;
-
- as = MK(ASum);
- if(as == nil)
- return;
- qlock(&sumlock);
- as->arena = arena;
- as->next = sumq;
- sumq = as;
- rwakeup(&sumwait);
- qunlock(&sumlock);
-}
-
-static void
-sumproc(void *unused)
-{
- ASum *as;
- Arena *arena;
-
- USED(unused);
-
- for(;;){
- qlock(&sumlock);
- while(sumq == nil)
- rsleep(&sumwait);
- as = sumq;
- sumq = as->next;
- qunlock(&sumlock);
- arena = as->arena;
- free(as);
-
- sumarena(arena);
- }
-}
-
-void
-sumarena(Arena *arena)
-{
- ZBlock *b;
- DigestState s;
- u64int a, e;
- u32int bs;
- u8int score[VtScoreSize];
-
- bs = MaxIoSize;
- if(bs < arena->blocksize)
- bs = arena->blocksize;
-
- /*
- * read & sum all blocks except the last one
- */
- memset(&s, 0, sizeof s);
- b = alloczblock(bs, 0);
- e = arena->base + arena->size;
- for(a = arena->base - arena->blocksize; a + arena->blocksize <= e; a += bs){
- if(a + bs > e)
- bs = arena->blocksize;
- if(readpart(arena->part, a, b->data, bs) < 0)
- goto ReadErr;
- sha1(b->data, bs, nil, &s);
- }
-
- /*
- * the last one is special, since it may already have the checksum included
- */
- bs = arena->blocksize;
- if(readpart(arena->part, e, b->data, bs) < 0){
-ReadErr:
- logerr(EOk, "sumarena can't sum %s, read at %lld failed: %r", arena->name, a);
- freezblock(b);
- return;
- }
-
- sha1(b->data, bs-VtScoreSize, nil, &s);
- sha1(zeroscore, VtScoreSize, nil, &s);
- sha1(nil, 0, score, &s);
-
- /*
- * check for no checksum or the same
- *
- * the writepart is okay because we flushed the dcache in sealarena
- */
- if(scorecmp(score, &b->data[bs - VtScoreSize]) != 0){
- if(scorecmp(zeroscore, &b->data[bs - VtScoreSize]) != 0)
- logerr(EOk, "overwriting mismatched checksums for arena=%s, found=%V calculated=%V",
- arena->name, &b->data[bs - VtScoreSize], score);
- scorecp(&b->data[bs - VtScoreSize], score);
- if(writepart(arena->part, e, b->data, bs) < 0)
- logerr(EOk, "sumarena can't write sum for %s: %r", arena->name);
- }
- freezblock(b);
-
- qlock(&arena->lock);
- scorecp(arena->score, score);
- qunlock(&arena->lock);
-}
-
-/*
- * write the arena trailer block to the partition
- */
-int
-wbarena(Arena *arena)
-{
- DBlock *b;
- int bad;
-
- if((b = getdblock(arena->part, arena->base + arena->size, 0)) == nil){
- logerr(EAdmin, "can't write arena trailer: %r");
- return -1;
- }
- dirtydblock(b, DirtyArenaTrailer);
- bad = okarena(arena)<0 || packarena(arena, b->data)<0;
- putdblock(b);
- if(bad)
- return -1;
- return 0;
-}
-
-int
-wbarenahead(Arena *arena)
-{
- ZBlock *b;
- ArenaHead head;
- int bad;
-
- namecp(head.name, arena->name);
- head.version = arena->version;
- head.size = arena->size + 2 * arena->blocksize;
- head.blocksize = arena->blocksize;
- b = alloczblock(arena->blocksize, 1);
- if(b == nil){
- logerr(EAdmin, "can't write arena header: %r");
-///ZZZ add error message?
- return -1;
- }
- /*
- * this writepart is okay because it only happens
- * during initialization.
- */
- bad = packarenahead(&head, b->data)<0 ||
- writepart(arena->part, arena->base - arena->blocksize, b->data, arena->blocksize)<0;
- freezblock(b);
- if(bad)
- return -1;
- return 0;
-}
-
-/*
- * read the arena header and trailer blocks from disk
- */
-static int
-loadarena(Arena *arena)
-{
- ArenaHead head;
- ZBlock *b;
-
- b = alloczblock(arena->blocksize, 0);
- if(b == nil)
- return -1;
- if(readpart(arena->part, arena->base + arena->size, b->data, arena->blocksize) < 0){
- freezblock(b);
- return -1;
- }
- if(unpackarena(arena, b->data) < 0){
- freezblock(b);
- return -1;
- }
- if(arena->version != ArenaVersion){
- seterr(EAdmin, "unknown arena version %d", arena->version);
- freezblock(b);
- return -1;
- }
- scorecp(arena->score, &b->data[arena->blocksize - VtScoreSize]);
-
- if(readpart(arena->part, arena->base - arena->blocksize, b->data, arena->blocksize) < 0){
- logerr(EAdmin, "can't read arena header: %r");
- freezblock(b);
- return 0;
- }
- if(unpackarenahead(&head, b->data) < 0)
- logerr(ECorrupt, "corrupted arena header: %r");
- else if(namecmp(arena->name, head.name)!=0
- || arena->version != head.version
- || arena->blocksize != head.blocksize
- || arena->size + 2 * arena->blocksize != head.size)
- logerr(ECorrupt, "arena header inconsistent with arena data");
- freezblock(b);
-
- return 0;
-}
-
-static int
-okarena(Arena *arena)
-{
- u64int dsize;
- int ok;
-
- ok = 0;
- dsize = arenadirsize(arena, arena->clumps);
- if(arena->used + dsize > arena->size){
- seterr(ECorrupt, "arena used > size");
- ok = -1;
- }
-
- if(arena->cclumps > arena->clumps)
- logerr(ECorrupt, "arena has more compressed clumps than total clumps");
-
- if(arena->uncsize + arena->clumps * ClumpSize + arena->blocksize < arena->used)
- logerr(ECorrupt, "arena uncompressed size inconsistent with used space %lld %d %lld", arena->uncsize, arena->clumps, arena->used);
-
- if(arena->ctime > arena->wtime)
- logerr(ECorrupt, "arena creation time after last write time");
-
- return ok;
-}
-
-static CIBlock*
-getcib(Arena *arena, int clump, int writing, CIBlock *rock)
-{
- int read;
- CIBlock *cib;
- u32int block, off;
-
- if(clump >= arena->clumps){
- seterr(EOk, "clump directory access out of range");
- return nil;
- }
- block = clump / arena->clumpmax;
- off = (clump - block * arena->clumpmax) * ClumpInfoSize;
-
-/*
- if(arena->cib.block == block
- && arena->cib.data != nil){
- arena->cib.offset = off;
- return &arena->cib;
- }
-
- if(writing){
- flushciblocks(arena);
- cib = &arena->cib;
- }else
- cib = rock;
-*/
- cib = rock;
-
- qlock(&stats.lock);
- stats.cireads++;
- qunlock(&stats.lock);
-
- cib->block = block;
- cib->offset = off;
-
- read = 1;
- if(writing && off == 0 && clump == arena->clumps-1)
- read = 0;
-
- cib->data = getdblock(arena->part, arena->base + arena->size - (block + 1) * arena->blocksize, read);
- if(cib->data == nil)
- return nil;
- return cib;
-}
-
-static void
-putcib(Arena *arena, CIBlock *cib)
-{
- if(cib != &arena->cib){
- putdblock(cib->data);
- cib->data = nil;
- }
-}
-
-/*
- * must be called with arena locked
- *
- * cache turned off now that dcache does write caching too.
- */
-int
-flushciblocks(Arena *arena)
-{
- int ok;
-
- if(arena->cib.data == nil)
- return 0;
- qlock(&stats.lock);
- stats.ciwrites++;
- qunlock(&stats.lock);
-// ok = writepart(arena->part, arena->base + arena->size - (arena->cib.block + 1) * arena->blocksize, arena->cib.data->data, arena->blocksize);
- ok = 0;
- if(ok < 0)
- seterr(EAdmin, "failed writing arena directory block");
- putdblock(arena->cib.data);
- arena->cib.data = nil;
- return ok;
-}
blob - 5275b938cf76762d7a90f05eb22280d4ea09e1c5 (mode 644)
blob + /dev/null
--- src/cmd/venti/arenas.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-typedef struct AHash AHash;
-
-/*
- * hash table for finding arena's based on their names.
- */
-struct AHash
-{
- AHash *next;
- Arena *arena;
-};
-
-enum
-{
- AHashSize = 512
-};
-
-static AHash *ahash[AHashSize];
-
-static u32int
-hashstr(char *s)
-{
- u32int h;
- int c;
-
- h = 0;
- for(; c = *s; s++){
- c ^= c << 6;
- h += (c << 11) ^ (c >> 1);
- c = *s;
- h ^= (c << 14) + (c << 7) + (c << 4) + c;
- }
- return h;
-}
-
-int
-addarena(Arena *arena)
-{
- AHash *a;
- u32int h;
-
- h = hashstr(arena->name) & (AHashSize - 1);
- a = MK(AHash);
- if(a == nil)
- return -1;
- a->arena = arena;
- a->next = ahash[h];
- ahash[h] = a;
- return 0;
-}
-
-Arena*
-findarena(char *name)
-{
- AHash *a;
- u32int h;
-
- h = hashstr(name) & (AHashSize - 1);
- for(a = ahash[h]; a != nil; a = a->next)
- if(strcmp(a->arena->name, name) == 0)
- return a->arena;
- return nil;
-}
-
-int
-delarena(Arena *arena)
-{
- AHash *a, *last;
- u32int h;
-
- h = hashstr(arena->name) & (AHashSize - 1);
- last = nil;
- for(a = ahash[h]; a != nil; a = a->next){
- if(a->arena == arena){
- if(last != nil)
- last->next = a->next;
- else
- ahash[h] = a->next;
- free(a);
- return 0;
- }
- last = a;
- }
- return -1;
-}
-
-ArenaPart*
-initarenapart(Part *part)
-{
- AMapN amn;
- ArenaPart *ap;
- ZBlock *b;
- u32int i;
- int ok;
-
- b = alloczblock(HeadSize, 0);
- if(b == nil || readpart(part, PartBlank, b->data, HeadSize) < 0){
- seterr(EAdmin, "can't read arena partition header: %r");
- return nil;
- }
-
- ap = MKZ(ArenaPart);
- if(ap == nil){
- freezblock(b);
- return nil;
- }
- ap->part = part;
- ok = unpackarenapart(ap, b->data);
- freezblock(b);
- if(ok < 0){
- seterr(ECorrupt, "corrupted arena partition header: %r");
- freearenapart(ap, 0);
- return nil;
- }
-
- ap->tabbase = (PartBlank + HeadSize + ap->blocksize - 1) & ~(ap->blocksize - 1);
- if(ap->version != ArenaPartVersion){
- seterr(ECorrupt, "unknown arena partition version %d", ap->version);
- freearenapart(ap, 0);
- return nil;
- }
- if(ap->blocksize & (ap->blocksize - 1)){
- seterr(ECorrupt, "illegal non-power-of-2 block size %d\n", ap->blocksize);
- freearenapart(ap, 0);
- return nil;
- }
- if(ap->tabbase >= ap->arenabase){
- seterr(ECorrupt, "arena partition table overlaps with arena storage");
- freearenapart(ap, 0);
- return nil;
- }
- ap->tabsize = ap->arenabase - ap->tabbase;
- partblocksize(part, ap->blocksize);
- ap->size = ap->part->size & ~(u64int)(ap->blocksize - 1);
-
- if(readarenamap(&amn, part, ap->tabbase, ap->tabsize) < 0){
- freearenapart(ap, 0);
- return nil;
- }
- ap->narenas = amn.n;
- ap->map = amn.map;
- if(okamap(ap->map, ap->narenas, ap->arenabase, ap->size, "arena table") < 0){
- freearenapart(ap, 0);
- return nil;
- }
-
- ap->arenas = MKNZ(Arena*, ap->narenas);
- for(i = 0; i < ap->narenas; i++){
- ap->arenas[i] = initarena(part, ap->map[i].start, ap->map[i].stop - ap->map[i].start, ap->blocksize);
- if(ap->arenas[i] == nil){
- freearenapart(ap, 1);
- return nil;
- }
- if(namecmp(ap->map[i].name, ap->arenas[i]->name) != 0){
- seterr(ECorrupt, "arena name mismatches with expected name: %s vs. %s",
- ap->map[i].name, ap->arenas[i]->name);
- freearenapart(ap, 1);
- return nil;
- }
- if(findarena(ap->arenas[i]->name)){
- seterr(ECorrupt, "duplicate arena name %s in %s",
- ap->map[i].name, ap->part->name);
- freearenapart(ap, 1);
- return nil;
- }
- }
-
- for(i = 0; i < ap->narenas; i++)
- addarena(ap->arenas[i]);
-
- return ap;
-}
-
-ArenaPart*
-newarenapart(Part *part, u32int blocksize, u32int tabsize)
-{
- ArenaPart *ap;
-
- if(blocksize & (blocksize - 1)){
- seterr(ECorrupt, "illegal non-power-of-2 block size %d\n", blocksize);
- return nil;
- }
- ap = MKZ(ArenaPart);
- if(ap == nil)
- return nil;
-
- ap->version = ArenaPartVersion;
- ap->part = part;
- ap->blocksize = blocksize;
- partblocksize(part, blocksize);
- ap->size = part->size & ~(u64int)(blocksize - 1);
- ap->tabbase = (PartBlank + HeadSize + blocksize - 1) & ~(blocksize - 1);
- ap->arenabase = (ap->tabbase + tabsize + blocksize - 1) & ~(blocksize - 1);
- ap->tabsize = ap->arenabase - ap->tabbase;
- ap->narenas = 0;
-
- if(wbarenapart(ap) < 0){
- freearenapart(ap, 0);
- return nil;
- }
-
- return ap;
-}
-
-int
-wbarenapart(ArenaPart *ap)
-{
- ZBlock *b;
-
- if(okamap(ap->map, ap->narenas, ap->arenabase, ap->size, "arena table") < 0)
- return -1;
- b = alloczblock(HeadSize, 1);
- if(b == nil)
-//ZZZ set error message?
- return -1;
-
- if(packarenapart(ap, b->data) < 0){
- seterr(ECorrupt, "can't make arena partition header: %r");
- freezblock(b);
- return -1;
- }
- if(writepart(ap->part, PartBlank, b->data, HeadSize) < 0){
- seterr(EAdmin, "can't write arena partition header: %r");
- freezblock(b);
- return -1;
- }
- freezblock(b);
-
- return wbarenamap(ap->map, ap->narenas, ap->part, ap->tabbase, ap->tabsize);
-}
-
-void
-freearenapart(ArenaPart *ap, int freearenas)
-{
- int i;
-
- if(ap == nil)
- return;
- if(freearenas){
- for(i = 0; i < ap->narenas; i++){
- if(ap->arenas[i] == nil)
- continue;
- delarena(ap->arenas[i]);
- freearena(ap->arenas[i]);
- }
- }
- free(ap->map);
- free(ap->arenas);
- free(ap);
-}
-
-int
-okamap(AMap *am, int n, u64int start, u64int stop, char *what)
-{
- u64int last;
- u32int i;
-
- last = start;
- for(i = 0; i < n; i++){
- if(am[i].start < last){
- if(i == 0)
- seterr(ECorrupt, "invalid start address in %s", what);
- else
- seterr(ECorrupt, "overlapping ranges in %s", what);
- return -1;
- }
- if(am[i].stop < am[i].start){
- seterr(ECorrupt, "invalid range in %s", what);
- return -1;
- }
- last = am[i].stop;
- }
- if(last > stop){
- seterr(ECorrupt, "invalid ending address in %s", what);
- return -1;
- }
- return 0;
-}
-
-int
-maparenas(AMap *am, Arena **arenas, int n, char *what)
-{
- u32int i;
-
- for(i = 0; i < n; i++){
- arenas[i] = findarena(am[i].name);
- if(arenas[i] == nil){
- seterr(EAdmin, "can't find arena '%s' for '%s'\n", am[i].name, what);
- return -1;
- }
- }
- return 0;
-}
-
-int
-readarenamap(AMapN *amn, Part *part, u64int base, u32int size)
-{
- IFile f;
- u32int ok;
-
- if(partifile(&f, part, base, size) < 0)
- return -1;
- ok = parseamap(&f, amn);
- freeifile(&f);
- return ok;
-}
-
-int
-wbarenamap(AMap *am, int n, Part *part, u64int base, u64int size)
-{
- Fmt f;
- ZBlock *b;
-
- b = alloczblock(size, 1);
- if(b == nil)
- return -1;
-
- fmtzbinit(&f, b);
-
- if(outputamap(&f, am, n) < 0){
- seterr(ECorrupt, "arena set size too small");
- freezblock(b);
- return -1;
- }
- if(writepart(part, base, b->data, size) < 0){
- seterr(EAdmin, "can't write arena set: %r");
- freezblock(b);
- return -1;
- }
- freezblock(b);
- return 0;
-}
-
-/*
- * amap: n '\n' amapelem * n
- * n: u32int
- * amapelem: name '\t' astart '\t' asize '\n'
- * astart, asize: u64int
- */
-int
-parseamap(IFile *f, AMapN *amn)
-{
- AMap *am;
- u64int v64;
- u32int v;
- char *s, *flds[4];
- int i, n;
-
- /*
- * arenas
- */
- if(ifileu32int(f, &v) < 0){
- seterr(ECorrupt, "syntax error: bad number of elements in %s", f->name);
- return -1;
- }
- n = v;
- if(n > MaxAMap){
- seterr(ECorrupt, "illegal number of elements in %s", f->name);
- return -1;
- }
- am = MKNZ(AMap, n);
- if(am == nil)
- return -1;
- for(i = 0; i < n; i++){
- s = ifileline(f);
- if(s == nil || getfields(s, flds, 4, 0, "\t") != 3)
- return -1;
- if(nameok(flds[0]) < 0)
- return -1;
- namecp(am[i].name, flds[0]);
- if(stru64int(flds[1], &v64) < 0){
- seterr(ECorrupt, "syntax error: bad arena base address in %s", f->name);
- free(am);
- return -1;
- }
- am[i].start = v64;
- if(stru64int(flds[2], &v64) < 0){
- seterr(ECorrupt, "syntax error: bad arena size in %s", f->name);
- free(am);
- return -1;
- }
- am[i].stop = v64;
- }
-
- amn->map = am;
- amn->n = n;
- return 0;
-}
-
-int
-outputamap(Fmt *f, AMap *am, int n)
-{
- int i;
-
- if(fmtprint(f, "%ud\n", n) < 0)
- return -1;
- for(i = 0; i < n; i++)
- if(fmtprint(f, "%s\t%llud\t%llud\n", am[i].name, am[i].start, am[i].stop) < 0)
- return -1;
- return 0;
-}
blob - e5aed260f741ed0df517b109d359b47f3445e714 (mode 644)
blob + /dev/null
--- src/cmd/venti/buildbuck.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-struct IEStream
-{
- Part *part;
- u64int off; /* read position within part */
- u64int n; /* number of valid ientries left to read */
- u32int size; /* allocated space in buffer */
- u8int *buf;
- u8int *pos; /* current place in buffer */
- u8int *epos; /* end of valid buffer contents */
-};
-
-IEStream*
-initiestream(Part *part, u64int off, u64int clumps, u32int size)
-{
- IEStream *ies;
-
-//ZZZ out of memory?
- ies = MKZ(IEStream);
- ies->buf = MKN(u8int, size);
- ies->epos = ies->buf;
- ies->pos = ies->epos;
- ies->off = off;
- ies->n = clumps;
- ies->size = size;
- ies->part = part;
- return ies;
-}
-
-void
-freeiestream(IEStream *ies)
-{
- if(ies == nil)
- return;
- free(ies->buf);
- free(ies);
-}
-
-static u8int*
-peekientry(IEStream *ies)
-{
- u32int n, nn;
-
- n = ies->epos - ies->pos;
- if(n < IEntrySize){
- memmove(ies->buf, ies->pos, n);
- ies->epos = &ies->buf[n];
- ies->pos = ies->buf;
- nn = ies->size;
- if(nn > ies->n * IEntrySize)
- nn = ies->n * IEntrySize;
- nn -= n;
- if(nn == 0)
- return nil;
- if(readpart(ies->part, ies->off, ies->epos, nn) < 0){
- seterr(EOk, "can't read sorted index entries: %r");
- return nil;
- }
- ies->epos += nn;
- ies->off += nn;
- }
- return ies->pos;
-}
-
-static u32int
-iebuck(Index *ix, u8int *b)
-{
- return hashbits(b, 32) / ix->div;
-}
-
-u32int
-buildbucket(Index *ix, IEStream *ies, IBucket *ib)
-{
- IEntry ie1, ie2;
- u8int *b;
- u32int buck;
-
- buck = TWID32;
- ib->n = 0;
- ib->depth = 0;
- while(ies->n){
- b = peekientry(ies);
- if(b == nil)
- return TWID32;
-//fprint(2, "b=%p ies->n=%lld ib.n=%d buck=%d score=%V\n", b, ies->n, ib->n, iebuck(ix, b), b);
- if(ib->n == 0)
- buck = iebuck(ix, b);
- else{
- if(buck != iebuck(ix, b))
- break;
- if(ientrycmp(&ib->data[(ib->n - 1)* IEntrySize], b) == 0){
- /*
- * guess that the larger address is the correct one to use
- */
- unpackientry(&ie1, &ib->data[(ib->n - 1)* IEntrySize]);
- unpackientry(&ie2, b);
- seterr(EOk, "duplicate index entry for score=%V type=%d\n", ie1.score, ie1.ia.type);
- ib->n--;
- if(ie1.ia.addr > ie2.ia.addr)
- memmove(b, &ib->data[ib->n * IEntrySize], IEntrySize);
- }
- }
- memmove(&ib->data[ib->n * IEntrySize], b, IEntrySize);
- ib->n++;
- ies->n--;
- ies->pos += IEntrySize;
- }
- return buck;
-}
blob - 8058ba098653cef50a84f99c22b5148590cd2116 (mode 644)
blob + /dev/null
--- src/cmd/venti/buildindex.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-static int
-writebucket(Index *ix, u32int buck, IBucket *ib, ZBlock *b)
-{
- ISect *is;
-
- is = findibucket(ix, buck, &buck);
- if(is == nil)
- return -1;
- qlock(&stats.lock);
- stats.indexwrites++;
- qunlock(&stats.lock);
- packibucket(ib, b->data);
- return writepart(is->part, is->blockbase + ((u64int)buck << is->blocklog), b->data, is->blocksize);
-}
-
-static int
-buildindex(Index *ix, Part *part, u64int off, u64int clumps, int zero)
-{
- IEStream *ies;
- IBucket ib, zib;
- ZBlock *z, *b;
- u32int next, buck;
- int ok;
- u64int found = 0;
-
-//ZZZ make buffer size configurable
- b = alloczblock(ix->blocksize, 0);
- z = alloczblock(ix->blocksize, 1);
- ies = initiestream(part, off, clumps, 64*1024);
- if(b == nil || z == nil || ies == nil){
- ok = 0;
- goto breakout;
- return -1;
- }
- ok = 0;
- next = 0;
- ib.data = b->data + IBucketSize;
- zib.data = z->data + IBucketSize;
- zib.n = 0;
- zib.depth = 0;
- for(;;){
- buck = buildbucket(ix, ies, &ib);
- found += ib.n;
- if(zero){
- for(; next != buck; next++){
- if(next == ix->buckets){
- if(buck != TWID32){
- fprint(2, "bucket out of range\n");
- ok = -1;
- }
- goto breakout;
- }
- if(writebucket(ix, next, &zib, z) < 0){
- fprint(2, "can't write zero bucket to buck=%d: %r", next);
- ok = -1;
- }
- }
- }
- if(buck >= ix->buckets){
- if(buck == TWID32)
- break;
- fprint(2, "bucket out of range\n");
- ok = -1;
- goto breakout;
- }
- if(writebucket(ix, buck, &ib, b) < 0){
- fprint(2, "bad bucket found=%lld: %r\n", found);
- ok = -1;
- }
- next = buck + 1;
- }
-breakout:;
- fprint(2, "constructed index with %lld entries\n", found);
- freeiestream(ies);
- freezblock(z);
- freezblock(b);
- return ok;
-}
-
-void
-usage(void)
-{
- fprint(2, "usage: buildindex [-Z] [-B blockcachesize] config tmppart\n");
- threadexitsall(0);
-}
-
-void
-threadmain(int argc, char *argv[])
-{
- Part *part;
- u64int clumps, base;
- u32int bcmem;
- int zero;
-
- zero = 1;
- bcmem = 0;
- ARGBEGIN{
- case 'B':
- bcmem = unittoull(ARGF());
- break;
- case 'Z':
- zero = 0;
- break;
- default:
- usage();
- break;
- }ARGEND
-
- if(argc != 2)
- usage();
-
- if(initventi(argv[0]) < 0)
- sysfatal("can't init venti: %r");
-
- if(bcmem < maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16))
- bcmem = maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16);
- fprint(2, "initialize %d bytes of disk block cache\n", bcmem);
- initdcache(bcmem);
-
- fprint(2, "building a new index %s using %s for temporary storage\n", mainindex->name, argv[1]);
-
- part = initpart(argv[1], 1);
- if(part == nil)
- sysfatal("can't initialize temporary partition: %r");
-
- clumps = sortrawientries(mainindex, part, &base);
- if(clumps == TWID64)
- sysfatal("can't build sorted index: %r");
- fprint(2, "found and sorted index entries for clumps=%lld at %lld\n", clumps, base);
-
- if(buildindex(mainindex, part, base, clumps, zero) < 0)
- sysfatal("can't build new index: %r");
-
- threadexitsall(0);
-}
blob - ae7c641acf9ab772ca8d0363cf78f0e112596dca (mode 644)
blob + /dev/null
--- src/cmd/venti/checkarenas.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-static int verbose;
-
-static void
-checkarena(Arena *arena, int scan, int fix)
-{
- Arena old;
- int err, e;
-
- if(verbose && arena->clumps)
- printarena(2, arena);
-
- old = *arena;
-
- if(scan){
- arena->used = 0;
- arena->clumps = 0;
- arena->cclumps = 0;
- arena->uncsize = 0;
- }
-
- err = 0;
- for(;;){
- e = syncarena(arena, 1000, 0, fix);
- err |= e;
- if(!(e & SyncHeader))
- break;
- if(verbose && arena->clumps)
- fprint(2, ".");
- }
- if(verbose && arena->clumps)
- fprint(2, "\n");
-
- err &= ~SyncHeader;
- if(arena->used != old.used
- || arena->clumps != old.clumps
- || arena->cclumps != old.cclumps
- || arena->uncsize != old.uncsize){
- fprint(2, "incorrect arena header fields\n");
- printarena(2, arena);
- err |= SyncHeader;
- }
-
- if(!err || !fix)
- return;
-
- fprint(2, "writing fixed arena header fields\n");
- if(wbarena(arena) < 0)
- fprint(2, "arena header write failed: %r\n");
-}
-
-void
-usage(void)
-{
- fprint(2, "usage: checkarenas [-afv] file\n");
- threadexitsall(0);
-}
-
-void
-threadmain(int argc, char *argv[])
-{
- ArenaPart *ap;
- Part *part;
- char *file;
- int i, fix, scan;
-
- fmtinstall('V', vtscorefmt);
-
- statsinit();
-
- fix = 0;
- scan = 0;
- ARGBEGIN{
- case 'f':
- fix++;
- break;
- case 'a':
- scan = 1;
- break;
- case 'v':
- verbose++;
- break;
- default:
- usage();
- break;
- }ARGEND
-
- if(!fix)
- readonly = 1;
-
- if(argc != 1)
- usage();
-
- file = argv[0];
-
- part = initpart(file, 0);
- if(part == nil)
- sysfatal("can't open partition %s: %r", file);
-
- ap = initarenapart(part);
- if(ap == nil)
- sysfatal("can't initialize arena partition in %s: %r", file);
-
- if(verbose > 1){
- printarenapart(2, ap);
- fprint(2, "\n");
- }
-
- initdcache(8 * MaxDiskBlock);
-
- for(i = 0; i < ap->narenas; i++)
- checkarena(ap->arenas[i], scan, fix);
-
- if(verbose > 1)
- printstats();
- threadexitsall(0);
-}
blob - 34edb3708475fc66bab98e394aac46fa84e49c73 (mode 644)
blob + /dev/null
--- src/cmd/venti/checkindex.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-static int
-checkbucket(Index *ix, u32int buck, IBucket *ib)
-{
- ISect *is;
- DBlock *eb;
- IBucket eib;
- IEntry ie, eie;
- int i, ei, ok, c;
-
- is = findibucket(ix, buck, &buck);
- eb = getdblock(is->part, is->blockbase + ((u64int)buck << is->blocklog), 1);
- if(eb == nil)
- return -1;
- unpackibucket(&eib, eb->data);
-
- ok = 0;
- ei = 0;
- for(i = 0; i < ib->n; i++){
- while(ei < eib.n){
- c = ientrycmp(&ib->data[i * IEntrySize], &eib.data[ei * IEntrySize]);
- if(c == 0){
- unpackientry(&ie, &ib->data[i * IEntrySize]);
- unpackientry(&eie, &eib.data[ei * IEntrySize]);
- if(iaddrcmp(&ie.ia, &eie.ia) != 0){
- fprint(2, "bad entry in index for score=%V\n", &ib->data[i * IEntrySize]);
- fprint(2, "\taddr=%lld type=%d size=%d blocks=%d\n",
- ie.ia.addr, ie.ia.type, ie.ia.size, ie.ia.blocks);
- fprint(2, "\taddr=%lld type=%d size=%d blocks=%d\n",
- eie.ia.addr, eie.ia.type, eie.ia.size, eie.ia.blocks);
- }
- ei++;
- goto cont;
- }
- if(c < 0)
- break;
-if(1)
- fprint(2, "spurious entry in index for score=%V type=%d\n",
- &eib.data[ei * IEntrySize], eib.data[ei * IEntrySize + IEntryTypeOff]);
- ei++;
- ok = -1;
- }
- fprint(2, "missing entry in index for score=%V type=%d\n",
- &ib->data[i * IEntrySize], ib->data[i * IEntrySize + IEntryTypeOff]);
- ok = -1;
- cont:;
- }
- for(; ei < eib.n; ei++){
-if(1) fprint(2, "spurious entry in index for score=%V; found %d entries expected %d\n",
- &eib.data[ei * IEntrySize], eib.n, ib->n);
- ok = -1;
- break;
- }
- putdblock(eb);
- return ok;
-}
-
-int
-checkindex(Index *ix, Part *part, u64int off, u64int clumps, int zero)
-{
- IEStream *ies;
- IBucket ib, zib;
- ZBlock *z, *b;
- u32int next, buck;
- int ok, bok;
-u64int found = 0;
-
-//ZZZ make buffer size configurable
- b = alloczblock(ix->blocksize, 0);
- z = alloczblock(ix->blocksize, 1);
- ies = initiestream(part, off, clumps, 64*1024);
- if(b == nil || z == nil || ies == nil){
- ok = -1;
- goto breakout;
- return -1;
- }
- ok = 0;
- next = 0;
- ib.data = b->data;
- zib.data = z->data;
- zib.n = 0;
- zib.depth = 0;
- for(;;){
- buck = buildbucket(ix, ies, &ib);
- found += ib.n;
- if(zero){
- for(; next != buck; next++){
- if(next == ix->buckets){
- if(buck != TWID32)
- fprint(2, "bucket out of range\n");
- goto breakout;
- }
- bok = checkbucket(ix, next, &zib);
- if(bok < 0){
- fprint(2, "bad bucket=%d found: %r\n", next);
- ok = -1;
- }
- }
- }
- if(buck >= ix->buckets){
- if(buck == TWID32)
- break;
- fprint(2, "bucket out of range\n");
- ok = -1;
- goto breakout;
- }
- bok = checkbucket(ix, buck, &ib);
- if(bok < 0){
- fprint(2, "bad bucket found=%lld: %r\n", found);
- ok = -1;
- }
- next = buck + 1;
- }
-breakout:;
-fprint(2, "found %lld entries in sorted list\n", found);
- freeiestream(ies);
- freezblock(z);
- freezblock(b);
- return ok;
-}
-
-void
-usage(void)
-{
- fprint(2, "usage: checkindex [-f] [-B blockcachesize] config tmp\n");
- threadexitsall(0);
-}
-
-void
-threadmain(int argc, char *argv[])
-{
- Part *part;
- u64int clumps, base;
- u32int bcmem;
- int fix, skipz;
-
- fix = 0;
- bcmem = 0;
- skipz = 0;
- ARGBEGIN{
- case 'B':
- bcmem = unittoull(ARGF());
- break;
- case 'f':
- fix++;
- break;
- case 'Z':
- skipz = 1;
- break;
- default:
- usage();
- break;
- }ARGEND
-
- if(!fix)
- readonly = 1;
-
- if(argc != 2)
- usage();
-
- if(initventi(argv[0]) < 0)
- sysfatal("can't init venti: %r");
-
- if(bcmem < maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16))
- bcmem = maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16);
- fprint(2, "initialize %d bytes of disk block cache\n", bcmem);
- initdcache(bcmem);
-
- part = initpart(argv[1], 1);
- if(part == nil)
- sysfatal("can't initialize temporary partition: %r");
-
- clumps = sortrawientries(mainindex, part, &base);
- if(clumps == TWID64)
- sysfatal("can't build sorted index: %r");
- fprint(2, "found and sorted index entries for clumps=%lld at %lld\n", clumps, base);
- checkindex(mainindex, part, base, clumps, !skipz);
-
- threadexitsall(0);
-}
blob - 1aa81bb1b019fce7913841c1c6828e1fca0a6278 (mode 644)
blob + /dev/null
--- src/cmd/venti/clump.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-#include "whack.h"
-
-/*
- * writes a lump to disk
- * returns the address in amap of the clump
- */
-int
-storeclump(Index *ix, ZBlock *zb, u8int *sc, int type, u32int creator, IAddr *ia)
-{
- ZBlock *cb;
- Clump cl;
- u64int a;
- u8int bh[VtScoreSize];
- int size, dsize;
-
-if(0)print("storeclump %08x %p\n", mainindex->arenas[0], &cl);
- size = zb->len;
- if(size > VtMaxLumpSize){
- seterr(EStrange, "lump too large");
- return -1;
- }
- if(vttypevalid(type) < 0){
- seterr(EStrange, "invalid lump type");
- return -1;
- }
-
- if(1){
- scoremem(bh, zb->data, size);
- if(scorecmp(sc, bh) != 0){
- seterr(ECorrupt, "storing clump: corrupted; expected=%V got=%V, size=%d", sc, bh, size);
- return -1;
- }
- }
-
- cb = alloczblock(size + ClumpSize, 0);
- if(cb == nil)
- return -1;
-
- cl.info.type = type;
- cl.info.uncsize = size;
- cl.creator = creator;
- cl.time = now();
- scorecp(cl.info.score, sc);
-
-if(0)print("whackblock %08x %p\n", mainindex->arenas[0], &cl);
- dsize=0; // dsize = whackblock(&cb->data[ClumpSize], zb->data, size);
-if(0)print("whackedblock %08x %p\n", mainindex->arenas[0], &cl);
- if(dsize > 0 && dsize < size){
- cl.encoding = ClumpECompress;
- }else{
- cl.encoding = ClumpENone;
- dsize = size;
- memmove(&cb->data[ClumpSize], zb->data, size);
- }
- cl.info.size = dsize;
-
- a = writeiclump(ix, &cl, cb->data);
-
- freezblock(cb);
- if(a == TWID64)
- return -1;
-
- qlock(&stats.lock);
- stats.clumpwrites++;
- stats.clumpbwrites += size;
- stats.clumpbcomp += dsize;
- qunlock(&stats.lock);
-
- ia->addr = a;
- ia->type = type;
- ia->size = size;
- ia->blocks = (dsize + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog;
-
- return 0;
-}
-
-u32int
-clumpmagic(Arena *arena, u64int aa)
-{
- u8int buf[U32Size];
-
- if(readarena(arena, aa, buf, U32Size) < 0)
- return TWID32;
- return unpackmagic(buf);
-}
-
-/*
- * fetch a block based at addr.
- * score is filled in with the block's score.
- * blocks is roughly the length of the clump on disk;
- * if zero, the length is unknown.
- */
-ZBlock*
-loadclump(Arena *arena, u64int aa, int blocks, Clump *cl, u8int *score, int verify)
-{
- Unwhack uw;
- ZBlock *zb, *cb;
- u8int bh[VtScoreSize], *buf;
- u32int n;
- int nunc;
-
- qlock(&stats.lock);
- stats.clumpreads++;
- qunlock(&stats.lock);
-
- if(blocks <= 0)
- blocks = 1;
-
- cb = alloczblock(blocks << ABlockLog, 0);
- if(cb == nil)
- return nil;
- n = readarena(arena, aa, cb->data, blocks << ABlockLog);
- if(n < ClumpSize){
- if(n != 0)
- seterr(ECorrupt, "loadclump read less than a header");
- freezblock(cb);
- return nil;
- }
- if(unpackclump(cl, cb->data) < 0){
- freezblock(cb);
- return nil;
- }
- n -= ClumpSize;
- if(n < cl->info.size){
- freezblock(cb);
- n = cl->info.size;
- cb = alloczblock(n, 0);
- if(cb == nil)
- return nil;
- if(readarena(arena, aa + ClumpSize, cb->data, n) != n){
- seterr(ECorrupt, "loadclump read too little data");
- freezblock(cb);
- return nil;
- }
- buf = cb->data;
- }else
- buf = cb->data + ClumpSize;
-
- scorecp(score, cl->info.score);
-
- zb = alloczblock(cl->info.uncsize, 0);
- if(zb == nil){
- freezblock(cb);
- return nil;
- }
- switch(cl->encoding){
- case ClumpECompress:
- unwhackinit(&uw);
- nunc = unwhack(&uw, zb->data, cl->info.uncsize, buf, cl->info.size);
- if(nunc != cl->info.uncsize){
- if(nunc < 0)
- seterr(ECorrupt, "decompression failed: %s", uw.err);
- else
- seterr(ECorrupt, "decompression gave partial block: %d/%d\n", nunc, cl->info.uncsize);
- freezblock(cb);
- freezblock(zb);
- return nil;
- }
- break;
- case ClumpENone:
- if(cl->info.size != cl->info.uncsize){
- seterr(ECorrupt, "loading clump: bad uncompressed size for uncompressed block");
- freezblock(cb);
- freezblock(zb);
- return nil;
- }
- memmove(zb->data, buf, cl->info.uncsize);
- break;
- default:
- seterr(ECorrupt, "unknown encoding in loadlump");
- freezblock(cb);
- freezblock(zb);
- return nil;
- }
- freezblock(cb);
-
- if(verify){
- scoremem(bh, zb->data, cl->info.uncsize);
- if(scorecmp(cl->info.score, bh) != 0){
- seterr(ECorrupt, "loading clump: corrupted; expected=%V got=%V", cl->info.score, bh);
- freezblock(zb);
- return nil;
- }
- if(vttypevalid(cl->info.type) < 0){
- seterr(ECorrupt, "loading lump: invalid lump type %d", cl->info.type);
- freezblock(zb);
- return nil;
- }
- }
-
- qlock(&stats.lock);
- stats.clumpbreads += cl->info.size;
- stats.clumpbuncomp += cl->info.uncsize;
- qunlock(&stats.lock);
-
- return zb;
-}
blob - 5a448b677b5f02df21e250386e4868c624c2ffd3 (mode 644)
blob + /dev/null
--- src/cmd/venti/clumpstats.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-int count[VtMaxLumpSize][VtMaxType];
-
-enum
-{
- ClumpChunks = 32*1024
-};
-
-static int
-readarenainfo(Arena *arena)
-{
- ClumpInfo *ci, *cis;
- u32int clump;
- int i, n, ok;
-
- if(arena->clumps)
- fprint(2, "reading directory for arena=%s with %d entries\n", arena->name, arena->clumps);
-
- cis = MKN(ClumpInfo, ClumpChunks);
- ok = 0;
- for(clump = 0; clump < arena->clumps; clump += n){
- n = ClumpChunks;
-
- if(n > arena->clumps - clump)
- n = arena->clumps - clump;
-
- if((i=readclumpinfos(arena, clump, cis, n)) != n){
- seterr(EOk, "arena directory read failed %d not %d: %r", i, n);
- ok = -1;
- break;
- }
-
- for(i = 0; i < n; i++){
- ci = &cis[i];
- if(ci->type >= VtMaxType || ci->uncsize >= VtMaxLumpSize) {
- fprint(2, "bad clump: %d: type = %d: size = %d\n", clump+i, ci->type, ci->uncsize);
- continue;
- }
-if(ci->uncsize == 422)
-print("%s: %d: %V\n", arena->name, clump+i, ci->score);
- count[ci->uncsize][ci->type]++;
- }
- }
- free(cis);
- if(ok < 0)
- return TWID32;
- return clump;
-}
-
-static void
-clumpstats(Index *ix)
-{
- int ok;
- ulong clumps, n;
- int i, j, t;
-
- ok = 0;
- clumps = 0;
- for(i = 0; i < ix->narenas; i++){
- n = readarenainfo(ix->arenas[i]);
- if(n == TWID32){
- ok = -1;
- break;
- }
- clumps += n;
- }
-
- if(ok < 0)
- return;
-
- print("clumps = %ld\n", clumps);
- for(i=0; i<VtMaxLumpSize; i++) {
- t = 0;
- for(j=0; j<VtMaxType; j++)
- t += count[i][j];
- if(t == 0)
- continue;
- print("%d\t%d", i, t);
- for(j=0; j<VtMaxType; j++)
- print("\t%d", count[i][j]);
- print("\n");
- }
-}
-
-
-void
-usage(void)
-{
- fprint(2, "usage: clumpstats [-B blockcachesize] config\n");
- threadexitsall(0);
-}
-
-void
-threadmain(int argc, char *argv[])
-{
- u32int bcmem;
-
- bcmem = 0;
-
- ARGBEGIN{
- case 'B':
- bcmem = unittoull(ARGF());
- break;
- default:
- usage();
- break;
- }ARGEND
-
- readonly = 1;
-
- if(argc != 1)
- usage();
-
- if(initventi(argv[0]) < 0)
- sysfatal("can't init venti: %r");
-
- if(bcmem < maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16))
- bcmem = maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16);
- fprint(2, "initialize %d bytes of disk block cache\n", bcmem);
- initdcache(bcmem);
-
- clumpstats(mainindex);
-
- threadexitsall(0);
-}
blob - cdcaad10ce97615649da45c5830d0049602f36b4 (mode 644)
blob + /dev/null
--- src/cmd/venti/config.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-Index *mainindex;
-int paranoid = 1; /* should verify hashes on disk read */
-
-static ArenaPart *configarenas(char *file);
-static ISect *configisect(char *file);
-
-int
-initventi(char *file)
-{
- Config conf;
-
- fmtinstall('V', vtscorefmt);
-
- statsinit();
-
- if(file == nil){
- seterr(EOk, "no configuration file");
- return -1;
- }
- if(runconfig(file, &conf) < 0){
- seterr(EOk, "can't initialize venti: %r");
- return -1;
- }
- mainindex = initindex(conf.index, conf.sects, conf.nsects);
- if(mainindex == nil)
- return -1;
- return 0;
-}
-
-/*
- * configs :
- * | configs config
- * config : "isect" filename
- * | "arenas" filename
- * | "index" name
- *
- * '#' and \n are comments
- */
-enum
-{
- MaxArgs = 2
-};
-int
-runconfig(char *file, Config *config)
-{
- ArenaPart **av;
- ISect **sv;
- IFile f;
- char *s, *line, *flds[MaxArgs + 1];
- int i, ok;
-
- if(readifile(&f, file) < 0)
- return -1;
- config->index = nil;
- config->naparts = 0;
- config->aparts = nil;
- config->nsects = 0;
- config->sects = nil;
- ok = -1;
- line = nil;
- for(;;){
- s = ifileline(&f);
- if(s == nil){
- ok = 0;
- break;
- }
- line = estrdup(s);
- i = getfields(s, flds, MaxArgs + 1, 1, " \t\r");
- if(i == 2 && strcmp(flds[0], "isect") == 0){
- sv = MKN(ISect*, config->nsects + 1);
- for(i = 0; i < config->nsects; i++)
- sv[i] = config->sects[i];
- free(config->sects);
- config->sects = sv;
- config->sects[config->nsects] = configisect(flds[1]);
- if(config->sects[config->nsects] == nil)
- break;
- config->nsects++;
- }else if(i == 2 && strcmp(flds[0], "arenas") == 0){
- av = MKN(ArenaPart*, config->naparts + 1);
- for(i = 0; i < config->naparts; i++)
- av[i] = config->aparts[i];
- free(config->aparts);
- config->aparts = av;
- config->aparts[config->naparts] = configarenas(flds[1]);
- if(config->aparts[config->naparts] == nil)
- break;
- config->naparts++;
- }else if(i == 2 && strcmp(flds[0], "index") == 0){
- if(nameok(flds[1]) < 0){
- seterr(EAdmin, "illegal index name %s in config file %s", flds[1], config);
- break;
- }
- if(config->index != nil){
- seterr(EAdmin, "duplicate indices in config file %s", config);
- break;
- }
- config->index = estrdup(flds[1]);
- }else{
- seterr(EAdmin, "illegal line '%s' in configuration file %s", line, config);
- break;
- }
- free(line);
- line = nil;
- }
- free(line);
- freeifile(&f);
- if(ok < 0){
- free(config->sects);
- config->sects = nil;
- free(config->aparts);
- config->aparts = nil;
- }
- return ok;
-}
-
-static ISect*
-configisect(char *file)
-{
- Part *part;
-
- fprint(2, "configure index section in %s\n", file);
-
- part = initpart(file, 0);
- if(part == nil)
- return nil;
- return initisect(part);
-}
-
-static ArenaPart*
-configarenas(char *file)
-{
- Part *part;
-
- fprint(2, "configure arenas in %s\n", file);
- part = initpart(file, 0);
- if(part == nil)
- return nil;
- return initarenapart(part);
-}
blob - 4688b076424cf03e28c99eaed0ca605b322b80c7 (mode 644)
blob + /dev/null
--- src/cmd/venti/conv.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-/*
- * disk structure conversion routines
- */
-#define U8GET(p) ((p)[0])
-#define U16GET(p) (((p)[0]<<8)|(p)[1])
-#define U32GET(p) ((u32int)(((p)[0]<<24)|((p)[1]<<16)|((p)[2]<<8)|(p)[3]))
-#define U64GET(p) (((u64int)U32GET(p)<<32)|(u64int)U32GET((p)+4))
-
-#define U8PUT(p,v) (p)[0]=(v)&0xFF
-#define U16PUT(p,v) (p)[0]=((v)>>8)&0xFF;(p)[1]=(v)&0xFF
-#define U32PUT(p,v) (p)[0]=((v)>>24)&0xFF;(p)[1]=((v)>>16)&0xFF;(p)[2]=((v)>>8)&0xFF;(p)[3]=(v)&0xFF
-#define U64PUT(p,v,t32) t32=(v)>>32;U32PUT(p,t32);t32=(v);U32PUT((p)+4,t32)
-
-static struct {
- u32int m;
- char *s;
-} magics[] = {
- ArenaPartMagic, "ArenaPartMagic",
- ArenaHeadMagic, "ArenaHeadMagic",
- ArenaMagic, "ArenaMagic",
- ISectMagic, "ISectMagic",
-};
-
-static char*
-fmtmagic(char *s, u32int m)
-{
- int i;
-
- for(i=0; i<nelem(magics); i++)
- if(magics[i].m == m)
- return magics[i].s;
- sprint(s, "0x%08lux", m);
- return s;
-}
-
-u32int
-unpackmagic(u8int *buf)
-{
- return U32GET(buf);
-}
-
-void
-packmagic(u32int magic, u8int *buf)
-{
- U32PUT(buf, magic);
-}
-
-int
-unpackarenapart(ArenaPart *ap, u8int *buf)
-{
- u8int *p;
- u32int m;
- char fbuf[20];
-
- p = buf;
-
- m = U32GET(p);
- if(m != ArenaPartMagic){
- seterr(ECorrupt, "arena set has wrong magic number: %s expected ArenaPartMagic (%lux)", fmtmagic(fbuf, m), ArenaPartMagic);
- return -1;
- }
- p += U32Size;
- ap->version = U32GET(p);
- p += U32Size;
- ap->blocksize = U32GET(p);
- p += U32Size;
- ap->arenabase = U32GET(p);
- p += U32Size;
-
- if(buf + ArenaPartSize != p)
- sysfatal("unpackarenapart unpacked wrong amount");
-
- return 0;
-}
-
-int
-packarenapart(ArenaPart *ap, u8int *buf)
-{
- u8int *p;
-
- p = buf;
-
- U32PUT(p, ArenaPartMagic);
- p += U32Size;
- U32PUT(p, ap->version);
- p += U32Size;
- U32PUT(p, ap->blocksize);
- p += U32Size;
- U32PUT(p, ap->arenabase);
- p += U32Size;
-
- if(buf + ArenaPartSize != p)
- sysfatal("packarenapart packed wrong amount");
-
- return 0;
-}
-
-int
-unpackarena(Arena *arena, u8int *buf)
-{
- u8int *p;
- u32int m;
- char fbuf[20];
-
- p = buf;
-
- m = U32GET(p);
- if(m != ArenaMagic){
- seterr(ECorrupt, "arena has wrong magic number: %s expected ArenaMagic (%lux)", fmtmagic(fbuf, m), m, ArenaMagic);
- return -1;
- }
- p += U32Size;
- arena->version = U32GET(p);
- p += U32Size;
- namecp(arena->name, (char*)p);
- p += ANameSize;
- arena->clumps = U32GET(p);
- p += U32Size;
- arena->cclumps = U32GET(p);
- p += U32Size;
- arena->ctime = U32GET(p);
- p += U32Size;
- arena->wtime = U32GET(p);
- p += U32Size;
- arena->used = U64GET(p);
- p += U64Size;
- arena->uncsize = U64GET(p);
- p += U64Size;
- arena->sealed = U8GET(p);
- p += U8Size;
-
- if(buf + ArenaSize != p)
- sysfatal("unpackarena unpacked wrong amount");
-
- return 0;
-}
-
-int
-packarena(Arena *arena, u8int *buf)
-{
- u8int *p;
- u32int t32;
-
- p = buf;
-
- U32PUT(p, ArenaMagic);
- p += U32Size;
- U32PUT(p, arena->version);
- p += U32Size;
- namecp((char*)p, arena->name);
- p += ANameSize;
- U32PUT(p, arena->clumps);
- p += U32Size;
- U32PUT(p, arena->cclumps);
- p += U32Size;
- U32PUT(p, arena->ctime);
- p += U32Size;
- U32PUT(p, arena->wtime);
- p += U32Size;
- U64PUT(p, arena->used, t32);
- p += U64Size;
- U64PUT(p, arena->uncsize, t32);
- p += U64Size;
- U8PUT(p, arena->sealed);
- p += U8Size;
-
- if(buf + ArenaSize != p)
- sysfatal("packarena packed wrong amount");
-
- return 0;
-}
-
-int
-unpackarenahead(ArenaHead *head, u8int *buf)
-{
- u8int *p;
- u32int m;
- char fbuf[20];
-
- p = buf;
-
- m = U32GET(p);
- if(m != ArenaHeadMagic){
- seterr(ECorrupt, "arena has wrong magic number: %s expected ArenaHeadMagic (%lux)",
- fmtmagic(fbuf, m), ArenaHeadMagic);
- return -1;
- }
- p += U32Size;
- head->version = U32GET(p);
- p += U32Size;
- namecp(head->name, (char*)p);
- p += ANameSize;
- head->blocksize = U32GET(p);
- p += U32Size;
- head->size = U64GET(p);
- p += U64Size;
-
- if(buf + ArenaHeadSize != p)
- sysfatal("unpackarenahead unpacked wrong amount");
-
- return 0;
-}
-
-int
-packarenahead(ArenaHead *head, u8int *buf)
-{
- u8int *p;
- u32int t32;
-
- p = buf;
-
- U32PUT(p, ArenaHeadMagic);
- p += U32Size;
- U32PUT(p, head->version);
- p += U32Size;
- namecp((char*)p, head->name);
- p += ANameSize;
- U32PUT(p, head->blocksize);
- p += U32Size;
- U64PUT(p, head->size, t32);
- p += U64Size;
-
- if(buf + ArenaHeadSize != p)
- sysfatal("packarenahead packed wrong amount");
-
- return 0;
-}
-
-static int
-checkclump(Clump *w)
-{
- if(w->encoding == ClumpENone){
- if(w->info.size != w->info.uncsize){
- seterr(ECorrupt, "uncompressed wad size mismatch");
- return -1;
- }
- }else if(w->encoding == ClumpECompress){
- if(w->info.size >= w->info.uncsize){
- seterr(ECorrupt, "compressed lump has inconsistent block sizes %d %d", w->info.size, w->info.uncsize);
- return -1;
- }
- }else{
- seterr(ECorrupt, "clump has illegal encoding");
- return -1;
- }
-
- return 0;
-}
-
-int
-unpackclump(Clump *c, u8int *buf)
-{
- u8int *p;
- u32int magic;
-
- p = buf;
- magic = U32GET(p);
- if(magic != ClumpMagic){
- seterr(ECorrupt, "clump has bad magic number=%#8.8ux", magic);
- return -1;
- }
- p += U32Size;
-
- c->info.type = vtfromdisktype(U8GET(p));
- p += U8Size;
- c->info.size = U16GET(p);
- p += U16Size;
- c->info.uncsize = U16GET(p);
- p += U16Size;
- scorecp(c->info.score, p);
- p += VtScoreSize;
-
- c->encoding = U8GET(p);
- p += U8Size;
- c->creator = U32GET(p);
- p += U32Size;
- c->time = U32GET(p);
- p += U32Size;
-
- if(buf + ClumpSize != p)
- sysfatal("unpackclump unpacked wrong amount");
-
- return checkclump(c);
-}
-
-int
-packclump(Clump *c, u8int *buf)
-{
- u8int *p;
-
- p = buf;
- U32PUT(p, ClumpMagic);
- p += U32Size;
-
- U8PUT(p, vttodisktype(c->info.type));
- p += U8Size;
- U16PUT(p, c->info.size);
- p += U16Size;
- U16PUT(p, c->info.uncsize);
- p += U16Size;
- scorecp(p, c->info.score);
- p += VtScoreSize;
-
- U8PUT(p, c->encoding);
- p += U8Size;
- U32PUT(p, c->creator);
- p += U32Size;
- U32PUT(p, c->time);
- p += U32Size;
-
- if(buf + ClumpSize != p)
- sysfatal("packclump packed wrong amount");
-
- return checkclump(c);
-}
-
-void
-unpackclumpinfo(ClumpInfo *ci, u8int *buf)
-{
- u8int *p;
-
- p = buf;
- ci->type = vtfromdisktype(U8GET(p));
- p += U8Size;
- ci->size = U16GET(p);
- p += U16Size;
- ci->uncsize = U16GET(p);
- p += U16Size;
- scorecp(ci->score, p);
- p += VtScoreSize;
-
- if(buf + ClumpInfoSize != p)
- sysfatal("unpackclumpinfo unpacked wrong amount");
-}
-
-void
-packclumpinfo(ClumpInfo *ci, u8int *buf)
-{
- u8int *p;
-
- p = buf;
- U8PUT(p, vttodisktype(ci->type));
- p += U8Size;
- U16PUT(p, ci->size);
- p += U16Size;
- U16PUT(p, ci->uncsize);
- p += U16Size;
- scorecp(p, ci->score);
- p += VtScoreSize;
-
- if(buf + ClumpInfoSize != p)
- sysfatal("packclumpinfo packed wrong amount");
-}
-
-int
-unpackisect(ISect *is, u8int *buf)
-{
- u8int *p;
- u32int m;
- char fbuf[20];
-
- p = buf;
-
-
- m = U32GET(p);
- if(m != ISectMagic){
- seterr(ECorrupt, "index section has wrong magic number: %s expected ISectMagic (%lux)",
- fmtmagic(fbuf, m), ISectMagic);
- return -1;
- }
- p += U32Size;
- is->version = U32GET(p);
- p += U32Size;
- namecp(is->name, (char*)p);
- p += ANameSize;
- namecp(is->index, (char*)p);
- p += ANameSize;
- is->blocksize = U32GET(p);
- p += U32Size;
- is->blockbase = U32GET(p);
- p += U32Size;
- is->blocks = U32GET(p);
- p += U32Size;
- is->start = U32GET(p);
- p += U32Size;
- is->stop = U32GET(p);
- p += U32Size;
-
- if(buf + ISectSize != p)
- sysfatal("unpackisect unpacked wrong amount");
-
- return 0;
-}
-
-int
-packisect(ISect *is, u8int *buf)
-{
- u8int *p;
-
- p = buf;
-
- U32PUT(p, ISectMagic);
- p += U32Size;
- U32PUT(p, is->version);
- p += U32Size;
- namecp((char*)p, is->name);
- p += ANameSize;
- namecp((char*)p, is->index);
- p += ANameSize;
- U32PUT(p, is->blocksize);
- p += U32Size;
- U32PUT(p, is->blockbase);
- p += U32Size;
- U32PUT(p, is->blocks);
- p += U32Size;
- U32PUT(p, is->start);
- p += U32Size;
- U32PUT(p, is->stop);
- p += U32Size;
-
- if(buf + ISectSize != p)
- sysfatal("packisect packed wrong amount");
-
- return 0;
-}
-
-void
-unpackientry(IEntry *ie, u8int *buf)
-{
- u8int *p;
-
- p = buf;
-
- scorecp(ie->score, p);
- p += VtScoreSize;
- ie->wtime = U32GET(p);
- p += U32Size;
- ie->train = U16GET(p);
- p += U16Size;
- ie->ia.addr = U64GET(p);
-if(ie->ia.addr>>56) print("%.8H => %llux\n", p, ie->ia.addr);
- p += U64Size;
- ie->ia.size = U16GET(p);
- p += U16Size;
- if(p - buf != IEntryTypeOff)
- sysfatal("unpackientry bad IEntryTypeOff amount");
- ie->ia.type = vtfromdisktype(U8GET(p));
- p += U8Size;
- ie->ia.blocks = U8GET(p);
- p += U8Size;
-
- if(p - buf != IEntrySize)
- sysfatal("unpackientry unpacked wrong amount");
-}
-
-void
-packientry(IEntry *ie, u8int *buf)
-{
- u32int t32;
- u8int *p;
-
- p = buf;
-
- scorecp(p, ie->score);
- p += VtScoreSize;
- U32PUT(p, ie->wtime);
- p += U32Size;
- U16PUT(p, ie->train);
- p += U16Size;
- U64PUT(p, ie->ia.addr, t32);
- p += U64Size;
- U16PUT(p, ie->ia.size);
- p += U16Size;
- U8PUT(p, vttodisktype(ie->ia.type));
- p += U8Size;
- U8PUT(p, ie->ia.blocks);
- p += U8Size;
-
- if(p - buf != IEntrySize)
- sysfatal("packientry packed wrong amount");
-}
-
-void
-unpackibucket(IBucket *b, u8int *buf)
-{
- b->n = U16GET(buf);
- b->depth = U32GET(&buf[U16Size]);
- b->data = buf + IBucketSize;
-}
-
-void
-packibucket(IBucket *b, u8int *buf)
-{
- U16PUT(buf, b->n);
- U32PUT(&buf[U16Size], b->depth);
-}
blob - 8b5bb84d13cb65c00bfa056557b41b26b07d009d (mode 644)
blob + /dev/null
--- src/cmd/venti/copy.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-int fast;
-
-VtConn *zsrc, *zdst;
-
-void
-usage(void)
-{
- fprint(2, "usage: copy src-host dst-host score [type]\n");
- threadexitsall("usage");
-}
-
-int
-parsescore(uchar *score, char *buf, int n)
-{
- int i, c;
-
- memset(score, 0, VtScoreSize);
-
- if(n < VtScoreSize*2)
- return -1;
- for(i=0; i<VtScoreSize*2; i++) {
- if(buf[i] >= '0' && buf[i] <= '9')
- c = buf[i] - '0';
- else if(buf[i] >= 'a' && buf[i] <= 'f')
- c = buf[i] - 'a' + 10;
- else if(buf[i] >= 'A' && buf[i] <= 'F')
- c = buf[i] - 'A' + 10;
- else {
- return -1;
- }
-
- if((i & 1) == 0)
- c <<= 4;
-
- score[i>>1] |= c;
- }
- return 0;
-}
-
-void
-walk(uchar score[VtScoreSize], uint type, int base)
-{
- int i, n, sub;
- uchar *buf;
- VtEntry e;
- VtRoot root;
-
- if(memcmp(score, vtzeroscore, VtScoreSize) == 0)
- return;
-
- buf = vtmallocz(VtMaxLumpSize);
- if(fast && vtread(zdst, score, type, buf, VtMaxLumpSize) >= 0){
- fprint(2, "skip %V\n", score);
- free(buf);
- return;
- }
-
- n = vtread(zsrc, score, type, buf, VtMaxLumpSize);
- if(n < 0){
- fprint(2, "warning: could not read block %V %d: %r", score, type);
- return;
- }
-
- switch(type){
- case VtRootType:
- if(vtrootunpack(&root, buf) < 0){
- fprint(2, "warning: could not unpack root in %V %d\n", score, type);
- break;
- }
- walk(root.score, VtDirType, 0);
- walk(root.prev, VtRootType, 0);
- break;
-
- case VtDirType:
- for(i=0; i<n/VtEntrySize; i++){
- if(vtentryunpack(&e, buf, i) < 0){
- fprint(2, "warning: could not unpack entry #%d in %V %d\n", i, score, type);
- continue;
- }
- if(!(e.flags & VtEntryActive))
- continue;
- if(e.flags&VtEntryDir)
- base = VtDirType;
- else
- base = VtDataType;
- sub = base | ((e.flags&VtEntryDepthMask)>>VtEntryDepthShift);
- walk(e.score, sub, base);
- }
- break;
-
- case VtDataType:
- break;
-
- default: /* pointers */
- for(i=0; i<n; i+=VtScoreSize)
- if(memcmp(buf+i, vtzeroscore, VtScoreSize) != 0)
- walk(buf+i, type-1, base);
- break;
- }
-
- if(vtwrite(zdst, score, type, buf, n) < 0)
- fprint(2, "warning: could not write block %V %d: %r", score, type);
- free(buf);
-}
-
-void
-threadmain(int argc, char *argv[])
-{
- int type, n;
- uchar score[VtScoreSize];
- uchar *buf;
-
- ARGBEGIN{
- case 'f':
- fast = 1;
- break;
- default:
- usage();
- break;
- }ARGEND
-
- if(argc != 3 && argc != 4)
- usage();
-
- fmtinstall('V', vtscorefmt);
-
- if(parsescore(score, argv[2], strlen(argv[2]) < 0))
- sysfatal("could not parse score: %r");
-
- buf = vtmallocz(VtMaxLumpSize);
-
- zsrc = vtdial(argv[0]);
- if(zsrc == nil)
- sysfatal("could not dial src server: %r");
- if(vtconnect(zsrc) < 0)
- sysfatal("vtconnect src: %r");
-
- zdst = vtdial(argv[1]);
- if(zdst == nil)
- sysfatal("could not dial dst server: %r");
- if(vtconnect(zdst) < 0)
- sysfatal("vtconnect dst: %r");
-
- if(argc == 4){
- type = atoi(argv[3]);
- n = vtread(zsrc, score, type, buf, VtMaxLumpSize);
- if(n < 0)
- sysfatal("could not read block: %r");
- }else{
- for(type=0; type<VtMaxType; type++){
- n = vtread(zsrc, score, type, buf, VtMaxLumpSize);
- if(n >= 0)
- break;
- }
- if(type == VtMaxType)
- sysfatal("could not find block %V of any type", score);
- }
-
- walk(score, type, VtDirType);
-
- if(vtsync(zdst) < 0)
- sysfatal("could not sync dst server: %r");
-
- threadexitsall(0);
-}
blob - 98c935ed16e8c8f1d572d47aec932d7c9091d60b (mode 644)
blob + /dev/null
--- src/cmd/venti/dat.h
+++ /dev/null
-typedef struct Config Config;
-typedef struct AMap AMap;
-typedef struct AMapN AMapN;
-typedef struct Arena Arena;
-typedef struct ArenaHead ArenaHead;
-typedef struct ArenaPart ArenaPart;
-typedef struct CIBlock CIBlock;
-typedef struct Clump Clump;
-typedef struct ClumpInfo ClumpInfo;
-typedef struct IAddr IAddr;
-typedef struct IBucket IBucket;
-typedef struct ICache ICache;
-typedef struct IEStream IEStream;
-typedef struct IEntry IEntry;
-typedef struct IFile IFile;
-typedef struct ISect ISect;
-typedef struct Index Index;
-typedef struct Lump Lump;
-typedef struct DBlock DBlock;
-typedef struct Part Part;
-typedef struct Stats Stats;
-typedef struct ZBlock ZBlock;
-
-#define TWID32 ((u32int)~(u32int)0)
-#define TWID64 ((u64int)~(u64int)0)
-#define TWID8 ((u8int)~(u8int)0)
-
-enum
-{
- ABlockLog = 9, /* log2(512), the quantum for reading arenas */
- ANameSize = 64,
- MaxDiskBlock = 64*1024, /* max. allowed size for a disk block */
- MaxIoSize = 64*1024, /* max. allowed size for a disk io operation */
- PartBlank = 256*1024, /* untouched section at beginning of partition */
- HeadSize = 512, /* size of a header after PartBlank */
- MinArenaSize = 1*1024*1024, /* smallest reasonable arena size */
- IndexBase = 1024*1024, /* initial address to use in an index */
- MaxIo = 64*1024, /* max size of a single read or write operation */
- ICacheBits = 16, /* default bits for indexing icache */
- ICacheDepth = 4, /* default depth of an icache hash chain */
- MaxAMap = 2*1024, /* max. allowed arenas in an address mapping; must be < 32*1024 */
-
- /*
- * return codes from syncarena
- */
- SyncDataErr = 1 << 0, /* problem reading the clump data */
- SyncCIErr = 1 << 1, /* found erroneous clump directory entries */
- SyncCIZero = 1 << 2, /* found unwritten clump directory entries */
- SyncFixErr = 1 << 3, /* error writing fixed data */
- SyncHeader = 1 << 4, /* altered header fields */
-
- /*
- * error severity
- */
- EOk = 0, /* error expected in normal operation */
- EStrange, /* strange error that should be logged */
- ECorrupt, /* corrupted data found in arenas */
- EICorrupt, /* corrupted data found in index */
- EAdmin, /* should be brought to administrators' attention */
- ECrash, /* really bad internal error */
- EBug, /* a limitation which should be fixed */
- EInconsist, /* inconsistencies between index and arena */
- EMax,
-
- /*
- * internal disk formats for the venti archival storage system
- */
- /*
- * magic numbers on disk
- */
- ClumpMagic = 0xd15cb10c, /* clump header */
- ClumpFreeMagic = 0, /* free clump; terminates active clump log */
-
- ArenaPartMagic = 0xa9e4a5e7, /* arena partition header */
- ArenaMagic = 0xf2a14ead, /* arena trailer */
- ArenaHeadMagic = 0xd15c4ead, /* arena header */
-
- ISectMagic = 0xd15c5ec7, /* index header */
-
- ArenaPartVersion = 3,
- ArenaVersion = 4,
- IndexVersion1 = 1,
- IndexVersion2 = 2,
- ISectVersion = 1,
-
- /*
- * encodings of clumps on disk
- */
- ClumpEErr = 0, /* can't happen */
- ClumpENone, /* plain */
- ClumpECompress, /* compressed */
- ClumpEMax,
-
- /*
- * marker for corrupted data on disk
- */
- VtTypeCorrupt = VtMaxType,
-
- /*
- * sizes in bytes on disk
- */
- U8Size = 1,
- U16Size = 2,
- U32Size = 4,
- U64Size = 8,
-
- ArenaPartSize = 4 * U32Size,
- ArenaSize = 2 * U64Size + 6 * U32Size + ANameSize + U8Size,
- ArenaHeadSize = U64Size + 3 * U32Size + ANameSize,
- ISectSize = 7 * U32Size + 2 * ANameSize,
- ClumpInfoSize = U8Size + 2 * U16Size + VtScoreSize,
- ClumpSize = ClumpInfoSize + U8Size + 3 * U32Size,
- IBucketSize = U32Size + U16Size,
- IEntrySize = U64Size + U32Size + 2*U16Size + 2*U8Size + VtScoreSize,
- IEntryTypeOff = VtScoreSize + U64Size + U32Size + 2 * U16Size,
-
- MaxClumpBlocks = (VtMaxLumpSize + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog,
-
- /*
- * dirty flags - order controls disk write order
- */
- DirtyArena = 1,
- DirtyIndexSplit,
- DirtyIndex,
- DirtyIndexBitmap,
- DirtyArenaCib,
- DirtyArenaTrailer,
- DirtyMax,
-
- VentiZZZZZZZZ
-};
-
-/*
- * results of parsing and initializing a config file
- */
-struct Config
-{
- char *index; /* name of the index to initialize */
- int naparts; /* arena partitions initialized */
- ArenaPart **aparts;
- int nsects; /* index sections initialized */
- ISect **sects;
-};
-
-/*
- * a Part is the low level interface to files or disks.
- * there are two main types of partitions
- * arena paritions, which some number of arenas, each in a sub-partition.
- * index partition, which only have one subpartition.
- */
-struct Part
-{
- int fd; /* rock for accessing the disk */
- u64int size; /* size of the partiton */
- u32int blocksize; /* block size for reads and writes */
- char *name;
- Channel *writechan; /* chan[dcache.nblock](DBlock*) */
-};
-
-/*
- * a cached block from the partition
- * yuck -- most of this is internal structure for the cache
- * all other routines should only use data
- */
-struct DBlock
-{
- u8int *data;
-
- Part *part; /* partition in which cached */
- u64int addr; /* base address on the partition */
- u16int size; /* amount of data available, not amount allocated; should go away */
- u32int dirty;
- u32int dirtying;
- DBlock *next; /* doubly linked hash chains */
- DBlock *prev;
- u32int heap; /* index in heap table */
- u32int used; /* last reference times */
- u32int used2;
- u32int ref; /* reference count */
- QLock lock; /* for access to data only */
- Channel writedonechan;
- void* chanbuf[1]; /* buffer for the chan! */
-};
-
-/*
- * a cached block from the partition
- * yuck -- most of this is internal structure for the cache
- * all other routines should only use data
- * double yuck -- this is mostly the same as a DBlock
- */
-struct Lump
-{
- Packet *data;
-
- Part *part; /* partition in which cached */
- u8int score[VtScoreSize]; /* score of packet */
- u8int type; /* type of packet */
- u16int size; /* amount of data allocated to hold packet */
- Lump *next; /* doubly linked hash chains */
- Lump *prev;
- u32int heap; /* index in heap table */
- u32int used; /* last reference times */
- u32int used2;
- u32int ref; /* reference count */
- QLock lock; /* for access to data only */
-};
-
-/*
- * mapping between names and address ranges
- */
-struct AMap
-{
- u64int start;
- u64int stop;
- char name[ANameSize];
-};
-
-/*
- * an AMap along with a length
- */
-struct AMapN
-{
- int n;
- AMap *map;
-};
-
-/*
- * an ArenaPart is a partition made up of Arenas
- * it exists because most os's don't support many partitions,
- * and we want to have many different Arenas
- */
-struct ArenaPart
-{
- Part *part;
- u64int size; /* size of underlying partition, rounded down to blocks */
- Arena **arenas;
- u32int tabbase; /* base address of arena table on disk */
- u32int tabsize; /* max. bytes in arena table */
-
- /*
- * fields stored on disk
- */
- u32int version;
- u32int blocksize; /* "optimal" block size for reads and writes */
- u32int arenabase; /* base address of first arena */
-
- /*
- * stored in the arena mapping table on disk
- */
- AMap *map;
- int narenas;
-};
-
-/*
- * info about one block in the clump info cache
- */
-struct CIBlock
-{
- u32int block; /* blocks in the directory */
- int offset; /* offsets of one clump in the data */
- DBlock *data;
-};
-
-/*
- * an Arena is a log of Clumps, preceeded by an ArenaHeader,
- * and followed by a Arena, each in one disk block.
- * struct on disk is not always up to date, but should be self-consistent.
- * to sync after reboot, follow clumps starting at used until ClumpFreeMagic if found.
- * <struct name="Arena" type="Arena *">
- * <field name="name" val="s->name" type="AName"/>
- * <field name="version" val="s->version" type="U32int"/>
- * <field name="partition" val="s->part->name" type="AName"/>
- * <field name="blocksize" val="s->blocksize" type="U32int"/>
- * <field name="start" val="s->base" type="U64int"/>
- * <field name="stop" val="s->base+2*s->blocksize" type="U64int"/>
- * <field name="created" val="s->ctime" type="U32int"/>
- * <field name="modified" val="s->wtime" type="U32int"/>
- * <field name="sealed" val="s->sealed" type="Sealed"/>
- * <field name="score" val="s->score" type="Score"/>
- * <field name="clumps" val="s->clumps" type="U32int"/>
- * <field name="compressedclumps" val="s->cclumps" type="U32int"/>
- * <field name="data" val="s->uncsize" type="U64int"/>
- * <field name="compresseddata" val="s->used - s->clumps * ClumpSize" type="U64int"/>
- * <field name="storage" val="s->used + s->clumps * ClumpInfoSize" type="U64int"/>
- * </struct>
- */
-struct Arena
-{
- QLock lock; /* lock for arena fields, writing to disk */
- Part *part; /* partition in which arena lives */
- int blocksize; /* size of block to read or write */
- u64int base; /* base address on disk */
- u64int size; /* total space in the arena */
- u64int limit; /* storage limit for clumps */
- u8int score[VtScoreSize]; /* score of the entire sealed & summed arena */
-
- int clumpmax; /* ClumpInfos per block */
- CIBlock cib; /* dirty clump directory block */
-
- /*
- * fields stored on disk
- */
- u32int version;
- char name[ANameSize]; /* text label */
- u32int clumps; /* number of allocated clumps */
- u32int cclumps; /* clumps which are compressed; informational only */
- u32int ctime; /* first time a block was written */
- u32int wtime; /* last time a block was written */
- u64int used; /* number of bytes currently used */
- u64int uncsize; /* total of all clumps's uncsize; informational only */
- u8int sealed; /* arena all filled up? */
-};
-
-/*
- * redundant storage of some fields at the beginning of each arena
- */
-struct ArenaHead
-{
- u32int version;
- char name[ANameSize];
- u32int blocksize;
- u64int size;
-};
-
-/*
- * most interesting meta information for a clump.
- * stored in each clump's header and in the Arena's directory,
- * stored in reverse order just prior to the arena trailer
- */
-struct ClumpInfo
-{
- u8int type;
- u16int size; /* size of disk data, not including header */
- u16int uncsize; /* size of uncompressed data */
- u8int score[VtScoreSize]; /* score of the uncompressed data only */
-};
-
-/*
- * header for an immutable clump of data
- */
-struct Clump
-{
- ClumpInfo info;
- u8int encoding;
- u32int creator; /* initial client which wrote the block */
- u32int time; /* creation at gmt seconds since 1/1/1970 */
-};
-
-/*
- * index of all clumps according to their score
- * this is just a wrapper to tie together the index sections
- * <struct name="Index" type="Index *">
- * <field name="name" val="s->name" type="AName"/>
- * <field name="version" val="s->version" type="U32int"/>
- * <field name="blocksize" val="s->blocksize" type="U32int"/>
- * <field name="tabsize" val="s->tabsize" type="U32int"/>
- * <field name="buckets" val="s->buckets" type="U32int"/>
- * <field name="buckdiv" val="s->div" type="U32int"/>
- * <field name="bitblocks" val="s->div" type="U32int"/>
- * <field name="maxdepth" val="s->div" type="U32int"/>
- * <field name="bitkeylog" val="s->div" type="U32int"/>
- * <field name="bitkeymask" val="s->div" type="U32int"/>
- * <array name="sect" val="&s->smap[i]" elems="s->nsects" type="Amap"/>
- * <array name="amap" val="&s->amap[i]" elems="s->narenas" type="Amap"/>
- * <array name="arena" val="s->arenas[i]" elems="s->narenas" type="Arena"/>
- * </struct>
- * <struct name="Amap" type="AMap *">
- * <field name="name" val="s->name" type="AName"/>
- * <field name="start" val="s->start" type="U64int"/>
- * <field name="stop" val="s->stop" type="U64int"/>
- * </struct>
- */
-struct Index
-{
- u32int div; /* divisor for mapping score to bucket */
- u32int buckets; /* last bucket used in disk hash table */
- u32int blocksize;
- u32int tabsize; /* max. bytes in index config */
- u32int bitblocks;
- u32int maxdepth;
- u32int bitkeylog;
- u32int bitkeymask;
-
- int mapalloc; /* first arena to check when adding a lump */
- Arena **arenas; /* arenas in the mapping */
- ISect **sects; /* sections which hold the buckets */
-
- /*
- * fields stored in config file
- */
- u32int version;
- char name[ANameSize]; /* text label */
- int nsects;
- AMap *smap; /* mapping of buckets to index sections */
- int narenas;
- AMap *amap; /* mapping from index addesses to arenas */
-};
-
-/*
- * one part of the bucket storage for an index.
- * the index blocks are sequentially allocated
- * across all of the sections.
- */
-struct ISect
-{
- Part *part;
- int blocklog; /* log2(blocksize) */
- int buckmax; /* max. entries in a index bucket */
- u32int tabbase; /* base address of index config table on disk */
- u32int tabsize; /* max. bytes in index config */
-
- /*
- * fields stored on disk
- */
- u32int version;
- char name[ANameSize]; /* text label */
- char index[ANameSize]; /* index owning the section */
- u32int blocksize; /* size of hash buckets in index */
- u32int blockbase; /* address of start of on disk index table */
- u32int blocks; /* total blocks on disk; some may be unused */
- u32int start; /* first bucket in this section */
- u32int stop; /* limit of buckets in this section */
-};
-
-/*
- * externally interesting part of an IEntry
- */
-struct IAddr
-{
- u64int addr;
- u16int size; /* uncompressed size */
- u8int type; /* type of block */
- u8int blocks; /* arena io quanta for Clump + data */
-};
-
-/*
- * entries in the index
- * kept in IBuckets in the disk index table,
- * cached in the memory ICache.
- */
-struct IEntry
-{
- u8int score[VtScoreSize];
- IEntry *next; /* next in hash chain */
- u32int wtime; /* last write time */
- u16int train; /* relative train containing the most recent ref; 0 if no ref, 1 if in same car */
- u8int rac; /* read ahead count */
- IAddr ia;
-};
-
-/*
- * buckets in the on disk index table
- */
-struct IBucket
-{
- u16int n; /* number of active indices */
- u32int depth; /* depth in version 2 (was overflow in v1) */
- u8int *data;
-};
-
-/*
- * temporary buffers used by individual threads
- */
-struct ZBlock
-{
- u32int len;
- u8int *data;
-};
-
-/*
- * simple input buffer for a '\0' terminated text file
- */
-struct IFile
-{
- char *name; /* name of the file */
- ZBlock *b; /* entire contents of file */
- u32int pos; /* current position in the file */
-};
-
-/*
- * statistics about the operation of the server
- * mainly for performance monitoring and profiling.
- */
-struct Stats
-{
- QLock lock;
- long lumpwrites; /* protocol block writes */
- long lumpreads; /* protocol block reads */
- long lumphit; /* lump cache hit */
- long lumpmiss; /* lump cache miss */
- long clumpwrites; /* clumps to disk */
- vlong clumpbwrites; /* clump data bytes to disk */
- vlong clumpbcomp; /* clump bytes compressed */
- long clumpreads; /* clumps from disk */
- vlong clumpbreads; /* clump data bytes from disk */
- vlong clumpbuncomp; /* clump bytes uncompressed */
- long ciwrites; /* clump directory to disk */
- long cireads; /* clump directory from disk */
- long indexwrites; /* index to disk */
- long indexreads; /* index from disk */
- long indexwreads; /* for writing a new entry */
- long indexareads; /* for allocating an overflow block */
- long indexsplits; /* index block splits */
- long diskwrites; /* total disk writes */
- long diskreads; /* total disk reads */
- vlong diskbwrites; /* total disk bytes written */
- vlong diskbreads; /* total disk bytes read */
- long pchit; /* partition cache hit */
- long pcmiss; /* partition cache miss */
- long pcreads; /* partition cache reads from disk */
- vlong pcbreads; /* partition cache bytes read */
- long icinserts; /* stores into index cache */
- long iclookups; /* index cache lookups */
- long ichits; /* hits in the cache */
- long icfills; /* successful fills from index */
- long absorbedwrites; /* disk writes absorbed by dcache */
- long dirtydblocks; /* blocks dirtied */
- long dcacheflushes; /* times dcache has flushed */
- long dcacheflushwrites; /* blocks written by those flushes */
-};
-
-extern Index *mainindex;
-extern u32int maxblocksize; /* max. block size used by any partition */
-extern int paranoid; /* should verify hashes on disk read */
-extern int queuewrites; /* put all lump writes on a queue and finish later */
-extern int readonly; /* only allowed to read the disk data */
-extern Stats stats;
-extern u8int zeroscore[VtScoreSize];
blob - 7e63323249a624bbbc2cc2aca15111c001bb6f07 (mode 644)
blob + /dev/null
--- src/cmd/venti/dcache.c
+++ /dev/null
-/*
- * The locking here is getting a little out of hand.
- */
-
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-typedef struct DCache DCache;
-
-enum
-{
- HashLog = 9,
- HashSize = 1<<HashLog,
- HashMask = HashSize - 1,
-};
-
-struct DCache
-{
- QLock lock;
- RWLock dirtylock; /* must be held to inspect or set b->dirty */
- u32int flushround;
- Rendez anydirty;
- Rendez full;
- Rendez flush;
- Rendez flushdone;
- DBlock *free; /* list of available lumps */
- u32int now; /* ticks for usage timestamps */
- int size; /* max. size of any block; allocated to each block */
- DBlock **heads; /* hash table for finding address */
- int nheap; /* number of available victims */
- DBlock **heap; /* heap for locating victims */
- int nblocks; /* number of blocks allocated */
- DBlock *blocks; /* array of block descriptors */
- DBlock **write; /* array of block pointers to be written */
- u8int *mem; /* memory for all block descriptors */
- int ndirty; /* number of dirty blocks */
- int maxdirty; /* max. number of dirty blocks */
-};
-
-static DCache dcache;
-
-static int downheap(int i, DBlock *b);
-static int upheap(int i, DBlock *b);
-static DBlock *bumpdblock(void);
-static void delheap(DBlock *db);
-static void fixheap(int i, DBlock *b);
-static void _flushdcache(void);
-static void flushproc(void*);
-static void flushtimerproc(void*);
-static void writeproc(void*);
-
-void
-initdcache(u32int mem)
-{
- DBlock *b, *last;
- u32int nblocks, blocksize;
- int i;
-
- if(mem < maxblocksize * 2)
- sysfatal("need at least %d bytes for the disk cache", maxblocksize * 2);
- if(maxblocksize == 0)
- sysfatal("no max. block size given for disk cache");
- blocksize = maxblocksize;
- nblocks = mem / blocksize;
- dcache.full.l = &dcache.lock;
- dcache.flush.l = &dcache.lock;
- dcache.anydirty.l = &dcache.lock;
- dcache.flushdone.l = &dcache.lock;
- dcache.nblocks = nblocks;
- dcache.maxdirty = (nblocks * 3) / 4;
- if(1)
- fprint(2, "initialize disk cache with %d blocks of %d bytes, maximum %d dirty blocks\n",
- nblocks, blocksize, dcache.maxdirty);
- dcache.size = blocksize;
- dcache.heads = MKNZ(DBlock*, HashSize);
- dcache.heap = MKNZ(DBlock*, nblocks);
- dcache.blocks = MKNZ(DBlock, nblocks);
- dcache.write = MKNZ(DBlock*, nblocks);
- dcache.mem = MKNZ(u8int, nblocks * blocksize);
-
- last = nil;
- for(i = 0; i < nblocks; i++){
- b = &dcache.blocks[i];
- b->data = &dcache.mem[i * blocksize];
- b->heap = TWID32;
- chaninit(&b->writedonechan, sizeof(void*), 1);
- b->next = last;
- last = b;
- }
- dcache.free = last;
- dcache.nheap = 0;
-
- vtproc(flushproc, nil);
- vtproc(flushtimerproc, nil);
-}
-
-static u32int
-pbhash(u64int addr)
-{
- u32int h;
-
-#define hashit(c) ((((c) * 0x6b43a9b5) >> (32 - HashLog)) & HashMask)
- h = (addr >> 32) ^ addr;
- return hashit(h);
-}
-
-DBlock*
-getdblock(Part *part, u64int addr, int read)
-{
- DBlock *b;
- u32int h, size;
-
- size = part->blocksize;
- if(size > dcache.size){
- seterr(EAdmin, "block size %d too big for cache", size);
- return nil;
- }
- h = pbhash(addr);
-
- /*
- * look for the block in the cache
- */
-//checkdcache();
- qlock(&dcache.lock);
-again:
- for(b = dcache.heads[h]; b != nil; b = b->next){
- if(b->part == part && b->addr == addr){
- qlock(&stats.lock);
- stats.pchit++;
- qunlock(&stats.lock);
- goto found;
- }
- }
- qlock(&stats.lock);
- stats.pcmiss++;
- qunlock(&stats.lock);
-
- /*
- * missed: locate the block with the oldest second to last use.
- * remove it from the heap, and fix up the heap.
- */
- b = bumpdblock();
- if(b == nil){
- logerr(EAdmin, "all disk cache blocks in use");
- rsleep(&dcache.full);
- goto again;
- }
-
- /*
- * the new block has no last use, so assume it happens sometime in the middle
-ZZZ this is not reasonable
- */
- b->used = (b->used2 + dcache.now) / 2;
-
- /*
- * rechain the block on the correct hash chain
- */
- b->next = dcache.heads[h];
- dcache.heads[h] = b;
- if(b->next != nil)
- b->next->prev = b;
- b->prev = nil;
-
- b->addr = addr;
- b->part = part;
- b->size = 0;
-
-found:
- b->ref++;
- b->used2 = b->used;
- b->used = dcache.now++;
- if(b->heap != TWID32)
- fixheap(b->heap, b);
-
- qunlock(&dcache.lock);
-//checkdcache();
-
- qlock(&b->lock);
- if(b->size != size){
- if(b->size < size){
- if(!read)
- memset(&b->data[b->size], 0, size - b->size);
- else{
- if(readpart(part, addr + b->size, &b->data[b->size], size - b->size) < 0){
- putdblock(b);
- return nil;
- }
- qlock(&stats.lock);
- stats.pcreads++;
- stats.pcbreads += size - b->size;
- qunlock(&stats.lock);
- }
- }
- b->size = size;
- }
-
- return b;
-}
-
-void
-putdblock(DBlock *b)
-{
- if(b == nil)
- return;
-
- if(b->dirtying){
- b->dirtying = 0;
- runlock(&dcache.dirtylock);
- }
- qunlock(&b->lock);
-
-//checkdcache();
- qlock(&dcache.lock);
- if(b->dirty)
- delheap(b);
- else if(--b->ref == 0){
- if(b->heap == TWID32)
- upheap(dcache.nheap++, b);
- rwakeupall(&dcache.full);
- }
-
- qunlock(&dcache.lock);
-//checkdcache();
-}
-
-void
-dirtydblock(DBlock *b, int dirty)
-{
- int odirty;
- Part *p;
-
-
- assert(b->ref != 0);
-
- /*
- * Because we use an rlock to keep track of how
- * many blocks are being dirtied (and a wlock to
- * stop dirtying), we cannot try to dirty two blocks
- * at the same time in the same thread -- if the
- * wlock happens between them, we get a deadlock.
- *
- * The only place in the code where we need to
- * dirty multiple blocks at once is in splitiblock, when
- * we split an index block. The new block has a dirty
- * flag of DirtyIndexSplit already, because it has to get
- * written to disk before the old block (DirtyIndex).
- * So when we see the DirtyIndexSplit block, we don't
- * acquire the rlock (and we don't set dirtying, so putdblock
- * won't drop the rlock). This kludginess means that
- * the caller will have to be sure to putdblock on the
- * new block before putdblock on the old block.
- */
- if(dirty == DirtyIndexSplit){
- /* caller should have another dirty block */
- assert(!canwlock(&dcache.dirtylock));
- /* this block should be clean */
- assert(b->dirtying == 0);
- assert(b->dirty == 0);
- }else{
- rlock(&dcache.dirtylock);
- assert(b->dirtying == 0);
- b->dirtying = 1;
- }
-
- qlock(&stats.lock);
- if(b->dirty)
- stats.absorbedwrites++;
- stats.dirtydblocks++;
- qunlock(&stats.lock);
-
- /*
- * In general, shouldn't mark any block as more than one
- * type, except that split index blocks are a subcase of index
- * blocks. Only clean blocks ever get marked DirtyIndexSplit,
- * though, so we don't need the opposite conjunction here.
- */
- odirty = b->dirty;
- if(b->dirty)
- assert(b->dirty == dirty
- || (b->dirty==DirtyIndexSplit && dirty==DirtyIndex));
- else
- b->dirty = dirty;
-
- p = b->part;
- if(p->writechan == nil){
-fprint(2, "allocate write proc for part %s\n", p->name);
- /* XXX hope this doesn't fail! */
- p->writechan = chancreate(sizeof(DBlock*), dcache.nblocks);
- vtproc(writeproc, p);
- }
- qlock(&dcache.lock);
- if(!odirty){
- dcache.ndirty++;
- rwakeupall(&dcache.anydirty);
- }
- qunlock(&dcache.lock);
-}
-
-/*
- * remove some block from use and update the free list and counters
- */
-static DBlock*
-bumpdblock(void)
-{
- int flushed;
- DBlock *b;
- ulong h;
-
- b = dcache.free;
- if(b != nil){
- dcache.free = b->next;
- return b;
- }
-
- if(dcache.ndirty >= dcache.maxdirty)
- _flushdcache();
-
- /*
- * remove blocks until we find one that is unused
- * referenced blocks are left in the heap even though
- * they can't be scavenged; this is simple a speed optimization
- */
- flushed = 0;
- for(;;){
- if(dcache.nheap == 0){
- _flushdcache();
- return nil;
- }
- b = dcache.heap[0];
- delheap(b);
- if(!b->ref)
- break;
- }
-
- fprint(2, "bump %s at %llud\n", b->part->name, b->addr);
-
- /*
- * unchain the block
- */
- if(b->prev == nil){
- h = pbhash(b->addr);
- if(dcache.heads[h] != b)
- sysfatal("bad hash chains in disk cache");
- dcache.heads[h] = b->next;
- }else
- b->prev->next = b->next;
- if(b->next != nil)
- b->next->prev = b->prev;
-
- return b;
-}
-
-/*
- * delete an arbitrary block from the heap
- */
-static void
-delheap(DBlock *db)
-{
- if(db->heap == TWID32)
- return;
- fixheap(db->heap, dcache.heap[--dcache.nheap]);
- db->heap = TWID32;
-}
-
-/*
- * push an element up or down to it's correct new location
- */
-static void
-fixheap(int i, DBlock *b)
-{
- if(upheap(i, b) == i)
- downheap(i, b);
-}
-
-static int
-upheap(int i, DBlock *b)
-{
- DBlock *bb;
- u32int now;
- int p;
-
- now = dcache.now;
- for(; i != 0; i = p){
- p = (i - 1) >> 1;
- bb = dcache.heap[p];
- if(b->used2 - now >= bb->used2 - now)
- break;
- dcache.heap[i] = bb;
- bb->heap = i;
- }
-
- dcache.heap[i] = b;
- b->heap = i;
- return i;
-}
-
-static int
-downheap(int i, DBlock *b)
-{
- DBlock *bb;
- u32int now;
- int k;
-
- now = dcache.now;
- for(; ; i = k){
- k = (i << 1) + 1;
- if(k >= dcache.nheap)
- break;
- if(k + 1 < dcache.nheap && dcache.heap[k]->used2 - now > dcache.heap[k + 1]->used2 - now)
- k++;
- bb = dcache.heap[k];
- if(b->used2 - now <= bb->used2 - now)
- break;
- dcache.heap[i] = bb;
- bb->heap = i;
- }
-
- dcache.heap[i] = b;
- b->heap = i;
- return i;
-}
-
-static void
-findblock(DBlock *bb)
-{
- DBlock *b, *last;
- int h;
-
- last = nil;
- h = pbhash(bb->addr);
- for(b = dcache.heads[h]; b != nil; b = b->next){
- if(last != b->prev)
- sysfatal("bad prev link");
- if(b == bb)
- return;
- last = b;
- }
- sysfatal("block missing from hash table");
-}
-
-void
-checkdcache(void)
-{
- DBlock *b;
- u32int size, now;
- int i, k, refed, nfree;
-
- qlock(&dcache.lock);
- size = dcache.size;
- now = dcache.now;
- for(i = 0; i < dcache.nheap; i++){
- if(dcache.heap[i]->heap != i)
- sysfatal("dc: mis-heaped at %d: %d", i, dcache.heap[i]->heap);
- if(i > 0 && dcache.heap[(i - 1) >> 1]->used2 - now > dcache.heap[i]->used2 - now)
- sysfatal("dc: bad heap ordering");
- k = (i << 1) + 1;
- if(k < dcache.nheap && dcache.heap[i]->used2 - now > dcache.heap[k]->used2 - now)
- sysfatal("dc: bad heap ordering");
- k++;
- if(k < dcache.nheap && dcache.heap[i]->used2 - now > dcache.heap[k]->used2 - now)
- sysfatal("dc: bad heap ordering");
- }
-
- refed = 0;
- for(i = 0; i < dcache.nblocks; i++){
- b = &dcache.blocks[i];
- if(b->data != &dcache.mem[i * size])
- sysfatal("dc: mis-blocked at %d", i);
- if(b->ref && b->heap == TWID32)
- refed++;
- if(b->addr)
- findblock(b);
- if(b->heap != TWID32
- && dcache.heap[b->heap] != b)
- sysfatal("dc: spurious heap value");
- }
-
- nfree = 0;
- for(b = dcache.free; b != nil; b = b->next){
- if(b->addr != 0 || b->heap != TWID32)
- sysfatal("dc: bad free list");
- nfree++;
- }
-
- if(dcache.nheap + nfree + refed != dcache.nblocks)
- sysfatal("dc: missing blocks: %d %d %d", dcache.nheap, refed, dcache.nblocks);
- qunlock(&dcache.lock);
-}
-
-void
-flushdcache(void)
-{
- u32int flushround;
-
- qlock(&dcache.lock);
- flushround = dcache.flushround;
- rwakeupall(&dcache.flush);
- while(flushround == dcache.flushround)
- rsleep(&dcache.flushdone);
- qunlock(&dcache.lock);
-}
-
-static void
-_flushdcache(void)
-{
- rwakeupall(&dcache.flush);
-}
-
-static int
-parallelwrites(DBlock **b, DBlock **eb, int dirty)
-{
- DBlock **p;
-
- for(p=b; p<eb && (*p)->dirty == dirty; p++)
- sendp((*p)->part->writechan, *p);
- for(p=b; p<eb && (*p)->dirty == dirty; p++)
- recvp(&(*p)->writedonechan);
-
- return p-b;
-}
-
-/*
- * Sort first by dirty flag, then by partition, then by address in partition.
- */
-static int
-writeblockcmp(const void *va, const void *vb)
-{
- DBlock *a, *b;
-
- a = *(DBlock**)va;
- b = *(DBlock**)vb;
-
- if(a->dirty != b->dirty)
- return a->dirty - b->dirty;
- if(a->part != b->part){
- if(a->part < b->part)
- return -1;
- if(a->part > b->part)
- return 1;
- }
- if(a->addr < b->addr)
- return -1;
- return 1;
-}
-
-static void
-flushtimerproc(void *v)
-{
- u32int round;
-
- for(;;){
- qlock(&dcache.lock);
- while(dcache.ndirty == 0)
- rsleep(&dcache.anydirty);
- round = dcache.flushround;
- qunlock(&dcache.lock);
-
- sleep(60*1000);
-
- qlock(&dcache.lock);
- if(round == dcache.flushround){
- rwakeupall(&dcache.flush);
- while(round == dcache.flushround)
- rsleep(&dcache.flushdone);
- }
- qunlock(&dcache.lock);
- }
-}
-
-static void
-flushproc(void *v)
-{
- int i, j, n;
- ulong t0;
- DBlock *b, **write;
-
- USED(v);
- for(;;){
- qlock(&dcache.lock);
- dcache.flushround++;
- rwakeupall(&dcache.flushdone);
- rsleep(&dcache.flush);
- qunlock(&dcache.lock);
-
- fprint(2, "flushing dcache: t=0 ms\n");
- t0 = nsec()/1000;
-
- /*
- * Because we don't record any dependencies at all, we must write out
- * all blocks currently dirty. Thus we must lock all the blocks that
- * are currently dirty.
- *
- * We grab dirtylock to stop the dirtying of new blocks.
- * Then we wait until all the current blocks finish being dirtied.
- * Now all the dirty blocks in the system are immutable (clean blocks
- * might still get recycled), so we can plan our disk writes.
- *
- * In a better scheme, dirtiers might lock the block for writing in getdblock,
- * so that flushproc could lock all the blocks here and then unlock them as it
- * finishes with them.
- */
-
- fprint(2, "flushproc: wlock at t=%lud\n", (ulong)(nsec()/1000) - t0);
- wlock(&dcache.dirtylock);
-
- fprint(2, "flushproc: build list at t=%lud\n", (ulong)(nsec()/1000) - t0);
- write = dcache.write;
- n = 0;
- for(i=0; i<dcache.nblocks; i++){
- b = &dcache.blocks[i];
- if(b->dirty)
- write[n++] = b;
- }
-
- qsort(write, n, sizeof(write[0]), writeblockcmp);
-
- /* Write each stage of blocks out. */
- fprint(2, "flushproc: write blocks at t=%lud\n", (ulong)(nsec()/1000) - t0);
- i = 0;
- for(j=1; j<DirtyMax; j++){
- fprint(2, "flushproc: flush dirty %d at t=%lud\n", j, (ulong)(nsec()/1000) - t0);
- i += parallelwrites(write+i, write+n, j);
- }
- assert(i == n);
-
- fprint(2, "flushproc: update dirty bits at t=%lud\n", (ulong)(nsec()/1000) - t0);
- qlock(&dcache.lock);
- for(i=0; i<n; i++){
- b = write[i];
- b->dirty = 0;
- --dcache.ndirty;
- if(b->ref == 0 && b->heap == TWID32){
- upheap(dcache.nheap++, b);
- rwakeupall(&dcache.full);
- }
- }
- qunlock(&dcache.lock);
- wunlock(&dcache.dirtylock);
-
- fprint(2, "flushproc: done at t=%lud\n", (ulong)(nsec()/1000) - t0);
-
- qlock(&stats.lock);
- stats.dcacheflushes++;
- stats.dcacheflushwrites += n;
- qunlock(&stats.lock);
- }
-}
-
-static void
-writeproc(void *v)
-{
- DBlock *b;
- Part *p;
-
- p = v;
-
- for(;;){
- b = recvp(p->writechan);
- if(writepart(p, b->addr, b->data, b->size) < 0)
- fprint(2, "write error: %r\n"); /* XXX details! */
- sendp(&b->writedonechan, b);
- }
-}
blob - 12fcddda3347824d45ae6392559e82f0217978f0 (mode 644)
blob + /dev/null
--- src/cmd/venti/dump.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-void
-printindex(int fd, Index *ix)
-{
- int i;
-
- fprint(fd, "index=%s version=%d blocksize=%d tabsize=%d\n",
- ix->name, ix->version, ix->blocksize, ix->tabsize);
- fprint(fd, "\tbuckets=%d div=%d\n", ix->buckets, ix->div);
- for(i = 0; i < ix->nsects; i++)
- fprint(fd, "\tsect=%s for buckets [%lld,%lld)\n", ix->smap[i].name, ix->smap[i].start, ix->smap[i].stop);
- for(i = 0; i < ix->narenas; i++)
- fprint(fd, "\tarena=%s at [%lld,%lld)\n", ix->amap[i].name, ix->amap[i].start, ix->amap[i].stop);
-}
-
-void
-printarenapart(int fd, ArenaPart *ap)
-{
- int i;
-
- fprint(fd, "arena partition=%s\n\tversion=%d blocksize=%d arenas=%d\n\tsetbase=%d setsize=%d\n",
- ap->part->name, ap->version, ap->blocksize, ap->narenas, ap->tabbase, ap->tabsize);
- for(i = 0; i < ap->narenas; i++)
- fprint(fd, "\tarena=%s at [%lld,%lld)\n", ap->map[i].name, ap->map[i].start, ap->map[i].stop);
-}
-
-void
-printarena(int fd, Arena *arena)
-{
- fprint(fd, "arena='%s' [%lld,%lld)\n\tversion=%d created=%d modified=%d",
- arena->name, arena->base, arena->base + arena->size + 2 * arena->blocksize,
- arena->version, arena->ctime, arena->wtime);
- if(arena->sealed)
- fprint(2, " sealed\n");
- else
- fprint(2, "\n");
- if(scorecmp(zeroscore, arena->score) != 0)
- fprint(2, "\tscore=%V\n", arena->score);
-
- fprint(fd, "\tclumps=%,d compressed clumps=%,d data=%,lld compressed data=%,lld disk storage=%,lld\n",
- arena->clumps, arena->cclumps, arena->uncsize,
- arena->used - arena->clumps * ClumpSize,
- arena->used + arena->clumps * ClumpInfoSize);
-}
blob - 7718adce1c4b84f00cd5148634539075ddd8ff7b (mode 644)
blob + /dev/null
--- src/cmd/venti/findscore.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-enum
-{
- ClumpChunks = 32*1024
-};
-
-static int verbose;
-
-int
-clumpinfoeq(ClumpInfo *c, ClumpInfo *d)
-{
- return c->type == d->type
- && c->size == d->size
- && c->uncsize == d->uncsize
- && scorecmp(c->score, d->score)==0;
-}
-
-int
-findscore(Arena *arena, uchar *score)
-{
- IEntry ie;
- ClumpInfo *ci, *cis;
- u64int a;
- u32int clump;
- int i, n, found;
-
-//ZZZ remove fprint?
- if(arena->clumps)
- fprint(2, "reading directory for arena=%s with %d entries\n", arena->name, arena->clumps);
-
- cis = MKN(ClumpInfo, ClumpChunks);
- found = 0;
- a = 0;
- memset(&ie, 0, sizeof(IEntry));
- for(clump = 0; clump < arena->clumps; clump += n){
- n = ClumpChunks;
- if(n > arena->clumps - clump)
- n = arena->clumps - clump;
- if(readclumpinfos(arena, clump, cis, n) != n){
- seterr(EOk, "arena directory read failed: %r");
- break;
- }
-
- for(i = 0; i < n; i++){
- ci = &cis[i];
- if(scorecmp(score, ci->score)==0){
- fprint(2, "found at clump=%d with type=%d size=%d csize=%d position=%lld\n",
- clump + i, ci->type, ci->uncsize, ci->size, a);
- found++;
- }
- a += ci->size + ClumpSize;
- }
- }
- free(cis);
- return found;
-}
-
-void
-usage(void)
-{
- fprint(2, "usage: findscore [-v] arenafile score\n");
- threadexitsall(0);
-}
-
-void
-threadmain(int argc, char *argv[])
-{
- ArenaPart *ap;
- Part *part;
- char *file;
- u8int score[VtScoreSize];
- int i, found;
-
- fmtinstall('V', vtscorefmt);
- statsinit();
-
- ARGBEGIN{
- case 'v':
- verbose++;
- break;
- default:
- usage();
- break;
- }ARGEND
-
- readonly = 1;
-
- if(argc != 2)
- usage();
-
- file = argv[0];
- if(strscore(argv[1], score) < 0)
- sysfatal("bad score %s\n", argv[1]);
-
- part = initpart(file, 0);
- if(part == nil)
- sysfatal("can't open partition %s: %r", file);
-
- ap = initarenapart(part);
- if(ap == nil)
- sysfatal("can't initialize arena partition in %s: %r", file);
-
- if(verbose > 1){
- printarenapart(2, ap);
- fprint(2, "\n");
- }
-
- initdcache(8 * MaxDiskBlock);
-
- found = 0;
- for(i = 0; i < ap->narenas; i++)
- found += findscore(ap->arenas[i], score);
-
- print("found %d occurances of %V\n", found, score);
-
- if(verbose > 1)
- printstats();
- threadexitsall(0);
-}
blob - 8e52c56be5951f569d17bb29ce745bd3bea6f32b (mode 644)
blob + /dev/null
--- src/cmd/venti/fmtarenas.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-void
-usage(void)
-{
- fprint(2, "usage: fmtarenas [-Z] [-b blocksize] [-a arenasize] name file\n");
- threadexitsall(0);
-}
-
-void
-threadmain(int argc, char *argv[])
-{
- ArenaPart *ap;
- Part *part;
- Arena *arena;
- u64int addr, limit, asize, apsize;
- char *file, *name, aname[ANameSize];
- int i, n, blocksize, tabsize, zero;
-
- fmtinstall('V', vtscorefmt);
-
- statsinit();
-
- blocksize = 8 * 1024;
- asize = 512 * 1024 *1024;
- tabsize = 64 * 1024; /* BUG: should be determine from number of arenas */
- zero = 1;
- ARGBEGIN{
- case 'a':
- asize = unittoull(ARGF());
- if(asize == TWID64)
- usage();
- break;
- case 'b':
- blocksize = unittoull(ARGF());
- if(blocksize == ~0)
- usage();
- if(blocksize > MaxDiskBlock){
- fprint(2, "block size too large, max %d\n", MaxDiskBlock);
- threadexitsall("usage");
- }
- break;
- case 'Z':
- zero = 0;
- break;
- default:
- usage();
- break;
- }ARGEND
-
- if(argc != 2)
- usage();
-
- name = argv[0];
- file = argv[1];
-
- if(nameok(name) < 0)
- sysfatal("illegal name template %s", name);
-
- part = initpart(file, 1);
- if(part == nil)
- sysfatal("can't open partition %s: %r", file);
-
- if(zero)
- zeropart(part, blocksize);
-
- ap = newarenapart(part, blocksize, tabsize);
- if(ap == nil)
- sysfatal("can't initialize arena: %r");
-
- apsize = ap->size - ap->arenabase;
- n = apsize / asize;
- if(apsize - (n * asize) >= MinArenaSize)
- n++;
-
- fprint(2, "configuring %s with arenas=%d for a total storage of bytes=%lld and directory bytes=%d\n",
- file, n, apsize, ap->tabsize);
-
- ap->narenas = n;
- ap->map = MKNZ(AMap, n);
- ap->arenas = MKNZ(Arena*, n);
-
- addr = ap->arenabase;
- for(i = 0; i < n; i++){
- limit = addr + asize;
- if(limit >= ap->size || ap->size - limit < MinArenaSize){
- limit = ap->size;
- if(limit - addr < MinArenaSize)
- sysfatal("bad arena set math: runt arena at %lld,%lld %lld\n", addr, limit, ap->size);
- }
-
- snprint(aname, ANameSize, "%s%d", name, i);
-
- fprint(2, "adding arena %s at [%lld,%lld)\n", aname, addr, limit);
-
- arena = newarena(part, aname, addr, limit - addr, blocksize);
- if(!arena)
- fprint(2, "can't make new arena %s: %r", aname);
- freearena(arena);
-
- ap->map[i].start = addr;
- ap->map[i].stop = limit;
- namecp(ap->map[i].name, aname);
-
- addr = limit;
- }
-
- if(wbarenapart(ap) < 0)
- fprint(2, "can't write back arena partition header for %s: %r\n", file);
-
- threadexitsall(0);
-}
blob - 6934e3d5da2d90f2fa24861ae959b5d6b1aa311d (mode 644)
blob + /dev/null
--- src/cmd/venti/fmtindex.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-void
-usage(void)
-{
- fprint(2, "usage: fmtindex config\n");
- threadexitsall(0);
-}
-
-void
-threadmain(int argc, char *argv[])
-{
- Config conf;
- Index *ix;
- ArenaPart *ap;
- Arena **arenas;
- AMap *amap;
- u64int addr;
- char *file;
- u32int i, j, n, narenas;
- int add;
-
- fmtinstall('V', vtscorefmt);
- statsinit();
-
- add = 0;
- ARGBEGIN{
- case 'a':
- add = 1;
- break;
- default:
- usage();
- break;
- }ARGEND
-
- if(argc != 1)
- usage();
-
- file = argv[0];
-
- if(runconfig(file, &conf) < 0)
- sysfatal("can't initialize config %s: %r", file);
- if(conf.index == nil)
- sysfatal("no index specified in %s", file);
- if(nameok(conf.index) < 0)
- sysfatal("illegal index name %s", conf.index);
-
- narenas = 0;
- for(i = 0; i < conf.naparts; i++){
- ap = conf.aparts[i];
- narenas += ap->narenas;
- }
-
- if(add){
- ix = initindex(conf.index, conf.sects, conf.nsects);
- if(ix == nil)
- sysfatal("can't initialize index %s: %r", conf.index);
- }else{
- ix = newindex(conf.index, conf.sects, conf.nsects);
- if(ix == nil)
- sysfatal("can't create new index %s: %r", conf.index);
-
- n = 0;
- for(i = 0; i < ix->nsects; i++)
- n += ix->sects[i]->blocks;
-
- fprint(2, "using %ud buckets of %ud; div=%d\n", ix->buckets, n, ix->div);
- }
- amap = MKNZ(AMap, narenas);
- arenas = MKNZ(Arena*, narenas);
-
- addr = IndexBase;
- n = 0;
- for(i = 0; i < conf.naparts; i++){
- ap = conf.aparts[i];
- for(j = 0; j < ap->narenas; j++){
- if(n >= narenas)
- sysfatal("too few slots in index's arena set");
-
- arenas[n] = ap->arenas[j];
- if(n < ix->narenas){
- if(arenas[n] != ix->arenas[n])
- sysfatal("mismatched arenas %s and %s at slot %d\n",
- arenas[n]->name, ix->arenas[n]->name, n);
- amap[n] = ix->amap[n];
- if(amap[n].start != addr)
- sysfatal("mis-located arena %s in index %s\n", arenas[n]->name, ix->name);
- addr = amap[n].stop;
- }else{
- amap[n].start = addr;
- addr += ap->arenas[j]->size;
- amap[n].stop = addr;
- namecp(amap[n].name, ap->arenas[j]->name);
- fprint(2, "add arena %s at [%lld,%lld)\n",
- amap[n].name, amap[n].start, amap[n].stop);
- }
-
- n++;
- }
- }
- fprint(2, "configured index=%s with arenas=%d and storage=%lld\n",
- ix->name, n, addr - IndexBase);
- fprint(2, "\tbitblocks=%d maxdepth=%d buckets=%d\n",
- ix->bitblocks, ix->maxdepth, ix->buckets);
-
- ix->amap = amap;
- ix->arenas = arenas;
- ix->narenas = narenas;
-
- if(wbindex(ix) < 0)
- fprint(2, "can't write back arena partition header for %s: %r\n", file);
-
- threadexitsall(0);
-}
blob - 77c929677a9b65de321682ca63d9e60e660c2cc5 (mode 644)
blob + /dev/null
--- src/cmd/venti/fmtisect.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-void
-usage(void)
-{
- fprint(2, "usage: fmtisect [-Z] [-b blocksize] name file\n");
- threadexitsall(0);
-}
-
-void
-threadmain(int argc, char *argv[])
-{
- ISect *is;
- Part *part;
- char *file, *name;
- int blocksize, setsize, zero;
-
- fmtinstall('V', vtscorefmt);
- statsinit();
-
- blocksize = 8 * 1024;
- setsize = 64 * 1024;
- zero = 1;
- ARGBEGIN{
- case 'b':
- blocksize = unittoull(ARGF());
- if(blocksize == ~0)
- usage();
- if(blocksize > MaxDiskBlock){
- fprint(2, "block size too large, max %d\n", MaxDiskBlock);
- threadexitsall("usage");
- }
- break;
- case 'Z':
- zero = 0;
- break;
- default:
- usage();
- break;
- }ARGEND
-
- if(argc != 2)
- usage();
-
- name = argv[0];
- file = argv[1];
-
- if(nameok(name) < 0)
- sysfatal("illegal name %s", name);
-
- part = initpart(file, 0);
- if(part == nil)
- sysfatal("can't open partition %s: %r", file);
-
- if(zero)
- zeropart(part, blocksize);
-
- fprint(2, "configuring index section %s with space for index config bytes=%d\n", name, setsize);
- is = newisect(part, name, blocksize, setsize);
- if(is == nil)
- sysfatal("can't initialize new index: %r");
-
- if(wbisect(is) < 0)
- fprint(2, "can't write back index section header for %s: %r\n", file);
-
- threadexitsall(0);
-}
blob - f8274397704e6ba802cee749f1aff508684c3660 (mode 644)
blob + /dev/null
--- src/cmd/venti/fns.h
+++ /dev/null
-/*
- * sorted by 4,/^$/|sort -bd +1
- */
-int addarena(Arena *name);
-ZBlock *alloczblock(u32int size, int zeroed);
-Arena *amapitoa(Index *index, u64int a, u64int *aa);
-u64int arenadirsize(Arena *arena, u32int clumps);
-void arenaupdate(Arena *arena, u32int size, u8int *score);
-void backsumarena(Arena *arena);
-u32int buildbucket(Index *ix, IEStream *ies, IBucket *ib);
-void checkdcache(void);
-void checklumpcache(void);
-int clumpinfoeq(ClumpInfo *c, ClumpInfo *d);
-int clumpinfoeq(ClumpInfo *c, ClumpInfo *d);
-u32int clumpmagic(Arena *arena, u64int aa);
-int delarena(Arena *arena);
-void dirtydblock(DBlock*, int);
-void *emalloc(ulong);
-void *erealloc(void *, ulong);
-char *estrdup(char*);
-void *ezmalloc(ulong);
-Arena *findarena(char *name);
-int flushciblocks(Arena *arena);
-void flushdcache(void);
-void flushqueue(void);
-void fmtzbinit(Fmt *f, ZBlock *b);
-void freearena(Arena *arena);
-void freearenapart(ArenaPart *ap, int freearenas);
-void freeiestream(IEStream *ies);
-void freeifile(IFile *f);
-void freeisect(ISect *is);
-void freeindex(Index *index);
-void freepart(Part *part);
-void freezblock(ZBlock *b);
-DBlock *getdblock(Part *part, u64int addr, int read);
-u32int hashbits(u8int *score, int nbits);
-int httpdinit(char *address);
-int iaddrcmp(IAddr *ia1, IAddr *ia2);
-int ientrycmp(const void *vie1, const void *vie2);
-char *ifileline(IFile *f);
-int ifilename(IFile *f, char *dst);
-int ifileu32int(IFile *f, u32int *r);
-int indexsect(Index *ix, u8int *score);
-Arena *initarena(Part *part, u64int base, u64int size, u32int blocksize);
-ArenaPart *initarenapart(Part *part);
-int initarenasum(void);
-void initdcache(u32int mem);
-void initicache(int bits, int depth);
-IEStream *initiestream(Part *part, u64int off, u64int clumps, u32int size);
-ISect *initisect(Part *part);
-Index *initindex(char *name, ISect **sects, int n);
-void initlumpcache(u32int size, u32int nblocks);
-int initlumpqueues(int nq);
-Part* initpart(char *name, int writable);
-int initventi(char *config);
-void insertlump(Lump *lump, Packet *p);
-int insertscore(u8int *score, IAddr *ia, int write);
-ZBlock *loadclump(Arena *arena, u64int aa, int blocks, Clump *cl, u8int *score, int verify);
-DBlock *loadibucket(Index *index, u8int *score, ISect **is, u32int *buck, IBucket *ib);
-int loadientry(Index *index, u8int *score, int type, IEntry *ie);
-void logerr(int severity, char *fmt, ...);
-Lump *lookuplump(u8int *score, int type);
-int lookupscore(u8int *score, int type, IAddr *ia, int *rac);
-int maparenas(AMap *am, Arena **arenas, int n, char *what);
-int namecmp(char *s, char *t);
-void namecp(char *dst, char *src);
-int nameok(char *name);
-Arena *newarena(Part *part, char *name, u64int base, u64int size, u32int blocksize);
-ArenaPart *newarenapart(Part *part, u32int blocksize, u32int tabsize);
-ISect *newisect(Part *part, char *name, u32int blocksize, u32int tabsize);
-Index *newindex(char *name, ISect **sects, int n);
-u32int now(void);
-int okamap(AMap *am, int n, u64int start, u64int stop, char *what);
-int outputamap(Fmt *f, AMap *am, int n);
-int outputindex(Fmt *f, Index *ix);
-int packarena(Arena *arena, u8int *buf);
-int packarenahead(ArenaHead *head, u8int *buf);
-int packarenapart(ArenaPart *as, u8int *buf);
-int packclump(Clump *c, u8int *buf);
-void packclumpinfo(ClumpInfo *ci, u8int *buf);
-void packibucket(IBucket *b, u8int *buf);
-void packientry(IEntry *i, u8int *buf);
-int packisect(ISect *is, u8int *buf);
-void packmagic(u32int magic, u8int *buf);
-ZBlock *packet2zblock(Packet *p, u32int size);
-int parseamap(IFile *f, AMapN *amn);
-int parseindex(IFile *f, Index *ix);
-void partblocksize(Part *part, u32int blocksize);
-int partifile(IFile *f, Part *part, u64int start, u32int size);
-void printarenapart(int fd, ArenaPart *ap);
-void printarena(int fd, Arena *arena);
-void printindex(int fd, Index *ix);
-void printstats(void);
-void putdblock(DBlock *b);
-void putlump(Lump *b);
-int queuewrite(Lump *b, Packet *p, int creator);
-u32int readarena(Arena *arena, u64int aa, u8int *buf, long n);
-int readarenamap(AMapN *amn, Part *part, u64int base, u32int size);
-int readclumpinfo(Arena *arena, int clump, ClumpInfo *ci);
-int readclumpinfos(Arena *arena, int clump, ClumpInfo *cis, int n);
-ZBlock *readfile(char *name);
-int readifile(IFile *f, char *name);
-Packet *readlump(u8int *score, int type, u32int size);
-int readpart(Part *part, u64int addr, u8int *buf, u32int n);
-int runconfig(char *config, Config*);
-int scorecmp(u8int *, u8int *);
-void scoremem(u8int *score, u8int *buf, int size);
-void seterr(int severity, char *fmt, ...);
-u64int sortrawientries(Index *ix, Part *tmp, u64int *tmpoff);
-void statsinit(void);
-int storeclump(Index *index, ZBlock *b, u8int *score, int type, u32int creator, IAddr *ia);
-int storeientry(Index *index, IEntry *m);
-int strscore(char *s, u8int *score);
-int stru32int(char *s, u32int *r);
-int stru64int(char *s, u64int *r);
-void sumarena(Arena *arena);
-int syncarena(Arena *arena, u32int n, int zok, int fix);
-int syncarenaindex(Index *ix, Arena *arena, u32int clump, u64int a, int fix);
-int syncindex(Index *ix, int fix);
-int u64log2(u64int v);
-u64int unittoull(char *s);
-int unpackarena(Arena *arena, u8int *buf);
-int unpackarenahead(ArenaHead *head, u8int *buf);
-int unpackarenapart(ArenaPart *as, u8int *buf);
-int unpackclump(Clump *c, u8int *buf);
-void unpackclumpinfo(ClumpInfo *ci, u8int *buf);
-void unpackibucket(IBucket *b, u8int *buf);
-void unpackientry(IEntry *i, u8int *buf);
-int unpackisect(ISect *is, u8int *buf);
-u32int unpackmagic(u8int *buf);
-int vtproc(void(*)(void*), void*);
-int vttypevalid(int type);
-int wbarena(Arena *arena);
-int wbarenahead(Arena *arena);
-int wbarenamap(AMap *am, int n, Part *part, u64int base, u64int size);
-int wbarenapart(ArenaPart *ap);
-int wbisect(ISect *is);
-int wbindex(Index *ix);
-int whackblock(u8int *dst, u8int *src, int ssize);
-u64int writeaclump(Arena *a, Clump *c, u8int *clbuf);
-u32int writearena(Arena *arena, u64int aa, u8int *clbuf, u32int n);
-int writeclumpinfo(Arena *arean, int clump, ClumpInfo *ci);
-u64int writeiclump(Index *ix, Clump *c, u8int *clbuf);
-int writelump(Packet *p, u8int *score, int type, u32int creator);
-int writepart(Part *part, u64int addr, u8int *buf, u32int n);
-int writeqlump(Lump *u, Packet *p, int creator);
-Packet *zblock2packet(ZBlock *zb, u32int size);
-void zeropart(Part *part, int blocksize);
-
-/*
-#pragma varargck argpos sysfatal 1
-#pragma varargck argpos logerr 2
-#pragma varargck argpos SetErr 2
-*/
-
-#define scorecmp(h1,h2) memcmp((h1),(h2),VtScoreSize)
-#define scorecp(h1,h2) memmove((h1),(h2),VtScoreSize)
-
-#define MK(t) ((t*)emalloc(sizeof(t)))
-#define MKZ(t) ((t*)ezmalloc(sizeof(t)))
-#define MKN(t,n) ((t*)emalloc((n)*sizeof(t)))
-#define MKNZ(t,n) ((t*)ezmalloc((n)*sizeof(t)))
-#define MKNA(t,at,n) ((t*)emalloc(sizeof(t) + (n)*sizeof(at)))
blob - 6bd637188235a79ee484e771669b747680e6cebf (mode 644)
blob + /dev/null
--- src/cmd/venti/httpd.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-#include "httpd.h"
-#include "xml.h"
-
-typedef struct HttpObj HttpObj;
-
-enum
-{
- ObjNameSize = 64,
- MaxObjs = 16
-};
-
-struct HttpObj
-{
- char name[ObjNameSize];
- int (*f)(HConnect*);
-};
-
-static HttpObj objs[MaxObjs];
-
-static void listenproc(void*);
-static int estats(HConnect *c);
-static int dindex(HConnect *c);
-static int xindex(HConnect *c);
-static int sindex(HConnect *c);
-static int notfound(HConnect *c);
-static int httpdobj(char *name, int (*f)(HConnect*));
-
-int
-httpdinit(char *address)
-{
- fmtinstall('D', hdatefmt);
- fmtinstall('H', httpfmt);
- fmtinstall('U', hurlfmt);
-
- if(address == nil)
- address = "tcp!*!http";
-
- httpdobj("/stats", estats);
- httpdobj("/index", dindex);
- httpdobj("/storage", sindex);
- httpdobj("/xindex", xindex);
-
- if(vtproc(listenproc, address) < 0)
- return -1;
- return 0;
-}
-
-static int
-httpdobj(char *name, int (*f)(HConnect*))
-{
- int i;
-
- if(name == nil || strlen(name) >= ObjNameSize)
- return -1;
- for(i = 0; i < MaxObjs; i++){
- if(objs[i].name[0] == '\0'){
- strcpy(objs[i].name, name);
- objs[i].f = f;
- return 0;
- }
- if(strcmp(objs[i].name, name) == 0)
- return -1;
- }
- return -1;
-}
-
-static HConnect*
-mkconnect(void)
-{
- HConnect *c;
-
- c = mallocz(sizeof(HConnect), 1);
- if(c == nil)
- sysfatal("out of memory");
- c->replog = nil;
- c->hpos = c->header;
- c->hstop = c->header;
- return c;
-}
-
-void httpproc(void*);
-
-static void
-listenproc(void *vaddress)
-{
- HConnect *c;
- char *address, ndir[NETPATHLEN], dir[NETPATHLEN];
- int ctl, nctl, data;
-
-//sleep(1000); /* let strace find us */
-
- address = vaddress;
- ctl = announce(address, dir);
- if(ctl < 0){
- fprint(2, "venti: httpd can't announce on %s: %r\n", address);
- return;
- }
-
-print("announce ctl %d dir %s\n", ctl, dir);
- for(;;){
- /*
- * wait for a call (or an error)
- */
- nctl = listen(dir, ndir);
-print("httpd listen %d %s...\n", nctl, ndir);
- if(nctl < 0){
- fprint(2, "venti: httpd can't listen on %s: %r\n", address);
- return;
- }
-
- data = accept(ctl, ndir);
-print("httpd accept %d...\n", data);
- if(data < 0){
- fprint(2, "venti: httpd accept: %r\n");
- close(nctl);
- continue;
- }
-print("httpd close nctl %d\n", nctl);
- close(nctl);
- c = mkconnect();
- hinit(&c->hin, data, Hread);
- hinit(&c->hout, data, Hwrite);
- vtproc(httpproc, c);
- }
-}
-
-void
-httpproc(void *v)
-{
- HConnect *c;
- int ok, t, i;
-
-//sleep(1000); /* let strace find us */
- c = v;
-
- for(t = 15*60*1000; ; t = 15*1000){
- if(hparsereq(c, t) < 0)
- break;
-
- ok = -1;
- for(i = 0; i < MaxObjs && objs[i].name[0]; i++){
- if(strcmp(c->req.uri, objs[i].name) == 0){
- ok = (*objs[i].f)(c);
- break;
- }
- }
- if(i == MaxObjs)
- ok = notfound(c);
- if(c->head.closeit)
- ok = -1;
- hreqcleanup(c);
-
- if(ok < 0)
- break;
- }
- hreqcleanup(c);
- close(c->hin.fd);
- free(c);
-}
-
-static int
-percent(long v, long total)
-{
- if(total == 0)
- total = 1;
- if(v < 1000*1000)
- return (v * 100) / total;
- total /= 100;
- if(total == 0)
- total = 1;
- return v / total;
-}
-
-static int
-preq(HConnect *c)
-{
- if(hparseheaders(c, 15*60*1000) < 0)
-{
-fprint(2, "hparseheaders failed\n");
- return -1;
-}
- if(strcmp(c->req.meth, "GET") != 0
- && strcmp(c->req.meth, "HEAD") != 0)
- return hunallowed(c, "GET, HEAD");
- if(c->head.expectother || c->head.expectcont)
- return hfail(c, HExpectFail, nil);
- return 0;
-}
-
-static int
-preqtext(HConnect *c)
-{
- Hio *hout;
- int r;
-
- r = preq(c);
- if(r < 0)
- return r;
-
- hout = &c->hout;
- if(c->req.vermaj){
- hokheaders(c);
- hprint(hout, "Content-type: text/plain\r\n");
- if(http11(c))
- hprint(hout, "Transfer-Encoding: chunked\r\n");
- hprint(hout, "\r\n");
- }
-
- if(http11(c))
- hxferenc(hout, 1);
- else
- c->head.closeit = 1;
- return 0;
-}
-
-static int
-notfound(HConnect *c)
-{
- int r;
-
- r = preq(c);
- if(r < 0)
- return r;
- return hfail(c, HNotFound, c->req.uri);
-}
-
-static int
-estats(HConnect *c)
-{
- Hio *hout;
- int r;
-
- r = preqtext(c);
- if(r < 0)
- return r;
-
-
- hout = &c->hout;
- hprint(hout, "lump writes=%,ld\n", stats.lumpwrites);
- hprint(hout, "lump reads=%,ld\n", stats.lumpreads);
- hprint(hout, "lump cache read hits=%,ld\n", stats.lumphit);
- hprint(hout, "lump cache read misses=%,ld\n", stats.lumpmiss);
-
- hprint(hout, "clump disk writes=%,ld\n", stats.clumpwrites);
- hprint(hout, "clump disk bytes written=%,lld\n", stats.clumpbwrites);
- hprint(hout, "clump disk bytes compressed=%,lld\n", stats.clumpbcomp);
- hprint(hout, "clump disk reads=%,ld\n", stats.clumpreads);
- hprint(hout, "clump disk bytes read=%,lld\n", stats.clumpbreads);
- hprint(hout, "clump disk bytes uncompressed=%,lld\n", stats.clumpbuncomp);
-
- hprint(hout, "clump directory disk writes=%,ld\n", stats.ciwrites);
- hprint(hout, "clump directory disk reads=%,ld\n", stats.cireads);
-
- hprint(hout, "index disk writes=%,ld\n", stats.indexwrites);
- hprint(hout, "index disk reads=%,ld\n", stats.indexreads);
- hprint(hout, "index disk reads for modify=%,ld\n", stats.indexwreads);
- hprint(hout, "index disk reads for allocation=%,ld\n", stats.indexareads);
- hprint(hout, "index block splits=%,ld\n", stats.indexsplits);
-
- hprint(hout, "index cache lookups=%,ld\n", stats.iclookups);
- hprint(hout, "index cache hits=%,ld %d%%\n", stats.ichits,
- percent(stats.ichits, stats.iclookups));
- hprint(hout, "index cache fills=%,ld %d%%\n", stats.icfills,
- percent(stats.icfills, stats.iclookups));
- hprint(hout, "index cache inserts=%,ld\n", stats.icinserts);
-
- hprint(hout, "disk cache hits=%,ld\n", stats.pchit);
- hprint(hout, "disk cache misses=%,ld\n", stats.pcmiss);
- hprint(hout, "disk cache reads=%,ld\n", stats.pcreads);
- hprint(hout, "disk cache bytes read=%,lld\n", stats.pcbreads);
-
- hprint(hout, "disk cache writes=%,ld\n", stats.dirtydblocks);
- hprint(hout, "disk cache writes absorbed=%,ld %d%%\n", stats.absorbedwrites,
- percent(stats.absorbedwrites, stats.dirtydblocks));
-
- hprint(hout, "disk cache flushes=%,ld\n", stats.dcacheflushes);
- hprint(hout, "disk cache flush writes=%,ld (%,ld per flush)\n",
- stats.dcacheflushwrites,
- stats.dcacheflushwrites/(stats.dcacheflushes ? stats.dcacheflushes : 1));
-
- hprint(hout, "disk writes=%,ld\n", stats.diskwrites);
- hprint(hout, "disk bytes written=%,lld\n", stats.diskbwrites);
- hprint(hout, "disk reads=%,ld\n", stats.diskreads);
- hprint(hout, "disk bytes read=%,lld\n", stats.diskbreads);
-
- hflush(hout);
- return 0;
-}
-
-static int
-sindex(HConnect *c)
-{
- Hio *hout;
- Index *ix;
- Arena *arena;
- vlong clumps, cclumps, uncsize, used, size;
- int i, r, active;
-
- r = preqtext(c);
- if(r < 0)
- return r;
- hout = &c->hout;
-
- ix = mainindex;
-
- hprint(hout, "index=%s\n", ix->name);
-
- active = 0;
- clumps = 0;
- cclumps = 0;
- uncsize = 0;
- used = 0;
- size = 0;
- for(i = 0; i < ix->narenas; i++){
- arena = ix->arenas[i];
- if(arena != nil && arena->clumps != 0){
- active++;
- clumps += arena->clumps;
- cclumps += arena->cclumps;
- uncsize += arena->uncsize;
- used += arena->used;
- }
- size += arena->size;
- }
- hprint(hout, "total arenas=%d active=%d\n", ix->narenas, active);
- hprint(hout, "total space=%lld used=%lld\n", size, used + clumps * ClumpInfoSize);
- hprint(hout, "clumps=%lld compressed clumps=%lld data=%lld compressed data=%lld\n",
- clumps, cclumps, uncsize, used - clumps * ClumpSize);
- hflush(hout);
- return 0;
-}
-
-static void
-darena(Hio *hout, Arena *arena)
-{
- hprint(hout, "arena='%s' on %s at [%lld,%lld)\n\tversion=%d created=%d modified=%d",
- arena->name, arena->part->name, arena->base, arena->base + arena->size + 2 * arena->blocksize,
- arena->version, arena->ctime, arena->wtime);
- if(arena->sealed)
- hprint(hout, " sealed\n");
- else
- hprint(hout, "\n");
- if(scorecmp(zeroscore, arena->score) != 0)
- hprint(hout, "\tscore=%V\n", arena->score);
-
- hprint(hout, "\tclumps=%d compressed clumps=%d data=%lld compressed data=%lld disk storage=%lld\n",
- arena->clumps, arena->cclumps, arena->uncsize,
- arena->used - arena->clumps * ClumpSize,
- arena->used + arena->clumps * ClumpInfoSize);
-}
-
-static int
-dindex(HConnect *c)
-{
- Hio *hout;
- Index *ix;
- int i, r;
-
- r = preqtext(c);
- if(r < 0)
- return r;
- hout = &c->hout;
-
-
- ix = mainindex;
- hprint(hout, "index=%s version=%d blocksize=%d tabsize=%d\n",
- ix->name, ix->version, ix->blocksize, ix->tabsize);
- hprint(hout, "\tbuckets=%d div=%d\n", ix->buckets, ix->div);
- for(i = 0; i < ix->nsects; i++)
- hprint(hout, "\tsect=%s for buckets [%lld,%lld) buckmax=%d\n", ix->smap[i].name, ix->smap[i].start, ix->smap[i].stop, ix->sects[i]->buckmax);
- for(i = 0; i < ix->narenas; i++){
- if(ix->arenas[i] != nil && ix->arenas[i]->clumps != 0){
- hprint(hout, "arena=%s at index [%lld,%lld)\n\t", ix->amap[i].name, ix->amap[i].start, ix->amap[i].stop);
- darena(hout, ix->arenas[i]);
- }
- }
- hflush(hout);
- return 0;
-}
-
-static int
-xindex(HConnect *c)
-{
- Hio *hout;
- int r;
-
- r = preq(c);
- if(r < 0)
- return r;
-
- hout = &c->hout;
- if(c->req.vermaj){
- hokheaders(c);
- hprint(hout, "Content-type: text/xml\r\n");
- if(http11(c))
- hprint(hout, "Transfer-Encoding: chunked\r\n");
- hprint(hout, "\r\n");
- }
-
- if(http11(c))
- hxferenc(hout, 1);
- else
- c->head.closeit = 1;
- xmlindex(hout, mainindex, "index", 0);
- hflush(hout);
- return 0;
-}
-
-void
-xmlindent(Hio *hout, int indent)
-{
- int i;
-
- for(i = 0; i < indent; i++)
- hputc(hout, '\t');
-}
-
-void
-xmlaname(Hio *hout, char *v, char *tag)
-{
- hprint(hout, " %s=\"%s\"", tag, v);
-}
-
-void
-xmlscore(Hio *hout, u8int *v, char *tag)
-{
- if(scorecmp(zeroscore, v) == 0)
- return;
- hprint(hout, " %s=\"%V\"", tag, v);
-}
-
-void
-xmlsealed(Hio *hout, int v, char *tag)
-{
- if(!v)
- return;
- hprint(hout, " %s=\"yes\"", tag);
-}
-
-void
-xmlu32int(Hio *hout, u32int v, char *tag)
-{
- hprint(hout, " %s=\"%ud\"", tag, v);
-}
-
-void
-xmlu64int(Hio *hout, u64int v, char *tag)
-{
- hprint(hout, " %s=\"%llud\"", tag, v);
-}
blob - fc1d32e7cd7616f23ff47607343f70e988b368ac (mode 644)
blob + /dev/null
--- src/cmd/venti/icache.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-struct ICache
-{
- QLock lock; /* locks hash table & all associated data */
- IEntry **heads; /* heads of all the hash chains */
- int bits; /* bits to use for indexing heads */
- u32int size; /* number of heads; == 1 << bits, should be < entries */
- IEntry *base; /* all allocated hash table entries */
- u32int entries; /* elements in base */
- u32int unused; /* index of first unused element in base */
- u32int stolen; /* last head from which an element was stolen */
-};
-
-static ICache icache;
-
-static IEntry *icachealloc(IAddr *ia, u8int *score);
-
-/*
- * bits is the number of bits in the icache hash table
- * depth is the average depth
- * memory usage is about (1<<bits) * depth * sizeof(IEntry) + (1<<bits) * sizeof(IEntry*)
- */
-void
-initicache(int bits, int depth)
-{
- icache.bits = bits;
- icache.size = 1 << bits;
- icache.entries = depth * icache.size;
- icache.base = MKNZ(IEntry, icache.entries);
- icache.heads = MKNZ(IEntry*, icache.size);
-}
-
-u32int
-hashbits(u8int *sc, int bits)
-{
- u32int v;
-
- v = (sc[0] << 24) | (sc[1] << 16) | (sc[2] << 8) | sc[3];
- if(bits < 32)
- v >>= (32 - bits);
- return v;
-}
-
-/*
-ZZZ need to think about evicting the correct IEntry,
-and writing back the wtime.
- * look up data score in the index cache
- * if this fails, pull it in from the disk index table, if it exists.
- *
- * must be called with the lump for this score locked
- */
-int
-lookupscore(u8int *score, int type, IAddr *ia, int *rac)
-{
- IEntry d, *ie, *last;
- u32int h;
-
- qlock(&stats.lock);
- stats.iclookups++;
- qunlock(&stats.lock);
-
- qlock(&icache.lock);
- h = hashbits(score, icache.bits);
- last = nil;
- for(ie = icache.heads[h]; ie != nil; ie = ie->next){
- if(ie->ia.type == type && scorecmp(ie->score, score)==0){
- if(last != nil)
- last->next = ie->next;
- else
- icache.heads[h] = ie->next;
- qlock(&stats.lock);
- stats.ichits++;
- qunlock(&stats.lock);
- ie->rac = 1;
- goto found;
- }
- last = ie;
- }
-
- qunlock(&icache.lock);
-
- if(loadientry(mainindex, score, type, &d) < 0)
- return -1;
-
- /*
- * no one else can load an entry for this score,
- * since we have the overall score lock.
- */
- qlock(&stats.lock);
- stats.icfills++;
- qunlock(&stats.lock);
-
- qlock(&icache.lock);
-
- ie = icachealloc(&d.ia, score);
-
-found:
- ie->next = icache.heads[h];
- icache.heads[h] = ie;
-
- *ia = ie->ia;
- *rac = ie->rac;
-
- qunlock(&icache.lock);
-
- return 0;
-}
-
-/*
- * insert a new element in the hash table.
- */
-int
-insertscore(u8int *score, IAddr *ia, int write)
-{
- IEntry *ie, se;
- u32int h;
-
- qlock(&stats.lock);
- stats.icinserts++;
- qunlock(&stats.lock);
-
- qlock(&icache.lock);
- h = hashbits(score, icache.bits);
-
- ie = icachealloc(ia, score);
-
- ie->next = icache.heads[h];
- icache.heads[h] = ie;
-
- se = *ie;
-
- qunlock(&icache.lock);
-
- if(!write)
- return 0;
-
- return storeientry(mainindex, &se);
-}
-
-/*
- * allocate a index cache entry which hasn't been used in a while.
- * must be called with icache.lock locked
- * if the score is already in the table, update the entry.
- */
-static IEntry *
-icachealloc(IAddr *ia, u8int *score)
-{
- IEntry *ie, *last, *next;
- u32int h;
-
- h = hashbits(score, icache.bits);
- last = nil;
- for(ie = icache.heads[h]; ie != nil; ie = ie->next){
- if(ie->ia.type == ia->type && scorecmp(ie->score, score)==9){
- if(last != nil)
- last->next = ie->next;
- else
- icache.heads[h] = ie->next;
- ie->rac = 1;
- return ie;
- }
- last = ie;
- }
-
- h = icache.unused;
- if(h < icache.entries){
- ie = &icache.base[h++];
- icache.unused = h;
- goto Found;
- }
-
- h = icache.stolen;
- for(;;){
- h++;
- if(h >= icache.size)
- h = 0;
- ie = icache.heads[h];
- if(ie != nil){
- last = nil;
- for(; next = ie->next; ie = next)
- last = ie;
- if(last != nil)
- last->next = nil;
- else
- icache.heads[h] = nil;
- icache.stolen = h;
- goto Found;
- }
- }
-Found:
- ie->ia = *ia;
- scorecp(ie->score, score);
- ie->rac = 0;
- return ie;
-}
blob - 811e487caf43702bcdd9ee201931e450f8a11115 (mode 644)
blob + /dev/null
--- src/cmd/venti/ifile.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-int
-readifile(IFile *f, char *name)
-{
- ZBlock *b;
-
- b = readfile(name);
- if(b == nil)
- return -1;
- f->name = name;
- f->b = b;
- f->pos = 0;
- return 0;
-}
-
-void
-freeifile(IFile *f)
-{
- freezblock(f->b);
- f->b = nil;
- f->pos = 0;
-}
-
-int
-partifile(IFile *f, Part *part, u64int start, u32int size)
-{
- ZBlock *b;
-
- b = alloczblock(size, 0);
- if(b == nil)
- return -1;
- if(readpart(part, start, b->data, size) < 0){
- seterr(EAdmin, "can't read %s: %r", part->name);
- freezblock(b);
- return -1;
- }
- f->name = part->name;
- f->b = b;
- f->pos = 0;
- return 0;
-}
-
-/*
- * return the next non-blank input line,
- * stripped of leading white space and with # comments eliminated
- */
-char*
-ifileline(IFile *f)
-{
- char *s, *e, *t;
- int c;
-
- for(;;){
- s = (char*)&f->b->data[f->pos];
- e = memchr(s, '\n', f->b->len - f->pos);
- if(e == nil)
- return nil;
- *e++ = '\0';
- f->pos = e - (char*)f->b->data;
- t = strchr(s, '#');
- if(t != nil)
- *t = '\0';
- for(; c = *s; s++)
- if(c != ' ' && c != '\t' && c != '\r')
- return s;
- }
-}
-
-int
-ifilename(IFile *f, char *dst)
-{
- char *s;
-
- s = ifileline(f);
- if(s == nil || strlen(s) >= ANameSize)
- return -1;
- namecp(dst, s);
- return 0;
-}
-
-int
-ifileu32int(IFile *f, u32int *r)
-{
- char *s;
-
- s = ifileline(f);
- if(s == nil)
- return -1;
- return stru32int(s, r);
-}
blob - c1fc141888576f502526ea8201a4fac56b99cd95 (mode 644)
blob + /dev/null
--- src/cmd/venti/index.c
+++ /dev/null
-/*
- * Index, mapping scores to log positions.
- *
- * The index is made up of some number of index sections, each of
- * which is typically stored on a different disk. The blocks in all the
- * index sections are logically numbered, with each index section
- * responsible for a range of blocks. Blocks are typically 8kB.
- *
- * Index Version 1:
- *
- * The N index blocks are treated as a giant hash table. The top 32 bits
- * of score are used as the key for a lookup. Each index block holds
- * one hash bucket, which is responsible for ceil(2^32 / N) of the key space.
- *
- * The index is sized so that a particular bucket is extraordinarily
- * unlikely to overflow: assuming compressed data blocks are 4kB
- * on disk, and assuming each block has a 40 byte index entry,
- * the index data will be 1% of the total data. Since scores are essentially
- * random, all buckets should be about the same fullness.
- * A factor of 5 gives us a wide comfort boundary to account for
- * random variation. So the index disk space should be 5% of the arena disk space.
- *
- * Problems with Index Version 1:
- *
- * Because the index size is chosen to handle the worst case load,
- * the index is very sparse, especially when the Venti server is mostly empty.
- * This has a few bad properties.
- *
- * Loading an index block (which typically requires a random disk seek)
- * is a very expensive operation, yet it yields only a couple index entries.
- * We're not making efficient use of the disk arm.
- *
- * Writing a block requires first checking to see if the block already
- * exists on the server, which in turn requires an index lookup. When
- * writing fresh data, these lookups will fail. The index entry cache
- * cannot serve these, so they must go to disk, which is expensive.
- *
- * Thus both the reading and the writing of blocks are held back by
- * the expense of accessing the index.
- *
- * Index Version 2:
- *
- * The index is sized to be exactly 2^M blocks. The index blocks are
- * logically arranged in a (not exactly balanced) binary tree of depth at
- * most M. The nodes are named by bit strings describing the path from
- * the root to the node. The root is . (dot). The left child of the root is .0,
- * the right child .1. The node you get to by starting at the root and going
- * left right right left is .0110. At the beginning, there is only the root block.
- * When a block with name .xxx fills, it splits into .xxx0 and .xxx1.
- * All the index data is kept in the leaves of the tree.
- *
- * Index leaf blocks are laid out on disk by interpreting the bitstring as a
- * binary fraction and multiplying by 2^M -- .0 is the first block, .1 is
- * the block halfway into the index, .0110 is at position 6/16, and
- * .xxx and .xxx0 map to the same block (but only one can be a leaf
- * node at any given time, so this is okay!). A cheap implementation of
- * this is to append zeros to the bit string to make it M bits long. That's
- * the integer index block number.
- *
- * To find the leaf block that should hold a score, use the bits of the
- * score one at a time to walk down the tree to a leaf node. If the tree
- * has leaf nodes .0, .10, and .11, then score 0110101... ends up in bucket
- * .0 while score 101110101... ends up in bucket .10. There is no leaf node
- * .1 because it has split into .10 and .11.
- *
- * If we know which disk blocks are in use, we can reconstruct the interior
- * of the tree: if .xxx1 is in use, then .xxx has been split. We keep an in-use
- * bitmap of all index disk blocks to aid in reconstructing the interior of the
- * tree. At one bit per index block, the bitmap is small enough to be kept
- * in memory even on the largest of Venti servers.
- *
- * After the root block splits, the index blocks being used will always be
- * at least half full (averaged across the entire index). So unlike Version 1,
- * Index Version 2 is quite dense, alleviating the two problems above.
- * Index block reads now return many index entries. More importantly,
- * small servers can keep most of the index in the disk cache, making them
- * more likely to handle negative lookups without going to disk.
- *
- * As the index becomes more full, Index Version 2's performance
- * degrades gracefully into Index Version 1. V2 is really an optimization
- * for little servers.
- *
- * Write Ordering for Index Version 2:
- *
- * Unlike Index Version 1, Version 2 must worry about write ordering
- * within the index. What happens if the in-use bitmap is out of sync
- * with the actual leaf nodes? What happens if .xxx splits into .xxx0 and
- * .xxx1 but only one of the new blocks gets written to disk?
- *
- * We arrange that when .xxx splits, the .xxx1 block is written first,
- * then the .xxx0 block, and finally the in-use bitmap entry for .xxx1.
- * The split is committed by the writing of .xxx0. This ordering leaves
- * two possible partial disk writes:
- *
- * (a) If .xxx1 is written but .xxx0 and the bitmap are not, then it's as if
- * the split never happened -- we won't think .xxx1 is in use, and we
- * won't go looking for it.
- *
- * (b) If .xxx1 and .xxx0 are written but the bitmap is not, then the first
- * time we try to load .xxx, we'll get .xxx0 instead, realize the bitmap is
- * out of date, and update the bitmap.
- *
- * Backwards Compatibility
- *
- * Because there are large Venti servers in use with Index V1, this code
- * will read either index version, but when asked to create a new index,
- * will only create V2.
- */
-
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-static int bucklook(u8int *score, int type, u8int *data, int n);
-static int writebucket(ISect *is, u32int buck, IBucket *ib, DBlock *b);
-static int okibucket(IBucket *ib, ISect *is);
-static int initindex1(Index*);
-static ISect *initisect1(ISect *is);
-static int splitiblock(Index *ix, DBlock *b, ISect *is, u32int buck, IBucket *ib);
-
-#define KEY(k,d) ((d) ? (k)>>(32-(d)) : 0)
-
-//static QLock indexlock; //ZZZ
-
-static char IndexMagic[] = "venti index configuration";
-
-Index*
-initindex(char *name, ISect **sects, int n)
-{
- IFile f;
- Index *ix;
- ISect *is;
- u32int last, blocksize, tabsize;
- int i;
-
- if(n <= 0){
- seterr(EOk, "no index sections to initialize index");
- return nil;
- }
- ix = MKZ(Index);
- if(ix == nil){
- seterr(EOk, "can't initialize index: out of memory");
- freeindex(ix);
- return nil;
- }
-
- tabsize = sects[0]->tabsize;
- if(partifile(&f, sects[0]->part, sects[0]->tabbase, tabsize) < 0)
- return nil;
- if(parseindex(&f, ix) < 0){
- freeifile(&f);
- freeindex(ix);
- return nil;
- }
- freeifile(&f);
- if(namecmp(ix->name, name) != 0){
- seterr(ECorrupt, "mismatched index name: found %s expected %s", ix->name, name);
- return nil;
- }
- if(ix->nsects != n){
- seterr(ECorrupt, "mismatched number index sections: found %d expected %d", n, ix->nsects);
- freeindex(ix);
- return nil;
- }
- ix->sects = sects;
- last = 0;
- blocksize = ix->blocksize;
- for(i = 0; i < ix->nsects; i++){
- is = sects[i];
- if(namecmp(ix->name, is->index) != 0
- || is->blocksize != blocksize
- || is->tabsize != tabsize
- || namecmp(is->name, ix->smap[i].name) != 0
- || is->start != ix->smap[i].start
- || is->stop != ix->smap[i].stop
- || last != is->start
- || is->start > is->stop){
- seterr(ECorrupt, "inconsistent index sections in %s", ix->name);
- freeindex(ix);
- return nil;
- }
- last = is->stop;
- }
- ix->tabsize = tabsize;
- ix->buckets = last;
-
- if(initindex1(ix) < 0){
- freeindex(ix);
- return nil;
- }
-
- ix->arenas = MKNZ(Arena*, ix->narenas);
- if(maparenas(ix->amap, ix->arenas, ix->narenas, ix->name) < 0){
- freeindex(ix);
- return nil;
- }
-
- return ix;
-}
-
-static int
-initindex1(Index *ix)
-{
- u32int buckets;
-
- switch(ix->version){
- case IndexVersion1:
- ix->div = (((u64int)1 << 32) + ix->buckets - 1) / ix->buckets;
- buckets = (((u64int)1 << 32) - 1) / ix->div + 1;
- if(buckets != ix->buckets){
- seterr(ECorrupt, "inconsistent math for divisor and buckets in %s", ix->name);
- return -1;
- }
- break;
-
- case IndexVersion2:
- buckets = ix->buckets - ix->bitblocks;
- if(ix->buckets < ix->bitblocks || (buckets&(buckets-1)))
- seterr(ECorrupt, "bucket count not a power of two in %s", ix->name);
- ix->maxdepth = u64log2(buckets);
- ix->bitkeylog = u64log2(ix->blocksize*8);
- ix->bitkeymask = (1<<ix->bitkeylog)-1;
- break;
- }
- return 0;
-}
-
-int
-wbindex(Index *ix)
-{
- Fmt f;
- ZBlock *b;
- int i;
-
- if(ix->nsects == 0){
- seterr(EOk, "no sections in index %s", ix->name);
- return -1;
- }
- b = alloczblock(ix->tabsize, 1);
- if(b == nil){
- seterr(EOk, "can't write index configuration: out of memory");
- return -1;
- }
- fmtzbinit(&f, b);
- if(outputindex(&f, ix) < 0){
- seterr(EOk, "can't make index configuration: table storage too small %d", ix->tabsize);
- freezblock(b);
- return -1;
- }
- for(i = 0; i < ix->nsects; i++){
- if(writepart(ix->sects[i]->part, ix->sects[i]->tabbase, b->data, ix->tabsize) < 0){
- seterr(EOk, "can't write index: %r");
- freezblock(b);
- return -1;
- }
- }
- freezblock(b);
-
- for(i = 0; i < ix->nsects; i++)
- if(wbisect(ix->sects[i]) < 0)
- return -1;
-
- return 0;
-}
-
-/*
- * index: IndexMagic '\n' version '\n' name '\n' blocksize '\n' [V2: bitblocks '\n'] sections arenas
- * version, blocksize: u32int
- * name: max. ANameSize string
- * sections, arenas: AMap
- */
-int
-outputindex(Fmt *f, Index *ix)
-{
- if(fmtprint(f, "%s\n%ud\n%s\n%ud\n", IndexMagic, ix->version, ix->name, ix->blocksize) < 0
- || (ix->version==IndexVersion2 && fmtprint(f, "%ud\n", ix->bitblocks) < 0)
- || outputamap(f, ix->smap, ix->nsects) < 0
- || outputamap(f, ix->amap, ix->narenas) < 0)
- return -1;
- return 0;
-}
-
-int
-parseindex(IFile *f, Index *ix)
-{
- AMapN amn;
- u32int v;
- char *s;
-
- /*
- * magic
- */
- s = ifileline(f);
- if(s == nil || strcmp(s, IndexMagic) != 0){
- seterr(ECorrupt, "bad index magic for %s", f->name);
- return -1;
- }
-
- /*
- * version
- */
- if(ifileu32int(f, &v) < 0){
- seterr(ECorrupt, "syntax error: bad version number in %s", f->name);
- return -1;
- }
- ix->version = v;
- if(ix->version != IndexVersion1 && ix->version != IndexVersion2){
- seterr(ECorrupt, "bad version number in %s", f->name);
- return -1;
- }
-
- /*
- * name
- */
- if(ifilename(f, ix->name) < 0){
- seterr(ECorrupt, "syntax error: bad index name in %s", f->name);
- return -1;
- }
-
- /*
- * block size
- */
- if(ifileu32int(f, &v) < 0){
- seterr(ECorrupt, "syntax error: bad block size number in %s", f->name);
- return -1;
- }
- ix->blocksize = v;
-
- if(ix->version == IndexVersion2){
- /*
- * free bitmap size
- */
- if(ifileu32int(f, &v) < 0){
- seterr(ECorrupt, "syntax error: bad bitmap size in %s", f->name);
- return -1;
- }
- ix->bitblocks = v;
- }
-
- if(parseamap(f, &amn) < 0)
- return -1;
- ix->nsects = amn.n;
- ix->smap = amn.map;
-
- if(parseamap(f, &amn) < 0)
- return -1;
- ix->narenas = amn.n;
- ix->amap = amn.map;
-
- return 0;
-}
-
-/*
- * initialize an entirely new index
- */
-Index *
-newindex(char *name, ISect **sects, int n)
-{
- Index *ix;
- AMap *smap;
- u64int nb;
- u32int div, ub, xb, fb, start, stop, blocksize, tabsize;
- int i, j, version;
-
- version = IndexVersion2;
-
- if(n < 1){
- seterr(EOk, "creating index with no index sections");
- return nil;
- }
-
- /*
- * compute the total buckets available in the index,
- * and the total buckets which are used.
- */
- nb = 0;
- blocksize = sects[0]->blocksize;
- tabsize = sects[0]->tabsize;
- for(i = 0; i < n; i++){
- if(sects[i]->start != 0 || sects[i]->stop != 0
- || sects[i]->index[0] != '\0'){
- seterr(EOk, "creating new index using non-empty section %s", sects[i]->name);
- return nil;
- }
- if(blocksize != sects[i]->blocksize){
- seterr(EOk, "mismatched block sizes in index sections");
- return nil;
- }
- if(tabsize != sects[i]->tabsize){
- seterr(EOk, "mismatched config table sizes in index sections");
- return nil;
- }
- nb += sects[i]->blocks;
- }
-
- /*
- * check for duplicate names
- */
- for(i = 0; i < n; i++){
- for(j = i + 1; j < n; j++){
- if(namecmp(sects[i]->name, sects[j]->name) == 0){
- seterr(EOk, "duplicate section name %s for index %s", sects[i]->name, name);
- return nil;
- }
- }
- }
-
- if(nb >= ((u64int)1 << 32)){
- seterr(EBug, "index too large");
- return nil;
- }
-
- div = 0;
- fb = 0;
- if(version == IndexVersion1){
- div = (((u64int)1 << 32) + nb - 1) / nb;
- ub = (((u64int)1 << 32) - 1) / div + 1;
- if(div < 100){
- seterr(EBug, "index divisor too coarse");
- return nil;
- }
- }else{
- fb = (nb+blocksize*8-1)/(blocksize*8);
- for(ub=1; ub<=((nb-fb)>>1); ub<<=1)
- ;
- ub += fb;
- }
- if(ub > nb){
- seterr(EBug, "index initialization math wrong");
- return nil;
- }
- xb = nb - ub;
-
- /*
- * initialize each of the index sections
- * and the section map table
- */
- smap = MKNZ(AMap, n);
- if(smap == nil){
- seterr(EOk, "can't create new index: out of memory");
- return nil;
- }
- start = 0;
- for(i = 0; i < n; i++){
- stop = start + sects[i]->blocks - xb / n;
- if(i == n - 1)
- stop = ub;
- sects[i]->start = start;
- sects[i]->stop = stop;
- namecp(sects[i]->index, name);
-
- smap[i].start = start;
- smap[i].stop = stop;
- namecp(smap[i].name, sects[i]->name);
- start = stop;
- }
-
- /*
- * initialize the index itself
- */
- ix = MKZ(Index);
- if(ix == nil){
- seterr(EOk, "can't create new index: out of memory");
- free(smap);
- return nil;
- }
- ix->version = version;
- namecp(ix->name, name);
- ix->sects = sects;
- ix->smap = smap;
- ix->nsects = n;
- ix->blocksize = blocksize;
- ix->buckets = ub;
- ix->tabsize = tabsize;
- ix->div = div;
- ix->bitblocks = fb;
-
- if(initindex1(ix) < 0){
- free(smap);
- return nil;
- }
-
- return ix;
-}
-
-ISect*
-initisect(Part *part)
-{
- ISect *is;
- ZBlock *b;
- int ok;
-
- b = alloczblock(HeadSize, 0);
- if(b == nil || readpart(part, PartBlank, b->data, HeadSize) < 0){
- seterr(EAdmin, "can't read index section header: %r");
- return nil;
- }
-
- is = MKZ(ISect);
- if(is == nil){
- freezblock(b);
- return nil;
- }
- is->part = part;
- ok = unpackisect(is, b->data);
- freezblock(b);
- if(ok < 0){
- seterr(ECorrupt, "corrupted index section header: %r");
- freeisect(is);
- return nil;
- }
-
- if(is->version != ISectVersion){
- seterr(EAdmin, "unknown index section version %d", is->version);
- freeisect(is);
- return nil;
- }
-
- return initisect1(is);
-}
-
-ISect*
-newisect(Part *part, char *name, u32int blocksize, u32int tabsize)
-{
- ISect *is;
- u32int tabbase;
-
- is = MKZ(ISect);
- if(is == nil)
- return nil;
-
- namecp(is->name, name);
- is->version = ISectVersion;
- is->part = part;
- is->blocksize = blocksize;
- is->start = 0;
- is->stop = 0;
- tabbase = (PartBlank + HeadSize + blocksize - 1) & ~(blocksize - 1);
- is->blockbase = (tabbase + tabsize + blocksize - 1) & ~(blocksize - 1);
- is->blocks = is->part->size / blocksize - is->blockbase / blocksize;
-
- is = initisect1(is);
- if(is == nil)
- return nil;
-
- return is;
-}
-
-/*
- * initialize the computed parameters for an index
- */
-static ISect*
-initisect1(ISect *is)
-{
- u64int v;
-
- is->buckmax = (is->blocksize - IBucketSize) / IEntrySize;
- is->blocklog = u64log2(is->blocksize);
- if(is->blocksize != (1 << is->blocklog)){
- seterr(ECorrupt, "illegal non-power-of-2 bucket size %d\n", is->blocksize);
- freeisect(is);
- return nil;
- }
- partblocksize(is->part, is->blocksize);
- is->tabbase = (PartBlank + HeadSize + is->blocksize - 1) & ~(is->blocksize - 1);
- if(is->tabbase >= is->blockbase){
- seterr(ECorrupt, "index section config table overlaps bucket storage");
- freeisect(is);
- return nil;
- }
- is->tabsize = is->blockbase - is->tabbase;
- v = is->part->size & ~(u64int)(is->blocksize - 1);
- if(is->blockbase + (u64int)is->blocks * is->blocksize != v){
- seterr(ECorrupt, "invalid blocks in index section %s", is->name);
-//ZZZZZZZZZ
-// freeisect(is);
-// return nil;
- }
-
- if(is->stop - is->start > is->blocks){
- seterr(ECorrupt, "index section overflows available space");
- freeisect(is);
- return nil;
- }
- if(is->start > is->stop){
- seterr(ECorrupt, "invalid index section range");
- freeisect(is);
- return nil;
- }
-
- return is;
-}
-
-int
-wbisect(ISect *is)
-{
- ZBlock *b;
-
- b = alloczblock(HeadSize, 1);
- if(b == nil)
-//ZZZ set error?
- return -1;
-
- if(packisect(is, b->data) < 0){
- seterr(ECorrupt, "can't make index section header: %r");
- freezblock(b);
- return -1;
- }
- if(writepart(is->part, PartBlank, b->data, HeadSize) < 0){
- seterr(EAdmin, "can't write index section header: %r");
- freezblock(b);
- return -1;
- }
- freezblock(b);
-
- return 0;
-}
-
-void
-freeisect(ISect *is)
-{
- if(is == nil)
- return;
- free(is);
-}
-
-void
-freeindex(Index *ix)
-{
- int i;
-
- if(ix == nil)
- return;
- free(ix->amap);
- free(ix->arenas);
- if(ix->sects)
- for(i = 0; i < ix->nsects; i++)
- freeisect(ix->sects[i]);
- free(ix->sects);
- free(ix->smap);
- free(ix);
-}
-
-/*
- * write a clump to an available arena in the index
- * and return the address of the clump within the index.
-ZZZ question: should this distinguish between an arena
-filling up and real errors writing the clump?
- */
-u64int
-writeiclump(Index *ix, Clump *c, u8int *clbuf)
-{
- u64int a;
- int i;
-
- for(i = ix->mapalloc; i < ix->narenas; i++){
- a = writeaclump(ix->arenas[i], c, clbuf);
- if(a != TWID64)
- return a + ix->amap[i].start;
- }
-
- seterr(EAdmin, "no space left in arenas");
- return TWID64;
-}
-
-/*
- * convert an arena index to an relative arena address
- */
-Arena*
-amapitoa(Index *ix, u64int a, u64int *aa)
-{
- int i, r, l, m;
-
- l = 1;
- r = ix->narenas - 1;
- while(l <= r){
- m = (r + l) / 2;
- if(ix->amap[m].start <= a)
- l = m + 1;
- else
- r = m - 1;
- }
- l--;
-
- if(a > ix->amap[l].stop){
-for(i=0; i<ix->narenas; i++)
- print("arena %d: %llux - %llux\n", i, ix->amap[i].start, ix->amap[i].stop);
-print("want arena %d for %llux\n", l, a);
- seterr(ECrash, "unmapped address passed to amapitoa");
- return nil;
- }
-
- if(ix->arenas[l] == nil){
- seterr(ECrash, "unmapped arena selected in amapitoa");
- return nil;
- }
- *aa = a - ix->amap[l].start;
- return ix->arenas[l];
-}
-
-int
-iaddrcmp(IAddr *ia1, IAddr *ia2)
-{
- return ia1->type != ia2->type
- || ia1->size != ia2->size
- || ia1->blocks != ia2->blocks
- || ia1->addr != ia2->addr;
-}
-
-/*
- * lookup the score in the partition
- *
- * nothing needs to be explicitly locked:
- * only static parts of ix are used, and
- * the bucket is locked by the DBlock lock.
- */
-int
-loadientry(Index *ix, u8int *score, int type, IEntry *ie)
-{
- ISect *is;
- DBlock *b;
- IBucket ib;
- u32int buck;
- int h, ok;
-
- ok = -1;
-
- qlock(&stats.lock);
- stats.indexreads++;
- qunlock(&stats.lock);
- b = loadibucket(ix, score, &is, &buck, &ib);
- if(b == nil)
- return -1;
-
- if(okibucket(&ib, is) < 0)
- goto out;
-
- h = bucklook(score, type, ib.data, ib.n);
- if(h & 1){
- h ^= 1;
- unpackientry(ie, &ib.data[h]);
- ok = 0;
- goto out;
- }
-
-out:
- putdblock(b);
- return ok;
-}
-
-/*
- * insert or update an index entry into the appropriate bucket
- */
-int
-storeientry(Index *ix, IEntry *ie)
-{
- ISect *is;
- DBlock *b;
- IBucket ib;
- u32int buck;
- int h, ok;
-
- ok = 0;
-
- qlock(&stats.lock);
- stats.indexwreads++;
- qunlock(&stats.lock);
-
-top:
- b = loadibucket(ix, ie->score, &is, &buck, &ib);
- if(b == nil)
- return -1;
-
- if(okibucket(&ib, is) < 0)
- goto out;
-
- h = bucklook(ie->score, ie->ia.type, ib.data, ib.n);
- if(h & 1){
- h ^= 1;
- dirtydblock(b, DirtyIndex);
- packientry(ie, &ib.data[h]);
- ok = writebucket(is, buck, &ib, b);
- goto out;
- }
-
- if(ib.n < is->buckmax){
- dirtydblock(b, DirtyIndex);
- memmove(&ib.data[h + IEntrySize], &ib.data[h], ib.n * IEntrySize - h);
- ib.n++;
-
- packientry(ie, &ib.data[h]);
- ok = writebucket(is, buck, &ib, b);
- goto out;
- }
-
- /* block is full -- not supposed to happen in V1 */
- if(ix->version == IndexVersion1){
- seterr(EAdmin, "index bucket %ud on %s is full\n", buck, is->part->name);
- ok = -1;
- goto out;
- }
-
- /* in V2, split the block and try again; splitiblock puts b */
- if(splitiblock(ix, b, is, buck, &ib) < 0)
- return -1;
- goto top;
-
-out:
- putdblock(b);
- return ok;
-}
-
-static int
-writebucket(ISect *is, u32int buck, IBucket *ib, DBlock *b)
-{
- assert(b->dirty == DirtyIndex || b->dirty == DirtyIndexSplit);
-
- if(buck >= is->blocks){
- seterr(EAdmin, "index write out of bounds: %d >= %d\n",
- buck, is->blocks);
- return -1;
- }
- qlock(&stats.lock);
- stats.indexwrites++;
- qunlock(&stats.lock);
- packibucket(ib, b->data);
- // return writepart(is->part, is->blockbase + ((u64int)buck << is->blocklog), b->data, is->blocksize);
- return 0;
-}
-
-static int
-okibucket(IBucket *ib, ISect *is)
-{
- if(ib->n <= is->buckmax)
- return 0;
-
- seterr(EICorrupt, "corrupted disk index bucket: n=%ud max=%ud, depth=%lud range=[%lud,%lud)",
- ib->n, is->buckmax, ib->depth, is->start, is->stop);
- return -1;
-}
-
-/*
- * look for score within data;
- * return 1 | byte index of matching index,
- * or 0 | index of least element > score
- */
-static int
-bucklook(u8int *score, int otype, u8int *data, int n)
-{
- int i, r, l, m, h, c, cc, type;
-
- type = vttodisktype(otype);
- l = 0;
- r = n - 1;
- while(l <= r){
- m = (r + l) >> 1;
- h = m * IEntrySize;
- for(i = 0; i < VtScoreSize; i++){
- c = score[i];
- cc = data[h + i];
- if(c != cc){
- if(c > cc)
- l = m + 1;
- else
- r = m - 1;
- goto cont;
- }
- }
- cc = data[h + IEntryTypeOff];
- if(type != cc){
- if(type > cc)
- l = m + 1;
- else
- r = m - 1;
- goto cont;
- }
- return h | 1;
- cont:;
- }
-
- return l * IEntrySize;
-}
-
-/*
- * compare two IEntries; consistent with bucklook
- */
-int
-ientrycmp(const void *vie1, const void *vie2)
-{
- u8int *ie1, *ie2;
- int i, v1, v2;
-
- ie1 = (u8int*)vie1;
- ie2 = (u8int*)vie2;
- for(i = 0; i < VtScoreSize; i++){
- v1 = ie1[i];
- v2 = ie2[i];
- if(v1 != v2){
- if(v1 < v2)
- return -1;
- return 0;
- }
- }
- v1 = ie1[IEntryTypeOff];
- v2 = ie2[IEntryTypeOff];
- if(v1 != v2){
- if(v1 < v2)
- return -1;
- return 0;
- }
- return -1;
-}
-
-/*
- * find the number of the index section holding bucket #buck
- */
-static int
-indexsect0(Index *ix, u32int buck)
-{
- int r, l, m;
-
- l = 1;
- r = ix->nsects - 1;
- while(l <= r){
- m = (r + l) >> 1;
- if(ix->sects[m]->start <= buck)
- l = m + 1;
- else
- r = m - 1;
- }
- return l - 1;
-}
-
-/*
- * load the index block at bucket #buck
- */
-static DBlock*
-loadibucket0(Index *ix, u32int buck, ISect **pis, u32int *pbuck, IBucket *ib, int read)
-{
- ISect *is;
- DBlock *b;
-
- is = ix->sects[indexsect0(ix, buck)];
- if(buck < is->start || is->stop <= buck){
- seterr(EAdmin, "index lookup out of range: %ud not found in index\n", buck);
- return nil;
- }
-
- buck -= is->start;
- if((b = getdblock(is->part, is->blockbase + ((u64int)buck << is->blocklog), read)) == nil)
- return nil;
-
- if(pis)
- *pis = is;
- if(pbuck)
- *pbuck = buck;
- if(ib)
- unpackibucket(ib, b->data);
- return b;
-}
-
-/*
- * find the number of the index section holding score
- */
-static int
-indexsect1(Index *ix, u8int *score)
-{
- return indexsect0(ix, hashbits(score, 32) / ix->div);
-}
-
-/*
- * load the index block responsible for score.
- */
-static DBlock*
-loadibucket1(Index *ix, u8int *score, ISect **pis, u32int *pbuck, IBucket *ib)
-{
- return loadibucket0(ix, hashbits(score, 32)/ix->div, pis, pbuck, ib, 1);
-}
-
-static u32int
-keytobuck(Index *ix, u32int key, int d)
-{
- /* clear all but top d bits; can't depend on boundary case shifts */
- if(d == 0)
- key = 0;
- else if(d != 32)
- key &= ~((1<<(32-d))-1);
-
- /* truncate to maxdepth bits */
- if(ix->maxdepth != 32)
- key >>= 32 - ix->maxdepth;
-
- return ix->bitblocks + key;
-}
-
-/*
- * to check whether .xxx has split, check whether .xxx1 is in use.
- * it might be worth caching the block for future lookups, but for now
- * let's assume the dcache is good enough.
- */
-static int
-bitmapop(Index *ix, u32int key, int d, int set)
-{
- DBlock *b;
- int inuse;
- u32int key1, buck1;
-
- if(d >= ix->maxdepth)
- return 0;
-
-
- /* construct .xxx1 in bucket number format */
- key1 = key | (1<<(32-d-1));
- buck1 = keytobuck(ix, key1, d+1);
-
-if(0) fprint(2, "key %d/%0*ub key1 %d/%0*ub buck1 %08ux\n",
- d, d, KEY(key, d), d+1, d+1, KEY(key1, d+1), buck1);
-
- /* check whether buck1 is in use */
-
- if((b = loadibucket0(ix, buck1 >> ix->bitkeylog, nil, nil, nil, 1)) == nil){
- seterr(ECorrupt, "cannot load in-use bitmap block");
-fprint(2, "loadibucket: %r\n");
- return -1;
- }
-if(0) fprint(2, "buck1 %08ux bitkeymask %08ux bitkeylog %d\n", buck1, ix->bitkeymask, ix->bitkeylog);
- buck1 &= ix->bitkeymask;
- inuse = ((u32int*)b->data)[buck1>>5] & (1<<(buck1&31));
- if(set && !inuse){
- dirtydblock(b, DirtyIndexBitmap);
- ((u32int*)b->data)[buck1>>5] |= (1<<(buck1&31));
- }
- putdblock(b);
- return inuse;
-}
-
-static int
-issplit(Index *ix, u32int key, int d)
-{
- return bitmapop(ix, key, d, 0);
-}
-
-static int
-marksplit(Index *ix, u32int key, int d)
-{
- return bitmapop(ix, key, d, 1);
-}
-
-/*
- * find the number of the index section holding score.
- * it's not terrible to be wrong once in a while, so we just
- * do what the bitmap tells us and don't worry about the
- * bitmap being out of date.
- */
-static int
-indexsect2(Index *ix, u8int *score)
-{
- u32int key;
- int d, is;
-
- key = hashbits(score, 32);
- for(d=0; d<=ix->maxdepth; d++){
- is = issplit(ix, key, d);
- if(is == -1)
- return 0; /* must return something valid! */
- if(!is)
- break;
- }
-
- if(d > ix->maxdepth){
- seterr(EBug, "index bitmap inconsistent with maxdepth");
- return 0; /* must return something valid! */
- }
-
- return indexsect0(ix, keytobuck(ix, key, d));
-}
-
-/*
- * load the index block responsible for score.
- * (unlike indexsect2, must be correct always.)
- */
-static DBlock*
-loadibucket2(Index *ix, u8int *score, ISect **pis, u32int *pbuck, IBucket *ib)
-{
- u32int key;
- int d, try, is;
- DBlock *b;
-
- for(try=0; try<32; try++){
- key = hashbits(score, 32);
- for(d=0; d<=ix->maxdepth; d++){
- is = issplit(ix, key, d);
- if(is == -1)
- return nil;
- if(!is)
- break;
- }
- if(d > ix->maxdepth){
- seterr(EBug, "index bitmap inconsistent with maxdepth");
- return nil;
- }
-
- if((b = loadibucket0(ix, keytobuck(ix, key, d), pis, pbuck, ib, 1)) == nil)
- return nil;
-
- if(ib->depth == d)
- return b;
-
- if(ib->depth < d){
- fprint(2, "loaded block %ud for %d/%0*ub got %d/%0*ub\n",
- *pbuck, d, d, KEY(key,d), ib->depth, ib->depth, KEY(key, ib->depth));
- seterr(EBug, "index block has smaller depth than expected -- cannot happen");
- putdblock(b);
- return nil;
- }
-
- /*
- * ib->depth > d, meaning the bitmap was out of date.
- * fix the bitmap and try again. if we actually updated
- * the bitmap in splitiblock, this would only happen if
- * venti crashed at an inopportune moment. but this way
- * the code gets tested more.
- */
-if(0) fprint(2, "update bitmap: %d/%0*ub is split\n", d, d, KEY(key,d));
- putdblock(b);
- if(marksplit(ix, key, d) < 0)
- return nil;
- }
- seterr(EBug, "loadibucket2 failed to sync bitmap with disk!");
- return nil;
-}
-
-static int
-splitiblock(Index *ix, DBlock *b, ISect *is, u32int buck, IBucket *ib)
-{
- int i, d;
- u8int *score;
- u32int buck0, buck1, key0, key1, key, dmask;
- DBlock *b0, *b1;
- IBucket ib0, ib1;
-
- if(ib->depth == ix->maxdepth){
-if(0) fprint(2, "depth=%d == maxdepth\n", ib->depth);
- seterr(EAdmin, "index bucket %ud on %s is full\n", buck, is->part->name);
- putdblock(b);
- return -1;
- }
-
- buck = is->start+buck - ix->bitblocks;
- d = ib->depth+1;
- buck0 = buck;
- buck1 = buck0 | (1<<(ix->maxdepth-d));
- if(ix->maxdepth == 32){
- key0 = buck0;
- key1 = buck1;
- }else{
- key0 = buck0 << (32-ix->maxdepth);
- key1 = buck1 << (32-ix->maxdepth);
- }
- buck0 += ix->bitblocks;
- buck1 += ix->bitblocks;
- USED(buck0);
- USED(key1);
-
- if(d == 32)
- dmask = TWID32;
- else
- dmask = TWID32 ^ ((1<<(32-d))-1);
-
- /*
- * Since we hold the lock for b, the bitmap
- * thinks buck1 doesn't exist, and the bit
- * for buck1 can't be updated without first
- * locking and splitting b, it's safe to try to
- * acquire the lock on buck1 without dropping b.
- * No one else will go after it too.
- *
- * Also, none of the rest of the code ever locks
- * more than one block at a time, so it's okay if
- * we do.
- */
- if((b1 = loadibucket0(ix, buck1, nil, nil, &ib1, 0)) == nil){
- putdblock(b);
- return -1;
- }
- b0 = b;
- ib0 = *ib;
-
- /*
- * Funny locking going on here -- see dirtydblock.
- * We must putdblock(b1) before putdblock(b0).
- */
- dirtydblock(b0, DirtyIndex);
- dirtydblock(b1, DirtyIndexSplit);
-
- /*
- * Split the block contents.
- * The block is already sorted, so it's pretty easy:
- * the first part stays, the second part goes to b1.
- */
- ib0.n = 0;
- ib0.depth = d;
- ib1.n = 0;
- ib1.depth = d;
- for(i=0; i<ib->n; i++){
- score = ib->data+i*IEntrySize;
- key = hashbits(score, 32);
- if((key&dmask) != key0)
- break;
- }
- ib0.n = i;
- ib1.n = ib->n - ib0.n;
- memmove(ib1.data, ib0.data+ib0.n*IEntrySize, ib1.n*IEntrySize);
-if(0) fprint(2, "splitiblock %d in %d/%0*ub => %d in %d/%0*ub + %d in %d/%0*ub\n",
- ib->n, d-1, d-1, key0>>(32-d+1),
- ib0.n, d, d, key0>>(32-d),
- ib1.n, d, d, key1>>(32-d));
-
- packibucket(&ib0, b0->data);
- packibucket(&ib1, b1->data);
-
- /* order matters! see comment above. */
- putdblock(b1);
- putdblock(b0);
-
- /*
- * let the recovery code take care of updating the bitmap.
- */
-
- qlock(&stats.lock);
- stats.indexsplits++;
- qunlock(&stats.lock);
-
- return 0;
-}
-
-int
-indexsect(Index *ix, u8int *score)
-{
- if(ix->version == IndexVersion1)
- return indexsect1(ix, score);
- return indexsect2(ix, score);
-}
-
-DBlock*
-loadibucket(Index *ix, u8int *score, ISect **pis, u32int *pbuck, IBucket *ib)
-{
- if(ix->version == IndexVersion1)
- return loadibucket1(ix, score, pis, pbuck, ib);
- return loadibucket2(ix, score, pis, pbuck, ib);
-}
-
-
blob - 746a89cab3216029d45c05f0647c975d3e157de4 (mode 644)
blob + /dev/null
--- src/cmd/venti/lump.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-int queuewrites = 0;
-
-static Packet *readilump(Lump *u, IAddr *ia, u8int *score, int rac);
-
-Packet*
-readlump(u8int *score, int type, u32int size)
-{
- Lump *u;
- Packet *p;
- IAddr ia;
- u32int n;
- int rac;
-
- qlock(&stats.lock);
- stats.lumpreads++;
- qunlock(&stats.lock);
- u = lookuplump(score, type);
- if(u->data != nil){
- n = packetsize(u->data);
- if(n > size){
- seterr(EOk, "read too small: asked for %d need at least %d", size, n);
- putlump(u);
-
- return nil;
- }
- p = packetdup(u->data, 0, n);
- putlump(u);
-
- return p;
- }
-
- if(lookupscore(score, type, &ia, &rac) < 0){
- //ZZZ place to check for someone trying to guess scores
- seterr(EOk, "no block with that score exists");
-
- putlump(u);
- return nil;
- }
- if(ia.size > size){
- seterr(EOk, "read too small 1: asked for %d need at least %d", size, ia.size);
-
- putlump(u);
- return nil;
- }
-
- p = readilump(u, &ia, score, rac);
- putlump(u);
-
- return p;
-}
-
-/*
- * save away a lump, and return it's score.
- * doesn't store duplicates, but checks that the data is really the same.
- */
-int
-writelump(Packet *p, u8int *score, int type, u32int creator)
-{
- Lump *u;
- int ok;
-
- qlock(&stats.lock);
- stats.lumpwrites++;
- qunlock(&stats.lock);
-
- packetsha1(p, score);
-
- u = lookuplump(score, type);
- if(u->data != nil){
- ok = 0;
- if(packetcmp(p, u->data) != 0){
- seterr(EStrange, "score collision");
- ok = -1;
- }
- packetfree(p);
- putlump(u);
- return ok;
- }
-
- if(queuewrites)
- return queuewrite(u, p, creator);
-
- ok = writeqlump(u, p, creator);
-
- putlump(u);
- return ok;
-}
-
-int
-writeqlump(Lump *u, Packet *p, int creator)
-{
- ZBlock *flat;
- Packet *old;
- IAddr ia;
- int ok;
- int rac;
-
- if(lookupscore(u->score, u->type, &ia, &rac) == 0){
- /*
- * if the read fails,
- * assume it was corrupted data and store the block again
- */
- old = readilump(u, &ia, u->score, rac);
- if(old != nil){
- ok = 0;
- if(packetcmp(p, old) != 0){
- seterr(EStrange, "score collision");
- ok = -1;
- }
- packetfree(p);
- packetfree(old);
-
- return ok;
- }
- logerr(EAdmin, "writelump: read %V failed, rewriting: %r\n", u->score);
- }
-
- flat = packet2zblock(p, packetsize(p));
- ok = storeclump(mainindex, flat, u->score, u->type, creator, &ia);
- freezblock(flat);
- if(ok == 0)
- ok = insertscore(u->score, &ia, 1);
- if(ok == 0)
- insertlump(u, p);
- else
- packetfree(p);
-
- return ok;
-}
-
-static void
-readahead(u64int a, Arena *arena, u64int aa, int n)
-{
- u8int buf[ClumpSize];
- Clump cl;
- IAddr ia;
-
- while(n > 0) {
- if (aa >= arena->used)
- break;
- if(readarena(arena, aa, buf, ClumpSize) < ClumpSize)
- break;
- if(unpackclump(&cl, buf) < 0)
- break;
- ia.addr = a;
- ia.type = cl.info.type;
- ia.size = cl.info.uncsize;
- ia.blocks = (cl.info.size + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog;
- insertscore(cl.info.score, &ia, 0);
- a += ClumpSize + cl.info.size;
- aa += ClumpSize + cl.info.size;
- n--;
- }
-}
-
-static Packet*
-readilump(Lump *u, IAddr *ia, u8int *score, int rac)
-{
- Arena *arena;
- ZBlock *zb;
- Packet *p, *pp;
- Clump cl;
- u64int a, aa;
- u8int sc[VtScoreSize];
-
- arena = amapitoa(mainindex, ia->addr, &aa);
- if(arena == nil)
- return nil;
-
- zb = loadclump(arena, aa, ia->blocks, &cl, sc, paranoid);
- if(zb == nil)
- return nil;
-
- if(ia->size != cl.info.uncsize){
- seterr(EInconsist, "index and clump size mismatch");
- freezblock(zb);
- return nil;
- }
- if(ia->type != cl.info.type){
- seterr(EInconsist, "index and clump type mismatch");
- freezblock(zb);
- return nil;
- }
- if(scorecmp(score, sc) != 0){
- seterr(ECrash, "score mismatch");
- freezblock(zb);
- return nil;
- }
-
- if(rac == 0) {
- a = ia->addr + ClumpSize + cl.info.size;
- aa += ClumpSize + cl.info.size;
- readahead(a, arena, aa, 20);
- }
-
- p = zblock2packet(zb, cl.info.uncsize);
- freezblock(zb);
- pp = packetdup(p, 0, packetsize(p));
- insertlump(u, pp);
- return p;
-}
blob - 979271af37ec76db1768e707296da7a7ad3ff4ee (mode 644)
blob + /dev/null
--- src/cmd/venti/lumpcache.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-typedef struct LumpCache LumpCache;
-
-enum
-{
- HashLog = 9,
- HashSize = 1<<HashLog,
- HashMask = HashSize - 1,
-};
-
-struct LumpCache
-{
- QLock lock;
- Rendez full;
- Lump *free; /* list of available lumps */
- u32int allowed; /* total allowable space for packets */
- u32int avail; /* remaining space for packets */
- u32int now; /* ticks for usage timestamps */
- Lump **heads; /* hash table for finding address */
- int nheap; /* number of available victims */
- Lump **heap; /* heap for locating victims */
- int nblocks; /* number of blocks allocated */
- Lump *blocks; /* array of block descriptors */
-};
-
-static LumpCache lumpcache;
-
-static void delheap(Lump *db);
-static int downheap(int i, Lump *b);
-static void fixheap(int i, Lump *b);
-static int upheap(int i, Lump *b);
-static Lump *bumplump(void);
-
-void
-initlumpcache(u32int size, u32int nblocks)
-{
- Lump *last, *b;
- int i;
-
- lumpcache.full.l = &lumpcache.lock;
- lumpcache.nblocks = nblocks;
- lumpcache.allowed = size;
- lumpcache.avail = size;
- lumpcache.heads = MKNZ(Lump*, HashSize);
- lumpcache.heap = MKNZ(Lump*, nblocks);
- lumpcache.blocks = MKNZ(Lump, nblocks);
-
- last = nil;
- for(i = 0; i < nblocks; i++){
- b = &lumpcache.blocks[i];
- b->type = TWID8;
- b->heap = TWID32;
- b->next = last;
- last = b;
- }
- lumpcache.free = last;
- lumpcache.nheap = 0;
-}
-
-Lump*
-lookuplump(u8int *score, int type)
-{
- Lump *b;
- u32int h;
-
- h = hashbits(score, HashLog);
-
- /*
- * look for the block in the cache
- */
-//checklumpcache();
- qlock(&lumpcache.lock);
-again:
- for(b = lumpcache.heads[h]; b != nil; b = b->next){
- if(scorecmp(score, b->score)==0 && type == b->type){
- qlock(&stats.lock);
- stats.lumphit++;
- qunlock(&stats.lock);
- goto found;
- }
- }
-
- /*
- * missed: locate the block with the oldest second to last use.
- * remove it from the heap, and fix up the heap.
- */
- while(lumpcache.free == nil){
- if(bumplump() == nil){
- logerr(EAdmin, "all lump cache blocks in use");
- rsleep(&lumpcache.full);
- goto again;
- }
- }
- qlock(&stats.lock);
- stats.lumpmiss++;
- qunlock(&stats.lock);
-
- b = lumpcache.free;
- lumpcache.free = b->next;
-
- /*
- * the new block has no last use, so assume it happens sometime in the middle
-ZZZ this is not reasonable
- */
- b->used = (b->used2 + lumpcache.now) / 2;
-
- /*
- * rechain the block on the correct hash chain
- */
- b->next = lumpcache.heads[h];
- lumpcache.heads[h] = b;
- if(b->next != nil)
- b->next->prev = b;
- b->prev = nil;
-
- scorecp(b->score, score);
- b->type = type;
- b->size = 0;
- b->data = nil;
-
-found:
- b->ref++;
- b->used2 = b->used;
- b->used = lumpcache.now++;
- if(b->heap != TWID32)
- fixheap(b->heap, b);
- qunlock(&lumpcache.lock);
-
-//checklumpcache();
-
- qlock(&b->lock);
-
- return b;
-}
-
-void
-insertlump(Lump *b, Packet *p)
-{
- u32int size;
-
- /*
- * look for the block in the cache
- */
-//checklumpcache();
- qlock(&lumpcache.lock);
-again:
-
- /*
- * missed: locate the block with the oldest second to last use.
- * remove it from the heap, and fix up the heap.
- */
- size = packetasize(p);
-//ZZZ
- while(lumpcache.avail < size){
- if(bumplump() == nil){
- logerr(EAdmin, "all lump cache blocks in use");
- rsleep(&lumpcache.full);
- goto again;
- }
- }
- b->data = p;
- b->size = size;
- lumpcache.avail -= size;
-
- qunlock(&lumpcache.lock);
-//checklumpcache();
-}
-
-void
-putlump(Lump *b)
-{
- if(b == nil)
- return;
-
- qunlock(&b->lock);
-//checklumpcache();
- qlock(&lumpcache.lock);
- if(--b->ref == 0){
- if(b->heap == TWID32)
- upheap(lumpcache.nheap++, b);
- rwakeup(&lumpcache.full);
- }
-
- qunlock(&lumpcache.lock);
-//checklumpcache();
-}
-
-/*
- * remove some lump from use and update the free list and counters
- */
-static Lump*
-bumplump(void)
-{
- Lump *b;
- u32int h;
-
- /*
- * remove blocks until we find one that is unused
- * referenced blocks are left in the heap even though
- * they can't be scavenged; this is simple a speed optimization
- */
- for(;;){
- if(lumpcache.nheap == 0)
- return nil;
- b = lumpcache.heap[0];
- delheap(b);
- if(!b->ref){
- rwakeup(&lumpcache.full);
- break;
- }
- }
-
- /*
- * unchain the block
- */
- if(b->prev == nil){
- h = hashbits(b->score, HashLog);
- if(lumpcache.heads[h] != b)
- sysfatal("bad hash chains in lump cache");
- lumpcache.heads[h] = b->next;
- }else
- b->prev->next = b->next;
- if(b->next != nil)
- b->next->prev = b->prev;
-
- if(b->data != nil){
- packetfree(b->data);
- b->data = nil;
- lumpcache.avail += b->size;
- b->size = 0;
- }
- b->type = TWID8;
-
- b->next = lumpcache.free;
- lumpcache.free = b;
-
- return b;
-}
-
-/*
- * delete an arbitrary block from the heap
- */
-static void
-delheap(Lump *db)
-{
- fixheap(db->heap, lumpcache.heap[--lumpcache.nheap]);
- db->heap = TWID32;
-}
-
-/*
- * push an element up or down to it's correct new location
- */
-static void
-fixheap(int i, Lump *b)
-{
- if(upheap(i, b) == i)
- downheap(i, b);
-}
-
-static int
-upheap(int i, Lump *b)
-{
- Lump *bb;
- u32int now;
- int p;
-
- now = lumpcache.now;
- for(; i != 0; i = p){
- p = (i - 1) >> 1;
- bb = lumpcache.heap[p];
- if(b->used2 - now >= bb->used2 - now)
- break;
- lumpcache.heap[i] = bb;
- bb->heap = i;
- }
-
- lumpcache.heap[i] = b;
- b->heap = i;
- return i;
-}
-
-static int
-downheap(int i, Lump *b)
-{
- Lump *bb;
- u32int now;
- int k;
-
- now = lumpcache.now;
- for(; ; i = k){
- k = (i << 1) + 1;
- if(k >= lumpcache.nheap)
- break;
- if(k + 1 < lumpcache.nheap && lumpcache.heap[k]->used2 - now > lumpcache.heap[k + 1]->used2 - now)
- k++;
- bb = lumpcache.heap[k];
- if(b->used2 - now <= bb->used2 - now)
- break;
- lumpcache.heap[i] = bb;
- bb->heap = i;
- }
-
- lumpcache.heap[i] = b;
- b->heap = i;
- return i;
-}
-
-static void
-findblock(Lump *bb)
-{
- Lump *b, *last;
- int h;
-
- last = nil;
- h = hashbits(bb->score, HashLog);
- for(b = lumpcache.heads[h]; b != nil; b = b->next){
- if(last != b->prev)
- sysfatal("bad prev link");
- if(b == bb)
- return;
- last = b;
- }
- sysfatal("block score=%V type=%#x missing from hash table", bb->score, bb->type);
-}
-
-void
-checklumpcache(void)
-{
- Lump *b;
- u32int size, now, nfree;
- int i, k, refed;
-
- qlock(&lumpcache.lock);
- now = lumpcache.now;
- for(i = 0; i < lumpcache.nheap; i++){
- if(lumpcache.heap[i]->heap != i)
- sysfatal("lc: mis-heaped at %d: %d", i, lumpcache.heap[i]->heap);
- if(i > 0 && lumpcache.heap[(i - 1) >> 1]->used2 - now > lumpcache.heap[i]->used2 - now)
- sysfatal("lc: bad heap ordering");
- k = (i << 1) + 1;
- if(k < lumpcache.nheap && lumpcache.heap[i]->used2 - now > lumpcache.heap[k]->used2 - now)
- sysfatal("lc: bad heap ordering");
- k++;
- if(k < lumpcache.nheap && lumpcache.heap[i]->used2 - now > lumpcache.heap[k]->used2 - now)
- sysfatal("lc: bad heap ordering");
- }
-
- refed = 0;
- size = 0;
- for(i = 0; i < lumpcache.nblocks; i++){
- b = &lumpcache.blocks[i];
- if(b->data == nil && b->size != 0)
- sysfatal("bad size: %d data=%p", b->size, b->data);
- if(b->ref && b->heap == TWID32)
- refed++;
- if(b->type != TWID8){
- findblock(b);
- size += b->size;
- }
- if(b->heap != TWID32
- && lumpcache.heap[b->heap] != b)
- sysfatal("lc: spurious heap value");
- }
- if(lumpcache.avail != lumpcache.allowed - size)
- sysfatal("mismatched available=%d and allowed=%d - used=%d space", lumpcache.avail, lumpcache.allowed, size);
-
- nfree = 0;
- for(b = lumpcache.free; b != nil; b = b->next){
- if(b->type != TWID8 || b->heap != TWID32)
- sysfatal("lc: bad free list");
- nfree++;
- }
-
- if(lumpcache.nheap + nfree + refed != lumpcache.nblocks)
- sysfatal("lc: missing blocks: %d %d %d %d", lumpcache.nheap, refed, nfree, lumpcache.nblocks);
- qunlock(&lumpcache.lock);
-}
blob - 1bb8841ee2111ce867bb925d555fa547b74dd49c (mode 644)
blob + /dev/null
--- src/cmd/venti/lumpqueue.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-typedef struct LumpQueue LumpQueue;
-typedef struct WLump WLump;
-
-enum
-{
- MaxLumpQ = 1 << 3 /* max. lumps on a single write queue, must be pow 2 */
-};
-
-struct WLump
-{
- Lump *u;
- Packet *p;
- int creator;
- int gen;
-};
-
-struct LumpQueue
-{
- QLock lock;
- Rendez flush;
- Rendez full;
- Rendez empty;
- WLump q[MaxLumpQ];
- int w;
- int r;
-};
-
-static LumpQueue *lumpqs;
-static int nqs;
-
-static QLock glk;
-static int gen;
-
-static void queueproc(void *vq);
-
-int
-initlumpqueues(int nq)
-{
- LumpQueue *q;
-
- int i;
- nqs = nq;
-
- lumpqs = MKNZ(LumpQueue, nq);
-
- for(i = 0; i < nq; i++){
- q = &lumpqs[i];
- q->full.l = &q->lock;
- q->empty.l = &q->lock;
- q->flush.l = &q->lock;
-
- if(vtproc(queueproc, q) < 0){
- seterr(EOk, "can't start write queue slave: %r");
- return -1;
- }
- }
-
- return 0;
-}
-
-/*
- * queue a lump & it's packet data for writing
- */
-int
-queuewrite(Lump *u, Packet *p, int creator)
-{
- LumpQueue *q;
- int i;
-
- i = indexsect(mainindex, u->score);
- if(i < 0 || i >= nqs){
- seterr(EBug, "internal error: illegal index section in queuewrite");
- return -1;
- }
-
- q = &lumpqs[i];
-
- qlock(&q->lock);
- while(q->r == ((q->w + 1) & (MaxLumpQ - 1)))
- rsleep(&q->full);
-
- q->q[q->w].u = u;
- q->q[q->w].p = p;
- q->q[q->w].creator = creator;
- q->q[q->w].gen = gen;
- q->w = (q->w + 1) & (MaxLumpQ - 1);
-
- rwakeup(&q->empty);
-
- qunlock(&q->lock);
-
- return 0;
-}
-
-void
-flushqueue(void)
-{
- int i;
- LumpQueue *q;
-
- if(!lumpqs)
- return;
-
- qlock(&glk);
- gen++;
- qunlock(&glk);
-
- for(i=0; i<mainindex->nsects; i++){
- q = &lumpqs[i];
- qlock(&q->lock);
- while(q->w != q->r && gen - q->q[q->r].gen > 0)
- rsleep(&q->flush);
- qunlock(&q->lock);
- }
-}
-
-static void
-queueproc(void *vq)
-{
- LumpQueue *q;
- Lump *u;
- Packet *p;
- int creator;
-
- q = vq;
- for(;;){
- qlock(&q->lock);
- while(q->w == q->r)
- rsleep(&q->empty);
-
- u = q->q[q->r].u;
- p = q->q[q->r].p;
- creator = q->q[q->r].creator;
-
- rwakeup(&q->full);
-
- qunlock(&q->lock);
-
- if(writeqlump(u, p, creator) < 0)
- fprint(2, "failed to write lump for %V: %r", u->score);
-
- qlock(&q->lock);
- q->r = (q->r + 1) & (MaxLumpQ - 1);
- rwakeup(&q->flush);
- qunlock(&q->lock);
-
- putlump(u);
- }
-}
blob - c6f9ef446fb3d0e3ad7276a5814a86d6cfa5c629
blob + 1a1b652da92eb4ef33db5b674f3a5b4826e3eba3
--- src/cmd/venti/mkfile
+++ src/cmd/venti/mkfile
-<$PLAN9/src/mkhdr
+%:VQ:
+ echo venti will return once it is debugged.
-LIBOFILES=\
- arena.$O\
- arenas.$O\
- buildbuck.$O\
- clump.$O\
- config.$O\
- conv.$O\
- dcache.$O\
- dump.$O\
- httpd.$O\
- icache.$O\
- ifile.$O\
- index.$O\
- lump.$O\
- lumpcache.$O\
- lumpqueue.$O\
- part.$O\
- score.$O\
- sortientry.$O\
- stats.$O\
- syncarena.$O\
- syncindex0.$O\
- unwhack.$O\
- utils.$O\
- unittoull.$O\
- whack.$O\
- xml.$O\
- zeropart.$O\
-
-SLIB=libvs.a
-
-LIB=$SLIB
-
-HFILES= dat.h\
- fns.h\
- stdinc.h\
-
-TARG=\
- venti\
- fmtarenas\
- fmtisect\
- fmtindex\
-# buildindex\
- checkarenas\
-# checkindex\
- clumpstats\
- findscore\
- rdarena\
- wrarena\
- syncindex\
- verifyarena\
- sync\
- read\
- write\
- copy\
- wb\
-
-BIN=$BIN/venti
-
-it:V: all
-
-<$PLAN9/src/mkmany
-
-$SLIB: $LIBOFILES
- $AR rvc $SLIB $LIBOFILES
-
-# xml.c:D: mkxml dat.h
-# ./mkxml dat.h > xml.c
-
-ainstall:V: ${TARG:%=%.ainstall}
-
-%.ainstall:V: $O.%
- scp $prereq amsterdam:/usr/local/bin/venti/$stem
-
blob - 67e5d0e98c65750e415268b54de1244047a2a0dd (mode 644)
blob + /dev/null
--- src/cmd/venti/mkroot.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-char *host;
-
-void
-usage(void)
-{
- fprint(2, "usage: mkroot [-h host] name type score blocksize prev\n");
- threadexitsall("usage");
-}
-
-void
-threadmain(int argc, char *argv[])
-{
- uchar score[VtScoreSize];
- uchar buf[VtRootSize];
- VtConn *z;
- VtRoot root;
-
- ARGBEGIN{
- case 'h':
- host = EARGF(usage());
- break;
- default:
- usage();
- break;
- }ARGEND
-
- if(argc != 5)
- usage();
-
- fmtinstall('V', vtscorefmt);
-
- strecpy(root.name, root.name+sizeof root.name, argv[0]);
- strecpy(root.type, root.type+sizeof root.type, argv[1]);
- if(vtparsescore(argv[2], strlen(argv[2]), nil, root.score) < 0)
- sysfatal("bad score '%s'", argv[2]);
- root.blocksize = atoi(argv[3]);
- if(vtparsescore(argv[4], strlen(argv[4]), nil, root.prev) < 0)
- sysfatal("bad score '%s'", argv[4]);
- vtrootpack(&root, buf);
-
- z = vtdial(host);
- if(z == nil)
- sysfatal("could not connect to server: %r");
-
- if(vtconnect(z) < 0)
- sysfatal("vtconnect: %r");
-
- if(vtwrite(z, score, VtRootType, buf, VtRootSize) < 0)
- sysfatal("vtwrite: %r");
- if(vtsync(z) < 0)
- sysfatal("vtsync: %r");
- vthangup(z);
- print("%V\n", score);
- threadexitsall(0);
-}
blob - 66e6c79a3b17b293bdb9258e27c22203426859ba (mode 644)
blob + /dev/null
--- src/cmd/venti/part.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-#define trace 1
-
-u32int maxblocksize;
-int readonly;
-
-Part*
-initpart(char *name, int writable)
-{
- Part *part;
- Dir *dir;
- int how;
-
- part = MKZ(Part);
- part->name = estrdup(name);
- if(!writable && readonly)
- how = OREAD;
- else
- how = ORDWR;
- part->fd = open(name, how);
- if(part->fd < 0){
- if(how == ORDWR)
- part->fd = open(name, OREAD);
- if(part->fd < 0){
- freepart(part);
- seterr(EOk, "can't open partition='%s': %r", name);
- return nil;
- }
- fprint(2, "warning: %s opened for reading only\n", name);
- }
- dir = dirfstat(part->fd);
- if(dir == nil){
- freepart(part);
- seterr(EOk, "can't stat partition='%s': %r", name);
- return nil;
- }
- part->size = dir->length;
- part->blocksize = 0;
- free(dir);
- return part;
-}
-
-void
-freepart(Part *part)
-{
- if(part == nil)
- return;
- close(part->fd);
- free(part->name);
- free(part);
-}
-
-void
-partblocksize(Part *part, u32int blocksize)
-{
- if(part->blocksize)
- sysfatal("resetting partition=%s's block size", part->name);
- part->blocksize = blocksize;
- if(blocksize > maxblocksize)
- maxblocksize = blocksize;
-}
-
-int
-writepart(Part *part, u64int addr, u8int *buf, u32int n)
-{
- long m, mm, nn;
-
- qlock(&stats.lock);
- stats.diskwrites++;
- stats.diskbwrites += n;
- qunlock(&stats.lock);
-
- if(addr > part->size || addr + n > part->size){
- seterr(ECorrupt, "out of bounds write to partition='%s'", part->name);
- return -1;
- }
-if(trace) print("write %s %lud at %llud\n", part->name, n, addr);
- for(nn = 0; nn < n; nn += m){
- mm = n - nn;
- if(mm > MaxIo)
- mm = MaxIo;
- m = pwrite(part->fd, &buf[nn], mm, addr + nn);
- if(m != mm){
- if(m < 0){
- seterr(EOk, "can't write partition='%s': %r", part->name);
- return -1;
- }
- logerr(EOk, "truncated write to partition='%s' n=%ld wrote=%ld", part->name, mm, m);
- }
- }
- return 0;
-}
-
-int
-readpart(Part *part, u64int addr, u8int *buf, u32int n)
-{
- long m, mm, nn;
- int i;
-
- qlock(&stats.lock);
- stats.diskreads++;
- stats.diskbreads += n;
- qunlock(&stats.lock);
-
- if(addr > part->size || addr + n > part->size){
- seterr(ECorrupt, "out of bounds read from partition='%s': addr=%lld n=%d size=%lld", part->name, addr, n, part->size);
- return -1;
- }
-if(trace) print("read %s %lud at %llud\n", part->name, n, addr);
- for(nn = 0; nn < n; nn += m){
- mm = n - nn;
- if(mm > MaxIo)
- mm = MaxIo;
- m = -1;
- for(i=0; i<4; i++) {
- m = pread(part->fd, &buf[nn], mm, addr + nn);
- if(m == mm)
- break;
- }
- if(m != mm){
- if(m < 0){
- seterr(EOk, "can't read partition='%s': %r", part->name);
- return -1;
- }
- logerr(EOk, "warning: truncated read from partition='%s' n=%ld read=%ld", part->name, mm, m);
- }
- }
- return 0;
-}
blob - 95121a9f6ac4b124552a6293cb076a8d48f8543e (mode 644)
blob + /dev/null
--- src/cmd/venti/printarena.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-int readonly = 1; /* for part.c */
-
-void
-usage(void)
-{
- fprint(2, "usage: printarena arenafile [offset]\n");
- threadexitsall("usage");
-}
-
-static void
-rdarena(Arena *arena, u64int offset)
-{
- u64int a, aa, e;
- u32int magic;
- Clump cl;
- uchar score[VtScoreSize];
- ZBlock *lump;
-
- printarena(2, arena);
-
- a = arena->base;
- e = arena->base + arena->size;
- if(offset != ~(u64int)0) {
- if(offset >= e-a)
- sysfatal("bad offset %llud >= %llud\n",
- offset, e-a);
- aa = offset;
- } else
- aa = 0;
-
- for(; aa < e; aa += ClumpSize+cl.info.size) {
- magic = clumpmagic(arena, aa);
- if(magic == ClumpFreeMagic)
- break;
- if(magic != ClumpMagic) {
- fprint(2, "illegal clump magic number %#8.8ux offset %llud\n",
- magic, aa);
- break;
- }
- lump = loadclump(arena, aa, 0, &cl, score, 0);
- if(lump == nil) {
- fprint(2, "clump %llud failed to read: %r\n", aa);
- break;
- }
- if(cl.info.type != VtTypeCorrupt) {
- scoremem(score, lump->data, cl.info.uncsize);
- if(scorecmp(cl.info.score, score) != 0) {
- fprint(2, "clump %llud has mismatched score\n", aa);
- break;
- }
- if(vttypevalid(cl.info.type) < 0) {
- fprint(2, "clump %llud has bad type %d\n", aa, cl.info.type);
- break;
- }
- }
- print("%llud %V %d %d\n", aa, score, cl.info.type, cl.info.uncsize);
- freezblock(lump);
- }
- print("end offset %llud\n", aa);
-}
-
-void
-threadmain(int argc, char *argv[])
-{
- char *file;
- Arena *arena;
- u64int offset, aoffset;
- Part *part;
- Dir *d;
- uchar buf[8192];
- ArenaHead head;
-
- aoffset = 0;
- ARGBEGIN{
- case 'o':
- aoffset = strtoull(EARGF(usage()), 0, 0);
- break;
- default:
- usage();
- break;
- }ARGEND
-
- offset = ~(u64int)0;
- switch(argc) {
- default:
- usage();
- case 2:
- offset = strtoull(argv[1], 0, 0);
- /* fall through */
- case 1:
- file = argv[0];
- }
-
-
- fmtinstall('V', vtscorefmt);
-
- statsinit();
-
- if((d = dirstat(file)) == nil)
- sysfatal("can't stat file %s: %r", file);
-
- part = initpart(file, 0);
- if(part == nil)
- sysfatal("can't open file %s: %r", file);
- if(readpart(part, aoffset, buf, sizeof buf) < 0)
- sysfatal("can't read file %s: %r", file);
-
- if(unpackarenahead(&head, buf) < 0)
- sysfatal("corrupted arena header: %r");
-
- if(aoffset+head.size > d->length)
- sysfatal("arena is truncated: want %llud bytes have %llud\n",
- head.size, d->length);
-
- partblocksize(part, head.blocksize);
- initdcache(8 * MaxDiskBlock);
-
- arena = initarena(part, aoffset, head.size, head.blocksize);
- if(arena == nil)
- sysfatal("initarena: %r");
-
- rdarena(arena, offset);
- threadexitsall(0);
-}
blob - 8d130def883a24afe8c837dd1a7fe1a128cdc8ce (mode 644)
blob + /dev/null
--- src/cmd/venti/rdarena.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-static int verbose;
-
-void
-usage(void)
-{
- fprint(2, "usage: rdarena [-v] arenapart arena\n");
- threadexitsall(0);
-}
-
-static void
-rdarena(Arena *arena)
-{
- ZBlock *b;
- u64int a, e;
- u32int bs;
-
- fprint(2, "copying %s to standard output\n", arena->name);
- printarena(2, arena);
-
- bs = MaxIoSize;
- if(bs < arena->blocksize)
- bs = arena->blocksize;
-
- b = alloczblock(bs, 0);
- e = arena->base + arena->size + arena->blocksize;
- for(a = arena->base - arena->blocksize; a + arena->blocksize <= e; a += bs){
- if(a + bs > e)
- bs = arena->blocksize;
- if(readpart(arena->part, a, b->data, bs) < 0)
- fprint(2, "can't copy %s, read at %lld failed: %r\n", arena->name, a);
- if(write(1, b->data, bs) != bs)
- sysfatal("can't copy %s, write at %lld failed: %r", arena->name, a);
- }
-
- freezblock(b);
-}
-
-void
-threadmain(int argc, char *argv[])
-{
- ArenaPart *ap;
- Part *part;
- char *file, *aname;
- int i;
-
- fmtinstall('V', vtscorefmt);
- statsinit();
-
- ARGBEGIN{
- case 'v':
- verbose++;
- break;
- default:
- usage();
- break;
- }ARGEND
-
- readonly = 1;
-
- if(argc != 2)
- usage();
-
- file = argv[0];
- aname = argv[1];
-
- part = initpart(file, 0);
- if(part == nil)
- sysfatal("can't open partition %s: %r", file);
-
- ap = initarenapart(part);
- if(ap == nil)
- sysfatal("can't initialize arena partition in %s: %r", file);
-
- if(verbose)
- printarenapart(2, ap);
-
- initdcache(8 * MaxDiskBlock);
-
- for(i = 0; i < ap->narenas; i++){
- if(strcmp(ap->arenas[i]->name, aname) == 0){
- rdarena(ap->arenas[i]);
- threadexitsall(0);
- }
- }
-
- sysfatal("couldn't find arena %s\n", aname);
-}
blob - 969a05934b6c47b2f870ca0157d454f542b31bd1 (mode 644)
blob + /dev/null
--- src/cmd/venti/read.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-char *host;
-
-void
-usage(void)
-{
- fprint(2, "usage: read [-h host] score [type]\n");
- threadexitsall("usage");
-}
-
-int
-parsescore(uchar *score, char *buf, int n)
-{
- int i, c;
-
- memset(score, 0, VtScoreSize);
-
- if(n != VtScoreSize*2){
- werrstr("score wrong length %d", n);
- return -1;
- }
- for(i=0; i<VtScoreSize*2; i++) {
- if(buf[i] >= '0' && buf[i] <= '9')
- c = buf[i] - '0';
- else if(buf[i] >= 'a' && buf[i] <= 'f')
- c = buf[i] - 'a' + 10;
- else if(buf[i] >= 'A' && buf[i] <= 'F')
- c = buf[i] - 'A' + 10;
- else {
- c = buf[i];
- werrstr("bad score char %d '%c'", c, c);
- return -1;
- }
-
- if((i & 1) == 0)
- c <<= 4;
-
- score[i>>1] |= c;
- }
- return 0;
-}
-
-void
-threadmain(int argc, char *argv[])
-{
- int type, n;
- uchar score[VtScoreSize];
- uchar *buf;
- VtConn *z;
-
- ARGBEGIN{
- case 'h':
- host = EARGF(usage());
- break;
- default:
- usage();
- break;
- }ARGEND
-
- if(argc != 1 && argc != 2)
- usage();
-
-
- fmtinstall('V', vtscorefmt);
-
- if(parsescore(score, argv[0], strlen(argv[0])) < 0)
- sysfatal("could not parse score '%s': %r", argv[0]);
-
- buf = vtmallocz(VtMaxLumpSize);
-
- z = vtdial(host);
- if(z == nil)
- sysfatal("could not connect to server: %r");
-
- if(vtconnect(z) < 0)
- sysfatal("vtconnect: %r");
-
- if(argc == 1){
- n = -1;
- for(type=0; type<VtMaxType; type++){
- n = vtread(z, score, type, buf, VtMaxLumpSize);
- if(n >= 0){
- fprint(2, "venti/read%s%s %V %d\n", host ? " -h" : "", host ? host : "",
- score, type);
- break;
- }
- }
- }else{
- type = atoi(argv[1]);
- n = vtread(z, score, type, buf, VtMaxLumpSize);
- }
- vthangup(z);
- if(n < 0)
- sysfatal("could not read block: %r");
- if(write(1, buf, n) != n)
- sysfatal("write: %r");
-
- threadexitsall(0);
-}
blob - 0809e84f6468380cf41efd74b55daab069c16ae6 (mode 644)
blob + /dev/null
--- src/cmd/venti/score.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-u8int zeroscore[VtScoreSize];
-
-void
-scoremem(u8int *score, u8int *buf, int n)
-{
- DigestState s;
-
- memset(&s, 0, sizeof s);
- sha1(buf, n, score, &s);
-}
-
-static int
-hexv(int c)
-{
- if(c >= '0' && c <= '9')
- return c - '0';
- if(c >= 'a' && c <= 'f')
- return c - 'a' + 10;
- if(c >= 'A' && c <= 'F')
- return c - 'A' + 10;
- return -1;
-}
-
-int
-strscore(char *s, u8int *score)
-{
- int i, c, d;
-
- for(i = 0; i < VtScoreSize; i++){
- c = hexv(s[2 * i]);
- if(c < 0)
- return -1;
- d = hexv(s[2 * i + 1]);
- if(d < 0)
- return -1;
- score[i] = (c << 4) + d;
- }
- return s[2 * i] == '\0';
-}
blob - 0e9f4b1e721d5741c76d972388f66559351b5cd1 (mode 644)
blob + /dev/null
--- src/cmd/venti/sortientry.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-typedef struct IEBuck IEBuck;
-typedef struct IEBucks IEBucks;
-
-enum
-{
- ClumpChunks = 32*1024
-};
-
-struct IEBuck
-{
- u32int head; /* head of chain of chunks on the disk */
- u32int used; /* usage of the last chunk */
- u64int total; /* total number of bytes in this bucket */
- u8int *buf; /* chunk of entries for this bucket */
-};
-
-struct IEBucks
-{
- Part *part;
- u64int off; /* offset for writing data in the partition */
- u32int chunks; /* total chunks written to fd */
- u64int max; /* max bytes entered in any one bucket */
- int bits; /* number of bits in initial bucket sort */
- int nbucks; /* 1 << bits, the number of buckets */
- u32int size; /* bytes in each of the buckets chunks */
- u32int usable; /* amount usable for IEntry data */
- u8int *buf; /* buffer for all chunks */
- IEBuck *bucks;
-};
-
-#define U32GET(p) (((p)[0]<<24)|((p)[1]<<16)|((p)[2]<<8)|(p)[3])
-#define U32PUT(p,v) (p)[0]=(v)>>24;(p)[1]=(v)>>16;(p)[2]=(v)>>8;(p)[3]=(v)
-
-static IEBucks *initiebucks(Part *part, int bits, u32int size);
-static int flushiebuck(IEBucks *ib, int b, int reset);
-static int flushiebucks(IEBucks *ib);
-static u32int sortiebuck(IEBucks *ib, int b);
-static u64int sortiebucks(IEBucks *ib);
-static int sprayientry(IEBucks *ib, IEntry *ie);
-static u32int readarenainfo(IEBucks *ib, Arena *arena, u64int a);
-static u32int readiebuck(IEBucks *ib, int b);
-static void freeiebucks(IEBucks *ib);
-
-/*
- * build a sorted file with all Ientries which should be in ix.
- * assumes the arenas' directories are up to date.
- * reads each, converts the entries to index entries,
- * and sorts them.
- */
-u64int
-sortrawientries(Index *ix, Part *tmp, u64int *base)
-{
- IEBucks *ib;
- u64int clumps, sorted;
- u32int n;
- int i, ok;
-
-//ZZZ should allow configuration of bits, bucket size
- ib = initiebucks(tmp, 8, 64*1024);
- if(ib == nil){
- seterr(EOk, "can't create sorting buckets: %r");
- return TWID64;
- }
- ok = 0;
- clumps = 0;
- for(i = 0; i < ix->narenas; i++){
- n = readarenainfo(ib, ix->arenas[i], ix->amap[i].start);
- if(n == TWID32){
- ok = -1;
- break;
- }
- clumps += n;
- }
-fprint(2, "got %lld clumps - starting sort\n", clumps);
- if(ok){
- sorted = sortiebucks(ib);
- *base = (u64int)ib->chunks * ib->size;
- if(sorted != clumps){
- fprint(2, "sorting messed up: clumps=%lld sorted=%lld\n", clumps, sorted);
- ok = -1;
- }
- }
- freeiebucks(ib);
- if(ok < 0)
- return TWID64;
- return clumps;
-}
-
-/*
- * read in all of the arena's clump directory,
- * convert to IEntry format, and bucket sort based
- * on the first few bits.
- */
-static u32int
-readarenainfo(IEBucks *ib, Arena *arena, u64int a)
-{
- IEntry ie;
- ClumpInfo *ci, *cis;
- u32int clump;
- int i, n, ok;
-
-//ZZZ remove fprint?
- if(arena->clumps)
- fprint(2, "reading directory for arena=%s with %d entries\n", arena->name, arena->clumps);
-
- cis = MKN(ClumpInfo, ClumpChunks);
- ok = 0;
- memset(&ie, 0, sizeof(IEntry));
- for(clump = 0; clump < arena->clumps; clump += n){
- n = ClumpChunks;
- if(n > arena->clumps - clump)
- n = arena->clumps - clump;
- if(readclumpinfos(arena, clump, cis, n) != n){
- seterr(EOk, "arena directory read failed: %r");
- ok = -1;
- break;
- }
-
- for(i = 0; i < n; i++){
- ci = &cis[i];
- ie.ia.type = ci->type;
- ie.ia.size = ci->uncsize;
- ie.ia.addr = a;
- a += ci->size + ClumpSize;
- ie.ia.blocks = (ci->size + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog;
- scorecp(ie.score, ci->score);
- sprayientry(ib, &ie);
- }
- }
- free(cis);
- if(ok < 0)
- return TWID32;
- return clump;
-}
-
-/*
- * initialize the external bucket sorting data structures
- */
-static IEBucks*
-initiebucks(Part *part, int bits, u32int size)
-{
- IEBucks *ib;
- int i;
-
- ib = MKZ(IEBucks);
- if(ib == nil){
- seterr(EOk, "out of memory");
- return nil;
- }
- ib->bits = bits;
- ib->nbucks = 1 << bits;
- ib->size = size;
- ib->usable = (size - U32Size) / IEntrySize * IEntrySize;
- ib->bucks = MKNZ(IEBuck, ib->nbucks);
- if(ib->bucks == nil){
- seterr(EOk, "out of memory allocation sorting buckets");
- freeiebucks(ib);
- return nil;
- }
- ib->buf = MKN(u8int, size * (1 << bits));
- if(ib->buf == nil){
- seterr(EOk, "out of memory allocating sorting buckets' buffers");
- freeiebucks(ib);
- return nil;
- }
- for(i = 0; i < ib->nbucks; i++){
- ib->bucks[i].head = TWID32;
- ib->bucks[i].buf = &ib->buf[i * size];
- }
- ib->part = part;
- return ib;
-}
-
-static void
-freeiebucks(IEBucks *ib)
-{
- if(ib == nil)
- return;
- free(ib->bucks);
- free(ib->buf);
- free(ib);
-}
-
-/*
- * initial sort: put the entry into the correct bucket
- */
-static int
-sprayientry(IEBucks *ib, IEntry *ie)
-{
- u32int n;
- int b;
-
- b = hashbits(ie->score, ib->bits);
- n = ib->bucks[b].used;
- packientry(ie, &ib->bucks[b].buf[n]);
- n += IEntrySize;
- ib->bucks[b].used = n;
- if(n + IEntrySize <= ib->usable)
- return 0;
- return flushiebuck(ib, b, 1);
-}
-
-/*
- * finish sorting:
- * for each bucket, read it in and sort it
- * write out the the final file
- */
-static u64int
-sortiebucks(IEBucks *ib)
-{
- u64int tot;
- u32int n;
- int i;
-
- if(flushiebucks(ib) < 0)
- return TWID64;
- for(i = 0; i < ib->nbucks; i++)
- ib->bucks[i].buf = nil;
- ib->off = (u64int)ib->chunks * ib->size;
- free(ib->buf);
-fprint(2, "ib->max = %lld\n", ib->max);
-fprint(2, "ib->chunks = %ud\n", ib->chunks);
- ib->buf = MKN(u8int, ib->max + U32Size);
- if(ib->buf == nil){
- seterr(EOk, "out of memory allocating final sorting buffer; try more buckets");
- return TWID64;
- }
- tot = 0;
- for(i = 0; i < ib->nbucks; i++){
- n = sortiebuck(ib, i);
- if(n == TWID32)
- return TWID64;
- tot += n;
- }
- return tot;
- return 0;
-}
-
-/*
- * sort from bucket b of ib into the output file to
- */
-static u32int
-sortiebuck(IEBucks *ib, int b)
-{
- u32int n;
-
- n = readiebuck(ib, b);
- if(n == TWID32)
- return TWID32;
- qsort(ib->buf, n, IEntrySize, ientrycmp);
- if(writepart(ib->part, ib->off, ib->buf, n * IEntrySize) < 0){
- seterr(EOk, "can't write sorted bucket: %r");
- return TWID32;
- }
- ib->off += n * IEntrySize;
- return n;
-}
-
-/*
- * write out a single bucket
- */
-static int
-flushiebuck(IEBucks *ib, int b, int reset)
-{
- u32int n;
-
- if(ib->bucks[b].used == 0)
- return 0;
- n = ib->bucks[b].used;
- U32PUT(&ib->bucks[b].buf[n], ib->bucks[b].head);
- n += U32Size;
- if(writepart(ib->part, (u64int)ib->chunks * ib->size, ib->bucks[b].buf, n) < 0){
- seterr(EOk, "can't write sorting bucket to file: %r");
- return -1;
- }
- ib->bucks[b].head = ib->chunks++;
- ib->bucks[b].total += ib->bucks[b].used;
- if(reset)
- ib->bucks[b].used = 0;
- return 0;
-}
-
-/*
- * write out all of the buckets, and compute
- * the maximum size of any bucket
- */
-static int
-flushiebucks(IEBucks *ib)
-{
- int i;
-
- for(i = 0; i < ib->nbucks; i++){
- if(flushiebuck(ib, i, 0) < 0)
- return -1;
- if(ib->bucks[i].total > ib->max)
- ib->max = ib->bucks[i].total;
- }
- return 0;
-}
-
-/*
- * read in the chained buffers for bucket b,
- * and return it's total number of IEntries
- */
-static u32int
-readiebuck(IEBucks *ib, int b)
-{
- u32int head, n, m;
-
- head = ib->bucks[b].head;
- n = 0;
- m = ib->bucks[b].used;
- if(m == 0)
- m = ib->usable;
-fprint(2, "%d total = %lld\n", b, ib->bucks[b].total);
- while(head != TWID32){
- if(readpart(ib->part, (u64int)head * ib->size, &ib->buf[n], m + U32Size) < 0){
-fprint(2, "n = %ud\n", n);
- seterr(EOk, "can't read index sort bucket: %r");
- return TWID32;
- }
- n += m;
- head = U32GET(&ib->buf[n]);
- m = ib->usable;
- }
-fprint(2, "n = %ud\n", n);
- return n / IEntrySize;
-}
blob - 079fd97b624b1749de8afb58d489cefbebe7e8fb (mode 644)
blob + /dev/null
--- src/cmd/venti/stats.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-Stats stats;
-
-void
-statsinit(void)
-{
-}
-
-static int
-percent(long v, long total)
-{
- if(total == 0)
- total = 1;
- if(v < 1000*1000)
- return (v * 100) / total;
- total /= 100;
- if(total == 0)
- total = 1;
- return v / total;
-}
-
-void
-printstats(void)
-{
- fprint(2, "lump writes=%,ld\n", stats.lumpwrites);
- fprint(2, "lump reads=%,ld\n", stats.lumpreads);
- fprint(2, "lump cache read hits=%,ld\n", stats.lumphit);
- fprint(2, "lump cache read misses=%,ld\n", stats.lumpmiss);
-
- fprint(2, "clump disk writes=%,ld\n", stats.clumpwrites);
- fprint(2, "clump disk bytes written=%,lld\n", stats.clumpbwrites);
- fprint(2, "clump disk bytes compressed=%,lld\n", stats.clumpbcomp);
- fprint(2, "clump disk reads=%,ld\n", stats.clumpreads);
- fprint(2, "clump disk bytes read=%,lld\n", stats.clumpbreads);
- fprint(2, "clump disk bytes uncompressed=%,lld\n", stats.clumpbuncomp);
-
- fprint(2, "clump directory disk writes=%,ld\n", stats.ciwrites);
- fprint(2, "clump directory disk reads=%,ld\n", stats.cireads);
-
- fprint(2, "index disk writes=%,ld\n", stats.indexwrites);
- fprint(2, "index disk reads=%,ld\n", stats.indexreads);
- fprint(2, "index disk reads for modify=%,ld\n", stats.indexwreads);
- fprint(2, "index disk reads for allocation=%,ld\n", stats.indexareads);
- fprint(2, "index block splits=%,ld\n", stats.indexsplits);
-
- fprint(2, "index cache lookups=%,ld\n", stats.iclookups);
- fprint(2, "index cache hits=%,ld %d%%\n", stats.ichits,
- percent(stats.ichits, stats.iclookups));
- fprint(2, "index cache fills=%,ld %d%%\n", stats.icfills,
- percent(stats.icfills, stats.iclookups));
- fprint(2, "index cache inserts=%,ld\n", stats.icinserts);
-
- fprint(2, "disk cache hits=%,ld\n", stats.pchit);
- fprint(2, "disk cache misses=%,ld\n", stats.pcmiss);
- fprint(2, "disk cache reads=%,ld\n", stats.pcreads);
- fprint(2, "disk cache bytes read=%,lld\n", stats.pcbreads);
-
- fprint(2, "disk cache writes=%,ld\n", stats.dirtydblocks);
- fprint(2, "disk cache writes absorbed=%,ld %d%%\n", stats.absorbedwrites,
- percent(stats.absorbedwrites, stats.dirtydblocks));
-
- fprint(2, "disk writes=%,ld\n", stats.diskwrites);
- fprint(2, "disk bytes written=%,lld\n", stats.diskbwrites);
- fprint(2, "disk reads=%,ld\n", stats.diskreads);
- fprint(2, "disk bytes read=%,lld\n", stats.diskbreads);
-
-}
blob - 6f8f5ba5cc8642459feb64b6e2330dbeee127b1d (mode 644)
blob + /dev/null
--- src/cmd/venti/stdinc.h
+++ /dev/null
-#include <u.h>
-#include <libc.h>
-#include <venti.h>
-#include <libsec.h>
-#include <thread.h>
-
blob - 7446140062c3b79799d88a65dc80b59a1c6f3b8c (mode 644)
blob + /dev/null
--- src/cmd/venti/sync.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-char *host;
-
-void
-usage(void)
-{
- fprint(2, "usage: sync [-h host]\n");
- threadexitsall("usage");
-}
-
-void
-threadmain(int argc, char *argv[])
-{
- VtConn *z;
-
- ARGBEGIN{
- case 'h':
- host = EARGF(usage());
- if(host == nil)
- usage();
- break;
- default:
- usage();
- break;
- }ARGEND
-
- if(argc != 0)
- usage();
-
-
- fmtinstall('V', vtscorefmt);
-
- z = vtdial(host);
- if(z == nil)
- sysfatal("could not connect to server: %r");
-
- if(vtconnect(z) < 0)
- sysfatal("vtconnect: %r");
-
- if(vtsync(z) < 0)
- sysfatal("vtsync: %r");
-
- vthangup(z);
- threadexitsall(0);
-}
blob - 7ba83aa1c880aacbc77510dec28bba2bd462c2a4 (mode 644)
blob + /dev/null
--- src/cmd/venti/syncarena.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-static int writeclumphead(Arena *arena, u64int aa, Clump *cl);
-static int writeclumpmagic(Arena *arena, u64int aa, u32int magic);
-
-int
-clumpinfocmp(ClumpInfo *c, ClumpInfo *d)
-{
- return c->type != d->type
- || c->size != d->size
- || c->uncsize != d->uncsize
- || scorecmp(c->score, d->score)!=0;
-}
-
-/*
- * synchronize the clump info directory with
- * with the clumps actually stored in the arena.
- * the directory should be at least as up to date
- * as the arena's trailer.
- *
- * checks/updates at most n clumps.
- *
- * returns 0 if ok, flags if error occurred
- */
-int
-syncarena(Arena *arena, u32int n, int zok, int fix)
-{
- ZBlock *lump;
- Clump cl;
- ClumpInfo ci;
- static ClumpInfo zci = { .type = -1 };
- u8int score[VtScoreSize];
- u64int uncsize, used, aa;
- u32int clump, clumps, cclumps, magic;
- int err, flush, broken;
-
- used = arena->used;
- clumps = arena->clumps;
- cclumps = arena->cclumps;
- uncsize = arena->uncsize;
-
- flush = 0;
- err = 0;
- for(; n; n--){
- aa = arena->used;
- clump = arena->clumps;
- magic = clumpmagic(arena, aa);
- if(magic == ClumpFreeMagic)
- break;
- if(magic != ClumpMagic){
- fprint(2, "illegal clump magic number=%#8.8ux at clump=%d\n", magic, clump);
- err |= SyncDataErr;
-//ZZZ write a zero here?
- if(0 && fix && writeclumpmagic(arena, aa, ClumpFreeMagic) < 0){
- fprint(2, "can't write corrected clump free magic: %r");
- err |= SyncFixErr;
- }
- break;
- }
- arena->clumps++;
-
- broken = 0;
- lump = loadclump(arena, aa, 0, &cl, score, 0);
- if(lump == nil){
- fprint(2, "clump=%d failed to read correctly: %r\n", clump);
- err |= SyncDataErr;
- }else if(cl.info.type != VtTypeCorrupt){
- scoremem(score, lump->data, cl.info.uncsize);
- if(scorecmp(cl.info.score, score) != 0){
- fprint(2, "clump=%d has mismatched score\n", clump);
- err |= SyncDataErr;
- broken = 1;
- }else if(vttypevalid(cl.info.type) < 0){
- fprint(2, "clump=%d has invalid type %d", clump, cl.info.type);
- err |= SyncDataErr;
- broken = 1;
- }
- if(broken && fix){
- cl.info.type = VtTypeCorrupt;
- if(writeclumphead(arena, aa, &cl) < 0){
- fprint(2, "can't write corrected clump header: %r");
- err |= SyncFixErr;
- }
- }
- }
- freezblock(lump);
- arena->used += ClumpSize + cl.info.size;
-
- if(!broken && readclumpinfo(arena, clump, &ci)<0){
- fprint(2, "arena directory read failed\n");
- broken = 1;
- }else if(!broken && clumpinfocmp(&ci, &cl.info)!=0){
- if(clumpinfocmp(&ci, &zci) == 0){
- err |= SyncCIZero;
- if(!zok)
- fprint(2, "unwritten clump info for clump=%d\n", clump);
- }else{
- err |= SyncCIErr;
- fprint(2, "bad clump info for clump=%d\n", clump);
- fprint(2, "\texpected score=%V type=%d size=%d uncsize=%d\n",
- cl.info.score, cl.info.type, cl.info.size, cl.info.uncsize);
- fprint(2, "\tfound score=%V type=%d size=%d uncsize=%d\n",
- ci.score, ci.type, ci.size, ci.uncsize);
- }
- broken = 1;
- }
- if(broken && fix){
- flush = 1;
- ci = cl.info;
- if(writeclumpinfo(arena, clump, &ci) < 0){
- fprint(2, "can't write correct clump directory: %r\n");
- err |= SyncFixErr;
- }
- }
-
- arena->uncsize += cl.info.uncsize;
- if(cl.info.size < cl.info.uncsize)
- arena->cclumps++;
- }
-
- if(flush){
- arena->wtime = now();
- if(arena->ctime == 0 && arena->clumps)
- arena->ctime = arena->wtime;
- if(flushciblocks(arena) < 0){
- fprint(2, "can't flush arena directory cache: %r");
- err |= SyncFixErr;
- }
- flushdcache();
- }
-
- if(used != arena->used
- || clumps != arena->clumps
- || cclumps != arena->cclumps
- || uncsize != arena->uncsize)
- err |= SyncHeader;
-
- return err;
-}
-
-static int
-writeclumphead(Arena *arena, u64int aa, Clump *cl)
-{
- ZBlock *zb;
- int bad;
-
- zb = alloczblock(ClumpSize, 0);
- if(zb == nil)
- return -1;
- bad = packclump(cl, zb->data)<0
- || writearena(arena, aa, zb->data, ClumpSize) != ClumpSize;
- freezblock(zb);
- return bad ? -1 : 0;
-}
-
-static int
-writeclumpmagic(Arena *arena, u64int aa, u32int magic)
-{
- u8int buf[U32Size];
-
- packmagic(magic, buf);
- return writearena(arena, aa, buf, U32Size) == U32Size;
-}
blob - 75bb0696360d20743cdbec2b171f4da7178b3d6c (mode 644)
blob + /dev/null
--- src/cmd/venti/syncindex.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-static int verbose;
-void
-usage(void)
-{
- fprint(2, "usage: syncindex [-fv] [-B blockcachesize] config\n");
- threadexitsall("usage");
-}
-
-void
-threadmain(int argc, char *argv[])
-{
- u32int bcmem;
- int fix;
-
- fix = 0;
- bcmem = 0;
- ARGBEGIN{
- case 'B':
- bcmem = unittoull(ARGF());
- break;
- case 'f':
- fix++;
- break;
- case 'v':
- verbose++;
- break;
- default:
- usage();
- break;
- }ARGEND
-
- if(!fix)
- readonly = 1;
-
- if(argc != 1)
- usage();
-
- if(initventi(argv[0]) < 0)
- sysfatal("can't init venti: %r");
-
- if(bcmem < maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16))
- bcmem = maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16);
- fprint(2, "initialize %d bytes of disk block cache\n", bcmem);
- initdcache(bcmem);
-
- if(verbose)
- printindex(2, mainindex);
- if(syncindex(mainindex, fix) < 0)
- sysfatal("failed to sync index=%s: %r\n", mainindex->name);
-
- threadexitsall(0);
-}
blob - 95db12081af7992db038b125cd0dfb1c0791c74e (mode 644)
blob + /dev/null
--- src/cmd/venti/syncindex0.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-enum
-{
- ClumpChunks = 32*1024
-};
-
-/*
- * shell sort is plenty good enough
- * because we're going to do a bunch of disk i/o's
- */
-static void
-sortclumpinfo(ClumpInfo *ci, int *s, int n)
-{
- int i, j, m, t;
-
- for(m = (n + 3) / 5; m > 0; m = (m + 1) / 3){
- for(i = n - m; i-- > 0;){
- for(j = i + m; j < n; j += m){
- if(memcmp(ci[s[j - m]].score, ci[s[j]].score, VtScoreSize) <= 0)
- break;
- t = s[j];
- s[j] = s[j - m];
- s[j - m] = t;
- }
- }
- }
-}
-
-int
-syncarenaindex(Index *ix, Arena *arena, u32int clump, u64int a, int fix)
-{
- Packet *pack;
- IEntry ie;
- IAddr ia;
- ClumpInfo *ci, *cis;
- u32int now;
- u64int *addrs;
- int i, n, ok, *s;
-
- now = time(nil);
- cis = MKN(ClumpInfo, ClumpChunks);
- addrs = MKN(u64int, ClumpChunks);
- s = MKN(int, ClumpChunks);
- ok = 0;
- for(; clump < arena->clumps; clump += n){
- n = ClumpChunks;
- if(n > arena->clumps - clump)
- n = arena->clumps - clump;
- n = readclumpinfos(arena, clump, cis, n);
- if(n <= 0){
- fprint(2, "arena directory read failed\n");
- ok = -1;
- break;
- }
-
- for(i = 0; i < n; i++){
- addrs[i] = a;
- a += cis[i].size + ClumpSize;
- s[i] = i;
- }
-
- sortclumpinfo(cis, s, n);
-
- for(i = 0; i < n; i++){
- ci = &cis[s[i]];
- ia.type = ci->type;
- ia.size = ci->uncsize;
- ia.addr = addrs[s[i]];
- ia.blocks = (ci->size + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog;
-
- if(loadientry(ix, ci->score, ci->type, &ie) < 0)
- fprint(2, "missing block type=%d score=%V\n", ci->type, ci->score);
- else if(iaddrcmp(&ia, &ie.ia) != 0){
- fprint(2, "\nmismatched index entry and clump at %d\n", clump + i);
- fprint(2, "\tclump: type=%d size=%d blocks=%d addr=%lld\n", ia.type, ia.size, ia.blocks, ia.addr);
- fprint(2, "\tindex: type=%d size=%d block=%d addr=%lld\n", ie.ia.type, ie.ia.size, ie.ia.blocks, ie.ia.addr);
- pack = readlump(ie.score, ie.ia.type, ie.ia.size);
- packetfree(pack);
- if(pack != nil){
- fprint(2, "duplicated lump\n");
- continue;
- }
- }else
- continue;
- if(!fix){
- ok = -1;
- continue;
- }
- ie.ia = ia;
- scorecp(ie.score, ci->score);
- ie.train = 0;
- ie.wtime = now;
- if(storeientry(ix, &ie) < 0){
- fprint(2, "can't fix index: %r");
- ok = -1;
- }
- }
-
- if(0 && clump / 1000 != (clump + n) / 1000)
- fprint(2, ".");
- }
- free(cis);
- free(addrs);
- free(s);
- return ok;
-}
-
-int
-syncindex(Index *ix, int fix)
-{
- Arena *arena;
- u64int a;
- u32int clump;
- int i, e, e1, ok, ok1;
-
- ok = 0;
- for(i = 0; i < ix->narenas; i++){
- arena = ix->arenas[i];
- clump = arena->clumps;
- a = arena->used;
- e = syncarena(arena, TWID32, fix, fix);
- e1 = e;
- if(fix)
- e1 &= ~(SyncHeader|SyncCIZero);
- if(e1 == SyncHeader)
- fprint(2, "arena %s: header is out-of-date\n", arena->name);
- if(e1)
- ok = -1;
- else{
- ok1 = syncarenaindex(ix, arena, clump, a + ix->amap[i].start, fix);
- if(fix && ok1==0 && (e & SyncHeader) && wbarena(arena) < 0)
- fprint(2, "arena=%s header write failed: %r\n", arena->name);
- ok |= ok1;
- }
- }
- if(fix && wbindex(ix) < 0){
- fprint(2, "can't write back index header for %s: %r\n", ix->name);
- return -1;
- }
- return ok;
-}
blob - db35aa0faebec9ae015d8909c7636789e1a13cca (mode 644)
blob + /dev/null
--- src/cmd/venti/unittoull.c
+++ /dev/null
-#include "stdinc.h"
-
-#define TWID64 ((u64int)~(u64int)0)
-
-u64int
-unittoull(char *s)
-{
- char *es;
- u64int n;
-
- if(s == nil)
- return TWID64;
- n = strtoul(s, &es, 0);
- if(*es == 'k' || *es == 'K'){
- n *= 1024;
- es++;
- }else if(*es == 'm' || *es == 'M'){
- n *= 1024*1024;
- es++;
- }else if(*es == 'g' || *es == 'G'){
- n *= 1024*1024*1024;
- es++;
- }
- if(*es != '\0')
- return TWID64;
- return n;
-}
blob - 5530bd07d003f43c444940d600c9b41cae07a809 (mode 644)
blob + /dev/null
--- src/cmd/venti/unwhack.c
+++ /dev/null
-#include "stdinc.h"
-#include "whack.h"
-
-enum
-{
- DMaxFastLen = 7,
- DBigLenCode = 0x3c, /* minimum code for large lenth encoding */
- DBigLenBits = 6,
- DBigLenBase = 1 /* starting items to encode for big lens */
-};
-
-static uchar lenval[1 << (DBigLenBits - 1)] =
-{
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 3, 3, 3, 3, 3, 3, 3, 3,
- 4, 4, 4, 4,
- 5,
- 6,
- 255,
- 255
-};
-
-static uchar lenbits[] =
-{
- 0, 0, 0,
- 2, 3, 5, 5,
-};
-
-static uchar offbits[16] =
-{
- 5, 5, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 12, 13
-};
-
-static ushort offbase[16] =
-{
- 0, 0x20,
- 0x40, 0x60,
- 0x80, 0xc0,
- 0x100, 0x180,
- 0x200, 0x300,
- 0x400, 0x600,
- 0x800, 0xc00,
- 0x1000,
- 0x2000
-};
-
-void
-unwhackinit(Unwhack *uw)
-{
- uw->err[0] = '\0';
-}
-
-int
-unwhack(Unwhack *uw, uchar *dst, int ndst, uchar *src, int nsrc)
-{
- uchar *s, *d, *dmax, *smax, lit;
- ulong uwbits, lithist;
- int i, off, len, bits, use, code, uwnbits, overbits;
-
- d = dst;
- dmax = d + ndst;
-
- smax = src + nsrc;
- uwnbits = 0;
- uwbits = 0;
- overbits = 0;
- lithist = ~0;
- while(src < smax || uwnbits - overbits >= MinDecode){
- while(uwnbits <= 24){
- uwbits <<= 8;
- if(src < smax)
- uwbits |= *src++;
- else
- overbits += 8;
- uwnbits += 8;
- }
-
- /*
- * literal
- */
- len = lenval[(uwbits >> (uwnbits - 5)) & 0x1f];
- if(len == 0){
- if(lithist & 0xf){
- uwnbits -= 9;
- lit = (uwbits >> uwnbits) & 0xff;
- lit &= 255;
- }else{
- uwnbits -= 8;
- lit = (uwbits >> uwnbits) & 0x7f;
- if(lit < 32){
- if(lit < 24){
- uwnbits -= 2;
- lit = (lit << 2) | ((uwbits >> uwnbits) & 3);
- }else{
- uwnbits -= 3;
- lit = (lit << 3) | ((uwbits >> uwnbits) & 7);
- }
- lit = (lit - 64) & 0xff;
- }
- }
- if(d >= dmax){
- snprint(uw->err, WhackErrLen, "too much output");
- return -1;
- }
- *d++ = lit;
- lithist = (lithist << 1) | (lit < 32) | (lit > 127);
- continue;
- }
-
- /*
- * length
- */
- if(len < 255)
- uwnbits -= lenbits[len];
- else{
- uwnbits -= DBigLenBits;
- code = ((uwbits >> uwnbits) & ((1 << DBigLenBits) - 1)) - DBigLenCode;
- len = DMaxFastLen;
- use = DBigLenBase;
- bits = (DBigLenBits & 1) ^ 1;
- while(code >= use){
- len += use;
- code -= use;
- code <<= 1;
- uwnbits--;
- if(uwnbits < 0){
- snprint(uw->err, WhackErrLen, "len out of range");
- return -1;
- }
- code |= (uwbits >> uwnbits) & 1;
- use <<= bits;
- bits ^= 1;
- }
- len += code;
-
- while(uwnbits <= 24){
- uwbits <<= 8;
- if(src < smax)
- uwbits |= *src++;
- else
- overbits += 8;
- uwnbits += 8;
- }
- }
-
- /*
- * offset
- */
- uwnbits -= 4;
- bits = (uwbits >> uwnbits) & 0xf;
- off = offbase[bits];
- bits = offbits[bits];
-
- uwnbits -= bits;
- off |= (uwbits >> uwnbits) & ((1 << bits) - 1);
- off++;
-
- if(off > d - dst){
- snprint(uw->err, WhackErrLen, "offset out of range: off=%d d=%ld len=%d nbits=%d", off, d - dst, len, uwnbits);
- return -1;
- }
- if(d + len > dmax){
- snprint(uw->err, WhackErrLen, "len out of range");
- return -1;
- }
- s = d - off;
- for(i = 0; i < len; i++)
- d[i] = s[i];
- d += len;
- }
- if(uwnbits < overbits){
- snprint(uw->err, WhackErrLen, "compressed data overrun");
- return -1;
- }
-
- len = d - dst;
-
- return len;
-}
blob - 26a0ef829be3e8e1ab78214d44037418a4cb0b18 (mode 644)
blob + /dev/null
--- src/cmd/venti/utils.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-int
-namecmp(char *s, char *t)
-{
- return strncmp(s, t, ANameSize);
-}
-
-void
-namecp(char *dst, char *src)
-{
- strncpy(dst, src, ANameSize - 1);
- dst[ANameSize - 1] = '\0';
-}
-
-int
-nameok(char *name)
-{
- char *t;
- int c;
-
- if(name == nil)
- return -1;
- for(t = name; c = *t; t++)
- if(t - name >= ANameSize
- || c < ' ' || c >= 0x7f)
- return -1;
- return 0;
-}
-
-int
-stru32int(char *s, u32int *r)
-{
- char *t;
- u32int n, nn, m;
- int c;
-
- m = TWID32 / 10;
- n = 0;
- for(t = s; ; t++){
- c = *t;
- if(c < '0' || c > '9')
- break;
- if(n > m)
- return -1;
- nn = n * 10 + c - '0';
- if(nn < n)
- return -1;
- n = nn;
- }
- *r = n;
- return s != t && *t == '\0';
-}
-
-int
-stru64int(char *s, u64int *r)
-{
- char *t;
- u64int n, nn, m;
- int c;
-
- m = TWID64 / 10;
- n = 0;
- for(t = s; ; t++){
- c = *t;
- if(c < '0' || c > '9')
- break;
- if(n > m)
- return -1;
- nn = n * 10 + c - '0';
- if(nn < n)
- return -1;
- n = nn;
- }
- *r = n;
- return s != t && *t == '\0';
-}
-
-int
-vttypevalid(int type)
-{
- return type < VtMaxType;
-}
-
-void
-fmtzbinit(Fmt *f, ZBlock *b)
-{
- f->runes = 0;
- f->start = b->data;
- f->to = f->start;
- f->stop = (char*)f->start + b->len;
- f->flush = nil;
- f->farg = nil;
- f->nfmt = 0;
- f->args = nil;
-}
-
-static int
-sflush(Fmt *f)
-{
- char *s;
- int n;
-
- n = (int)f->farg;
- n += 256;
- f->farg = (void*)n;
- s = f->start;
- f->start = realloc(s, n);
- if(f->start == nil){
- f->start = s;
- return 0;
- }
- f->to = (char*)f->start + ((char*)f->to - s);
- f->stop = (char*)f->start + n - 1;
- return 1;
-}
-
-static char*
-logit(int severity, char *fmt, va_list args)
-{
- Fmt f;
- int n;
-
- f.runes = 0;
- n = 32;
- f.start = malloc(n);
- if(f.start == nil)
- return nil;
- f.to = f.start;
- f.stop = (char*)f.start + n - 1;
- f.flush = sflush;
- f.farg = (void*)n;
- f.nfmt = 0;
- f.args = args;
- n = dofmt(&f, fmt);
- if(n < 0)
-{
-fprint(2, "dofmt %s failed\n", fmt);
- return nil;
-}
- *(char*)f.to = '\0';
-
- if(argv0 == nil)
- fprint(2, "%s: err %d: %s\n", argv0, severity, f.start);
- else
- fprint(2, "err %d: %s\n", severity, f.start);
- return f.start;
-}
-
-void
-seterr(int severity, char *fmt, ...)
-{
- char *s;
- va_list args;
-
- va_start(args, fmt);
- s = logit(severity, fmt, args);
- va_end(args);
- if(s == nil)
- werrstr("error setting error");
- else{
- werrstr("%s", s);
- free(s);
- }
-}
-
-void
-logerr(int severity, char *fmt, ...)
-{
- char *s;
- va_list args;
-
- va_start(args, fmt);
- s = logit(severity, fmt, args);
- va_end(args);
- free(s);
-}
-
-u32int
-now(void)
-{
- return time(nil);
-}
-
-void
-fatal(char *fmt, ...)
-{
- Fmt f;
- char buf[256];
-
- f.runes = 0;
- f.start = buf;
- f.to = buf;
- f.stop = buf + sizeof(buf);
- f.flush = fmtfdflush;
- f.farg = (void*)2;
- f.nfmt = 0;
- fmtprint(&f, "fatal %s error:", argv0);
- va_start(f.args, fmt);
- dofmt(&f, fmt);
- va_end(f.args);
- fmtprint(&f, "\n");
- fmtfdflush(&f);
- if(0)
- abort();
- threadexitsall(buf);
-}
-
-ZBlock *
-alloczblock(u32int size, int zeroed)
-{
- ZBlock *b;
- static ZBlock z;
-
- b = malloc(sizeof(ZBlock) + size);
- if(b == nil){
- seterr(EOk, "out of memory");
- return nil;
- }
-
- *b = z;
- b->data = (u8int*)&b[1];
- b->len = size;
- if(zeroed)
- memset(b->data, 0, size);
- return b;
-}
-
-void
-freezblock(ZBlock *b)
-{
- free(b);
-}
-
-ZBlock*
-packet2zblock(Packet *p, u32int size)
-{
- ZBlock *b;
-
- if(p == nil)
- return nil;
- b = alloczblock(size, 0);
- if(b == nil)
- return nil;
- b->len = size;
- if(packetcopy(p, b->data, 0, size) < 0){
- freezblock(b);
- return nil;
- }
- return b;
-}
-
-Packet*
-zblock2packet(ZBlock *zb, u32int size)
-{
- Packet *p;
-
- if(zb == nil)
- return nil;
- p = packetalloc();
- packetappend(p, zb->data, size);
- return p;
-}
-
-void *
-emalloc(ulong n)
-{
- void *p;
-
- p = malloc(n);
- if(p == nil)
- sysfatal("out of memory");
- memset(p, 0xa5, n);
-if(0)print("emalloc %p-%p by %lux\n", p, (char*)p+n, getcallerpc(&n));
- return p;
-}
-
-void *
-ezmalloc(ulong n)
-{
- void *p;
-
- p = malloc(n);
- if(p == nil)
- sysfatal("out of memory");
- memset(p, 0, n);
-if(0)print("ezmalloc %p-%p by %lux\n", p, (char*)p+n, getcallerpc(&n));
- return p;
-}
-
-void *
-erealloc(void *p, ulong n)
-{
- p = realloc(p, n);
- if(p == nil)
- sysfatal("out of memory");
-if(0)print("erealloc %p-%p by %lux\n", p, (char*)p+n, getcallerpc(&p));
- return p;
-}
-
-char *
-estrdup(char *s)
-{
- char *t;
- int n;
-
- n = strlen(s) + 1;
- t = emalloc(n);
- memmove(t, s, n);
-if(0)print("estrdup %p-%p by %lux\n", t, (char*)t+n, getcallerpc(&s));
- return t;
-}
-
-ZBlock*
-readfile(char *name)
-{
- Part *p;
- ZBlock *b;
-
- p = initpart(name, 1);
- if(p == nil)
- return nil;
- b = alloczblock(p->size, 0);
- if(b == nil){
- seterr(EOk, "can't alloc %s: %r", name);
- freepart(p);
- return nil;
- }
- if(readpart(p, 0, b->data, p->size) < 0){
- seterr(EOk, "can't read %s: %r", name);
- freepart(p);
- freezblock(b);
- return nil;
- }
- freepart(p);
- return b;
-}
-
-/*
- * return floor(log2(v))
- */
-int
-u64log2(u64int v)
-{
- int i;
-
- for(i = 0; i < 64; i++)
- if((v >> i) <= 1)
- break;
- return i;
-}
-
-int
-vtproc(void (*fn)(void*), void *arg)
-{
- proccreate(fn, arg, 256*1024);
- return 0;
-}
-
blob - 4c533cc626c782d719e4792aff43b28e218eee7d (mode 644)
blob + /dev/null
--- src/cmd/venti/venti.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-#include "whack.h"
-
-int debug;
-int mainstacksize = 256*1024;
-
-static void ventiserver(char *vaddr);
-
-void
-threadmain(int argc, char *argv[])
-{
- char *config, *haddr, *vaddr;
- u32int mem, icmem, bcmem, minbcmem;
-
- vaddr = "tcp!*!venti";
- haddr = nil;
- config = nil;
- mem = 0xffffffffUL;
- icmem = 0;
- bcmem = 0;
- ARGBEGIN{
- case 'a':
- vaddr = ARGF();
- if(vaddr == nil)
- goto usage;
- break;
- case 'B':
- bcmem = unittoull(ARGF());
- break;
- case 'c':
- config = ARGF();
- if(config == nil)
- goto usage;
- break;
- case 'C':
- mem = unittoull(ARGF());
- break;
- case 'd':
- debug = 1;
- break;
- case 'h':
- haddr = ARGF();
- if(haddr == nil)
- goto usage;
- break;
- case 'I':
- icmem = unittoull(ARGF());
- break;
- case 'w':
- queuewrites = 1;
- break;
- default:
- goto usage;
- }ARGEND
-
-print("whack %d\n", sizeof(Whack));
-
- if(argc){
- usage:
- fprint(2, "usage: venti [-dw] [-a ventiaddress] [-h httpaddress] [-c config] [-C cachesize] [-I icachesize] [-B blockcachesize]\n");
- threadexitsall("usage");
- }
-
- fmtinstall('V', vtscorefmt);
- fmtinstall('H', encodefmt);
- fmtinstall('F', vtfcallfmt);
-
- if(config == nil)
- config = "venti.conf";
-
-
- if(initarenasum() < 0)
- fprint(2, "warning: can't initialize arena summing process: %r");
-
- if(initventi(config) < 0)
- sysfatal("can't init server: %r");
-
- if(mem == 0xffffffffUL)
- mem = 1 * 1024 * 1024;
- fprint(2, "initialize %d bytes of lump cache for %d lumps\n",
- mem, mem / (8 * 1024));
- initlumpcache(mem, mem / (8 * 1024));
-
- icmem = u64log2(icmem / (sizeof(IEntry)+sizeof(IEntry*)) / ICacheDepth);
- if(icmem < 4)
- icmem = 4;
- fprint(2, "initialize %d bytes of index cache for %d index entries\n",
- (sizeof(IEntry)+sizeof(IEntry*)) * (1 << icmem) * ICacheDepth,
- (1 << icmem) * ICacheDepth);
- initicache(icmem, ICacheDepth);
-
- /*
- * need a block for every arena and every process
- */
- minbcmem = maxblocksize *
- (mainindex->narenas + mainindex->nsects*4 + 16);
- if(bcmem < minbcmem)
- bcmem = minbcmem;
-
- fprint(2, "initialize %d bytes of disk block cache\n", bcmem);
- initdcache(bcmem);
-
- fprint(2, "sync arenas and index...\n");
- if(syncindex(mainindex, 1) < 0)
- sysfatal("can't sync server: %r");
-
- if(queuewrites){
- fprint(2, "initialize write queue...\n");
- if(initlumpqueues(mainindex->nsects) < 0){
- fprint(2, "can't initialize lump queues,"
- " disabling write queueing: %r");
- queuewrites = 0;
- }
- }
-
- if(haddr){
- fprint(2, "starting http server at %s\n", haddr);
- if(httpdinit(haddr) < 0)
- fprint(2, "warning: can't start http server: %r");
- }
-
- ventiserver(vaddr);
- threadexitsall(0);
-}
-
-static void
-vtrerror(VtReq *r, char *error)
-{
- r->rx.type = VtRerror;
- r->rx.error = estrdup(error);
-}
-
-static void
-ventiserver(char *addr)
-{
- Packet *p;
- VtReq *r;
- VtSrv *s;
- char err[ERRMAX];
-
- s = vtlisten(addr);
- if(s == nil)
- sysfatal("can't announce %s: %r", addr);
-
- while((r = vtgetreq(s)) != nil){
- r->rx.type = r->tx.type+1;
- // print("req (arenas[0]=%p sects[0]=%p) %F\n",
- // mainindex->arenas[0], mainindex->sects[0], &r->tx);
- switch(r->tx.type){
- default:
- vtrerror(r, "unknown request");
- break;
- case VtTread:
- if((r->rx.data = readlump(r->tx.score, r->tx.dtype, r->tx.count)) == nil){
- rerrstr(err, sizeof err);
- vtrerror(r, err);
- }
- break;
- case VtTwrite:
- p = r->tx.data;
- r->tx.data = nil;
- if(writelump(p, r->rx.score, r->tx.dtype, 0) < 0){
- rerrstr(err, sizeof err);
- vtrerror(r, err);
- }
- break;
- case VtTsync:
- flushqueue();
- flushdcache();
- break;
- }
- vtrespond(r);
- }
-}
-
blob - 9e7dc31b9b8f11403240eb1c7e84e251dc813e6c (mode 644)
blob + /dev/null
--- src/cmd/venti/verifyarena.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-static int verbose;
-
-void
-usage(void)
-{
- fprint(2, "usage: verifyarena [-v]\n");
- threadexitsall(0);
-}
-
-static void
-readblock(uchar *buf, int n)
-{
- int nr, m;
-
- for(nr = 0; nr < n; nr += m){
- m = n - nr;
- m = read(0, &buf[nr], m);
- if(m <= 0)
- sysfatal("can't read arena from standard input: %r");
- }
-}
-
-static void
-verifyarena(void)
-{
- Arena arena;
- ArenaHead head;
- ZBlock *b;
- DigestState s;
- u64int n, e;
- u32int bs;
- u8int score[VtScoreSize];
-
- fprint(2, "verify arena from standard input\n");
-
- memset(&arena, 0, sizeof arena);
- memset(&s, 0, sizeof s);
-
- /*
- * read the little bit, which will included the header
- */
- bs = MaxIoSize;
- b = alloczblock(bs, 0);
- readblock(b->data, HeadSize);
- sha1(b->data, HeadSize, nil, &s);
- if(unpackarenahead(&head, b->data) < 0)
- sysfatal("corrupted arena header: %r");
- if(head.version != ArenaVersion)
- fprint(2, "warning: unknown arena version %d\n", head.version);
-
- /*
- * now we know how much to read
- * read everything but the last block, which is special
- */
- e = head.size - head.blocksize;
- for(n = HeadSize; n < e; n += bs){
- if(n + bs > e)
- bs = e - n;
- readblock(b->data, bs);
- sha1(b->data, bs, nil, &s);
- }
-
- /*
- * read the last block update the sum.
- * the sum is calculated assuming the slot for the sum is zero.
- */
- bs = head.blocksize;
- readblock(b->data, bs);
- sha1(b->data, bs-VtScoreSize, nil, &s);
- sha1(zeroscore, VtScoreSize, nil, &s);
- sha1(nil, 0, score, &s);
-
- /*
- * validity check on the trailer
- */
- arena.blocksize = head.blocksize;
- if(unpackarena(&arena, b->data) < 0)
- sysfatal("corrupted arena trailer: %r");
- scorecp(arena.score, &b->data[arena.blocksize - VtScoreSize]);
-
- if(namecmp(arena.name, head.name) != 0)
- sysfatal("arena header and trailer names clash: %s vs. %s\n", head.name, arena.name);
- if(arena.version != head.version)
- sysfatal("arena header and trailer versions clash: %d vs. %d\n", head.version, arena.version);
- arena.size = head.size - 2 * head.blocksize;
-
- /*
- * check for no checksum or the same
- */
- if(scorecmp(score, arena.score) != 0){
- if(scorecmp(zeroscore, arena.score) != 0)
- fprint(2, "warning: mismatched checksums for arena=%s, found=%V calculated=%V",
- arena.name, arena.score, score);
- scorecp(arena.score, score);
- }else
- fprint(2, "matched score\n");
-
- printarena(2, &arena);
-}
-
-void
-threadmain(int argc, char *argv[])
-{
- fmtinstall('V', vtscorefmt);
- statsinit();
-
- ARGBEGIN{
- case 'v':
- verbose++;
- break;
- default:
- usage();
- break;
- }ARGEND
-
- readonly = 1;
-
- if(argc != 0)
- usage();
-
- verifyarena();
- threadexitsall(0);
-}
blob - 9aa6b6d2fbb0a968cc2733684c30f744497e5085 (mode 644)
blob + /dev/null
--- src/cmd/venti/whack.c
+++ /dev/null
-#include "stdinc.h"
-#include "whack.h"
-
-typedef struct Huff Huff;
-
-enum
-{
- MaxFastLen = 9,
- BigLenCode = 0x1f4, /* minimum code for large lenth encoding */
- BigLenBits = 9,
- BigLenBase = 4, /* starting items to encode for big lens */
-
- MinOffBits = 6,
- MaxOffBits = MinOffBits + 8,
-
- MaxLen = 2051 /* max. length encodable in 24 bits */
-};
-
-enum
-{
- StatBytes,
- StatOutBytes,
- StatLits,
- StatMatches,
- StatLitBits,
- StatOffBits,
- StatLenBits,
-
- MaxStat
-};
-
-struct Huff
-{
- short bits; /* length of the code */
- ulong encode; /* the code */
-};
-
-static Huff lentab[MaxFastLen] =
-{
- {2, 0x2}, /* 10 */
- {3, 0x6}, /* 110 */
- {5, 0x1c}, /* 11100 */
- {5, 0x1d}, /* 11101 */
- {6, 0x3c}, /* 111100 */
- {7, 0x7a}, /* 1111010 */
- {7, 0x7b}, /* 1111011 */
- {8, 0xf8}, /* 11111000 */
- {8, 0xf9}, /* 11111001 */
-};
-
-static int thwmaxcheck;
-
-void
-whackinit(Whack *tw, int level)
-{
- thwmaxcheck = (1 << level);
- thwmaxcheck -= thwmaxcheck >> 2;
- if(thwmaxcheck < 2)
- thwmaxcheck = 2;
- else if(thwmaxcheck > 1024)
- thwmaxcheck = 1024;
- memset(tw, 0, sizeof *tw);
- tw->begin = 2 * WhackMaxOff;
-}
-
-/*
- * find a string in the dictionary
- */
-static int
-whackmatch(Whack *b, uchar **ss, uchar *esrc, ulong h, ulong now)
-{
- ushort then, off, last;
- int bestoff, bestlen, check;
- uchar *s, *t;
-
- s = *ss;
- if(esrc < s + MinMatch)
- return -1;
- if(s + MaxLen < esrc)
- esrc = s + MaxLen;
-
- bestoff = 0;
- bestlen = 0;
- check = thwmaxcheck;
- last = 0;
- for(then = b->hash[h]; check-- > 0; then = b->next[then & (WhackMaxOff - 1)]){
- off = now - then;
- if(off <= last || off > WhackMaxOff)
- break;
-
- /*
- * don't need to check for the end because
- * 1) s too close check above
- */
- t = s - off;
- if(s[0] == t[0] && s[1] == t[1] && s[2] == t[2]){
- if(!bestlen || esrc - s > bestlen && s[bestlen] == t[bestlen]){
- t += 3;
- for(s += 3; s < esrc; s++){
- if(*s != *t)
- break;
- t++;
- }
- if(s - *ss > bestlen){
- bestlen = s - *ss;
- bestoff = off;
- if(bestlen > thwmaxcheck)
- break;
- }
- }
- }
- s = *ss;
- last = off;
- }
- *ss += bestlen;
- return bestoff;
-}
-
-/*
- * knuth vol. 3 multiplicative hashing
- * each byte x chosen according to rules
- * 1/4 < x < 3/10, 1/3 x < < 3/7, 4/7 < x < 2/3, 7/10 < x < 3/4
- * with reasonable spread between the bytes & their complements
- *
- * the 3 byte value appears to be as almost good as the 4 byte value,
- * and might be faster on some machines
- */
-/*
-#define hashit(c) ((((ulong)(c) * 0x6b43a9) >> (24 - HashLog)) & HashMask)
-*/
-#define hashit(c) (((((ulong)(c) & 0xffffff) * 0x6b43a9b5) >> (32 - HashLog)) & HashMask)
-
-/*
- * lz77 compression with single lookup in a hash table for each block
- */
-int
-whack(Whack *w, uchar *dst, uchar *src, int n, ulong stats[WhackStats])
-{
- uchar *s, *ss, *sss, *esrc, *half, *wdst, *wdmax;
- ulong cont, code, wbits;
- ushort now;
- int toff, lithist, h, len, bits, use, wnbits, lits, matches, offbits, lenbits;
-
- if(n < MinMatch)
- return -1;
-
- wdst = dst;
- wdmax = dst + n;
-
- now = w->begin;
- s = src;
- w->data = s;
-
- cont = (s[0] << 16) | (s[1] << 8) | s[2];
-
- esrc = s + n;
- half = s + (n >> 1);
- wnbits = 0;
- wbits = 0;
- lits = 0;
- matches = 0;
- offbits = 0;
- lenbits = 0;
- lithist = ~0;
- while(s < esrc){
- h = hashit(cont);
-
- sss = s;
- toff = whackmatch(w, &sss, esrc, h, now);
- ss = sss;
-
- len = ss - s;
- for(; wnbits >= 8; wnbits -= 8){
- if(wdst >= wdmax){
- w->begin = now;
- return -1;
- }
- *wdst++ = wbits >> (wnbits - 8);
- }
- if(len < MinMatch){
- toff = *s;
- lithist = (lithist << 1) | toff < 32 | toff > 127;
- if(lithist & 0x1e){
- wbits = (wbits << 9) | toff;
- wnbits += 9;
- }else if(lithist & 1){
- toff = (toff + 64) & 0xff;
- if(toff < 96){
- wbits = (wbits << 10) | toff;
- wnbits += 10;
- }else{
- wbits = (wbits << 11) | toff;
- wnbits += 11;
- }
- }else{
- wbits = (wbits << 8) | toff;
- wnbits += 8;
- }
- lits++;
-
- /*
- * speed hack
- * check for compression progress, bail if none achieved
- */
- if(s > half){
- if(4 * (s - src) < 5 * lits){
- w->begin = now;
- return -1;
- }
- half = esrc;
- }
-
- if(s + MinMatch <= esrc){
- w->next[now & (WhackMaxOff - 1)] = w->hash[h];
- w->hash[h] = now;
- if(s + MinMatch < esrc)
- cont = (cont << 8) | s[MinMatch];
- }
- now++;
- s++;
- continue;
- }
-
- matches++;
-
- /*
- * length of match
- */
- if(len > MaxLen){
- len = MaxLen;
- ss = s + len;
- }
- len -= MinMatch;
- if(len < MaxFastLen){
- bits = lentab[len].bits;
- wbits = (wbits << bits) | lentab[len].encode;
- wnbits += bits;
- lenbits += bits;
- }else{
- code = BigLenCode;
- bits = BigLenBits;
- use = BigLenBase;
- len -= MaxFastLen;
- while(len >= use){
- len -= use;
- code = (code + use) << 1;
- use <<= (bits & 1) ^ 1;
- bits++;
- }
-
- wbits = (wbits << bits) | (code + len);
- wnbits += bits;
- lenbits += bits;
-
- for(; wnbits >= 8; wnbits -= 8){
- if(wdst >= wdmax){
- w->begin = now;
- return -1;
- }
- *wdst++ = wbits >> (wnbits - 8);
- }
- }
-
- /*
- * offset in history
- */
- toff--;
- for(bits = MinOffBits; toff >= (1 << bits); bits++)
- ;
- if(bits < MaxOffBits-1){
- wbits = (wbits << 3) | (bits - MinOffBits);
- if(bits != MinOffBits)
- bits--;
- wnbits += bits + 3;
- offbits += bits + 3;
- }else{
- wbits = (wbits << 4) | 0xe | (bits - (MaxOffBits-1));
- bits--;
- wnbits += bits + 4;
- offbits += bits + 4;
- }
- wbits = (wbits << bits) | toff & ((1 << bits) - 1);
-
- for(; s != ss; s++){
- if(s + MinMatch <= esrc){
- h = hashit(cont);
- w->next[now & (WhackMaxOff - 1)] = w->hash[h];
- w->hash[h] = now;
- if(s + MinMatch < esrc)
- cont = (cont << 8) | s[MinMatch];
- }
- now++;
- }
- }
-
- w->begin = now;
-
- stats[StatBytes] += esrc - src;
- stats[StatLits] += lits;
- stats[StatMatches] += matches;
- stats[StatLitBits] += (wdst - (dst + 2)) * 8 + wnbits - offbits - lenbits;
- stats[StatOffBits] += offbits;
- stats[StatLenBits] += lenbits;
-
- if(wnbits & 7){
- wbits <<= 8 - (wnbits & 7);
- wnbits += 8 - (wnbits & 7);
- }
- for(; wnbits >= 8; wnbits -= 8){
- if(wdst >= wdmax)
- return -1;
- *wdst++ = wbits >> (wnbits - 8);
- }
-
- stats[StatOutBytes] += wdst - dst;
-
- return wdst - dst;
-}
-
-int
-whackblock(uchar *dst, uchar *src, int ssize)
-{
- Whack w;
- ulong stats[MaxStat];
- int r;
-
- whackinit(&w, 6);
- r = whack(&w, dst, src, ssize, stats);
- return r;
-}
blob - fb966169c073eb5f0e7c69885312e5acc44dfc5c (mode 644)
blob + /dev/null
--- src/cmd/venti/whack.h
+++ /dev/null
-typedef struct Whack Whack;
-typedef struct Unwhack Unwhack;
-
-enum
-{
- WhackStats = 8,
- WhackErrLen = 64, /* max length of error message from thwack or unthwack */
- WhackMaxOff = 16*1024, /* max allowed offset */
-
- HashLog = 14,
- HashSize = 1<<HashLog,
- HashMask = HashSize - 1,
-
- MinMatch = 3, /* shortest match possible */
-
- MinDecode = 8, /* minimum bits to decode a match or lit; >= 8 */
-
- MaxSeqMask = 8, /* number of bits in coding block mask */
- MaxSeqStart = 256 /* max offset of initial coding block */
-};
-
-struct Whack
-{
- ushort begin; /* time of first byte in hash */
- ushort hash[HashSize];
- ushort next[WhackMaxOff];
- uchar *data;
-};
-
-struct Unwhack
-{
- char err[WhackErrLen];
-};
-
-void whackinit(Whack*, int level);
-void unwhackinit(Unwhack*);
-int whack(Whack*, uchar *dst, uchar *src, int nsrc, ulong stats[WhackStats]);
-int unwhack(Unwhack*, uchar *dst, int ndst, uchar *src, int nsrc);
-
-int whackblock(uchar *dst, uchar *src, int ssize);
blob - f232928ae0e9c23dc3cd49c0de565b3def5af17c (mode 644)
blob + /dev/null
--- src/cmd/venti/wrarena.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-char *host;
-int readonly = 1; /* for part.c */
-
-void
-usage(void)
-{
- fprint(2, "usage: wrarena [-h host] arenafile [offset]\n");
- threadexitsall("usage");
-}
-
-static void
-rdarena(VtConn *z, Arena *arena, u64int offset)
-{
- u64int a, aa, e;
- u32int magic;
- Clump cl;
- uchar score[VtScoreSize];
- ZBlock *lump;
-
- fprint(2, "copying %s to venti\n", arena->name);
- printarena(2, arena);
-
- a = arena->base;
- e = arena->base + arena->size;
- if(offset != ~(u64int)0) {
- if(offset >= e-a)
- sysfatal("bad offset %llud >= %llud\n",
- offset, e-a);
- aa = offset;
- } else
- aa = 0;
-
- for(; aa < e; aa += ClumpSize+cl.info.size) {
- magic = clumpmagic(arena, aa);
- if(magic == ClumpFreeMagic)
- break;
- if(magic != ClumpMagic) {
- fprint(2, "illegal clump magic number %#8.8ux offset %llud\n",
- magic, aa);
- break;
- }
- lump = loadclump(arena, aa, 0, &cl, score, 0);
- if(lump == nil) {
- fprint(2, "clump %llud failed to read: %r\n", aa);
- break;
- }
- if(cl.info.type != VtTypeCorrupt) {
- scoremem(score, lump->data, cl.info.uncsize);
- if(scorecmp(cl.info.score, score) != 0) {
- fprint(2, "clump %llud has mismatched score\n", aa);
- break;
- }
- if(vttypevalid(cl.info.type) < 0) {
- fprint(2, "clump %llud has bad type %d\n", aa, cl.info.type);
- break;
- }
- }
- if(z && vtwrite(z, score, cl.info.type, lump->data, cl.info.uncsize) < 0)
- sysfatal("failed writing clump %llud: %r", aa);
- freezblock(lump);
- }
- if(z && vtsync(z) < 0)
- sysfatal("failed executing sync: %r");
-
- print("end offset %llud\n", aa);
-}
-
-void
-threadmain(int argc, char *argv[])
-{
- char *file;
- VtConn *z;
- Arena *arena;
- u64int offset, aoffset;
- Part *part;
- Dir *d;
- uchar buf[8192];
- ArenaHead head;
-
- aoffset = 0;
- ARGBEGIN{
- case 'h':
- host = EARGF(usage());
- break;
- case 'o':
- aoffset = strtoull(EARGF(usage()), 0, 0);
- break;
- default:
- usage();
- break;
- }ARGEND
-
- offset = ~(u64int)0;
- switch(argc) {
- default:
- usage();
- case 2:
- offset = strtoull(argv[1], 0, 0);
- /* fall through */
- case 1:
- file = argv[0];
- }
-
-
- fmtinstall('V', vtscorefmt);
-
- statsinit();
-
- if((d = dirstat(file)) == nil)
- sysfatal("can't stat file %s: %r", file);
-
- part = initpart(file, 0);
- if(part == nil)
- sysfatal("can't open file %s: %r", file);
- if(readpart(part, aoffset, buf, sizeof buf) < 0)
- sysfatal("can't read file %s: %r", file);
-
- if(unpackarenahead(&head, buf) < 0)
- sysfatal("corrupted arena header: %r");
-
- if(aoffset+head.size > d->length)
- sysfatal("arena is truncated: want %llud bytes have %llud\n",
- head.size, d->length);
-
- partblocksize(part, head.blocksize);
- initdcache(8 * MaxDiskBlock);
-
- arena = initarena(part, aoffset, head.size, head.blocksize);
- if(arena == nil)
- sysfatal("initarena: %r");
-
- if(host && strcmp(host, "/dev/null") != 0){
- z = vtdial(host);
- if(z == nil)
- sysfatal("could not connect to server: %r");
- if(vtconnect(z) < 0)
- sysfatal("vtconnect: %r");
- }else
- z = nil;
-
- rdarena(z, arena, offset);
- vthangup(z);
- threadexitsall(0);
-}
blob - 94b61be44d95a07e8a6e52cf105b7d44671f5350 (mode 644)
blob + /dev/null
--- src/cmd/venti/write.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-char *host;
-
-void
-usage(void)
-{
- fprint(2, "usage: write [-z] [-h host] [-t type] <datablock\n");
- threadexitsall("usage");
-}
-
-void
-threadmain(int argc, char *argv[])
-{
- uchar *p, score[VtScoreSize];
- int type, n, dotrunc;
- VtConn *z;
-
- dotrunc = 0;
- type = VtDataType;
- ARGBEGIN{
- case 'z':
- dotrunc = 1;
- break;
- case 'h':
- host = EARGF(usage());
- break;
- case 't':
- type = atoi(EARGF(usage()));
- break;
- default:
- usage();
- break;
- }ARGEND
-
- if(argc != 0)
- usage();
-
-
- fmtinstall('V', vtscorefmt);
-
- p = ezmalloc(VtMaxLumpSize+1);
- n = readn(0, p, VtMaxLumpSize+1);
- if(n > VtMaxLumpSize)
- sysfatal("data too big");
- z = vtdial(host);
- if(z == nil)
- sysfatal("could not connect to server: %r");
- if(vtconnect(z) < 0)
- sysfatal("vtconnect: %r");
- if(dotrunc)
- n = vtzerotruncate(type, p, n);
- if(vtwrite(z, score, type, p, n) < 0)
- sysfatal("vtwrite: %r");
- vthangup(z);
- print("%V\n", score);
- threadexitsall(0);
-}
blob - d50d5bc99555069eda8586d02544612bf65d627b (mode 644)
blob + /dev/null
--- src/cmd/venti/xml.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-#include "httpd.h"
-#include "xml.h"
-
-void xmlarena(Hio *hout, Arena *s, char *tag, int indent){
- xmlindent(hout, indent);
- hprint(hout, "<%s", tag);
- xmlaname(hout, s->name, "name");
- xmlu32int(hout, s->version, "version");
- xmlaname(hout, s->part->name, "partition");
- xmlu32int(hout, s->blocksize, "blocksize");
- xmlu64int(hout, s->base, "start");
- xmlu64int(hout, s->base+2*s->blocksize, "stop");
- xmlu32int(hout, s->ctime, "created");
- xmlu32int(hout, s->wtime, "modified");
- xmlsealed(hout, s->sealed, "sealed");
- xmlscore(hout, s->score, "score");
- xmlu32int(hout, s->clumps, "clumps");
- xmlu32int(hout, s->cclumps, "compressedclumps");
- xmlu64int(hout, s->uncsize, "data");
- xmlu64int(hout, s->used - s->clumps * ClumpSize, "compresseddata");
- xmlu64int(hout, s->used + s->clumps * ClumpInfoSize, "storage");
- hprint(hout, "/>\n");
-}
-
-void xmlindex(Hio *hout, Index *s, char *tag, int indent){
- int i;
- xmlindent(hout, indent);
- hprint(hout, "<%s", tag);
- xmlaname(hout, s->name, "name");
- xmlu32int(hout, s->version, "version");
- xmlu32int(hout, s->blocksize, "blocksize");
- xmlu32int(hout, s->tabsize, "tabsize");
- xmlu32int(hout, s->buckets, "buckets");
- xmlu32int(hout, s->div, "buckdiv");
- hprint(hout, ">\n");
- xmlindent(hout, indent + 1);
- hprint(hout, "<sects>\n");
- for(i = 0; i < s->nsects; i++)
- xmlamap(hout, &s->smap[i], "sect", indent + 2);
- xmlindent(hout, indent + 1);
- hprint(hout, "</sects>\n");
- xmlindent(hout, indent + 1);
- hprint(hout, "<amaps>\n");
- for(i = 0; i < s->narenas; i++)
- xmlamap(hout, &s->amap[i], "amap", indent + 2);
- xmlindent(hout, indent + 1);
- hprint(hout, "</amaps>\n");
- xmlindent(hout, indent + 1);
- hprint(hout, "<arenas>\n");
- for(i = 0; i < s->narenas; i++)
- xmlarena(hout, s->arenas[i], "arena", indent + 2);
- xmlindent(hout, indent + 1);
- hprint(hout, "</arenas>\n");
- xmlindent(hout, indent);
- hprint(hout, "</%s>\n", tag);
-}
-
-void xmlamap(Hio *hout, AMap *s, char *tag, int indent){
- xmlindent(hout, indent);
- hprint(hout, "<%s", tag);
- xmlaname(hout, s->name, "name");
- xmlu64int(hout, s->start, "start");
- xmlu64int(hout, s->stop, "stop");
- hprint(hout, "/>\n");
-}
-
blob - c9e52b0bb666b62ae04903313c9e4e1b09ff0d36 (mode 644)
blob + /dev/null
--- src/cmd/venti/xml.h
+++ /dev/null
-void xmlamap(Hio *hout, AMap *v, char *tag, int indent);
-void xmlarena(Hio *hout, Arena *v, char *tag, int indent);
-void xmlindex(Hio *hout, Index *v, char *tag, int indent);
-
-void xmlaname(Hio *hout, char *v, char *tag);
-void xmlscore(Hio *hout, u8int *v, char *tag);
-void xmlsealed(Hio *hout, int v, char *tag);
-void xmlu32int(Hio *hout, u32int v, char *tag);
-void xmlu64int(Hio *hout, u64int v, char *tag);
-
-void xmlindent(Hio *hout, int indent);
blob - b24c98c631e44ada7cfb114ab0a0e13b63d3ce08 (mode 644)
blob + /dev/null
--- src/cmd/venti/zeropart.c
+++ /dev/null
-#include "stdinc.h"
-#include "dat.h"
-#include "fns.h"
-
-void
-zeropart(Part *part, int blocksize)
-{
- ZBlock *b;
- u64int addr;
- int w;
-
- fprint(2, "clearing the partition\n");
-
- b = alloczblock(MaxIoSize, 1);
-
- w = 0;
- for(addr = PartBlank; addr + MaxIoSize <= part->size; addr += MaxIoSize){
- if(writepart(part, addr, b->data, MaxIoSize) < 0)
- sysfatal("can't initialize %s, writing block %d failed: %r", part->name, w);
- w++;
- }
-
- for(; addr + blocksize <= part->size; addr += blocksize)
- if(writepart(part, addr, b->data, blocksize) < 0)
- sysfatal("can't initialize %s: %r", part->name);
-
- freezblock(b);
-}
