commit a0d146edd7a7de6236a0d60baafeeb59f8452aae from: rsc date: Tue Jul 12 15:23:36 2005 UTC return of venti commit - 88bb285e3d87ec2508840af33f7e0af53ec3c13c commit + a0d146edd7a7de6236a0d60baafeeb59f8452aae blob - 1a1b652da92eb4ef33db5b674f3a5b4826e3eba3 blob + bba8d2eb824648e2e9b95fa8703609c7ac4b35bb --- src/cmd/venti/mkfile +++ src/cmd/venti/mkfile @@ -1,3 +1,13 @@ -%:VQ: - echo venti will return once it is debugged. +<$PLAN9/src/mkhdr +DIRS=srv + +TARG=\ + copy\ + read\ + sync\ + write\ + +<$PLAN9/src/mkmany +<$PLAN9/src/mkdirs + blob - /dev/null blob + 89fbbac9d6d35e9b94fb3e9660563989b511c117 (mode 644) --- /dev/null +++ src/cmd/venti/copy.c @@ -0,0 +1,170 @@ +#include +#include +#include +#include +#include + +int changes; +int rewrite; +int ignoreerrors; +int fast; +int verbose; +VtConn *zsrc, *zdst; + +void +usage(void) +{ + fprint(2, "usage: copy [-fir] [-t type] srchost dsthost score\n"); + threadexitsall("usage"); +} + +void +walk(uchar score[VtScoreSize], uint type, int base) +{ + int i, n; + uchar *buf; + VtEntry e; + VtRoot root; + + if(memcmp(score, vtzeroscore, VtScoreSize) == 0) + return; + + buf = vtmallocz(VtMaxLumpSize); + if(fast && vtread(zdst, score, type, buf, VtMaxLumpSize) >= 0){ + if(verbose) + fprint(2, "skip %V\n", score); + free(buf); + return; + } + + n = vtread(zsrc, score, type, buf, VtMaxLumpSize); + if(n < 0){ + if(rewrite){ + changes++; + memmove(score, vtzeroscore, VtScoreSize); + }else if(!ignoreerrors) + sysfatal("reading block %V (type %d): %r", type, score); + return; + } + + switch(type){ + case VtRootType: + if(vtrootunpack(&root, buf) < 0){ + fprint(2, "warning: could not unpack root in %V %d\n", score, type); + break; + } + walk(root.score, VtDirType, 0); + walk(root.prev, VtRootType, 0); + vtrootpack(&root, buf); /* walk might have changed score */ + break; + + case VtDirType: + for(i=0; i= 0) + break; + } + if(type == VtMaxType) + sysfatal("could not find block %V of any type", score); + } + + walk(score, type, VtDirType); + if(changes) + print("%s:%V (%d pointers rewritten)\n", prefix, score, changes); + + if(vtsync(zdst) < 0) + sysfatal("could not sync dst server: %r"); + + threadexitsall(0); +} blob - /dev/null blob + 29cf9ecf357ee848a501062d669c61e3dfc299b2 (mode 644) --- /dev/null +++ src/cmd/venti/devnull.c @@ -0,0 +1,80 @@ +/* Copyright (c) 2004 Russ Cox */ +#include +#include +#include +#include +#include + +#ifndef _UNISTD_H_ +#pragma varargck type "F" VtFcall* +#pragma varargck type "T" void +#endif + +int verbose; + +enum +{ + STACK = 8192, +}; + +void +usage(void) +{ + fprint(2, "usage: venti/devnull [-v] [-a address]\n"); + threadexitsall("usage"); +} + +void +threadmain(int argc, char **argv) +{ + VtReq *r; + VtSrv *srv; + char *address; + Packet *p; + + fmtinstall('V', vtscorefmt); + fmtinstall('F', vtfcallfmt); + + address = "tcp!*!venti"; + + ARGBEGIN{ + case 'v': + verbose++; + break; + case 'a': + address = EARGF(usage()); + break; + default: + usage(); + }ARGEND + + srv = vtlisten(address); + if(srv == nil) + sysfatal("vtlisten %s: %r", argv[1]); + + while((r = vtgetreq(srv)) != nil){ + r->rx.msgtype = r->tx.msgtype+1; + if(verbose) + fprint(2, "<- %F\n", &r->tx); + switch(r->tx.msgtype){ + case VtTping: + break; + case VtTgoodbye: + break; + case VtTread: + r->rx.error = vtstrdup("no such block"); + r->rx.msgtype = VtRerror; + break; + case VtTwrite: + packetsha1(r->tx.data, r->rx.score); + break; + case VtTsync: + break; + } + if(verbose) + fprint(2, "-> %F\n", &r->rx); + vtrespond(r); + } + threadexitsall(nil); +} + blob - /dev/null blob + f18cbf3500fe09d041030297d382a87412bfe06d (mode 644) --- /dev/null +++ src/cmd/venti/mkroot.c @@ -0,0 +1,59 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +char *host; + +void +usage(void) +{ + fprint(2, "usage: mkroot [-h host] name type score blocksize prev\n"); + threadexitsall("usage"); +} + +void +threadmain(int argc, char *argv[]) +{ + uchar score[VtScoreSize]; + uchar buf[VtRootSize]; + VtConn *z; + VtRoot root; + + ARGBEGIN{ + case 'h': + host = EARGF(usage()); + break; + default: + usage(); + break; + }ARGEND + + if(argc != 5) + usage(); + + ventifmtinstall(); + + strecpy(root.name, root.name+sizeof root.name, argv[0]); + strecpy(root.type, root.type+sizeof root.type, argv[1]); + if(vtparsescore(argv[2], strlen(argv[2]), nil, root.score) < 0) + sysfatal("bad score '%s'", argv[2]); + root.blocksize = atoi(argv[3]); + if(vtparsescore(argv[4], strlen(argv[4]), nil, root.prev) < 0) + sysfatal("bad score '%s'", argv[4]); + vtrootpack(&root, buf); + + z = vtdial(host); + if(z == nil) + sysfatal("could not connect to server: %r"); + + if(vtconnect(z) < 0) + sysfatal("vtconnect: %r"); + + if(vtwrite(z, score, VtRootType, buf, VtRootSize) < 0) + sysfatal("vtwrite: %r"); + if(vtsync(z) < 0) + sysfatal("vtsync: %r"); + vthangup(z); + print("%V\n", score); + threadexitsall(0); +} blob - /dev/null blob + b7a09ef869da881443ecce1e4b81796d66b0bc16 (mode 644) --- /dev/null +++ src/cmd/venti/randtest.c @@ -0,0 +1,334 @@ +#include +#include +#include +#include +#include + + +enum { STACK = 32768 }; +void xxxsrand(long); +long xxxlrand(void); + +Channel *cw; +Channel *cr; +char *host; +int blocksize, seed, randpct; +int doread, dowrite, packets, permute; +vlong totalbytes, cur; +VtConn *z; +int multi; +int maxpackets; +int sequence; +int doublecheck = 1; +uint *order; + +void +usage(void) +{ + fprint(2, "usage: randtest [-q] [-h host] [-s seed] [-b blocksize] [-p randpct] [-n totalbytes] [-M maxblocks] [-P] [-r] [-w]\n"); + threadexitsall("usage"); +} + +void +wr(char *buf, char *buf2) +{ + uchar score[VtScoreSize], score2[VtScoreSize]; + DigestState ds; + + memset(&ds, 0, sizeof ds); + if(doublecheck) + sha1((uchar*)buf, blocksize, score, &ds); + if(vtwrite(z, score2, VtDataType, (uchar*)buf, blocksize) < 0) + sysfatal("vtwrite %V at %,lld: %r", score, cur); + if(doublecheck && memcmp(score, score2, VtScoreSize) != 0) + sysfatal("score mismatch! %V %V", score, score2); +} + +void +wrthread(void *v) +{ + char *p; + + USED(v); + while((p = recvp(cw)) != nil){ + wr(p, nil); + free(p); + } +} + +void +rd(char *buf, char *buf2) +{ + uchar score[VtScoreSize]; + DigestState ds; + + memset(&ds, 0, sizeof ds); + sha1((uchar*)buf, blocksize, score, &ds); + if(vtread(z, score, VtDataType, (uchar*)buf2, blocksize) < 0) + sysfatal("vtread %V at %,lld: %r", score, cur); + if(memcmp(buf, buf2, blocksize) != 0) + sysfatal("bad data read! %V", score); +} + +void +rdthread(void *v) +{ + char *p, *buf2; + + buf2 = vtmalloc(blocksize); + USED(v); + while((p = recvp(cr)) != nil){ + rd(p, buf2); + free(p); + } +} + +char *template; + +void +run(void (*fn)(char*, char*), Channel *c) +{ + int i, t, j, packets; + char *buf2, *buf; + + buf2 = vtmalloc(blocksize); + buf = vtmalloc(blocksize); + cur = 0; + packets = totalbytes/blocksize; + if(maxpackets == 0) + maxpackets = packets; + order = vtmalloc(packets*sizeof order[0]); + for(i=0; i= 0) + rng_vec[i] = x; + } +} + +void +xxxsrand(long seed) +{ + isrand(seed); +} + +long +xxxlrand(void) +{ + ulong x; + + rng_tap--; + if(rng_tap < rng_vec) { + if(rng_feed == 0) { + isrand(1); + rng_tap--; + } + rng_tap += LEN; + } + rng_feed--; + if(rng_feed < rng_vec) + rng_feed += LEN; + x = (*rng_feed + *rng_tap) & MASK; + *rng_feed = x; + + return x; +} + blob - /dev/null blob + 3f3441e731ae4a9e22a575002c5de2964d625a19 (mode 644) --- /dev/null +++ src/cmd/venti/read.c @@ -0,0 +1,75 @@ +#include +#include +#include +#include +#include + +void +usage(void) +{ + fprint(2, "usage: read [-h host] [-t type] score\n"); + threadexitsall("usage"); +} + +void +threadmain(int argc, char *argv[]) +{ + int type, n; + uchar score[VtScoreSize]; + uchar *buf; + VtConn *z; + char *host; + + fmtinstall('F', vtfcallfmt); + fmtinstall('V', vtscorefmt); + + host = nil; + type = -1; + ARGBEGIN{ + case 'h': + host = EARGF(usage()); + break; + case 't': + type = atoi(argv[1]); + break; + default: + usage(); + break; + }ARGEND + + if(argc != 1) + usage(); + + if(vtparsescore(argv[0], nil, score) < 0) + sysfatal("could not parse score '%s': %r", argv[0]); + + buf = vtmallocz(VtMaxLumpSize); + + z = vtdial(host); + if(z == nil) + sysfatal("could not connect to server: %r"); + + if(vtconnect(z) < 0) + sysfatal("vtconnect: %r"); + + if(type == -1){ + n = -1; + for(type=0; type= 0){ + fprint(2, "venti/read%s%s %V %d\n", host ? " -h" : "", host ? host : "", + score, type); + break; + } + } + }else{ + type = atoi(argv[1]); + n = vtread(z, score, type, buf, VtMaxLumpSize); + } + vthangup(z); + if(n < 0) + sysfatal("could not read block: %r"); + if(write(1, buf, n) != n) + sysfatal("write: %r"); + threadexitsall(0); +} blob - /dev/null blob + bb1d9b6b50c2b0dcd612d8747b5f38c694784eb1 (mode 644) --- /dev/null +++ src/cmd/venti/readlist.c @@ -0,0 +1,112 @@ +#include +#include +#include +#include +#include + +char *host; +Biobuf b; +VtConn *z; +uchar *buf; +void run(Biobuf*); +int nn; + +void +usage(void) +{ + fprint(2, "usage: readlist [-h host] list\n"); + threadexitsall("usage"); +} + +int +parsescore(uchar *score, char *buf, int n) +{ + int i, c; + + memset(score, 0, VtScoreSize); + + if(n != VtScoreSize*2){ + werrstr("score wrong length %d", n); + return -1; + } + for(i=0; i= '0' && buf[i] <= '9') + c = buf[i] - '0'; + else if(buf[i] >= 'a' && buf[i] <= 'f') + c = buf[i] - 'a' + 10; + else if(buf[i] >= 'A' && buf[i] <= 'F') + c = buf[i] - 'A' + 10; + else { + c = buf[i]; + werrstr("bad score char %d '%c'", c, c); + return -1; + } + + if((i & 1) == 0) + c <<= 4; + + score[i>>1] |= c; + } + return 0; +} + +void +threadmain(int argc, char *argv[]) +{ + int fd, i; + + ARGBEGIN{ + case 'h': + host = EARGF(usage()); + break; + default: + usage(); + break; + }ARGEND + + fmtinstall('V', vtscorefmt); + buf = vtmallocz(VtMaxLumpSize); + z = vtdial(host); + if(z == nil) + sysfatal("could not connect to server: %r"); + if(vtconnect(z) < 0) + sysfatal("vtconnect: %r"); + + if(argc == 0){ + Binit(&b, 0, OREAD); + run(&b); + }else{ + for(i=0; i +#include +#include +#include +#include + +#ifndef _UNISTD_H_ +#pragma varargck type "F" VtFcall* +#pragma varargck type "T" void +#endif + +VtConn *z; +int verbose; + +enum +{ + STACK = 8192, +}; + +void +usage(void) +{ + fprint(2, "usage: venti/ro [-v] [-a address] [-h address]\n"); + threadexitsall("usage"); +} + +void +readthread(void *v) +{ + char err[ERRMAX]; + VtReq *r; + uchar *buf; + int n; + + r = v; + buf = vtmalloc(r->tx.count); + if((n=vtread(z, r->tx.score, r->tx.blocktype, buf, r->tx.count)) < 0){ + r->rx.msgtype = VtRerror; + rerrstr(err, sizeof err); + r->rx.error = vtstrdup(err); + free(buf); + }else{ + r->rx.data = packetforeign(buf, n, free, buf); + } + if(verbose) + fprint(2, "-> %F\n", &r->rx); + vtrespond(r); +} + +void +threadmain(int argc, char **argv) +{ + VtReq *r; + VtSrv *srv; + char *address, *ventiaddress; + + fmtinstall('F', vtfcallfmt); + fmtinstall('V', vtscorefmt); + + address = "tcp!*!venti"; + ventiaddress = nil; + + ARGBEGIN{ + case 'v': + verbose++; + break; + case 'a': + address = EARGF(usage()); + break; + case 'h': + ventiaddress = EARGF(usage()); + break; + default: + usage(); + }ARGEND + + if((z = vtdial(ventiaddress)) == nil) + sysfatal("vtdial %s: %r", ventiaddress); + if(vtconnect(z) < 0) + sysfatal("vtconnect: %r"); + + srv = vtlisten(address); + if(srv == nil) + sysfatal("vtlisten %s: %r", address); + + while((r = vtgetreq(srv)) != nil){ + r->rx.msgtype = r->tx.msgtype+1; + if(verbose) + fprint(2, "<- %F\n", &r->tx); + switch(r->tx.msgtype){ + case VtTping: + break; + case VtTgoodbye: + break; + case VtTread: + threadcreate(readthread, r, 16384); + continue; + case VtTwrite: + r->rx.error = vtstrdup("read-only server"); + r->rx.msgtype = VtRerror; + break; + case VtTsync: + break; + } + if(verbose) + fprint(2, "-> %F\n", &r->rx); + vtrespond(r); + } + threadexitsall(nil); +} + blob - /dev/null blob + 15bf44d22ceb8c396a599da98f18ce0a83b420b1 (mode 644) --- /dev/null +++ src/cmd/venti/srv/arena.c @@ -0,0 +1,737 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +typedef struct ASum ASum; + +struct ASum +{ + Arena *arena; + ASum *next; +}; + +static void sealarena(Arena *arena); +static int okarena(Arena *arena); +static int loadarena(Arena *arena); +static CIBlock *getcib(Arena *arena, int clump, int writing, CIBlock *rock); +static void putcib(Arena *arena, CIBlock *cib); +static void sumproc(void *); + +static QLock sumlock; +static Rendez sumwait; +static ASum *sumq; +static uchar zero[8192]; + +int arenasumsleeptime; + +int +initarenasum(void) +{ + sumwait.l = &sumlock; + + if(vtproc(sumproc, nil) < 0){ + seterr(EOk, "can't start arena checksum slave: %r"); + return -1; + } + return 0; +} + +/* + * make an Arena, and initialize it based upon the disk header and trailer. + */ +Arena* +initarena(Part *part, u64int base, u64int size, u32int blocksize) +{ + Arena *arena; + + arena = MKZ(Arena); + arena->part = part; + arena->blocksize = blocksize; + arena->clumpmax = arena->blocksize / ClumpInfoSize; + arena->base = base + blocksize; + arena->size = size - 2 * blocksize; + + if(loadarena(arena) < 0){ + seterr(ECorrupt, "arena header or trailer corrupted"); + freearena(arena); + return nil; + } + if(okarena(arena) < 0){ + freearena(arena); + return nil; + } + + if(arena->diskstats.sealed && scorecmp(zeroscore, arena->score)==0) + backsumarena(arena); + + return arena; +} + +void +freearena(Arena *arena) +{ + if(arena == nil) + return; + free(arena); +} + +Arena* +newarena(Part *part, u32int vers, char *name, u64int base, u64int size, u32int blocksize) +{ + int bsize; + Arena *arena; + + if(nameok(name) < 0){ + seterr(EOk, "illegal arena name", name); + return nil; + } + arena = MKZ(Arena); + arena->part = part; + arena->version = vers; + if(vers == ArenaVersion4) + arena->clumpmagic = _ClumpMagic; + else{ + do + arena->clumpmagic = fastrand(); + while(arena->clumpmagic==_ClumpMagic || arena->clumpmagic==0); + } + arena->blocksize = blocksize; + arena->clumpmax = arena->blocksize / ClumpInfoSize; + arena->base = base + blocksize; + arena->size = size - 2 * blocksize; + + namecp(arena->name, name); + + bsize = sizeof zero; + if(bsize > arena->blocksize) + bsize = arena->blocksize; + + if(wbarena(arena)<0 || wbarenahead(arena)<0 + || writepart(arena->part, arena->base, zero, bsize)<0){ + freearena(arena); + return nil; + } + + return arena; +} + +int +readclumpinfo(Arena *arena, int clump, ClumpInfo *ci) +{ + CIBlock *cib, r; + + cib = getcib(arena, clump, 0, &r); + if(cib == nil) + return -1; + unpackclumpinfo(ci, &cib->data->data[cib->offset]); + putcib(arena, cib); + return 0; +} + +int +readclumpinfos(Arena *arena, int clump, ClumpInfo *cis, int n) +{ + CIBlock *cib, r; + int i; + + for(i = 0; i < n; i++){ + cib = getcib(arena, clump + i, 0, &r); + if(cib == nil) + break; + unpackclumpinfo(&cis[i], &cib->data->data[cib->offset]); + putcib(arena, cib); + } + return i; +} + +/* + * write directory information for one clump + * must be called the arena locked + */ +int +writeclumpinfo(Arena *arena, int clump, ClumpInfo *ci) +{ + CIBlock *cib, r; + + cib = getcib(arena, clump, 1, &r); + if(cib == nil) + return -1; + dirtydblock(cib->data, DirtyArenaCib); + packclumpinfo(ci, &cib->data->data[cib->offset]); + putcib(arena, cib); + return 0; +} + +u64int +arenadirsize(Arena *arena, u32int clumps) +{ + return ((clumps / arena->clumpmax) + 1) * arena->blocksize; +} + +/* + * read a clump of data + * n is a hint of the size of the data, not including the header + * make sure it won't run off the end, then return the number of bytes actually read + */ +u32int +readarena(Arena *arena, u64int aa, u8int *buf, long n) +{ + DBlock *b; + u64int a; + u32int blocksize, off, m; + long nn; + + if(n == 0) + return -1; + + qlock(&arena->lock); + a = arena->size - arenadirsize(arena, arena->memstats.clumps); + qunlock(&arena->lock); + if(aa >= a){ + seterr(EOk, "reading beyond arena clump storage: clumps=%d aa=%lld a=%lld -1 clumps=%lld\n", arena->memstats.clumps, aa, a, arena->size - arenadirsize(arena, arena->memstats.clumps - 1)); + return -1; + } + if(aa + n > a) + n = a - aa; + + blocksize = arena->blocksize; + a = arena->base + aa; + off = a & (blocksize - 1); + a -= off; + nn = 0; + for(;;){ + b = getdblock(arena->part, a, OREAD); + if(b == nil) + return -1; + m = blocksize - off; + if(m > n - nn) + m = n - nn; + memmove(&buf[nn], &b->data[off], m); + putdblock(b); + nn += m; + if(nn == n) + break; + off = 0; + a += blocksize; + } + return n; +} + +/* + * write some data to the clump section at a given offset + * used to fix up corrupted arenas. + */ +u32int +writearena(Arena *arena, u64int aa, u8int *clbuf, u32int n) +{ + DBlock *b; + u64int a; + u32int blocksize, off, m; + long nn; + int ok; + + if(n == 0) + return -1; + + qlock(&arena->lock); + a = arena->size - arenadirsize(arena, arena->memstats.clumps); + if(aa >= a || aa + n > a){ + qunlock(&arena->lock); + seterr(EOk, "writing beyond arena clump storage"); + return -1; + } + + blocksize = arena->blocksize; + a = arena->base + aa; + off = a & (blocksize - 1); + a -= off; + nn = 0; + for(;;){ + b = getdblock(arena->part, a, off != 0 || off + n < blocksize ? ORDWR : OWRITE); + if(b == nil){ + qunlock(&arena->lock); + return -1; + } + dirtydblock(b, DirtyArena); + m = blocksize - off; + if(m > n - nn) + m = n - nn; + memmove(&b->data[off], &clbuf[nn], m); + // ok = writepart(arena->part, a, b->data, blocksize); + ok = 0; + putdblock(b); + if(ok < 0){ + qunlock(&arena->lock); + return -1; + } + nn += m; + if(nn == n) + break; + off = 0; + a += blocksize; + } + qunlock(&arena->lock); + return n; +} + +/* + * allocate space for the clump and write it, + * updating the arena directory +ZZZ question: should this distinguish between an arena +filling up and real errors writing the clump? + */ +u64int +writeaclump(Arena *arena, Clump *c, u8int *clbuf, u64int start, u64int *pa) +{ + DBlock *b; + u64int a, aa; + u32int clump, n, nn, m, off, blocksize; + int ok; + AState as; + + n = c->info.size + ClumpSize + U32Size; + qlock(&arena->lock); + aa = arena->memstats.used; + if(arena->memstats.sealed + || aa + n + U32Size + arenadirsize(arena, arena->memstats.clumps + 1) > arena->size){ + if(!arena->memstats.sealed){ + trace(0, "seal memstats %s", arena->name); + arena->memstats.sealed = 1; + as.arena = arena; + as.aa = start+aa; + as.stats = arena->memstats; + setdcachestate(&as); + } + qunlock(&arena->lock); + return TWID64; + } + if(packclump(c, &clbuf[0], arena->clumpmagic) < 0){ + qunlock(&arena->lock); + return TWID64; + } + + /* + * write the data out one block at a time + */ + blocksize = arena->blocksize; + a = arena->base + aa; + off = a & (blocksize - 1); + a -= off; + nn = 0; + for(;;){ + b = getdblock(arena->part, a, off != 0 ? ORDWR : OWRITE); + if(b == nil){ + qunlock(&arena->lock); + return TWID64; + } + dirtydblock(b, DirtyArena); + m = blocksize - off; + if(m > n - nn) + m = n - nn; + memmove(&b->data[off], &clbuf[nn], m); + // ok = writepart(arena->part, a, b->data, blocksize); + ok = 0; + putdblock(b); + if(ok < 0){ + qunlock(&arena->lock); + return TWID64; + } + nn += m; + if(nn == n) + break; + off = 0; + a += blocksize; + } + + arena->memstats.used += c->info.size + ClumpSize; + arena->memstats.uncsize += c->info.uncsize; + if(c->info.size < c->info.uncsize) + arena->memstats.cclumps++; + + clump = arena->memstats.clumps++; + if(arena->memstats.clumps == 0) + sysfatal("clumps wrapped"); + arena->wtime = now(); + if(arena->ctime == 0) + arena->ctime = arena->wtime; + + writeclumpinfo(arena, clump, &c->info); + + /* set up for call to setdcachestate */ + as.arena = arena; + as.aa = start+arena->memstats.used; + as.stats = arena->memstats; + + /* update this before calling setdcachestate so it cannot be behind dcache.diskstate */ + *pa = start+aa; + setdcachestate(&as); + qunlock(&arena->lock); + + return aa; +} + +int +atailcmp(ATailStats *a, ATailStats *b) +{ + /* good test */ + if(a->used < b->used) + return -1; + if(a->used > b->used) + return 1; + + /* suspect tests - why order this way? (no one cares) */ + if(a->clumps < b->clumps) + return -1; + if(a->clumps > b->clumps) + return 1; + if(a->cclumps < b->cclumps) + return -1; + if(a->cclumps > b->cclumps) + return 1; + if(a->uncsize < b->uncsize) + return -1; + if(a->uncsize > b->uncsize) + return 1; + if(a->sealed < b->sealed) + return -1; + if(a->sealed > b->sealed) + return 1; + + /* everything matches */ + return 0; +} + +void +setatailstate(AState *as) +{ + int i, j, osealed; + Arena *a; + Index *ix; + + trace(0, "setatailstate %s 0x%llux clumps %d", as->arena->name, as->aa, as->stats.clumps); + + ix = mainindex; + for(i=0; inarenas; i++) + if(ix->arenas[i] == as->arena) + break; + if(i==ix->narenas || as->aa < ix->amap[i].start || as->aa >= ix->amap[i].stop || as->arena != ix->arenas[i]){ + fprint(2, "funny settailstate 0x%llux\n", as->aa); + return; + } + + for(j=i; --j>=0; ){ + a = ix->arenas[j]; + if(atailcmp(&a->diskstats, &a->memstats) == 0) + break; + } + for(j++; j<=i; j++){ + a = ix->arenas[j]; + qlock(&a->lock); + osealed = a->diskstats.sealed; + if(j == i) + a->diskstats = as->stats; + else + a->diskstats = a->memstats; + wbarena(a); + if(a->diskstats.sealed != osealed && !a->inqueue) + sealarena(a); + qunlock(&a->lock); + } +} + +/* + * once sealed, an arena never has any data added to it. + * it should only be changed to fix errors. + * this also syncs the clump directory. + */ +static void +sealarena(Arena *arena) +{ + arena->inqueue = 1; + backsumarena(arena); +} + +void +backsumarena(Arena *arena) +{ + ASum *as; + + if(sumwait.l == nil) + return; + + as = MK(ASum); + if(as == nil) + return; + qlock(&sumlock); + as->arena = arena; + as->next = sumq; + sumq = as; + rwakeup(&sumwait); + qunlock(&sumlock); +} + +static void +sumproc(void *unused) +{ + ASum *as; + Arena *arena; + + USED(unused); + + for(;;){ + qlock(&sumlock); + while(sumq == nil) + rsleep(&sumwait); + as = sumq; + sumq = as->next; + qunlock(&sumlock); + arena = as->arena; + free(as); + + sumarena(arena); + } +} + +void +sumarena(Arena *arena) +{ + ZBlock *b; + DigestState s; + u64int a, e; + u32int bs; + u8int score[VtScoreSize]; + + bs = MaxIoSize; + if(bs < arena->blocksize) + bs = arena->blocksize; + + /* + * read & sum all blocks except the last one + */ + memset(&s, 0, sizeof s); + b = alloczblock(bs, 0, arena->part->blocksize); + e = arena->base + arena->size; + for(a = arena->base - arena->blocksize; a + arena->blocksize <= e; a += bs){ + sleep(arenasumsleeptime); + if(a + bs > e) + bs = arena->blocksize; + if(readpart(arena->part, a, b->data, bs) < 0) + goto ReadErr; + addstat(StatSumRead, 1); + addstat(StatSumReadBytes, bs); + sha1(b->data, bs, nil, &s); + } + + /* + * the last one is special, since it may already have the checksum included + */ + bs = arena->blocksize; + if(readpart(arena->part, e, b->data, bs) < 0){ +ReadErr: + logerr(EOk, "sumarena can't sum %s, read at %lld failed: %r", arena->name, a); + freezblock(b); + return; + } + addstat(StatSumRead, 1); + addstat(StatSumReadBytes, bs); + + sha1(b->data, bs-VtScoreSize, nil, &s); + sha1(zeroscore, VtScoreSize, nil, &s); + sha1(nil, 0, score, &s); + + /* + * check for no checksum or the same + * + * the writepart is okay because we flushed the dcache in sealarena + */ + if(scorecmp(score, &b->data[bs - VtScoreSize]) != 0){ + if(scorecmp(zeroscore, &b->data[bs - VtScoreSize]) != 0) + logerr(EOk, "overwriting mismatched checksums for arena=%s, found=%V calculated=%V", + arena->name, &b->data[bs - VtScoreSize], score); + scorecp(&b->data[bs - VtScoreSize], score); + if(writepart(arena->part, e, b->data, bs) < 0) + logerr(EOk, "sumarena can't write sum for %s: %r", arena->name); + } + freezblock(b); + + qlock(&arena->lock); + scorecp(arena->score, score); + qunlock(&arena->lock); +} + +/* + * write the arena trailer block to the partition + */ +int +wbarena(Arena *arena) +{ + DBlock *b; + int bad; + + if((b = getdblock(arena->part, arena->base + arena->size, OWRITE)) == nil){ + logerr(EAdmin, "can't write arena trailer: %r"); + return -1; + } + dirtydblock(b, DirtyArenaTrailer); + bad = okarena(arena)<0 || packarena(arena, b->data)<0; + putdblock(b); + if(bad) + return -1; + return 0; +} + +int +wbarenahead(Arena *arena) +{ + ZBlock *b; + ArenaHead head; + int bad; + + namecp(head.name, arena->name); + head.version = arena->version; + head.size = arena->size + 2 * arena->blocksize; + head.blocksize = arena->blocksize; + head.clumpmagic = arena->clumpmagic; + b = alloczblock(arena->blocksize, 1, arena->part->blocksize); + if(b == nil){ + logerr(EAdmin, "can't write arena header: %r"); +///ZZZ add error message? + return -1; + } + /* + * this writepart is okay because it only happens + * during initialization. + */ + bad = packarenahead(&head, b->data)<0 || + writepart(arena->part, arena->base - arena->blocksize, b->data, arena->blocksize)<0; + freezblock(b); + if(bad) + return -1; + return 0; +} + +/* + * read the arena header and trailer blocks from disk + */ +static int +loadarena(Arena *arena) +{ + ArenaHead head; + ZBlock *b; + + b = alloczblock(arena->blocksize, 0, arena->part->blocksize); + if(b == nil) + return -1; + if(readpart(arena->part, arena->base + arena->size, b->data, arena->blocksize) < 0){ + freezblock(b); + return -1; + } + if(unpackarena(arena, b->data) < 0){ + freezblock(b); + return -1; + } + if(arena->version != ArenaVersion4 && arena->version != ArenaVersion5){ + seterr(EAdmin, "unknown arena version %d", arena->version); + freezblock(b); + return -1; + } + scorecp(arena->score, &b->data[arena->blocksize - VtScoreSize]); + + if(readpart(arena->part, arena->base - arena->blocksize, b->data, arena->blocksize) < 0){ + logerr(EAdmin, "can't read arena header: %r"); + freezblock(b); + return 0; + } + if(unpackarenahead(&head, b->data) < 0) + logerr(ECorrupt, "corrupted arena header: %r"); + else if(namecmp(arena->name, head.name)!=0 + || arena->clumpmagic != head.clumpmagic + || arena->version != head.version + || arena->blocksize != head.blocksize + || arena->size + 2 * arena->blocksize != head.size){ + if(namecmp(arena->name, head.name)!=0) + logerr(ECorrupt, "arena tail name %s head %s", + arena->name, head.name); + else if(arena->clumpmagic != head.clumpmagic) + logerr(ECorrupt, "arena tail clumpmagic 0x%lux head 0x%lux", + (ulong)arena->clumpmagic, (ulong)head.clumpmagic); + else if(arena->version != head.version) + logerr(ECorrupt, "arena tail version %d head version %d", + arena->version, head.version); + else if(arena->blocksize != head.blocksize) + logerr(ECorrupt, "arena tail block size %d head %d", + arena->blocksize, head.blocksize); + else if(arena->size+2*arena->blocksize != head.size) + logerr(ECorrupt, "arena tail size %lud head %lud", + (ulong)arena->size+2*arena->blocksize, head.size); + else + logerr(ECorrupt, "arena header inconsistent with arena data"); + } + freezblock(b); + + return 0; +} + +static int +okarena(Arena *arena) +{ + u64int dsize; + int ok; + + ok = 0; + dsize = arenadirsize(arena, arena->diskstats.clumps); + if(arena->diskstats.used + dsize > arena->size){ + seterr(ECorrupt, "arena used > size"); + ok = -1; + } + + if(arena->diskstats.cclumps > arena->diskstats.clumps) + logerr(ECorrupt, "arena has more compressed clumps than total clumps"); + + if(arena->diskstats.uncsize + arena->diskstats.clumps * ClumpSize + arena->blocksize < arena->diskstats.used) + logerr(ECorrupt, "arena uncompressed size inconsistent with used space %lld %d %lld", arena->diskstats.uncsize, arena->diskstats.clumps, arena->diskstats.used); + + if(arena->ctime > arena->wtime) + logerr(ECorrupt, "arena creation time after last write time"); + + return ok; +} + +static CIBlock* +getcib(Arena *arena, int clump, int writing, CIBlock *rock) +{ + int mode; + CIBlock *cib; + u32int block, off; + + if(clump >= arena->memstats.clumps){ + seterr(EOk, "clump directory access out of range"); + return nil; + } + block = clump / arena->clumpmax; + off = (clump - block * arena->clumpmax) * ClumpInfoSize; + cib = rock; + cib->block = block; + cib->offset = off; + + if(writing){ + if(off == 0 && clump == arena->memstats.clumps-1) + mode = OWRITE; + else + mode = ORDWR; + }else + mode = OREAD; + + cib->data = getdblock(arena->part, + arena->base + arena->size - (block + 1) * arena->blocksize, mode); + if(cib->data == nil) + return nil; + return cib; +} + +static void +putcib(Arena *arena, CIBlock *cib) +{ + putdblock(cib->data); + cib->data = nil; +} blob - /dev/null blob + 2ad1bb02c0d1d274dedf3af49dd5f96612498292 (mode 644) --- /dev/null +++ src/cmd/venti/srv/arenas.c @@ -0,0 +1,414 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +typedef struct AHash AHash; + +/* + * hash table for finding arena's based on their names. + */ +struct AHash +{ + AHash *next; + Arena *arena; +}; + +enum +{ + AHashSize = 512 +}; + +static AHash *ahash[AHashSize]; + +static u32int +hashstr(char *s) +{ + u32int h; + int c; + + h = 0; + for(; c = *s; s++){ + c ^= c << 6; + h += (c << 11) ^ (c >> 1); + c = *s; + h ^= (c << 14) + (c << 7) + (c << 4) + c; + } + return h; +} + +int +addarena(Arena *arena) +{ + AHash *a; + u32int h; + + h = hashstr(arena->name) & (AHashSize - 1); + a = MK(AHash); + if(a == nil) + return -1; + a->arena = arena; + a->next = ahash[h]; + ahash[h] = a; + return 0; +} + +Arena* +findarena(char *name) +{ + AHash *a; + u32int h; + + h = hashstr(name) & (AHashSize - 1); + for(a = ahash[h]; a != nil; a = a->next) + if(strcmp(a->arena->name, name) == 0) + return a->arena; + return nil; +} + +int +delarena(Arena *arena) +{ + AHash *a, *last; + u32int h; + + h = hashstr(arena->name) & (AHashSize - 1); + last = nil; + for(a = ahash[h]; a != nil; a = a->next){ + if(a->arena == arena){ + if(last != nil) + last->next = a->next; + else + ahash[h] = a->next; + free(a); + return 0; + } + last = a; + } + return -1; +} + +ArenaPart* +initarenapart(Part *part) +{ + AMapN amn; + ArenaPart *ap; + ZBlock *b; + u32int i; + int ok; + + b = alloczblock(HeadSize, 0, 0); + if(b == nil || readpart(part, PartBlank, b->data, HeadSize) < 0){ + seterr(EAdmin, "can't read arena partition header: %r"); + return nil; + } + + ap = MKZ(ArenaPart); + if(ap == nil){ + freezblock(b); + return nil; + } + ap->part = part; + ok = unpackarenapart(ap, b->data); + freezblock(b); + if(ok < 0){ + freearenapart(ap, 0); + return nil; + } + + ap->tabbase = (PartBlank + HeadSize + ap->blocksize - 1) & ~(ap->blocksize - 1); + if(ap->version != ArenaPartVersion){ + seterr(ECorrupt, "unknown arena partition version %d", ap->version); + freearenapart(ap, 0); + return nil; + } + if(ap->blocksize & (ap->blocksize - 1)){ + seterr(ECorrupt, "illegal non-power-of-2 block size %d\n", ap->blocksize); + freearenapart(ap, 0); + return nil; + } + if(ap->tabbase >= ap->arenabase){ + seterr(ECorrupt, "arena partition table overlaps with arena storage"); + freearenapart(ap, 0); + return nil; + } + ap->tabsize = ap->arenabase - ap->tabbase; + partblocksize(part, ap->blocksize); + ap->size = ap->part->size & ~(u64int)(ap->blocksize - 1); + + if(readarenamap(&amn, part, ap->tabbase, ap->tabsize) < 0){ + freearenapart(ap, 0); + return nil; + } + ap->narenas = amn.n; + ap->map = amn.map; + if(okamap(ap->map, ap->narenas, ap->arenabase, ap->size, "arena table") < 0){ + freearenapart(ap, 0); + return nil; + } + + ap->arenas = MKNZ(Arena*, ap->narenas); + for(i = 0; i < ap->narenas; i++){ + ap->arenas[i] = initarena(part, ap->map[i].start, ap->map[i].stop - ap->map[i].start, ap->blocksize); + if(ap->arenas[i] == nil){ + seterr(ECorrupt, "%s: %r", ap->map[i].name); + freearenapart(ap, 1); + return nil; + } + if(namecmp(ap->map[i].name, ap->arenas[i]->name) != 0){ + seterr(ECorrupt, "arena name mismatches with expected name: %s vs. %s", + ap->map[i].name, ap->arenas[i]->name); + freearenapart(ap, 1); + return nil; + } + if(findarena(ap->arenas[i]->name)){ + seterr(ECorrupt, "duplicate arena name %s in %s", + ap->map[i].name, ap->part->name); + freearenapart(ap, 1); + return nil; + } + } + + for(i = 0; i < ap->narenas; i++) + addarena(ap->arenas[i]); + + return ap; +} + +ArenaPart* +newarenapart(Part *part, u32int blocksize, u32int tabsize) +{ + ArenaPart *ap; + + if(blocksize & (blocksize - 1)){ + seterr(ECorrupt, "illegal non-power-of-2 block size %d\n", blocksize); + return nil; + } + ap = MKZ(ArenaPart); + if(ap == nil) + return nil; + + ap->version = ArenaPartVersion; + ap->part = part; + ap->blocksize = blocksize; + partblocksize(part, blocksize); + ap->size = part->size & ~(u64int)(blocksize - 1); + ap->tabbase = (PartBlank + HeadSize + blocksize - 1) & ~(blocksize - 1); + ap->arenabase = (ap->tabbase + tabsize + blocksize - 1) & ~(blocksize - 1); + ap->tabsize = ap->arenabase - ap->tabbase; + ap->narenas = 0; + + if(wbarenapart(ap) < 0){ + freearenapart(ap, 0); + return nil; + } + + return ap; +} + +int +wbarenapart(ArenaPart *ap) +{ + ZBlock *b; + + if(okamap(ap->map, ap->narenas, ap->arenabase, ap->size, "arena table") < 0) + return -1; + b = alloczblock(HeadSize, 1, 0); + if(b == nil) +//ZZZ set error message? + return -1; + + if(packarenapart(ap, b->data) < 0){ + seterr(ECorrupt, "can't make arena partition header: %r"); + freezblock(b); + return -1; + } + if(writepart(ap->part, PartBlank, b->data, HeadSize) < 0){ + seterr(EAdmin, "can't write arena partition header: %r"); + freezblock(b); + return -1; + } + freezblock(b); + + return wbarenamap(ap->map, ap->narenas, ap->part, ap->tabbase, ap->tabsize); +} + +void +freearenapart(ArenaPart *ap, int freearenas) +{ + int i; + + if(ap == nil) + return; + if(freearenas){ + for(i = 0; i < ap->narenas; i++){ + if(ap->arenas[i] == nil) + continue; + delarena(ap->arenas[i]); + freearena(ap->arenas[i]); + } + } + free(ap->map); + free(ap->arenas); + free(ap); +} + +int +okamap(AMap *am, int n, u64int start, u64int stop, char *what) +{ + u64int last; + u32int i; + + last = start; + for(i = 0; i < n; i++){ + if(am[i].start < last){ + if(i == 0) + seterr(ECorrupt, "invalid start address in %s", what); + else + seterr(ECorrupt, "overlapping ranges in %s", what); + return -1; + } + if(am[i].stop < am[i].start){ + seterr(ECorrupt, "invalid range in %s", what); + return -1; + } + last = am[i].stop; + } + if(last > stop){ + seterr(ECorrupt, "invalid ending address in %s", what); + return -1; + } + return 0; +} + +int +maparenas(AMap *am, Arena **arenas, int n, char *what) +{ + u32int i; + + for(i = 0; i < n; i++){ + arenas[i] = findarena(am[i].name); + if(arenas[i] == nil){ + seterr(EAdmin, "can't find arena '%s' for '%s'\n", am[i].name, what); + return -1; + } + } + return 0; +} + +int +readarenamap(AMapN *amn, Part *part, u64int base, u32int size) +{ + IFile f; + u32int ok; + + if(partifile(&f, part, base, size) < 0) + return -1; + ok = parseamap(&f, amn); + freeifile(&f); + return ok; +} + +int +wbarenamap(AMap *am, int n, Part *part, u64int base, u64int size) +{ + Fmt f; + ZBlock *b; + + b = alloczblock(size, 1, part->blocksize); + if(b == nil) + return -1; + + fmtzbinit(&f, b); + + if(outputamap(&f, am, n) < 0){ + seterr(ECorrupt, "arena set size too small"); + freezblock(b); + return -1; + } + if(writepart(part, base, b->data, size) < 0){ + seterr(EAdmin, "can't write arena set: %r"); + freezblock(b); + return -1; + } + freezblock(b); + return 0; +} + +/* + * amap: n '\n' amapelem * n + * n: u32int + * amapelem: name '\t' astart '\t' asize '\n' + * astart, asize: u64int + */ +int +parseamap(IFile *f, AMapN *amn) +{ + AMap *am; + u64int v64; + u32int v; + char *s, *t, *flds[4]; + int i, n; + + /* + * arenas + */ + if(ifileu32int(f, &v) < 0){ + seterr(ECorrupt, "syntax error: bad number of elements in %s", f->name); + return -1; + } + n = v; + if(n > MaxAMap){ + seterr(ECorrupt, "illegal number of elements in %s", f->name); + return -1; + } + am = MKNZ(AMap, n); + if(am == nil){ + fprint(2, "out of memory\n"); + return -1; + } + for(i = 0; i < n; i++){ + s = ifileline(f); + if(s) + t = estrdup(s); + else + t = nil; + if(s == nil || getfields(s, flds, 4, 0, "\t") != 3){ + fprint(2, "early eof after %d of %d, %s:#%d: %s\n", i, n, f->name, f->pos, t); + free(t); + return -1; + } + free(t); + if(nameok(flds[0]) < 0) + return -1; + namecp(am[i].name, flds[0]); + if(stru64int(flds[1], &v64) < 0){ + seterr(ECorrupt, "syntax error: bad arena base address in %s", f->name); + free(am); + return -1; + } + am[i].start = v64; + if(stru64int(flds[2], &v64) < 0){ + seterr(ECorrupt, "syntax error: bad arena size in %s", f->name); + free(am); + return -1; + } + am[i].stop = v64; + } + + amn->map = am; + amn->n = n; + return 0; +} + +int +outputamap(Fmt *f, AMap *am, int n) +{ + int i; + + if(fmtprint(f, "%ud\n", n) < 0) + return -1; + for(i = 0; i < n; i++) + if(fmtprint(f, "%s\t%llud\t%llud\n", am[i].name, am[i].start, am[i].stop) < 0) + return -1; + return 0; +} blob - /dev/null blob + 5c50a0df314c36c46a85a2df5a5447cd63ec5935 (mode 644) --- /dev/null +++ src/cmd/venti/srv/bloom.c @@ -0,0 +1,210 @@ +/* + * Bloom filter tracking which scores are present in our arenas + * and (more importantly) which are not. + */ + +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +int +bloominit(Bloom *b, vlong vsize, u8int *data) +{ + ulong size; + + size = vsize; + if(size != vsize){ /* truncation */ + werrstr("bloom data too big"); + return -1; + } + + b->size = size; + b->nhash = 32; /* will be fixed by caller on initialization */ + if(data != nil) + if(unpackbloomhead(b, data) < 0) + return -1; + +fprint(2, "bloom size %lud nhash %d\n", b->size, b->nhash); + b->mask = b->size-1; + b->data = data; + return 0; +} + +void +wbbloomhead(Bloom *b) +{ + packbloomhead(b, b->data); +} + +Bloom* +readbloom(Part *p) +{ + int i, n; + uint ones; + uchar buf[512]; + uchar *data; + u32int *a; + Bloom *b; + + b = vtmallocz(sizeof *b); + if(readpart(p, 0, buf, sizeof buf) < 0) + return nil; +fprint(2, "header %.16H\n", buf); + if(bloominit(b, 0, buf) < 0){ + vtfree(b); + return nil; + } + data = vtmallocz(b->size); + if(readpart(p, 0, data, b->size) < 0){ + vtfree(b); + vtfree(data); + return nil; + } + b->data = data; + b->part = p; + + a = (u32int*)b->data; + n = b->size/4; + ones = 0; + for(i=0; isize == MaxBloomSize) /* 2^32 overflows ulong */ + addstat(StatBloomBits, b->size*8-1); + else + addstat(StatBloomBits, b->size*8); + + return b; +} + +int +writebloom(Bloom *b) +{ + wbbloomhead(b); + return writepart(b->part, 0, b->data, b->size); +} + +/* + * Derive two random 32-bit quantities a, b from the score + * and then use a+b*i as a sequence of bloom filter indices. + * Michael Mitzenmacher has a recent (2005) paper saying this is okay. + * We reserve the bottom bytes (BloomHeadSize*8 bits) for the header. + */ +static void +gethashes(u8int *score, ulong *h) +{ + int i; + u32int a, b; + + a = 0; + b = 0; + for(i=4; i+8<=VtScoreSize; i+=8){ + a ^= *(u32int*)(score+i); + b ^= *(u32int*)(score+i+4); + } + if(i+4 <= VtScoreSize) /* 20 is not 4-aligned */ + a ^= *(u32int*)(score+i); + for(i=0; idata; + for(i=0; inhash; i++){ + x = h[i]; + y = &tab[(x&b->mask)>>5]; + z = 1<<(x&31); + if(!(*y&z)){ + nnew++; + *y |= z; + } + } + if(nnew) + addstat(StatBloomOnes, nnew); + + trace("markbloomfilter", "markbloomfilter exit"); +} + +static int +_inbloomfilter(Bloom *b, u8int *score) +{ + int i; + ulong h[BloomMaxHash], x; + u32int *tab; + + gethashes(score, h); + tab = (u32int*)b->data; + for(i=0; inhash; i++){ + x = h[i]; + if(!(tab[(x&b->mask)>>5] & (1<<(x&31)))) + return 0; + } + return 1; +} + +int +inbloomfilter(Bloom *b, u8int *score) +{ + int r; + uint ms; + + if(b == nil) + return 1; + + ms = msec(); + rlock(&b->lk); + r = _inbloomfilter(b, score); + runlock(&b->lk); + ms = ms - msec(); + addstat2(StatBloomLookup, 1, StatBloomLookupTime, ms); + if(r) + addstat(StatBloomMiss, 1); + else + addstat(StatBloomHit, 1); + return r; +} + +void +markbloomfilter(Bloom *b, u8int *score) +{ + if(b == nil) + return; + + rlock(&b->lk); + qlock(&b->mod); + _markbloomfilter(b, score); + qunlock(&b->mod); + runlock(&b->lk); +} + +static void +bloomwriteproc(void *v) +{ + Bloom *b; + + b = v; + for(;;){ + recv(b->writechan, 0); + if(writebloom(b) < 0) + fprint(2, "oops! writing bloom: %r\n"); + send(b->writedonechan, 0); + } +} + +void +startbloomproc(Bloom *b) +{ + b->writechan = chancreate(sizeof(void*), 0); + b->writedonechan = chancreate(sizeof(void*), 0); + vtproc(bloomwriteproc, b); +} blob - /dev/null blob + 240e77d726d97d3079515b44e62c54b3ee446ef3 (mode 644) --- /dev/null +++ src/cmd/venti/srv/buildbuck.c @@ -0,0 +1,132 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +/* + * An IEStream is a sorted list of index entries. + */ +struct IEStream +{ + Part *part; + u64int off; /* read position within part */ + u64int n; /* number of valid ientries left to read */ + u32int size; /* allocated space in buffer */ + u8int *buf; + u8int *pos; /* current place in buffer */ + u8int *epos; /* end of valid buffer contents */ +}; + +IEStream* +initiestream(Part *part, u64int off, u64int clumps, u32int size) +{ + IEStream *ies; + +//ZZZ out of memory? + ies = MKZ(IEStream); + ies->buf = MKN(u8int, size); + ies->epos = ies->buf; + ies->pos = ies->epos; + ies->off = off; + ies->n = clumps; + ies->size = size; + ies->part = part; + return ies; +} + +void +freeiestream(IEStream *ies) +{ + if(ies == nil) + return; + free(ies->buf); + free(ies); +} + +/* + * Return the next IEntry (still packed) in the stream. + */ +static u8int* +peekientry(IEStream *ies) +{ + u32int n, nn; + + n = ies->epos - ies->pos; + if(n < IEntrySize){ + memmove(ies->buf, ies->pos, n); + ies->epos = &ies->buf[n]; + ies->pos = ies->buf; + nn = ies->size; + if(nn > ies->n * IEntrySize) + nn = ies->n * IEntrySize; + nn -= n; + if(nn == 0) + return nil; +//fprint(2, "peek %d from %llud into %p\n", nn, ies->off, ies->epos); + if(readpart(ies->part, ies->off, ies->epos, nn) < 0){ + seterr(EOk, "can't read sorted index entries: %r"); + return nil; + } + ies->epos += nn; + ies->off += nn; + } + return ies->pos; +} + +/* + * Compute the bucket number for the given IEntry. + * Knows that the score is the first thing in the packed + * representation. + */ +static u32int +iebuck(Index *ix, u8int *b, IBucket *ib, IEStream *ies) +{ + USED(ies); + USED(ib); + return hashbits(b, 32) / ix->div; +} + +/* + * Fill ib with the next bucket in the stream. + */ +u32int +buildbucket(Index *ix, IEStream *ies, IBucket *ib, uint maxdata) +{ + IEntry ie1, ie2; + u8int *b; + u32int buck; + + buck = TWID32; + ib->n = 0; + while(ies->n){ + b = peekientry(ies); + if(b == nil) + return TWID32; +//fprint(2, "b=%p ies->n=%lld ib.n=%d buck=%d score=%V\n", b, ies->n, ib->n, iebuck(ix, b, ib, ies), b); + if(ib->n == 0) + buck = iebuck(ix, b, ib, ies); + else{ + if(buck != iebuck(ix, b, ib, ies)) + break; + if(ientrycmp(&ib->data[(ib->n - 1)* IEntrySize], b) == 0){ + /* + * guess that the larger address is the correct one to use + */ + unpackientry(&ie1, &ib->data[(ib->n - 1)* IEntrySize]); + unpackientry(&ie2, b); + seterr(EOk, "duplicate index entry for score=%V type=%d", ie1.score, ie1.ia.type); + ib->n--; + if(ie1.ia.addr > ie2.ia.addr) + memmove(b, &ib->data[ib->n * IEntrySize], IEntrySize); + } + } + if((ib->n+1)*IEntrySize > maxdata){ + seterr(EOk, "bucket overflow"); + return TWID32; + } + memmove(&ib->data[ib->n * IEntrySize], b, IEntrySize); + ib->n++; + ies->n--; + ies->pos += IEntrySize; + } + return buck; +} blob - /dev/null blob + 8714474a2f4d34bedf5ee37c3c2b12138112e561 (mode 644) --- /dev/null +++ src/cmd/venti/srv/buildindex.c @@ -0,0 +1,160 @@ +/* + * Rebuild the Venti index from scratch. + */ + +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +/* + * Write a single bucket. Could profit from a big buffer here + * so that we can absorb sporadic runs of blocks into one write, + * avoiding disk seeks. + */ +static int +writebucket(Index *ix, u32int buck, IBucket *ib, ZBlock *b) +{ + ISect *is; + + is = ix->sects[indexsect0(ix, buck)]; + if(buck < is->start || buck >= is->stop){ + seterr(EAdmin, "cannot find index section for bucket %lud\n", (ulong)buck); + return -1; + } + buck -= is->start; + +/* + qlock(&stats.lock); + stats.indexwrites++; + qunlock(&stats.lock); +*/ + packibucket(ib, b->data, is->bucketmagic); + return writepart(is->part, is->blockbase + ((u64int)buck << is->blocklog), b->data, is->blocksize); +} + +static int +buildindex(Index *ix, Part *part, u64int off, u64int clumps, int zero) +{ + IEStream *ies; + IBucket ib, zib; + ZBlock *z, *b; + u32int next, buck; + int ok; + uint nbuck; + u64int found = 0; + +//ZZZ make buffer size configurable + b = alloczblock(ix->blocksize, 0, ix->blocksize); + z = alloczblock(ix->blocksize, 1, ix->blocksize); + ies = initiestream(part, off, clumps, 64*1024); + if(b == nil || z == nil || ies == nil){ + ok = 0; + goto breakout; + return -1; + } + ok = 0; + next = 0; + memset(&ib, 0, sizeof ib); + ib.data = b->data + IBucketSize; + zib.data = z->data + IBucketSize; + zib.n = 0; + nbuck = 0; + for(;;){ + buck = buildbucket(ix, ies, &ib, ix->blocksize-IBucketSize); + found += ib.n; + if(zero){ + for(; next != buck; next++){ + if(next == ix->buckets){ + if(buck != TWID32){ + fprint(2, "bucket out of range\n"); + ok = -1; + } + goto breakout; + } + if(writebucket(ix, next, &zib, z) < 0){ + fprint(2, "can't write zero bucket to buck=%d: %r", next); + ok = -1; + } + } + } + if(buck >= ix->buckets){ + if(buck == TWID32) + break; + fprint(2, "bucket out of range\n"); + ok = -1; + goto breakout; + } + if(writebucket(ix, buck, &ib, b) < 0){ + fprint(2, "bad bucket found=%lld: %r\n", found); + ok = -1; + } + next = buck + 1; + if(++nbuck%10000 == 0) + fprint(2, "\t%,d buckets written...\n", nbuck); + } +breakout:; + fprint(2, "wrote index with %lld entries\n", found); + freeiestream(ies); + freezblock(z); + freezblock(b); + return ok; +} + +void +usage(void) +{ + fprint(2, "usage: buildindex [-Z] [-B blockcachesize] config tmppart\n"); + threadexitsall(0); +} + +Config conf; + +void +threadmain(int argc, char *argv[]) +{ + Part *part; + u64int clumps, base; + u32int bcmem; + int zero; + + zero = 1; + bcmem = 0; + ARGBEGIN{ + case 'B': + bcmem = unittoull(ARGF()); + break; + case 'Z': + zero = 0; + break; + default: + usage(); + break; + }ARGEND + + if(argc != 2) + usage(); + + if(initventi(argv[0], &conf) < 0) + sysfatal("can't init venti: %r"); + + if(bcmem < maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16)) + bcmem = maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16); + if(0) fprint(2, "initialize %d bytes of disk block cache\n", bcmem); + initdcache(bcmem); + + fprint(2, "building a new index %s using %s for temporary storage\n", mainindex->name, argv[1]); + + part = initpart(argv[1], ORDWR|ODIRECT); + if(part == nil) + sysfatal("can't initialize temporary partition: %r"); + + clumps = sortrawientries(mainindex, part, &base, mainindex->bloom); + if(clumps == TWID64) + sysfatal("can't build sorted index: %r"); + fprint(2, "found and sorted index entries for clumps=%lld at %lld\n", clumps, base); + + if(buildindex(mainindex, part, base, clumps, zero) < 0) + sysfatal("can't build new index: %r"); + + threadexitsall(0); +} blob - /dev/null blob + 525a634cb2a345fc26cbed8c62471b9cabe858c9 (mode 644) --- /dev/null +++ src/cmd/venti/srv/checkarenas.c @@ -0,0 +1,135 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +static int verbose; + +static void +checkarena(Arena *arena, int scan, int fix) +{ + ATailStats old; + int err, e; + + if(verbose && arena->memstats.clumps) + printarena(2, arena); + + old = arena->memstats; + + if(scan){ + arena->memstats.used = 0; + arena->memstats.clumps = 0; + arena->memstats.cclumps = 0; + arena->memstats.uncsize = 0; + } + + err = 0; + for(;;){ + e = syncarena(arena, 0, 1000, 0, fix); + err |= e; + if(!(e & SyncHeader)) + break; + if(verbose && arena->memstats.clumps) + fprint(2, "."); + } + if(verbose && arena->memstats.clumps) + fprint(2, "\n"); + + err &= ~SyncHeader; + if(arena->memstats.used != old.used + || arena->memstats.clumps != old.clumps + || arena->memstats.cclumps != old.cclumps + || arena->memstats.uncsize != old.uncsize){ + fprint(2, "%s: incorrect arena header fields\n", arena->name); + printarena(2, arena); + err |= SyncHeader; + } + + if(!err || !fix) + return; + + fprint(2, "%s: writing fixed arena header fields\n", arena->name); + arena->diskstats = arena->memstats; + if(wbarena(arena) < 0) + fprint(2, "arena header write failed: %r\n"); + flushdcache(); +} + +void +usage(void) +{ + fprint(2, "usage: checkarenas [-afv] file [arenaname...]\n"); + threadexitsall(0); +} + +int +should(char *name, int argc, char **argv) +{ + int i; + + if(argc == 0) + return 1; + for(i=0; i 1){ + printarenapart(2, ap); + fprint(2, "\n"); + } + + initdcache(8 * MaxDiskBlock); + + for(i = 0; i < ap->narenas; i++) + if(should(ap->arenas[i]->name, argc, argv)) + checkarena(ap->arenas[i], scan, fix); + + if(verbose > 1) + printstats(); + threadexitsall(0); +} blob - /dev/null blob + f7040d12ec5c7eba19a9fdd081c2557ff14cf519 (mode 644) --- /dev/null +++ src/cmd/venti/srv/checkindex.c @@ -0,0 +1,293 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +static int extra, missing, wrong; + +static void +phdr(DBlock *eb) +{ + static int did; + + if(!did){ + did = 1; + print("# diff actual correct\n"); + } + print("%s block 0x%llux\n", eb->part->name, eb->addr); +} + +static void +pie(IEntry *ie, char c) +{ + print("%c %V %22lld %3d %5d %3d\n", + c, ie->score, ie->ia.addr, ie->ia.type, ie->ia.size, ie->ia.blocks); +} + +static int +checkbucket(Index *ix, u32int buck, IBucket *ib) +{ + ISect *is; + DBlock *eb; + IBucket eib; + IEntry ie, eie; + int i, ei, ok, c, hdr; + + is = ix->sects[indexsect0(ix, buck)]; + if(buck < is->start || buck >= is->stop){ + seterr(EAdmin, "cannot find index section for bucket %lud\n", (ulong)buck); + return -1; + } + buck -= is->start; + eb = getdblock(is->part, is->blockbase + ((u64int)buck << is->blocklog), OREAD); + if(eb == nil) + return -1; + unpackibucket(&eib, eb->data, is->bucketmagic); + + ok = 0; + ei = 0; + hdr = 0; + for(i = 0; i < ib->n; i++){ + while(ei < eib.n){ + c = ientrycmp(&ib->data[i * IEntrySize], &eib.data[ei * IEntrySize]); + if(c == 0){ + unpackientry(&ie, &ib->data[i * IEntrySize]); + unpackientry(&eie, &eib.data[ei * IEntrySize]); + if(iaddrcmp(&ie.ia, &eie.ia) != 0){ + if(!hdr){ + phdr(eb); + hdr = 1; + } + wrong++; + pie(&eie, '<'); + pie(&ie, '>'); + } + ei++; + goto cont; + } + if(c < 0) + break; + if(!hdr){ + phdr(eb); + hdr = 1; + } + unpackientry(&eie, &eib.data[ei*IEntrySize]); + extra++; + pie(&eie, '<'); + ei++; + ok = -1; + } + if(!hdr){ + phdr(eb); + hdr = 1; + } + unpackientry(&ie, &ib->data[i*IEntrySize]); + missing++; + pie(&ie, '>'); + ok = -1; + cont:; + } + for(; ei < eib.n; ei++){ + if(!hdr){ + phdr(eb); + hdr = 1; + } + unpackientry(&eie, &eib.data[ei*IEntrySize]); + pie(&eie, '<'); + ok = -1; + } + putdblock(eb); + return ok; +} + +int +checkindex(Index *ix, Part *part, u64int off, u64int clumps, int zero) +{ + IEStream *ies; + IBucket ib, zib; + ZBlock *z, *b; + u32int next, buck; + int ok, bok; +u64int found = 0; + +//ZZZ make buffer size configurable + b = alloczblock(ix->blocksize, 0, ix->blocksize); + z = alloczblock(ix->blocksize, 1, ix->blocksize); + ies = initiestream(part, off, clumps, 64*1024); + if(b == nil || z == nil || ies == nil){ + werrstr("allocating: %r"); + ok = -1; + goto breakout; + return -1; + } + ok = 0; + next = 0; + memset(&ib, 0, sizeof ib); + ib.data = b->data; + zib.data = z->data; + zib.n = 0; + zib.buck = 0; + for(;;){ + buck = buildbucket(ix, ies, &ib, ix->blocksize-IBucketSize); + found += ib.n; + if(zero){ + for(; next != buck; next++){ + if(next == ix->buckets){ + if(buck != TWID32){ + ok = -1; + werrstr("internal error: bucket out of range"); + } + if(ok < 0) + werrstr("%d spurious entries, %d missing, %d wrong", extra, missing, wrong); + goto breakout; + } + bok = checkbucket(ix, next, &zib); + if(bok < 0) + ok = -1; + } + } + if(buck >= ix->buckets){ + if(buck == TWID32) + break; + werrstr("internal error: bucket out of range"); + ok = -1; + goto breakout; + } + bok = checkbucket(ix, buck, &ib); + if(bok < 0) + ok = -1; + next = buck + 1; + } +breakout: + freeiestream(ies); + freezblock(z); + freezblock(b); + return ok; +} + +int +checkbloom(Bloom *b1, Bloom *b2, int fix) +{ + u32int *a1, *a2; + int i, n, extra, missing; + + if(b1==nil && b2==nil) + return 0; + if(b1==nil || b2==nil){ + werrstr("nil/non-nil"); + return -1; + } + wbbloomhead(b1); + wbbloomhead(b2); + if(memcmp(b1->data, b2->data, BloomHeadSize) != 0){ + werrstr("bloom header mismatch"); + return -1; + } + a1 = (u32int*)b1->data; + a2 = (u32int*)b2->data; + n = b1->size/4; + extra = 0; + missing = 0; + for(i=BloomHeadSize/4; idata, b2->data, b1->size); + return writebloom(b1); + } + return 0; +} + + +void +usage(void) +{ + fprint(2, "usage: checkindex [-f] [-B blockcachesize] config tmp\n"); + threadexitsall(0); +} + +Config conf; + +void +threadmain(int argc, char *argv[]) +{ + Bloom *oldbloom, *newbloom; + Part *part; + u64int clumps, base; + u32int bcmem; + int fix, skipz, ok; + + fix = 0; + bcmem = 0; + skipz = 0; + ARGBEGIN{ + case 'B': + bcmem = unittoull(ARGF()); + break; + case 'f': + fix++; + break; + case 'Z': + skipz = 1; + break; + default: + usage(); + break; + }ARGEND + + if(argc != 2) + usage(); + + ventifmtinstall(); + + part = initpart(argv[1], ORDWR|ODIRECT); + if(part == nil) + sysfatal("can't initialize temporary partition: %r"); + + if(!fix) + readonly = 1; + + if(initventi(argv[0], &conf) < 0) + sysfatal("can't init venti: %r"); + oldbloom = mainindex->bloom; + newbloom = nil; + if(oldbloom){ + newbloom = vtmallocz(sizeof *newbloom); + bloominit(newbloom, oldbloom->size, nil); + newbloom->data = vtmallocz(oldbloom->size); + } + if(bcmem < maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16)) + bcmem = maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16); + if(0) fprint(2, "initialize %d bytes of disk block cache\n", bcmem); + initdcache(bcmem); + + fprint(2, "checkindex: building entry list\n"); + clumps = sortrawientries(mainindex, part, &base, newbloom); + if(clumps == TWID64) + sysfatal("can't build sorted index: %r"); + fprint(2, "checkindex: checking %lld entries at %lld\n", clumps, base); + ok = 0; + if(checkindex(mainindex, part, base, clumps, !skipz) < 0){ + fprint(2, "checkindex: %r\n"); + ok = -1; + } + if(checkbloom(oldbloom, newbloom, fix) < 0){ + fprint(2, "checkbloom: %r\n"); + ok = -1; + } + if(ok < 0) + sysfatal("errors found"); + fprint(2, "checkindex: index is correct\n"); + threadexitsall(0); +} blob - /dev/null blob + 88ebdb50ab0740ddce38bdcd21a239fdd86ed0b7 (mode 644) --- /dev/null +++ src/cmd/venti/srv/clump.c @@ -0,0 +1,222 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" +#include "whack.h" + +/* + * Write a lump to disk. Updates ia with an index address + * for the newly-written lump. Upon return, the lump will + * have been placed in the disk cache but will likely not be on disk yet. + */ +int +storeclump(Index *ix, ZBlock *zb, u8int *sc, int type, u32int creator, IAddr *ia) +{ + ZBlock *cb; + Clump cl; + u64int a; + u8int bh[VtScoreSize]; + int size, dsize; + + trace(TraceLump, "storeclump enter", sc, type); + size = zb->len; + if(size > VtMaxLumpSize){ + seterr(EStrange, "lump too large"); + return -1; + } + if(vttypevalid(type) < 0){ + seterr(EStrange, "invalid lump type"); + return -1; + } + + if(0){ + scoremem(bh, zb->data, size); + if(scorecmp(sc, bh) != 0){ + seterr(ECorrupt, "storing clump: corrupted; expected=%V got=%V, size=%d", sc, bh, size); + return -1; + } + } + + cb = alloczblock(size + ClumpSize + U32Size, 0, 0); + if(cb == nil) + return -1; + + cl.info.type = type; + cl.info.uncsize = size; + cl.creator = creator; + cl.time = now(); + scorecp(cl.info.score, sc); + + trace(TraceLump, "storeclump whackblock"); + dsize = whackblock(&cb->data[ClumpSize], zb->data, size); + if(dsize > 0 && dsize < size){ + cl.encoding = ClumpECompress; + }else{ + if(dsize > size){ + fprint(2, "whack error: dsize=%d size=%d\n", dsize, size); + abort(); + } + cl.encoding = ClumpENone; + dsize = size; + memmove(&cb->data[ClumpSize], zb->data, size); + } + memset(cb->data+ClumpSize+dsize, 0, 4); + cl.info.size = dsize; + + ia->addr = 0; + ia->type = type; + ia->size = size; + ia->blocks = (dsize + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog; + + a = writeiclump(ix, &cl, cb->data, &ia->addr); + + trace(TraceLump, "storeclump exit %lld", a); + + freezblock(cb); + if(a == TWID64) + return -1; + +/* + qlock(&stats.lock); + stats.clumpwrites++; + stats.clumpbwrites += size; + stats.clumpbcomp += dsize; + qunlock(&stats.lock); +*/ + + return 0; +} + +u32int +clumpmagic(Arena *arena, u64int aa) +{ + u8int buf[U32Size]; + + if(readarena(arena, aa, buf, U32Size) < 0) + return TWID32; + return unpackmagic(buf); +} + +/* + * fetch a block based at addr. + * score is filled in with the block's score. + * blocks is roughly the length of the clump on disk; + * if zero, the length is unknown. + */ +ZBlock* +loadclump(Arena *arena, u64int aa, int blocks, Clump *cl, u8int *score, int verify) +{ + Unwhack uw; + ZBlock *zb, *cb; + u8int bh[VtScoreSize], *buf; + u32int n; + int nunc; + +/* + qlock(&stats.lock); + stats.clumpreads++; + qunlock(&stats.lock); +*/ + + if(blocks <= 0) + blocks = 1; + + trace(TraceLump, "loadclump enter"); + + cb = alloczblock(blocks << ABlockLog, 0, 0); + if(cb == nil) + return nil; + n = readarena(arena, aa, cb->data, blocks << ABlockLog); + if(n < ClumpSize){ + if(n != 0) + seterr(ECorrupt, "loadclump read less than a header"); + freezblock(cb); + return nil; + } + trace(TraceLump, "loadclump unpack"); + if(unpackclump(cl, cb->data, arena->clumpmagic) < 0){ + seterr(ECorrupt, "loadclump %s %llud: %r", arena->name, aa); + freezblock(cb); + return nil; + } + n -= ClumpSize; + if(n < cl->info.size){ + freezblock(cb); + n = cl->info.size; + cb = alloczblock(n, 0, 0); + if(cb == nil) + return nil; + if(readarena(arena, aa + ClumpSize, cb->data, n) != n){ + seterr(ECorrupt, "loadclump read too little data"); + freezblock(cb); + return nil; + } + buf = cb->data; + }else + buf = cb->data + ClumpSize; + + scorecp(score, cl->info.score); + + zb = alloczblock(cl->info.uncsize, 0, 0); + if(zb == nil){ + freezblock(cb); + return nil; + } + switch(cl->encoding){ + case ClumpECompress: + trace(TraceLump, "loadclump decompress"); + unwhackinit(&uw); + nunc = unwhack(&uw, zb->data, cl->info.uncsize, buf, cl->info.size); + if(nunc != cl->info.uncsize){ + if(nunc < 0) + seterr(ECorrupt, "decompression of %llud failed: %s", aa, uw.err); + else + seterr(ECorrupt, "decompression of %llud gave partial block: %d/%d\n", aa, nunc, cl->info.uncsize); + freezblock(cb); + freezblock(zb); + return nil; + } + break; + case ClumpENone: + if(cl->info.size != cl->info.uncsize){ + seterr(ECorrupt, "loading clump: bad uncompressed size for uncompressed block %llud", aa); + freezblock(cb); + freezblock(zb); + return nil; + } + scoremem(bh, buf, cl->info.uncsize); + if(scorecmp(cl->info.score, bh) != 0) + seterr(ECorrupt, "pre-copy sha1 wrong at %s %llud: expected=%V got=%V", arena->name, aa, cl->info.score, bh); + memmove(zb->data, buf, cl->info.uncsize); + break; + default: + seterr(ECorrupt, "unknown encoding in loadlump %llud", aa); + freezblock(cb); + freezblock(zb); + return nil; + } + freezblock(cb); + + if(verify){ + trace(TraceLump, "loadclump verify"); + scoremem(bh, zb->data, cl->info.uncsize); + if(scorecmp(cl->info.score, bh) != 0){ + seterr(ECorrupt, "loading clump: corrupted at %s %llud; expected=%V got=%V", arena->name, aa, cl->info.score, bh); + freezblock(zb); + return nil; + } + if(vttypevalid(cl->info.type) < 0){ + seterr(ECorrupt, "loading lump at %s %llud: invalid lump type %d", arena->name, aa, cl->info.type); + freezblock(zb); + return nil; + } + } + + trace(TraceLump, "loadclump exit"); +/* + qlock(&stats.lock); + stats.clumpbreads += cl->info.size; + stats.clumpbuncomp += cl->info.uncsize; + qunlock(&stats.lock); +*/ + return zb; +} blob - /dev/null blob + d2cfe251c3d6697bf886d734761bd964d92e23cd (mode 644) --- /dev/null +++ src/cmd/venti/srv/clumpstats.c @@ -0,0 +1,127 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +int count[VtMaxLumpSize][VtMaxType]; +Config conf; + +enum +{ + ClumpChunks = 32*1024 +}; + +static int +readarenainfo(Arena *arena) +{ + ClumpInfo *ci, *cis; + u32int clump; + int i, n, ok; + + if(arena->memstats.clumps) + fprint(2, "reading directory for arena=%s with %d entries\n", arena->name, arena->memstats.clumps); + + cis = MKN(ClumpInfo, ClumpChunks); + ok = 0; + for(clump = 0; clump < arena->memstats.clumps; clump += n){ + n = ClumpChunks; + + if(n > arena->memstats.clumps - clump) + n = arena->memstats.clumps - clump; + + if((i=readclumpinfos(arena, clump, cis, n)) != n){ + seterr(EOk, "arena directory read failed %d not %d: %r", i, n); + ok = -1; + break; + } + + for(i = 0; i < n; i++){ + ci = &cis[i]; + if(ci->type >= VtMaxType || ci->uncsize >= VtMaxLumpSize) { + fprint(2, "bad clump: %d: type = %d: size = %d\n", clump+i, ci->type, ci->uncsize); + continue; + } + count[ci->uncsize][ci->type]++; + } + } + free(cis); + if(ok < 0) + return TWID32; + return clump; +} + +static void +clumpstats(Index *ix) +{ + int ok; + ulong clumps, n; + int i, j, t; + + ok = 0; + clumps = 0; + for(i = 0; i < ix->narenas; i++){ + n = readarenainfo(ix->arenas[i]); + if(n == TWID32){ + ok = -1; + break; + } + clumps += n; + } + + if(ok < 0) + return; + + print("clumps = %ld\n", clumps); + for(i=0; inarenas + mainindex->nsects * 4 + 16)) + bcmem = maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16); + if(0) fprint(2, "initialize %d bytes of disk block cache\n", bcmem); + initdcache(bcmem); + + clumpstats(mainindex); + + threadexitsall(0); +} blob - /dev/null blob + e6232d53407a4f084648e7630b96ca2acd090784 (mode 644) --- /dev/null +++ src/cmd/venti/srv/config.c @@ -0,0 +1,245 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +Index *mainindex; +int paranoid = 1; /* should verify hashes on disk read */ + +static ArenaPart *configarenas(char *file); +static ISect *configisect(char *file); +static Bloom *configbloom(char *file); + +int +initventi(char *file, Config *conf) +{ + statsinit(); + + if(file == nil){ + seterr(EOk, "no configuration file"); + return -1; + } + if(runconfig(file, conf) < 0){ + seterr(EOk, "can't initialize venti: %r"); + return -1; + } + mainindex = initindex(conf->index, conf->sects, conf->nsects); + if(mainindex == nil) + return -1; + mainindex->bloom = conf->bloom; + return 0; +} + +static int +numok(char *s) +{ + char *p; + + strtoull(s, &p, 0); + if(p == s) + return -1; + if(*p == 0) + return 0; + if(p[1] == 0 && strchr("MmGgKk", *p)) + return 0; + return 0; +} + +/* + * configs : + * | configs config + * config : "isect" filename + * | "arenas" filename + * | "index" name + * | "bcmem" num + * | "mem" num + * | "icmem" num + * | "queuewrites" + * | "httpaddr" address + * | "addr" address + * + * '#' and \n delimit comments + */ +enum +{ + MaxArgs = 2 +}; +int +runconfig(char *file, Config *config) +{ + ArenaPart **av; + ISect **sv; + IFile f; + char *s, *line, *flds[MaxArgs + 1]; + int i, ok; + + if(readifile(&f, file) < 0) + return -1; + memset(config, 0, sizeof *config); + config->mem = 0xFFFFFFFFUL; + ok = -1; + line = nil; + for(;;){ + s = ifileline(&f); + if(s == nil){ + ok = 0; + break; + } + line = estrdup(s); + i = getfields(s, flds, MaxArgs + 1, 1, " \t\r"); + if(i == 2 && strcmp(flds[0], "isect") == 0){ + sv = MKN(ISect*, config->nsects + 1); + for(i = 0; i < config->nsects; i++) + sv[i] = config->sects[i]; + free(config->sects); + config->sects = sv; + config->sects[config->nsects] = configisect(flds[1]); + if(config->sects[config->nsects] == nil) + break; + config->nsects++; + }else if(i == 2 && strcmp(flds[0], "arenas") == 0){ + av = MKN(ArenaPart*, config->naparts + 1); + for(i = 0; i < config->naparts; i++) + av[i] = config->aparts[i]; + free(config->aparts); + config->aparts = av; + config->aparts[config->naparts] = configarenas(flds[1]); + if(config->aparts[config->naparts] == nil) + break; + config->naparts++; + }else if(i == 2 && strcmp(flds[0], "bloom") == 0){ + if(config->bloom){ + seterr(EAdmin, "duplicate bloom lines in configuration file %s", file); + break; + } + if((config->bloom = configbloom(flds[1])) == nil) + break; + }else if(i == 2 && strcmp(flds[0], "index") == 0){ + if(nameok(flds[1]) < 0){ + seterr(EAdmin, "illegal index name %s in config file %s", flds[1], file); + break; + } + if(config->index != nil){ + seterr(EAdmin, "duplicate indices in config file %s", file); + break; + } + config->index = estrdup(flds[1]); + }else if(i == 2 && strcmp(flds[0], "bcmem") == 0){ + if(numok(flds[1]) < 0){ + seterr(EAdmin, "illegal size %s in config file %s", + flds[1], file); + break; + } + if(config->bcmem != 0){ + seterr(EAdmin, "duplicate bcmem lines in config file %s", file); + break; + } + config->bcmem = unittoull(flds[1]); + }else if(i == 2 && strcmp(flds[0], "mem") == 0){ + if(numok(flds[1]) < 0){ + seterr(EAdmin, "illegal size %s in config file %s", + flds[1], file); + break; + } + if(config->mem != 0xFFFFFFFFUL){ + seterr(EAdmin, "duplicate mem lines in config file %s", file); + break; + } + config->mem = unittoull(flds[1]); + }else if(i == 2 && strcmp(flds[0], "icmem") == 0){ + if(numok(flds[1]) < 0){ + seterr(EAdmin, "illegal size %s in config file %s", + flds[1], file); + break; + } + if(config->icmem != 0){ + seterr(EAdmin, "duplicate icmem lines in config file %s", file); + break; + } + config->icmem = unittoull(flds[1]); + }else if(i == 1 && strcmp(flds[0], "queuewrites") == 0){ + config->queuewrites = 1; + }else if(i == 2 && strcmp(flds[0], "httpaddr") == 0){ + if(config->haddr){ + seterr(EAdmin, "duplicate httpaddr lines in configuration file %s", file); + break; + } + config->haddr = estrdup(flds[1]); + }else if(i == 2 && strcmp(flds[0], "webroot") == 0){ + if(config->webroot){ + seterr(EAdmin, "duplicate webroot lines in configuration file %s", file); + break; + } + config->webroot = estrdup(flds[1]); + }else if(i == 2 && strcmp(flds[0], "addr") == 0){ + if(config->vaddr){ + seterr(EAdmin, "duplicate addr lines in configuration file %s", file); + break; + } + config->vaddr = estrdup(flds[1]); + }else{ + seterr(EAdmin, "illegal line '%s' in configuration file %s", line, file); + break; + } + free(line); + line = nil; + } + free(line); + freeifile(&f); + if(ok < 0){ + free(config->sects); + config->sects = nil; + free(config->aparts); + config->aparts = nil; + } + return ok; +} + +static ISect* +configisect(char *file) +{ + Part *part; + ISect *is; + + if(0) fprint(2, "configure index section in %s\n", file); + + part = initpart(file, ORDWR|ODIRECT); + if(part == nil) + return nil; + is = initisect(part); + if(is == nil) + werrstr("%s: %r", file); + return is; +} + +static ArenaPart* +configarenas(char *file) +{ + ArenaPart *ap; + Part *part; + + if(0) fprint(2, "configure arenas in %s\n", file); + part = initpart(file, ORDWR|ODIRECT); + if(part == nil) + return nil; + ap = initarenapart(part); + if(ap == nil) + werrstr("%s: %r", file); + return ap; +} + +static Bloom* +configbloom(char *file) +{ + Bloom *b; + Part *part; + + if(0) fprint(2, "configure bloom in %s\n", file); + part = initpart(file, ORDWR|ODIRECT); + if(part == nil) + return nil; + b = readbloom(part); + if(b == nil) + werrstr("%s: %r", file); + return b; +} + blob - /dev/null blob + 13afc7d24d8389698397b8ffff911acfd60ba715 (mode 644) --- /dev/null +++ src/cmd/venti/srv/conv.c @@ -0,0 +1,632 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +/* + * disk structure conversion routines + */ +#define U8GET(p) ((p)[0]) +#define U16GET(p) (((p)[0]<<8)|(p)[1]) +#define U32GET(p) ((u32int)(((p)[0]<<24)|((p)[1]<<16)|((p)[2]<<8)|(p)[3])) +#define U64GET(p) (((u64int)U32GET(p)<<32)|(u64int)U32GET((p)+4)) + +#define U8PUT(p,v) (p)[0]=(v)&0xFF +#define U16PUT(p,v) (p)[0]=((v)>>8)&0xFF;(p)[1]=(v)&0xFF +#define U32PUT(p,v) (p)[0]=((v)>>24)&0xFF;(p)[1]=((v)>>16)&0xFF;(p)[2]=((v)>>8)&0xFF;(p)[3]=(v)&0xFF +#define U64PUT(p,v,t32) t32=(v)>>32;U32PUT(p,t32);t32=(v);U32PUT((p)+4,t32) + +static struct { + u32int m; + char *s; +} magics[] = { + ArenaPartMagic, "ArenaPartMagic", + ArenaHeadMagic, "ArenaHeadMagic", + ArenaMagic, "ArenaMagic", + ISectMagic, "ISectMagic", + BloomMagic, "BloomMagic", +}; + +static char* +fmtmagic(char *s, u32int m) +{ + int i; + + for(i=0; iversion = U32GET(p); + p += U32Size; + ap->blocksize = U32GET(p); + p += U32Size; + ap->arenabase = U32GET(p); + p += U32Size; + + if(buf + ArenaPartSize != p) + sysfatal("unpackarenapart unpacked wrong amount"); + + return 0; +} + +int +packarenapart(ArenaPart *ap, u8int *buf) +{ + u8int *p; + + p = buf; + + U32PUT(p, ArenaPartMagic); + p += U32Size; + U32PUT(p, ap->version); + p += U32Size; + U32PUT(p, ap->blocksize); + p += U32Size; + U32PUT(p, ap->arenabase); + p += U32Size; + + if(buf + ArenaPartSize != p) + sysfatal("packarenapart packed wrong amount"); + + return 0; +} + +int +unpackarena(Arena *arena, u8int *buf) +{ + int sz; + u8int *p; + u32int m; + char fbuf[20]; + + p = buf; + + m = U32GET(p); + if(m != ArenaMagic){ + seterr(ECorrupt, "arena has wrong magic number: %s expected ArenaMagic (%lux)", fmtmagic(fbuf, m), ArenaMagic); + return -1; + } + p += U32Size; + arena->version = U32GET(p); + p += U32Size; + namecp(arena->name, (char*)p); + p += ANameSize; + arena->diskstats.clumps = U32GET(p); + p += U32Size; + arena->diskstats.cclumps = U32GET(p); + p += U32Size; + arena->ctime = U32GET(p); + p += U32Size; + arena->wtime = U32GET(p); + p += U32Size; + if(arena->version == ArenaVersion5){ + arena->clumpmagic = U32GET(p); + p += U32Size; + } + arena->diskstats.used = U64GET(p); + p += U64Size; + arena->diskstats.uncsize = U64GET(p); + p += U64Size; + arena->diskstats.sealed = U8GET(p); + p += U8Size; + + arena->memstats = arena->diskstats; + + switch(arena->version){ + case ArenaVersion4: + sz = ArenaSize4; + arena->clumpmagic = _ClumpMagic; + break; + case ArenaVersion5: + sz = ArenaSize5; + break; + default: + seterr(ECorrupt, "arena has bad version number %d", arena->version); + return -1; + } + if(buf + sz != p) + sysfatal("unpackarena unpacked wrong amount"); + + return 0; +} + +int +packarena(Arena *arena, u8int *buf) +{ + int sz; + u8int *p; + u32int t32; + + switch(arena->version){ + case ArenaVersion4: + sz = ArenaSize4; + if(arena->clumpmagic != _ClumpMagic) + fprint(2, "warning: writing old arena tail loses clump magic 0x%lux != 0x%lux\n", + (ulong)arena->clumpmagic, (ulong)_ClumpMagic); + break; + case ArenaVersion5: + sz = ArenaSize5; + break; + default: + sysfatal("packarena unknown version %d", arena->version); + return -1; + } + + p = buf; + + U32PUT(p, ArenaMagic); + p += U32Size; + U32PUT(p, arena->version); + p += U32Size; + namecp((char*)p, arena->name); + p += ANameSize; + U32PUT(p, arena->diskstats.clumps); + p += U32Size; + U32PUT(p, arena->diskstats.cclumps); + p += U32Size; + U32PUT(p, arena->ctime); + p += U32Size; + U32PUT(p, arena->wtime); + p += U32Size; + if(arena->version == ArenaVersion5){ + U32PUT(p, arena->clumpmagic); + p += U32Size; + } + U64PUT(p, arena->diskstats.used, t32); + p += U64Size; + U64PUT(p, arena->diskstats.uncsize, t32); + p += U64Size; + U8PUT(p, arena->diskstats.sealed); + p += U8Size; + + if(buf + sz != p) + sysfatal("packarena packed wrong amount"); + + return 0; +} + +int +unpackarenahead(ArenaHead *head, u8int *buf) +{ + u8int *p; + u32int m; + int sz; + + p = buf; + + m = U32GET(p); + /* XXX check magic! */ + + p += U32Size; + head->version = U32GET(p); + p += U32Size; + namecp(head->name, (char*)p); + p += ANameSize; + head->blocksize = U32GET(p); + p += U32Size; + head->size = U64GET(p); + p += U64Size; + if(head->version == ArenaVersion5){ + head->clumpmagic = U32GET(p); + p += U32Size; + } + + switch(head->version){ + case ArenaVersion4: + sz = ArenaHeadSize4; + head->clumpmagic = _ClumpMagic; + break; + case ArenaVersion5: + sz = ArenaHeadSize5; + break; + default: + seterr(ECorrupt, "arena head has unexpected version %d", head->version); + return -1; + } + + if(buf + sz != p) + sysfatal("unpackarenahead unpacked wrong amount"); + + return 0; +} + +int +packarenahead(ArenaHead *head, u8int *buf) +{ + u8int *p; + int sz; + u32int t32; + + switch(head->version){ + case ArenaVersion4: + sz = ArenaHeadSize4; + if(head->clumpmagic != _ClumpMagic) + fprint(2, "warning: writing old arena header loses clump magic 0x%lux != 0x%lux\n", + (ulong)head->clumpmagic, (ulong)_ClumpMagic); + break; + case ArenaVersion5: + sz = ArenaHeadSize5; + break; + default: + sysfatal("packarenahead unknown version %d", head->version); + return -1; + } + + p = buf; + + U32PUT(p, ArenaHeadMagic); + p += U32Size; + U32PUT(p, head->version); + p += U32Size; + namecp((char*)p, head->name); + p += ANameSize; + U32PUT(p, head->blocksize); + p += U32Size; + U64PUT(p, head->size, t32); + p += U64Size; + if(head->version == ArenaVersion5){ + U32PUT(p, head->clumpmagic); + p += U32Size; + } + if(buf + sz != p) + sysfatal("packarenahead packed wrong amount"); + + return 0; +} + +static int +checkclump(Clump *w) +{ + if(w->encoding == ClumpENone){ + if(w->info.size != w->info.uncsize){ + seterr(ECorrupt, "uncompressed wad size mismatch"); + return -1; + } + }else if(w->encoding == ClumpECompress){ + if(w->info.size >= w->info.uncsize){ + seterr(ECorrupt, "compressed lump has inconsistent block sizes %d %d", w->info.size, w->info.uncsize); + return -1; + } + }else{ + seterr(ECorrupt, "clump has illegal encoding"); + return -1; + } + + return 0; +} + +int +unpackclump(Clump *c, u8int *buf, u32int cmagic) +{ + u8int *p; + u32int magic; + + p = buf; + magic = U32GET(p); + if(magic != cmagic){ + seterr(ECorrupt, "clump has bad magic number=%#8.8ux != %#8.8ux", magic, cmagic); + return -1; + } + p += U32Size; + + c->info.type = vtfromdisktype(U8GET(p)); + p += U8Size; + c->info.size = U16GET(p); + p += U16Size; + c->info.uncsize = U16GET(p); + p += U16Size; + scorecp(c->info.score, p); + p += VtScoreSize; + + c->encoding = U8GET(p); + p += U8Size; + c->creator = U32GET(p); + p += U32Size; + c->time = U32GET(p); + p += U32Size; + + if(buf + ClumpSize != p) + sysfatal("unpackclump unpacked wrong amount"); + + return checkclump(c); +} + +int +packclump(Clump *c, u8int *buf, u32int magic) +{ + u8int *p; + + p = buf; + U32PUT(p, magic); + p += U32Size; + + U8PUT(p, vttodisktype(c->info.type)); + p += U8Size; + U16PUT(p, c->info.size); + p += U16Size; + U16PUT(p, c->info.uncsize); + p += U16Size; + scorecp(p, c->info.score); + p += VtScoreSize; + + U8PUT(p, c->encoding); + p += U8Size; + U32PUT(p, c->creator); + p += U32Size; + U32PUT(p, c->time); + p += U32Size; + + if(buf + ClumpSize != p) + sysfatal("packclump packed wrong amount"); + + return checkclump(c); +} + +void +unpackclumpinfo(ClumpInfo *ci, u8int *buf) +{ + u8int *p; + + p = buf; + ci->type = vtfromdisktype(U8GET(p)); + p += U8Size; + ci->size = U16GET(p); + p += U16Size; + ci->uncsize = U16GET(p); + p += U16Size; + scorecp(ci->score, p); + p += VtScoreSize; + + if(buf + ClumpInfoSize != p) + sysfatal("unpackclumpinfo unpacked wrong amount"); +} + +void +packclumpinfo(ClumpInfo *ci, u8int *buf) +{ + u8int *p; + + p = buf; + U8PUT(p, vttodisktype(ci->type)); + p += U8Size; + U16PUT(p, ci->size); + p += U16Size; + U16PUT(p, ci->uncsize); + p += U16Size; + scorecp(p, ci->score); + p += VtScoreSize; + + if(buf + ClumpInfoSize != p) + sysfatal("packclumpinfo packed wrong amount"); +} + +int +unpackisect(ISect *is, u8int *buf) +{ + u8int *p; + u32int m; + char fbuf[20]; + + p = buf; + + + m = U32GET(p); + if(m != ISectMagic){ + seterr(ECorrupt, "index section has wrong magic number: %s expected ISectMagic (%lux)", + fmtmagic(fbuf, m), ISectMagic); + return -1; + } + p += U32Size; + is->version = U32GET(p); + p += U32Size; + namecp(is->name, (char*)p); + p += ANameSize; + namecp(is->index, (char*)p); + p += ANameSize; + is->blocksize = U32GET(p); + p += U32Size; + is->blockbase = U32GET(p); + p += U32Size; + is->blocks = U32GET(p); + p += U32Size; + is->start = U32GET(p); + p += U32Size; + is->stop = U32GET(p); + p += U32Size; + if(buf + ISectSize1 != p) + sysfatal("unpackisect unpacked wrong amount"); + is->bucketmagic = 0; + if(is->version == ISectVersion2){ + is->bucketmagic = U32GET(p); + p += U32Size; + if(buf + ISectSize2 != p) + sysfatal("unpackisect unpacked wrong amount"); + } + + return 0; +} + +int +packisect(ISect *is, u8int *buf) +{ + u8int *p; + + p = buf; + + U32PUT(p, ISectMagic); + p += U32Size; + U32PUT(p, is->version); + p += U32Size; + namecp((char*)p, is->name); + p += ANameSize; + namecp((char*)p, is->index); + p += ANameSize; + U32PUT(p, is->blocksize); + p += U32Size; + U32PUT(p, is->blockbase); + p += U32Size; + U32PUT(p, is->blocks); + p += U32Size; + U32PUT(p, is->start); + p += U32Size; + U32PUT(p, is->stop); + p += U32Size; + if(buf + ISectSize1 != p) + sysfatal("packisect packed wrong amount"); + if(is->version == ISectVersion2){ + U32PUT(p, is->bucketmagic); + p += U32Size; + if(buf + ISectSize2 != p) + sysfatal("packisect packed wrong amount"); + } + + return 0; +} + +void +unpackientry(IEntry *ie, u8int *buf) +{ + u8int *p; + + p = buf; + + scorecp(ie->score, p); + p += VtScoreSize; + ie->wtime = U32GET(p); + p += U32Size; + ie->train = U16GET(p); + p += U16Size; + ie->ia.addr = U64GET(p); +if(ie->ia.addr>>56) print("%.8H => %llux\n", p, ie->ia.addr); + p += U64Size; + ie->ia.size = U16GET(p); + p += U16Size; + if(p - buf != IEntryTypeOff) + sysfatal("unpackientry bad IEntryTypeOff amount"); + ie->ia.type = vtfromdisktype(U8GET(p)); + p += U8Size; + ie->ia.blocks = U8GET(p); + p += U8Size; + + if(p - buf != IEntrySize) + sysfatal("unpackientry unpacked wrong amount"); +} + +void +packientry(IEntry *ie, u8int *buf) +{ + u32int t32; + u8int *p; + + p = buf; + + scorecp(p, ie->score); + p += VtScoreSize; + U32PUT(p, ie->wtime); + p += U32Size; + U16PUT(p, ie->train); + p += U16Size; + U64PUT(p, ie->ia.addr, t32); + p += U64Size; + U16PUT(p, ie->ia.size); + p += U16Size; + U8PUT(p, vttodisktype(ie->ia.type)); + p += U8Size; + U8PUT(p, ie->ia.blocks); + p += U8Size; + + if(p - buf != IEntrySize) + sysfatal("packientry packed wrong amount"); +} + +void +unpackibucket(IBucket *b, u8int *buf, u32int magic) +{ + b->n = U16GET(buf); + b->data = buf + IBucketSize; + if(magic && magic != U32GET(buf+U16Size)) + b->n = 0; +} + +void +packibucket(IBucket *b, u8int *buf, u32int magic) +{ + U16PUT(buf, b->n); + U32PUT(buf+U16Size, magic); +} + +void +packbloomhead(Bloom *b, u8int *buf) +{ + u8int *p; + + p = buf; + U32PUT(p, BloomMagic); + U32PUT(p+4, BloomVersion); + U32PUT(p+8, b->nhash); + U32PUT(p+12, b->size); +} + +int +unpackbloomhead(Bloom *b, u8int *buf) +{ + u8int *p; + u32int m; + char fbuf[20]; + + p = buf; + + m = U32GET(p); + if(m != BloomMagic){ + seterr(ECorrupt, "bloom filter has wrong magic number: %s expected BloomMagic (%lux)", fmtmagic(fbuf, m), (ulong)BloomMagic); + return -1; + } + p += U32Size; + + m = U32GET(p); + if(m != BloomVersion){ + seterr(ECorrupt, "bloom filter has wrong version %ud expected %ud", (uint)m, (uint)BloomVersion); + return -1; + } + p += U32Size; + + b->nhash = U32GET(p); + p += U32Size; + + b->size = U32GET(p); + p += U32Size; + + if(buf + BloomHeadSize != p) + sysfatal("unpackarena unpacked wrong amount"); + + return 0; +} blob - /dev/null blob + 5f6d1a3f7c4b212b3ee9fd2a7adb153ef661c0c5 (mode 644) --- /dev/null +++ src/cmd/venti/srv/dat.h @@ -0,0 +1,718 @@ +typedef struct Config Config; +typedef struct AMap AMap; +typedef struct AMapN AMapN; +typedef struct Arena Arena; +typedef struct AState AState; +typedef struct ArenaHead ArenaHead; +typedef struct ArenaPart ArenaPart; +typedef struct ArenaTail ArenaTail; +typedef struct ATailStats ATailStats; +typedef struct CIBlock CIBlock; +typedef struct Clump Clump; +typedef struct ClumpInfo ClumpInfo; +typedef struct Graph Graph; +typedef struct IAddr IAddr; +typedef struct IBucket IBucket; +typedef struct IEStream IEStream; +typedef struct IEntry IEntry; +typedef struct IFile IFile; +typedef struct ISect ISect; +typedef struct Index Index; +typedef struct Lump Lump; +typedef struct DBlock DBlock; +typedef struct Part Part; +typedef struct Statbin Statbin; +typedef struct Statdesc Statdesc; +typedef struct Stats Stats; +typedef struct ZBlock ZBlock; +typedef struct Round Round; +typedef struct Bloom Bloom; + +#define TWID32 ((u32int)~(u32int)0) +#define TWID64 ((u64int)~(u64int)0) +#define TWID8 ((u8int)~(u8int)0) + +enum +{ + ABlockLog = 9, /* log2(512), the quantum for reading arenas */ + ANameSize = 64, + MaxDiskBlock = 64*1024, /* max. allowed size for a disk block */ + MaxIoSize = 64*1024, /* max. allowed size for a disk io operation */ + PartBlank = 256*1024, /* untouched section at beginning of partition */ + HeadSize = 512, /* size of a header after PartBlank */ + MinArenaSize = 1*1024*1024, /* smallest reasonable arena size */ + IndexBase = 1024*1024, /* initial address to use in an index */ + MaxIo = 64*1024, /* max size of a single read or write operation */ + ICacheBits = 16, /* default bits for indexing icache */ + ICacheDepth = 4, /* default depth of an icache hash chain */ + MaxAMap = 2*1024, /* max. allowed arenas in an address mapping; must be < 32*1024 */ + + /* + * return codes from syncarena + */ + SyncDataErr = 1 << 0, /* problem reading the clump data */ + SyncCIErr = 1 << 1, /* found erroneous clump directory entries */ + SyncCIZero = 1 << 2, /* found unwritten clump directory entries */ + SyncFixErr = 1 << 3, /* error writing fixed data */ + SyncHeader = 1 << 4, /* altered header fields */ + + /* + * error severity + */ + EOk = 0, /* error expected in normal operation */ + EStrange, /* strange error that should be logged */ + ECorrupt, /* corrupted data found in arenas */ + EICorrupt, /* corrupted data found in index */ + EAdmin, /* should be brought to administrators' attention */ + ECrash, /* really bad internal error */ + EBug, /* a limitation which should be fixed */ + EInconsist, /* inconsistencies between index and arena */ + EMax, + + /* + * internal disk formats for the venti archival storage system + */ + /* + * magic numbers on disk + */ + _ClumpMagic = 0xd15cb10c, /* clump header, deprecated */ + ClumpFreeMagic = 0, /* free clump; terminates active clump log */ + + ArenaPartMagic = 0xa9e4a5e7, /* arena partition header */ + ArenaMagic = 0xf2a14ead, /* arena trailer */ + ArenaHeadMagic = 0xd15c4ead, /* arena header */ + + BloomMagic = 0xb1004ead, /* bloom filter header */ + BloomMaxHash = 32, + + ISectMagic = 0xd15c5ec7, /* index header */ + + ArenaPartVersion = 3, + ArenaVersion4 = 4, + ArenaVersion5 = 5, + BloomVersion = 1, + IndexVersion = 1, + ISectVersion1 = 1, + ISectVersion2 = 2, + + /* + * encodings of clumps on disk + */ + ClumpEErr = 0, /* can't happen */ + ClumpENone, /* plain */ + ClumpECompress, /* compressed */ + ClumpEMax, + + /* + * sizes in bytes on disk + */ + U8Size = 1, + U16Size = 2, + U32Size = 4, + U64Size = 8, + + ArenaPartSize = 4 * U32Size, + ArenaSize4 = 2 * U64Size + 6 * U32Size + ANameSize + U8Size, + ArenaSize5 = ArenaSize4 + U32Size, + ArenaHeadSize4 = U64Size + 3 * U32Size + ANameSize, + ArenaHeadSize5 = ArenaHeadSize4 + U32Size, + BloomHeadSize = 4 * U32Size, + ISectSize1 = 7 * U32Size + 2 * ANameSize, + ISectSize2 = ISectSize1 + U32Size, + ClumpInfoSize = U8Size + 2 * U16Size + VtScoreSize, + ClumpSize = ClumpInfoSize + U8Size + 3 * U32Size, + MaxBloomSize = 1<<(32-3), /* 2^32 bits */ + MaxBloomHash = 32, /* bits per score */ + /* + * BUG - The various block copies that manipulate entry buckets + * would be faster if we bumped IBucketSize up to 8 and IEntrySize up to 40, + * so that everything is word-aligned. Buildindex is actually cpu-bound + * by the (byte at a time) copying in qsort. + */ + IBucketSize = U32Size + U16Size, + IEntrySize = U64Size + U32Size + 2*U16Size + 2*U8Size + VtScoreSize, + IEntryTypeOff = VtScoreSize + U64Size + U32Size + 2 * U16Size, + + MaxClumpBlocks = (VtMaxLumpSize + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog, + + /* + * dirty flags - order controls disk write order + */ + DirtyArena = 1, + DirtyArenaCib, + DirtyArenaTrailer, + DirtyMax, + + VentiZZZZZZZZ +}; + +extern char TraceDisk[]; +extern char TraceLump[]; +extern char TraceBlock[]; +extern char TraceProc[]; +extern char TraceWork[]; +extern char TraceQuiet[]; +extern char TraceRpc[]; + +/* + * results of parsing and initializing a config file + */ +struct Config +{ + char *index; /* name of the index to initialize */ + int naparts; /* arena partitions initialized */ + ArenaPart **aparts; + int nsects; /* index sections initialized */ + ISect **sects; + Bloom *bloom; /* bloom filter */ + u32int bcmem; + u32int mem; + u32int icmem; + int queuewrites; + char* haddr; + char* vaddr; + char* webroot; +}; + +/* + * a Part is the low level interface to files or disks. + * there are two main types of partitions + * arena paritions, which some number of arenas, each in a sub-partition. + * index partition, which only have one subpartition. + */ +struct Part +{ + int fd; /* rock for accessing the disk */ + int mode; + u64int offset; + u64int size; /* size of the partiton */ + u32int blocksize; /* block size for reads and writes */ + u32int fsblocksize; /* minimum file system block size */ + char *name; + char *filename; + Channel *writechan; /* chan[dcache.nblock](DBlock*) */ +}; + +/* + * a cached block from the partition + * yuck -- most of this is internal structure for the cache + * all other routines should only use data + */ +struct DBlock +{ + u8int *data; + + Part *part; /* partition in which cached */ + u64int addr; /* base address on the partition */ + u32int size; /* amount of data available, not amount allocated; should go away */ + u32int mode; + u32int dirty; + u32int dirtying; + DBlock *next; /* doubly linked hash chains */ + DBlock *prev; + u32int heap; /* index in heap table */ + u32int used; /* last reference times */ + u32int used2; + u32int ref; /* reference count */ + RWLock lock; /* for access to data only */ + Channel *writedonechan; + void* chanbuf[1]; /* buffer for the chan! */ +}; + +/* + * a cached block from the partition + * yuck -- most of this is internal structure for the cache + * all other routines should only use data + * double yuck -- this is mostly the same as a DBlock + */ +struct Lump +{ + Packet *data; + + Part *part; /* partition in which cached */ + u8int score[VtScoreSize]; /* score of packet */ + u8int type; /* type of packet */ + u32int size; /* amount of data allocated to hold packet */ + Lump *next; /* doubly linked hash chains */ + Lump *prev; + u32int heap; /* index in heap table */ + u32int used; /* last reference times */ + u32int used2; + u32int ref; /* reference count */ + QLock lock; /* for access to data only */ +}; + +/* + * mapping between names and address ranges + */ +struct AMap +{ + u64int start; + u64int stop; + char name[ANameSize]; +}; + +/* + * an AMap along with a length + */ +struct AMapN +{ + int n; + AMap *map; +}; + +/* + * an ArenaPart is a partition made up of Arenas + * it exists because most os's don't support many partitions, + * and we want to have many different Arenas + */ +struct ArenaPart +{ + Part *part; + u64int size; /* size of underlying partition, rounded down to blocks */ + Arena **arenas; + u32int tabbase; /* base address of arena table on disk */ + u32int tabsize; /* max. bytes in arena table */ + + /* + * fields stored on disk + */ + u32int version; + u32int blocksize; /* "optimal" block size for reads and writes */ + u32int arenabase; /* base address of first arena */ + + /* + * stored in the arena mapping table on disk + */ + AMap *map; + int narenas; +}; + +/* + * info about one block in the clump info cache + */ +struct CIBlock +{ + u32int block; /* blocks in the directory */ + int offset; /* offsets of one clump in the data */ + DBlock *data; +}; + +/* + * Statistics kept in the tail. + */ +struct ATailStats +{ + u32int clumps; /* number of clumps */ + u32int cclumps; /* number of compressed clumps */ + u64int used; + u64int uncsize; + u8int sealed; +}; + +/* + * Arena state - represents a point in the data log + */ +struct AState +{ + Arena *arena; + u64int aa; /* index address */ + ATailStats stats; +}; + +/* + * an Arena is a log of Clumps, preceeded by an ArenaHeader, + * and followed by a Arena, each in one disk block. + * struct on disk is not always up to date, but should be self-consistent. + * to sync after reboot, follow clumps starting at used until ClumpFreeMagic if found. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + */ +struct Arena +{ + QLock lock; /* lock for arena fields, writing to disk */ + Part *part; /* partition in which arena lives */ + int blocksize; /* size of block to read or write */ + u64int base; /* base address on disk */ + u64int size; /* total space in the arena */ + u64int limit; /* storage limit for clumps */ + u8int score[VtScoreSize]; /* score of the entire sealed & summed arena */ + + int clumpmax; /* ClumpInfos per block */ + AState mem; + int inqueue; + DigestState sha1; + + /* + * fields stored on disk + */ + u32int version; + char name[ANameSize]; /* text label */ + ATailStats memstats; + ATailStats diskstats; + u32int ctime; /* first time a block was written */ + u32int wtime; /* last time a block was written */ + u32int clumpmagic; +}; + +/* + * redundant storage of some fields at the beginning of each arena + */ +struct ArenaHead +{ + u32int version; + char name[ANameSize]; + u32int blocksize; + u64int size; + u32int clumpmagic; +}; + +/* + * most interesting meta information for a clump. + * stored in each clump's header and in the Arena's directory, + * stored in reverse order just prior to the arena trailer + */ +struct ClumpInfo +{ + u8int type; + u16int size; /* size of disk data, not including header */ + u16int uncsize; /* size of uncompressed data */ + u8int score[VtScoreSize]; /* score of the uncompressed data only */ +}; + +/* + * header for an immutable clump of data + */ +struct Clump +{ + ClumpInfo info; + u8int encoding; + u32int creator; /* initial client which wrote the block */ + u32int time; /* creation at gmt seconds since 1/1/1970 */ +}; + +/* + * index of all clumps according to their score + * this is just a wrapper to tie together the index sections + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + */ +struct Index +{ + u32int div; /* divisor for mapping score to bucket */ + u32int buckets; /* last bucket used in disk hash table */ + u32int blocksize; + u32int tabsize; /* max. bytes in index config */ + u32int bitblocks; //XXX remove these fields + u32int maxdepth; + u32int bitkeylog; + u32int bitkeymask; + + int mapalloc; /* first arena to check when adding a lump */ + Arena **arenas; /* arenas in the mapping */ + ISect **sects; /* sections which hold the buckets */ + Bloom *bloom; /* bloom filter */ + + /* + * fields stored in config file + */ + u32int version; + char name[ANameSize]; /* text label */ + int nsects; + AMap *smap; /* mapping of buckets to index sections */ + int narenas; + AMap *amap; /* mapping from index addesses to arenas */ +}; + +/* + * one part of the bucket storage for an index. + * the index blocks are sequentially allocated + * across all of the sections. + */ +struct ISect +{ + Part *part; + int blocklog; /* log2(blocksize) */ + int buckmax; /* max. entries in a index bucket */ + u32int tabbase; /* base address of index config table on disk */ + u32int tabsize; /* max. bytes in index config */ + Channel *writechan; + Channel *writedonechan; + + /* + * fields stored on disk + */ + u32int version; + u32int bucketmagic; + char name[ANameSize]; /* text label */ + char index[ANameSize]; /* index owning the section */ + u32int blocksize; /* size of hash buckets in index */ + u32int blockbase; /* address of start of on disk index table */ + u32int blocks; /* total blocks on disk; some may be unused */ + u32int start; /* first bucket in this section */ + u32int stop; /* limit of buckets in this section */ +}; + +/* + * externally interesting part of an IEntry + */ +struct IAddr +{ + u64int addr; + u16int size; /* uncompressed size */ + u8int type; /* type of block */ + u8int blocks; /* arena io quanta for Clump + data */ +}; + +/* + * entries in the index + * kept in IBuckets in the disk index table, + * cached in the memory ICache. + */ +struct IEntry +{ + u8int score[VtScoreSize]; + IEntry *next; /* next in hash chain */ + IEntry *nextdirty; /* next in dirty chain */ + u32int wtime; /* last write time */ + u16int train; /* relative train containing the most recent ref; 0 if no ref, 1 if in same car */ + u8int rac; /* read ahead count */ + u8int dirty; /* is dirty */ + IAddr ia; +}; + +/* + * buckets in the on disk index table + */ +struct IBucket +{ + u16int n; /* number of active indices */ + u32int buck; /* used by buildindex/checkindex only */ + u8int *data; +}; + +/* + * temporary buffers used by individual threads + */ +struct ZBlock +{ + u32int len; + u32int _size; + u8int *data; + u8int *free; +}; + +/* + * simple input buffer for a '\0' terminated text file + */ +struct IFile +{ + char *name; /* name of the file */ + ZBlock *b; /* entire contents of file */ + u32int pos; /* current position in the file */ +}; + +struct Statdesc +{ + char *name; + ulong max; +}; + +/* keep in sync with stats.c:/statdesc and httpd.c:/graphname*/ +enum +{ + StatRpcTotal, + StatRpcRead, + StatRpcReadOk, + StatRpcReadFail, + StatRpcReadBytes, + StatRpcReadTime, + StatRpcReadCached, + StatRpcReadCachedTime, + StatRpcReadUncached, + StatRpcReadUncachedTime, + StatRpcWrite, + StatRpcWriteNew, + StatRpcWriteOld, + StatRpcWriteFail, + StatRpcWriteBytes, + StatRpcWriteTime, + StatRpcWriteNewTime, + StatRpcWriteOldTime, + + StatLcacheHit, + StatLcacheMiss, + StatLcacheRead, + StatLcacheWrite, + StatLcacheSize, + StatLcacheStall, + StatLcacheReadTime, + + StatDcacheHit, + StatDcacheMiss, + StatDcacheLookup, + StatDcacheRead, + StatDcacheWrite, + StatDcacheDirty, + StatDcacheSize, + StatDcacheFlush, + StatDcacheStall, + StatDcacheLookupTime, + + StatDblockStall, + StatLumpStall, + + StatIcacheHit, + StatIcacheMiss, + StatIcacheRead, + StatIcacheWrite, + StatIcacheFill, + StatIcachePrefetch, + StatIcacheDirty, + StatIcacheSize, + StatIcacheFlush, + StatIcacheStall, + StatIcacheReadTime, + + StatBloomHit, + StatBloomMiss, + StatBloomFalseMiss, + StatBloomLookup, + StatBloomOnes, + StatBloomBits, + StatBloomLookupTime, + + StatApartRead, + StatApartReadBytes, + StatApartWrite, + StatApartWriteBytes, + + StatIsectRead, + StatIsectReadBytes, + StatIsectWrite, + StatIsectWriteBytes, + + StatSumRead, + StatSumReadBytes, + + NStat +}; + +extern Statdesc statdesc[NStat]; + +/* + * statistics about the operation of the server + * mainly for performance monitoring and profiling. + */ +struct Stats +{ + ulong now; + ulong n[NStat]; +}; + +struct Statbin +{ + uint nsamp; + uint min; + uint max; + uint avg; +}; + +struct Graph +{ + long (*fn)(Stats*, Stats*, void*); + void *arg; + long t0; + long t1; + long min; + long max; + long wid; + long ht; + int fill; +}; + +/* + * for kicking background processes that run one round after another after another + */ +struct Round +{ + QLock lock; + Rendez start; + Rendez finish; + Rendez delaywait; + int delaytime; + int delaykick; + char* name; + int last; + int current; + int next; + int doanother; +}; + +/* + * Bloom filter of stored block hashes + */ +struct Bloom +{ + RWLock lk; /* protects nhash, nbits, tab, mb */ + QLock mod; /* one marker at a time, protects nb */ + int nhash; + ulong size; /* bytes in tab */ + ulong mask; /* to produce index */ + u8int *data; + Part *part; + Channel *writechan; + Channel *writedonechan; +}; + +extern Index *mainindex; +extern u32int maxblocksize; /* max. block size used by any partition */ +extern int paranoid; /* should verify hashes on disk read */ +extern int queuewrites; /* put all lump writes on a queue and finish later */ +extern int readonly; /* only allowed to read the disk data */ +extern Stats stats; +extern u8int zeroscore[VtScoreSize]; +extern int compressblocks; +extern int writestodevnull; /* dangerous - for performance debugging */ +extern int collectstats; +extern QLock memdrawlock; +extern int icachesleeptime; +extern int arenasumsleeptime; + +#ifndef PLAN9PORT +#pragma varargck type "V" uchar* +#define ODIRECT 0 +#endif blob - /dev/null blob + 72aaafaf67c6720813263e6f9cb16e39bc15aa98 (mode 644) --- /dev/null +++ src/cmd/venti/srv/dcache.c @@ -0,0 +1,816 @@ +/* + * Disk cache. + * + * Caches raw disk blocks. Getdblock() gets a block, putdblock puts it back. + * Getdblock has a mode parameter that determines i/o and access to a block: + * if mode is OREAD or ORDWR, it is read from disk if not already in memory. + * If mode is ORDWR or OWRITE, it is locked for exclusive use before being returned. + * It is *not* marked dirty -- once changes have been made, they should be noted + * by using dirtydblock() before putdblock(). + * + * There is a global cache lock as well as a lock on each block. + * Within a thread, the cache lock can be acquired while holding a block lock, + * but not vice versa; and a block cannot be locked if you already hold the lock + * on another block. + * + * The flush proc writes out dirty blocks in batches, one batch per dirty tag. + * For example, the DirtyArena blocks are all written to disk before any of the + * DirtyArenaCib blocks. + * + * This code used to be in charge of flushing the dirty index blocks out to + * disk, but updating the index turned out to benefit from extra care. + * Now cached index blocks are never marked dirty. The index.c code takes + * care of updating them behind our back, and uses _getdblock to update any + * cached copies of the blocks as it changes them on disk. + */ + +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +typedef struct DCache DCache; + +enum +{ + HashLog = 9, + HashSize = 1<dirty */ + Rendez full; + Round round; + DBlock *free; /* list of available lumps */ + u32int now; /* ticks for usage timestamps */ + int size; /* max. size of any block; allocated to each block */ + DBlock **heads; /* hash table for finding address */ + int nheap; /* number of available victims */ + DBlock **heap; /* heap for locating victims */ + int nblocks; /* number of blocks allocated */ + DBlock *blocks; /* array of block descriptors */ + DBlock **write; /* array of block pointers to be written */ + u8int *mem; /* memory for all block descriptors */ + int ndirty; /* number of dirty blocks */ + int maxdirty; /* max. number of dirty blocks */ + Channel *ra; + u8int *rabuf; + u32int ramax; + u32int rasize; + u64int raaddr; + Part *rapart; + + AState diskstate; + AState state; +}; + +typedef struct Ra Ra; +struct Ra +{ + Part *part; + u64int addr; +}; + +static DCache dcache; + +static int downheap(int i, DBlock *b); +static int upheap(int i, DBlock *b); +static DBlock *bumpdblock(void); +static void delheap(DBlock *db); +static void fixheap(int i, DBlock *b); +static void flushproc(void*); +static void writeproc(void*); +static void raproc(void*); + +void +initdcache(u32int mem) +{ + DBlock *b, *last; + u32int nblocks, blocksize; + int i; + u8int *p; + + if(mem < maxblocksize * 2) + sysfatal("need at least %d bytes for the disk cache", maxblocksize * 2); + if(maxblocksize == 0) + sysfatal("no max. block size given for disk cache"); + blocksize = maxblocksize; + nblocks = mem / blocksize; + dcache.full.l = &dcache.lock; + dcache.nblocks = nblocks; + dcache.maxdirty = (nblocks * 2) / 3; + trace(TraceProc, "initialize disk cache with %d blocks of %d bytes, maximum %d dirty blocks\n", + nblocks, blocksize, dcache.maxdirty); + dcache.size = blocksize; + dcache.heads = MKNZ(DBlock*, HashSize); + dcache.heap = MKNZ(DBlock*, nblocks); + dcache.blocks = MKNZ(DBlock, nblocks); + dcache.write = MKNZ(DBlock*, nblocks); + dcache.mem = MKNZ(u8int, (nblocks+1+128) * blocksize); + dcache.ra = chancreate(sizeof(Ra), 0); + + last = nil; + p = (u8int*)(((ulong)dcache.mem+blocksize-1)&~(ulong)(blocksize-1)); + for(i = 0; i < nblocks; i++){ + b = &dcache.blocks[i]; + b->data = &p[i * blocksize]; + b->heap = TWID32; + b->writedonechan = chancreate(sizeof(void*), 1); + b->next = last; + last = b; + } + dcache.rabuf = &p[i*blocksize]; + dcache.ramax = 128*blocksize; + dcache.raaddr = 0; + dcache.rapart = nil; + + dcache.free = last; + dcache.nheap = 0; + setstat(StatDcacheSize, nblocks); + initround(&dcache.round, "dcache", 120*1000); + + vtproc(flushproc, nil); + vtproc(delaykickroundproc, &dcache.round); + vtproc(raproc, nil); +} + +void +setdcachestate(AState *a) +{ + trace(TraceBlock, "setdcachestate %s 0x%llux clumps %d", a->arena ? a->arena->name : nil, a->aa, a->stats.clumps); + qlock(&dcache.lock); + dcache.state = *a; + qunlock(&dcache.lock); +} + +AState +diskstate(void) +{ + AState a; + + qlock(&dcache.lock); + a = dcache.diskstate; + qunlock(&dcache.lock); + return a; +} + +static void +raproc(void *v) +{ + Ra ra; + DBlock *b; + + USED(v); + while(recv(dcache.ra, &ra) == 1){ + if(ra.part->size <= ra.addr) + continue; + b = _getdblock(ra.part, ra.addr, OREAD, 2); + putdblock(b); + } +} + +void +dreadahead(Part *part, u64int addr, int miss) +{ + Ra ra; + static struct { + Part *part; + u64int addr; + } lastmiss; + static struct { + Part *part; + u64int addr; + int dir; + } lastra; + +return; + if(miss){ + if(lastmiss.part==part && lastmiss.addr==addr-dcache.size){ + XRa: + lastra.part = part; + lastra.dir = addr-lastmiss.addr; + lastra.addr = addr+lastra.dir; + ra.part = part; + ra.addr = lastra.addr; + nbsend(dcache.ra, &ra); + }else if(lastmiss.part==part && lastmiss.addr==addr+dcache.size){ + addr -= dcache.size; + goto XRa; + } + }else{ + if(lastra.part==part && lastra.addr==addr){ + lastra.addr += lastra.dir; + ra.part = part; + ra.addr = lastra.addr; + nbsend(dcache.ra, &ra); + } + } + + if(miss){ + lastmiss.part = part; + lastmiss.addr = addr; + } + +// fprint(2, "%s %llx %s\n", part->name, addr, miss ? "miss" : "hit"); +} + +int +rareadpart(Part *part, u64int addr, u8int *buf, uint n, int load) +{ + uint nn; + static RWLock ralock; + + rlock(&ralock); + if(dcache.rapart==part && dcache.raaddr <= addr && addr+n <= dcache.raaddr+dcache.rasize){ + memmove(buf, dcache.rabuf+(addr-dcache.raaddr), n); + runlock(&ralock); + return 0; + } + if(load != 2 || addr >= part->size){ /* addr >= part->size: let readpart do the error */ + runlock(&ralock); + return readpart(part, addr, buf, n); + } + + runlock(&ralock); + wlock(&ralock); +fprint(2, "raread %s %llx\n", part->name, addr); + nn = dcache.ramax; + if(addr+nn > part->size) + nn = part->size - addr; + if(readpart(part, addr, dcache.rabuf, nn) < 0){ + wunlock(&ralock); + return -1; + } + memmove(buf, dcache.rabuf, n); + dcache.rapart = part; + dcache.rasize = nn; + dcache.raaddr = addr; + wunlock(&ralock); + + addstat(StatApartReadBytes, nn-n); + return 0; +} + +static u32int +pbhash(u64int addr) +{ + u32int h; + +#define hashit(c) ((((c) * 0x6b43a9b5) >> (32 - HashLog)) & HashMask) + h = (addr >> 32) ^ addr; + return hashit(h); +} + +DBlock* +getdblock(Part *part, u64int addr, int mode) +{ + DBlock *b; + uint ms; + + ms = msec(); + b = _getdblock(part, addr, mode, 1); + if(mode == OREAD || mode == ORDWR) + addstat(StatDcacheRead, 1); + if(mode == OWRITE || mode == ORDWR) + addstat(StatDcacheWrite, 1); + ms = msec() - ms; + addstat2(StatDcacheLookup, 1, StatDcacheLookupTime, ms); + return b; +} + +DBlock* +_getdblock(Part *part, u64int addr, int mode, int load) +{ + DBlock *b; + u32int h, size; + + trace(TraceBlock, "getdblock enter %s 0x%llux", part->name, addr); + size = part->blocksize; + if(size > dcache.size){ + seterr(EAdmin, "block size %d too big for cache with size %d", size, dcache.size); + return nil; + } + h = pbhash(addr); + + /* + * look for the block in the cache + */ +//checkdcache(); + qlock(&dcache.lock); +again: + for(b = dcache.heads[h]; b != nil; b = b->next){ + if(b->part == part && b->addr == addr){ + /* + qlock(&stats.lock); + stats.pchit++; + qunlock(&stats.lock); + */ + if(load){ + addstat(StatDcacheHit, 1); + if(load != 2 && mode != OWRITE) + dreadahead(part, b->addr, 0); + } + goto found; + } + } + + /* + * missed: locate the block with the oldest second to last use. + * remove it from the heap, and fix up the heap. + */ + if(!load){ + qunlock(&dcache.lock); + return nil; + } + + addstat(StatDcacheMiss, 1); + + b = bumpdblock(); + if(b == nil){ + trace(TraceBlock, "all disk cache blocks in use"); + addstat(StatDcacheStall, 1); + rsleep(&dcache.full); + addstat(StatDcacheStall, -1); + goto again; + } + + assert(!b->dirty); + + /* + * the new block has no last use, so assume it happens sometime in the middle +ZZZ this is not reasonable + */ + b->used = (b->used2 + dcache.now) / 2; + + /* + * rechain the block on the correct hash chain + */ + b->next = dcache.heads[h]; + dcache.heads[h] = b; + if(b->next != nil) + b->next->prev = b; + b->prev = nil; + + b->addr = addr; + b->part = part; + b->size = 0; + if(load != 2 && mode != OWRITE) + dreadahead(part, b->addr, 1); + +found: + b->ref++; + b->used2 = b->used; + b->used = dcache.now++; + if(b->heap != TWID32) + fixheap(b->heap, b); + + qunlock(&dcache.lock); +//checkdcache(); + + trace(TraceBlock, "getdblock lock"); + addstat(StatDblockStall, 1); + if(mode == OREAD) + rlock(&b->lock); + else + wlock(&b->lock); + addstat(StatDblockStall, -1); + trace(TraceBlock, "getdblock locked"); + + if(b->size != size){ + if(mode == OREAD){ + addstat(StatDblockStall, 1); + runlock(&b->lock); + wlock(&b->lock); + addstat(StatDblockStall, -1); + } + if(b->size < size){ + if(mode == OWRITE) + memset(&b->data[b->size], 0, size - b->size); + else{ + trace(TraceBlock, "getdblock readpart %s 0x%llux", part->name, addr); + if(rareadpart(part, addr + b->size, &b->data[b->size], size - b->size, load) < 0){ + b->mode = ORDWR; /* so putdblock wunlocks */ + putdblock(b); + return nil; + } + trace(TraceBlock, "getdblock readpartdone"); + addstat(StatApartRead, 1); + addstat(StatApartReadBytes, size-b->size); + } + } + b->size = size; + if(mode == OREAD){ + addstat(StatDblockStall, 1); + wunlock(&b->lock); + rlock(&b->lock); + addstat(StatDblockStall, -1); + } + } + + b->mode = mode; + trace(TraceBlock, "getdblock exit"); + return b; +} + +void +putdblock(DBlock *b) +{ + if(b == nil) + return; + + trace(TraceBlock, "putdblock %s 0x%llux", b->part->name, b->addr); + + if(b->mode == OREAD) + runlock(&b->lock); + else + wunlock(&b->lock); + +//checkdcache(); + qlock(&dcache.lock); + if(--b->ref == 0 && !b->dirty){ + if(b->heap == TWID32) + upheap(dcache.nheap++, b); + rwakeupall(&dcache.full); + } + qunlock(&dcache.lock); +//checkdcache(); +} + +void +dirtydblock(DBlock *b, int dirty) +{ + int odirty; + Part *p; + + + trace(TraceBlock, "dirtydblock enter %s 0x%llux %d from 0x%lux", b->part->name, b->addr, dirty, getcallerpc(&b)); + assert(b->ref != 0); + assert(b->mode==ORDWR || b->mode==OWRITE); + + odirty = b->dirty; + if(b->dirty) + assert(b->dirty == dirty); + else + b->dirty = dirty; + + p = b->part; + if(p->writechan == nil){ + trace(TraceBlock, "dirtydblock allocwriteproc %s", p->name); + /* XXX hope this doesn't fail! */ + p->writechan = chancreate(sizeof(DBlock*), dcache.nblocks); + vtproc(writeproc, p); + } + qlock(&dcache.lock); + if(!odirty){ + dcache.ndirty++; + setstat(StatDcacheDirty, dcache.ndirty); + if(dcache.ndirty >= dcache.maxdirty) + kickround(&dcache.round, 0); + else + delaykickround(&dcache.round); + } + qunlock(&dcache.lock); +} + +/* + * remove some block from use and update the free list and counters + */ +static DBlock* +bumpdblock(void) +{ + DBlock *b; + ulong h; + + trace(TraceBlock, "bumpdblock enter"); + b = dcache.free; + if(b != nil){ + dcache.free = b->next; + return b; + } + + if(dcache.ndirty >= dcache.maxdirty) + kickdcache(); + + /* + * remove blocks until we find one that is unused + * referenced blocks are left in the heap even though + * they can't be scavenged; this is simple a speed optimization + */ + for(;;){ + if(dcache.nheap == 0){ + kickdcache(); + trace(TraceBlock, "bumpdblock gotnothing"); + return nil; + } + b = dcache.heap[0]; + delheap(b); + if(!b->ref && !b->dirty) + break; + } + + trace(TraceBlock, "bumpdblock bumping %s 0x%llux", b->part->name, b->addr); + + /* + * unchain the block + */ + if(b->prev == nil){ + h = pbhash(b->addr); + if(dcache.heads[h] != b) + sysfatal("bad hash chains in disk cache"); + dcache.heads[h] = b->next; + }else + b->prev->next = b->next; + if(b->next != nil) + b->next->prev = b->prev; + + return b; +} + +/* + * delete an arbitrary block from the heap + */ +static void +delheap(DBlock *db) +{ + if(db->heap == TWID32) + return; + fixheap(db->heap, dcache.heap[--dcache.nheap]); + db->heap = TWID32; +} + +/* + * push an element up or down to it's correct new location + */ +static void +fixheap(int i, DBlock *b) +{ + if(upheap(i, b) == i) + downheap(i, b); +} + +static int +upheap(int i, DBlock *b) +{ + DBlock *bb; + u32int now; + int p; + + now = dcache.now; + for(; i != 0; i = p){ + p = (i - 1) >> 1; + bb = dcache.heap[p]; + if(b->used2 - now >= bb->used2 - now) + break; + dcache.heap[i] = bb; + bb->heap = i; + } + + dcache.heap[i] = b; + b->heap = i; + return i; +} + +static int +downheap(int i, DBlock *b) +{ + DBlock *bb; + u32int now; + int k; + + now = dcache.now; + for(; ; i = k){ + k = (i << 1) + 1; + if(k >= dcache.nheap) + break; + if(k + 1 < dcache.nheap && dcache.heap[k]->used2 - now > dcache.heap[k + 1]->used2 - now) + k++; + bb = dcache.heap[k]; + if(b->used2 - now <= bb->used2 - now) + break; + dcache.heap[i] = bb; + bb->heap = i; + } + + dcache.heap[i] = b; + b->heap = i; + return i; +} + +static void +findblock(DBlock *bb) +{ + DBlock *b, *last; + int h; + + last = nil; + h = pbhash(bb->addr); + for(b = dcache.heads[h]; b != nil; b = b->next){ + if(last != b->prev) + sysfatal("bad prev link"); + if(b == bb) + return; + last = b; + } + sysfatal("block missing from hash table"); +} + +void +checkdcache(void) +{ + DBlock *b; + u32int size, now; + int i, k, refed, nfree; + + qlock(&dcache.lock); + size = dcache.size; + now = dcache.now; + for(i = 0; i < dcache.nheap; i++){ + if(dcache.heap[i]->heap != i) + sysfatal("dc: mis-heaped at %d: %d", i, dcache.heap[i]->heap); + if(i > 0 && dcache.heap[(i - 1) >> 1]->used2 - now > dcache.heap[i]->used2 - now) + sysfatal("dc: bad heap ordering"); + k = (i << 1) + 1; + if(k < dcache.nheap && dcache.heap[i]->used2 - now > dcache.heap[k]->used2 - now) + sysfatal("dc: bad heap ordering"); + k++; + if(k < dcache.nheap && dcache.heap[i]->used2 - now > dcache.heap[k]->used2 - now) + sysfatal("dc: bad heap ordering"); + } + + refed = 0; + for(i = 0; i < dcache.nblocks; i++){ + b = &dcache.blocks[i]; + if(b->data != &dcache.mem[i * size]) + sysfatal("dc: mis-blocked at %d", i); + if(b->ref && b->heap == TWID32) + refed++; + if(b->addr) + findblock(b); + if(b->heap != TWID32 + && dcache.heap[b->heap] != b) + sysfatal("dc: spurious heap value"); + } + + nfree = 0; + for(b = dcache.free; b != nil; b = b->next){ + if(b->addr != 0 || b->heap != TWID32) + sysfatal("dc: bad free list"); + nfree++; + } + + if(dcache.nheap + nfree + refed != dcache.nblocks) + sysfatal("dc: missing blocks: %d %d %d", dcache.nheap, refed, dcache.nblocks); + qunlock(&dcache.lock); +} + +void +flushdcache(void) +{ + trace(TraceProc, "flushdcache enter"); + kickround(&dcache.round, 1); + trace(TraceProc, "flushdcache exit"); +} + +void +kickdcache(void) +{ + kickround(&dcache.round, 0); +} + +static int +parallelwrites(DBlock **b, DBlock **eb, int dirty) +{ + DBlock **p, **q; + for(p=b; pdirty == dirty; p++){ + assert(b<=p && ppart->writechan, *p); + } + q = p; + for(p=b; pwritedonechan); + } + + return p-b; +} + +/* + * Sort first by dirty flag, then by partition, then by address in partition. + */ +static int +writeblockcmp(const void *va, const void *vb) +{ + DBlock *a, *b; + + a = *(DBlock**)va; + b = *(DBlock**)vb; + + if(a->dirty != b->dirty) + return a->dirty - b->dirty; + if(a->part != b->part){ + if(a->part < b->part) + return -1; + if(a->part > b->part) + return 1; + } + if(a->addr < b->addr) + return -1; + return 1; +} + +static void +flushproc(void *v) +{ + int i, j, n; + ulong t0; + DBlock *b, **write; + AState as; + + USED(v); + threadsetname("flushproc"); + for(;;){ + waitforkick(&dcache.round); + + trace(TraceWork, "start"); + qlock(&dcache.lock); + as = dcache.state; + qunlock(&dcache.lock); + + t0 = nsec()/1000; + + trace(TraceProc, "build t=%lud", (ulong)(nsec()/1000)-t0); + write = dcache.write; + n = 0; + for(i=0; idirty) + write[n++] = b; + } + + qsort(write, n, sizeof(write[0]), writeblockcmp); + + /* Write each stage of blocks out. */ + trace(TraceProc, "writeblocks t=%lud", (ulong)(nsec()/1000)-t0); + i = 0; + for(j=1; jdirty); + abort(); + } + +/* XXX +* the locking here is suspect. what if a block is redirtied +* after the write happens? we'll still decrement dcache.ndirty here. +*/ + trace(TraceProc, "undirty.%d t=%lud", j, (ulong)(nsec()/1000)-t0); + qlock(&dcache.lock); + dcache.diskstate = as; + for(i=0; iref == 0 && b->heap == TWID32){ + upheap(dcache.nheap++, b); + rwakeupall(&dcache.full); + } + } + setstat(StatDcacheDirty, dcache.ndirty); + qunlock(&dcache.lock); + addstat(StatDcacheFlush, 1); + trace(TraceWork, "finish"); + } +} + +static void +writeproc(void *v) +{ + DBlock *b; + Part *p; + + p = v; + + threadsetname("writeproc:%s", p->name); + for(;;){ + b = recvp(p->writechan); + trace(TraceWork, "start"); + assert(b->part == p); + trace(TraceProc, "wlock %s 0x%llux", p->name, b->addr); + wlock(&b->lock); + trace(TraceProc, "writepart %s 0x%llux", p->name, b->addr); + if(writepart(p, b->addr, b->data, b->size) < 0) + fprint(2, "write error: %r\n"); /* XXX details! */ + addstat(StatApartWrite, 1); + addstat(StatApartWriteBytes, b->size); + b->dirty = 0; + wunlock(&b->lock); + trace(TraceProc, "finish %s 0x%llux", p->name, b->addr); + trace(TraceWork, "finish"); + sendp(b->writedonechan, b); + } +} blob - /dev/null blob + fa2bfb7d2528af18d0153233a90589965bc90d6a (mode 644) --- /dev/null +++ src/cmd/venti/srv/dump.c @@ -0,0 +1,47 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +void +printindex(int fd, Index *ix) +{ + int i; + + fprint(fd, "index=%s version=%d blocksize=%d tabsize=%d\n", + ix->name, ix->version, ix->blocksize, ix->tabsize); + fprint(fd, "\tbuckets=%d div=%d\n", ix->buckets, ix->div); + for(i = 0; i < ix->nsects; i++) + fprint(fd, "\tsect=%s for buckets [%lld,%lld)\n", ix->smap[i].name, ix->smap[i].start, ix->smap[i].stop); + for(i = 0; i < ix->narenas; i++) + fprint(fd, "\tarena=%s at [%lld,%lld)\n", ix->amap[i].name, ix->amap[i].start, ix->amap[i].stop); +} + +void +printarenapart(int fd, ArenaPart *ap) +{ + int i; + + fprint(fd, "arena partition=%s\n\tversion=%d blocksize=%d arenas=%d\n\tsetbase=%d setsize=%d\n", + ap->part->name, ap->version, ap->blocksize, ap->narenas, ap->tabbase, ap->tabsize); + for(i = 0; i < ap->narenas; i++) + fprint(fd, "\tarena=%s at [%lld,%lld)\n", ap->map[i].name, ap->map[i].start, ap->map[i].stop); +} + +void +printarena(int fd, Arena *arena) +{ + fprint(fd, "arena='%s' [%lld,%lld)\n\tversion=%d created=%d modified=%d", + arena->name, arena->base, arena->base + arena->size + 2 * arena->blocksize, + arena->version, arena->ctime, arena->wtime); + if(arena->memstats.sealed) + fprint(2, " sealed\n"); + else + fprint(2, "\n"); + if(scorecmp(zeroscore, arena->score) != 0) + fprint(2, "\tscore=%V\n", arena->score); + + fprint(fd, "\tclumps=%,d compressed clumps=%,d data=%,lld compressed data=%,lld disk storage=%,lld\n", + arena->memstats.clumps, arena->memstats.cclumps, arena->memstats.uncsize, + arena->memstats.used - arena->memstats.clumps * ClumpSize, + arena->memstats.used + arena->memstats.clumps * ClumpInfoSize); +} blob - /dev/null blob + 226d97aef20e794ce196f496f89e764c9245ddd4 (mode 644) --- /dev/null +++ src/cmd/venti/srv/findscore.c @@ -0,0 +1,121 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +enum +{ + ClumpChunks = 32*1024 +}; + +static int verbose; + +int +clumpinfoeq(ClumpInfo *c, ClumpInfo *d) +{ + return c->type == d->type + && c->size == d->size + && c->uncsize == d->uncsize + && scorecmp(c->score, d->score)==0; +} + +int +findscore(Arena *arena, uchar *score) +{ + IEntry ie; + ClumpInfo *ci, *cis; + u64int a; + u32int clump; + int i, n, found; + +//ZZZ remove fprint? + if(arena->memstats.clumps) + fprint(2, "reading directory for arena=%s with %d entries\n", arena->name, arena->memstats.clumps); + + cis = MKN(ClumpInfo, ClumpChunks); + found = 0; + a = 0; + memset(&ie, 0, sizeof(IEntry)); + for(clump = 0; clump < arena->memstats.clumps; clump += n){ + n = ClumpChunks; + if(n > arena->memstats.clumps - clump) + n = arena->memstats.clumps - clump; + if(readclumpinfos(arena, clump, cis, n) != n){ + seterr(EOk, "arena directory read failed: %r"); + break; + } + + for(i = 0; i < n; i++){ + ci = &cis[i]; + if(scorecmp(score, ci->score)==0){ + fprint(2, "found at clump=%d with type=%d size=%d csize=%d position=%lld\n", + clump + i, ci->type, ci->uncsize, ci->size, a); + found++; + } + a += ci->size + ClumpSize; + } + } + free(cis); + return found; +} + +void +usage(void) +{ + fprint(2, "usage: findscore [-v] arenafile score\n"); + threadexitsall(0); +} + +void +threadmain(int argc, char *argv[]) +{ + ArenaPart *ap; + Part *part; + char *file; + u8int score[VtScoreSize]; + int i, found; + + ventifmtinstall(); + + ARGBEGIN{ + case 'v': + verbose++; + break; + default: + usage(); + break; + }ARGEND + + readonly = 1; + + if(argc != 2) + usage(); + + file = argv[0]; + if(strscore(argv[1], score) < 0) + sysfatal("bad score %s\n", argv[1]); + + part = initpart(file, OREAD|ODIRECT); + if(part == nil) + sysfatal("can't open partition %s: %r", file); + + ap = initarenapart(part); + if(ap == nil) + sysfatal("can't initialize arena partition in %s: %r", file); + + if(verbose > 1){ + printarenapart(2, ap); + fprint(2, "\n"); + } + + initdcache(8 * MaxDiskBlock); + + found = 0; + for(i = 0; i < ap->narenas; i++) + found += findscore(ap->arenas[i], score); + + print("found %d occurrences of %V\n", found, score); + + if(verbose > 1) + printstats(); + threadexitsall(0); +} blob - /dev/null blob + 44c975ecaf13a2e36dc972cddc98addd45ffb3ee (mode 644) --- /dev/null +++ src/cmd/venti/srv/fmtarenas.c @@ -0,0 +1,135 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" +#ifndef ODIRECT +#define ODIRECT 0 +#endif + +void +usage(void) +{ + fprint(2, "usage: fmtarenas [-Z] [-b blocksize] [-a arenasize] name file\n"); + threadexitsall(0); +} + +void +threadmain(int argc, char *argv[]) +{ + int vers; + ArenaPart *ap; + Part *part; + Arena *arena; + u64int addr, limit, asize, apsize; + char *file, *name, aname[ANameSize]; + int i, n, blocksize, tabsize, zero; + + ventifmtinstall(); + statsinit(); + + blocksize = 8 * 1024; + asize = 512 * 1024 *1024; + tabsize = 512 * 1024; /* BUG: should be determine from number of arenas */ + zero = -1; + vers = ArenaVersion5; + ARGBEGIN{ + case 'D': + settrace(EARGF(usage())); + break; + case 'a': + asize = unittoull(ARGF()); + if(asize == TWID64) + usage(); + break; + case 'b': + blocksize = unittoull(ARGF()); + if(blocksize == ~0) + usage(); + if(blocksize > MaxDiskBlock){ + fprint(2, "block size too large, max %d\n", MaxDiskBlock); + threadexitsall("usage"); + } + break; + case '4': + vers = ArenaVersion4; + break; + case 'Z': + zero = 0; + break; + default: + usage(); + break; + }ARGEND + + if(zero == -1){ + if(vers == ArenaVersion4) + zero = 1; + else + zero = 0; + } + + if(argc != 2) + usage(); + + name = argv[0]; + file = argv[1]; + + if(nameok(name) < 0) + sysfatal("illegal name template %s", name); + + part = initpart(file, ORDWR|ODIRECT); + if(part == nil) + sysfatal("can't open partition %s: %r", file); + + if(zero) + zeropart(part, blocksize); + + maxblocksize = blocksize; + initdcache(20*blocksize); + + ap = newarenapart(part, blocksize, tabsize); + if(ap == nil) + sysfatal("can't initialize arena: %r"); + + apsize = ap->size - ap->arenabase; + n = apsize / asize; + if(apsize - (n * asize) >= MinArenaSize) + n++; + + fprint(2, "fmtarenas %s: %,d arenas, %,lld bytes storage, %,d bytes for index map\n", + file, n, apsize, ap->tabsize); + + ap->narenas = n; + ap->map = MKNZ(AMap, n); + ap->arenas = MKNZ(Arena*, n); + + addr = ap->arenabase; + for(i = 0; i < n; i++){ + limit = addr + asize; + if(limit >= ap->size || ap->size - limit < MinArenaSize){ + limit = ap->size; + if(limit - addr < MinArenaSize) + sysfatal("bad arena set math: runt arena at %lld,%lld %lld\n", addr, limit, ap->size); + } + + snprint(aname, ANameSize, "%s%d", name, i); + + if(0) fprint(2, "adding arena %s at [%lld,%lld)\n", aname, addr, limit); + + arena = newarena(part, vers, aname, addr, limit - addr, blocksize); + if(!arena) + fprint(2, "can't make new arena %s: %r", aname); + freearena(arena); + + ap->map[i].start = addr; + ap->map[i].stop = limit; + namecp(ap->map[i].name, aname); + + addr = limit; + } + + if(wbarenapart(ap) < 0) + fprint(2, "can't write back arena partition header for %s: %r\n", file); + + flushdcache(); + threadexitsall(0); +} blob - /dev/null blob + 3c50d82fea62416ff6049aaf83ba155e471643e8 (mode 644) --- /dev/null +++ src/cmd/venti/srv/fmtbloom.c @@ -0,0 +1,115 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +Bloom b; + +void +usage(void) +{ + fprint(2, "usage: fmtbloom [-s size] [-n nblocks | -N nhash] file\n"); + threadexitsall(0); +} + +void +threadmain(int argc, char *argv[]) +{ + Part *part; + char *file; + vlong bits, size, size2; + int nhash; + vlong nblocks; + + ventifmtinstall(); + statsinit(); + + size = 0; + nhash = nblocks = 0; + ARGBEGIN{ + case 'n': + if(nhash || nblocks) + usage(); + nblocks = unittoull(EARGF(usage())); + break; + case 'N': + if(nhash || nblocks) + usage(); + nhash = unittoull(EARGF(usage())); + if(nhash > BloomMaxHash){ + fprint(2, "maximum possible is -N %d", BloomMaxHash); + usage(); + } + break; + case 's': + size = unittoull(ARGF()); + if(size == ~0) + usage(); + break; + default: + usage(); + break; + }ARGEND + + if(argc != 1) + usage(); + + file = argv[0]; + + part = initpart(file, ORDWR|ODIRECT); + if(part == nil) + sysfatal("can't open partition %s: %r", file); + + if(size == 0) + size = part->size; + + if(size < 1024*1024) + sysfatal("bloom filter too small"); + + if(size > MaxBloomSize){ + fprint(2, "warning: not using entire %,lld bytes; using only %,lld bytes\n", + size, MaxBloomSize); + size = MaxBloomSize; + } + if(size&(size-1)){ + for(size2=1; size2= nblocks*64) + size2 >>= 1; + if(size2 != size){ + size = size2; + fprint(2, "warning: using only %lldMB - not enough blocks to warrant more\n", + size/1024/1024); + } + + /* + * optimal is to use ln 2 times as many hash functions as we have bits per blocks. + */ + bits = (8*size)/nblocks; + nhash = bits*7/10; + if(nhash > BloomMaxHash) + nhash = BloomMaxHash; + } + if(!nhash) + nhash = BloomMaxHash; + if(bloominit(&b, size, nil) < 0) + sysfatal("bloominit: %r"); + b.nhash = nhash; + bits = nhash*10/7; + nblocks = (8*size)/bits; + fprint(2, "fmtbloom: using %lldMB, %d hashes/score, best up to %,lld blocks\n", size, nhash, nblocks); + b.data = vtmallocz(size); + b.part = part; + if(writebloom(&b) < 0) + sysfatal("writing %s: %r", file); + threadexitsall(0); +} blob - /dev/null blob + a0eb6b1645f2b342e0d1a86e8149b027b35c3a49 (mode 644) --- /dev/null +++ src/cmd/venti/srv/fmtindex.c @@ -0,0 +1,120 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +void +usage(void) +{ + fprint(2, "usage: fmtindex [-a] config\n"); + threadexitsall(0); +} + +void +threadmain(int argc, char *argv[]) +{ + Config conf; + Index *ix; + ArenaPart *ap; + Arena **arenas; + AMap *amap; + u64int addr; + char *file; + u32int i, j, n, narenas; + int add; + + ventifmtinstall(); + statsinit(); + + add = 0; + ARGBEGIN{ + case 'a': + add = 1; + break; + default: + usage(); + break; + }ARGEND + + if(argc != 1) + usage(); + + file = argv[0]; + + if(runconfig(file, &conf) < 0) + sysfatal("can't initialize config %s: %r", file); + if(conf.index == nil) + sysfatal("no index specified in %s", file); + if(nameok(conf.index) < 0) + sysfatal("illegal index name %s", conf.index); + + narenas = 0; + for(i = 0; i < conf.naparts; i++){ + ap = conf.aparts[i]; + narenas += ap->narenas; + } + + if(add){ + ix = initindex(conf.index, conf.sects, conf.nsects); + if(ix == nil) + sysfatal("can't initialize index %s: %r", conf.index); + }else{ + ix = newindex(conf.index, conf.sects, conf.nsects); + if(ix == nil) + sysfatal("can't create new index %s: %r", conf.index); + + n = 0; + for(i = 0; i < ix->nsects; i++) + n += ix->sects[i]->blocks; + + if(0) fprint(2, "using %ud buckets of %ud; div=%d\n", ix->buckets, n, ix->div); + } + amap = MKNZ(AMap, narenas); + arenas = MKNZ(Arena*, narenas); + + addr = IndexBase; + n = 0; + for(i = 0; i < conf.naparts; i++){ + ap = conf.aparts[i]; + for(j = 0; j < ap->narenas; j++){ + if(n >= narenas) + sysfatal("too few slots in index's arena set"); + + arenas[n] = ap->arenas[j]; + if(n < ix->narenas){ + if(arenas[n] != ix->arenas[n]) + sysfatal("mismatched arenas %s and %s at slot %d\n", + arenas[n]->name, ix->arenas[n]->name, n); + amap[n] = ix->amap[n]; + if(amap[n].start != addr) + sysfatal("mis-located arena %s in index %s\n", arenas[n]->name, ix->name); + addr = amap[n].stop; + }else{ + amap[n].start = addr; + addr += ap->arenas[j]->size; + amap[n].stop = addr; + namecp(amap[n].name, ap->arenas[j]->name); + if(0) fprint(2, "add arena %s at [%lld,%lld)\n", + amap[n].name, amap[n].start, amap[n].stop); + } + + n++; + } + } + if(0){ + fprint(2, "configured index=%s with arenas=%d and storage=%lld\n", + ix->name, n, addr - IndexBase); + fprint(2, "\tbitblocks=%d maxdepth=%d buckets=%d\n", + ix->bitblocks, ix->maxdepth, ix->buckets); + } + fprint(2, "fmtindex: %,d arenas, %,d index buckets, %,lld bytes storage\n", + n, ix->buckets, addr-IndexBase); + + ix->amap = amap; + ix->arenas = arenas; + ix->narenas = narenas; + + if(wbindex(ix) < 0) + fprint(2, "can't write back arena partition header for %s: %r\n", file); + + threadexitsall(0); +} blob - /dev/null blob + 28b88de61520c3807b095a720c05a3a0aed1569d (mode 644) --- /dev/null +++ src/cmd/venti/srv/fmtisect.c @@ -0,0 +1,83 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +void +usage(void) +{ + fprint(2, "usage: fmtisect [-Z] [-b blocksize] name file\n"); + threadexitsall(0); +} + +void +threadmain(int argc, char *argv[]) +{ + int vers; + ISect *is; + Part *part; + char *file, *name; + int blocksize, setsize, zero; + + ventifmtinstall(); + statsinit(); + + blocksize = 8 * 1024; + setsize = 512 * 1024; + zero = -1; + vers = ISectVersion2; + ARGBEGIN{ + case 'b': + blocksize = unittoull(ARGF()); + if(blocksize == ~0) + usage(); + if(blocksize > MaxDiskBlock){ + fprint(2, "block size too large, max %d\n", MaxDiskBlock); + threadexitsall("usage"); + } + break; + case '1': + vers = ISectVersion1; + break; + case 'Z': + zero = 0; + break; + default: + usage(); + break; + }ARGEND + + if(zero == -1){ + if(vers == ISectVersion1) + zero = 1; + else + zero = 0; + } + + if(argc != 2) + usage(); + + name = argv[0]; + file = argv[1]; + + if(nameok(name) < 0) + sysfatal("illegal name %s", name); + + part = initpart(file, ORDWR|ODIRECT); + if(part == nil) + sysfatal("can't open partition %s: %r", file); + + if(zero) + zeropart(part, blocksize); + + is = newisect(part, vers, name, blocksize, setsize); + if(is == nil) + sysfatal("can't initialize new index: %r"); + + fprint(2, "fmtisect %s: %,d buckets of %,d entries, %,d bytes for index map\n", + file, is->blocks, is->buckmax, setsize); + + if(wbisect(is) < 0) + fprint(2, "can't write back index section header for %s: %r\n", file); + + threadexitsall(0); +} blob - /dev/null blob + f35580ed48467e6f9d1cffa8210a82d491b0d731 (mode 644) --- /dev/null +++ src/cmd/venti/srv/fns.h @@ -0,0 +1,206 @@ +/* + * sorted by 4,/^$/|sort -bd +1 + */ +int addarena(Arena *name); +void addstat(int, int); +void addstat2(int, int, int, int); +ZBlock *alloczblock(u32int size, int zeroed, uint alignment); +Arena *amapitoa(Index *index, u64int a, u64int *aa); +u64int arenadirsize(Arena *arena, u32int clumps); +void arenaupdate(Arena *arena, u32int size, u8int *score); +void backsumarena(Arena *arena); +void binstats(long (*fn)(Stats *s0, Stats *s1, void*), void *arg, long t0, long t1, Statbin *bin, int nbin); +int bloominit(Bloom*, vlong, uchar*); +int bucklook(u8int*, int, u8int*, int); +u32int buildbucket(Index *ix, IEStream *ies, IBucket *ib, uint); +void checkdcache(void); +void checklumpcache(void); +int clumpinfoeq(ClumpInfo *c, ClumpInfo *d); +int clumpinfoeq(ClumpInfo *c, ClumpInfo *d); +u32int clumpmagic(Arena *arena, u64int aa); +uint countbits(uint n); +int delarena(Arena *arena); +void delaykickicache(void); +void delaykickround(Round*); +void delaykickroundproc(void*); +void dirtydblock(DBlock*, int); +AState diskstate(void); +void *emalloc(ulong); +void *erealloc(void *, ulong); +char *estrdup(char*); +void *ezmalloc(ulong); +Arena *findarena(char *name); +int flushciblocks(Arena *arena); +void flushdcache(void); +void flushicache(void); +void flushqueue(void); +void fmtzbinit(Fmt *f, ZBlock *b); +void freearena(Arena *arena); +void freearenapart(ArenaPart *ap, int freearenas); +void freeiestream(IEStream *ies); +void freeifile(IFile *f); +void freeisect(ISect *is); +void freeindex(Index *index); +void freepart(Part *part); +void freezblock(ZBlock *b); +DBlock *_getdblock(Part *part, u64int addr, int mode, int load); +DBlock *getdblock(Part *part, u64int addr, int mode); +u32int hashbits(u8int *score, int nbits); +int httpdinit(char *address, char *webroot); +int iaddrcmp(IAddr *ia1, IAddr *ia2); +IEntry* icachedirty(u32int, u32int, u64int); +void icacheclean(IEntry*); +int ientrycmp(const void *vie1, const void *vie2); +char *ifileline(IFile *f); +int ifilename(IFile *f, char *dst); +int ifileu32int(IFile *f, u32int *r); +int inbloomfilter(Bloom*, u8int*); +int indexsect(Index *ix, u8int *score); +int indexsect0(Index *ix, u32int buck); +Arena *initarena(Part *part, u64int base, u64int size, u32int blocksize); +ArenaPart *initarenapart(Part *part); +int initarenasum(void); +void initbloomfilter(Index*); +void initdcache(u32int mem); +void initicache(int bits, int depth); +void initicachewrite(void); +IEStream *initiestream(Part *part, u64int off, u64int clumps, u32int size); +ISect *initisect(Part *part); +Index *initindex(char *name, ISect **sects, int n); +void initlumpcache(u32int size, u32int nblocks); +int initlumpqueues(int nq); +Part* initpart(char *name, int mode); +void initround(Round*, char*, int); +int initventi(char *config, Config *conf); +void insertlump(Lump *lump, Packet *p); +int insertscore(u8int *score, IAddr *ia, int write); +void kickdcache(void); +void kickicache(void); +void kickround(Round*, int wait); +ZBlock *loadclump(Arena *arena, u64int aa, int blocks, Clump *cl, u8int *score, int verify); +DBlock *loadibucket(Index *index, u8int *score, ISect **is, u32int *buck, IBucket *ib); +int loadientry(Index *index, u8int *score, int type, IEntry *ie); +void logerr(int severity, char *fmt, ...); +Lump *lookuplump(u8int *score, int type); +int lookupscore(u8int *score, int type, IAddr *ia, int *rac); +int maparenas(AMap *am, Arena **arenas, int n, char *what); +void markbloomfilter(Bloom*, u8int*); +uint msec(void); +int namecmp(char *s, char *t); +void namecp(char *dst, char *src); +int nameok(char *name); +Arena *newarena(Part *part, u32int, char *name, u64int base, u64int size, u32int blocksize); +ArenaPart *newarenapart(Part *part, u32int blocksize, u32int tabsize); +ISect *newisect(Part *part, u32int vers, char *name, u32int blocksize, u32int tabsize); +Index *newindex(char *name, ISect **sects, int n); +u32int now(void); +int okamap(AMap *am, int n, u64int start, u64int stop, char *what); +int okibucket(IBucket*, ISect*); +int outputamap(Fmt *f, AMap *am, int n); +int outputindex(Fmt *f, Index *ix); +int packarena(Arena *arena, u8int *buf); +int packarenahead(ArenaHead *head, u8int *buf); +int packarenapart(ArenaPart *as, u8int *buf); +void packbloomhead(Bloom*, u8int*); +int packclump(Clump *c, u8int *buf, u32int); +void packclumpinfo(ClumpInfo *ci, u8int *buf); +void packibucket(IBucket *b, u8int *buf, u32int magic); +void packientry(IEntry *i, u8int *buf); +int packisect(ISect *is, u8int *buf); +void packmagic(u32int magic, u8int *buf); +ZBlock *packet2zblock(Packet *p, u32int size); +int parseamap(IFile *f, AMapN *amn); +int parseindex(IFile *f, Index *ix); +void partblocksize(Part *part, u32int blocksize); +int partifile(IFile *f, Part *part, u64int start, u32int size); +void printarenapart(int fd, ArenaPart *ap); +void printarena(int fd, Arena *arena); +void printindex(int fd, Index *ix); +void printstats(void); +void putdblock(DBlock *b); +void putlump(Lump *b); +int queuewrite(Lump *b, Packet *p, int creator, uint ms); +u32int readarena(Arena *arena, u64int aa, u8int *buf, long n); +int readarenamap(AMapN *amn, Part *part, u64int base, u32int size); +Bloom *readbloom(Part*); +int readclumpinfo(Arena *arena, int clump, ClumpInfo *ci); +int readclumpinfos(Arena *arena, int clump, ClumpInfo *cis, int n); +ZBlock *readfile(char *name); +int readifile(IFile *f, char *name); +Packet *readlump(u8int *score, int type, u32int size, int *cached); +int readpart(Part *part, u64int addr, u8int *buf, u32int n); +int runconfig(char *config, Config*); +int scorecmp(u8int *, u8int *); +void scoremem(u8int *score, u8int *buf, int size); +void setatailstate(AState*); +void setdcachestate(AState*); +void seterr(int severity, char *fmt, ...); +void setstat(int, long); +void settrace(char *type); +u64int sortrawientries(Index *ix, Part *tmp, u64int *tmpoff, Bloom *bloom); +void startbloomproc(Bloom*); +Memimage* statgraph(Graph *g); +void statsinit(void); +int storeclump(Index *index, ZBlock *b, u8int *score, int type, u32int creator, IAddr *ia); +int storeientry(Index *index, IEntry *m); +int strscore(char *s, u8int *score); +int stru32int(char *s, u32int *r); +int stru64int(char *s, u64int *r); +void sumarena(Arena *arena); +int syncarena(Arena *arena, u64int start, u32int n, int zok, int fix); +int syncarenaindex(Index *ix, Arena *arena, u32int clump, u64int a, int fix, int *pflush, int check); +int syncindex(Index *ix, int fix, int mustflushicache, int check); +void trace(char *type, char*, ...); +void traceinit(void); +int u64log2(u64int v); +u64int unittoull(char *s); +int unpackarena(Arena *arena, u8int *buf); +int unpackarenahead(ArenaHead *head, u8int *buf); +int unpackarenapart(ArenaPart *as, u8int *buf); +int unpackbloomhead(Bloom*, u8int*); +int unpackclump(Clump *c, u8int *buf, u32int); +void unpackclumpinfo(ClumpInfo *ci, u8int *buf); +void unpackibucket(IBucket *b, u8int *buf, u32int magic); +void unpackientry(IEntry *i, u8int *buf); +int unpackisect(ISect *is, u8int *buf); +u32int unpackmagic(u8int *buf); +void ventifmtinstall(void); +void vtloghdump(Hio*, VtLog*); +void vtloghlist(Hio*); +int vtproc(void(*)(void*), void*); +int vttypevalid(int type); +void waitforkick(Round*); +int wbarena(Arena *arena); +int wbarenahead(Arena *arena); +int wbarenamap(AMap *am, int n, Part *part, u64int base, u64int size); +int wbarenapart(ArenaPart *ap); +void wbbloomhead(Bloom*); +int wbisect(ISect *is); +int wbindex(Index *ix); +int whackblock(u8int *dst, u8int *src, int ssize); +u64int writeaclump(Arena *a, Clump *c, u8int *clbuf, u64int, u64int*); +u32int writearena(Arena *arena, u64int aa, u8int *clbuf, u32int n); +int writebloom(Bloom*); +int writeclumpinfo(Arena *arean, int clump, ClumpInfo *ci); +int writepng(Hio*, Memimage*); +u64int writeiclump(Index *ix, Clump *c, u8int *clbuf, u64int*); +int writelump(Packet *p, u8int *score, int type, u32int creator, uint ms); +int writepart(Part *part, u64int addr, u8int *buf, u32int n); +int writeqlump(Lump *u, Packet *p, int creator, uint ms); +Packet *zblock2packet(ZBlock *zb, u32int size); +void zeropart(Part *part, int blocksize); + +/* +#pragma varargck argpos sysfatal 1 +#pragma varargck argpos logerr 2 +#pragma varargck argpos SetErr 2 +*/ + +#define scorecmp(h1,h2) memcmp((h1),(h2),VtScoreSize) +#define scorecp(h1,h2) memmove((h1),(h2),VtScoreSize) + +#define MK(t) ((t*)emalloc(sizeof(t))) +#define MKZ(t) ((t*)ezmalloc(sizeof(t))) +#define MKN(t,n) ((t*)emalloc((n)*sizeof(t))) +#define MKNZ(t,n) ((t*)ezmalloc((n)*sizeof(t))) +#define MKNA(t,at,n) ((t*)emalloc(sizeof(t) + (n)*sizeof(at))) blob - /dev/null blob + 157b82db18931519306a5edf5ba4523ae437e065 (mode 644) --- /dev/null +++ src/cmd/venti/srv/graph.c @@ -0,0 +1,202 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +enum +{ + Top = 1, + Bottom = 1, + Left = 40, + Right = 0, + MinWidth = Left+Right+2, + MinHeight = Top+Bottom+2, + DefaultWidth = Left+Right+500, + DefaultHeight = Top+Bottom+40 +}; + +QLock memdrawlock; +static Memsubfont *smallfont; +static Memimage *black; +static Memimage *blue; +static Memimage *red; +static Memimage *lofill[6]; +static Memimage *hifill[6]; +static Memimage *grid; + +static ulong fill[] = { + 0xFFAAAAFF, 0xBB5D5DFF, /* peach */ + DPalegreygreen, DPurpleblue, /* aqua */ + DDarkyellow, DYellowgreen, /* yellow */ + DMedgreen, DDarkgreen, /* green */ + 0x00AAFFFF, 0x0088CCFF, /* blue */ + 0xCCCCCCFF, 0x888888FF, /* grey */ +}; + +Memimage* +allocrepl(ulong color) +{ + Memimage *m; + + m = allocmemimage(Rect(0,0,1,1), RGB24); + memfillcolor(m, color); + m->flags |= Frepl; + m->clipr = Rect(-1000000, -1000000, 1000000, 1000000); + return m; +} + +static void +ginit(void) +{ + static int first = 1; + int i; + + if(!first) + return; + + first = 0; + memimageinit(); + smallfont = openmemsubfont(unsharp("#9/font/lucidasans/lstr.10")); + black = memblack; + blue = allocrepl(DBlue); + red = allocrepl(DRed); + grid = allocrepl(0x77777777); + for(i=0; i= valmax) + val = valmax; + if(valmax == valmin) + valmax++; + return ptmin + (vlong)(val-valmin)*(ptmax-ptmin)/(valmax-valmin); +} + +Memimage* +statgraph(Graph *g) +{ + int i, lastlo, nbin, x, lo, hi, min, max, first; + Memimage *m; + Rectangle r; + Statbin *b, bin[2000]; /* 32 kB, but whack is worse */ + + needstack(8192); /* double check that bin didn't kill us */ + + if(g->wid <= MinWidth) + g->wid = DefaultWidth; + if(g->ht <= MinHeight) + g->ht = DefaultHeight; + if(g->wid > nelem(bin)) + g->wid = nelem(bin); + if(g->fill < 0) + g->fill = ((uint)g->arg>>8)%nelem(lofill); + if(g->fill > nelem(lofill)) + g->fill %= nelem(lofill); + + nbin = g->wid - (Left+Right); + binstats(g->fn, g->arg, g->t0, g->t1, bin, nbin); + + /* + * compute bounds + */ + min = g->min; + max = g->max; + if(min < 0 || max <= min){ + min = max = 0; + first = 1; + for(i=0; insamp == 0) + continue; + if(first || b->min < min) + min = b->min; + if(first || b->max > max) + max = b->max; + first = 0; + } + } + + qlock(&memdrawlock); + ginit(); + if(smallfont==nil || black==nil || blue==nil || red==nil || hifill==nil || lofill==nil){ + werrstr("graphics initialization failed"); + qunlock(&memdrawlock); + return nil; + } + + /* fresh image */ + m = allocmemimage(Rect(0,0,g->wid,g->ht), ABGR32); + if(m == nil){ + qunlock(&memdrawlock); + return nil; + } + r = Rect(Left, Top, g->wid-Right, g->ht-Bottom); + memfillcolor(m, DTransparent); + + /* x axis */ + memimagedraw(m, Rect(r.min.x, r.max.y, r.max.x, r.max.y+1), black, ZP, memopaque, ZP, S); + + /* y labels */ + drawlabel(m, r.min, max); + if(min != 0) + drawlabel(m, Pt(r.min.x, r.max.y-smallfont->height), min); + + /* actual data */ + lastlo = -1; + for(i=0; insamp == 0) + continue; + lo = scalept(b->min, min, max, r.max.y, r.min.y); + hi = scalept(b->max, min, max, r.max.y, r.min.y); + x = r.min.x+i; + hi-=2; + if(0) + if(lastlo != -1){ + if(lastlo < lo) + memimagedraw(m, Rect(x-1, lastlo, x, lo), hifill[g->fill], ZP, memopaque, ZP, S); + else if(lastlo > lo) + memimagedraw(m, Rect(x-1, lo, x, lastlo), hifill[g->fill], ZP, memopaque, ZP, S); + } + memimagedraw(m, Rect(x, hi, x+1,lo), hifill[g->fill], ZP, memopaque, ZP, S); + memimagedraw(m, Rect(x, lo, x+1, r.max.y), lofill[g->fill], ZP, memopaque, ZP, S); + lastlo = lo; + } + + if(bin[nbin-1].nsamp) + drawlabel(m, Pt(r.max.x, r.min.y+(Dy(r)-smallfont->height)/2), bin[nbin-1].avg); + qunlock(&memdrawlock); + return m; +} blob - /dev/null blob + 5f1a00e16dce1430eb1f94a307b020e3d42db077 (mode 644) --- /dev/null +++ src/cmd/venti/srv/httpd.c @@ -0,0 +1,988 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" +#include "xml.h" + +typedef struct HttpObj HttpObj; +extern QLock memdrawlock; + +enum +{ + ObjNameSize = 64, + MaxObjs = 16 +}; + +struct HttpObj +{ + char name[ObjNameSize]; + int (*f)(HConnect*); +}; + +static HttpObj objs[MaxObjs]; + +static char *webroot; + +static void listenproc(void*); +static int estats(HConnect *c); +static int dindex(HConnect *c); +static int xindex(HConnect *c); +static int xlog(HConnect *c); +static int sindex(HConnect *c); +static int hicacheflush(HConnect *c); +static int hdcacheflush(HConnect *c); +static int notfound(HConnect *c); +static int httpdobj(char *name, int (*f)(HConnect*)); +static int xgraph(HConnect *c); +static int xset(HConnect *c); +static int fromwebdir(HConnect *c); + +int +httpdinit(char *address, char *dir) +{ + fmtinstall('D', hdatefmt); +/* fmtinstall('H', httpfmt); */ + fmtinstall('U', hurlfmt); + + if(address == nil) + address = "tcp!*!http"; + webroot = dir; + + httpdobj("/stats", estats); + httpdobj("/index", dindex); + httpdobj("/storage", sindex); + httpdobj("/xindex", xindex); + httpdobj("/flushicache", hicacheflush); + httpdobj("/flushdcache", hdcacheflush); + httpdobj("/graph/", xgraph); + httpdobj("/set/", xset); + httpdobj("/log", xlog); + httpdobj("/log/", xlog); + + if(vtproc(listenproc, address) < 0) + return -1; + return 0; +} + +static int +httpdobj(char *name, int (*f)(HConnect*)) +{ + int i; + + if(name == nil || strlen(name) >= ObjNameSize) + return -1; + for(i = 0; i < MaxObjs; i++){ + if(objs[i].name[0] == '\0'){ + strcpy(objs[i].name, name); + objs[i].f = f; + return 0; + } + if(strcmp(objs[i].name, name) == 0) + return -1; + } + return -1; +} + +static HConnect* +mkconnect(void) +{ + HConnect *c; + + c = mallocz(sizeof(HConnect), 1); + if(c == nil) + sysfatal("out of memory"); + c->replog = nil; + c->hpos = c->header; + c->hstop = c->header; + return c; +} + +void httpproc(void*); + +static void +listenproc(void *vaddress) +{ + HConnect *c; + char *address, ndir[NETPATHLEN], dir[NETPATHLEN]; + int ctl, nctl, data; + +//sleep(1000); /* let strace find us */ + + address = vaddress; + ctl = announce(address, dir); + if(ctl < 0){ + fprint(2, "venti: httpd can't announce on %s: %r\n", address); + return; + } + + if(0) print("announce ctl %d dir %s\n", ctl, dir); + for(;;){ + /* + * wait for a call (or an error) + */ + nctl = listen(dir, ndir); + if(0) print("httpd listen %d %s...\n", nctl, ndir); + if(nctl < 0){ + fprint(2, "venti: httpd can't listen on %s: %r\n", address); + return; + } + + data = accept(ctl, ndir); + if(0) print("httpd accept %d...\n", data); + if(data < 0){ + fprint(2, "venti: httpd accept: %r\n"); + close(nctl); + continue; + } + if(0) print("httpd close nctl %d\n", nctl); + close(nctl); + c = mkconnect(); + hinit(&c->hin, data, Hread); + hinit(&c->hout, data, Hwrite); + vtproc(httpproc, c); + } +} + +void +httpproc(void *v) +{ + HConnect *c; + int ok, i, n; + +//sleep(1000); /* let strace find us */ + c = v; + + for(;;){ + /* + * No timeout because the signal appears to hit every + * proc, not just us. + */ + if(hparsereq(c, 0) < 0) + break; + + ok = -1; + for(i = 0; i < MaxObjs && objs[i].name[0]; i++){ + n = strlen(objs[i].name); + if((objs[i].name[n-1] == '/' && strncmp(c->req.uri, objs[i].name, n) == 0) + || (objs[i].name[n-1] != '/' && strcmp(c->req.uri, objs[i].name) == 0)){ + ok = (*objs[i].f)(c); + goto found; + } + } + ok = fromwebdir(c); + found: + if(c->head.closeit) + ok = -1; + hreqcleanup(c); + + if(ok < 0) + break; + } + hreqcleanup(c); + close(c->hin.fd); + free(c); +} + +static int +percent(long v, long total) +{ + if(total == 0) + total = 1; + if(v < 1000*1000) + return (v * 100) / total; + total /= 100; + if(total == 0) + total = 1; + return v / total; +} + +static int +preq(HConnect *c) +{ + if(hparseheaders(c, 0) < 0) + return -1; + if(strcmp(c->req.meth, "GET") != 0 + && strcmp(c->req.meth, "HEAD") != 0) + return hunallowed(c, "GET, HEAD"); + if(c->head.expectother || c->head.expectcont) + return hfail(c, HExpectFail, nil); + return 0; +} + +static int +preqtype(HConnect *c, char *type) +{ + Hio *hout; + int r; + + r = preq(c); + if(r < 0) + return r; + + hout = &c->hout; + if(c->req.vermaj){ + hokheaders(c); + hprint(hout, "Content-type: %s\r\n", type); + if(http11(c)) + hprint(hout, "Transfer-Encoding: chunked\r\n"); + hprint(hout, "\r\n"); + } + + if(http11(c)) + hxferenc(hout, 1); + else + c->head.closeit = 1; + return 0; +} + +static int +preqtext(HConnect *c) +{ + return preqtype(c, "text/plain"); +} + +static int +notfound(HConnect *c) +{ + int r; + + r = preq(c); + if(r < 0) + return r; + return hfail(c, HNotFound, c->req.uri); +} + +struct { + char *ext; + char *type; +} exttab[] = { + ".html", "text/html", + ".txt", "text/plain", + ".xml", "text/xml", + ".png", "image/png", + ".gif", "image/gif", + 0 +}; + +static int +fromwebdir(HConnect *c) +{ + char buf[4096], *p, *ext, *type; + int i, fd, n, defaulted; + Dir *d; + + if(webroot == nil || strstr(c->req.uri, "..")) + return notfound(c); + snprint(buf, sizeof buf-20, "%s/%s", webroot, c->req.uri+1); + defaulted = 0; +reopen: + if((fd = open(buf, OREAD)) < 0) + return notfound(c); + d = dirfstat(fd); + if(d == nil){ + close(fd); + return notfound(c); + } + if(d->mode&DMDIR){ + if(!defaulted){ + defaulted = 1; + strcat(buf, "/index.html"); + free(d); + close(fd); + goto reopen; + } + free(d); + return notfound(c); + } + free(d); + p = buf+strlen(buf); + type = "application/octet-stream"; + for(i=0; exttab[i].ext; i++){ + ext = exttab[i].ext; + if(p-strlen(ext) >= buf && strcmp(p-strlen(ext), ext) == 0){ + type = exttab[i].type; + break; + } + } + if(preqtype(c, type) < 0){ + close(fd); + return 0; + } + while((n = read(fd, buf, sizeof buf)) > 0) + if(hwrite(&c->hout, buf, n) < 0) + break; + close(fd); + hflush(&c->hout); + return 0; +} + +static struct +{ + char *name; + int *p; +} namedints[] = +{ + "compress", &compressblocks, + "devnull", &writestodevnull, + "logging", &ventilogging, + "stats", &collectstats, + "icachesleeptime", &icachesleeptime, + "arenasumsleeptime", &arenasumsleeptime, + 0 +}; + +static int +xset(HConnect *c) +{ + int i, nf, r; + char *f[10], *s; + + s = estrdup(c->req.uri); + nf = getfields(s+strlen("/set/"), f, nelem(f), 1, "/"); + + if(nf < 1) + return notfound(c); + for(i=0; namedints[i].name; i++){ + if(strcmp(f[0], namedints[i].name) == 0){ + if(nf >= 2) + *namedints[i].p = atoi(f[1]); + r = preqtext(c); + if(r < 0) + return r; + hprint(&c->hout, "%s = %d\n", f[0], *namedints[i].p); + hflush(&c->hout); + return 0; + } + } + return notfound(c); +} + +static int +estats(HConnect *c) +{ + Hio *hout; + int r; + + r = preqtext(c); + if(r < 0) + return r; + + + hout = &c->hout; +/* + hprint(hout, "lump writes=%,ld\n", stats.lumpwrites); + hprint(hout, "lump reads=%,ld\n", stats.lumpreads); + hprint(hout, "lump cache read hits=%,ld\n", stats.lumphit); + hprint(hout, "lump cache read misses=%,ld\n", stats.lumpmiss); + + hprint(hout, "clump disk writes=%,ld\n", stats.clumpwrites); + hprint(hout, "clump disk bytes written=%,lld\n", stats.clumpbwrites); + hprint(hout, "clump disk bytes compressed=%,lld\n", stats.clumpbcomp); + hprint(hout, "clump disk reads=%,ld\n", stats.clumpreads); + hprint(hout, "clump disk bytes read=%,lld\n", stats.clumpbreads); + hprint(hout, "clump disk bytes uncompressed=%,lld\n", stats.clumpbuncomp); + + hprint(hout, "clump directory disk writes=%,ld\n", stats.ciwrites); + hprint(hout, "clump directory disk reads=%,ld\n", stats.cireads); + + hprint(hout, "index disk writes=%,ld\n", stats.indexwrites); + hprint(hout, "index disk reads=%,ld\n", stats.indexreads); + hprint(hout, "index disk bloom filter hits=%,ld %d%% falsemisses=%,ld %d%%\n", + stats.indexbloomhits, + percent(stats.indexbloomhits, stats.indexreads), + stats.indexbloomfalsemisses, + percent(stats.indexbloomfalsemisses, stats.indexreads)); + hprint(hout, "bloom filter bits=%,ld of %,ld %d%%\n", + stats.bloomones, stats.bloombits, percent(stats.bloomones, stats.bloombits)); + hprint(hout, "index disk reads for modify=%,ld\n", stats.indexwreads); + hprint(hout, "index disk reads for allocation=%,ld\n", stats.indexareads); + hprint(hout, "index block splits=%,ld\n", stats.indexsplits); + + hprint(hout, "index cache lookups=%,ld\n", stats.iclookups); + hprint(hout, "index cache hits=%,ld %d%%\n", stats.ichits, + percent(stats.ichits, stats.iclookups)); + hprint(hout, "index cache fills=%,ld %d%%\n", stats.icfills, + percent(stats.icfills, stats.iclookups)); + hprint(hout, "index cache inserts=%,ld\n", stats.icinserts); + + hprint(hout, "disk cache hits=%,ld\n", stats.pchit); + hprint(hout, "disk cache misses=%,ld\n", stats.pcmiss); + hprint(hout, "disk cache reads=%,ld\n", stats.pcreads); + hprint(hout, "disk cache bytes read=%,lld\n", stats.pcbreads); + + hprint(hout, "disk cache writes=%,ld\n", stats.dirtydblocks); + hprint(hout, "disk cache writes absorbed=%,ld %d%%\n", stats.absorbedwrites, + percent(stats.absorbedwrites, stats.dirtydblocks)); + + hprint(hout, "disk cache flushes=%,ld\n", stats.dcacheflushes); + hprint(hout, "disk cache flush writes=%,ld (%,ld per flush)\n", + stats.dcacheflushwrites, + stats.dcacheflushwrites/(stats.dcacheflushes ? stats.dcacheflushes : 1)); + + hprint(hout, "disk writes=%,ld\n", stats.diskwrites); + hprint(hout, "disk bytes written=%,lld\n", stats.diskbwrites); + hprint(hout, "disk reads=%,ld\n", stats.diskreads); + hprint(hout, "disk bytes read=%,lld\n", stats.diskbreads); +*/ + + hflush(hout); + return 0; +} + +static int +sindex(HConnect *c) +{ + Hio *hout; + Index *ix; + Arena *arena; + vlong clumps, cclumps, uncsize, used, size; + int i, r, active; + + r = preqtext(c); + if(r < 0) + return r; + hout = &c->hout; + + ix = mainindex; + + hprint(hout, "index=%s\n", ix->name); + + active = 0; + clumps = 0; + cclumps = 0; + uncsize = 0; + used = 0; + size = 0; + for(i = 0; i < ix->narenas; i++){ + arena = ix->arenas[i]; + if(arena != nil && arena->memstats.clumps != 0){ + active++; + clumps += arena->memstats.clumps; + cclumps += arena->memstats.cclumps; + uncsize += arena->memstats.uncsize; + used += arena->memstats.used; + } + size += arena->size; + } + hprint(hout, "total arenas=%,d active=%,d\n", ix->narenas, active); + hprint(hout, "total space=%,lld used=%,lld\n", size, used + clumps * ClumpInfoSize); + hprint(hout, "clumps=%,lld compressed clumps=%,lld data=%,lld compressed data=%,lld\n", + clumps, cclumps, uncsize, used - clumps * ClumpSize); + hflush(hout); + return 0; +} + +static void +darena(Hio *hout, Arena *arena) +{ + hprint(hout, "arena='%s' on %s at [%lld,%lld)\n\tversion=%d created=%d modified=%d", + arena->name, arena->part->name, arena->base, arena->base + arena->size + 2 * arena->blocksize, + arena->version, arena->ctime, arena->wtime); + if(arena->memstats.sealed) + hprint(hout, " mem=sealed"); + if(arena->diskstats.sealed) + hprint(hout, " disk=sealed"); + hprint(hout, "\n"); + if(scorecmp(zeroscore, arena->score) != 0) + hprint(hout, "\tscore=%V\n", arena->score); + + hprint(hout, "\tmem: clumps=%d compressed clumps=%d data=%,lld compressed data=%,lld storage=%,lld\n", + arena->memstats.clumps, arena->memstats.cclumps, arena->memstats.uncsize, + arena->memstats.used - arena->memstats.clumps * ClumpSize, + arena->memstats.used + arena->memstats.clumps * ClumpInfoSize); + hprint(hout, "\tdisk: clumps=%d compressed clumps=%d data=%,lld compressed data=%,lld storage=%,lld\n", + arena->diskstats.clumps, arena->diskstats.cclumps, arena->diskstats.uncsize, + arena->diskstats.used - arena->diskstats.clumps * ClumpSize, + arena->diskstats.used + arena->diskstats.clumps * ClumpInfoSize); +} + +static int +hicacheflush(HConnect *c) +{ + Hio *hout; + int r; + + r = preqtext(c); + if(r < 0) + return r; + hout = &c->hout; + + flushicache(); + hprint(hout, "flushed icache\n"); + hflush(hout); + return 0; +} + +static int +hdcacheflush(HConnect *c) +{ + Hio *hout; + int r; + + r = preqtext(c); + if(r < 0) + return r; + hout = &c->hout; + + flushdcache(); + hprint(hout, "flushed dcache\n"); + hflush(hout); + return 0; +} + +static int +dindex(HConnect *c) +{ + Hio *hout; + Index *ix; + int i, r; + + r = preqtext(c); + if(r < 0) + return r; + hout = &c->hout; + + + ix = mainindex; + hprint(hout, "index=%s version=%d blocksize=%d tabsize=%d\n", + ix->name, ix->version, ix->blocksize, ix->tabsize); + hprint(hout, "\tbuckets=%d div=%d\n", ix->buckets, ix->div); + for(i = 0; i < ix->nsects; i++) + hprint(hout, "\tsect=%s for buckets [%lld,%lld) buckmax=%d\n", ix->smap[i].name, ix->smap[i].start, ix->smap[i].stop, ix->sects[i]->buckmax); + for(i = 0; i < ix->narenas; i++){ + if(ix->arenas[i] != nil && ix->arenas[i]->memstats.clumps != 0){ + hprint(hout, "arena=%s at index [%lld,%lld)\n\t", ix->amap[i].name, ix->amap[i].start, ix->amap[i].stop); + darena(hout, ix->arenas[i]); + } + } + hflush(hout); + return 0; +} + +typedef struct Arg Arg; +struct Arg +{ + int index; + int index2; +}; + +static long +rawgraph(Stats *s, Stats *t, void *va) +{ + Arg *a; + + a = va; + return t->n[a->index]; +} + +static long +diffgraph(Stats *s, Stats *t, void *va) +{ + Arg *a; + + a = va; + return t->n[a->index] - s->n[a->index]; +} + +static long +pctgraph(Stats *s, Stats *t, void *va) +{ + Arg *a; + + a = va; + return percent(t->n[a->index], t->n[a->index2]); +} + +static long +pctdiffgraph(Stats *s, Stats *t, void *va) +{ + Arg *a; + + a = va; + return percent(t->n[a->index]-s->n[a->index], t->n[a->index2]-s->n[a->index2]); +} + +static long +netbw(Stats *s) +{ + ulong *n; + + n = s->n; + return n[StatRpcReadBytes]+n[StatRpcWriteBytes]; /* not exactly right */ +} + +static long +diskbw(Stats *s) +{ + ulong *n; + + n = s->n; + return n[StatApartReadBytes]+n[StatApartWriteBytes] + + n[StatIsectReadBytes]+n[StatIsectWriteBytes] + + n[StatSumReadBytes]; +} + +static long +iobw(Stats *s) +{ + return netbw(s)+diskbw(s); +} + +static long +diskgraph(Stats *s, Stats *t, void *va) +{ + USED(va); + return diskbw(t)-diskbw(s); +} + +static long +netgraph(Stats *s, Stats *t, void *va) +{ + USED(va); + return netbw(t)-netbw(s); +} + +static long +iograph(Stats *s, Stats *t, void *va) +{ + USED(va); + return iobw(t)-iobw(s); +} + + +static char* graphname[] = +{ + "rpctotal", + "rpcread", + "rpcreadok", + "rpcreadfail", + "rpcreadbyte", + "rpcreadtime", + "rpcreadcached", + "rpcreadcachedtime", + "rpcreaduncached", + "rpcreaduncachedtime", + "rpcwrite", + "rpcwritenew", + "rpcwriteold", + "rpcwritefail", + "rpcwritebyte", + "rpcwritetime", + "rpcwritenewtime", + "rpcwriteoldtime", + + "lcachehit", + "lcachemiss", + "lcachelookup", + "lcachewrite", + "lcachesize", + "lcachestall", + "lcachelookuptime", + + "dcachehit", + "dcachemiss", + "dcachelookup", + "dcacheread", + "dcachewrite", + "dcachedirty", + "dcachesize", + "dcacheflush", + "dcachestall", + "dcachelookuptime", + + "dblockstall", + "lumpstall", + + "icachehit", + "icachemiss", + "icachelookup", + "icachewrite", + "icachefill", + "icacheprefetch", + "icachedirty", + "icachesize", + "icacheflush", + "icachestall", + "icachelookuptime", + + "bloomhit", + "bloommiss", + "bloomfalsemiss", + "bloomlookup", + "bloomones", + "bloombits", + "bloomlookuptime", + + "apartread", + "apartreadbyte", + "apartwrite", + "apartwritebyte", + + "isectread", + "isectreadbyte", + "isectwrite", + "isectwritebyte", + + "sumread", + "sumreadbyte", +}; + +static int +findname(char *s) +{ + int i; + + for(i=0; ifn, g->arg, g->t0, g->t1, bin, nbin); + + hprint(io, "stats\n\n"); + for(i=0; insamp, b->min, b->max, b->avg); + } +} + +static int +xgraph(HConnect *c) +{ + char *f[20], *s; + Hio *hout; + Memimage *m; + int i, nf, dotext; + Graph g; + Arg arg; + + s = estrdup(c->req.uri); +if(0) fprint(2, "graph %s\n" ,s); + memset(&g, 0, sizeof g); + nf = getfields(s+strlen("/graph/"), f, nelem(f), 1, "/"); + if(nf < 1) + goto notfound; + if((arg.index = findname(f[0])) == -1 && strcmp(f[0], "*") != 0) + goto notfound; + g.arg = &arg; + g.t0 = -120; + g.t1 = 0; + g.min = -1; + g.max = -1; + g.fn = rawgraph; + g.wid = -1; + g.ht = -1; + dotext = 0; + g.fill = -1; + for(i=1; ihout, &g); + hflush(&c->hout); + return 0; + } + + m = statgraph(&g); + if(m == nil) + goto notfound; + + if(preqtype(c, "image/png") < 0) + return -1; + hout = &c->hout; + writepng(hout, m); + qlock(&memdrawlock); + freememimage(m); + qunlock(&memdrawlock); + hflush(hout); + free(s); + return 0; + +notfound: + free(s); + return notfound(c); +} + +static int +xloglist(HConnect *c) +{ + if(preqtype(c, "text/html") < 0) + return -1; + vtloghlist(&c->hout); + hflush(&c->hout); + return 0; +} + +static int +xlog(HConnect *c) +{ + char *name; + VtLog *l; + + if(strcmp(c->req.uri, "/log") == 0 || strcmp(c->req.uri, "/log/") == 0) + return xloglist(c); + if(strncmp(c->req.uri, "/log/", 5) != 0) + return notfound(c); + name = c->req.uri + strlen("/log/"); + l = vtlogopen(name, 0); + if(l == nil) + return notfound(c); + if(preqtype(c, "text/html") < 0){ + vtlogclose(l); + return -1; + } + vtloghdump(&c->hout, l); + vtlogclose(l); + hflush(&c->hout); + return 0; +} + +static int +xindex(HConnect *c) +{ + if(preqtype(c, "text/xml") < 0) + return -1; + xmlindex(&c->hout, mainindex, "index", 0); + hflush(&c->hout); + return 0; +} + +void +xmlindent(Hio *hout, int indent) +{ + int i; + + for(i = 0; i < indent; i++) + hputc(hout, '\t'); +} + +void +xmlaname(Hio *hout, char *v, char *tag) +{ + hprint(hout, " %s=\"%s\"", tag, v); +} + +void +xmlscore(Hio *hout, u8int *v, char *tag) +{ + if(scorecmp(zeroscore, v) == 0) + return; + hprint(hout, " %s=\"%V\"", tag, v); +} + +void +xmlsealed(Hio *hout, int v, char *tag) +{ + if(!v) + return; + hprint(hout, " %s=\"yes\"", tag); +} + +void +xmlu32int(Hio *hout, u32int v, char *tag) +{ + hprint(hout, " %s=\"%ud\"", tag, v); +} + +void +xmlu64int(Hio *hout, u64int v, char *tag) +{ + hprint(hout, " %s=\"%llud\"", tag, v); +} + +void +vtloghdump(Hio *h, VtLog *l) +{ + int i; + VtLogChunk *c; + char *name; + + name = l ? l->name : "<nil>"; + +fprint(2, "hdump xfer %d\n", h->xferenc); + hprint(h, "\n"); + hprint(h, "Venti Server Log: %s\n", name); + hprint(h, "\n"); + hprint(h, "Venti Server Log: %s\n

\n", name); + + if(l){ + c = l->w; + for(i=0; inchunk; i++){ + if(++c == l->chunk+l->nchunk) + c = l->chunk; + hwrite(h, c->p, c->wp-c->p); + } + } + hprint(h, "\n"); +} + +static int +strpcmp(const void *va, const void *vb) +{ + return strcmp(*(char**)va, *(char**)vb); +} + +void +vtloghlist(Hio *h) +{ + char **p; + int i, n; + + hprint(h, "\n"); + hprint(h, "Venti Server Logs\n"); + hprint(h, "\n"); + hprint(h, "Venti Server Logs\n

\n"); + + p = vtlognames(&n); + qsort(p, n, sizeof(p[0]), strpcmp); + for(i=0; i%s
\n", p[i], p[i]); + vtfree(p); + hprint(h, "\n"); +} blob - /dev/null blob + 46d411e584473e1f20cf665ca5751b2fa032cf51 (mode 644) --- /dev/null +++ src/cmd/venti/srv/icache.c @@ -0,0 +1,348 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +typedef struct ICache ICache; +struct ICache +{ + QLock lock; /* locks hash table & all associated data */ + Rendez full; + IEntry **heads; /* heads of all the hash chains */ + int bits; /* bits to use for indexing heads */ + u32int size; /* number of heads; == 1 << bits, should be < entries */ + IEntry *base; /* all allocated hash table entries */ + u32int entries; /* elements in base */ + IEntry *dirty; /* chain of dirty elements */ + u32int ndirty; + u32int maxdirty; + u32int unused; /* index of first unused element in base */ + u32int stolen; /* last head from which an element was stolen */ + + Arena *last[4]; + Arena *lastload; + int nlast; +}; + +static ICache icache; + +static IEntry *icachealloc(IAddr *ia, u8int *score); + +/* + * bits is the number of bits in the icache hash table + * depth is the average depth + * memory usage is about (1<>= (32 - bits); + return v; +} + +static void +loadarenaclumps(Arena *arena, u64int aa) +{ + ulong i; + ClumpInfo ci; + IAddr ia; + +fprint(2, "seed index cache with arena @%llud, (map %llud), %d clumps\n", arena->base, aa, arena->memstats.clumps); + for(i=0; imemstats.clumps; i++){ + if(readclumpinfo(arena, i, &ci) < 0) + break; + ia.type = ci.type; + ia.size = ci.uncsize; + ia.blocks = (ci.size + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog; + ia.addr = aa; + aa += ClumpSize + ci.size; + if(ia.type != VtCorruptType) + insertscore(ci.score, &ia, 0); + } +} + +/* +ZZZ need to think about evicting the correct IEntry, +and writing back the wtime. + * look up data score in the index cache + * if this fails, pull it in from the disk index table, if it exists. + * + * must be called with the lump for this score locked + */ +int +lookupscore(u8int *score, int type, IAddr *ia, int *rac) +{ + IEntry d, *ie, *last; + u32int h; + u64int aa; + Arena *load; + int i; + uint ms; + + load = nil; + aa = 0; + ms = msec(); + + trace(TraceLump, "lookupscore %V.%d", score, type); + + qlock(&icache.lock); + h = hashbits(score, icache.bits); + last = nil; + for(ie = icache.heads[h]; ie != nil; ie = ie->next){ + if(ie->ia.type == type && scorecmp(ie->score, score)==0){ + if(last != nil) + last->next = ie->next; + else + icache.heads[h] = ie->next; + addstat(StatIcacheHit, 1); + ie->rac = 1; + trace(TraceLump, "lookupscore incache"); + goto found; + } + last = ie; + } + addstat(StatIcacheMiss, 1); + qunlock(&icache.lock); + + if(loadientry(mainindex, score, type, &d) < 0){ + ms = msec() - ms; + addstat2(StatIcacheRead, 1, StatIcacheReadTime, ms); + return -1; + } + + addstat(StatIcacheFill, 1); + + trace(TraceLump, "lookupscore loaded"); + + /* + * no one else can load an entry for this score, + * since we have the overall score lock. + */ + qlock(&icache.lock); + + /* + * If we notice that all the hits are coming from one arena, + * load the table of contents for that arena into the cache. + */ + ie = icachealloc(&d.ia, score); + icache.last[icache.nlast++%nelem(icache.last)] = amapitoa(mainindex, ie->ia.addr, &aa); + aa = ie->ia.addr - aa; /* compute base addr of arena */ + for(i=0; inext = icache.heads[h]; + icache.heads[h] = ie; + + *ia = ie->ia; + *rac = ie->rac; + + qunlock(&icache.lock); + + if(load){ + trace(TraceProc, "preload 0x%llux", aa); + loadarenaclumps(load, aa); + } + ms = msec() - ms; + addstat2(StatIcacheRead, 1, StatIcacheReadTime, ms); + + return 0; +} + +/* + * insert a new element in the hash table. + */ +int +insertscore(u8int *score, IAddr *ia, int write) +{ + IEntry *ie, se; + u32int h; + + trace(TraceLump, "insertscore enter"); + if(write) + addstat(StatIcacheWrite, 1); + else + addstat(StatIcachePrefetch, 1); + + qlock(&icache.lock); + h = hashbits(score, icache.bits); + + ie = icachealloc(ia, score); + if(write){ + icache.ndirty++; + setstat(StatIcacheDirty, icache.ndirty); + delaykickicache(); + ie->dirty = 1; + } + ie->next = icache.heads[h]; + icache.heads[h] = ie; + + se = *ie; + qunlock(&icache.lock); + + if(write && icache.ndirty >= icache.maxdirty) + kickicache(); + + /* + * It's okay not to do this under icache.lock. + * Calling insertscore only happens when we hold + * the lump, meaning any searches for this block + * will hit in the lump cache until after we return. + */ + markbloomfilter(mainindex->bloom, score); + + return 0; +} + +/* + * allocate a index cache entry which hasn't been used in a while. + * must be called with icache.lock locked + * if the score is already in the table, update the entry. + */ +static IEntry * +icachealloc(IAddr *ia, u8int *score) +{ + int i; + IEntry *ie, *last, *clean, *lastclean; + u32int h; + + h = hashbits(score, icache.bits); + last = nil; + for(ie = icache.heads[h]; ie != nil; ie = ie->next){ + if(ie->ia.type == ia->type && scorecmp(ie->score, score)==0){ + if(last != nil) + last->next = ie->next; + else + icache.heads[h] = ie->next; + trace(TraceLump, "icachealloc hit"); + ie->rac = 1; + return ie; + } + last = ie; + } + + h = icache.unused; + if(h < icache.entries){ + ie = &icache.base[h++]; + icache.unused = h; + trace(TraceLump, "icachealloc unused"); + goto Found; + } + + h = icache.stolen; + for(i=0;; i++){ + h++; + if(h >= icache.size) + h = 0; + if(i == icache.size){ + trace(TraceLump, "icachealloc sleep"); + addstat(StatIcacheStall, 1); + while(icache.ndirty == icache.entries){ + /* + * This is a bit suspect. Kickicache will wake up the + * icachewritecoord, but if all the index entries are for + * unflushed disk blocks, icachewritecoord won't be + * able to do much. It always rewakes everyone when + * it thinks it is done, though, so at least we'll go around + * the while loop again. Also, if icachewritecoord sees + * that the disk state hasn't change at all since the last + * time around, it kicks the disk. This needs to be + * rethought, but it shouldn't deadlock anymore. + */ + kickicache(); + rsleep(&icache.full); + } + addstat(StatIcacheStall, -1); + i = 0; + } + lastclean = nil; + clean = nil; + last = nil; + for(ie=icache.heads[h]; ie; last=ie, ie=ie->next){ + if(!ie->dirty){ + clean = ie; + lastclean = last; + } + } + if(clean){ + if(lastclean) + lastclean->next = clean->next; + else + icache.heads[h] = clean->next; + clean->next = nil; + icache.stolen = h; + ie = clean; + trace(TraceLump, "icachealloc steal"); + goto Found; + } + } + +Found: + ie->ia = *ia; + scorecp(ie->score, score); + ie->rac = 0; + return ie; +} + +IEntry* +icachedirty(u32int lo, u32int hi, u64int limit) +{ + int i; + u32int h; + IEntry *ie, *dirty; + + dirty = nil; + trace(TraceProc, "icachedirty enter"); + qlock(&icache.lock); + for(i=0; inext) + if(ie->dirty && ie->ia.addr != 0 && ie->ia.addr < limit){ + h = hashbits(ie->score, 32); + if(lo <= h && h <= hi){ + ie->nextdirty = dirty; + dirty = ie; + } + } + qunlock(&icache.lock); + trace(TraceProc, "icachedirty exit"); + if(dirty == nil) + flushdcache(); + return dirty; +} + +void +icacheclean(IEntry *ie) +{ + trace(TraceProc, "icachedirty enter"); + qlock(&icache.lock); + for(; ie; ie=ie->nextdirty){ + icache.ndirty--; + ie->dirty = 0; + } + setstat(StatIcacheDirty, icache.ndirty); + rwakeupall(&icache.full); + qunlock(&icache.lock); + trace(TraceProc, "icachedirty exit"); +} + blob - /dev/null blob + 900af871eccd69e2e8514cdf50a0dfbb31305ffb (mode 644) --- /dev/null +++ src/cmd/venti/srv/icachewrite.c @@ -0,0 +1,318 @@ +/* + * Write the dirty icache entries to disk. Random seeks are + * so expensive that it makes sense to wait until we have + * a lot and then just make a sequential pass over the disk. + */ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +static void icachewriteproc(void*); +static void icachewritecoord(void*); +static IEntry *iesort(IEntry*); + +int icachesleeptime = 1000; /* milliseconds */ + +enum +{ + Bufsize = 8*1024*1024 +}; + +typedef struct IWrite IWrite; +struct IWrite +{ + Round round; + AState as; +}; + +static IWrite iwrite; + +void +initicachewrite(void) +{ + int i; + Index *ix; + + initround(&iwrite.round, "icache", 120*60*1000); + ix = mainindex; + for(i=0; insects; i++){ + ix->sects[i]->writechan = chancreate(sizeof(ulong), 1); + ix->sects[i]->writedonechan = chancreate(sizeof(ulong), 1); + vtproc(icachewriteproc, ix->sects[i]); + } + vtproc(icachewritecoord, nil); + vtproc(delaykickroundproc, &iwrite.round); +} + +static IEntry* +nextchunk(Index *ix, ISect *is, IEntry **pie, u64int *paddr, uint *pnbuf) +{ + u64int addr, naddr; + uint nbuf; + int bsize; + IEntry *iefirst, *ie, **l; + + bsize = 1<blocklog; + iefirst = *pie; + addr = is->blockbase + ((u64int)(hashbits(iefirst->score, 32) / ix->div - is->start) << is->blocklog); + nbuf = 0; + for(l=&iefirst->nextdirty; (ie=*l)!=nil; l=&(*l)->nextdirty){ + naddr = is->blockbase + ((u64int)(hashbits(ie->score, 32) / ix->div - is->start) << is->blocklog); + if(naddr - addr >= Bufsize) + break; + nbuf = naddr-addr; + } + nbuf += bsize; + + *l = nil; + *pie = ie; + *paddr = addr; + *pnbuf = nbuf; + return iefirst; +} + +static int +icachewritesect(Index *ix, ISect *is, u8int *buf) +{ + int err, h, bsize; + u32int lo, hi; + u64int addr, naddr; + uint nbuf, off; + DBlock *b; + IBucket ib; + IEntry *ie, *iedirty, **l, *chunk; + + lo = is->start * ix->div; + if(TWID32/ix->div < is->stop) + hi = TWID32; + else + hi = is->stop * ix->div - 1; + + trace(TraceProc, "icachewritesect enter %ud %ud %llud", lo, hi, iwrite.as.aa); + + iedirty = icachedirty(lo, hi, iwrite.as.aa); + iedirty = iesort(iedirty); + bsize = 1<blocklog; + err = 0; + + while(iedirty){ + sleep(icachesleeptime); + trace(TraceProc, "icachewritesect nextchunk"); + chunk = nextchunk(ix, is, &iedirty, &addr, &nbuf); + + trace(TraceProc, "icachewritesect readpart 0x%llux+0x%ux", addr, nbuf); + if(readpart(is->part, addr, buf, nbuf) < 0){ + // XXX + fprint(2, "icachewriteproc readpart: %r\n"); + err = -1; + continue; + } + trace(TraceProc, "icachewritesect updatebuf"); + addstat(StatIsectReadBytes, nbuf); + addstat(StatIsectRead, 1); + + for(l=&chunk; (ie=*l)!=nil; l=&ie->nextdirty){ + again: + naddr = is->blockbase + ((u64int)(hashbits(ie->score, 32) / ix->div - is->start) << is->blocklog); + off = naddr - addr; + if(off+bsize > nbuf){ + fprint(2, "whoops! addr=0x%llux nbuf=%ud addr+nbuf=0x%llux naddr=0x%llux\n", + addr, nbuf, addr+nbuf, naddr); + assert(off+bsize <= nbuf); + } + unpackibucket(&ib, buf+off, is->bucketmagic); + if(okibucket(&ib, is) < 0){ + fprint(2, "bad bucket XXX\n"); + goto skipit; + } + trace(TraceProc, "icachewritesect add %V at 0x%llux", ie->score, naddr); + h = bucklook(ie->score, ie->ia.type, ib.data, ib.n); + if(h & 1){ + h ^= 1; + packientry(ie, &ib.data[h]); + }else if(ib.n < is->buckmax){ + memmove(&ib.data[h+IEntrySize], &ib.data[h], ib.n*IEntrySize - h); + ib.n++; + packientry(ie, &ib.data[h]); + }else{ + fprint(2, "bucket overflow XXX\n"); + skipit: + err = -1; + *l = ie->nextdirty; + ie = *l; + if(ie) + goto again; + else + break; + } + packibucket(&ib, buf+off, is->bucketmagic); + if((b = _getdblock(is->part, naddr, ORDWR, 0)) != nil){ + memmove(b->data, buf+off, bsize); + putdblock(b); + } + } + + trace(TraceProc, "icachewritesect writepart", addr, nbuf); + if(writepart(is->part, addr, buf, nbuf) < 0){ + // XXX + fprint(2, "icachewriteproc writepart: %r\n"); + err = -1; + continue; + } + addstat(StatIsectWriteBytes, nbuf); + addstat(StatIsectWrite, 1); + icacheclean(chunk); + } + + trace(TraceProc, "icachewritesect done"); + return err; +} + +static void +icachewriteproc(void *v) +{ + uint bsize; + ISect *is; + Index *ix; + u8int *buf; + + ix = mainindex; + is = v; + threadsetname("icachewriteproc:%s", is->part->name); + + bsize = 1<blocklog; + buf = emalloc(Bufsize+bsize); + buf = (u8int*)(((ulong)buf+bsize-1)&~(ulong)(bsize-1)); + + for(;;){ + trace(TraceProc, "icachewriteproc recv"); + recv(is->writechan, 0); + trace(TraceWork, "start"); + icachewritesect(ix, is, buf); + trace(TraceProc, "icachewriteproc send"); + trace(TraceWork, "finish"); + send(is->writedonechan, 0); + } +} + +static void +icachewritecoord(void *v) +{ + int i; + Index *ix; + AState as; + + USED(v); + + threadsetname("icachewritecoord"); + + ix = mainindex; + iwrite.as = diskstate(); + + for(;;){ + trace(TraceProc, "icachewritecoord sleep"); + waitforkick(&iwrite.round); + trace(TraceWork, "start"); + as = diskstate(); + if(as.arena==iwrite.as.arena && as.aa==iwrite.as.aa){ + /* will not be able to do anything more than last flush - kick disk */ + trace(TraceProc, "icachewritecoord flush dcache"); + kickdcache(); + trace(TraceProc, "icachewritecoord flushed dcache"); + } + iwrite.as = as; + + trace(TraceProc, "icachewritecoord start flush"); + if(iwrite.as.arena){ + for(i=0; insects; i++) + send(ix->sects[i]->writechan, 0); + if(ix->bloom) + send(ix->bloom->writechan, 0); + + for(i=0; insects; i++) + recv(ix->sects[i]->writedonechan, 0); + if(ix->bloom) + recv(ix->bloom->writedonechan, 0); + + trace(TraceProc, "icachewritecoord donewrite"); + setatailstate(&iwrite.as); + } + icacheclean(nil); /* wake up anyone waiting */ + trace(TraceWork, "finish"); + addstat(StatIcacheFlush, 1); + } +} + +void +flushicache(void) +{ + trace(TraceProc, "flushicache enter"); + kickround(&iwrite.round, 1); + trace(TraceProc, "flushicache exit"); +} + +void +kickicache(void) +{ + kickround(&iwrite.round, 0); +} + +void +delaykickicache(void) +{ + delaykickround(&iwrite.round); +} + +static IEntry* +iesort(IEntry *ie) +{ + int cmp; + IEntry **l; + IEntry *ie1, *ie2, *sorted; + + if(ie == nil || ie->nextdirty == nil) + return ie; + + /* split the lists */ + ie1 = ie; + ie2 = ie; + if(ie2) + ie2 = ie2->nextdirty; + if(ie2) + ie2 = ie2->nextdirty; + while(ie1 && ie2){ + ie1 = ie1->nextdirty; + ie2 = ie2->nextdirty; + if(ie2) + ie2 = ie2->nextdirty; + } + if(ie1){ + ie2 = ie1->nextdirty; + ie1->nextdirty = nil; + } + + /* sort the lists */ + ie1 = iesort(ie); + ie2 = iesort(ie2); + + /* merge the lists */ + sorted = nil; + l = &sorted; + cmp = 0; + while(ie1 || ie2){ + if(ie1 && ie2) + cmp = scorecmp(ie1->score, ie2->score); + if(ie1==nil || (ie2 && cmp > 0)){ + *l = ie2; + l = &ie2->nextdirty; + ie2 = ie2->nextdirty; + }else{ + *l = ie1; + l = &ie1->nextdirty; + ie1 = ie1->nextdirty; + } + } + *l = nil; + return sorted; +} + blob - /dev/null blob + fc784c9bb4c8ecfc9e08ec8f79658c97e293596e (mode 644) --- /dev/null +++ src/cmd/venti/srv/ifile.c @@ -0,0 +1,93 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +int +readifile(IFile *f, char *name) +{ + ZBlock *b; + + b = readfile(name); + if(b == nil) + return -1; + f->name = name; + f->b = b; + f->pos = 0; + return 0; +} + +void +freeifile(IFile *f) +{ + freezblock(f->b); + f->b = nil; + f->pos = 0; +} + +int +partifile(IFile *f, Part *part, u64int start, u32int size) +{ + ZBlock *b; + + b = alloczblock(size, 0, part->blocksize); + if(b == nil) + return -1; + if(readpart(part, start, b->data, size) < 0){ + seterr(EAdmin, "can't read %s: %r", part->name); + freezblock(b); + return -1; + } + f->name = part->name; + f->b = b; + f->pos = 0; + return 0; +} + +/* + * return the next non-blank input line, + * stripped of leading white space and with # comments eliminated + */ +char* +ifileline(IFile *f) +{ + char *s, *e, *t; + int c; + + for(;;){ + s = (char*)&f->b->data[f->pos]; + e = memchr(s, '\n', f->b->len - f->pos); + if(e == nil) + return nil; + *e++ = '\0'; + f->pos = e - (char*)f->b->data; + t = strchr(s, '#'); + if(t != nil) + *t = '\0'; + for(; c = *s; s++) + if(c != ' ' && c != '\t' && c != '\r') + return s; + } +} + +int +ifilename(IFile *f, char *dst) +{ + char *s; + + s = ifileline(f); + if(s == nil || strlen(s) >= ANameSize) + return -1; + namecp(dst, s); + return 0; +} + +int +ifileu32int(IFile *f, u32int *r) +{ + char *s; + + s = ifileline(f); + if(s == nil) + return -1; + return stru32int(s, r); +} blob - /dev/null blob + 46bf91e2b337e9ae71fecb2b73743471fb4ee85b (mode 644) --- /dev/null +++ src/cmd/venti/srv/index.c @@ -0,0 +1,819 @@ +/* + * Index, mapping scores to log positions. + * + * The index is made up of some number of index sections, each of + * which is typically stored on a different disk. The blocks in all the + * index sections are logically numbered, with each index section + * responsible for a range of blocks. Blocks are typically 8kB. + * + * The N index blocks are treated as a giant hash table. The top 32 bits + * of score are used as the key for a lookup. Each index block holds + * one hash bucket, which is responsible for ceil(2^32 / N) of the key space. + * + * The index is sized so that a particular bucket is extraordinarily + * unlikely to overflow: assuming compressed data blocks are 4kB + * on disk, and assuming each block has a 40 byte index entry, + * the index data will be 1% of the total data. Since scores are essentially + * random, all buckets should be about the same fullness. + * A factor of 5 gives us a wide comfort boundary to account for + * random variation. So the index disk space should be 5% of the arena disk space. + */ + +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +//static int bucklook(u8int *score, int type, u8int *data, int n); +//static int writebucket(ISect *is, u32int buck, IBucket *ib, DBlock *b); +//static int okibucket(IBucket *ib, ISect *is); +static int initindex1(Index*); +static ISect *initisect1(ISect *is); +//static int splitiblock(Index *ix, DBlock *b, ISect *is, u32int buck, IBucket *ib); + +#define KEY(k,d) ((d) ? (k)>>(32-(d)) : 0) + +//static QLock indexlock; //ZZZ + +static char IndexMagic[] = "venti index configuration"; + +Index* +initindex(char *name, ISect **sects, int n) +{ + IFile f; + Index *ix; + ISect *is; + u32int last, blocksize, tabsize; + int i; + + if(n <= 0){ +fprint(2, "bad n\n"); + seterr(EOk, "no index sections to initialize index"); + return nil; + } + ix = MKZ(Index); + if(ix == nil){ +fprint(2, "no mem\n"); + seterr(EOk, "can't initialize index: out of memory"); + freeindex(ix); + return nil; + } + + tabsize = sects[0]->tabsize; + if(partifile(&f, sects[0]->part, sects[0]->tabbase, tabsize) < 0) + return nil; + if(parseindex(&f, ix) < 0){ + freeifile(&f); + freeindex(ix); + return nil; + } + freeifile(&f); + if(namecmp(ix->name, name) != 0){ + seterr(ECorrupt, "mismatched index name: found %s expected %s", ix->name, name); + return nil; + } + if(ix->nsects != n){ + seterr(ECorrupt, "mismatched number index sections: found %d expected %d", n, ix->nsects); + freeindex(ix); + return nil; + } + ix->sects = sects; + last = 0; + blocksize = ix->blocksize; + for(i = 0; i < ix->nsects; i++){ + is = sects[i]; + if(namecmp(ix->name, is->index) != 0 + || is->blocksize != blocksize + || is->tabsize != tabsize + || namecmp(is->name, ix->smap[i].name) != 0 + || is->start != ix->smap[i].start + || is->stop != ix->smap[i].stop + || last != is->start + || is->start > is->stop){ + seterr(ECorrupt, "inconsistent index sections in %s", ix->name); + freeindex(ix); + return nil; + } + last = is->stop; + } + ix->tabsize = tabsize; + ix->buckets = last; + + if(initindex1(ix) < 0){ + freeindex(ix); + return nil; + } + + ix->arenas = MKNZ(Arena*, ix->narenas); + if(maparenas(ix->amap, ix->arenas, ix->narenas, ix->name) < 0){ + freeindex(ix); + return nil; + } + + return ix; +} + +static int +initindex1(Index *ix) +{ + u32int buckets; + + ix->div = (((u64int)1 << 32) + ix->buckets - 1) / ix->buckets; + buckets = (((u64int)1 << 32) - 1) / ix->div + 1; + if(buckets != ix->buckets){ + seterr(ECorrupt, "inconsistent math for divisor and buckets in %s", ix->name); + return -1; + } + + return 0; +} + +int +wbindex(Index *ix) +{ + Fmt f; + ZBlock *b; + int i; + + if(ix->nsects == 0){ + seterr(EOk, "no sections in index %s", ix->name); + return -1; + } + b = alloczblock(ix->tabsize, 1, ix->blocksize); + if(b == nil){ + seterr(EOk, "can't write index configuration: out of memory"); + return -1; + } + fmtzbinit(&f, b); + if(outputindex(&f, ix) < 0){ + seterr(EOk, "can't make index configuration: table storage too small %d", ix->tabsize); + freezblock(b); + return -1; + } + for(i = 0; i < ix->nsects; i++){ + if(writepart(ix->sects[i]->part, ix->sects[i]->tabbase, b->data, ix->tabsize) < 0){ + seterr(EOk, "can't write index: %r"); + freezblock(b); + return -1; + } + } + freezblock(b); + + for(i = 0; i < ix->nsects; i++) + if(wbisect(ix->sects[i]) < 0) + return -1; + + return 0; +} + +/* + * index: IndexMagic '\n' version '\n' name '\n' blocksize '\n' [V2: bitblocks '\n'] sections arenas + * version, blocksize: u32int + * name: max. ANameSize string + * sections, arenas: AMap + */ +int +outputindex(Fmt *f, Index *ix) +{ + if(fmtprint(f, "%s\n%ud\n%s\n%ud\n", IndexMagic, ix->version, ix->name, ix->blocksize) < 0 + || outputamap(f, ix->smap, ix->nsects) < 0 + || outputamap(f, ix->amap, ix->narenas) < 0) + return -1; + return 0; +} + +int +parseindex(IFile *f, Index *ix) +{ + AMapN amn; + u32int v; + char *s; + + /* + * magic + */ + s = ifileline(f); + if(s == nil || strcmp(s, IndexMagic) != 0){ + seterr(ECorrupt, "bad index magic for %s", f->name); + return -1; + } + + /* + * version + */ + if(ifileu32int(f, &v) < 0){ + seterr(ECorrupt, "syntax error: bad version number in %s", f->name); + return -1; + } + ix->version = v; + if(ix->version != IndexVersion){ + seterr(ECorrupt, "bad version number in %s", f->name); + return -1; + } + + /* + * name + */ + if(ifilename(f, ix->name) < 0){ + seterr(ECorrupt, "syntax error: bad index name in %s", f->name); + return -1; + } + + /* + * block size + */ + if(ifileu32int(f, &v) < 0){ + seterr(ECorrupt, "syntax error: bad block size number in %s", f->name); + return -1; + } + ix->blocksize = v; + + if(parseamap(f, &amn) < 0) + return -1; + ix->nsects = amn.n; + ix->smap = amn.map; + + if(parseamap(f, &amn) < 0) + return -1; + ix->narenas = amn.n; + ix->amap = amn.map; + + return 0; +} + +/* + * initialize an entirely new index + */ +Index * +newindex(char *name, ISect **sects, int n) +{ + Index *ix; + AMap *smap; + u64int nb; + u32int div, ub, xb, fb, start, stop, blocksize, tabsize; + int i, j; + + if(n < 1){ + seterr(EOk, "creating index with no index sections"); + return nil; + } + + /* + * compute the total buckets available in the index, + * and the total buckets which are used. + */ + nb = 0; + blocksize = sects[0]->blocksize; + tabsize = sects[0]->tabsize; + for(i = 0; i < n; i++){ + if(sects[i]->start != 0 || sects[i]->stop != 0 + || sects[i]->index[0] != '\0'){ + seterr(EOk, "creating new index using non-empty section %s", sects[i]->name); + return nil; + } + if(blocksize != sects[i]->blocksize){ + seterr(EOk, "mismatched block sizes in index sections"); + return nil; + } + if(tabsize != sects[i]->tabsize){ + seterr(EOk, "mismatched config table sizes in index sections"); + return nil; + } + nb += sects[i]->blocks; + } + + /* + * check for duplicate names + */ + for(i = 0; i < n; i++){ + for(j = i + 1; j < n; j++){ + if(namecmp(sects[i]->name, sects[j]->name) == 0){ + seterr(EOk, "duplicate section name %s for index %s", sects[i]->name, name); + return nil; + } + } + } + + if(nb >= ((u64int)1 << 32)){ + seterr(EBug, "index too large"); + return nil; + } + + fb = 0; + div = (((u64int)1 << 32) + nb - 1) / nb; + ub = (((u64int)1 << 32) - 1) / div + 1; + if(div < 100){ + seterr(EBug, "index divisor too coarse"); + return nil; + } + if(ub > nb){ + seterr(EBug, "index initialization math wrong"); + return nil; + } + xb = nb - ub; + + /* + * initialize each of the index sections + * and the section map table + */ + smap = MKNZ(AMap, n); + if(smap == nil){ + seterr(EOk, "can't create new index: out of memory"); + return nil; + } + start = 0; + for(i = 0; i < n; i++){ + stop = start + sects[i]->blocks - xb / n; + if(i == n - 1) + stop = ub; + sects[i]->start = start; + sects[i]->stop = stop; + namecp(sects[i]->index, name); + + smap[i].start = start; + smap[i].stop = stop; + namecp(smap[i].name, sects[i]->name); + start = stop; + } + + /* + * initialize the index itself + */ + ix = MKZ(Index); + if(ix == nil){ + seterr(EOk, "can't create new index: out of memory"); + free(smap); + return nil; + } + ix->version = IndexVersion; + namecp(ix->name, name); + ix->sects = sects; + ix->smap = smap; + ix->nsects = n; + ix->blocksize = blocksize; + ix->buckets = ub; + ix->tabsize = tabsize; + ix->div = div; + ix->bitblocks = fb; + + if(initindex1(ix) < 0){ + free(smap); + return nil; + } + + return ix; +} + +ISect* +initisect(Part *part) +{ + ISect *is; + ZBlock *b; + int ok; + + b = alloczblock(HeadSize, 0, 0); + if(b == nil || readpart(part, PartBlank, b->data, HeadSize) < 0){ + seterr(EAdmin, "can't read index section header: %r"); + return nil; + } + + is = MKZ(ISect); + if(is == nil){ + freezblock(b); + return nil; + } + is->part = part; + ok = unpackisect(is, b->data); + freezblock(b); + if(ok < 0){ + seterr(ECorrupt, "corrupted index section header: %r"); + freeisect(is); + return nil; + } + + if(is->version != ISectVersion1 && is->version != ISectVersion2){ + seterr(EAdmin, "unknown index section version %d", is->version); + freeisect(is); + return nil; + } + + return initisect1(is); +} + +ISect* +newisect(Part *part, u32int vers, char *name, u32int blocksize, u32int tabsize) +{ + ISect *is; + u32int tabbase; + + is = MKZ(ISect); + if(is == nil) + return nil; + + namecp(is->name, name); + is->version = vers; + is->part = part; + is->blocksize = blocksize; + is->start = 0; + is->stop = 0; + tabbase = (PartBlank + HeadSize + blocksize - 1) & ~(blocksize - 1); + is->blockbase = (tabbase + tabsize + blocksize - 1) & ~(blocksize - 1); + is->blocks = is->part->size / blocksize - is->blockbase / blocksize; + is->bucketmagic = 0; + if(is->version == ISectVersion2){ + do{ + is->bucketmagic = fastrand(); + }while(is->bucketmagic==0); + } + is = initisect1(is); + if(is == nil) + return nil; + + return is; +} + +/* + * initialize the computed parameters for an index + */ +static ISect* +initisect1(ISect *is) +{ + u64int v; + + is->buckmax = (is->blocksize - IBucketSize) / IEntrySize; + is->blocklog = u64log2(is->blocksize); + if(is->blocksize != (1 << is->blocklog)){ + seterr(ECorrupt, "illegal non-power-of-2 bucket size %d\n", is->blocksize); + freeisect(is); + return nil; + } + partblocksize(is->part, is->blocksize); + is->tabbase = (PartBlank + HeadSize + is->blocksize - 1) & ~(is->blocksize - 1); + if(is->tabbase >= is->blockbase){ + seterr(ECorrupt, "index section config table overlaps bucket storage"); + freeisect(is); + return nil; + } + is->tabsize = is->blockbase - is->tabbase; + v = is->part->size & ~(u64int)(is->blocksize - 1); + if(is->blockbase + (u64int)is->blocks * is->blocksize != v){ + seterr(ECorrupt, "invalid blocks in index section %s", is->name); +//ZZZZZZZZZ +// freeisect(is); +// return nil; + } + + if(is->stop - is->start > is->blocks){ + seterr(ECorrupt, "index section overflows available space"); + freeisect(is); + return nil; + } + if(is->start > is->stop){ + seterr(ECorrupt, "invalid index section range"); + freeisect(is); + return nil; + } + + return is; +} + +int +wbisect(ISect *is) +{ + ZBlock *b; + + b = alloczblock(HeadSize, 1, 0); + if(b == nil) +//ZZZ set error? + return -1; + + if(packisect(is, b->data) < 0){ + seterr(ECorrupt, "can't make index section header: %r"); + freezblock(b); + return -1; + } + if(writepart(is->part, PartBlank, b->data, HeadSize) < 0){ + seterr(EAdmin, "can't write index section header: %r"); + freezblock(b); + return -1; + } + freezblock(b); + + return 0; +} + +void +freeisect(ISect *is) +{ + if(is == nil) + return; + free(is); +} + +void +freeindex(Index *ix) +{ + int i; + + if(ix == nil) + return; + free(ix->amap); + free(ix->arenas); + if(ix->sects) + for(i = 0; i < ix->nsects; i++) + freeisect(ix->sects[i]); + free(ix->sects); + free(ix->smap); + free(ix); +} + +/* + * write a clump to an available arena in the index + * and return the address of the clump within the index. +ZZZ question: should this distinguish between an arena +filling up and real errors writing the clump? + */ +u64int +writeiclump(Index *ix, Clump *c, u8int *clbuf, u64int *pa) +{ + u64int a; + int i; + + trace(TraceLump, "writeiclump enter"); + for(i = ix->mapalloc; i < ix->narenas; i++){ + a = writeaclump(ix->arenas[i], c, clbuf, ix->amap[i].start, pa); + if(a != TWID64){ + ix->mapalloc = i; /* assuming write is atomic, race is okay */ + trace(TraceLump, "writeiclump exit"); + return a; + } + } + + seterr(EAdmin, "no space left in arenas"); + trace(TraceLump, "writeiclump failed"); + return TWID64; +} + +/* + * convert an arena index to an relative arena address + */ +Arena* +amapitoa(Index *ix, u64int a, u64int *aa) +{ + int i, r, l, m; + + l = 1; + r = ix->narenas - 1; + while(l <= r){ + m = (r + l) / 2; + if(ix->amap[m].start <= a) + l = m + 1; + else + r = m - 1; + } + l--; + + if(a > ix->amap[l].stop){ +for(i=0; inarenas; i++) + print("arena %d: %llux - %llux\n", i, ix->amap[i].start, ix->amap[i].stop); +print("want arena %d for %llux\n", l, a); + seterr(ECrash, "unmapped address passed to amapitoa"); + return nil; + } + + if(ix->arenas[l] == nil){ + seterr(ECrash, "unmapped arena selected in amapitoa"); + return nil; + } + *aa = a - ix->amap[l].start; + return ix->arenas[l]; +} + +int +iaddrcmp(IAddr *ia1, IAddr *ia2) +{ + return ia1->type != ia2->type + || ia1->size != ia2->size + || ia1->blocks != ia2->blocks + || ia1->addr != ia2->addr; +} + +/* + * lookup the score in the partition + * + * nothing needs to be explicitly locked: + * only static parts of ix are used, and + * the bucket is locked by the DBlock lock. + */ +int +loadientry(Index *ix, u8int *score, int type, IEntry *ie) +{ + ISect *is; + DBlock *b; + IBucket ib; + u32int buck; + int h, ok; + + ok = -1; + + trace(TraceLump, "loadientry enter"); + + /* + qlock(&stats.lock); + stats.indexreads++; + qunlock(&stats.lock); + */ + + if(!inbloomfilter(mainindex->bloom, score)){ + trace(TraceLump, "loadientry bloomhit"); + return -1; + } + + trace(TraceLump, "loadientry loadibucket"); + b = loadibucket(ix, score, &is, &buck, &ib); + trace(TraceLump, "loadientry loadedibucket"); + if(b == nil) + return -1; + + if(okibucket(&ib, is) < 0){ + trace(TraceLump, "loadientry badbucket"); + goto out; + } + + h = bucklook(score, type, ib.data, ib.n); + if(h & 1){ + h ^= 1; + trace(TraceLump, "loadientry found"); + unpackientry(ie, &ib.data[h]); + ok = 0; + goto out; + } + trace(TraceLump, "loadientry notfound"); + addstat(StatBloomFalseMiss, 1); +out: + putdblock(b); + trace(TraceLump, "loadientry exit"); + return ok; +} + +int +okibucket(IBucket *ib, ISect *is) +{ + if(ib->n <= is->buckmax) + return 0; + + seterr(EICorrupt, "corrupted disk index bucket: n=%ud max=%ud, range=[%lud,%lud)", + ib->n, is->buckmax, is->start, is->stop); + return -1; +} + +/* + * look for score within data; + * return 1 | byte index of matching index, + * or 0 | index of least element > score + */ +int +bucklook(u8int *score, int otype, u8int *data, int n) +{ + int i, r, l, m, h, c, cc, type; + + type = vttodisktype(otype); + l = 0; + r = n - 1; + while(l <= r){ + m = (r + l) >> 1; + h = m * IEntrySize; + for(i = 0; i < VtScoreSize; i++){ + c = score[i]; + cc = data[h + i]; + if(c != cc){ + if(c > cc) + l = m + 1; + else + r = m - 1; + goto cont; + } + } + cc = data[h + IEntryTypeOff]; + if(type != cc){ + if(type > cc) + l = m + 1; + else + r = m - 1; + goto cont; + } + return h | 1; + cont:; + } + + return l * IEntrySize; +} + +/* + * compare two IEntries; consistent with bucklook + */ +int +ientrycmp(const void *vie1, const void *vie2) +{ + u8int *ie1, *ie2; + int i, v1, v2; + + ie1 = (u8int*)vie1; + ie2 = (u8int*)vie2; + for(i = 0; i < VtScoreSize; i++){ + v1 = ie1[i]; + v2 = ie2[i]; + if(v1 != v2){ + if(v1 < v2) + return -1; + return 1; + } + } + v1 = ie1[IEntryTypeOff]; + v2 = ie2[IEntryTypeOff]; + if(v1 != v2){ + if(v1 < v2) + return -1; + return 1; + } + return 0; +} + +/* + * find the number of the index section holding bucket #buck + */ +int +indexsect0(Index *ix, u32int buck) +{ + int r, l, m; + + l = 1; + r = ix->nsects - 1; + while(l <= r){ + m = (r + l) >> 1; + if(ix->sects[m]->start <= buck) + l = m + 1; + else + r = m - 1; + } + return l - 1; +} + +/* + * load the index block at bucket #buck + */ +static DBlock* +loadibucket0(Index *ix, u32int buck, ISect **pis, u32int *pbuck, IBucket *ib, int mode) +{ + ISect *is; + DBlock *b; + + is = ix->sects[indexsect0(ix, buck)]; + if(buck < is->start || is->stop <= buck){ + seterr(EAdmin, "index lookup out of range: %ud not found in index\n", buck); + return nil; + } + + buck -= is->start; + if((b = getdblock(is->part, is->blockbase + ((u64int)buck << is->blocklog), mode)) == nil) + return nil; + + if(pis) + *pis = is; + if(pbuck) + *pbuck = buck; + if(ib) + unpackibucket(ib, b->data, is->bucketmagic); + return b; +} + +/* + * find the number of the index section holding score + */ +static int +indexsect1(Index *ix, u8int *score) +{ + return indexsect0(ix, hashbits(score, 32) / ix->div); +} + +/* + * load the index block responsible for score. + */ +static DBlock* +loadibucket1(Index *ix, u8int *score, ISect **pis, u32int *pbuck, IBucket *ib) +{ + return loadibucket0(ix, hashbits(score, 32)/ix->div, pis, pbuck, ib, OREAD); +} + +int +indexsect(Index *ix, u8int *score) +{ + return indexsect1(ix, score); +} + +DBlock* +loadibucket(Index *ix, u8int *score, ISect **pis, u32int *pbuck, IBucket *ib) +{ + return loadibucket1(ix, score, pis, pbuck, ib); +} + + blob - /dev/null blob + d1e58a6df13b450676f354a8f3faaf8d41ed7d7f (mode 644) --- /dev/null +++ src/cmd/venti/srv/lump.c @@ -0,0 +1,249 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +int queuewrites = 0; +int writestodevnull = 0; + +static Packet *readilump(Lump *u, IAddr *ia, u8int *score, int rac); + +Packet* +readlump(u8int *score, int type, u32int size, int *cached) +{ + Lump *u; + Packet *p; + IAddr ia; + u32int n; + int rac; + + trace(TraceLump, "readlump enter"); +/* + qlock(&stats.lock); + stats.lumpreads++; + qunlock(&stats.lock); +*/ + if(scorecmp(score, zeroscore) == 0) + return packetalloc(); + u = lookuplump(score, type); + if(u->data != nil){ + trace(TraceLump, "readlump lookuplump hit"); + if(cached) + *cached = 1; + n = packetsize(u->data); + if(n > size){ + seterr(EOk, "read too small: asked for %d need at least %d", size, n); + putlump(u); + + return nil; + } + p = packetdup(u->data, 0, n); + putlump(u); + return p; + } + + if(cached) + *cached = 0; + + if(lookupscore(score, type, &ia, &rac) < 0){ + //ZZZ place to check for someone trying to guess scores + seterr(EOk, "no block with score %V/%d exists", score, type); + + putlump(u); + return nil; + } + if(ia.size > size){ + seterr(EOk, "read too small 1: asked for %d need at least %d", size, ia.size); + + putlump(u); + return nil; + } + + trace(TraceLump, "readlump readilump"); + p = readilump(u, &ia, score, rac); + putlump(u); + + trace(TraceLump, "readlump exit"); + return p; +} + +/* + * save away a lump, and return it's score. + * doesn't store duplicates, but checks that the data is really the same. + */ +int +writelump(Packet *p, u8int *score, int type, u32int creator, uint ms) +{ + Lump *u; + int ok; + +/* + qlock(&stats.lock); + stats.lumpwrites++; + qunlock(&stats.lock); +*/ + + packetsha1(p, score); + if(packetsize(p) == 0 || writestodevnull==1){ + packetfree(p); + return 0; + } + + u = lookuplump(score, type); + if(u->data != nil){ + ok = 0; + if(packetcmp(p, u->data) != 0){ + seterr(EStrange, "score collision"); + ok = -1; + } + packetfree(p); + putlump(u); + return ok; + } + + if(writestodevnull==2){ + packetfree(p); + return 0; + } + + if(queuewrites) + return queuewrite(u, p, creator, ms); + + ok = writeqlump(u, p, creator, ms); + + putlump(u); + return ok; +} + +int +writeqlump(Lump *u, Packet *p, int creator, uint ms) +{ + ZBlock *flat; + Packet *old; + IAddr ia; + int ok; + int rac; + + if(lookupscore(u->score, u->type, &ia, &rac) == 0){ + /* assume the data is here! XXX */ + packetfree(p); + ms = msec() - ms; + addstat2(StatRpcWriteOld, 1, StatRpcWriteOldTime, ms); + return 0; + + /* + * if the read fails, + * assume it was corrupted data and store the block again + */ + old = readilump(u, &ia, u->score, rac); + if(old != nil){ + ok = 0; + if(packetcmp(p, old) != 0){ + seterr(EStrange, "score collision"); + ok = -1; + } + packetfree(p); + packetfree(old); + + ms = msec() - ms; + addstat2(StatRpcWriteOld, 1, StatRpcWriteOldTime, ms); + return ok; + } + logerr(EAdmin, "writelump: read %V failed, rewriting: %r\n", u->score); + } + + flat = packet2zblock(p, packetsize(p)); + ok = storeclump(mainindex, flat, u->score, u->type, creator, &ia); + freezblock(flat); + if(ok == 0) + ok = insertscore(u->score, &ia, 1); + if(ok == 0) + insertlump(u, p); + else + packetfree(p); + + ms = msec() - ms; + addstat2(StatRpcWriteNew, 1, StatRpcWriteNewTime, ms); + return ok; +} + +static void +lreadahead(u64int a, Arena *arena, u64int aa, int n) +{ + u8int buf[ClumpSize]; + Clump cl; + IAddr ia; + + while(n > 0) { + if (aa >= arena->memstats.used) + break; + if(readarena(arena, aa, buf, ClumpSize) < ClumpSize) + break; + if(unpackclump(&cl, buf, arena->clumpmagic) < 0) + break; + ia.addr = a; + ia.type = cl.info.type; + ia.size = cl.info.uncsize; + ia.blocks = (cl.info.size + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog; + insertscore(cl.info.score, &ia, 0); + a += ClumpSize + cl.info.size; + aa += ClumpSize + cl.info.size; + n--; + } +} + +static Packet* +readilump(Lump *u, IAddr *ia, u8int *score, int rac) +{ + Arena *arena; + ZBlock *zb; + Packet *p, *pp; + Clump cl; + u64int a, aa; + u8int sc[VtScoreSize]; + + trace(TraceLump, "readilump enter"); + arena = amapitoa(mainindex, ia->addr, &aa); + if(arena == nil){ + trace(TraceLump, "readilump amapitoa failed"); + return nil; + } + + trace(TraceLump, "readilump loadclump"); + zb = loadclump(arena, aa, ia->blocks, &cl, sc, paranoid); + if(zb == nil){ + trace(TraceLump, "readilump loadclump failed"); + return nil; + } + + if(ia->size != cl.info.uncsize){ + seterr(EInconsist, "index and clump size mismatch"); + freezblock(zb); + return nil; + } + if(ia->type != cl.info.type){ + seterr(EInconsist, "index and clump type mismatch"); + freezblock(zb); + return nil; + } + if(scorecmp(score, sc) != 0){ + seterr(ECrash, "score mismatch"); + freezblock(zb); + return nil; + } + + if(rac == 0) { + trace(TraceLump, "readilump readahead"); + a = ia->addr + ClumpSize + cl.info.size; + aa += ClumpSize + cl.info.size; + lreadahead(a, arena, aa, 20); + } + + trace(TraceLump, "readilump success"); + p = zblock2packet(zb, cl.info.uncsize); + freezblock(zb); + pp = packetdup(p, 0, packetsize(p)); + trace(TraceLump, "readilump insertlump"); + insertlump(u, pp); + trace(TraceLump, "readilump exit"); + return p; +} blob - /dev/null blob + 8a1e2d95a1620db98e5c29d4674386aad7d1e0cf (mode 644) --- /dev/null +++ src/cmd/venti/srv/lumpcache.c @@ -0,0 +1,417 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +/* #define CHECK(x) x */ +#define CHECK(x) + +typedef struct LumpCache LumpCache; + +enum +{ + HashLog = 9, + HashSize = 1<type = TWID8; + b->heap = TWID32; + b->next = last; + last = b; + } + lumpcache.free = last; + lumpcache.nheap = 0; +} + +Lump* +lookuplump(u8int *score, int type) +{ + uint ms; + Lump *b; + u32int h; + + ms = msec(); + trace(TraceLump, "lookuplump enter"); + + h = hashbits(score, HashLog); + + /* + * look for the block in the cache + */ + qlock(&lumpcache.lock); + CHECK(checklumpcache()); +again: + for(b = lumpcache.heads[h]; b != nil; b = b->next){ + if(scorecmp(score, b->score)==0 && type == b->type){ + addstat(StatLcacheHit, 1); + trace(TraceLump, "lookuplump hit"); + goto found; + } + } + + trace(TraceLump, "lookuplump miss"); + + /* + * missed: locate the block with the oldest second to last use. + * remove it from the heap, and fix up the heap. + */ + while(lumpcache.free == nil){ + trace(TraceLump, "lookuplump bump"); + CHECK(checklumpcache()); + if(bumplump() == nil){ + CHECK(checklumpcache()); + logerr(EAdmin, "all lump cache blocks in use"); + addstat(StatLcacheStall, 1); + CHECK(checklumpcache()); + rsleep(&lumpcache.full); + CHECK(checklumpcache()); + addstat(StatLcacheStall, -1); + goto again; + } + CHECK(checklumpcache()); + } + + addstat(StatLcacheMiss, 1); + b = lumpcache.free; + lumpcache.free = b->next; + + /* + * the new block has no last use, so assume it happens sometime in the middle +ZZZ this is not reasonable + */ + b->used = (b->used2 + lumpcache.now) / 2; + + /* + * rechain the block on the correct hash chain + */ + b->next = lumpcache.heads[h]; + lumpcache.heads[h] = b; + if(b->next != nil) + b->next->prev = b; + b->prev = nil; + + scorecp(b->score, score); + b->type = type; + b->size = 0; + b->data = nil; + +found: + b->ref++; + b->used2 = b->used; + b->used = lumpcache.now++; + if(b->heap != TWID32) + fixheap(b->heap, b); + CHECK(checklumpcache()); + qunlock(&lumpcache.lock); + + + addstat(StatLumpStall, 1); + qlock(&b->lock); + addstat(StatLumpStall, -1); + + trace(TraceLump, "lookuplump exit"); + addstat2(StatLcacheRead, 1, StatLcacheReadTime, msec()-ms); + return b; +} + +void +insertlump(Lump *b, Packet *p) +{ + u32int size; + + /* + * look for the block in the cache + */ + trace(TraceLump, "insertlump enter"); + qlock(&lumpcache.lock); + CHECK(checklumpcache()); +again: + + addstat(StatLcacheWrite, 1); + + /* + * missed: locate the block with the oldest second to last use. + * remove it from the heap, and fix up the heap. + */ + size = packetasize(p); +//ZZZ + while(lumpcache.avail < size){ + trace(TraceLump, "insertlump bump"); + CHECK(checklumpcache()); + if(bumplump() == nil){ + logerr(EAdmin, "all lump cache blocks in use"); + addstat(StatLcacheStall, 1); + CHECK(checklumpcache()); + rsleep(&lumpcache.full); + CHECK(checklumpcache()); + addstat(StatLcacheStall, -1); + goto again; + } + CHECK(checklumpcache()); + } + b->data = p; + b->size = size; + lumpcache.avail -= size; + CHECK(checklumpcache()); + qunlock(&lumpcache.lock); + trace(TraceLump, "insertlump exit"); +} + +void +putlump(Lump *b) +{ + if(b == nil) + return; + + trace(TraceLump, "putlump"); + qunlock(&b->lock); + qlock(&lumpcache.lock); + CHECK(checklumpcache()); + if(--b->ref == 0){ + if(b->heap == TWID32) + upheap(lumpcache.nheap++, b); + trace(TraceLump, "putlump wakeup"); + rwakeupall(&lumpcache.full); + } + CHECK(checklumpcache()); + qunlock(&lumpcache.lock); +} + +/* + * remove some lump from use and update the free list and counters + */ +static Lump* +bumplump(void) +{ + Lump *b; + u32int h; + + /* + * remove blocks until we find one that is unused + * referenced blocks are left in the heap even though + * they can't be scavenged; this is simple a speed optimization + */ + CHECK(checklumpcache()); + for(;;){ + if(lumpcache.nheap == 0){ + trace(TraceLump, "bumplump emptyheap"); + return nil; + } + b = lumpcache.heap[0]; + delheap(b); + if(!b->ref){ + trace(TraceLump, "bumplump wakeup"); + rwakeupall(&lumpcache.full); + break; + } + } + + /* + * unchain the block + */ + trace(TraceLump, "bumplump unchain"); + if(b->prev == nil){ + h = hashbits(b->score, HashLog); + if(lumpcache.heads[h] != b) + sysfatal("bad hash chains in lump cache"); + lumpcache.heads[h] = b->next; + }else + b->prev->next = b->next; + if(b->next != nil) + b->next->prev = b->prev; + + if(b->data != nil){ + packetfree(b->data); + b->data = nil; + lumpcache.avail += b->size; + b->size = 0; + } + b->type = TWID8; + + b->next = lumpcache.free; + lumpcache.free = b; + + CHECK(checklumpcache()); + trace(TraceLump, "bumplump exit"); + return b; +} + +/* + * delete an arbitrary block from the heap + */ +static void +delheap(Lump *db) +{ + fixheap(db->heap, lumpcache.heap[--lumpcache.nheap]); + db->heap = TWID32; +} + +/* + * push an element up or down to it's correct new location + */ +static void +fixheap(int i, Lump *b) +{ + if(upheap(i, b) == i) + downheap(i, b); +} + +static int +upheap(int i, Lump *b) +{ + Lump *bb; + u32int now; + int p; + + now = lumpcache.now; + for(; i != 0; i = p){ + p = (i - 1) >> 1; + bb = lumpcache.heap[p]; + if(b->used2 - now >= bb->used2 - now) + break; + lumpcache.heap[i] = bb; + bb->heap = i; + } + + lumpcache.heap[i] = b; + b->heap = i; + return i; +} + +static int +downheap(int i, Lump *b) +{ + Lump *bb; + u32int now; + int k; + + now = lumpcache.now; + for(; ; i = k){ + k = (i << 1) + 1; + if(k >= lumpcache.nheap) + break; + if(k + 1 < lumpcache.nheap && lumpcache.heap[k]->used2 - now > lumpcache.heap[k + 1]->used2 - now) + k++; + bb = lumpcache.heap[k]; + if(b->used2 - now <= bb->used2 - now) + break; + lumpcache.heap[i] = bb; + bb->heap = i; + } + + lumpcache.heap[i] = b; + b->heap = i; + return i; +} + +static void +findblock(Lump *bb) +{ + Lump *b, *last; + int h; + + last = nil; + h = hashbits(bb->score, HashLog); + for(b = lumpcache.heads[h]; b != nil; b = b->next){ + if(last != b->prev) + sysfatal("bad prev link"); + if(b == bb) + return; + last = b; + } + sysfatal("block score=%V type=%#x missing from hash table", bb->score, bb->type); +} + +void +checklumpcache(void) +{ + Lump *b; + u32int size, now, nfree; + int i, k, refed; + + now = lumpcache.now; + for(i = 0; i < lumpcache.nheap; i++){ + if(lumpcache.heap[i]->heap != i) + sysfatal("lc: mis-heaped at %d: %d", i, lumpcache.heap[i]->heap); + if(i > 0 && lumpcache.heap[(i - 1) >> 1]->used2 - now > lumpcache.heap[i]->used2 - now) + sysfatal("lc: bad heap ordering"); + k = (i << 1) + 1; + if(k < lumpcache.nheap && lumpcache.heap[i]->used2 - now > lumpcache.heap[k]->used2 - now) + sysfatal("lc: bad heap ordering"); + k++; + if(k < lumpcache.nheap && lumpcache.heap[i]->used2 - now > lumpcache.heap[k]->used2 - now) + sysfatal("lc: bad heap ordering"); + } + + refed = 0; + size = 0; + for(i = 0; i < lumpcache.nblocks; i++){ + b = &lumpcache.blocks[i]; + if(b->data == nil && b->size != 0) + sysfatal("bad size: %d data=%p", b->size, b->data); + if(b->ref && b->heap == TWID32) + refed++; + if(b->type != TWID8){ + findblock(b); + size += b->size; + } + if(b->heap != TWID32 + && lumpcache.heap[b->heap] != b) + sysfatal("lc: spurious heap value"); + } + if(lumpcache.avail != lumpcache.allowed - size){ + fprint(2, "mismatched available=%d and allowed=%d - used=%d space", lumpcache.avail, lumpcache.allowed, size); + *(int*)0=0; + } + + nfree = 0; + for(b = lumpcache.free; b != nil; b = b->next){ + if(b->type != TWID8 || b->heap != TWID32) + sysfatal("lc: bad free list"); + nfree++; + } + + if(lumpcache.nheap + nfree + refed != lumpcache.nblocks) + sysfatal("lc: missing blocks: %d %d %d %d", lumpcache.nheap, refed, nfree, lumpcache.nblocks); +} blob - /dev/null blob + 1b03f41ce7a65bee9625e3fb0dbe6659f386a10e (mode 644) --- /dev/null +++ src/cmd/venti/srv/lumpqueue.c @@ -0,0 +1,187 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +typedef struct LumpQueue LumpQueue; +typedef struct WLump WLump; + +enum +{ + MaxLumpQ = 1 << 3 /* max. lumps on a single write queue, must be pow 2 */ +}; + +struct WLump +{ + Lump *u; + Packet *p; + int creator; + int gen; + uint ms; +}; + +struct LumpQueue +{ + QLock lock; + Rendez flush; + Rendez full; + Rendez empty; + WLump q[MaxLumpQ]; + int w; + int r; +}; + +static LumpQueue *lumpqs; +static int nqs; + +static QLock glk; +static int gen; + +static void queueproc(void *vq); + +int +initlumpqueues(int nq) +{ + LumpQueue *q; + + int i; + nqs = nq; + + lumpqs = MKNZ(LumpQueue, nq); + + for(i = 0; i < nq; i++){ + q = &lumpqs[i]; + q->full.l = &q->lock; + q->empty.l = &q->lock; + q->flush.l = &q->lock; + + if(vtproc(queueproc, q) < 0){ + seterr(EOk, "can't start write queue slave: %r"); + return -1; + } + if(vtproc(queueproc, q) < 0){ + seterr(EOk, "can't start write queue slave: %r"); + return -1; + } + if(vtproc(queueproc, q) < 0){ + seterr(EOk, "can't start write queue slave: %r"); + return -1; + } + if(vtproc(queueproc, q) < 0){ + seterr(EOk, "can't start write queue slave: %r"); + return -1; + } + if(vtproc(queueproc, q) < 0){ + seterr(EOk, "can't start write queue slave: %r"); + return -1; + } + } + + return 0; +} + +/* + * queue a lump & it's packet data for writing + */ +int +queuewrite(Lump *u, Packet *p, int creator, uint ms) +{ + LumpQueue *q; + int i; + + trace(TraceProc, "queuewrite"); + i = indexsect(mainindex, u->score); + if(i < 0 || i >= nqs){ + seterr(EBug, "internal error: illegal index section in queuewrite"); + return -1; + } + + q = &lumpqs[i]; + + qlock(&q->lock); + while(q->r == ((q->w + 1) & (MaxLumpQ - 1))){ + trace(TraceProc, "queuewrite sleep"); + rsleep(&q->full); + } + + q->q[q->w].u = u; + q->q[q->w].p = p; + q->q[q->w].creator = creator; + q->q[q->w].ms = ms; + q->q[q->w].gen = gen; + q->w = (q->w + 1) & (MaxLumpQ - 1); + + trace(TraceProc, "queuewrite wakeup"); + rwakeup(&q->empty); + + qunlock(&q->lock); + + return 0; +} + +void +flushqueue(void) +{ + int i; + LumpQueue *q; + + if(!lumpqs) + return; + + trace(TraceProc, "flushqueue"); + + qlock(&glk); + gen++; + qunlock(&glk); + + for(i=0; insects; i++){ + q = &lumpqs[i]; + qlock(&q->lock); + while(q->w != q->r && gen - q->q[q->r].gen > 0){ + trace(TraceProc, "flushqueue sleep q%d", i); + rsleep(&q->flush); + } + qunlock(&q->lock); + } +} + +static void +queueproc(void *vq) +{ + LumpQueue *q; + Lump *u; + Packet *p; + int creator; + uint ms; + + threadsetname("queueproc"); + + q = vq; + for(;;){ + qlock(&q->lock); + while(q->w == q->r){ + trace(TraceProc, "queueproc sleep empty"); + rsleep(&q->empty); + } + + u = q->q[q->r].u; + p = q->q[q->r].p; + creator = q->q[q->r].creator; + ms = q->q[q->r].ms; + + q->r = (q->r + 1) & (MaxLumpQ - 1); + trace(TraceProc, "queueproc wakeup flush"); + rwakeupall(&q->flush); + + trace(TraceProc, "queueproc wakeup full"); + rwakeup(&q->full); + + qunlock(&q->lock); + + trace(TraceProc, "queueproc writelump %V", u->score); + if(writeqlump(u, p, creator, ms) < 0) + fprint(2, "failed to write lump for %V: %r", u->score); + trace(TraceProc, "queueproc wrotelump %V", u->score); + + putlump(u); + } +} blob - /dev/null blob + 2fd4508e888a4477c741bca2aa227aad0687116e (mode 644) --- /dev/null +++ src/cmd/venti/srv/mkfile @@ -0,0 +1,146 @@ +<$PLAN9/src/mkhdr +CC=9c + +AR=ar + +LIBOFILES=\ + arena.$O\ + arenas.$O\ + bloom.$O\ + buildbuck.$O\ + clump.$O\ + config.$O\ + conv.$O\ + dcache.$O\ + dump.$O\ + graph.$O\ + httpd.$O\ + icache.$O\ + icachewrite.$O\ + ifile.$O\ + index.$O\ + lump.$O\ + lumpcache.$O\ + lumpqueue.$O\ + part.$O\ + png.$O\ + round.$O\ + score.$O\ + sortientry.$O\ + stats.$O\ + syncarena.$O\ + syncindex0.$O\ + trace.$O\ + unwhack.$O\ + utils.$O\ + unittoull.$O\ + whack.$O\ + xml.$O\ + zblock.$O\ + zeropart.$O\ + +SLIB=libvs.a + +LIB=$SLIB + +HFILES= dat.h\ + fns.h\ + stdinc.h\ + +TARG=\ + venti\ + fmtarenas\ + fmtbloom\ + fmtisect\ + fmtindex\ + buildindex\ + checkarenas\ + checkindex\ + clumpstats\ + findscore\ + rdarena\ + wrarena\ + syncindex\ + printarena\ + verifyarena\ + +OFILES= + +BIN=$BIN/venti + +it:V: $O.venti + +$O.venti: # debugmalloc2.$O # debugmalloc.$O #_p9dir.$O debugmalloc.$O + +CLEANFILES=$CLEANFILES $SLIB + +<$PLAN9/src/mkmany + +$SLIB: $LIBOFILES + $AR rvc $SLIB $LIBOFILES + +# xml.c:D: mkxml dat.h +# ./mkxml dat.h > xml.c + +ainstall:V: ${TARG:%=%.ainstall} + +%.ainstall:V: $O.% + scp $prereq amsterdam:/usr/local/bin/venti/$stem + +test:VQ: ${TARG:%=o.%} + slay o.venti|rc + vtmp=/home/tmp + echo '**********' FMTARENAS + ./o.fmtarenas -a 40M -b 8k arenas $vtmp/arena + echo '**********' FMTBLOOM + ./o.fmtbloom -s 10M $vtmp/bloom + echo '**********' FMTISECT + ./o.fmtisect -b 8k isect $vtmp/isect + ( + echo index main + echo isect $vtmp/isect + echo arenas $vtmp/arena + echo bloom $vtmp/bloom + echo webroot $HOME/src/venti/www + echo mem 64M + echo icmem 64M + echo bcmem 64M + ) >vtmp.conf + echo '**********' FMTINDEX + ./o.fmtindex vtmp.conf + echo '**********' VENTI + # ./o.venti -c vtmp.conf -B 64M -I 64M -C 64M -a 'tcp!*!17034' -h 'tcp!*!8001' >a 2>&1 & + ./o.venti -c vtmp.conf -a 'tcp!*!17034' -h 'tcp!*!8001' >a 2>&1 & + sleep 5 + echo '**********' VAC + venti='tcp!127.0.0.1!17034' export venti + 9 time vac /usr/local/plan9 >a.vac + case ${websync:-no} in + yes) + echo '**********' SYNC VIA WEB + hget http://127.0.0.1:8001/flushdcache + hget http://127.0.0.1:8001/flushicache + hget http://127.0.0.1:8001/flushdcache + echo '**********' KILL VENTI + killall -9 o.venti + ;; + no) + echo '**********' KILL VENTI + killall -9 o.venti + echo '**********' SYNCINDEX + ./o.syncindex -B64M -I64M -f vtmp.conf + ;; + esac + echo '**********' CHECKINDEX + ./o.checkindex -B64M vtmp.conf /home/tmp/check >check.out + wc check.out + +luadisk.o: luadisk.c + gcc -c -ggdb -Wall -I/usr/include/lua50 luadisk.c + +libluadisk.so: luadisk.o + gcc -shared -o $target luadisk.o -llua50 -llualib50 + +$O.xwrarena: xwrarena.$O + $LD -o $target xwrarena.$O + blob - /dev/null blob + 9b80b6e691bfd3e6ee94946fc825623966d0661b (mode 644) --- /dev/null +++ src/cmd/venti/srv/part.c @@ -0,0 +1,383 @@ +#ifdef PLAN9PORT /* SORRY! */ +#include +#include +#include +#endif +#include "stdinc.h" +#include +#include "dat.h" +#include "fns.h" + +u32int maxblocksize; +int readonly; + +static int +strtoullsuf(char *p, char **pp, int rad, u64int *u) +{ + u64int v; + + if(!isdigit(*p)) + return -1; + v = strtoull(p, &p, rad); + switch(*p){ + case 'k': + case 'K': + v *= 1024; + p++; + break; + case 'm': + case 'M': + v *= 1024*1024; + p++; + break; + case 'g': + case 'G': + v *= 1024*1024*1024; + p++; + break; + case 't': + case 'T': + v *= 1024*1024; + v *= 1024*1024; + p++; + break; + } + *pp = p; + *u = v; + return 0; +} + +static int +parsepart(char *name, char **file, u64int *lo, u64int *hi) +{ + char *p; + + *file = estrdup(name); + if((p = strrchr(*file, ':')) == nil){ + *lo = 0; + *hi = 0; + return 0; + } + *p++ = 0; + if(*p == '-') + *lo = 0; + else{ + if(strtoullsuf(p, &p, 0, lo) < 0){ + free(*file); + return -1; + } + } + if(*p == '-') + p++; + if(*p == 0){ + *hi = 0; + return 0; + } + if(strtoullsuf(p, &p, 0, hi) < 0 || *p != 0){ + free(*file); + return -1; + } + return 0; +} + +Part* +initpart(char *name, int mode) +{ + Part *part; + Dir *dir; + char *file; + u64int lo, hi; + + if(parsepart(name, &file, &lo, &hi) < 0) + return nil; + trace(TraceDisk, "initpart %s file %s lo 0x%llx hi 0x%llx", name, file, lo, hi); + part = MKZ(Part); + part->name = estrdup(name); + part->filename = estrdup(file); + if(readonly){ + mode &= (OREAD|OWRITE|ORDWR); + mode |= OREAD; + } + part->fd = open(file, mode); + if(part->fd < 0){ + if((mode&(OREAD|OWRITE|ORDWR)) == ORDWR) + part->fd = open(file, (mode&~ORDWR)|OREAD); + if(part->fd < 0){ + freepart(part); + fprint(2, "can't open partition='%s': %r\n", file); + seterr(EOk, "can't open partition='%s': %r", file); + fprint(2, "%r\n"); + free(file); + return nil; + } + fprint(2, "warning: %s opened for reading only\n", name); + } + part->offset = lo; + dir = dirfstat(part->fd); + if(dir == nil){ + freepart(part); + seterr(EOk, "can't stat partition='%s': %r", file); + free(file); + return nil; + } + if(dir->length == 0){ + free(dir); + freepart(part); + seterr(EOk, "can't determine size of partition %s", file); + free(file); + return nil; + } + if(dir->length < hi || dir->length < lo){ + freepart(part); + seterr(EOk, "partition '%s': bounds out of range (max %lld)", name, dir->length); + free(dir); + free(file); + return nil; + } + if(hi == 0) + hi = dir->length; + part->size = hi - part->offset; +#ifdef _LIBC_H_ + { + struct statfs sfs; + if(fstatfs(part->fd, &sfs) >= 0) + part->fsblocksize = sfs.f_bsize; + } +#endif + free(dir); + return part; +} + +void +freepart(Part *part) +{ + if(part == nil) + return; + if(part->fd >= 0) + close(part->fd); + free(part->name); + free(part); +} + +void +partblocksize(Part *part, u32int blocksize) +{ + if(part->blocksize) + sysfatal("resetting partition=%s's block size", part->name); + part->blocksize = blocksize; + if(blocksize > maxblocksize) + maxblocksize = blocksize; +} + +/* + * Read/write some amount of data between a block device or file and a memory buffer. + * + * Most Unix systems require that when accessing a block device directly, + * the buffer, offset, and count are all multiples of the device block size, + * making this a lot more complicated than it otherwise would be. + * + * Most of our callers will make things easy on us, but for some callers it's best + * if we just do the work here, with only one place to get it right (hopefully). + * + * If everything is aligned properly, prwb will try to do big transfers in the main + * body of the loop: up to MaxIo bytes at a time. If everything isn't aligned properly, + * we work one block at a time. + */ +#undef min +#define min(a, b) ((a) < (b) ? (a) : (b)) +int +prwb(char *name, int fd, int isread, u64int offset, void *vbuf, u32int count, u32int blocksize) +{ + char *op; + u8int *buf, *tmp, *freetmp, *dst; + u32int c, delta, icount, opsize; + int r; + + buf = vbuf; + tmp = nil; + freetmp = nil; + icount = count; + opsize = blocksize; + + if(count == 0){ + logerr(EStrange, "pwrb %s called to %s 0 bytes", name, isread ? "read" : "write"); + return 0; + } + + assert(blocksize > 0); + + /* allocate blocksize-aligned temp buffer if needed */ + if((ulong)offset%blocksize || (ulong)buf%blocksize || count%blocksize){ + if((freetmp = malloc(blocksize*2)) == nil) + return -1; + tmp = freetmp; + tmp += blocksize - (ulong)tmp%blocksize; + } + + /* handle beginning fringe */ + if((delta = (ulong)offset%blocksize) != 0){ + assert(tmp != nil); + if((r=pread(fd, tmp, blocksize, offset-delta)) != blocksize){ + dst = tmp; + offset = offset-delta; + op = "read"; + goto Error; + } + c = min(count, blocksize-delta); + assert(c > 0 && c < blocksize); + if(isread) + memmove(buf, tmp+delta, c); + else{ + memmove(tmp+delta, buf, c); + if((r=pwrite(fd, tmp, blocksize, offset-delta)) != blocksize){ + dst = tmp; + offset = offset-delta; + op = "read"; + goto Error; + } + } + assert(c > 0); + offset += c; + buf += c; + count -= c; + } + + /* handle full blocks */ + while(count >= blocksize){ + assert((ulong)offset%blocksize == 0); + if((ulong)buf%blocksize){ + assert(tmp != nil); + dst = tmp; + opsize = blocksize; + }else{ + dst = buf; + opsize = count - count%blocksize; + if(opsize > MaxIo) + opsize = MaxIo; + } + if(isread){ + if((r=pread(fd, dst, opsize, offset))<=0 || r%blocksize){ + op = "read"; + goto Error; + } + if(dst == tmp){ + assert(r == blocksize); + memmove(buf, tmp, blocksize); + } + }else{ + if(dst == tmp){ + assert(opsize == blocksize); + memmove(dst, buf, blocksize); + } + if((r=pwrite(fd, dst, opsize, offset))<=0 || r%blocksize){ + op = "write"; + goto Error; + } + if(dst == tmp) + assert(r == blocksize); + } + assert(r > 0); + offset += r; + buf += r; + count -= r; + } + + /* handle ending fringe */ + if(count > 0){ + assert((ulong)offset%blocksize == 0); + assert(tmp != nil); + /* + * Complicated condition: if we're reading it's okay to get less than + * a block as long as it's enough to satisfy the read - maybe this is + * a normal file. (We never write to normal files, or else things would + * be even more complicated.) + */ + r = pread(fd, tmp, blocksize, offset); + if((isread && r < count) || (!isread && r != blocksize)){ +print("FAILED isread=%d r=%d count=%d blocksize=%d\n", isread, r, count, blocksize); + dst = tmp; + op = "read"; + goto Error; + } + if(isread) + memmove(buf, tmp, count); + else{ + memmove(tmp, buf, count); + if(pwrite(fd, tmp, opsize, offset) != blocksize){ + dst = tmp; + op = "write"; + goto Error; + } + } + } + if(freetmp) + free(freetmp); + return icount; + +Error: + seterr(EAdmin, "%s %s offset 0x%llux count %ud buf %p returned %d: %r", + op, name, offset, opsize, dst, r); + if(freetmp) + free(freetmp); + return -1; +} + +int +rwpart(Part *part, int isread, u64int offset, u8int *buf, u32int count) +{ + u32int blocksize; + + trace(TraceDisk, "%s %s %ud at 0x%llx", + isread ? "read" : "write", part->name, count, offset); + if(offset >= part->size || offset+count > part->size){ + seterr(EStrange, "out of bounds %s offset 0x%llux count %ud to partition %s size 0x%llux", + isread ? "read" : "write", offset, count, part->name, part->size); + return -1; + } + + blocksize = part->fsblocksize; + if(blocksize == 0) + blocksize = part->blocksize; + if(blocksize == 0) + blocksize = 4096; + + return prwb(part->filename, part->fd, isread, part->offset+offset, buf, count, blocksize); +} + +int +readpart(Part *part, u64int offset, u8int *buf, u32int count) +{ + return rwpart(part, 1, offset, buf, count); +} + +int +writepart(Part *part, u64int offset, u8int *buf, u32int count) +{ + return rwpart(part, 0, offset, buf, count); +} + +ZBlock* +readfile(char *name) +{ + Part *p; + ZBlock *b; + + p = initpart(name, OREAD); + if(p == nil) + return nil; + b = alloczblock(p->size, 0, p->blocksize); + if(b == nil){ + seterr(EOk, "can't alloc %s: %r", name); + freepart(p); + return nil; + } + if(readpart(p, 0, b->data, p->size) < 0){ + seterr(EOk, "can't read %s: %r", name); + freepart(p); + freezblock(b); + return nil; + } + freepart(p); + return b; +} + blob - /dev/null blob + 966b7e96b60dbb249746636f8e640e3bfa036765 (mode 644) --- /dev/null +++ src/cmd/venti/srv/png.c @@ -0,0 +1,241 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +enum +{ + IDATSIZE = 20000, + FilterNone = 0 +}; + +typedef struct ZlibR ZlibR; +typedef struct ZlibW ZlibW; + +struct ZlibR +{ + uchar *data; + int width; + int dx; + int dy; + int x; + int y; + int pixwid; +}; + +struct ZlibW +{ + Hio *io; + uchar *buf; + uchar *b; + uchar *e; +}; + +static ulong *crctab; +static uchar PNGmagic[] = { 137, 'P', 'N', 'G', '\r', '\n', 26, '\n'}; + +static void +put4(uchar *a, ulong v) +{ + a[0] = v>>24; + a[1] = v>>16; + a[2] = v>>8; + a[3] = v; +} + +static void +chunk(Hio *io, char *type, uchar *d, int n) +{ + uchar buf[4]; + ulong crc = 0; + + if(strlen(type) != 4) + return; + put4(buf, n); + hwrite(io, buf, 4); + hwrite(io, type, 4); + hwrite(io, d, n); + crc = blockcrc(crctab, crc, type, 4); + crc = blockcrc(crctab, crc, d, n); + put4(buf, crc); + hwrite(io, buf, 4); +} + +static int +zread(void *va, void *buf, int n) +{ + int a, i, pixels, pixwid; + uchar *b, *e, *img; + ZlibR *z; + + z = va; + pixwid = z->pixwid; + b = buf; + e = b+n; + while(b+pixwid <= e){ + if(z->y >= z->dy) + break; + if(z->x == 0) + *b++ = FilterNone; + pixels = (e-b)/pixwid; + if(pixels > z->dx - z->x) + pixels = z->dx - z->x; + img = z->data + z->width*z->y + pixwid*z->x; + memmove(b, img, pixwid*pixels); + if(pixwid == 4){ + /* + * Convert to non-premultiplied alpha. + */ + for(i=0; i= a) + b[0] = a; + b[0] = (b[0]*255)/a; + if(b[1] >= a) + b[1] = a; + b[1] = (b[1]*255)/a; + if(b[2] >= a) + b[2] = a; + b[2] = (b[2]*255)/a; + } + } + }else + b += pixwid*pixels; + + z->x += pixels; + if(z->x >= z->dx){ + z->x = 0; + z->y++; + } + } + return b - (uchar*)buf; +} + +static void +IDAT(ZlibW *z) +{ + chunk(z->io, "IDAT", z->buf, z->b - z->buf); + z->b = z->buf; +} + +static int +zwrite(void *va, void *buf, int n) +{ + int m; + uchar *b, *e; + ZlibW *z; + + z = va; + b = buf; + e = b+n; + + while(b < e){ + m = z->e - z->b; + if(m > e - b) + m = e - b; + memmove(z->b, b, m); + z->b += m; + b += m; + if(z->b >= z->e) + IDAT(z); + } + return n; +} + +static Memimage* +memRGBA(Memimage *i) +{ + Memimage *ni; + char buf[32]; + ulong dst; + + /* + * [A]BGR because we want R,G,B,[A] in big-endian order. Sigh. + */ + chantostr(buf, i->chan); + if(strchr(buf, 'a')) + dst = ABGR32; + else + dst = BGR24; + + if(i->chan == dst) + return i; + + qlock(&memdrawlock); + ni = allocmemimage(i->r, dst); + if(ni) + memimagedraw(ni, ni->r, i, i->r.min, nil, i->r.min, S); + qunlock(&memdrawlock); + return ni; +} + +int +writepng(Hio *io, Memimage *m) +{ + static int first = 1; + static QLock lk; + uchar buf[200], *h; + Memimage *rgb; + ZlibR zr; + ZlibW zw; + + if(first){ + qlock(&lk); + if(first){ + deflateinit(); + crctab = mkcrctab(0xedb88320); + first = 0; + } + qunlock(&lk); + } + + rgb = memRGBA(m); + if(rgb == nil) + return -1; + + hwrite(io, PNGmagic, sizeof PNGmagic); + + /* IHDR chunk */ + h = buf; + put4(h, Dx(m->r)); h += 4; + put4(h, Dy(m->r)); h += 4; + *h++ = 8; /* 8 bits per channel */ + if(rgb->chan == BGR24) + *h++ = 2; /* RGB */ + else + *h++ = 6; /* RGBA */ + *h++ = 0; /* compression - deflate */ + *h++ = 0; /* filter - none */ + *h++ = 0; /* interlace - none */ + chunk(io, "IHDR", buf, h-buf); + + /* image data */ + zr.dx = Dx(m->r); + zr.dy = Dy(m->r); + zr.width = rgb->width * sizeof(ulong); + zr.data = rgb->data->bdata; + zr.x = 0; + zr.y = 0; + zr.pixwid = chantodepth(rgb->chan)/8; + zw.io = io; + zw.buf = vtmalloc(IDATSIZE); + zw.b = zw.buf; + zw.e = zw.b + IDATSIZE; + if(deflatezlib(&zw, zwrite, &zr, zread, 6, 0) < 0){ + free(zw.buf); + return -1; + } + if(zw.b > zw.buf) + IDAT(&zw); + free(zw.buf); + chunk(io, "IEND", nil, 0); + + if(m != rgb){ + qlock(&memdrawlock); + freememimage(rgb); + qunlock(&memdrawlock); + } + return 0; +} blob - /dev/null blob + c305e81804e48e96685e0a0571bc1e283809eb9a (mode 644) --- /dev/null +++ src/cmd/venti/srv/printarena.c @@ -0,0 +1,130 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +void +usage(void) +{ + fprint(2, "usage: printarena arenafile [offset]\n"); + threadexitsall("usage"); +} + +static void +rdarena(Arena *arena, u64int offset) +{ + u64int a, aa, e; + u32int magic; + Clump cl; + uchar score[VtScoreSize]; + ZBlock *lump; + + printarena(2, arena); + + a = arena->base; + e = arena->base + arena->size; + if(offset != ~(u64int)0) { + if(offset >= e-a) + sysfatal("bad offset %llud >= %llud\n", + offset, e-a); + aa = offset; + } else + aa = 0; + + for(; aa < e; aa += ClumpSize+cl.info.size) { + magic = clumpmagic(arena, aa); + if(magic == ClumpFreeMagic) + break; + if(magic != arena->clumpmagic) { + fprint(2, "illegal clump magic number %#8.8ux offset %llud\n", + magic, aa); + break; + } + lump = loadclump(arena, aa, 0, &cl, score, 0); + if(lump == nil) { + fprint(2, "clump %llud failed to read: %r\n", aa); + break; + } + if(cl.info.type != VtCorruptType) { + scoremem(score, lump->data, cl.info.uncsize); + if(scorecmp(cl.info.score, score) != 0) { + fprint(2, "clump %llud has mismatched score\n", aa); + break; + } + if(vttypevalid(cl.info.type) < 0) { + fprint(2, "clump %llud has bad type %d\n", aa, cl.info.type); + break; + } + } + print("%22llud %V %3d %5d\n", aa, score, cl.info.type, cl.info.uncsize); + freezblock(lump); + } + print("end offset %llud\n", aa); +} + +void +threadmain(int argc, char *argv[]) +{ + char *file; + Arena *arena; + u64int offset, aoffset; + Part *part; + Dir *d; + uchar buf[8192]; + ArenaHead head; + + readonly = 1; /* for part.c */ + aoffset = 0; + ARGBEGIN{ + case 'o': + aoffset = strtoull(EARGF(usage()), 0, 0); + break; + default: + usage(); + break; + }ARGEND + + offset = ~(u64int)0; + switch(argc) { + default: + usage(); + case 2: + offset = strtoull(argv[1], 0, 0); + /* fall through */ + case 1: + file = argv[0]; + } + + + ventifmtinstall(); + statsinit(); + + if((d = dirstat(file)) == nil) + sysfatal("can't stat file %s: %r", file); + + part = initpart(file, OREAD|ODIRECT); + if(part == nil) + sysfatal("can't open file %s: %r", file); + if(readpart(part, aoffset, buf, sizeof buf) < 0) + sysfatal("can't read file %s: %r", file); + + if(unpackarenahead(&head, buf) < 0) + sysfatal("corrupted arena header: %r"); + + print("# arena head version=%d name=%.*s blocksize=%d size=%lld clumpmagic=0x%.8ux\n", + head.version, ANameSize, head.name, head.blocksize, + head.size, head.clumpmagic); + + if(aoffset+head.size > d->length) + sysfatal("arena is truncated: want %llud bytes have %llud\n", + head.size, d->length); + + partblocksize(part, head.blocksize); + initdcache(8 * MaxDiskBlock); + + arena = initarena(part, aoffset, head.size, head.blocksize); + if(arena == nil) + sysfatal("initarena: %r"); + + rdarena(arena, offset); + threadexitsall(0); +} blob - /dev/null blob + 111db0187fa47017c41c628517fbafdd4602b060 (mode 644) --- /dev/null +++ src/cmd/venti/srv/printarenas.c @@ -0,0 +1,113 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" +#include + +Biobuf bout; + +static void +pie(IEntry *ie) +{ + Bprint(&bout, "%22lld %V %3d %5d\n", + ie->ia.addr, ie->score, ie->ia.type, ie->ia.size); +} + +void +usage(void) +{ + fprint(2, "usage: printarenas [-B blockcachesize] config [arenaname...]\n"); + threadexitsall(0); +} + +Config conf; + +int +shoulddump(char *name, int argc, char **argv) +{ + int i; + + if(argc == 0) + return 1; + for(i=0; imemstats.clumps; clump += n){ + n = ClumpChunks; + if(n > arena->memstats.clumps - clump) + n = arena->memstats.clumps - clump; + if(readclumpinfos(arena, clump, cis, n) != n){ + fprint(2, "arena directory read failed: %r\n"); + break; + } + + for(i = 0; i < n; i++){ + ci = &cis[i]; + ie.ia.type = ci->type; + ie.ia.size = ci->uncsize; + ie.ia.addr = a; + a += ci->size + ClumpSize; + ie.ia.blocks = (ci->size + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog; + scorecp(ie.score, ci->score); + pie(&ie); + } + } + free(cis); +} + +void +threadmain(int argc, char *argv[]) +{ + int i; + Index *ix; + u32int bcmem; + + bcmem = 0; + ARGBEGIN{ + case 'B': + bcmem = unittoull(ARGF()); + break; + default: + usage(); + break; + }ARGEND + + if(argc < 1) + usage(); + + ventifmtinstall(); + + if(initventi(argv[0], &conf) < 0) + sysfatal("can't init venti: %r"); + + if(bcmem < maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16)) + bcmem = maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16); + if(0) fprint(2, "initialize %d bytes of disk block cache\n", bcmem); + initdcache(bcmem); + + Binit(&bout, 1, OWRITE); + ix = mainindex; + for(i=0; inarenas; i++) + if(shoulddump(ix->arenas[i]->name, argc-1, argv+1)) + dumparena(ix->arenas[i], ix->amap[i].start); + Bterm(&bout); + threadexitsall(0); +} blob - /dev/null blob + edbcf7934c142fafec45bf43d62cd229f7a61d80 (mode 644) --- /dev/null +++ src/cmd/venti/srv/printindex.c @@ -0,0 +1,99 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" +#include + +Biobuf bout; + +static void +pie(IEntry *ie) +{ + Bprint(&bout, "%22lld %V %3d %5d\n", + ie->ia.addr, ie->score, ie->ia.type, ie->ia.size); +} + +void +usage(void) +{ + fprint(2, "usage: printindex [-B blockcachesize] config [isectname...]\n"); + threadexitsall(0); +} + +Config conf; + +int +shoulddump(char *name, int argc, char **argv) +{ + int i; + + if(argc == 0) + return 1; + for(i=0; iblocksize); + for(i=0; iblocks; i++){ + off = is->blockbase+(u64int)is->blocksize*i; + if(readpart(is->part, off, buf, is->blocksize) < 0) + fprint(2, "read %s at 0x%llux: %r\n", is->part->name, off); + else{ + unpackibucket(&ib, buf, is->bucketmagic); + for(j=0; jnarenas + mainindex->nsects * 4 + 16)) + bcmem = maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16); + if(0) fprint(2, "initialize %d bytes of disk block cache\n", bcmem); + initdcache(bcmem); + + ix = mainindex; + Binit(&bout, 1, OWRITE); + for(i=0; insects; i++) + if(shoulddump(ix->sects[i]->name, argc-1, argv+1)) + dumpisect(ix->sects[i]); + Bterm(&bout); + threadexitsall(0); +} blob - /dev/null blob + f3392ef8131ee0fb3ad964176462a186da0e47cd (mode 644) --- /dev/null +++ src/cmd/venti/srv/printmap.c @@ -0,0 +1,42 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +void +usage(void) +{ + fprint(2, "usage: printmap [-B blockcachesize] config\n"); + threadexitsall("usage"); +} + +Config conf; + +void +threadmain(int argc, char *argv[]) +{ + u32int bcmem; + int fix; + + fix = 0; + bcmem = 0; + ARGBEGIN{ + case 'B': + bcmem = unittoull(ARGF()); + break; + default: + usage(); + break; + }ARGEND + + if(!fix) + readonly = 1; + + if(argc != 1) + usage(); + + if(initventi(argv[0], &conf) < 0) + sysfatal("can't init venti: %r"); + + printindex(1, mainindex); + threadexitsall(0); +} blob - /dev/null blob + 909cc2067dfba7d272cd6372bddb7d0567c7dbd7 (mode 644) --- /dev/null +++ src/cmd/venti/srv/rdarena.c @@ -0,0 +1,91 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +static int verbose; + +void +usage(void) +{ + fprint(2, "usage: rdarena [-v] arenapart arena\n"); + threadexitsall(0); +} + +static void +rdarena(Arena *arena) +{ + ZBlock *b; + u64int a, e; + u32int bs; + + fprint(2, "copying %s to standard output\n", arena->name); + printarena(2, arena); + + bs = MaxIoSize; + if(bs < arena->blocksize) + bs = arena->blocksize; + + b = alloczblock(bs, 0, arena->blocksize); + e = arena->base + arena->size + arena->blocksize; + for(a = arena->base - arena->blocksize; a + arena->blocksize <= e; a += bs){ + if(a + bs > e) + bs = arena->blocksize; + if(readpart(arena->part, a, b->data, bs) < 0) + fprint(2, "can't copy %s, read at %lld failed: %r\n", arena->name, a); + if(write(1, b->data, bs) != bs) + sysfatal("can't copy %s, write at %lld failed: %r", arena->name, a); + } + + freezblock(b); +} + +void +threadmain(int argc, char *argv[]) +{ + ArenaPart *ap; + Part *part; + char *file, *aname; + int i; + + ventifmtinstall(); + statsinit(); + + ARGBEGIN{ + case 'v': + verbose++; + break; + default: + usage(); + break; + }ARGEND + + readonly = 1; + + if(argc != 2) + usage(); + + file = argv[0]; + aname = argv[1]; + + part = initpart(file, OREAD|ODIRECT); + if(part == nil) + sysfatal("can't open partition %s: %r", file); + + ap = initarenapart(part); + if(ap == nil) + sysfatal("can't initialize arena partition in %s: %r", file); + + if(verbose) + printarenapart(2, ap); + + initdcache(8 * MaxDiskBlock); + + for(i = 0; i < ap->narenas; i++){ + if(strcmp(ap->arenas[i]->name, aname) == 0){ + rdarena(ap->arenas[i]); + threadexitsall(0); + } + } + + sysfatal("couldn't find arena %s\n", aname); +} blob - /dev/null blob + bbf4a478a1276fc07c80b6c98f611cc74ded5a36 (mode 644) --- /dev/null +++ src/cmd/venti/srv/round.c @@ -0,0 +1,102 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +void +waitforkick(Round *r) +{ + int n; + + qlock(&r->lock); + r->last = r->current; + assert(r->current+1 == r->next); + rwakeupall(&r->finish); + while(!r->doanother) + rsleep(&r->start); + n = r->next++; + r->current = n; + r->doanother = 0; + qunlock(&r->lock); +} + +static void +_kickround(Round *r, int wait) +{ + int n; + + if(!r->doanother) + trace(TraceProc, "kick %s", r->name); + r->doanother = 1; + rwakeup(&r->start); + if(wait){ + n = r->next; + while((int)(n - r->last) > 0){ + r->doanother = 1; + rwakeup(&r->start); + rsleep(&r->finish); + } + } +} + +void +kickround(Round *r, int wait) +{ + qlock(&r->lock); + _kickround(r, wait); + qunlock(&r->lock); +} + +void +initround(Round *r, char *name, int delay) +{ + memset(r, 0, sizeof *r); + r->name = name; + r->start.l = &r->lock; + r->finish.l = &r->lock; + r->delaywait.l = &r->lock; + r->last = 0; + r->current = 0; + r->next = 1; + r->doanother = 0; + r->delaytime = delay; +} + +void +delaykickround(Round *r) +{ + qlock(&r->lock); + r->delaykick = 1; + rwakeup(&r->delaywait); + qunlock(&r->lock); +} + +void +delaykickroundproc(void *v) +{ + Round *r = v; + int n; + + threadsetname("delaykickproc %s", r->name); + qlock(&r->lock); + for(;;){ + while(r->delaykick == 0){ + trace(TraceProc, "sleep"); + rsleep(&r->delaywait); + } + + n = r->next; + qunlock(&r->lock); + + trace(TraceProc, "waitround 0x%ux", (uint)n); + sleep(r->delaytime); + + qlock(&r->lock); + if(n == r->next){ + trace(TraceProc, "kickround 0x%ux", (uint)n); + _kickround(r, 1); + } + + trace(TraceProc, "finishround 0x%ux", (uint)n); + } +} + blob - /dev/null blob + 0809e84f6468380cf41efd74b55daab069c16ae6 (mode 644) --- /dev/null +++ src/cmd/venti/srv/score.c @@ -0,0 +1,43 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +u8int zeroscore[VtScoreSize]; + +void +scoremem(u8int *score, u8int *buf, int n) +{ + DigestState s; + + memset(&s, 0, sizeof s); + sha1(buf, n, score, &s); +} + +static int +hexv(int c) +{ + if(c >= '0' && c <= '9') + return c - '0'; + if(c >= 'a' && c <= 'f') + return c - 'a' + 10; + if(c >= 'A' && c <= 'F') + return c - 'A' + 10; + return -1; +} + +int +strscore(char *s, u8int *score) +{ + int i, c, d; + + for(i = 0; i < VtScoreSize; i++){ + c = hexv(s[2 * i]); + if(c < 0) + return -1; + d = hexv(s[2 * i + 1]); + if(d < 0) + return -1; + score[i] = (c << 4) + d; + } + return s[2 * i] == '\0'; +} blob - /dev/null blob + e1fc24b09097954d091619b2c96a1920f44e68ca (mode 644) --- /dev/null +++ src/cmd/venti/srv/sortientry.c @@ -0,0 +1,376 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" +#include + +typedef struct IEBuck IEBuck; +typedef struct IEBucks IEBucks; + +enum +{ + ClumpChunks = 32*1024 +}; + +struct IEBuck +{ + u32int head; /* head of chain of chunks on the disk */ + u32int used; /* usage of the last chunk */ + u64int total; /* total number of bytes in this bucket */ + u8int *buf; /* chunk of entries for this bucket */ +}; + +struct IEBucks +{ + Part *part; + u64int off; /* offset for writing data in the partition */ + u32int chunks; /* total chunks written to fd */ + u64int max; /* max bytes entered in any one bucket */ + int bits; /* number of bits in initial bucket sort */ + int nbucks; /* 1 << bits, the number of buckets */ + u32int size; /* bytes in each of the buckets chunks */ + u32int usable; /* amount usable for IEntry data */ + u8int *buf; /* buffer for all chunks */ + u8int *xbuf; + IEBuck *bucks; +}; + +#define U32GET(p) (((p)[0]<<24)|((p)[1]<<16)|((p)[2]<<8)|(p)[3]) +#define U32PUT(p,v) (p)[0]=(v)>>24;(p)[1]=(v)>>16;(p)[2]=(v)>>8;(p)[3]=(v) + +static IEBucks *initiebucks(Part *part, int bits, u32int size); +static int flushiebuck(IEBucks *ib, int b, int reset); +static int flushiebucks(IEBucks *ib); +static u32int sortiebuck(IEBucks *ib, int b); +static u64int sortiebucks(IEBucks *ib); +static int sprayientry(IEBucks *ib, IEntry *ie); +static u32int readarenainfo(IEBucks *ib, Arena *arena, u64int a, Bloom *b); +static u32int readiebuck(IEBucks *ib, int b); +static void freeiebucks(IEBucks *ib); + +/* + * build a sorted file with all IEntries which should be in ix. + * assumes the arenas' directories are up to date. + * reads each, converts the entries to index entries, + * and sorts them. + */ +u64int +sortrawientries(Index *ix, Part *tmp, u64int *base, Bloom *bloom) +{ + IEBucks *ib; + u64int clumps, sorted; + u32int n; + int i, ok; + +//ZZZ should allow configuration of bits, bucket size + ib = initiebucks(tmp, 8, 64*1024); + if(ib == nil){ + seterr(EOk, "can't create sorting buckets: %r"); + return TWID64; + } + ok = 0; + clumps = 0; + fprint(2, "constructing entry list\n"); + for(i = 0; i < ix->narenas; i++){ + n = readarenainfo(ib, ix->arenas[i], ix->amap[i].start, bloom); + if(n == TWID32){ + ok = -1; + break; + } + clumps += n; + } + fprint(2, "sorting %lld entries\n", clumps); + if(ok == 0){ + sorted = sortiebucks(ib); + *base = (u64int)ib->chunks * ib->size; + if(sorted != clumps){ + fprint(2, "sorting messed up: clumps=%lld sorted=%lld\n", clumps, sorted); + ok = -1; + } + } + freeiebucks(ib); + if(ok < 0) + return TWID64; + return clumps; +} + +#define CHECK(cis) if(((ulong*)cis)[-4] != 0xA110C09) xabort(); + +void +xabort(void) +{ + int *x; + + x = 0; + *x = 0; +} + +/* + * read in all of the arena's clump directory, + * convert to IEntry format, and bucket sort based + * on the first few bits. + */ +static u32int +readarenainfo(IEBucks *ib, Arena *arena, u64int a, Bloom *b) +{ + IEntry ie; + ClumpInfo *ci, *cis; + u32int clump; + int i, n, ok, nskip; +// static Biobuf bout; + +//ZZZ remove fprint? +//fprint(2, "ra %s %d %d\n", arena->name, arena->memstats.clumps, arena->diskstats.clumps); + if(arena->memstats.clumps) + fprint(2, "\tarena %s: %d entries\n", arena->name, arena->memstats.clumps); + else + fprint(2, "[%s] ", arena->name); + + cis = MKN(ClumpInfo, ClumpChunks); + ok = 0; + nskip = 0; + memset(&ie, 0, sizeof(IEntry)); +// Binit(&bout, 1, OWRITE); + for(clump = 0; clump < arena->memstats.clumps; clump += n){ + n = ClumpChunks; + if(n > arena->memstats.clumps - clump) + n = arena->memstats.clumps - clump; + if(readclumpinfos(arena, clump, cis, n) != n){ + seterr(EOk, "arena directory read failed: %r"); + ok = -1; + break; + } + + for(i = 0; i < n; i++){ + ci = &cis[i]; + ie.ia.type = ci->type; + ie.ia.size = ci->uncsize; + ie.ia.addr = a; + a += ci->size + ClumpSize; + ie.ia.blocks = (ci->size + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog; + scorecp(ie.score, ci->score); + // Bprint(&bout, "%22lld %V %3d %5d\n", + // ie.ia.addr, ie.score, ie.ia.type, ie.ia.size); + if(ci->type == VtCorruptType){ + // print("! %V %22lld %3d %5d %3d\n", + // ie.score, ie.ia.addr, ie.ia.type, ie.ia.size, ie.ia.blocks); + nskip++; + }else + sprayientry(ib, &ie); + markbloomfilter(b, ie.score); + } + } +// Bterm(&bout); + free(cis); + if(ok < 0) + return TWID32; + return clump - nskip; +} + +/* + * initialize the external bucket sorting data structures + */ +static IEBucks* +initiebucks(Part *part, int bits, u32int size) +{ + IEBucks *ib; + int i; + + ib = MKZ(IEBucks); + if(ib == nil){ + seterr(EOk, "out of memory"); + return nil; + } + ib->bits = bits; + ib->nbucks = 1 << bits; + ib->size = size; + ib->usable = (size - U32Size) / IEntrySize * IEntrySize; + ib->bucks = MKNZ(IEBuck, ib->nbucks); + if(ib->bucks == nil){ + seterr(EOk, "out of memory allocation sorting buckets"); + freeiebucks(ib); + return nil; + } + ib->xbuf = MKN(u8int, size * ((1 << bits)+1)); + ib->buf = (u8int*)(((ulong)ib->xbuf+size-1)&~(ulong)(size-1)); + if(ib->buf == nil){ + seterr(EOk, "out of memory allocating sorting buckets' buffers"); + freeiebucks(ib); + return nil; + } + for(i = 0; i < ib->nbucks; i++){ + ib->bucks[i].head = TWID32; + ib->bucks[i].buf = &ib->buf[i * size]; + } + ib->part = part; + return ib; +} + +static void +freeiebucks(IEBucks *ib) +{ + if(ib == nil) + return; + free(ib->bucks); + free(ib->buf); + free(ib); +} + +/* + * initial sort: put the entry into the correct bucket + */ +static int +sprayientry(IEBucks *ib, IEntry *ie) +{ + u32int n; + int b; + + b = hashbits(ie->score, ib->bits); + n = ib->bucks[b].used; + if(n + IEntrySize > ib->usable){ + /* should be flushed below, but if flush fails, this can happen */ + seterr(EOk, "out of space in bucket"); + return -1; + } + packientry(ie, &ib->bucks[b].buf[n]); + n += IEntrySize; + ib->bucks[b].used = n; + if(n + IEntrySize <= ib->usable) + return 0; + return flushiebuck(ib, b, 1); +} + +/* + * finish sorting: + * for each bucket, read it in and sort it + * write out the the final file + */ +static u64int +sortiebucks(IEBucks *ib) +{ + u64int tot; + u32int n; + int i; + + if(flushiebucks(ib) < 0) + return TWID64; + for(i = 0; i < ib->nbucks; i++) + ib->bucks[i].buf = nil; + ib->off = (u64int)ib->chunks * ib->size; + free(ib->xbuf); +if(0){ + fprint(2, "ib->max = %lld\n", ib->max); + fprint(2, "ib->chunks = %ud\n", ib->chunks); +} + ib->buf = MKN(u8int, ib->max + U32Size); + if(ib->buf == nil){ + seterr(EOk, "out of memory allocating final sorting buffer; try more buckets"); + return TWID64; + } + tot = 0; + for(i = 0; i < ib->nbucks; i++){ + n = sortiebuck(ib, i); + if(n == TWID32) + return TWID64; + if(n != ib->bucks[i].total/IEntrySize) + fprint(2, "bucket %d changed count %d => %d\n", + i, (int)(ib->bucks[i].total/IEntrySize), n); + tot += n; + } + return tot; + return 0; +} + +/* + * sort from bucket b of ib into the output file to + */ +static u32int +sortiebuck(IEBucks *ib, int b) +{ + u32int n; + + n = readiebuck(ib, b); + if(n == TWID32) + return TWID32; + qsort(ib->buf, n, IEntrySize, ientrycmp); + if(writepart(ib->part, ib->off, ib->buf, n * IEntrySize) < 0){ + seterr(EOk, "can't write sorted bucket: %r"); + return TWID32; + } + ib->off += n * IEntrySize; + return n; +} + +/* + * write out a single bucket + */ +static int +flushiebuck(IEBucks *ib, int b, int reset) +{ + u32int n; + + if(ib->bucks[b].used == 0) + return 0; + n = ib->bucks[b].used; + U32PUT(&ib->bucks[b].buf[n], ib->bucks[b].head); + n += U32Size; + USED(n); + if(writepart(ib->part, (u64int)ib->chunks * ib->size, ib->bucks[b].buf, ib->size) < 0){ + seterr(EOk, "can't write sorting bucket to file: %r"); +xabort(); + return -1; + } + ib->bucks[b].head = ib->chunks++; + ib->bucks[b].total += ib->bucks[b].used; + if(reset) + ib->bucks[b].used = 0; + return 0; +} + +/* + * write out all of the buckets, and compute + * the maximum size of any bucket + */ +static int +flushiebucks(IEBucks *ib) +{ + int i; + + for(i = 0; i < ib->nbucks; i++){ + if(flushiebuck(ib, i, 0) < 0) + return -1; + if(ib->bucks[i].total > ib->max) + ib->max = ib->bucks[i].total; + } + return 0; +} + +/* + * read in the chained buffers for bucket b, + * and return it's total number of IEntries + */ +static u32int +readiebuck(IEBucks *ib, int b) +{ + u32int head, m, n; + + head = ib->bucks[b].head; + n = 0; + m = ib->bucks[b].used; + if(m == 0) + m = ib->usable; +// if(ib->bucks[b].total) +// fprint(2, "\tbucket %d: %d entries\n", b, ib->bucks[b].total/IEntrySize); + while(head != TWID32){ + if(readpart(ib->part, (u64int)head * ib->size, &ib->buf[n], m + U32Size) < 0){ + seterr(EOk, "can't read index sort bucket: %r"); + return TWID32; + } + n += m; + head = U32GET(&ib->buf[n]); + m = ib->usable; + } + if(n != ib->bucks[b].total) + fprint(2, "\tbucket %d: expected %d entries, got %d\n", + b, (int)ib->bucks[b].total/IEntrySize, n/IEntrySize); + return n / IEntrySize; +} blob - /dev/null blob + 874f7d27065398e0d6277cd5d18d54f235ba856c (mode 644) --- /dev/null +++ src/cmd/venti/srv/stats.c @@ -0,0 +1,212 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +int collectstats = 1; + +/* keep in sync with dat.h:/NStat */ +Statdesc statdesc[NStat] = +{ + { "rpc total", }, + { "rpc reads", }, + { "rpc reads ok", }, + { "rpc reads failed", }, + { "rpc read bytes", }, + { "rpc read time", }, + { "rpc read cached", }, + { "rpc read cached time", }, + { "rpc read uncached", }, + { "rpc read uncached time "}, + + { "rpc writes", }, + { "rpc writes new", }, + { "rpc writes old", }, + { "rpc writes failed", }, + { "rpc write bytes", }, + { "rpc write time", }, + { "rpc write new time", }, + { "rpc write old time", }, + + { "lump cache hits", }, + { "lump cache misses", }, + { "lump cache reads", }, + { "lump cache writes", }, + { "lump cache size", }, + { "lump cache stall", }, + { "lump cache read time", }, + + { "disk cache hits", }, + { "disk cache misses", }, + { "disk cache lookups", }, + { "disk cache reads", }, + { "disk cache writes", }, + { "disk cache dirty", }, + { "disk cache size", }, + { "disk cache flushes", }, + { "disk cache stalls", }, + { "disk cache lookup time", }, + + { "disk block stalls", }, + { "lump stalls", }, + + { "index cache hits", }, + { "index cache misses", }, + { "index cache reads", }, + { "index cache writes", }, + { "index cache fills", }, + { "index cache prefetches", }, + { "index cache dirty", }, + { "index cache size", }, + { "index cache flushes", }, + { "index cache stalls", }, + { "index cache read time", }, + + { "bloom filter hits", }, + { "bloom filter misses", }, + { "bloom filter false misses", }, + { "bloom filter lookups", }, + { "bloom filter ones", }, + { "bloom filter bits", }, + { "bloom filter lookup time", }, + + { "arena block reads", }, + { "arena block read bytes", }, + { "arena block writes", }, + { "arena block write bytes", }, + + { "isect block reads", }, + { "isect block read bytes", }, + { "isect block writes", }, + { "isect block write bytes", }, + + { "sum reads", }, + { "sum read bytes", }, +}; + +QLock statslock; +Stats stats; +Stats *stathist; +int nstathist; +ulong statind; +ulong stattime; + +void +statsproc(void *v) +{ + USED(v); + + for(;;){ + stats.now = time(0); + stathist[stattime%nstathist] = stats; + stattime++; + sleep(1000); + } +} + +void +statsinit(void) +{ + nstathist = 90000; + stathist = MKNZ(Stats, nstathist); + vtproc(statsproc, nil); +} + +void +setstat(int index, long val) +{ + qlock(&statslock); + stats.n[index] = val; + qunlock(&statslock); +} + +void +addstat(int index, int inc) +{ + if(!collectstats) + return; + qlock(&statslock); + stats.n[index] += inc; + qunlock(&statslock); +} + +void +addstat2(int index, int inc, int index1, int inc1) +{ + if(!collectstats) + return; + qlock(&statslock); + stats.n[index] += inc; + stats.n[index1] += inc1; + qunlock(&statslock); +} + +void +printstats(void) +{ +} + +void +binstats(long (*fn)(Stats *s0, Stats *s1, void *arg), void *arg, + long t0, long t1, Statbin *bin, int nbin) +{ + long t, xt0, te, v; + int i, j, lo, hi, m, oj; + vlong tot; + Statbin *b; + + t = stats.now; + + /* negative times mean relative to now. */ + if(t0 <= 0) + t0 += t; + if(t1 <= 0) + t1 += t; + /* ten minute range if none given */ + if(t1 <= t0) + t0 = t1 - 60*10; + if(0) fprint(2, "stats %ld-%ld\n", t0, t1); + + /* binary search to find t0-1 or close */ + lo = stattime; + hi = stattime+nstathist; + while(lo+1 < hi){ + m = (lo+hi)/2; + if(stathist[m%nstathist].now >= t0) + hi = m; + else + lo = m; + } + xt0 = stathist[lo%nstathist].now; + if(0) fprint(2, "bsearch found %ld\n", xt0); + if(xt0 >= t1){ + /* no samples */ + memset(bin, 0, nbin*sizeof bin[0]); + return; + } + + hi = stattime+nstathist; + te = t0; + j = lo+1; + for(i=0; insamp==0 || v < b->min) + b->min = v; + if(b->nsamp==0 || v > b->max) + b->max = v; + tot += v; + b->nsamp++; + } + if(0) fprint(2, "bin%d: %ld to %ld; %d to %d - %d samples\n", i, t, te, oj, j, b->nsamp); + if(b->nsamp) + b->avg = tot / b->nsamp; + if(b->nsamp==0 && i>0) + *b = bin[i-1]; + } +} blob - /dev/null blob + 3fd06ccd7e7c4294918ea03c537ff8f3d8cc42d4 (mode 644) --- /dev/null +++ src/cmd/venti/srv/stdinc.h @@ -0,0 +1,9 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include blob - /dev/null blob + d11ca4f3b421285637788f7a7bbeab915efdf59f (mode 644) --- /dev/null +++ src/cmd/venti/srv/syncarena.c @@ -0,0 +1,174 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +static int writeclumphead(Arena *arena, u64int aa, Clump *cl); +static int writeclumpmagic(Arena *arena, u64int aa, u32int magic); + +int +clumpinfocmp(ClumpInfo *c, ClumpInfo *d) +{ + return c->type != d->type + || c->size != d->size + || c->uncsize != d->uncsize + || scorecmp(c->score, d->score)!=0; +} + +/* + * synchronize the clump info directory with + * with the clumps actually stored in the arena. + * the directory should be at least as up to date + * as the arena's trailer. + * + * checks/updates at most n clumps. + * + * returns 0 if ok, flags if error occurred + */ +int +syncarena(Arena *arena, u64int start, u32int n, int zok, int fix) +{ + ZBlock *lump; + Clump cl; + ClumpInfo ci; + static ClumpInfo zci = { .type = -1 }; + u8int score[VtScoreSize]; + u64int uncsize, used, aa; + u32int clump, clumps, cclumps, magic; + int err, flush, broken; + AState as; + + used = arena->memstats.used; + clumps = arena->memstats.clumps; + cclumps = arena->memstats.cclumps; + uncsize = arena->memstats.uncsize; + trace(TraceProc, "syncarena start"); + flush = 0; + err = 0; + for(; n; n--){ + aa = arena->memstats.used; + clump = arena->memstats.clumps; + magic = clumpmagic(arena, aa); + if(magic == ClumpFreeMagic) + break; + if(magic != arena->clumpmagic){ + fprint(2, "%s: illegal clump magic number=%#8.8ux at clump=%d\n", arena->name, magic, clump); + /* err |= SyncDataErr; */ + if(fix && writeclumpmagic(arena, aa, ClumpFreeMagic) < 0){ + fprint(2, "can't write corrected clump free magic: %r"); + err |= SyncFixErr; + } + break; + } + + broken = 0; + lump = loadclump(arena, aa, 0, &cl, score, 0); + if(lump == nil){ + fprint(2, "%s: clump=%d failed to read correctly: %r\n", arena->name, clump); + break; + err |= SyncDataErr; + }else if(cl.info.type != VtCorruptType){ + scoremem(score, lump->data, cl.info.uncsize); + if(scorecmp(cl.info.score, score) != 0){ + /* ignore partially written block */ + if(cl.encoding == ClumpENone) + break; + fprint(2, "%s: clump=%d has mismatched score\n", arena->name, clump); + err |= SyncDataErr; + broken = 1; + }else if(vttypevalid(cl.info.type) < 0){ + fprint(2, "%s: clump=%d has invalid type %d", arena->name, clump, cl.info.type); + err |= SyncDataErr; + broken = 1; + } + if(broken && fix){ + cl.info.type = VtCorruptType; + if(writeclumphead(arena, aa, &cl) < 0){ + fprint(2, "%s: can't write corrected clump header: %r", arena->name); + err |= SyncFixErr; + } + } + } + freezblock(lump); + arena->memstats.used += ClumpSize + cl.info.size; + + arena->memstats.clumps++; + if(!broken && readclumpinfo(arena, clump, &ci)<0){ + fprint(2, "%s: arena directory read failed\n", arena->name); + broken = 1; + }else if(!broken && clumpinfocmp(&ci, &cl.info)!=0){ + if(clumpinfocmp(&ci, &zci) == 0){ + err |= SyncCIZero; + if(!zok) + fprint(2, "%s: unwritten clump info for clump=%d\n", arena->name, clump); + }else{ + err |= SyncCIErr; + fprint(2, "%s: bad clump info for clump=%d\n", arena->name, clump); + fprint(2, "\texpected score=%V type=%d size=%d uncsize=%d\n", + cl.info.score, cl.info.type, cl.info.size, cl.info.uncsize); + fprint(2, "\tfound score=%V type=%d size=%d uncsize=%d\n", + ci.score, ci.type, ci.size, ci.uncsize); + } + broken = 1; + } + if(broken && fix){ + flush = 1; + ci = cl.info; + if(writeclumpinfo(arena, clump, &ci) < 0){ + fprint(2, "%s: can't write correct clump directory: %r\n", arena->name); + err |= SyncFixErr; + } + } + trace(TraceProc, "syncarena unindexed clump %V %d", cl.info.score, arena->memstats.clumps); + + arena->memstats.uncsize += cl.info.uncsize; + if(cl.info.size < cl.info.uncsize) + arena->memstats.cclumps++; + } + + if(flush){ + trace(TraceProc, "syncarena flush"); + arena->wtime = now(); + if(arena->ctime == 0 && arena->memstats.clumps) + arena->ctime = arena->wtime; + flushdcache(); + } + + if(used != arena->memstats.used + || clumps != arena->memstats.clumps + || cclumps != arena->memstats.cclumps + || uncsize != arena->memstats.uncsize) + err |= SyncHeader; + if(start && (err&SyncHeader)){ + trace(TraceProc, "syncarena setdcachestate"); + as.arena = arena; + as.aa = start+arena->memstats.used; + as.stats = arena->memstats; + setdcachestate(&as); + } + + return err; +} + +static int +writeclumphead(Arena *arena, u64int aa, Clump *cl) +{ + ZBlock *zb; + int bad; + + zb = alloczblock(ClumpSize, 0, arena->blocksize); + if(zb == nil) + return -1; + bad = packclump(cl, zb->data, arena->clumpmagic)<0 + || writearena(arena, aa, zb->data, ClumpSize) != ClumpSize; + freezblock(zb); + return bad ? -1 : 0; +} + +static int +writeclumpmagic(Arena *arena, u64int aa, u32int magic) +{ + u8int buf[U32Size]; + + packmagic(magic, buf); + return writearena(arena, aa, buf, U32Size) == U32Size; +} blob - /dev/null blob + b35ca2a425b4a6ffe4a3da3e78089f98b1f465f9 (mode 644) --- /dev/null +++ src/cmd/venti/srv/syncindex.c @@ -0,0 +1,73 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +static int verbose; +void +usage(void) +{ + fprint(2, "usage: syncindex [-fv] [-B blockcachesize] config\n"); + threadexitsall("usage"); +} + +Config conf; + +void +threadmain(int argc, char *argv[]) +{ + u32int bcmem, icmem; + int fix; + + fix = 0; + bcmem = 0; + icmem = 0; + ARGBEGIN{ + case 'B': + bcmem = unittoull(EARGF(usage())); + break; + case 'I': + icmem = unittoull(EARGF(usage())); + break; + case 'f': + fix++; + break; + case 'v': + verbose++; + break; + default: + usage(); + break; + }ARGEND + + if(!fix) + readonly = 1; + + if(argc != 1) + usage(); + + if(initventi(argv[0], &conf) < 0) + sysfatal("can't init venti: %r"); + + if(bcmem < maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16)) + bcmem = maxblocksize * (mainindex->narenas + mainindex->nsects * 4 + 16); + if(0) fprint(2, "initialize %d bytes of disk block cache\n", bcmem); + initdcache(bcmem); + initlumpcache(1*1024*1024, 1024/8); + icmem = u64log2(icmem / (sizeof(IEntry)+sizeof(IEntry*)) / ICacheDepth); + if(icmem < 4) + icmem = 4; + if(1) fprint(2, "initialize %d bytes of index cache for %d index entries\n", + (sizeof(IEntry)+sizeof(IEntry*)) * (1 << icmem) * ICacheDepth, + (1 << icmem) * ICacheDepth); + initicache(icmem, ICacheDepth); + initicachewrite(); + if(mainindex->bloom) + startbloomproc(mainindex->bloom); + + if(verbose) + printindex(2, mainindex); + if(syncindex(mainindex, fix, 1, 0) < 0) + sysfatal("failed to sync index=%s: %r\n", mainindex->name); + + threadexitsall(0); +} blob - /dev/null blob + 12b69ed2c3aa5a0f8ea2be71b7769903bf4473e4 (mode 644) --- /dev/null +++ src/cmd/venti/srv/syncindex0.c @@ -0,0 +1,167 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +enum +{ + ClumpChunks = 32*1024 +}; + +static int missing, wrong; + +/* + * shell sort is plenty good enough + * because we're going to do a bunch of disk i/o's + */ +static void +sortclumpinfo(ClumpInfo *ci, int *s, int n) +{ + int i, j, m, t; + + for(m = (n + 3) / 5; m > 0; m = (m + 1) / 3){ + for(i = n - m; i-- > 0;){ + for(j = i + m; j < n; j += m){ + if(memcmp(ci[s[j - m]].score, ci[s[j]].score, VtScoreSize) <= 0) + break; + t = s[j]; + s[j] = s[j - m]; + s[j - m] = t; + } + } + } +} + +int +syncarenaindex(Index *ix, Arena *arena, u32int clump, u64int a, int fix, int *pflush, int check) +{ + Packet *pack; + IEntry ie; + IAddr ia; + ClumpInfo *ci, *cis; + u64int *addrs; + int i, n, ok, *s, flush; + + trace(TraceProc, "syncarenaindex enter"); + + flush = 0; + cis = MKN(ClumpInfo, ClumpChunks); + addrs = MKN(u64int, ClumpChunks); + s = MKN(int, ClumpChunks); + ok = 0; + for(; clump < arena->memstats.clumps; clump += n){ + n = ClumpChunks; + if(n > arena->memstats.clumps - clump) + n = arena->memstats.clumps - clump; + n = readclumpinfos(arena, clump, cis, n); + if(n <= 0){ + fprint(2, "arena directory read failed\n"); + ok = -1; + break; + } + + for(i = 0; i < n; i++){ + addrs[i] = a; + a += cis[i].size + ClumpSize; + s[i] = i; + } + + sortclumpinfo(cis, s, n); + + for(i = 0; i < n; i++){ + ci = &cis[s[i]]; + ia.type = ci->type; + ia.size = ci->uncsize; + ia.addr = addrs[s[i]]; + ia.blocks = (ci->size + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog; + + if(!check) + goto Add; + if(loadientry(ix, ci->score, ci->type, &ie) < 0){ + trace(TraceProc, "syncarenaindex missing block %V.%d", ci->score, ci->type); + missing++; + if(0) fprint(2, "missing block type=%d score=%V\n", ci->type, ci->score); + }else if(iaddrcmp(&ia, &ie.ia) != 0){ + trace(TraceProc, "syncarenaindex mismatched entry"); + fprint(2, "\nmismatched index entry and clump at %d\n", clump + i); + fprint(2, "\tclump: type=%d size=%d blocks=%d addr=%lld\n", ia.type, ia.size, ia.blocks, ia.addr); + fprint(2, "\tindex: type=%d size=%d block=%d addr=%lld\n", ie.ia.type, ie.ia.size, ie.ia.blocks, ie.ia.addr); + pack = readlump(ie.score, ie.ia.type, ie.ia.size, nil); + packetfree(pack); + if(pack != nil){ + fprint(2, "duplicated lump\n"); + continue; + } + wrong++; + }else + continue; + Add: + if(!fix){ + ok = -1; + continue; + } + flush = 1; + trace(TraceProc, "syncarenaindex insert %V", ci->score); + insertscore(ci->score, &ia, 1); + } + + if(0 && clump / 1000 != (clump + n) / 1000) + fprint(2, "."); + } + free(cis); + free(addrs); + free(s); + if(flush){ + flushdcache(); + *pflush = 1; + } + return ok; +} + +int +syncindex(Index *ix, int fix, int mustflush, int check) +{ + Arena *arena; + u64int a; + u32int clump; + int i, e, e1, ok, ok1, flush; + + ok = 0; + flush = 0; + for(i = 0; i < ix->narenas; i++){ + trace(TraceProc, "syncindex start %d", i); + arena = ix->arenas[i]; + clump = arena->memstats.clumps; + a = arena->memstats.used; + e = syncarena(arena, ix->amap[i].start, TWID32, fix, fix); + e1 = e; + if(fix) + e1 &= ~(SyncHeader|SyncCIZero|SyncCIErr); + if(e1 == SyncHeader) + fprint(2, "arena %s: header is out-of-date\n", arena->name); + if(e1) + ok = -1; + else{ + ok1 = syncarenaindex(ix, arena, clump, a + ix->amap[i].start, fix, &flush, check); + if(ok1 < 0) + fprint(2, "syncarenaindex: %r\n"); + if(fix && ok1==0 && (e & SyncHeader) && wbarena(arena) < 0) + fprint(2, "arena=%s header write failed: %r\n", arena->name); + ok |= ok1; + } + } + if(missing || wrong) + fprint(2, "syncindex: %d missing entries, %d wrong entries (flush=%d)\n", missing, wrong, flush); + if(fix && wbindex(ix) < 0){ + fprint(2, "can't write back index header for %s: %r\n", ix->name); + return -1; + } + if(fix && flush){ + flushdcache(); + if(mustflush){ + flushicache(); + flushdcache(); + }else + kickicache(); + } + return ok; +} blob - /dev/null blob + f8669b3dc40d3d462883c40008989edabd35cfb7 (mode 644) --- /dev/null +++ src/cmd/venti/srv/trace.c @@ -0,0 +1,38 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +char TraceDisk[] = "disk"; +char TraceLump[] = "lump"; +char TraceBlock[] = "block"; +char TraceProc[] = "proc"; +char TraceWork[] = "work"; +char TraceQuiet[] = "quiet"; +char TraceRpc[] = "rpc"; + +void +trace(char *level, char *fmt, ...) +{ + char buf[512]; + va_list arg; + + if(level == nil || !ventilogging) + return; + va_start(arg, fmt); + vsnprint(buf, sizeof buf, fmt, arg); + va_end(arg); + vtlog(level, "%T %s: %s
\n", + threadgetname(), buf); + vtlog("all", "%T %s %s: %s
\n", + level, threadgetname(), buf); +} + +void +traceinit(void) +{ +} + +void +settrace(char *trace) +{ +} blob - /dev/null blob + 1f741170247098c37269db63067cd2a97a5d79e5 (mode 644) --- /dev/null +++ src/cmd/venti/srv/unittoull.c @@ -0,0 +1,30 @@ +#include "stdinc.h" + +#define TWID64 ((u64int)~(u64int)0) + +u64int +unittoull(char *s) +{ + char *es; + u64int n; + + if(s == nil) + return TWID64; + n = strtoul(s, &es, 0); + if(*es == 'k' || *es == 'K'){ + n *= 1024; + es++; + }else if(*es == 'm' || *es == 'M'){ + n *= 1024*1024; + es++; + }else if(*es == 'g' || *es == 'G'){ + n *= 1024*1024*1024; + es++; + }else if(*es == 't' || *es == 'T'){ + n *= 1024*1024; + n *= 1024*1024; + } + if(*es != '\0') + return TWID64; + return n; +} blob - /dev/null blob + 5530bd07d003f43c444940d600c9b41cae07a809 (mode 644) --- /dev/null +++ src/cmd/venti/srv/unwhack.c @@ -0,0 +1,179 @@ +#include "stdinc.h" +#include "whack.h" + +enum +{ + DMaxFastLen = 7, + DBigLenCode = 0x3c, /* minimum code for large lenth encoding */ + DBigLenBits = 6, + DBigLenBase = 1 /* starting items to encode for big lens */ +}; + +static uchar lenval[1 << (DBigLenBits - 1)] = +{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, + 5, + 6, + 255, + 255 +}; + +static uchar lenbits[] = +{ + 0, 0, 0, + 2, 3, 5, 5, +}; + +static uchar offbits[16] = +{ + 5, 5, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 12, 13 +}; + +static ushort offbase[16] = +{ + 0, 0x20, + 0x40, 0x60, + 0x80, 0xc0, + 0x100, 0x180, + 0x200, 0x300, + 0x400, 0x600, + 0x800, 0xc00, + 0x1000, + 0x2000 +}; + +void +unwhackinit(Unwhack *uw) +{ + uw->err[0] = '\0'; +} + +int +unwhack(Unwhack *uw, uchar *dst, int ndst, uchar *src, int nsrc) +{ + uchar *s, *d, *dmax, *smax, lit; + ulong uwbits, lithist; + int i, off, len, bits, use, code, uwnbits, overbits; + + d = dst; + dmax = d + ndst; + + smax = src + nsrc; + uwnbits = 0; + uwbits = 0; + overbits = 0; + lithist = ~0; + while(src < smax || uwnbits - overbits >= MinDecode){ + while(uwnbits <= 24){ + uwbits <<= 8; + if(src < smax) + uwbits |= *src++; + else + overbits += 8; + uwnbits += 8; + } + + /* + * literal + */ + len = lenval[(uwbits >> (uwnbits - 5)) & 0x1f]; + if(len == 0){ + if(lithist & 0xf){ + uwnbits -= 9; + lit = (uwbits >> uwnbits) & 0xff; + lit &= 255; + }else{ + uwnbits -= 8; + lit = (uwbits >> uwnbits) & 0x7f; + if(lit < 32){ + if(lit < 24){ + uwnbits -= 2; + lit = (lit << 2) | ((uwbits >> uwnbits) & 3); + }else{ + uwnbits -= 3; + lit = (lit << 3) | ((uwbits >> uwnbits) & 7); + } + lit = (lit - 64) & 0xff; + } + } + if(d >= dmax){ + snprint(uw->err, WhackErrLen, "too much output"); + return -1; + } + *d++ = lit; + lithist = (lithist << 1) | (lit < 32) | (lit > 127); + continue; + } + + /* + * length + */ + if(len < 255) + uwnbits -= lenbits[len]; + else{ + uwnbits -= DBigLenBits; + code = ((uwbits >> uwnbits) & ((1 << DBigLenBits) - 1)) - DBigLenCode; + len = DMaxFastLen; + use = DBigLenBase; + bits = (DBigLenBits & 1) ^ 1; + while(code >= use){ + len += use; + code -= use; + code <<= 1; + uwnbits--; + if(uwnbits < 0){ + snprint(uw->err, WhackErrLen, "len out of range"); + return -1; + } + code |= (uwbits >> uwnbits) & 1; + use <<= bits; + bits ^= 1; + } + len += code; + + while(uwnbits <= 24){ + uwbits <<= 8; + if(src < smax) + uwbits |= *src++; + else + overbits += 8; + uwnbits += 8; + } + } + + /* + * offset + */ + uwnbits -= 4; + bits = (uwbits >> uwnbits) & 0xf; + off = offbase[bits]; + bits = offbits[bits]; + + uwnbits -= bits; + off |= (uwbits >> uwnbits) & ((1 << bits) - 1); + off++; + + if(off > d - dst){ + snprint(uw->err, WhackErrLen, "offset out of range: off=%d d=%ld len=%d nbits=%d", off, d - dst, len, uwnbits); + return -1; + } + if(d + len > dmax){ + snprint(uw->err, WhackErrLen, "len out of range"); + return -1; + } + s = d - off; + for(i = 0; i < len; i++) + d[i] = s[i]; + d += len; + } + if(uwnbits < overbits){ + snprint(uw->err, WhackErrLen, "compressed data overrun"); + return -1; + } + + len = d - dst; + + return len; +} blob - /dev/null blob + 03fd9065117709e039ad8c4c5c9f18d1f0f8b5cb (mode 644) --- /dev/null +++ src/cmd/venti/srv/utils.c @@ -0,0 +1,252 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +int +namecmp(char *s, char *t) +{ + return strncmp(s, t, ANameSize); +} + +void +namecp(char *dst, char *src) +{ + strncpy(dst, src, ANameSize - 1); + dst[ANameSize - 1] = '\0'; +} + +int +nameok(char *name) +{ + char *t; + int c; + + if(name == nil) + return -1; + for(t = name; c = *t; t++) + if(t - name >= ANameSize + || c < ' ' || c >= 0x7f) + return -1; + return 0; +} + +int +stru32int(char *s, u32int *r) +{ + char *t; + u32int n, nn, m; + int c; + + m = TWID32 / 10; + n = 0; + for(t = s; ; t++){ + c = *t; + if(c < '0' || c > '9') + break; + if(n > m) + return -1; + nn = n * 10 + c - '0'; + if(nn < n) + return -1; + n = nn; + } + *r = n; + return s != t && *t == '\0'; +} + +int +stru64int(char *s, u64int *r) +{ + char *t; + u64int n, nn, m; + int c; + + m = TWID64 / 10; + n = 0; + for(t = s; ; t++){ + c = *t; + if(c < '0' || c > '9') + break; + if(n > m) + return -1; + nn = n * 10 + c - '0'; + if(nn < n) + return -1; + n = nn; + } + *r = n; + return s != t && *t == '\0'; +} + +int +vttypevalid(int type) +{ + return type < VtMaxType; +} + +static char* +logit(int severity, char *fmt, va_list args) +{ + char *s; + + s = vsmprint(fmt, args); + if(s == nil) + return nil; + if(argv0 == nil) + fprint(2, "%s: err %d: %s\n", argv0, severity, s); + else + fprint(2, "err %d: %s\n", severity, s); + return s; +} + +void +seterr(int severity, char *fmt, ...) +{ + char *s; + va_list args; + + va_start(args, fmt); + s = logit(severity, fmt, args); + va_end(args); + if(s == nil) + werrstr("error setting error"); + else{ + werrstr("%s", s); + free(s); + } +} + +void +logerr(int severity, char *fmt, ...) +{ + char *s; + va_list args; + + va_start(args, fmt); + s = logit(severity, fmt, args); + va_end(args); + free(s); +} + +u32int +now(void) +{ + return time(nil); +} + +int abortonmem = 1; + +void * +emalloc(ulong n) +{ + void *p; + + p = malloc(n); + if(p == nil){ + if(abortonmem) + abort(); + sysfatal("out of memory allocating %lud", n); + } + memset(p, 0xa5, n); +if(0)print("emalloc %p-%p by %lux\n", p, (char*)p+n, getcallerpc(&n)); + return p; +} + +void * +ezmalloc(ulong n) +{ + void *p; + + p = malloc(n); + if(p == nil){ + if(abortonmem) + abort(); + sysfatal("out of memory allocating %lud", n); + } + memset(p, 0, n); +if(0)print("ezmalloc %p-%p by %lux\n", p, (char*)p+n, getcallerpc(&n)); + return p; +} + +void * +erealloc(void *p, ulong n) +{ + p = realloc(p, n); + if(p == nil){ + if(abortonmem) + abort(); + sysfatal("out of memory allocating %lud", n); + } +if(0)print("erealloc %p-%p by %lux\n", p, (char*)p+n, getcallerpc(&p)); + return p; +} + +char * +estrdup(char *s) +{ + char *t; + int n; + + n = strlen(s) + 1; + t = emalloc(n); + memmove(t, s, n); +if(0)print("estrdup %p-%p by %lux\n", t, (char*)t+n, getcallerpc(&s)); + return t; +} + +/* + * return floor(log2(v)) + */ +int +u64log2(u64int v) +{ + int i; + + for(i = 0; i < 64; i++) + if((v >> i) <= 1) + break; + return i; +} + +int +vtproc(void (*fn)(void*), void *arg) +{ + proccreate(fn, arg, 256*1024); + return 0; +} + +int +ientryfmt(Fmt *fmt) +{ + IEntry *ie; + + ie = va_arg(fmt->args, IEntry*); + return fmtprint(fmt, "%V %22lld %3d %5d %3d", + ie->score, ie->ia.addr, ie->ia.type, ie->ia.size, ie->ia.blocks); +} + +void +ventifmtinstall(void) +{ + fmtinstall('F', vtfcallfmt); + fmtinstall('H', encodefmt); + fmtinstall('I', ientryfmt); + fmtinstall('V', vtscorefmt); +} + +uint +msec(void) +{ + return nsec()/1000000; +} + +uint +countbits(uint n) +{ + n = (n&0x55555555)+((n>>1)&0x55555555); + n = (n&0x33333333)+((n>>2)&0x33333333); + n = (n&0x0F0F0F0F)+((n>>4)&0x0F0F0F0F); + n = (n&0x00FF00FF)+((n>>8)&0x00FF00FF); + n = (n&0x0000FFFF)+((n>>16)&0x0000FFFF); + return n; +} blob - /dev/null blob + 91ded95c1f254507b104a450c8abc6f04753b4db (mode 644) --- /dev/null +++ src/cmd/venti/srv/venti.c @@ -0,0 +1,266 @@ +#ifdef PLAN9PORT +#include +#include +#endif +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +#include "whack.h" + +int debug; +int nofork; +int mainstacksize = 256*1024; +VtSrv *ventisrv; + +static void ventiserver(void*); + +void +usage(void) +{ + fprint(2, "usage: venti [-dw] [-a ventiaddress] [-h httpaddress] [-c config] [-C cachesize] [-I icachesize] [-B blockcachesize]\n"); + threadexitsall("usage"); +} +void +threadmain(int argc, char *argv[]) +{ + char *configfile, *haddr, *vaddr, *webroot; + u32int mem, icmem, bcmem, minbcmem; + Config config; + + traceinit(); + threadsetname("main"); + vaddr = nil; + haddr = nil; + configfile = nil; + webroot = nil; + mem = 0; + icmem = 0; + bcmem = 0; + ARGBEGIN{ + case 'a': + vaddr = EARGF(usage()); + break; + case 'B': + bcmem = unittoull(EARGF(usage())); + break; + case 'c': + configfile = EARGF(usage()); + break; + case 'C': + mem = unittoull(EARGF(usage())); + break; + case 'D': + settrace(EARGF(usage())); + break; + case 'd': + debug = 1; + nofork = 1; + break; + case 'h': + haddr = EARGF(usage()); + break; + case 'I': + icmem = unittoull(EARGF(usage())); + break; + case 'L': + ventilogging = 1; + break; + case 's': + nofork = 1; + break; + case 'W': + webroot = EARGF(usage()); + break; + default: + usage(); + }ARGEND + + if(argc) + usage(); + + if(!nofork) + rfork(RFNOTEG); + +#ifdef PLAN9PORT + { + /* sigh - needed to avoid signals when writing to hungup networks */ + struct sigaction sa; + memset(&sa, 0, sizeof sa); + sa.sa_handler = SIG_IGN; + sigaction(SIGPIPE, &sa, nil); + } +#endif + + trace(TraceQuiet, "venti started"); + fprint(2, "venti: "); + + ventifmtinstall(); + if(configfile == nil) + configfile = "venti.conf"; + + if(initarenasum() < 0) + fprint(2, "warning: can't initialize arena summing process: %r"); + + fprint(2, "conf..."); + if(initventi(configfile, &config) < 0) + sysfatal("can't init server: %r"); + + if(mem == 0) + mem = config.mem; + if(bcmem == 0) + bcmem = config.bcmem; + if(icmem == 0) + icmem = config.icmem; + if(haddr == nil) + haddr = config.haddr; + if(vaddr == nil) + vaddr = config.vaddr; + if(vaddr == nil) + vaddr = "tcp!*!venti"; + if(webroot == nil) + webroot = config.webroot; + if(queuewrites == 0) + queuewrites = config.queuewrites; + + if(haddr){ + fprint(2, "httpd %s...", haddr); + if(httpdinit(haddr, webroot) < 0) + fprint(2, "warning: can't start http server: %r"); + } + + fprint(2, "init..."); + + if(mem == 0xffffffffUL) + mem = 1 * 1024 * 1024; + if(0) fprint(2, "initialize %d bytes of lump cache for %d lumps\n", + mem, mem / (8 * 1024)); + initlumpcache(mem, mem / (8 * 1024)); + + icmem = u64log2(icmem / (sizeof(IEntry)+sizeof(IEntry*)) / ICacheDepth); + if(icmem < 4) + icmem = 4; + if(0) fprint(2, "initialize %d bytes of index cache for %d index entries\n", + (sizeof(IEntry)+sizeof(IEntry*)) * (1 << icmem) * ICacheDepth, + (1 << icmem) * ICacheDepth); + initicache(icmem, ICacheDepth); + initicachewrite(); + + /* + * need a block for every arena and every process + */ + minbcmem = maxblocksize * + (mainindex->narenas + mainindex->nsects*4 + 16); + if(bcmem < minbcmem) + bcmem = minbcmem; + + if(0) fprint(2, "initialize %d bytes of disk block cache\n", bcmem); + initdcache(bcmem); + + if(mainindex->bloom) + startbloomproc(mainindex->bloom); + + fprint(2, "sync..."); + if(syncindex(mainindex, 1, 0, 0) < 0) + sysfatal("can't sync server: %r"); + + if(queuewrites){ + fprint(2, "queue..."); + if(initlumpqueues(mainindex->nsects) < 0){ + fprint(2, "can't initialize lump queues," + " disabling write queueing: %r"); + queuewrites = 0; + } + } + + fprint(2, "announce %s...", vaddr); + ventisrv = vtlisten(vaddr); + if(ventisrv == nil) + sysfatal("can't announce %s: %r", vaddr); + + fprint(2, "serving.\n"); + if(nofork) + ventiserver(nil); + else + vtproc(ventiserver, nil); +} + +static void +vtrerror(VtReq *r, char *error) +{ + r->rx.msgtype = VtRerror; + r->rx.error = estrdup(error); +} + +static void +ventiserver(void *v) +{ + Packet *p; + VtReq *r; + char err[ERRMAX]; + uint ms; + int cached, ok; + + USED(v); + threadsetname("ventiserver"); + trace(TraceWork, "start"); + while((r = vtgetreq(ventisrv)) != nil){ + trace(TraceWork, "finish"); + trace(TraceWork, "start request %F", &r->tx); + trace(TraceRpc, "<- %F", &r->tx); + r->rx.msgtype = r->tx.msgtype+1; + addstat(StatRpcTotal, 1); + // print("req (arenas[0]=%p sects[0]=%p) %F\n", + // mainindex->arenas[0], mainindex->sects[0], &r->tx); + switch(r->tx.msgtype){ + default: + vtrerror(r, "unknown request"); + break; + case VtTread: + ms = msec(); + r->rx.data = readlump(r->tx.score, r->tx.blocktype, r->tx.count, &cached); + ms = msec() - ms; + addstat2(StatRpcRead, 1, StatRpcReadTime, ms); + if(r->rx.data == nil){ + addstat(StatRpcReadFail, 1); + rerrstr(err, sizeof err); + vtrerror(r, err); + }else{ + addstat(StatRpcReadBytes, packetsize(r->rx.data)); + addstat(StatRpcReadOk, 1); + if(cached) + addstat2(StatRpcReadCached, 1, StatRpcReadCachedTime, ms); + else + addstat2(StatRpcReadUncached, 1, StatRpcReadUncachedTime, ms); + } + break; + case VtTwrite: + p = r->tx.data; + r->tx.data = nil; + addstat(StatRpcWriteBytes, packetsize(p)); + ms = msec(); + ok = writelump(p, r->rx.score, r->tx.blocktype, 0, ms); + ms = msec() - ms; + addstat2(StatRpcWrite, 1, StatRpcWriteTime, ms); + + if(ok < 0){ + addstat(StatRpcWriteFail, 1); + rerrstr(err, sizeof err); + vtrerror(r, err); + } + break; + case VtTsync: + flushqueue(); + flushdcache(); + break; + } + trace(TraceRpc, "-> %F", &r->rx); + vtrespond(r); + trace(TraceWork, "start"); + } + flushdcache(); + flushicache(); + threadexitsall(0); +} + + blob - /dev/null blob + 5236c0934d87c1a1a1255f372f89aef895b773a3 (mode 644) --- /dev/null +++ src/cmd/venti/srv/verifyarena.c @@ -0,0 +1,127 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +static int verbose; + +void +usage(void) +{ + fprint(2, "usage: verifyarena [-v]\n"); + threadexitsall(0); +} + +static void +readblock(uchar *buf, int n) +{ + int nr, m; + + for(nr = 0; nr < n; nr += m){ + m = n - nr; + m = read(0, &buf[nr], m); + if(m <= 0) + sysfatal("can't read arena from standard input: %r"); + } +} + +static void +verifyarena(void) +{ + Arena arena; + ArenaHead head; + ZBlock *b; + DigestState s; + u64int n, e; + u32int bs; + u8int score[VtScoreSize]; + + fprint(2, "verify arena from standard input\n"); + + memset(&arena, 0, sizeof arena); + memset(&s, 0, sizeof s); + + /* + * read the little bit, which will included the header + */ + bs = MaxIoSize; + b = alloczblock(bs, 0, 0); + readblock(b->data, HeadSize); + sha1(b->data, HeadSize, nil, &s); + if(unpackarenahead(&head, b->data) < 0) + sysfatal("corrupted arena header: %r"); + if(head.version != ArenaVersion4 && head.version != ArenaVersion5) + fprint(2, "warning: unknown arena version %d\n", head.version); + + /* + * now we know how much to read + * read everything but the last block, which is special + */ + e = head.size - head.blocksize; + for(n = HeadSize; n < e; n += bs){ + if(n + bs > e) + bs = e - n; + readblock(b->data, bs); + sha1(b->data, bs, nil, &s); + } + + /* + * read the last block update the sum. + * the sum is calculated assuming the slot for the sum is zero. + */ + bs = head.blocksize; + readblock(b->data, bs); + sha1(b->data, bs-VtScoreSize, nil, &s); + sha1(zeroscore, VtScoreSize, nil, &s); + sha1(nil, 0, score, &s); + + /* + * validity check on the trailer + */ + arena.blocksize = head.blocksize; + if(unpackarena(&arena, b->data) < 0) + sysfatal("corrupted arena trailer: %r"); + scorecp(arena.score, &b->data[arena.blocksize - VtScoreSize]); + + if(namecmp(arena.name, head.name) != 0) + sysfatal("arena header and trailer names clash: %s vs. %s\n", head.name, arena.name); + if(arena.version != head.version) + sysfatal("arena header and trailer versions clash: %d vs. %d\n", head.version, arena.version); + arena.size = head.size - 2 * head.blocksize; + + /* + * check for no checksum or the same + */ + if(scorecmp(score, arena.score) != 0){ + if(scorecmp(zeroscore, arena.score) != 0) + fprint(2, "warning: mismatched checksums for arena=%s, found=%V calculated=%V", + arena.name, arena.score, score); + scorecp(arena.score, score); + }else + fprint(2, "matched score\n"); + + printarena(2, &arena); +} + +void +threadmain(int argc, char *argv[]) +{ + ventifmtinstall(); + statsinit(); + + ARGBEGIN{ + case 'v': + verbose++; + break; + default: + usage(); + break; + }ARGEND + + readonly = 1; + + if(argc != 0) + usage(); + + verifyarena(); + threadexitsall(0); +} blob - /dev/null blob + ecd29033936a96fc125dcfaf8faff73ac08de0bc (mode 644) --- /dev/null +++ src/cmd/venti/srv/whack.c @@ -0,0 +1,331 @@ +#include "stdinc.h" +#include "whack.h" + +typedef struct Huff Huff; +int compressblocks = 1; + +enum +{ + MaxFastLen = 9, + BigLenCode = 0x1f4, /* minimum code for large lenth encoding */ + BigLenBits = 9, + BigLenBase = 4, /* starting items to encode for big lens */ + + MinOffBits = 6, + MaxOffBits = MinOffBits + 8, + + MaxLen = 2051 /* max. length encodable in 24 bits */ +}; + +enum +{ + StatBytes, + StatOutBytes, + StatLits, + StatMatches, + StatLitBits, + StatOffBits, + StatLenBits, + + MaxStat +}; + +struct Huff +{ + short bits; /* length of the code */ + ulong encode; /* the code */ +}; + +static Huff lentab[MaxFastLen] = +{ + {2, 0x2}, /* 10 */ + {3, 0x6}, /* 110 */ + {5, 0x1c}, /* 11100 */ + {5, 0x1d}, /* 11101 */ + {6, 0x3c}, /* 111100 */ + {7, 0x7a}, /* 1111010 */ + {7, 0x7b}, /* 1111011 */ + {8, 0xf8}, /* 11111000 */ + {8, 0xf9}, /* 11111001 */ +}; + +static int thwmaxcheck; + +void +whackinit(Whack *tw, int level) +{ + thwmaxcheck = (1 << level); + thwmaxcheck -= thwmaxcheck >> 2; + if(thwmaxcheck < 2) + thwmaxcheck = 2; + else if(thwmaxcheck > 1024) + thwmaxcheck = 1024; + memset(tw, 0, sizeof *tw); + tw->begin = 2 * WhackMaxOff; +} + +/* + * find a string in the dictionary + */ +static int +whackmatch(Whack *b, uchar **ss, uchar *esrc, ulong h, ulong now) +{ + ushort then, off, last; + int bestoff, bestlen, check; + uchar *s, *t; + + s = *ss; + if(esrc < s + MinMatch) + return -1; + if(s + MaxLen < esrc) + esrc = s + MaxLen; + + bestoff = 0; + bestlen = 0; + check = thwmaxcheck; + last = 0; + for(then = b->hash[h]; check-- > 0; then = b->next[then & (WhackMaxOff - 1)]){ + off = now - then; + if(off <= last || off > WhackMaxOff) + break; + + /* + * don't need to check for the end because + * 1) s too close check above + */ + t = s - off; + if(s[0] == t[0] && s[1] == t[1] && s[2] == t[2]){ + if(!bestlen || esrc - s > bestlen && s[bestlen] == t[bestlen]){ + t += 3; + for(s += 3; s < esrc; s++){ + if(*s != *t) + break; + t++; + } + if(s - *ss > bestlen){ + bestlen = s - *ss; + bestoff = off; + if(bestlen > thwmaxcheck) + break; + } + } + } + s = *ss; + last = off; + } + *ss += bestlen; + return bestoff; +} + +/* + * knuth vol. 3 multiplicative hashing + * each byte x chosen according to rules + * 1/4 < x < 3/10, 1/3 x < < 3/7, 4/7 < x < 2/3, 7/10 < x < 3/4 + * with reasonable spread between the bytes & their complements + * + * the 3 byte value appears to be as almost good as the 4 byte value, + * and might be faster on some machines + */ +/* +#define hashit(c) ((((ulong)(c) * 0x6b43a9) >> (24 - HashLog)) & HashMask) +*/ +#define hashit(c) (((((ulong)(c) & 0xffffff) * 0x6b43a9b5) >> (32 - HashLog)) & HashMask) + +/* + * lz77 compression with single lookup in a hash table for each block + */ +int +whack(Whack *w, uchar *dst, uchar *src, int n, ulong stats[WhackStats]) +{ + uchar *s, *ss, *sss, *esrc, *half, *wdst, *wdmax; + ulong cont, code, wbits; + ushort now; + int toff, lithist, h, len, bits, use, wnbits, lits, matches, offbits, lenbits; + + if(!compressblocks || n < MinMatch) + return -1; + + wdst = dst; + wdmax = dst + n; + + now = w->begin; + s = src; + w->data = s; + + cont = (s[0] << 16) | (s[1] << 8) | s[2]; + + esrc = s + n; + half = s + (n >> 1); + wnbits = 0; + wbits = 0; + lits = 0; + matches = 0; + offbits = 0; + lenbits = 0; + lithist = ~0; + while(s < esrc){ + h = hashit(cont); + + sss = s; + toff = whackmatch(w, &sss, esrc, h, now); + ss = sss; + + len = ss - s; + for(; wnbits >= 8; wnbits -= 8){ + if(wdst >= wdmax){ + w->begin = now; + return -1; + } + *wdst++ = wbits >> (wnbits - 8); + } + if(len < MinMatch){ + toff = *s; + lithist = (lithist << 1) | toff < 32 | toff > 127; + if(lithist & 0x1e){ + wbits = (wbits << 9) | toff; + wnbits += 9; + }else if(lithist & 1){ + toff = (toff + 64) & 0xff; + if(toff < 96){ + wbits = (wbits << 10) | toff; + wnbits += 10; + }else{ + wbits = (wbits << 11) | toff; + wnbits += 11; + } + }else{ + wbits = (wbits << 8) | toff; + wnbits += 8; + } + lits++; + + /* + * speed hack + * check for compression progress, bail if none achieved + */ + if(s > half){ + if(4 * (s - src) < 5 * lits){ + w->begin = now; + return -1; + } + half = esrc; + } + + if(s + MinMatch <= esrc){ + w->next[now & (WhackMaxOff - 1)] = w->hash[h]; + w->hash[h] = now; + if(s + MinMatch < esrc) + cont = (cont << 8) | s[MinMatch]; + } + now++; + s++; + continue; + } + + matches++; + + /* + * length of match + */ + if(len > MaxLen){ + len = MaxLen; + ss = s + len; + } + len -= MinMatch; + if(len < MaxFastLen){ + bits = lentab[len].bits; + wbits = (wbits << bits) | lentab[len].encode; + wnbits += bits; + lenbits += bits; + }else{ + code = BigLenCode; + bits = BigLenBits; + use = BigLenBase; + len -= MaxFastLen; + while(len >= use){ + len -= use; + code = (code + use) << 1; + use <<= (bits & 1) ^ 1; + bits++; + } + + wbits = (wbits << bits) | (code + len); + wnbits += bits; + lenbits += bits; + + for(; wnbits >= 8; wnbits -= 8){ + if(wdst >= wdmax){ + w->begin = now; + return -1; + } + *wdst++ = wbits >> (wnbits - 8); + } + } + + /* + * offset in history + */ + toff--; + for(bits = MinOffBits; toff >= (1 << bits); bits++) + ; + if(bits < MaxOffBits-1){ + wbits = (wbits << 3) | (bits - MinOffBits); + if(bits != MinOffBits) + bits--; + wnbits += bits + 3; + offbits += bits + 3; + }else{ + wbits = (wbits << 4) | 0xe | (bits - (MaxOffBits-1)); + bits--; + wnbits += bits + 4; + offbits += bits + 4; + } + wbits = (wbits << bits) | toff & ((1 << bits) - 1); + + for(; s != ss; s++){ + if(s + MinMatch <= esrc){ + h = hashit(cont); + w->next[now & (WhackMaxOff - 1)] = w->hash[h]; + w->hash[h] = now; + if(s + MinMatch < esrc) + cont = (cont << 8) | s[MinMatch]; + } + now++; + } + } + + w->begin = now; + + stats[StatBytes] += esrc - src; + stats[StatLits] += lits; + stats[StatMatches] += matches; + stats[StatLitBits] += (wdst - (dst + 2)) * 8 + wnbits - offbits - lenbits; + stats[StatOffBits] += offbits; + stats[StatLenBits] += lenbits; + + if(wnbits & 7){ + wbits <<= 8 - (wnbits & 7); + wnbits += 8 - (wnbits & 7); + } + for(; wnbits >= 8; wnbits -= 8){ + if(wdst >= wdmax) + return -1; + *wdst++ = wbits >> (wnbits - 8); + } + + stats[StatOutBytes] += wdst - dst; + + return wdst - dst; +} + +int +whackblock(uchar *dst, uchar *src, int ssize) +{ + Whack w; + ulong stats[MaxStat]; + int r; + + whackinit(&w, 6); + r = whack(&w, dst, src, ssize, stats); + return r; +} blob - /dev/null blob + fb966169c073eb5f0e7c69885312e5acc44dfc5c (mode 644) --- /dev/null +++ src/cmd/venti/srv/whack.h @@ -0,0 +1,40 @@ +typedef struct Whack Whack; +typedef struct Unwhack Unwhack; + +enum +{ + WhackStats = 8, + WhackErrLen = 64, /* max length of error message from thwack or unthwack */ + WhackMaxOff = 16*1024, /* max allowed offset */ + + HashLog = 14, + HashSize = 1<= 8 */ + + MaxSeqMask = 8, /* number of bits in coding block mask */ + MaxSeqStart = 256 /* max offset of initial coding block */ +}; + +struct Whack +{ + ushort begin; /* time of first byte in hash */ + ushort hash[HashSize]; + ushort next[WhackMaxOff]; + uchar *data; +}; + +struct Unwhack +{ + char err[WhackErrLen]; +}; + +void whackinit(Whack*, int level); +void unwhackinit(Unwhack*); +int whack(Whack*, uchar *dst, uchar *src, int nsrc, ulong stats[WhackStats]); +int unwhack(Unwhack*, uchar *dst, int ndst, uchar *src, int nsrc); + +int whackblock(uchar *dst, uchar *src, int ssize); blob - /dev/null blob + 4b8358ca261de4035620a1fc79a4d9ef25a68694 (mode 644) --- /dev/null +++ src/cmd/venti/srv/wrarena.c @@ -0,0 +1,217 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +QLock godot; +char *host; +int readonly = 1; /* for part.c */ +int mainstacksize = 256*1024; +Channel *c; +VtConn *z; +int fast; /* and a bit unsafe; only for benchmarking */ +int haveaoffset; +int maxwrites = -1; + +typedef struct ZClump ZClump; +struct ZClump +{ + ZBlock *lump; + Clump cl; + u64int aa; +}; + +void +usage(void) +{ + fprint(2, "usage: wrarena [-h host] arenafile [offset]\n"); + threadexitsall("usage"); +} + +void +vtsendthread(void *v) +{ + ZClump zcl; + + USED(v); + while(recv(c, &zcl) == 1){ + if(zcl.lump == nil) + break; + if(vtwrite(z, zcl.cl.info.score, zcl.cl.info.type, zcl.lump->data, zcl.cl.info.uncsize) < 0) + sysfatal("failed writing clump %llud: %r", zcl.aa); + freezblock(zcl.lump); + } + /* + * All the send threads try to exit right when + * threadmain is calling threadexitsall. + * Either libthread or the Linux NPTL pthreads library + * can't handle this condition (I suspect NPTL but have + * not confirmed this) and we get a seg fault in exit. + * I spent a day tracking this down with no success, + * so we're going to work around it instead by just + * sitting here and waiting for the threadexitsall to + * take effect. + */ + qlock(&godot); +} + +static void +rdarena(Arena *arena, u64int offset) +{ + u64int a, aa, e; + u32int magic; + Clump cl; + uchar score[VtScoreSize]; + ZBlock *lump; + ZClump zcl; + + fprint(2, "wrarena: copying %s to venti\n", arena->name); + printarena(2, arena); + + a = arena->base; + e = arena->base + arena->size; + if(offset != ~(u64int)0) { + if(offset >= e-a) + sysfatal("bad offset %llud >= %llud\n", + offset, e-a); + aa = offset; + } else + aa = 0; + + if(maxwrites != 0) + for(; aa < e; aa += ClumpSize+cl.info.size) { + magic = clumpmagic(arena, aa); + if(magic == ClumpFreeMagic) + break; + if(magic != arena->clumpmagic) { + // fprint(2, "illegal clump magic number %#8.8ux offset %llud\n", + // magic, aa); + break; + } + lump = loadclump(arena, aa, 0, &cl, score, 0); + if(lump == nil) { + fprint(2, "clump %llud failed to read: %r\n", aa); + break; + } + if(!fast && cl.info.type != VtCorruptType) { + scoremem(score, lump->data, cl.info.uncsize); + if(scorecmp(cl.info.score, score) != 0) { + fprint(2, "clump %llud has mismatched score\n", aa); + break; + } + if(vttypevalid(cl.info.type) < 0) { + fprint(2, "clump %llud has bad type %d\n", aa, cl.info.type); + break; + } + } + if(z && cl.info.type != VtCorruptType){ + zcl.cl = cl; + zcl.lump = lump; + zcl.aa = aa; + send(c, &zcl); + }else + freezblock(lump); + if(maxwrites>0 && --maxwrites == 0) + break; + } + if(haveaoffset) + print("end offset %llud\n", aa); +} + +void +threadmain(int argc, char *argv[]) +{ + int i; + char *file; + Arena *arena; + u64int offset, aoffset; + Part *part; + Dir *d; + uchar buf[8192]; + ArenaHead head; + ZClump zerocl; + + qlock(&godot); + aoffset = 0; + ARGBEGIN{ + case 'f': + fast = 1; + ventidoublechecksha1 = 0; + break; + case 'h': + host = EARGF(usage()); + break; + case 'o': + haveaoffset = 1; + aoffset = strtoull(EARGF(usage()), 0, 0); + break; + case 'M': + maxwrites = atoi(EARGF(usage())); + break; + default: + usage(); + break; + }ARGEND + + offset = ~(u64int)0; + switch(argc) { + default: + usage(); + case 2: + offset = strtoull(argv[1], 0, 0); + /* fall through */ + case 1: + file = argv[0]; + } + + fmtinstall('V', vtscorefmt); + + statsinit(); + + if((d = dirstat(file)) == nil) + sysfatal("can't stat file %s: %r", file); + + part = initpart(file, OREAD); + if(part == nil) + sysfatal("can't open file %s: %r", file); + if(readpart(part, aoffset, buf, sizeof buf) < 0) + sysfatal("can't read file %s: %r", file); + + if(unpackarenahead(&head, buf) < 0) + sysfatal("corrupted arena header: %r"); + + if(aoffset+head.size > d->length) + sysfatal("arena is truncated: want %llud bytes have %llud\n", + head.size, d->length); + + partblocksize(part, head.blocksize); + initdcache(8 * MaxDiskBlock); + + arena = initarena(part, aoffset, head.size, head.blocksize); + if(arena == nil) + sysfatal("initarena: %r"); + + if(host && strcmp(host, "/dev/null") != 0){ + z = vtdial(host); + if(z == nil) + sysfatal("could not connect to server: %r"); + if(vtconnect(z) < 0) + sysfatal("vtconnect: %r"); + }else + z = nil; + + c = chancreate(sizeof(ZClump), 0); + for(i=0; i<12; i++) + vtproc(vtsendthread, nil); + + rdarena(arena, offset); + if(vtsync(z) < 0) + sysfatal("executing sync: %r"); + + memset(&zerocl, 0, sizeof zerocl); + for(i=0; i<12; i++) + send(c, &zerocl); + if(z){ + vthangup(z); + } + threadexitsall(0); +} blob - /dev/null blob + e91afa0547fafcc06bbaae1813f028a753944d73 (mode 644) --- /dev/null +++ src/cmd/venti/srv/xml.c @@ -0,0 +1,68 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" +#include "xml.h" + +void xmlarena(Hio *hout, Arena *s, char *tag, int indent){ + xmlindent(hout, indent); + hprint(hout, "<%s", tag); + xmlaname(hout, s->name, "name"); + xmlu32int(hout, s->version, "version"); + xmlaname(hout, s->part->name, "partition"); + xmlu32int(hout, s->blocksize, "blocksize"); + xmlu64int(hout, s->base, "start"); + xmlu64int(hout, s->base+2*s->blocksize, "stop"); + xmlu32int(hout, s->ctime, "created"); + xmlu32int(hout, s->wtime, "modified"); + xmlsealed(hout, s->memstats.sealed, "sealed"); + xmlscore(hout, s->score, "score"); + xmlu32int(hout, s->memstats.clumps, "clumps"); + xmlu32int(hout, s->memstats.cclumps, "compressedclumps"); + xmlu64int(hout, s->memstats.uncsize, "data"); + xmlu64int(hout, s->memstats.used - s->memstats.clumps * ClumpSize, "compresseddata"); + xmlu64int(hout, s->memstats.used + s->memstats.clumps * ClumpInfoSize, "storage"); + hprint(hout, "/>\n"); +} + +void xmlindex(Hio *hout, Index *s, char *tag, int indent){ + int i; + xmlindent(hout, indent); + hprint(hout, "<%s", tag); + xmlaname(hout, s->name, "name"); + xmlu32int(hout, s->version, "version"); + xmlu32int(hout, s->blocksize, "blocksize"); + xmlu32int(hout, s->tabsize, "tabsize"); + xmlu32int(hout, s->buckets, "buckets"); + xmlu32int(hout, s->div, "buckdiv"); + hprint(hout, ">\n"); + xmlindent(hout, indent + 1); + hprint(hout, "\n"); + for(i = 0; i < s->nsects; i++) + xmlamap(hout, &s->smap[i], "sect", indent + 2); + xmlindent(hout, indent + 1); + hprint(hout, "\n"); + xmlindent(hout, indent + 1); + hprint(hout, "\n"); + for(i = 0; i < s->narenas; i++) + xmlamap(hout, &s->amap[i], "amap", indent + 2); + xmlindent(hout, indent + 1); + hprint(hout, "\n"); + xmlindent(hout, indent + 1); + hprint(hout, "\n"); + for(i = 0; i < s->narenas; i++) + xmlarena(hout, s->arenas[i], "arena", indent + 2); + xmlindent(hout, indent + 1); + hprint(hout, "\n"); + xmlindent(hout, indent); + hprint(hout, "\n", tag); +} + +void xmlamap(Hio *hout, AMap *s, char *tag, int indent){ + xmlindent(hout, indent); + hprint(hout, "<%s", tag); + xmlaname(hout, s->name, "name"); + xmlu64int(hout, s->start, "start"); + xmlu64int(hout, s->stop, "stop"); + hprint(hout, "/>\n"); +} + blob - /dev/null blob + c9e52b0bb666b62ae04903313c9e4e1b09ff0d36 (mode 644) --- /dev/null +++ src/cmd/venti/srv/xml.h @@ -0,0 +1,11 @@ +void xmlamap(Hio *hout, AMap *v, char *tag, int indent); +void xmlarena(Hio *hout, Arena *v, char *tag, int indent); +void xmlindex(Hio *hout, Index *v, char *tag, int indent); + +void xmlaname(Hio *hout, char *v, char *tag); +void xmlscore(Hio *hout, u8int *v, char *tag); +void xmlsealed(Hio *hout, int v, char *tag); +void xmlu32int(Hio *hout, u32int v, char *tag); +void xmlu64int(Hio *hout, u64int v, char *tag); + +void xmlindent(Hio *hout, int indent); blob - /dev/null blob + b33cdd25d47ecc8c022e92d3999aea6a39364cb8 (mode 644) --- /dev/null +++ src/cmd/venti/srv/zblock.c @@ -0,0 +1,93 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +void +fmtzbinit(Fmt *f, ZBlock *b) +{ + f->runes = 0; + f->start = b->data; + f->to = f->start; + f->stop = (char*)f->start + b->len; + f->flush = nil; + f->farg = nil; + f->nfmt = 0; +} + +#define ROUNDUP(p, n) ((void*)(((ulong)(p)+(n)-1)&~(ulong)((n)-1))) + +static char zmagic[] = "1234567890abcdefghijkl"; + +ZBlock * +alloczblock(u32int size, int zeroed, uint blocksize) +{ + uchar *p, *data; + ZBlock *b; + static ZBlock z; + int n; + + if(blocksize == 0) + blocksize = 32; /* try for cache line alignment */ + + n = size+32/*XXX*/+sizeof(ZBlock)+blocksize+8; + p = malloc(n); + if(p == nil){ + seterr(EOk, "out of memory"); + return nil; + } + + data = ROUNDUP(p, blocksize); + b = ROUNDUP(data+size+32/*XXX*/, 8); + if(0) fprint(2, "alloc %p-%p data %p-%p b %p-%p\n", + p, p+n, data, data+size, b, b+1); + *b = z; + b->data = data; + b->free = p; + b->len = size; + b->_size = size; + if(zeroed) + memset(b->data, 0, size); + memmove(b->data+size, zmagic, 32/*XXX*/); + return b; +} + +void +freezblock(ZBlock *b) +{ + if(b){ + if(memcmp(b->data+b->_size, zmagic, 32) != 0) + abort(); + memset(b->data+b->_size, 0, 32); + free(b->free); + } +} + +ZBlock* +packet2zblock(Packet *p, u32int size) +{ + ZBlock *b; + + if(p == nil) + return nil; + b = alloczblock(size, 0, 0); + if(b == nil) + return nil; + if(packetcopy(p, b->data, 0, size) < 0){ + freezblock(b); + return nil; + } + return b; +} + +Packet* +zblock2packet(ZBlock *zb, u32int size) +{ + Packet *p; + + if(zb == nil) + return nil; + p = packetalloc(); + packetappend(p, zb->data, size); + return p; +} + blob - /dev/null blob + fe75c81df9ee152941a3e16a479904cc5b373b0d (mode 644) --- /dev/null +++ src/cmd/venti/srv/zeropart.c @@ -0,0 +1,31 @@ +#include "stdinc.h" +#include "dat.h" +#include "fns.h" + +void +zeropart(Part *part, int blocksize) +{ + ZBlock *b; + u64int addr; + int w; + + fprint(2, "clearing the partition\n"); +//fprint(2, "NOT!\n"); +//return; +//b=alloczblock(MaxIoSize, 1, blocksize); +//freezblock(b); + b = alloczblock(MaxIoSize, 1, blocksize); + + w = 0; + for(addr = PartBlank; addr + MaxIoSize <= part->size; addr += MaxIoSize){ + if(writepart(part, addr, b->data, MaxIoSize) < 0) + sysfatal("can't initialize %s, writing block %d failed: %r", part->name, w); + w++; + } + + for(; addr + blocksize <= part->size; addr += blocksize) + if(writepart(part, addr, b->data, blocksize) < 0) + sysfatal("can't initialize %s: %r", part->name); + + freezblock(b); +} blob - /dev/null blob + 9d817a72ebdb9c79e8c4cc0a6ba49e654aeb313d (mode 644) --- /dev/null +++ src/cmd/venti/sync.c @@ -0,0 +1,54 @@ +#include +#include +#include +#include + +char *host; +int donothing; + +void +usage(void) +{ + fprint(2, "usage: sync [-h host]\n"); + threadexitsall("usage"); +} + +void +threadmain(int argc, char *argv[]) +{ + VtConn *z; + + fmtinstall('V', vtscorefmt); + fmtinstall('F', vtfcallfmt); + + ARGBEGIN{ + case 'h': + host = EARGF(usage()); + if(host == nil) + usage(); + break; + case 'x': + donothing = 1; + break; + default: + usage(); + break; + }ARGEND + + if(argc != 0) + usage(); + + z = vtdial(host); + if(z == nil) + sysfatal("could not connect to server: %r"); + + if(vtconnect(z) < 0) + sysfatal("vtconnect: %r"); + + if(!donothing) + if(vtsync(z) < 0) + sysfatal("vtsync: %r"); + + vthangup(z); + threadexitsall(0); +} blob - /dev/null blob + c11a5a3143a22947b55bbd2f1e00974ce56b1afa (mode 644) --- /dev/null +++ src/cmd/venti/write.c @@ -0,0 +1,62 @@ +#include +#include +#include +#include +#include + +void +usage(void) +{ + fprint(2, "usage: write [-z] [-h host] [-t type] VtMaxLumpSize) + sysfatal("input too big: max block size is %d", VtMaxLumpSize); + z = vtdial(host); + if(z == nil) + sysfatal("could not connect to server: %r"); + if(vtconnect(z) < 0) + sysfatal("vtconnect: %r"); + if(dotrunc) + n = vtzerotruncate(type, p, n); + if(vtwrite(z, score, type, p, n) < 0) + sysfatal("vtwrite: %r"); + vthangup(z); + print("%V\n", score); + threadexitsall(0); +}