commit - c5a183de108e5685305734d5cf984b58bb0d614a
commit + 45ac814c8609174199cadb6f1bbb4baf7c12c94a
blob - c576e9aaf4485e7457af1e716ca93e7ed8648a01
blob + 1a75c695b07f6722122c1e2b65217d735b924b62
--- src/cmd/venti/srv/arena.c
+++ src/cmd/venti/srv/arena.c
filling up and real errors writing the clump?
*/
u64int
-writeaclump(Arena *arena, Clump *c, u8int *clbuf, u64int start, u64int *pa)
+writeaclump(Arena *arena, Clump *c, u8int *clbuf)
{
DBlock *b;
u64int a, aa;
u32int clump, n, nn, m, off, blocksize;
int ok;
- AState as;
n = c->info.size + ClumpSize + U32Size;
qlock(&arena->lock);
if(!arena->memstats.sealed){
logerr(EOk, "seal memstats %s", arena->name);
arena->memstats.sealed = 1;
- as.arena = arena;
- as.aa = start+aa;
- as.stats = arena->memstats;
- setdcachestate(&as);
}
qunlock(&arena->lock);
return TWID64;
writeclumpinfo(arena, clump, &c->info);
wbarena(arena);
- /* set up for call to setdcachestate */
- as.arena = arena;
- as.aa = start+arena->memstats.used;
- as.stats = arena->memstats;
-
- /* update this before calling setdcachestate so it cannot be behind dcache.diskstate */
- *pa = start+aa;
- setdcachestate(&as);
qunlock(&arena->lock);
return aa;
blob - 917780b422321610bdfeabb6e2a0e046171346de
blob + 4b7b9bdda196839762a5f66e5f38b03df006cd74
--- src/cmd/venti/srv/buildindex.c
+++ src/cmd/venti/srv/buildindex.c
void
usage(void)
{
- fprint(2, "usage: buildindex [-b] [-i isect]... [-M imem] venti.conf\n");
+ fprint(2, "usage: buildindex [-bd] [-i isect]... [-M imem] venti.conf\n");
threadexitsall("usage");
}
case 'b':
bloom = 1;
break;
+ case 'd': /* debugging - make sure to run all 3 passes */
+ dumb = 1;
+ break;
case 'i':
isect = vtrealloc(isect, (nisect+1)*sizeof(isect[0]));
isect[nisect++] = EARGF(usage());
break;
- case 'd': /* debugging - make sure to run all 3 passes */
- dumb = 1;
- break;
case 'M':
imem = unittoull(EARGF(usage()));
break;
if(a->memstats.clumps)
fprint(2, "%T arena %s: %d entries\n",
a->name, a->memstats.clumps);
- addr = ix->amap[i].start;
- for(clump=0; clump<a->memstats.clumps; clump+=n){
+ /*
+ * Running the loop backwards accesses the
+ * clump info blocks forwards, since they are
+ * stored in reverse order at the end of the arena.
+ * This speeds things slightly.
+ */
+ addr = ix->amap[i].start + a->memstats.used;
+ for(clump=a->memstats.clumps; clump > 0; clump-=n){
n = ClumpChunks;
- if(n > a->memstats.clumps - clump)
- n = a->memstats.clumps - clump;
- if(readclumpinfos(a, clump, cis, n) != n){
+ if(n > clump)
+ n = clump;
+ if(readclumpinfos(a, clump-n, cis, n) != n){
fprint(2, "%T arena %s: directory read: %r\n", a->name);
errors = 1;
break;
}
- for(j=0; j<n; j++){
+ for(j=n-1; j>=0; j--){
ci = &cis[j];
ie.ia.type = ci->type;
ie.ia.size = ci->uncsize;
+ addr -= ci->size + ClumpSize;
ie.ia.addr = addr;
- addr += ci->size + ClumpSize;
ie.ia.blocks = (ci->size + ClumpSize + (1<<ABlockLog)-1) >> ABlockLog;
scorecp(ie.score, ci->score);
if(ci->type == VtCorruptType)
}
}
}
+ if(addr != ix->amap[i].start)
+ fprint(2, "%T arena %s: clump miscalculation %lld != %lld\n", a->name, addr, ix->amap[i].start);
}
add(&arenaentries, tot);
add(&skipentries, nskip);
blob - ea527126b51ceb39d216f7b2d7de6ec560c63984
blob + 54eb2e4481d1ffbd43bc2f672051b08efcf034c1
--- src/cmd/venti/srv/checkarenas.c
+++ src/cmd/venti/srv/checkarenas.c
err = 0;
for(;;){
- e = syncarena(arena, 0, 1000, 0, fix);
+ e = syncarena(arena, 1000, 0, fix);
err |= e;
if(!(e & SyncHeader))
break;
blob - ec277864c056eec8bcf6d6e89c672bfb91e60d80
blob + ed4de34d9bf8408f8ac2011b7e36ac7a5ec924c5
--- src/cmd/venti/srv/clump.c
+++ src/cmd/venti/srv/clump.c
memset(cb->data+ClumpSize+dsize, 0, 4);
cl.info.size = dsize;
- ia->addr = 0;
- ia->type = type;
- ia->size = size;
- ia->blocks = (dsize + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog;
-
- a = writeiclump(ix, &cl, cb->data, &ia->addr);
-
+ a = writeiclump(ix, &cl, cb->data);
trace(TraceLump, "storeclump exit %lld", a);
-
freezblock(cb);
if(a == TWID64)
return -1;
+ ia->addr = a;
+ ia->type = type;
+ ia->size = size;
+ ia->blocks = (dsize + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog;
+
/*
qlock(&stats.lock);
stats.clumpwrites++;
blob - e0a9e18a538aa0abd449716267afed83c71128d3
blob + 4ed8832cfba867a2882394f7090aefb0a35278cb
--- src/cmd/venti/srv/dat.h
+++ src/cmd/venti/srv/dat.h
AMap *smap; /* mapping of buckets to index sections */
int narenas;
AMap *amap; /* mapping from index addesses to arenas */
+
+ QLock writing;
};
/*
blob - f0cb531848c9a2b2456f1d94508f5a326ee7696a
blob + abbd26a403c75397a037adbe9f0adae20a084a29
--- src/cmd/venti/srv/dcache.c
+++ src/cmd/venti/srv/dcache.c
u8int *mem; /* memory for all block descriptors */
int ndirty; /* number of dirty blocks */
int maxdirty; /* max. number of dirty blocks */
-
- AState diskstate;
- AState state;
};
typedef struct Ra Ra;
vtproc(delaykickroundproc, &dcache.round);
}
-void
-setdcachestate(AState *a)
-{
- trace(TraceBlock, "setdcachestate %s 0x%llux clumps %d", a->arena ? a->arena->name : nil, a->aa, a->stats.clumps);
- qlock(&dcache.lock);
- dcache.state = *a;
- qunlock(&dcache.lock);
-}
-
-AState
-diskstate(void)
-{
- AState a;
-
- qlock(&dcache.lock);
- a = dcache.diskstate;
- qunlock(&dcache.lock);
- return a;
-}
-
static u32int
pbhash(u64int addr)
{
int i, j, n;
ulong t0;
DBlock *b, **write;
- AState as;
USED(v);
threadsetname("flushproc");
trace(TraceWork, "start");
t0 = nsec()/1000;
trace(TraceProc, "build t=%lud", (ulong)(nsec()/1000)-t0);
-
- qlock(&dcache.lock);
- as = dcache.state;
- qunlock(&dcache.lock);
write = dcache.write;
n = 0;
*/
trace(TraceProc, "undirty.%d t=%lud", j, (ulong)(nsec()/1000)-t0);
qlock(&dcache.lock);
- dcache.diskstate = as;
for(i=0; i<n; i++){
b = write[i];
--dcache.ndirty;
blob - fd9dccc5eef0c12208c3f621808999a5919e07db
blob + ac05ab8cd1be619c12edaf8a27152ccbf3e55de7
--- src/cmd/venti/srv/fixarenas.c
+++ src/cmd/venti/srv/fixarenas.c
return u32(pagein(0, Block)) == ArenaHeadMagic;
}
-static int tabsizes[] = { 16*1024, 64*1024, 512*1024, };
+static int tabsizes[] = { 16*1024, 64*1024, 512*1024, 768*1024, };
/*
* Poke around on the disk to guess what the ArenaPart numbers are.
*/
* Fmtarenas used to use 64k tab, now uses 512k tab.
*/
if(ap.arenabase == 0){
+ print("trying standard arena bases...\n");
for(i=0; i<nelem(tabsizes); i++){
- ap.arenabase = ROUNDUP(PartBlank+HeadSize, ap.blocksize);
+ ap.arenabase = ROUNDUP(PartBlank+HeadSize+tabsizes[i], ap.blocksize);
p = pagein(ap.arenabase, Block);
if(u32(p) == ArenaHeadMagic)
break;
bcit = cibuf;
ecit = cibuf+ncibuf;
- smart = 1;
+ smart = 0; /* Somehow the smart code doesn't do corrupt clumps right. */
Again:
nbad = 0;
ci = bci;
blob - 05a3e93726697ba8f843c5cc4a87792cf1b7a795
blob + 398562c2766ca3f9e51f068f18d90bee5ea7cdd3
--- src/cmd/venti/srv/fns.h
+++ src/cmd/venti/srv/fns.h
void dirtydblock(DBlock*, int);
void diskaccess(int);
void disksched(void);
-AState diskstate(void);
void *emalloc(ulong);
void emptydcache(void);
void emptyicache(void);
int hdebug(HConnect*);
int hdisk(HConnect*);
int hnotfound(HConnect*);
+int hproc(HConnect*);
int hsethtml(HConnect*);
int hsettext(HConnect*);
int httpdinit(char *address, char *webroot);
ulong icachedirtyfrac(void);
void icacheclean(IEntry*);
int icachelookup(u8int *score, int type, IAddr *ia);
+AState icachestate(void);
int ientrycmp(const void *vie1, const void *vie2);
char *ifileline(IFile *f);
int ifilename(IFile *f, char *dst);
void initround(Round*, char*, int);
int initventi(char *config, Config *conf);
void insertlump(Lump *lump, Packet *p);
-int insertscore(u8int *score, IAddr *ia, int state);
+int insertscore(u8int *score, IAddr *ia, int state, AState *as);
void kickdcache(void);
void kickicache(void);
void kickround(Round*, int wait);
int scorecmp(u8int *, u8int *);
void scoremem(u8int *score, u8int *buf, int size);
void setatailstate(AState*);
-void setdcachestate(AState*);
void seterr(int severity, char *fmt, ...);
void setstat(int, long);
void settrace(char *type);
int stru32int(char *s, u32int *r);
int stru64int(char *s, u64int *r);
void sumarena(Arena *arena);
-int syncarena(Arena *arena, u64int start, u32int n, int zok, int fix);
-int syncarenaindex(Index *ix, Arena *arena, u32int clump, u64int a, int fix, int *pflush, int check);
-int syncindex(Index *ix, int fix, int mustflushicache, int check);
+int syncarena(Arena *arena, u32int n, int zok, int fix);
+int syncindex(Index *ix);
void trace(char *type, char*, ...);
void traceinit(void);
int u64log2(u64int v);
int wbisect(ISect *is);
int wbindex(Index *ix);
int whackblock(u8int *dst, u8int *src, int ssize);
-u64int writeaclump(Arena *a, Clump *c, u8int *clbuf, u64int, u64int*);
+u64int writeaclump(Arena *a, Clump *c, u8int *clbuf);
u32int writearena(Arena *arena, u64int aa, u8int *clbuf, u32int n);
int writebloom(Bloom*);
int writeclumpinfo(Arena *arean, int clump, ClumpInfo *ci);
int writepng(Hio*, Memimage*);
-u64int writeiclump(Index *ix, Clump *c, u8int *clbuf, u64int*);
+u64int writeiclump(Index *ix, Clump *c, u8int *clbuf);
int writelump(Packet *p, u8int *score, int type, u32int creator, uint ms);
int writepart(Part *part, u64int addr, u8int *buf, u32int n);
int writeqlump(Lump *u, Packet *p, int creator, uint ms);
blob - 51d8b9a1fe18efb0d1e7aaea212d97da373f4e7b
blob + 5e4bbe74152e542e8b142c82d9cbe15b97b3f6f1
--- src/cmd/venti/srv/httpd.c
+++ src/cmd/venti/srv/httpd.c
httpdobj("/emptydcache", hdcacheempty);
httpdobj("/disk", hdisk);
httpdobj("/debug", hdebug);
+ httpdobj("/proc/", hproc);
if(vtproc(listenproc, address) < 0)
return -1;
if(scorecmp(zeroscore, arena->score) != 0)
hprint(hout, "\tscore=%V\n", arena->score);
- hprint(hout, "\tmem: clumps=%d compressed clumps=%d data=%,lld compressed data=%,lld storage=%,lld\n",
+ hprint(hout, "\twritten: clumps=%d compressed clumps=%d data=%,lld compressed data=%,lld storage=%,lld\n",
arena->memstats.clumps, arena->memstats.cclumps, arena->memstats.uncsize,
arena->memstats.used - arena->memstats.clumps * ClumpSize,
arena->memstats.used + arena->memstats.clumps * ClumpInfoSize);
- hprint(hout, "\tdisk: clumps=%d compressed clumps=%d data=%,lld compressed data=%,lld storage=%,lld\n",
+ hprint(hout, "\tindexed: clumps=%d compressed clumps=%d data=%,lld compressed data=%,lld storage=%,lld\n",
arena->diskstats.clumps, arena->diskstats.cclumps, arena->diskstats.uncsize,
arena->diskstats.used - arena->diskstats.clumps * ClumpSize,
arena->diskstats.used + arena->diskstats.clumps * ClumpInfoSize);
blob - /dev/null
blob + 42933920aa0f37e4d9e7835ae1b5acd99845327d (mode 644)
--- /dev/null
+++ src/cmd/venti/srv/hproc.c
+#include "stdinc.h"
+#include "dat.h"
+#include "fns.h"
+#include "xml.h"
+
+int
+hproc(HConnect *c)
+{
+ int r;
+
+ if((r = hsettext(c)) < 0)
+ return r;
+ hprint(&c->hout, "/proc only implemented on Plan 9\n");
+ hflush(&c->hout);
+ return 0;
+}
+
blob - 384fd2c1437a09aa33ab7c19bc53e9a252dbf583
blob + b1935d42c84d6bc42442f36a53f567e8b9e2db06
--- src/cmd/venti/srv/icache.c
+++ src/cmd/venti/srv/icache.c
IEntry dirty;
u32int maxdirty;
u32int ndirty;
+ AState as;
ISum **sum;
int nsum;
}
int
-insertscore(u8int score[VtScoreSize], IAddr *ia, int state)
+insertscore(u8int score[VtScoreSize], IAddr *ia, int state, AState *as)
{
ISum *toload;
else{
assert(state == IEDirty);
toload = nil;
+ if(as == nil)
+ fprint(2, "%T insertscore IEDirty without as; called from %lux\n", getcallerpc(&score));
+ else{
+ if(icache.as.aa > as->aa)
+ fprint(2, "%T insertscore: aa moving backward: %#llux -> %#llux\n", icache.as.aa, as->aa);
+ icache.as = *as;
+ }
}
qunlock(&icache.lock);
if(toload){
if(loadientry(mainindex, score, type, &d) < 0)
return -1;
- insertscore(score, &d.ia, IEClean);
+ insertscore(score, &d.ia, IEClean, nil);
*ia = d.ia;
return 0;
}
return dirty;
}
+AState
+icachestate(void)
+{
+ AState as;
+ qlock(&icache.lock);
+ as = icache.as;
+ qunlock(&icache.lock);
+ return as;
+}
+
/*
* The singly-linked non-circular list of index entries ie
* has been written to disk. Move them to the clean list.
blob - 0805001c65aa70f92dffb8fc197778ff13d1e925
blob + 1cb9fc0e25aad1c512c6b84009e11b3487ecac80
--- src/cmd/venti/srv/icachewrite.c
+++ src/cmd/venti/srv/icachewrite.c
static IEntry *iesort(IEntry*);
int icachesleeptime = 1000; /* milliseconds */
-int minicachesleeptime = 50;
+int minicachesleeptime = 0;
enum
{
threadsetname("icachewritecoord");
ix = mainindex;
- iwrite.as = diskstate();
+ iwrite.as = icachestate();
for(;;){
trace(TraceProc, "icachewritecoord sleep");
waitforkick(&iwrite.round);
trace(TraceWork, "start");
- as = diskstate();
+ as = icachestate();
if(as.arena==iwrite.as.arena && as.aa==iwrite.as.aa){
/* will not be able to do anything more than last flush - kick disk */
+ fprint(2, "icache: nothing to do - kick dcache\n");
trace(TraceProc, "icachewritecoord kick dcache");
kickdcache();
trace(TraceProc, "icachewritecoord kicked dcache");
+ goto SkipWork; /* won't do anything; don't bother rewriting bloom filter */
}
iwrite.as = as;
err |= recvul(ix->bloom->writedonechan);
trace(TraceProc, "icachewritecoord donewrite err=%d", err);
- if(err == 0)
+ if(err == 0){
setatailstate(&iwrite.as);
+ }
}
+ SkipWork:
icacheclean(nil); /* wake up anyone waiting */
trace(TraceWork, "finish");
addstat(StatIcacheFlush, 1);
blob - dd49e05505945c6cbe456ca79eadd7c2a134869e
blob + d751b98f619f3a31112298f3d39fdcfec41c6e44
--- src/cmd/venti/srv/index.c
+++ src/cmd/venti/srv/index.c
filling up and real errors writing the clump?
*/
u64int
-writeiclump(Index *ix, Clump *c, u8int *clbuf, u64int *pa)
+writeiclump(Index *ix, Clump *c, u8int *clbuf)
{
u64int a;
int i;
+ IAddr ia;
+ AState as;
trace(TraceLump, "writeiclump enter");
- for(i = ix->mapalloc; i < ix->narenas; i++){
- a = writeaclump(ix->arenas[i], c, clbuf, ix->amap[i].start, pa);
+ qlock(&ix->writing);
+ for(i = ix->mapalloc; i < ix->narenas; i++){
+ a = writeaclump(ix->arenas[i], c, clbuf);
if(a != TWID64){
- ix->mapalloc = i; /* assuming write is atomic, race is okay */
+ ix->mapalloc = i;
+ ia.addr = ix->amap[i].start + a;
+ ia.type = c->info.type;
+ ia.size = c->info.uncsize;
+ ia.blocks = (c->info.size + ClumpSize + (1<<ABlockLog) - 1) >> ABlockLog;
+ as.arena = ix->arenas[i];
+ as.aa = ia.addr;
+ as.stats = as.arena->memstats;
+ insertscore(c->info.score, &ia, IEDirty, &as);
+ qunlock(&ix->writing);
trace(TraceLump, "writeiclump exit");
- return a;
+ return ia.addr;
}
}
+ qunlock(&ix->writing);
seterr(EAdmin, "no space left in arenas");
trace(TraceLump, "writeiclump failed");
blob - 206d5e064352953fc097a3c95e2440e9d440ed0f
blob + 9b244948b118351d6ce3e686b710de25fa5bc1c6
--- src/cmd/venti/srv/lump.c
+++ src/cmd/venti/srv/lump.c
ok = storeclump(mainindex, flat, u->score, u->type, creator, &ia);
freezblock(flat);
if(ok == 0)
- ok = insertscore(u->score, &ia, IEDirty);
- if(ok == 0)
insertlump(u, p);
else
packetfree(p);
blob - 2c55495148d73d7a94a0874393b4af00300034ea
blob + dc54187e4fe00c4fa43a75aa7c0c6edd97bbe725
--- src/cmd/venti/srv/mkfile
+++ src/cmd/venti/srv/mkfile
dump.$O\
graph.$O\
hdisk.$O\
+ hproc.$O\
httpd.$O\
icache.$O\
icachewrite.$O\
blob - fe2c78143a65cc012985793661b82bc8860718e8
blob + 0e6cc2019110c132082c5c635c257022029e79c7
--- src/cmd/venti/srv/syncarena.c
+++ src/cmd/venti/srv/syncarena.c
* returns 0 if ok, flags if error occurred
*/
int
-syncarena(Arena *arena, u64int start, u32int n, int zok, int fix)
+syncarena(Arena *arena, u32int n, int zok, int fix)
{
ZBlock *lump;
Clump cl;
fprint(2, "%s: illegal clump magic number=%#8.8ux at clump=%d\n", arena->name, magic, clump);
/* err |= SyncDataErr; */
if(fix && writeclumpmagic(arena, aa, ClumpFreeMagic) < 0){
- fprint(2, "can't write corrected clump free magic: %r");
+ fprint(2, "%s: can't write corrected clump free magic: %r", arena->name);
err |= SyncFixErr;
}
break;
|| cclumps != arena->memstats.cclumps
|| uncsize != arena->memstats.uncsize){
err |= SyncHeader;
- fprint(2, "arena %s: start=%lld fix=%d flush=%d %lld->%lld %ud->%ud %ud->%ud %lld->%lld\n",
+ fprint(2, "arena %s: fix=%d flush=%d %lld->%lld %ud->%ud %ud->%ud %lld->%lld\n",
arena->name,
- start,
fix,
flush,
used, arena->memstats.used,
blob - fb3c4ce284077e58148da0916b0dbc3adf501e01
blob + 8f521558990d54d653b01a3f5d093b5c5516e41a
--- src/cmd/venti/srv/syncindex.c
+++ src/cmd/venti/srv/syncindex.c
void
usage(void)
{
- fprint(2, "usage: syncindex [-fv] [-B blockcachesize] config\n");
+ fprint(2, "usage: syncindex [-v] [-B blockcachesize] config\n");
threadexitsall("usage");
}
threadmain(int argc, char *argv[])
{
u32int bcmem, icmem;
- int fix;
- fix = 0;
bcmem = 0;
icmem = 0;
ARGBEGIN{
case 'I':
icmem = unittoull(EARGF(usage()));
break;
- case 'f':
- fix++;
- break;
case 'v':
verbose++;
break;
break;
}ARGEND
- if(!fix)
- readonly = 1;
-
if(argc != 1)
usage();
if(verbose)
printindex(2, mainindex);
- if(syncindex(mainindex, fix, 1, 0) < 0)
+ if(syncindex(mainindex) < 0)
sysfatal("failed to sync index=%s: %r\n", mainindex->name);
+ flushicache();
+ flushdcache();
threadexitsall(0);
}
blob - 98f6adf11db9d584fc0fc74a81c3775e3230c663
blob + be3a2ea064f61a84d45b3acdce4c854b273c4934
--- src/cmd/venti/srv/syncindex0.c
+++ src/cmd/venti/srv/syncindex0.c
#include "dat.h"
#include "fns.h"
-enum
+static int
+syncarenaindex(Arena *arena, u64int a0)
{
- ClumpChunks = 32*1024
-};
-
-static int missing, wrong;
-
-/*
- * shell sort is plenty good enough
- * because we're going to do a bunch of disk i/o's
- */
-static void
-sortclumpinfo(ClumpInfo *ci, int *s, int n)
-{
- int i, j, m, t;
-
- for(m = (n + 3) / 5; m > 0; m = (m + 1) / 3){
- for(i = n - m; i-- > 0;){
- for(j = i + m; j < n; j += m){
- if(memcmp(ci[s[j - m]].score, ci[s[j]].score, VtScoreSize) <= 0)
- break;
- t = s[j];
- s[j] = s[j - m];
- s[j - m] = t;
- }
- }
- }
-}
-
-int
-syncarenaindex(Index *ix, Arena *arena, u32int clump, u64int a, int fix, int *pflush, int check)
-{
- Packet *pack;
- IEntry ie;
+ int ok;
+ u32int clump;
+ u64int a;
+ ClumpInfo ci;
IAddr ia;
- ClumpInfo *ci, *cis;
- u64int *addrs;
- int i, n, ok, *s, flush;
+ AState as;
+
+ if(arena->diskstats.clumps == arena->memstats.clumps)
+ return 0;
+
+ memset(&as, 0, sizeof as);
+ as.arena = arena;
+ as.stats = arena->diskstats;
- trace(TraceProc, "syncarenaindex enter");
-
- flush = 0;
- cis = MKN(ClumpInfo, ClumpChunks);
- addrs = MKN(u64int, ClumpChunks);
- s = MKN(int, ClumpChunks);
ok = 0;
- for(; clump < arena->memstats.clumps; clump += n){
- n = ClumpChunks;
- if(n > arena->memstats.clumps - clump)
- n = arena->memstats.clumps - clump;
- n = readclumpinfos(arena, clump, cis, n);
- if(n <= 0){
- fprint(2, "arena directory read failed\n");
+ a = a0 + arena->diskstats.used;
+ for(clump=arena->diskstats.clumps; clump < arena->memstats.clumps; clump++){
+ if(readclumpinfo(arena, clump, &ci) < 0){
+ fprint(2, "%s: clump %d: cannot read clumpinfo\n",
+ arena->name, clump);
ok = -1;
break;
}
- for(i = 0; i < n; i++){
- addrs[i] = a;
- a += cis[i].size + ClumpSize;
- s[i] = i;
- }
+ ia.type = ci.type;
+ ia.size = ci.uncsize;
+ ia.addr = a;
+ ia.blocks = (ClumpSize + ci.size + (1 << ABlockLog) - 1) >> ABlockLog;
+ a += ClumpSize + ci.size;
- sortclumpinfo(cis, s, n);
-
- for(i = 0; i < n; i++){
- ci = &cis[s[i]];
- ia.type = ci->type;
- ia.size = ci->uncsize;
- ia.addr = addrs[s[i]];
- ia.blocks = (ci->size + ClumpSize + (1 << ABlockLog) - 1) >> ABlockLog;
-
- if(!check)
- goto Add;
- if(loadientry(ix, ci->score, ci->type, &ie) < 0){
- trace(TraceProc, "syncarenaindex missing block %V.%d", ci->score, ci->type);
- missing++;
- if(0) fprint(2, "missing block type=%d score=%V\n", ci->type, ci->score);
- }else if(iaddrcmp(&ia, &ie.ia) != 0){
- trace(TraceProc, "syncarenaindex mismatched entry");
- fprint(2, "\nmismatched index entry and clump at %d\n", clump + i);
- fprint(2, "\tclump: type=%d size=%d blocks=%d addr=%lld\n", ia.type, ia.size, ia.blocks, ia.addr);
- fprint(2, "\tindex: type=%d size=%d block=%d addr=%lld\n", ie.ia.type, ie.ia.size, ie.ia.blocks, ie.ia.addr);
- pack = readlump(ie.score, ie.ia.type, ie.ia.size, nil);
- packetfree(pack);
- if(pack != nil){
- fprint(2, "duplicated lump\n");
- continue;
- }
- wrong++;
- }else
- continue;
- Add:
- if(!fix){
- ok = -1;
- continue;
- }
- flush = 1;
- trace(TraceProc, "syncarenaindex insert %V", ci->score);
- insertscore(ci->score, &ia, IEDirty);
- }
-
- if(0 && clump / 1000 != (clump + n) / 1000)
- fprint(2, ".");
+ as.stats.used += ClumpSize + ci.size;
+ as.stats.uncsize += ia.size;
+ as.stats.clumps++;
+ if(ci.uncsize > ci.size)
+ as.stats.cclumps++;
+ as.aa = a;
+ insertscore(ci.score, &ia, IEDirty, &as);
}
- free(cis);
- free(addrs);
- free(s);
- if(flush){
- flushdcache();
- *pflush = 1;
- }
+ flushdcache();
return ok;
}
int
-syncindex(Index *ix, int fix, int mustflush, int check)
+syncindex(Index *ix)
{
Arena *arena;
- AState as;
- u64int a;
- int i, e, e1, ok, ok1, flush;
+ int i, e, e1, ok;
ok = 0;
- flush = 0;
for(i = 0; i < ix->narenas; i++){
trace(TraceProc, "syncindex start %d", i);
arena = ix->arenas[i];
- /*
- * Syncarena will scan through the arena looking for blocks
- * that have been forgotten. It will update arena->memstats.used,
- * so save the currenct copy as the place to start the
- * syncarenaindex scan.
- */
- a = arena->memstats.used;
- e = syncarena(arena, ix->amap[i].start, TWID32, fix, fix);
+ e = syncarena(arena, TWID32, 1, 1);
e1 = e;
- if(fix)
- e1 &= ~(SyncHeader|SyncCIZero|SyncCIErr);
- if(e1 == SyncHeader)
+ e1 &= ~(SyncHeader|SyncCIZero|SyncCIErr);
+ if(e & SyncHeader)
fprint(2, "arena %s: header is out-of-date\n", arena->name);
- if(e1)
+ if(e1){
+ fprint(2, "arena %s: %x\n", arena->name, e1);
ok = -1;
- else{
- /*
- * use diskstats not memstats here, because diskstats
- * is what has been indexed; memstats is what has
- * made it to disk (confusing names).
- */
- ok1 = syncarenaindex(ix, arena,
- arena->diskstats.clumps,
- ix->amap[i].start + arena->diskstats.used,
- fix, &flush, check);
- if(ok1 < 0)
- fprint(2, "syncarenaindex: %r\n");
- if(fix && ok1==0 && (e & SyncHeader) && wbarena(arena) < 0)
- fprint(2, "arena=%s header write failed: %r\n", arena->name);
- ok |= ok1;
+ continue;
+ }
+ flushdcache();
+
+ if(arena->memstats.clumps == arena->diskstats.clumps)
+ continue;
+
+ fprint(2, "%T %s: indexing %d clumps...\n",
+ arena->name,
+ arena->memstats.clumps - arena->diskstats.clumps);
- as.arena = arena;
- as.aa = ix->amap[i].start + arena->memstats.used;
- as.stats = arena->memstats;
- setdcachestate(&as);
+ if(syncarenaindex(arena, ix->amap[i].start) < 0){
+ fprint(2, "arena %s: syncarenaindex: %r\n", arena->name);
+ ok = -1;
+ continue;
}
- }
- if(missing || wrong)
- fprint(2, "syncindex: %d missing entries, %d wrong entries (flush=%d)\n", missing, wrong, flush);
- if(fix && wbindex(ix) < 0){
- fprint(2, "can't write back index header for %s: %r\n", ix->name);
- return -1;
- }
- if(fix && flush){
+ if(wbarena(arena) < 0){
+ fprint(2, "arena %s: wbarena: %r\n", arena->name);
+ ok = -1;
+ continue;
+ }
flushdcache();
- if(mustflush){
- flushicache();
- flushdcache();
- }else
- kickicache();
+ delaykickicache();
}
return ok;
}
blob - 4d59dfc6d483735ab4be20600894ec472e43a021
blob + 8149537698c17a0bff9b9aaec7a23862be4177ca
--- src/cmd/venti/srv/venti.c
+++ src/cmd/venti/srv/venti.c
startbloomproc(mainindex->bloom);
fprint(2, "sync...");
- if(!readonly && syncindex(mainindex, 1, 0, 0) < 0)
+ if(!readonly && syncindex(mainindex) < 0)
sysfatal("can't sync server: %r");
if(!readonly && queuewrites){
blob - a9f67b11619726b45d3f8dad9b61bd527e11447d
blob + cbd65127179c95a39404cc6f71d0a6013d7b5b8b
--- src/cmd/venti/srv/wrarena.c
+++ src/cmd/venti/srv/wrarena.c
Arena *arena;
u64int offset, aoffset;
Part *part;
- Dir *d;
uchar buf[8192];
ArenaHead head;
ZClump zerocl;
statsinit();
- if((d = dirstat(file)) == nil)
- sysfatal("can't stat file %s: %r", file);
-
part = initpart(file, OREAD);
if(part == nil)
sysfatal("can't open file %s: %r", file);
if(unpackarenahead(&head, buf) < 0)
sysfatal("corrupted arena header: %r");
- if(aoffset+head.size > d->length)
+ if(aoffset+head.size > part->size)
sysfatal("arena is truncated: want %llud bytes have %llud\n",
- head.size, d->length);
+ head.size, part->size);
partblocksize(part, head.blocksize);
initdcache(8 * MaxDiskBlock);
