2 * vbackup [-Dnv] fspartition [score]
4 * Copy a file system to a disk image stored on Venti.
5 * Prints a vnfs config line for the copied image.
9 * -n nop -- don't actually write blocks
10 * -s print status updates
11 * -v print debugging trace
12 * -w write parallelism
14 * If score is given on the command line, it should be the
15 * score from a previous vbackup on this fspartition.
16 * In this mode, only the new blocks are stored to Venti.
17 * The result is still a complete image, but requires many
18 * fewer Venti writes in the common case.
20 * This program is structured as three processes connected
23 * fsysproc | cmpproc | ventiproc
25 * Fsysproc reads the disk and queues the blocks.
26 * Cmpproc compares the blocks against the SHA1 hashes
27 * in the old image, if any. It discards the unchanged blocks
28 * and queues the changed ones. Ventiproc writes blocks to Venti.
30 * There is a fourth proc, statusproc, which prints status
31 * updates about how the various procs are progressing.
48 typedef struct WriteReq WriteReq;
55 Biobuf bscores; /* biobuf filled with block scores */
56 int debug; /* debugging flag (not used) */
57 Disk* disk; /* disk being backed up */
58 RWLock endlk; /* silly synchonization */
59 int errors; /* are we exiting with an error status? */
60 int fsscanblock; /* last block scanned */
61 Fsys* fsys; /* file system being backed up */
62 int nchange; /* number of changed blocks */
63 int nop; /* don't actually send blocks to venti */
64 int nwrite; /* number of write-behind threads */
65 Queue* qcmp; /* queue fsys->cmp */
66 Queue* qventi; /* queue cmp->venti */
67 int statustime; /* print status every _ seconds */
68 int verbose; /* print extra stuff */
69 VtFile* vfile; /* venti file being written */
70 Channel* writechan; /* chan(WriteReq) */
71 VtConn* z; /* connection to venti */
72 VtCache* zcache; /* cache of venti blocks */
73 uchar* zero; /* blocksize zero bytes */
75 extern int ncopy, nread, nwrite; /* hidden in libventi */
79 void statusproc(void*);
80 void ventiproc(void*);
82 char* mountplace(char *dev);
87 fprint(2, "usage: vbackup [-DVnv] [-m mtpt] [-s secs] [-w n] disk [score]\n");
88 threadexitsall("usage");
92 threadmain(int argc, char **argv)
94 char *pref, *mountname;
95 uchar score[VtScoreSize], prev[VtScoreSize];
105 fmtinstall('F', vtfcallfmt);
106 fmtinstall('H', encodefmt);
107 fmtinstall('T', timefmt);
108 fmtinstall('V', vtscorefmt);
110 mountname = sysname();
122 mountname = EARGF(usage());
128 statustime = atoi(EARGF(usage()));
134 nwrite = atoi(EARGF(usage()));
138 if(argc != 1 && argc != 2)
142 print("# %T vbackup %s %s\n", argv[0], argc>=2 ? argv[1] : "");
146 if((disk = diskopenfile(argv[0])) == nil)
147 sysfatal("diskopen: %r");
148 if((disk = diskcache(disk, 16384, 2*MAXQ+16)) == nil)
149 sysfatal("diskcache: %r");
150 if((fsys = fsysopen(disk)) == nil)
151 sysfatal("ffsopen: %r");
156 if((z = vtdial(nil)) == nil)
157 sysfatal("vtdial: %r");
159 sysfatal("vtconnect: %r");
162 * set up venti block cache
164 zero = vtmallocz(fsys->blocksize);
165 bsize = fsys->blocksize;
166 csize = 50; /* plenty; could probably do with 5 */
169 fprint(2, "cache %d blocks\n", csize);
170 c = vtcachealloc(z, bsize, csize);
174 * parse starting score
176 memset(prev, 0, sizeof prev);
178 vfile = vtfilecreateroot(c, (fsys->blocksize/VtScoreSize)*VtScoreSize,
179 fsys->blocksize, VtDataType);
181 sysfatal("vtfilecreateroot: %r");
182 vtfilelock(vfile, VtORDWR);
183 if(vtfilewrite(vfile, zero, 1, bsize*fsys->nblock-1) != 1)
184 sysfatal("vtfilewrite: %r");
185 if(vtfileflush(vfile) < 0)
186 sysfatal("vtfileflush: %r");
188 if(vtparsescore(argv[1], &pref, score) < 0)
189 sysfatal("bad score: %r");
190 if(pref!=nil && strcmp(pref, fsys->type) != 0)
191 sysfatal("score is %s but fsys is %s", pref, fsys->type);
192 b = vtcacheglobal(c, score, VtRootType);
194 if(vtrootunpack(&root, b->data) < 0)
195 sysfatal("bad root: %r");
196 if(strcmp(root.type, fsys->type) != 0)
197 sysfatal("root is %s but fsys is %s", root.type, fsys->type);
198 memmove(prev, score, VtScoreSize);
199 memmove(score, root.score, VtScoreSize);
202 b = vtcacheglobal(c, score, VtDirType);
204 sysfatal("vtcacheglobal %V: %r", score);
205 if(vtentryunpack(&e, b->data, 0) < 0)
206 sysfatal("%V: vtentryunpack failed", score);
208 fprint(2, "entry: size %llud psize %d dsize %d\n",
209 e.size, e.psize, e.dsize);
211 if((vfile = vtfileopenroot(c, &e)) == nil)
212 sysfatal("vtfileopenroot: %r");
213 vtfilelock(vfile, VtORDWR);
215 sysfatal("file system block sizes don't match %d %lld", e.dsize, bsize);
216 if(e.size != fsys->nblock*bsize)
217 sysfatal("file system block counts don't match %lld %lld", e.size, fsys->nblock*bsize);
221 * write scores of blocks into temporary file
223 if((tmp = getenv("TMP")) != nil){
225 }else if(access("/var/tmp", 0) >= 0)
229 tmpnam = smprint("%s/vbackup.XXXXXX", tmp);
231 sysfatal("smprint: %r");
233 if((fd = opentemp(tmpnam)) < 0)
234 sysfatal("opentemp %s: %r", tmpnam);
236 print("# %T reading scores into %s\n", tmpnam);
238 fprint(2, "read scores into %s...\n", tmpnam);
240 Binit(&bscores, fd, OWRITE);
241 for(i=0; i<fsys->nblock; i++){
242 if(vtfileblockscore(vfile, i, score) < 0)
243 sysfatal("vtfileblockhash %d: %r", i);
244 if(Bwrite(&bscores, score, VtScoreSize) != VtScoreSize)
245 sysfatal("Bwrite: %r");
251 * prep scores for rereading
254 Binit(&bscores, fd, OREAD);
257 * start the main processes
260 print("# %T starting procs\n");
265 proccreate(fsysproc, nil, STACK);
267 proccreate(ventiproc, nil, STACK);
269 proccreate(cmpproc, nil, STACK);
272 proccreate(statusproc, nil, STACK);
276 * wait for processes to finish
281 print("# %T procs exited: %d blocks changed, %d read, %d written, %d copied\n",
282 nchange, nread, nwrite, ncopy);
287 vtfilelock(vfile, -1);
288 if(vtfileflush(vfile) < 0)
289 sysfatal("vtfileflush: %r");
290 if(vtfilegetentry(vfile, &e) < 0)
291 sysfatal("vtfilegetentry: %r");
293 b = vtcacheallocblock(c, VtDirType);
295 sysfatal("vtcacheallocblock: %r");
296 vtentrypack(&e, b->data, 0);
297 if(vtblockwrite(b) < 0)
298 sysfatal("vtblockwrite: %r");
300 memset(&root, 0, sizeof root);
301 strecpy(root.name, root.name+sizeof root.name, argv[0]);
302 strecpy(root.type, root.type+sizeof root.type, fsys->type);
303 memmove(root.score, b->score, VtScoreSize);
304 root.blocksize = fsys->blocksize;
305 memmove(root.prev, prev, VtScoreSize);
308 b = vtcacheallocblock(c, VtRootType);
310 sysfatal("vtcacheallocblock: %r");
311 vtrootpack(&root, b->data);
312 if(vtblockwrite(b) < 0)
313 sysfatal("vtblockwrite: %r");
315 tm = *localtime(time(0));
318 print("mount /%s/%d/%02d%02d%s %s:%V %d/%02d%02d/%02d%02d\n",
319 mountname, tm.year, tm.mon, tm.mday,
322 tm.year, tm.mon, tm.mday, tm.hour, tm.min);
323 print("# %T %s %s:%V\n", argv[0], root.type, b->score);
325 print("# %T venti sync\n");
328 sysfatal("vtsync: %r");
330 print("# %T synced\n");
335 fsysproc(void *dummy)
342 for(i=0; i<fsys->nblock; i++){
344 if((db = fsysreadblock(fsys, i)) != nil)
350 print("# %T fsys proc exiting\n");
360 uchar score[VtScoreSize];
361 uchar score1[VtScoreSize];
365 bsize = fsys->blocksize;
366 while((db = qread(qcmp, &bno)) != nil){
368 sha1(data, vtzerotruncate(VtDataType, data, bsize), score, nil);
369 if(Bseek(&bscores, (vlong)bno*VtScoreSize, 0) < 0)
370 sysfatal("cmpproc Bseek: %r");
371 if(Bread(&bscores, score1, VtScoreSize) != VtScoreSize)
372 sysfatal("cmpproc Bread: %r");
373 if(memcmp(score, score1, VtScoreSize) != 0){
376 print("# block %ud: old %V new %V\n", bno, score1, score);
377 qwrite(qventi, db, bno);
389 uchar score[VtScoreSize];
393 while(recv(writechan, &wr) == 1){
396 if(vtwritepacket(z, score, wr.type, wr.p) < 0)
397 sysfatal("vtwritepacket: %r");
402 myvtwrite(VtConn *z, uchar score[VtScoreSize], uint type, uchar *buf, int n)
407 return vtwrite(z, score, type, buf, n);
409 wr.p = packetalloc();
410 packetappend(wr.p, buf, n);
411 packetsha1(wr.p, score);
413 send(writechan, &wr);
418 ventiproc(void *dummy)
427 proccreate(vtsendproc, z, STACK);
428 proccreate(vtrecvproc, z, STACK);
430 writechan = chancreate(sizeof(WriteReq), 0);
431 for(i=0; i<nwrite; i++)
432 threadcreate(writethread, nil, STACK);
433 vtcachesetwrite(zcache, myvtwrite);
435 bsize = fsys->blocksize;
436 vtfilelock(vfile, -1);
437 while((db = qread(qventi, &bno)) != nil){
442 if(vtfilewrite(vfile, db->data, bsize, bno*bsize) != bsize)
443 sysfatal("ventiproc vtfilewrite: %r");
444 if(vtfileflushbefore(vfile, (bno+1)*bsize) < 0)
445 sysfatal("ventiproc vtfileflushbefore: %r");
449 vtcachesetwrite(zcache, nil);
450 for(i=0; i<nwrite; i++)
451 send(writechan, nil);
456 percent(u32int a, u32int b)
458 return (vlong)a*100/b;
462 statusproc(void *dummy)
469 if(qcmp->closed && qcmp->nel==0 && qventi->closed && qventi->nel==0)
474 print("# %T fsscan=%d%% cmpq=%d%% ventiq=%d%%\n",
475 percent(fsscanblock, fsys->nblock),
476 percent(qcmp->nel, MAXQ),
477 percent(qventi->nel, MAXQ));
488 tm = *localtime(time(0));
489 return fmtprint(fmt, "%04d/%02d%02d %02d:%02d:%02d.%03d",
490 tm.year+1900, tm.mon+1, tm.mday, tm.hour, tm.min, tm.sec,
491 (int)(ns%1000000000)/1000000);
495 mountplace(char *dev)
502 sysfatal("pipe: %r");
507 cmd = smprint("mount | awk '$1==\"%s\" && $2 == \"on\" {print $3}'", dev);
508 if(threadspawnl(fd, "sh", "sh", "-c", cmd, nil) < 0)
509 sysfatal("exec mount|awk (to find mtpt of %s): %r", dev);
510 /* threadspawnl closed p[1] */
511 n = readn(p[0], buf, sizeof buf-1);
516 if((q = strchr(buf, '\n')) == nil)
520 if(q>buf && *(q-1) == '/')