1 /*
2  * vbackup [-Dnv] fspartition [score]
3  *
4  * Copy a file system to a disk image stored on Venti.
5  * Prints a vnfs config line for the copied image.
6  *
7  *	-D	print debugging
8  *	-m	set mount name
9  *	-n	nop -- don't actually write blocks
10  *	-s	print status updates
11  *	-v	print debugging trace
12  *	-w	write parallelism
13  * 
14  * If score is given on the command line, it should be the
15  * score from a previous vbackup on this fspartition.
16  * In this mode, only the new blocks are stored to Venti.
17  * The result is still a complete image, but requires many
18  * fewer Venti writes in the common case.
19  *
20  * This program is structured as three processes connected
21  * by buffered queues:
22  *
23  * 	fsysproc | cmpproc | ventiproc
24  * 
25  * Fsysproc reads the disk and queues the blocks.
26  * Cmpproc compares the blocks against the SHA1 hashes
27  * in the old image, if any.  It discards the unchanged blocks
28  * and queues the changed ones.  Ventiproc writes blocks to Venti.
29  * 
30  * There is a fourth proc, statusproc, which prints status
31  * updates about how the various procs are progressing.
32  */
33 
34 #include <u.h>
35 #include <libc.h>
36 #include <bio.h>
37 #include <thread.h>
38 #include <libsec.h>
39 #include <venti.h>
40 #include <diskfs.h>
41 #include "queue.h"
42 
43 enum
44 {
45 	STACK = 32768,
46 };
47 
48 typedef struct WriteReq WriteReq;
49 struct WriteReq
50 {
51 	Packet *p;
52 	uint type;
53 };
54 
55 Biobuf	bscores;		/* biobuf filled with block scores */
56 int		debug;		/* debugging flag (not used) */
57 Disk*	disk;			/* disk being backed up */
58 RWLock	endlk;		/* silly synchonization */
59 int		errors;		/* are we exiting with an error status? */
60 int		fsscanblock;	/* last block scanned */
61 Fsys*	fsys;			/* file system being backed up */
62 int		nchange;		/* number of changed blocks */
63 int		nop;			/* don't actually send blocks to venti */
64 int		nwrite;		/* number of write-behind threads */
65 Queue*	qcmp;		/* queue fsys->cmp */
66 Queue*	qventi;		/* queue cmp->venti */
67 int		statustime;	/* print status every _ seconds */
68 int		verbose;		/* print extra stuff */
69 VtFile*	vfile;			/* venti file being written */
70 Channel*	writechan;	/* chan(WriteReq) */
71 VtConn*	z;			/* connection to venti */
72 VtCache*	zcache;		/* cache of venti blocks */
73 uchar*	zero;			/* blocksize zero bytes */
74 
75 extern	int	ncopy, nread, nwrite;	/* hidden in libventi */
76 
77 void		cmpproc(void*);
78 void		fsysproc(void*);
79 void		statusproc(void*);
80 void		ventiproc(void*);
81 int		timefmt(Fmt*);
82 char*	mountplace(char *dev);
83 
84 void
85 usage(void)
86 {
87 	fprint(2, "usage: vbackup [-DVnv] [-m mtpt] [-s secs] [-w n] disk [score]\n");
88 	threadexitsall("usage");
89 }
90 
91 void
92 threadmain(int argc, char **argv)
93 {
94 	char *pref, *mountname;
95 	uchar score[VtScoreSize], prev[VtScoreSize];
96 	int i, fd, csize;
97 	vlong bsize;
98 	Tm tm;
99 	VtEntry e;
100 	VtBlock *b;
101 	VtCache *c;
102 	VtRoot root;
103 	char *tmp, *tmpnam;
104 
105 	fmtinstall('F', vtfcallfmt);
106 	fmtinstall('H', encodefmt);
107 	fmtinstall('T', timefmt);
108 	fmtinstall('V', vtscorefmt);
109 
110 	mountname = sysname();
111 	ARGBEGIN{
112 	default:
113 		usage();
114 		break;
115 	case 'D':
116 		debug++;
117 		break;
118 	case 'V':
119 		chattyventi = 1;
120 		break;
121 	case 'm':
122 		mountname = EARGF(usage());
123 		break;
124 	case 'n':
125 		nop = 1;
126 		break;
127 	case 's':
128 		statustime = atoi(EARGF(usage()));
129 		break;
130 	case 'v':
131 		verbose = 1;
132 		break;
133 	case 'w':
134 		nwrite = atoi(EARGF(usage()));
135 		break;
136 	}ARGEND
137 
138 	if(argc != 1 && argc != 2)
139 		usage();
140 
141 	if(statustime)
142 		print("# %T vbackup %s %s\n", argv[0], argc>=2 ? argv[1] : "");
143 	/*
144 	 * open fs
145 	 */
146 	if((disk = diskopenfile(argv[0])) == nil)
147 		sysfatal("diskopen: %r");
148 	if((disk = diskcache(disk, 16384, 2*MAXQ+16)) == nil)
149 		sysfatal("diskcache: %r");
150 	if((fsys = fsysopen(disk)) == nil)
151 		sysfatal("ffsopen: %r");
152 
153 	/*
154 	 * connect to venti
155 	 */
156 	if((z = vtdial(nil)) == nil)
157 		sysfatal("vtdial: %r");
158 	if(vtconnect(z) < 0)
159 		sysfatal("vtconnect: %r");
160 
161 	/*
162 	 * set up venti block cache
163 	 */
164 	zero = vtmallocz(fsys->blocksize);
165 	bsize = fsys->blocksize;
166 	csize = 50;	/* plenty; could probably do with 5 */
167 
168 	if(verbose)
169 		fprint(2, "cache %d blocks\n", csize);
170 	c = vtcachealloc(z, bsize, csize);
171 	zcache = c;
172 
173 	/*
174 	 * parse starting score
175 	 */
176 	memset(prev, 0, sizeof prev);
177 	if(argc == 1){
178 		vfile = vtfilecreateroot(c, (fsys->blocksize/VtScoreSize)*VtScoreSize,
179 			fsys->blocksize, VtDataType);
180 		if(vfile == nil)
181 			sysfatal("vtfilecreateroot: %r");
182 		vtfilelock(vfile, VtORDWR);
183 		if(vtfilewrite(vfile, zero, 1, bsize*fsys->nblock-1) != 1)
184 			sysfatal("vtfilewrite: %r");
185 		if(vtfileflush(vfile) < 0)
186 			sysfatal("vtfileflush: %r");
187 	}else{
188 		if(vtparsescore(argv[1], &pref, score) < 0)
189 			sysfatal("bad score: %r");
190 		if(pref!=nil && strcmp(pref, fsys->type) != 0)
191 			sysfatal("score is %s but fsys is %s", pref, fsys->type);
192 		b = vtcacheglobal(c, score, VtRootType);
193 		if(b){
194 			if(vtrootunpack(&root, b->data) < 0)
195 				sysfatal("bad root: %r");
196 			if(strcmp(root.type, fsys->type) != 0)
197 				sysfatal("root is %s but fsys is %s", root.type, fsys->type);
198 			memmove(prev, score, VtScoreSize);
199 			memmove(score, root.score, VtScoreSize);
200 			vtblockput(b);
201 		}
202 		b = vtcacheglobal(c, score, VtDirType);
203 		if(b == nil)
204 			sysfatal("vtcacheglobal %V: %r", score);
205 		if(vtentryunpack(&e, b->data, 0) < 0)
206 			sysfatal("%V: vtentryunpack failed", score);
207 		if(verbose)
208 			fprint(2, "entry: size %llud psize %d dsize %d\n",
209 				e.size, e.psize, e.dsize);
210 		vtblockput(b);
211 		if((vfile = vtfileopenroot(c, &e)) == nil)
212 			sysfatal("vtfileopenroot: %r");
213 		vtfilelock(vfile, VtORDWR);
214 		if(e.dsize != bsize)
215 			sysfatal("file system block sizes don't match %d %lld", e.dsize, bsize);
216 		if(e.size != fsys->nblock*bsize)
217 			sysfatal("file system block counts don't match %lld %lld", e.size, fsys->nblock*bsize);
218 	}
219 
220 	/*
221 	 * write scores of blocks into temporary file
222 	 */
223 	if((tmp = getenv("TMP")) != nil){
224 		/* okay, good */
225 	}else if(access("/var/tmp", 0) >= 0)
226 		tmp = "/var/tmp";
227 	else
228 		tmp = "/tmp";
229 	tmpnam = smprint("%s/vbackup.XXXXXX", tmp);
230 	if(tmpnam == nil)
231 		sysfatal("smprint: %r");
232 
233 	if((fd = opentemp(tmpnam)) < 0)
234 		sysfatal("opentemp %s: %r", tmpnam);
235 	if(statustime)
236 		print("# %T reading scores into %s\n", tmpnam);
237 	if(verbose)
238 		fprint(2, "read scores into %s...\n", tmpnam);
239 
240 	Binit(&bscores, fd, OWRITE);
241 	for(i=0; i<fsys->nblock; i++){
242 		if(vtfileblockscore(vfile, i, score) < 0)
243 			sysfatal("vtfileblockhash %d: %r", i);
244 		if(Bwrite(&bscores, score, VtScoreSize) != VtScoreSize)
245 			sysfatal("Bwrite: %r");
246 	}
247 	Bterm(&bscores);
248 	vtfileunlock(vfile);
249 
250 	/*
251 	 * prep scores for rereading
252 	 */
253 	seek(fd, 0, 0);
254 	Binit(&bscores, fd, OREAD);
255 
256 	/*
257 	 * start the main processes 
258 	 */
259 	if(statustime)
260 		print("# %T starting procs\n");
261 	qcmp = qalloc();
262 	qventi = qalloc();
263 
264 	rlock(&endlk);
265 	proccreate(fsysproc, nil, STACK);
266 	rlock(&endlk);
267 	proccreate(ventiproc, nil, STACK);
268 	rlock(&endlk);
269 	proccreate(cmpproc, nil, STACK);
270 	if(statustime){
271 		rlock(&endlk);
272 		proccreate(statusproc, nil, STACK);
273 	}
274 
275 	/*
276 	 * wait for processes to finish
277 	 */
278 	wlock(&endlk);
279 
280 	if(statustime)
281 		print("# %T procs exited: %d blocks changed, %d read, %d written, %d copied\n",
282 			nchange, nread, nwrite, ncopy);
283 
284 	/*
285 	 * prepare root block
286 	 */
287 	vtfilelock(vfile, -1);
288 	if(vtfileflush(vfile) < 0)
289 		sysfatal("vtfileflush: %r");
290 	if(vtfilegetentry(vfile, &e) < 0)
291 		sysfatal("vtfilegetentry: %r");
292 
293 	b = vtcacheallocblock(c, VtDirType);
294 	if(b == nil)
295 		sysfatal("vtcacheallocblock: %r");
296 	vtentrypack(&e, b->data, 0);
297 	if(vtblockwrite(b) < 0)
298 		sysfatal("vtblockwrite: %r");
299 
300 	memset(&root, 0, sizeof root);
301 	strecpy(root.name, root.name+sizeof root.name, argv[0]);
302 	strecpy(root.type, root.type+sizeof root.type, fsys->type);
303 	memmove(root.score, b->score, VtScoreSize);
304 	root.blocksize = fsys->blocksize;
305 	memmove(root.prev, prev, VtScoreSize);
306 	vtblockput(b);
307 
308 	b = vtcacheallocblock(c, VtRootType);
309 	if(b == nil)
310 		sysfatal("vtcacheallocblock: %r");
311 	vtrootpack(&root, b->data);
312 	if(vtblockwrite(b) < 0)
313 		sysfatal("vtblockwrite: %r");
314 
315 	tm = *localtime(time(0));
316 	tm.year += 1900;
317 	tm.mon++;
318 	print("mount /%s/%d/%02d%02d%s %s:%V %d/%02d%02d/%02d%02d\n",
319 		mountname, tm.year, tm.mon, tm.mday, 
320 		mountplace(argv[0]),
321 		root.type, b->score,
322 		tm.year, tm.mon, tm.mday, tm.hour, tm.min);
323 	print("# %T %s %s:%V\n", argv[0], root.type, b->score);
324 	if(statustime)
325 		print("# %T venti sync\n");
326 	vtblockput(b);
327 	if(vtsync(z) < 0)
328 		sysfatal("vtsync: %r");
329 	if(statustime)
330 		print("# %T synced\n");
331 	threadexitsall(nil);
332 }
333 
334 void
335 fsysproc(void *dummy)
336 {
337 	u32int i;
338 	Block *db;
339 
340 	USED(dummy);
341 
342 	for(i=0; i<fsys->nblock; i++){
343 		fsscanblock = i;
344 		if((db = fsysreadblock(fsys, i)) != nil)
345 			qwrite(qcmp, db, i);
346 	}
347 	fsscanblock = i;
348 	qclose(qcmp);
349 
350 	print("# %T fsys proc exiting\n");
351 	runlock(&endlk);
352 }
353 
354 void
355 cmpproc(void *dummy)
356 {
357 	uchar *data;
358 	Block *db;
359 	u32int bno, bsize;
360 	uchar score[VtScoreSize];
361 	uchar score1[VtScoreSize];
362 
363 	USED(dummy);
364 
365 	bsize = fsys->blocksize;
366 	while((db = qread(qcmp, &bno)) != nil){
367 		data = db->data;
368 		sha1(data, vtzerotruncate(VtDataType, data, bsize), score, nil);
369 		if(Bseek(&bscores, (vlong)bno*VtScoreSize, 0) < 0)
370 			sysfatal("cmpproc Bseek: %r");
371 		if(Bread(&bscores, score1, VtScoreSize) != VtScoreSize)
372 			sysfatal("cmpproc Bread: %r");
373 		if(memcmp(score, score1, VtScoreSize) != 0){
374 			nchange++;
375 			if(verbose)
376 				print("# block %ud: old %V new %V\n", bno, score1, score);
377 			qwrite(qventi, db, bno);
378 		}else
379 			blockput(db);
380 	}
381 	qclose(qventi);
382 	runlock(&endlk);
383 }
384 
385 void
386 writethread(void *v)
387 {
388 	WriteReq wr;
389 	uchar score[VtScoreSize];
390 
391 	USED(v);
392 
393 	while(recv(writechan, &wr) == 1){
394 		if(wr.p == nil)
395 			break;
396 		if(vtwritepacket(z, score, wr.type, wr.p) < 0)
397 			sysfatal("vtwritepacket: %r");
398 	}
399 }
400 
401 int
402 myvtwrite(VtConn *z, uchar score[VtScoreSize], uint type, uchar *buf, int n)
403 {
404 	WriteReq wr;
405 
406 	if(nwrite == 0)
407 		return vtwrite(z, score, type, buf, n);
408 
409 	wr.p = packetalloc();
410 	packetappend(wr.p, buf, n);
411 	packetsha1(wr.p, score);
412 	wr.type = type;
413 	send(writechan, &wr);
414 	return 0;
415 }
416 
417 void
418 ventiproc(void *dummy)
419 {
420 	int i;
421 	Block *db;
422 	u32int bno;
423 	u64int bsize;
424 
425 	USED(dummy);
426 
427 	proccreate(vtsendproc, z, STACK);
428 	proccreate(vtrecvproc, z, STACK);
429 
430 	writechan = chancreate(sizeof(WriteReq), 0);
431 	for(i=0; i<nwrite; i++)
432 		threadcreate(writethread, nil, STACK);
433 	vtcachesetwrite(zcache, myvtwrite);
434 
435 	bsize = fsys->blocksize;
436 	vtfilelock(vfile, -1);
437 	while((db = qread(qventi, &bno)) != nil){
438 		if(nop){
439 			blockput(db);
440 			continue;
441 		}
442 		if(vtfilewrite(vfile, db->data, bsize, bno*bsize) != bsize)
443 			sysfatal("ventiproc vtfilewrite: %r");
444 		if(vtfileflushbefore(vfile, (bno+1)*bsize) < 0)
445 			sysfatal("ventiproc vtfileflushbefore: %r");
446 		blockput(db);
447 	}
448 	vtfileunlock(vfile);
449 	vtcachesetwrite(zcache, nil);
450 	for(i=0; i<nwrite; i++)
451 		send(writechan, nil);
452 	runlock(&endlk);
453 }
454 
455 static int
456 percent(u32int a, u32int b)
457 {
458 	return (vlong)a*100/b;
459 }
460 
461 void
462 statusproc(void *dummy)
463 {
464 	int n;
465 	USED(dummy);
466 
467 	for(n=0;;n++){
468 		sleep(1000);
469 		if(qcmp->closed && qcmp->nel==0 && qventi->closed && qventi->nel==0)
470 			break;
471 		if(n < statustime)
472 			continue;
473 		n = 0;
474 		print("# %T fsscan=%d%% cmpq=%d%% ventiq=%d%%\n",
475 			percent(fsscanblock, fsys->nblock),
476 			percent(qcmp->nel, MAXQ),
477 			percent(qventi->nel, MAXQ));
478 	}
479 	runlock(&endlk);
480 }
481 
482 int
483 timefmt(Fmt *fmt)
484 {
485 	vlong ns;
486 	Tm tm;
487 	ns = nsec();
488 	tm = *localtime(time(0));
489 	return fmtprint(fmt, "%04d/%02d%02d %02d:%02d:%02d.%03d", 
490 		tm.year+1900, tm.mon+1, tm.mday, tm.hour, tm.min, tm.sec,
491 		(int)(ns%1000000000)/1000000);
492 }
493 
494 char*
495 mountplace(char *dev)
496 {
497 	char *cmd, *q;
498 	int p[2], fd[3], n;
499 	char buf[100];
500 	
501 	if(pipe(p) < 0)
502 		sysfatal("pipe: %r");
503 
504 	fd[0] = -1;
505 	fd[1] = p[1];
506 	fd[2] = -1;
507 	cmd = smprint("mount | awk '$1==\"%s\" && $2 == \"on\" {print $3}'", dev);
508 	if(threadspawnl(fd, "sh", "sh", "-c", cmd, nil) < 0)
509 		sysfatal("exec mount|awk (to find mtpt of %s): %r", dev);
510 	/* threadspawnl closed p[1] */
511 	n = readn(p[0], buf, sizeof buf-1);
512 	close(p[0]);
513 	if(n <= 0)
514 		return dev;
515 	buf[n] = 0;
516 	if((q = strchr(buf, '\n')) == nil)
517 		return dev;
518 	*q = 0;
519 	q = buf+strlen(buf);
520 	if(q>buf && *(q-1) == '/')
521 		*--q = 0;
522 	return strdup(buf);
523 }
524