1 004aa293 2005-07-13 devnull /*
  2 004aa293 2005-07-13 devnull  * vbackup [-Dnv] fspartition [score]
  3 004aa293 2005-07-13 devnull  *
  4 004aa293 2005-07-13 devnull  * Copy a file system to a disk image stored on Venti.
  5 004aa293 2005-07-13 devnull  * Prints a vnfs config line for the copied image.
  6 004aa293 2005-07-13 devnull  *
  7 004aa293 2005-07-13 devnull  *	-D	print debugging
  8 3bc4abf4 2005-11-07 devnull  *	-f	'fast' writes - skip write if block exists on server
  9 a4b5da0b 2007-11-27 rsc  *	-i	read old scores incrementally
 10 004aa293 2005-07-13 devnull  *	-m	set mount name
 11 9437f55b 2007-06-12 devnull  *	-M	set mount place
 12 004aa293 2005-07-13 devnull  *	-n	nop -- don't actually write blocks
 13 004aa293 2005-07-13 devnull  *	-s	print status updates
 14 004aa293 2005-07-13 devnull  *	-v	print debugging trace
 15 004aa293 2005-07-13 devnull  *	-w	write parallelism
 16 fa325e9b 2020-01-10 cross  *
 17 004aa293 2005-07-13 devnull  * If score is given on the command line, it should be the
 18 004aa293 2005-07-13 devnull  * score from a previous vbackup on this fspartition.
 19 004aa293 2005-07-13 devnull  * In this mode, only the new blocks are stored to Venti.
 20 004aa293 2005-07-13 devnull  * The result is still a complete image, but requires many
 21 004aa293 2005-07-13 devnull  * fewer Venti writes in the common case.
 22 004aa293 2005-07-13 devnull  *
 23 004aa293 2005-07-13 devnull  * This program is structured as three processes connected
 24 004aa293 2005-07-13 devnull  * by buffered queues:
 25 004aa293 2005-07-13 devnull  *
 26 004aa293 2005-07-13 devnull  * 	fsysproc | cmpproc | ventiproc
 27 fa325e9b 2020-01-10 cross  *
 28 004aa293 2005-07-13 devnull  * Fsysproc reads the disk and queues the blocks.
 29 004aa293 2005-07-13 devnull  * Cmpproc compares the blocks against the SHA1 hashes
 30 004aa293 2005-07-13 devnull  * in the old image, if any.  It discards the unchanged blocks
 31 004aa293 2005-07-13 devnull  * and queues the changed ones.  Ventiproc writes blocks to Venti.
 32 fa325e9b 2020-01-10 cross  *
 33 004aa293 2005-07-13 devnull  * There is a fourth proc, statusproc, which prints status
 34 004aa293 2005-07-13 devnull  * updates about how the various procs are progressing.
 35 004aa293 2005-07-13 devnull  */
 36 004aa293 2005-07-13 devnull 
 37 004aa293 2005-07-13 devnull #include <u.h>
 38 004aa293 2005-07-13 devnull #include <libc.h>
 39 004aa293 2005-07-13 devnull #include <bio.h>
 40 004aa293 2005-07-13 devnull #include <thread.h>
 41 004aa293 2005-07-13 devnull #include <libsec.h>
 42 004aa293 2005-07-13 devnull #include <venti.h>
 43 004aa293 2005-07-13 devnull #include <diskfs.h>
 44 004aa293 2005-07-13 devnull #include "queue.h"
 45 004aa293 2005-07-13 devnull 
 46 004aa293 2005-07-13 devnull enum
 47 004aa293 2005-07-13 devnull {
 48 cbeb0b26 2006-04-01 devnull 	STACK = 32768
 49 004aa293 2005-07-13 devnull };
 50 004aa293 2005-07-13 devnull 
 51 004aa293 2005-07-13 devnull typedef struct WriteReq WriteReq;
 52 004aa293 2005-07-13 devnull struct WriteReq
 53 004aa293 2005-07-13 devnull {
 54 004aa293 2005-07-13 devnull 	Packet *p;
 55 004aa293 2005-07-13 devnull 	uint type;
 56 3bc4abf4 2005-11-07 devnull 	uchar score[VtScoreSize];
 57 004aa293 2005-07-13 devnull };
 58 004aa293 2005-07-13 devnull 
 59 004aa293 2005-07-13 devnull Biobuf	bscores;		/* biobuf filled with block scores */
 60 004aa293 2005-07-13 devnull int		debug;		/* debugging flag (not used) */
 61 004aa293 2005-07-13 devnull Disk*	disk;			/* disk being backed up */
 62 004aa293 2005-07-13 devnull RWLock	endlk;		/* silly synchonization */
 63 004aa293 2005-07-13 devnull int		errors;		/* are we exiting with an error status? */
 64 3bc4abf4 2005-11-07 devnull int		fastwrites;		/* do not write blocks already on server */
 65 004aa293 2005-07-13 devnull int		fsscanblock;	/* last block scanned */
 66 004aa293 2005-07-13 devnull Fsys*	fsys;			/* file system being backed up */
 67 a4b5da0b 2007-11-27 rsc int		incremental;	/* use vscores rather than bscores */
 68 004aa293 2005-07-13 devnull int		nchange;		/* number of changed blocks */
 69 004aa293 2005-07-13 devnull int		nop;			/* don't actually send blocks to venti */
 70 3bc4abf4 2005-11-07 devnull int		nskip;		/* number of blocks skipped (already on server) */
 71 3bc4abf4 2005-11-07 devnull int		nwritethread;	/* number of write-behind threads */
 72 004aa293 2005-07-13 devnull Queue*	qcmp;		/* queue fsys->cmp */
 73 004aa293 2005-07-13 devnull Queue*	qventi;		/* queue cmp->venti */
 74 004aa293 2005-07-13 devnull int		statustime;	/* print status every _ seconds */
 75 004aa293 2005-07-13 devnull int		verbose;		/* print extra stuff */
 76 004aa293 2005-07-13 devnull VtFile*	vfile;			/* venti file being written */
 77 a4b5da0b 2007-11-27 rsc VtFile*	vscores;		/* venti file with block scores */
 78 004aa293 2005-07-13 devnull Channel*	writechan;	/* chan(WriteReq) */
 79 004aa293 2005-07-13 devnull VtConn*	z;			/* connection to venti */
 80 004aa293 2005-07-13 devnull VtCache*	zcache;		/* cache of venti blocks */
 81 004aa293 2005-07-13 devnull uchar*	zero;			/* blocksize zero bytes */
 82 004aa293 2005-07-13 devnull 
 83 3bc4abf4 2005-11-07 devnull int nsend, nrecv;
 84 004aa293 2005-07-13 devnull 
 85 004aa293 2005-07-13 devnull void		cmpproc(void*);
 86 004aa293 2005-07-13 devnull void		fsysproc(void*);
 87 004aa293 2005-07-13 devnull void		statusproc(void*);
 88 004aa293 2005-07-13 devnull void		ventiproc(void*);
 89 004aa293 2005-07-13 devnull int		timefmt(Fmt*);
 90 9437f55b 2007-06-12 devnull char*	guessmountplace(char *dev);
 91 004aa293 2005-07-13 devnull 
 92 004aa293 2005-07-13 devnull void
 93 004aa293 2005-07-13 devnull usage(void)
 94 004aa293 2005-07-13 devnull {
 95 9437f55b 2007-06-12 devnull 	fprint(2, "usage: vbackup [-DVnv] [-m mtpt] [-M mtpl] [-s secs] [-w n] disk [score]\n");
 96 004aa293 2005-07-13 devnull 	threadexitsall("usage");
 97 004aa293 2005-07-13 devnull }
 98 004aa293 2005-07-13 devnull 
 99 004aa293 2005-07-13 devnull void
100 004aa293 2005-07-13 devnull threadmain(int argc, char **argv)
101 004aa293 2005-07-13 devnull {
102 9437f55b 2007-06-12 devnull 	char *pref, *mountname, *mountplace;
103 004aa293 2005-07-13 devnull 	uchar score[VtScoreSize], prev[VtScoreSize];
104 004aa293 2005-07-13 devnull 	int i, fd, csize;
105 004aa293 2005-07-13 devnull 	vlong bsize;
106 004aa293 2005-07-13 devnull 	Tm tm;
107 004aa293 2005-07-13 devnull 	VtEntry e;
108 004aa293 2005-07-13 devnull 	VtBlock *b;
109 004aa293 2005-07-13 devnull 	VtCache *c;
110 004aa293 2005-07-13 devnull 	VtRoot root;
111 004aa293 2005-07-13 devnull 	char *tmp, *tmpnam;
112 004aa293 2005-07-13 devnull 
113 004aa293 2005-07-13 devnull 	fmtinstall('F', vtfcallfmt);
114 004aa293 2005-07-13 devnull 	fmtinstall('H', encodefmt);
115 004aa293 2005-07-13 devnull 	fmtinstall('T', timefmt);
116 004aa293 2005-07-13 devnull 	fmtinstall('V', vtscorefmt);
117 004aa293 2005-07-13 devnull 
118 004aa293 2005-07-13 devnull 	mountname = sysname();
119 9437f55b 2007-06-12 devnull 	mountplace = nil;
120 004aa293 2005-07-13 devnull 	ARGBEGIN{
121 004aa293 2005-07-13 devnull 	default:
122 004aa293 2005-07-13 devnull 		usage();
123 004aa293 2005-07-13 devnull 		break;
124 004aa293 2005-07-13 devnull 	case 'D':
125 004aa293 2005-07-13 devnull 		debug++;
126 004aa293 2005-07-13 devnull 		break;
127 004aa293 2005-07-13 devnull 	case 'V':
128 004aa293 2005-07-13 devnull 		chattyventi = 1;
129 004aa293 2005-07-13 devnull 		break;
130 3bc4abf4 2005-11-07 devnull 	case 'f':
131 3bc4abf4 2005-11-07 devnull 		fastwrites = 1;
132 3bc4abf4 2005-11-07 devnull 		break;
133 a4b5da0b 2007-11-27 rsc 	case 'i':
134 a4b5da0b 2007-11-27 rsc 		incremental = 1;
135 a4b5da0b 2007-11-27 rsc 		break;
136 004aa293 2005-07-13 devnull 	case 'm':
137 004aa293 2005-07-13 devnull 		mountname = EARGF(usage());
138 9437f55b 2007-06-12 devnull 		break;
139 9437f55b 2007-06-12 devnull 	case 'M':
140 9437f55b 2007-06-12 devnull 		mountplace = EARGF(usage());
141 9437f55b 2007-06-12 devnull 		i = strlen(mountplace);
142 9437f55b 2007-06-12 devnull 		if(i > 0 && mountplace[i-1] == '/')
143 9437f55b 2007-06-12 devnull 			mountplace[i-1] = 0;
144 004aa293 2005-07-13 devnull 		break;
145 004aa293 2005-07-13 devnull 	case 'n':
146 004aa293 2005-07-13 devnull 		nop = 1;
147 004aa293 2005-07-13 devnull 		break;
148 004aa293 2005-07-13 devnull 	case 's':
149 004aa293 2005-07-13 devnull 		statustime = atoi(EARGF(usage()));
150 004aa293 2005-07-13 devnull 		break;
151 004aa293 2005-07-13 devnull 	case 'v':
152 004aa293 2005-07-13 devnull 		verbose = 1;
153 004aa293 2005-07-13 devnull 		break;
154 004aa293 2005-07-13 devnull 	case 'w':
155 3bc4abf4 2005-11-07 devnull 		nwritethread = atoi(EARGF(usage()));
156 004aa293 2005-07-13 devnull 		break;
157 004aa293 2005-07-13 devnull 	}ARGEND
158 004aa293 2005-07-13 devnull 
159 004aa293 2005-07-13 devnull 	if(argc != 1 && argc != 2)
160 004aa293 2005-07-13 devnull 		usage();
161 004aa293 2005-07-13 devnull 
162 004aa293 2005-07-13 devnull 	if(statustime)
163 004aa293 2005-07-13 devnull 		print("# %T vbackup %s %s\n", argv[0], argc>=2 ? argv[1] : "");
164 3bc4abf4 2005-11-07 devnull 
165 004aa293 2005-07-13 devnull 	/*
166 004aa293 2005-07-13 devnull 	 * open fs
167 004aa293 2005-07-13 devnull 	 */
168 004aa293 2005-07-13 devnull 	if((disk = diskopenfile(argv[0])) == nil)
169 004aa293 2005-07-13 devnull 		sysfatal("diskopen: %r");
170 04692ecc 2006-01-07 devnull 	if((disk = diskcache(disk, 32768, 2*MAXQ+16)) == nil)
171 004aa293 2005-07-13 devnull 		sysfatal("diskcache: %r");
172 004aa293 2005-07-13 devnull 	if((fsys = fsysopen(disk)) == nil)
173 83ab7d88 2007-11-27 rsc 		sysfatal("fsysopen: %r");
174 004aa293 2005-07-13 devnull 
175 004aa293 2005-07-13 devnull 	/*
176 004aa293 2005-07-13 devnull 	 * connect to venti
177 004aa293 2005-07-13 devnull 	 */
178 004aa293 2005-07-13 devnull 	if((z = vtdial(nil)) == nil)
179 004aa293 2005-07-13 devnull 		sysfatal("vtdial: %r");
180 004aa293 2005-07-13 devnull 	if(vtconnect(z) < 0)
181 004aa293 2005-07-13 devnull 		sysfatal("vtconnect: %r");
182 004aa293 2005-07-13 devnull 
183 004aa293 2005-07-13 devnull 	/*
184 004aa293 2005-07-13 devnull 	 * set up venti block cache
185 004aa293 2005-07-13 devnull 	 */
186 004aa293 2005-07-13 devnull 	zero = vtmallocz(fsys->blocksize);
187 004aa293 2005-07-13 devnull 	bsize = fsys->blocksize;
188 004aa293 2005-07-13 devnull 	csize = 50;	/* plenty; could probably do with 5 */
189 004aa293 2005-07-13 devnull 
190 004aa293 2005-07-13 devnull 	if(verbose)
191 004aa293 2005-07-13 devnull 		fprint(2, "cache %d blocks\n", csize);
192 b8f45719 2009-06-10 rsc 	c = vtcachealloc(z, bsize*csize);
193 004aa293 2005-07-13 devnull 	zcache = c;
194 004aa293 2005-07-13 devnull 
195 004aa293 2005-07-13 devnull 	/*
196 004aa293 2005-07-13 devnull 	 * parse starting score
197 004aa293 2005-07-13 devnull 	 */
198 004aa293 2005-07-13 devnull 	memset(prev, 0, sizeof prev);
199 004aa293 2005-07-13 devnull 	if(argc == 1){
200 004aa293 2005-07-13 devnull 		vfile = vtfilecreateroot(c, (fsys->blocksize/VtScoreSize)*VtScoreSize,
201 004aa293 2005-07-13 devnull 			fsys->blocksize, VtDataType);
202 004aa293 2005-07-13 devnull 		if(vfile == nil)
203 004aa293 2005-07-13 devnull 			sysfatal("vtfilecreateroot: %r");
204 004aa293 2005-07-13 devnull 		vtfilelock(vfile, VtORDWR);
205 004aa293 2005-07-13 devnull 		if(vtfilewrite(vfile, zero, 1, bsize*fsys->nblock-1) != 1)
206 004aa293 2005-07-13 devnull 			sysfatal("vtfilewrite: %r");
207 004aa293 2005-07-13 devnull 		if(vtfileflush(vfile) < 0)
208 004aa293 2005-07-13 devnull 			sysfatal("vtfileflush: %r");
209 004aa293 2005-07-13 devnull 	}else{
210 004aa293 2005-07-13 devnull 		if(vtparsescore(argv[1], &pref, score) < 0)
211 004aa293 2005-07-13 devnull 			sysfatal("bad score: %r");
212 004aa293 2005-07-13 devnull 		if(pref!=nil && strcmp(pref, fsys->type) != 0)
213 004aa293 2005-07-13 devnull 			sysfatal("score is %s but fsys is %s", pref, fsys->type);
214 b8f45719 2009-06-10 rsc 		b = vtcacheglobal(c, score, VtRootType, VtRootSize);
215 004aa293 2005-07-13 devnull 		if(b){
216 004aa293 2005-07-13 devnull 			if(vtrootunpack(&root, b->data) < 0)
217 004aa293 2005-07-13 devnull 				sysfatal("bad root: %r");
218 004aa293 2005-07-13 devnull 			if(strcmp(root.type, fsys->type) != 0)
219 004aa293 2005-07-13 devnull 				sysfatal("root is %s but fsys is %s", root.type, fsys->type);
220 004aa293 2005-07-13 devnull 			memmove(prev, score, VtScoreSize);
221 004aa293 2005-07-13 devnull 			memmove(score, root.score, VtScoreSize);
222 004aa293 2005-07-13 devnull 			vtblockput(b);
223 004aa293 2005-07-13 devnull 		}
224 b8f45719 2009-06-10 rsc 		b = vtcacheglobal(c, score, VtDirType, VtEntrySize);
225 004aa293 2005-07-13 devnull 		if(b == nil)
226 004aa293 2005-07-13 devnull 			sysfatal("vtcacheglobal %V: %r", score);
227 004aa293 2005-07-13 devnull 		if(vtentryunpack(&e, b->data, 0) < 0)
228 004aa293 2005-07-13 devnull 			sysfatal("%V: vtentryunpack failed", score);
229 004aa293 2005-07-13 devnull 		if(verbose)
230 004aa293 2005-07-13 devnull 			fprint(2, "entry: size %llud psize %d dsize %d\n",
231 004aa293 2005-07-13 devnull 				e.size, e.psize, e.dsize);
232 004aa293 2005-07-13 devnull 		vtblockput(b);
233 004aa293 2005-07-13 devnull 		if((vfile = vtfileopenroot(c, &e)) == nil)
234 004aa293 2005-07-13 devnull 			sysfatal("vtfileopenroot: %r");
235 004aa293 2005-07-13 devnull 		vtfilelock(vfile, VtORDWR);
236 004aa293 2005-07-13 devnull 		if(e.dsize != bsize)
237 004aa293 2005-07-13 devnull 			sysfatal("file system block sizes don't match %d %lld", e.dsize, bsize);
238 004aa293 2005-07-13 devnull 		if(e.size != fsys->nblock*bsize)
239 004aa293 2005-07-13 devnull 			sysfatal("file system block counts don't match %lld %lld", e.size, fsys->nblock*bsize);
240 004aa293 2005-07-13 devnull 	}
241 004aa293 2005-07-13 devnull 
242 b8861903 2007-12-04 rsc 	tmpnam = nil;
243 a4b5da0b 2007-11-27 rsc 	if(incremental){
244 a4b5da0b 2007-11-27 rsc 		if(vtfilegetentry(vfile, &e) < 0)
245 a4b5da0b 2007-11-27 rsc 			sysfatal("vtfilegetentry: %r");
246 a4b5da0b 2007-11-27 rsc 		if((vscores = vtfileopenroot(c, &e)) == nil)
247 a4b5da0b 2007-11-27 rsc 			sysfatal("vtfileopenroot: %r");
248 a4b5da0b 2007-11-27 rsc 		vtfileunlock(vfile);
249 a4b5da0b 2007-11-27 rsc 	}else{
250 a4b5da0b 2007-11-27 rsc 		/*
251 a4b5da0b 2007-11-27 rsc 		 * write scores of blocks into temporary file
252 a4b5da0b 2007-11-27 rsc 		 */
253 a4b5da0b 2007-11-27 rsc 		if((tmp = getenv("TMP")) != nil){
254 a4b5da0b 2007-11-27 rsc 			/* okay, good */
255 a4b5da0b 2007-11-27 rsc 		}else if(access("/var/tmp", 0) >= 0)
256 a4b5da0b 2007-11-27 rsc 			tmp = "/var/tmp";
257 a4b5da0b 2007-11-27 rsc 		else
258 a4b5da0b 2007-11-27 rsc 			tmp = "/tmp";
259 a4b5da0b 2007-11-27 rsc 		tmpnam = smprint("%s/vbackup.XXXXXX", tmp);
260 a4b5da0b 2007-11-27 rsc 		if(tmpnam == nil)
261 a4b5da0b 2007-11-27 rsc 			sysfatal("smprint: %r");
262 fa325e9b 2020-01-10 cross 
263 a58a827f 2008-07-10 rsc 		if((fd = opentemp(tmpnam, ORDWR|ORCLOSE)) < 0)
264 a4b5da0b 2007-11-27 rsc 			sysfatal("opentemp %s: %r", tmpnam);
265 a4b5da0b 2007-11-27 rsc 		if(statustime)
266 a4b5da0b 2007-11-27 rsc 			print("# %T reading scores into %s\n", tmpnam);
267 a4b5da0b 2007-11-27 rsc 		if(verbose)
268 a4b5da0b 2007-11-27 rsc 			fprint(2, "read scores into %s...\n", tmpnam);
269 fa325e9b 2020-01-10 cross 
270 a4b5da0b 2007-11-27 rsc 		Binit(&bscores, fd, OWRITE);
271 a4b5da0b 2007-11-27 rsc 		for(i=0; i<fsys->nblock; i++){
272 a4b5da0b 2007-11-27 rsc 			if(vtfileblockscore(vfile, i, score) < 0)
273 a4b5da0b 2007-11-27 rsc 				sysfatal("vtfileblockhash %d: %r", i);
274 a4b5da0b 2007-11-27 rsc 			if(Bwrite(&bscores, score, VtScoreSize) != VtScoreSize)
275 a4b5da0b 2007-11-27 rsc 				sysfatal("Bwrite: %r");
276 a4b5da0b 2007-11-27 rsc 		}
277 a4b5da0b 2007-11-27 rsc 		Bterm(&bscores);
278 a4b5da0b 2007-11-27 rsc 		vtfileunlock(vfile);
279 fa325e9b 2020-01-10 cross 
280 a4b5da0b 2007-11-27 rsc 		/*
281 a4b5da0b 2007-11-27 rsc 		 * prep scores for rereading
282 a4b5da0b 2007-11-27 rsc 		 */
283 a4b5da0b 2007-11-27 rsc 		seek(fd, 0, 0);
284 a4b5da0b 2007-11-27 rsc 		Binit(&bscores, fd, OREAD);
285 004aa293 2005-07-13 devnull 	}
286 004aa293 2005-07-13 devnull 
287 004aa293 2005-07-13 devnull 	/*
288 fa325e9b 2020-01-10 cross 	 * start the main processes
289 004aa293 2005-07-13 devnull 	 */
290 004aa293 2005-07-13 devnull 	if(statustime)
291 004aa293 2005-07-13 devnull 		print("# %T starting procs\n");
292 004aa293 2005-07-13 devnull 	qcmp = qalloc();
293 004aa293 2005-07-13 devnull 	qventi = qalloc();
294 004aa293 2005-07-13 devnull 
295 004aa293 2005-07-13 devnull 	rlock(&endlk);
296 004aa293 2005-07-13 devnull 	proccreate(fsysproc, nil, STACK);
297 004aa293 2005-07-13 devnull 	rlock(&endlk);
298 004aa293 2005-07-13 devnull 	proccreate(ventiproc, nil, STACK);
299 004aa293 2005-07-13 devnull 	rlock(&endlk);
300 004aa293 2005-07-13 devnull 	proccreate(cmpproc, nil, STACK);
301 004aa293 2005-07-13 devnull 	if(statustime){
302 004aa293 2005-07-13 devnull 		rlock(&endlk);
303 004aa293 2005-07-13 devnull 		proccreate(statusproc, nil, STACK);
304 004aa293 2005-07-13 devnull 	}
305 004aa293 2005-07-13 devnull 
306 004aa293 2005-07-13 devnull 	/*
307 004aa293 2005-07-13 devnull 	 * wait for processes to finish
308 004aa293 2005-07-13 devnull 	 */
309 004aa293 2005-07-13 devnull 	wlock(&endlk);
310 fa325e9b 2020-01-10 cross 
311 6fc7da3c 2006-10-19 devnull 	qfree(qcmp);
312 6fc7da3c 2006-10-19 devnull 	qfree(qventi);
313 004aa293 2005-07-13 devnull 
314 004aa293 2005-07-13 devnull 	if(statustime)
315 9437f55b 2007-06-12 devnull 		print("# %T procs exited: %d of %d %d-byte blocks changed, "
316 9437f55b 2007-06-12 devnull 			"%d read, %d written, %d skipped, %d copied\n",
317 9437f55b 2007-06-12 devnull 			nchange, fsys->nblock, fsys->blocksize,
318 9437f55b 2007-06-12 devnull 			vtcachenread, vtcachenwrite, nskip, vtcachencopy);
319 004aa293 2005-07-13 devnull 
320 004aa293 2005-07-13 devnull 	/*
321 004aa293 2005-07-13 devnull 	 * prepare root block
322 004aa293 2005-07-13 devnull 	 */
323 a4b5da0b 2007-11-27 rsc 	if(incremental)
324 a4b5da0b 2007-11-27 rsc 		vtfileclose(vscores);
325 004aa293 2005-07-13 devnull 	vtfilelock(vfile, -1);
326 004aa293 2005-07-13 devnull 	if(vtfileflush(vfile) < 0)
327 004aa293 2005-07-13 devnull 		sysfatal("vtfileflush: %r");
328 004aa293 2005-07-13 devnull 	if(vtfilegetentry(vfile, &e) < 0)
329 004aa293 2005-07-13 devnull 		sysfatal("vtfilegetentry: %r");
330 6fc7da3c 2006-10-19 devnull 	vtfileunlock(vfile);
331 6fc7da3c 2006-10-19 devnull 	vtfileclose(vfile);
332 004aa293 2005-07-13 devnull 
333 b8f45719 2009-06-10 rsc 	b = vtcacheallocblock(c, VtDirType, VtEntrySize);
334 004aa293 2005-07-13 devnull 	if(b == nil)
335 004aa293 2005-07-13 devnull 		sysfatal("vtcacheallocblock: %r");
336 004aa293 2005-07-13 devnull 	vtentrypack(&e, b->data, 0);
337 004aa293 2005-07-13 devnull 	if(vtblockwrite(b) < 0)
338 004aa293 2005-07-13 devnull 		sysfatal("vtblockwrite: %r");
339 004aa293 2005-07-13 devnull 
340 004aa293 2005-07-13 devnull 	memset(&root, 0, sizeof root);
341 004aa293 2005-07-13 devnull 	strecpy(root.name, root.name+sizeof root.name, argv[0]);
342 004aa293 2005-07-13 devnull 	strecpy(root.type, root.type+sizeof root.type, fsys->type);
343 004aa293 2005-07-13 devnull 	memmove(root.score, b->score, VtScoreSize);
344 004aa293 2005-07-13 devnull 	root.blocksize = fsys->blocksize;
345 004aa293 2005-07-13 devnull 	memmove(root.prev, prev, VtScoreSize);
346 004aa293 2005-07-13 devnull 	vtblockput(b);
347 004aa293 2005-07-13 devnull 
348 b8f45719 2009-06-10 rsc 	b = vtcacheallocblock(c, VtRootType, VtRootSize);
349 004aa293 2005-07-13 devnull 	if(b == nil)
350 004aa293 2005-07-13 devnull 		sysfatal("vtcacheallocblock: %r");
351 004aa293 2005-07-13 devnull 	vtrootpack(&root, b->data);
352 004aa293 2005-07-13 devnull 	if(vtblockwrite(b) < 0)
353 004aa293 2005-07-13 devnull 		sysfatal("vtblockwrite: %r");
354 004aa293 2005-07-13 devnull 
355 004aa293 2005-07-13 devnull 	tm = *localtime(time(0));
356 004aa293 2005-07-13 devnull 	tm.year += 1900;
357 004aa293 2005-07-13 devnull 	tm.mon++;
358 9437f55b 2007-06-12 devnull 	if(mountplace == nil)
359 9437f55b 2007-06-12 devnull 		mountplace = guessmountplace(argv[0]);
360 004aa293 2005-07-13 devnull 	print("mount /%s/%d/%02d%02d%s %s:%V %d/%02d%02d/%02d%02d\n",
361 fa325e9b 2020-01-10 cross 		mountname, tm.year, tm.mon, tm.mday,
362 9437f55b 2007-06-12 devnull 		mountplace,
363 004aa293 2005-07-13 devnull 		root.type, b->score,
364 004aa293 2005-07-13 devnull 		tm.year, tm.mon, tm.mday, tm.hour, tm.min);
365 004aa293 2005-07-13 devnull 	print("# %T %s %s:%V\n", argv[0], root.type, b->score);
366 004aa293 2005-07-13 devnull 	if(statustime)
367 004aa293 2005-07-13 devnull 		print("# %T venti sync\n");
368 004aa293 2005-07-13 devnull 	vtblockput(b);
369 004aa293 2005-07-13 devnull 	if(vtsync(z) < 0)
370 004aa293 2005-07-13 devnull 		sysfatal("vtsync: %r");
371 004aa293 2005-07-13 devnull 	if(statustime)
372 004aa293 2005-07-13 devnull 		print("# %T synced\n");
373 fa325e9b 2020-01-10 cross 
374 6fc7da3c 2006-10-19 devnull 	fsysclose(fsys);
375 6fc7da3c 2006-10-19 devnull 	diskclose(disk);
376 6fc7da3c 2006-10-19 devnull 	vtcachefree(zcache);
377 e8471ef5 2007-01-25 devnull 
378 e8471ef5 2007-01-25 devnull 	// Vtgoodbye hangs on Linux - not sure why.
379 e8471ef5 2007-01-25 devnull 	// Probably vtfcallrpc doesn't quite get the
380 e8471ef5 2007-01-25 devnull 	// remote hangup right.  Also probably related
381 e8471ef5 2007-01-25 devnull 	// to the vtrecvproc problem below.
382 e8471ef5 2007-01-25 devnull 	// vtgoodbye(z);
383 e8471ef5 2007-01-25 devnull 
384 6fc7da3c 2006-10-19 devnull 	// Leak here, because I can't seem to make
385 6fc7da3c 2006-10-19 devnull 	// the vtrecvproc exit.
386 6fc7da3c 2006-10-19 devnull 	// vtfreeconn(z);
387 e8471ef5 2007-01-25 devnull 
388 6fc7da3c 2006-10-19 devnull 	free(tmpnam);
389 6fc7da3c 2006-10-19 devnull 	z = nil;
390 6fc7da3c 2006-10-19 devnull 	zcache = nil;
391 6fc7da3c 2006-10-19 devnull 	fsys = nil;
392 6fc7da3c 2006-10-19 devnull 	disk = nil;
393 004aa293 2005-07-13 devnull 	threadexitsall(nil);
394 004aa293 2005-07-13 devnull }
395 004aa293 2005-07-13 devnull 
396 004aa293 2005-07-13 devnull void
397 004aa293 2005-07-13 devnull fsysproc(void *dummy)
398 004aa293 2005-07-13 devnull {
399 004aa293 2005-07-13 devnull 	u32int i;
400 004aa293 2005-07-13 devnull 	Block *db;
401 004aa293 2005-07-13 devnull 
402 004aa293 2005-07-13 devnull 	USED(dummy);
403 004aa293 2005-07-13 devnull 
404 004aa293 2005-07-13 devnull 	for(i=0; i<fsys->nblock; i++){
405 004aa293 2005-07-13 devnull 		fsscanblock = i;
406 004aa293 2005-07-13 devnull 		if((db = fsysreadblock(fsys, i)) != nil)
407 004aa293 2005-07-13 devnull 			qwrite(qcmp, db, i);
408 004aa293 2005-07-13 devnull 	}
409 004aa293 2005-07-13 devnull 	fsscanblock = i;
410 004aa293 2005-07-13 devnull 	qclose(qcmp);
411 004aa293 2005-07-13 devnull 
412 3bc4abf4 2005-11-07 devnull 	if(statustime)
413 3bc4abf4 2005-11-07 devnull 		print("# %T fsys proc exiting\n");
414 004aa293 2005-07-13 devnull 	runlock(&endlk);
415 004aa293 2005-07-13 devnull }
416 004aa293 2005-07-13 devnull 
417 004aa293 2005-07-13 devnull void
418 004aa293 2005-07-13 devnull cmpproc(void *dummy)
419 004aa293 2005-07-13 devnull {
420 004aa293 2005-07-13 devnull 	uchar *data;
421 004aa293 2005-07-13 devnull 	Block *db;
422 004aa293 2005-07-13 devnull 	u32int bno, bsize;
423 004aa293 2005-07-13 devnull 	uchar score[VtScoreSize];
424 004aa293 2005-07-13 devnull 	uchar score1[VtScoreSize];
425 004aa293 2005-07-13 devnull 
426 004aa293 2005-07-13 devnull 	USED(dummy);
427 004aa293 2005-07-13 devnull 
428 a4b5da0b 2007-11-27 rsc 	if(incremental)
429 a4b5da0b 2007-11-27 rsc 		vtfilelock(vscores, VtOREAD);
430 004aa293 2005-07-13 devnull 	bsize = fsys->blocksize;
431 004aa293 2005-07-13 devnull 	while((db = qread(qcmp, &bno)) != nil){
432 004aa293 2005-07-13 devnull 		data = db->data;
433 004aa293 2005-07-13 devnull 		sha1(data, vtzerotruncate(VtDataType, data, bsize), score, nil);
434 a4b5da0b 2007-11-27 rsc 		if(incremental){
435 a4b5da0b 2007-11-27 rsc 			if(vtfileblockscore(vscores, bno, score1) < 0)
436 a4b5da0b 2007-11-27 rsc 				sysfatal("cmpproc vtfileblockscore %d: %r", bno);
437 a4b5da0b 2007-11-27 rsc 		}else{
438 a4b5da0b 2007-11-27 rsc 			if(Bseek(&bscores, (vlong)bno*VtScoreSize, 0) < 0)
439 a4b5da0b 2007-11-27 rsc 				sysfatal("cmpproc Bseek: %r");
440 a4b5da0b 2007-11-27 rsc 			if(Bread(&bscores, score1, VtScoreSize) != VtScoreSize)
441 a4b5da0b 2007-11-27 rsc 				sysfatal("cmpproc Bread: %r");
442 a4b5da0b 2007-11-27 rsc 		}
443 004aa293 2005-07-13 devnull 		if(memcmp(score, score1, VtScoreSize) != 0){
444 004aa293 2005-07-13 devnull 			nchange++;
445 004aa293 2005-07-13 devnull 			if(verbose)
446 004aa293 2005-07-13 devnull 				print("# block %ud: old %V new %V\n", bno, score1, score);
447 004aa293 2005-07-13 devnull 			qwrite(qventi, db, bno);
448 004aa293 2005-07-13 devnull 		}else
449 004aa293 2005-07-13 devnull 			blockput(db);
450 004aa293 2005-07-13 devnull 	}
451 004aa293 2005-07-13 devnull 	qclose(qventi);
452 a4b5da0b 2007-11-27 rsc 	if(incremental)
453 a4b5da0b 2007-11-27 rsc 		vtfileunlock(vscores);
454 3bc4abf4 2005-11-07 devnull 	if(statustime)
455 3bc4abf4 2005-11-07 devnull 		print("# %T cmp proc exiting\n");
456 004aa293 2005-07-13 devnull 	runlock(&endlk);
457 004aa293 2005-07-13 devnull }
458 004aa293 2005-07-13 devnull 
459 004aa293 2005-07-13 devnull void
460 004aa293 2005-07-13 devnull writethread(void *v)
461 004aa293 2005-07-13 devnull {
462 004aa293 2005-07-13 devnull 	WriteReq wr;
463 3bc4abf4 2005-11-07 devnull 	char err[ERRMAX];
464 004aa293 2005-07-13 devnull 
465 004aa293 2005-07-13 devnull 	USED(v);
466 004aa293 2005-07-13 devnull 
467 004aa293 2005-07-13 devnull 	while(recv(writechan, &wr) == 1){
468 3bc4abf4 2005-11-07 devnull 		nrecv++;
469 004aa293 2005-07-13 devnull 		if(wr.p == nil)
470 004aa293 2005-07-13 devnull 			break;
471 fa325e9b 2020-01-10 cross 
472 3bc4abf4 2005-11-07 devnull 		if(fastwrites && vtread(z, wr.score, wr.type, nil, 0) < 0){
473 3bc4abf4 2005-11-07 devnull 			rerrstr(err, sizeof err);
474 3bc4abf4 2005-11-07 devnull 			if(strstr(err, "read too small")){	/* already exists */
475 3bc4abf4 2005-11-07 devnull 				nskip++;
476 3bc4abf4 2005-11-07 devnull 				packetfree(wr.p);
477 3bc4abf4 2005-11-07 devnull 				continue;
478 3bc4abf4 2005-11-07 devnull 			}
479 3bc4abf4 2005-11-07 devnull 		}
480 3bc4abf4 2005-11-07 devnull 		if(vtwritepacket(z, wr.score, wr.type, wr.p) < 0)
481 004aa293 2005-07-13 devnull 			sysfatal("vtwritepacket: %r");
482 6fc7da3c 2006-10-19 devnull 		packetfree(wr.p);
483 004aa293 2005-07-13 devnull 	}
484 004aa293 2005-07-13 devnull }
485 004aa293 2005-07-13 devnull 
486 004aa293 2005-07-13 devnull int
487 004aa293 2005-07-13 devnull myvtwrite(VtConn *z, uchar score[VtScoreSize], uint type, uchar *buf, int n)
488 004aa293 2005-07-13 devnull {
489 004aa293 2005-07-13 devnull 	WriteReq wr;
490 004aa293 2005-07-13 devnull 
491 86190ea3 2007-04-08 devnull 	if(nwritethread == 0){
492 86190ea3 2007-04-08 devnull 		n = vtwrite(z, score, type, buf, n);
493 86190ea3 2007-04-08 devnull 		if(n < 0)
494 86190ea3 2007-04-08 devnull 			sysfatal("vtwrite: %r");
495 86190ea3 2007-04-08 devnull 		return n;
496 86190ea3 2007-04-08 devnull 	}
497 004aa293 2005-07-13 devnull 
498 004aa293 2005-07-13 devnull 	wr.p = packetalloc();
499 004aa293 2005-07-13 devnull 	packetappend(wr.p, buf, n);
500 004aa293 2005-07-13 devnull 	packetsha1(wr.p, score);
501 3bc4abf4 2005-11-07 devnull 	memmove(wr.score, score, VtScoreSize);
502 004aa293 2005-07-13 devnull 	wr.type = type;
503 3bc4abf4 2005-11-07 devnull 	nsend++;
504 004aa293 2005-07-13 devnull 	send(writechan, &wr);
505 004aa293 2005-07-13 devnull 	return 0;
506 004aa293 2005-07-13 devnull }
507 004aa293 2005-07-13 devnull 
508 004aa293 2005-07-13 devnull void
509 004aa293 2005-07-13 devnull ventiproc(void *dummy)
510 004aa293 2005-07-13 devnull {
511 004aa293 2005-07-13 devnull 	int i;
512 004aa293 2005-07-13 devnull 	Block *db;
513 004aa293 2005-07-13 devnull 	u32int bno;
514 004aa293 2005-07-13 devnull 	u64int bsize;
515 004aa293 2005-07-13 devnull 
516 004aa293 2005-07-13 devnull 	USED(dummy);
517 004aa293 2005-07-13 devnull 
518 004aa293 2005-07-13 devnull 	proccreate(vtsendproc, z, STACK);
519 004aa293 2005-07-13 devnull 	proccreate(vtrecvproc, z, STACK);
520 004aa293 2005-07-13 devnull 
521 004aa293 2005-07-13 devnull 	writechan = chancreate(sizeof(WriteReq), 0);
522 3bc4abf4 2005-11-07 devnull 	for(i=0; i<nwritethread; i++)
523 004aa293 2005-07-13 devnull 		threadcreate(writethread, nil, STACK);
524 004aa293 2005-07-13 devnull 	vtcachesetwrite(zcache, myvtwrite);
525 004aa293 2005-07-13 devnull 
526 004aa293 2005-07-13 devnull 	bsize = fsys->blocksize;
527 004aa293 2005-07-13 devnull 	vtfilelock(vfile, -1);
528 004aa293 2005-07-13 devnull 	while((db = qread(qventi, &bno)) != nil){
529 004aa293 2005-07-13 devnull 		if(nop){
530 004aa293 2005-07-13 devnull 			blockput(db);
531 004aa293 2005-07-13 devnull 			continue;
532 004aa293 2005-07-13 devnull 		}
533 004aa293 2005-07-13 devnull 		if(vtfilewrite(vfile, db->data, bsize, bno*bsize) != bsize)
534 004aa293 2005-07-13 devnull 			sysfatal("ventiproc vtfilewrite: %r");
535 004aa293 2005-07-13 devnull 		if(vtfileflushbefore(vfile, (bno+1)*bsize) < 0)
536 004aa293 2005-07-13 devnull 			sysfatal("ventiproc vtfileflushbefore: %r");
537 004aa293 2005-07-13 devnull 		blockput(db);
538 004aa293 2005-07-13 devnull 	}
539 004aa293 2005-07-13 devnull 	vtfileunlock(vfile);
540 004aa293 2005-07-13 devnull 	vtcachesetwrite(zcache, nil);
541 3bc4abf4 2005-11-07 devnull 	for(i=0; i<nwritethread; i++)
542 004aa293 2005-07-13 devnull 		send(writechan, nil);
543 6fc7da3c 2006-10-19 devnull 	chanfree(writechan);
544 3bc4abf4 2005-11-07 devnull 	if(statustime)
545 3bc4abf4 2005-11-07 devnull 		print("# %T venti proc exiting - nsend %d nrecv %d\n", nsend, nrecv);
546 004aa293 2005-07-13 devnull 	runlock(&endlk);
547 004aa293 2005-07-13 devnull }
548 004aa293 2005-07-13 devnull 
549 004aa293 2005-07-13 devnull static int
550 004aa293 2005-07-13 devnull percent(u32int a, u32int b)
551 004aa293 2005-07-13 devnull {
552 004aa293 2005-07-13 devnull 	return (vlong)a*100/b;
553 004aa293 2005-07-13 devnull }
554 004aa293 2005-07-13 devnull 
555 004aa293 2005-07-13 devnull void
556 004aa293 2005-07-13 devnull statusproc(void *dummy)
557 004aa293 2005-07-13 devnull {
558 004aa293 2005-07-13 devnull 	int n;
559 004aa293 2005-07-13 devnull 	USED(dummy);
560 004aa293 2005-07-13 devnull 
561 004aa293 2005-07-13 devnull 	for(n=0;;n++){
562 004aa293 2005-07-13 devnull 		sleep(1000);
563 004aa293 2005-07-13 devnull 		if(qcmp->closed && qcmp->nel==0 && qventi->closed && qventi->nel==0)
564 004aa293 2005-07-13 devnull 			break;
565 004aa293 2005-07-13 devnull 		if(n < statustime)
566 004aa293 2005-07-13 devnull 			continue;
567 004aa293 2005-07-13 devnull 		n = 0;
568 004aa293 2005-07-13 devnull 		print("# %T fsscan=%d%% cmpq=%d%% ventiq=%d%%\n",
569 004aa293 2005-07-13 devnull 			percent(fsscanblock, fsys->nblock),
570 004aa293 2005-07-13 devnull 			percent(qcmp->nel, MAXQ),
571 004aa293 2005-07-13 devnull 			percent(qventi->nel, MAXQ));
572 004aa293 2005-07-13 devnull 	}
573 3bc4abf4 2005-11-07 devnull 	print("# %T status proc exiting\n");
574 004aa293 2005-07-13 devnull 	runlock(&endlk);
575 004aa293 2005-07-13 devnull }
576 004aa293 2005-07-13 devnull 
577 004aa293 2005-07-13 devnull int
578 004aa293 2005-07-13 devnull timefmt(Fmt *fmt)
579 004aa293 2005-07-13 devnull {
580 004aa293 2005-07-13 devnull 	vlong ns;
581 004aa293 2005-07-13 devnull 	Tm tm;
582 004aa293 2005-07-13 devnull 	ns = nsec();
583 004aa293 2005-07-13 devnull 	tm = *localtime(time(0));
584 fa325e9b 2020-01-10 cross 	return fmtprint(fmt, "%04d/%02d%02d %02d:%02d:%02d.%03d",
585 004aa293 2005-07-13 devnull 		tm.year+1900, tm.mon+1, tm.mday, tm.hour, tm.min, tm.sec,
586 004aa293 2005-07-13 devnull 		(int)(ns%1000000000)/1000000);
587 004aa293 2005-07-13 devnull }
588 004aa293 2005-07-13 devnull 
589 004aa293 2005-07-13 devnull char*
590 9437f55b 2007-06-12 devnull guessmountplace(char *dev)
591 004aa293 2005-07-13 devnull {
592 004aa293 2005-07-13 devnull 	char *cmd, *q;
593 004aa293 2005-07-13 devnull 	int p[2], fd[3], n;
594 004aa293 2005-07-13 devnull 	char buf[100];
595 fa325e9b 2020-01-10 cross 
596 004aa293 2005-07-13 devnull 	if(pipe(p) < 0)
597 004aa293 2005-07-13 devnull 		sysfatal("pipe: %r");
598 004aa293 2005-07-13 devnull 
599 004aa293 2005-07-13 devnull 	fd[0] = -1;
600 004aa293 2005-07-13 devnull 	fd[1] = p[1];
601 004aa293 2005-07-13 devnull 	fd[2] = -1;
602 9437f55b 2007-06-12 devnull 	cmd = smprint("mount | awk 'BEGIN{v=\"%s\"; u=v; sub(/rdisk/, \"disk\", u);} ($1==v||$1==u) && $2 == \"on\" {print $3}'", dev);
603 004aa293 2005-07-13 devnull 	if(threadspawnl(fd, "sh", "sh", "-c", cmd, nil) < 0)
604 004aa293 2005-07-13 devnull 		sysfatal("exec mount|awk (to find mtpt of %s): %r", dev);
605 004aa293 2005-07-13 devnull 	/* threadspawnl closed p[1] */
606 6fc7da3c 2006-10-19 devnull 	free(cmd);
607 004aa293 2005-07-13 devnull 	n = readn(p[0], buf, sizeof buf-1);
608 004aa293 2005-07-13 devnull 	close(p[0]);
609 004aa293 2005-07-13 devnull 	if(n <= 0)
610 004aa293 2005-07-13 devnull 		return dev;
611 004aa293 2005-07-13 devnull 	buf[n] = 0;
612 004aa293 2005-07-13 devnull 	if((q = strchr(buf, '\n')) == nil)
613 004aa293 2005-07-13 devnull 		return dev;
614 004aa293 2005-07-13 devnull 	*q = 0;
615 004aa293 2005-07-13 devnull 	q = buf+strlen(buf);
616 004aa293 2005-07-13 devnull 	if(q>buf && *(q-1) == '/')
617 004aa293 2005-07-13 devnull 		*--q = 0;
618 004aa293 2005-07-13 devnull 	return strdup(buf);
619 004aa293 2005-07-13 devnull }