1 a0d146ed 2005-07-12 devnull #include "stdinc.h"
  2 a0d146ed 2005-07-12 devnull #include "dat.h"
  3 a0d146ed 2005-07-12 devnull #include "fns.h"
  4 a0d146ed 2005-07-12 devnull 
  5 a0d146ed 2005-07-12 devnull QLock godot;
  6 a0d146ed 2005-07-12 devnull char *host;
  7 a0d146ed 2005-07-12 devnull int readonly = 1;	/* for part.c */
  8 a0d146ed 2005-07-12 devnull int mainstacksize = 256*1024;
  9 a0d146ed 2005-07-12 devnull Channel *c;
 10 a0d146ed 2005-07-12 devnull VtConn *z;
 11 a0d146ed 2005-07-12 devnull int fast;	/* and a bit unsafe; only for benchmarking */
 12 a0d146ed 2005-07-12 devnull int haveaoffset;
 13 a0d146ed 2005-07-12 devnull int maxwrites = -1;
 14 703c2d41 2007-05-03 devnull int verbose;
 15 a0d146ed 2005-07-12 devnull 
 16 a0d146ed 2005-07-12 devnull typedef struct ZClump ZClump;
 17 a0d146ed 2005-07-12 devnull struct ZClump
 18 a0d146ed 2005-07-12 devnull {
 19 a0d146ed 2005-07-12 devnull 	ZBlock *lump;
 20 a0d146ed 2005-07-12 devnull 	Clump cl;
 21 a0d146ed 2005-07-12 devnull 	u64int aa;
 22 a0d146ed 2005-07-12 devnull };
 23 a0d146ed 2005-07-12 devnull 
 24 a0d146ed 2005-07-12 devnull void
 25 a0d146ed 2005-07-12 devnull usage(void)
 26 a0d146ed 2005-07-12 devnull {
 27 9ee00732 2011-06-07 rsc 	fprint(2, "usage: wrarena [-o fileoffset] [-h host] arenafile [clumpoffset]\n");
 28 a0d146ed 2005-07-12 devnull 	threadexitsall("usage");
 29 a0d146ed 2005-07-12 devnull }
 30 a0d146ed 2005-07-12 devnull 
 31 a0d146ed 2005-07-12 devnull void
 32 a0d146ed 2005-07-12 devnull vtsendthread(void *v)
 33 a0d146ed 2005-07-12 devnull {
 34 a0d146ed 2005-07-12 devnull 	ZClump zcl;
 35 a0d146ed 2005-07-12 devnull 
 36 a0d146ed 2005-07-12 devnull 	USED(v);
 37 a0d146ed 2005-07-12 devnull 	while(recv(c, &zcl) == 1){
 38 a0d146ed 2005-07-12 devnull 		if(zcl.lump == nil)
 39 a0d146ed 2005-07-12 devnull 			break;
 40 a0d146ed 2005-07-12 devnull 		if(vtwrite(z, zcl.cl.info.score, zcl.cl.info.type, zcl.lump->data, zcl.cl.info.uncsize) < 0)
 41 a0d146ed 2005-07-12 devnull 			sysfatal("failed writing clump %llud: %r", zcl.aa);
 42 703c2d41 2007-05-03 devnull 		if(verbose)
 43 703c2d41 2007-05-03 devnull 			print("%V\n", zcl.cl.info.score);
 44 a0d146ed 2005-07-12 devnull 		freezblock(zcl.lump);
 45 a0d146ed 2005-07-12 devnull 	}
 46 a0d146ed 2005-07-12 devnull 	/*
 47 a0d146ed 2005-07-12 devnull 	 * All the send threads try to exit right when
 48 fa325e9b 2020-01-10 cross 	 * threadmain is calling threadexitsall.
 49 a0d146ed 2005-07-12 devnull 	 * Either libthread or the Linux NPTL pthreads library
 50 a0d146ed 2005-07-12 devnull 	 * can't handle this condition (I suspect NPTL but have
 51 a0d146ed 2005-07-12 devnull 	 * not confirmed this) and we get a seg fault in exit.
 52 a0d146ed 2005-07-12 devnull 	 * I spent a day tracking this down with no success,
 53 a0d146ed 2005-07-12 devnull 	 * so we're going to work around it instead by just
 54 a0d146ed 2005-07-12 devnull 	 * sitting here and waiting for the threadexitsall to
 55 a0d146ed 2005-07-12 devnull 	 * take effect.
 56 a0d146ed 2005-07-12 devnull 	 */
 57 a0d146ed 2005-07-12 devnull 	qlock(&godot);
 58 a0d146ed 2005-07-12 devnull }
 59 a0d146ed 2005-07-12 devnull 
 60 a0d146ed 2005-07-12 devnull static void
 61 a0d146ed 2005-07-12 devnull rdarena(Arena *arena, u64int offset)
 62 a0d146ed 2005-07-12 devnull {
 63 703c2d41 2007-05-03 devnull 	int i;
 64 a0d146ed 2005-07-12 devnull 	u64int a, aa, e;
 65 952f137f 2007-09-25 rsc 	uchar score[VtScoreSize];
 66 a0d146ed 2005-07-12 devnull 	Clump cl;
 67 703c2d41 2007-05-03 devnull 	ClumpInfo ci;
 68 a0d146ed 2005-07-12 devnull 	ZBlock *lump;
 69 a0d146ed 2005-07-12 devnull 	ZClump zcl;
 70 a0d146ed 2005-07-12 devnull 
 71 a0d146ed 2005-07-12 devnull 	fprint(2, "wrarena: copying %s to venti\n", arena->name);
 72 a0d146ed 2005-07-12 devnull 	printarena(2, arena);
 73 a0d146ed 2005-07-12 devnull 
 74 a0d146ed 2005-07-12 devnull 	a = arena->base;
 75 a0d146ed 2005-07-12 devnull 	e = arena->base + arena->size;
 76 a0d146ed 2005-07-12 devnull 	if(offset != ~(u64int)0) {
 77 952f137f 2007-09-25 rsc 		if(offset >= e - a)
 78 f5a8ea6f 2011-06-02 rsc 			sysfatal("bad offset %#llx >= %#llx", offset, e - a);
 79 a0d146ed 2005-07-12 devnull 		aa = offset;
 80 a0d146ed 2005-07-12 devnull 	} else
 81 a0d146ed 2005-07-12 devnull 		aa = 0;
 82 a0d146ed 2005-07-12 devnull 
 83 952f137f 2007-09-25 rsc 	i = 0;
 84 952f137f 2007-09-25 rsc 	for(a = 0; maxwrites != 0 && i < arena->memstats.clumps;
 85 952f137f 2007-09-25 rsc 	    a += ClumpSize + ci.size){
 86 952f137f 2007-09-25 rsc 		if(readclumpinfo(arena, i++, &ci) < 0)
 87 a0d146ed 2005-07-12 devnull 			break;
 88 703c2d41 2007-05-03 devnull 		if(a < aa || ci.type == VtCorruptType){
 89 703c2d41 2007-05-03 devnull 			if(ci.type == VtCorruptType)
 90 f5a8ea6f 2011-06-02 rsc 				fprint(2, "%s: corrupt clump read at %#llx: +%d\n",
 91 f5a8ea6f 2011-06-02 rsc 					argv0, a, ClumpSize+ci.size);
 92 703c2d41 2007-05-03 devnull 			continue;
 93 a0d146ed 2005-07-12 devnull 		}
 94 703c2d41 2007-05-03 devnull 		lump = loadclump(arena, a, 0, &cl, score, 0);
 95 a0d146ed 2005-07-12 devnull 		if(lump == nil) {
 96 703c2d41 2007-05-03 devnull 			fprint(2, "clump %#llx failed to read: %r\n", a);
 97 703c2d41 2007-05-03 devnull 			continue;
 98 a0d146ed 2005-07-12 devnull 		}
 99 a0d146ed 2005-07-12 devnull 		if(!fast && cl.info.type != VtCorruptType) {
100 a0d146ed 2005-07-12 devnull 			scoremem(score, lump->data, cl.info.uncsize);
101 a0d146ed 2005-07-12 devnull 			if(scorecmp(cl.info.score, score) != 0) {
102 952f137f 2007-09-25 rsc 				fprint(2, "clump %#llx has mismatched score\n",
103 952f137f 2007-09-25 rsc 					a);
104 a0d146ed 2005-07-12 devnull 				break;
105 a0d146ed 2005-07-12 devnull 			}
106 a0d146ed 2005-07-12 devnull 			if(vttypevalid(cl.info.type) < 0) {
107 952f137f 2007-09-25 rsc 				fprint(2, "clump %#llx has bad type %d\n",
108 952f137f 2007-09-25 rsc 					a, cl.info.type);
109 a0d146ed 2005-07-12 devnull 				break;
110 a0d146ed 2005-07-12 devnull 			}
111 a0d146ed 2005-07-12 devnull 		}
112 a0d146ed 2005-07-12 devnull 		if(z && cl.info.type != VtCorruptType){
113 a0d146ed 2005-07-12 devnull 			zcl.cl = cl;
114 a0d146ed 2005-07-12 devnull 			zcl.lump = lump;
115 703c2d41 2007-05-03 devnull 			zcl.aa = a;
116 a0d146ed 2005-07-12 devnull 			send(c, &zcl);
117 a0d146ed 2005-07-12 devnull 		}else
118 a0d146ed 2005-07-12 devnull 			freezblock(lump);
119 952f137f 2007-09-25 rsc 		if(maxwrites > 0)
120 952f137f 2007-09-25 rsc 			--maxwrites;
121 a0d146ed 2005-07-12 devnull 	}
122 703c2d41 2007-05-03 devnull 	if(a > aa)
123 703c2d41 2007-05-03 devnull 		aa = a;
124 a0d146ed 2005-07-12 devnull 	if(haveaoffset)
125 703c2d41 2007-05-03 devnull 		print("end offset %#llx\n", aa);
126 a0d146ed 2005-07-12 devnull }
127 a0d146ed 2005-07-12 devnull 
128 a0d146ed 2005-07-12 devnull void
129 a0d146ed 2005-07-12 devnull threadmain(int argc, char *argv[])
130 a0d146ed 2005-07-12 devnull {
131 a0d146ed 2005-07-12 devnull 	int i;
132 a0d146ed 2005-07-12 devnull 	char *file;
133 a0d146ed 2005-07-12 devnull 	Arena *arena;
134 20271fed 2012-04-07 rsc 	ArenaPart *ap;
135 a0d146ed 2005-07-12 devnull 	u64int offset, aoffset;
136 a0d146ed 2005-07-12 devnull 	Part *part;
137 a0d146ed 2005-07-12 devnull 	uchar buf[8192];
138 a0d146ed 2005-07-12 devnull 	ArenaHead head;
139 a0d146ed 2005-07-12 devnull 	ZClump zerocl;
140 a0d146ed 2005-07-12 devnull 
141 a0d146ed 2005-07-12 devnull 	qlock(&godot);
142 a0d146ed 2005-07-12 devnull 	aoffset = 0;
143 a0d146ed 2005-07-12 devnull 	ARGBEGIN{
144 a0d146ed 2005-07-12 devnull 	case 'f':
145 a0d146ed 2005-07-12 devnull 		fast = 1;
146 a0d146ed 2005-07-12 devnull 		ventidoublechecksha1 = 0;
147 a0d146ed 2005-07-12 devnull 		break;
148 a0d146ed 2005-07-12 devnull 	case 'h':
149 a0d146ed 2005-07-12 devnull 		host = EARGF(usage());
150 a0d146ed 2005-07-12 devnull 		break;
151 a0d146ed 2005-07-12 devnull 	case 'o':
152 a0d146ed 2005-07-12 devnull 		haveaoffset = 1;
153 a0d146ed 2005-07-12 devnull 		aoffset = strtoull(EARGF(usage()), 0, 0);
154 a0d146ed 2005-07-12 devnull 		break;
155 a0d146ed 2005-07-12 devnull 	case 'M':
156 a0d146ed 2005-07-12 devnull 		maxwrites = atoi(EARGF(usage()));
157 a0d146ed 2005-07-12 devnull 		break;
158 703c2d41 2007-05-03 devnull 	case 'v':
159 703c2d41 2007-05-03 devnull 		verbose = 1;
160 703c2d41 2007-05-03 devnull 		break;
161 a0d146ed 2005-07-12 devnull 	default:
162 a0d146ed 2005-07-12 devnull 		usage();
163 a0d146ed 2005-07-12 devnull 		break;
164 a0d146ed 2005-07-12 devnull 	}ARGEND
165 a0d146ed 2005-07-12 devnull 
166 a0d146ed 2005-07-12 devnull 	offset = ~(u64int)0;
167 a0d146ed 2005-07-12 devnull 	switch(argc) {
168 a0d146ed 2005-07-12 devnull 	default:
169 a0d146ed 2005-07-12 devnull 		usage();
170 a0d146ed 2005-07-12 devnull 	case 2:
171 a0d146ed 2005-07-12 devnull 		offset = strtoull(argv[1], 0, 0);
172 a0d146ed 2005-07-12 devnull 		/* fall through */
173 a0d146ed 2005-07-12 devnull 	case 1:
174 a0d146ed 2005-07-12 devnull 		file = argv[0];
175 a0d146ed 2005-07-12 devnull 	}
176 a0d146ed 2005-07-12 devnull 
177 d4bf606d 2008-07-04 rsc 	ventifmtinstall();
178 a0d146ed 2005-07-12 devnull 
179 a0d146ed 2005-07-12 devnull 	statsinit();
180 a0d146ed 2005-07-12 devnull 
181 a0d146ed 2005-07-12 devnull 	part = initpart(file, OREAD);
182 a0d146ed 2005-07-12 devnull 	if(part == nil)
183 a0d146ed 2005-07-12 devnull 		sysfatal("can't open file %s: %r", file);
184 20271fed 2012-04-07 rsc 
185 20271fed 2012-04-07 rsc 	// Try as arena partition.
186 20271fed 2012-04-07 rsc 	arena = nil;
187 20271fed 2012-04-07 rsc 	ap = initarenapart(part);
188 0798d6b7 2013-01-03 0intro 	if(ap != nil)
189 20271fed 2012-04-07 rsc 		goto loaded;
190 20271fed 2012-04-07 rsc 
191 a0d146ed 2005-07-12 devnull 	if(readpart(part, aoffset, buf, sizeof buf) < 0)
192 a0d146ed 2005-07-12 devnull 		sysfatal("can't read file %s: %r", file);
193 a0d146ed 2005-07-12 devnull 
194 a0d146ed 2005-07-12 devnull 	if(unpackarenahead(&head, buf) < 0)
195 a0d146ed 2005-07-12 devnull 		sysfatal("corrupted arena header: %r");
196 a0d146ed 2005-07-12 devnull 
197 45ac814c 2007-10-29 rsc 	if(aoffset+head.size > part->size)
198 f5a8ea6f 2011-06-02 rsc 		sysfatal("arena is truncated: want %llud bytes have %llud",
199 45ac814c 2007-10-29 rsc 			head.size, part->size);
200 a0d146ed 2005-07-12 devnull 
201 a0d146ed 2005-07-12 devnull 	partblocksize(part, head.blocksize);
202 a0d146ed 2005-07-12 devnull 
203 a0d146ed 2005-07-12 devnull 	arena = initarena(part, aoffset, head.size, head.blocksize);
204 a0d146ed 2005-07-12 devnull 	if(arena == nil)
205 a0d146ed 2005-07-12 devnull 		sysfatal("initarena: %r");
206 a0d146ed 2005-07-12 devnull 
207 20271fed 2012-04-07 rsc loaded:
208 23fb2edb 2005-07-24 devnull 	z = nil;
209 23fb2edb 2005-07-24 devnull 	if(host==nil || strcmp(host, "/dev/null") != 0){
210 a0d146ed 2005-07-12 devnull 		z = vtdial(host);
211 a0d146ed 2005-07-12 devnull 		if(z == nil)
212 a0d146ed 2005-07-12 devnull 			sysfatal("could not connect to server: %r");
213 a0d146ed 2005-07-12 devnull 		if(vtconnect(z) < 0)
214 a0d146ed 2005-07-12 devnull 			sysfatal("vtconnect: %r");
215 23fb2edb 2005-07-24 devnull 	}
216 0798d6b7 2013-01-03 0intro 
217 d4bf606d 2008-07-04 rsc 	print("%T starting to send data\n");
218 a0d146ed 2005-07-12 devnull 	c = chancreate(sizeof(ZClump), 0);
219 a0d146ed 2005-07-12 devnull 	for(i=0; i<12; i++)
220 a0d146ed 2005-07-12 devnull 		vtproc(vtsendthread, nil);
221 a0d146ed 2005-07-12 devnull 
222 0798d6b7 2013-01-03 0intro 	initdcache(8 * MaxDiskBlock);
223 0798d6b7 2013-01-03 0intro 
224 20271fed 2012-04-07 rsc 	if(ap != nil) {
225 a2df31f4 2012-04-07 rsc 		for(i=0; i<ap->narenas; i++)
226 20271fed 2012-04-07 rsc 			rdarena(ap->arenas[i], 0);
227 20271fed 2012-04-07 rsc 	} else
228 20271fed 2012-04-07 rsc 		rdarena(arena, offset);
229 20271fed 2012-04-07 rsc 
230 a0d146ed 2005-07-12 devnull 	memset(&zerocl, 0, sizeof zerocl);
231 a0d146ed 2005-07-12 devnull 	for(i=0; i<12; i++)
232 a0d146ed 2005-07-12 devnull 		send(c, &zerocl);
233 d4bf606d 2008-07-04 rsc 	if(vtsync(z) < 0)
234 d4bf606d 2008-07-04 rsc 		sysfatal("executing sync: %r");
235 a0d146ed 2005-07-12 devnull 	if(z){
236 a0d146ed 2005-07-12 devnull 		vthangup(z);
237 a0d146ed 2005-07-12 devnull 	}
238 d4bf606d 2008-07-04 rsc 	print("%T sent all data\n");
239 d4bf606d 2008-07-04 rsc 
240 a0d146ed 2005-07-12 devnull 	threadexitsall(0);
241 a0d146ed 2005-07-12 devnull }