op public repos

Blob

Date:: Mon Dec 24 13:18:56 2018 UTC
Message:: eliminate got_object_open() round-trip when opening blobs
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/types.h>
18 #include <sys/queue.h>
19 #include <sys/uio.h>
20 #include <sys/time.h>
21 #include <sys/limits.h>
22 #include <sys/syslimits.h>
23 #include <sys/mman.h>
24 
25 #include <limits.h>
26 #include <signal.h>
27 #include <stdint.h>
28 #include <imsg.h>
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <sha1.h>
33 #include <zlib.h>
34 
35 #include "got_error.h"
36 #include "got_object.h"
37 
38 #include "got_lib_delta.h"
39 #include "got_lib_inflate.h"
40 #include "got_lib_object.h"
41 #include "got_lib_object_cache.h"
42 #include "got_lib_object_parse.h"
43 #include "got_lib_privsep.h"
44 #include "got_lib_pack.h"
45 
46 static volatile sig_atomic_t sigint_received;
47 
48 static void
49 catch_sigint(int signo)
50 {
51 	sigint_received = 1;
52 }
53 
54 static const struct got_error *
55 object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
56     struct got_packidx *packidx, struct got_object_cache *objcache)
57 {
58 	const struct got_error *err = NULL;
59 	struct got_imsg_packed_object iobj;
60 	struct got_object *obj;
61 	struct got_object_id id;
62 	size_t datalen;
63 
64 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
65 	if (datalen != sizeof(iobj))
66 		return got_error(GOT_ERR_PRIVSEP_LEN);
67 	memcpy(&iobj, imsg->data, sizeof(iobj));
68 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
69 
70 	err = got_packfile_open_object(&obj, pack, packidx, iobj.idx, &id);
71 	if (err)
72 		return err;
73 	obj->refcnt++;
74 
75 	err = got_object_cache_add(objcache, &obj->id, obj);
76 	if (err)
77 		goto done;
78 	obj->refcnt++;
79 
80 	err = got_privsep_send_obj(ibuf, obj);
81 done:
82 	got_object_close(obj);
83 	return err;
84 }
85 
86 static const struct got_error *
87 commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
88     struct got_packidx *packidx, struct got_object_cache *objcache)
89 {
90 	const struct got_error *err = NULL;
91 	struct got_imsg_packed_object iobj;
92 	struct got_object *obj;
93 	struct got_commit_object *commit = NULL;
94 	uint8_t *buf;
95 	size_t len;
96 	struct got_object_id id;
97 	size_t datalen;
98 
99 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
100 	if (datalen != sizeof(iobj))
101 		return got_error(GOT_ERR_PRIVSEP_LEN);
102 	memcpy(&iobj, imsg->data, sizeof(iobj));
103 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
104 
105 	err = got_packfile_open_object(&obj, pack, packidx, iobj.idx, &id);
106 	if (err)
107 		return err;
108 
109 	err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
110 	if (err)
111 		return err;
112 
113 	obj->size = len;
114 	err = got_object_parse_commit(&commit, buf, len);
115 	free(buf);
116 	if (err) {
117 		got_object_close(obj);
118 		return err;
119 	}
120 
121 	err = got_privsep_send_commit(ibuf, commit);
122 	got_object_commit_close(commit);
123 	if (err) {
124 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
125 			err = NULL;
126 		else
127 			got_privsep_send_error(ibuf, err);
128 	}
129 
130 	return err;
131 }
132 
133 static const struct got_error *
134 tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
135     struct got_packidx *packidx, struct got_object_cache *objcache)
136 {
137 	const struct got_error *err = NULL;
138 	struct got_imsg_packed_object iobj;
139 	struct got_object *obj = NULL;
140 	struct got_tree_object *tree = NULL;
141 	uint8_t *buf;
142 	size_t len;
143 	struct got_object_id id;
144 	size_t datalen;
145 
146 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
147 	if (datalen != sizeof(iobj))
148 		return got_error(GOT_ERR_PRIVSEP_LEN);
149 	memcpy(&iobj, imsg->data, sizeof(iobj));
150 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
151 
152 	err = got_packfile_open_object(&obj, pack, packidx, iobj.idx, &id);
153 	if (err)
154 		return err;
155 
156 	err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
157 	if (err)
158 		return err;
159 
160 	obj->size = len;
161 	err = got_object_parse_tree(&tree, buf, len);
162 	free(buf);
163 
164 	err = got_privsep_send_tree(ibuf, tree);
165 	if (obj)
166 		got_object_close(obj);
167 	got_object_tree_close(tree);
168 	if (err) {
169 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
170 			err = NULL;
171 		else
172 			got_privsep_send_error(ibuf, err);
173 	}
174 
175 	return err;
176 }
177 
178 static const struct got_error *
179 receive_file(FILE **f, struct imsgbuf *ibuf, int imsg_code)
180 {
181 	const struct got_error *err;
182 	struct imsg imsg;
183 	size_t datalen;
184 
185 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
186 	if (err)
187 		return err;
188 
189 	if (imsg.hdr.type != imsg_code) {
190 		err = got_error(GOT_ERR_PRIVSEP_MSG);
191 		goto done;
192 	}
193 
194 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
195 	if (datalen != 0) {
196 		err = got_error(GOT_ERR_PRIVSEP_LEN);
197 		goto done;
198 	}
199 	if (imsg.fd == -1) {
200 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
201 		goto done;
202 	}
203 
204 	*f = fdopen(imsg.fd, "w+");
205 	if (*f == NULL) {
206 		close(imsg.fd);
207 		err = got_error_from_errno();
208 		goto done;
209 	}
210 done:
211 	imsg_free(&imsg);
212 	return err;
213 }
214 
215 static const struct got_error *
216 blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
217     struct got_packidx *packidx, struct got_object_cache *objcache)
218 {
219 	const struct got_error *err = NULL;
220 	struct got_imsg_packed_object iobj;
221 	struct got_object *obj = NULL;
222 	FILE *outfile = NULL, *basefile = NULL, *accumfile = NULL;
223 	struct got_object_id id;
224 	size_t datalen;
225 
226 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
227 	if (datalen != sizeof(iobj))
228 		return got_error(GOT_ERR_PRIVSEP_LEN);
229 	memcpy(&iobj, imsg->data, sizeof(iobj));
230 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
231 
232 	err = got_packfile_open_object(&obj, pack, packidx, iobj.idx, &id);
233 	if (err)
234 		return err;
235 
236 	err = receive_file(&outfile, ibuf, GOT_IMSG_BLOB_OUTFD);
237 	if (err)
238 		return err;
239 	err = receive_file(&basefile, ibuf, GOT_IMSG_TMPFD);
240 	if (err)
241 		return err;
242 	err = receive_file(&accumfile, ibuf, GOT_IMSG_TMPFD);
243 	if (err)
244 		return err;
245 
246 	err = got_packfile_extract_object(pack, obj, outfile, basefile,
247 	    accumfile);
248 	if (err)
249 		goto done;
250 
251 	err = got_privsep_send_blob(ibuf, obj->size, obj->hdrlen);
252 done:
253 	if (outfile)
254 		fclose(outfile);
255 	if (basefile)
256 		fclose(basefile);
257 	if (accumfile)
258 		fclose(accumfile);
259 	if (obj)
260 		got_object_close(obj);
261 	if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
262 		got_privsep_send_error(ibuf, err);
263 
264 	return err;
265 }
266 
267 static const struct got_error *
268 tag_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
269     struct got_packidx *packidx, struct got_object_cache *objcache)
270 {
271 	const struct got_error *err = NULL;
272 	struct got_imsg_packed_object iobj;
273 	struct got_object *obj = NULL;
274 	struct got_tag_object *tag = NULL;
275 	uint8_t *buf;
276 	size_t len;
277 	struct got_object_id id;
278 	size_t datalen;
279 
280 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
281 	if (datalen != sizeof(iobj))
282 		return got_error(GOT_ERR_PRIVSEP_LEN);
283 	memcpy(&iobj, imsg->data, sizeof(iobj));
284 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
285 
286 	err = got_packfile_open_object(&obj, pack, packidx, iobj.idx, &id);
287 	if (err)
288 		return err;
289 
290 	err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
291 	if (err)
292 		return err;
293 
294 	obj->size = len;
295 	err = got_object_parse_tag(&tag, buf, len);
296 	free(buf);
297 
298 	err = got_privsep_send_tag(ibuf, tag);
299 	if (obj)
300 		got_object_close(obj);
301 	got_object_tag_close(tag);
302 	if (err) {
303 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
304 			err = NULL;
305 		else
306 			got_privsep_send_error(ibuf, err);
307 	}
308 
309 	return err;
310 }
311 
312 static const struct got_error *
313 receive_packidx(struct got_packidx **packidx, struct imsgbuf *ibuf)
314 {
315 	const struct got_error *err = NULL;
316 	struct imsg imsg;
317 	struct got_imsg_packidx ipackidx;
318 	size_t datalen;
319 	struct got_packidx *p;
320 
321 	*packidx = NULL;
322 
323 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
324 	if (err)
325 		return err;
326 
327 	p = calloc(1, sizeof(*p));
328 	if (p == NULL) {
329 		err = got_error_from_errno();
330 		goto done;
331 	}
332 
333 	if (imsg.hdr.type != GOT_IMSG_PACKIDX) {
334 		err = got_error(GOT_ERR_PRIVSEP_MSG);
335 		goto done;
336 	}
337 
338 	if (imsg.fd == -1) {
339 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
340 		goto done;
341 	}
342 
343 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
344 	if (datalen != sizeof(ipackidx)) {
345 		err = got_error(GOT_ERR_PRIVSEP_LEN);
346 		goto done;
347 	}
348 	memcpy(&ipackidx, imsg.data, sizeof(ipackidx));
349 
350 	p->len = ipackidx.len;
351 	p->fd = dup(imsg.fd);
352 	if (p->fd == -1) {
353 		err = got_error_from_errno();
354 		goto done;
355 	}
356 	if (lseek(p->fd, 0, SEEK_SET) == -1) {
357 		err = got_error_from_errno();
358 		goto done;
359 	}
360 
361 #ifndef GOT_PACK_NO_MMAP
362 	p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0);
363 	if (p->map == MAP_FAILED)
364 		p->map = NULL; /* fall back to read(2) */
365 #endif
366 	err = got_packidx_init_hdr(p, 1);
367 done:
368 	if (err) {
369 		if (imsg.fd != -1)
370 			close(imsg.fd);
371 		got_packidx_close(p);
372 	} else
373 		*packidx = p;
374 	imsg_free(&imsg);
375 	return err;
376 }
377 
378 static const struct got_error *
379 receive_pack(struct got_pack **packp, struct imsgbuf *ibuf)
380 {
381 	const struct got_error *err = NULL;
382 	struct imsg imsg;
383 	struct got_imsg_pack ipack;
384 	size_t datalen;
385 	struct got_pack *pack;
386 
387 	*packp = NULL;
388 
389 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
390 	if (err)
391 		return err;
392 
393 	pack = calloc(1, sizeof(*pack));
394 	if (pack == NULL) {
395 		err = got_error_from_errno();
396 		goto done;
397 	}
398 
399 	if (imsg.hdr.type != GOT_IMSG_PACK) {
400 		err = got_error(GOT_ERR_PRIVSEP_MSG);
401 		goto done;
402 	}
403 
404 	if (imsg.fd == -1) {
405 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
406 		goto done;
407 	}
408 
409 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
410 	if (datalen != sizeof(ipack)) {
411 		err = got_error(GOT_ERR_PRIVSEP_LEN);
412 		goto done;
413 	}
414 	memcpy(&ipack, imsg.data, sizeof(ipack));
415 
416 	pack->filesize = ipack.filesize;
417 	pack->fd = dup(imsg.fd);
418 	if (pack->fd == -1) {
419 		err = got_error_from_errno();
420 		goto done;
421 	}
422 	if (lseek(pack->fd, 0, SEEK_SET) == -1) {
423 		err = got_error_from_errno();
424 		goto done;
425 	}
426 	pack->path_packfile = strdup(ipack.path_packfile);
427 	if (pack->path_packfile == NULL) {
428 		err = got_error_from_errno();
429 		goto done;
430 	}
431 
432 #ifndef GOT_PACK_NO_MMAP
433 	pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
434 	    pack->fd, 0);
435 	if (pack->map == MAP_FAILED)
436 		pack->map = NULL; /* fall back to read(2) */
437 #endif
438 done:
439 	if (err) {
440 		if (imsg.fd != -1)
441 			close(imsg.fd);
442 		free(pack);
443 	} else
444 		*packp = pack;
445 	imsg_free(&imsg);
446 	return err;
447 }
448 
449 int
450 main(int argc, char *argv[])
451 {
452 	const struct got_error *err = NULL;
453 	struct imsgbuf ibuf;
454 	struct imsg imsg;
455 	struct got_packidx *packidx = NULL;
456 	struct got_pack *pack = NULL;
457 	struct got_object_cache objcache;
458 
459 	//static int attached;
460 	//while (!attached) sleep(1);
461 
462 	signal(SIGINT, catch_sigint);
463 
464 	imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
465 
466 	err = got_object_cache_init(&objcache, GOT_OBJECT_CACHE_TYPE_OBJ);
467 	if (err) {
468 		err = got_error_from_errno();
469 		got_privsep_send_error(&ibuf, err);
470 		return 1;
471 	}
472 
473 #ifndef PROFILE
474 	/* revoke access to most system calls */
475 	if (pledge("stdio recvfd", NULL) == -1) {
476 		err = got_error_from_errno();
477 		got_privsep_send_error(&ibuf, err);
478 		return 1;
479 	}
480 #endif
481 
482 	err = receive_packidx(&packidx, &ibuf);
483 	if (err) {
484 		got_privsep_send_error(&ibuf, err);
485 		return 1;
486 	}
487 
488 	err = receive_pack(&pack, &ibuf);
489 	if (err) {
490 		got_privsep_send_error(&ibuf, err);
491 		return 1;
492 	}
493 
494 	while (1) {
495 		imsg.fd = -1;
496 
497 		if (sigint_received) {
498 			err = got_error(GOT_ERR_CANCELLED);
499 			break;
500 		}
501 
502 		err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
503 		if (err) {
504 			if (err->code == GOT_ERR_PRIVSEP_PIPE)
505 				err = NULL;
506 			break;
507 		}
508 
509 		if (imsg.hdr.type == GOT_IMSG_STOP)
510 			break;
511 
512 		switch (imsg.hdr.type) {
513 		case GOT_IMSG_PACKED_OBJECT_REQUEST:
514 			err = object_request(&imsg, &ibuf, pack, packidx,
515 			    &objcache);
516 			break;
517 		case GOT_IMSG_COMMIT_REQUEST:
518 			err = commit_request(&imsg, &ibuf, pack, packidx,
519 			    &objcache);
520 			break;
521 		case GOT_IMSG_TREE_REQUEST:
522 			err = tree_request(&imsg, &ibuf, pack, packidx,
523 			   &objcache);
524 			break;
525 		case GOT_IMSG_BLOB_REQUEST:
526 			err = blob_request(&imsg, &ibuf, pack, packidx,
527 			   &objcache);
528 			break;
529 		case GOT_IMSG_TAG_REQUEST:
530 			err = tag_request(&imsg, &ibuf, pack, packidx,
531 			   &objcache);
532 			break;
533 		default:
534 			err = got_error(GOT_ERR_PRIVSEP_MSG);
535 			break;
536 		}
537 
538 		if (imsg.fd != -1)
539 			close(imsg.fd);
540 		imsg_free(&imsg);
541 		if (err)
542 			break;
543 	}
544 
545 	if (packidx)
546 		got_packidx_close(packidx);
547 	if (pack)
548 		got_pack_close(pack);
549 	got_object_cache_close(&objcache);
550 	imsg_clear(&ibuf);
551 	if (err) {
552 		if (!sigint_received && err->code != GOT_ERR_PRIVSEP_PIPE) {
553 			fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
554 			got_privsep_send_error(&ibuf, err);
555 		}
556 	}
557 	close(GOT_IMSG_FD_CHILD);
558 	return err ? 1 : 0;
559 }