op public repos

Blob

Date:: Sun Feb 13 00:10:25 2022 UTC
Message:: reuse existing deltas when creating pack files tested by thomas, naddy, and myself
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/types.h>
18 #include <sys/queue.h>
19 #include <sys/uio.h>
20 #include <sys/time.h>
21 #include <sys/mman.h>
22 
23 #include <limits.h>
24 #include <signal.h>
25 #include <stdint.h>
26 #include <imsg.h>
27 #include <stdio.h>
28 #include <stdlib.h>
29 #include <string.h>
30 #include <sha1.h>
31 #include <unistd.h>
32 #include <zlib.h>
33 
34 #include "got_error.h"
35 #include "got_object.h"
36 #include "got_path.h"
37 
38 #include "got_lib_delta.h"
39 #include "got_lib_delta_cache.h"
40 #include "got_lib_object.h"
41 #include "got_lib_object_cache.h"
42 #include "got_lib_object_parse.h"
43 #include "got_lib_privsep.h"
44 #include "got_lib_pack.h"
45 
46 static volatile sig_atomic_t sigint_received;
47 
48 static void
49 catch_sigint(int signo)
50 {
51 	sigint_received = 1;
52 }
53 
54 static const struct got_error *
55 open_object(struct got_object **obj, struct got_pack *pack,
56     struct got_packidx *packidx, int idx, struct got_object_id *id,
57     struct got_object_cache *objcache)
58 {
59 	const struct got_error *err;
60 
61 	err = got_packfile_open_object(obj, pack, packidx, idx, id);
62 	if (err)
63 		return err;
64 	(*obj)->refcnt++;
65 
66 	err = got_object_cache_add(objcache, id, *obj);
67 	if (err) {
68 		if (err->code == GOT_ERR_OBJ_EXISTS ||
69 		    err->code == GOT_ERR_OBJ_TOO_LARGE)
70 			err = NULL;
71 		return err;
72 	}
73 	(*obj)->refcnt++;
74 	return NULL;
75 }
76 
77 static const struct got_error *
78 object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
79     struct got_packidx *packidx, struct got_object_cache *objcache)
80 {
81 	const struct got_error *err = NULL;
82 	struct got_imsg_packed_object iobj;
83 	struct got_object *obj;
84 	struct got_object_id id;
85 	size_t datalen;
86 
87 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
88 	if (datalen != sizeof(iobj))
89 		return got_error(GOT_ERR_PRIVSEP_LEN);
90 	memcpy(&iobj, imsg->data, sizeof(iobj));
91 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
92 
93 	obj = got_object_cache_get(objcache, &id);
94 	if (obj) {
95 		obj->refcnt++;
96 	} else {
97 		err = open_object(&obj, pack, packidx, iobj.idx, &id,
98 		    objcache);
99 		if (err)
100 			goto done;
101 	}
102 
103 	err = got_privsep_send_obj(ibuf, obj);
104 done:
105 	got_object_close(obj);
106 	return err;
107 }
108 
109 static const struct got_error *
110 open_commit(struct got_commit_object **commit, struct got_pack *pack,
111     struct got_packidx *packidx, int obj_idx, struct got_object_id *id,
112     struct got_object_cache *objcache)
113 {
114 	const struct got_error *err = NULL;
115 	struct got_object *obj = NULL;
116 	uint8_t *buf = NULL;
117 	size_t len;
118 
119 	*commit = NULL;
120 
121 	obj = got_object_cache_get(objcache, id);
122 	if (obj) {
123 		obj->refcnt++;
124 	} else {
125 		err = open_object(&obj, pack, packidx, obj_idx, id,
126 		    objcache);
127 		if (err)
128 			return err;
129 	}
130 
131 	err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
132 	if (err)
133 		goto done;
134 
135 	obj->size = len;
136 
137 	err = got_object_parse_commit(commit, buf, len);
138 done:
139 	got_object_close(obj);
140 	free(buf);
141 	return err;
142 }
143 
144 static const struct got_error *
145 commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
146     struct got_packidx *packidx, struct got_object_cache *objcache)
147 {
148 	const struct got_error *err = NULL;
149 	struct got_imsg_packed_object iobj;
150 	struct got_commit_object *commit = NULL;
151 	struct got_object_id id;
152 	size_t datalen;
153 
154 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
155 	if (datalen != sizeof(iobj))
156 		return got_error(GOT_ERR_PRIVSEP_LEN);
157 	memcpy(&iobj, imsg->data, sizeof(iobj));
158 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
159 
160 	err = open_commit(&commit, pack, packidx, iobj.idx, &id, objcache);
161 	if (err)
162 		goto done;
163 
164 	err = got_privsep_send_commit(ibuf, commit);
165 done:
166 	if (commit)
167 		got_object_commit_close(commit);
168 	if (err) {
169 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
170 			err = NULL;
171 		else
172 			got_privsep_send_error(ibuf, err);
173 	}
174 
175 	return err;
176 }
177 
178 static const struct got_error *
179 open_tree(uint8_t **buf, struct got_pathlist_head *entries, int *nentries,
180     struct got_pack *pack, struct got_packidx *packidx, int obj_idx,
181     struct got_object_id *id, struct got_object_cache *objcache)
182 {
183 	const struct got_error *err = NULL;
184 	struct got_object *obj = NULL;
185 	size_t len;
186 
187 	*buf = NULL;
188 	*nentries = 0;
189 
190 	obj = got_object_cache_get(objcache, id);
191 	if (obj) {
192 		obj->refcnt++;
193 	} else {
194 		err = open_object(&obj, pack, packidx, obj_idx, id,
195 		    objcache);
196 		if (err)
197 			return err;
198 	}
199 
200 	err = got_packfile_extract_object_to_mem(buf, &len, obj, pack);
201 	if (err)
202 		goto done;
203 
204 	obj->size = len;
205 
206 	err = got_object_parse_tree(entries, nentries, *buf, len);
207 done:
208 	got_object_close(obj);
209 	if (err) {
210 		free(*buf);
211 		*buf = NULL;
212 	}
213 	return err;
214 }
215 
216 static const struct got_error *
217 tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
218     struct got_packidx *packidx, struct got_object_cache *objcache)
219 {
220 	const struct got_error *err = NULL;
221 	struct got_imsg_packed_object iobj;
222 	struct got_pathlist_head entries;
223 	int nentries = 0;
224 	uint8_t *buf = NULL;
225 	struct got_object_id id;
226 	size_t datalen;
227 
228 	TAILQ_INIT(&entries);
229 
230 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
231 	if (datalen != sizeof(iobj))
232 		return got_error(GOT_ERR_PRIVSEP_LEN);
233 	memcpy(&iobj, imsg->data, sizeof(iobj));
234 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
235 
236 	err = open_tree(&buf, &entries, &nentries, pack, packidx, iobj.idx,
237 	    &id, objcache);
238 	if (err)
239 		return err;
240 
241 	err = got_privsep_send_tree(ibuf, &entries, nentries);
242 	got_object_parsed_tree_entries_free(&entries);
243 	free(buf);
244 	if (err) {
245 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
246 			err = NULL;
247 		else
248 			got_privsep_send_error(ibuf, err);
249 	}
250 
251 	return err;
252 }
253 
254 static const struct got_error *
255 receive_file(FILE **f, struct imsgbuf *ibuf, uint32_t imsg_code)
256 {
257 	const struct got_error *err;
258 	struct imsg imsg;
259 	size_t datalen;
260 
261 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
262 	if (err)
263 		return err;
264 
265 	if (imsg.hdr.type != imsg_code) {
266 		err = got_error(GOT_ERR_PRIVSEP_MSG);
267 		goto done;
268 	}
269 
270 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
271 	if (datalen != 0) {
272 		err = got_error(GOT_ERR_PRIVSEP_LEN);
273 		goto done;
274 	}
275 	if (imsg.fd == -1) {
276 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
277 		goto done;
278 	}
279 
280 	*f = fdopen(imsg.fd, "w+");
281 	if (*f == NULL) {
282 		err = got_error_from_errno("fdopen");
283 		close(imsg.fd);
284 		goto done;
285 	}
286 done:
287 	imsg_free(&imsg);
288 	return err;
289 }
290 
291 static const struct got_error *
292 receive_tempfile(FILE **f, const char *mode, struct imsg *imsg,
293     struct imsgbuf *ibuf)
294 {
295 	size_t datalen;
296 
297 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
298 	if (datalen != 0)
299 		return got_error(GOT_ERR_PRIVSEP_LEN);
300 
301 	if (imsg->fd == -1)
302 		return got_error(GOT_ERR_PRIVSEP_NO_FD);
303 
304 	*f = fdopen(imsg->fd, mode);
305 	if (*f == NULL)
306 		return got_error_from_errno("fdopen");
307 	imsg->fd = -1;
308 
309 	return NULL;
310 }
311 
312 static const struct got_error *
313 blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
314     struct got_packidx *packidx, struct got_object_cache *objcache,
315     FILE *basefile, FILE *accumfile)
316 {
317 	const struct got_error *err = NULL;
318 	struct got_imsg_packed_object iobj;
319 	struct got_object *obj = NULL;
320 	FILE *outfile = NULL;
321 	struct got_object_id id;
322 	size_t datalen;
323 	uint64_t blob_size;
324 	uint8_t *buf = NULL;
325 
326 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
327 	if (datalen != sizeof(iobj))
328 		return got_error(GOT_ERR_PRIVSEP_LEN);
329 	memcpy(&iobj, imsg->data, sizeof(iobj));
330 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
331 
332 	obj = got_object_cache_get(objcache, &id);
333 	if (obj) {
334 		obj->refcnt++;
335 	} else {
336 		err = open_object(&obj, pack, packidx, iobj.idx, &id,
337 		    objcache);
338 		if (err)
339 			return err;
340 	}
341 
342 	err = receive_file(&outfile, ibuf, GOT_IMSG_BLOB_OUTFD);
343 	if (err)
344 		goto done;
345 
346 	if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
347 		err = got_pack_get_max_delta_object_size(&blob_size, obj, pack);
348 		if (err)
349 			goto done;
350 	} else
351 		blob_size = obj->size;
352 
353 	if (blob_size <= GOT_PRIVSEP_INLINE_BLOB_DATA_MAX)
354 		err = got_packfile_extract_object_to_mem(&buf, &obj->size,
355 		    obj, pack);
356 	else
357 		err = got_packfile_extract_object(pack, obj, outfile, basefile,
358 		    accumfile);
359 	if (err)
360 		goto done;
361 
362 	err = got_privsep_send_blob(ibuf, obj->size, obj->hdrlen, buf);
363 done:
364 	free(buf);
365 	if (outfile && fclose(outfile) == EOF && err == NULL)
366 		err = got_error_from_errno("fclose");
367 	got_object_close(obj);
368 	if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
369 		got_privsep_send_error(ibuf, err);
370 
371 	return err;
372 }
373 
374 static const struct got_error *
375 tag_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
376     struct got_packidx *packidx, struct got_object_cache *objcache)
377 {
378 	const struct got_error *err = NULL;
379 	struct got_imsg_packed_object iobj;
380 	struct got_object *obj = NULL;
381 	struct got_tag_object *tag = NULL;
382 	uint8_t *buf = NULL;
383 	size_t len;
384 	struct got_object_id id;
385 	size_t datalen;
386 
387 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
388 	if (datalen != sizeof(iobj))
389 		return got_error(GOT_ERR_PRIVSEP_LEN);
390 	memcpy(&iobj, imsg->data, sizeof(iobj));
391 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
392 
393 	obj = got_object_cache_get(objcache, &id);
394 	if (obj) {
395 		obj->refcnt++;
396 	} else {
397 		err = open_object(&obj, pack, packidx, iobj.idx, &id,
398 		    objcache);
399 		if (err)
400 			return err;
401 	}
402 
403 	err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
404 	if (err)
405 		goto done;
406 
407 	obj->size = len;
408 	err = got_object_parse_tag(&tag, buf, len);
409 	if (err)
410 		goto done;
411 
412 	err = got_privsep_send_tag(ibuf, tag);
413 done:
414 	free(buf);
415 	got_object_close(obj);
416 	if (tag)
417 		got_object_tag_close(tag);
418 	if (err) {
419 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
420 			err = NULL;
421 		else
422 			got_privsep_send_error(ibuf, err);
423 	}
424 
425 	return err;
426 }
427 
428 static struct got_parsed_tree_entry *
429 find_entry_by_name(struct got_pathlist_head *entries, int nentries,
430     const char *name, size_t len)
431 {
432 	struct got_pathlist_entry *pe;
433 
434 	/* Note that tree entries are sorted in strncmp() order. */
435 	TAILQ_FOREACH(pe, entries, entry) {
436 		int cmp = strncmp(pe->path, name, len);
437 		if (cmp < 0)
438 			continue;
439 		if (cmp > 0)
440 			break;
441 		if (pe->path[len] == '\0')
442 			return (struct got_parsed_tree_entry *)pe->data;
443 	}
444 	return NULL;
445 }
446 
447 static const struct got_error *
448 tree_path_changed(int *changed, uint8_t **buf1, uint8_t **buf2,
449     struct got_pathlist_head *entries1, int *nentries1,
450     struct got_pathlist_head *entries2, int *nentries2,
451     const char *path, struct got_pack *pack, struct got_packidx *packidx,
452     struct imsgbuf *ibuf, struct got_object_cache *objcache)
453 {
454 	const struct got_error *err = NULL;
455 	struct got_parsed_tree_entry *pte1 = NULL, *pte2 = NULL;
456 	const char *seg, *s;
457 	size_t seglen;
458 
459 	*changed = 0;
460 
461 	/* We not do support comparing the root path. */
462 	if (got_path_is_root_dir(path))
463 		return got_error_path(path, GOT_ERR_BAD_PATH);
464 
465 	s = path;
466 	while (*s == '/')
467 		s++;
468 	seg = s;
469 	seglen = 0;
470 	while (*s) {
471 		if (*s != '/') {
472 			s++;
473 			seglen++;
474 			if (*s)
475 				continue;
476 		}
477 
478 		pte1 = find_entry_by_name(entries1, *nentries1, seg, seglen);
479 		if (pte1 == NULL) {
480 			err = got_error(GOT_ERR_NO_OBJ);
481 			break;
482 		}
483 
484 		pte2 = find_entry_by_name(entries2, *nentries2, seg, seglen);
485 		if (pte2 == NULL) {
486 			*changed = 1;
487 			break;
488 		}
489 
490 		if (pte1->mode != pte2->mode) {
491 			*changed = 1;
492 			break;
493 		}
494 
495 		if (memcmp(pte1->id, pte2->id, SHA1_DIGEST_LENGTH) == 0) {
496 			*changed = 0;
497 			break;
498 		}
499 
500 		if (*s == '\0') { /* final path element */
501 			*changed = 1;
502 			break;
503 		}
504 
505 		seg = s + 1;
506 		s++;
507 		seglen = 0;
508 		if (*s) {
509 			struct got_object_id id1, id2;
510 			int idx;
511 
512 			memcpy(id1.sha1, pte1->id, SHA1_DIGEST_LENGTH);
513 			idx = got_packidx_get_object_idx(packidx, &id1);
514 			if (idx == -1) {
515 				err = got_error_no_obj(&id1);
516 				break;
517 			}
518 			got_object_parsed_tree_entries_free(entries1);
519 			*nentries1 = 0;
520 			free(*buf1);
521 			*buf1 = NULL;
522 			err = open_tree(buf1, entries1, nentries1, pack,
523 			    packidx, idx, &id1, objcache);
524 			pte1 = NULL;
525 			if (err)
526 				break;
527 
528 			memcpy(id2.sha1, pte2->id, SHA1_DIGEST_LENGTH);
529 			idx = got_packidx_get_object_idx(packidx, &id2);
530 			if (idx == -1) {
531 				err = got_error_no_obj(&id2);
532 				break;
533 			}
534 			got_object_parsed_tree_entries_free(entries2);
535 			*nentries2 = 0;
536 			free(*buf2);
537 			*buf2 = NULL;
538 			err = open_tree(buf2, entries2, nentries2, pack,
539 			    packidx, idx, &id2, objcache);
540 			pte2 = NULL;
541 			if (err)
542 				break;
543 		}
544 	}
545 
546 	return err;
547 }
548 
549 static const struct got_error *
550 send_traversed_commits(struct got_object_id *commit_ids, size_t ncommits,
551     struct imsgbuf *ibuf)
552 {
553 	const struct got_error *err;
554 	struct ibuf *wbuf;
555 	size_t i;
556 
557 	wbuf = imsg_create(ibuf, GOT_IMSG_TRAVERSED_COMMITS, 0, 0,
558 	    sizeof(struct got_imsg_traversed_commits) +
559 	    ncommits * SHA1_DIGEST_LENGTH);
560 	if (wbuf == NULL)
561 		return got_error_from_errno("imsg_create TRAVERSED_COMMITS");
562 
563 	if (imsg_add(wbuf, &ncommits, sizeof(ncommits)) == -1) {
564 		err = got_error_from_errno("imsg_add TRAVERSED_COMMITS");
565 		ibuf_free(wbuf);
566 		return err;
567 	}
568 	for (i = 0; i < ncommits; i++) {
569 		struct got_object_id *id = &commit_ids[i];
570 		if (imsg_add(wbuf, id->sha1, SHA1_DIGEST_LENGTH) == -1) {
571 			err = got_error_from_errno(
572 			    "imsg_add TRAVERSED_COMMITS");
573 			ibuf_free(wbuf);
574 			return err;
575 		}
576 	}
577 
578 	wbuf->fd = -1;
579 	imsg_close(ibuf, wbuf);
580 
581 	return got_privsep_flush_imsg(ibuf);
582 }
583 
584 static const struct got_error *
585 send_commit_traversal_done(struct imsgbuf *ibuf)
586 {
587 	if (imsg_compose(ibuf, GOT_IMSG_COMMIT_TRAVERSAL_DONE, 0, 0, -1,
588 	    NULL, 0) == -1)
589 		return got_error_from_errno("imsg_compose TRAVERSAL_DONE");
590 
591 	return got_privsep_flush_imsg(ibuf);
592 }
593 
594 
595 static const struct got_error *
596 commit_traversal_request(struct imsg *imsg, struct imsgbuf *ibuf,
597     struct got_pack *pack, struct got_packidx *packidx,
598     struct got_object_cache *objcache)
599 {
600 	const struct got_error *err = NULL;
601 	struct got_imsg_packed_object iobj;
602 	struct got_object_qid *pid;
603 	struct got_commit_object *commit = NULL, *pcommit = NULL;
604 	struct got_pathlist_head entries, pentries;
605 	int nentries = 0, pnentries = 0;
606 	struct got_object_id id;
607 	size_t datalen, path_len;
608 	char *path = NULL;
609 	const int min_alloc = 64;
610 	int changed = 0, ncommits = 0, nallocated = 0;
611 	struct got_object_id *commit_ids = NULL;
612 
613 	TAILQ_INIT(&entries);
614 	TAILQ_INIT(&pentries);
615 
616 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
617 	if (datalen < sizeof(iobj))
618 		return got_error(GOT_ERR_PRIVSEP_LEN);
619 	memcpy(&iobj, imsg->data, sizeof(iobj));
620 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
621 
622 	path_len = datalen - sizeof(iobj) - 1;
623 	if (path_len < 0)
624 		return got_error(GOT_ERR_PRIVSEP_LEN);
625 	if (path_len > 0) {
626 		path = imsg->data + sizeof(iobj);
627 		if (path[path_len] != '\0')
628 			return got_error(GOT_ERR_PRIVSEP_LEN);
629 	}
630 
631 	nallocated = min_alloc;
632 	commit_ids = reallocarray(NULL, nallocated, sizeof(*commit_ids));
633 	if (commit_ids == NULL)
634 		return got_error_from_errno("reallocarray");
635 
636 	do {
637 		const size_t max_datalen = MAX_IMSGSIZE - IMSG_HEADER_SIZE;
638 		int idx;
639 
640 		if (sigint_received) {
641 			err = got_error(GOT_ERR_CANCELLED);
642 			goto done;
643 		}
644 
645 		if (commit == NULL) {
646 			idx = got_packidx_get_object_idx(packidx, &id);
647 			if (idx == -1)
648 				break;
649 			err = open_commit(&commit, pack, packidx,
650 			    idx, &id, objcache);
651 			if (err) {
652 				if (err->code != GOT_ERR_NO_OBJ)
653 					goto done;
654 				err = NULL;
655 				break;
656 			}
657 		}
658 
659 		if (sizeof(struct got_imsg_traversed_commits) +
660 		    ncommits * SHA1_DIGEST_LENGTH >= max_datalen) {
661 			err = send_traversed_commits(commit_ids, ncommits,
662 			    ibuf);
663 			if (err)
664 				goto done;
665 			ncommits = 0;
666 		}
667 		ncommits++;
668 		if (ncommits > nallocated) {
669 			struct got_object_id *new;
670 			nallocated += min_alloc;
671 			new = reallocarray(commit_ids, nallocated,
672 			    sizeof(*commit_ids));
673 			if (new == NULL) {
674 				err = got_error_from_errno("reallocarray");
675 				goto done;
676 			}
677 			commit_ids = new;
678 		}
679 		memcpy(commit_ids[ncommits - 1].sha1, id.sha1,
680 		    SHA1_DIGEST_LENGTH);
681 
682 		pid = STAILQ_FIRST(&commit->parent_ids);
683 		if (pid == NULL)
684 			break;
685 
686 		idx = got_packidx_get_object_idx(packidx, pid->id);
687 		if (idx == -1)
688 			break;
689 
690 		err = open_commit(&pcommit, pack, packidx, idx, pid->id,
691 		    objcache);
692 		if (err) {
693 			if (err->code != GOT_ERR_NO_OBJ)
694 				goto done;
695 			err = NULL;
696 			break;
697 		}
698 
699 		if (path[0] == '/' && path[1] == '\0') {
700 			if (got_object_id_cmp(pcommit->tree_id,
701 			    commit->tree_id) != 0) {
702 				changed = 1;
703 				break;
704 			}
705 		} else {
706 			int pidx;
707 			uint8_t *buf = NULL, *pbuf = NULL;
708 
709 			idx = got_packidx_get_object_idx(packidx,
710 			    commit->tree_id);
711 			if (idx == -1)
712 				break;
713 			pidx = got_packidx_get_object_idx(packidx,
714 			    pcommit->tree_id);
715 			if (pidx == -1)
716 				break;
717 
718 			err = open_tree(&buf, &entries, &nentries, pack,
719 			    packidx, idx, commit->tree_id, objcache);
720 			if (err)
721 				goto done;
722 			err = open_tree(&pbuf, &pentries, &pnentries, pack,
723 			    packidx, pidx, pcommit->tree_id, objcache);
724 			if (err) {
725 				free(buf);
726 				goto done;
727 			}
728 
729 			err = tree_path_changed(&changed, &buf, &pbuf,
730 			    &entries, &nentries, &pentries, &pnentries, path,
731 			    pack, packidx, ibuf, objcache);
732 
733 			got_object_parsed_tree_entries_free(&entries);
734 			nentries = 0;
735 			free(buf);
736 			got_object_parsed_tree_entries_free(&pentries);
737 			pnentries = 0;
738 			free(pbuf);
739 			if (err) {
740 				if (err->code != GOT_ERR_NO_OBJ)
741 					goto done;
742 				err = NULL;
743 				break;
744 			}
745 		}
746 
747 		if (!changed) {
748 			memcpy(id.sha1, pid->id->sha1, SHA1_DIGEST_LENGTH);
749 			got_object_commit_close(commit);
750 			commit = pcommit;
751 			pcommit = NULL;
752 		}
753 	} while (!changed);
754 
755 	if (ncommits > 0) {
756 		err = send_traversed_commits(commit_ids, ncommits, ibuf);
757 		if (err)
758 			goto done;
759 
760 		if (changed) {
761 			err = got_privsep_send_commit(ibuf, commit);
762 			if (err)
763 				goto done;
764 		}
765 	}
766 	err = send_commit_traversal_done(ibuf);
767 done:
768 	free(commit_ids);
769 	if (commit)
770 		got_object_commit_close(commit);
771 	if (pcommit)
772 		got_object_commit_close(pcommit);
773 	if (nentries != 0)
774 		got_object_parsed_tree_entries_free(&entries);
775 	if (pnentries != 0)
776 		got_object_parsed_tree_entries_free(&pentries);
777 	if (err) {
778 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
779 			err = NULL;
780 		else
781 			got_privsep_send_error(ibuf, err);
782 	}
783 
784 	return err;
785 }
786 
787 static const struct got_error *
788 raw_object_request(struct imsg *imsg, struct imsgbuf *ibuf,
789     struct got_pack *pack, struct got_packidx *packidx,
790     struct got_object_cache *objcache, FILE *basefile, FILE *accumfile)
791 {
792 	const struct got_error *err = NULL;
793 	uint8_t *buf = NULL;
794 	uint64_t size = 0;
795 	FILE *outfile = NULL;
796 	struct got_imsg_packed_object iobj;
797 	struct got_object *obj;
798 	struct got_object_id id;
799 	size_t datalen;
800 
801 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
802 	if (datalen != sizeof(iobj))
803 		return got_error(GOT_ERR_PRIVSEP_LEN);
804 	memcpy(&iobj, imsg->data, sizeof(iobj));
805 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
806 
807 	obj = got_object_cache_get(objcache, &id);
808 	if (obj) {
809 		obj->refcnt++;
810 	} else {
811 		err = open_object(&obj, pack, packidx, iobj.idx, &id,
812 		    objcache);
813 		if (err)
814 			return err;
815 	}
816 
817 	err = receive_file(&outfile, ibuf, GOT_IMSG_RAW_OBJECT_OUTFD);
818 	if (err)
819 		return err;
820 
821 	if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
822 		err = got_pack_get_max_delta_object_size(&size, obj, pack);
823 		if (err)
824 			goto done;
825 	} else
826 		size = obj->size;
827 
828 	if (size <= GOT_PRIVSEP_INLINE_OBJECT_DATA_MAX)
829 		err = got_packfile_extract_object_to_mem(&buf, &obj->size,
830 		    obj, pack);
831 	else
832 		err = got_packfile_extract_object(pack, obj, outfile, basefile,
833 		    accumfile);
834 	if (err)
835 		goto done;
836 
837 	err = got_privsep_send_raw_obj(ibuf, obj->size, obj->hdrlen, buf);
838 done:
839 	free(buf);
840 	if (outfile && fclose(outfile) == EOF && err == NULL)
841 		err = got_error_from_errno("fclose");
842 	got_object_close(obj);
843 	if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
844 		got_privsep_send_error(ibuf, err);
845 
846 	return err;
847 }
848 
849 static const struct got_error *
850 get_base_object_id(struct got_object_id *base_id, struct got_packidx *packidx,
851     off_t base_offset)
852 {
853 	const struct got_error *err;
854 	int idx;
855 
856 	err = got_packidx_get_offset_idx(&idx, packidx, base_offset); 
857 	if (err)
858 		return err;
859 	if (idx == -1)
860 		return got_error(GOT_ERR_BAD_PACKIDX);
861 
862 	return got_packidx_get_object_id(base_id, packidx, idx);
863 }
864 
865 static const struct got_error *
866 raw_delta_request(struct imsg *imsg, struct imsgbuf *ibuf,
867     FILE *delta_outfile, struct got_pack *pack,
868     struct got_packidx *packidx)
869 {
870 	const struct got_error *err = NULL;
871 	struct got_imsg_raw_delta_request req;
872 	size_t datalen, delta_size;
873 	off_t delta_offset;
874 	uint8_t *delta_buf = NULL;
875 	struct got_object_id id, base_id;
876 	off_t base_offset, delta_out_offset = 0;
877 	uint64_t base_size = 0, result_size = 0;
878 	size_t w;
879 
880 	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
881 	if (datalen != sizeof(req))
882 		return got_error(GOT_ERR_PRIVSEP_LEN);
883 	memcpy(&req, imsg->data, sizeof(req));
884 	memcpy(id.sha1, req.id, SHA1_DIGEST_LENGTH);
885 
886 	imsg->fd = -1;
887 
888 	err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
889 	    &delta_offset, &base_offset, &base_id, &base_size, &result_size,
890 	    pack, packidx, req.idx);
891 	if (err)
892 		goto done;
893 
894 	/*
895 	 * If this is an offset delta we must determine the base
896 	 * object ID ourselves.
897 	 */
898 	if (base_offset != 0) {
899 		err = get_base_object_id(&base_id, packidx, base_offset);
900 		if (err)
901 			goto done;
902 	}
903 
904 	delta_out_offset = ftello(delta_outfile);
905 	w = fwrite(delta_buf, 1, delta_size, delta_outfile);
906 	if (w != delta_size) {
907 		err = got_ferror(delta_outfile, GOT_ERR_IO);
908 		goto done;
909 	}
910 	if (fflush(delta_outfile) == -1) {
911 		err = got_error_from_errno("fflush");
912 		goto done;
913 	}
914 
915 	err = got_privsep_send_raw_delta(ibuf, base_size, result_size,
916 	    delta_size, delta_offset, delta_out_offset, &base_id);
917 done:
918 	free(delta_buf);
919 	return err;
920 }
921 
922 static const struct got_error *
923 receive_packidx(struct got_packidx **packidx, struct imsgbuf *ibuf)
924 {
925 	const struct got_error *err = NULL;
926 	struct imsg imsg;
927 	struct got_imsg_packidx ipackidx;
928 	size_t datalen;
929 	struct got_packidx *p;
930 
931 	*packidx = NULL;
932 
933 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
934 	if (err)
935 		return err;
936 
937 	p = calloc(1, sizeof(*p));
938 	if (p == NULL) {
939 		err = got_error_from_errno("calloc");
940 		goto done;
941 	}
942 
943 	if (imsg.hdr.type != GOT_IMSG_PACKIDX) {
944 		err = got_error(GOT_ERR_PRIVSEP_MSG);
945 		goto done;
946 	}
947 
948 	if (imsg.fd == -1) {
949 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
950 		goto done;
951 	}
952 
953 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
954 	if (datalen != sizeof(ipackidx)) {
955 		err = got_error(GOT_ERR_PRIVSEP_LEN);
956 		goto done;
957 	}
958 	memcpy(&ipackidx, imsg.data, sizeof(ipackidx));
959 
960 	p->len = ipackidx.len;
961 	p->fd = dup(imsg.fd);
962 	if (p->fd == -1) {
963 		err = got_error_from_errno("dup");
964 		goto done;
965 	}
966 	if (lseek(p->fd, 0, SEEK_SET) == -1) {
967 		err = got_error_from_errno("lseek");
968 		goto done;
969 	}
970 
971 #ifndef GOT_PACK_NO_MMAP
972 	p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0);
973 	if (p->map == MAP_FAILED)
974 		p->map = NULL; /* fall back to read(2) */
975 #endif
976 	err = got_packidx_init_hdr(p, 1, ipackidx.packfile_size);
977 done:
978 	if (err) {
979 		if (imsg.fd != -1)
980 			close(imsg.fd);
981 		got_packidx_close(p);
982 	} else
983 		*packidx = p;
984 	imsg_free(&imsg);
985 	return err;
986 }
987 
988 static const struct got_error *
989 receive_pack(struct got_pack **packp, struct imsgbuf *ibuf)
990 {
991 	const struct got_error *err = NULL;
992 	struct imsg imsg;
993 	struct got_imsg_pack ipack;
994 	size_t datalen;
995 	struct got_pack *pack;
996 
997 	*packp = NULL;
998 
999 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1000 	if (err)
1001 		return err;
1002 
1003 	pack = calloc(1, sizeof(*pack));
1004 	if (pack == NULL) {
1005 		err = got_error_from_errno("calloc");
1006 		goto done;
1007 	}
1008 
1009 	if (imsg.hdr.type != GOT_IMSG_PACK) {
1010 		err = got_error(GOT_ERR_PRIVSEP_MSG);
1011 		goto done;
1012 	}
1013 
1014 	if (imsg.fd == -1) {
1015 		err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1016 		goto done;
1017 	}
1018 
1019 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1020 	if (datalen != sizeof(ipack)) {
1021 		err = got_error(GOT_ERR_PRIVSEP_LEN);
1022 		goto done;
1023 	}
1024 	memcpy(&ipack, imsg.data, sizeof(ipack));
1025 
1026 	pack->filesize = ipack.filesize;
1027 	pack->fd = dup(imsg.fd);
1028 	if (pack->fd == -1) {
1029 		err = got_error_from_errno("dup");
1030 		goto done;
1031 	}
1032 	if (lseek(pack->fd, 0, SEEK_SET) == -1) {
1033 		err = got_error_from_errno("lseek");
1034 		goto done;
1035 	}
1036 	pack->path_packfile = strdup(ipack.path_packfile);
1037 	if (pack->path_packfile == NULL) {
1038 		err = got_error_from_errno("strdup");
1039 		goto done;
1040 	}
1041 
1042 	pack->delta_cache = got_delta_cache_alloc(100,
1043 	    GOT_DELTA_RESULT_SIZE_CACHED_MAX);
1044 	if (pack->delta_cache == NULL) {
1045 		err = got_error_from_errno("got_delta_cache_alloc");
1046 		goto done;
1047 	}
1048 
1049 #ifndef GOT_PACK_NO_MMAP
1050 	pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
1051 	    pack->fd, 0);
1052 	if (pack->map == MAP_FAILED)
1053 		pack->map = NULL; /* fall back to read(2) */
1054 #endif
1055 done:
1056 	if (err) {
1057 		if (imsg.fd != -1)
1058 			close(imsg.fd);
1059 		free(pack);
1060 	} else
1061 		*packp = pack;
1062 	imsg_free(&imsg);
1063 	return err;
1064 }
1065 
1066 int
1067 main(int argc, char *argv[])
1068 {
1069 	const struct got_error *err = NULL;
1070 	struct imsgbuf ibuf;
1071 	struct imsg imsg;
1072 	struct got_packidx *packidx = NULL;
1073 	struct got_pack *pack = NULL;
1074 	struct got_object_cache objcache;
1075 	FILE *basefile = NULL, *accumfile = NULL, *delta_outfile = NULL;
1076 
1077 	//static int attached;
1078 	//while (!attached) sleep(1);
1079 
1080 	signal(SIGINT, catch_sigint);
1081 
1082 	imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
1083 
1084 	err = got_object_cache_init(&objcache, GOT_OBJECT_CACHE_TYPE_OBJ);
1085 	if (err) {
1086 		err = got_error_from_errno("got_object_cache_init");
1087 		got_privsep_send_error(&ibuf, err);
1088 		return 1;
1089 	}
1090 
1091 #ifndef PROFILE
1092 	/* revoke access to most system calls */
1093 	if (pledge("stdio recvfd", NULL) == -1) {
1094 		err = got_error_from_errno("pledge");
1095 		got_privsep_send_error(&ibuf, err);
1096 		return 1;
1097 	}
1098 #endif
1099 
1100 	err = receive_packidx(&packidx, &ibuf);
1101 	if (err) {
1102 		got_privsep_send_error(&ibuf, err);
1103 		return 1;
1104 	}
1105 
1106 	err = receive_pack(&pack, &ibuf);
1107 	if (err) {
1108 		got_privsep_send_error(&ibuf, err);
1109 		return 1;
1110 	}
1111 
1112 	for (;;) {
1113 		imsg.fd = -1;
1114 
1115 		if (sigint_received) {
1116 			err = got_error(GOT_ERR_CANCELLED);
1117 			break;
1118 		}
1119 
1120 		err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1121 		if (err) {
1122 			if (err->code == GOT_ERR_PRIVSEP_PIPE)
1123 				err = NULL;
1124 			break;
1125 		}
1126 
1127 		if (imsg.hdr.type == GOT_IMSG_STOP)
1128 			break;
1129 
1130 		switch (imsg.hdr.type) {
1131 		case GOT_IMSG_TMPFD:
1132 			if (basefile == NULL) {
1133 				err = receive_tempfile(&basefile, "w+",
1134 				   &imsg, &ibuf);
1135 			} else if (accumfile == NULL) {
1136 				err = receive_tempfile(&accumfile, "w+",
1137 				   &imsg, &ibuf);
1138 			} else
1139 				err = got_error(GOT_ERR_PRIVSEP_MSG);
1140 			break;
1141 		case GOT_IMSG_PACKED_OBJECT_REQUEST:
1142 			err = object_request(&imsg, &ibuf, pack, packidx,
1143 			    &objcache);
1144 			break;
1145 		case GOT_IMSG_PACKED_RAW_OBJECT_REQUEST:
1146 			if (basefile == NULL || accumfile == NULL) {
1147 				err = got_error(GOT_ERR_PRIVSEP_MSG);
1148 				break;
1149 			}
1150 			err = raw_object_request(&imsg, &ibuf, pack, packidx,
1151 			    &objcache, basefile, accumfile);
1152 			break;
1153 		case GOT_IMSG_RAW_DELTA_OUTFD:
1154 			if (delta_outfile != NULL) {
1155 				err = got_error(GOT_ERR_PRIVSEP_MSG);
1156 				break;
1157 			}
1158 			err = receive_tempfile(&delta_outfile, "w",
1159 			    &imsg, &ibuf);
1160 			break;
1161 		case GOT_IMSG_RAW_DELTA_REQUEST:
1162 			if (delta_outfile == NULL) {
1163 				err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1164 				break;
1165 			}
1166 			err = raw_delta_request(&imsg, &ibuf, delta_outfile,
1167 			    pack, packidx);
1168 			break;
1169 		case GOT_IMSG_COMMIT_REQUEST:
1170 			err = commit_request(&imsg, &ibuf, pack, packidx,
1171 			    &objcache);
1172 			break;
1173 		case GOT_IMSG_TREE_REQUEST:
1174 			err = tree_request(&imsg, &ibuf, pack, packidx,
1175 			    &objcache);
1176 			break;
1177 		case GOT_IMSG_BLOB_REQUEST:
1178 			if (basefile == NULL || accumfile == NULL) {
1179 				err = got_error(GOT_ERR_PRIVSEP_MSG);
1180 				break;
1181 			}
1182 			err = blob_request(&imsg, &ibuf, pack, packidx,
1183 			    &objcache, basefile, accumfile);
1184 			break;
1185 		case GOT_IMSG_TAG_REQUEST:
1186 			err = tag_request(&imsg, &ibuf, pack, packidx,
1187 			    &objcache);
1188 			break;
1189 		case GOT_IMSG_COMMIT_TRAVERSAL_REQUEST:
1190 			err = commit_traversal_request(&imsg, &ibuf, pack,
1191 			    packidx, &objcache);
1192 			break;
1193 		default:
1194 			err = got_error(GOT_ERR_PRIVSEP_MSG);
1195 			break;
1196 		}
1197 
1198 		if (imsg.fd != -1 && close(imsg.fd) == -1 && err == NULL)
1199 			err = got_error_from_errno("close");
1200 		imsg_free(&imsg);
1201 		if (err)
1202 			break;
1203 	}
1204 
1205 	if (packidx)
1206 		got_packidx_close(packidx);
1207 	if (pack)
1208 		got_pack_close(pack);
1209 	got_object_cache_close(&objcache);
1210 	imsg_clear(&ibuf);
1211 	if (basefile && fclose(basefile) == EOF && err == NULL)
1212 		err = got_error_from_errno("fclose");
1213 	if (accumfile && fclose(accumfile) == EOF && err == NULL)
1214 		err = got_error_from_errno("fclose");
1215 	if (delta_outfile && fclose(delta_outfile) == EOF && err == NULL)
1216 		err = got_error_from_errno("fclose");
1217 	if (err) {
1218 		if (!sigint_received && err->code != GOT_ERR_PRIVSEP_PIPE) {
1219 			fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
1220 			got_privsep_send_error(&ibuf, err);
1221 		}
1222 	}
1223 	if (close(GOT_IMSG_FD_CHILD) == -1 && err == NULL)
1224 		err = got_error_from_errno("close");
1225 	return err ? 1 : 0;
1226 }