Blob


1 /*
2 * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
3 *
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 */
17 #include <sys/stat.h>
18 #include <sys/types.h>
19 #include <sys/queue.h>
20 #include <sys/uio.h>
21 #include <sys/time.h>
22 #include <sys/mman.h>
24 #include <inttypes.h>
25 #include <limits.h>
26 #include <signal.h>
27 #include <stdint.h>
28 #include <imsg.h>
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <sha1.h>
33 #include <unistd.h>
34 #include <zlib.h>
36 #include "got_error.h"
37 #include "got_object.h"
38 #include "got_path.h"
40 #include "got_lib_delta.h"
41 #include "got_lib_delta_cache.h"
42 #include "got_lib_object.h"
43 #include "got_lib_object_cache.h"
44 #include "got_lib_object_parse.h"
45 #include "got_lib_object_idset.h"
46 #include "got_lib_privsep.h"
47 #include "got_lib_pack.h"
49 #ifndef nitems
50 #define nitems(_a) (sizeof((_a)) / sizeof((_a)[0]))
51 #endif
53 static volatile sig_atomic_t sigint_received;
55 static void
56 catch_sigint(int signo)
57 {
58 sigint_received = 1;
59 }
61 static const struct got_error *
62 open_object(struct got_object **obj, struct got_pack *pack,
63 struct got_packidx *packidx, int idx, struct got_object_id *id,
64 struct got_object_cache *objcache)
65 {
66 const struct got_error *err;
68 err = got_packfile_open_object(obj, pack, packidx, idx, id);
69 if (err)
70 return err;
71 (*obj)->refcnt++;
73 err = got_object_cache_add(objcache, id, *obj);
74 if (err) {
75 if (err->code == GOT_ERR_OBJ_EXISTS ||
76 err->code == GOT_ERR_OBJ_TOO_LARGE)
77 err = NULL;
78 return err;
79 }
80 (*obj)->refcnt++;
81 return NULL;
82 }
84 static const struct got_error *
85 object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
86 struct got_packidx *packidx, struct got_object_cache *objcache)
87 {
88 const struct got_error *err = NULL;
89 struct got_imsg_packed_object iobj;
90 struct got_object *obj;
91 struct got_object_id id;
92 size_t datalen;
94 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
95 if (datalen != sizeof(iobj))
96 return got_error(GOT_ERR_PRIVSEP_LEN);
97 memcpy(&iobj, imsg->data, sizeof(iobj));
98 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
100 obj = got_object_cache_get(objcache, &id);
101 if (obj) {
102 obj->refcnt++;
103 } else {
104 err = open_object(&obj, pack, packidx, iobj.idx, &id,
105 objcache);
106 if (err)
107 goto done;
110 err = got_privsep_send_obj(ibuf, obj);
111 done:
112 got_object_close(obj);
113 return err;
116 static const struct got_error *
117 open_commit(struct got_commit_object **commit, struct got_pack *pack,
118 struct got_packidx *packidx, int obj_idx, struct got_object_id *id,
119 struct got_object_cache *objcache)
121 const struct got_error *err = NULL;
122 struct got_object *obj = NULL;
123 uint8_t *buf = NULL;
124 size_t len;
126 *commit = NULL;
128 obj = got_object_cache_get(objcache, id);
129 if (obj) {
130 obj->refcnt++;
131 } else {
132 err = open_object(&obj, pack, packidx, obj_idx, id,
133 objcache);
134 if (err)
135 return err;
138 err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
139 if (err)
140 goto done;
142 obj->size = len;
144 err = got_object_parse_commit(commit, buf, len);
145 done:
146 got_object_close(obj);
147 free(buf);
148 return err;
151 static const struct got_error *
152 commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
153 struct got_packidx *packidx, struct got_object_cache *objcache)
155 const struct got_error *err = NULL;
156 struct got_imsg_packed_object iobj;
157 struct got_commit_object *commit = NULL;
158 struct got_object_id id;
159 size_t datalen;
161 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
162 if (datalen != sizeof(iobj))
163 return got_error(GOT_ERR_PRIVSEP_LEN);
164 memcpy(&iobj, imsg->data, sizeof(iobj));
165 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
167 err = open_commit(&commit, pack, packidx, iobj.idx, &id, objcache);
168 if (err)
169 goto done;
171 err = got_privsep_send_commit(ibuf, commit);
172 done:
173 if (commit)
174 got_object_commit_close(commit);
175 if (err) {
176 if (err->code == GOT_ERR_PRIVSEP_PIPE)
177 err = NULL;
178 else
179 got_privsep_send_error(ibuf, err);
182 return err;
185 static const struct got_error *
186 open_tree(uint8_t **buf, struct got_parsed_tree_entry **entries, size_t *nentries,
187 size_t *nentries_alloc, struct got_pack *pack, struct got_packidx *packidx,
188 int obj_idx, struct got_object_id *id, struct got_object_cache *objcache)
190 const struct got_error *err = NULL;
191 struct got_object *obj = NULL;
192 size_t len;
194 *buf = NULL;
195 *nentries = 0;
197 obj = got_object_cache_get(objcache, id);
198 if (obj) {
199 obj->refcnt++;
200 } else {
201 err = open_object(&obj, pack, packidx, obj_idx, id,
202 objcache);
203 if (err)
204 return err;
207 err = got_packfile_extract_object_to_mem(buf, &len, obj, pack);
208 if (err)
209 goto done;
211 obj->size = len;
213 err = got_object_parse_tree(entries, nentries, nentries_alloc,
214 *buf, len);
215 done:
216 got_object_close(obj);
217 if (err) {
218 free(*buf);
219 *buf = NULL;
221 return err;
224 static const struct got_error *
225 tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
226 struct got_packidx *packidx, struct got_object_cache *objcache,
227 struct got_parsed_tree_entry **entries, size_t *nentries,
228 size_t *nentries_alloc)
230 const struct got_error *err = NULL;
231 struct got_imsg_packed_object iobj;
232 uint8_t *buf = NULL;
233 struct got_object_id id;
234 size_t datalen;
236 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
237 if (datalen != sizeof(iobj))
238 return got_error(GOT_ERR_PRIVSEP_LEN);
239 memcpy(&iobj, imsg->data, sizeof(iobj));
240 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
242 err = open_tree(&buf, entries, nentries, nentries_alloc,
243 pack, packidx, iobj.idx, &id, objcache);
244 if (err)
245 return err;
247 err = got_privsep_send_tree(ibuf, *entries, *nentries);
248 free(buf);
249 if (err) {
250 if (err->code == GOT_ERR_PRIVSEP_PIPE)
251 err = NULL;
252 else
253 got_privsep_send_error(ibuf, err);
256 return err;
259 static const struct got_error *
260 receive_file(FILE **f, struct imsgbuf *ibuf, uint32_t imsg_code)
262 const struct got_error *err;
263 struct imsg imsg;
264 size_t datalen;
266 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
267 if (err)
268 return err;
270 if (imsg.hdr.type != imsg_code) {
271 err = got_error(GOT_ERR_PRIVSEP_MSG);
272 goto done;
275 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
276 if (datalen != 0) {
277 err = got_error(GOT_ERR_PRIVSEP_LEN);
278 goto done;
280 if (imsg.fd == -1) {
281 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
282 goto done;
285 *f = fdopen(imsg.fd, "w+");
286 if (*f == NULL) {
287 err = got_error_from_errno("fdopen");
288 close(imsg.fd);
289 goto done;
291 done:
292 imsg_free(&imsg);
293 return err;
296 static const struct got_error *
297 receive_tempfile(FILE **f, const char *mode, struct imsg *imsg,
298 struct imsgbuf *ibuf)
300 size_t datalen;
302 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
303 if (datalen != 0)
304 return got_error(GOT_ERR_PRIVSEP_LEN);
306 if (imsg->fd == -1)
307 return got_error(GOT_ERR_PRIVSEP_NO_FD);
309 *f = fdopen(imsg->fd, mode);
310 if (*f == NULL)
311 return got_error_from_errno("fdopen");
312 imsg->fd = -1;
314 return NULL;
317 static const struct got_error *
318 blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
319 struct got_packidx *packidx, struct got_object_cache *objcache,
320 FILE *basefile, FILE *accumfile)
322 const struct got_error *err = NULL;
323 struct got_imsg_packed_object iobj;
324 struct got_object *obj = NULL;
325 FILE *outfile = NULL;
326 struct got_object_id id;
327 size_t datalen;
328 uint64_t blob_size;
329 uint8_t *buf = NULL;
331 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
332 if (datalen != sizeof(iobj))
333 return got_error(GOT_ERR_PRIVSEP_LEN);
334 memcpy(&iobj, imsg->data, sizeof(iobj));
335 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
337 obj = got_object_cache_get(objcache, &id);
338 if (obj) {
339 obj->refcnt++;
340 } else {
341 err = open_object(&obj, pack, packidx, iobj.idx, &id,
342 objcache);
343 if (err)
344 return err;
347 err = receive_file(&outfile, ibuf, GOT_IMSG_BLOB_OUTFD);
348 if (err)
349 goto done;
351 if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
352 err = got_pack_get_max_delta_object_size(&blob_size, obj, pack);
353 if (err)
354 goto done;
355 } else
356 blob_size = obj->size;
358 if (blob_size <= GOT_PRIVSEP_INLINE_BLOB_DATA_MAX)
359 err = got_packfile_extract_object_to_mem(&buf, &obj->size,
360 obj, pack);
361 else
362 err = got_packfile_extract_object(pack, obj, outfile, basefile,
363 accumfile);
364 if (err)
365 goto done;
367 err = got_privsep_send_blob(ibuf, obj->size, obj->hdrlen, buf);
368 done:
369 free(buf);
370 if (outfile && fclose(outfile) == EOF && err == NULL)
371 err = got_error_from_errno("fclose");
372 got_object_close(obj);
373 if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
374 got_privsep_send_error(ibuf, err);
376 return err;
379 static const struct got_error *
380 tag_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
381 struct got_packidx *packidx, struct got_object_cache *objcache)
383 const struct got_error *err = NULL;
384 struct got_imsg_packed_object iobj;
385 struct got_object *obj = NULL;
386 struct got_tag_object *tag = NULL;
387 uint8_t *buf = NULL;
388 size_t len;
389 struct got_object_id id;
390 size_t datalen;
392 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
393 if (datalen != sizeof(iobj))
394 return got_error(GOT_ERR_PRIVSEP_LEN);
395 memcpy(&iobj, imsg->data, sizeof(iobj));
396 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
398 obj = got_object_cache_get(objcache, &id);
399 if (obj) {
400 obj->refcnt++;
401 } else {
402 err = open_object(&obj, pack, packidx, iobj.idx, &id,
403 objcache);
404 if (err)
405 return err;
408 err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
409 if (err)
410 goto done;
412 obj->size = len;
413 err = got_object_parse_tag(&tag, buf, len);
414 if (err)
415 goto done;
417 err = got_privsep_send_tag(ibuf, tag);
418 done:
419 free(buf);
420 got_object_close(obj);
421 if (tag)
422 got_object_tag_close(tag);
423 if (err) {
424 if (err->code == GOT_ERR_PRIVSEP_PIPE)
425 err = NULL;
426 else
427 got_privsep_send_error(ibuf, err);
430 return err;
433 static struct got_parsed_tree_entry *
434 find_entry_by_name(struct got_parsed_tree_entry *entries, int nentries,
435 const char *name, size_t len)
437 struct got_parsed_tree_entry *pte;
438 int cmp, i;
440 /* Note that tree entries are sorted in strncmp() order. */
441 for (i = 0; i < nentries; i++) {
442 pte = &entries[i];
443 cmp = strncmp(pte->name, name, len);
444 if (cmp < 0)
445 continue;
446 if (cmp > 0)
447 break;
448 if (pte->name[len] == '\0')
449 return pte;
451 return NULL;
454 static const struct got_error *
455 tree_path_changed(int *changed, uint8_t **buf1, uint8_t **buf2,
456 struct got_parsed_tree_entry **entries1, size_t *nentries1,
457 size_t *nentries_alloc1,
458 struct got_parsed_tree_entry **entries2, size_t *nentries2,
459 size_t *nentries_alloc2,
460 const char *path, struct got_pack *pack, struct got_packidx *packidx,
461 struct imsgbuf *ibuf, struct got_object_cache *objcache)
463 const struct got_error *err = NULL;
464 struct got_parsed_tree_entry *pte1 = NULL, *pte2 = NULL;
465 const char *seg, *s;
466 size_t seglen;
468 *changed = 0;
470 /* We not do support comparing the root path. */
471 if (got_path_is_root_dir(path))
472 return got_error_path(path, GOT_ERR_BAD_PATH);
474 s = path;
475 while (*s == '/')
476 s++;
477 seg = s;
478 seglen = 0;
479 while (*s) {
480 if (*s != '/') {
481 s++;
482 seglen++;
483 if (*s)
484 continue;
487 pte1 = find_entry_by_name(*entries1, *nentries1, seg, seglen);
488 if (pte1 == NULL) {
489 err = got_error(GOT_ERR_NO_OBJ);
490 break;
493 pte2 = find_entry_by_name(*entries2, *nentries2, seg, seglen);
494 if (pte2 == NULL) {
495 *changed = 1;
496 break;
499 if (pte1->mode != pte2->mode) {
500 *changed = 1;
501 break;
504 if (memcmp(pte1->id, pte2->id, SHA1_DIGEST_LENGTH) == 0) {
505 *changed = 0;
506 break;
509 if (*s == '\0') { /* final path element */
510 *changed = 1;
511 break;
514 seg = s + 1;
515 s++;
516 seglen = 0;
517 if (*s) {
518 struct got_object_id id1, id2;
519 int idx;
521 memcpy(id1.sha1, pte1->id, SHA1_DIGEST_LENGTH);
522 idx = got_packidx_get_object_idx(packidx, &id1);
523 if (idx == -1) {
524 err = got_error_no_obj(&id1);
525 break;
527 *nentries1 = 0;
528 free(*buf1);
529 *buf1 = NULL;
530 err = open_tree(buf1, entries1, nentries1,
531 nentries_alloc1, pack, packidx, idx, &id1,
532 objcache);
533 pte1 = NULL;
534 if (err)
535 break;
537 memcpy(id2.sha1, pte2->id, SHA1_DIGEST_LENGTH);
538 idx = got_packidx_get_object_idx(packidx, &id2);
539 if (idx == -1) {
540 err = got_error_no_obj(&id2);
541 break;
543 *nentries2 = 0;
544 free(*buf2);
545 *buf2 = NULL;
546 err = open_tree(buf2, entries2, nentries2,
547 nentries_alloc2, pack, packidx, idx, &id2, objcache);
548 pte2 = NULL;
549 if (err)
550 break;
554 return err;
557 static const struct got_error *
558 send_traversed_commits(struct got_object_id *commit_ids, size_t ncommits,
559 struct imsgbuf *ibuf)
561 struct ibuf *wbuf;
562 size_t i;
564 wbuf = imsg_create(ibuf, GOT_IMSG_TRAVERSED_COMMITS, 0, 0,
565 sizeof(struct got_imsg_traversed_commits) +
566 ncommits * SHA1_DIGEST_LENGTH);
567 if (wbuf == NULL)
568 return got_error_from_errno("imsg_create TRAVERSED_COMMITS");
570 if (imsg_add(wbuf, &ncommits, sizeof(ncommits)) == -1)
571 return got_error_from_errno("imsg_add TRAVERSED_COMMITS");
573 for (i = 0; i < ncommits; i++) {
574 struct got_object_id *id = &commit_ids[i];
575 if (imsg_add(wbuf, id->sha1, SHA1_DIGEST_LENGTH) == -1) {
576 return got_error_from_errno(
577 "imsg_add TRAVERSED_COMMITS");
581 wbuf->fd = -1;
582 imsg_close(ibuf, wbuf);
584 return got_privsep_flush_imsg(ibuf);
587 static const struct got_error *
588 send_commit_traversal_done(struct imsgbuf *ibuf)
590 if (imsg_compose(ibuf, GOT_IMSG_COMMIT_TRAVERSAL_DONE, 0, 0, -1,
591 NULL, 0) == -1)
592 return got_error_from_errno("imsg_compose TRAVERSAL_DONE");
594 return got_privsep_flush_imsg(ibuf);
597 static const struct got_error *
598 commit_traversal_request(struct imsg *imsg, struct imsgbuf *ibuf,
599 struct got_pack *pack, struct got_packidx *packidx,
600 struct got_object_cache *objcache)
602 const struct got_error *err = NULL;
603 struct got_imsg_packed_object iobj;
604 struct got_object_qid *pid;
605 struct got_commit_object *commit = NULL, *pcommit = NULL;
606 struct got_parsed_tree_entry *entries = NULL, *pentries = NULL;
607 size_t nentries = 0, nentries_alloc = 0;
608 size_t pnentries = 0, pnentries_alloc = 0;
609 struct got_object_id id;
610 size_t datalen, path_len;
611 char *path = NULL;
612 const int min_alloc = 64;
613 int changed = 0, ncommits = 0, nallocated = 0;
614 struct got_object_id *commit_ids = NULL;
616 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
617 if (datalen < sizeof(iobj))
618 return got_error(GOT_ERR_PRIVSEP_LEN);
619 memcpy(&iobj, imsg->data, sizeof(iobj));
620 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
622 path_len = datalen - sizeof(iobj) - 1;
623 if (path_len < 0)
624 return got_error(GOT_ERR_PRIVSEP_LEN);
625 if (path_len > 0) {
626 path = imsg->data + sizeof(iobj);
627 if (path[path_len] != '\0')
628 return got_error(GOT_ERR_PRIVSEP_LEN);
631 nallocated = min_alloc;
632 commit_ids = reallocarray(NULL, nallocated, sizeof(*commit_ids));
633 if (commit_ids == NULL)
634 return got_error_from_errno("reallocarray");
636 do {
637 const size_t max_datalen = MAX_IMSGSIZE - IMSG_HEADER_SIZE;
638 int idx;
640 if (sigint_received) {
641 err = got_error(GOT_ERR_CANCELLED);
642 goto done;
645 if (commit == NULL) {
646 idx = got_packidx_get_object_idx(packidx, &id);
647 if (idx == -1)
648 break;
649 err = open_commit(&commit, pack, packidx,
650 idx, &id, objcache);
651 if (err) {
652 if (err->code != GOT_ERR_NO_OBJ)
653 goto done;
654 err = NULL;
655 break;
659 if (sizeof(struct got_imsg_traversed_commits) +
660 ncommits * SHA1_DIGEST_LENGTH >= max_datalen) {
661 err = send_traversed_commits(commit_ids, ncommits,
662 ibuf);
663 if (err)
664 goto done;
665 ncommits = 0;
667 ncommits++;
668 if (ncommits > nallocated) {
669 struct got_object_id *new;
670 nallocated += min_alloc;
671 new = reallocarray(commit_ids, nallocated,
672 sizeof(*commit_ids));
673 if (new == NULL) {
674 err = got_error_from_errno("reallocarray");
675 goto done;
677 commit_ids = new;
679 memcpy(commit_ids[ncommits - 1].sha1, id.sha1,
680 SHA1_DIGEST_LENGTH);
682 pid = STAILQ_FIRST(&commit->parent_ids);
683 if (pid == NULL)
684 break;
686 idx = got_packidx_get_object_idx(packidx, &pid->id);
687 if (idx == -1)
688 break;
690 err = open_commit(&pcommit, pack, packidx, idx, &pid->id,
691 objcache);
692 if (err) {
693 if (err->code != GOT_ERR_NO_OBJ)
694 goto done;
695 err = NULL;
696 break;
699 if (path[0] == '/' && path[1] == '\0') {
700 if (got_object_id_cmp(pcommit->tree_id,
701 commit->tree_id) != 0) {
702 changed = 1;
703 break;
705 } else {
706 int pidx;
707 uint8_t *buf = NULL, *pbuf = NULL;
709 idx = got_packidx_get_object_idx(packidx,
710 commit->tree_id);
711 if (idx == -1)
712 break;
713 pidx = got_packidx_get_object_idx(packidx,
714 pcommit->tree_id);
715 if (pidx == -1)
716 break;
718 err = open_tree(&buf, &entries, &nentries,
719 &nentries_alloc, pack, packidx, idx,
720 commit->tree_id, objcache);
721 if (err)
722 goto done;
723 err = open_tree(&pbuf, &pentries, &pnentries,
724 &pnentries_alloc, pack, packidx, pidx,
725 pcommit->tree_id, objcache);
726 if (err) {
727 free(buf);
728 goto done;
731 err = tree_path_changed(&changed, &buf, &pbuf,
732 &entries, &nentries, &nentries_alloc,
733 &pentries, &pnentries, &pnentries_alloc,
734 path, pack, packidx, ibuf, objcache);
736 nentries = 0;
737 free(buf);
738 pnentries = 0;
739 free(pbuf);
740 if (err) {
741 if (err->code != GOT_ERR_NO_OBJ)
742 goto done;
743 err = NULL;
744 break;
748 if (!changed) {
749 memcpy(id.sha1, pid->id.sha1, SHA1_DIGEST_LENGTH);
750 got_object_commit_close(commit);
751 commit = pcommit;
752 pcommit = NULL;
754 } while (!changed);
756 if (ncommits > 0) {
757 err = send_traversed_commits(commit_ids, ncommits, ibuf);
758 if (err)
759 goto done;
761 if (changed) {
762 err = got_privsep_send_commit(ibuf, commit);
763 if (err)
764 goto done;
767 err = send_commit_traversal_done(ibuf);
768 done:
769 free(commit_ids);
770 if (commit)
771 got_object_commit_close(commit);
772 if (pcommit)
773 got_object_commit_close(pcommit);
774 free(entries);
775 free(pentries);
776 if (err) {
777 if (err->code == GOT_ERR_PRIVSEP_PIPE)
778 err = NULL;
779 else
780 got_privsep_send_error(ibuf, err);
783 return err;
786 static const struct got_error *
787 raw_object_request(struct imsg *imsg, struct imsgbuf *ibuf,
788 struct got_pack *pack, struct got_packidx *packidx,
789 struct got_object_cache *objcache, FILE *basefile, FILE *accumfile)
791 const struct got_error *err = NULL;
792 uint8_t *buf = NULL;
793 uint64_t size = 0;
794 FILE *outfile = NULL;
795 struct got_imsg_packed_object iobj;
796 struct got_object *obj;
797 struct got_object_id id;
798 size_t datalen;
800 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
801 if (datalen != sizeof(iobj))
802 return got_error(GOT_ERR_PRIVSEP_LEN);
803 memcpy(&iobj, imsg->data, sizeof(iobj));
804 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
806 obj = got_object_cache_get(objcache, &id);
807 if (obj) {
808 obj->refcnt++;
809 } else {
810 err = open_object(&obj, pack, packidx, iobj.idx, &id,
811 objcache);
812 if (err)
813 return err;
816 err = receive_file(&outfile, ibuf, GOT_IMSG_RAW_OBJECT_OUTFD);
817 if (err)
818 return err;
820 if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
821 err = got_pack_get_max_delta_object_size(&size, obj, pack);
822 if (err)
823 goto done;
824 } else
825 size = obj->size;
827 if (size <= GOT_PRIVSEP_INLINE_OBJECT_DATA_MAX)
828 err = got_packfile_extract_object_to_mem(&buf, &obj->size,
829 obj, pack);
830 else
831 err = got_packfile_extract_object(pack, obj, outfile, basefile,
832 accumfile);
833 if (err)
834 goto done;
836 err = got_privsep_send_raw_obj(ibuf, obj->size, obj->hdrlen, buf);
837 done:
838 free(buf);
839 if (outfile && fclose(outfile) == EOF && err == NULL)
840 err = got_error_from_errno("fclose");
841 got_object_close(obj);
842 if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
843 got_privsep_send_error(ibuf, err);
845 return err;
848 static const struct got_error *
849 get_base_object_id(struct got_object_id *base_id, struct got_packidx *packidx,
850 off_t base_offset)
852 const struct got_error *err;
853 int idx;
855 err = got_packidx_get_offset_idx(&idx, packidx, base_offset);
856 if (err)
857 return err;
858 if (idx == -1)
859 return got_error(GOT_ERR_BAD_PACKIDX);
861 return got_packidx_get_object_id(base_id, packidx, idx);
864 static const struct got_error *
865 raw_delta_request(struct imsg *imsg, struct imsgbuf *ibuf,
866 FILE *delta_outfile, struct got_pack *pack,
867 struct got_packidx *packidx)
869 const struct got_error *err = NULL;
870 struct got_imsg_raw_delta_request req;
871 size_t datalen, delta_size, delta_compressed_size;
872 off_t delta_offset;
873 uint8_t *delta_buf = NULL;
874 struct got_object_id id, base_id;
875 off_t base_offset, delta_out_offset = 0;
876 uint64_t base_size = 0, result_size = 0;
877 size_t w;
879 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
880 if (datalen != sizeof(req))
881 return got_error(GOT_ERR_PRIVSEP_LEN);
882 memcpy(&req, imsg->data, sizeof(req));
883 memcpy(id.sha1, req.id, SHA1_DIGEST_LENGTH);
885 imsg->fd = -1;
887 err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
888 &delta_compressed_size, &delta_offset, &base_offset, &base_id,
889 &base_size, &result_size, pack, packidx, req.idx);
890 if (err)
891 goto done;
893 /*
894 * If this is an offset delta we must determine the base
895 * object ID ourselves.
896 */
897 if (base_offset != 0) {
898 err = get_base_object_id(&base_id, packidx, base_offset);
899 if (err)
900 goto done;
903 delta_out_offset = ftello(delta_outfile);
904 w = fwrite(delta_buf, 1, delta_compressed_size, delta_outfile);
905 if (w != delta_compressed_size) {
906 err = got_ferror(delta_outfile, GOT_ERR_IO);
907 goto done;
909 if (fflush(delta_outfile) == -1) {
910 err = got_error_from_errno("fflush");
911 goto done;
914 err = got_privsep_send_raw_delta(ibuf, base_size, result_size,
915 delta_size, delta_compressed_size, delta_offset, delta_out_offset,
916 &base_id);
917 done:
918 free(delta_buf);
919 return err;
922 struct search_deltas_arg {
923 struct imsgbuf *ibuf;
924 struct got_packidx *packidx;
925 struct got_pack *pack;
926 struct got_object_idset *idset;
927 FILE *delta_outfile;
928 struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS];
929 size_t ndeltas;
930 };
932 static const struct got_error *
933 search_delta_for_object(struct got_object_id *id, void *data, void *arg)
935 const struct got_error *err;
936 struct search_deltas_arg *a = arg;
937 int obj_idx;
938 uint8_t *delta_buf = NULL;
939 uint64_t base_size, result_size;
940 size_t delta_size, delta_compressed_size;
941 off_t delta_offset, base_offset;
942 struct got_object_id base_id;
944 if (sigint_received)
945 return got_error(GOT_ERR_CANCELLED);
947 obj_idx = got_packidx_get_object_idx(a->packidx, id);
948 if (obj_idx == -1)
949 return NULL; /* object not present in our pack file */
951 err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
952 &delta_compressed_size, &delta_offset, &base_offset, &base_id,
953 &base_size, &result_size, a->pack, a->packidx, obj_idx);
954 if (err) {
955 if (err->code == GOT_ERR_OBJ_TYPE)
956 return NULL; /* object not stored as a delta */
957 return err;
960 /*
961 * If this is an offset delta we must determine the base
962 * object ID ourselves.
963 */
964 if (base_offset != 0) {
965 err = get_base_object_id(&base_id, a->packidx, base_offset);
966 if (err)
967 goto done;
970 if (got_object_idset_contains(a->idset, &base_id)) {
971 struct got_imsg_reused_delta *delta;
972 off_t delta_out_offset = ftello(a->delta_outfile);
973 size_t w;
975 w = fwrite(delta_buf, 1, delta_compressed_size,
976 a->delta_outfile);
977 if (w != delta_compressed_size) {
978 err = got_ferror(a->delta_outfile, GOT_ERR_IO);
979 goto done;
982 delta = &a->deltas[a->ndeltas++];
983 memcpy(&delta->id, id, sizeof(delta->id));
984 memcpy(&delta->base_id, &base_id, sizeof(delta->base_id));
985 delta->base_size = base_size;
986 delta->result_size = result_size;
987 delta->delta_size = delta_size;
988 delta->delta_compressed_size = delta_compressed_size;
989 delta->delta_offset = delta_offset;
990 delta->delta_out_offset = delta_out_offset;
992 if (a->ndeltas >= GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS) {
993 err = got_privsep_send_reused_deltas(a->ibuf,
994 a->deltas, a->ndeltas);
995 if (err)
996 goto done;
997 a->ndeltas = 0;
1000 done:
1001 free(delta_buf);
1002 return err;
1005 static const struct got_error *
1006 recv_object_ids(struct got_object_idset *idset, struct imsgbuf *ibuf)
1008 const struct got_error *err = NULL;
1009 int done = 0;
1010 struct got_object_id *ids;
1011 size_t nids, i;
1013 for (;;) {
1014 err = got_privsep_recv_object_idlist(&done, &ids, &nids, ibuf);
1015 if (err || done)
1016 break;
1017 for (i = 0; i < nids; i++) {
1018 err = got_object_idset_add(idset, &ids[i], NULL);
1019 if (err) {
1020 free(ids);
1021 return err;
1024 free(ids);
1027 return err;
1030 static const struct got_error *
1031 recv_object_id_queue(struct got_object_id_queue *queue, struct imsgbuf *ibuf)
1033 const struct got_error *err = NULL;
1034 int done = 0;
1035 struct got_object_qid *qid;
1036 struct got_object_id *ids;
1037 size_t nids, i;
1039 for (;;) {
1040 err = got_privsep_recv_object_idlist(&done, &ids, &nids, ibuf);
1041 if (err || done)
1042 break;
1043 for (i = 0; i < nids; i++) {
1044 err = got_object_qid_alloc_partial(&qid);
1045 if (err)
1046 return err;
1047 memcpy(&qid->id, &ids[i], sizeof(qid->id));
1048 STAILQ_INSERT_TAIL(queue, qid, entry);
1052 return err;
1055 static const struct got_error *
1056 delta_reuse_request(struct imsg *imsg, struct imsgbuf *ibuf,
1057 FILE *delta_outfile, struct got_pack *pack, struct got_packidx *packidx)
1059 const struct got_error *err = NULL;
1060 struct got_object_idset *idset;
1061 struct search_deltas_arg sda;
1063 idset = got_object_idset_alloc();
1064 if (idset == NULL)
1065 return got_error_from_errno("got_object_idset_alloc");
1067 err = recv_object_ids(idset, ibuf);
1068 if (err)
1069 return err;
1071 memset(&sda, 0, sizeof(sda));
1072 sda.ibuf = ibuf;
1073 sda.idset = idset;
1074 sda.pack = pack;
1075 sda.packidx = packidx;
1076 sda.delta_outfile = delta_outfile;
1077 err = got_object_idset_for_each(idset, search_delta_for_object, &sda);
1078 if (err)
1079 goto done;
1081 if (sda.ndeltas > 0) {
1082 err = got_privsep_send_reused_deltas(ibuf, sda.deltas,
1083 sda.ndeltas);
1084 if (err)
1085 goto done;
1088 if (fflush(delta_outfile) == -1) {
1089 err = got_error_from_errno("fflush");
1090 goto done;
1093 err = got_privsep_send_reused_deltas_done(ibuf);
1094 done:
1095 got_object_idset_free(idset);
1096 return err;
1099 static const struct got_error *
1100 receive_packidx(struct got_packidx **packidx, struct imsgbuf *ibuf)
1102 const struct got_error *err = NULL;
1103 struct imsg imsg;
1104 struct got_imsg_packidx ipackidx;
1105 size_t datalen;
1106 struct got_packidx *p;
1108 *packidx = NULL;
1110 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1111 if (err)
1112 return err;
1114 p = calloc(1, sizeof(*p));
1115 if (p == NULL) {
1116 err = got_error_from_errno("calloc");
1117 goto done;
1120 if (imsg.hdr.type != GOT_IMSG_PACKIDX) {
1121 err = got_error(GOT_ERR_PRIVSEP_MSG);
1122 goto done;
1125 if (imsg.fd == -1) {
1126 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1127 goto done;
1130 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1131 if (datalen != sizeof(ipackidx)) {
1132 err = got_error(GOT_ERR_PRIVSEP_LEN);
1133 goto done;
1135 memcpy(&ipackidx, imsg.data, sizeof(ipackidx));
1137 p->len = ipackidx.len;
1138 p->fd = dup(imsg.fd);
1139 if (p->fd == -1) {
1140 err = got_error_from_errno("dup");
1141 goto done;
1143 if (lseek(p->fd, 0, SEEK_SET) == -1) {
1144 err = got_error_from_errno("lseek");
1145 goto done;
1148 #ifndef GOT_PACK_NO_MMAP
1149 if (p->len > 0 && p->len <= SIZE_MAX) {
1150 p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0);
1151 if (p->map == MAP_FAILED)
1152 p->map = NULL; /* fall back to read(2) */
1154 #endif
1155 err = got_packidx_init_hdr(p, 1, ipackidx.packfile_size);
1156 done:
1157 if (err) {
1158 if (imsg.fd != -1)
1159 close(imsg.fd);
1160 got_packidx_close(p);
1161 } else
1162 *packidx = p;
1163 imsg_free(&imsg);
1164 return err;
1167 static const struct got_error *
1168 send_tree_enumeration_done(struct imsgbuf *ibuf)
1170 if (imsg_compose(ibuf, GOT_IMSG_TREE_ENUMERATION_DONE, 0, 0, -1,
1171 NULL, 0) == -1)
1172 return got_error_from_errno("imsg_compose TREE_ENUMERATION_DONE");
1174 return got_privsep_flush_imsg(ibuf);
1177 struct enumerated_tree {
1178 struct got_object_id id;
1179 char *path;
1180 uint8_t *buf;
1181 struct got_parsed_tree_entry *entries;
1182 int nentries;
1185 static const struct got_error *
1186 enumerate_tree(int *have_all_entries, struct imsgbuf *ibuf, size_t *totlen,
1187 struct got_object_id *tree_id,
1188 const char *path, struct got_pack *pack, struct got_packidx *packidx,
1189 struct got_object_cache *objcache, struct got_object_idset *idset,
1190 struct enumerated_tree **trees, size_t *nalloc, size_t *ntrees)
1192 const struct got_error *err = NULL;
1193 struct got_object_id_queue ids;
1194 struct got_object_qid *qid;
1195 uint8_t *buf = NULL;
1196 struct got_parsed_tree_entry *entries = NULL;
1197 size_t nentries = 0, nentries_alloc = 0, i;
1198 struct enumerated_tree *tree;
1200 *ntrees = 0;
1201 *have_all_entries = 1;
1202 STAILQ_INIT(&ids);
1204 err = got_object_qid_alloc_partial(&qid);
1205 if (err)
1206 return err;
1207 memcpy(&qid->id.sha1, tree_id, SHA1_DIGEST_LENGTH);
1208 qid->data = strdup(path);
1209 if (qid->data == NULL) {
1210 err = got_error_from_errno("strdup");
1211 goto done;
1213 STAILQ_INSERT_TAIL(&ids, qid, entry);
1214 qid = NULL;
1216 /* Traverse the tree hierarchy, gather tree object IDs and paths. */
1217 do {
1218 const char *path;
1219 int idx, i;
1221 if (sigint_received) {
1222 err = got_error(GOT_ERR_CANCELLED);
1223 goto done;
1226 qid = STAILQ_FIRST(&ids);
1227 STAILQ_REMOVE_HEAD(&ids, entry);
1228 path = qid->data;
1230 idx = got_packidx_get_object_idx(packidx, &qid->id);
1231 if (idx == -1) {
1232 *have_all_entries = 0;
1233 break;
1236 err = open_tree(&buf, &entries, &nentries, &nentries_alloc,
1237 pack, packidx, idx, &qid->id, objcache);
1238 if (err) {
1239 if (err->code != GOT_ERR_NO_OBJ)
1240 goto done;
1243 err = got_object_idset_add(idset, &qid->id, NULL);
1244 if (err)
1245 goto done;
1247 for (i = 0; i < nentries; i++) {
1248 struct got_object_qid *eqid = NULL;
1249 struct got_parsed_tree_entry *pte = &entries[i];
1250 char *p;
1252 if (!S_ISDIR(pte->mode))
1253 continue;
1255 err = got_object_qid_alloc_partial(&eqid);
1256 if (err)
1257 goto done;
1258 memcpy(eqid->id.sha1, pte->id, sizeof(eqid->id.sha1));
1260 if (got_object_idset_contains(idset, &eqid->id)) {
1261 got_object_qid_free(eqid);
1262 continue;
1265 if (asprintf(&p, "%s%s%s", path,
1266 got_path_is_root_dir(path) ? "" : "/",
1267 pte->name) == -1) {
1268 err = got_error_from_errno("asprintf");
1269 got_object_qid_free(eqid);
1270 goto done;
1272 eqid->data = p;
1273 STAILQ_INSERT_TAIL(&ids, eqid, entry);
1276 if (*ntrees >= *nalloc) {
1277 struct enumerated_tree *new;
1278 new = recallocarray(*trees, *nalloc, *nalloc + 16,
1279 sizeof(*new));
1280 if (new == NULL) {
1281 err = got_error_from_errno("malloc");
1282 goto done;
1284 *trees = new;
1285 *nalloc += 16;
1287 tree = &(*trees)[*ntrees];
1288 (*ntrees)++;
1289 memcpy(&tree->id, &qid->id, sizeof(tree->id));
1290 tree->path = qid->data;
1291 tree->buf = buf;
1292 buf = NULL;
1293 tree->entries = entries;
1294 entries = NULL;
1295 nentries_alloc = 0;
1296 tree->nentries = nentries;
1297 nentries = 0;
1299 got_object_qid_free(qid);
1300 qid = NULL;
1301 } while (!STAILQ_EMPTY(&ids));
1303 if (*have_all_entries) {
1304 int i;
1306 * We have managed to traverse all entries in the hierarchy.
1307 * Tell the main process what we have found.
1309 for (i = 0; i < *ntrees; i++) {
1310 tree = &(*trees)[i];
1311 err = got_privsep_send_enumerated_tree(totlen,
1312 ibuf, &tree->id, tree->path, tree->entries,
1313 tree->nentries);
1314 if (err)
1315 goto done;
1316 free(tree->buf);
1317 tree->buf = NULL;
1318 free(tree->path);
1319 tree->path = NULL;
1320 free(tree->entries);
1321 tree->entries = NULL;
1323 *ntrees = 0; /* don't loop again below to free memory */
1325 err = send_tree_enumeration_done(ibuf);
1326 } else {
1328 * We can only load fully packed tree hierarchies on
1329 * behalf of the main process, otherwise the main process
1330 * gets a wrong idea about which tree objects have
1331 * already been traversed.
1332 * Indicate a missing entry for the root of this tree.
1333 * The main process should continue by loading this
1334 * entire tree the slow way.
1336 err = got_privsep_send_enumerated_tree(totlen, ibuf,
1337 tree_id, "/", NULL, -1);
1338 if (err)
1339 goto done;
1341 done:
1342 free(buf);
1343 free(entries);
1344 for (i = 0; i < *ntrees; i++) {
1345 tree = &(*trees)[i];
1346 free(tree->buf);
1347 tree->buf = NULL;
1348 free(tree->path);
1349 tree->path = NULL;
1350 free(tree->entries);
1351 tree->entries = NULL;
1353 if (qid)
1354 free(qid->data);
1355 got_object_qid_free(qid);
1356 got_object_id_queue_free(&ids);
1357 if (err) {
1358 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1359 err = NULL;
1360 else
1361 got_privsep_send_error(ibuf, err);
1364 return err;
1367 static const struct got_error *
1368 enumeration_request(struct imsg *imsg, struct imsgbuf *ibuf,
1369 struct got_pack *pack, struct got_packidx *packidx,
1370 struct got_object_cache *objcache)
1372 const struct got_error *err = NULL;
1373 struct got_object_id_queue commit_ids;
1374 const struct got_object_id_queue *parents = NULL;
1375 struct got_object_qid *qid = NULL;
1376 struct got_object *obj = NULL;
1377 struct got_commit_object *commit = NULL;
1378 struct got_object_id *tree_id = NULL;
1379 size_t totlen = 0;
1380 struct got_object_idset *idset;
1381 int i, idx, have_all_entries = 1;
1382 struct enumerated_tree *trees = NULL;
1383 size_t ntrees = 0, nalloc = 16;
1385 STAILQ_INIT(&commit_ids);
1387 trees = calloc(nalloc, sizeof(*trees));
1388 if (trees == NULL)
1389 return got_error_from_errno("calloc");
1391 idset = got_object_idset_alloc();
1392 if (idset == NULL) {
1393 err = got_error_from_errno("got_object_idset_alloc");
1394 goto done;
1397 err = recv_object_id_queue(&commit_ids, ibuf);
1398 if (err)
1399 goto done;
1401 if (STAILQ_EMPTY(&commit_ids)) {
1402 err = got_error(GOT_ERR_PRIVSEP_MSG);
1403 goto done;
1406 err = recv_object_ids(idset, ibuf);
1407 if (err)
1408 goto done;
1410 while (!STAILQ_EMPTY(&commit_ids)) {
1411 if (sigint_received) {
1412 err = got_error(GOT_ERR_CANCELLED);
1413 goto done;
1416 qid = STAILQ_FIRST(&commit_ids);
1417 STAILQ_REMOVE_HEAD(&commit_ids, entry);
1419 if (got_object_idset_contains(idset, &qid->id)) {
1420 got_object_qid_free(qid);
1421 qid = NULL;
1422 continue;
1425 idx = got_packidx_get_object_idx(packidx, &qid->id);
1426 if (idx == -1) {
1427 have_all_entries = 0;
1428 break;
1431 err = open_object(&obj, pack, packidx, idx, &qid->id,
1432 objcache);
1433 if (err)
1434 goto done;
1435 if (obj->type == GOT_OBJ_TYPE_TAG) {
1436 struct got_tag_object *tag;
1437 uint8_t *buf;
1438 size_t len;
1439 err = got_packfile_extract_object_to_mem(&buf,
1440 &len, obj, pack);
1441 if (err)
1442 goto done;
1443 obj->size = len;
1444 err = got_object_parse_tag(&tag, buf, len);
1445 if (err) {
1446 free(buf);
1447 goto done;
1449 idx = got_packidx_get_object_idx(packidx, &tag->id);
1450 if (idx == -1) {
1451 have_all_entries = 0;
1452 break;
1454 err = open_commit(&commit, pack, packidx, idx,
1455 &tag->id, objcache);
1456 got_object_tag_close(tag);
1457 free(buf);
1458 if (err)
1459 goto done;
1460 } else if (obj->type == GOT_OBJ_TYPE_COMMIT) {
1461 err = open_commit(&commit, pack, packidx, idx,
1462 &qid->id, objcache);
1463 if (err)
1464 goto done;
1465 } else {
1466 err = got_error(GOT_ERR_OBJ_TYPE);
1467 goto done;
1469 got_object_close(obj);
1470 obj = NULL;
1472 err = got_privsep_send_enumerated_commit(ibuf, &qid->id,
1473 got_object_commit_get_committer_time(commit));
1474 if (err)
1475 goto done;
1477 tree_id = got_object_commit_get_tree_id(commit);
1478 idx = got_packidx_get_object_idx(packidx, tree_id);
1479 if (idx == -1) {
1480 have_all_entries = 0;
1481 err = got_privsep_send_enumerated_tree(&totlen, ibuf,
1482 tree_id, "/", NULL, -1);
1483 if (err)
1484 goto done;
1485 break;
1488 if (got_object_idset_contains(idset, tree_id)) {
1489 got_object_qid_free(qid);
1490 qid = NULL;
1491 continue;
1494 err = enumerate_tree(&have_all_entries, ibuf, &totlen,
1495 tree_id, "/", pack, packidx, objcache, idset,
1496 &trees, &nalloc, &ntrees);
1497 if (err)
1498 goto done;
1500 if (!have_all_entries)
1501 break;
1503 got_object_qid_free(qid);
1504 qid = NULL;
1506 parents = got_object_commit_get_parent_ids(commit);
1507 if (parents) {
1508 struct got_object_qid *pid;
1509 STAILQ_FOREACH(pid, parents, entry) {
1510 if (got_object_idset_contains(idset, &pid->id))
1511 continue;
1512 err = got_object_qid_alloc_partial(&qid);
1513 if (err)
1514 goto done;
1515 memcpy(&qid->id, &pid->id, sizeof(qid->id));
1516 STAILQ_INSERT_TAIL(&commit_ids, qid, entry);
1517 qid = NULL;
1521 got_object_commit_close(commit);
1522 commit = NULL;
1525 if (have_all_entries) {
1526 err = got_privsep_send_object_enumeration_done(ibuf);
1527 if (err)
1528 goto done;
1529 } else {
1530 err = got_privsep_send_object_enumeration_incomplete(ibuf);
1531 if (err)
1532 goto done;
1534 done:
1535 if (obj)
1536 got_object_close(obj);
1537 if (commit)
1538 got_object_commit_close(commit);
1539 got_object_qid_free(qid);
1540 got_object_id_queue_free(&commit_ids);
1541 if (idset)
1542 got_object_idset_free(idset);
1543 for (i = 0; i < ntrees; i++) {
1544 struct enumerated_tree *tree = &trees[i];
1545 free(tree->buf);
1546 free(tree->path);
1547 free(tree->entries);
1549 free(trees);
1550 return err;
1553 enum findtwixt_color {
1554 COLOR_KEEP = 0,
1555 COLOR_DROP,
1556 COLOR_SKIP,
1557 COLOR_MAX,
1560 static const struct got_error *
1561 paint_commit(struct got_object_qid *qid, intptr_t color)
1563 if (color < 0 || color >= COLOR_MAX)
1564 return got_error(GOT_ERR_RANGE);
1566 qid->data = (void *)color;
1567 return NULL;
1570 static const struct got_error *
1571 queue_commit_id(struct got_object_id_queue *ids, struct got_object_id *id,
1572 intptr_t color)
1574 const struct got_error *err;
1575 struct got_object_qid *qid;
1577 err = got_object_qid_alloc_partial(&qid);
1578 if (err)
1579 return err;
1581 memcpy(&qid->id, id, sizeof(qid->id));
1582 STAILQ_INSERT_TAIL(ids, qid, entry);
1583 return paint_commit(qid, color);
1586 static const struct got_error *
1587 paint_commits(struct got_object_id_queue *ids, int *nids,
1588 struct got_object_idset *keep, struct got_object_idset *drop,
1589 struct got_object_idset *skip, struct got_pack *pack,
1590 struct got_packidx *packidx, struct imsgbuf *ibuf,
1591 struct got_object_cache *objcache)
1593 const struct got_error *err = NULL;
1594 struct got_commit_object *commit = NULL;
1595 struct got_object_id_queue painted;
1596 const struct got_object_id_queue *parents;
1597 struct got_object_qid *qid = NULL;
1598 int nqueued = *nids, nskip = 0, npainted = 0;
1600 STAILQ_INIT(&painted);
1602 while (!STAILQ_EMPTY(ids) && nskip != nqueued) {
1603 int idx;
1604 intptr_t color;
1606 if (sigint_received) {
1607 err = got_error(GOT_ERR_CANCELLED);
1608 goto done;
1611 qid = STAILQ_FIRST(ids);
1612 idx = got_packidx_get_object_idx(packidx, &qid->id);
1613 if (idx == -1) {
1614 qid = NULL;
1615 break;
1618 STAILQ_REMOVE_HEAD(ids, entry);
1619 nqueued--;
1620 color = (intptr_t)qid->data;
1621 if (color == COLOR_SKIP)
1622 nskip--;
1624 if (got_object_idset_contains(skip, &qid->id)) {
1625 got_object_qid_free(qid);
1626 qid = NULL;
1627 continue;
1630 switch (color) {
1631 case COLOR_KEEP:
1632 if (got_object_idset_contains(keep, &qid->id)) {
1633 got_object_qid_free(qid);
1634 qid = NULL;
1635 continue;
1637 if (got_object_idset_contains(drop, &qid->id)) {
1638 err = paint_commit(qid, COLOR_SKIP);
1639 if (err)
1640 goto done;
1642 err = got_object_idset_add(keep, &qid->id, NULL);
1643 if (err)
1644 goto done;
1645 break;
1646 case COLOR_DROP:
1647 if (got_object_idset_contains(drop, &qid->id)) {
1648 got_object_qid_free(qid);
1649 qid = NULL;
1650 continue;
1652 if (got_object_idset_contains(keep, &qid->id)) {
1653 err = paint_commit(qid, COLOR_SKIP);
1654 if (err)
1655 goto done;
1657 err = got_object_idset_add(drop, &qid->id, NULL);
1658 if (err)
1659 goto done;
1660 break;
1661 case COLOR_SKIP:
1662 if (!got_object_idset_contains(skip, &qid->id)) {
1663 err = got_object_idset_add(skip, &qid->id,
1664 NULL);
1665 if (err)
1666 goto done;
1668 break;
1669 default:
1670 /* should not happen */
1671 err = got_error_fmt(GOT_ERR_NOT_IMPL,
1672 "%s invalid commit color %"PRIdPTR, __func__,
1673 color);
1674 goto done;
1677 err = open_commit(&commit, pack, packidx, idx, &qid->id,
1678 objcache);
1679 if (err)
1680 goto done;
1682 parents = got_object_commit_get_parent_ids(commit);
1683 if (parents) {
1684 struct got_object_qid *pid;
1685 color = (intptr_t)qid->data;
1686 STAILQ_FOREACH(pid, parents, entry) {
1687 err = queue_commit_id(ids, &pid->id, color);
1688 if (err)
1689 goto done;
1690 nqueued++;
1691 if (color == COLOR_SKIP)
1692 nskip++;
1696 got_object_commit_close(commit);
1697 commit = NULL;
1699 STAILQ_INSERT_TAIL(&painted, qid, entry);
1700 qid = NULL;
1701 npainted++;
1703 err = got_privsep_send_painted_commits(ibuf, &painted,
1704 &npainted, 1, 0);
1705 if (err)
1706 goto done;
1709 err = got_privsep_send_painted_commits(ibuf, &painted, &npainted, 1, 1);
1710 if (err)
1711 goto done;
1713 *nids = nqueued;
1714 done:
1715 if (commit)
1716 got_object_commit_close(commit);
1717 got_object_qid_free(qid);
1718 return err;
1721 static void
1722 commit_painting_free(struct got_object_idset **keep,
1723 struct got_object_idset **drop,
1724 struct got_object_idset **skip)
1726 if (*keep) {
1727 got_object_idset_free(*keep);
1728 *keep = NULL;
1730 if (*drop) {
1731 got_object_idset_free(*drop);
1732 *drop = NULL;
1734 if (*skip) {
1735 got_object_idset_free(*skip);
1736 *skip = NULL;
1740 static const struct got_error *
1741 commit_painting_init(struct imsgbuf *ibuf, struct got_object_idset **keep,
1742 struct got_object_idset **drop, struct got_object_idset **skip)
1744 const struct got_error *err = NULL;
1746 *keep = got_object_idset_alloc();
1747 if (*keep == NULL) {
1748 err = got_error_from_errno("got_object_idset_alloc");
1749 goto done;
1751 *drop = got_object_idset_alloc();
1752 if (*drop == NULL) {
1753 err = got_error_from_errno("got_object_idset_alloc");
1754 goto done;
1756 *skip = got_object_idset_alloc();
1757 if (*skip == NULL) {
1758 err = got_error_from_errno("got_object_idset_alloc");
1759 goto done;
1762 err = recv_object_ids(*keep, ibuf);
1763 if (err)
1764 goto done;
1765 err = recv_object_ids(*drop, ibuf);
1766 if (err)
1767 goto done;
1768 err = recv_object_ids(*skip, ibuf);
1769 if (err)
1770 goto done;
1772 done:
1773 if (err)
1774 commit_painting_free(keep, drop, skip);
1776 return err;
1779 static const struct got_error *
1780 commit_painting_request(struct imsg *imsg, struct imsgbuf *ibuf,
1781 struct got_pack *pack, struct got_packidx *packidx,
1782 struct got_object_cache *objcache, struct got_object_idset *keep,
1783 struct got_object_idset *drop, struct got_object_idset *skip)
1785 const struct got_error *err = NULL;
1786 struct got_imsg_commit_painting_request ireq;
1787 struct got_object_id id;
1788 size_t datalen;
1789 struct got_object_id_queue ids;
1790 int nids = 0;
1792 STAILQ_INIT(&ids);
1794 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
1795 if (datalen != sizeof(ireq))
1796 return got_error(GOT_ERR_PRIVSEP_LEN);
1797 memcpy(&ireq, imsg->data, sizeof(ireq));
1798 memcpy(id.sha1, ireq.id, SHA1_DIGEST_LENGTH);
1800 err = queue_commit_id(&ids, &id, ireq.color);
1801 if (err)
1802 return err;
1803 nids = 1;
1805 err = paint_commits(&ids, &nids, keep, drop, skip,
1806 pack, packidx, ibuf, objcache);
1807 if (err)
1808 goto done;
1810 err = got_privsep_send_painted_commits(ibuf, &ids, &nids, 0, 1);
1811 if (err)
1812 goto done;
1814 err = got_privsep_send_painting_commits_done(ibuf);
1815 done:
1816 got_object_id_queue_free(&ids);
1817 return err;
1820 static const struct got_error *
1821 receive_pack(struct got_pack **packp, struct imsgbuf *ibuf)
1823 const struct got_error *err = NULL;
1824 struct imsg imsg;
1825 struct got_imsg_pack ipack;
1826 size_t datalen;
1827 struct got_pack *pack;
1829 *packp = NULL;
1831 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1832 if (err)
1833 return err;
1835 pack = calloc(1, sizeof(*pack));
1836 if (pack == NULL) {
1837 err = got_error_from_errno("calloc");
1838 goto done;
1841 if (imsg.hdr.type != GOT_IMSG_PACK) {
1842 err = got_error(GOT_ERR_PRIVSEP_MSG);
1843 goto done;
1846 if (imsg.fd == -1) {
1847 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1848 goto done;
1851 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1852 if (datalen != sizeof(ipack)) {
1853 err = got_error(GOT_ERR_PRIVSEP_LEN);
1854 goto done;
1856 memcpy(&ipack, imsg.data, sizeof(ipack));
1858 pack->filesize = ipack.filesize;
1859 pack->fd = dup(imsg.fd);
1860 if (pack->fd == -1) {
1861 err = got_error_from_errno("dup");
1862 goto done;
1864 if (lseek(pack->fd, 0, SEEK_SET) == -1) {
1865 err = got_error_from_errno("lseek");
1866 goto done;
1868 pack->path_packfile = strdup(ipack.path_packfile);
1869 if (pack->path_packfile == NULL) {
1870 err = got_error_from_errno("strdup");
1871 goto done;
1874 err = got_delta_cache_alloc(&pack->delta_cache);
1875 if (err)
1876 goto done;
1878 #ifndef GOT_PACK_NO_MMAP
1879 if (pack->filesize > 0 && pack->filesize <= SIZE_MAX) {
1880 pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
1881 pack->fd, 0);
1882 if (pack->map == MAP_FAILED)
1883 pack->map = NULL; /* fall back to read(2) */
1885 #endif
1886 done:
1887 if (err) {
1888 if (imsg.fd != -1)
1889 close(imsg.fd);
1890 free(pack);
1891 } else
1892 *packp = pack;
1893 imsg_free(&imsg);
1894 return err;
1897 int
1898 main(int argc, char *argv[])
1900 const struct got_error *err = NULL;
1901 struct imsgbuf ibuf;
1902 struct imsg imsg;
1903 struct got_packidx *packidx = NULL;
1904 struct got_pack *pack = NULL;
1905 struct got_object_cache objcache;
1906 FILE *basefile = NULL, *accumfile = NULL, *delta_outfile = NULL;
1907 struct got_object_idset *keep = NULL, *drop = NULL, *skip = NULL;
1908 struct got_parsed_tree_entry *entries = NULL;
1909 size_t nentries = 0, nentries_alloc = 0;
1911 //static int attached;
1912 //while (!attached) sleep(1);
1914 signal(SIGINT, catch_sigint);
1916 imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
1918 err = got_object_cache_init(&objcache, GOT_OBJECT_CACHE_TYPE_OBJ);
1919 if (err) {
1920 err = got_error_from_errno("got_object_cache_init");
1921 got_privsep_send_error(&ibuf, err);
1922 return 1;
1925 #ifndef PROFILE
1926 /* revoke access to most system calls */
1927 if (pledge("stdio recvfd", NULL) == -1) {
1928 err = got_error_from_errno("pledge");
1929 got_privsep_send_error(&ibuf, err);
1930 return 1;
1932 #endif
1934 err = receive_packidx(&packidx, &ibuf);
1935 if (err) {
1936 got_privsep_send_error(&ibuf, err);
1937 return 1;
1940 err = receive_pack(&pack, &ibuf);
1941 if (err) {
1942 got_privsep_send_error(&ibuf, err);
1943 return 1;
1946 for (;;) {
1947 imsg.fd = -1;
1949 if (sigint_received) {
1950 err = got_error(GOT_ERR_CANCELLED);
1951 break;
1954 err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1955 if (err) {
1956 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1957 err = NULL;
1958 break;
1961 if (imsg.hdr.type == GOT_IMSG_STOP)
1962 break;
1964 switch (imsg.hdr.type) {
1965 case GOT_IMSG_TMPFD:
1966 if (basefile == NULL) {
1967 err = receive_tempfile(&basefile, "w+",
1968 &imsg, &ibuf);
1969 } else if (accumfile == NULL) {
1970 err = receive_tempfile(&accumfile, "w+",
1971 &imsg, &ibuf);
1972 } else
1973 err = got_error(GOT_ERR_PRIVSEP_MSG);
1974 break;
1975 case GOT_IMSG_PACKED_OBJECT_REQUEST:
1976 err = object_request(&imsg, &ibuf, pack, packidx,
1977 &objcache);
1978 break;
1979 case GOT_IMSG_PACKED_RAW_OBJECT_REQUEST:
1980 if (basefile == NULL || accumfile == NULL) {
1981 err = got_error(GOT_ERR_PRIVSEP_MSG);
1982 break;
1984 err = raw_object_request(&imsg, &ibuf, pack, packidx,
1985 &objcache, basefile, accumfile);
1986 break;
1987 case GOT_IMSG_RAW_DELTA_OUTFD:
1988 if (delta_outfile != NULL) {
1989 err = got_error(GOT_ERR_PRIVSEP_MSG);
1990 break;
1992 err = receive_tempfile(&delta_outfile, "w",
1993 &imsg, &ibuf);
1994 break;
1995 case GOT_IMSG_RAW_DELTA_REQUEST:
1996 if (delta_outfile == NULL) {
1997 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1998 break;
2000 err = raw_delta_request(&imsg, &ibuf, delta_outfile,
2001 pack, packidx);
2002 break;
2003 case GOT_IMSG_DELTA_REUSE_REQUEST:
2004 if (delta_outfile == NULL) {
2005 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
2006 break;
2008 err = delta_reuse_request(&imsg, &ibuf,
2009 delta_outfile, pack, packidx);
2010 break;
2011 case GOT_IMSG_COMMIT_REQUEST:
2012 err = commit_request(&imsg, &ibuf, pack, packidx,
2013 &objcache);
2014 break;
2015 case GOT_IMSG_TREE_REQUEST:
2016 err = tree_request(&imsg, &ibuf, pack, packidx,
2017 &objcache, &entries, &nentries, &nentries_alloc);
2018 break;
2019 case GOT_IMSG_BLOB_REQUEST:
2020 if (basefile == NULL || accumfile == NULL) {
2021 err = got_error(GOT_ERR_PRIVSEP_MSG);
2022 break;
2024 err = blob_request(&imsg, &ibuf, pack, packidx,
2025 &objcache, basefile, accumfile);
2026 break;
2027 case GOT_IMSG_TAG_REQUEST:
2028 err = tag_request(&imsg, &ibuf, pack, packidx,
2029 &objcache);
2030 break;
2031 case GOT_IMSG_COMMIT_TRAVERSAL_REQUEST:
2032 err = commit_traversal_request(&imsg, &ibuf, pack,
2033 packidx, &objcache);
2034 break;
2035 case GOT_IMSG_OBJECT_ENUMERATION_REQUEST:
2036 err = enumeration_request(&imsg, &ibuf, pack,
2037 packidx, &objcache);
2038 break;
2039 case GOT_IMSG_COMMIT_PAINTING_INIT:
2040 commit_painting_free(&keep, &drop, &skip);
2041 err = commit_painting_init(&ibuf, &keep, &drop, &skip);
2042 break;
2043 case GOT_IMSG_COMMIT_PAINTING_REQUEST:
2044 if (keep == NULL || drop == NULL || skip == NULL) {
2045 err = got_error(GOT_ERR_PRIVSEP_MSG);
2046 break;
2048 err = commit_painting_request(&imsg, &ibuf, pack,
2049 packidx, &objcache, keep, drop, skip);
2050 break;
2051 case GOT_IMSG_COMMIT_PAINTING_DONE:
2052 commit_painting_free(&keep, &drop, &skip);
2053 break;
2054 default:
2055 err = got_error(GOT_ERR_PRIVSEP_MSG);
2056 break;
2059 if (imsg.fd != -1 && close(imsg.fd) == -1 && err == NULL)
2060 err = got_error_from_errno("close");
2061 imsg_free(&imsg);
2062 if (err)
2063 break;
2066 free(entries);
2067 commit_painting_free(&keep, &drop, &skip);
2068 if (packidx)
2069 got_packidx_close(packidx);
2070 if (pack)
2071 got_pack_close(pack);
2072 got_object_cache_close(&objcache);
2073 imsg_clear(&ibuf);
2074 if (basefile && fclose(basefile) == EOF && err == NULL)
2075 err = got_error_from_errno("fclose");
2076 if (accumfile && fclose(accumfile) == EOF && err == NULL)
2077 err = got_error_from_errno("fclose");
2078 if (delta_outfile && fclose(delta_outfile) == EOF && err == NULL)
2079 err = got_error_from_errno("fclose");
2080 if (err) {
2081 if (!sigint_received && err->code != GOT_ERR_PRIVSEP_PIPE) {
2082 fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
2083 got_privsep_send_error(&ibuf, err);
2086 if (close(GOT_IMSG_FD_CHILD) == -1 && err == NULL)
2087 err = got_error_from_errno("close");
2088 return err ? 1 : 0;