Blob


1 /*
2 * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
3 *
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 */
17 #include <sys/types.h>
18 #include <sys/queue.h>
19 #include <sys/uio.h>
20 #include <sys/time.h>
21 #include <sys/mman.h>
23 #include <limits.h>
24 #include <signal.h>
25 #include <stdint.h>
26 #include <imsg.h>
27 #include <stdio.h>
28 #include <stdlib.h>
29 #include <string.h>
30 #include <sha1.h>
31 #include <unistd.h>
32 #include <zlib.h>
34 #include "got_error.h"
35 #include "got_object.h"
36 #include "got_path.h"
38 #include "got_lib_delta.h"
39 #include "got_lib_delta_cache.h"
40 #include "got_lib_object.h"
41 #include "got_lib_object_cache.h"
42 #include "got_lib_object_parse.h"
43 #include "got_lib_object_idset.h"
44 #include "got_lib_privsep.h"
45 #include "got_lib_pack.h"
47 static volatile sig_atomic_t sigint_received;
49 static void
50 catch_sigint(int signo)
51 {
52 sigint_received = 1;
53 }
55 static const struct got_error *
56 open_object(struct got_object **obj, struct got_pack *pack,
57 struct got_packidx *packidx, int idx, struct got_object_id *id,
58 struct got_object_cache *objcache)
59 {
60 const struct got_error *err;
62 err = got_packfile_open_object(obj, pack, packidx, idx, id);
63 if (err)
64 return err;
65 (*obj)->refcnt++;
67 err = got_object_cache_add(objcache, id, *obj);
68 if (err) {
69 if (err->code == GOT_ERR_OBJ_EXISTS ||
70 err->code == GOT_ERR_OBJ_TOO_LARGE)
71 err = NULL;
72 return err;
73 }
74 (*obj)->refcnt++;
75 return NULL;
76 }
78 static const struct got_error *
79 object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
80 struct got_packidx *packidx, struct got_object_cache *objcache)
81 {
82 const struct got_error *err = NULL;
83 struct got_imsg_packed_object iobj;
84 struct got_object *obj;
85 struct got_object_id id;
86 size_t datalen;
88 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
89 if (datalen != sizeof(iobj))
90 return got_error(GOT_ERR_PRIVSEP_LEN);
91 memcpy(&iobj, imsg->data, sizeof(iobj));
92 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
94 obj = got_object_cache_get(objcache, &id);
95 if (obj) {
96 obj->refcnt++;
97 } else {
98 err = open_object(&obj, pack, packidx, iobj.idx, &id,
99 objcache);
100 if (err)
101 goto done;
104 err = got_privsep_send_obj(ibuf, obj);
105 done:
106 got_object_close(obj);
107 return err;
110 static const struct got_error *
111 open_commit(struct got_commit_object **commit, struct got_pack *pack,
112 struct got_packidx *packidx, int obj_idx, struct got_object_id *id,
113 struct got_object_cache *objcache)
115 const struct got_error *err = NULL;
116 struct got_object *obj = NULL;
117 uint8_t *buf = NULL;
118 size_t len;
120 *commit = NULL;
122 obj = got_object_cache_get(objcache, id);
123 if (obj) {
124 obj->refcnt++;
125 } else {
126 err = open_object(&obj, pack, packidx, obj_idx, id,
127 objcache);
128 if (err)
129 return err;
132 err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
133 if (err)
134 goto done;
136 obj->size = len;
138 err = got_object_parse_commit(commit, buf, len);
139 done:
140 got_object_close(obj);
141 free(buf);
142 return err;
145 static const struct got_error *
146 commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
147 struct got_packidx *packidx, struct got_object_cache *objcache)
149 const struct got_error *err = NULL;
150 struct got_imsg_packed_object iobj;
151 struct got_commit_object *commit = NULL;
152 struct got_object_id id;
153 size_t datalen;
155 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
156 if (datalen != sizeof(iobj))
157 return got_error(GOT_ERR_PRIVSEP_LEN);
158 memcpy(&iobj, imsg->data, sizeof(iobj));
159 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
161 err = open_commit(&commit, pack, packidx, iobj.idx, &id, objcache);
162 if (err)
163 goto done;
165 err = got_privsep_send_commit(ibuf, commit);
166 done:
167 if (commit)
168 got_object_commit_close(commit);
169 if (err) {
170 if (err->code == GOT_ERR_PRIVSEP_PIPE)
171 err = NULL;
172 else
173 got_privsep_send_error(ibuf, err);
176 return err;
179 static const struct got_error *
180 open_tree(uint8_t **buf, struct got_parsed_tree_entry **entries, int *nentries,
181 struct got_pack *pack, struct got_packidx *packidx, int obj_idx,
182 struct got_object_id *id, struct got_object_cache *objcache)
184 const struct got_error *err = NULL;
185 struct got_object *obj = NULL;
186 size_t len;
188 *buf = NULL;
189 *nentries = 0;
191 obj = got_object_cache_get(objcache, id);
192 if (obj) {
193 obj->refcnt++;
194 } else {
195 err = open_object(&obj, pack, packidx, obj_idx, id,
196 objcache);
197 if (err)
198 return err;
201 err = got_packfile_extract_object_to_mem(buf, &len, obj, pack);
202 if (err)
203 goto done;
205 obj->size = len;
207 err = got_object_parse_tree(entries, nentries, *buf, len);
208 done:
209 got_object_close(obj);
210 if (err) {
211 free(*buf);
212 *buf = NULL;
214 return err;
217 static const struct got_error *
218 tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
219 struct got_packidx *packidx, struct got_object_cache *objcache)
221 const struct got_error *err = NULL;
222 struct got_imsg_packed_object iobj;
223 struct got_parsed_tree_entry *entries = NULL;
224 int nentries = 0;
225 uint8_t *buf = NULL;
226 struct got_object_id id;
227 size_t datalen;
229 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
230 if (datalen != sizeof(iobj))
231 return got_error(GOT_ERR_PRIVSEP_LEN);
232 memcpy(&iobj, imsg->data, sizeof(iobj));
233 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
235 err = open_tree(&buf, &entries, &nentries, pack, packidx, iobj.idx,
236 &id, objcache);
237 if (err)
238 return err;
240 err = got_privsep_send_tree(ibuf, entries, nentries);
241 free(entries);
242 free(buf);
243 if (err) {
244 if (err->code == GOT_ERR_PRIVSEP_PIPE)
245 err = NULL;
246 else
247 got_privsep_send_error(ibuf, err);
250 return err;
253 static const struct got_error *
254 receive_file(FILE **f, struct imsgbuf *ibuf, uint32_t imsg_code)
256 const struct got_error *err;
257 struct imsg imsg;
258 size_t datalen;
260 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
261 if (err)
262 return err;
264 if (imsg.hdr.type != imsg_code) {
265 err = got_error(GOT_ERR_PRIVSEP_MSG);
266 goto done;
269 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
270 if (datalen != 0) {
271 err = got_error(GOT_ERR_PRIVSEP_LEN);
272 goto done;
274 if (imsg.fd == -1) {
275 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
276 goto done;
279 *f = fdopen(imsg.fd, "w+");
280 if (*f == NULL) {
281 err = got_error_from_errno("fdopen");
282 close(imsg.fd);
283 goto done;
285 done:
286 imsg_free(&imsg);
287 return err;
290 static const struct got_error *
291 receive_tempfile(FILE **f, const char *mode, struct imsg *imsg,
292 struct imsgbuf *ibuf)
294 size_t datalen;
296 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
297 if (datalen != 0)
298 return got_error(GOT_ERR_PRIVSEP_LEN);
300 if (imsg->fd == -1)
301 return got_error(GOT_ERR_PRIVSEP_NO_FD);
303 *f = fdopen(imsg->fd, mode);
304 if (*f == NULL)
305 return got_error_from_errno("fdopen");
306 imsg->fd = -1;
308 return NULL;
311 static const struct got_error *
312 blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
313 struct got_packidx *packidx, struct got_object_cache *objcache,
314 FILE *basefile, FILE *accumfile)
316 const struct got_error *err = NULL;
317 struct got_imsg_packed_object iobj;
318 struct got_object *obj = NULL;
319 FILE *outfile = NULL;
320 struct got_object_id id;
321 size_t datalen;
322 uint64_t blob_size;
323 uint8_t *buf = NULL;
325 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
326 if (datalen != sizeof(iobj))
327 return got_error(GOT_ERR_PRIVSEP_LEN);
328 memcpy(&iobj, imsg->data, sizeof(iobj));
329 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
331 obj = got_object_cache_get(objcache, &id);
332 if (obj) {
333 obj->refcnt++;
334 } else {
335 err = open_object(&obj, pack, packidx, iobj.idx, &id,
336 objcache);
337 if (err)
338 return err;
341 err = receive_file(&outfile, ibuf, GOT_IMSG_BLOB_OUTFD);
342 if (err)
343 goto done;
345 if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
346 err = got_pack_get_max_delta_object_size(&blob_size, obj, pack);
347 if (err)
348 goto done;
349 } else
350 blob_size = obj->size;
352 if (blob_size <= GOT_PRIVSEP_INLINE_BLOB_DATA_MAX)
353 err = got_packfile_extract_object_to_mem(&buf, &obj->size,
354 obj, pack);
355 else
356 err = got_packfile_extract_object(pack, obj, outfile, basefile,
357 accumfile);
358 if (err)
359 goto done;
361 err = got_privsep_send_blob(ibuf, obj->size, obj->hdrlen, buf);
362 done:
363 free(buf);
364 if (outfile && fclose(outfile) == EOF && err == NULL)
365 err = got_error_from_errno("fclose");
366 got_object_close(obj);
367 if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
368 got_privsep_send_error(ibuf, err);
370 return err;
373 static const struct got_error *
374 tag_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
375 struct got_packidx *packidx, struct got_object_cache *objcache)
377 const struct got_error *err = NULL;
378 struct got_imsg_packed_object iobj;
379 struct got_object *obj = NULL;
380 struct got_tag_object *tag = NULL;
381 uint8_t *buf = NULL;
382 size_t len;
383 struct got_object_id id;
384 size_t datalen;
386 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
387 if (datalen != sizeof(iobj))
388 return got_error(GOT_ERR_PRIVSEP_LEN);
389 memcpy(&iobj, imsg->data, sizeof(iobj));
390 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
392 obj = got_object_cache_get(objcache, &id);
393 if (obj) {
394 obj->refcnt++;
395 } else {
396 err = open_object(&obj, pack, packidx, iobj.idx, &id,
397 objcache);
398 if (err)
399 return err;
402 err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack);
403 if (err)
404 goto done;
406 obj->size = len;
407 err = got_object_parse_tag(&tag, buf, len);
408 if (err)
409 goto done;
411 err = got_privsep_send_tag(ibuf, tag);
412 done:
413 free(buf);
414 got_object_close(obj);
415 if (tag)
416 got_object_tag_close(tag);
417 if (err) {
418 if (err->code == GOT_ERR_PRIVSEP_PIPE)
419 err = NULL;
420 else
421 got_privsep_send_error(ibuf, err);
424 return err;
427 static struct got_parsed_tree_entry *
428 find_entry_by_name(struct got_parsed_tree_entry *entries, int nentries,
429 const char *name, size_t len)
431 struct got_parsed_tree_entry *pte;
432 int cmp, i;
434 /* Note that tree entries are sorted in strncmp() order. */
435 for (i = 0; i < nentries; i++) {
436 pte = &entries[i];
437 cmp = strncmp(pte->name, name, len);
438 if (cmp < 0)
439 continue;
440 if (cmp > 0)
441 break;
442 if (pte->name[len] == '\0')
443 return pte;
445 return NULL;
448 static const struct got_error *
449 tree_path_changed(int *changed, uint8_t **buf1, uint8_t **buf2,
450 struct got_parsed_tree_entry **entries1, int *nentries1,
451 struct got_parsed_tree_entry **entries2, int *nentries2,
452 const char *path, struct got_pack *pack, struct got_packidx *packidx,
453 struct imsgbuf *ibuf, struct got_object_cache *objcache)
455 const struct got_error *err = NULL;
456 struct got_parsed_tree_entry *pte1 = NULL, *pte2 = NULL;
457 const char *seg, *s;
458 size_t seglen;
460 *changed = 0;
462 /* We not do support comparing the root path. */
463 if (got_path_is_root_dir(path))
464 return got_error_path(path, GOT_ERR_BAD_PATH);
466 s = path;
467 while (*s == '/')
468 s++;
469 seg = s;
470 seglen = 0;
471 while (*s) {
472 if (*s != '/') {
473 s++;
474 seglen++;
475 if (*s)
476 continue;
479 pte1 = find_entry_by_name(*entries1, *nentries1, seg, seglen);
480 if (pte1 == NULL) {
481 err = got_error(GOT_ERR_NO_OBJ);
482 break;
485 pte2 = find_entry_by_name(*entries2, *nentries2, seg, seglen);
486 if (pte2 == NULL) {
487 *changed = 1;
488 break;
491 if (pte1->mode != pte2->mode) {
492 *changed = 1;
493 break;
496 if (memcmp(pte1->id, pte2->id, SHA1_DIGEST_LENGTH) == 0) {
497 *changed = 0;
498 break;
501 if (*s == '\0') { /* final path element */
502 *changed = 1;
503 break;
506 seg = s + 1;
507 s++;
508 seglen = 0;
509 if (*s) {
510 struct got_object_id id1, id2;
511 int idx;
513 memcpy(id1.sha1, pte1->id, SHA1_DIGEST_LENGTH);
514 idx = got_packidx_get_object_idx(packidx, &id1);
515 if (idx == -1) {
516 err = got_error_no_obj(&id1);
517 break;
519 free(*entries1);
520 *nentries1 = 0;
521 free(*buf1);
522 *buf1 = NULL;
523 err = open_tree(buf1, entries1, nentries1, pack,
524 packidx, idx, &id1, objcache);
525 pte1 = NULL;
526 if (err)
527 break;
529 memcpy(id2.sha1, pte2->id, SHA1_DIGEST_LENGTH);
530 idx = got_packidx_get_object_idx(packidx, &id2);
531 if (idx == -1) {
532 err = got_error_no_obj(&id2);
533 break;
535 free(*entries2);
536 *nentries2 = 0;
537 free(*buf2);
538 *buf2 = NULL;
539 err = open_tree(buf2, entries2, nentries2, pack,
540 packidx, idx, &id2, objcache);
541 pte2 = NULL;
542 if (err)
543 break;
547 return err;
550 static const struct got_error *
551 send_traversed_commits(struct got_object_id *commit_ids, size_t ncommits,
552 struct imsgbuf *ibuf)
554 struct ibuf *wbuf;
555 size_t i;
557 wbuf = imsg_create(ibuf, GOT_IMSG_TRAVERSED_COMMITS, 0, 0,
558 sizeof(struct got_imsg_traversed_commits) +
559 ncommits * SHA1_DIGEST_LENGTH);
560 if (wbuf == NULL)
561 return got_error_from_errno("imsg_create TRAVERSED_COMMITS");
563 if (imsg_add(wbuf, &ncommits, sizeof(ncommits)) == -1)
564 return got_error_from_errno("imsg_add TRAVERSED_COMMITS");
566 for (i = 0; i < ncommits; i++) {
567 struct got_object_id *id = &commit_ids[i];
568 if (imsg_add(wbuf, id->sha1, SHA1_DIGEST_LENGTH) == -1) {
569 return got_error_from_errno(
570 "imsg_add TRAVERSED_COMMITS");
574 wbuf->fd = -1;
575 imsg_close(ibuf, wbuf);
577 return got_privsep_flush_imsg(ibuf);
580 static const struct got_error *
581 send_commit_traversal_done(struct imsgbuf *ibuf)
583 if (imsg_compose(ibuf, GOT_IMSG_COMMIT_TRAVERSAL_DONE, 0, 0, -1,
584 NULL, 0) == -1)
585 return got_error_from_errno("imsg_compose TRAVERSAL_DONE");
587 return got_privsep_flush_imsg(ibuf);
591 static const struct got_error *
592 commit_traversal_request(struct imsg *imsg, struct imsgbuf *ibuf,
593 struct got_pack *pack, struct got_packidx *packidx,
594 struct got_object_cache *objcache)
596 const struct got_error *err = NULL;
597 struct got_imsg_packed_object iobj;
598 struct got_object_qid *pid;
599 struct got_commit_object *commit = NULL, *pcommit = NULL;
600 struct got_parsed_tree_entry *entries = NULL, *pentries = NULL;
601 int nentries = 0, pnentries = 0;
602 struct got_object_id id;
603 size_t datalen, path_len;
604 char *path = NULL;
605 const int min_alloc = 64;
606 int changed = 0, ncommits = 0, nallocated = 0;
607 struct got_object_id *commit_ids = NULL;
609 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
610 if (datalen < sizeof(iobj))
611 return got_error(GOT_ERR_PRIVSEP_LEN);
612 memcpy(&iobj, imsg->data, sizeof(iobj));
613 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
615 path_len = datalen - sizeof(iobj) - 1;
616 if (path_len < 0)
617 return got_error(GOT_ERR_PRIVSEP_LEN);
618 if (path_len > 0) {
619 path = imsg->data + sizeof(iobj);
620 if (path[path_len] != '\0')
621 return got_error(GOT_ERR_PRIVSEP_LEN);
624 nallocated = min_alloc;
625 commit_ids = reallocarray(NULL, nallocated, sizeof(*commit_ids));
626 if (commit_ids == NULL)
627 return got_error_from_errno("reallocarray");
629 do {
630 const size_t max_datalen = MAX_IMSGSIZE - IMSG_HEADER_SIZE;
631 int idx;
633 if (sigint_received) {
634 err = got_error(GOT_ERR_CANCELLED);
635 goto done;
638 if (commit == NULL) {
639 idx = got_packidx_get_object_idx(packidx, &id);
640 if (idx == -1)
641 break;
642 err = open_commit(&commit, pack, packidx,
643 idx, &id, objcache);
644 if (err) {
645 if (err->code != GOT_ERR_NO_OBJ)
646 goto done;
647 err = NULL;
648 break;
652 if (sizeof(struct got_imsg_traversed_commits) +
653 ncommits * SHA1_DIGEST_LENGTH >= max_datalen) {
654 err = send_traversed_commits(commit_ids, ncommits,
655 ibuf);
656 if (err)
657 goto done;
658 ncommits = 0;
660 ncommits++;
661 if (ncommits > nallocated) {
662 struct got_object_id *new;
663 nallocated += min_alloc;
664 new = reallocarray(commit_ids, nallocated,
665 sizeof(*commit_ids));
666 if (new == NULL) {
667 err = got_error_from_errno("reallocarray");
668 goto done;
670 commit_ids = new;
672 memcpy(commit_ids[ncommits - 1].sha1, id.sha1,
673 SHA1_DIGEST_LENGTH);
675 pid = STAILQ_FIRST(&commit->parent_ids);
676 if (pid == NULL)
677 break;
679 idx = got_packidx_get_object_idx(packidx, &pid->id);
680 if (idx == -1)
681 break;
683 err = open_commit(&pcommit, pack, packidx, idx, &pid->id,
684 objcache);
685 if (err) {
686 if (err->code != GOT_ERR_NO_OBJ)
687 goto done;
688 err = NULL;
689 break;
692 if (path[0] == '/' && path[1] == '\0') {
693 if (got_object_id_cmp(pcommit->tree_id,
694 commit->tree_id) != 0) {
695 changed = 1;
696 break;
698 } else {
699 int pidx;
700 uint8_t *buf = NULL, *pbuf = NULL;
702 idx = got_packidx_get_object_idx(packidx,
703 commit->tree_id);
704 if (idx == -1)
705 break;
706 pidx = got_packidx_get_object_idx(packidx,
707 pcommit->tree_id);
708 if (pidx == -1)
709 break;
711 err = open_tree(&buf, &entries, &nentries, pack,
712 packidx, idx, commit->tree_id, objcache);
713 if (err)
714 goto done;
715 err = open_tree(&pbuf, &pentries, &pnentries, pack,
716 packidx, pidx, pcommit->tree_id, objcache);
717 if (err) {
718 free(buf);
719 goto done;
722 err = tree_path_changed(&changed, &buf, &pbuf,
723 &entries, &nentries, &pentries, &pnentries, path,
724 pack, packidx, ibuf, objcache);
726 free(entries);
727 entries = NULL;
728 nentries = 0;
729 free(buf);
730 free(pentries);
731 pentries = NULL;
732 pnentries = 0;
733 free(pbuf);
734 if (err) {
735 if (err->code != GOT_ERR_NO_OBJ)
736 goto done;
737 err = NULL;
738 break;
742 if (!changed) {
743 memcpy(id.sha1, pid->id.sha1, SHA1_DIGEST_LENGTH);
744 got_object_commit_close(commit);
745 commit = pcommit;
746 pcommit = NULL;
748 } while (!changed);
750 if (ncommits > 0) {
751 err = send_traversed_commits(commit_ids, ncommits, ibuf);
752 if (err)
753 goto done;
755 if (changed) {
756 err = got_privsep_send_commit(ibuf, commit);
757 if (err)
758 goto done;
761 err = send_commit_traversal_done(ibuf);
762 done:
763 free(commit_ids);
764 if (commit)
765 got_object_commit_close(commit);
766 if (pcommit)
767 got_object_commit_close(pcommit);
768 free(entries);
769 free(pentries);
770 if (err) {
771 if (err->code == GOT_ERR_PRIVSEP_PIPE)
772 err = NULL;
773 else
774 got_privsep_send_error(ibuf, err);
777 return err;
780 static const struct got_error *
781 raw_object_request(struct imsg *imsg, struct imsgbuf *ibuf,
782 struct got_pack *pack, struct got_packidx *packidx,
783 struct got_object_cache *objcache, FILE *basefile, FILE *accumfile)
785 const struct got_error *err = NULL;
786 uint8_t *buf = NULL;
787 uint64_t size = 0;
788 FILE *outfile = NULL;
789 struct got_imsg_packed_object iobj;
790 struct got_object *obj;
791 struct got_object_id id;
792 size_t datalen;
794 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
795 if (datalen != sizeof(iobj))
796 return got_error(GOT_ERR_PRIVSEP_LEN);
797 memcpy(&iobj, imsg->data, sizeof(iobj));
798 memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
800 obj = got_object_cache_get(objcache, &id);
801 if (obj) {
802 obj->refcnt++;
803 } else {
804 err = open_object(&obj, pack, packidx, iobj.idx, &id,
805 objcache);
806 if (err)
807 return err;
810 err = receive_file(&outfile, ibuf, GOT_IMSG_RAW_OBJECT_OUTFD);
811 if (err)
812 return err;
814 if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
815 err = got_pack_get_max_delta_object_size(&size, obj, pack);
816 if (err)
817 goto done;
818 } else
819 size = obj->size;
821 if (size <= GOT_PRIVSEP_INLINE_OBJECT_DATA_MAX)
822 err = got_packfile_extract_object_to_mem(&buf, &obj->size,
823 obj, pack);
824 else
825 err = got_packfile_extract_object(pack, obj, outfile, basefile,
826 accumfile);
827 if (err)
828 goto done;
830 err = got_privsep_send_raw_obj(ibuf, obj->size, obj->hdrlen, buf);
831 done:
832 free(buf);
833 if (outfile && fclose(outfile) == EOF && err == NULL)
834 err = got_error_from_errno("fclose");
835 got_object_close(obj);
836 if (err && err->code != GOT_ERR_PRIVSEP_PIPE)
837 got_privsep_send_error(ibuf, err);
839 return err;
842 static const struct got_error *
843 get_base_object_id(struct got_object_id *base_id, struct got_packidx *packidx,
844 off_t base_offset)
846 const struct got_error *err;
847 int idx;
849 err = got_packidx_get_offset_idx(&idx, packidx, base_offset);
850 if (err)
851 return err;
852 if (idx == -1)
853 return got_error(GOT_ERR_BAD_PACKIDX);
855 return got_packidx_get_object_id(base_id, packidx, idx);
858 static const struct got_error *
859 raw_delta_request(struct imsg *imsg, struct imsgbuf *ibuf,
860 FILE *delta_outfile, struct got_pack *pack,
861 struct got_packidx *packidx)
863 const struct got_error *err = NULL;
864 struct got_imsg_raw_delta_request req;
865 size_t datalen, delta_size, delta_compressed_size;
866 off_t delta_offset;
867 uint8_t *delta_buf = NULL;
868 struct got_object_id id, base_id;
869 off_t base_offset, delta_out_offset = 0;
870 uint64_t base_size = 0, result_size = 0;
871 size_t w;
873 datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
874 if (datalen != sizeof(req))
875 return got_error(GOT_ERR_PRIVSEP_LEN);
876 memcpy(&req, imsg->data, sizeof(req));
877 memcpy(id.sha1, req.id, SHA1_DIGEST_LENGTH);
879 imsg->fd = -1;
881 err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
882 &delta_compressed_size, &delta_offset, &base_offset, &base_id,
883 &base_size, &result_size, pack, packidx, req.idx);
884 if (err)
885 goto done;
887 /*
888 * If this is an offset delta we must determine the base
889 * object ID ourselves.
890 */
891 if (base_offset != 0) {
892 err = get_base_object_id(&base_id, packidx, base_offset);
893 if (err)
894 goto done;
897 delta_out_offset = ftello(delta_outfile);
898 w = fwrite(delta_buf, 1, delta_compressed_size, delta_outfile);
899 if (w != delta_compressed_size) {
900 err = got_ferror(delta_outfile, GOT_ERR_IO);
901 goto done;
903 if (fflush(delta_outfile) == -1) {
904 err = got_error_from_errno("fflush");
905 goto done;
908 err = got_privsep_send_raw_delta(ibuf, base_size, result_size,
909 delta_size, delta_compressed_size, delta_offset, delta_out_offset,
910 &base_id);
911 done:
912 free(delta_buf);
913 return err;
916 struct search_deltas_arg {
917 struct imsgbuf *ibuf;
918 struct got_packidx *packidx;
919 struct got_pack *pack;
920 struct got_object_idset *idset;
921 FILE *delta_outfile;
922 struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS];
923 size_t ndeltas;
924 };
926 static const struct got_error *
927 search_delta_for_object(struct got_object_id *id, void *data, void *arg)
929 const struct got_error *err;
930 struct search_deltas_arg *a = arg;
931 int obj_idx;
932 uint8_t *delta_buf = NULL;
933 uint64_t base_size, result_size;
934 size_t delta_size, delta_compressed_size;
935 off_t delta_offset, base_offset;
936 struct got_object_id base_id;
938 if (sigint_received)
939 return got_error(GOT_ERR_CANCELLED);
941 obj_idx = got_packidx_get_object_idx(a->packidx, id);
942 if (obj_idx == -1)
943 return NULL; /* object not present in our pack file */
945 err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
946 &delta_compressed_size, &delta_offset, &base_offset, &base_id,
947 &base_size, &result_size, a->pack, a->packidx, obj_idx);
948 if (err) {
949 if (err->code == GOT_ERR_OBJ_TYPE)
950 return NULL; /* object not stored as a delta */
951 return err;
954 /*
955 * If this is an offset delta we must determine the base
956 * object ID ourselves.
957 */
958 if (base_offset != 0) {
959 err = get_base_object_id(&base_id, a->packidx, base_offset);
960 if (err)
961 goto done;
964 if (got_object_idset_contains(a->idset, &base_id)) {
965 struct got_imsg_reused_delta *delta;
966 off_t delta_out_offset = ftello(a->delta_outfile);
967 size_t w;
969 w = fwrite(delta_buf, 1, delta_compressed_size,
970 a->delta_outfile);
971 if (w != delta_compressed_size) {
972 err = got_ferror(a->delta_outfile, GOT_ERR_IO);
973 goto done;
976 delta = &a->deltas[a->ndeltas++];
977 memcpy(&delta->id, id, sizeof(delta->id));
978 memcpy(&delta->base_id, &base_id, sizeof(delta->base_id));
979 delta->base_size = base_size;
980 delta->result_size = result_size;
981 delta->delta_size = delta_size;
982 delta->delta_compressed_size = delta_compressed_size;
983 delta->delta_offset = delta_offset;
984 delta->delta_out_offset = delta_out_offset;
986 if (a->ndeltas >= GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS) {
987 err = got_privsep_send_reused_deltas(a->ibuf,
988 a->deltas, a->ndeltas);
989 if (err)
990 goto done;
991 a->ndeltas = 0;
994 done:
995 free(delta_buf);
996 return err;
999 static const struct got_error *
1000 recv_object_ids(struct got_object_idset *idset, struct imsgbuf *ibuf)
1002 const struct got_error *err = NULL;
1003 int done = 0;
1004 struct got_object_id *ids;
1005 size_t nids, i;
1007 for (;;) {
1008 err = got_privsep_recv_object_idlist(&done, &ids, &nids, ibuf);
1009 if (err || done)
1010 break;
1011 for (i = 0; i < nids; i++) {
1012 err = got_object_idset_add(idset, &ids[i], NULL);
1013 if (err) {
1014 free(ids);
1015 return err;
1018 free(ids);
1021 return err;
1024 static const struct got_error *
1025 delta_reuse_request(struct imsg *imsg, struct imsgbuf *ibuf,
1026 FILE *delta_outfile, struct got_pack *pack, struct got_packidx *packidx)
1028 const struct got_error *err = NULL;
1029 struct got_object_idset *idset;
1030 struct search_deltas_arg sda;
1032 idset = got_object_idset_alloc();
1033 if (idset == NULL)
1034 return got_error_from_errno("got_object_idset_alloc");
1036 err = recv_object_ids(idset, ibuf);
1037 if (err)
1038 return err;
1040 memset(&sda, 0, sizeof(sda));
1041 sda.ibuf = ibuf;
1042 sda.idset = idset;
1043 sda.pack = pack;
1044 sda.packidx = packidx;
1045 sda.delta_outfile = delta_outfile;
1046 err = got_object_idset_for_each(idset, search_delta_for_object, &sda);
1047 if (err)
1048 goto done;
1050 if (sda.ndeltas > 0) {
1051 err = got_privsep_send_reused_deltas(ibuf, sda.deltas,
1052 sda.ndeltas);
1053 if (err)
1054 goto done;
1057 if (fflush(delta_outfile) == -1) {
1058 err = got_error_from_errno("fflush");
1059 goto done;
1062 err = got_privsep_send_reused_deltas_done(ibuf);
1063 done:
1064 got_object_idset_free(idset);
1065 return err;
1068 static const struct got_error *
1069 receive_packidx(struct got_packidx **packidx, struct imsgbuf *ibuf)
1071 const struct got_error *err = NULL;
1072 struct imsg imsg;
1073 struct got_imsg_packidx ipackidx;
1074 size_t datalen;
1075 struct got_packidx *p;
1077 *packidx = NULL;
1079 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1080 if (err)
1081 return err;
1083 p = calloc(1, sizeof(*p));
1084 if (p == NULL) {
1085 err = got_error_from_errno("calloc");
1086 goto done;
1089 if (imsg.hdr.type != GOT_IMSG_PACKIDX) {
1090 err = got_error(GOT_ERR_PRIVSEP_MSG);
1091 goto done;
1094 if (imsg.fd == -1) {
1095 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1096 goto done;
1099 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1100 if (datalen != sizeof(ipackidx)) {
1101 err = got_error(GOT_ERR_PRIVSEP_LEN);
1102 goto done;
1104 memcpy(&ipackidx, imsg.data, sizeof(ipackidx));
1106 p->len = ipackidx.len;
1107 p->fd = dup(imsg.fd);
1108 if (p->fd == -1) {
1109 err = got_error_from_errno("dup");
1110 goto done;
1112 if (lseek(p->fd, 0, SEEK_SET) == -1) {
1113 err = got_error_from_errno("lseek");
1114 goto done;
1117 #ifndef GOT_PACK_NO_MMAP
1118 p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0);
1119 if (p->map == MAP_FAILED)
1120 p->map = NULL; /* fall back to read(2) */
1121 #endif
1122 err = got_packidx_init_hdr(p, 1, ipackidx.packfile_size);
1123 done:
1124 if (err) {
1125 if (imsg.fd != -1)
1126 close(imsg.fd);
1127 got_packidx_close(p);
1128 } else
1129 *packidx = p;
1130 imsg_free(&imsg);
1131 return err;
1134 static const struct got_error *
1135 receive_pack(struct got_pack **packp, struct imsgbuf *ibuf)
1137 const struct got_error *err = NULL;
1138 struct imsg imsg;
1139 struct got_imsg_pack ipack;
1140 size_t datalen;
1141 struct got_pack *pack;
1143 *packp = NULL;
1145 err = got_privsep_recv_imsg(&imsg, ibuf, 0);
1146 if (err)
1147 return err;
1149 pack = calloc(1, sizeof(*pack));
1150 if (pack == NULL) {
1151 err = got_error_from_errno("calloc");
1152 goto done;
1155 if (imsg.hdr.type != GOT_IMSG_PACK) {
1156 err = got_error(GOT_ERR_PRIVSEP_MSG);
1157 goto done;
1160 if (imsg.fd == -1) {
1161 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1162 goto done;
1165 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1166 if (datalen != sizeof(ipack)) {
1167 err = got_error(GOT_ERR_PRIVSEP_LEN);
1168 goto done;
1170 memcpy(&ipack, imsg.data, sizeof(ipack));
1172 pack->filesize = ipack.filesize;
1173 pack->fd = dup(imsg.fd);
1174 if (pack->fd == -1) {
1175 err = got_error_from_errno("dup");
1176 goto done;
1178 if (lseek(pack->fd, 0, SEEK_SET) == -1) {
1179 err = got_error_from_errno("lseek");
1180 goto done;
1182 pack->path_packfile = strdup(ipack.path_packfile);
1183 if (pack->path_packfile == NULL) {
1184 err = got_error_from_errno("strdup");
1185 goto done;
1188 err = got_delta_cache_alloc(&pack->delta_cache);
1189 if (err)
1190 goto done;
1192 #ifndef GOT_PACK_NO_MMAP
1193 pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
1194 pack->fd, 0);
1195 if (pack->map == MAP_FAILED)
1196 pack->map = NULL; /* fall back to read(2) */
1197 #endif
1198 done:
1199 if (err) {
1200 if (imsg.fd != -1)
1201 close(imsg.fd);
1202 free(pack);
1203 } else
1204 *packp = pack;
1205 imsg_free(&imsg);
1206 return err;
1209 int
1210 main(int argc, char *argv[])
1212 const struct got_error *err = NULL;
1213 struct imsgbuf ibuf;
1214 struct imsg imsg;
1215 struct got_packidx *packidx = NULL;
1216 struct got_pack *pack = NULL;
1217 struct got_object_cache objcache;
1218 FILE *basefile = NULL, *accumfile = NULL, *delta_outfile = NULL;
1220 //static int attached;
1221 //while (!attached) sleep(1);
1223 signal(SIGINT, catch_sigint);
1225 imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
1227 err = got_object_cache_init(&objcache, GOT_OBJECT_CACHE_TYPE_OBJ);
1228 if (err) {
1229 err = got_error_from_errno("got_object_cache_init");
1230 got_privsep_send_error(&ibuf, err);
1231 return 1;
1234 #ifndef PROFILE
1235 /* revoke access to most system calls */
1236 if (pledge("stdio recvfd", NULL) == -1) {
1237 err = got_error_from_errno("pledge");
1238 got_privsep_send_error(&ibuf, err);
1239 return 1;
1241 #endif
1243 err = receive_packidx(&packidx, &ibuf);
1244 if (err) {
1245 got_privsep_send_error(&ibuf, err);
1246 return 1;
1249 err = receive_pack(&pack, &ibuf);
1250 if (err) {
1251 got_privsep_send_error(&ibuf, err);
1252 return 1;
1255 for (;;) {
1256 imsg.fd = -1;
1258 if (sigint_received) {
1259 err = got_error(GOT_ERR_CANCELLED);
1260 break;
1263 err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1264 if (err) {
1265 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1266 err = NULL;
1267 break;
1270 if (imsg.hdr.type == GOT_IMSG_STOP)
1271 break;
1273 switch (imsg.hdr.type) {
1274 case GOT_IMSG_TMPFD:
1275 if (basefile == NULL) {
1276 err = receive_tempfile(&basefile, "w+",
1277 &imsg, &ibuf);
1278 } else if (accumfile == NULL) {
1279 err = receive_tempfile(&accumfile, "w+",
1280 &imsg, &ibuf);
1281 } else
1282 err = got_error(GOT_ERR_PRIVSEP_MSG);
1283 break;
1284 case GOT_IMSG_PACKED_OBJECT_REQUEST:
1285 err = object_request(&imsg, &ibuf, pack, packidx,
1286 &objcache);
1287 break;
1288 case GOT_IMSG_PACKED_RAW_OBJECT_REQUEST:
1289 if (basefile == NULL || accumfile == NULL) {
1290 err = got_error(GOT_ERR_PRIVSEP_MSG);
1291 break;
1293 err = raw_object_request(&imsg, &ibuf, pack, packidx,
1294 &objcache, basefile, accumfile);
1295 break;
1296 case GOT_IMSG_RAW_DELTA_OUTFD:
1297 if (delta_outfile != NULL) {
1298 err = got_error(GOT_ERR_PRIVSEP_MSG);
1299 break;
1301 err = receive_tempfile(&delta_outfile, "w",
1302 &imsg, &ibuf);
1303 break;
1304 case GOT_IMSG_RAW_DELTA_REQUEST:
1305 if (delta_outfile == NULL) {
1306 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1307 break;
1309 err = raw_delta_request(&imsg, &ibuf, delta_outfile,
1310 pack, packidx);
1311 break;
1312 case GOT_IMSG_DELTA_REUSE_REQUEST:
1313 if (delta_outfile == NULL) {
1314 err = got_error(GOT_ERR_PRIVSEP_NO_FD);
1315 break;
1317 err = delta_reuse_request(&imsg, &ibuf,
1318 delta_outfile, pack, packidx);
1319 break;
1320 case GOT_IMSG_COMMIT_REQUEST:
1321 err = commit_request(&imsg, &ibuf, pack, packidx,
1322 &objcache);
1323 break;
1324 case GOT_IMSG_TREE_REQUEST:
1325 err = tree_request(&imsg, &ibuf, pack, packidx,
1326 &objcache);
1327 break;
1328 case GOT_IMSG_BLOB_REQUEST:
1329 if (basefile == NULL || accumfile == NULL) {
1330 err = got_error(GOT_ERR_PRIVSEP_MSG);
1331 break;
1333 err = blob_request(&imsg, &ibuf, pack, packidx,
1334 &objcache, basefile, accumfile);
1335 break;
1336 case GOT_IMSG_TAG_REQUEST:
1337 err = tag_request(&imsg, &ibuf, pack, packidx,
1338 &objcache);
1339 break;
1340 case GOT_IMSG_COMMIT_TRAVERSAL_REQUEST:
1341 err = commit_traversal_request(&imsg, &ibuf, pack,
1342 packidx, &objcache);
1343 break;
1344 default:
1345 err = got_error(GOT_ERR_PRIVSEP_MSG);
1346 break;
1349 if (imsg.fd != -1 && close(imsg.fd) == -1 && err == NULL)
1350 err = got_error_from_errno("close");
1351 imsg_free(&imsg);
1352 if (err)
1353 break;
1356 if (packidx)
1357 got_packidx_close(packidx);
1358 if (pack)
1359 got_pack_close(pack);
1360 got_object_cache_close(&objcache);
1361 imsg_clear(&ibuf);
1362 if (basefile && fclose(basefile) == EOF && err == NULL)
1363 err = got_error_from_errno("fclose");
1364 if (accumfile && fclose(accumfile) == EOF && err == NULL)
1365 err = got_error_from_errno("fclose");
1366 if (delta_outfile && fclose(delta_outfile) == EOF && err == NULL)
1367 err = got_error_from_errno("fclose");
1368 if (err) {
1369 if (!sigint_received && err->code != GOT_ERR_PRIVSEP_PIPE) {
1370 fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
1371 got_privsep_send_error(&ibuf, err);
1374 if (close(GOT_IMSG_FD_CHILD) == -1 && err == NULL)
1375 err = got_error_from_errno("close");
1376 return err ? 1 : 0;