Blob


1 /*
2 * Copyright (c) 2020 Ori Bernstein
3 * Copyright (c) 2021, 2022 Stefan Sperling <stsp@openbsd.org>
4 *
5 * Permission to use, copy, modify, and distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
8 *
9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16 */
18 #include <sys/types.h>
19 #include <sys/queue.h>
20 #include <sys/tree.h>
21 #include <sys/uio.h>
23 #include <sha1.h>
24 #include <limits.h>
25 #include <stdio.h>
26 #include <stdint.h>
27 #include <stdlib.h>
28 #include <string.h>
29 #include <time.h>
30 #include <imsg.h>
31 #include <inttypes.h>
32 #include <unistd.h>
34 #include "got_error.h"
35 #include "got_cancel.h"
36 #include "got_object.h"
37 #include "got_reference.h"
38 #include "got_repository_admin.h"
39 #include "got_path.h"
41 #include "got_lib_delta.h"
42 #include "got_lib_object.h"
43 #include "got_lib_object_cache.h"
44 #include "got_lib_object_idset.h"
45 #include "got_lib_privsep.h"
46 #include "got_lib_ratelimit.h"
47 #include "got_lib_pack.h"
48 #include "got_lib_pack_create.h"
49 #include "got_lib_repository.h"
51 struct send_id_arg {
52 struct imsgbuf *ibuf;
53 struct got_object_id *ids[GOT_IMSG_OBJ_ID_LIST_MAX_NIDS];
54 size_t nids;
55 };
57 static const struct got_error *
58 send_id(struct got_object_id *id, void *data, void *arg)
59 {
60 const struct got_error *err = NULL;
61 struct send_id_arg *a = arg;
63 a->ids[a->nids++] = id;
65 if (a->nids >= GOT_IMSG_OBJ_ID_LIST_MAX_NIDS) {
66 err = got_privsep_send_object_idlist(a->ibuf, a->ids, a->nids);
67 if (err)
68 return err;
69 a->nids = 0;
70 }
72 return NULL;
73 }
75 static const struct got_error *
76 send_idset(struct imsgbuf *ibuf, struct got_object_idset *idset)
77 {
78 const struct got_error *err;
79 struct send_id_arg sia;
81 memset(&sia, 0, sizeof(sia));
82 sia.ibuf = ibuf;
83 err = got_object_idset_for_each(idset, send_id, &sia);
84 if (err)
85 return err;
87 if (sia.nids > 0) {
88 err = got_privsep_send_object_idlist(ibuf, sia.ids, sia.nids);
89 if (err)
90 return err;
91 }
93 return got_privsep_send_object_idlist_done(ibuf);
94 }
96 static const struct got_error *
97 recv_reused_delta(struct got_imsg_reused_delta *delta,
98 struct got_object_idset *idset, struct got_pack_metavec *v)
99 {
100 struct got_pack_meta *m, *base;
102 if (delta->delta_offset + delta->delta_size < delta->delta_offset ||
103 delta->delta_offset +
104 delta->delta_compressed_size < delta->delta_offset)
105 return got_error(GOT_ERR_BAD_PACKFILE);
107 m = got_object_idset_get(idset, &delta->id);
108 if (m == NULL)
109 return got_error(GOT_ERR_NO_OBJ);
111 base = got_object_idset_get(idset, &delta->base_id);
112 if (base == NULL)
113 return got_error(GOT_ERR_NO_OBJ);
115 m->delta_len = delta->delta_size;
116 m->delta_compressed_len = delta->delta_compressed_size;
117 m->delta_offset = delta->delta_out_offset;
118 m->prev = base;
119 m->size = delta->result_size;
120 m->reused_delta_offset = delta->delta_offset;
121 m->base_obj_id = got_object_id_dup(&delta->base_id);
122 if (m->base_obj_id == NULL)
123 return got_error_from_errno("got_object_id_dup");
125 return got_pack_add_meta(m, v);
128 static const struct got_error *
129 prepare_delta_reuse(struct got_pack *pack, struct got_packidx *packidx,
130 int delta_outfd, struct got_repository *repo)
132 const struct got_error *err = NULL;
134 if (!pack->child_has_delta_outfd) {
135 int outfd_child;
136 outfd_child = dup(delta_outfd);
137 if (outfd_child == -1) {
138 err = got_error_from_errno("dup");
139 goto done;
141 err = got_privsep_send_raw_delta_outfd(
142 pack->privsep_child->ibuf, outfd_child);
143 if (err)
144 goto done;
145 pack->child_has_delta_outfd = 1;
148 err = got_privsep_send_delta_reuse_req(pack->privsep_child->ibuf);
149 done:
150 return err;
153 const struct got_error *
154 got_pack_search_deltas(struct got_pack_metavec *v,
155 struct got_object_idset *idset, int delta_cache_fd,
156 int ncolored, int nfound, int ntrees, int ncommits,
157 struct got_repository *repo,
158 got_pack_progress_cb progress_cb, void *progress_arg,
159 struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
161 const struct got_error *err = NULL;
162 struct got_packidx *packidx;
163 struct got_pack *pack;
164 struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS];
165 size_t ndeltas, i;
167 err = got_pack_find_pack_for_reuse(&packidx, repo);
168 if (err)
169 return err;
171 if (packidx == NULL)
172 return NULL;
174 err = got_pack_cache_pack_for_packidx(&pack, packidx, repo);
175 if (err)
176 return err;
178 if (pack->privsep_child == NULL) {
179 err = got_pack_start_privsep_child(pack, packidx);
180 if (err)
181 return err;
184 err = prepare_delta_reuse(pack, packidx, delta_cache_fd, repo);
185 if (err)
186 return err;
188 err = send_idset(pack->privsep_child->ibuf, idset);
189 if (err)
190 return err;
192 for (;;) {
193 int done = 0;
195 if (cancel_cb) {
196 err = (*cancel_cb)(cancel_arg);
197 if (err)
198 break;
201 err = got_privsep_recv_reused_deltas(&done, deltas, &ndeltas,
202 pack->privsep_child->ibuf);
203 if (err || done)
204 break;
206 for (i = 0; i < ndeltas; i++) {
207 struct got_imsg_reused_delta *delta = &deltas[i];
208 err = recv_reused_delta(delta, idset, v);
209 if (err)
210 goto done;
213 err = got_pack_report_progress(progress_cb, progress_arg, rl,
214 ncolored, nfound, ntrees, 0L, ncommits,
215 got_object_idset_num_elements(idset), v->nmeta, 0);
216 if (err)
217 break;
219 done:
220 return err;
223 struct recv_painted_commit_arg {
224 int *ncolored;
225 int *nqueued;
226 int *nskip;
227 struct got_object_id_queue *ids;
228 struct got_object_idset *keep;
229 struct got_object_idset *drop;
230 struct got_object_idset *skip;
231 got_pack_progress_cb progress_cb;
232 void *progress_arg;
233 struct got_ratelimit *rl;
234 got_cancel_cb cancel_cb;
235 void *cancel_arg;
236 };
238 static const struct got_error *
239 recv_painted_commit(void *arg, struct got_object_id *id, intptr_t color)
241 const struct got_error *err = NULL;
242 struct recv_painted_commit_arg *a = arg;
243 struct got_object_qid *qid, *tmp;
245 if (a->cancel_cb) {
246 err = a->cancel_cb(a->cancel_arg);
247 if (err)
248 return err;
251 switch (color) {
252 case COLOR_KEEP:
253 err = got_object_idset_add(a->keep, id, NULL);
254 if (err)
255 return err;
256 (*a->ncolored)++;
257 break;
258 case COLOR_DROP:
259 err = got_object_idset_add(a->drop, id, NULL);
260 if (err)
261 return err;
262 (*a->ncolored)++;
263 break;
264 case COLOR_SKIP:
265 err = got_object_idset_add(a->skip, id, NULL);
266 if (err)
267 return err;
268 break;
269 default:
270 /* should not happen */
271 return got_error_fmt(GOT_ERR_NOT_IMPL,
272 "%s invalid commit color %"PRIdPTR, __func__, color);
275 STAILQ_FOREACH_SAFE(qid, a->ids, entry, tmp) {
276 if (got_object_id_cmp(&qid->id, id) != 0)
277 continue;
278 STAILQ_REMOVE(a->ids, qid, got_object_qid, entry);
279 color = (intptr_t)qid->data;
280 got_object_qid_free(qid);
281 (*a->nqueued)--;
282 if (color == COLOR_SKIP)
283 (*a->nskip)--;
284 break;
287 return got_pack_report_progress(a->progress_cb, a->progress_arg, a->rl,
288 *a->ncolored, 0, 0, 0L, 0, 0, 0, 0);
291 static const struct got_error *
292 paint_packed_commits(struct got_pack *pack, struct got_object_id *id,
293 int idx, intptr_t color, int *ncolored, int *nqueued, int *nskip,
294 struct got_object_id_queue *ids,
295 struct got_object_idset *keep, struct got_object_idset *drop,
296 struct got_object_idset *skip, struct got_repository *repo,
297 got_pack_progress_cb progress_cb, void *progress_arg,
298 struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
300 const struct got_error *err = NULL;
301 struct got_object_id_queue next_ids;
302 struct got_object_qid *qid, *tmp;
303 struct recv_painted_commit_arg arg;
305 STAILQ_INIT(&next_ids);
307 err = got_privsep_send_painting_request(pack->privsep_child->ibuf,
308 idx, id, color);
309 if (err)
310 return err;
312 arg.ncolored = ncolored;
313 arg.nqueued = nqueued;
314 arg.nskip = nskip;
315 arg.ids = ids;
316 arg.keep = keep;
317 arg.drop = drop;
318 arg.skip = skip;
319 arg.progress_cb = progress_cb;
320 arg.progress_arg = progress_arg;
321 arg.rl = rl;
322 arg.cancel_cb = cancel_cb;
323 arg.cancel_arg = cancel_arg;
324 err = got_privsep_recv_painted_commits(&next_ids,
325 recv_painted_commit, &arg, pack->privsep_child->ibuf);
326 if (err)
327 return err;
329 STAILQ_FOREACH_SAFE(qid, &next_ids, entry, tmp) {
330 struct got_object_qid *old_id;
331 intptr_t qcolor, ocolor;
332 STAILQ_FOREACH(old_id, ids, entry) {
333 if (got_object_id_cmp(&qid->id, &old_id->id))
334 continue;
335 qcolor = (intptr_t)qid->data;
336 ocolor = (intptr_t)old_id->data;
337 STAILQ_REMOVE(&next_ids, qid, got_object_qid, entry);
338 got_object_qid_free(qid);
339 qid = NULL;
340 if (qcolor != ocolor) {
341 got_pack_paint_commit(old_id, qcolor);
342 if (ocolor == COLOR_SKIP)
343 (*nskip)--;
344 else if (qcolor == COLOR_SKIP)
345 (*nskip)++;
347 break;
350 while (!STAILQ_EMPTY(&next_ids)) {
351 qid = STAILQ_FIRST(&next_ids);
352 STAILQ_REMOVE_HEAD(&next_ids, entry);
353 got_pack_paint_commit(qid, color);
354 STAILQ_INSERT_TAIL(ids, qid, entry);
355 (*nqueued)++;
356 if (color == COLOR_SKIP)
357 (*nskip)++;
360 return err;
363 const struct got_error *
364 got_pack_paint_commits(int *ncolored, struct got_object_id_queue *ids, int nids,
365 struct got_object_idset *keep, struct got_object_idset *drop,
366 struct got_object_idset *skip, struct got_repository *repo,
367 got_pack_progress_cb progress_cb, void *progress_arg,
368 struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
370 const struct got_error *err = NULL;
371 struct got_commit_object *commit = NULL;
372 struct got_packidx *packidx = NULL;
373 struct got_pack *pack = NULL;
374 const struct got_object_id_queue *parents;
375 struct got_object_qid *qid = NULL;
376 int nqueued = nids, nskip = 0;
377 int idx;
379 while (!STAILQ_EMPTY(ids) && nskip != nqueued) {
380 intptr_t color;
382 if (cancel_cb) {
383 err = cancel_cb(cancel_arg);
384 if (err)
385 break;
388 qid = STAILQ_FIRST(ids);
389 STAILQ_REMOVE_HEAD(ids, entry);
390 nqueued--;
391 color = (intptr_t)qid->data;
392 if (color == COLOR_SKIP)
393 nskip--;
395 if (got_object_idset_contains(skip, &qid->id)) {
396 got_object_qid_free(qid);
397 qid = NULL;
398 continue;
400 if (color == COLOR_KEEP &&
401 got_object_idset_contains(keep, &qid->id)) {
402 got_object_qid_free(qid);
403 qid = NULL;
404 continue;
406 if (color == COLOR_DROP &&
407 got_object_idset_contains(drop, &qid->id)) {
408 got_object_qid_free(qid);
409 qid = NULL;
410 continue;
413 /* Pinned pack may have moved to different cache slot. */
414 pack = got_repo_get_pinned_pack(repo);
416 if (packidx && pack) {
417 idx = got_packidx_get_object_idx(packidx, &qid->id);
418 if (idx != -1) {
419 err = paint_packed_commits(pack, &qid->id,
420 idx, color, ncolored, &nqueued, &nskip,
421 ids, keep, drop, skip, repo,
422 progress_cb, progress_arg, rl,
423 cancel_cb, cancel_arg);
424 if (err)
425 break;
426 got_object_qid_free(qid);
427 qid = NULL;
428 continue;
432 switch (color) {
433 case COLOR_KEEP:
434 if (got_object_idset_contains(drop, &qid->id)) {
435 err = got_pack_paint_commit(qid, COLOR_SKIP);
436 if (err)
437 goto done;
438 } else
439 (*ncolored)++;
440 err = got_object_idset_add(keep, &qid->id, NULL);
441 if (err)
442 goto done;
443 break;
444 case COLOR_DROP:
445 if (got_object_idset_contains(keep, &qid->id)) {
446 err = got_pack_paint_commit(qid, COLOR_SKIP);
447 if (err)
448 goto done;
449 } else
450 (*ncolored)++;
451 err = got_object_idset_add(drop, &qid->id, NULL);
452 if (err)
453 goto done;
454 break;
455 case COLOR_SKIP:
456 if (!got_object_idset_contains(skip, &qid->id)) {
457 err = got_object_idset_add(skip, &qid->id,
458 NULL);
459 if (err)
460 goto done;
462 break;
463 default:
464 /* should not happen */
465 err = got_error_fmt(GOT_ERR_NOT_IMPL,
466 "%s invalid commit color %"PRIdPTR, __func__,
467 color);
468 goto done;
471 err = got_pack_report_progress(progress_cb, progress_arg, rl,
472 *ncolored, 0, 0, 0L, 0, 0, 0, 0);
473 if (err)
474 break;
476 err = got_object_open_as_commit(&commit, repo, &qid->id);
477 if (err)
478 break;
480 parents = got_object_commit_get_parent_ids(commit);
481 if (parents) {
482 struct got_object_qid *pid;
483 color = (intptr_t)qid->data;
484 STAILQ_FOREACH(pid, parents, entry) {
485 err = got_pack_queue_commit_id(ids, &pid->id,
486 color, repo);
487 if (err)
488 break;
489 nqueued++;
490 if (color == COLOR_SKIP)
491 nskip++;
495 if (pack == NULL && (commit->flags & GOT_COMMIT_FLAG_PACKED)) {
496 if (packidx == NULL) {
497 err = got_pack_find_pack_for_commit_painting(
498 &packidx, ids, nqueued, repo);
499 if (err)
500 goto done;
502 if (packidx != NULL) {
503 err = got_pack_cache_pack_for_packidx(&pack,
504 packidx, repo);
505 if (err)
506 goto done;
507 if (pack->privsep_child == NULL) {
508 err = got_pack_start_privsep_child(
509 pack, packidx);
510 if (err)
511 goto done;
513 err = got_privsep_init_commit_painting(
514 pack->privsep_child->ibuf);
515 if (err)
516 goto done;
517 err = send_idset(pack->privsep_child->ibuf,
518 keep);
519 if (err)
520 goto done;
521 err = send_idset(pack->privsep_child->ibuf, drop);
522 if (err)
523 goto done;
524 err = send_idset(pack->privsep_child->ibuf, skip);
525 if (err)
526 goto done;
527 err = got_repo_pin_pack(repo, packidx, pack);
528 if (err)
529 goto done;
533 got_object_commit_close(commit);
534 commit = NULL;
536 got_object_qid_free(qid);
537 qid = NULL;
539 done:
540 if (pack) {
541 const struct got_error *pack_err;
542 pack_err = got_privsep_send_painting_commits_done(
543 pack->privsep_child->ibuf);
544 if (err == NULL)
545 err = pack_err;
547 if (commit)
548 got_object_commit_close(commit);
549 got_object_qid_free(qid);
550 got_repo_unpin_pack(repo);
551 return err;
554 struct load_packed_obj_arg {
555 /* output parameters: */
556 struct got_object_id *id;
557 char *dpath;
558 time_t mtime;
560 /* input parameters: */
561 uint32_t seed;
562 int want_meta;
563 struct got_object_idset *idset;
564 struct got_object_idset *idset_exclude;
565 int loose_obj_only;
566 int *ncolored;
567 int *nfound;
568 int *ntrees;
569 got_pack_progress_cb progress_cb;
570 void *progress_arg;
571 struct got_ratelimit *rl;
572 got_cancel_cb cancel_cb;
573 void *cancel_arg;
574 };
576 static const struct got_error *
577 load_packed_commit_id(void *arg, time_t mtime, struct got_object_id *id,
578 struct got_repository *repo)
580 struct load_packed_obj_arg *a = arg;
582 if (got_object_idset_contains(a->idset, id) ||
583 got_object_idset_contains(a->idset_exclude, id))
584 return NULL;
586 return got_pack_add_object(a->want_meta,
587 a->want_meta ? a->idset : a->idset_exclude,
588 id, "", GOT_OBJ_TYPE_COMMIT, mtime, a->seed, a->loose_obj_only,
589 repo, a->ncolored, a->nfound, a->ntrees,
590 a->progress_cb, a->progress_arg, a->rl);
593 static const struct got_error *
594 load_packed_tree_ids(void *arg, struct got_tree_object *tree, time_t mtime,
595 struct got_object_id *id, const char *dpath, struct got_repository *repo)
597 const struct got_error *err;
598 struct load_packed_obj_arg *a = arg;
599 const char *relpath;
601 /*
602 * When we receive a tree's ID and path but not the tree itself,
603 * this tree object was not found in the pack file. This is the
604 * last time we are being called for this optimized traversal.
605 * Return from here and switch to loading objects the slow way.
606 */
607 if (tree == NULL) {
608 free(a->id);
609 a->id = got_object_id_dup(id);
610 if (a->id == NULL) {
611 err = got_error_from_errno("got_object_id_dup");
612 free(a->dpath);
613 a->dpath = NULL;
614 return err;
617 free(a->dpath);
618 a->dpath = strdup(dpath);
619 if (a->dpath == NULL) {
620 err = got_error_from_errno("strdup");
621 free(a->id);
622 a->id = NULL;
623 return err;
626 a->mtime = mtime;
627 return NULL;
630 if (got_object_idset_contains(a->idset, id) ||
631 got_object_idset_contains(a->idset_exclude, id))
632 return NULL;
634 relpath = dpath;
635 while (relpath[0] == '/')
636 relpath++;
638 err = got_pack_add_object(a->want_meta,
639 a->want_meta ? a->idset : a->idset_exclude,
640 id, relpath, GOT_OBJ_TYPE_TREE, mtime, a->seed,
641 a->loose_obj_only, repo, a->ncolored, a->nfound, a->ntrees,
642 a->progress_cb, a->progress_arg, a->rl);
643 if (err)
644 return err;
646 return got_pack_load_tree_entries(NULL, a->want_meta, a->idset,
647 a->idset_exclude, tree, dpath, mtime, a->seed, repo,
648 a->loose_obj_only, a->ncolored, a->nfound, a->ntrees,
649 a->progress_cb, a->progress_arg, a->rl,
650 a->cancel_cb, a->cancel_arg);
653 const struct got_error *
654 got_pack_load_packed_object_ids(int *found_all_objects,
655 struct got_object_id **ours, int nours,
656 struct got_object_id **theirs, int ntheirs,
657 int want_meta, uint32_t seed, struct got_object_idset *idset,
658 struct got_object_idset *idset_exclude, int loose_obj_only,
659 struct got_repository *repo, struct got_packidx *packidx,
660 int *ncolored, int *nfound, int *ntrees,
661 got_pack_progress_cb progress_cb, void *progress_arg,
662 struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
664 const struct got_error *err = NULL;
665 struct load_packed_obj_arg lpa;
667 memset(&lpa, 0, sizeof(lpa));
668 lpa.seed = seed;
669 lpa.want_meta = want_meta;
670 lpa.idset = idset;
671 lpa.idset_exclude = idset_exclude;
672 lpa.loose_obj_only = loose_obj_only;
673 lpa.ncolored = ncolored;
674 lpa.nfound = nfound;
675 lpa.ntrees = ntrees;
676 lpa.progress_cb = progress_cb;
677 lpa.progress_arg = progress_arg;
678 lpa.rl = rl;
679 lpa.cancel_cb = cancel_cb;
680 lpa.cancel_arg = cancel_arg;
682 /* Attempt to load objects via got-read-pack, as far as possible. */
683 err = got_object_enumerate(found_all_objects, load_packed_commit_id,
684 load_packed_tree_ids, &lpa, ours, nours, theirs, ntheirs,
685 packidx, repo);
686 if (err)
687 return err;
689 if (lpa.id == NULL)
690 return NULL;
692 /*
693 * An incomplete tree hierarchy was present in the pack file
694 * and caused loading to be aborted.
695 * Continue loading trees the slow way.
696 */
697 err = got_pack_load_tree(want_meta, idset, idset_exclude,
698 lpa.id, lpa.dpath, lpa.mtime, seed, repo, loose_obj_only,
699 ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
700 cancel_cb, cancel_arg);
701 free(lpa.id);
702 free(lpa.dpath);
703 return err;