Blob


1 /*
2 * Copyright (c) 2020 Ori Bernstein
3 * Copyright (c) 2021, 2022 Stefan Sperling <stsp@openbsd.org>
4 *
5 * Permission to use, copy, modify, and distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
8 *
9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16 */
18 #include <sys/types.h>
19 #include <sys/queue.h>
20 #include <sys/tree.h>
21 #include <sys/uio.h>
23 #include <sha1.h>
24 #include <sha2.h>
25 #include <limits.h>
26 #include <stdio.h>
27 #include <stdint.h>
28 #include <stdlib.h>
29 #include <string.h>
30 #include <time.h>
31 #include <imsg.h>
32 #include <inttypes.h>
33 #include <unistd.h>
35 #include "got_error.h"
36 #include "got_cancel.h"
37 #include "got_object.h"
38 #include "got_reference.h"
39 #include "got_repository_admin.h"
40 #include "got_path.h"
42 #include "got_lib_delta.h"
43 #include "got_lib_object.h"
44 #include "got_lib_object_cache.h"
45 #include "got_lib_object_idset.h"
46 #include "got_lib_privsep.h"
47 #include "got_lib_ratelimit.h"
48 #include "got_lib_pack.h"
49 #include "got_lib_pack_create.h"
50 #include "got_lib_repository.h"
52 struct send_id_arg {
53 struct imsgbuf *ibuf;
54 struct got_object_id *ids[GOT_IMSG_OBJ_ID_LIST_MAX_NIDS];
55 size_t nids;
56 };
58 static const struct got_error *
59 send_id(struct got_object_id *id, void *data, void *arg)
60 {
61 const struct got_error *err = NULL;
62 struct send_id_arg *a = arg;
64 a->ids[a->nids++] = id;
66 if (a->nids >= GOT_IMSG_OBJ_ID_LIST_MAX_NIDS) {
67 err = got_privsep_send_object_idlist(a->ibuf, a->ids, a->nids);
68 if (err)
69 return err;
70 a->nids = 0;
71 }
73 return NULL;
74 }
76 static const struct got_error *
77 send_idset(struct imsgbuf *ibuf, struct got_object_idset *idset)
78 {
79 const struct got_error *err;
80 struct send_id_arg sia;
82 memset(&sia, 0, sizeof(sia));
83 sia.ibuf = ibuf;
84 err = got_object_idset_for_each(idset, send_id, &sia);
85 if (err)
86 return err;
88 if (sia.nids > 0) {
89 err = got_privsep_send_object_idlist(ibuf, sia.ids, sia.nids);
90 if (err)
91 return err;
92 }
94 return got_privsep_send_object_idlist_done(ibuf);
95 }
97 static const struct got_error *
98 recv_reused_delta(struct got_imsg_reused_delta *delta,
99 struct got_object_idset *idset, struct got_pack_metavec *v)
101 struct got_pack_meta *m, *base;
103 if (delta->delta_offset + delta->delta_size < delta->delta_offset ||
104 delta->delta_offset +
105 delta->delta_compressed_size < delta->delta_offset)
106 return got_error(GOT_ERR_BAD_PACKFILE);
108 m = got_object_idset_get(idset, &delta->id);
109 if (m == NULL)
110 return got_error(GOT_ERR_NO_OBJ);
112 base = got_object_idset_get(idset, &delta->base_id);
113 if (base == NULL)
114 return got_error(GOT_ERR_NO_OBJ);
116 m->delta_len = delta->delta_size;
117 m->delta_compressed_len = delta->delta_compressed_size;
118 m->delta_offset = 0;
119 m->prev = base;
120 m->size = delta->result_size;
121 m->reused_delta_offset = delta->delta_offset;
122 m->base_obj_id = got_object_id_dup(&delta->base_id);
123 if (m->base_obj_id == NULL)
124 return got_error_from_errno("got_object_id_dup");
126 return got_pack_add_meta(m, v);
129 const struct got_error *
130 got_pack_search_deltas(struct got_packidx **packidx, struct got_pack **pack,
131 struct got_pack_metavec *v, struct got_object_idset *idset,
132 int ncolored, int nfound, int ntrees, int ncommits,
133 struct got_repository *repo,
134 got_pack_progress_cb progress_cb, void *progress_arg,
135 struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
137 const struct got_error *err = NULL;
138 struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS];
139 size_t ndeltas, i;
141 *packidx = NULL;
142 *pack = NULL;
144 err = got_pack_find_pack_for_reuse(packidx, repo);
145 if (err)
146 return err;
148 if (*packidx == NULL)
149 return NULL;
151 err = got_pack_cache_pack_for_packidx(pack, *packidx, repo);
152 if (err)
153 goto done;
155 if ((*pack)->privsep_child == NULL) {
156 err = got_pack_start_privsep_child(*pack, *packidx);
157 if (err)
158 goto done;
161 err = got_privsep_send_delta_reuse_req((*pack)->privsep_child->ibuf);
162 if (err)
163 goto done;
165 err = send_idset((*pack)->privsep_child->ibuf, idset);
166 if (err)
167 goto done;
169 for (;;) {
170 int done = 0;
172 if (cancel_cb) {
173 err = (*cancel_cb)(cancel_arg);
174 if (err)
175 break;
178 err = got_privsep_recv_reused_deltas(&done, deltas, &ndeltas,
179 (*pack)->privsep_child->ibuf);
180 if (err || done)
181 break;
183 for (i = 0; i < ndeltas; i++) {
184 struct got_imsg_reused_delta *delta = &deltas[i];
185 err = recv_reused_delta(delta, idset, v);
186 if (err)
187 goto done;
190 err = got_pack_report_progress(progress_cb, progress_arg, rl,
191 ncolored, nfound, ntrees, 0L, ncommits,
192 got_object_idset_num_elements(idset), v->nmeta, 0);
193 if (err)
194 break;
196 done:
197 return err;
200 struct recv_painted_commit_arg {
201 int *ncolored;
202 int *nqueued;
203 int *nskip;
204 struct got_object_id_queue *ids;
205 struct got_object_idset *keep;
206 struct got_object_idset *drop;
207 struct got_object_idset *skip;
208 got_pack_progress_cb progress_cb;
209 void *progress_arg;
210 struct got_ratelimit *rl;
211 got_cancel_cb cancel_cb;
212 void *cancel_arg;
213 };
215 static const struct got_error *
216 recv_painted_commit(void *arg, struct got_object_id *id, intptr_t color)
218 const struct got_error *err = NULL;
219 struct recv_painted_commit_arg *a = arg;
220 struct got_object_qid *qid, *tmp;
222 if (a->cancel_cb) {
223 err = a->cancel_cb(a->cancel_arg);
224 if (err)
225 return err;
228 switch (color) {
229 case COLOR_KEEP:
230 err = got_object_idset_add(a->keep, id, NULL);
231 if (err)
232 return err;
233 (*a->ncolored)++;
234 break;
235 case COLOR_DROP:
236 err = got_object_idset_add(a->drop, id, NULL);
237 if (err)
238 return err;
239 (*a->ncolored)++;
240 break;
241 case COLOR_SKIP:
242 err = got_object_idset_add(a->skip, id, NULL);
243 if (err)
244 return err;
245 break;
246 default:
247 /* should not happen */
248 return got_error_fmt(GOT_ERR_NOT_IMPL,
249 "%s invalid commit color %"PRIdPTR, __func__, color);
252 STAILQ_FOREACH_SAFE(qid, a->ids, entry, tmp) {
253 if (got_object_id_cmp(&qid->id, id) != 0)
254 continue;
255 STAILQ_REMOVE(a->ids, qid, got_object_qid, entry);
256 color = (intptr_t)qid->data;
257 got_object_qid_free(qid);
258 (*a->nqueued)--;
259 if (color == COLOR_SKIP)
260 (*a->nskip)--;
261 break;
264 return got_pack_report_progress(a->progress_cb, a->progress_arg, a->rl,
265 *a->ncolored, 0, 0, 0L, 0, 0, 0, 0);
268 static const struct got_error *
269 paint_packed_commits(struct got_pack *pack, struct got_object_id *id,
270 int idx, intptr_t color, int *ncolored, int *nqueued, int *nskip,
271 struct got_object_id_queue *ids,
272 struct got_object_idset *keep, struct got_object_idset *drop,
273 struct got_object_idset *skip, struct got_repository *repo,
274 got_pack_progress_cb progress_cb, void *progress_arg,
275 struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
277 const struct got_error *err = NULL;
278 struct got_object_id_queue next_ids;
279 struct got_object_qid *qid, *tmp;
280 struct recv_painted_commit_arg arg;
282 STAILQ_INIT(&next_ids);
284 err = got_privsep_send_painting_request(pack->privsep_child->ibuf,
285 idx, id, color);
286 if (err)
287 return err;
289 arg.ncolored = ncolored;
290 arg.nqueued = nqueued;
291 arg.nskip = nskip;
292 arg.ids = ids;
293 arg.keep = keep;
294 arg.drop = drop;
295 arg.skip = skip;
296 arg.progress_cb = progress_cb;
297 arg.progress_arg = progress_arg;
298 arg.rl = rl;
299 arg.cancel_cb = cancel_cb;
300 arg.cancel_arg = cancel_arg;
301 err = got_privsep_recv_painted_commits(&next_ids,
302 recv_painted_commit, &arg, pack->privsep_child->ibuf);
303 if (err)
304 return err;
306 STAILQ_FOREACH_SAFE(qid, &next_ids, entry, tmp) {
307 struct got_object_qid *old_id;
308 intptr_t qcolor, ocolor;
309 STAILQ_FOREACH(old_id, ids, entry) {
310 if (got_object_id_cmp(&qid->id, &old_id->id))
311 continue;
312 qcolor = (intptr_t)qid->data;
313 ocolor = (intptr_t)old_id->data;
314 STAILQ_REMOVE(&next_ids, qid, got_object_qid, entry);
315 got_object_qid_free(qid);
316 qid = NULL;
317 if (qcolor != ocolor) {
318 got_pack_paint_commit(old_id, qcolor);
319 if (ocolor == COLOR_SKIP)
320 (*nskip)--;
321 else if (qcolor == COLOR_SKIP)
322 (*nskip)++;
324 break;
327 while (!STAILQ_EMPTY(&next_ids)) {
328 qid = STAILQ_FIRST(&next_ids);
329 STAILQ_REMOVE_HEAD(&next_ids, entry);
330 got_pack_paint_commit(qid, color);
331 STAILQ_INSERT_TAIL(ids, qid, entry);
332 (*nqueued)++;
333 if (color == COLOR_SKIP)
334 (*nskip)++;
337 return err;
340 const struct got_error *
341 got_pack_paint_commits(int *ncolored, struct got_object_id_queue *ids, int nids,
342 struct got_object_idset *keep, struct got_object_idset *drop,
343 struct got_object_idset *skip, struct got_repository *repo,
344 got_pack_progress_cb progress_cb, void *progress_arg,
345 struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
347 const struct got_error *err = NULL;
348 struct got_commit_object *commit = NULL;
349 struct got_packidx *packidx = NULL;
350 struct got_pack *pack = NULL;
351 const struct got_object_id_queue *parents;
352 struct got_object_qid *qid = NULL;
353 int nqueued = nids, nskip = 0;
354 int idx;
356 while (!STAILQ_EMPTY(ids) && nskip != nqueued) {
357 intptr_t color;
359 if (cancel_cb) {
360 err = cancel_cb(cancel_arg);
361 if (err)
362 break;
365 qid = STAILQ_FIRST(ids);
366 STAILQ_REMOVE_HEAD(ids, entry);
367 nqueued--;
368 color = (intptr_t)qid->data;
369 if (color == COLOR_SKIP)
370 nskip--;
372 if (got_object_idset_contains(skip, &qid->id)) {
373 got_object_qid_free(qid);
374 qid = NULL;
375 continue;
377 if (color == COLOR_KEEP &&
378 got_object_idset_contains(keep, &qid->id)) {
379 got_object_qid_free(qid);
380 qid = NULL;
381 continue;
383 if (color == COLOR_DROP &&
384 got_object_idset_contains(drop, &qid->id)) {
385 got_object_qid_free(qid);
386 qid = NULL;
387 continue;
390 /* Pinned pack may have moved to different cache slot. */
391 pack = got_repo_get_pinned_pack(repo);
393 if (packidx && pack) {
394 idx = got_packidx_get_object_idx(packidx, &qid->id);
395 if (idx != -1) {
396 err = paint_packed_commits(pack, &qid->id,
397 idx, color, ncolored, &nqueued, &nskip,
398 ids, keep, drop, skip, repo,
399 progress_cb, progress_arg, rl,
400 cancel_cb, cancel_arg);
401 if (err)
402 break;
403 got_object_qid_free(qid);
404 qid = NULL;
405 continue;
409 switch (color) {
410 case COLOR_KEEP:
411 if (got_object_idset_contains(drop, &qid->id)) {
412 err = got_pack_paint_commit(qid, COLOR_SKIP);
413 if (err)
414 goto done;
415 } else
416 (*ncolored)++;
417 err = got_object_idset_add(keep, &qid->id, NULL);
418 if (err)
419 goto done;
420 break;
421 case COLOR_DROP:
422 if (got_object_idset_contains(keep, &qid->id)) {
423 err = got_pack_paint_commit(qid, COLOR_SKIP);
424 if (err)
425 goto done;
426 } else
427 (*ncolored)++;
428 err = got_object_idset_add(drop, &qid->id, NULL);
429 if (err)
430 goto done;
431 break;
432 case COLOR_SKIP:
433 if (!got_object_idset_contains(skip, &qid->id)) {
434 err = got_object_idset_add(skip, &qid->id,
435 NULL);
436 if (err)
437 goto done;
439 break;
440 default:
441 /* should not happen */
442 err = got_error_fmt(GOT_ERR_NOT_IMPL,
443 "%s invalid commit color %"PRIdPTR, __func__,
444 color);
445 goto done;
448 err = got_pack_report_progress(progress_cb, progress_arg, rl,
449 *ncolored, 0, 0, 0L, 0, 0, 0, 0);
450 if (err)
451 break;
453 err = got_object_open_as_commit(&commit, repo, &qid->id);
454 if (err)
455 break;
457 parents = got_object_commit_get_parent_ids(commit);
458 if (parents) {
459 struct got_object_qid *pid;
460 color = (intptr_t)qid->data;
461 STAILQ_FOREACH(pid, parents, entry) {
462 err = got_pack_queue_commit_id(ids, &pid->id,
463 color, repo);
464 if (err)
465 break;
466 nqueued++;
467 if (color == COLOR_SKIP)
468 nskip++;
472 if (pack == NULL && (commit->flags & GOT_COMMIT_FLAG_PACKED)) {
473 if (packidx == NULL) {
474 err = got_pack_find_pack_for_commit_painting(
475 &packidx, ids, nqueued, repo);
476 if (err)
477 goto done;
479 if (packidx != NULL) {
480 err = got_pack_cache_pack_for_packidx(&pack,
481 packidx, repo);
482 if (err)
483 goto done;
484 if (pack->privsep_child == NULL) {
485 err = got_pack_start_privsep_child(
486 pack, packidx);
487 if (err)
488 goto done;
490 err = got_privsep_init_commit_painting(
491 pack->privsep_child->ibuf);
492 if (err)
493 goto done;
494 err = send_idset(pack->privsep_child->ibuf,
495 keep);
496 if (err)
497 goto done;
498 err = send_idset(pack->privsep_child->ibuf, drop);
499 if (err)
500 goto done;
501 err = send_idset(pack->privsep_child->ibuf, skip);
502 if (err)
503 goto done;
504 err = got_repo_pin_pack(repo, packidx, pack);
505 if (err)
506 goto done;
510 got_object_commit_close(commit);
511 commit = NULL;
513 got_object_qid_free(qid);
514 qid = NULL;
516 done:
517 if (pack) {
518 const struct got_error *pack_err;
519 pack_err = got_privsep_send_painting_commits_done(
520 pack->privsep_child->ibuf);
521 if (err == NULL)
522 err = pack_err;
524 if (commit)
525 got_object_commit_close(commit);
526 got_object_qid_free(qid);
527 got_repo_unpin_pack(repo);
528 return err;
531 struct load_packed_obj_arg {
532 /* output parameters: */
533 struct got_object_id *id;
534 char *dpath;
535 time_t mtime;
537 /* input parameters: */
538 uint32_t seed;
539 int want_meta;
540 struct got_object_idset *idset;
541 struct got_object_idset *idset_exclude;
542 int loose_obj_only;
543 int *ncolored;
544 int *nfound;
545 int *ntrees;
546 got_pack_progress_cb progress_cb;
547 void *progress_arg;
548 struct got_ratelimit *rl;
549 got_cancel_cb cancel_cb;
550 void *cancel_arg;
551 };
553 static const struct got_error *
554 load_packed_commit_id(void *arg, time_t mtime, struct got_object_id *id,
555 struct got_repository *repo)
557 struct load_packed_obj_arg *a = arg;
559 if (got_object_idset_contains(a->idset, id) ||
560 got_object_idset_contains(a->idset_exclude, id))
561 return NULL;
563 return got_pack_add_object(a->want_meta,
564 a->want_meta ? a->idset : a->idset_exclude,
565 id, "", GOT_OBJ_TYPE_COMMIT, mtime, a->seed, a->loose_obj_only,
566 repo, a->ncolored, a->nfound, a->ntrees,
567 a->progress_cb, a->progress_arg, a->rl);
570 static const struct got_error *
571 load_packed_tree_ids(void *arg, struct got_tree_object *tree, time_t mtime,
572 struct got_object_id *id, const char *dpath, struct got_repository *repo)
574 const struct got_error *err;
575 struct load_packed_obj_arg *a = arg;
576 const char *relpath;
578 /*
579 * When we receive a tree's ID and path but not the tree itself,
580 * this tree object was not found in the pack file. This is the
581 * last time we are being called for this optimized traversal.
582 * Return from here and switch to loading objects the slow way.
583 */
584 if (tree == NULL) {
585 free(a->id);
586 a->id = got_object_id_dup(id);
587 if (a->id == NULL) {
588 err = got_error_from_errno("got_object_id_dup");
589 free(a->dpath);
590 a->dpath = NULL;
591 return err;
594 free(a->dpath);
595 a->dpath = strdup(dpath);
596 if (a->dpath == NULL) {
597 err = got_error_from_errno("strdup");
598 free(a->id);
599 a->id = NULL;
600 return err;
603 a->mtime = mtime;
604 return NULL;
607 if (got_object_idset_contains(a->idset, id) ||
608 got_object_idset_contains(a->idset_exclude, id))
609 return NULL;
611 relpath = dpath;
612 while (relpath[0] == '/')
613 relpath++;
615 err = got_pack_add_object(a->want_meta,
616 a->want_meta ? a->idset : a->idset_exclude,
617 id, relpath, GOT_OBJ_TYPE_TREE, mtime, a->seed,
618 a->loose_obj_only, repo, a->ncolored, a->nfound, a->ntrees,
619 a->progress_cb, a->progress_arg, a->rl);
620 if (err)
621 return err;
623 return got_pack_load_tree_entries(NULL, a->want_meta, a->idset,
624 a->idset_exclude, tree, dpath, mtime, a->seed, repo,
625 a->loose_obj_only, a->ncolored, a->nfound, a->ntrees,
626 a->progress_cb, a->progress_arg, a->rl,
627 a->cancel_cb, a->cancel_arg);
630 const struct got_error *
631 got_pack_load_packed_object_ids(int *found_all_objects,
632 struct got_object_id **ours, int nours,
633 struct got_object_id **theirs, int ntheirs,
634 int want_meta, uint32_t seed, struct got_object_idset *idset,
635 struct got_object_idset *idset_exclude, int loose_obj_only,
636 struct got_repository *repo, struct got_packidx *packidx,
637 int *ncolored, int *nfound, int *ntrees,
638 got_pack_progress_cb progress_cb, void *progress_arg,
639 struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
641 const struct got_error *err = NULL;
642 struct load_packed_obj_arg lpa;
644 memset(&lpa, 0, sizeof(lpa));
645 lpa.seed = seed;
646 lpa.want_meta = want_meta;
647 lpa.idset = idset;
648 lpa.idset_exclude = idset_exclude;
649 lpa.loose_obj_only = loose_obj_only;
650 lpa.ncolored = ncolored;
651 lpa.nfound = nfound;
652 lpa.ntrees = ntrees;
653 lpa.progress_cb = progress_cb;
654 lpa.progress_arg = progress_arg;
655 lpa.rl = rl;
656 lpa.cancel_cb = cancel_cb;
657 lpa.cancel_arg = cancel_arg;
659 /* Attempt to load objects via got-read-pack, as far as possible. */
660 err = got_object_enumerate(found_all_objects, load_packed_commit_id,
661 load_packed_tree_ids, &lpa, ours, nours, theirs, ntheirs,
662 packidx, repo);
663 if (err)
664 return err;
666 if (lpa.id == NULL)
667 return NULL;
669 /*
670 * An incomplete tree hierarchy was present in the pack file
671 * and caused loading to be aborted.
672 * Continue loading trees the slow way.
673 */
674 err = got_pack_load_tree(want_meta, idset, idset_exclude,
675 lpa.id, lpa.dpath, lpa.mtime, seed, repo, loose_obj_only,
676 ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
677 cancel_cb, cancel_arg);
678 free(lpa.id);
679 free(lpa.dpath);
680 return err;