Blob


1 /*
2 * Copyright (c) 2022 Stefan Sperling <stsp@openbsd.org>
3 *
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 */
17 #include <sys/queue.h>
18 #include <sys/tree.h>
19 #include <sys/stat.h>
21 #include <errno.h>
22 #include <limits.h>
23 #include <sha1.h>
24 #include <sha2.h>
25 #include <stdio.h>
26 #include <stdlib.h>
27 #include <string.h>
28 #include <unistd.h>
30 #include "got_error.h"
31 #include "got_object.h"
32 #include "got_repository.h"
33 #include "got_path.h"
35 #include "got_lib_delta.h"
36 #include "got_lib_object.h"
37 #include "got_lib_object_cache.h"
38 #include "got_lib_object_parse.h"
39 #include "got_lib_pack.h"
40 #include "got_lib_repository.h"
42 const struct got_error *
43 got_object_open_packed(struct got_object **obj, struct got_object_id *id,
44 struct got_repository *repo)
45 {
46 const struct got_error *err = NULL;
47 struct got_pack *pack = NULL;
48 struct got_packidx *packidx = NULL;
49 int idx;
50 char *path_packfile;
52 err = got_repo_search_packidx(&packidx, &idx, repo, id);
53 if (err)
54 return err;
56 err = got_packidx_get_packfile_path(&path_packfile,
57 packidx->path_packidx);
58 if (err)
59 return err;
61 pack = got_repo_get_cached_pack(repo, path_packfile);
62 if (pack == NULL) {
63 err = got_repo_cache_pack(&pack, repo, path_packfile, packidx);
64 if (err)
65 goto done;
66 }
68 err = got_packfile_open_object(obj, pack, packidx, idx, id);
69 if (err)
70 return err;
71 (*obj)->refcnt++;
73 err = got_repo_cache_object(repo, id, *obj);
74 if (err) {
75 if (err->code == GOT_ERR_OBJ_EXISTS ||
76 err->code == GOT_ERR_OBJ_TOO_LARGE)
77 err = NULL;
78 }
79 done:
80 free(path_packfile);
81 return err;
82 }
84 const struct got_error *
85 got_object_open_from_packfile(struct got_object **obj, struct got_object_id *id,
86 struct got_pack *pack, struct got_packidx *packidx, int obj_idx,
87 struct got_repository *repo)
88 {
89 return got_error(GOT_ERR_NOT_IMPL);
90 }
92 const struct got_error *
93 got_object_read_raw_delta(uint64_t *base_size, uint64_t *result_size,
94 off_t *delta_size, off_t *delta_compressed_size, off_t *delta_offset,
95 off_t *delta_out_offset, struct got_object_id **base_id, int delta_cache_fd,
96 struct got_packidx *packidx, int obj_idx, struct got_object_id *id,
97 struct got_repository *repo)
98 {
99 return got_error(GOT_ERR_NOT_IMPL);
102 const struct got_error *
103 got_object_open(struct got_object **obj, struct got_repository *repo,
104 struct got_object_id *id)
106 const struct got_error *err = NULL;
107 int fd;
109 *obj = got_repo_get_cached_object(repo, id);
110 if (*obj != NULL) {
111 (*obj)->refcnt++;
112 return NULL;
115 err = got_object_open_packed(obj, id, repo);
116 if (err) {
117 if (err->code != GOT_ERR_NO_OBJ)
118 return err;
119 } else
120 return NULL;
122 err = got_object_open_loose_fd(&fd, id, repo);
123 if (err) {
124 if (err->code == GOT_ERR_ERRNO && errno == ENOENT)
125 err = got_error_no_obj(id);
126 return err;
129 err = got_object_read_header(obj, fd);
130 if (err)
131 goto done;
133 memcpy(&(*obj)->id, id, sizeof((*obj)->id));
134 (*obj)->refcnt++;
136 err = got_repo_cache_object(repo, id, *obj);
137 if (err) {
138 if (err->code == GOT_ERR_OBJ_EXISTS ||
139 err->code == GOT_ERR_OBJ_TOO_LARGE)
140 err = NULL;
142 done:
143 if (close(fd) == -1 && err == NULL)
144 err = got_error_from_errno("close");
145 return err;
148 static const struct got_error *
149 wrap_fd(FILE **f, int wrapped_fd)
151 const struct got_error *err = NULL;
152 int fd;
154 if (ftruncate(wrapped_fd, 0L) == -1)
155 return got_error_from_errno("ftruncate");
157 if (lseek(wrapped_fd, 0L, SEEK_SET) == -1)
158 return got_error_from_errno("lseek");
160 fd = dup(wrapped_fd);
161 if (fd == -1)
162 return got_error_from_errno("dup");
164 *f = fdopen(fd, "w+");
165 if (*f == NULL) {
166 err = got_error_from_errno("fdopen");
167 close(fd);
169 return err;
172 static const struct got_error *
173 read_packed_object_raw(uint8_t **outbuf, off_t *size, size_t *hdrlen,
174 int outfd, struct got_pack *pack, struct got_packidx *packidx, int idx,
175 struct got_object_id *id)
177 const struct got_error *err = NULL;
178 uint64_t raw_size = 0;
179 struct got_object *obj;
180 FILE *outfile = NULL, *basefile = NULL, *accumfile = NULL;
182 *outbuf = NULL;
183 *size = 0;
184 *hdrlen = 0;
186 err = got_packfile_open_object(&obj, pack, packidx, idx, id);
187 if (err)
188 return err;
190 if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
191 err = got_pack_get_max_delta_object_size(&raw_size, obj, pack);
192 if (err)
193 goto done;
194 } else
195 raw_size = obj->size;
197 if (raw_size <= GOT_DELTA_RESULT_SIZE_CACHED_MAX) {
198 size_t len;
199 err = got_packfile_extract_object_to_mem(outbuf, &len,
200 obj, pack);
201 if (err)
202 goto done;
203 *size = (off_t)len;
204 } else {
205 /*
206 * XXX This uses 3 file extra descriptors for no good reason.
207 * We should have got_packfile_extract_object_to_fd().
208 */
209 err = wrap_fd(&outfile, outfd);
210 if (err)
211 goto done;
212 err = wrap_fd(&basefile, pack->basefd);
213 if (err)
214 goto done;
215 err = wrap_fd(&accumfile, pack->accumfd);
216 if (err)
217 goto done;
218 err = got_packfile_extract_object(pack, obj, outfile, basefile,
219 accumfile);
220 if (err)
221 goto done;
222 *size = obj->size;
225 *hdrlen = obj->hdrlen;
226 done:
227 got_object_close(obj);
228 if (outfile && fclose(outfile) == EOF && err == NULL)
229 err = got_error_from_errno("fclose");
230 if (basefile && fclose(basefile) == EOF && err == NULL)
231 err = got_error_from_errno("fclose");
232 if (accumfile && fclose(accumfile) == EOF && err == NULL)
233 err = got_error_from_errno("fclose");
234 return err;
238 static void
239 put_raw_object_tempfile(struct got_raw_object *obj)
241 struct got_repository *repo = obj->close_arg;
243 if (obj->tempfile_idx != -1)
244 got_repo_temp_fds_put(obj->tempfile_idx, repo);
247 /* *outfd must be initialized to -1 by caller */
248 const struct got_error *
249 got_object_raw_open(struct got_raw_object **obj, int *outfd,
250 struct got_repository *repo, struct got_object_id *id)
252 const struct got_error *err = NULL;
253 struct got_packidx *packidx = NULL;
254 int idx, tempfd, tempfile_idx;
255 uint8_t *outbuf = NULL;
256 off_t size = 0;
257 size_t hdrlen = 0;
258 char *path_packfile = NULL;
260 *obj = got_repo_get_cached_raw_object(repo, id);
261 if (*obj != NULL) {
262 (*obj)->refcnt++;
263 return NULL;
266 err = got_repo_temp_fds_get(&tempfd, &tempfile_idx, repo);
267 if (err)
268 return err;
270 err = got_repo_search_packidx(&packidx, &idx, repo, id);
271 if (err == NULL) {
272 struct got_pack *pack = NULL;
274 err = got_packidx_get_packfile_path(&path_packfile,
275 packidx->path_packidx);
276 if (err)
277 goto done;
279 pack = got_repo_get_cached_pack(repo, path_packfile);
280 if (pack == NULL) {
281 err = got_repo_cache_pack(&pack, repo, path_packfile,
282 packidx);
283 if (err)
284 goto done;
286 err = read_packed_object_raw(&outbuf, &size, &hdrlen,
287 tempfd, pack, packidx, idx, id);
288 if (err)
289 goto done;
290 } else if (err->code == GOT_ERR_NO_OBJ) {
291 int fd;
293 err = got_object_open_loose_fd(&fd, id, repo);
294 if (err)
295 goto done;
296 err = got_object_read_raw(&outbuf, &size, &hdrlen,
297 GOT_DELTA_RESULT_SIZE_CACHED_MAX, tempfd, id, fd);
298 if (close(fd) == -1 && err == NULL)
299 err = got_error_from_errno("close");
300 if (err)
301 goto done;
304 if (outbuf == NULL) {
305 if (*outfd != -1) {
306 err = got_error_msg(GOT_ERR_NOT_IMPL, "bad outfd");
307 goto done;
310 /*
311 * Duplicate tempfile descriptor to allow use of
312 * fdopen(3) inside got_object_raw_alloc().
313 */
314 *outfd = dup(tempfd);
315 if (*outfd == -1) {
316 err = got_error_from_errno("dup");
317 goto done;
321 err = got_object_raw_alloc(obj, outbuf, outfd,
322 GOT_DELTA_RESULT_SIZE_CACHED_MAX, hdrlen, size);
323 if (err)
324 goto done;
326 err = got_repo_cache_raw_object(repo, id, *obj);
327 if (err) {
328 if (err->code == GOT_ERR_OBJ_EXISTS ||
329 err->code == GOT_ERR_OBJ_TOO_LARGE)
330 err = NULL;
332 done:
333 free(path_packfile);
334 if (err) {
335 if (*obj) {
336 got_object_raw_close(*obj);
337 *obj = NULL;
339 free(outbuf);
340 got_repo_temp_fds_put(tempfile_idx, repo);
341 if (*outfd != -1) {
342 close(*outfd);
343 *outfd = -1;
345 } else {
346 if (((*obj)->f == NULL && (*obj)->fd == -1)) {
347 /* This raw object is not backed by a file. */
348 got_repo_temp_fds_put(tempfile_idx, repo);
349 if (*outfd != -1) {
350 close(*outfd);
351 *outfd = -1;
353 } else {
354 (*obj)->tempfile_idx = tempfile_idx;
355 (*obj)->close_cb = put_raw_object_tempfile;
356 (*obj)->close_arg = repo;
359 return err;
362 static const struct got_error *
363 open_commit(struct got_commit_object **commit,
364 struct got_repository *repo, struct got_object_id *id, int check_cache)
366 const struct got_error *err = NULL;
367 struct got_packidx *packidx = NULL;
368 int idx;
369 char *path_packfile = NULL;
371 if (check_cache) {
372 *commit = got_repo_get_cached_commit(repo, id);
373 if (*commit != NULL) {
374 (*commit)->refcnt++;
375 return NULL;
377 } else
378 *commit = NULL;
380 err = got_repo_search_packidx(&packidx, &idx, repo, id);
381 if (err == NULL) {
382 struct got_pack *pack = NULL;
383 struct got_object *obj;
384 uint8_t *buf;
385 size_t len;
387 err = got_packidx_get_packfile_path(&path_packfile,
388 packidx->path_packidx);
389 if (err)
390 return err;
392 pack = got_repo_get_cached_pack(repo, path_packfile);
393 if (pack == NULL) {
394 err = got_repo_cache_pack(&pack, repo, path_packfile,
395 packidx);
396 if (err)
397 goto done;
399 err = got_packfile_open_object(&obj, pack, packidx, idx, id);
400 if (err)
401 goto done;
402 err = got_packfile_extract_object_to_mem(&buf, &len,
403 obj, pack);
404 got_object_close(obj);
405 if (err)
406 goto done;
407 err = got_object_parse_commit(commit, buf, len, id->algo);
408 free(buf);
409 } else if (err->code == GOT_ERR_NO_OBJ) {
410 int fd;
412 err = got_object_open_loose_fd(&fd, id, repo);
413 if (err)
414 return err;
415 err = got_object_read_commit(commit, fd, id, 0);
416 if (close(fd) == -1 && err == NULL)
417 err = got_error_from_errno("close");
418 if (err)
419 return err;
422 if (err == NULL) {
423 (*commit)->refcnt++;
424 err = got_repo_cache_commit(repo, id, *commit);
425 if (err) {
426 if (err->code == GOT_ERR_OBJ_EXISTS ||
427 err->code == GOT_ERR_OBJ_TOO_LARGE)
428 err = NULL;
431 done:
432 free(path_packfile);
433 return err;
436 const struct got_error *
437 got_object_open_as_commit(struct got_commit_object **commit,
438 struct got_repository *repo, struct got_object_id *id)
440 *commit = got_repo_get_cached_commit(repo, id);
441 if (*commit != NULL) {
442 (*commit)->refcnt++;
443 return NULL;
446 return open_commit(commit, repo, id, 0);
449 const struct got_error *
450 got_object_commit_open(struct got_commit_object **commit,
451 struct got_repository *repo, struct got_object *obj)
453 return open_commit(commit, repo, got_object_get_id(obj), 1);
456 static const struct got_error *
457 open_tree(struct got_tree_object **tree,
458 struct got_repository *repo, struct got_object_id *id, int check_cache)
460 const struct got_error *err = NULL;
461 struct got_packidx *packidx = NULL;
462 int idx;
463 char *path_packfile = NULL;
464 struct got_parsed_tree_entry *entries = NULL;
465 size_t nentries = 0, nentries_alloc = 0, i;
466 uint8_t *buf = NULL;
468 if (check_cache) {
469 *tree = got_repo_get_cached_tree(repo, id);
470 if (*tree != NULL) {
471 (*tree)->refcnt++;
472 return NULL;
474 } else
475 *tree = NULL;
477 err = got_repo_search_packidx(&packidx, &idx, repo, id);
478 if (err == NULL) {
479 struct got_pack *pack = NULL;
480 struct got_object *obj;
481 size_t len;
483 err = got_packidx_get_packfile_path(&path_packfile,
484 packidx->path_packidx);
485 if (err)
486 return err;
488 pack = got_repo_get_cached_pack(repo, path_packfile);
489 if (pack == NULL) {
490 err = got_repo_cache_pack(&pack, repo, path_packfile,
491 packidx);
492 if (err)
493 goto done;
495 err = got_packfile_open_object(&obj, pack, packidx, idx, id);
496 if (err)
497 goto done;
498 err = got_packfile_extract_object_to_mem(&buf, &len,
499 obj, pack);
500 got_object_close(obj);
501 if (err)
502 goto done;
503 err = got_object_parse_tree(&entries, &nentries,
504 &nentries_alloc, buf, len, id->algo);
505 if (err)
506 goto done;
507 } else if (err->code == GOT_ERR_NO_OBJ) {
508 int fd;
510 err = got_object_open_loose_fd(&fd, id, repo);
511 if (err)
512 return err;
513 err = got_object_read_tree(&entries, &nentries,
514 &nentries_alloc, &buf, fd, id);
515 if (close(fd) == -1 && err == NULL)
516 err = got_error_from_errno("close");
517 if (err)
518 goto done;
519 } else
520 goto done;
522 *tree = malloc(sizeof(**tree));
523 if (*tree == NULL) {
524 err = got_error_from_errno("malloc");
525 goto done;
527 (*tree)->entries = calloc(nentries, sizeof(struct got_tree_entry));
528 if ((*tree)->entries == NULL) {
529 err = got_error_from_errno("malloc");
530 goto done;
532 (*tree)->nentries = nentries;
533 (*tree)->refcnt = 0;
535 for (i = 0; i < nentries; i++) {
536 struct got_parsed_tree_entry *pe = &entries[i];
537 struct got_tree_entry *te = &(*tree)->entries[i];
539 if (strlcpy(te->name, pe->name,
540 sizeof(te->name)) >= sizeof(te->name)) {
541 err = got_error(GOT_ERR_NO_SPACE);
542 goto done;
544 memcpy(te->id.hash, pe->id, SHA1_DIGEST_LENGTH);
545 te->mode = pe->mode;
546 te->idx = i;
548 done:
549 free(path_packfile);
550 free(entries);
551 free(buf);
552 if (err == NULL) {
553 (*tree)->refcnt++;
554 err = got_repo_cache_tree(repo, id, *tree);
555 if (err) {
556 if (err->code == GOT_ERR_OBJ_EXISTS ||
557 err->code == GOT_ERR_OBJ_TOO_LARGE)
558 err = NULL;
561 if (err) {
562 if (*tree)
563 free((*tree)->entries);
564 free(*tree);
565 *tree = NULL;
567 return err;
570 const struct got_error *
571 got_object_open_as_tree(struct got_tree_object **tree,
572 struct got_repository *repo, struct got_object_id *id)
574 *tree = got_repo_get_cached_tree(repo, id);
575 if (*tree != NULL) {
576 (*tree)->refcnt++;
577 return NULL;
580 return open_tree(tree, repo, id, 0);
583 const struct got_error *
584 got_object_tree_open(struct got_tree_object **tree,
585 struct got_repository *repo, struct got_object *obj)
587 return open_tree(tree, repo, got_object_get_id(obj), 1);
590 const struct got_error *
591 got_object_open_as_blob(struct got_blob_object **blob,
592 struct got_repository *repo, struct got_object_id *id, size_t blocksize,
593 int outfd)
595 return got_error(GOT_ERR_NOT_IMPL);
598 const struct got_error *
599 got_object_blob_open(struct got_blob_object **blob,
600 struct got_repository *repo, struct got_object *obj, size_t blocksize,
601 int outfd)
603 return got_error(GOT_ERR_NOT_IMPL);
606 static const struct got_error *
607 open_tag(struct got_tag_object **tag, struct got_repository *repo,
608 struct got_object_id *id, int check_cache)
610 const struct got_error *err = NULL;
611 struct got_packidx *packidx = NULL;
612 int idx;
613 char *path_packfile = NULL;
614 struct got_object *obj = NULL;
615 int obj_type = GOT_OBJ_TYPE_ANY;
617 if (check_cache) {
618 *tag = got_repo_get_cached_tag(repo, id);
619 if (*tag != NULL) {
620 (*tag)->refcnt++;
621 return NULL;
623 } else
624 *tag = NULL;
626 err = got_repo_search_packidx(&packidx, &idx, repo, id);
627 if (err == NULL) {
628 struct got_pack *pack = NULL;
629 uint8_t *buf = NULL;
630 size_t len;
632 err = got_packidx_get_packfile_path(&path_packfile,
633 packidx->path_packidx);
634 if (err)
635 return err;
637 pack = got_repo_get_cached_pack(repo, path_packfile);
638 if (pack == NULL) {
639 err = got_repo_cache_pack(&pack, repo, path_packfile,
640 packidx);
641 if (err)
642 goto done;
645 /* Beware of "lightweight" tags: Check object type first. */
646 err = got_packfile_open_object(&obj, pack, packidx, idx, id);
647 if (err)
648 goto done;
649 obj_type = obj->type;
650 if (obj_type != GOT_OBJ_TYPE_TAG) {
651 err = got_error(GOT_ERR_OBJ_TYPE);
652 got_object_close(obj);
653 goto done;
655 err = got_packfile_extract_object_to_mem(&buf, &len,
656 obj, pack);
657 got_object_close(obj);
658 if (err)
659 goto done;
660 err = got_object_parse_tag(tag, buf, len, id->algo);
661 free(buf);
662 } else if (err->code == GOT_ERR_NO_OBJ) {
663 int fd;
665 err = got_object_open_loose_fd(&fd, id, repo);
666 if (err)
667 return err;
668 err = got_object_read_header(&obj, fd);
669 if (close(fd) == -1 && err == NULL)
670 err = got_error_from_errno("close");
671 if (err)
672 return err;
673 obj_type = obj->type;
674 got_object_close(obj);
675 if (obj_type != GOT_OBJ_TYPE_TAG)
676 return got_error(GOT_ERR_OBJ_TYPE);
678 err = got_object_open_loose_fd(&fd, id, repo);
679 if (err)
680 return err;
681 err = got_object_read_tag(tag, fd, id, 0);
682 if (close(fd) == -1 && err == NULL)
683 err = got_error_from_errno("close");
684 if (err)
685 return err;
688 if (err == NULL) {
689 (*tag)->refcnt++;
690 err = got_repo_cache_tag(repo, id, *tag);
691 if (err) {
692 if (err->code == GOT_ERR_OBJ_EXISTS ||
693 err->code == GOT_ERR_OBJ_TOO_LARGE)
694 err = NULL;
697 done:
698 free(path_packfile);
699 return err;
702 const struct got_error *
703 got_object_open_as_tag(struct got_tag_object **tag,
704 struct got_repository *repo, struct got_object_id *id)
706 *tag = got_repo_get_cached_tag(repo, id);
707 if (*tag != NULL) {
708 (*tag)->refcnt++;
709 return NULL;
712 return open_tag(tag, repo, id, 0);
715 const struct got_error *
716 got_object_tag_open(struct got_tag_object **tag,
717 struct got_repository *repo, struct got_object *obj)
719 return open_tag(tag, repo, got_object_get_id(obj), 1);