op public repos

Blob

Date:: Sat Mar 17 17:13:19 2018 UTC
Message:: remove GOT_ERR_NO_MEM, we can just use errno in those cases
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/types.h>
18 #include <sys/stat.h>
19 #include <sys/queue.h>
20 
21 #include <dirent.h>
22 #include <fcntl.h>
23 #include <errno.h>
24 #include <stdio.h>
25 #include <stdint.h>
26 #include <stdlib.h>
27 #include <string.h>
28 #include <limits.h>
29 #include <sha1.h>
30 #include <endian.h>
31 #include <zlib.h>
32 
33 #include "got_error.h"
34 #include "got_object.h"
35 #include "got_repository.h"
36 
37 #include "got_lib_sha1.h"
38 #include "got_lib_pack.h"
39 #include "got_lib_path.h"
40 #include "got_lib_delta.h"
41 #include "got_lib_zbuf.h"
42 #include "got_lib_object.h"
43 #include "got_lib_repository.h"
44 
45 #ifndef nitems
46 #define nitems(_a) (sizeof(_a) / sizeof((_a)[0]))
47 #endif
48 
49 #define GOT_PACK_PREFIX		"pack-"
50 #define GOT_PACKFILE_SUFFIX	".pack"
51 #define GOT_PACKIDX_SUFFIX		".idx"
52 #define GOT_PACKFILE_NAMELEN	(strlen(GOT_PACK_PREFIX) + \
53 				SHA1_DIGEST_STRING_LENGTH - 1 + \
54 				strlen(GOT_PACKFILE_SUFFIX))
55 #define GOT_PACKIDX_NAMELEN	(strlen(GOT_PACK_PREFIX) + \
56 				SHA1_DIGEST_STRING_LENGTH - 1 + \
57 				strlen(GOT_PACKIDX_SUFFIX))
58 
59 #ifndef MIN
60 #define	MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
61 #endif
62 
63 static const struct got_error *
64 verify_fanout_table(uint32_t *fanout_table)
65 {
66 	int i;
67 
68 	for (i = 0; i < 0xff - 1; i++) {
69 		if (be32toh(fanout_table[i]) > be32toh(fanout_table[i + 1]))
70 			return got_error(GOT_ERR_BAD_PACKIDX);
71 	}
72 
73 	return NULL;
74 }
75 
76 static const struct got_error *
77 get_packfile_size(size_t *size, const char *path_idx)
78 {
79 	struct stat sb;
80 	char *path_pack;
81 	char base_path[PATH_MAX];
82 	char *dot;
83 
84 	if (strlcpy(base_path, path_idx, PATH_MAX) > PATH_MAX)
85 		return got_error(GOT_ERR_NO_SPACE);
86 
87 	dot = strrchr(base_path, '.');
88 	if (dot == NULL)
89 		return got_error(GOT_ERR_BAD_PATH);
90 	*dot = '\0';
91 	if (asprintf(&path_pack, "%s.pack", base_path) == -1)
92 		return got_error_from_errno();
93 
94 	if (stat(path_pack, &sb) != 0) {
95 		free(path_pack);
96 		return got_error_from_errno();
97 	}
98 
99 	free(path_pack);
100 	*size = sb.st_size;
101 	return 0;
102 }
103 
104 const struct got_error *
105 got_packidx_open(struct got_packidx_v2_hdr **packidx, const char *path)
106 {
107 	struct got_packidx_v2_hdr *p;
108 	FILE *f;
109 	const struct got_error *err = NULL;
110 	size_t n, nobj, packfile_size;
111 	SHA1_CTX ctx;
112 	uint8_t sha1[SHA1_DIGEST_LENGTH];
113 
114 	SHA1Init(&ctx);
115 
116 	f = fopen(path, "rb");
117 	if (f == NULL)
118 		return got_error_from_errno();
119 
120 	err = get_packfile_size(&packfile_size, path);
121 	if (err)
122 		return err;
123 
124 	p = calloc(1, sizeof(*p));
125 	if (p == NULL) {
126 		err = got_error_from_errno();
127 		goto done;
128 	}
129 
130 	n = fread(&p->magic, sizeof(p->magic), 1, f);
131 	if (n != 1) {
132 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
133 		goto done;
134 	}
135 
136 	if (betoh32(p->magic) != GOT_PACKIDX_V2_MAGIC) {
137 		err = got_error(GOT_ERR_BAD_PACKIDX);
138 		goto done;
139 	}
140 
141 	SHA1Update(&ctx, (uint8_t *)&p->magic, sizeof(p->magic));
142 
143 	n = fread(&p->version, sizeof(p->version), 1, f);
144 	if (n != 1) {
145 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
146 		goto done;
147 	}
148 
149 	if (betoh32(p->version) != GOT_PACKIDX_VERSION) {
150 		err = got_error(GOT_ERR_BAD_PACKIDX);
151 		goto done;
152 	}
153 
154 	SHA1Update(&ctx, (uint8_t *)&p->version, sizeof(p->version));
155 
156 	n = fread(&p->fanout_table, sizeof(p->fanout_table), 1, f);
157 	if (n != 1) {
158 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
159 		goto done;
160 	}
161 
162 	err = verify_fanout_table(p->fanout_table);
163 	if (err)
164 		goto done;
165 
166 	SHA1Update(&ctx, (uint8_t *)p->fanout_table, sizeof(p->fanout_table));
167 
168 	nobj = betoh32(p->fanout_table[0xff]);
169 
170 	p->sorted_ids = calloc(nobj, sizeof(*p->sorted_ids));
171 	if (p->sorted_ids == NULL) {
172 		err = got_error_from_errno();
173 		goto done;
174 	}
175 
176 	n = fread(p->sorted_ids, sizeof(*p->sorted_ids), nobj, f);
177 	if (n != nobj) {
178 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
179 		goto done;
180 	}
181 
182 	SHA1Update(&ctx, (uint8_t *)p->sorted_ids,
183 	    nobj * sizeof(*p->sorted_ids));
184 
185 	p->crc32 = calloc(nobj, sizeof(*p->crc32));
186 	if (p->crc32 == NULL) {
187 		err = got_error_from_errno();
188 		goto done;
189 	}
190 
191 	n = fread(p->crc32, sizeof(*p->crc32), nobj, f);
192 	if (n != nobj) {
193 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
194 		goto done;
195 	}
196 
197 	SHA1Update(&ctx, (uint8_t *)p->crc32, nobj * sizeof(*p->crc32));
198 
199 	p->offsets = calloc(nobj, sizeof(*p->offsets));
200 	if (p->offsets == NULL) {
201 		err = got_error_from_errno();
202 		goto done;
203 	}
204 
205 	n = fread(p->offsets, sizeof(*p->offsets), nobj, f);
206 	if (n != nobj) {
207 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
208 		goto done;
209 	}
210 
211 	SHA1Update(&ctx, (uint8_t *)p->offsets, nobj * sizeof(*p->offsets));
212 
213 	/* Large file offsets are contained only in files > 2GB. */
214 	if (packfile_size <= 0x80000000)
215 		goto checksum;
216 
217 	p->large_offsets = calloc(nobj, sizeof(*p->large_offsets));
218 	if (p->large_offsets == NULL) {
219 		err = got_error_from_errno();
220 		goto done;
221 	}
222 
223 	n = fread(p->large_offsets, sizeof(*p->large_offsets), nobj, f);
224 	if (n != nobj) {
225 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
226 		goto done;
227 	}
228 
229 	SHA1Update(&ctx, (uint8_t*)p->large_offsets,
230 	    nobj * sizeof(*p->large_offsets));
231 
232 checksum:
233 	n = fread(&p->trailer, sizeof(p->trailer), 1, f);
234 	if (n != 1) {
235 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
236 		goto done;
237 	}
238 
239 	SHA1Update(&ctx, p->trailer.packfile_sha1, SHA1_DIGEST_LENGTH);
240 	SHA1Final(sha1, &ctx);
241 	if (memcmp(p->trailer.packidx_sha1, sha1, SHA1_DIGEST_LENGTH) != 0)
242 		err = got_error(GOT_ERR_PACKIDX_CSUM);
243 done:
244 	fclose(f);
245 	if (err)
246 		got_packidx_close(p);
247 	else
248 		*packidx = p;
249 	return err;
250 }
251 
252 void
253 got_packidx_close(struct got_packidx_v2_hdr *packidx)
254 {
255 	free(packidx->sorted_ids);
256 	free(packidx->offsets);
257 	free(packidx->crc32);
258 	free(packidx->large_offsets);
259 	free(packidx);
260 }
261 
262 static int
263 is_packidx_filename(const char *name, size_t len)
264 {
265 	if (len != GOT_PACKIDX_NAMELEN)
266 		return 0;
267 
268 	if (strncmp(name, GOT_PACK_PREFIX, strlen(GOT_PACK_PREFIX)) != 0)
269 		return 0;
270 
271 	if (strcmp(name + strlen(GOT_PACK_PREFIX) +
272 	    SHA1_DIGEST_STRING_LENGTH - 1, GOT_PACKIDX_SUFFIX) != 0)
273 		return 0;
274 
275 	return 1;
276 }
277 
278 static off_t
279 get_object_offset(struct got_packidx_v2_hdr *packidx, int idx)
280 {
281 	uint32_t totobj = betoh32(packidx->fanout_table[0xff]);
282 	uint32_t offset = betoh32(packidx->offsets[idx]);
283 	if (offset & GOT_PACKIDX_OFFSET_VAL_IS_LARGE_IDX) {
284 		uint64_t loffset;
285 		idx = offset & GOT_PACKIDX_OFFSET_VAL_MASK;
286 		if (idx < 0 || idx > totobj || packidx->large_offsets == NULL)
287 			return -1;
288 		loffset = betoh64(packidx->large_offsets[idx]);
289 		return (loffset > INT64_MAX ? -1 : (off_t)loffset);
290 	}
291 	return (off_t)(offset & GOT_PACKIDX_OFFSET_VAL_MASK);
292 }
293 
294 static int
295 get_object_idx(struct got_packidx_v2_hdr *packidx, struct got_object_id *id)
296 {
297 	u_int8_t id0 = id->sha1[0];
298 	uint32_t totobj = betoh32(packidx->fanout_table[0xff]);
299 	int i = 0;
300 
301 	if (id0 > 0)
302 		i = betoh32(packidx->fanout_table[id0 - 1]);
303 
304 	while (i < totobj) {
305 		struct got_object_id *oid = &packidx->sorted_ids[i];
306 		int cmp = got_object_id_cmp(id, oid);
307 
308 		if (cmp == 0)
309 			return i;
310 		i++;
311 	}
312 
313 	return -1;
314 }
315 
316 static struct got_packidx_v2_hdr *
317 dup_packidx(struct got_packidx_v2_hdr *packidx)
318 {
319 	struct got_packidx_v2_hdr *p;
320 	size_t nobj;
321 
322 	p = calloc(1, sizeof(*p));
323 	if (p == NULL)
324 		return NULL;
325 
326 	memcpy(p, packidx, sizeof(*p));
327 	p->sorted_ids = NULL;
328 	p->crc32 = NULL;
329 	p->offsets = NULL;
330 	p->large_offsets = NULL;
331 
332 	nobj = betoh32(p->fanout_table[0xff]);
333 
334 	p->sorted_ids = calloc(nobj, sizeof(*p->sorted_ids));
335 	if (p->sorted_ids == NULL)
336 		goto err;
337 	memcpy(p->sorted_ids, packidx->sorted_ids, nobj * sizeof(*p->sorted_ids));
338 
339 	p->crc32 = calloc(nobj, sizeof(*p->crc32));
340 	if (p->crc32 == NULL)
341 		goto err;
342 	memcpy(p->crc32, packidx->crc32, nobj * sizeof(*p->crc32));
343 
344 	p->offsets = calloc(nobj, sizeof(*p->offsets));
345 	if (p->offsets == NULL)
346 		goto err;
347 	memcpy(p->offsets, packidx->offsets, nobj * sizeof(*p->offsets));
348 
349 	if (p->large_offsets) {
350 		p->large_offsets = calloc(nobj, sizeof(*p->large_offsets));
351 		if (p->large_offsets == NULL)
352 			goto err;
353 		memcpy(p->large_offsets, packidx->large_offsets,
354 		    nobj * sizeof(*p->large_offsets));
355 	}
356 
357 	return p;
358 
359 err:
360 	free(p->large_offsets);
361 	free(p->offsets);
362 	free(p->crc32);
363 	free(p->sorted_ids);
364 	free(p);
365 	return NULL;
366 }
367 
368 static void
369 cache_packidx(struct got_packidx_v2_hdr *packidx, struct got_repository *repo)
370 {
371 	int i;
372 
373 	for (i = 0; i < nitems(repo->packidx_cache); i++) {
374 		if (repo->packidx_cache[i] == NULL)
375 			break;
376 	}
377 
378 	if (i == nitems(repo->packidx_cache)) {
379 		got_packidx_close(repo->packidx_cache[i - 1]);
380 		memmove(&repo->packidx_cache[1], &repo->packidx_cache[0],
381 		    sizeof(repo->packidx_cache) -
382 		    sizeof(repo->packidx_cache[0]));
383 		i = 0;
384 	}
385 
386 	repo->packidx_cache[i] = dup_packidx(packidx);
387 }
388 
389 static const struct got_error *
390 search_packidx(struct got_packidx_v2_hdr **packidx, int *idx,
391     struct got_repository *repo, struct got_object_id *id)
392 {
393 	const struct got_error *err;
394 	char *path_packdir;
395 	DIR *packdir;
396 	struct dirent *dent;
397 	char *path_packidx;
398 	int i;
399 
400 	/* Search pack index cache. */
401 	for (i = 0; i < nitems(repo->packidx_cache); i++) {
402 		if (repo->packidx_cache[i] == NULL)
403 			break;
404 		*idx = get_object_idx(repo->packidx_cache[i], id);
405 		if (*idx != -1) {
406 			*packidx = repo->packidx_cache[i];
407 			return NULL;
408 		}
409 	}
410 	/* No luck. Search the filesystem. */
411 
412 	path_packdir = got_repo_get_path_objects_pack(repo);
413 	if (path_packdir == NULL)
414 		return got_error_from_errno();
415 
416 	packdir = opendir(path_packdir);
417 	if (packdir == NULL) {
418 		err = got_error_from_errno();
419 		goto done;
420 	}
421 
422 	while ((dent = readdir(packdir)) != NULL) {
423 		if (!is_packidx_filename(dent->d_name, dent->d_namlen))
424 			continue;
425 
426 		if (asprintf(&path_packidx, "%s/%s", path_packdir,
427 		    dent->d_name) == -1) {
428 			err = got_error_from_errno();
429 			goto done;
430 		}
431 
432 		err = got_packidx_open(packidx, path_packidx);
433 		free(path_packidx);
434 		if (err)
435 			goto done;
436 
437 		*idx = get_object_idx(*packidx, id);
438 		if (*idx != -1) {
439 			err = NULL; /* found the object */
440 			cache_packidx(*packidx, repo);
441 			goto done;
442 		}
443 
444 		got_packidx_close(*packidx);
445 		*packidx = NULL;
446 	}
447 
448 	err = got_error(GOT_ERR_NO_OBJ);
449 done:
450 	free(path_packdir);
451 	if (packdir && closedir(packdir) != 0 && err == 0)
452 		err = got_error_from_errno();
453 	return err;
454 }
455 
456 static const struct got_error *
457 get_packfile_path(char **path_packfile, struct got_repository *repo,
458     struct got_packidx_v2_hdr *packidx)
459 {
460 	char *path_packdir;
461 	char hex[SHA1_DIGEST_STRING_LENGTH];
462 	char *sha1str;
463 
464 	path_packdir = got_repo_get_path_objects_pack(repo);
465 	if (path_packdir == NULL)
466 		return got_error_from_errno();
467 
468 	sha1str = got_sha1_digest_to_str(packidx->trailer.packfile_sha1,
469 	    hex, sizeof(hex));
470 	if (sha1str == NULL)
471 		return got_error(GOT_ERR_PACKIDX_CSUM);
472 
473 	if (asprintf(path_packfile, "%s/%s%s%s", path_packdir,
474 	    GOT_PACK_PREFIX, sha1str, GOT_PACKFILE_SUFFIX) == -1) {
475 		*path_packfile = NULL;
476 		return got_error_from_errno();
477 	}
478 
479 	return NULL;
480 }
481 
482 const struct got_error *
483 read_packfile_hdr(FILE *f, struct got_packidx_v2_hdr *packidx)
484 {
485 	const struct got_error *err = NULL;
486 	uint32_t totobj = betoh32(packidx->fanout_table[0xff]);
487 	struct got_packfile_hdr hdr;
488 	size_t n;
489 
490 	n = fread(&hdr, sizeof(hdr), 1, f);
491 	if (n != 1)
492 		return got_ferror(f, GOT_ERR_BAD_PACKIDX);
493 
494 	if (betoh32(hdr.signature) != GOT_PACKFILE_SIGNATURE ||
495 	    betoh32(hdr.version) != GOT_PACKFILE_VERSION ||
496 	    betoh32(hdr.nobjects) != totobj)
497 		err = got_error(GOT_ERR_BAD_PACKFILE);
498 
499 	return err;
500 }
501 
502 static const struct got_error *
503 open_packfile(FILE **packfile, const char *path_packfile,
504     struct got_repository *repo, struct got_packidx_v2_hdr *packidx)
505 {
506 	const struct got_error *err = NULL;
507 
508 	*packfile = NULL;
509 
510 	*packfile = fopen(path_packfile, "rb");
511 	if (*packfile == NULL) {
512 		err = got_error_from_errno();
513 		return err;
514 	}
515 
516 	if (packidx) {
517 		err = read_packfile_hdr(*packfile, packidx);
518 		if (err) {
519 			fclose(*packfile);
520 			*packfile = NULL;
521 		}
522 	}
523 	return err;
524 }
525 
526 void
527 got_pack_close(struct got_pack *pack)
528 {
529 	fclose(pack->packfile);
530 	pack->packfile = NULL;
531 	free(pack->path_packfile);
532 	pack->path_packfile = NULL;
533 	pack->filesize = 0;
534 }
535 
536 static const struct got_error *
537 cache_pack(struct got_pack **packp, const char *path_packfile,
538     struct got_packidx_v2_hdr *packidx, struct got_repository *repo)
539 {
540 	const struct got_error *err = NULL;
541 	struct got_pack *pack = NULL;
542 	int i;
543 
544 	if (packp)
545 		*packp = NULL;
546 
547 	for (i = 0; i < nitems(repo->packs); i++) {
548 		pack = &repo->packs[i];
549 		if (pack->path_packfile == NULL)
550 			break;
551 		if (strcmp(pack->path_packfile, path_packfile) == 0)
552 			return NULL;
553 	}
554 
555 	if (i == nitems(repo->packs) - 1) {
556 		got_pack_close(&repo->packs[i - 1]);
557 		memmove(&repo->packs[1], &repo->packs[0],
558 		    sizeof(repo->packs) - sizeof(repo->packs[0]));
559 		i = 0;
560 	}
561 
562 	pack = &repo->packs[i];
563 
564 	pack->path_packfile = strdup(path_packfile);
565 	if (pack->path_packfile == NULL) {
566 		err = got_error_from_errno();
567 		goto done;
568 	}
569 
570 	err = open_packfile(&pack->packfile, path_packfile, repo, packidx);
571 	if (err)
572 		goto done;
573 
574 	err = get_packfile_size(&pack->filesize, path_packfile);
575 done:
576 	if (err) {
577 		if (pack)
578 			free(pack->path_packfile);
579 		free(pack);
580 	} else if (packp)
581 		*packp = pack;
582 	return err;
583 }
584 
585 struct got_pack *
586 get_cached_pack(const char *path_packfile, struct got_repository *repo)
587 {
588 	struct got_pack *pack = NULL;
589 	int i;
590 
591 	for (i = 0; i < nitems(repo->packs); i++) {
592 		pack = &repo->packs[i];
593 		if (pack->path_packfile == NULL)
594 			break;
595 		if (strcmp(pack->path_packfile, path_packfile) == 0)
596 			return pack;
597 	}
598 
599 	return NULL;
600 }
601 
602 static const struct got_error *
603 parse_object_type_and_size(uint8_t *type, uint64_t *size, size_t *len,
604     FILE *packfile)
605 {
606 	uint8_t t = 0;
607 	uint64_t s = 0;
608 	uint8_t sizeN;
609 	size_t n;
610 	int i = 0;
611 
612 	do {
613 		/* We do not support size values which don't fit in 64 bit. */
614 		if (i > 9)
615 			return got_error(GOT_ERR_NO_SPACE);
616 
617 		n = fread(&sizeN, sizeof(sizeN), 1, packfile);
618 		if (n != 1)
619 			return got_ferror(packfile, GOT_ERR_BAD_PACKIDX);
620 
621 		if (i == 0) {
622 			t = (sizeN & GOT_PACK_OBJ_SIZE0_TYPE_MASK) >>
623 			    GOT_PACK_OBJ_SIZE0_TYPE_MASK_SHIFT;
624 			s = (sizeN & GOT_PACK_OBJ_SIZE0_VAL_MASK);
625 		} else {
626 			size_t shift = 4 + 7 * (i - 1);
627 			s |= ((sizeN & GOT_PACK_OBJ_SIZE_VAL_MASK) << shift);
628 		}
629 		i++;
630 	} while (sizeN & GOT_PACK_OBJ_SIZE_MORE);
631 
632 	*type = t;
633 	*size = s;
634 	*len = i * sizeof(sizeN);
635 	return NULL;
636 }
637 
638 static const struct got_error *
639 open_plain_object(struct got_object **obj, const char *path_packfile,
640     struct got_object_id *id, uint8_t type, off_t offset, size_t size)
641 {
642 	*obj = calloc(1, sizeof(**obj));
643 	if (*obj == NULL)
644 		return got_error_from_errno();
645 
646 	(*obj)->path_packfile = strdup(path_packfile);
647 	if ((*obj)->path_packfile == NULL) {
648 		const struct got_error *err = got_error_from_errno();
649 		free(*obj);
650 		*obj = NULL;
651 		return err;
652 	}
653 
654 	(*obj)->type = type;
655 	(*obj)->flags = GOT_OBJ_FLAG_PACKED;
656 	(*obj)->hdrlen = 0;
657 	(*obj)->size = size;
658 	memcpy(&(*obj)->id, id, sizeof((*obj)->id));
659 	(*obj)->pack_offset = offset;
660 
661 	return NULL;
662 }
663 
664 static const struct got_error *
665 parse_negative_offset(int64_t *offset, size_t *len, FILE *packfile)
666 {
667 	int64_t o = 0;
668 	uint8_t offN;
669 	size_t n;
670 	int i = 0;
671 
672 	do {
673 		/* We do not support offset values which don't fit in 64 bit. */
674 		if (i > 8)
675 			return got_error(GOT_ERR_NO_SPACE);
676 
677 		n = fread(&offN, sizeof(offN), 1, packfile);
678 		if (n != 1)
679 			return got_ferror(packfile, GOT_ERR_BAD_PACKIDX);
680 
681 		if (i == 0)
682 			o = (offN & GOT_PACK_OBJ_DELTA_OFF_VAL_MASK);
683 		else {
684 			o++;
685 			o <<= 7;
686 			o += (offN & GOT_PACK_OBJ_DELTA_OFF_VAL_MASK);
687 		}
688 		i++;
689 	} while (offN & GOT_PACK_OBJ_DELTA_OFF_MORE);
690 
691 	*offset = o;
692 	*len = i * sizeof(offN);
693 	return NULL;
694 }
695 
696 static const struct got_error *
697 parse_offset_delta(off_t *base_offset, FILE *packfile, off_t offset)
698 {
699 	const struct got_error *err;
700 	int64_t negoffset;
701 	size_t negofflen;
702 
703 	err = parse_negative_offset(&negoffset, &negofflen, packfile);
704 	if (err)
705 		return err;
706 
707 	/* Compute the base object's offset (must be in the same pack file). */
708 	*base_offset = (offset - negoffset);
709 	if (*base_offset <= 0)
710 		return got_error(GOT_ERR_BAD_PACKFILE);
711 
712 	return NULL;
713 }
714 
715 static const struct got_error *
716 resolve_delta_chain(struct got_delta_chain *, struct got_repository *,
717     FILE *, const char *, off_t, size_t, int, size_t);
718 
719 static const struct got_error *
720 add_delta(struct got_delta_chain *deltas, const char *path_packfile,
721     off_t delta_offset, size_t tslen, int delta_type, size_t delta_size,
722     size_t delta_data_offset, uint8_t *delta_buf, size_t delta_len)
723 {
724 	struct got_delta *delta;
725 
726 	delta = got_delta_open(path_packfile, delta_offset, tslen,
727 	    delta_type, delta_size, delta_data_offset, delta_buf,
728 	    delta_len);
729 	if (delta == NULL)
730 		return got_error_from_errno();
731 	/* delta is freed in got_object_close() */
732 	deltas->nentries++;
733 	SIMPLEQ_INSERT_HEAD(&deltas->entries, delta, entry);
734 	return NULL;
735 }
736 
737 static const struct got_error *
738 resolve_offset_delta(struct got_delta_chain *deltas,
739     struct got_repository *repo, FILE *packfile, const char *path_packfile,
740     off_t delta_offset,size_t tslen, int delta_type, size_t delta_size)
741 
742 {
743 	const struct got_error *err;
744 	off_t base_offset;
745 	uint8_t base_type;
746 	uint64_t base_size;
747 	size_t base_tslen;
748 	off_t delta_data_offset;
749 	uint8_t *delta_buf;
750 	size_t delta_len;
751 
752 	err = parse_offset_delta(&base_offset, packfile, delta_offset);
753 	if (err)
754 		return err;
755 
756 	delta_data_offset = ftello(packfile);
757 	if (delta_data_offset == -1)
758 		return got_error_from_errno();
759 
760 	err = got_inflate_to_mem(&delta_buf, &delta_len, packfile);
761 	if (err)
762 		return err;
763 
764 	err = add_delta(deltas, path_packfile, delta_offset, tslen,
765 	    delta_type, delta_size, delta_data_offset, delta_buf, delta_len);
766 	if (err)
767 		return err;
768 
769 	/* An offset delta must be in the same packfile. */
770 	if (fseeko(packfile, base_offset, SEEK_SET) != 0)
771 		return got_error_from_errno();
772 
773 	err = parse_object_type_and_size(&base_type, &base_size, &base_tslen,
774 	    packfile);
775 	if (err)
776 		return err;
777 
778 	return resolve_delta_chain(deltas, repo, packfile, path_packfile,
779 	    base_offset, base_tslen, base_type, base_size);
780 }
781 
782 static const struct got_error *
783 resolve_ref_delta(struct got_delta_chain *deltas, struct got_repository *repo,
784     FILE *packfile, const char *path_packfile, off_t delta_offset,
785     size_t tslen, int delta_type, size_t delta_size)
786 {
787 	const struct got_error *err;
788 	struct got_object_id id;
789 	struct got_packidx_v2_hdr *packidx;
790 	int idx;
791 	off_t base_offset;
792 	uint8_t base_type;
793 	uint64_t base_size;
794 	size_t base_tslen;
795 	size_t n;
796 	char *path_base_packfile;
797 	struct got_pack *base_pack;
798 	off_t delta_data_offset;
799 	uint8_t *delta_buf;
800 	size_t delta_len;
801 
802 	n = fread(&id, sizeof(id), 1, packfile);
803 	if (n != 1)
804 		return got_ferror(packfile, GOT_ERR_IO);
805 
806 	delta_data_offset = ftello(packfile);
807 	if (delta_data_offset == -1)
808 		return got_error_from_errno();
809 
810 	err = got_inflate_to_mem(&delta_buf, &delta_len, packfile);
811 	if (err)
812 		return err;
813 
814 	err = add_delta(deltas, path_packfile, delta_offset, tslen,
815 	    delta_type, delta_size, delta_data_offset, delta_buf, delta_len);
816 	if (err)
817 		return err;
818 
819 	err = search_packidx(&packidx, &idx, repo, &id);
820 	if (err)
821 		return err;
822 
823 	base_offset = get_object_offset(packidx, idx);
824 	if (base_offset == (uint64_t)-1) {
825 		return got_error(GOT_ERR_BAD_PACKIDX);
826 	}
827 
828 	err = get_packfile_path(&path_base_packfile, repo, packidx);
829 	if (err)
830 		return err;
831 
832 	base_pack = get_cached_pack(path_base_packfile, repo);
833 	if (base_pack == NULL) {
834 		err = cache_pack(&base_pack, path_base_packfile, NULL, repo);
835 		if (err)
836 			goto done;
837 	}
838 
839 	if (fseeko(base_pack->packfile, base_offset, SEEK_SET) != 0) {
840 		err = got_error_from_errno();
841 		goto done;
842 	}
843 
844 	err = parse_object_type_and_size(&base_type, &base_size, &base_tslen,
845 	    base_pack->packfile);
846 	if (err)
847 		goto done;
848 
849 	err = resolve_delta_chain(deltas, repo, base_pack->packfile,
850 	    path_base_packfile, base_offset, base_tslen, base_type,
851 	    base_size);
852 done:
853 	free(path_base_packfile);
854 	return err;
855 }
856 
857 static const struct got_error *
858 resolve_delta_chain(struct got_delta_chain *deltas, struct got_repository *repo,
859     FILE *packfile, const char *path_packfile, off_t delta_offset, size_t tslen,
860     int delta_type, size_t delta_size)
861 {
862 	const struct got_error *err = NULL;
863 
864 	switch (delta_type) {
865 	case GOT_OBJ_TYPE_COMMIT:
866 	case GOT_OBJ_TYPE_TREE:
867 	case GOT_OBJ_TYPE_BLOB:
868 	case GOT_OBJ_TYPE_TAG:
869 		/* Plain types are the final delta base. Recursion ends. */
870 		err = add_delta(deltas, path_packfile, delta_offset, tslen,
871 		    delta_type, delta_size, 0, NULL, 0);
872 		break;
873 	case GOT_OBJ_TYPE_OFFSET_DELTA:
874 		err = resolve_offset_delta(deltas, repo, packfile,
875 		    path_packfile, delta_offset, tslen, delta_type,
876 		    delta_size);
877 		break;
878 	case GOT_OBJ_TYPE_REF_DELTA:
879 		err = resolve_ref_delta(deltas, repo, packfile,
880 		    path_packfile, delta_offset, tslen, delta_type,
881 		    delta_size);
882 		break;
883 	default:
884 		return got_error(GOT_ERR_NOT_IMPL);
885 	}
886 
887 	return err;
888 }
889 
890 static const struct got_error *
891 open_delta_object(struct got_object **obj, struct got_repository *repo,
892     struct got_packidx_v2_hdr *packidx, const char *path_packfile,
893     FILE *packfile, struct got_object_id *id, off_t offset, size_t tslen,
894     int delta_type, size_t delta_size)
895 {
896 	const struct got_error *err = NULL;
897 	int resolved_type;
898 
899 	*obj = calloc(1, sizeof(**obj));
900 	if (*obj == NULL)
901 		return got_error_from_errno();
902 
903 	(*obj)->flags = 0;
904 	(*obj)->hdrlen = 0;
905 	(*obj)->size = 0; /* Not known because deltas aren't applied yet. */
906 	memcpy(&(*obj)->id, id, sizeof((*obj)->id));
907 	(*obj)->pack_offset = offset + tslen;
908 
909 	(*obj)->path_packfile = strdup(path_packfile);
910 	if ((*obj)->path_packfile == NULL) {
911 		err = got_error_from_errno();
912 		goto done;
913 	}
914 	(*obj)->flags |= GOT_OBJ_FLAG_PACKED;
915 
916 	SIMPLEQ_INIT(&(*obj)->deltas.entries);
917 	(*obj)->flags |= GOT_OBJ_FLAG_DELTIFIED;
918 
919 	err = resolve_delta_chain(&(*obj)->deltas, repo, packfile,
920 	    path_packfile, offset, tslen, delta_type, delta_size);
921 	if (err)
922 		goto done;
923 
924 	err = got_delta_chain_get_base_type(&resolved_type, &(*obj)->deltas);
925 	if (err)
926 		goto done;
927 	(*obj)->type = resolved_type;
928 
929 done:
930 	if (err) {
931 		got_object_close(*obj);
932 		*obj = NULL;
933 	}
934 	return err;
935 }
936 
937 static const struct got_error *
938 open_packed_object(struct got_object **obj, struct got_repository *repo,
939     struct got_packidx_v2_hdr *packidx, int idx, struct got_object_id *id)
940 {
941 	const struct got_error *err = NULL;
942 	off_t offset;
943 	char *path_packfile;
944 	struct got_pack *pack;
945 	uint8_t type;
946 	uint64_t size;
947 	size_t tslen;
948 
949 	*obj = NULL;
950 
951 	offset = get_object_offset(packidx, idx);
952 	if (offset == (uint64_t)-1)
953 		return got_error(GOT_ERR_BAD_PACKIDX);
954 
955 	err = get_packfile_path(&path_packfile, repo, packidx);
956 	if (err)
957 		return err;
958 
959 	pack = get_cached_pack(path_packfile, repo);
960 	if (pack == NULL) {
961 		err = cache_pack(&pack, path_packfile, packidx, repo);
962 		if (err)
963 			goto done;
964 	}
965 
966 	if (fseeko(pack->packfile, offset, SEEK_SET) != 0) {
967 		err = got_error_from_errno();
968 		goto done;
969 	}
970 
971 	err = parse_object_type_and_size(&type, &size, &tslen, pack->packfile);
972 	if (err)
973 		goto done;
974 
975 	switch (type) {
976 	case GOT_OBJ_TYPE_COMMIT:
977 	case GOT_OBJ_TYPE_TREE:
978 	case GOT_OBJ_TYPE_BLOB:
979 	case GOT_OBJ_TYPE_TAG:
980 		err = open_plain_object(obj, path_packfile, id, type,
981 		    offset + tslen, size);
982 		break;
983 
984 	case GOT_OBJ_TYPE_OFFSET_DELTA:
985 	case GOT_OBJ_TYPE_REF_DELTA:
986 		err = open_delta_object(obj, repo, packidx, path_packfile,
987 		    pack->packfile, id, offset, tslen, type, size);
988 		break;
989 
990 	default:
991 		err = got_error(GOT_ERR_NOT_IMPL);
992 		goto done;
993 	}
994 done:
995 	free(path_packfile);
996 	return err;
997 }
998 
999 const struct got_error *
1000 got_packfile_open_object(struct got_object **obj, struct got_object_id *id,
1001     struct got_repository *repo)
1002 {
1003 	const struct got_error *err = NULL;
1004 	struct got_packidx_v2_hdr *packidx = NULL;
1005 	int idx;
1006 
1007 	err = search_packidx(&packidx, &idx, repo, id);
1008 	if (err)
1009 		return err;
1010 
1011 	err = open_packed_object(obj, repo, packidx, idx, id);
1012 	if (err)
1013 		return err;
1014 
1015 	err = cache_pack(NULL, (*obj)->path_packfile, packidx, repo);
1016 	return err;
1017 }
1018 
1019 static const struct got_error *
1020 get_delta_chain_max_size(uint64_t *max_size, struct got_delta_chain *deltas)
1021 {
1022 	struct got_delta *delta;
1023 	uint64_t base_size = 0, result_size = 0;
1024 
1025 	*max_size = 0;
1026 	SIMPLEQ_FOREACH(delta, &deltas->entries, entry) {
1027 		/* Plain object types are the delta base. */
1028 		if (delta->type != GOT_OBJ_TYPE_COMMIT &&
1029 		    delta->type != GOT_OBJ_TYPE_TREE &&
1030 		    delta->type != GOT_OBJ_TYPE_BLOB &&
1031 		    delta->type != GOT_OBJ_TYPE_TAG) {
1032 			const struct got_error *err;
1033 			err = got_delta_get_sizes(&base_size, &result_size,
1034 			    delta->delta_buf, delta->delta_len);
1035 			if (err)
1036 				return err;
1037 		} else
1038 			base_size = delta->size;
1039 		if (base_size > *max_size)
1040 			*max_size = base_size;
1041 		if (result_size > *max_size)
1042 			*max_size = result_size;
1043 	}
1044 
1045 	return NULL;
1046 }
1047 
1048 static const struct got_error *
1049 dump_delta_chain_to_file(size_t *result_size, struct got_delta_chain *deltas,
1050     FILE *outfile, struct got_pack *pack, struct got_repository *repo)
1051 {
1052 	const struct got_error *err = NULL;
1053 	struct got_delta *delta;
1054 	FILE *base_file = NULL, *accum_file = NULL;
1055 	uint8_t *base_buf = NULL, *accum_buf = NULL;
1056 	size_t accum_size = 0;
1057 	uint64_t max_size;
1058 	int n = 0;
1059 
1060 	*result_size = 0;
1061 
1062 	if (SIMPLEQ_EMPTY(&deltas->entries))
1063 		return got_error(GOT_ERR_BAD_DELTA_CHAIN);
1064 
1065 	/* We process small enough files entirely in memory for speed. */
1066 	err = get_delta_chain_max_size(&max_size, deltas);
1067 	if (err)
1068 		return err;
1069 	if (max_size < GOT_DELTA_RESULT_SIZE_CACHED_MAX) {
1070 		accum_buf = malloc(max_size);
1071 		if (accum_buf == NULL)
1072 			return got_error_from_errno();
1073 	} else {
1074 		base_file = got_opentemp();
1075 		if (base_file == NULL)
1076 			return got_error_from_errno();
1077 
1078 		accum_file = got_opentemp();
1079 		if (accum_file == NULL) {
1080 			err = got_error_from_errno();
1081 			fclose(base_file);
1082 			return err;
1083 		}
1084 	}
1085 
1086 	/* Deltas are ordered in ascending order. */
1087 	SIMPLEQ_FOREACH(delta, &deltas->entries, entry) {
1088 		if (n == 0) {
1089 			size_t base_len;
1090 
1091 			/* Plain object types are the delta base. */
1092 			if (delta->type != GOT_OBJ_TYPE_COMMIT &&
1093 			    delta->type != GOT_OBJ_TYPE_TREE &&
1094 			    delta->type != GOT_OBJ_TYPE_BLOB &&
1095 			    delta->type != GOT_OBJ_TYPE_TAG) {
1096 				err = got_error(GOT_ERR_BAD_DELTA_CHAIN);
1097 				goto done;
1098 			}
1099 
1100 			if (fseeko(pack->packfile, delta->offset + delta->tslen,
1101 			    SEEK_SET) != 0) {
1102 				err = got_error_from_errno();
1103 				goto done;
1104 			}
1105 			if (base_file)
1106 				err = got_inflate_to_file(&base_len,
1107 				    pack->packfile, base_file);
1108 			else {
1109 				err = got_inflate_to_mem(&base_buf, &base_len,
1110 				    pack->packfile);
1111 				if (base_len < max_size) {
1112 					uint8_t *p;
1113 					p = reallocarray(base_buf, 1, max_size);
1114 					if (p == NULL) {
1115 						err = got_error_from_errno();
1116 						goto done;
1117 					}
1118 					base_buf = p;
1119 				}
1120 			}
1121 			if (err)
1122 				goto done;
1123 			n++;
1124 			if (base_file)
1125 				rewind(base_file);
1126 			continue;
1127 		}
1128 
1129 		if (base_buf) {
1130 			err = got_delta_apply_in_mem(base_buf, delta->delta_buf,
1131 			    delta->delta_len, accum_buf, &accum_size);
1132 			n++;
1133 		} else {
1134 			err = got_delta_apply(base_file, delta->delta_buf,
1135 			    delta->delta_len,
1136 			    /* Final delta application writes to output file. */
1137 			    ++n < deltas->nentries ? accum_file : outfile,
1138 			    &accum_size);
1139 		}
1140 		if (err)
1141 			goto done;
1142 
1143 		if (n < deltas->nentries) {
1144 			/* Accumulated delta becomes the new base. */
1145 			if (base_buf) {
1146 				uint8_t *tmp = accum_buf;
1147 				accum_buf = base_buf;
1148 				base_buf = tmp;
1149 			} else {
1150 				FILE *tmp = accum_file;
1151 				accum_file = base_file;
1152 				base_file = tmp;
1153 				rewind(base_file);
1154 				rewind(accum_file);
1155 			}
1156 		}
1157 	}
1158 
1159 done:
1160 	free(base_buf);
1161 	if (accum_buf) {
1162 		size_t len = fwrite(accum_buf, 1, accum_size, outfile);
1163 		free(accum_buf);
1164 		if (len != accum_size)
1165 			return got_ferror(outfile, GOT_ERR_IO);
1166 	}
1167 	if (base_file)
1168 		fclose(base_file);
1169 	if (accum_file)
1170 		fclose(accum_file);
1171 	rewind(outfile);
1172 	if (err == NULL)
1173 		*result_size = accum_size;
1174 	return err;
1175 }
1176 
1177 static const struct got_error *
1178 dump_delta_chain_to_mem(uint8_t **outbuf, size_t *outlen,
1179     struct got_delta_chain *deltas, struct got_pack *pack,
1180     struct got_repository *repo)
1181 {
1182 	const struct got_error *err = NULL;
1183 	struct got_delta *delta;
1184 	uint8_t *base_buf = NULL, *accum_buf = NULL;
1185 	size_t accum_size;
1186 	uint64_t max_size;
1187 	int n = 0;
1188 
1189 	*outbuf = NULL;
1190 	*outlen = 0;
1191 
1192 	if (SIMPLEQ_EMPTY(&deltas->entries))
1193 		return got_error(GOT_ERR_BAD_DELTA_CHAIN);
1194 
1195 	err = get_delta_chain_max_size(&max_size, deltas);
1196 	if (err)
1197 		return err;
1198 	accum_buf = malloc(max_size);
1199 	if (accum_buf == NULL)
1200 		return got_error_from_errno();
1201 
1202 	/* Deltas are ordered in ascending order. */
1203 	SIMPLEQ_FOREACH(delta, &deltas->entries, entry) {
1204 		if (n == 0) {
1205 			size_t base_len;
1206 
1207 			/* Plain object types are the delta base. */
1208 			if (delta->type != GOT_OBJ_TYPE_COMMIT &&
1209 			    delta->type != GOT_OBJ_TYPE_TREE &&
1210 			    delta->type != GOT_OBJ_TYPE_BLOB &&
1211 			    delta->type != GOT_OBJ_TYPE_TAG) {
1212 				err = got_error(GOT_ERR_BAD_DELTA_CHAIN);
1213 				goto done;
1214 			}
1215 
1216 			if (fseeko(pack->packfile, delta->offset + delta->tslen,
1217 			    SEEK_SET) != 0) {
1218 				err = got_error_from_errno();
1219 				goto done;
1220 			}
1221 			err = got_inflate_to_mem(&base_buf, &base_len,
1222 			    pack->packfile);
1223 			if (base_len < max_size) {
1224 				uint8_t *p;
1225 				p = reallocarray(base_buf, 1, max_size);
1226 				if (p == NULL) {
1227 					err = got_error_from_errno();
1228 					goto done;
1229 				}
1230 				base_buf = p;
1231 			}
1232 			if (err)
1233 				goto done;
1234 			n++;
1235 			continue;
1236 		}
1237 
1238 		err = got_delta_apply_in_mem(base_buf, delta->delta_buf,
1239 		    delta->delta_len, accum_buf, &accum_size);
1240 		n++;
1241 		if (err)
1242 			goto done;
1243 
1244 		if (n < deltas->nentries) {
1245 			/* Accumulated delta becomes the new base. */
1246 			uint8_t *tmp = accum_buf;
1247 			accum_buf = base_buf;
1248 			base_buf = tmp;
1249 		}
1250 	}
1251 
1252 done:
1253 	free(base_buf);
1254 	if (err) {
1255 		free(accum_buf);
1256 		*outbuf = NULL;
1257 		*outlen = 0;
1258 	} else {
1259 		*outbuf = accum_buf;
1260 		*outlen = accum_size;
1261 	}
1262 	return err;
1263 }
1264 
1265 const struct got_error *
1266 got_packfile_extract_object(FILE **f, struct got_object *obj,
1267     struct got_repository *repo)
1268 {
1269 	const struct got_error *err = NULL;
1270 	struct got_pack *pack;
1271 
1272 	if ((obj->flags & GOT_OBJ_FLAG_PACKED) == 0)
1273 		return got_error(GOT_ERR_OBJ_NOT_PACKED);
1274 
1275 	pack = get_cached_pack(obj->path_packfile, repo);
1276 	if (pack == NULL) {
1277 		err = cache_pack(&pack, obj->path_packfile, NULL, repo);
1278 		if (err)
1279 			goto done;
1280 	}
1281 
1282 	if ((obj->flags & GOT_OBJ_FLAG_DELTIFIED) == 0) {
1283 		if (fseeko(pack->packfile, obj->pack_offset, SEEK_SET) != 0) {
1284 			err = got_error_from_errno();
1285 			goto done;
1286 		}
1287 
1288 		if (obj->size < GOT_DELTA_RESULT_SIZE_CACHED_MAX) {
1289 			size_t size = obj->size;
1290 			if (size == 0) /* empty file */
1291 				size = 1;
1292 			*f = fmemopen(NULL, size, "w+");
1293 		} else
1294 			*f = got_opentemp();
1295 		if (*f == NULL) {
1296 			err = got_error(GOT_ERR_FILE_OPEN);
1297 			goto done;
1298 		}
1299 		err = got_inflate_to_file(&obj->size, pack->packfile, *f);
1300 	} else {
1301 		uint64_t max_size;
1302 
1303 		err = get_delta_chain_max_size(&max_size, &obj->deltas);
1304 		if (err)
1305 			return err;
1306 		if (max_size < GOT_DELTA_RESULT_SIZE_CACHED_MAX)
1307 			*f = fmemopen(NULL, max_size, "w+");
1308 		else
1309 			*f = got_opentemp();
1310 		if (*f == NULL) {
1311 			err = got_error(GOT_ERR_FILE_OPEN);
1312 			goto done;
1313 		}
1314 		err = dump_delta_chain_to_file(&obj->size, &obj->deltas, *f,
1315 		    pack, repo);
1316 	}
1317 done:
1318 	if (err && *f) {
1319 		fclose(*f);
1320 		*f = NULL;
1321 	}
1322 	return err;
1323 }
1324 
1325 const struct got_error *
1326 got_packfile_extract_object_to_mem(uint8_t **buf, size_t *len,
1327     struct got_object *obj, struct got_repository *repo)
1328 {
1329 	const struct got_error *err = NULL;
1330 	FILE *packfile = NULL;
1331 	struct got_pack *pack;
1332 
1333 	if ((obj->flags & GOT_OBJ_FLAG_PACKED) == 0)
1334 		return got_error(GOT_ERR_OBJ_NOT_PACKED);
1335 
1336 	pack = get_cached_pack(obj->path_packfile, repo);
1337 	if (pack == NULL) {
1338 		err = cache_pack(&pack, obj->path_packfile, NULL, repo);
1339 		if (err)
1340 			goto done;
1341 	}
1342 
1343 	if ((obj->flags & GOT_OBJ_FLAG_DELTIFIED) == 0) {
1344 		if (fseeko(pack->packfile, obj->pack_offset, SEEK_SET) != 0) {
1345 			err = got_error_from_errno();
1346 			goto done;
1347 		}
1348 
1349 		err = got_inflate_to_mem(buf, len, pack->packfile);
1350 	} else
1351 		err = dump_delta_chain_to_mem(buf, len, &obj->deltas, pack,
1352 		    repo);
1353 done:
1354 	if (packfile)
1355 		fclose(packfile);
1356 	return err;
1357 }