op public repos

Blob

Date:: Thu May 23 19:04:23 2019 UTC
Message:: replace got_pack_get_packfile_size() with simple fstat()
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018, 2019 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/types.h>
18 #include <sys/stat.h>
19 #include <sys/queue.h>
20 #include <sys/uio.h>
21 #include <sys/mman.h>
22 
23 #include <fcntl.h>
24 #include <errno.h>
25 #include <stdio.h>
26 #include <stdint.h>
27 #include <stdlib.h>
28 #include <string.h>
29 #include <limits.h>
30 #include <sha1.h>
31 #include <endian.h>
32 #include <zlib.h>
33 #include <imsg.h>
34 
35 #include "got_error.h"
36 #include "got_object.h"
37 #include "got_opentemp.h"
38 #include "got_path.h"
39 
40 #include "got_lib_sha1.h"
41 #include "got_lib_delta.h"
42 #include "got_lib_inflate.h"
43 #include "got_lib_object.h"
44 #include "got_lib_privsep.h"
45 #include "got_lib_pack.h"
46 
47 #ifndef nitems
48 #define nitems(_a) (sizeof(_a) / sizeof((_a)[0]))
49 #endif
50 
51 #ifndef MIN
52 #define	MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
53 #endif
54 
55 static const struct got_error *
56 verify_fanout_table(uint32_t *fanout_table)
57 {
58 	int i;
59 
60 	for (i = 0; i < 0xff - 1; i++) {
61 		if (be32toh(fanout_table[i]) > be32toh(fanout_table[i + 1]))
62 			return got_error(GOT_ERR_BAD_PACKIDX);
63 	}
64 
65 	return NULL;
66 }
67 
68 const struct got_error *
69 got_packidx_init_hdr(struct got_packidx *p, int verify)
70 {
71 	const struct got_error *err = NULL;
72 	struct got_packidx_v2_hdr *h;
73 	SHA1_CTX ctx;
74 	uint8_t sha1[SHA1_DIGEST_LENGTH];
75 	size_t nobj, len_fanout, len_ids, offset, remain;
76 	ssize_t n;
77 
78 	SHA1Init(&ctx);
79 
80 	h = &p->hdr;
81 	offset = 0;
82 	remain = p->len;
83 
84 	if (remain < sizeof(*h->magic)) {
85 		err = got_error(GOT_ERR_BAD_PACKIDX);
86 		goto done;
87 	}
88 	if (p->map)
89 		h->magic = (uint32_t *)(p->map + offset);
90 	else {
91 		h->magic = malloc(sizeof(*h->magic));
92 		if (h->magic == NULL) {
93 			err = got_error_from_errno("malloc");
94 			goto done;
95 		}
96 		n = read(p->fd, h->magic, sizeof(*h->magic));
97 		if (n < 0)
98 			err = got_error_from_errno("read");
99 		else if (n != sizeof(*h->magic)) {
100 			err = got_error(GOT_ERR_BAD_PACKIDX);
101 			goto done;
102 		}
103 	}
104 	if (betoh32(*h->magic) != GOT_PACKIDX_V2_MAGIC) {
105 		err = got_error(GOT_ERR_BAD_PACKIDX);
106 		goto done;
107 	}
108 	offset += sizeof(*h->magic);
109 	remain -= sizeof(*h->magic);
110 
111 	if (verify)
112 		SHA1Update(&ctx, (uint8_t *)h->magic, sizeof(*h->magic));
113 
114 	if (remain < sizeof(*h->version)) {
115 		err = got_error(GOT_ERR_BAD_PACKIDX);
116 		goto done;
117 	}
118 	if (p->map)
119 		h->version = (uint32_t *)(p->map + offset);
120 	else {
121 		h->version = malloc(sizeof(*h->version));
122 		if (h->version == NULL) {
123 			err = got_error_from_errno("malloc");
124 			goto done;
125 		}
126 		n = read(p->fd, h->version, sizeof(*h->version));
127 		if (n < 0)
128 			err = got_error_from_errno("read");
129 		else if (n != sizeof(*h->version)) {
130 			err = got_error(GOT_ERR_BAD_PACKIDX);
131 			goto done;
132 		}
133 	}
134 	if (betoh32(*h->version) != GOT_PACKIDX_VERSION) {
135 		err = got_error(GOT_ERR_BAD_PACKIDX);
136 		goto done;
137 	}
138 	offset += sizeof(*h->version);
139 	remain -= sizeof(*h->version);
140 
141 	if (verify)
142 		SHA1Update(&ctx, (uint8_t *)h->version, sizeof(*h->version));
143 
144 	len_fanout =
145 	    sizeof(*h->fanout_table) * GOT_PACKIDX_V2_FANOUT_TABLE_ITEMS;
146 	if (remain < len_fanout) {
147 		err = got_error(GOT_ERR_BAD_PACKIDX);
148 		goto done;
149 	}
150 	if (p->map)
151 		h->fanout_table = (uint32_t *)(p->map + offset);
152 	else {
153 		h->fanout_table = malloc(len_fanout);
154 		if (h->fanout_table == NULL) {
155 			err = got_error_from_errno("malloc");
156 			goto done;
157 		}
158 		n = read(p->fd, h->fanout_table, len_fanout);
159 		if (n < 0)
160 			err = got_error_from_errno("read");
161 		else if (n != len_fanout) {
162 			err = got_error(GOT_ERR_BAD_PACKIDX);
163 			goto done;
164 		}
165 	}
166 	err = verify_fanout_table(h->fanout_table);
167 	if (err)
168 		goto done;
169 	if (verify)
170 		SHA1Update(&ctx, (uint8_t *)h->fanout_table, len_fanout);
171 	offset += len_fanout;
172 	remain -= len_fanout;
173 
174 	nobj = betoh32(h->fanout_table[0xff]);
175 	len_ids = nobj * sizeof(*h->sorted_ids);
176 	if (len_ids <= nobj || len_ids > remain) {
177 		err = got_error(GOT_ERR_BAD_PACKIDX);
178 		goto done;
179 	}
180 	if (p->map)
181 		h->sorted_ids =
182 		    (struct got_packidx_object_id *)((uint8_t*)(p->map + offset));
183 	else {
184 		h->sorted_ids = malloc(len_ids);
185 		if (h->sorted_ids == NULL) {
186 			err = got_error(GOT_ERR_BAD_PACKIDX);
187 			goto done;
188 		}
189 		n = read(p->fd, h->sorted_ids, len_ids);
190 		if (n < 0)
191 			err = got_error_from_errno("read");
192 		else if (n != len_ids) {
193 			err = got_error(GOT_ERR_BAD_PACKIDX);
194 			goto done;
195 		}
196 	}
197 	if (verify)
198 		SHA1Update(&ctx, (uint8_t *)h->sorted_ids, len_ids);
199 	offset += len_ids;
200 	remain -= len_ids;
201 
202 	if (remain < nobj * sizeof(*h->crc32)) {
203 		err = got_error(GOT_ERR_BAD_PACKIDX);
204 		goto done;
205 	}
206 	if (p->map)
207 		h->crc32 = (uint32_t *)((uint8_t*)(p->map + offset));
208 	else {
209 		h->crc32 = malloc(nobj * sizeof(*h->crc32));
210 		if (h->crc32 == NULL) {
211 			err = got_error_from_errno("malloc");
212 			goto done;
213 		}
214 		n = read(p->fd, h->crc32, nobj * sizeof(*h->crc32));
215 		if (n < 0)
216 			err = got_error_from_errno("read");
217 		else if (n != nobj * sizeof(*h->crc32)) {
218 			err = got_error(GOT_ERR_BAD_PACKIDX);
219 			goto done;
220 		}
221 	}
222 	if (verify)
223 		SHA1Update(&ctx, (uint8_t *)h->crc32, nobj * sizeof(*h->crc32));
224 	remain -= nobj * sizeof(*h->crc32);
225 	offset += nobj * sizeof(*h->crc32);
226 
227 	if (remain < nobj * sizeof(*h->offsets)) {
228 		err = got_error(GOT_ERR_BAD_PACKIDX);
229 		goto done;
230 	}
231 	if (p->map)
232 		h->offsets = (uint32_t *)((uint8_t*)(p->map + offset));
233 	else {
234 		h->offsets = malloc(nobj * sizeof(*h->offsets));
235 		if (h->offsets == NULL) {
236 			err = got_error_from_errno("malloc");
237 			goto done;
238 		}
239 		n = read(p->fd, h->offsets, nobj * sizeof(*h->offsets));
240 		if (n < 0)
241 			err = got_error_from_errno("read");
242 		else if (n != nobj * sizeof(*h->offsets)) {
243 			err = got_error(GOT_ERR_BAD_PACKIDX);
244 			goto done;
245 		}
246 	}
247 	if (verify)
248 		SHA1Update(&ctx, (uint8_t *)h->offsets,
249 		    nobj * sizeof(*h->offsets));
250 	remain -= nobj * sizeof(*h->offsets);
251 	offset += nobj * sizeof(*h->offsets);
252 
253 	/* Large file offsets are contained only in files > 2GB. */
254 	if (p->len <= 0x80000000)
255 		goto checksum;
256 
257 	if (remain < nobj * sizeof(*h->large_offsets)) {
258 		err = got_error(GOT_ERR_BAD_PACKIDX);
259 		goto done;
260 	}
261 	if (p->map)
262 		h->large_offsets = (uint64_t *)((uint8_t*)(p->map + offset));
263 	else {
264 		h->offsets = malloc(nobj * sizeof(*h->large_offsets));
265 		if (h->offsets == NULL) {
266 			err = got_error_from_errno("malloc");
267 			goto done;
268 		}
269 		n = read(p->fd, h->large_offsets,
270 		    nobj * sizeof(*h->large_offsets));
271 		if (n < 0)
272 			err = got_error_from_errno("read");
273 		else if (n != nobj * sizeof(*h->large_offsets)) {
274 			err = got_error(GOT_ERR_BAD_PACKIDX);
275 			goto done;
276 		}
277 	}
278 	if (verify)
279 		SHA1Update(&ctx, (uint8_t*)h->large_offsets,
280 		    nobj * sizeof(*h->large_offsets));
281 	remain -= nobj * sizeof(*h->large_offsets);
282 	offset += nobj * sizeof(*h->large_offsets);
283 
284 checksum:
285 	if (remain < sizeof(*h->trailer)) {
286 		err = got_error(GOT_ERR_BAD_PACKIDX);
287 		goto done;
288 	}
289 	if (p->map)
290 		h->trailer =
291 		    (struct got_packidx_trailer *)((uint8_t*)(p->map + offset));
292 	else {
293 		h->trailer = malloc(sizeof(*h->trailer));
294 		if (h->trailer == NULL) {
295 			err = got_error_from_errno("malloc");
296 			goto done;
297 		}
298 		n = read(p->fd, h->trailer, sizeof(*h->trailer));
299 		if (n < 0)
300 			err = got_error_from_errno("read");
301 		else if (n != sizeof(*h->trailer)) {
302 			err = got_error(GOT_ERR_BAD_PACKIDX);
303 			goto done;
304 		}
305 	}
306 	if (verify) {
307 		SHA1Update(&ctx, h->trailer->packfile_sha1, SHA1_DIGEST_LENGTH);
308 		SHA1Final(sha1, &ctx);
309 		if (memcmp(h->trailer->packidx_sha1, sha1,
310 		    SHA1_DIGEST_LENGTH) != 0)
311 			err = got_error(GOT_ERR_PACKIDX_CSUM);
312 	}
313 done:
314 	return err;
315 }
316 
317 const struct got_error *
318 got_packidx_open(struct got_packidx **packidx, const char *path, int verify)
319 {
320 	const struct got_error *err = NULL;
321 	struct got_packidx *p;
322 	struct stat sb;
323 
324 	*packidx = NULL;
325 
326 	p = calloc(1, sizeof(*p));
327 	if (p == NULL)
328 		return got_error_from_errno("calloc");
329 
330 	p->fd = open(path, O_RDONLY | O_NOFOLLOW);
331 	if (p->fd == -1)
332 		return got_error_from_errno2("open", path);
333 
334 	if (fstat(p->fd, &sb) != 0) {
335 		err = got_error_from_errno2("fstat", path);
336 		close(p->fd);
337 		free(p);
338 		return err;
339 	}
340 	p->len = sb.st_size;
341 	if (p->len < sizeof(p->hdr)) {
342 		err = got_error(GOT_ERR_BAD_PACKIDX);
343 		close(p->fd);
344 		free(p);
345 		return err;
346 	}
347 
348 	p->path_packidx = strdup(path);
349 	if (p->path_packidx == NULL) {
350 		err = got_error_from_errno("strdup");
351 		goto done;
352 	}
353 
354 #ifndef GOT_PACK_NO_MMAP
355 	p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0);
356 	if (p->map == MAP_FAILED) {
357 		if (errno != ENOMEM) {
358 			err = got_error_from_errno("mmap");
359 			goto done;
360 		}
361 		p->map = NULL; /* fall back to read(2) */
362 	}
363 #endif
364 
365 	err = got_packidx_init_hdr(p, verify);
366 done:
367 	if (err)
368 		got_packidx_close(p);
369 	else
370 		*packidx = p;
371 
372 	return err;
373 }
374 
375 const struct got_error *
376 got_packidx_close(struct got_packidx *packidx)
377 {
378 	const struct got_error *err = NULL;
379 
380 	free(packidx->path_packidx);
381 	if (packidx->map) {
382 		if (munmap(packidx->map, packidx->len) == -1)
383 			err = got_error_from_errno("munmap");
384 	} else {
385 		free(packidx->hdr.magic);
386 		free(packidx->hdr.version);
387 		free(packidx->hdr.fanout_table);
388 		free(packidx->hdr.sorted_ids);
389 		free(packidx->hdr.crc32);
390 		free(packidx->hdr.offsets);
391 		free(packidx->hdr.large_offsets);
392 		free(packidx->hdr.trailer);
393 	}
394 	if (close(packidx->fd) != 0 && err == NULL)
395 		err = got_error_from_errno("close");
396 	free(packidx);
397 
398 	return err;
399 }
400 
401 static off_t
402 get_object_offset(struct got_packidx *packidx, int idx)
403 {
404 	uint32_t totobj = betoh32(packidx->hdr.fanout_table[0xff]);
405 	uint32_t offset = betoh32(packidx->hdr.offsets[idx]);
406 	if (offset & GOT_PACKIDX_OFFSET_VAL_IS_LARGE_IDX) {
407 		uint64_t loffset;
408 		idx = offset & GOT_PACKIDX_OFFSET_VAL_MASK;
409 		if (idx < 0 || idx > totobj ||
410 		    packidx->hdr.large_offsets == NULL)
411 			return -1;
412 		loffset = betoh64(packidx->hdr.large_offsets[idx]);
413 		return (loffset > INT64_MAX ? -1 : (off_t)loffset);
414 	}
415 	return (off_t)(offset & GOT_PACKIDX_OFFSET_VAL_MASK);
416 }
417 
418 int
419 got_packidx_get_object_idx(struct got_packidx *packidx, struct got_object_id *id)
420 {
421 	u_int8_t id0 = id->sha1[0];
422 	uint32_t totobj = betoh32(packidx->hdr.fanout_table[0xff]);
423 	int left = 0, right = totobj - 1;
424 
425 	if (id0 > 0)
426 		left = betoh32(packidx->hdr.fanout_table[id0 - 1]);
427 
428 	while (left <= right) {
429 		struct got_packidx_object_id *oid;
430 		int i, cmp;
431 
432 		i = ((left + right) / 2);
433 		oid = &packidx->hdr.sorted_ids[i];
434 		cmp = memcmp(id->sha1, oid->sha1, SHA1_DIGEST_LENGTH);
435 		if (cmp == 0)
436 			return i;
437 		else if (cmp > 0)
438 			left = i + 1;
439 		else if (cmp < 0)
440 			right = i - 1;
441 	}
442 
443 	return -1;
444 }
445 
446 const struct got_error *
447 got_pack_stop_privsep_child(struct got_pack *pack)
448 {
449 	const struct got_error *err = NULL;
450 
451 	if (pack->privsep_child == NULL)
452 		return NULL;
453 
454 	err = got_privsep_send_stop(pack->privsep_child->imsg_fd);
455 	if (err)
456 		return err;
457 	err = got_privsep_wait_for_child(pack->privsep_child->pid);
458 	if (close(pack->privsep_child->imsg_fd) != 0 && err == NULL)
459 		err = got_error_from_errno("close");
460 	free(pack->privsep_child);
461 	pack->privsep_child = NULL;
462 	return err;
463 }
464 
465 const struct got_error *
466 got_pack_close(struct got_pack *pack)
467 {
468 	const struct got_error *err = NULL;
469 
470 	err = got_pack_stop_privsep_child(pack);
471 	if (pack->map && munmap(pack->map, pack->filesize) == -1 && !err)
472 		err = got_error_from_errno("munmap");
473 	if (pack->fd != -1 && close(pack->fd) != 0 && err == NULL)
474 		err = got_error_from_errno("close");
475 	pack->fd = -1;
476 	free(pack->path_packfile);
477 	pack->path_packfile = NULL;
478 	pack->filesize = 0;
479 
480 	return err;
481 }
482 
483 static const struct got_error *
484 parse_object_type_and_size(uint8_t *type, uint64_t *size, size_t *len,
485     struct got_pack *pack, off_t offset)
486 {
487 	uint8_t t = 0;
488 	uint64_t s = 0;
489 	uint8_t sizeN;
490 	size_t mapoff = 0;
491 	int i = 0;
492 
493 	*len = 0;
494 
495 	if (offset >= pack->filesize)
496 		return got_error(GOT_ERR_PACK_OFFSET);
497 
498 	if (pack->map) {
499 		mapoff = (size_t)offset;
500 	} else {
501 		if (lseek(pack->fd, offset, SEEK_SET) == -1)
502 			return got_error_from_errno("lseek");
503 	}
504 
505 	do {
506 		/* We do not support size values which don't fit in 64 bit. */
507 		if (i > 9)
508 			return got_error(GOT_ERR_NO_SPACE);
509 
510 		if (pack->map) {
511 			sizeN = *(pack->map + mapoff);
512 			mapoff += sizeof(sizeN);
513 		} else {
514 			ssize_t n = read(pack->fd, &sizeN, sizeof(sizeN));
515 			if (n < 0)
516 				return got_error_from_errno("read");
517 			if (n != sizeof(sizeN))
518 				return got_error(GOT_ERR_BAD_PACKFILE);
519 		}
520 		*len += sizeof(sizeN);
521 
522 		if (i == 0) {
523 			t = (sizeN & GOT_PACK_OBJ_SIZE0_TYPE_MASK) >>
524 			    GOT_PACK_OBJ_SIZE0_TYPE_MASK_SHIFT;
525 			s = (sizeN & GOT_PACK_OBJ_SIZE0_VAL_MASK);
526 		} else {
527 			size_t shift = 4 + 7 * (i - 1);
528 			s |= ((sizeN & GOT_PACK_OBJ_SIZE_VAL_MASK) << shift);
529 		}
530 		i++;
531 	} while (sizeN & GOT_PACK_OBJ_SIZE_MORE);
532 
533 	*type = t;
534 	*size = s;
535 	return NULL;
536 }
537 
538 static const struct got_error *
539 open_plain_object(struct got_object **obj, const char *path_packfile,
540     struct got_object_id *id, uint8_t type, off_t offset, size_t size, int idx)
541 {
542 	*obj = calloc(1, sizeof(**obj));
543 	if (*obj == NULL)
544 		return got_error_from_errno("calloc");
545 
546 	(*obj)->path_packfile = strdup(path_packfile);
547 	if ((*obj)->path_packfile == NULL) {
548 		const struct got_error *err = got_error_from_errno("strdup");
549 		free(*obj);
550 		*obj = NULL;
551 		return err;
552 	}
553 
554 	(*obj)->type = type;
555 	(*obj)->flags = GOT_OBJ_FLAG_PACKED;
556 	(*obj)->pack_idx = idx;
557 	(*obj)->hdrlen = 0;
558 	(*obj)->size = size;
559 	memcpy(&(*obj)->id, id, sizeof((*obj)->id));
560 	(*obj)->pack_offset = offset;
561 
562 	return NULL;
563 }
564 
565 static const struct got_error *
566 parse_negative_offset(int64_t *offset, size_t *len, struct got_pack *pack,
567     off_t delta_offset)
568 {
569 	int64_t o = 0;
570 	uint8_t offN;
571 	int i = 0;
572 
573 	*len = 0;
574 
575 	do {
576 		/* We do not support offset values which don't fit in 64 bit. */
577 		if (i > 8)
578 			return got_error(GOT_ERR_NO_SPACE);
579 
580 		if (pack->map) {
581 			size_t mapoff;
582 			if (delta_offset >= pack->filesize)
583 				return got_error(GOT_ERR_PACK_OFFSET);
584 			mapoff = (size_t)delta_offset + *len;
585 			offN = *(pack->map + mapoff);
586 		} else {
587 			ssize_t n;
588 			n = read(pack->fd, &offN, sizeof(offN));
589 			if (n < 0)
590 				return got_error_from_errno("read");
591 			if (n != sizeof(offN))
592 				return got_error(GOT_ERR_BAD_PACKFILE);
593 		}
594 		*len += sizeof(offN);
595 
596 		if (i == 0)
597 			o = (offN & GOT_PACK_OBJ_DELTA_OFF_VAL_MASK);
598 		else {
599 			o++;
600 			o <<= 7;
601 			o += (offN & GOT_PACK_OBJ_DELTA_OFF_VAL_MASK);
602 		}
603 		i++;
604 	} while (offN & GOT_PACK_OBJ_DELTA_OFF_MORE);
605 
606 	*offset = o;
607 	return NULL;
608 }
609 
610 static const struct got_error *
611 parse_offset_delta(off_t *base_offset, size_t *len, struct got_pack *pack,
612     off_t offset, int tslen)
613 {
614 	const struct got_error *err;
615 	int64_t negoffset;
616 	size_t negofflen;
617 
618 	*len = 0;
619 
620 	err = parse_negative_offset(&negoffset, &negofflen, pack,
621 	    offset + tslen);
622 	if (err)
623 		return err;
624 
625 	/* Compute the base object's offset (must be in the same pack file). */
626 	*base_offset = (offset - negoffset);
627 	if (*base_offset <= 0)
628 		return got_error(GOT_ERR_BAD_PACKFILE);
629 
630 	*len = negofflen;
631 	return NULL;
632 }
633 
634 static const struct got_error *
635 resolve_delta_chain(struct got_delta_chain *, struct got_packidx *,
636     struct got_pack *, off_t, size_t, int, size_t, unsigned int);
637 
638 static const struct got_error *
639 add_delta(struct got_delta_chain *deltas, off_t delta_offset, size_t tslen,
640     int delta_type, size_t delta_size, size_t delta_data_offset,
641     uint8_t *delta_buf, size_t delta_len)
642 {
643 	struct got_delta *delta;
644 
645 	delta = got_delta_open(delta_offset, tslen, delta_type, delta_size,
646 	    delta_data_offset, delta_buf,
647 	    delta_len);
648 	if (delta == NULL)
649 		return got_error_from_errno("got_delta_open");
650 	/* delta is freed in got_object_close() */
651 	deltas->nentries++;
652 	SIMPLEQ_INSERT_HEAD(&deltas->entries, delta, entry);
653 	return NULL;
654 }
655 
656 static const struct got_error *
657 resolve_offset_delta(struct got_delta_chain *deltas,
658     struct got_packidx *packidx, struct got_pack *pack, off_t delta_offset,
659     size_t tslen, int delta_type, size_t delta_size, unsigned int recursion)
660 
661 {
662 	const struct got_error *err;
663 	off_t base_offset;
664 	uint8_t base_type;
665 	uint64_t base_size;
666 	size_t base_tslen;
667 	off_t delta_data_offset;
668 	uint8_t *delta_buf;
669 	size_t delta_len, consumed;
670 
671 	err = parse_offset_delta(&base_offset, &consumed, pack,
672 	    delta_offset, tslen);
673 	if (err)
674 		return err;
675 
676 	delta_data_offset = delta_offset + tslen + consumed;
677 	if (delta_data_offset >= pack->filesize)
678 		return got_error(GOT_ERR_PACK_OFFSET);
679 
680 	if (pack->map == NULL) {
681 		delta_data_offset = lseek(pack->fd, 0, SEEK_CUR);
682 		if (delta_data_offset == -1)
683 			return got_error_from_errno("lseek");
684 	}
685 
686 	if (pack->map) {
687 		size_t mapoff = (size_t)delta_data_offset;
688 		err = got_inflate_to_mem_mmap(&delta_buf, &delta_len, pack->map,
689 		    mapoff, pack->filesize - mapoff);
690 		if (err)
691 			return err;
692 	} else {
693 
694 		err = got_inflate_to_mem_fd(&delta_buf, &delta_len, pack->fd);
695 		if (err)
696 			return err;
697 	}
698 
699 	err = add_delta(deltas, delta_offset, tslen, delta_type, delta_size,
700 	    delta_data_offset, delta_buf, delta_len);
701 	if (err)
702 		return err;
703 
704 	/* An offset delta must be in the same packfile. */
705 	if (base_offset >= pack->filesize)
706 		return got_error(GOT_ERR_PACK_OFFSET);
707 
708 	err = parse_object_type_and_size(&base_type, &base_size, &base_tslen,
709 	    pack, base_offset);
710 	if (err)
711 		return err;
712 
713 	return resolve_delta_chain(deltas, packidx, pack, base_offset,
714 	    base_tslen, base_type, base_size, recursion - 1);
715 }
716 
717 static const struct got_error *
718 resolve_ref_delta(struct got_delta_chain *deltas, struct got_packidx *packidx,
719     struct got_pack *pack, off_t delta_offset, size_t tslen, int delta_type,
720     size_t delta_size, unsigned int recursion)
721 {
722 	const struct got_error *err;
723 	struct got_object_id id;
724 	int idx;
725 	off_t base_offset;
726 	uint8_t base_type;
727 	uint64_t base_size;
728 	size_t base_tslen;
729 	off_t delta_data_offset;
730 	uint8_t *delta_buf;
731 	size_t delta_len;
732 
733 	if (delta_offset >= pack->filesize)
734 		return got_error(GOT_ERR_PACK_OFFSET);
735 	delta_data_offset = delta_offset + tslen;
736 	if (delta_data_offset >= pack->filesize)
737 		return got_error(GOT_ERR_PACK_OFFSET);
738 
739 	if (pack->map == NULL) {
740 		delta_data_offset = lseek(pack->fd, 0, SEEK_CUR);
741 		if (delta_data_offset == -1)
742 			return got_error_from_errno("lseek");
743 	}
744 
745 	if (pack->map) {
746 		size_t mapoff = (size_t)delta_data_offset;
747 		memcpy(&id, pack->map + mapoff, sizeof(id));
748 		mapoff += sizeof(id);
749 		err = got_inflate_to_mem_mmap(&delta_buf, &delta_len, pack->map,
750 		    mapoff, pack->filesize - mapoff);
751 		if (err)
752 			return err;
753 	} else {
754 		ssize_t n = read(pack->fd, &id, sizeof(id));
755 		if (n < 0)
756 			return got_error_from_errno("read");
757 		if (n != sizeof(id))
758 			return got_error(GOT_ERR_BAD_PACKFILE);
759 		err = got_inflate_to_mem_fd(&delta_buf, &delta_len, pack->fd);
760 		if (err)
761 			return err;
762 	}
763 
764 	err = add_delta(deltas, delta_offset, tslen, delta_type, delta_size,
765 	    delta_data_offset, delta_buf, delta_len);
766 	if (err)
767 		return err;
768 
769 	/* Delta base must be in the same pack file. */
770 	idx = got_packidx_get_object_idx(packidx, &id);
771 	if (idx == -1)
772 		return got_error(GOT_ERR_BAD_PACKFILE);
773 
774 	base_offset = get_object_offset(packidx, idx);
775 	if (base_offset == (uint64_t)-1)
776 		return got_error(GOT_ERR_BAD_PACKIDX);
777 
778 	if (base_offset >= pack->filesize)
779 		return got_error(GOT_ERR_PACK_OFFSET);
780 
781 	err = parse_object_type_and_size(&base_type, &base_size, &base_tslen,
782 	    pack, base_offset);
783 	if (err)
784 		return err;
785 
786 	return resolve_delta_chain(deltas, packidx, pack, base_offset,
787 	    base_tslen, base_type, base_size, recursion - 1);
788 }
789 
790 static const struct got_error *
791 resolve_delta_chain(struct got_delta_chain *deltas, struct got_packidx *packidx,
792     struct got_pack *pack, off_t delta_offset, size_t tslen, int delta_type,
793     size_t delta_size, unsigned int recursion)
794 {
795 	const struct got_error *err = NULL;
796 
797 	if (--recursion == 0)
798 		return got_error(GOT_ERR_RECURSION);
799 
800 	switch (delta_type) {
801 	case GOT_OBJ_TYPE_COMMIT:
802 	case GOT_OBJ_TYPE_TREE:
803 	case GOT_OBJ_TYPE_BLOB:
804 	case GOT_OBJ_TYPE_TAG:
805 		/* Plain types are the final delta base. Recursion ends. */
806 		err = add_delta(deltas, delta_offset, tslen, delta_type,
807 		    delta_size, 0, NULL, 0);
808 		break;
809 	case GOT_OBJ_TYPE_OFFSET_DELTA:
810 		err = resolve_offset_delta(deltas, packidx, pack,
811 		    delta_offset, tslen, delta_type, delta_size, recursion - 1);
812 		break;
813 	case GOT_OBJ_TYPE_REF_DELTA:
814 		err = resolve_ref_delta(deltas, packidx, pack,
815 		    delta_offset, tslen, delta_type, delta_size, recursion - 1);
816 		break;
817 	default:
818 		return got_error(GOT_ERR_OBJ_TYPE);
819 	}
820 
821 	return err;
822 }
823 
824 static const struct got_error *
825 open_delta_object(struct got_object **obj, struct got_packidx *packidx,
826     struct got_pack *pack, struct got_object_id *id, off_t offset,
827     size_t tslen, int delta_type, size_t delta_size, int idx)
828 {
829 	const struct got_error *err = NULL;
830 	int resolved_type;
831 
832 	*obj = calloc(1, sizeof(**obj));
833 	if (*obj == NULL)
834 		return got_error_from_errno("calloc");
835 
836 	(*obj)->flags = 0;
837 	(*obj)->hdrlen = 0;
838 	(*obj)->size = 0; /* Not known because deltas aren't applied yet. */
839 	memcpy(&(*obj)->id, id, sizeof((*obj)->id));
840 	(*obj)->pack_offset = offset + tslen;
841 
842 	SIMPLEQ_INIT(&(*obj)->deltas.entries);
843 	(*obj)->flags |= GOT_OBJ_FLAG_DELTIFIED;
844 
845 	(*obj)->path_packfile = strdup(pack->path_packfile);
846 	if ((*obj)->path_packfile == NULL) {
847 		err = got_error_from_errno("strdup");
848 		goto done;
849 	}
850 	(*obj)->flags |= GOT_OBJ_FLAG_PACKED;
851 	(*obj)->pack_idx = idx;
852 
853 	err = resolve_delta_chain(&(*obj)->deltas, packidx, pack, offset,
854 	    tslen, delta_type, delta_size, GOT_DELTA_CHAIN_RECURSION_MAX);
855 	if (err)
856 		goto done;
857 
858 	err = got_delta_chain_get_base_type(&resolved_type, &(*obj)->deltas);
859 	if (err)
860 		goto done;
861 	(*obj)->type = resolved_type;
862 done:
863 	if (err) {
864 		got_object_close(*obj);
865 		*obj = NULL;
866 	}
867 	return err;
868 }
869 
870 const struct got_error *
871 got_packfile_open_object(struct got_object **obj, struct got_pack *pack,
872     struct got_packidx *packidx, int idx, struct got_object_id *id)
873 {
874 	const struct got_error *err = NULL;
875 	off_t offset;
876 	uint8_t type;
877 	uint64_t size;
878 	size_t tslen;
879 
880 	*obj = NULL;
881 
882 	offset = get_object_offset(packidx, idx);
883 	if (offset == (uint64_t)-1)
884 		return got_error(GOT_ERR_BAD_PACKIDX);
885 
886 	err = parse_object_type_and_size(&type, &size, &tslen, pack, offset);
887 	if (err)
888 		return err;
889 
890 	switch (type) {
891 	case GOT_OBJ_TYPE_COMMIT:
892 	case GOT_OBJ_TYPE_TREE:
893 	case GOT_OBJ_TYPE_BLOB:
894 	case GOT_OBJ_TYPE_TAG:
895 		err = open_plain_object(obj, pack->path_packfile, id, type,
896 		    offset + tslen, size, idx);
897 		break;
898 	case GOT_OBJ_TYPE_OFFSET_DELTA:
899 	case GOT_OBJ_TYPE_REF_DELTA:
900 		err = open_delta_object(obj, packidx, pack, id, offset,
901 		    tslen, type, size, idx);
902 		break;
903 	default:
904 		err = got_error(GOT_ERR_OBJ_TYPE);
905 		break;
906 	}
907 
908 	return err;
909 }
910 
911 static const struct got_error *
912 get_delta_chain_max_size(uint64_t *max_size, struct got_delta_chain *deltas)
913 {
914 	struct got_delta *delta;
915 	uint64_t base_size = 0, result_size = 0;
916 
917 	*max_size = 0;
918 	SIMPLEQ_FOREACH(delta, &deltas->entries, entry) {
919 		/* Plain object types are the delta base. */
920 		if (delta->type != GOT_OBJ_TYPE_COMMIT &&
921 		    delta->type != GOT_OBJ_TYPE_TREE &&
922 		    delta->type != GOT_OBJ_TYPE_BLOB &&
923 		    delta->type != GOT_OBJ_TYPE_TAG) {
924 			const struct got_error *err;
925 			err = got_delta_get_sizes(&base_size, &result_size,
926 			    delta->delta_buf, delta->delta_len);
927 			if (err)
928 				return err;
929 		} else
930 			base_size = delta->size;
931 		if (base_size > *max_size)
932 			*max_size = base_size;
933 		if (result_size > *max_size)
934 			*max_size = result_size;
935 	}
936 
937 	return NULL;
938 }
939 
940 const struct got_error *
941 got_pack_get_max_delta_object_size(uint64_t *size, struct got_object *obj)
942 {
943 	if ((obj->flags & GOT_OBJ_FLAG_DELTIFIED) == 0)
944 		return got_error(GOT_ERR_OBJ_TYPE);
945 
946 	return get_delta_chain_max_size(size, &obj->deltas);
947 }
948 
949 static const struct got_error *
950 dump_delta_chain_to_file(size_t *result_size, struct got_delta_chain *deltas,
951     struct got_pack *pack, FILE *outfile, FILE *base_file, FILE *accum_file)
952 {
953 	const struct got_error *err = NULL;
954 	struct got_delta *delta;
955 	uint8_t *base_buf = NULL, *accum_buf = NULL;
956 	size_t base_bufsz = 0, accum_size = 0;
957 	uint64_t max_size;
958 	int n = 0;
959 
960 	*result_size = 0;
961 
962 	if (SIMPLEQ_EMPTY(&deltas->entries))
963 		return got_error(GOT_ERR_BAD_DELTA_CHAIN);
964 
965 	/* We process small enough files entirely in memory for speed. */
966 	err = get_delta_chain_max_size(&max_size, deltas);
967 	if (err)
968 		return err;
969 	if (max_size < GOT_DELTA_RESULT_SIZE_CACHED_MAX) {
970 		accum_buf = malloc(max_size);
971 		if (accum_buf == NULL)
972 			return got_error_from_errno("malloc");
973 		base_file = NULL;
974 		accum_file = NULL;
975 	}
976 
977 	/* Deltas are ordered in ascending order. */
978 	SIMPLEQ_FOREACH(delta, &deltas->entries, entry) {
979 		if (n == 0) {
980 			size_t mapoff;
981 			off_t delta_data_offset;
982 
983 			/* Plain object types are the delta base. */
984 			if (delta->type != GOT_OBJ_TYPE_COMMIT &&
985 			    delta->type != GOT_OBJ_TYPE_TREE &&
986 			    delta->type != GOT_OBJ_TYPE_BLOB &&
987 			    delta->type != GOT_OBJ_TYPE_TAG) {
988 				err = got_error(GOT_ERR_BAD_DELTA_CHAIN);
989 				goto done;
990 			}
991 
992 			delta_data_offset = delta->offset + delta->tslen;
993 			if (delta_data_offset >= pack->filesize) {
994 				err = got_error(GOT_ERR_PACK_OFFSET);
995 				goto done;
996 			}
997 			if (pack->map == NULL) {
998 				if (lseek(pack->fd, delta_data_offset, SEEK_SET)
999 				    == -1) {
1000 					err = got_error_from_errno("lseek");
1001 					goto done;
1002 				}
1003 			}
1004 			if (base_file) {
1005 				if (pack->map) {
1006 					mapoff = (size_t)delta_data_offset;
1007 					err = got_inflate_to_file_mmap(
1008 					    &base_bufsz, pack->map, mapoff,
1009 					    pack->filesize - mapoff, base_file);
1010 				} else
1011 					err = got_inflate_to_file_fd(
1012 					    &base_bufsz, pack->fd, base_file);
1013 			} else {
1014 				if (pack->map) {
1015 					mapoff = (size_t)delta_data_offset;
1016 					err = got_inflate_to_mem_mmap(&base_buf,
1017 					    &base_bufsz, pack->map, mapoff,
1018 					    pack->filesize - mapoff);
1019 				} else
1020 					err = got_inflate_to_mem_fd(&base_buf,
1021 					    &base_bufsz, pack->fd);
1022 			}
1023 			if (err)
1024 				goto done;
1025 			n++;
1026 			if (base_file)
1027 				rewind(base_file);
1028 			continue;
1029 		}
1030 
1031 		if (base_buf) {
1032 			err = got_delta_apply_in_mem(base_buf, base_bufsz,
1033 			    delta->delta_buf, delta->delta_len, accum_buf,
1034 			    &accum_size, max_size);
1035 			n++;
1036 		} else {
1037 			err = got_delta_apply(base_file, delta->delta_buf,
1038 			    delta->delta_len,
1039 			    /* Final delta application writes to output file. */
1040 			    ++n < deltas->nentries ? accum_file : outfile,
1041 			    &accum_size);
1042 		}
1043 		if (err)
1044 			goto done;
1045 
1046 		if (n < deltas->nentries) {
1047 			/* Accumulated delta becomes the new base. */
1048 			if (base_buf) {
1049 				uint8_t *tmp = accum_buf;
1050 				/*
1051 				 * Base buffer switches roles with accumulation
1052 				 * buffer. Ensure it can hold the largest
1053 				 * result in the delta chain. The initial
1054 				 * allocation might have been smaller.
1055 				 */
1056 				if (base_bufsz < max_size) {
1057 					uint8_t *p;
1058 					p = reallocarray(base_buf, 1, max_size);
1059 					if (p == NULL) {
1060 						err = got_error_from_errno(
1061 						    "reallocarray");
1062 						goto done;
1063 					}
1064 					base_buf = p;
1065 					base_bufsz = max_size;
1066 				}
1067 				accum_buf = base_buf;
1068 				base_buf = tmp;
1069 			} else {
1070 				FILE *tmp = accum_file;
1071 				accum_file = base_file;
1072 				base_file = tmp;
1073 				rewind(base_file);
1074 				rewind(accum_file);
1075 			}
1076 		}
1077 	}
1078 
1079 done:
1080 	free(base_buf);
1081 	if (accum_buf) {
1082 		size_t len = fwrite(accum_buf, 1, accum_size, outfile);
1083 		free(accum_buf);
1084 		if (len != accum_size)
1085 			err = got_ferror(outfile, GOT_ERR_IO);
1086 	}
1087 	rewind(outfile);
1088 	if (err == NULL)
1089 		*result_size = accum_size;
1090 	return err;
1091 }
1092 
1093 static const struct got_error *
1094 dump_delta_chain_to_mem(uint8_t **outbuf, size_t *outlen,
1095     struct got_delta_chain *deltas, struct got_pack *pack)
1096 {
1097 	const struct got_error *err = NULL;
1098 	struct got_delta *delta;
1099 	uint8_t *base_buf = NULL, *accum_buf = NULL;
1100 	size_t base_bufsz = 0, accum_size = 0;
1101 	uint64_t max_size;
1102 	int n = 0;
1103 
1104 	*outbuf = NULL;
1105 	*outlen = 0;
1106 
1107 	if (SIMPLEQ_EMPTY(&deltas->entries))
1108 		return got_error(GOT_ERR_BAD_DELTA_CHAIN);
1109 
1110 	err = get_delta_chain_max_size(&max_size, deltas);
1111 	if (err)
1112 		return err;
1113 	accum_buf = malloc(max_size);
1114 	if (accum_buf == NULL)
1115 		return got_error_from_errno("malloc");
1116 
1117 	/* Deltas are ordered in ascending order. */
1118 	SIMPLEQ_FOREACH(delta, &deltas->entries, entry) {
1119 		if (n == 0) {
1120 			size_t delta_data_offset;
1121 
1122 			/* Plain object types are the delta base. */
1123 			if (delta->type != GOT_OBJ_TYPE_COMMIT &&
1124 			    delta->type != GOT_OBJ_TYPE_TREE &&
1125 			    delta->type != GOT_OBJ_TYPE_BLOB &&
1126 			    delta->type != GOT_OBJ_TYPE_TAG) {
1127 				err = got_error(GOT_ERR_BAD_DELTA_CHAIN);
1128 				goto done;
1129 			}
1130 
1131 			delta_data_offset = delta->offset + delta->tslen;
1132 			if (delta_data_offset >= pack->filesize) {
1133 				err = got_error(GOT_ERR_PACK_OFFSET);
1134 				goto done;
1135 			}
1136 			if (pack->map) {
1137 				size_t mapoff = (size_t)delta_data_offset;
1138 				err = got_inflate_to_mem_mmap(&base_buf,
1139 				    &base_bufsz, pack->map, mapoff,
1140 				    pack->filesize - mapoff);
1141 			} else {
1142 				if (lseek(pack->fd, delta_data_offset, SEEK_SET)
1143 				    == -1) {
1144 					err = got_error_from_errno("lseek");
1145 					goto done;
1146 				}
1147 				err = got_inflate_to_mem_fd(&base_buf,
1148 				    &base_bufsz, pack->fd);
1149 			}
1150 			if (err)
1151 				goto done;
1152 			n++;
1153 			continue;
1154 		}
1155 
1156 		err = got_delta_apply_in_mem(base_buf, base_bufsz,
1157 		    delta->delta_buf, delta->delta_len, accum_buf,
1158 		    &accum_size, max_size);
1159 		n++;
1160 		if (err)
1161 			goto done;
1162 
1163 		if (n < deltas->nentries) {
1164 			/* Accumulated delta becomes the new base. */
1165 			uint8_t *tmp = accum_buf;
1166 			/*
1167 			 * Base buffer switches roles with accumulation buffer.
1168 			 * Ensure it can hold the largest result in the delta
1169 			 * chain. Initial allocation might have been smaller.
1170 			 */
1171 			if (base_bufsz < max_size) {
1172 				uint8_t *p;
1173 				p = reallocarray(base_buf, 1, max_size);
1174 				if (p == NULL) {
1175 					err = got_error_from_errno(
1176 					    "reallocarray");
1177 					goto done;
1178 				}
1179 				base_buf = p;
1180 				base_bufsz = max_size;
1181 			}
1182 			accum_buf = base_buf;
1183 			base_buf = tmp;
1184 		}
1185 	}
1186 
1187 done:
1188 	free(base_buf);
1189 	if (err) {
1190 		free(accum_buf);
1191 		*outbuf = NULL;
1192 		*outlen = 0;
1193 	} else {
1194 		*outbuf = accum_buf;
1195 		*outlen = accum_size;
1196 	}
1197 	return err;
1198 }
1199 
1200 const struct got_error *
1201 got_packfile_extract_object(struct got_pack *pack, struct got_object *obj,
1202     FILE *outfile, FILE *base_file, FILE *accum_file)
1203 {
1204 	const struct got_error *err = NULL;
1205 
1206 	if ((obj->flags & GOT_OBJ_FLAG_PACKED) == 0)
1207 		return got_error(GOT_ERR_OBJ_NOT_PACKED);
1208 
1209 	if ((obj->flags & GOT_OBJ_FLAG_DELTIFIED) == 0) {
1210 		if (obj->pack_offset >= pack->filesize)
1211 			return got_error(GOT_ERR_PACK_OFFSET);
1212 
1213 		if (pack->map) {
1214 			size_t mapoff = (size_t)obj->pack_offset;
1215 			err = got_inflate_to_file_mmap(&obj->size, pack->map,
1216 			    mapoff, pack->filesize - mapoff, outfile);
1217 		} else {
1218 			if (lseek(pack->fd, obj->pack_offset, SEEK_SET) == -1)
1219 				return got_error_from_errno("lseek");
1220 			err = got_inflate_to_file_fd(&obj->size, pack->fd,
1221 			    outfile);
1222 		}
1223 	} else
1224 		err = dump_delta_chain_to_file(&obj->size, &obj->deltas, pack,
1225 		    outfile, base_file, accum_file);
1226 
1227 	return err;
1228 }
1229 
1230 const struct got_error *
1231 got_packfile_extract_object_to_mem(uint8_t **buf, size_t *len,
1232     struct got_object *obj, struct got_pack *pack)
1233 {
1234 	const struct got_error *err = NULL;
1235 
1236 	if ((obj->flags & GOT_OBJ_FLAG_PACKED) == 0)
1237 		return got_error(GOT_ERR_OBJ_NOT_PACKED);
1238 
1239 	if ((obj->flags & GOT_OBJ_FLAG_DELTIFIED) == 0) {
1240 		if (obj->pack_offset >= pack->filesize)
1241 			return got_error(GOT_ERR_PACK_OFFSET);
1242 		if (pack->map) {
1243 			size_t mapoff = (size_t)obj->pack_offset;
1244 			err = got_inflate_to_mem_mmap(buf, len, pack->map,
1245 			    mapoff, pack->filesize - mapoff);
1246 		} else {
1247 			if (lseek(pack->fd, obj->pack_offset, SEEK_SET) == -1)
1248 				return got_error_from_errno("lseek");
1249 			err = got_inflate_to_mem_fd(buf, len, pack->fd);
1250 		}
1251 	} else
1252 		err = dump_delta_chain_to_mem(buf, len, &obj->deltas, pack);
1253 
1254 	return err;
1255 }