op public repos

Blob

Date:: Wed Mar 18 16:13:46 2020 UTC
Message:: extract large objects to a temporary file in got-index-pack
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/queue.h>
18 
19 #include <errno.h>
20 #include <stdio.h>
21 #include <stdlib.h>
22 #include <string.h>
23 #include <sha1.h>
24 #include <zlib.h>
25 #include <time.h>
26 
27 #include "got_error.h"
28 #include "got_object.h"
29 #include "got_path.h"
30 
31 #include "got_lib_inflate.h"
32 
33 #ifndef MIN
34 #define	MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
35 #endif
36 
37 const struct got_error *
38 got_inflate_init(struct got_inflate_buf *zb, uint8_t *outbuf, size_t bufsize,
39     uint32_t *input_crc)
40 {
41 	const struct got_error *err = NULL;
42 	int zerr;
43 
44 	memset(&zb->z, 0, sizeof(zb->z));
45 
46 	zb->z.zalloc = Z_NULL;
47 	zb->z.zfree = Z_NULL;
48 	zerr = inflateInit(&zb->z);
49 	if (zerr != Z_OK) {
50 		if  (zerr == Z_ERRNO)
51 			return got_error_from_errno("inflateInit");
52 		if  (zerr == Z_MEM_ERROR) {
53 			errno = ENOMEM;
54 			return got_error_from_errno("inflateInit");
55 		}
56 		return got_error(GOT_ERR_DECOMPRESSION);
57 	}
58 
59 	zb->inlen = zb->outlen = bufsize;
60 
61 	zb->inbuf = calloc(1, zb->inlen);
62 	if (zb->inbuf == NULL) {
63 		err = got_error_from_errno("calloc");
64 		goto done;
65 	}
66 
67 	zb->flags = 0;
68 	if (outbuf == NULL) {
69 		zb->outbuf = calloc(1, zb->outlen);
70 		if (zb->outbuf == NULL) {
71 			err = got_error_from_errno("calloc");
72 			goto done;
73 		}
74 		zb->flags |= GOT_INFLATE_F_OWN_OUTBUF;
75 	} else
76 		zb->outbuf = outbuf;
77 
78 	zb->input_crc = input_crc;
79 done:
80 	if (err)
81 		got_inflate_end(zb);
82 	return err;
83 }
84 
85 const struct got_error *
86 got_inflate_read(struct got_inflate_buf *zb, FILE *f, size_t *outlenp,
87    size_t *consumed)
88 {
89 	size_t last_total_out = zb->z.total_out;
90 	size_t last_total_in = zb->z.total_in;
91 	z_stream *z = &zb->z;
92 	int ret = Z_ERRNO;
93 
94 	z->next_out = zb->outbuf;
95 	z->avail_out = zb->outlen;
96 
97 	*outlenp = 0;
98 	if (consumed)
99 		*consumed = 0;
100 	do {
101 		char *crc_in = NULL;
102 		size_t crc_avail = 0;
103 
104 		if (z->avail_in == 0) {
105 			size_t n = fread(zb->inbuf, 1, zb->inlen, f);
106 			if (n == 0) {
107 				if (ferror(f))
108 					return got_ferror(f, GOT_ERR_IO);
109 				/* EOF */
110 				ret = Z_STREAM_END;
111 				break;
112 			}
113 			z->next_in = zb->inbuf;
114 			z->avail_in = n;
115 		}
116 		if (zb->input_crc) {
117 			crc_in = z->next_in;
118 			crc_avail = z->avail_in;
119 		}
120 		ret = inflate(z, Z_SYNC_FLUSH);
121 		if (zb->input_crc) {
122 			*zb->input_crc = crc32(*zb->input_crc,
123 			    crc_in, crc_avail - z->avail_in);
124 		}
125 	} while (ret == Z_OK && z->avail_out > 0);
126 
127 	if (ret == Z_OK || ret == Z_BUF_ERROR) {
128 		zb->flags |= GOT_INFLATE_F_HAVE_MORE;
129 	} else {
130 		if (ret != Z_STREAM_END)
131 			return got_error(GOT_ERR_DECOMPRESSION);
132 		zb->flags &= ~GOT_INFLATE_F_HAVE_MORE;
133 	}
134 
135 	*outlenp = z->total_out - last_total_out;
136 	if (consumed)
137 		*consumed += z->total_in - last_total_in;
138 	return NULL;
139 }
140 
141 const struct got_error *
142 got_inflate_read_fd(struct got_inflate_buf *zb, int fd, size_t *outlenp,
143     size_t *consumed)
144 {
145 	size_t last_total_out = zb->z.total_out;
146 	size_t last_total_in = zb->z.total_in;
147 	z_stream *z = &zb->z;
148 	int ret = Z_ERRNO;
149 
150 	z->next_out = zb->outbuf;
151 	z->avail_out = zb->outlen;
152 
153 	*outlenp = 0;
154 	if (consumed)
155 		*consumed = 0;
156 	do {
157 		char *crc_in = NULL;
158 		size_t crc_avail = 0;
159 
160 		if (z->avail_in == 0) {
161 			ssize_t n = read(fd, zb->inbuf, zb->inlen);
162 			if (n < 0)
163 				return got_error_from_errno("read");
164 			else if (n == 0) {
165 				/* EOF */
166 				ret = Z_STREAM_END;
167 				break;
168 			}
169 			z->next_in = zb->inbuf;
170 			z->avail_in = n;
171 		}
172 		if (zb->input_crc) {
173 			crc_in = z->next_in;
174 			crc_avail = z->avail_in;
175 		}
176 		ret = inflate(z, Z_SYNC_FLUSH);
177 		if (zb->input_crc) {
178 			*zb->input_crc = crc32(*zb->input_crc,
179 			    crc_in, crc_avail - z->avail_in);
180 		}
181 	} while (ret == Z_OK && z->avail_out > 0);
182 
183 	if (ret == Z_OK || ret == Z_BUF_ERROR) {
184 		zb->flags |= GOT_INFLATE_F_HAVE_MORE;
185 	} else {
186 		if (ret != Z_STREAM_END)
187 			return got_error(GOT_ERR_DECOMPRESSION);
188 		zb->flags &= ~GOT_INFLATE_F_HAVE_MORE;
189 	}
190 
191 	*outlenp = z->total_out - last_total_out;
192 	if (consumed)
193 		*consumed += z->total_in - last_total_in;
194 	return NULL;
195 }
196 
197 const struct got_error *
198 got_inflate_read_mmap(struct got_inflate_buf *zb, uint8_t *map, size_t offset,
199     size_t len, size_t *outlenp, size_t *consumed)
200 {
201 	size_t last_total_out = zb->z.total_out;
202 	z_stream *z = &zb->z;
203 	int ret = Z_ERRNO;
204 
205 	z->next_out = zb->outbuf;
206 	z->avail_out = zb->outlen;
207 
208 	*outlenp = 0;
209 	*consumed = 0;
210 
211 	do {
212 		char *crc_in = NULL;
213 		size_t crc_avail = 0;
214 		size_t last_total_in = zb->z.total_in;
215 
216 		if (z->avail_in == 0) {
217 			if (len == 0) {
218 				/* EOF */
219 				ret = Z_STREAM_END;
220 				break;
221 			}
222 			z->next_in = map + offset + *consumed;
223 			z->avail_in = len - *consumed;
224 		}
225 		if (zb->input_crc) {
226 			crc_in = z->next_in;
227 			crc_avail = z->avail_in;
228 		}
229 		ret = inflate(z, Z_SYNC_FLUSH);
230 		if (zb->input_crc) {
231 			*zb->input_crc = crc32(*zb->input_crc,
232 			    crc_in, crc_avail - z->avail_in);
233 		}
234 		*consumed += z->total_in - last_total_in;
235 	} while (ret == Z_OK && z->avail_out > 0);
236 
237 	if (ret == Z_OK || ret == Z_BUF_ERROR) {
238 		zb->flags |= GOT_INFLATE_F_HAVE_MORE;
239 	} else {
240 		if (ret != Z_STREAM_END)
241 			return got_error(GOT_ERR_DECOMPRESSION);
242 		zb->flags &= ~GOT_INFLATE_F_HAVE_MORE;
243 	}
244 
245 	*outlenp = z->total_out - last_total_out;
246 	return NULL;
247 }
248 
249 void
250 got_inflate_end(struct got_inflate_buf *zb)
251 {
252 	free(zb->inbuf);
253 	if (zb->flags & GOT_INFLATE_F_OWN_OUTBUF)
254 		free(zb->outbuf);
255 	inflateEnd(&zb->z);
256 }
257 
258 const struct got_error *
259 got_inflate_to_mem(uint8_t **outbuf, size_t *outlen,
260     size_t *consumed_total, FILE *f)
261 {
262 	const struct got_error *err;
263 	size_t avail, consumed;
264 	struct got_inflate_buf zb;
265 	void *newbuf;
266 	int nbuf = 1;
267 
268 	if (outbuf) {
269 		*outbuf = malloc(GOT_INFLATE_BUFSIZE);
270 		if (*outbuf == NULL)
271 			return got_error_from_errno("malloc");
272 		err = got_inflate_init(&zb, *outbuf, GOT_INFLATE_BUFSIZE, NULL);
273 	} else
274 		err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, NULL);
275 	if (err)
276 		return err;
277 
278 	*outlen = 0;
279 	if (consumed_total)
280 		*consumed_total = 0;
281 
282 	do {
283 		err = got_inflate_read(&zb, f, &avail, &consumed);
284 		if (err)
285 			goto done;
286 		*outlen += avail;
287 		if (consumed_total)
288 			*consumed_total += consumed;
289 		if (zb.flags & GOT_INFLATE_F_HAVE_MORE) {
290 			if (outbuf == NULL)
291 				continue;
292 			zb.outlen = (nbuf * GOT_INFLATE_BUFSIZE) - *outlen;
293 			newbuf = reallocarray(*outbuf, ++nbuf,
294 			   GOT_INFLATE_BUFSIZE);
295 			if (newbuf == NULL) {
296 				err = got_error_from_errno("reallocarray");
297 				free(*outbuf);
298 				*outbuf = NULL;
299 				*outlen = 0;
300 				goto done;
301 			}
302 			*outbuf = newbuf;
303 			zb.outbuf = newbuf + *outlen;
304 		}
305 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
306 
307 done:
308 	got_inflate_end(&zb);
309 	return err;
310 }
311 
312 const struct got_error *
313 got_inflate_to_mem_fd(uint8_t **outbuf, size_t *outlen,
314     size_t *consumed_total, uint32_t *input_crc, size_t expected_size, int infd)
315 {
316 	const struct got_error *err;
317 	size_t avail, consumed;
318 	struct got_inflate_buf zb;
319 	void *newbuf;
320 	int nbuf = 1;
321 	size_t bufsize = GOT_INFLATE_BUFSIZE;
322 
323 	/* Optimize buffer size in case short reads should suffice. */
324 	if (expected_size > 0 && expected_size < bufsize)
325 		bufsize = expected_size;
326 		
327 	if (outbuf) {
328 		*outbuf = malloc(bufsize);
329 		if (*outbuf == NULL)
330 			return got_error_from_errno("malloc");
331 		err = got_inflate_init(&zb, *outbuf, GOT_INFLATE_BUFSIZE,
332 		    input_crc);
333 	} else
334 		err = got_inflate_init(&zb, NULL, bufsize, input_crc);
335 	if (err)
336 		goto done;
337 
338 	*outlen = 0;
339 	if (consumed_total)
340 		*consumed_total = 0;
341 
342 	do {
343 		err = got_inflate_read_fd(&zb, infd, &avail, &consumed);
344 		if (err)
345 			goto done;
346 		*outlen += avail;
347 		if (consumed_total)
348 			*consumed_total += consumed;
349 		if (zb.flags & GOT_INFLATE_F_HAVE_MORE) {
350 			if (outbuf == NULL)
351 				continue;
352 			zb.outlen = (nbuf * GOT_INFLATE_BUFSIZE) - *outlen;
353 			newbuf = reallocarray(*outbuf, ++nbuf,
354 			    GOT_INFLATE_BUFSIZE);
355 			if (newbuf == NULL) {
356 				err = got_error_from_errno("reallocarray");
357 				free(*outbuf);
358 				*outbuf = NULL;
359 				*outlen = 0;
360 				goto done;
361 			}
362 			*outbuf = newbuf;
363 			zb.outbuf = newbuf + *outlen;
364 		}
365 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
366 
367 done:
368 	got_inflate_end(&zb);
369 	return err;
370 }
371 
372 const struct got_error *
373 got_inflate_to_mem_mmap(uint8_t **outbuf, size_t *outlen,
374     size_t *consumed_total, uint32_t *input_crc, uint8_t *map, size_t offset,
375     size_t len)
376 {
377 	const struct got_error *err;
378 	size_t avail, consumed;
379 	struct got_inflate_buf zb;
380 	void *newbuf;
381 	int nbuf = 1;
382 
383 	if (outbuf) {
384 		*outbuf = malloc(GOT_INFLATE_BUFSIZE);
385 		if (*outbuf == NULL)
386 			return got_error_from_errno("malloc");
387 		err = got_inflate_init(&zb, *outbuf, GOT_INFLATE_BUFSIZE,
388 		    input_crc);
389 		if (err) {
390 			free(*outbuf);
391 			*outbuf = NULL;
392 			return err;
393 		}
394 	} else {
395 		err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE,
396 		    input_crc);
397 	}
398 
399 	*outlen = 0;
400 	if (consumed_total)
401 		*consumed_total = 0;
402 	do {
403 		err = got_inflate_read_mmap(&zb, map, offset, len, &avail,
404 		    &consumed);
405 		if (err)
406 			goto done;
407 		offset += consumed;
408 		if (consumed_total)
409 			*consumed_total += consumed;
410 		len -= consumed;
411 		*outlen += avail;
412 		if (len == 0)
413 			break;
414 		if (zb.flags & GOT_INFLATE_F_HAVE_MORE) {
415 			if (outbuf == NULL)
416 				continue;
417 			newbuf = reallocarray(*outbuf, ++nbuf,
418 			    GOT_INFLATE_BUFSIZE);
419 			if (newbuf == NULL) {
420 				err = got_error_from_errno("reallocarray");
421 				free(*outbuf);
422 				*outbuf = NULL;
423 				*outlen = 0;
424 				goto done;
425 			}
426 			*outbuf = newbuf;
427 			zb.outbuf = newbuf + *outlen;
428 			zb.outlen = (nbuf * GOT_INFLATE_BUFSIZE) - *outlen;
429 		}
430 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
431 done:
432 	got_inflate_end(&zb);
433 	return err;
434 }
435 
436 const struct got_error *
437 got_inflate_to_fd(size_t *outlen, FILE *infile, int outfd)
438 {
439 	const struct got_error *err = NULL;
440 	size_t avail;
441 	struct got_inflate_buf zb;
442 
443 	err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, NULL);
444 	if (err)
445 		goto done;
446 
447 	*outlen = 0;
448 
449 	do {
450 		err = got_inflate_read(&zb, infile, &avail, NULL);
451 		if (err)
452 			goto done;
453 		if (avail > 0) {
454 			ssize_t n;
455 			n = write(outfd, zb.outbuf, avail);
456 			if (n != avail) {
457 				err = got_error_from_errno("write");
458 				goto done;
459 			}
460 			*outlen += avail;
461 		}
462 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
463 
464 done:
465 	if (err == NULL) {
466 		if (lseek(outfd, SEEK_SET, 0) == -1)
467 			err = got_error_from_errno("lseek");
468 	}
469 	got_inflate_end(&zb);
470 	return err;
471 }
472 
473 const struct got_error *
474 got_inflate_to_file(size_t *outlen, FILE *infile, FILE *outfile)
475 {
476 	const struct got_error *err;
477 	size_t avail;
478 	struct got_inflate_buf zb;
479 
480 	err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, NULL);
481 	if (err)
482 		goto done;
483 
484 	*outlen = 0;
485 
486 	do {
487 		err = got_inflate_read(&zb, infile, &avail, NULL);
488 		if (err)
489 			goto done;
490 		if (avail > 0) {
491 			size_t n;
492 			n = fwrite(zb.outbuf, avail, 1, outfile);
493 			if (n != 1) {
494 				err = got_ferror(outfile, GOT_ERR_IO);
495 				goto done;
496 			}
497 			*outlen += avail;
498 		}
499 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
500 
501 done:
502 	if (err == NULL)
503 		rewind(outfile);
504 	got_inflate_end(&zb);
505 	return err;
506 }
507 
508 const struct got_error *
509 got_inflate_to_file_fd(size_t *outlen, size_t *consumed_total,
510     uint32_t *input_crc, int infd, FILE *outfile)
511 {
512 	const struct got_error *err;
513 	size_t avail, consumed;
514 	struct got_inflate_buf zb;
515 
516 	err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, input_crc);
517 	if (err)
518 		goto done;
519 
520 	*outlen = 0;
521 	if (consumed_total)
522 		*consumed_total = 0;
523 	do {
524 		err = got_inflate_read_fd(&zb, infd, &avail, &consumed);
525 		if (err)
526 			goto done;
527 		if (avail > 0) {
528 			size_t n;
529 			n = fwrite(zb.outbuf, avail, 1, outfile);
530 			if (n != 1) {
531 				err = got_ferror(outfile, GOT_ERR_IO);
532 				goto done;
533 			}
534 			*outlen += avail;
535 			if (consumed_total)
536 				*consumed_total += consumed;
537 		}
538 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
539 
540 done:
541 	if (err == NULL)
542 		rewind(outfile);
543 	got_inflate_end(&zb);
544 	return err;
545 }
546 
547 const struct got_error *
548 got_inflate_to_file_mmap(size_t *outlen, size_t *consumed_total,
549     uint32_t *input_crc, uint8_t *map, size_t offset, size_t len,
550     FILE *outfile)
551 {
552 	const struct got_error *err;
553 	size_t avail, consumed;
554 	struct got_inflate_buf zb;
555 
556 	err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, input_crc);
557 	if (err)
558 		goto done;
559 
560 	*outlen = 0;
561 	if (consumed_total)
562 		*consumed_total = 0;
563 	do {
564 		err = got_inflate_read_mmap(&zb, map, offset, len, &avail,
565 		    &consumed);
566 		if (err)
567 			goto done;
568 		offset += consumed;
569 		if (consumed_total)
570 			*consumed_total += consumed;
571 		len -= consumed;
572 		if (avail > 0) {
573 			size_t n;
574 			n = fwrite(zb.outbuf, avail, 1, outfile);
575 			if (n != 1) {
576 				err = got_ferror(outfile, GOT_ERR_IO);
577 				goto done;
578 			}
579 			*outlen += avail;
580 		}
581 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
582 
583 done:
584 	if (err == NULL)
585 		rewind(outfile);
586 	got_inflate_end(&zb);
587 	return err;
588 }