op public repos

Blob

Date:: Sat Feb 4 14:00:31 2023 UTC
Message:: inflate: support sha256
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/queue.h>
18 
19 #include <errno.h>
20 #include <stdio.h>
21 #include <stdlib.h>
22 #include <string.h>
23 #include <sha1.h>
24 #include <sha2.h>
25 #include <poll.h>
26 #include <unistd.h>
27 #include <zlib.h>
28 #include <time.h>
29 
30 #include "got_error.h"
31 #include "got_object.h"
32 #include "got_path.h"
33 
34 #include "got_lib_inflate.h"
35 #include "got_lib_poll.h"
36 
37 #ifndef MIN
38 #define	MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
39 #endif
40 
41 const struct got_error *
42 got_inflate_init(struct got_inflate_buf *zb, uint8_t *outbuf, size_t bufsize,
43     struct got_inflate_checksum *csum)
44 {
45 	const struct got_error *err = NULL;
46 	int zerr;
47 
48 	memset(zb, 0, sizeof(*zb));
49 
50 	zb->z.zalloc = Z_NULL;
51 	zb->z.zfree = Z_NULL;
52 	zerr = inflateInit(&zb->z);
53 	if (zerr != Z_OK) {
54 		if  (zerr == Z_ERRNO)
55 			return got_error_from_errno("inflateInit");
56 		if  (zerr == Z_MEM_ERROR) {
57 			errno = ENOMEM;
58 			return got_error_from_errno("inflateInit");
59 		}
60 		return got_error(GOT_ERR_DECOMPRESSION);
61 	}
62 
63 	zb->inlen = zb->outlen = bufsize;
64 
65 	zb->inbuf = calloc(1, zb->inlen);
66 	if (zb->inbuf == NULL) {
67 		err = got_error_from_errno("calloc");
68 		goto done;
69 	}
70 
71 	zb->flags = 0;
72 	if (outbuf == NULL) {
73 		zb->outbuf = calloc(1, zb->outlen);
74 		if (zb->outbuf == NULL) {
75 			err = got_error_from_errno("calloc");
76 			goto done;
77 		}
78 		zb->flags |= GOT_INFLATE_F_OWN_OUTBUF;
79 	} else
80 		zb->outbuf = outbuf;
81 
82 	zb->csum = csum;
83 done:
84 	if (err)
85 		got_inflate_end(zb);
86 	return err;
87 }
88 
89 static void
90 csum_input(struct got_inflate_checksum *csum, const uint8_t *buf, size_t len)
91 {
92 	if (csum->input_crc)
93 		*csum->input_crc = crc32(*csum->input_crc, buf, len);
94 
95 	if (csum->input_sha1)
96 		SHA1Update(csum->input_sha1, buf, len);
97 
98 	if (csum->input_sha256)
99 		SHA256Update(csum->input_sha256, buf, len);
100 }
101 
102 static void
103 csum_output(struct got_inflate_checksum *csum, const uint8_t *buf, size_t len)
104 {
105 	if (csum->output_crc)
106 		*csum->output_crc = crc32(*csum->output_crc, buf, len);
107 
108 	if (csum->output_sha1)
109 		SHA1Update(csum->output_sha1, buf, len);
110 
111 	if (csum->output_sha256)
112 		SHA256Update(csum->output_sha256, buf, len);
113 }
114 
115 const struct got_error *
116 got_inflate_read(struct got_inflate_buf *zb, FILE *f, size_t *outlenp,
117     size_t *consumed)
118 {
119 	size_t last_total_out = zb->z.total_out;
120 	size_t last_total_in = zb->z.total_in;
121 	z_stream *z = &zb->z;
122 	int ret = Z_ERRNO;
123 
124 	z->next_out = zb->outbuf;
125 	z->avail_out = zb->outlen;
126 
127 	*outlenp = 0;
128 	if (consumed)
129 		*consumed = 0;
130 	do {
131 		uint8_t *csum_in = NULL, *csum_out = NULL;
132 		size_t csum_avail_in = 0, csum_avail_out = 0;
133 
134 		if (z->avail_in == 0) {
135 			size_t n = fread(zb->inbuf, 1, zb->inlen, f);
136 			if (n == 0) {
137 				if (ferror(f))
138 					return got_ferror(f, GOT_ERR_IO);
139 				/* EOF */
140 				ret = Z_STREAM_END;
141 				break;
142 			}
143 			z->next_in = zb->inbuf;
144 			z->avail_in = n;
145 		}
146 		if (zb->csum) {
147 			csum_in = z->next_in;
148 			csum_avail_in = z->avail_in;
149 			csum_out = z->next_out;
150 			csum_avail_out = z->avail_out;
151 		}
152 		ret = inflate(z, Z_SYNC_FLUSH);
153 		if (zb->csum) {
154 			csum_input(zb->csum, csum_in,
155 			    csum_avail_in - z->avail_in);
156 			csum_output(zb->csum, csum_out,
157 			    csum_avail_out - z->avail_out);
158 		}
159 	} while (ret == Z_OK && z->avail_out > 0);
160 
161 	if (ret == Z_OK || ret == Z_BUF_ERROR) {
162 		zb->flags |= GOT_INFLATE_F_HAVE_MORE;
163 	} else {
164 		if (ret != Z_STREAM_END)
165 			return got_error(GOT_ERR_DECOMPRESSION);
166 		zb->flags &= ~GOT_INFLATE_F_HAVE_MORE;
167 	}
168 
169 	*outlenp = z->total_out - last_total_out;
170 	if (consumed)
171 		*consumed += z->total_in - last_total_in;
172 	return NULL;
173 }
174 
175 const struct got_error *
176 got_inflate_read_fd(struct got_inflate_buf *zb, int fd, size_t *outlenp,
177     size_t *consumed)
178 {
179 	const struct got_error *err = NULL;
180 	size_t last_total_out = zb->z.total_out;
181 	size_t last_total_in = zb->z.total_in;
182 	z_stream *z = &zb->z;
183 	int ret = Z_ERRNO;
184 
185 	z->next_out = zb->outbuf;
186 	z->avail_out = zb->outlen;
187 
188 	*outlenp = 0;
189 	if (consumed)
190 		*consumed = 0;
191 	do {
192 		uint8_t *csum_in = NULL, *csum_out = NULL;
193 		size_t csum_avail_in = 0, csum_avail_out = 0;
194 
195 		if (z->avail_in == 0) {
196 			ssize_t n;
197 			err = got_poll_fd(fd, POLLIN, INFTIM);
198 			if (err) {
199 				if (err->code == GOT_ERR_EOF) {
200 					ret = Z_STREAM_END;
201 					break;
202 				}
203 				return err;
204 			}
205 			n = read(fd, zb->inbuf, zb->inlen);
206 			if (n < 0)
207 				return got_error_from_errno("read");
208 			else if (n == 0) {
209 				/* EOF */
210 				ret = Z_STREAM_END;
211 				break;
212 			}
213 			z->next_in = zb->inbuf;
214 			z->avail_in = n;
215 		}
216 		if (zb->csum) {
217 			csum_in = z->next_in;
218 			csum_avail_in = z->avail_in;
219 			csum_out = z->next_out;
220 			csum_avail_out = z->avail_out;
221 		}
222 		ret = inflate(z, Z_SYNC_FLUSH);
223 		if (zb->csum) {
224 			csum_input(zb->csum, csum_in,
225 			    csum_avail_in - z->avail_in);
226 			csum_output(zb->csum, csum_out,
227 			    csum_avail_out - z->avail_out);
228 		}
229 	} while (ret == Z_OK && z->avail_out > 0);
230 
231 	if (ret == Z_OK || ret == Z_BUF_ERROR) {
232 		zb->flags |= GOT_INFLATE_F_HAVE_MORE;
233 	} else {
234 		if (ret != Z_STREAM_END)
235 			return got_error(GOT_ERR_DECOMPRESSION);
236 		zb->flags &= ~GOT_INFLATE_F_HAVE_MORE;
237 	}
238 
239 	*outlenp = z->total_out - last_total_out;
240 	if (consumed)
241 		*consumed += z->total_in - last_total_in;
242 	return NULL;
243 }
244 
245 const struct got_error *
246 got_inflate_read_mmap(struct got_inflate_buf *zb, uint8_t *map, size_t offset,
247     size_t len, size_t *outlenp, size_t *consumed)
248 {
249 	size_t last_total_out = zb->z.total_out;
250 	z_stream *z = &zb->z;
251 	int ret = Z_ERRNO;
252 
253 	z->next_out = zb->outbuf;
254 	z->avail_out = zb->outlen;
255 
256 	*outlenp = 0;
257 	*consumed = 0;
258 
259 	do {
260 		uint8_t *csum_in = NULL, *csum_out = NULL;
261 		size_t csum_avail_in = 0, csum_avail_out = 0;
262 		size_t last_total_in = zb->z.total_in;
263 
264 		if (z->avail_in == 0) {
265 			if (len == 0) {
266 				/* EOF */
267 				ret = Z_STREAM_END;
268 				break;
269 			}
270 			z->next_in = map + offset + *consumed;
271 			if (len - *consumed > UINT_MAX)
272 				z->avail_in = UINT_MAX;
273 			else
274 				z->avail_in = len - *consumed;
275 		}
276 		if (zb->csum) {
277 			csum_in = z->next_in;
278 			csum_avail_in = z->avail_in;
279 			csum_out = z->next_out;
280 			csum_avail_out = z->avail_out;
281 		}
282 		ret = inflate(z, Z_SYNC_FLUSH);
283 		if (zb->csum) {
284 			csum_input(zb->csum, csum_in,
285 			    csum_avail_in - z->avail_in);
286 			csum_output(zb->csum, csum_out,
287 			    csum_avail_out - z->avail_out);
288 		}
289 		*consumed += z->total_in - last_total_in;
290 	} while (ret == Z_OK && z->avail_out > 0);
291 
292 	if (ret == Z_OK || ret == Z_BUF_ERROR) {
293 		zb->flags |= GOT_INFLATE_F_HAVE_MORE;
294 	} else {
295 		if (ret != Z_STREAM_END)
296 			return got_error(GOT_ERR_DECOMPRESSION);
297 		zb->flags &= ~GOT_INFLATE_F_HAVE_MORE;
298 	}
299 
300 	*outlenp = z->total_out - last_total_out;
301 	return NULL;
302 }
303 
304 void
305 got_inflate_end(struct got_inflate_buf *zb)
306 {
307 	free(zb->inbuf);
308 	if (zb->flags & GOT_INFLATE_F_OWN_OUTBUF)
309 		free(zb->outbuf);
310 	inflateEnd(&zb->z);
311 }
312 
313 const struct got_error *
314 got_inflate_to_mem(uint8_t **outbuf, size_t *outlen,
315     size_t *consumed_total, struct got_inflate_checksum *csum, FILE *f)
316 {
317 	const struct got_error *err;
318 	size_t avail, consumed;
319 	struct got_inflate_buf zb;
320 	void *newbuf;
321 	int nbuf = 1;
322 
323 	if (outbuf) {
324 		*outbuf = malloc(GOT_INFLATE_BUFSIZE);
325 		if (*outbuf == NULL)
326 			return got_error_from_errno("malloc");
327 		err = got_inflate_init(&zb, *outbuf, GOT_INFLATE_BUFSIZE, csum);
328 	} else
329 		err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, csum);
330 	if (err)
331 		return err;
332 
333 	*outlen = 0;
334 	if (consumed_total)
335 		*consumed_total = 0;
336 
337 	do {
338 		err = got_inflate_read(&zb, f, &avail, &consumed);
339 		if (err)
340 			goto done;
341 		*outlen += avail;
342 		if (consumed_total)
343 			*consumed_total += consumed;
344 		if (zb.flags & GOT_INFLATE_F_HAVE_MORE) {
345 			if (outbuf == NULL)
346 				continue;
347 			newbuf = reallocarray(*outbuf, ++nbuf,
348 			    GOT_INFLATE_BUFSIZE);
349 			if (newbuf == NULL) {
350 				err = got_error_from_errno("reallocarray");
351 				free(*outbuf);
352 				*outbuf = NULL;
353 				*outlen = 0;
354 				goto done;
355 			}
356 			*outbuf = newbuf;
357 			zb.outbuf = newbuf + *outlen;
358 			zb.outlen = (nbuf * GOT_INFLATE_BUFSIZE) - *outlen;
359 		}
360 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
361 
362 done:
363 	got_inflate_end(&zb);
364 	return err;
365 }
366 
367 const struct got_error *
368 got_inflate_to_mem_fd(uint8_t **outbuf, size_t *outlen,
369     size_t *consumed_total, struct got_inflate_checksum *csum,
370     size_t expected_size, int infd)
371 {
372 	const struct got_error *err;
373 	size_t avail, consumed;
374 	struct got_inflate_buf zb;
375 	void *newbuf;
376 	int nbuf = 1;
377 	size_t bufsize = GOT_INFLATE_BUFSIZE;
378 
379 	/* Optimize buffer size in case short reads should suffice. */
380 	if (expected_size > 0 && expected_size < bufsize)
381 		bufsize = expected_size;
382 
383 	if (outbuf) {
384 		*outbuf = malloc(bufsize);
385 		if (*outbuf == NULL)
386 			return got_error_from_errno("malloc");
387 		err = got_inflate_init(&zb, *outbuf, GOT_INFLATE_BUFSIZE, csum);
388 	} else
389 		err = got_inflate_init(&zb, NULL, bufsize, csum);
390 	if (err)
391 		goto done;
392 
393 	*outlen = 0;
394 	if (consumed_total)
395 		*consumed_total = 0;
396 
397 	do {
398 		err = got_inflate_read_fd(&zb, infd, &avail, &consumed);
399 		if (err)
400 			goto done;
401 		*outlen += avail;
402 		if (consumed_total)
403 			*consumed_total += consumed;
404 		if (zb.flags & GOT_INFLATE_F_HAVE_MORE) {
405 			if (outbuf == NULL)
406 				continue;
407 			newbuf = reallocarray(*outbuf, ++nbuf,
408 			    GOT_INFLATE_BUFSIZE);
409 			if (newbuf == NULL) {
410 				err = got_error_from_errno("reallocarray");
411 				free(*outbuf);
412 				*outbuf = NULL;
413 				*outlen = 0;
414 				goto done;
415 			}
416 			*outbuf = newbuf;
417 			zb.outbuf = newbuf + *outlen;
418 			zb.outlen = (nbuf * GOT_INFLATE_BUFSIZE) - *outlen;
419 		}
420 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
421 
422 done:
423 	got_inflate_end(&zb);
424 	return err;
425 }
426 
427 const struct got_error *
428 got_inflate_to_mem_mmap(uint8_t **outbuf, size_t *outlen,
429     size_t *consumed_total, struct got_inflate_checksum *csum, uint8_t *map,
430     size_t offset, size_t len)
431 {
432 	const struct got_error *err;
433 	size_t avail, consumed;
434 	struct got_inflate_buf zb;
435 	void *newbuf;
436 	int nbuf = 1;
437 
438 	if (outbuf) {
439 		*outbuf = malloc(GOT_INFLATE_BUFSIZE);
440 		if (*outbuf == NULL)
441 			return got_error_from_errno("malloc");
442 		err = got_inflate_init(&zb, *outbuf, GOT_INFLATE_BUFSIZE, csum);
443 		if (err) {
444 			free(*outbuf);
445 			*outbuf = NULL;
446 			return err;
447 		}
448 	} else {
449 		err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, csum);
450 		if (err)
451 			return err;
452 	}
453 
454 	*outlen = 0;
455 	if (consumed_total)
456 		*consumed_total = 0;
457 	do {
458 		err = got_inflate_read_mmap(&zb, map, offset, len, &avail,
459 		    &consumed);
460 		if (err)
461 			goto done;
462 		offset += consumed;
463 		if (consumed_total)
464 			*consumed_total += consumed;
465 		len -= consumed;
466 		*outlen += avail;
467 		if (len == 0)
468 			break;
469 		if (zb.flags & GOT_INFLATE_F_HAVE_MORE) {
470 			if (outbuf == NULL)
471 				continue;
472 			newbuf = reallocarray(*outbuf, ++nbuf,
473 			    GOT_INFLATE_BUFSIZE);
474 			if (newbuf == NULL) {
475 				err = got_error_from_errno("reallocarray");
476 				free(*outbuf);
477 				*outbuf = NULL;
478 				*outlen = 0;
479 				goto done;
480 			}
481 			*outbuf = newbuf;
482 			zb.outbuf = newbuf + *outlen;
483 			zb.outlen = (nbuf * GOT_INFLATE_BUFSIZE) - *outlen;
484 		}
485 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
486 done:
487 	got_inflate_end(&zb);
488 	return err;
489 }
490 
491 const struct got_error *
492 got_inflate_to_fd(size_t *outlen, FILE *infile,
493     struct got_inflate_checksum *csum, int outfd)
494 {
495 	const struct got_error *err = NULL;
496 	size_t avail;
497 	struct got_inflate_buf zb;
498 
499 	err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, csum);
500 	if (err)
501 		goto done;
502 
503 	*outlen = 0;
504 
505 	do {
506 		err = got_inflate_read(&zb, infile, &avail, NULL);
507 		if (err)
508 			goto done;
509 		if (avail > 0) {
510 			ssize_t n;
511 			n = write(outfd, zb.outbuf, avail);
512 			if (n != avail) {
513 				err = got_error_from_errno("write");
514 				goto done;
515 			}
516 			*outlen += avail;
517 		}
518 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
519 
520 done:
521 	if (err == NULL) {
522 		if (lseek(outfd, SEEK_SET, 0) == -1)
523 			err = got_error_from_errno("lseek");
524 	}
525 	got_inflate_end(&zb);
526 	return err;
527 }
528 
529 const struct got_error *
530 got_inflate_to_file(size_t *outlen, FILE *infile,
531     struct got_inflate_checksum *csum, FILE *outfile)
532 {
533 	const struct got_error *err;
534 	size_t avail;
535 	struct got_inflate_buf zb;
536 
537 	err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, csum);
538 	if (err)
539 		goto done;
540 
541 	*outlen = 0;
542 
543 	do {
544 		err = got_inflate_read(&zb, infile, &avail, NULL);
545 		if (err)
546 			goto done;
547 		if (avail > 0) {
548 			size_t n;
549 			n = fwrite(zb.outbuf, avail, 1, outfile);
550 			if (n != 1) {
551 				err = got_ferror(outfile, GOT_ERR_IO);
552 				goto done;
553 			}
554 			*outlen += avail;
555 		}
556 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
557 
558 done:
559 	if (err == NULL)
560 		rewind(outfile);
561 	got_inflate_end(&zb);
562 	return err;
563 }
564 
565 const struct got_error *
566 got_inflate_to_file_fd(size_t *outlen, size_t *consumed_total,
567     struct got_inflate_checksum *csum, int infd, FILE *outfile)
568 {
569 	const struct got_error *err;
570 	size_t avail, consumed;
571 	struct got_inflate_buf zb;
572 
573 	err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, csum);
574 	if (err)
575 		goto done;
576 
577 	*outlen = 0;
578 	if (consumed_total)
579 		*consumed_total = 0;
580 	do {
581 		err = got_inflate_read_fd(&zb, infd, &avail, &consumed);
582 		if (err)
583 			goto done;
584 		if (avail > 0) {
585 			size_t n;
586 			n = fwrite(zb.outbuf, avail, 1, outfile);
587 			if (n != 1) {
588 				err = got_ferror(outfile, GOT_ERR_IO);
589 				goto done;
590 			}
591 			*outlen += avail;
592 			if (consumed_total)
593 				*consumed_total += consumed;
594 		}
595 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
596 
597 done:
598 	if (err == NULL)
599 		rewind(outfile);
600 	got_inflate_end(&zb);
601 	return err;
602 }
603 
604 const struct got_error *
605 got_inflate_to_file_mmap(size_t *outlen, size_t *consumed_total,
606     struct got_inflate_checksum *csum, uint8_t *map, size_t offset,
607     size_t len, FILE *outfile)
608 {
609 	const struct got_error *err;
610 	size_t avail, consumed;
611 	struct got_inflate_buf zb;
612 
613 	err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, csum);
614 	if (err)
615 		goto done;
616 
617 	*outlen = 0;
618 	if (consumed_total)
619 		*consumed_total = 0;
620 	do {
621 		err = got_inflate_read_mmap(&zb, map, offset, len, &avail,
622 		    &consumed);
623 		if (err)
624 			goto done;
625 		offset += consumed;
626 		if (consumed_total)
627 			*consumed_total += consumed;
628 		len -= consumed;
629 		if (avail > 0) {
630 			size_t n;
631 			n = fwrite(zb.outbuf, avail, 1, outfile);
632 			if (n != 1) {
633 				err = got_ferror(outfile, GOT_ERR_IO);
634 				goto done;
635 			}
636 			*outlen += avail;
637 		}
638 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
639 
640 done:
641 	if (err == NULL)
642 		rewind(outfile);
643 	got_inflate_end(&zb);
644 	return err;
645 }