op public repos

Blob

Date:: Wed Mar 18 16:13:43 2020 UTC
Message:: do not buffer more data than necessary in got_inflate_to_mem_fd()
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/queue.h>
18 
19 #include <errno.h>
20 #include <stdio.h>
21 #include <stdlib.h>
22 #include <string.h>
23 #include <sha1.h>
24 #include <zlib.h>
25 #include <time.h>
26 
27 #include "got_error.h"
28 #include "got_object.h"
29 #include "got_path.h"
30 
31 #include "got_lib_inflate.h"
32 
33 #ifndef MIN
34 #define	MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
35 #endif
36 
37 const struct got_error *
38 got_inflate_init(struct got_inflate_buf *zb, uint8_t *outbuf, size_t bufsize,
39     uint32_t *input_crc)
40 {
41 	const struct got_error *err = NULL;
42 	int zerr;
43 
44 	memset(&zb->z, 0, sizeof(zb->z));
45 
46 	zb->z.zalloc = Z_NULL;
47 	zb->z.zfree = Z_NULL;
48 	zerr = inflateInit(&zb->z);
49 	if (zerr != Z_OK) {
50 		if  (zerr == Z_ERRNO)
51 			return got_error_from_errno("inflateInit");
52 		if  (zerr == Z_MEM_ERROR) {
53 			errno = ENOMEM;
54 			return got_error_from_errno("inflateInit");
55 		}
56 		return got_error(GOT_ERR_DECOMPRESSION);
57 	}
58 
59 	zb->inlen = zb->outlen = bufsize;
60 
61 	zb->inbuf = calloc(1, zb->inlen);
62 	if (zb->inbuf == NULL) {
63 		err = got_error_from_errno("calloc");
64 		goto done;
65 	}
66 
67 	zb->flags = 0;
68 	if (outbuf == NULL) {
69 		zb->outbuf = calloc(1, zb->outlen);
70 		if (zb->outbuf == NULL) {
71 			err = got_error_from_errno("calloc");
72 			goto done;
73 		}
74 		zb->flags |= GOT_INFLATE_F_OWN_OUTBUF;
75 	} else
76 		zb->outbuf = outbuf;
77 
78 	zb->input_crc = input_crc;
79 done:
80 	if (err)
81 		got_inflate_end(zb);
82 	return err;
83 }
84 
85 const struct got_error *
86 got_inflate_read(struct got_inflate_buf *zb, FILE *f, size_t *outlenp,
87    size_t *consumed)
88 {
89 	size_t last_total_out = zb->z.total_out;
90 	size_t last_total_in = zb->z.total_in;
91 	z_stream *z = &zb->z;
92 	int ret = Z_ERRNO;
93 
94 	z->next_out = zb->outbuf;
95 	z->avail_out = zb->outlen;
96 
97 	*outlenp = 0;
98 	if (consumed)
99 		*consumed = 0;
100 	do {
101 		char *crc_in = NULL;
102 		size_t crc_avail = 0;
103 
104 		if (z->avail_in == 0) {
105 			size_t n = fread(zb->inbuf, 1, zb->inlen, f);
106 			if (n == 0) {
107 				if (ferror(f))
108 					return got_ferror(f, GOT_ERR_IO);
109 				/* EOF */
110 				ret = Z_STREAM_END;
111 				break;
112 			}
113 			z->next_in = zb->inbuf;
114 			z->avail_in = n;
115 		}
116 		if (zb->input_crc) {
117 			crc_in = z->next_in;
118 			crc_avail = z->avail_in;
119 		}
120 		ret = inflate(z, Z_SYNC_FLUSH);
121 		if (zb->input_crc) {
122 			*zb->input_crc = crc32(*zb->input_crc,
123 			    crc_in, crc_avail - z->avail_in);
124 		}
125 	} while (ret == Z_OK && z->avail_out > 0);
126 
127 	if (ret == Z_OK || ret == Z_BUF_ERROR) {
128 		zb->flags |= GOT_INFLATE_F_HAVE_MORE;
129 	} else {
130 		if (ret != Z_STREAM_END)
131 			return got_error(GOT_ERR_DECOMPRESSION);
132 		zb->flags &= ~GOT_INFLATE_F_HAVE_MORE;
133 	}
134 
135 	*outlenp = z->total_out - last_total_out;
136 	if (consumed)
137 		*consumed += z->total_in - last_total_in;
138 	return NULL;
139 }
140 
141 const struct got_error *
142 got_inflate_read_fd(struct got_inflate_buf *zb, int fd, size_t *outlenp,
143     size_t *consumed)
144 {
145 	size_t last_total_out = zb->z.total_out;
146 	size_t last_total_in = zb->z.total_in;
147 	z_stream *z = &zb->z;
148 	int ret = Z_ERRNO;
149 
150 	z->next_out = zb->outbuf;
151 	z->avail_out = zb->outlen;
152 
153 	*outlenp = 0;
154 	if (consumed)
155 		*consumed = 0;
156 	do {
157 		char *crc_in = NULL;
158 		size_t crc_avail = 0;
159 
160 		if (z->avail_in == 0) {
161 			ssize_t n = read(fd, zb->inbuf, zb->inlen);
162 			if (n < 0)
163 				return got_error_from_errno("read");
164 			else if (n == 0) {
165 				/* EOF */
166 				ret = Z_STREAM_END;
167 				break;
168 			}
169 			z->next_in = zb->inbuf;
170 			z->avail_in = n;
171 		}
172 		if (zb->input_crc) {
173 			crc_in = z->next_in;
174 			crc_avail = z->avail_in;
175 		}
176 		ret = inflate(z, Z_SYNC_FLUSH);
177 		if (zb->input_crc) {
178 			*zb->input_crc = crc32(*zb->input_crc,
179 			    crc_in, crc_avail - z->avail_in);
180 		}
181 	} while (ret == Z_OK && z->avail_out > 0);
182 
183 	if (ret == Z_OK || ret == Z_BUF_ERROR) {
184 		zb->flags |= GOT_INFLATE_F_HAVE_MORE;
185 	} else {
186 		if (ret != Z_STREAM_END)
187 			return got_error(GOT_ERR_DECOMPRESSION);
188 		zb->flags &= ~GOT_INFLATE_F_HAVE_MORE;
189 	}
190 
191 	*outlenp = z->total_out - last_total_out;
192 	if (consumed)
193 		*consumed += z->total_in - last_total_in;
194 	return NULL;
195 }
196 
197 const struct got_error *
198 got_inflate_read_mmap(struct got_inflate_buf *zb, uint8_t *map, size_t offset,
199     size_t len, size_t *outlenp, size_t *consumed)
200 {
201 	size_t last_total_out = zb->z.total_out;
202 	z_stream *z = &zb->z;
203 	int ret = Z_ERRNO;
204 
205 	z->next_out = zb->outbuf;
206 	z->avail_out = zb->outlen;
207 
208 	*outlenp = 0;
209 	*consumed = 0;
210 
211 	do {
212 		char *crc_in = NULL;
213 		size_t crc_avail = 0;
214 		size_t last_total_in = zb->z.total_in;
215 
216 		if (z->avail_in == 0) {
217 			if (len == 0) {
218 				/* EOF */
219 				ret = Z_STREAM_END;
220 				break;
221 			}
222 			z->next_in = map + offset + *consumed;
223 			z->avail_in = len - *consumed;
224 		}
225 		if (zb->input_crc) {
226 			crc_in = z->next_in;
227 			crc_avail = z->avail_in;
228 		}
229 		ret = inflate(z, Z_SYNC_FLUSH);
230 		if (zb->input_crc) {
231 			*zb->input_crc = crc32(*zb->input_crc,
232 			    crc_in, crc_avail - z->avail_in);
233 		}
234 		*consumed += z->total_in - last_total_in;
235 	} while (ret == Z_OK && z->avail_out > 0);
236 
237 	if (ret == Z_OK || ret == Z_BUF_ERROR) {
238 		zb->flags |= GOT_INFLATE_F_HAVE_MORE;
239 	} else {
240 		if (ret != Z_STREAM_END)
241 			return got_error(GOT_ERR_DECOMPRESSION);
242 		zb->flags &= ~GOT_INFLATE_F_HAVE_MORE;
243 	}
244 
245 	*outlenp = z->total_out - last_total_out;
246 	return NULL;
247 }
248 
249 void
250 got_inflate_end(struct got_inflate_buf *zb)
251 {
252 	free(zb->inbuf);
253 	if (zb->flags & GOT_INFLATE_F_OWN_OUTBUF)
254 		free(zb->outbuf);
255 	inflateEnd(&zb->z);
256 }
257 
258 const struct got_error *
259 got_inflate_to_mem(uint8_t **outbuf, size_t *outlen,
260     size_t *consumed_total, FILE *f)
261 {
262 	const struct got_error *err;
263 	size_t avail, consumed;
264 	struct got_inflate_buf zb;
265 	void *newbuf;
266 	int nbuf = 1;
267 
268 	if (outbuf) {
269 		*outbuf = malloc(GOT_INFLATE_BUFSIZE);
270 		if (*outbuf == NULL)
271 			return got_error_from_errno("malloc");
272 		err = got_inflate_init(&zb, *outbuf, GOT_INFLATE_BUFSIZE, NULL);
273 	} else
274 		err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, NULL);
275 	if (err)
276 		return err;
277 
278 	*outlen = 0;
279 	if (consumed_total)
280 		*consumed_total = 0;
281 
282 	do {
283 		err = got_inflate_read(&zb, f, &avail, &consumed);
284 		if (err)
285 			goto done;
286 		*outlen += avail;
287 		if (consumed_total)
288 			*consumed_total += consumed;
289 		if (zb.flags & GOT_INFLATE_F_HAVE_MORE) {
290 			if (outbuf == NULL)
291 				continue;
292 			zb.outlen = (nbuf * GOT_INFLATE_BUFSIZE) - *outlen;
293 			newbuf = reallocarray(*outbuf, ++nbuf,
294 			   GOT_INFLATE_BUFSIZE);
295 			if (newbuf == NULL) {
296 				err = got_error_from_errno("reallocarray");
297 				free(*outbuf);
298 				*outbuf = NULL;
299 				*outlen = 0;
300 				goto done;
301 			}
302 			*outbuf = newbuf;
303 			zb.outbuf = newbuf + *outlen;
304 		}
305 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
306 
307 done:
308 	got_inflate_end(&zb);
309 	return err;
310 }
311 
312 const struct got_error *
313 got_inflate_to_mem_fd(uint8_t **outbuf, size_t *outlen,
314     size_t *consumed_total, uint32_t *input_crc, size_t expected_size, int infd)
315 {
316 	const struct got_error *err;
317 	size_t avail, consumed;
318 	struct got_inflate_buf zb;
319 	void *newbuf;
320 	int nbuf = 1;
321 	size_t bufsize = GOT_INFLATE_BUFSIZE;
322 
323 	/* Optimize buffer size in case short reads should suffice. */
324 	if (expected_size > 0 && expected_size < bufsize)
325 		bufsize = expected_size;
326 		
327 	if (outbuf) {
328 		*outbuf = malloc(bufsize);
329 		if (*outbuf == NULL)
330 			return got_error_from_errno("malloc");
331 		err = got_inflate_init(&zb, *outbuf, GOT_INFLATE_BUFSIZE,
332 		    input_crc);
333 	} else
334 		err = got_inflate_init(&zb, NULL, bufsize, input_crc);
335 	if (err)
336 		goto done;
337 
338 	*outlen = 0;
339 	if (consumed_total)
340 		*consumed_total = 0;
341 
342 	do {
343 		err = got_inflate_read_fd(&zb, infd, &avail, &consumed);
344 		if (err)
345 			goto done;
346 		*outlen += avail;
347 		if (consumed_total)
348 			*consumed_total += consumed;
349 		if (zb.flags & GOT_INFLATE_F_HAVE_MORE) {
350 			if (outbuf == NULL)
351 				continue;
352 			zb.outlen = (nbuf * GOT_INFLATE_BUFSIZE) - *outlen;
353 			newbuf = reallocarray(*outbuf, ++nbuf,
354 			    GOT_INFLATE_BUFSIZE);
355 			if (newbuf == NULL) {
356 				err = got_error_from_errno("reallocarray");
357 				free(*outbuf);
358 				*outbuf = NULL;
359 				*outlen = 0;
360 				goto done;
361 			}
362 			*outbuf = newbuf;
363 			zb.outbuf = newbuf + *outlen;
364 		}
365 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
366 
367 done:
368 	got_inflate_end(&zb);
369 	return err;
370 }
371 
372 const struct got_error *
373 got_inflate_to_mem_mmap(uint8_t **outbuf, size_t *outlen, uint8_t *map,
374     size_t offset, size_t len)
375 {
376 	const struct got_error *err;
377 	size_t avail, consumed;
378 	struct got_inflate_buf zb;
379 	void *newbuf;
380 	int nbuf = 1;
381 
382 	*outbuf = malloc(GOT_INFLATE_BUFSIZE);
383 	if (*outbuf == NULL)
384 		return got_error_from_errno("malloc");
385 	err = got_inflate_init(&zb, *outbuf, GOT_INFLATE_BUFSIZE, NULL);
386 	if (err) {
387 		free(*outbuf);
388 		*outbuf = NULL;
389 		return err;
390 	}
391 
392 	*outlen = 0;
393 
394 	do {
395 		err = got_inflate_read_mmap(&zb, map, offset, len, &avail,
396 		    &consumed);
397 		if (err)
398 			goto done;
399 		offset += consumed;
400 		len -= consumed;
401 		*outlen += avail;
402 		if (len == 0)
403 			break;
404 		if (zb.flags & GOT_INFLATE_F_HAVE_MORE) {
405 			newbuf = reallocarray(*outbuf, ++nbuf,
406 			    GOT_INFLATE_BUFSIZE);
407 			if (newbuf == NULL) {
408 				err = got_error_from_errno("reallocarray");
409 				free(*outbuf);
410 				*outbuf = NULL;
411 				*outlen = 0;
412 				goto done;
413 			}
414 			*outbuf = newbuf;
415 			zb.outbuf = newbuf + *outlen;
416 			zb.outlen = (nbuf * GOT_INFLATE_BUFSIZE) - *outlen;
417 		}
418 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
419 done:
420 	got_inflate_end(&zb);
421 	return err;
422 }
423 
424 const struct got_error *
425 got_inflate_to_fd(size_t *outlen, FILE *infile, int outfd)
426 {
427 	const struct got_error *err = NULL;
428 	size_t avail;
429 	struct got_inflate_buf zb;
430 
431 	err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, NULL);
432 	if (err)
433 		goto done;
434 
435 	*outlen = 0;
436 
437 	do {
438 		err = got_inflate_read(&zb, infile, &avail, NULL);
439 		if (err)
440 			goto done;
441 		if (avail > 0) {
442 			ssize_t n;
443 			n = write(outfd, zb.outbuf, avail);
444 			if (n != avail) {
445 				err = got_error_from_errno("write");
446 				goto done;
447 			}
448 			*outlen += avail;
449 		}
450 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
451 
452 done:
453 	if (err == NULL) {
454 		if (lseek(outfd, SEEK_SET, 0) == -1)
455 			err = got_error_from_errno("lseek");
456 	}
457 	got_inflate_end(&zb);
458 	return err;
459 }
460 
461 const struct got_error *
462 got_inflate_to_file(size_t *outlen, FILE *infile, FILE *outfile)
463 {
464 	const struct got_error *err;
465 	size_t avail;
466 	struct got_inflate_buf zb;
467 
468 	err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, NULL);
469 	if (err)
470 		goto done;
471 
472 	*outlen = 0;
473 
474 	do {
475 		err = got_inflate_read(&zb, infile, &avail, NULL);
476 		if (err)
477 			goto done;
478 		if (avail > 0) {
479 			size_t n;
480 			n = fwrite(zb.outbuf, avail, 1, outfile);
481 			if (n != 1) {
482 				err = got_ferror(outfile, GOT_ERR_IO);
483 				goto done;
484 			}
485 			*outlen += avail;
486 		}
487 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
488 
489 done:
490 	if (err == NULL)
491 		rewind(outfile);
492 	got_inflate_end(&zb);
493 	return err;
494 }
495 
496 const struct got_error *
497 got_inflate_to_file_fd(size_t *outlen, int infd, FILE *outfile)
498 {
499 	const struct got_error *err;
500 	size_t avail;
501 	struct got_inflate_buf zb;
502 
503 	err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, NULL);
504 	if (err)
505 		goto done;
506 
507 	*outlen = 0;
508 
509 	do {
510 		err = got_inflate_read_fd(&zb, infd, &avail, NULL);
511 		if (err)
512 			goto done;
513 		if (avail > 0) {
514 			size_t n;
515 			n = fwrite(zb.outbuf, avail, 1, outfile);
516 			if (n != 1) {
517 				err = got_ferror(outfile, GOT_ERR_IO);
518 				goto done;
519 			}
520 			*outlen += avail;
521 		}
522 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
523 
524 done:
525 	if (err == NULL)
526 		rewind(outfile);
527 	got_inflate_end(&zb);
528 	return err;
529 }
530 
531 const struct got_error *
532 got_inflate_to_file_mmap(size_t *outlen, uint8_t *map, size_t offset,
533     size_t len, FILE *outfile)
534 {
535 	const struct got_error *err;
536 	size_t avail;
537 	struct got_inflate_buf zb;
538 	size_t consumed;
539 
540 	err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, NULL);
541 	if (err)
542 		goto done;
543 
544 	*outlen = 0;
545 
546 	do {
547 		err = got_inflate_read_mmap(&zb, map, offset, len, &avail,
548 		    &consumed);
549 		if (err)
550 			goto done;
551 		offset += consumed;
552 		len -= consumed;
553 		if (avail > 0) {
554 			size_t n;
555 			n = fwrite(zb.outbuf, avail, 1, outfile);
556 			if (n != 1) {
557 				err = got_ferror(outfile, GOT_ERR_IO);
558 				goto done;
559 			}
560 			*outlen += avail;
561 		}
562 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
563 
564 done:
565 	if (err == NULL)
566 		rewind(outfile);
567 	got_inflate_end(&zb);
568 	return err;
569 }