op public repos

Blob

Date:: Wed Mar 18 16:13:42 2020 UTC
Message:: avoid re-reading the entirety of an object's data to calculate the CRC
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/queue.h>
18 
19 #include <errno.h>
20 #include <stdio.h>
21 #include <stdlib.h>
22 #include <string.h>
23 #include <sha1.h>
24 #include <zlib.h>
25 #include <time.h>
26 
27 #include "got_error.h"
28 #include "got_object.h"
29 #include "got_path.h"
30 
31 #include "got_lib_inflate.h"
32 
33 #ifndef MIN
34 #define	MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
35 #endif
36 
37 const struct got_error *
38 got_inflate_init(struct got_inflate_buf *zb, uint8_t *outbuf, size_t bufsize,
39     uint32_t *input_crc)
40 {
41 	const struct got_error *err = NULL;
42 	int zerr;
43 
44 	memset(&zb->z, 0, sizeof(zb->z));
45 
46 	zb->z.zalloc = Z_NULL;
47 	zb->z.zfree = Z_NULL;
48 	zerr = inflateInit(&zb->z);
49 	if (zerr != Z_OK) {
50 		if  (zerr == Z_ERRNO)
51 			return got_error_from_errno("inflateInit");
52 		if  (zerr == Z_MEM_ERROR) {
53 			errno = ENOMEM;
54 			return got_error_from_errno("inflateInit");
55 		}
56 		return got_error(GOT_ERR_DECOMPRESSION);
57 	}
58 
59 	zb->inlen = zb->outlen = bufsize;
60 
61 	zb->inbuf = calloc(1, zb->inlen);
62 	if (zb->inbuf == NULL) {
63 		err = got_error_from_errno("calloc");
64 		goto done;
65 	}
66 
67 	zb->flags = 0;
68 	if (outbuf == NULL) {
69 		zb->outbuf = calloc(1, zb->outlen);
70 		if (zb->outbuf == NULL) {
71 			err = got_error_from_errno("calloc");
72 			goto done;
73 		}
74 		zb->flags |= GOT_INFLATE_F_OWN_OUTBUF;
75 	} else
76 		zb->outbuf = outbuf;
77 
78 	zb->input_crc = input_crc;
79 done:
80 	if (err)
81 		got_inflate_end(zb);
82 	return err;
83 }
84 
85 const struct got_error *
86 got_inflate_read(struct got_inflate_buf *zb, FILE *f, size_t *outlenp,
87    size_t *consumed)
88 {
89 	size_t last_total_out = zb->z.total_out;
90 	size_t last_total_in = zb->z.total_in;
91 	z_stream *z = &zb->z;
92 	int ret = Z_ERRNO;
93 
94 	z->next_out = zb->outbuf;
95 	z->avail_out = zb->outlen;
96 
97 	*outlenp = 0;
98 	if (consumed)
99 		*consumed = 0;
100 	do {
101 		char *crc_in = NULL;
102 		size_t crc_avail = 0;
103 
104 		if (z->avail_in == 0) {
105 			size_t n = fread(zb->inbuf, 1, zb->inlen, f);
106 			if (n == 0) {
107 				if (ferror(f))
108 					return got_ferror(f, GOT_ERR_IO);
109 				/* EOF */
110 				ret = Z_STREAM_END;
111 				break;
112 			}
113 			z->next_in = zb->inbuf;
114 			z->avail_in = n;
115 		}
116 		if (zb->input_crc) {
117 			crc_in = z->next_in;
118 			crc_avail = z->avail_in;
119 		}
120 		ret = inflate(z, Z_SYNC_FLUSH);
121 		if (zb->input_crc) {
122 			*zb->input_crc = crc32(*zb->input_crc,
123 			    crc_in, crc_avail - z->avail_in);
124 		}
125 	} while (ret == Z_OK && z->avail_out > 0);
126 
127 	if (ret == Z_OK || ret == Z_BUF_ERROR) {
128 		zb->flags |= GOT_INFLATE_F_HAVE_MORE;
129 	} else {
130 		if (ret != Z_STREAM_END)
131 			return got_error(GOT_ERR_DECOMPRESSION);
132 		zb->flags &= ~GOT_INFLATE_F_HAVE_MORE;
133 	}
134 
135 	*outlenp = z->total_out - last_total_out;
136 	if (consumed)
137 		*consumed += z->total_in - last_total_in;
138 	return NULL;
139 }
140 
141 const struct got_error *
142 got_inflate_read_fd(struct got_inflate_buf *zb, int fd, size_t *outlenp,
143     size_t *consumed)
144 {
145 	size_t last_total_out = zb->z.total_out;
146 	size_t last_total_in = zb->z.total_in;
147 	z_stream *z = &zb->z;
148 	int ret = Z_ERRNO;
149 
150 	z->next_out = zb->outbuf;
151 	z->avail_out = zb->outlen;
152 
153 	*outlenp = 0;
154 	if (consumed)
155 		*consumed = 0;
156 	do {
157 		char *crc_in = NULL;
158 		size_t crc_avail = 0;
159 
160 		if (z->avail_in == 0) {
161 			ssize_t n = read(fd, zb->inbuf, zb->inlen);
162 			if (n < 0)
163 				return got_error_from_errno("read");
164 			else if (n == 0) {
165 				/* EOF */
166 				ret = Z_STREAM_END;
167 				break;
168 			}
169 			z->next_in = zb->inbuf;
170 			z->avail_in = n;
171 		}
172 		if (zb->input_crc) {
173 			crc_in = z->next_in;
174 			crc_avail = z->avail_in;
175 		}
176 		ret = inflate(z, Z_SYNC_FLUSH);
177 		if (zb->input_crc) {
178 			*zb->input_crc = crc32(*zb->input_crc,
179 			    crc_in, crc_avail - z->avail_in);
180 		}
181 	} while (ret == Z_OK && z->avail_out > 0);
182 
183 	if (ret == Z_OK || ret == Z_BUF_ERROR) {
184 		zb->flags |= GOT_INFLATE_F_HAVE_MORE;
185 	} else {
186 		if (ret != Z_STREAM_END)
187 			return got_error(GOT_ERR_DECOMPRESSION);
188 		zb->flags &= ~GOT_INFLATE_F_HAVE_MORE;
189 	}
190 
191 	*outlenp = z->total_out - last_total_out;
192 	if (consumed)
193 		*consumed += z->total_in - last_total_in;
194 	return NULL;
195 }
196 
197 const struct got_error *
198 got_inflate_read_mmap(struct got_inflate_buf *zb, uint8_t *map, size_t offset,
199     size_t len, size_t *outlenp, size_t *consumed)
200 {
201 	size_t last_total_out = zb->z.total_out;
202 	z_stream *z = &zb->z;
203 	int ret = Z_ERRNO;
204 
205 	z->next_out = zb->outbuf;
206 	z->avail_out = zb->outlen;
207 
208 	*outlenp = 0;
209 	*consumed = 0;
210 
211 	do {
212 		char *crc_in = NULL;
213 		size_t crc_avail = 0;
214 		size_t last_total_in = zb->z.total_in;
215 
216 		if (z->avail_in == 0) {
217 			if (len == 0) {
218 				/* EOF */
219 				ret = Z_STREAM_END;
220 				break;
221 			}
222 			z->next_in = map + offset + *consumed;
223 			z->avail_in = len - *consumed;
224 		}
225 		if (zb->input_crc) {
226 			crc_in = z->next_in;
227 			crc_avail = z->avail_in;
228 		}
229 		ret = inflate(z, Z_SYNC_FLUSH);
230 		if (zb->input_crc) {
231 			*zb->input_crc = crc32(*zb->input_crc,
232 			    crc_in, crc_avail - z->avail_in);
233 		}
234 		*consumed += z->total_in - last_total_in;
235 	} while (ret == Z_OK && z->avail_out > 0);
236 
237 	if (ret == Z_OK || ret == Z_BUF_ERROR) {
238 		zb->flags |= GOT_INFLATE_F_HAVE_MORE;
239 	} else {
240 		if (ret != Z_STREAM_END)
241 			return got_error(GOT_ERR_DECOMPRESSION);
242 		zb->flags &= ~GOT_INFLATE_F_HAVE_MORE;
243 	}
244 
245 	*outlenp = z->total_out - last_total_out;
246 	return NULL;
247 }
248 
249 void
250 got_inflate_end(struct got_inflate_buf *zb)
251 {
252 	free(zb->inbuf);
253 	if (zb->flags & GOT_INFLATE_F_OWN_OUTBUF)
254 		free(zb->outbuf);
255 	inflateEnd(&zb->z);
256 }
257 
258 const struct got_error *
259 got_inflate_to_mem(uint8_t **outbuf, size_t *outlen,
260     size_t *consumed_total, FILE *f)
261 {
262 	const struct got_error *err;
263 	size_t avail, consumed;
264 	struct got_inflate_buf zb;
265 	void *newbuf;
266 	int nbuf = 1;
267 
268 	if (outbuf) {
269 		*outbuf = malloc(GOT_INFLATE_BUFSIZE);
270 		if (*outbuf == NULL)
271 			return got_error_from_errno("malloc");
272 		err = got_inflate_init(&zb, *outbuf, GOT_INFLATE_BUFSIZE, NULL);
273 	} else
274 		err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, NULL);
275 	if (err)
276 		return err;
277 
278 	*outlen = 0;
279 	if (consumed_total)
280 		*consumed_total = 0;
281 
282 	do {
283 		err = got_inflate_read(&zb, f, &avail, &consumed);
284 		if (err)
285 			goto done;
286 		*outlen += avail;
287 		if (consumed_total)
288 			*consumed_total += consumed;
289 		if (zb.flags & GOT_INFLATE_F_HAVE_MORE) {
290 			zb.outlen = (nbuf * GOT_INFLATE_BUFSIZE) - *outlen;
291 			if (outbuf == NULL)
292 				continue;
293 			newbuf = reallocarray(*outbuf, ++nbuf,
294 			   GOT_INFLATE_BUFSIZE);
295 			if (newbuf == NULL) {
296 				err = got_error_from_errno("reallocarray");
297 				free(*outbuf);
298 				*outbuf = NULL;
299 				*outlen = 0;
300 				goto done;
301 			}
302 			*outbuf = newbuf;
303 			zb.outbuf = newbuf + *outlen;
304 		}
305 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
306 
307 done:
308 	got_inflate_end(&zb);
309 	return err;
310 }
311 
312 const struct got_error *
313 got_inflate_to_mem_fd(uint8_t **outbuf, size_t *outlen,
314     size_t *consumed_total, uint32_t *input_crc, int infd)
315 {
316 	const struct got_error *err;
317 	size_t avail, consumed;
318 	struct got_inflate_buf zb;
319 	void *newbuf;
320 	int nbuf = 1;
321 
322 	if (outbuf) {
323 		*outbuf = malloc(GOT_INFLATE_BUFSIZE);
324 		if (*outbuf == NULL)
325 			return got_error_from_errno("malloc");
326 		err = got_inflate_init(&zb, *outbuf, GOT_INFLATE_BUFSIZE,
327 		    input_crc);
328 	} else
329 		err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE,
330 		    input_crc);
331 	if (err)
332 		goto done;
333 
334 	*outlen = 0;
335 	if (consumed_total)
336 		*consumed_total = 0;
337 
338 	do {
339 		err = got_inflate_read_fd(&zb, infd, &avail, &consumed);
340 		if (err)
341 			goto done;
342 		*outlen += avail;
343 		if (consumed_total)
344 			*consumed_total += consumed;
345 		if (zb.flags & GOT_INFLATE_F_HAVE_MORE) {
346 			zb.outlen = (nbuf * GOT_INFLATE_BUFSIZE) - *outlen;
347 			if (outbuf == NULL)
348 				continue;
349 			newbuf = reallocarray(*outbuf, ++nbuf,
350 			    GOT_INFLATE_BUFSIZE);
351 			if (newbuf == NULL) {
352 				err = got_error_from_errno("reallocarray");
353 				free(*outbuf);
354 				*outbuf = NULL;
355 				*outlen = 0;
356 				goto done;
357 			}
358 			*outbuf = newbuf;
359 			zb.outbuf = newbuf + *outlen;
360 		}
361 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
362 
363 done:
364 	got_inflate_end(&zb);
365 	return err;
366 }
367 
368 const struct got_error *
369 got_inflate_to_mem_mmap(uint8_t **outbuf, size_t *outlen, uint8_t *map,
370     size_t offset, size_t len)
371 {
372 	const struct got_error *err;
373 	size_t avail, consumed;
374 	struct got_inflate_buf zb;
375 	void *newbuf;
376 	int nbuf = 1;
377 
378 	*outbuf = malloc(GOT_INFLATE_BUFSIZE);
379 	if (*outbuf == NULL)
380 		return got_error_from_errno("malloc");
381 	err = got_inflate_init(&zb, *outbuf, GOT_INFLATE_BUFSIZE, NULL);
382 	if (err) {
383 		free(*outbuf);
384 		*outbuf = NULL;
385 		return err;
386 	}
387 
388 	*outlen = 0;
389 
390 	do {
391 		err = got_inflate_read_mmap(&zb, map, offset, len, &avail,
392 		    &consumed);
393 		if (err)
394 			goto done;
395 		offset += consumed;
396 		len -= consumed;
397 		*outlen += avail;
398 		if (len == 0)
399 			break;
400 		if (zb.flags & GOT_INFLATE_F_HAVE_MORE) {
401 			newbuf = reallocarray(*outbuf, ++nbuf,
402 			    GOT_INFLATE_BUFSIZE);
403 			if (newbuf == NULL) {
404 				err = got_error_from_errno("reallocarray");
405 				free(*outbuf);
406 				*outbuf = NULL;
407 				*outlen = 0;
408 				goto done;
409 			}
410 			*outbuf = newbuf;
411 			zb.outbuf = newbuf + *outlen;
412 			zb.outlen = (nbuf * GOT_INFLATE_BUFSIZE) - *outlen;
413 		}
414 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
415 done:
416 	got_inflate_end(&zb);
417 	return err;
418 }
419 
420 const struct got_error *
421 got_inflate_to_fd(size_t *outlen, FILE *infile, int outfd)
422 {
423 	const struct got_error *err = NULL;
424 	size_t avail;
425 	struct got_inflate_buf zb;
426 
427 	err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, NULL);
428 	if (err)
429 		goto done;
430 
431 	*outlen = 0;
432 
433 	do {
434 		err = got_inflate_read(&zb, infile, &avail, NULL);
435 		if (err)
436 			goto done;
437 		if (avail > 0) {
438 			ssize_t n;
439 			n = write(outfd, zb.outbuf, avail);
440 			if (n != avail) {
441 				err = got_error_from_errno("write");
442 				goto done;
443 			}
444 			*outlen += avail;
445 		}
446 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
447 
448 done:
449 	if (err == NULL) {
450 		if (lseek(outfd, SEEK_SET, 0) == -1)
451 			err = got_error_from_errno("lseek");
452 	}
453 	got_inflate_end(&zb);
454 	return err;
455 }
456 
457 const struct got_error *
458 got_inflate_to_file(size_t *outlen, FILE *infile, FILE *outfile)
459 {
460 	const struct got_error *err;
461 	size_t avail;
462 	struct got_inflate_buf zb;
463 
464 	err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, NULL);
465 	if (err)
466 		goto done;
467 
468 	*outlen = 0;
469 
470 	do {
471 		err = got_inflate_read(&zb, infile, &avail, NULL);
472 		if (err)
473 			goto done;
474 		if (avail > 0) {
475 			size_t n;
476 			n = fwrite(zb.outbuf, avail, 1, outfile);
477 			if (n != 1) {
478 				err = got_ferror(outfile, GOT_ERR_IO);
479 				goto done;
480 			}
481 			*outlen += avail;
482 		}
483 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
484 
485 done:
486 	if (err == NULL)
487 		rewind(outfile);
488 	got_inflate_end(&zb);
489 	return err;
490 }
491 
492 const struct got_error *
493 got_inflate_to_file_fd(size_t *outlen, int infd, FILE *outfile)
494 {
495 	const struct got_error *err;
496 	size_t avail;
497 	struct got_inflate_buf zb;
498 
499 	err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, NULL);
500 	if (err)
501 		goto done;
502 
503 	*outlen = 0;
504 
505 	do {
506 		err = got_inflate_read_fd(&zb, infd, &avail, NULL);
507 		if (err)
508 			goto done;
509 		if (avail > 0) {
510 			size_t n;
511 			n = fwrite(zb.outbuf, avail, 1, outfile);
512 			if (n != 1) {
513 				err = got_ferror(outfile, GOT_ERR_IO);
514 				goto done;
515 			}
516 			*outlen += avail;
517 		}
518 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
519 
520 done:
521 	if (err == NULL)
522 		rewind(outfile);
523 	got_inflate_end(&zb);
524 	return err;
525 }
526 
527 const struct got_error *
528 got_inflate_to_file_mmap(size_t *outlen, uint8_t *map, size_t offset,
529     size_t len, FILE *outfile)
530 {
531 	const struct got_error *err;
532 	size_t avail;
533 	struct got_inflate_buf zb;
534 	size_t consumed;
535 
536 	err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE, NULL);
537 	if (err)
538 		goto done;
539 
540 	*outlen = 0;
541 
542 	do {
543 		err = got_inflate_read_mmap(&zb, map, offset, len, &avail,
544 		    &consumed);
545 		if (err)
546 			goto done;
547 		offset += consumed;
548 		len -= consumed;
549 		if (avail > 0) {
550 			size_t n;
551 			n = fwrite(zb.outbuf, avail, 1, outfile);
552 			if (n != 1) {
553 				err = got_ferror(outfile, GOT_ERR_IO);
554 				goto done;
555 			}
556 			*outlen += avail;
557 		}
558 	} while (zb.flags & GOT_INFLATE_F_HAVE_MORE);
559 
560 done:
561 	if (err == NULL)
562 		rewind(outfile);
563 	got_inflate_end(&zb);
564 	return err;
565 }