commit 55fdd2574a8da106021dd4857d3727185bda6c99
from: Stefan Sperling <stsp@stsp.name>
date: Wed Mar 18 16:13:43 2020 UTC

do not buffer more data than necessary in got_inflate_to_mem_fd()

commit - 55f98ccb9920b8e02a64b7dc937db99a85c7eb61
commit + 55fdd2574a8da106021dd4857d3727185bda6c99
blob - e97951c3fe40ff57563fcd8b71cac2022417eae9
blob + 1d8212ee1d7b2dff406fc259b64350befec9ebca
--- lib/got_lib_inflate.h
+++ lib/got_lib_inflate.h
@@ -40,7 +40,7 @@ void got_inflate_end(struct got_inflate_buf *);
 const struct got_error *got_inflate_to_mem(uint8_t **, size_t *, size_t *,
     FILE *);
 const struct got_error *got_inflate_to_mem_fd(uint8_t **, size_t *, size_t *,
-    uint32_t *, int);
+    uint32_t *, size_t, int);
 const struct got_error *got_inflate_to_mem_mmap(uint8_t **, size_t *, uint8_t *,
     size_t, size_t);
 const struct got_error *got_inflate_to_file(size_t *, FILE *, FILE *);
blob - adb65f7160aeb80a3a64d7178286e66c5c9815e5
blob + e1e7cd506205b1d5951f680a97acc6c747d2fa59
--- lib/inflate.c
+++ lib/inflate.c
@@ -311,23 +311,27 @@ done:
 
 const struct got_error *
 got_inflate_to_mem_fd(uint8_t **outbuf, size_t *outlen,
-    size_t *consumed_total, uint32_t *input_crc, int infd)
+    size_t *consumed_total, uint32_t *input_crc, size_t expected_size, int infd)
 {
 	const struct got_error *err;
 	size_t avail, consumed;
 	struct got_inflate_buf zb;
 	void *newbuf;
 	int nbuf = 1;
+	size_t bufsize = GOT_INFLATE_BUFSIZE;
 
+	/* Optimize buffer size in case short reads should suffice. */
+	if (expected_size > 0 && expected_size < bufsize)
+		bufsize = expected_size;
+		
 	if (outbuf) {
-		*outbuf = malloc(GOT_INFLATE_BUFSIZE);
+		*outbuf = malloc(bufsize);
 		if (*outbuf == NULL)
 			return got_error_from_errno("malloc");
 		err = got_inflate_init(&zb, *outbuf, GOT_INFLATE_BUFSIZE,
 		    input_crc);
 	} else
-		err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE,
-		    input_crc);
+		err = got_inflate_init(&zb, NULL, bufsize, input_crc);
 	if (err)
 		goto done;
 
blob - 28dd5296a74ec449099ea0735ace1978b81b2567
blob + e5817bcdb4d07c0921e5917a0ca6502ae3d96c55
--- lib/pack.c
+++ lib/pack.c
@@ -720,7 +720,7 @@ read_delta_data(uint8_t **delta_buf, size_t *delta_len
 		if (lseek(pack->fd, delta_data_offset, SEEK_SET) == -1)
 			return got_error_from_errno("lseek");
 		err = got_inflate_to_mem_fd(delta_buf, delta_len, NULL,
-		    NULL, pack->fd);
+		    NULL, 0, pack->fd);
 	}
 	return err;
 }
@@ -1101,7 +1101,8 @@ dump_delta_chain_to_file(size_t *result_size, struct g
 					    pack->filesize - mapoff);
 				} else
 					err = got_inflate_to_mem_fd(&base_buf,
-					    &base_bufsz, NULL, NULL, pack->fd);
+					    &base_bufsz, NULL, NULL, max_size,
+					    pack->fd);
 			}
 			if (err)
 				goto done;
@@ -1248,7 +1249,8 @@ got_pack_dump_delta_chain_to_mem(uint8_t **outbuf, siz
 					goto done;
 				}
 				err = got_inflate_to_mem_fd(&base_buf,
-				    &base_bufsz, NULL, NULL, pack->fd);
+				    &base_bufsz, NULL, NULL, max_size,
+				    pack->fd);
 			}
 			if (err)
 				goto done;
@@ -1369,7 +1371,7 @@ got_packfile_extract_object_to_mem(uint8_t **buf, size
 			if (lseek(pack->fd, obj->pack_offset, SEEK_SET) == -1)
 				return got_error_from_errno("lseek");
 			err = got_inflate_to_mem_fd(buf, len, NULL, NULL,
-			    pack->fd);
+			    obj->size, pack->fd);
 		}
 	} else
 		err = got_pack_dump_delta_chain_to_mem(buf, len, &obj->deltas,
blob - 45930556cd110e5ed911da0e3c9cce993446a6d4
blob + fc35ff4dc3aa95542ad3f9141e8fdf2970b229e6
--- libexec/got-index-pack/got-index-pack.c
+++ libexec/got-index-pack/got-index-pack.c
@@ -180,7 +180,7 @@ read_packed_object(struct got_pack *pack, struct got_i
 	case GOT_OBJ_TYPE_TAG:
 		/* XXX TODO reading large objects into memory is bad! */
 		err = got_inflate_to_mem_fd(&data, &datalen, &obj->len,
-		    &obj->crc, pack->fd);
+		    &obj->crc, obj->size, pack->fd);
 		if (err)
 			break;
 		SHA1Init(&ctx);
@@ -213,7 +213,7 @@ read_packed_object(struct got_pack *pack, struct got_i
 		obj->crc = crc32(obj->crc, obj->ref_id.sha1,
 		    SHA1_DIGEST_LENGTH);
 		err = got_inflate_to_mem_fd(NULL, &datalen, &obj->len,
-		    &obj->crc, pack->fd);
+		    &obj->crc, obj->size, pack->fd);
 		if (err)
 			break;
 		obj->len += SHA1_DIGEST_LENGTH;
@@ -235,7 +235,7 @@ read_packed_object(struct got_pack *pack, struct got_i
 			break;
 
 		err = got_inflate_to_mem_fd(NULL, &datalen, &obj->len,
-		    &obj->crc, pack->fd);
+		    &obj->crc, obj->size, pack->fd);
 		if (err)
 			break;
 		obj->len += obj->base_offsetlen;