commit 224848656b45718ffd238961a636ef83dd006e63
from: Stefan Sperling <stsp@stsp.name>
date: Tue Mar 13 17:28:40 2018 UTC

process delta chains in memory if max size is < 32 MB

commit - 39ff877fbb5335dfcf380e186a76b54074407112
commit + 224848656b45718ffd238961a636ef83dd006e63
blob - f8ab225fffcd98374fba89b150774262431fbb13
blob + 8b8243244f2258d9a028e56b322c67023310b69e
--- lib/delta.c
+++ lib/delta.c
@@ -225,7 +225,42 @@ copy_from_delta(const uint8_t **p, size_t *remain, siz
 	return NULL;
 }
 
+static const struct got_error *
+parse_delta_sizes(uint64_t *base_size, uint64_t *result_size,
+    const uint8_t **p, size_t *remain)
+{
+	const struct got_error *err;
+
+	/* Read the two size fields at the beginning of the stream. */
+	err = parse_size(base_size, p, remain);
+	if (err)
+		return err;
+	err = next_delta_byte(p, remain);
+	if (err)
+		return err;
+	err = parse_size(result_size, p, remain);
+	if (err)
+		return err;
+
+	return NULL;
+}
+
 const struct got_error *
+got_delta_get_sizes(uint64_t *base_size, uint64_t *result_size,
+    const uint8_t *delta_buf, size_t delta_len)
+{
+	size_t remain;
+	const uint8_t *p;
+
+	if (delta_len < GOT_DELTA_STREAM_LENGTH_MIN)
+		return got_error(GOT_ERR_BAD_DELTA);
+
+	p = delta_buf;
+	remain = delta_len;
+	return parse_delta_sizes(base_size, result_size, &p, &remain);
+}
+
+const struct got_error *
 got_delta_apply(FILE *base_file, const uint8_t *delta_buf,
     size_t delta_len, FILE *outfile)
 {
@@ -242,17 +277,9 @@ got_delta_apply(FILE *base_file, const uint8_t *delta_
 
 	p = delta_buf;
 	remain = delta_len;
-
-	/* Read the two size fields at the beginning of the stream. */
-	err = parse_size(&base_size, &p, &remain);
+	err = parse_delta_sizes(&base_size, &result_size, &p, &remain);
 	if (err)
 		return err;
-	err = next_delta_byte(&p, &remain);
-	if (err)
-		return err;
-	err = parse_size(&result_size, &p, &remain);
-	if (err)
-		return err;
 
 	if (result_size < GOT_DELTA_RESULT_SIZE_CACHED_MAX)
 		memstream = open_memstream(&memstream_buf, &memstream_size);
blob - c0738c0ca2196c671aa755529b9011445ce366e5
blob + c92a6d049d7ede2f488c9b093835e27fd6221c5d
--- lib/got_delta_lib.h
+++ lib/got_delta_lib.h
@@ -34,6 +34,8 @@ struct got_delta *got_delta_open(const char *, off_t, 
 void got_delta_close(struct got_delta *);
 const struct got_error *got_delta_chain_get_base_type(int *,
     struct got_delta_chain *);
+const struct got_error *got_delta_get_sizes(uint64_t *, uint64_t *,
+    const uint8_t *, size_t);
 const struct got_error *got_delta_apply(FILE *, const uint8_t *, size_t,
     FILE *);
 
blob - 1c07f6346b919626ec160b7bcc0acfa4d83b5e21
blob + 52d060271b4c99be982509935b482e278279e203
--- lib/pack.c
+++ lib/pack.c
@@ -920,21 +920,89 @@ got_packfile_open_object(struct got_object **obj, stru
 }
 
 static const struct got_error *
+get_delta_sizes(uint64_t *base_size, uint64_t *result_size,
+    struct got_delta *delta)
+{
+	const struct got_error *err;
+	uint8_t *delta_buf = NULL;
+	size_t delta_len = 0;
+	FILE *delta_file;
+
+	delta_file = fopen(delta->path_packfile, "rb");
+	if (delta_file == NULL)
+		return got_error_from_errno();
+
+	if (fseeko(delta_file, delta->data_offset, SEEK_CUR) != 0) {
+		err = got_error_from_errno();
+		fclose(delta_file);
+		return err;
+	}
+
+	err = got_inflate_to_mem(&delta_buf, &delta_len, delta_file);
+	fclose(delta_file);
+	if (err)
+		return err;
+
+	err = got_delta_get_sizes(base_size, result_size, delta_buf, delta_len);
+	free(delta_buf);
+	return err;
+}
+
+static const struct got_error *
+get_delta_chain_max_size(uint64_t *max_size, struct got_delta_chain *deltas)
+{
+	struct got_delta *delta;
+	uint64_t base_size = 0, result_size = 0;
+
+	*max_size = 0;
+	SIMPLEQ_FOREACH(delta, &deltas->entries, entry) {
+		/* Plain object types are the delta base. */
+		if (delta->type != GOT_OBJ_TYPE_COMMIT &&
+		    delta->type != GOT_OBJ_TYPE_TREE &&
+		    delta->type != GOT_OBJ_TYPE_BLOB &&
+		    delta->type != GOT_OBJ_TYPE_TAG) {
+			const struct got_error *err;
+			err = get_delta_sizes(&base_size, &result_size, delta);
+			if (err)
+				return err;
+		} else
+			base_size = delta->size;
+		if (base_size > *max_size)
+			*max_size = base_size;
+		if (result_size > *max_size)
+			*max_size = result_size;
+	}
+
+	return NULL;
+}
+
+static const struct got_error *
 dump_delta_chain(struct got_delta_chain *deltas, FILE *outfile)
 {
 	const struct got_error *err = NULL;
 	struct got_delta *delta;
-	FILE *base_file, *accum_file;
+	FILE *base_file = NULL, *accum_file = NULL;
+	uint64_t max_size;
 	int n = 0;
 
 	if (SIMPLEQ_EMPTY(&deltas->entries))
 		return got_error(GOT_ERR_BAD_DELTA_CHAIN);
 
-	base_file = got_opentemp();
+	err = get_delta_chain_max_size(&max_size, deltas);
+	if (err)
+		return err;
+
+	if (max_size < GOT_DELTA_RESULT_SIZE_CACHED_MAX)
+		base_file = fmemopen(NULL, max_size, "w+");
+	else
+		base_file = got_opentemp();
 	if (base_file == NULL)
 		return got_error_from_errno();
 
-	accum_file = got_opentemp();
+	if (max_size < GOT_DELTA_RESULT_SIZE_CACHED_MAX)
+		accum_file = fmemopen(NULL, max_size, "w+");
+	else
+		accum_file = got_opentemp();
 	if (accum_file == NULL) {
 		err = got_error_from_errno();
 		fclose(base_file);
@@ -953,7 +1021,6 @@ dump_delta_chain(struct got_delta_chain *deltas, FILE 
 			goto done;
 		}
 
-
 		if (n == 0) {
 			/* Plain object types are the delta base. */
 			if (delta->type != GOT_OBJ_TYPE_COMMIT &&