Commit Diff

Commit:
42c69117cce2e1658d5b5aabbc383ce7252cf167
From:
Stefan Sperling <stsp@stsp.name>
Date:
Message:
stop storing decompressed delta data in struct got_delta; fetch it on demand
Actions:
Patch | Tree
commit - a74f7e836f1414f6c06fca610a6231e37c1e68fd
commit + 42c69117cce2e1658d5b5aabbc383ce7252cf167
blob - 42f628d13edc1ddbaa6844b6156d41a9438d9e4b
blob + 9abb0d4dc56c7c09ebf32461a9eb717f79cbca05
--- lib/delta.c
+++ lib/delta.c
@@ -40,7 +40,7 @@
 
 struct got_delta *
 got_delta_open(off_t offset, size_t tslen, int type, size_t size,
-    off_t data_offset, uint8_t *delta_buf, size_t delta_len)
+    off_t data_offset)
 {
 	struct got_delta *delta;
 
@@ -53,8 +53,6 @@ got_delta_open(off_t offset, size_t tslen, int type, s
 	delta->tslen = tslen;
 	delta->size = size;
 	delta->data_offset = data_offset;
-	delta->delta_buf = delta_buf;
-	delta->delta_len = delta_len;
 	return delta;
 }
 
blob - 8b79413858bc4609d31a7e5b7910f8419f709365
blob + b04e243f7e28e091a99469856203d981d23691d5
--- lib/got_lib_delta.h
+++ lib/got_lib_delta.h
@@ -21,8 +21,6 @@ struct got_delta {
 	int type;
 	size_t size;
 	off_t data_offset;
-	uint8_t *delta_buf;
-	size_t delta_len;
 };
 
 struct got_delta_chain {
@@ -32,8 +30,7 @@ struct got_delta_chain {
 
 #define GOT_DELTA_CHAIN_RECURSION_MAX	500
 
-struct got_delta *got_delta_open(off_t, size_t, int, size_t, off_t,
-    uint8_t *, size_t);
+struct got_delta *got_delta_open(off_t, size_t, int, size_t, off_t);
 const struct got_error *got_delta_chain_get_base_type(int *,
     struct got_delta_chain *);
 const struct got_error *got_delta_get_sizes(uint64_t *, uint64_t *,
blob - 654dfa86d4de464fcdc3b68b5b8469f50f476a4c
blob + 5ccd979e0bc59829534dc075f1e50f9ef8d069d0
--- lib/got_lib_pack.h
+++ lib/got_lib_pack.h
@@ -168,7 +168,7 @@ const struct got_error *got_packidx_match_id_str_prefi
 const struct got_error *got_packfile_open_object(struct got_object **,
     struct got_pack *, struct got_packidx *, int, struct got_object_id *);
 const struct got_error *got_pack_get_max_delta_object_size(uint64_t *,
-    struct got_object *);
+    struct got_object *, struct got_pack *);
 const struct got_error *got_packfile_extract_object(struct got_pack *,
     struct got_object *, FILE *, FILE *, FILE *);
 const struct got_error *got_packfile_extract_object_to_mem(uint8_t **, size_t *,
blob - 6d876216ae17f82d4f9dee9e675e1577abd93a08
blob + c67ff751c1fbd694ef012e6ec8f54283f24acdf3
--- lib/object_cache.c
+++ lib/object_cache.c
@@ -84,9 +84,9 @@ get_size_obj(struct got_object *obj)
 		return size;
 
 	SIMPLEQ_FOREACH(delta, &obj->deltas.entries, entry) {
-		if (SIZE_MAX - (sizeof(*delta) + delta->delta_len) < size)
+		if (SIZE_MAX - sizeof(*delta) < size)
 			return SIZE_MAX;
-		size += sizeof(*delta) + delta->delta_len;
+		size += sizeof(*delta);
 	}
 
 	return size;
blob - a037ceb7fcaca7f9358cecf93fb644ff7f35ccc9
blob + 9a37c6f1d7c6414382484dbde632c0fb6bb7591e
--- lib/object_parse.c
+++ lib/object_parse.c
@@ -113,7 +113,6 @@ got_object_close(struct got_object *obj)
 		while (!SIMPLEQ_EMPTY(&obj->deltas.entries)) {
 			delta = SIMPLEQ_FIRST(&obj->deltas.entries);
 			SIMPLEQ_REMOVE_HEAD(&obj->deltas.entries, entry);
-			free(delta->delta_buf);
 			free(delta);
 		}
 	}
blob - fb5f114d8a5ce54615d80a1f1a73b8ab0dc65a9a
blob + 433976a663e5eb022cd09dd6498cb25a651150e3
--- lib/pack.c
+++ lib/pack.c
@@ -705,17 +705,34 @@ parse_offset_delta(off_t *base_offset, size_t *len, st
 static const struct got_error *
 resolve_delta_chain(struct got_delta_chain *, struct got_packidx *,
     struct got_pack *, off_t, size_t, int, size_t, unsigned int);
+
+static const struct got_error *
+read_delta_data(uint8_t **delta_buf, size_t *delta_len,
+    size_t delta_data_offset, struct got_pack *pack)
+{
+	const struct got_error *err = NULL;
+
+	if (pack->map) {
+		if (delta_data_offset >= pack->filesize)
+			return got_error(GOT_ERR_PACK_OFFSET);
+		err = got_inflate_to_mem_mmap(delta_buf, delta_len, pack->map,
+		    delta_data_offset, pack->filesize - delta_data_offset);
+	} else {
+		if (lseek(pack->fd, delta_data_offset, SEEK_SET) == -1)
+			return got_error_from_errno("lseek");
+		err = got_inflate_to_mem_fd(delta_buf, delta_len, pack->fd);
+	}
+	return err;
+}
 
 static const struct got_error *
 add_delta(struct got_delta_chain *deltas, off_t delta_offset, size_t tslen,
-    int delta_type, size_t delta_size, size_t delta_data_offset,
-    uint8_t *delta_buf, size_t delta_len)
+    int delta_type, size_t delta_size, size_t delta_data_offset)
 {
 	struct got_delta *delta;
 
 	delta = got_delta_open(delta_offset, tslen, delta_type, delta_size,
-	    delta_data_offset, delta_buf,
-	    delta_len);
+	    delta_data_offset);
 	if (delta == NULL)
 		return got_error_from_errno("got_delta_open");
 	/* delta is freed in got_object_close() */
@@ -736,8 +753,7 @@ resolve_offset_delta(struct got_delta_chain *deltas,
 	uint64_t base_size;
 	size_t base_tslen;
 	off_t delta_data_offset;
-	uint8_t *delta_buf;
-	size_t delta_len, consumed;
+	size_t consumed;
 
 	err = parse_offset_delta(&base_offset, &consumed, pack,
 	    delta_offset, tslen);
@@ -754,21 +770,8 @@ resolve_offset_delta(struct got_delta_chain *deltas,
 			return got_error_from_errno("lseek");
 	}
 
-	if (pack->map) {
-		size_t mapoff = (size_t)delta_data_offset;
-		err = got_inflate_to_mem_mmap(&delta_buf, &delta_len, pack->map,
-		    mapoff, pack->filesize - mapoff);
-		if (err)
-			return err;
-	} else {
-
-		err = got_inflate_to_mem_fd(&delta_buf, &delta_len, pack->fd);
-		if (err)
-			return err;
-	}
-
 	err = add_delta(deltas, delta_offset, tslen, delta_type, delta_size,
-	    delta_data_offset, delta_buf, delta_len);
+	    delta_data_offset);
 	if (err)
 		return err;
 
@@ -801,28 +804,27 @@ resolve_ref_delta(struct got_delta_chain *deltas, stru
 	uint8_t *delta_buf;
 	size_t delta_len;
 
-	if (delta_offset >= pack->filesize)
+	if (delta_offset + tslen >= pack->filesize)
 		return got_error(GOT_ERR_PACK_OFFSET);
-	delta_data_offset = delta_offset + tslen;
-	if (delta_data_offset >= pack->filesize)
-		return got_error(GOT_ERR_PACK_OFFSET);
 
 	if (pack->map == NULL) {
-		delta_data_offset = lseek(pack->fd, 0, SEEK_CUR);
-		if (delta_data_offset == -1)
-			return got_error_from_errno("lseek");
 	}
 
 	if (pack->map) {
-		size_t mapoff = (size_t)delta_data_offset;
+		size_t mapoff = delta_offset + tslen;
 		memcpy(&id, pack->map + mapoff, sizeof(id));
 		mapoff += sizeof(id);
+		delta_data_offset = (off_t)mapoff;
 		err = got_inflate_to_mem_mmap(&delta_buf, &delta_len, pack->map,
 		    mapoff, pack->filesize - mapoff);
 		if (err)
 			return err;
 	} else {
-		ssize_t n = read(pack->fd, &id, sizeof(id));
+		ssize_t n;
+		delta_data_offset = lseek(pack->fd, 0, SEEK_CUR);
+		if (delta_data_offset == -1)
+			return got_error_from_errno("lseek");
+		n = read(pack->fd, &id, sizeof(id));
 		if (n < 0)
 			return got_error_from_errno("read");
 		if (n != sizeof(id))
@@ -833,7 +835,7 @@ resolve_ref_delta(struct got_delta_chain *deltas, stru
 	}
 
 	err = add_delta(deltas, delta_offset, tslen, delta_type, delta_size,
-	    delta_data_offset, delta_buf, delta_len);
+	    delta_data_offset);
 	if (err)
 		return err;
 
@@ -875,7 +877,7 @@ resolve_delta_chain(struct got_delta_chain *deltas, st
 	case GOT_OBJ_TYPE_TAG:
 		/* Plain types are the final delta base. Recursion ends. */
 		err = add_delta(deltas, delta_offset, tslen, delta_type,
-		    delta_size, 0, NULL, 0);
+		    delta_size, 0);
 		break;
 	case GOT_OBJ_TYPE_OFFSET_DELTA:
 		err = resolve_offset_delta(deltas, packidx, pack,
@@ -980,7 +982,8 @@ got_packfile_open_object(struct got_object **obj, stru
 }
 
 static const struct got_error *
-get_delta_chain_max_size(uint64_t *max_size, struct got_delta_chain *deltas)
+get_delta_chain_max_size(uint64_t *max_size, struct got_delta_chain *deltas,
+    struct got_pack *pack)
 {
 	struct got_delta *delta;
 	uint64_t base_size = 0, result_size = 0;
@@ -993,8 +996,16 @@ get_delta_chain_max_size(uint64_t *max_size, struct go
 		    delta->type != GOT_OBJ_TYPE_BLOB &&
 		    delta->type != GOT_OBJ_TYPE_TAG) {
 			const struct got_error *err;
+			uint8_t *delta_buf;
+			size_t delta_len;
+
+			err = read_delta_data(&delta_buf, &delta_len,
+			    delta->data_offset, pack);
+			if (err)
+				return err;
 			err = got_delta_get_sizes(&base_size, &result_size,
-			    delta->delta_buf, delta->delta_len);
+			    delta_buf, delta_len);
+			free(delta_buf);
 			if (err)
 				return err;
 		} else
@@ -1009,12 +1020,13 @@ get_delta_chain_max_size(uint64_t *max_size, struct go
 }
 
 const struct got_error *
-got_pack_get_max_delta_object_size(uint64_t *size, struct got_object *obj)
+got_pack_get_max_delta_object_size(uint64_t *size, struct got_object *obj,
+    struct got_pack *pack)
 {
 	if ((obj->flags & GOT_OBJ_FLAG_DELTIFIED) == 0)
 		return got_error(GOT_ERR_OBJ_TYPE);
 
-	return get_delta_chain_max_size(size, &obj->deltas);
+	return get_delta_chain_max_size(size, &obj->deltas, pack);
 }
 
 static const struct got_error *
@@ -1023,8 +1035,8 @@ dump_delta_chain_to_file(size_t *result_size, struct g
 {
 	const struct got_error *err = NULL;
 	struct got_delta *delta;
-	uint8_t *base_buf = NULL, *accum_buf = NULL;
-	size_t base_bufsz = 0, accum_size = 0;
+	uint8_t *base_buf = NULL, *accum_buf = NULL, *delta_buf;
+	size_t base_bufsz = 0, accum_size = 0, delta_len;
 	uint64_t max_size;
 	int n = 0;
 
@@ -1034,7 +1046,7 @@ dump_delta_chain_to_file(size_t *result_size, struct g
 		return got_error(GOT_ERR_BAD_DELTA_CHAIN);
 
 	/* We process small enough files entirely in memory for speed. */
-	err = get_delta_chain_max_size(&max_size, deltas);
+	err = get_delta_chain_max_size(&max_size, deltas, pack);
 	if (err)
 		return err;
 	if (max_size < GOT_DELTA_RESULT_SIZE_CACHED_MAX) {
@@ -1099,18 +1111,23 @@ dump_delta_chain_to_file(size_t *result_size, struct g
 			continue;
 		}
 
+		err = read_delta_data(&delta_buf, &delta_len,
+		    delta->data_offset, pack);
+		if (err)
+			goto done;
 		if (base_buf) {
 			err = got_delta_apply_in_mem(base_buf, base_bufsz,
-			    delta->delta_buf, delta->delta_len, accum_buf,
+			    delta_buf, delta_len, accum_buf,
 			    &accum_size, max_size);
 			n++;
 		} else {
-			err = got_delta_apply(base_file, delta->delta_buf,
-			    delta->delta_len,
+			err = got_delta_apply(base_file, delta_buf,
+			    delta_len,
 			    /* Final delta application writes to output file. */
 			    ++n < deltas->nentries ? accum_file : outfile,
 			    &accum_size);
 		}
+		free(delta_buf);
 		if (err)
 			goto done;
 
@@ -1167,8 +1184,8 @@ dump_delta_chain_to_mem(uint8_t **outbuf, size_t *outl
 {
 	const struct got_error *err = NULL;
 	struct got_delta *delta;
-	uint8_t *base_buf = NULL, *accum_buf = NULL;
-	size_t base_bufsz = 0, accum_size = 0;
+	uint8_t *base_buf = NULL, *accum_buf = NULL, *delta_buf;
+	size_t base_bufsz = 0, accum_size = 0, delta_len;
 	uint64_t max_size;
 	int n = 0;
 
@@ -1178,7 +1195,7 @@ dump_delta_chain_to_mem(uint8_t **outbuf, size_t *outl
 	if (SIMPLEQ_EMPTY(&deltas->entries))
 		return got_error(GOT_ERR_BAD_DELTA_CHAIN);
 
-	err = get_delta_chain_max_size(&max_size, deltas);
+	err = get_delta_chain_max_size(&max_size, deltas, pack);
 	if (err)
 		return err;
 	accum_buf = malloc(max_size);
@@ -1224,9 +1241,14 @@ dump_delta_chain_to_mem(uint8_t **outbuf, size_t *outl
 			continue;
 		}
 
+		err = read_delta_data(&delta_buf, &delta_len,
+		    delta->data_offset, pack);
+		if (err)
+			goto done;
 		err = got_delta_apply_in_mem(base_buf, base_bufsz,
-		    delta->delta_buf, delta->delta_len, accum_buf,
+		    delta_buf, delta_len, accum_buf,
 		    &accum_size, max_size);
+		free(delta_buf);
 		n++;
 		if (err)
 			goto done;
blob - faa1ca0fe5d71391f0f5fe9cb3b683725741ca18
blob + cb7464ca665181ed509062b13f114a4acc24e5e1
--- libexec/got-read-pack/got-read-pack.c
+++ libexec/got-read-pack/got-read-pack.c
@@ -289,7 +289,7 @@ blob_request(struct imsg *imsg, struct imsgbuf *ibuf, 
 		goto done;
 
 	if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) {
-		err = got_pack_get_max_delta_object_size(&blob_size, obj);
+		err = got_pack_get_max_delta_object_size(&blob_size, obj, pack);
 		if (err)
 			goto done;
 	} else