op public repos

Commit Diff

Commit:: a53d2f137940289a6db04a438b9b945a65ab8524
From:: Stefan Sperling <stsp@stsp.name>
Date:: Sat Mar 17 13:19:13 2018 UTC
Message:: cache delta data in struct got_delta directly
Actions:: Patch | Tree
commit - 6d89869a98049341af4b0a79db823d11ac59d444
commit + a53d2f137940289a6db04a438b9b945a65ab8524
blob - f28e5f73c740eaf73d8a41a888a8f0de42c8d69e
blob + a1de367be2f98b15140f7347f7a435580803b917
--- lib/delta.c
+++ lib/delta.c
@@ -36,7 +36,8 @@
 
 struct got_delta *
 got_delta_open(const char *path_packfile, off_t offset, size_t tslen,
-    int type, size_t size, off_t data_offset)
+    int type, size_t size, off_t data_offset, uint8_t *delta_buf,
+    size_t delta_len)
 {
 	struct got_delta *delta;
 
@@ -49,12 +50,15 @@ got_delta_open(const char *path_packfile, off_t offset
 	delta->tslen = tslen;
 	delta->size = size;
 	delta->data_offset = data_offset;
+	delta->delta_buf = delta_buf;
+	delta->delta_len = delta_len;
 	return delta;
 }
 
 void
 got_delta_close(struct got_delta *delta)
 {
+	free(delta->delta_buf);
 	free(delta);
 }
 
blob - e1098e1de940ff52efdb5088309e86ac591b44d7
blob + bc986fd9f13fbac49dfd74e76e0039dea07c5f74
--- lib/got_delta_lib.h
+++ lib/got_delta_lib.h
@@ -21,6 +21,8 @@ struct got_delta {
 	int type;
 	size_t size;
 	off_t data_offset;
+	uint8_t *delta_buf;
+	size_t delta_len;
 };
 
 struct got_delta_chain {
@@ -29,7 +31,7 @@ struct got_delta_chain {
 };
 
 struct got_delta *got_delta_open(const char *, off_t, size_t, int, size_t,
-    off_t);
+    off_t, uint8_t *, size_t);
 void got_delta_close(struct got_delta *);
 const struct got_error *got_delta_chain_get_base_type(int *,
     struct got_delta_chain *);
blob - 0cea059d4070e73bec61397b7463e60ca1cf8a40
blob + 0e20e105346024022013f73bc6715c61bd7deb98
--- lib/got_pack_lib.h
+++ lib/got_pack_lib.h
@@ -14,19 +14,6 @@
  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  */
 
-struct got_delta_cache_entry {
-	off_t data_offset;
-	uint8_t *delta_buf;
-	size_t delta_len;
-};
-
-#define GOT_DELTA_CACHE_SIZE	1024
-
-struct got_delta_cache {
-	int nentries;
-	struct got_delta_cache_entry deltas[GOT_DELTA_CACHE_SIZE];
-};
-
 /* A pack file segment mapped with mmap(2). */
 struct got_pack_mapping {
 	TAILQ_ENTRY(got_pack_mapping) entry;
@@ -48,8 +35,6 @@ struct got_pack {
 	int nmappings;
 
 	TAILQ_HEAD(, got_pack_mapping) mappings;
-
-	struct got_delta_cache delta_cache;
 };
 
 const struct got_error *got_pack_close(struct got_pack *);
blob - 2690ec66c0c1716c8dfee8d7340003dbb2f39c98
blob + 74ddaf8441706156d1194062a44e04ab9e7da89b
--- lib/pack.c
+++ lib/pack.c
@@ -580,7 +580,6 @@ const struct got_error *
 got_pack_close(struct got_pack *pack)
 {
 	const struct got_error *err = NULL;
-	int i;
 
 	while (!TAILQ_EMPTY(&pack->mappings)) {
 		struct got_pack_mapping *map = TAILQ_FIRST(&pack->mappings);
@@ -598,16 +597,6 @@ got_pack_close(struct got_pack *pack)
 		pack->path_packfile = NULL;
 		pack->filesize = 0;
 	}
-
-	for (i = 0; i < pack->delta_cache.nentries; i++) {
-		struct got_delta_cache_entry *entry;
-		entry = &pack->delta_cache.deltas[i];
-		entry->data_offset = 0;
-		free(entry->delta_buf);
-		entry->delta_buf = NULL;
-		entry->delta_len = 0;
-	}
-	pack->delta_cache.nentries = 0;
 
 	return err;
 }
@@ -799,12 +788,13 @@ resolve_delta_chain(struct got_delta_chain *, struct g
 static const struct got_error *
 add_delta(struct got_delta_chain *deltas, const char *path_packfile,
     off_t delta_offset, size_t tslen, int delta_type, size_t delta_size,
-    size_t delta_data_offset)
+    size_t delta_data_offset, uint8_t *delta_buf, size_t delta_len)
 {
 	struct got_delta *delta;
 
 	delta = got_delta_open(path_packfile, delta_offset, tslen,
-	    delta_type, delta_size, delta_data_offset);
+	    delta_type, delta_size, delta_data_offset, delta_buf,
+	    delta_len);
 	if (delta == NULL)
 		return got_error(GOT_ERR_NO_MEM);
 	/* delta is freed in got_object_close() */
@@ -825,6 +815,8 @@ resolve_offset_delta(struct got_delta_chain *deltas,
 	uint64_t base_size;
 	size_t base_tslen;
 	off_t delta_data_offset;
+	uint8_t *delta_buf;
+	size_t delta_len;
 
 	err = parse_offset_delta(&base_offset, packfile, delta_offset);
 	if (err)
@@ -833,9 +825,13 @@ resolve_offset_delta(struct got_delta_chain *deltas,
 	delta_data_offset = ftello(packfile);
 	if (delta_data_offset == -1)
 		return got_error_from_errno();
+
+	err = got_inflate_to_mem(&delta_buf, &delta_len, packfile);
+	if (err)
+		return err;
 
 	err = add_delta(deltas, path_packfile, delta_offset, tslen,
-	    delta_type, delta_size, delta_data_offset);
+	    delta_type, delta_size, delta_data_offset, delta_buf, delta_len);
 	if (err)
 		return err;
 
@@ -869,6 +865,8 @@ resolve_ref_delta(struct got_delta_chain *deltas, stru
 	char *path_base_packfile;
 	struct got_pack *base_pack;
 	off_t delta_data_offset;
+	uint8_t *delta_buf;
+	size_t delta_len;
 
 	n = fread(&id, sizeof(id), 1, packfile);
 	if (n != 1)
@@ -878,8 +876,12 @@ resolve_ref_delta(struct got_delta_chain *deltas, stru
 	if (delta_data_offset == -1)
 		return got_error_from_errno();
 
+	err = got_inflate_to_mem(&delta_buf, &delta_len, packfile);
+	if (err)
+		return err;
+
 	err = add_delta(deltas, path_packfile, delta_offset, tslen,
-	    delta_type, delta_size, delta_data_offset);
+	    delta_type, delta_size, delta_data_offset, delta_buf, delta_len);
 	if (err)
 		return err;
 
@@ -936,7 +938,7 @@ resolve_delta_chain(struct got_delta_chain *deltas, st
 	case GOT_OBJ_TYPE_TAG:
 		/* Plain types are the final delta base. Recursion ends. */
 		err = add_delta(deltas, path_packfile, delta_offset, tslen,
-		    delta_type, delta_size, 0);
+		    delta_type, delta_size, 0, NULL, 0);
 		break;
 	case GOT_OBJ_TYPE_OFFSET_DELTA:
 		err = resolve_offset_delta(deltas, repo, packfile,
@@ -1086,28 +1088,6 @@ got_packfile_open_object(struct got_object **obj, stru
 }
 
 static const struct got_error *
-get_delta_sizes(uint64_t *base_size, uint64_t *result_size,
-    struct got_delta *delta, FILE *packfile)
-{
-	const struct got_error *err;
-	uint8_t *delta_buf = NULL;
-	size_t delta_len = 0;
-
-	if (fseeko(packfile, delta->data_offset, SEEK_SET) != 0) {
-		err = got_error_from_errno();
-		return err;
-	}
-
-	err = got_inflate_to_mem(&delta_buf, &delta_len, packfile);
-	if (err)
-		return err;
-
-	err = got_delta_get_sizes(base_size, result_size, delta_buf, delta_len);
-	free(delta_buf);
-	return err;
-}
-
-static const struct got_error *
 get_delta_chain_max_size(uint64_t *max_size, struct got_delta_chain *deltas,
     FILE *packfile)
 {
@@ -1122,8 +1102,8 @@ get_delta_chain_max_size(uint64_t *max_size, struct go
 		    delta->type != GOT_OBJ_TYPE_BLOB &&
 		    delta->type != GOT_OBJ_TYPE_TAG) {
 			const struct got_error *err;
-			err = get_delta_sizes(&base_size, &result_size, delta,
-			    packfile);
+			err = got_delta_get_sizes(&base_size, &result_size,
+			    delta->delta_buf, delta->delta_len);
 			if (err)
 				return err;
 		} else
@@ -1133,67 +1113,10 @@ get_delta_chain_max_size(uint64_t *max_size, struct go
 		if (result_size > *max_size)
 			*max_size = result_size;
 	}
-
-	return NULL;
-}
-
-void
-clear_delta_cache_entry(struct got_delta_cache_entry *entry)
-{
-	entry->data_offset = 0;
-	free(entry->delta_buf);
-	entry->delta_buf = NULL;
-	entry->delta_len = 0;
-}
-
-const struct got_error *
-cache_delta(off_t data_offset, uint8_t *delta_buf, size_t delta_len,
-    struct got_pack *pack)
-{
-	int i;
-	struct got_delta_cache *cache = &pack->delta_cache;
-	struct got_delta_cache_entry *entry;
-
-	if (cache->nentries == nitems(cache->deltas)) {
-		entry = &cache->deltas[cache->nentries - 1];
-		clear_delta_cache_entry(entry);
-		cache->nentries--;
-		memmove(&cache->deltas[1], &cache->deltas[0],
-		    sizeof(cache->deltas) - sizeof(cache->deltas[0]));
-		i = 0;
-	} else
-		i = cache->nentries;
 
-	entry = &cache->deltas[i];
-	entry->data_offset = data_offset;
-	entry->delta_buf = delta_buf;
-	entry->delta_len = delta_len;
-	cache->nentries++;
 	return NULL;
 }
 
-void
-get_cached_delta(uint8_t **delta_buf, size_t *delta_len, off_t data_offset,
-    struct got_pack *pack)
-{
-	struct got_delta_cache_entry *entry;
-	int i;
-
-	*delta_buf = NULL;
-	*delta_len = 0;
-
-	for (i = 0; i < nitems(pack->delta_cache.deltas); i++) {
-		entry = &pack->delta_cache.deltas[i];
-		if (entry->data_offset == 0)
-			break;
-		if (entry->data_offset == data_offset) {
-			*delta_buf = entry->delta_buf;
-			*delta_len = entry->delta_len;
-			break;
-		}
-	}
-}
-
 static const struct got_error *
 dump_delta_chain_to_file(size_t *result_size, struct got_delta_chain *deltas,
     FILE *outfile, struct got_pack *pack, struct got_repository *repo)
@@ -1234,9 +1157,6 @@ dump_delta_chain_to_file(size_t *result_size, struct g
 
 	/* Deltas are ordered in ascending order. */
 	SIMPLEQ_FOREACH(delta, &deltas->entries, entry) {
-		uint8_t *delta_buf = NULL;
-		size_t delta_len = 0;
-
 		if (n == 0) {
 			size_t base_len;
 
@@ -1276,36 +1196,15 @@ dump_delta_chain_to_file(size_t *result_size, struct g
 			if (base_file)
 				rewind(base_file);
 			continue;
-		}
-
-		get_cached_delta(&delta_buf, &delta_len, delta->data_offset,
-		    pack);
-		if (delta_buf == NULL) {
-			if (fseeko(pack->packfile, delta->data_offset, SEEK_SET)
-			    != 0) {
-				err = got_error_from_errno();
-				goto done;
-			}
-
-			/* Delta streams should always fit in memory. */
-			err = got_inflate_to_mem(&delta_buf, &delta_len,
-			    pack->packfile);
-			if (err)
-				goto done;
-
-			err = cache_delta(delta->data_offset, delta_buf,
-			    delta_len, pack);
-			if (err)
-				goto done;
 		}
-		/* delta_buf is now cached */
 
 		if (base_buf) {
-			err = got_delta_apply_in_mem(base_buf, delta_buf,
-			    delta_len, accum_buf, &accum_size);
+			err = got_delta_apply_in_mem(base_buf, delta->delta_buf,
+			    delta->delta_len, accum_buf, &accum_size);
 			n++;
 		} else {
-			err = got_delta_apply(base_file, delta_buf, delta_len,
+			err = got_delta_apply(base_file, delta->delta_buf,
+			    delta->delta_len,
 			    /* Final delta application writes to output file. */
 			    ++n < deltas->nentries ? accum_file : outfile,
 			    &accum_size);
@@ -1374,9 +1273,6 @@ dump_delta_chain_to_mem(uint8_t **outbuf, size_t *outl
 
 	/* Deltas are ordered in ascending order. */
 	SIMPLEQ_FOREACH(delta, &deltas->entries, entry) {
-		uint8_t *delta_buf = NULL;
-		size_t delta_len = 0;
-
 		if (n == 0) {
 			size_t base_len;
 
@@ -1411,30 +1307,8 @@ dump_delta_chain_to_mem(uint8_t **outbuf, size_t *outl
 			continue;
 		}
 
-		get_cached_delta(&delta_buf, &delta_len, delta->data_offset,
-		    pack);
-		if (delta_buf == NULL) {
-			if (fseeko(pack->packfile, delta->data_offset, SEEK_SET)
-			    != 0) {
-				err = got_error_from_errno();
-				goto done;
-			}
-
-			/* Delta streams should always fit in memory. */
-			err = got_inflate_to_mem(&delta_buf, &delta_len,
-			    pack->packfile);
-			if (err)
-				goto done;
-
-			err = cache_delta(delta->data_offset, delta_buf,
-			    delta_len, pack);
-			if (err)
-				goto done;
-		}
-		/* delta_buf is now cached */
-
-		err = got_delta_apply_in_mem(base_buf, delta_buf,
-		    delta_len, accum_buf, &accum_size);
+		err = got_delta_apply_in_mem(base_buf, delta->delta_buf,
+		    delta->delta_len, accum_buf, &accum_size);
 		n++;
 		if (err)
 			goto done;