op public repos

Commit Diff

Commit:: e32baab7e125247f6fe514984040708eb0fc1d0c
From:: Stefan Sperling <stsp@stsp.name>
Date:: Mon Nov 5 11:39:49 2018 UTC
Message:: add an object cache for mini commits
Actions:: Patch | Tree
commit - 7762fe12f8608c112ccb3bbada7235a2c4208fd0
commit + e32baab7e125247f6fe514984040708eb0fc1d0c
blob - be5add975f075e866935ce17ed6496c3e1439b20
blob + 08c1ddd1b73298b410791afccf5d5c931d412f72
--- lib/commit_graph.c
+++ lib/commit_graph.c
@@ -194,7 +194,7 @@ detect_changed_path(int *changed, struct got_commit_ob
 			*changed = 1; /* The path was created in this commit. */
 		free(obj_id);
 	} else {
-		err = got_object_open_mini_commit(&pcommit, repo, pid->id);
+		err = got_object_open_as_mini_commit(&pcommit, repo, pid->id);
 		if (err)
 			goto done;
 
@@ -460,7 +460,7 @@ got_commit_graph_open(struct got_commit_graph **graph,
 
 	*graph = NULL;
 
-	err = got_object_open_mini_commit(&commit, repo, commit_id);
+	err = got_object_open_as_mini_commit(&commit, repo, commit_id);
 	if (err)
 		return err;
 
@@ -553,7 +553,7 @@ fetch_commits_from_open_branches(int *ncommits,
 		commit_id = &graph->tips[i].id;
 		child_node = graph->tips[i].node;
 
-		err = got_object_open_mini_commit(&commit, repo, commit_id);
+		err = got_object_open_as_mini_commit(&commit, repo, commit_id);
 		if (err)
 			break;
 
@@ -635,7 +635,7 @@ got_commit_graph_iter_start(struct got_commit_graph *g
 	if (start_node == NULL)
 		return got_error(GOT_ERR_NO_OBJ);
 
-	err = got_object_open_mini_commit(&commit, repo, &start_node->id);
+	err = got_object_open_as_mini_commit(&commit, repo, &start_node->id);
 	if (err)
 		return err;
 
blob - 95f019e89afcbce1f16aa9d9641ca1a175c5eec1
blob + 5eccdfbebc694f89e02859e3abc9331c23fd1776
--- lib/got_lib_object.h
+++ lib/got_lib_object.h
@@ -55,9 +55,13 @@ struct got_commit_object_mini {
 	unsigned int nparents;
 	struct got_object_id_queue parent_ids;
 	struct tm tm_committer;	/* UTC */
+	int refcnt;		/* > 0 if open and/or cached */
 };
 
 const struct got_error *
-got_object_open_mini_commit(struct got_commit_object_mini **,
-    struct got_repository *, struct got_object_id *);
+got_object_mini_commit_open(struct got_commit_object_mini **,
+    struct got_repository *, struct got_object *);
 void got_object_mini_commit_close(struct got_commit_object_mini *);
+const struct got_error *got_object_open_as_mini_commit(
+    struct got_commit_object_mini **, struct got_repository *,
+    struct got_object_id *);
blob - d574268b30620f1eb67c2f20275e6987b2248aaa
blob + 424cbaa1930612b2fe7b26bb46afd6575d3788f3
--- lib/got_lib_object_cache.h
+++ lib/got_lib_object_cache.h
@@ -18,6 +18,7 @@ enum got_object_cache_type {
 	GOT_OBJECT_CACHE_TYPE_OBJ,
 	GOT_OBJECT_CACHE_TYPE_TREE,
 	GOT_OBJECT_CACHE_TYPE_COMMIT,
+	GOT_OBJECT_CACHE_TYPE_MINI_COMMIT,
 };
 
 struct got_object_cache_entry {
@@ -26,6 +27,7 @@ struct got_object_cache_entry {
 		struct got_object *obj;
 		struct got_tree_object *tree;
 		struct got_commit_object *commit;
+		struct got_commit_object_mini *mini_commit;
 	} data;
 };
 
blob - feabca184d20dfd1ca4c681d9ccb24e0c7717d12
blob + 3eb7a0d15f87c4fbed21b13c884e3b13e83ed6be
--- lib/got_lib_repository.h
+++ lib/got_lib_repository.h
@@ -38,6 +38,7 @@ struct got_repository {
 	struct got_object_cache objcache;
 	struct got_object_cache treecache;
 	struct got_object_cache commitcache;
+	struct got_object_cache minicommitcache;
 };
 
 const struct got_error*got_repo_cache_object(struct got_repository *,
@@ -52,6 +53,10 @@ const struct got_error*got_repo_cache_commit(struct go
     struct got_object_id *, struct got_commit_object *);
 struct got_commit_object *got_repo_get_cached_commit(struct got_repository *,
     struct got_object_id *);
+const struct got_error*got_repo_cache_mini_commit(struct got_repository *,
+    struct got_object_id *, struct got_commit_object_mini *);
+struct got_commit_object_mini *got_repo_get_cached_mini_commit(
+    struct got_repository *, struct got_object_id *);
 const struct got_error *got_repo_cache_packidx(struct got_repository *,
     struct got_packidx *);
 const struct got_error *got_repo_search_packidx(struct got_packidx **, int *,
blob - de22dce4277a47abcc56218844d0eb77281f29be
blob + cdf402815e07307ece15bae1ad771e079b6dd695
--- lib/object.c
+++ lib/object.c
@@ -311,18 +311,59 @@ open_commit(struct got_commit_object **commit,
 	if (err == NULL) {
 		(*commit)->refcnt++;
 		err = got_repo_cache_commit(repo, &obj->id, *commit);
+	}
+
+	return err;
+}
+
+const struct got_error *
+got_object_open_as_commit(struct got_commit_object **commit,
+    struct got_repository *repo, struct got_object_id *id)
+{
+	const struct got_error *err;
+	struct got_object *obj;
+
+	*commit = got_repo_get_cached_commit(repo, id);
+	if (*commit != NULL) {
+		(*commit)->refcnt++;
+		return NULL;
+	}
+
+	err = got_object_open(&obj, repo, id);
+	if (err)
+		return err;
+	if (got_object_get_type(obj) != GOT_OBJ_TYPE_COMMIT) {
+		err = got_error(GOT_ERR_OBJ_TYPE);
+		goto done;
 	}
 
+	err = open_commit(commit, repo, obj, 0);
+done:
+	got_object_close(obj);
 	return err;
 }
 
+const struct got_error *
+got_object_commit_open(struct got_commit_object **commit,
+    struct got_repository *repo, struct got_object *obj)
+{
+	return open_commit(commit, repo, obj, 1);
+}
+
 static const struct got_error *
 open_mini_commit(struct got_commit_object_mini **commit,
-    struct got_repository *repo, struct got_object *obj)
+    struct got_repository *repo, struct got_object *obj, int check_cache)
 {
 	const struct got_error *err = NULL;
 
-	*commit = NULL;
+	if (check_cache) {
+		*commit = got_repo_get_cached_mini_commit(repo, &obj->id);
+		if (*commit != NULL) {
+			(*commit)->refcnt++;
+			return NULL;
+		}
+	} else
+		*commit = NULL;
 
 	if (obj->type != GOT_OBJ_TYPE_COMMIT)
 		return got_error(GOT_ERR_OBJ_TYPE);
@@ -348,17 +389,22 @@ open_mini_commit(struct got_commit_object_mini **commi
 		close(fd);
 	}
 
+	if (err == NULL) {
+		(*commit)->refcnt++;
+		err = got_repo_cache_mini_commit(repo, &obj->id, *commit);
+	}
+
 	return err;
 }
 
 const struct got_error *
-got_object_open_as_commit(struct got_commit_object **commit,
+got_object_open_as_mini_commit(struct got_commit_object_mini **commit,
     struct got_repository *repo, struct got_object_id *id)
 {
 	const struct got_error *err;
 	struct got_object *obj;
 
-	*commit = got_repo_get_cached_commit(repo, id);
+	*commit = got_repo_get_cached_mini_commit(repo, id);
 	if (*commit != NULL) {
 		(*commit)->refcnt++;
 		return NULL;
@@ -372,38 +418,17 @@ got_object_open_as_commit(struct got_commit_object **c
 		goto done;
 	}
 
-	err = open_commit(commit, repo, obj, 0);
+	err = open_mini_commit(commit, repo, obj, 0);
 done:
 	got_object_close(obj);
 	return err;
 }
 
 const struct got_error *
-got_object_commit_open(struct got_commit_object **commit,
+got_object_mini_commit_open(struct got_commit_object_mini **commit,
     struct got_repository *repo, struct got_object *obj)
 {
-	return open_commit(commit, repo, obj, 1);
-}
-
-const struct got_error *
-got_object_open_mini_commit(struct got_commit_object_mini **commit,
-    struct got_repository *repo, struct got_object_id *id)
-{
-	const struct got_error *err;
-	struct got_object *obj;
-
-	err = got_object_open(&obj, repo, id);
-	if (err)
-		return err;
-	if (got_object_get_type(obj) != GOT_OBJ_TYPE_COMMIT) {
-		err = got_error(GOT_ERR_OBJ_TYPE);
-		goto done;
-	}
-
-	err = open_mini_commit(commit, repo, obj);
-done:
-	got_object_close(obj);
-	return err;
+	return open_mini_commit(commit, repo, obj, 1);
 }
 
 const struct got_error *
blob - 24b34b5a240fdcde737f55f54a4f6755371faffc
blob + 883a84a15e357e4e6f3b5ba193c7c8fea843d4c1
--- lib/object_cache.c
+++ lib/object_cache.c
@@ -32,9 +32,10 @@
 #include "got_lib_object_idcache.h"
 #include "got_lib_object_cache.h"
 
-#define GOT_OBJECT_CACHE_SIZE_OBJ	1024
-#define GOT_OBJECT_CACHE_SIZE_TREE	2048
-#define GOT_OBJECT_CACHE_SIZE_COMMIT	512
+#define GOT_OBJECT_CACHE_SIZE_OBJ		1024
+#define GOT_OBJECT_CACHE_SIZE_TREE		2048
+#define GOT_OBJECT_CACHE_SIZE_COMMIT		512
+#define GOT_OBJECT_CACHE_SIZE_MINI_COMMIT	512
 
 const struct got_error *
 got_object_cache_init(struct got_object_cache *cache,
@@ -52,6 +53,9 @@ got_object_cache_init(struct got_object_cache *cache,
 	case GOT_OBJECT_CACHE_TYPE_COMMIT:
 		size = GOT_OBJECT_CACHE_SIZE_COMMIT;
 		break;
+	case GOT_OBJECT_CACHE_TYPE_MINI_COMMIT:
+		size = GOT_OBJECT_CACHE_SIZE_MINI_COMMIT;
+		break;
 	}
 
 	cache->idcache = got_object_idcache_alloc(size);
@@ -85,6 +89,9 @@ got_object_cache_add(struct got_object_cache *cache, s
 		case GOT_OBJECT_CACHE_TYPE_COMMIT:
 			got_object_commit_close(ce->data.commit);
 			break;
+		case GOT_OBJECT_CACHE_TYPE_MINI_COMMIT:
+			got_object_mini_commit_close(ce->data.mini_commit);
+			break;
 		}
 		free(ce);
 		cache->cache_evict++;
@@ -104,6 +111,9 @@ got_object_cache_add(struct got_object_cache *cache, s
 	case GOT_OBJECT_CACHE_TYPE_COMMIT:
 		ce->data.commit = (struct got_commit_object *)item;
 		break;
+	case GOT_OBJECT_CACHE_TYPE_MINI_COMMIT:
+		ce->data.mini_commit = (struct got_commit_object_mini *)item;
+		break;
 	}
 
 	err = got_object_idcache_add(cache->idcache, id, ce);
@@ -132,6 +142,8 @@ got_object_cache_get(struct got_object_cache *cache, s
 			return ce->data.tree;
 		case GOT_OBJECT_CACHE_TYPE_COMMIT:
 			return ce->data.commit;
+		case GOT_OBJECT_CACHE_TYPE_MINI_COMMIT:
+			return ce->data.mini_commit;
 		}
 	}
 
@@ -157,6 +169,7 @@ void check_refcount(struct got_object_id *id, void *da
 	struct got_object *obj;
 	struct got_tree_object *tree;
 	struct got_commit_object *commit;
+	struct got_commit_object_mini *mini_commit;
 	char *id_str;
 
 	if (got_object_id_str(&id_str, id) != NULL)
@@ -184,6 +197,13 @@ void check_refcount(struct got_object_id *id, void *da
 		fprintf(stderr, "commit %s has %d unclaimed references\n",
 		    id_str, commit->refcnt - 1);
 		break;
+	case GOT_OBJECT_CACHE_TYPE_MINI_COMMIT:
+		mini_commit = ce->data.mini_commit;
+		if (mini_commit->refcnt == 1)
+			break;
+		fprintf(stderr, "commit %s has %d unclaimed references\n",
+		    id_str, mini_commit->refcnt - 1);
+		break;
 	}
 	free(id_str);
 }
@@ -203,6 +223,9 @@ got_object_cache_close(struct got_object_cache *cache)
 	case GOT_OBJECT_CACHE_TYPE_COMMIT:
 		print_cache_stats(cache, "commit");
 		break;
+	case GOT_OBJECT_CACHE_TYPE_MINI_COMMIT:
+		print_cache_stats(cache, "mini-commit");
+		break;
 	}
 
 	got_object_idcache_for_each(cache->idcache, check_refcount, cache);
blob - ebf46135eb8eed76cb6a8e760c20294fcdb3cc06
blob + ce15a31927ace3c21e8ca473c1e0718e612288a0
--- lib/object_parse.c
+++ lib/object_parse.c
@@ -313,6 +313,12 @@ void
 got_object_mini_commit_close(struct got_commit_object_mini *commit)
 {
 	struct got_object_qid *qid;
+
+	if (commit->refcnt > 0) {
+		commit->refcnt--;
+		if (commit->refcnt > 0)
+			return;
+	}
 
 	while (!SIMPLEQ_EMPTY(&commit->parent_ids)) {
 		qid = SIMPLEQ_FIRST(&commit->parent_ids);
blob - d82e38e345a904ababe7f97b02142d026947c304
blob + 751a7ce53f5493154336f1f3a6417d9afb42d2fe
--- lib/repository.c
+++ lib/repository.c
@@ -232,6 +232,28 @@ got_repo_get_cached_commit(struct got_repository *repo
 }
 
 const struct got_error *
+got_repo_cache_mini_commit(struct got_repository *repo,
+    struct got_object_id *id, struct got_commit_object_mini *commit)
+{
+#ifndef GOT_NO_OBJ_CACHE
+	const struct got_error *err = NULL;
+	err = got_object_cache_add(&repo->minicommitcache, id, commit);
+	if (err)
+		return err;
+	commit->refcnt++;
+#endif
+	return NULL;
+}
+
+struct got_commit_object_mini *
+got_repo_get_cached_mini_commit(struct got_repository *repo,
+    struct got_object_id *id)
+{
+	return (struct got_commit_object_mini *)got_object_cache_get(
+	    &repo->minicommitcache, id);
+}
+
+const struct got_error *
 open_repo(struct got_repository *repo, const char *path)
 {
 	const struct got_error *err = NULL;
@@ -352,6 +374,10 @@ got_repo_open(struct got_repository **repop, const cha
 		goto done;
 	err = got_object_cache_init(&repo->commitcache,
 	    GOT_OBJECT_CACHE_TYPE_COMMIT);
+	if (err)
+		goto done;
+	err = got_object_cache_init(&repo->minicommitcache,
+	    GOT_OBJECT_CACHE_TYPE_MINI_COMMIT);
 	if (err)
 		goto done;
 
@@ -413,6 +439,7 @@ got_repo_close(struct got_repository *repo)
 	got_object_cache_close(&repo->objcache);
 	got_object_cache_close(&repo->treecache);
 	got_object_cache_close(&repo->commitcache);
+	got_object_cache_close(&repo->minicommitcache);
 
 	for (i = 0; i < nitems(repo->privsep_children); i++) {
 		if (repo->privsep_children[i].imsg_fd == -1)