commit e32baab7e125247f6fe514984040708eb0fc1d0c from: Stefan Sperling date: Mon Nov 05 11:39:49 2018 UTC add an object cache for mini commits commit - 7762fe12f8608c112ccb3bbada7235a2c4208fd0 commit + e32baab7e125247f6fe514984040708eb0fc1d0c blob - be5add975f075e866935ce17ed6496c3e1439b20 blob + 08c1ddd1b73298b410791afccf5d5c931d412f72 --- lib/commit_graph.c +++ lib/commit_graph.c @@ -194,7 +194,7 @@ detect_changed_path(int *changed, struct got_commit_ob *changed = 1; /* The path was created in this commit. */ free(obj_id); } else { - err = got_object_open_mini_commit(&pcommit, repo, pid->id); + err = got_object_open_as_mini_commit(&pcommit, repo, pid->id); if (err) goto done; @@ -460,7 +460,7 @@ got_commit_graph_open(struct got_commit_graph **graph, *graph = NULL; - err = got_object_open_mini_commit(&commit, repo, commit_id); + err = got_object_open_as_mini_commit(&commit, repo, commit_id); if (err) return err; @@ -553,7 +553,7 @@ fetch_commits_from_open_branches(int *ncommits, commit_id = &graph->tips[i].id; child_node = graph->tips[i].node; - err = got_object_open_mini_commit(&commit, repo, commit_id); + err = got_object_open_as_mini_commit(&commit, repo, commit_id); if (err) break; @@ -635,7 +635,7 @@ got_commit_graph_iter_start(struct got_commit_graph *g if (start_node == NULL) return got_error(GOT_ERR_NO_OBJ); - err = got_object_open_mini_commit(&commit, repo, &start_node->id); + err = got_object_open_as_mini_commit(&commit, repo, &start_node->id); if (err) return err; blob - 95f019e89afcbce1f16aa9d9641ca1a175c5eec1 blob + 5eccdfbebc694f89e02859e3abc9331c23fd1776 --- lib/got_lib_object.h +++ lib/got_lib_object.h @@ -55,9 +55,13 @@ struct got_commit_object_mini { unsigned int nparents; struct got_object_id_queue parent_ids; struct tm tm_committer; /* UTC */ + int refcnt; /* > 0 if open and/or cached */ }; const struct got_error * -got_object_open_mini_commit(struct got_commit_object_mini **, - struct got_repository *, struct got_object_id *); +got_object_mini_commit_open(struct got_commit_object_mini **, + struct got_repository *, struct got_object *); void got_object_mini_commit_close(struct got_commit_object_mini *); +const struct got_error *got_object_open_as_mini_commit( + struct got_commit_object_mini **, struct got_repository *, + struct got_object_id *); blob - d574268b30620f1eb67c2f20275e6987b2248aaa blob + 424cbaa1930612b2fe7b26bb46afd6575d3788f3 --- lib/got_lib_object_cache.h +++ lib/got_lib_object_cache.h @@ -18,6 +18,7 @@ enum got_object_cache_type { GOT_OBJECT_CACHE_TYPE_OBJ, GOT_OBJECT_CACHE_TYPE_TREE, GOT_OBJECT_CACHE_TYPE_COMMIT, + GOT_OBJECT_CACHE_TYPE_MINI_COMMIT, }; struct got_object_cache_entry { @@ -26,6 +27,7 @@ struct got_object_cache_entry { struct got_object *obj; struct got_tree_object *tree; struct got_commit_object *commit; + struct got_commit_object_mini *mini_commit; } data; }; blob - feabca184d20dfd1ca4c681d9ccb24e0c7717d12 blob + 3eb7a0d15f87c4fbed21b13c884e3b13e83ed6be --- lib/got_lib_repository.h +++ lib/got_lib_repository.h @@ -38,6 +38,7 @@ struct got_repository { struct got_object_cache objcache; struct got_object_cache treecache; struct got_object_cache commitcache; + struct got_object_cache minicommitcache; }; const struct got_error*got_repo_cache_object(struct got_repository *, @@ -52,6 +53,10 @@ const struct got_error*got_repo_cache_commit(struct go struct got_object_id *, struct got_commit_object *); struct got_commit_object *got_repo_get_cached_commit(struct got_repository *, struct got_object_id *); +const struct got_error*got_repo_cache_mini_commit(struct got_repository *, + struct got_object_id *, struct got_commit_object_mini *); +struct got_commit_object_mini *got_repo_get_cached_mini_commit( + struct got_repository *, struct got_object_id *); const struct got_error *got_repo_cache_packidx(struct got_repository *, struct got_packidx *); const struct got_error *got_repo_search_packidx(struct got_packidx **, int *, blob - de22dce4277a47abcc56218844d0eb77281f29be blob + cdf402815e07307ece15bae1ad771e079b6dd695 --- lib/object.c +++ lib/object.c @@ -311,18 +311,59 @@ open_commit(struct got_commit_object **commit, if (err == NULL) { (*commit)->refcnt++; err = got_repo_cache_commit(repo, &obj->id, *commit); + } + + return err; +} + +const struct got_error * +got_object_open_as_commit(struct got_commit_object **commit, + struct got_repository *repo, struct got_object_id *id) +{ + const struct got_error *err; + struct got_object *obj; + + *commit = got_repo_get_cached_commit(repo, id); + if (*commit != NULL) { + (*commit)->refcnt++; + return NULL; + } + + err = got_object_open(&obj, repo, id); + if (err) + return err; + if (got_object_get_type(obj) != GOT_OBJ_TYPE_COMMIT) { + err = got_error(GOT_ERR_OBJ_TYPE); + goto done; } + err = open_commit(commit, repo, obj, 0); +done: + got_object_close(obj); return err; } +const struct got_error * +got_object_commit_open(struct got_commit_object **commit, + struct got_repository *repo, struct got_object *obj) +{ + return open_commit(commit, repo, obj, 1); +} + static const struct got_error * open_mini_commit(struct got_commit_object_mini **commit, - struct got_repository *repo, struct got_object *obj) + struct got_repository *repo, struct got_object *obj, int check_cache) { const struct got_error *err = NULL; - *commit = NULL; + if (check_cache) { + *commit = got_repo_get_cached_mini_commit(repo, &obj->id); + if (*commit != NULL) { + (*commit)->refcnt++; + return NULL; + } + } else + *commit = NULL; if (obj->type != GOT_OBJ_TYPE_COMMIT) return got_error(GOT_ERR_OBJ_TYPE); @@ -348,17 +389,22 @@ open_mini_commit(struct got_commit_object_mini **commi close(fd); } + if (err == NULL) { + (*commit)->refcnt++; + err = got_repo_cache_mini_commit(repo, &obj->id, *commit); + } + return err; } const struct got_error * -got_object_open_as_commit(struct got_commit_object **commit, +got_object_open_as_mini_commit(struct got_commit_object_mini **commit, struct got_repository *repo, struct got_object_id *id) { const struct got_error *err; struct got_object *obj; - *commit = got_repo_get_cached_commit(repo, id); + *commit = got_repo_get_cached_mini_commit(repo, id); if (*commit != NULL) { (*commit)->refcnt++; return NULL; @@ -372,38 +418,17 @@ got_object_open_as_commit(struct got_commit_object **c goto done; } - err = open_commit(commit, repo, obj, 0); + err = open_mini_commit(commit, repo, obj, 0); done: got_object_close(obj); return err; } const struct got_error * -got_object_commit_open(struct got_commit_object **commit, +got_object_mini_commit_open(struct got_commit_object_mini **commit, struct got_repository *repo, struct got_object *obj) { - return open_commit(commit, repo, obj, 1); -} - -const struct got_error * -got_object_open_mini_commit(struct got_commit_object_mini **commit, - struct got_repository *repo, struct got_object_id *id) -{ - const struct got_error *err; - struct got_object *obj; - - err = got_object_open(&obj, repo, id); - if (err) - return err; - if (got_object_get_type(obj) != GOT_OBJ_TYPE_COMMIT) { - err = got_error(GOT_ERR_OBJ_TYPE); - goto done; - } - - err = open_mini_commit(commit, repo, obj); -done: - got_object_close(obj); - return err; + return open_mini_commit(commit, repo, obj, 1); } const struct got_error * blob - 24b34b5a240fdcde737f55f54a4f6755371faffc blob + 883a84a15e357e4e6f3b5ba193c7c8fea843d4c1 --- lib/object_cache.c +++ lib/object_cache.c @@ -32,9 +32,10 @@ #include "got_lib_object_idcache.h" #include "got_lib_object_cache.h" -#define GOT_OBJECT_CACHE_SIZE_OBJ 1024 -#define GOT_OBJECT_CACHE_SIZE_TREE 2048 -#define GOT_OBJECT_CACHE_SIZE_COMMIT 512 +#define GOT_OBJECT_CACHE_SIZE_OBJ 1024 +#define GOT_OBJECT_CACHE_SIZE_TREE 2048 +#define GOT_OBJECT_CACHE_SIZE_COMMIT 512 +#define GOT_OBJECT_CACHE_SIZE_MINI_COMMIT 512 const struct got_error * got_object_cache_init(struct got_object_cache *cache, @@ -52,6 +53,9 @@ got_object_cache_init(struct got_object_cache *cache, case GOT_OBJECT_CACHE_TYPE_COMMIT: size = GOT_OBJECT_CACHE_SIZE_COMMIT; break; + case GOT_OBJECT_CACHE_TYPE_MINI_COMMIT: + size = GOT_OBJECT_CACHE_SIZE_MINI_COMMIT; + break; } cache->idcache = got_object_idcache_alloc(size); @@ -85,6 +89,9 @@ got_object_cache_add(struct got_object_cache *cache, s case GOT_OBJECT_CACHE_TYPE_COMMIT: got_object_commit_close(ce->data.commit); break; + case GOT_OBJECT_CACHE_TYPE_MINI_COMMIT: + got_object_mini_commit_close(ce->data.mini_commit); + break; } free(ce); cache->cache_evict++; @@ -104,6 +111,9 @@ got_object_cache_add(struct got_object_cache *cache, s case GOT_OBJECT_CACHE_TYPE_COMMIT: ce->data.commit = (struct got_commit_object *)item; break; + case GOT_OBJECT_CACHE_TYPE_MINI_COMMIT: + ce->data.mini_commit = (struct got_commit_object_mini *)item; + break; } err = got_object_idcache_add(cache->idcache, id, ce); @@ -132,6 +142,8 @@ got_object_cache_get(struct got_object_cache *cache, s return ce->data.tree; case GOT_OBJECT_CACHE_TYPE_COMMIT: return ce->data.commit; + case GOT_OBJECT_CACHE_TYPE_MINI_COMMIT: + return ce->data.mini_commit; } } @@ -157,6 +169,7 @@ void check_refcount(struct got_object_id *id, void *da struct got_object *obj; struct got_tree_object *tree; struct got_commit_object *commit; + struct got_commit_object_mini *mini_commit; char *id_str; if (got_object_id_str(&id_str, id) != NULL) @@ -184,6 +197,13 @@ void check_refcount(struct got_object_id *id, void *da fprintf(stderr, "commit %s has %d unclaimed references\n", id_str, commit->refcnt - 1); break; + case GOT_OBJECT_CACHE_TYPE_MINI_COMMIT: + mini_commit = ce->data.mini_commit; + if (mini_commit->refcnt == 1) + break; + fprintf(stderr, "commit %s has %d unclaimed references\n", + id_str, mini_commit->refcnt - 1); + break; } free(id_str); } @@ -203,6 +223,9 @@ got_object_cache_close(struct got_object_cache *cache) case GOT_OBJECT_CACHE_TYPE_COMMIT: print_cache_stats(cache, "commit"); break; + case GOT_OBJECT_CACHE_TYPE_MINI_COMMIT: + print_cache_stats(cache, "mini-commit"); + break; } got_object_idcache_for_each(cache->idcache, check_refcount, cache); blob - ebf46135eb8eed76cb6a8e760c20294fcdb3cc06 blob + ce15a31927ace3c21e8ca473c1e0718e612288a0 --- lib/object_parse.c +++ lib/object_parse.c @@ -313,6 +313,12 @@ void got_object_mini_commit_close(struct got_commit_object_mini *commit) { struct got_object_qid *qid; + + if (commit->refcnt > 0) { + commit->refcnt--; + if (commit->refcnt > 0) + return; + } while (!SIMPLEQ_EMPTY(&commit->parent_ids)) { qid = SIMPLEQ_FIRST(&commit->parent_ids); blob - d82e38e345a904ababe7f97b02142d026947c304 blob + 751a7ce53f5493154336f1f3a6417d9afb42d2fe --- lib/repository.c +++ lib/repository.c @@ -232,6 +232,28 @@ got_repo_get_cached_commit(struct got_repository *repo } const struct got_error * +got_repo_cache_mini_commit(struct got_repository *repo, + struct got_object_id *id, struct got_commit_object_mini *commit) +{ +#ifndef GOT_NO_OBJ_CACHE + const struct got_error *err = NULL; + err = got_object_cache_add(&repo->minicommitcache, id, commit); + if (err) + return err; + commit->refcnt++; +#endif + return NULL; +} + +struct got_commit_object_mini * +got_repo_get_cached_mini_commit(struct got_repository *repo, + struct got_object_id *id) +{ + return (struct got_commit_object_mini *)got_object_cache_get( + &repo->minicommitcache, id); +} + +const struct got_error * open_repo(struct got_repository *repo, const char *path) { const struct got_error *err = NULL; @@ -352,6 +374,10 @@ got_repo_open(struct got_repository **repop, const cha goto done; err = got_object_cache_init(&repo->commitcache, GOT_OBJECT_CACHE_TYPE_COMMIT); + if (err) + goto done; + err = got_object_cache_init(&repo->minicommitcache, + GOT_OBJECT_CACHE_TYPE_MINI_COMMIT); if (err) goto done; @@ -413,6 +439,7 @@ got_repo_close(struct got_repository *repo) got_object_cache_close(&repo->objcache); got_object_cache_close(&repo->treecache); got_object_cache_close(&repo->commitcache); + got_object_cache_close(&repo->minicommitcache); for (i = 0; i < nitems(repo->privsep_children); i++) { if (repo->privsep_children[i].imsg_fd == -1)