commit 9b576444ae8786f3cc494c57cb8e7a64b74970db from: Stefan Sperling date: Mon Mar 14 13:22:20 2022 UTC cache a list of known pack index files when the repository is opened Avoids overhead due to readdir calls while searching a pack index. ok op@ commit - 899fcfdf4534887848844329dae7327bd810a432 commit + 9b576444ae8786f3cc494c57cb8e7a64b74970db blob - 798762e89557feb475f650e62e549abe41eb43c4 blob + 5fb86bb46e7a4329db5383980128ed5c1b627aec --- lib/got_lib_repository.h +++ lib/got_lib_repository.h @@ -51,6 +51,8 @@ struct got_repository { char *path_git_dir; int gitdir_fd; + struct got_pathlist_head packidx_paths; + /* The pack index cache speeds up search for packed objects. */ struct got_packidx *packidx_cache[GOT_PACK_CACHE_SIZE]; blob - 3b6af380cd3ae1ac103c402a5365107c232529dd blob + 5a375985f96945d5f42e1d0b1b1f051e5df5c329 --- lib/pack_create.c +++ lib/pack_create.c @@ -488,20 +488,14 @@ static const struct got_error * find_pack_for_reuse(struct got_packidx **best_packidx, struct got_repository *repo) { - const struct got_error *err; - struct got_pathlist_head packidx_paths; + const struct got_error *err = NULL; struct got_pathlist_entry *pe; const char *best_packidx_path = NULL; int nobj_max = 0; - TAILQ_INIT(&packidx_paths); *best_packidx = NULL; - err = got_repo_list_packidx(&packidx_paths, repo); - if (err) - return err; - - TAILQ_FOREACH(pe, &packidx_paths, entry) { + TAILQ_FOREACH(pe, &repo->packidx_paths, entry) { const char *path_packidx = pe->path; struct got_packidx *packidx; int nobj; @@ -522,9 +516,6 @@ find_pack_for_reuse(struct got_packidx **best_packidx, repo); } - TAILQ_FOREACH(pe, &packidx_paths, entry) - free((void *)pe->path); - got_pathlist_free(&packidx_paths); return err; } blob - d4175545865a20ba9a3cff43736298aaae5e8c37 blob + bea8c8ad1e87cf8b7547d55c5b292380e389a38f --- lib/repository.c +++ lib/repository.c @@ -670,6 +670,7 @@ got_repo_open(struct got_repository **repop, const cha } RB_INIT(&repo->packidx_bloom_filters); + TAILQ_INIT(&repo->packidx_paths); for (i = 0; i < nitems(repo->privsep_children); i++) { memset(&repo->privsep_children[i], 0, @@ -750,6 +751,8 @@ got_repo_open(struct got_repository **repop, const cha goto done; } } + + err = got_repo_list_packidx(&repo->packidx_paths, repo); done: if (err) got_repo_close(repo); @@ -764,6 +767,7 @@ got_repo_close(struct got_repository *repo) { const struct got_error *err = NULL, *child_err; struct got_packidx_bloom_filter *bf; + struct got_pathlist_entry *pe; size_t i; for (i = 0; i < repo->pack_cache_size; i++) { @@ -824,6 +828,10 @@ got_repo_close(struct got_repository *repo) for (i = 0; i < repo->nextensions; i++) free(repo->extensions[i]); free(repo->extensions); + + TAILQ_FOREACH(pe, &repo->packidx_paths, entry) + free((void *)pe->path); + got_pathlist_free(&repo->packidx_paths); free(repo); return err; @@ -1092,11 +1100,8 @@ got_repo_search_packidx(struct got_packidx **packidx, struct got_repository *repo, struct got_object_id *id) { const struct got_error *err; - DIR *packdir = NULL; - struct dirent *dent; - char *path_packidx; + struct got_pathlist_entry *pe; size_t i; - int packdir_fd; /* Search pack index cache. */ for (i = 0; i < repo->pack_cache_size; i++) { @@ -1122,42 +1127,14 @@ got_repo_search_packidx(struct got_packidx **packidx, } } /* No luck. Search the filesystem. */ - - packdir_fd = openat(got_repo_get_fd(repo), - GOT_OBJECTS_PACK_DIR, O_DIRECTORY | O_CLOEXEC); - if (packdir_fd == -1) { - if (errno == ENOENT) - err = got_error_no_obj(id); - else - err = got_error_from_errno_fmt("openat: %s/%s", - got_repo_get_path_git_dir(repo), - GOT_OBJECTS_PACK_DIR); - goto done; - } - packdir = fdopendir(packdir_fd); - if (packdir == NULL) { - err = got_error_from_errno("fdopendir"); - goto done; - } - - while ((dent = readdir(packdir)) != NULL) { + TAILQ_FOREACH(pe, &repo->packidx_paths, entry) { + const char *path_packidx = pe->path; int is_cached = 0; - if (!got_repo_is_packidx_filename(dent->d_name, dent->d_namlen)) - continue; - - if (asprintf(&path_packidx, "%s/%s", GOT_OBJECTS_PACK_DIR, - dent->d_name) == -1) { - err = got_error_from_errno("asprintf"); - goto done; - } - if (!got_repo_check_packidx_bloom_filter(repo, - path_packidx, id)) { - free(path_packidx); + pe->path, id)) continue; /* object will not be found in this index */ - } for (i = 0; i < repo->pack_cache_size; i++) { if (repo->packidx_cache[i] == NULL) @@ -1168,26 +1145,19 @@ got_repo_search_packidx(struct got_packidx **packidx, break; } } - if (is_cached) { - free(path_packidx); + if (is_cached) continue; /* already searched */ - } err = got_packidx_open(packidx, got_repo_get_fd(repo), path_packidx, 0); - if (err) { - free(path_packidx); + if (err) goto done; - } err = add_packidx_bloom_filter(repo, *packidx, path_packidx); - if (err) { - free(path_packidx); + if (err) goto done; - } err = cache_packidx(repo, *packidx, path_packidx); - free(path_packidx); if (err) goto done; @@ -1200,8 +1170,6 @@ got_repo_search_packidx(struct got_packidx **packidx, err = got_error_no_obj(id); done: - if (packdir && closedir(packdir) != 0 && err == NULL) - err = got_error_from_errno("closedir"); return err; } @@ -1485,46 +1453,18 @@ match_packed_object(struct got_object_id **unique_id, struct got_repository *repo, const char *id_str_prefix, int obj_type) { const struct got_error *err = NULL; - DIR *packdir = NULL; - struct dirent *dent; - char *path_packidx; - struct got_object_id_queue matched_ids; - int packdir_fd; + struct got_object_id_queue matched_ids; + struct got_pathlist_entry *pe; STAILQ_INIT(&matched_ids); - packdir_fd = openat(got_repo_get_fd(repo), - GOT_OBJECTS_PACK_DIR, O_DIRECTORY | O_CLOEXEC); - if (packdir_fd == -1) { - if (errno != ENOENT) { - err = got_error_from_errno2("openat", - GOT_OBJECTS_PACK_DIR); - } - goto done; - } - - packdir = fdopendir(packdir_fd); - if (packdir == NULL) { - err = got_error_from_errno("fdopendir"); - goto done; - } - - while ((dent = readdir(packdir)) != NULL) { + TAILQ_FOREACH(pe, &repo->packidx_paths, entry) { + const char *path_packidx = pe->path; struct got_packidx *packidx; struct got_object_qid *qid; - if (!got_repo_is_packidx_filename(dent->d_name, dent->d_namlen)) - continue; - - if (asprintf(&path_packidx, "%s/%s", GOT_OBJECTS_PACK_DIR, - dent->d_name) == -1) { - err = got_error_from_errno("strdup"); - break; - } - err = got_packidx_open(&packidx, got_repo_get_fd(repo), path_packidx, 0); - free(path_packidx); if (err) break; @@ -1564,8 +1504,6 @@ match_packed_object(struct got_object_id **unique_id, } done: got_object_id_queue_free(&matched_ids); - if (packdir && closedir(packdir) != 0 && err == NULL) - err = got_error_from_errno("closedir"); if (err) { free(*unique_id); *unique_id = NULL;