commit 7e656b930dec54dd96e304ea8e3427a5531abaf9 from: Stefan Sperling date: Sat Mar 17 00:37:48 2018 UTC add struct got_pack; some preparation for mmap, which isn't used yet commit - b29656e27eca86259a4fc9099e9b5b31433e0065 commit + 7e656b930dec54dd96e304ea8e3427a5531abaf9 blob - f84a8a1c1403584c887369ec6db77e41699af2b7 blob + 7810cfb121d405b56681297a6f9a287f8a8dcbb8 --- lib/got_pack_lib.h +++ lib/got_pack_lib.h @@ -14,6 +14,30 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +/* A pack file segment mapped with mmap(2). */ +struct got_pack_mapping { + TAILQ_ENTRY(got_pack_mapping) entry; + int fd; + uint8_t *addr; + off_t offset; + size_t len; +}; + +#define GOT_PACK_MAX_OPEN_MAPPINGS 512 +#define GOT_PACK_MAPPING_MIN_SIZE 8192 +#define GOT_PACK_MAPPING_MAX_SIZE (2048 * GOT_PACK_MAPPING_MIN_SIZE) + +/* An open pack file. */ +struct got_pack { + char *path_packfile; + FILE *packfile; + size_t filesize; + int nmappings; + TAILQ_HEAD(, got_pack_mapping) mappings; +}; + +const struct got_error *got_pack_close(struct got_pack *); + /* See Documentation/technical/pack-format.txt in Git. */ struct got_packidx_trailer { blob - a8e4cedf77becb8285b6605ada60fbe549208fbb blob + 410d1e0bb6f59fae0200836154c3bb98cfb8267b --- lib/got_repository_lib.h +++ lib/got_repository_lib.h @@ -28,6 +28,7 @@ struct got_delta_cache { }; #define GOT_PACKIDX_CACHE_SIZE 64 +#define GOT_PACK_CACHE_SIZE GOT_PACKIDX_CACHE_SIZE struct got_repository { char *path; @@ -36,7 +37,9 @@ struct got_repository { /* The pack index cache speeds up search for packed objects. */ struct got_packidx_v2_hdr *packidx_cache[GOT_PACKIDX_CACHE_SIZE]; - /* The delta cache speeds up reconstruction of packed objects. */ - struct got_delta_cache delta_cache[GOT_PACKIDX_CACHE_SIZE]; -}; + /* Open file handles, memory maps, and cached deltas for pack files. */ + struct got_pack packs[GOT_PACK_CACHE_SIZE]; + /* XXX TODO move into packs[] */ + struct got_delta_cache delta_cache[GOT_DELTA_CACHE_SIZE]; +}; blob - b9e653bff893a26dfec127374b1cf76741196ac3 blob + 2554b19dc189cb2849e0fd142279c320e10a6a3c --- lib/pack.c +++ lib/pack.c @@ -17,8 +17,10 @@ #include #include #include +#include #include +#include #include #include #include @@ -500,33 +502,172 @@ read_packfile_hdr(FILE *f, struct got_packidx_v2_hdr * return err; } +#ifdef notyet static const struct got_error * -open_packfile(FILE **packfile, char **path_packfile, - struct got_repository *repo, struct got_packidx_v2_hdr *packidx) +map_packfile_segment(struct got_pack_mapping **map, const char *path_packfile, + off_t offset, size_t len) { - const struct got_error *err; + const struct got_error *err = NULL; - *packfile = NULL; + if (len < GOT_PACK_MAPPING_MIN_SIZE) + len = GOT_PACK_MAPPING_MIN_SIZE; - err = get_packfile_path(path_packfile, repo, packidx); - if (err) - return err; + if (len > GOT_PACK_MAPPING_MAX_SIZE) + return got_error(GOT_ERR_NO_SPACE); - *packfile = fopen(*path_packfile, "rb"); - if (*packfile == NULL) { + *map = calloc(1, sizeof(**map)); + if (*map == NULL) + return got_error(GOT_ERR_NO_MEM); + + (*map)->fd = open(path_packfile, O_RDONLY | O_NOFOLLOW); + if ((*map)->fd == -1) { err = got_error_from_errno(); - free(*path_packfile); + free((*map)); + *map = NULL; return err; } - err = read_packfile_hdr(*packfile, packidx); - if (err) { - fclose(*packfile); - *packfile = NULL; + (*map)->addr = mmap(NULL, len, PROT_READ, MAP_PRIVATE, (*map)->fd, offset); + if ((*map)->addr == NULL) { + err = got_error_from_errno(); + close((*map)->fd); + free((*map)); + *map = NULL; + return err; + } + + (*map)->offset = offset; + (*map)->len = len; + + return NULL; +} +#endif + +static const struct got_error * +unmap_packfile_segment(struct got_pack_mapping *map) +{ + if (munmap(map->addr, map->len) == -1 || close(map->fd) == -1) + return got_error_from_errno(); + free(map); + return NULL; +} + +static const struct got_error * +open_packfile(FILE **packfile, const char *path_packfile, + struct got_repository *repo, struct got_packidx_v2_hdr *packidx) +{ + const struct got_error *err = NULL; + + *packfile = NULL; + + *packfile = fopen(path_packfile, "rb"); + if (*packfile == NULL) { + err = got_error_from_errno(); + return err; + } + + if (packidx) { + err = read_packfile_hdr(*packfile, packidx); + if (err) { + fclose(*packfile); + *packfile = NULL; + } + } + return err; +} + +const struct got_error * +got_pack_close(struct got_pack *pack) +{ + const struct got_error *err = NULL; + + while (!TAILQ_EMPTY(&pack->mappings)) { + struct got_pack_mapping *map = TAILQ_FIRST(&pack->mappings); + err = unmap_packfile_segment(map); + if (err) + break; + TAILQ_REMOVE(&pack->mappings, map, entry); + pack->nmappings--; + } + + if (err == NULL) { + fclose(pack->packfile); + pack->packfile = NULL; + free(pack->path_packfile); + pack->path_packfile = NULL; + pack->filesize = 0; + } + return err; +} + +static const struct got_error * +cache_pack(struct got_pack **packp, const char *path_packfile, + struct got_packidx_v2_hdr *packidx, struct got_repository *repo) +{ + const struct got_error *err = NULL; + struct got_pack *pack = NULL; + int i; + + if (packp) + *packp = NULL; + + for (i = 0; i < nitems(repo->packs); i++) { + pack = &repo->packs[i]; + if (pack->path_packfile == NULL) + break; + if (strcmp(pack->path_packfile, path_packfile) == 0) + return NULL; } + + if (i == nitems(repo->packs) - 1) { + got_pack_close(&repo->packs[i - 1]); + memmove(&repo->packs[1], &repo->packs[0], + sizeof(repo->packs) - sizeof(repo->packs[0])); + i = 0; + } + + pack = &repo->packs[i]; + + TAILQ_INIT(&pack->mappings); + + pack->path_packfile = strdup(path_packfile); + if (pack->path_packfile == NULL) { + err = got_error(GOT_ERR_NO_MEM); + goto done; + } + + err = open_packfile(&pack->packfile, path_packfile, repo, packidx); + if (err) + goto done; + + err = get_packfile_size(&pack->filesize, path_packfile); +done: + if (err) { + if (pack) + free(pack->path_packfile); + free(pack); + } else if (packp) + *packp = pack; return err; } +struct got_pack * +get_cached_pack(const char *path_packfile, struct got_repository *repo) +{ + struct got_pack *pack = NULL; + int i; + + for (i = 0; i < nitems(repo->packs); i++) { + pack = &repo->packs[i]; + if (pack->path_packfile == NULL) + break; + if (strcmp(pack->path_packfile, path_packfile) == 0) + return pack; + } + + return NULL; +} + static const struct got_error * parse_object_type_and_size(uint8_t *type, uint64_t *size, size_t *len, FILE *packfile) @@ -740,7 +881,11 @@ resolve_ref_delta(struct got_delta_chain *deltas, stru return got_error(GOT_ERR_BAD_PACKIDX); } - err = open_packfile(&base_packfile, &path_base_packfile, repo, packidx); + err = get_packfile_path(&path_base_packfile, repo, packidx); + if (err) + return err; + + err = open_packfile(&base_packfile, path_base_packfile, repo, packidx); got_packidx_close(packidx); if (err) return err; @@ -863,10 +1008,14 @@ open_packed_object(struct got_object **obj, struct got if (offset == (uint64_t)-1) return got_error(GOT_ERR_BAD_PACKIDX); - err = open_packfile(&packfile, &path_packfile, repo, packidx); + err = get_packfile_path(&path_packfile, repo, packidx); if (err) return err; + err = open_packfile(&packfile, path_packfile, repo, packidx); + if (err) + return err; + if (fseeko(packfile, offset, SEEK_SET) != 0) { err = got_error_from_errno(); goto done; @@ -915,6 +1064,10 @@ got_packfile_open_object(struct got_object **obj, stru return err; err = open_packed_object(obj, repo, packidx, idx, id); + if (err) + return err; + + err = cache_pack(NULL, (*obj)->path_packfile, packidx, repo); got_packidx_close(packidx); return err; } @@ -1143,7 +1296,7 @@ dump_delta_chain_to_file(size_t *result_size, struct g goto done; } if (base_file) - err = got_inflate_to_file(&delta_len, + err = got_inflate_to_file(&base_len, packfile, base_file); else { err = got_inflate_to_mem(&base_buf, &base_len, @@ -1353,7 +1506,7 @@ got_packfile_extract_object(FILE **f, struct got_objec struct got_repository *repo) { const struct got_error *err = NULL; - FILE *packfile = NULL; + struct got_pack *pack; if ((obj->flags & GOT_OBJ_FLAG_PACKED) == 0) return got_error(GOT_ERR_OBJ_NOT_PACKED); @@ -1364,25 +1517,24 @@ got_packfile_extract_object(FILE **f, struct got_objec goto done; } - packfile = fopen(obj->path_packfile, "rb"); - if (packfile == NULL) { - err = got_error_from_errno(); - goto done; + pack = get_cached_pack(obj->path_packfile, repo); + if (pack == NULL) { + err = cache_pack(&pack, obj->path_packfile, NULL, repo); + if (err) + goto done; } if ((obj->flags & GOT_OBJ_FLAG_DELTIFIED) == 0) { - if (fseeko(packfile, obj->pack_offset, SEEK_SET) != 0) { + if (fseeko(pack->packfile, obj->pack_offset, SEEK_SET) != 0) { err = got_error_from_errno(); goto done; } - err = got_inflate_to_file(&obj->size, packfile, *f); + err = got_inflate_to_file(&obj->size, pack->packfile, *f); } else err = dump_delta_chain_to_file(&obj->size, &obj->deltas, *f, - packfile, obj->path_packfile, repo); + pack->packfile, obj->path_packfile, repo); done: - if (packfile) - fclose(packfile); if (err && *f) fclose(*f); return err; @@ -1394,26 +1546,28 @@ got_packfile_extract_object_to_mem(uint8_t **buf, size { const struct got_error *err = NULL; FILE *packfile = NULL; + struct got_pack *pack; if ((obj->flags & GOT_OBJ_FLAG_PACKED) == 0) return got_error(GOT_ERR_OBJ_NOT_PACKED); - packfile = fopen(obj->path_packfile, "rb"); - if (packfile == NULL) { - err = got_error_from_errno(); - goto done; + pack = get_cached_pack(obj->path_packfile, repo); + if (pack == NULL) { + err = cache_pack(&pack, obj->path_packfile, NULL, repo); + if (err) + goto done; } if ((obj->flags & GOT_OBJ_FLAG_DELTIFIED) == 0) { - if (fseeko(packfile, obj->pack_offset, SEEK_SET) != 0) { + if (fseeko(pack->packfile, obj->pack_offset, SEEK_SET) != 0) { err = got_error_from_errno(); goto done; } - err = got_inflate_to_mem(buf, len, packfile); + err = got_inflate_to_mem(buf, len, pack->packfile); } else - err = dump_delta_chain_to_mem(buf, len, &obj->deltas, packfile, - obj->path_packfile, repo); + err = dump_delta_chain_to_mem(buf, len, &obj->deltas, + pack->packfile, obj->path_packfile, repo); done: if (packfile) fclose(packfile); blob - c19c01392d63487dbce490ca2de4bbd2576af1d0 blob + c8d78504f16a5dffe3306c31059fb3c8ac289cab --- lib/repository.c +++ lib/repository.c @@ -29,11 +29,11 @@ #include "got_repository.h" #include "got_path_lib.h" -#include "got_repository_lib.h" -#include "got_zbuf_lib.h" #include "got_delta_lib.h" +#include "got_zbuf_lib.h" #include "got_object_lib.h" #include "got_pack_lib.h" +#include "got_repository_lib.h" #ifndef nitems #define nitems(_a) (sizeof(_a) / sizeof((_a)[0])) @@ -208,6 +208,12 @@ got_repo_close(struct got_repository *repo) got_packidx_close(repo->packidx_cache[i]); } + for (i = 0; i < nitems(repo->packs); i++) { + if (repo->packs[i].path_packfile == NULL) + break; + got_pack_close(&repo->packs[i]); + } + for (i = 0; i < nitems(repo->delta_cache); i++) { struct got_delta_cache *cache = &repo->delta_cache[i]; int j;