commit - b29656e27eca86259a4fc9099e9b5b31433e0065
commit + 7e656b930dec54dd96e304ea8e3427a5531abaf9
blob - f84a8a1c1403584c887369ec6db77e41699af2b7
blob + 7810cfb121d405b56681297a6f9a287f8a8dcbb8
--- lib/got_pack_lib.h
+++ lib/got_pack_lib.h
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
+/* A pack file segment mapped with mmap(2). */
+struct got_pack_mapping {
+ TAILQ_ENTRY(got_pack_mapping) entry;
+ int fd;
+ uint8_t *addr;
+ off_t offset;
+ size_t len;
+};
+
+#define GOT_PACK_MAX_OPEN_MAPPINGS 512
+#define GOT_PACK_MAPPING_MIN_SIZE 8192
+#define GOT_PACK_MAPPING_MAX_SIZE (2048 * GOT_PACK_MAPPING_MIN_SIZE)
+
+/* An open pack file. */
+struct got_pack {
+ char *path_packfile;
+ FILE *packfile;
+ size_t filesize;
+ int nmappings;
+ TAILQ_HEAD(, got_pack_mapping) mappings;
+};
+
+const struct got_error *got_pack_close(struct got_pack *);
+
/* See Documentation/technical/pack-format.txt in Git. */
struct got_packidx_trailer {
blob - a8e4cedf77becb8285b6605ada60fbe549208fbb
blob + 410d1e0bb6f59fae0200836154c3bb98cfb8267b
--- lib/got_repository_lib.h
+++ lib/got_repository_lib.h
};
#define GOT_PACKIDX_CACHE_SIZE 64
+#define GOT_PACK_CACHE_SIZE GOT_PACKIDX_CACHE_SIZE
struct got_repository {
char *path;
/* The pack index cache speeds up search for packed objects. */
struct got_packidx_v2_hdr *packidx_cache[GOT_PACKIDX_CACHE_SIZE];
- /* The delta cache speeds up reconstruction of packed objects. */
- struct got_delta_cache delta_cache[GOT_PACKIDX_CACHE_SIZE];
-};
+ /* Open file handles, memory maps, and cached deltas for pack files. */
+ struct got_pack packs[GOT_PACK_CACHE_SIZE];
+ /* XXX TODO move into packs[] */
+ struct got_delta_cache delta_cache[GOT_DELTA_CACHE_SIZE];
+};
blob - b9e653bff893a26dfec127374b1cf76741196ac3
blob + 2554b19dc189cb2849e0fd142279c320e10a6a3c
--- lib/pack.c
+++ lib/pack.c
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/queue.h>
+#include <sys/mman.h>
#include <dirent.h>
+#include <fcntl.h>
#include <errno.h>
#include <stdio.h>
#include <stdint.h>
return err;
}
+#ifdef notyet
static const struct got_error *
-open_packfile(FILE **packfile, char **path_packfile,
- struct got_repository *repo, struct got_packidx_v2_hdr *packidx)
+map_packfile_segment(struct got_pack_mapping **map, const char *path_packfile,
+ off_t offset, size_t len)
{
- const struct got_error *err;
+ const struct got_error *err = NULL;
- *packfile = NULL;
+ if (len < GOT_PACK_MAPPING_MIN_SIZE)
+ len = GOT_PACK_MAPPING_MIN_SIZE;
- err = get_packfile_path(path_packfile, repo, packidx);
- if (err)
- return err;
+ if (len > GOT_PACK_MAPPING_MAX_SIZE)
+ return got_error(GOT_ERR_NO_SPACE);
- *packfile = fopen(*path_packfile, "rb");
- if (*packfile == NULL) {
+ *map = calloc(1, sizeof(**map));
+ if (*map == NULL)
+ return got_error(GOT_ERR_NO_MEM);
+
+ (*map)->fd = open(path_packfile, O_RDONLY | O_NOFOLLOW);
+ if ((*map)->fd == -1) {
err = got_error_from_errno();
- free(*path_packfile);
+ free((*map));
+ *map = NULL;
return err;
}
- err = read_packfile_hdr(*packfile, packidx);
- if (err) {
- fclose(*packfile);
- *packfile = NULL;
+ (*map)->addr = mmap(NULL, len, PROT_READ, MAP_PRIVATE, (*map)->fd, offset);
+ if ((*map)->addr == NULL) {
+ err = got_error_from_errno();
+ close((*map)->fd);
+ free((*map));
+ *map = NULL;
+ return err;
+ }
+
+ (*map)->offset = offset;
+ (*map)->len = len;
+
+ return NULL;
+}
+#endif
+
+static const struct got_error *
+unmap_packfile_segment(struct got_pack_mapping *map)
+{
+ if (munmap(map->addr, map->len) == -1 || close(map->fd) == -1)
+ return got_error_from_errno();
+ free(map);
+ return NULL;
+}
+
+static const struct got_error *
+open_packfile(FILE **packfile, const char *path_packfile,
+ struct got_repository *repo, struct got_packidx_v2_hdr *packidx)
+{
+ const struct got_error *err = NULL;
+
+ *packfile = NULL;
+
+ *packfile = fopen(path_packfile, "rb");
+ if (*packfile == NULL) {
+ err = got_error_from_errno();
+ return err;
+ }
+
+ if (packidx) {
+ err = read_packfile_hdr(*packfile, packidx);
+ if (err) {
+ fclose(*packfile);
+ *packfile = NULL;
+ }
+ }
+ return err;
+}
+
+const struct got_error *
+got_pack_close(struct got_pack *pack)
+{
+ const struct got_error *err = NULL;
+
+ while (!TAILQ_EMPTY(&pack->mappings)) {
+ struct got_pack_mapping *map = TAILQ_FIRST(&pack->mappings);
+ err = unmap_packfile_segment(map);
+ if (err)
+ break;
+ TAILQ_REMOVE(&pack->mappings, map, entry);
+ pack->nmappings--;
+ }
+
+ if (err == NULL) {
+ fclose(pack->packfile);
+ pack->packfile = NULL;
+ free(pack->path_packfile);
+ pack->path_packfile = NULL;
+ pack->filesize = 0;
+ }
+ return err;
+}
+
+static const struct got_error *
+cache_pack(struct got_pack **packp, const char *path_packfile,
+ struct got_packidx_v2_hdr *packidx, struct got_repository *repo)
+{
+ const struct got_error *err = NULL;
+ struct got_pack *pack = NULL;
+ int i;
+
+ if (packp)
+ *packp = NULL;
+
+ for (i = 0; i < nitems(repo->packs); i++) {
+ pack = &repo->packs[i];
+ if (pack->path_packfile == NULL)
+ break;
+ if (strcmp(pack->path_packfile, path_packfile) == 0)
+ return NULL;
}
+
+ if (i == nitems(repo->packs) - 1) {
+ got_pack_close(&repo->packs[i - 1]);
+ memmove(&repo->packs[1], &repo->packs[0],
+ sizeof(repo->packs) - sizeof(repo->packs[0]));
+ i = 0;
+ }
+
+ pack = &repo->packs[i];
+
+ TAILQ_INIT(&pack->mappings);
+
+ pack->path_packfile = strdup(path_packfile);
+ if (pack->path_packfile == NULL) {
+ err = got_error(GOT_ERR_NO_MEM);
+ goto done;
+ }
+
+ err = open_packfile(&pack->packfile, path_packfile, repo, packidx);
+ if (err)
+ goto done;
+
+ err = get_packfile_size(&pack->filesize, path_packfile);
+done:
+ if (err) {
+ if (pack)
+ free(pack->path_packfile);
+ free(pack);
+ } else if (packp)
+ *packp = pack;
return err;
}
+struct got_pack *
+get_cached_pack(const char *path_packfile, struct got_repository *repo)
+{
+ struct got_pack *pack = NULL;
+ int i;
+
+ for (i = 0; i < nitems(repo->packs); i++) {
+ pack = &repo->packs[i];
+ if (pack->path_packfile == NULL)
+ break;
+ if (strcmp(pack->path_packfile, path_packfile) == 0)
+ return pack;
+ }
+
+ return NULL;
+}
+
static const struct got_error *
parse_object_type_and_size(uint8_t *type, uint64_t *size, size_t *len,
FILE *packfile)
return got_error(GOT_ERR_BAD_PACKIDX);
}
- err = open_packfile(&base_packfile, &path_base_packfile, repo, packidx);
+ err = get_packfile_path(&path_base_packfile, repo, packidx);
+ if (err)
+ return err;
+
+ err = open_packfile(&base_packfile, path_base_packfile, repo, packidx);
got_packidx_close(packidx);
if (err)
return err;
if (offset == (uint64_t)-1)
return got_error(GOT_ERR_BAD_PACKIDX);
- err = open_packfile(&packfile, &path_packfile, repo, packidx);
+ err = get_packfile_path(&path_packfile, repo, packidx);
if (err)
return err;
+ err = open_packfile(&packfile, path_packfile, repo, packidx);
+ if (err)
+ return err;
+
if (fseeko(packfile, offset, SEEK_SET) != 0) {
err = got_error_from_errno();
goto done;
return err;
err = open_packed_object(obj, repo, packidx, idx, id);
+ if (err)
+ return err;
+
+ err = cache_pack(NULL, (*obj)->path_packfile, packidx, repo);
got_packidx_close(packidx);
return err;
}
goto done;
}
if (base_file)
- err = got_inflate_to_file(&delta_len,
+ err = got_inflate_to_file(&base_len,
packfile, base_file);
else {
err = got_inflate_to_mem(&base_buf, &base_len,
struct got_repository *repo)
{
const struct got_error *err = NULL;
- FILE *packfile = NULL;
+ struct got_pack *pack;
if ((obj->flags & GOT_OBJ_FLAG_PACKED) == 0)
return got_error(GOT_ERR_OBJ_NOT_PACKED);
goto done;
}
- packfile = fopen(obj->path_packfile, "rb");
- if (packfile == NULL) {
- err = got_error_from_errno();
- goto done;
+ pack = get_cached_pack(obj->path_packfile, repo);
+ if (pack == NULL) {
+ err = cache_pack(&pack, obj->path_packfile, NULL, repo);
+ if (err)
+ goto done;
}
if ((obj->flags & GOT_OBJ_FLAG_DELTIFIED) == 0) {
- if (fseeko(packfile, obj->pack_offset, SEEK_SET) != 0) {
+ if (fseeko(pack->packfile, obj->pack_offset, SEEK_SET) != 0) {
err = got_error_from_errno();
goto done;
}
- err = got_inflate_to_file(&obj->size, packfile, *f);
+ err = got_inflate_to_file(&obj->size, pack->packfile, *f);
} else
err = dump_delta_chain_to_file(&obj->size, &obj->deltas, *f,
- packfile, obj->path_packfile, repo);
+ pack->packfile, obj->path_packfile, repo);
done:
- if (packfile)
- fclose(packfile);
if (err && *f)
fclose(*f);
return err;
{
const struct got_error *err = NULL;
FILE *packfile = NULL;
+ struct got_pack *pack;
if ((obj->flags & GOT_OBJ_FLAG_PACKED) == 0)
return got_error(GOT_ERR_OBJ_NOT_PACKED);
- packfile = fopen(obj->path_packfile, "rb");
- if (packfile == NULL) {
- err = got_error_from_errno();
- goto done;
+ pack = get_cached_pack(obj->path_packfile, repo);
+ if (pack == NULL) {
+ err = cache_pack(&pack, obj->path_packfile, NULL, repo);
+ if (err)
+ goto done;
}
if ((obj->flags & GOT_OBJ_FLAG_DELTIFIED) == 0) {
- if (fseeko(packfile, obj->pack_offset, SEEK_SET) != 0) {
+ if (fseeko(pack->packfile, obj->pack_offset, SEEK_SET) != 0) {
err = got_error_from_errno();
goto done;
}
- err = got_inflate_to_mem(buf, len, packfile);
+ err = got_inflate_to_mem(buf, len, pack->packfile);
} else
- err = dump_delta_chain_to_mem(buf, len, &obj->deltas, packfile,
- obj->path_packfile, repo);
+ err = dump_delta_chain_to_mem(buf, len, &obj->deltas,
+ pack->packfile, obj->path_packfile, repo);
done:
if (packfile)
fclose(packfile);
blob - c19c01392d63487dbce490ca2de4bbd2576af1d0
blob + c8d78504f16a5dffe3306c31059fb3c8ac289cab
--- lib/repository.c
+++ lib/repository.c
#include "got_repository.h"
#include "got_path_lib.h"
-#include "got_repository_lib.h"
-#include "got_zbuf_lib.h"
#include "got_delta_lib.h"
+#include "got_zbuf_lib.h"
#include "got_object_lib.h"
#include "got_pack_lib.h"
+#include "got_repository_lib.h"
#ifndef nitems
#define nitems(_a) (sizeof(_a) / sizeof((_a)[0]))
got_packidx_close(repo->packidx_cache[i]);
}
+ for (i = 0; i < nitems(repo->packs); i++) {
+ if (repo->packs[i].path_packfile == NULL)
+ break;
+ got_pack_close(&repo->packs[i]);
+ }
+
for (i = 0; i < nitems(repo->delta_cache); i++) {
struct got_delta_cache *cache = &repo->delta_cache[i];
int j;
