commit fc79a48d6d45b863032ca7d3a161781cc8e89f95
from: Stefan Sperling <stsp@stsp.name>
date: Mon Jul 09 15:00:51 2018 UTC

add a compile time switch to read pack index with read(2)

commit - af9b7fee80c56b28b68b06d4452580ec51066242
commit + fc79a48d6d45b863032ca7d3a161781cc8e89f95
blob - 8c76a73674eff72a0574331795351df57050510c
blob + 2563d2de603cf0699d20c0a5f5a4144c115d5bf7
--- got/Makefile
+++ got/Makefile
@@ -17,6 +17,7 @@ DEBUG = -O0 -pg
 DEBUG = -O0 -g
 .endif
 CFLAGS += -Werror -Wall -Wstrict-prototypes -Wunused-variable
+#CPLAGS += -DGOT_PACK_NO_MMAP
 
 # For now, default to installing binary in ~/bin
 GROUP!=id -g -n
blob - 856b6608a054f295377b1a15f34c5394f9f77c72
blob + a603e1a81cf8e0bd501567e926adb3470545bec8
--- lib/pack.c
+++ lib/pack.c
@@ -125,6 +125,7 @@ got_packidx_open(struct got_packidx **packidx, const c
 	struct got_packidx_v2_hdr *h;
 	const struct got_error *err = NULL;
 	size_t nobj, len_fanout, len_ids, offset, remain;
+	ssize_t n;
 	SHA1_CTX ctx;
 	uint8_t sha1[SHA1_DIGEST_LENGTH];
 
@@ -159,11 +160,12 @@ got_packidx_open(struct got_packidx **packidx, const c
 		goto done;
 	}
 
+#ifndef GOT_PACK_NO_MMAP
 	p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0);
-	if (p->map == MAP_FAILED) {
-		err = got_error_from_errno();
-		goto done;
-	}
+	if (p->map == MAP_FAILED)
+		p->map = NULL; /* fall back to read(2) */
+#endif
+
 	h = &p->hdr;
 	offset = 0;
 	remain = p->len;
@@ -172,7 +174,20 @@ got_packidx_open(struct got_packidx **packidx, const c
 		err = got_error(GOT_ERR_BAD_PACKIDX);
 		goto done;
 	}
-	h->magic = (uint32_t *)(p->map + offset);
+	if (p->map)
+		h->magic = (uint32_t *)(p->map + offset);
+	else {
+		h->magic = malloc(sizeof(*h->magic));
+		if (h->magic == NULL) {
+			err = got_error_from_errno();
+			goto done;
+		}
+		n = read(p->fd, h->magic, sizeof(*h->magic));
+		if (n != sizeof(*h->magic)) {
+			err = got_error(GOT_ERR_BAD_PACKIDX);
+			goto done;
+		}
+	}
 	if (betoh32(*h->magic) != GOT_PACKIDX_V2_MAGIC) {
 		err = got_error(GOT_ERR_BAD_PACKIDX);
 		goto done;
@@ -187,7 +202,20 @@ got_packidx_open(struct got_packidx **packidx, const c
 		err = got_error(GOT_ERR_BAD_PACKIDX);
 		goto done;
 	}
-	h->version = (uint32_t *)(p->map + offset);
+	if (p->map)
+		h->version = (uint32_t *)(p->map + offset);
+	else {
+		h->version = malloc(sizeof(*h->version));
+		if (h->version == NULL) {
+			err = got_error_from_errno();
+			goto done;
+		}
+		n = read(p->fd, h->version, sizeof(*h->version));
+		if (n != sizeof(*h->version)) {
+			err = got_error(GOT_ERR_BAD_PACKIDX);
+			goto done;
+		}
+	}
 	if (betoh32(*h->version) != GOT_PACKIDX_VERSION) {
 		err = got_error(GOT_ERR_BAD_PACKIDX);
 		goto done;
@@ -204,7 +232,20 @@ got_packidx_open(struct got_packidx **packidx, const c
 		err = got_error(GOT_ERR_BAD_PACKIDX);
 		goto done;
 	}
-	h->fanout_table = (uint32_t *)(p->map + offset);
+	if (p->map)
+		h->fanout_table = (uint32_t *)(p->map + offset);
+	else {
+		h->fanout_table = malloc(len_fanout);
+		if (h->fanout_table == NULL) {
+			err = got_error_from_errno();
+			goto done;
+		}
+		n = read(p->fd, h->fanout_table, len_fanout);
+		if (n != len_fanout) {
+			err = got_error(GOT_ERR_BAD_PACKIDX);
+			goto done;
+		}
+	}
 	err = verify_fanout_table(h->fanout_table);
 	if (err)
 		goto done;
@@ -219,8 +260,21 @@ got_packidx_open(struct got_packidx **packidx, const c
 		err = got_error(GOT_ERR_BAD_PACKIDX);
 		goto done;
 	}
-	h->sorted_ids =
-	    (struct got_packidx_object_id *)((uint8_t*)(p->map + offset));
+	if (p->map)
+		h->sorted_ids =
+		    (struct got_packidx_object_id *)((uint8_t*)(p->map + offset));
+	else {
+		h->sorted_ids = malloc(len_ids);
+		if (h->sorted_ids == NULL) {
+			err = got_error(GOT_ERR_BAD_PACKIDX);
+			goto done;
+		}
+		n = read(p->fd, h->sorted_ids, len_ids);
+		if (n != len_ids) {
+			err = got_error(GOT_ERR_BAD_PACKIDX);
+			goto done;
+		}
+	}
 	if (verify)
 		SHA1Update(&ctx, (uint8_t *)h->sorted_ids, len_ids);
 	offset += len_ids;
@@ -230,7 +284,20 @@ got_packidx_open(struct got_packidx **packidx, const c
 		err = got_error(GOT_ERR_BAD_PACKIDX);
 		goto done;
 	}
-	h->crc32 = (uint32_t *)((uint8_t*)(p->map + offset));
+	if (p->map)
+		h->crc32 = (uint32_t *)((uint8_t*)(p->map + offset));
+	else {
+		h->crc32 = malloc(nobj * sizeof(*h->crc32));
+		if (h->crc32 == NULL) {
+			err = got_error_from_errno();
+			goto done;
+		}
+		n = read(p->fd, h->crc32, nobj * sizeof(*h->crc32));
+		if (n != nobj * sizeof(*h->crc32)) {
+			err = got_error(GOT_ERR_BAD_PACKIDX);
+			goto done;
+		}
+	}
 	if (verify)
 		SHA1Update(&ctx, (uint8_t *)h->crc32, nobj * sizeof(*h->crc32));
 	remain -= nobj * sizeof(*h->crc32);
@@ -240,7 +307,20 @@ got_packidx_open(struct got_packidx **packidx, const c
 		err = got_error(GOT_ERR_BAD_PACKIDX);
 		goto done;
 	}
-	h->offsets = (uint32_t *)((uint8_t*)(p->map + offset));
+	if (p->map)
+		h->offsets = (uint32_t *)((uint8_t*)(p->map + offset));
+	else {
+		h->offsets = malloc(nobj * sizeof(*h->offsets));
+		if (h->offsets == NULL) {
+			err = got_error_from_errno();
+			goto done;
+		}
+		n = read(p->fd, h->offsets, nobj * sizeof(*h->offsets));
+		if (n != nobj * sizeof(*h->offsets)) {
+			err = got_error(GOT_ERR_BAD_PACKIDX);
+			goto done;
+		}
+	}
 	if (verify)
 		SHA1Update(&ctx, (uint8_t *)h->offsets,
 		    nobj * sizeof(*h->offsets));
@@ -255,7 +335,21 @@ got_packidx_open(struct got_packidx **packidx, const c
 		err = got_error(GOT_ERR_BAD_PACKIDX);
 		goto done;
 	}
-	h->large_offsets = (uint64_t *)((uint8_t*)(p->map + offset));
+	if (p->map)
+		h->large_offsets = (uint64_t *)((uint8_t*)(p->map + offset));
+	else {
+		h->offsets = malloc(nobj * sizeof(*h->large_offsets));
+		if (h->offsets == NULL) {
+			err = got_error_from_errno();
+			goto done;
+		}
+		n = read(p->fd, h->large_offsets,
+		    nobj * sizeof(*h->large_offsets));
+		if (n != nobj * sizeof(*h->large_offsets)) {
+			err = got_error(GOT_ERR_BAD_PACKIDX);
+			goto done;
+		}
+	}
 	if (verify)
 		SHA1Update(&ctx, (uint8_t*)h->large_offsets,
 		    nobj * sizeof(*h->large_offsets));
@@ -267,8 +361,21 @@ checksum:
 		err = got_error(GOT_ERR_BAD_PACKIDX);
 		goto done;
 	}
-	h->trailer =
-	    (struct got_packidx_trailer *)((uint8_t*)(p->map + offset));
+	if (p->map)
+		h->trailer =
+		    (struct got_packidx_trailer *)((uint8_t*)(p->map + offset));
+	else {
+		h->trailer = malloc(sizeof(*h->trailer));
+		if (h->trailer == NULL) {
+			err = got_error_from_errno();
+			goto done;
+		}
+		n = read(p->fd, h->trailer, sizeof(*h->trailer));
+		if (n != sizeof(*h->trailer)) {
+			err = got_error(GOT_ERR_BAD_PACKIDX);
+			goto done;
+		}
+	}
 	if (verify) {
 		SHA1Update(&ctx, h->trailer->packfile_sha1, SHA1_DIGEST_LENGTH);
 		SHA1Final(sha1, &ctx);
@@ -290,9 +397,18 @@ got_packidx_close(struct got_packidx *packidx)
 	const struct got_error *err = NULL;
 
 	free(packidx->path_packidx);
-	if (packidx->map != NULL && packidx->map != MAP_FAILED) {
+	if (packidx->map) {
 		if (munmap(packidx->map, packidx->len) == -1)
 			err = got_error_from_errno();
+	} else {
+		free(packidx->hdr.magic);
+		free(packidx->hdr.version);
+		free(packidx->hdr.fanout_table);
+		free(packidx->hdr.sorted_ids);
+		free(packidx->hdr.crc32);
+		free(packidx->hdr.offsets);
+		free(packidx->hdr.large_offsets);
+		free(packidx->hdr.trailer);
 	}
 	close(packidx->fd);
 	free(packidx);
blob - c8f2d7e9d3b04208321ffaa6a29020d7b147e132
blob + fff00c3507ab78c3b9aab9cca4628b60b28c94a3
--- regress/repository/Makefile
+++ regress/repository/Makefile
@@ -9,6 +9,7 @@ CPPFLAGS = -I${.CURDIR}/../../include -I${.CURDIR}/../
 LDADD = -lutil -lz
 DEBUG = -O0 -g
 CFLAGS += -Werror -Wall -Wstrict-prototypes -Wunused-variable
+#CFLAGS += -DGOT_PACK_NO_MMAP
 
 NOMAN = yes
 
blob - 4c90c64fc7ae7d0c6e12a7cfcd6aae6e59a22245
blob + 699a6f2edc3886636e3055d238df66eaf6f338fa
--- regress/worktree/Makefile
+++ regress/worktree/Makefile
@@ -9,6 +9,7 @@ CPPFLAGS = -I${.CURDIR}/../../include -I${.CURDIR}/../
 LDADD = -lutil -lz
 DEBUG = -O0 -g
 CFLAGS += -Werror -Wall -Wstrict-prototypes -Wunused-variable
+#CFLAGS += -DGOT_PACK_NO_MMAP
 
 NOMAN = yes
 
blob - f3f595818068357c40dd403d4732cde1697a36dc
blob + 636134505907c48c61f1d68a1f39a814ef6e14a6
--- tog/Makefile
+++ tog/Makefile
@@ -17,6 +17,7 @@ DEBUG = -O0 -pg
 DEBUG = -O0 -g
 .endif
 CFLAGS += -Werror -Wall -Wstrict-prototypes -Wunused-variable
+#CPLAGS += -DGOT_PACK_NO_MMAP
 
 # For now, default to installing binary in ~/bin
 GROUP!=id -g -n