Commit Diff

Commit:
6fd11751dcadfc1e752c54a1f7880b006e6b3ae8
From:
Stefan Sperling <stsp@stsp.name>
Date:
Message:
turns out on-disk path of pack file may not match its hash Fixes reading a repo with a pack idx and file dated June 2011, where the filename does not contain the actual packfile SHA1.
Actions:
Patch | Tree
commit - f5feadcc2d9dc4fb3f64ed21683f11df58eb73d7
commit + 6fd11751dcadfc1e752c54a1f7880b006e6b3ae8
blob - fdc4674cc9c28b74b91e37266f1fea6402e9dc7b
blob + 6658cfd0374119b887476174a22c3e5eb671d444
--- lib/got_lib_pack.h
+++ lib/got_lib_pack.h
@@ -65,6 +65,12 @@ struct got_packidx_v2_hdr {
 	struct got_packidx_trailer trailer;
 };
 
+/* An open pack index file. */
+struct got_packidx {
+	char *path_packidx; /* actual on-disk path */
+	struct got_packidx_v2_hdr hdr;
+};
+
 struct got_packfile_hdr {
 	uint32_t	signature;
 #define GOT_PACKFILE_SIGNATURE	0x5041434b	/* 'P' 'A' 'C' 'K' */
@@ -129,9 +135,9 @@ struct got_packfile_obj_data {
 	} __attribute__((__packed__));
 } __attribute__((__packed__));
 
-const struct got_error *got_packidx_open(struct got_packidx_v2_hdr **,
+const struct got_error *got_packidx_open(struct got_packidx **,
     const char *);
-void got_packidx_close(struct got_packidx_v2_hdr *);
+void got_packidx_close(struct got_packidx *);
 
 const struct got_error *got_packfile_open_object(struct got_object **,
     struct got_object_id *, struct got_repository *);
blob - b3bd69b5b9828aeddd3b0617185c445f17027b6e
blob + 3a9c223be1d3414bc6e7fb4f23b22fd12686d95d
--- lib/got_lib_repository.h
+++ lib/got_lib_repository.h
@@ -22,7 +22,7 @@ struct got_repository {
 	char *path_git_dir;
 
 	/* The pack index cache speeds up search for packed objects. */
-	struct got_packidx_v2_hdr *packidx_cache[GOT_PACKIDX_CACHE_SIZE];
+	struct got_packidx *packidx_cache[GOT_PACKIDX_CACHE_SIZE];
 
 	/* Open file handles for pack files. */
 	struct got_pack packs[GOT_PACK_CACHE_SIZE];
blob - ce51140a9103308797ff08128ff48a30ef3a4334
blob + 53f6829b40e18097a1f4813c3ffc8cba09178e6d
--- lib/pack.c
+++ lib/pack.c
@@ -118,15 +118,17 @@ get_packfile_size(size_t *size, const char *path)
 }
 
 const struct got_error *
-got_packidx_open(struct got_packidx_v2_hdr **packidx, const char *path)
+got_packidx_open(struct got_packidx **packidx, const char *path)
 {
-	struct got_packidx_v2_hdr *p;
+	struct got_packidx *p;
 	FILE *f;
 	const struct got_error *err = NULL;
 	size_t n, nobj, packfile_size;
 	SHA1_CTX ctx;
 	uint8_t sha1[SHA1_DIGEST_LENGTH];
 
+	*packidx = NULL;
+
 	SHA1Init(&ctx);
 
 	f = fopen(path, "rb");
@@ -138,123 +140,128 @@ got_packidx_open(struct got_packidx_v2_hdr **packidx, 
 		return err;
 
 	p = calloc(1, sizeof(*p));
-	if (p == NULL) {
+	if (p == NULL)
+		return got_error_from_errno();
+	p->path_packidx = strdup(path);
+	if (p->path_packidx == NULL) {
 		err = got_error_from_errno();
-		goto done;
+		free(p->path_packidx);
+		free(p);
+		return err;
 	}
 
-	n = fread(&p->magic, sizeof(p->magic), 1, f);
+	n = fread(&p->hdr.magic, sizeof(p->hdr.magic), 1, f);
 	if (n != 1) {
 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
 		goto done;
 	}
 
-	if (betoh32(p->magic) != GOT_PACKIDX_V2_MAGIC) {
+	if (betoh32(p->hdr.magic) != GOT_PACKIDX_V2_MAGIC) {
 		err = got_error(GOT_ERR_BAD_PACKIDX);
 		goto done;
 	}
 
-	SHA1Update(&ctx, (uint8_t *)&p->magic, sizeof(p->magic));
+	SHA1Update(&ctx, (uint8_t *)&p->hdr.magic, sizeof(p->hdr.magic));
 
-	n = fread(&p->version, sizeof(p->version), 1, f);
+	n = fread(&p->hdr.version, sizeof(p->hdr.version), 1, f);
 	if (n != 1) {
 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
 		goto done;
 	}
 
-	if (betoh32(p->version) != GOT_PACKIDX_VERSION) {
+	if (betoh32(p->hdr.version) != GOT_PACKIDX_VERSION) {
 		err = got_error(GOT_ERR_BAD_PACKIDX);
 		goto done;
 	}
 
-	SHA1Update(&ctx, (uint8_t *)&p->version, sizeof(p->version));
+	SHA1Update(&ctx, (uint8_t *)&p->hdr.version, sizeof(p->hdr.version));
 
-	n = fread(&p->fanout_table, sizeof(p->fanout_table), 1, f);
+	n = fread(&p->hdr.fanout_table, sizeof(p->hdr.fanout_table), 1, f);
 	if (n != 1) {
 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
 		goto done;
 	}
 
-	err = verify_fanout_table(p->fanout_table);
+	err = verify_fanout_table(p->hdr.fanout_table);
 	if (err)
 		goto done;
 
-	SHA1Update(&ctx, (uint8_t *)p->fanout_table, sizeof(p->fanout_table));
+	SHA1Update(&ctx, (uint8_t *)p->hdr.fanout_table, sizeof(p->hdr.fanout_table));
 
-	nobj = betoh32(p->fanout_table[0xff]);
+	nobj = betoh32(p->hdr.fanout_table[0xff]);
 
-	p->sorted_ids = calloc(nobj, sizeof(*p->sorted_ids));
-	if (p->sorted_ids == NULL) {
+	p->hdr.sorted_ids = calloc(nobj, sizeof(*p->hdr.sorted_ids));
+	if (p->hdr.sorted_ids == NULL) {
 		err = got_error_from_errno();
 		goto done;
 	}
 
-	n = fread(p->sorted_ids, sizeof(*p->sorted_ids), nobj, f);
+	n = fread(p->hdr.sorted_ids, sizeof(*p->hdr.sorted_ids), nobj, f);
 	if (n != nobj) {
 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
 		goto done;
 	}
 
-	SHA1Update(&ctx, (uint8_t *)p->sorted_ids,
-	    nobj * sizeof(*p->sorted_ids));
+	SHA1Update(&ctx, (uint8_t *)p->hdr.sorted_ids,
+	    nobj * sizeof(*p->hdr.sorted_ids));
 
-	p->crc32 = calloc(nobj, sizeof(*p->crc32));
-	if (p->crc32 == NULL) {
+	p->hdr.crc32 = calloc(nobj, sizeof(*p->hdr.crc32));
+	if (p->hdr.crc32 == NULL) {
 		err = got_error_from_errno();
 		goto done;
 	}
 
-	n = fread(p->crc32, sizeof(*p->crc32), nobj, f);
+	n = fread(p->hdr.crc32, sizeof(*p->hdr.crc32), nobj, f);
 	if (n != nobj) {
 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
 		goto done;
 	}
 
-	SHA1Update(&ctx, (uint8_t *)p->crc32, nobj * sizeof(*p->crc32));
+	SHA1Update(&ctx, (uint8_t *)p->hdr.crc32, nobj * sizeof(*p->hdr.crc32));
 
-	p->offsets = calloc(nobj, sizeof(*p->offsets));
-	if (p->offsets == NULL) {
+	p->hdr.offsets = calloc(nobj, sizeof(*p->hdr.offsets));
+	if (p->hdr.offsets == NULL) {
 		err = got_error_from_errno();
 		goto done;
 	}
 
-	n = fread(p->offsets, sizeof(*p->offsets), nobj, f);
+	n = fread(p->hdr.offsets, sizeof(*p->hdr.offsets), nobj, f);
 	if (n != nobj) {
 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
 		goto done;
 	}
 
-	SHA1Update(&ctx, (uint8_t *)p->offsets, nobj * sizeof(*p->offsets));
+	SHA1Update(&ctx, (uint8_t *)p->hdr.offsets, nobj * sizeof(*p->hdr.offsets));
 
 	/* Large file offsets are contained only in files > 2GB. */
 	if (packfile_size <= 0x80000000)
 		goto checksum;
 
-	p->large_offsets = calloc(nobj, sizeof(*p->large_offsets));
-	if (p->large_offsets == NULL) {
+	p->hdr.large_offsets = calloc(nobj, sizeof(*p->hdr.large_offsets));
+	if (p->hdr.large_offsets == NULL) {
 		err = got_error_from_errno();
 		goto done;
 	}
 
-	n = fread(p->large_offsets, sizeof(*p->large_offsets), nobj, f);
+	n = fread(p->hdr.large_offsets, sizeof(*p->hdr.large_offsets), nobj, f);
 	if (n != nobj) {
 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
 		goto done;
 	}
 
-	SHA1Update(&ctx, (uint8_t*)p->large_offsets,
-	    nobj * sizeof(*p->large_offsets));
+	SHA1Update(&ctx, (uint8_t*)p->hdr.large_offsets,
+	    nobj * sizeof(*p->hdr.large_offsets));
 
 checksum:
-	n = fread(&p->trailer, sizeof(p->trailer), 1, f);
+	n = fread(&p->hdr.trailer, sizeof(p->hdr.trailer), 1, f);
 	if (n != 1) {
 		err = got_ferror(f, GOT_ERR_BAD_PACKIDX);
 		goto done;
 	}
 
-	SHA1Update(&ctx, p->trailer.packfile_sha1, SHA1_DIGEST_LENGTH);
+	SHA1Update(&ctx, p->hdr.trailer.packfile_sha1, SHA1_DIGEST_LENGTH);
 	SHA1Final(sha1, &ctx);
-	if (memcmp(p->trailer.packidx_sha1, sha1, SHA1_DIGEST_LENGTH) != 0)
+	if (memcmp(p->hdr.trailer.packidx_sha1, sha1, SHA1_DIGEST_LENGTH) != 0)
 		err = got_error(GOT_ERR_PACKIDX_CSUM);
 done:
 	fclose(f);
@@ -266,12 +273,13 @@ done:
 }
 
 void
-got_packidx_close(struct got_packidx_v2_hdr *packidx)
+got_packidx_close(struct got_packidx *packidx)
 {
-	free(packidx->sorted_ids);
-	free(packidx->offsets);
-	free(packidx->crc32);
-	free(packidx->large_offsets);
+	free(packidx->hdr.sorted_ids);
+	free(packidx->hdr.offsets);
+	free(packidx->hdr.crc32);
+	free(packidx->hdr.large_offsets);
+	free(packidx->path_packidx);
 	free(packidx);
 }
 
@@ -292,16 +300,17 @@ is_packidx_filename(const char *name, size_t len)
 }
 
 static off_t
-get_object_offset(struct got_packidx_v2_hdr *packidx, int idx)
+get_object_offset(struct got_packidx *packidx, int idx)
 {
-	uint32_t totobj = betoh32(packidx->fanout_table[0xff]);
-	uint32_t offset = betoh32(packidx->offsets[idx]);
+	uint32_t totobj = betoh32(packidx->hdr.fanout_table[0xff]);
+	uint32_t offset = betoh32(packidx->hdr.offsets[idx]);
 	if (offset & GOT_PACKIDX_OFFSET_VAL_IS_LARGE_IDX) {
 		uint64_t loffset;
 		idx = offset & GOT_PACKIDX_OFFSET_VAL_MASK;
-		if (idx < 0 || idx > totobj || packidx->large_offsets == NULL)
+		if (idx < 0 || idx > totobj ||
+		    packidx->hdr.large_offsets == NULL)
 			return -1;
-		loffset = betoh64(packidx->large_offsets[idx]);
+		loffset = betoh64(packidx->hdr.large_offsets[idx]);
 		return (loffset > INT64_MAX ? -1 : (off_t)loffset);
 	}
 	return (off_t)(offset & GOT_PACKIDX_OFFSET_VAL_MASK);
@@ -309,20 +318,21 @@ get_object_offset(struct got_packidx_v2_hdr *packidx, 
 
 static const struct got_error *
 get_packfile_path(char **path_packfile, struct got_repository *repo,
-    struct got_packidx_v2_hdr *packidx);
+    struct got_packidx *packidx);
 
 static int
-get_object_idx(struct got_packidx_v2_hdr *packidx, struct got_object_id *id, struct got_repository *repo)
+get_object_idx(struct got_packidx *packidx, struct got_object_id *id,
+    struct got_repository *repo)
 {
 	u_int8_t id0 = id->sha1[0];
-	uint32_t totobj = betoh32(packidx->fanout_table[0xff]);
+	uint32_t totobj = betoh32(packidx->hdr.fanout_table[0xff]);
 	int i = 0;
 
 	if (id0 > 0)
-		i = betoh32(packidx->fanout_table[id0 - 1]);
+		i = betoh32(packidx->hdr.fanout_table[id0 - 1]);
 
 	while (i < totobj) {
-		struct got_object_id *oid = &packidx->sorted_ids[i];
+		struct got_object_id *oid = &packidx->hdr.sorted_ids[i];
 		int cmp = got_object_id_cmp(id, oid);
 
 		if (cmp == 0)
@@ -333,61 +343,68 @@ get_object_idx(struct got_packidx_v2_hdr *packidx, str
 	return -1;
 }
 
-static struct got_packidx_v2_hdr *
-dup_packidx(struct got_packidx_v2_hdr *packidx)
+static struct got_packidx *
+dup_packidx(struct got_packidx *packidx)
 {
-	struct got_packidx_v2_hdr *p;
+	struct got_packidx *p;
 	size_t nobj;
 
 	p = calloc(1, sizeof(*p));
 	if (p == NULL)
 		return NULL;
 
-	memcpy(p, packidx, sizeof(*p));
-	p->sorted_ids = NULL;
-	p->crc32 = NULL;
-	p->offsets = NULL;
-	p->large_offsets = NULL;
-
-	nobj = betoh32(p->fanout_table[0xff]);
-
-	p->sorted_ids = calloc(nobj, sizeof(*p->sorted_ids));
-	if (p->sorted_ids == NULL)
+	p->path_packidx = strdup(packidx->path_packidx);
+	if (p->path_packidx == NULL) {
+		free(p);
+		return NULL;
+	}
+	memcpy(&p->hdr, &packidx->hdr, sizeof(p->hdr));
+	p->hdr.sorted_ids = NULL;
+	p->hdr.crc32 = NULL;
+	p->hdr.offsets = NULL;
+	p->hdr.large_offsets = NULL;
+
+	nobj = betoh32(p->hdr.fanout_table[0xff]);
+
+	p->hdr.sorted_ids = calloc(nobj, sizeof(*p->hdr.sorted_ids));
+	if (p->hdr.sorted_ids == NULL)
 		goto err;
-	memcpy(p->sorted_ids, packidx->sorted_ids,
-	    nobj * sizeof(*p->sorted_ids));
+	memcpy(p->hdr.sorted_ids, packidx->hdr.sorted_ids,
+	    nobj * sizeof(*p->hdr.sorted_ids));
 
-	p->crc32 = calloc(nobj, sizeof(*p->crc32));
-	if (p->crc32 == NULL)
+	p->hdr.crc32 = calloc(nobj, sizeof(*p->hdr.crc32));
+	if (p->hdr.crc32 == NULL)
 		goto err;
-	memcpy(p->crc32, packidx->crc32, nobj * sizeof(*p->crc32));
+	memcpy(p->hdr.crc32, packidx->hdr.crc32, nobj * sizeof(*p->hdr.crc32));
 
-	p->offsets = calloc(nobj, sizeof(*p->offsets));
-	if (p->offsets == NULL)
+	p->hdr.offsets = calloc(nobj, sizeof(*p->hdr.offsets));
+	if (p->hdr.offsets == NULL)
 		goto err;
-	memcpy(p->offsets, packidx->offsets, nobj * sizeof(*p->offsets));
+	memcpy(p->hdr.offsets, packidx->hdr.offsets,
+	    nobj * sizeof(*p->hdr.offsets));
 
-	if (p->large_offsets) {
-		p->large_offsets = calloc(nobj, sizeof(*p->large_offsets));
-		if (p->large_offsets == NULL)
+	if (p->hdr.large_offsets) {
+		p->hdr.large_offsets = calloc(nobj, sizeof(*p->hdr.large_offsets));
+		if (p->hdr.large_offsets == NULL)
 			goto err;
-		memcpy(p->large_offsets, packidx->large_offsets,
-		    nobj * sizeof(*p->large_offsets));
+		memcpy(p->hdr.large_offsets, packidx->hdr.large_offsets,
+		    nobj * sizeof(*p->hdr.large_offsets));
 	}
 
 	return p;
 
 err:
-	free(p->large_offsets);
-	free(p->offsets);
-	free(p->crc32);
-	free(p->sorted_ids);
+	free(p->hdr.large_offsets);
+	free(p->hdr.offsets);
+	free(p->hdr.crc32);
+	free(p->hdr.sorted_ids);
+	free(p->path_packidx);
 	free(p);
 	return NULL;
 }
 
 static void
-cache_packidx(struct got_packidx_v2_hdr *packidx, struct got_repository *repo)
+cache_packidx(struct got_packidx *packidx, struct got_repository *repo)
 {
 	int i;
 
@@ -408,7 +425,7 @@ cache_packidx(struct got_packidx_v2_hdr *packidx, stru
 }
 
 static const struct got_error *
-search_packidx(struct got_packidx_v2_hdr **packidx, int *idx,
+search_packidx(struct got_packidx **packidx, int *idx,
     struct got_repository *repo, struct got_object_id *id)
 {
 	const struct got_error *err;
@@ -476,35 +493,33 @@ done:
 
 static const struct got_error *
 get_packfile_path(char **path_packfile, struct got_repository *repo,
-    struct got_packidx_v2_hdr *packidx)
+    struct got_packidx *packidx)
 {
-	char *path_packdir;
-	char hex[SHA1_DIGEST_STRING_LENGTH];
-	char *sha1str;
+	size_t size;
 
-	path_packdir = got_repo_get_path_objects_pack(repo);
-	if (path_packdir == NULL)
+	/* Packfile path contains ".pack" instead of ".idx", so add one byte. */
+	size = strlen(packidx->path_packidx) + 2;
+	if (size < GOT_PACKFILE_NAMELEN + 1)
+		return got_error(GOT_ERR_BAD_PATH);
+
+	*path_packfile = calloc(size, sizeof(**path_packfile));
+	if (*path_packfile == NULL)
 		return got_error_from_errno();
 
-	sha1str = got_sha1_digest_to_str(packidx->trailer.packfile_sha1,
-	    hex, sizeof(hex));
-	if (sha1str == NULL)
-		return got_error(GOT_ERR_PACKIDX_CSUM);
+	/* Copy up to and excluding ".idx". */
+	strncpy(*path_packfile, packidx->path_packidx, size - strlen(".idx") - 2);
 
-	if (asprintf(path_packfile, "%s/%s%s%s", path_packdir,
-	    GOT_PACK_PREFIX, sha1str, GOT_PACKFILE_SUFFIX) == -1) {
-		*path_packfile = NULL;
-		return got_error_from_errno();
-	}
+	if (strlcat(*path_packfile, GOT_PACKFILE_SUFFIX, size) >= size)
+		return got_error(GOT_ERR_NO_SPACE);
 
 	return NULL;
 }
 
 const struct got_error *
-read_packfile_hdr(int fd, struct got_packidx_v2_hdr *packidx)
+read_packfile_hdr(int fd, struct got_packidx *packidx)
 {
 	const struct got_error *err = NULL;
-	uint32_t totobj = betoh32(packidx->fanout_table[0xff]);
+	uint32_t totobj = betoh32(packidx->hdr.fanout_table[0xff]);
 	struct got_packfile_hdr hdr;
 	ssize_t n;
 
@@ -524,7 +539,7 @@ read_packfile_hdr(int fd, struct got_packidx_v2_hdr *p
 
 static const struct got_error *
 open_packfile(int *fd, const char *path_packfile,
-    struct got_repository *repo, struct got_packidx_v2_hdr *packidx)
+    struct got_repository *repo, struct got_packidx *packidx)
 {
 	const struct got_error *err = NULL;
 
@@ -554,7 +569,7 @@ got_pack_close(struct got_pack *pack)
 
 static const struct got_error *
 cache_pack(struct got_pack **packp, const char *path_packfile,
-    struct got_packidx_v2_hdr *packidx, struct got_repository *repo)
+    struct got_packidx *packidx, struct got_repository *repo)
 {
 	const struct got_error *err = NULL;
 	struct got_pack *pack = NULL;
@@ -813,7 +828,7 @@ resolve_ref_delta(struct got_delta_chain *deltas, stru
 {
 	const struct got_error *err;
 	struct got_object_id id;
-	struct got_packidx_v2_hdr *packidx;
+	struct got_packidx *packidx;
 	int idx;
 	off_t base_offset;
 	uint8_t base_type;
@@ -924,9 +939,9 @@ resolve_delta_chain(struct got_delta_chain *deltas, st
 
 static const struct got_error *
 open_delta_object(struct got_object **obj, struct got_repository *repo,
-    struct got_packidx_v2_hdr *packidx, const char *path_packfile,
-    int fd, size_t packfile_size, struct got_object_id *id, off_t offset,
-    size_t tslen, int delta_type, size_t delta_size)
+    const char *path_packfile, int fd, size_t packfile_size,
+    struct got_object_id *id, off_t offset, size_t tslen,
+    int delta_type, size_t delta_size)
 {
 	const struct got_error *err = NULL;
 	int resolved_type;
@@ -972,7 +987,7 @@ done:
 
 static const struct got_error *
 open_packed_object(struct got_object **obj, struct got_repository *repo,
-    struct got_packidx_v2_hdr *packidx, int idx, struct got_object_id *id)
+    struct got_packidx *packidx, int idx, struct got_object_id *id)
 {
 	const struct got_error *err = NULL;
 	off_t offset;
@@ -1023,8 +1038,8 @@ open_packed_object(struct got_object **obj, struct got
 
 	case GOT_OBJ_TYPE_OFFSET_DELTA:
 	case GOT_OBJ_TYPE_REF_DELTA:
-		err = open_delta_object(obj, repo, packidx, path_packfile,
-		    pack->fd, pack->filesize, id, offset, tslen, type, size);
+		err = open_delta_object(obj, repo, path_packfile, pack->fd,
+		    pack->filesize, id, offset, tslen, type, size);
 		break;
 
 	default:
@@ -1041,7 +1056,7 @@ got_packfile_open_object(struct got_object **obj, stru
     struct got_repository *repo)
 {
 	const struct got_error *err = NULL;
-	struct got_packidx_v2_hdr *packidx = NULL;
+	struct got_packidx *packidx = NULL;
 	int idx;
 
 	err = search_packidx(&packidx, &idx, repo, id);