commit f57598a25f07ea6b68f64ef9369142b9c056eb05
from: Omar Polo <op@omarpolo.com>
date: Sun Feb 12 14:09:59 2023 UTC

handle both sha1 and sha256 object ids in pack files

bump got_packidx_object_id so it can hold a sha256 digest and adapt
the code to read the needed type of hash.

commit - d6720956975e97012cad5147fc6fa9b599a72b1b
commit + f57598a25f07ea6b68f64ef9369142b9c056eb05
blob - 92c6f66d7d519ba568fc92f2cfcd067988da49b6
blob + 9e0f5450c83d862479c43c0409e69c88745d5341
--- lib/got_lib_pack.h
+++ lib/got_lib_pack.h
@@ -69,7 +69,7 @@ struct got_packidx_trailer {
 } __attribute__((__packed__));
 
 struct got_packidx_object_id {
-	u_int8_t hash[SHA1_DIGEST_LENGTH];
+	u_int8_t hash[GOT_OBJECT_ID_MAXLEN];
 } __attribute__((__packed__));
 
 /* Ignore pack index version 1 which is no longer written by Git. */
@@ -91,8 +91,11 @@ struct got_packidx_v2_hdr {
 	uint32_t	*fanout_table;	/* values are big endian */
 #define GOT_PACKIDX_V2_FANOUT_TABLE_ITEMS (0xff + 1)
 
-	/* Sorted SHA1 checksums for each object in the pack file. */
-	struct got_packidx_object_id *sorted_ids;
+	/*
+	 * Sorted hash checksums for each object in the pack file.
+	 * Exact size depends on the repository object format.
+	 */
+	void *sorted_ids;
 
 	/* CRC32 of the packed representation of each object. */
 	uint32_t	*crc32;
blob - 49053c6e02a0f77183b4f81d91af1743137e194a
blob + b67ad40ec2c31fe24e90f390d0eb3ffb1f047d5b
--- lib/pack.c
+++ lib/pack.c
@@ -182,14 +182,13 @@ got_packidx_init_hdr(struct got_packidx *p, int verify
 	remain -= len_fanout;
 
 	nobj = be32toh(h->fanout_table[0xff]);
-	len_ids = nobj * sizeof(*h->sorted_ids);
+	len_ids = nobj * got_hash_digest_length(algo);
 	if (len_ids <= nobj || len_ids > remain) {
 		err = got_error(GOT_ERR_BAD_PACKIDX);
 		goto done;
 	}
 	if (p->map)
-		h->sorted_ids =
-		    (struct got_packidx_object_id *)((uint8_t*)(p->map + offset));
+		h->sorted_ids = p->map + offset;
 	else {
 		h->sorted_ids = malloc(len_ids);
 		if (h->sorted_ids == NULL) {
@@ -492,6 +491,9 @@ got_packidx_get_object_idx(struct got_packidx *packidx
 	u_int8_t id0 = id->hash[0];
 	uint32_t totobj = be32toh(packidx->hdr.fanout_table[0xff]);
 	int left = 0, right = totobj - 1;
+	size_t idlen;
+
+	idlen = got_hash_digest_length(GOT_HASH_SHA1);
 
 	if (id0 > 0)
 		left = be32toh(packidx->hdr.fanout_table[id0 - 1]);
@@ -501,8 +503,8 @@ got_packidx_get_object_idx(struct got_packidx *packidx
 		int i, cmp;
 
 		i = ((left + right) / 2);
-		oid = &packidx->hdr.sorted_ids[i];
-		cmp = memcmp(id->hash, oid->hash, SHA1_DIGEST_LENGTH);
+		oid = packidx->hdr.sorted_ids + idlen * i;
+		cmp = memcmp(id->hash, oid->hash, idlen);
 		if (cmp == 0)
 			return i;
 		else if (cmp > 0)
@@ -652,13 +654,17 @@ got_packidx_get_object_id(struct got_object_id *id,
     struct got_packidx *packidx, int idx)
 {
 	uint32_t totobj = be32toh(packidx->hdr.fanout_table[0xff]);
+	enum got_hash_algorithm algo = GOT_HASH_SHA1;
 	struct got_packidx_object_id *oid;
+	size_t idlen;
 
+	idlen = got_hash_digest_length(algo);
+
 	if (idx < 0 || idx >= totobj)
 		return got_error(GOT_ERR_NO_OBJ);
 
-	oid = &packidx->hdr.sorted_ids[idx];
-	memcpy(id->hash, oid->hash, SHA1_DIGEST_LENGTH);
+	oid = packidx->hdr.sorted_ids + (idx * idlen);
+	memcpy(id->hash, oid->hash, idlen);
 	return NULL;
 }
 
@@ -671,10 +677,12 @@ got_packidx_match_id_str_prefix(struct got_object_id_q
 	u_int8_t id0;
 	uint32_t totobj = be32toh(packidx->hdr.fanout_table[0xff]);
 	char hex[3];
-	size_t prefix_len = strlen(id_str_prefix);
+	size_t idlen, prefix_len = strlen(id_str_prefix);
 	struct got_packidx_object_id *oid;
 	uint32_t i = 0;
 
+	idlen = got_hash_digest_length(algo);
+
 	STAILQ_INIT(matched_ids);
 
 	if (prefix_len < 2)
@@ -688,7 +696,7 @@ got_packidx_match_id_str_prefix(struct got_object_id_q
 
 	if (id0 > 0)
 		i = be32toh(packidx->hdr.fanout_table[id0 - 1]);
-	oid = &packidx->hdr.sorted_ids[i];
+	oid = packidx->hdr.sorted_ids + i * idlen;
 	while (i < totobj && oid->hash[0] == id0) {
 		char id_str[GOT_OBJECT_ID_HEX_MAXLEN];
 		struct got_object_qid *qid;
@@ -708,10 +716,11 @@ got_packidx_match_id_str_prefix(struct got_object_id_q
 		err = got_object_qid_alloc_partial(&qid);
 		if (err)
 			break;
-		memcpy(qid->id.hash, oid->hash, SHA1_DIGEST_LENGTH);
+		memcpy(qid->id.hash, oid->hash, idlen);
 		STAILQ_INSERT_TAIL(matched_ids, qid, entry);
 
-		oid = &packidx->hdr.sorted_ids[++i];
+		i++;
+		oid = packidx->hdr.sorted_ids + i * idlen;
 	}
 
 	if (err)
blob - 5870c50d33ebc831012351325d3c1f891cc564e6
blob + 02bee8a7044af6efd5c1837b240e0941d9a1549a
--- lib/pack_index.c
+++ lib/pack_index.c
@@ -474,7 +474,10 @@ find_object_idx(struct got_packidx *packidx, uint8_t *
 	uint32_t nindexed = be32toh(packidx->hdr.fanout_table[0xff]);
 	int left = 0, right = nindexed - 1;
 	int cmp = 0, i = 0;
+	size_t idlen;
 
+	idlen = got_hash_digest_length(GOT_HASH_SHA1);
+
 	if (id0 > 0)
 		left = be32toh(packidx->hdr.fanout_table[id0 - 1]);
 
@@ -482,9 +485,9 @@ find_object_idx(struct got_packidx *packidx, uint8_t *
 		struct got_packidx_object_id *oid;
 
 		i = ((left + right) / 2);
-		oid = &packidx->hdr.sorted_ids[i];
+		oid = packidx->hdr.sorted_ids + idlen * i;
 
-		cmp = memcmp(sha1, oid->hash, SHA1_DIGEST_LENGTH);
+		cmp = memcmp(sha1, oid->hash, idlen);
 		if (cmp == 0)
 			return -1; /* object already indexed */
 		else if (cmp > 0)
@@ -501,13 +504,19 @@ static void
 print_packidx(struct got_packidx *packidx)
 {
 	uint32_t nindexed = be32toh(packidx->hdr.fanout_table[0xff]);
+	enum got_hash_algorithm algo = GOT_HASH_SHA1;
+	struct got_packidx_object_id *oid;
+	size_t idlen;
 	int i;
 
+	idlen = got_hash_digest_length(algo);
+
 	fprintf(stderr, "object IDs:\n");
 	for (i = 0; i < nindexed; i++) {
-		char hex[SHA1_DIGEST_STRING_LENGTH];
-		got_sha1_digest_to_str(packidx->hdr.sorted_ids[i].hash,
-		    hex, sizeof(hex));
+		char hex[GOT_OBJECT_ID_HEX_MAXLEN];
+
+		oid = packidx->hdr.sorted_ids + idlen * i;
+		got_hash_digest_to_str(oid->hash, hex, sizeof(hex), algo);
 		fprintf(stderr, "%s\n", hex);
 	}
 	fprintf(stderr, "\n");
@@ -535,10 +544,14 @@ static void
 add_indexed_object(struct got_packidx *packidx, uint32_t idx,
     struct got_indexed_object *obj)
 {
+	enum got_hash_algorithm algo = GOT_HASH_SHA1;
+	size_t idlen;
 	int i;
 
-	memcpy(packidx->hdr.sorted_ids[idx].hash, obj->id.hash,
-	    SHA1_DIGEST_LENGTH);
+	idlen = got_hash_digest_length(algo);
+
+	memcpy(packidx->hdr.sorted_ids + idx * idlen, obj->id.hash,
+	    idlen);
 	packidx->hdr.crc32[idx] = htobe32(obj->crc);
 	if (obj->off < GOT_PACKIDX_OFFSET_VAL_IS_LARGE_IDX)
 		packidx->hdr.offsets[idx] = htobe32(obj->off);
@@ -593,15 +606,19 @@ update_packidx(struct got_packidx *packidx, uint32_t n
     struct got_indexed_object *obj)
 {
 	int idx;
+	size_t idlen;
+	enum got_hash_algorithm algo = GOT_HASH_SHA1;
 	uint32_t nindexed = be32toh(packidx->hdr.fanout_table[0xff]);
 
+	idlen = got_hash_digest_length(algo);
+
 	idx = find_object_idx(packidx, obj->id.hash);
 	if (idx == -1)
 		return; /* object already indexed */
 
-	memmove(&packidx->hdr.sorted_ids[idx + 1],
-	    &packidx->hdr.sorted_ids[idx],
-	    sizeof(struct got_packidx_object_id) * (nindexed - idx));
+	memmove(packidx->hdr.sorted_ids + (idx + 1) * idlen,
+	    packidx->hdr.sorted_ids + idx * idlen,
+	    idlen * (nindexed - idx));
 	memmove(&packidx->hdr.offsets[idx + 1], &packidx->hdr.offsets[idx],
 	    sizeof(uint32_t) * (nindexed - idx));
 
@@ -702,8 +719,7 @@ got_pack_index(struct got_pack *pack, int idxfd, FILE 
 		err = got_error_from_errno("calloc");
 		goto done;
 	}
-	packidx.hdr.sorted_ids = calloc(nobj,
-	    sizeof(struct got_packidx_object_id));
+	packidx.hdr.sorted_ids = calloc(nobj, SHA1_DIGEST_LENGTH);
 	if (packidx.hdr.sorted_ids == NULL) {
 		err = got_error_from_errno("calloc");
 		goto done;
@@ -945,7 +961,7 @@ got_pack_index(struct got_pack *pack, int idxfd, FILE 
 	    GOT_PACKIDX_V2_FANOUT_TABLE_ITEMS * sizeof(uint32_t), &ctx);
 	if (err)
 		goto done;
-	err = got_pack_hwrite(idxfd, packidx.hdr.sorted_ids,
+	got_pack_hwrite(idxfd, packidx.hdr.sorted_ids,
 	    nobj * SHA1_DIGEST_LENGTH, &ctx);
 	if (err)
 		goto done;
blob - 99a99106cf364e5149e36643733a0b3a5b3a51ff
blob + 39881d55e76da49d2e37a089ca50e5d0c345a31f
--- lib/repository.c
+++ lib/repository.c
@@ -1123,7 +1123,9 @@ add_packidx_bloom_filter(struct got_repository *repo,
 {
 	int i, nobjects = be32toh(packidx->hdr.fanout_table[0xff]);
 	struct got_packidx_bloom_filter *bf;
-	size_t len;
+	size_t len, idlen;
+
+	idlen = got_hash_digest_length(repo->algo);
 
 	/*
 	 * Don't use bloom filters for very large pack index files.
@@ -1162,8 +1164,8 @@ add_packidx_bloom_filter(struct got_repository *repo,
 	bloom_init(bf->bloom, nobjects < 1000 ? 1000 : nobjects, 0.1);
 	for (i = 0; i < nobjects; i++) {
 		struct got_packidx_object_id *id;
-		id = &packidx->hdr.sorted_ids[i];
-		bloom_add(bf->bloom, id->hash, sizeof(id->hash));
+		id = packidx->hdr.sorted_ids + i * idlen;
+		bloom_add(bf->bloom, id->hash, idlen);
 	}
 
 	RB_INSERT(got_packidx_bloom_filter_tree,
blob - e77be5b3d71e288c7b91770b3d71f373f26ead78
blob + 9f11106c2c45a73b5745d8715c16bec6e2cf610d
--- lib/repository_admin.c
+++ lib/repository_admin.c
@@ -569,7 +569,7 @@ got_repo_list_pack(FILE *packfile, struct got_object_i
 			if (err)
 				break;
 		}
-		oid = &packidx->hdr.sorted_ids[i];
+		oid = packidx->hdr.sorted_ids + i * SHA1_DIGEST_LENGTH;
 		memcpy(id.hash, oid->hash, SHA1_DIGEST_LENGTH);
 
 		offset = got_packidx_get_object_offset(packidx, i);