commit 3428a4637c1b22375936cca37656bd2d67ade1b2 from: Omar Polo date: Sun Feb 05 10:59:01 2023 UTC hacks to get got-read-pack working on sha256 repositories commit - 01f02e603b4cddeaf18b46197eacc16a88ec3d67 commit + 3428a4637c1b22375936cca37656bd2d67ade1b2 blob - 3ca37674822674c80444158b4083d669c36da1d3 blob + 8a177274384d56727ce53de667f430947054a7b8 --- lib/got_lib_pack.h +++ lib/got_lib_pack.h @@ -57,18 +57,18 @@ const struct got_error *got_pack_parse_object_type_and SHA1_DIGEST_STRING_LENGTH - 1 + \ strlen(GOT_PACKFILE_SUFFIX)) #define GOT_PACKIDX_NAMELEN (strlen(GOT_PACK_PREFIX) + \ - SHA1_DIGEST_STRING_LENGTH - 1 + \ + SHA256_DIGEST_STRING_LENGTH - 1 + \ strlen(GOT_PACKIDX_SUFFIX)) /* See Documentation/technical/pack-format.txt in Git. */ struct got_packidx_trailer { - u_int8_t packfile_hash[SHA1_DIGEST_LENGTH]; - u_int8_t packidx_hash[SHA1_DIGEST_LENGTH]; + u_int8_t packfile_hash[SHA256_DIGEST_LENGTH]; + u_int8_t packidx_hash[SHA256_DIGEST_LENGTH]; } __attribute__((__packed__)); struct got_packidx_object_id { - u_int8_t hash[SHA1_DIGEST_LENGTH]; + u_int8_t hash[SHA256_DIGEST_LENGTH]; } __attribute__((__packed__)); /* Ignore pack index version 1 which is no longer written by Git. */ @@ -193,9 +193,10 @@ struct got_packfile_obj_data { } __attribute__((__packed__)); } __attribute__((__packed__)); -const struct got_error *got_packidx_init_hdr(struct got_packidx *, int, off_t); +const struct got_error *got_packidx_init_hdr(struct got_packidx *, int, off_t, + enum got_hash_algorithm); const struct got_error *got_packidx_open(struct got_packidx **, - int, const char *, int); + int, const char *, int, enum got_hash_algorithm); const struct got_error *got_packidx_close(struct got_packidx *); const struct got_error *got_packidx_get_packfile_path(char **, const char *); off_t got_packidx_get_object_offset(struct got_packidx *, int idx); blob - 0048ae629a38010d410c22c71c21498ebe088357 blob + 2eaa51dadeaa199ad49d97bdb86a772cb2b16ba3 --- lib/pack.c +++ lib/pack.c @@ -57,6 +57,47 @@ #define MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b)) #endif +static inline void +hash_init(SHA1_CTX *sha1_ctx, SHA2_CTX *sha256_ctx, + enum got_hash_algorithm algo) +{ + if (algo == GOT_HASH_SHA256) + SHA256Init(sha256_ctx); + else + SHA1Init(sha1_ctx); +} + +static inline void +hash_update(SHA1_CTX *sha1_ctx, SHA2_CTX *sha256_ctx, void *data, size_t len, + enum got_hash_algorithm algo) +{ + if (algo == GOT_HASH_SHA256) + SHA256Update(sha256_ctx, data, len); + else + SHA1Update(sha1_ctx, data, len); +} + +static inline void +hash_final(SHA1_CTX *sha1_ctx, SHA2_CTX *sha256_ctx, uint8_t *out, + enum got_hash_algorithm algo) +{ + if (algo == GOT_HASH_SHA256) + SHA256Final(out, sha256_ctx); + else + SHA1Final(out, sha1_ctx); +} + +static inline int +hash_cmp(uint8_t *orig, uint8_t *comp, enum got_hash_algorithm algo) +{ + size_t len = SHA1_DIGEST_LENGTH; + + if (algo == GOT_HASH_SHA256) + len = SHA256_DIGEST_LENGTH; + + return memcmp(orig, comp, len); +} + static const struct got_error * verify_fanout_table(uint32_t *fanout_table) { @@ -71,17 +112,19 @@ verify_fanout_table(uint32_t *fanout_table) } const struct got_error * -got_packidx_init_hdr(struct got_packidx *p, int verify, off_t packfile_size) +got_packidx_init_hdr(struct got_packidx *p, int verify, off_t packfile_size, + enum got_hash_algorithm algo) { const struct got_error *err = NULL; struct got_packidx_v2_hdr *h; - SHA1_CTX ctx; - uint8_t hash[SHA1_DIGEST_LENGTH]; + SHA2_CTX sha256_ctx; + SHA1_CTX sha1_ctx; + uint8_t hash[SHA256_DIGEST_LENGTH]; size_t nobj, len_fanout, len_ids, offset, remain; ssize_t n; int i; - SHA1Init(&ctx); + hash_init(&sha1_ctx, &sha256_ctx, algo); h = &p->hdr; offset = 0; @@ -116,7 +159,8 @@ got_packidx_init_hdr(struct got_packidx *p, int verify remain -= sizeof(*h->magic); if (verify) - SHA1Update(&ctx, (uint8_t *)h->magic, sizeof(*h->magic)); + hash_update(&sha1_ctx, &sha256_ctx, h->magic, sizeof(*h->magic), + algo); if (remain < sizeof(*h->version)) { err = got_error(GOT_ERR_BAD_PACKIDX); @@ -147,7 +191,8 @@ got_packidx_init_hdr(struct got_packidx *p, int verify remain -= sizeof(*h->version); if (verify) - SHA1Update(&ctx, (uint8_t *)h->version, sizeof(*h->version)); + hash_update(&sha1_ctx, &sha256_ctx, + h->version, sizeof(*h->version), algo); len_fanout = sizeof(*h->fanout_table) * GOT_PACKIDX_V2_FANOUT_TABLE_ITEMS; @@ -176,7 +221,8 @@ got_packidx_init_hdr(struct got_packidx *p, int verify if (err) goto done; if (verify) - SHA1Update(&ctx, (uint8_t *)h->fanout_table, len_fanout); + hash_update(&sha1_ctx, &sha256_ctx, h->fanout_table, + len_fanout, algo); offset += len_fanout; remain -= len_fanout; @@ -204,7 +250,8 @@ got_packidx_init_hdr(struct got_packidx *p, int verify } } if (verify) - SHA1Update(&ctx, (uint8_t *)h->sorted_ids, len_ids); + hash_update(&sha1_ctx, &sha256_ctx, h->sorted_ids, len_ids, + algo); offset += len_ids; remain -= len_ids; @@ -229,7 +276,8 @@ got_packidx_init_hdr(struct got_packidx *p, int verify } } if (verify) - SHA1Update(&ctx, (uint8_t *)h->crc32, nobj * sizeof(*h->crc32)); + hash_update(&sha1_ctx, &sha256_ctx, h->crc32, + nobj * sizeof(*h->crc32), algo); remain -= nobj * sizeof(*h->crc32); offset += nobj * sizeof(*h->crc32); @@ -254,8 +302,8 @@ got_packidx_init_hdr(struct got_packidx *p, int verify } } if (verify) - SHA1Update(&ctx, (uint8_t *)h->offsets, - nobj * sizeof(*h->offsets)); + hash_update(&sha1_ctx, &sha256_ctx, h->offsets, + nobj * sizeof(*h->offsets), algo); remain -= nobj * sizeof(*h->offsets); offset += nobj * sizeof(*h->offsets); @@ -297,8 +345,8 @@ got_packidx_init_hdr(struct got_packidx *p, int verify } } if (verify) - SHA1Update(&ctx, (uint8_t*)h->large_offsets, - p->nlargeobj * sizeof(*h->large_offsets)); + hash_update(&sha1_ctx, &sha256_ctx, h->large_offsets, + p->nlargeobj * sizeof(*h->large_offsets), algo); remain -= p->nlargeobj * sizeof(*h->large_offsets); offset += p->nlargeobj * sizeof(*h->large_offsets); @@ -325,10 +373,13 @@ checksum: } } if (verify) { - SHA1Update(&ctx, h->trailer->packfile_hash, SHA1_DIGEST_LENGTH); - SHA1Final(hash, &ctx); - if (memcmp(h->trailer->packidx_hash, hash, - SHA1_DIGEST_LENGTH) != 0) + hash_update(&sha1_ctx, &sha256_ctx, h->trailer->packfile_hash, + algo == GOT_HASH_SHA256 + ? SHA256_DIGEST_LENGTH + : SHA1_DIGEST_LENGTH, + algo); + hash_final(&sha1_ctx, &sha256_ctx, hash, algo); + if (hash_cmp(h->trailer->packidx_hash, hash, algo) != 0) err = got_error(GOT_ERR_PACKIDX_CSUM); } done: @@ -337,7 +388,7 @@ done: const struct got_error * got_packidx_open(struct got_packidx **packidx, - int dir_fd, const char *relpath, int verify) + int dir_fd, const char *relpath, int verify, enum got_hash_algorithm algo) { const struct got_error *err = NULL; struct got_packidx *p = NULL; @@ -405,7 +456,7 @@ got_packidx_open(struct got_packidx **packidx, } #endif - err = got_packidx_init_hdr(p, verify, pack_sb.st_size); + err = got_packidx_init_hdr(p, verify, pack_sb.st_size, algo); done: if (err) { if (p) @@ -489,10 +540,14 @@ int got_packidx_get_object_idx(struct got_packidx *packidx, struct got_object_id *id) { + size_t len = SHA1_DIGEST_LENGTH; u_int8_t id0 = id->hash[0]; uint32_t totobj = be32toh(packidx->hdr.fanout_table[0xff]); int left = 0, right = totobj - 1; + if (id->algo == GOT_HASH_SHA256) + len = SHA256_DIGEST_LENGTH; + if (id0 > 0) left = be32toh(packidx->hdr.fanout_table[id0 - 1]); @@ -502,7 +557,7 @@ got_packidx_get_object_idx(struct got_packidx *packidx i = ((left + right) / 2); oid = &packidx->hdr.sorted_ids[i]; - cmp = memcmp(id->hash, oid->hash, SHA1_DIGEST_LENGTH); + cmp = memcmp(id->hash, oid->hash, len); if (cmp == 0) return i; else if (cmp > 0) @@ -658,7 +713,8 @@ got_packidx_get_object_id(struct got_object_id *id, return got_error(GOT_ERR_NO_OBJ); oid = &packidx->hdr.sorted_ids[idx]; - memcpy(id->hash, oid->hash, SHA1_DIGEST_LENGTH); + memcpy(id->hash, oid->hash, SHA256_DIGEST_LENGTH); + id->algo = GOT_HASH_SHA256; return NULL; } @@ -689,11 +745,12 @@ got_packidx_match_id_str_prefix(struct got_object_id_q i = be32toh(packidx->hdr.fanout_table[id0 - 1]); oid = &packidx->hdr.sorted_ids[i]; while (i < totobj && oid->hash[0] == id0) { - char id_str[SHA1_DIGEST_STRING_LENGTH]; + char id_str[SHA256_DIGEST_STRING_LENGTH]; struct got_object_qid *qid; int cmp; - if (!got_sha1_digest_to_str(oid->hash, id_str, sizeof(id_str))) + if (!got_sha256_digest_to_str(oid->hash, id_str, + sizeof(id_str))) return got_error(GOT_ERR_NO_SPACE); cmp = strncmp(id_str, id_str_prefix, prefix_len); @@ -706,7 +763,8 @@ got_packidx_match_id_str_prefix(struct got_object_id_q err = got_object_qid_alloc_partial(&qid); if (err) break; - memcpy(qid->id.hash, oid->hash, SHA1_DIGEST_LENGTH); + memcpy(qid->id.hash, oid->hash, SHA256_DIGEST_LENGTH); + qid->id.algo = GOT_HASH_SHA256; STAILQ_INSERT_TAIL(matched_ids, qid, entry); oid = &packidx->hdr.sorted_ids[++i]; @@ -1945,7 +2003,7 @@ got_packfile_extract_raw_delta(uint8_t **delta_buf, si err = got_pack_parse_ref_delta(base_id, pack, offset, tslen); if (err) return err; - delta_hdrlen = SHA1_DIGEST_LENGTH; + delta_hdrlen = SHA256_DIGEST_LENGTH; break; default: return got_error_fmt(GOT_ERR_OBJ_TYPE, blob - 58654437b08e0b54366f259acaa637015d7e2767 blob + 9bc761dbfaecdcbc57a85b7f922bfc98eab9bbe9 --- lib/repository.c +++ lib/repository.c @@ -1256,7 +1256,7 @@ got_repo_search_packidx(struct got_packidx **packidx, continue; /* already searched */ err = got_packidx_open(packidx, got_repo_get_fd(repo), - path_packidx, 0); + path_packidx, 0, repo->algo); if (err) goto done; @@ -1358,7 +1358,7 @@ got_repo_get_packidx(struct got_packidx **packidx, con /* No luck. Search the filesystem. */ err = got_packidx_open(packidx, got_repo_get_fd(repo), - path_packidx, 0); + path_packidx, 0, repo->algo); if (err) return err; @@ -1655,7 +1655,7 @@ match_packed_object(struct got_object_id **unique_id, struct got_object_qid *qid; err = got_packidx_open(&packidx, got_repo_get_fd(repo), - path_packidx, 0); + path_packidx, 0, repo->algo); if (err) break; @@ -2370,7 +2370,7 @@ got_repo_get_packfile_info(int *npackfiles, int *nobje } err = got_packidx_open(&packidx, got_repo_get_fd(repo), - path_packidx, 0); + path_packidx, 0, repo->algo); free(path_packidx); if (err) goto done; blob - eccc6809e30b6faec649f38baf03d8d883381d2c blob + bb843d5c6f24f6ba90698df04cc5510caec5b91a --- lib/repository_admin.c +++ lib/repository_admin.c @@ -547,7 +547,8 @@ got_repo_list_pack(FILE *packfile, struct got_object_i goto done; } - err = got_packidx_open(&packidx, got_repo_get_fd(repo), idxpath, 1); + err = got_packidx_open(&packidx, got_repo_get_fd(repo), idxpath, 1, + repo->algo); if (err) goto done; blob - fa7e848b2a4b1ace5b5ac7c12b5d44e8474b980b blob + ed28b97017a1d45c58f9702965fc5cd63d2b38a7 --- libexec/got-read-pack/got-read-pack.c +++ libexec/got-read-pack/got-read-pack.c @@ -142,7 +142,7 @@ open_commit(struct got_commit_object **commit, struct obj->size = len; - err = got_object_parse_commit(commit, buf, len, GOT_HASH_SHA1); + err = got_object_parse_commit(commit, buf, len, id->algo); done: got_object_close(obj); free(buf); @@ -1142,7 +1142,8 @@ receive_packidx(struct got_packidx **packidx, struct i p->map = NULL; /* fall back to read(2) */ } #endif - err = got_packidx_init_hdr(p, 1, ipackidx.packfile_size); + err = got_packidx_init_hdr(p, 1, ipackidx.packfile_size, + GOT_HASH_SHA256); /* XXX! */ done: if (err) { if (imsg.fd != -1)