commit a1fd68d829c4e5eb0496107b9ac3d94968f81a8e from: Stefan Sperling date: Fri Jan 12 19:59:51 2018 UTC Extract non-deltified objects from pack files. commit - cb672fbdfe95d95d806a21fa0db320aa2636c927 commit + a1fd68d829c4e5eb0496107b9ac3d94968f81a8e blob - 79deadb93b5f53f5dc7e8ed1b432d74c47ed11e5 blob + 8f4ae1a3a30d63c209f8f08cd86d333b4b0fd863 --- include/got_error.h +++ include/got_error.h @@ -15,22 +15,24 @@ */ /* Error codes */ -#define GOT_ERR_UNKNOWN 0x0000 -#define GOT_ERR_NO_MEM 0x0001 -#define GOT_ERR_NOT_GIT_REPO 0x0002 -#define GOT_ERR_NOT_ABSPATH 0x0003 -#define GOT_ERR_BAD_PATH 0x0004 -#define GOT_ERR_NOT_REF 0x0005 -#define GOT_ERR_IO 0x0006 -#define GOT_ERR_EOF 0x0007 -#define GOT_ERR_DECOMPRESSION 0x0008 -#define GOT_ERR_NO_SPACE 0x0009 -#define GOT_ERR_BAD_OBJ_HDR 0x0010 -#define GOT_ERR_OBJ_TYPE 0x0011 -#define GOT_ERR_BAD_OBJ_DATA 0x0012 -#define GOT_ERR_FILE_OPEN 0x0013 -#define GOT_ERR_BAD_PACKIDX 0x0014 -#define GOT_ERR_PACKIDX_CSUM 0x0015 +#define GOT_ERR_UNKNOWN 0 +#define GOT_ERR_NO_MEM 1 +#define GOT_ERR_NOT_GIT_REPO 2 +#define GOT_ERR_NOT_ABSPATH 3 +#define GOT_ERR_BAD_PATH 4 +#define GOT_ERR_NOT_REF 5 +#define GOT_ERR_IO 6 +#define GOT_ERR_EOF 7 +#define GOT_ERR_DECOMPRESSION 8 +#define GOT_ERR_NO_SPACE 9 +#define GOT_ERR_BAD_OBJ_HDR 10 +#define GOT_ERR_OBJ_TYPE 11 +#define GOT_ERR_BAD_OBJ_DATA 12 +#define GOT_ERR_FILE_OPEN 13 +#define GOT_ERR_BAD_PACKIDX 14 +#define GOT_ERR_PACKIDX_CSUM 15 +#define GOT_ERR_BAD_PACKFILE 16 +#define GOT_ERR_NO_OBJ 17 static const struct got_error { int code; @@ -52,6 +54,8 @@ static const struct got_error { { GOT_ERR_FILE_OPEN, "could not open file" }, { GOT_ERR_BAD_PACKIDX, "bad pack index file" }, { GOT_ERR_PACKIDX_CSUM, "pack index file checksum error" }, + { GOT_ERR_BAD_PACKFILE, "bad pack file" }, + { GOT_ERR_NO_OBJ, "object not found" }, }; const struct got_error * got_error(int code); blob - 6854ce112fc51fe9f38a696a11921af6e3dea29f blob + 9e3f32b04c48fb04b77e6f8312deb951ae55ea31 --- include/got_object.h +++ include/got_object.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 Stefan Sperling + * Copyright (c) 2018 Stefan Sperling * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -80,7 +80,9 @@ struct got_object { struct got_repository; -char * got_object_id_str(struct got_object_id *, char *, size_t); +char *got_object_id_str(struct got_object_id *, char *, size_t); +int got_object_id_cmp(struct got_object_id *, struct got_object_id *); +const char *got_object_get_type_tag(int); const struct got_error *got_object_open(struct got_object **, struct got_repository *, struct got_object_id *); void got_object_close(struct got_object *); blob - 65e7c610c725361e0c22696225499962bdad27aa blob + 49c27b10f99d94b1ac7bced8c2a7d30c7c724446 --- include/got_repository.h +++ include/got_repository.h @@ -24,6 +24,7 @@ void got_repo_close(struct got_repository*); char *got_repo_get_path_git_dir(struct got_repository *); char *got_repo_get_path_objects(struct got_repository *); +char *got_repo_get_path_objects_pack(struct got_repository *); char *got_repo_get_path_refs(struct got_repository *); struct got_reference; blob - 668013da54ca7a913d9378e2413398b9dd4a0185 blob + df56c0dec9b4f886d0894e14ad05aea9fde11040 --- include/got_sha1.h +++ include/got_sha1.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 Stefan Sperling + * Copyright (c) 2018 Stefan Sperling * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -15,3 +15,4 @@ */ int got_parse_sha1_digest(uint8_t *, const char *); +char *got_sha1_digest_to_str(const uint8_t *, char *, size_t); blob - 0f5eb9be742c6f3c7540471e463276a101a096e0 blob + d99ed9f7c19a52512573c812b0f54ccfe0218861 --- lib/diff.c +++ lib/diff.c @@ -30,31 +30,8 @@ #include "got_diff.h" #include "diff.h" - -static FILE * -opentemp(void) -{ - char name[PATH_MAX]; - int fd; - FILE *f; - - if (strlcpy(name, "/tmp/got.XXXXXXXX", sizeof(name)) >= sizeof(name)) - return NULL; - - fd = mkstemp(name); - if (fd < 0) - return NULL; - - unlink(name); - f = fdopen(fd, "w+"); - if (f == NULL) { - close(fd); - return NULL; - } +#include "path.h" - return f; -} - const struct got_error * got_diff_blob(struct got_blob_object *blob1, struct got_blob_object *blob2, const char *label1, const char *label2, FILE *outfile) @@ -70,14 +47,14 @@ got_diff_blob(struct got_blob_object *blob1, struct go int res, flags = 0; if (blob1) { - f1 = opentemp(); + f1 = got_opentemp(); if (f1 == NULL) return got_error(GOT_ERR_FILE_OPEN); } else flags |= D_EMPTY1; if (blob2) { - f2 = opentemp(); + f2 = got_opentemp(); if (f2 == NULL) { fclose(f1); return got_error(GOT_ERR_FILE_OPEN); @@ -156,12 +133,6 @@ match_entry_by_name(struct got_tree_entry *te1, struct return NULL; } -static int -same_id(struct got_object_id *id1, struct got_object_id *id2) -{ - return (memcmp(id1->sha1, id2->sha1, SHA1_DIGEST_LENGTH) == 0); -} - static const struct got_error * diff_added_blob(struct got_object_id *id, struct got_repository *repo) { @@ -383,12 +354,11 @@ diff_entry_old_new(struct got_tree_entry *te1, struct } if (S_ISDIR(te1->mode) && S_ISDIR(te2->mode)) { - if (!same_id(&te1->id, &te2->id)) + if (got_object_id_cmp(&te1->id, &te2->id) != 0) return diff_modified_tree(&te1->id, &te2->id, repo); } else if (S_ISREG(te1->mode) && S_ISREG(te2->mode)) { - if (!same_id(&te1->id, &te2->id)) + if (got_object_id_cmp(&te1->id, &te2->id) != 0) return diff_modified_blob(&te1->id, &te2->id, repo); - } return diff_kind_mismatch(&te1->id, &te2->id); blob - 87f6288f8d365111bb6ea7fd97603fe058c40bfa blob + 2beacfb382bf9e479a9039320ec767d7564a03da --- lib/object.c +++ lib/object.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 Stefan Sperling + * Copyright (c) 2018 Stefan Sperling * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -29,6 +30,7 @@ #include "got_object.h" #include "got_repository.h" #include "got_sha1.h" +#include "pack.h" #ifndef MIN #define MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b)) @@ -50,22 +52,28 @@ char * got_object_id_str(struct got_object_id *id, char *buf, size_t size) { - char *p = buf; - char hex[3]; - int i; + return got_sha1_digest_to_str(id->sha1, buf, size); +} - if (size < SHA1_DIGEST_STRING_LENGTH) - return NULL; +int +got_object_id_cmp(struct got_object_id *id1, struct got_object_id *id2) +{ + return memcmp(id1->sha1, id2->sha1, SHA1_DIGEST_LENGTH); +} - for (i = 0; i < SHA1_DIGEST_LENGTH; i++) { - snprintf(hex, sizeof(hex), "%.2x", id->sha1[i]); - p[0] = hex[0]; - p[1] = hex[1]; - p += 2; +const char * +got_object_get_type_tag(int type) +{ + switch (type) { + case GOT_OBJ_TYPE_COMMIT: + return GOT_OBJ_TAG_COMMIT; + case GOT_OBJ_TYPE_TREE: + return GOT_OBJ_TAG_TREE; + case GOT_OBJ_TYPE_BLOB: + return GOT_OBJ_TAG_BLOB; } - p[0] = '\0'; - return buf; + return NULL; } static void @@ -198,10 +206,9 @@ parse_object_header(struct got_object **obj, char *buf static const struct got_error * read_object_header(struct got_object **obj, struct got_repository *repo, - const char *path) + FILE *f) { const struct got_error *err; - FILE *f; struct got_zstream_buf zb; char *buf; size_t len; @@ -209,19 +216,13 @@ read_object_header(struct got_object **obj, struct got size_t outlen, totlen; int i, ret; - f = fopen(path, "rb"); - if (f == NULL) - return got_error(GOT_ERR_BAD_PATH); - buf = calloc(zbsize, sizeof(char)); if (buf == NULL) return got_error(GOT_ERR_NO_MEM); err = inflate_init(&zb, zbsize); - if (err) { - fclose(f); + if (err) return err; - } i = 0; totlen = 0; @@ -244,7 +245,6 @@ read_object_header(struct got_object **obj, struct got err = parse_object_header(obj, buf, totlen); done: inflate_end(&zb); - fclose(f); return err; } @@ -274,17 +274,35 @@ got_object_open(struct got_object **obj, struct got_re struct got_object_id *id) { const struct got_error *err = NULL; - char *path = NULL; + char *path; + FILE *f = NULL; err = object_path(&path, id, repo); if (err) return err; - err = read_object_header(obj, repo, path); + f = fopen(path, "rb"); + if (f == NULL) { + if (errno != ENOENT) { + err = got_error(GOT_ERR_BAD_PATH); + goto done; + } + err = got_packfile_extract_object(&f, id, repo); + if (err) + goto done; + if (f == NULL) { + err = got_error(GOT_ERR_NO_OBJ); + goto done; + } + } + + err = read_object_header(obj, repo, f); if (err == NULL) memcpy((*obj)->id.sha1, id->sha1, SHA1_DIGEST_LENGTH); done: free(path); + if (f != NULL) + fclose(f); return err; } @@ -723,4 +741,3 @@ got_object_blob_read_block(struct got_blob_object *blo { return inflate_read(&blob->zb, blob->f, outlenp); } - blob - 5f38af5b0c6763a70545931c03d88f214de45956 blob + 6f0e06cbee2c22585c03b3bb96b3396835605179 --- lib/pack.c +++ lib/pack.c @@ -14,25 +14,49 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include #include +#include +#include +#include #include +#include #include #include #include #include #include +#include #include "got_error.h" +#include "got_object.h" +#include "got_repository.h" +#include "got_sha1.h" #include "pack.h" +#include "path.h" +#define GOT_PACK_PREFIX "pack-" +#define GOT_PACKFILE_SUFFIX ".pack" +#define GOT_PACKIDX_SUFFIX ".idx" +#define GOT_PACKFILE_NAMELEN (strlen(GOT_PACK_PREFIX) + \ + SHA1_DIGEST_STRING_LENGTH - 1 + \ + strlen(GOT_PACKFILE_SUFFIX)) +#define GOT_PACKIDX_NAMELEN (strlen(GOT_PACK_PREFIX) + \ + SHA1_DIGEST_STRING_LENGTH - 1 + \ + strlen(GOT_PACKIDX_SUFFIX)) + +#ifndef MIN +#define MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b)) +#endif + static const struct got_error * verify_fanout_table(uint32_t *fanout_table) { int i; for (i = 0; i < 0xff - 1; i++) { - if (fanout_table[i] > fanout_table[i + 1]) + if (be32toh(fanout_table[i]) > be32toh(fanout_table[i + 1])) return got_error(GOT_ERR_BAD_PACKIDX); } @@ -149,33 +173,33 @@ got_packidx_open(struct got_packidx_v2_hdr **packidx, SHA1Update(&ctx, (uint8_t *)p->sorted_ids, nobj * sizeof(*p->sorted_ids)); - p->offsets = calloc(nobj, sizeof(*p->offsets)); - if (p->offsets == NULL) { + p->crc32 = calloc(nobj, sizeof(*p->crc32)); + if (p->crc32 == NULL) { err = got_error(GOT_ERR_NO_MEM); goto done; } - n = fread(p->offsets, sizeof(*p->offsets), nobj, f); + n = fread(p->crc32, sizeof(*p->crc32), nobj, f); if (n != nobj) { err = got_error(ferror(f) ? GOT_ERR_IO : GOT_ERR_BAD_PACKIDX); goto done; } - SHA1Update(&ctx, (uint8_t *)p->offsets, nobj * sizeof(*p->offsets)); + SHA1Update(&ctx, (uint8_t *)p->crc32, nobj * sizeof(*p->crc32)); - p->crc32 = calloc(nobj, sizeof(*p->crc32)); - if (p->crc32 == NULL) { + p->offsets = calloc(nobj, sizeof(*p->offsets)); + if (p->offsets == NULL) { err = got_error(GOT_ERR_NO_MEM); goto done; } - n = fread(p->crc32, sizeof(*p->crc32), nobj, f); + n = fread(p->offsets, sizeof(*p->offsets), nobj, f); if (n != nobj) { err = got_error(ferror(f) ? GOT_ERR_IO : GOT_ERR_BAD_PACKIDX); goto done; } - SHA1Update(&ctx, (uint8_t *)p->crc32, nobj * sizeof(*p->crc32)); + SHA1Update(&ctx, (uint8_t *)p->offsets, nobj * sizeof(*p->offsets)); /* Large file offsets are contained only in files > 2GB. */ if (packfile_size <= 0x80000000) @@ -203,9 +227,9 @@ checksum: goto done; } - SHA1Update(&ctx, p->trailer.pack_file_sha1, SHA1_DIGEST_LENGTH); + SHA1Update(&ctx, p->trailer.packfile_sha1, SHA1_DIGEST_LENGTH); SHA1Final(sha1, &ctx); - if (memcmp(p->trailer.pack_idx_sha1, sha1, SHA1_DIGEST_LENGTH) != 0) + if (memcmp(p->trailer.packidx_sha1, sha1, SHA1_DIGEST_LENGTH) != 0) err = got_error(GOT_ERR_PACKIDX_CSUM); done: fclose(f); @@ -224,4 +248,276 @@ got_packidx_close(struct got_packidx_v2_hdr *packidx) free(packidx->crc32); free(packidx->large_offsets); free(packidx); +} + +static int +is_packidx_filename(const char *name, size_t len) +{ + if (len != GOT_PACKIDX_NAMELEN) + return 0; + + if (strncmp(name, GOT_PACK_PREFIX, strlen(GOT_PACK_PREFIX)) != 0) + return 0; + + if (strcmp(name + strlen(GOT_PACK_PREFIX) + + SHA1_DIGEST_STRING_LENGTH - 1, GOT_PACKIDX_SUFFIX) != 0) + return 0; + + return 1; +} + +static off_t +get_object_offset(struct got_packidx_v2_hdr *packidx, int idx) +{ + uint32_t totobj = betoh32(packidx->fanout_table[0xff]); + uint32_t offset = betoh32(packidx->offsets[idx]); + if (offset & GOT_PACKIDX_OFFSET_VAL_IS_LARGE_IDX) { + uint64_t loffset; + idx = offset & GOT_PACKIDX_OFFSET_VAL_MASK; + if (idx < 0 || idx > totobj || packidx->large_offsets == NULL) + return -1; + loffset = betoh64(packidx->large_offsets[idx]); + return (loffset > INT64_MAX ? -1 : (off_t)loffset); + } + return (off_t)(offset & GOT_PACKIDX_OFFSET_VAL_MASK); +} + +static int +get_object_idx(struct got_packidx_v2_hdr *packidx, struct got_object_id *id) +{ + u_int8_t id0 = id->sha1[0]; + uint32_t totobj = betoh32(packidx->fanout_table[0xff]); + int i = 0; + + if (id0 > 0) + i = betoh32(packidx->fanout_table[id0 - 1]); + + while (i < totobj) { + struct got_object_id *oid = &packidx->sorted_ids[i++]; + uint32_t offset; + + if (got_object_id_cmp(id, oid) < 0) + continue; + if (got_object_id_cmp(id, oid) > 0) + break; + + return i; + } + + return -1; +} + +const struct got_error * +read_packfile_hdr(FILE *f, struct got_packidx_v2_hdr *packidx) +{ + const struct got_error *err = NULL; + uint32_t totobj = betoh32(packidx->fanout_table[0xff]); + struct got_packfile_hdr hdr; + size_t n; + + n = fread(&hdr, sizeof(hdr), 1, f); + if (n != 1) + return got_error(ferror(f) ? GOT_ERR_IO : GOT_ERR_BAD_PACKFILE); + + if (betoh32(hdr.signature) != GOT_PACKFILE_SIGNATURE || + betoh32(hdr.version) != GOT_PACKFILE_VERSION || + betoh32(hdr.nobjects) != totobj) + err = got_error(GOT_ERR_BAD_PACKFILE); + + return err; +} + +static const struct got_error * +dump_packed_object(FILE **f, FILE *packfile, off_t offset) +{ + const struct got_error *err = NULL; + const char *template = "/tmp/got.XXXXXXXXXX"; + uint64_t size = 0; + uint8_t type = 0; + uint8_t sizeN; + int i; + size_t n; + const char *type_tag; + + *f = got_opentemp(); + if (*f == NULL) { + err = got_error(GOT_ERR_FILE_OPEN); + goto done; + } + + if (fseeko(packfile, offset, SEEK_SET) != 0) { + err = got_error(errno == EIO ? GOT_ERR_IO : GOT_ERR_BAD_PATH); + goto done; + } + + i = 0; + do { + /* We do not support size values which don't fit in 64 bit. */ + if (i > 9) { + err = got_error(GOT_ERR_NO_SPACE); + goto done; + } + + n = fread(&sizeN, sizeof(sizeN), 1, packfile); + if (n != 1) { + err = got_error(ferror(packfile) ? + GOT_ERR_IO : GOT_ERR_BAD_PACKFILE); + goto done; + } + if (i == 0) { + type = (sizeN & GOT_PACK_OBJ_SIZE0_TYPE_MASK) >> + GOT_PACK_OBJ_SIZE0_TYPE_MASK_SHIFT; + size = (sizeN & GOT_PACK_OBJ_SIZE0_VAL_MASK); + } else { + size_t shift = 4 + 7 * (i - 1); + size |= ((sizeN & GOT_PACK_OBJ_SIZE_VAL_MASK) << shift); + } + i++; + } while (sizeN & GOT_PACK_OBJ_SIZE_MORE); + + if (type == GOT_OBJ_TYPE_OFFSET_DELTA) + printf("object type OFFSET_DELTA not yet implemented\n"); + else if (type == GOT_OBJ_TYPE_REF_DELTA) + printf("object type REF_DELTA not yet implemented\n"); + else if (type == GOT_OBJ_TYPE_TAG) + printf("object type TAG not yet implemented\n"); + + type_tag = got_object_get_type_tag(type); + if (type_tag == NULL) { + err = got_error(GOT_ERR_BAD_OBJ_HDR); + goto done; + } + + fprintf(*f, "%s %llu", type_tag, size); + fputc('\0', *f); + + while (size > 0) { + uint8_t data[2048]; + size_t len = MIN(size, sizeof(data)); + + n = fread(data, len, 1, packfile); + if (n != 1) { + err = got_error(ferror(packfile) ? + GOT_ERR_IO : GOT_ERR_BAD_PACKFILE); + goto done; + } + + n = fwrite(data, len, 1, *f); + if (n != 1) { + err = got_error(ferror(*f) ? + GOT_ERR_IO : GOT_ERR_BAD_PACKFILE); + goto done; + } + + size -= len; + } + + printf("object type is %d\n", type); + rewind(*f); +done: + if (err && *f) + fclose(*f); + return err; } +static const struct got_error * +extract_object(FILE **f, const char *path_packdir, + struct got_packidx_v2_hdr *packidx, struct got_object_id *id) +{ + const struct got_error *err = NULL; + int idx = get_object_idx(packidx, id); + off_t offset; + char *path_packfile; + FILE *packfile; + char hex[SHA1_DIGEST_STRING_LENGTH]; + char *sha1str; + + *f = NULL; + if (idx == -1) /* object not found in pack index */ + return NULL; + + offset = get_object_offset(packidx, idx); + if (offset == (uint64_t)-1) + return got_error(GOT_ERR_BAD_PACKIDX); + + sha1str = got_sha1_digest_to_str(packidx->trailer.packfile_sha1, + hex, sizeof(hex)); + if (sha1str == NULL) + return got_error(GOT_ERR_PACKIDX_CSUM); + + if (asprintf(&path_packfile, "%s/%s%s%s", path_packdir, + GOT_PACK_PREFIX, sha1str, GOT_PACKFILE_SUFFIX) == -1) + return got_error(GOT_ERR_NO_MEM); + + packfile = fopen(path_packfile, "rb"); + if (packfile == NULL) { + err = got_error(errno == EIO ? GOT_ERR_IO : GOT_ERR_BAD_PATH); + goto done; + } + + err = read_packfile_hdr(packfile, packidx); + if (err) + goto done; + + printf("Dumping object at offset %llu\n", offset); + err = dump_packed_object(f, packfile, offset); + if (err) + goto done; + +done: + free(path_packfile); + if (packfile && fclose(packfile) == -1 && errno == EIO && err == 0) + err = got_error(GOT_ERR_IO); + return err; +} + +const struct got_error * +got_packfile_extract_object(FILE **f, struct got_object_id *id, + struct got_repository *repo) +{ + const struct got_error *err = NULL; + DIR *packdir = NULL; + struct dirent *dent; + char *path_packdir = got_repo_get_path_objects_pack(repo); + + if (path_packdir == NULL) { + err = got_error(GOT_ERR_NO_MEM); + goto done; + } + + packdir = opendir(path_packdir); + if (packdir == NULL) { + err = got_error(errno == EIO ? GOT_ERR_IO : GOT_ERR_BAD_PATH); + goto done; + } + + while ((dent = readdir(packdir)) != NULL) { + struct got_packidx_v2_hdr *packidx; + char *path_packidx, *path_object; + + if (!is_packidx_filename(dent->d_name, dent->d_namlen)) + continue; + + if (asprintf(&path_packidx, "%s/%s", path_packdir, + dent->d_name) == -1) { + err = got_error(GOT_ERR_NO_MEM); + goto done; + } + + err = got_packidx_open(&packidx, path_packidx); + free(path_packidx); + if (err) + goto done; + + err = extract_object(f, path_packdir, packidx, id); + if (err) + goto done; + if (*f != NULL) + break; + } + +done: + free(path_packdir); + if (packdir && closedir(packdir) != 0 && errno == EIO && err == 0) + err = got_error(GOT_ERR_IO); + return err; +} blob - 4d508b318d7934d6ae1c48eeb2d580a8f6e8b80a blob + 9e034eb2f7ab046e1448b158af3478bdf73aee1b --- lib/pack.h +++ lib/pack.h @@ -16,13 +16,9 @@ /* See Documentation/technical/pack-format.txt in Git. */ -struct got_pack_obj_id { - u_int8_t sha1[SHA1_DIGEST_LENGTH]; -} __attribute__((__packed__)); - struct got_packidx_trailer { - u_int8_t pack_file_sha1[SHA1_DIGEST_LENGTH]; - u_int8_t pack_idx_sha1[SHA1_DIGEST_LENGTH]; + u_int8_t packfile_sha1[SHA1_DIGEST_LENGTH]; + u_int8_t packidx_sha1[SHA1_DIGEST_LENGTH]; } __attribute__((__packed__)); /* Ignore pack index version 1 which is no longer written by Git. */ @@ -44,16 +40,16 @@ struct got_packidx_v2_hdr { uint32_t fanout_table[0xff + 1]; /* values are big endian */ /* Sorted SHA1 checksums for each object in the pack file. */ - struct got_pack_obj_id *sorted_ids; + struct got_object_id *sorted_ids; + /* CRC32 of the packed representation of each object. */ + uint32_t *crc32; + /* Offset into the pack file for each object. */ uint32_t *offsets; /* values are big endian */ #define GOT_PACKIDX_OFFSET_VAL_MASK 0x7fffffff #define GOT_PACKIDX_OFFSET_VAL_IS_LARGE_IDX 0x80000000 - /* CRC32 of the packed representation of each object. */ - uint32_t *crc32; - /* Large offsets table is empty for pack files < 2 GB. */ uint64_t *large_offsets; /* values are big endian */ @@ -80,8 +76,9 @@ struct got_packfile_obj_hdr { uint8_t *size; /* variable length */ #define GOT_PACK_OBJ_SIZE_MORE 0x80 #define GOT_PACK_OBJ_SIZE0_TYPE_MASK 0x70 /* See struct got_object->type */ +#define GOT_PACK_OBJ_SIZE0_TYPE_MASK_SHIFT 4 #define GOT_PACK_OBJ_SIZE0_VAL_MASK 0x0f -#define GOT_PACK_OBJ_SIZEN_VAL_MASK 0x7f +#define GOT_PACK_OBJ_SIZE_VAL_MASK 0x7f }; /* If object is not a DELTA type. */ @@ -91,7 +88,7 @@ struct got_packfile_object_data { /* If object is of type GOT_OBJ_TYPE_REF_DELTA. */ struct got_packfile_object_data_ref_delta { - struct got_pack_obj_id id; + struct got_object_id id; uint8_t *delta_data; /* compressed */ }; @@ -123,3 +120,6 @@ struct got_packfile_obj_data { const struct got_error *got_packidx_open(struct got_packidx_v2_hdr **, const char *); void got_packidx_close(struct got_packidx_v2_hdr *); + +const struct got_error *got_packfile_extract_object(FILE **, + struct got_object_id *, struct got_repository *); blob - 3ca3b590a1c62b865c6be7a84c4826d3f5266e79 blob + 7ec32c1f45c6d2006e0355e0c06d728bfee555ee --- lib/path.c +++ lib/path.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 Stefan Sperling + * Copyright (c) 2018 Stefan Sperling * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -58,3 +58,27 @@ got_path_normalize(const char *path) return resolved; } + +FILE * +got_opentemp(void) +{ + char name[PATH_MAX]; + int fd; + FILE *f; + + if (strlcpy(name, "/tmp/got.XXXXXXXX", sizeof(name)) >= sizeof(name)) + return NULL; + + fd = mkstemp(name); + if (fd < 0) + return NULL; + + /* unlink(name); */ + f = fdopen(fd, "w+"); + if (f == NULL) { + close(fd); + return NULL; + } + + return f; +} blob - 80111d04b9214ae2860466b3cadba2b2ae2c7fa6 blob + 3bca95b3c18a88dff7ba4f0da9738a1e898be44b --- lib/path.h +++ lib/path.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 Stefan Sperling + * Copyright (c) 2018 Stefan Sperling * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -30,3 +30,7 @@ char *got_path_get_absolute(const char *); * The result is allocated with malloc(3). */ char *got_path_normalize(const char *); + +/* Open a new temporary file for writing. + * The file is not visible in the filesystem. */ +FILE *got_opentemp(void); blob - 162c91475aa774b4a122e8cfec1eef07f938db6e blob + 35425149aec6f9f82de230270de16cf37bb95705 --- lib/repository.c +++ lib/repository.c @@ -33,8 +33,10 @@ #define GOT_REFS_DIR "refs" #define GOT_HEAD_FILE "HEAD" +/* Other files and directories inside the git directory. */ #define GOT_FETCH_HEAD_FILE "FETCH_HEAD" #define GOT_ORIG_HEAD_FILE "ORIG_HEAD" +#define GOT_OBJECTS_PACK_DIR "objects/pack" char * got_repo_get_path_git_dir(struct got_repository *repo) @@ -66,6 +68,12 @@ got_repo_get_path_objects(struct got_repository *repo) } char * +got_repo_get_path_objects_pack(struct got_repository *repo) +{ + return get_path_git_child(repo, GOT_OBJECTS_PACK_DIR); +} + +char * got_repo_get_path_refs(struct got_repository *repo) { return get_path_git_child(repo, GOT_REFS_DIR); blob - e6058f894daa451ecd640fb5d2a0e836c8237a19 blob + a4ddd9d3a24fd55f76b3178e99a7ee145660c9a7 --- lib/sha1.c +++ lib/sha1.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 Stefan Sperling + * Copyright (c) 2018 Stefan Sperling * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -58,3 +59,24 @@ got_parse_sha1_digest(uint8_t *digest, const char *lin return 1; } + +char * +got_sha1_digest_to_str(const uint8_t *digest, char *buf, size_t size) +{ + char *p = buf; + char hex[3]; + int i; + + if (size < SHA1_DIGEST_STRING_LENGTH) + return NULL; + + for (i = 0; i < SHA1_DIGEST_LENGTH; i++) { + snprintf(hex, sizeof(hex), "%.2x", digest[i]); + p[0] = hex[0]; + p[1] = hex[1]; + p += 2; + } + p[0] = '\0'; + + return buf; +} blob - 829b06d7cb1532e0343af4928d7ce406f568369b blob + 21b28bc395ccfabde390d43e0eccb28fe26e1dee --- regress/packfiles/Makefile +++ regress/packfiles/Makefile @@ -1,10 +1,10 @@ .PATH:${.CURDIR}/../../lib PROG = packfile_test -SRCS = error.c pack.c packfile_test.c +SRCS = error.c pack.c repository.c object.c path.c sha1.c packfile_test.c CPPFLAGS = -I${.CURDIR}/../../include -I${.CURDIR}/../../lib -LDADD = +LDADD = -lutil -lz DEBUG = -O0 -g NOMAN = yes blob - fbe9c7437fe13d12c10a9584e75e2894a0943d46 blob + 4217a2737bf95376fc1e1b96daed4ea4ec97123e --- regress/packfiles/packfile_test.c +++ regress/packfiles/packfile_test.c @@ -15,6 +15,7 @@ */ #include +#include #include #include @@ -22,10 +23,11 @@ #include #include "got_error.h" +#include "got_object.h" #include "pack.h" #define RUN_TEST(expr, name) \ - if (!(expr)) { printf("test %s failed", (name)); failure = 1; } + if (!(expr)) { printf("test %s failed\n", (name)); failure = 1; } #define GOT_REPO_PATH "../../../" blob - c1000921691f14bd6c3e1465b6e913321d140b5b blob + 29207702c74a55aadd942a3cbc380e667852e237 --- regress/repository/Makefile +++ regress/repository/Makefile @@ -2,7 +2,7 @@ PROG = repository_test SRCS = path.c repository.c error.c refs.c object.c sha1.c diff.c \ - diffreg.c xmalloc.c repository_test.c + diffreg.c xmalloc.c pack.c repository_test.c CPPFLAGS = -I${.CURDIR}/../../include LDADD = -lutil -lz blob - 6abb0c9084000310bda6227a507778b6316756c2 blob + 8fd30992c9434f52ee5debdc68bdf0bbf2a550c3 --- regress/repository/repository_test.c +++ regress/repository/repository_test.c @@ -30,7 +30,7 @@ #include "got_diff.h" #define RUN_TEST(expr, name) \ - if (!(expr)) { printf("test %s failed", (name)); failure = 1; } + if (!(expr)) { printf("test %s failed\n", (name)); failure = 1; } #define GOT_REPO_PATH "../../../"