Blob


1 /*
2 * Copyright (c) 2018 Stefan Sperling <stsp@openbsd.org>
3 *
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 */
17 /* An open pack file. */
18 struct got_pack {
19 char *path_packfile;
20 int fd;
21 uint8_t *map;
22 size_t filesize;
23 };
25 const struct got_error *got_pack_close(struct got_pack *);
27 #define GOT_PACK_PREFIX "pack-"
28 #define GOT_PACKFILE_SUFFIX ".pack"
29 #define GOT_PACKIDX_SUFFIX ".idx"
30 #define GOT_PACKFILE_NAMELEN (strlen(GOT_PACK_PREFIX) + \
31 SHA1_DIGEST_STRING_LENGTH - 1 + \
32 strlen(GOT_PACKFILE_SUFFIX))
33 #define GOT_PACKIDX_NAMELEN (strlen(GOT_PACK_PREFIX) + \
34 SHA1_DIGEST_STRING_LENGTH - 1 + \
35 strlen(GOT_PACKIDX_SUFFIX))
37 /* See Documentation/technical/pack-format.txt in Git. */
39 struct got_packidx_trailer {
40 u_int8_t packfile_sha1[SHA1_DIGEST_LENGTH];
41 u_int8_t packidx_sha1[SHA1_DIGEST_LENGTH];
42 } __attribute__((__packed__));
44 struct got_packidx_object_id {
45 u_int8_t sha1[SHA1_DIGEST_LENGTH];
46 } __attribute__((__packed__));
48 /* Ignore pack index version 1 which is no longer written by Git. */
49 #define GOT_PACKIDX_VERSION 2
51 struct got_packidx_v2_hdr {
52 uint32_t *magic; /* big endian */
53 #define GOT_PACKIDX_V2_MAGIC 0xff744f63 /* "\377t0c" */
54 uint32_t *version;
56 /*
57 * Each entry N in the fanout table contains the number of objects in
58 * the packfile whose SHA1 begins with a byte less than or equal to N.
59 * The last entry (index 255) contains the number of objects in the
60 * pack file whose first SHA1 byte is <= 0xff, and thus records the
61 * total number of objects in the pack file. All pointer variables
62 * below point to tables with a corresponding number of entries.
63 */
64 uint32_t *fanout_table; /* values are big endian */
65 #define GOT_PACKIDX_V2_FANOUT_TABLE_ITEMS (0xff + 1)
67 /* Sorted SHA1 checksums for each object in the pack file. */
68 struct got_packidx_object_id *sorted_ids;
70 /* CRC32 of the packed representation of each object. */
71 uint32_t *crc32;
73 /* Offset into the pack file for each object. */
74 uint32_t *offsets; /* values are big endian */
75 #define GOT_PACKIDX_OFFSET_VAL_MASK 0x7fffffff
76 #define GOT_PACKIDX_OFFSET_VAL_IS_LARGE_IDX 0x80000000
78 /* Large offsets table is empty for pack files < 2 GB. */
79 uint64_t *large_offsets; /* values are big endian */
81 struct got_packidx_trailer *trailer;
82 };
84 /* An open pack index file. */
85 struct got_packidx {
86 char *path_packidx; /* actual on-disk path */
87 int fd;
88 uint8_t *map;
89 size_t len;
90 struct got_packidx_v2_hdr hdr; /* convenient pointers into map */
91 };
93 struct got_packfile_hdr {
94 uint32_t signature;
95 #define GOT_PACKFILE_SIGNATURE 0x5041434b /* 'P' 'A' 'C' 'K' */
96 uint32_t version; /* big endian */
97 #define GOT_PACKFILE_VERSION 2
98 uint32_t nobjects; /* big endian */
99 };
101 struct got_packfile_obj_hdr {
102 /*
103 * The object size field uses a variable length encoding:
104 * size0...sizeN form a 4+7+7+...+7 bit integer, where size0 is the
105 * least significant part and sizeN is the most significant part.
106 * If the MSB of a size byte is set, an additional size byte follows.
107 * Of the 7 remaining bits of size0, the first 3 bits indicate the
108 * object's type, and the remaining 4 bits contribute to the size.
109 */
110 uint8_t *size; /* variable length */
111 #define GOT_PACK_OBJ_SIZE_MORE 0x80
112 #define GOT_PACK_OBJ_SIZE0_TYPE_MASK 0x70 /* See struct got_object->type */
113 #define GOT_PACK_OBJ_SIZE0_TYPE_MASK_SHIFT 4
114 #define GOT_PACK_OBJ_SIZE0_VAL_MASK 0x0f
115 #define GOT_PACK_OBJ_SIZE_VAL_MASK 0x7f
116 };
118 /* If object is not a DELTA type. */
119 struct got_packfile_object_data {
120 uint8_t *data; /* compressed */
121 };
123 /* If object is of type GOT_OBJ_TYPE_REF_DELTA. */
124 struct got_packfile_object_data_ref_delta {
125 uint8_t sha1[SHA1_DIGEST_LENGTH];
126 uint8_t *delta_data; /* compressed */
127 };
129 /* If object is of type GOT_OBJ_TYPE_OFFSET_DELTA. */
130 struct got_packfile_object_data_offset_delta {
131 /*
132 * This offset is interpreted as a negative offset from
133 * the got_packfile_obj_hdr corresponding to this object.
134 * The size provided in the header specifies the amount
135 * of compressed delta data that follows.
137 * This field uses a variable length encoding of N bytes,
138 * where the MSB is always set except for the last byte.
139 * The value is encoded as a series of N 7 bit integers,
140 * which are concatenated, and if N > 1 the value 2^7 +
141 * 2^14 + ... + 2^(7 * (n-1)) is added to the result.
142 */
143 uint8_t *offset; /* variable length */
144 #define GOT_PACK_OBJ_DELTA_OFF_MORE 0x80
145 #define GOT_PACK_OBJ_DELTA_OFF_VAL_MASK 0x7f
146 uint8_t *delta_data; /* compressed */
147 };
149 struct got_packfile_obj_data {
150 union {
151 struct got_packfile_object_data data;
152 struct got_packfile_object_data_ref_delta ref_delta;
153 struct got_packfile_object_data_offset_delta offset_delta;
154 } __attribute__((__packed__));
155 } __attribute__((__packed__));
157 const struct got_error *got_packidx_init_hdr(struct got_packidx *, int);
158 const struct got_error *got_packidx_open(struct got_packidx **,
159 const char *, int);
160 const struct got_error* got_packidx_close(struct got_packidx *);
161 int got_packidx_get_object_idx(struct got_packidx *, struct got_object_id *);
163 const struct got_error *got_packfile_open_object(struct got_object **,
164 struct got_pack *, struct got_packidx *, int, struct got_object_id *);
165 const struct got_error *got_packfile_extract_object(struct got_pack *,
166 struct got_object *, FILE *);
167 const struct got_error *got_packfile_extract_object_to_mem(uint8_t **, size_t *,
168 struct got_object *, struct got_pack *);
169 const struct got_error *got_pack_get_packfile_size(size_t *, const char *);
170 struct got_pack *got_repo_get_cached_pack(struct got_repository *, const char *);