1 /*
2  * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 struct got_pack_privsep_child {
18 	int imsg_fd;
19 	pid_t pid;
20 	struct imsgbuf *ibuf;
21 };
22 
23 /* An open pack file. */
24 struct got_pack {
25 	char *path_packfile;
26 	int fd;
27 	enum got_hash_algorithm algo;
28 	uint8_t *map;
29 	off_t filesize;
30 	struct got_pack_privsep_child *privsep_child;
31 	int basefd;
32 	int accumfd;
33 	int child_has_tempfiles;
34 	int child_has_delta_outfd;
35 	struct got_delta_cache *delta_cache;
36 };
37 
38 struct got_packidx;
39 
40 const struct got_error *got_pack_start_privsep_child(struct got_pack *,
41     struct got_packidx *);
42 const struct got_error *got_pack_close(struct got_pack *);
43 
44 const struct got_error *got_pack_parse_offset_delta(off_t *, size_t *,
45     struct got_pack *, off_t, size_t);
46 const struct got_error *got_pack_parse_ref_delta(struct got_object_id *,
47     struct got_pack *, off_t, int);
48 const struct got_error *got_pack_resolve_delta_chain(struct got_delta_chain *,
49     struct got_packidx *, struct got_pack *, off_t, size_t, int, size_t,
50     unsigned int);
51 const struct got_error *got_pack_parse_object_type_and_size(uint8_t *,
52     uint64_t *, size_t *, struct got_pack *, off_t);
53 
54 #define GOT_PACK_PREFIX		"pack-"
55 #define GOT_PACKFILE_SUFFIX	".pack"
56 #define GOT_PACKIDX_SUFFIX	".idx"
57 #define GOT_PACKFILE_NAMELEN	(strlen(GOT_PACK_PREFIX) + \
58 				SHA1_DIGEST_STRING_LENGTH - 1 + \
59 				strlen(GOT_PACKFILE_SUFFIX))
60 #define GOT_PACKIDX_NAMELEN(digest_len) \
61 				(strlen(GOT_PACK_PREFIX) + \
62 				digest_len - 1 + strlen(GOT_PACKIDX_SUFFIX))
63 
64 /* See Documentation/technical/pack-format.txt in Git. */
65 
66 struct got_packidx_trailer {
67 	uint8_t		packfile_hash[GOT_OBJECT_ID_MAXLEN];
68 	uint8_t		packidx_hash[GOT_OBJECT_ID_MAXLEN];
69 } __attribute__((__packed__));
70 
71 struct got_packidx_object_id {
72 	u_int8_t hash[GOT_OBJECT_ID_MAXLEN];
73 } __attribute__((__packed__));
74 
75 /* Ignore pack index version 1 which is no longer written by Git. */
76 #define GOT_PACKIDX_VERSION 2
77 
78 struct got_packidx_v2_hdr {
79 	uint32_t	*magic;		/* big endian */
80 #define GOT_PACKIDX_V2_MAGIC 0xff744f63	/* "\377t0c" */
81 	uint32_t	*version;
82 
83 	/*
84 	 * Each entry N in the fanout table contains the number of objects in
85 	 * the packfile whose SHA1 begins with a byte less than or equal to N.
86 	 * The last entry (index 255) contains the number of objects in the
87 	 * pack file whose first SHA1 byte is <= 0xff, and thus records the
88 	 * total number of objects in the pack file. All pointer variables
89 	 * below point to tables with a corresponding number of entries.
90 	 */
91 	uint32_t	*fanout_table;	/* values are big endian */
92 #define GOT_PACKIDX_V2_FANOUT_TABLE_ITEMS (0xff + 1)
93 
94 	/*
95 	 * Sorted hash checksums for each object in the pack file.
96 	 * Exact size depends on the repository object format.
97 	 */
98 	void *sorted_ids;
99 
100 	/* CRC32 of the packed representation of each object. */
101 	uint32_t	*crc32;
102 
103 	/* Offset into the pack file for each object. */
104 	uint32_t	*offsets;		/* values are big endian */
105 #define GOT_PACKIDX_OFFSET_VAL_MASK		0x7fffffff
106 #define GOT_PACKIDX_OFFSET_VAL_IS_LARGE_IDX	0x80000000
107 
108 	/* Large offsets table is empty for pack files < 2 GB. */
109 	uint64_t	*large_offsets;		/* values are big endian */
110 
111 	struct got_packidx_trailer trailer;
112 };
113 
114 struct got_pack_offset_index {
115 	uint32_t offset;
116 	uint32_t idx;
117 };
118 
119 struct got_pack_large_offset_index {
120 	uint64_t offset;
121 	uint32_t idx;
122 };
123 
124 /* An open pack index file. */
125 struct got_packidx {
126 	char *path_packidx; /* actual on-disk path */
127 	int fd;
128 	enum got_hash_algorithm algo;
129 	uint8_t *map;
130 	size_t len;
131 	size_t nlargeobj;
132 	struct got_packidx_v2_hdr hdr; /* convenient pointers into map */
133 	struct got_pack_offset_index *sorted_offsets;
134 	struct got_pack_large_offset_index *sorted_large_offsets;
135 };
136 
137 struct got_packfile_hdr {
138 	uint32_t	signature;
139 #define GOT_PACKFILE_SIGNATURE	0x5041434b	/* 'P' 'A' 'C' 'K' */
140 	uint32_t	version;	/* big endian */
141 #define GOT_PACKFILE_VERSION 2
142 	uint32_t	nobjects;	/* big endian */
143 };
144 
145 struct got_packfile_obj_hdr {
146 	/*
147 	 * The object size field uses a variable length encoding:
148 	 * size0...sizeN form a 4+7+7+...+7 bit integer, where size0 is the
149 	 * least significant part and sizeN is the most significant part.
150 	 * If the MSB of a size byte is set, an additional size byte follows.
151 	 * Of the 7 remaining bits of size0, the first 3 bits indicate the
152 	 * object's type, and the remaining 4 bits contribute to the size.
153 	 */
154 	uint8_t *size;		/* variable length */
155 #define GOT_PACK_OBJ_SIZE_MORE		0x80
156 #define GOT_PACK_OBJ_SIZE0_TYPE_MASK	0x70 /* See struct got_object->type */
157 #define GOT_PACK_OBJ_SIZE0_TYPE_MASK_SHIFT	4
158 #define GOT_PACK_OBJ_SIZE0_VAL_MASK	0x0f
159 #define GOT_PACK_OBJ_SIZE_VAL_MASK	0x7f
160 };
161 
162 /* If object is not a DELTA type. */
163 struct got_packfile_object_data {
164 	uint8_t *data;	/* compressed */
165 };
166 
167 /* If object is of type	GOT_OBJ_TYPE_REF_DELTA. */
168 struct got_packfile_object_data_ref_delta {
169 	uint8_t sha1[SHA1_DIGEST_LENGTH];
170 	uint8_t *delta_data;		/* compressed */
171 };
172 
173 /* If object is of type GOT_OBJ_TYPE_OFFSET_DELTA. */
174 struct got_packfile_object_data_offset_delta {
175 	/*
176 	 * This offset is interpreted as a negative offset from
177 	 * the got_packfile_obj_hdr corresponding to this object.
178 	 * The size provided in the header specifies the amount
179 	 * of compressed delta data that follows.
180 	 *
181 	 * This field uses a variable length encoding of N bytes,
182 	 * where the MSB is always set except for the last byte.
183 	 * The value is encoded as a series of N 7 bit integers,
184 	 * which are concatenated, and if N > 1 the value 2^7 +
185 	 * 2^14 + ... + 2^(7 * (n-1)) is added to the result.
186 	 */
187 	uint8_t *offset;	/* variable length */
188 #define GOT_PACK_OBJ_DELTA_OFF_MORE		0x80
189 #define GOT_PACK_OBJ_DELTA_OFF_VAL_MASK		0x7f
190 	uint8_t *delta_data;		/* compressed */
191 };
192 
193 struct got_packfile_obj_data {
194 	union {
195 		struct got_packfile_object_data data;
196 		struct got_packfile_object_data_ref_delta ref_delta;
197 		struct got_packfile_object_data_offset_delta offset_delta;
198 	} __attribute__((__packed__));
199 } __attribute__((__packed__));
200 
201 const struct got_error *got_packidx_init_hdr(struct got_packidx *, int, off_t);
202 const struct got_error *got_packidx_open(struct got_packidx **,
203     int, const char *, int, enum got_hash_algorithm);
204 const struct got_error *got_packidx_close(struct got_packidx *);
205 const struct got_error *got_packidx_get_packfile_path(char **, const char *);
206 off_t got_packidx_get_object_offset(struct got_packidx *, int idx);
207 int got_packidx_get_object_idx(struct got_packidx *, struct got_object_id *);
208 const struct got_error *got_packidx_get_offset_idx(int *, struct got_packidx *,
209     off_t);
210 const struct got_error *got_packidx_get_object_id(struct got_object_id *,
211     struct got_packidx *, int);
212 const struct got_error *got_packidx_match_id_str_prefix(
213     struct got_object_id_queue *, struct got_packidx *, const char *);
214 
215 const struct got_error *got_packfile_open_object(struct got_object **,
216     struct got_pack *, struct got_packidx *, int, struct got_object_id *);
217 const struct got_error *got_pack_get_delta_chain_max_size(uint64_t *,
218     struct got_delta_chain *, struct got_pack *);
219 const struct got_error *got_pack_get_max_delta_object_size(uint64_t *,
220     struct got_object *, struct got_pack *);
221 const struct got_error *got_pack_dump_delta_chain_to_file(size_t *,
222     struct got_delta_chain *, struct got_pack *, FILE *, FILE *, FILE *);
223 const struct got_error *got_pack_dump_delta_chain_to_mem(uint8_t **, size_t *,
224     struct got_delta_chain *, struct got_pack *);
225 const struct got_error *got_packfile_extract_object(struct got_pack *,
226     struct got_object *, FILE *, FILE *, FILE *);
227 const struct got_error *got_packfile_extract_object_to_mem(uint8_t **, size_t *,
228     struct got_object *, struct got_pack *);
229 const struct got_error *got_packfile_extract_raw_delta(uint8_t **, size_t *,
230     size_t *, off_t *, off_t *, off_t *, struct got_object_id *, uint64_t *,
231     uint64_t *, struct got_pack *, struct got_packidx *, int);