1 /*
2  * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 /* An open pack file. */
18 struct got_pack {
19 	char *path_packfile;
20 	int fd;
21 	uint8_t *map;
22 	size_t filesize;
23 	struct got_privsep_child *privsep_child;
24 	struct got_delta_cache *delta_cache;
25 };
26 
27 const struct got_error *got_pack_stop_privsep_child(struct got_pack *);
28 const struct got_error *got_pack_close(struct got_pack *);
29 
30 const struct got_error *got_pack_parse_offset_delta(off_t *, size_t *,
31     struct got_pack *, off_t, int);
32 const struct got_error *got_pack_resolve_delta_chain(struct got_delta_chain *,
33     struct got_packidx *, struct got_pack *, off_t, size_t, int, size_t,
34     unsigned int);
35 const struct got_error *got_pack_parse_object_type_and_size(uint8_t *,
36     uint64_t *, size_t *, struct got_pack *, off_t);
37 
38 #define GOT_PACK_PREFIX		"pack-"
39 #define GOT_PACKFILE_SUFFIX	".pack"
40 #define GOT_PACKIDX_SUFFIX		".idx"
41 #define GOT_PACKFILE_NAMELEN	(strlen(GOT_PACK_PREFIX) + \
42 				SHA1_DIGEST_STRING_LENGTH - 1 + \
43 				strlen(GOT_PACKFILE_SUFFIX))
44 #define GOT_PACKIDX_NAMELEN	(strlen(GOT_PACK_PREFIX) + \
45 				SHA1_DIGEST_STRING_LENGTH - 1 + \
46 				strlen(GOT_PACKIDX_SUFFIX))
47 
48 /* See Documentation/technical/pack-format.txt in Git. */
49 
50 struct got_packidx_trailer {
51 	u_int8_t	packfile_sha1[SHA1_DIGEST_LENGTH];
52 	u_int8_t	packidx_sha1[SHA1_DIGEST_LENGTH];
53 } __attribute__((__packed__));
54 
55 struct got_packidx_object_id {
56 	u_int8_t sha1[SHA1_DIGEST_LENGTH];
57 } __attribute__((__packed__));
58 
59 /* Ignore pack index version 1 which is no longer written by Git. */
60 #define GOT_PACKIDX_VERSION 2
61 
62 struct got_packidx_v2_hdr {
63 	uint32_t	*magic;		/* big endian */
64 #define GOT_PACKIDX_V2_MAGIC 0xff744f63	/* "\377t0c" */
65 	uint32_t	*version;
66 
67 	/* 
68 	 * Each entry N in the fanout table contains the number of objects in
69 	 * the packfile whose SHA1 begins with a byte less than or equal to N.
70 	 * The last entry (index 255) contains the number of objects in the
71 	 * pack file whose first SHA1 byte is <= 0xff, and thus records the
72 	 * total number of objects in the pack file. All pointer variables
73 	 * below point to tables with a corresponding number of entries.
74 	 */
75 	uint32_t	*fanout_table;	/* values are big endian */
76 #define GOT_PACKIDX_V2_FANOUT_TABLE_ITEMS (0xff + 1)
77 
78 	/* Sorted SHA1 checksums for each object in the pack file. */
79 	struct got_packidx_object_id *sorted_ids;
80 
81 	/* CRC32 of the packed representation of each object. */
82 	uint32_t	*crc32;
83 
84 	/* Offset into the pack file for each object. */
85 	uint32_t	*offsets;		/* values are big endian */
86 #define GOT_PACKIDX_OFFSET_VAL_MASK		0x7fffffff
87 #define GOT_PACKIDX_OFFSET_VAL_IS_LARGE_IDX	0x80000000
88 
89 	/* Large offsets table is empty for pack files < 2 GB. */
90 	uint64_t	*large_offsets;		/* values are big endian */
91 
92 	struct got_packidx_trailer *trailer;
93 };
94 
95 /* An open pack index file. */
96 struct got_packidx {
97 	char *path_packidx; /* actual on-disk path */
98 	int fd;
99 	uint8_t *map;
100 	size_t len;
101 	size_t nlargeobj;
102 	struct got_packidx_v2_hdr hdr; /* convenient pointers into map */
103 };
104 
105 struct got_packfile_hdr {
106 	uint32_t	signature;
107 #define GOT_PACKFILE_SIGNATURE	0x5041434b	/* 'P' 'A' 'C' 'K' */
108 	uint32_t	version;	/* big endian */
109 #define GOT_PACKFILE_VERSION 2
110 	uint32_t	nobjects;	/* big endian */
111 };
112 
113 struct got_packfile_obj_hdr {
114 	/* 
115 	 * The object size field uses a variable length encoding:
116 	 * size0...sizeN form a 4+7+7+...+7 bit integer, where size0 is the
117 	 * least significant part and sizeN is the most significant part.
118 	 * If the MSB of a size byte is set, an additional size byte follows.
119 	 * Of the 7 remaining bits of size0, the first 3 bits indicate the
120 	 * object's type, and the remaining 4 bits contribute to the size.
121 	 */
122 	uint8_t *size;		/* variable length */
123 #define GOT_PACK_OBJ_SIZE_MORE		0x80
124 #define GOT_PACK_OBJ_SIZE0_TYPE_MASK	0x70 /* See struct got_object->type */
125 #define GOT_PACK_OBJ_SIZE0_TYPE_MASK_SHIFT	4
126 #define GOT_PACK_OBJ_SIZE0_VAL_MASK	0x0f
127 #define GOT_PACK_OBJ_SIZE_VAL_MASK	0x7f
128 };
129 
130 /* If object is not a DELTA type. */
131 struct got_packfile_object_data {
132 	uint8_t *data;	/* compressed */
133 };
134 
135 /* If object is of type	GOT_OBJ_TYPE_REF_DELTA. */
136 struct got_packfile_object_data_ref_delta {
137 	uint8_t sha1[SHA1_DIGEST_LENGTH];
138 	uint8_t *delta_data;		/* compressed */
139 };
140 
141 /* If object is of type GOT_OBJ_TYPE_OFFSET_DELTA. */
142 struct got_packfile_object_data_offset_delta {
143 	/* 
144 	 * This offset is interpreted as a negative offset from
145 	 * the got_packfile_obj_hdr corresponding to this object.
146 	 * The size provided in the header specifies the amount
147 	 * of compressed delta data that follows.
148 	 *
149 	 * This field uses a variable length encoding of N bytes,
150 	 * where the MSB is always set except for the last byte.
151 	 * The value is encoded as a series of N 7 bit integers,
152 	 * which are concatenated, and if N > 1 the value 2^7 +
153 	 * 2^14 + ... + 2^(7 * (n-1)) is added to the result.
154 	 */
155 	uint8_t *offset;	/* variable length */
156 #define GOT_PACK_OBJ_DELTA_OFF_MORE		0x80
157 #define GOT_PACK_OBJ_DELTA_OFF_VAL_MASK		0x7f
158 	uint8_t *delta_data;		/* compressed */
159 };
160 
161 struct got_packfile_obj_data {
162 	union {
163 		struct got_packfile_object_data data;
164 		struct got_packfile_object_data_ref_delta ref_delta;
165 		struct got_packfile_object_data_offset_delta offset_delta;
166 	} __attribute__((__packed__));
167 } __attribute__((__packed__));
168 
169 const struct got_error *got_packidx_init_hdr(struct got_packidx *, int);
170 const struct got_error *got_packidx_open(struct got_packidx **,
171     int, const char *, int);
172 const struct got_error *got_packidx_close(struct got_packidx *);
173 const struct got_error *got_packidx_get_packfile_path(char **, struct got_packidx *);
174 int got_packidx_get_object_idx(struct got_packidx *, struct got_object_id *);
175 const struct got_error *got_packidx_match_id_str_prefix(
176     struct got_object_id_queue *, struct got_packidx *, const char *);
177 
178 const struct got_error *got_packfile_open_object(struct got_object **,
179     struct got_pack *, struct got_packidx *, int, struct got_object_id *);
180 const struct got_error *got_pack_get_delta_chain_max_size(uint64_t *,
181     struct got_delta_chain *, struct got_pack *);
182 const struct got_error *got_pack_get_max_delta_object_size(uint64_t *,
183     struct got_object *, struct got_pack *);
184 const struct got_error *got_pack_dump_delta_chain_to_file(size_t *,
185     struct got_delta_chain *, struct got_pack *, FILE *, FILE *, FILE *);
186 const struct got_error *got_pack_dump_delta_chain_to_mem(uint8_t **, size_t *,
187     struct got_delta_chain *, struct got_pack *);
188 const struct got_error *got_packfile_extract_object(struct got_pack *,
189     struct got_object *, FILE *, FILE *, FILE *);
190 const struct got_error *got_packfile_extract_object_to_mem(uint8_t **, size_t *,
191     struct got_object *, struct got_pack *);
192 struct got_pack *got_repo_get_cached_pack(struct got_repository *, const char *);