1 /*
2  * Copyright (c) 2020 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/types.h>
18 #include <sys/queue.h>
19 #include <sys/uio.h>
20 #include <sys/stat.h>
21 #include <sys/socket.h>
22 #include <sys/wait.h>
23 
24 #include <dirent.h>
25 #include <errno.h>
26 #include <fcntl.h>
27 #include <stdint.h>
28 #include <sha1.h>
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <limits.h>
33 #include <unistd.h>
34 #include <imsg.h>
35 
36 #include "got_error.h"
37 #include "got_cancel.h"
38 #include "got_object.h"
39 #include "got_reference.h"
40 #include "got_repository.h"
41 #include "got_repository_admin.h"
42 #include "got_opentemp.h"
43 #include "got_path.h"
44 
45 #include "got_lib_delta.h"
46 #include "got_lib_object.h"
47 #include "got_lib_object_cache.h"
48 #include "got_lib_pack.h"
49 #include "got_lib_privsep.h"
50 #include "got_lib_repository.h"
51 #include "got_lib_pack_create.h"
52 #include "got_lib_sha1.h"
53 
54 #ifndef nitems
55 #define nitems(_a)	(sizeof((_a)) / sizeof((_a)[0]))
56 #endif
57 
58 static const struct got_error *
59 get_reflist_object_ids(struct got_object_id ***ids, int *nobjects,
60     unsigned int wanted_obj_type_mask, struct got_reflist_head *refs,
61     struct got_repository *repo,
62     got_cancel_cb cancel_cb, void *cancel_arg)
63 {
64 	const struct got_error *err = NULL;
65 	const size_t alloc_chunksz = 256;
66 	size_t nalloc;
67 	struct got_reflist_entry *re;
68 	int i;
69 
70 	*ids = NULL;
71 	*nobjects = 0;
72 
73 	*ids = reallocarray(NULL, alloc_chunksz, sizeof(struct got_object_id *));
74 	if (*ids == NULL)
75 		return got_error_from_errno("reallocarray");
76 	nalloc = alloc_chunksz;
77 
78 	TAILQ_FOREACH(re, refs, entry) {
79 		struct got_object_id *id;
80 
81 		if (cancel_cb) {
82 			err = cancel_cb(cancel_arg);
83 			if (err)
84 				goto done;
85 		}
86 
87 		err = got_ref_resolve(&id, repo, re->ref);
88 		if (err)
89 			goto done;
90 
91 		if (wanted_obj_type_mask != GOT_OBJ_TYPE_ANY) {
92 			int obj_type;
93 			err = got_object_get_type(&obj_type, repo, id);
94 			if (err)
95 				goto done;
96 			if ((wanted_obj_type_mask & (1 << obj_type)) == 0) {
97 				free(id);
98 				id = NULL;
99 				continue;
100 			}
101 		}
102 
103 		if (nalloc >= *nobjects) {
104 			struct got_object_id **new;
105 			new = recallocarray(*ids, nalloc,
106 			    nalloc + alloc_chunksz,
107 			    sizeof(struct got_object_id *));
108 			if (new == NULL) {
109 				err = got_error_from_errno(
110 				    "recallocarray");
111 				goto done;
112 			}
113 			*ids = new;
114 			nalloc += alloc_chunksz;
115 		}
116 		(*ids)[*nobjects] = id;
117 		if ((*ids)[*nobjects] == NULL) {
118 			err = got_error_from_errno("got_object_id_dup");
119 			goto done;
120 		}
121 		(*nobjects)++;
122 	}
123 done:
124 	if (err) {
125 		for (i = 0; i < *nobjects; i++)
126 			free((*ids)[i]);
127 		free(*ids);
128 		*ids = NULL;
129 		*nobjects = 0;
130 	}
131 	return err;
132 }
133 
134 const struct got_error *
135 got_repo_pack_objects(FILE **packfile, struct got_object_id **pack_hash,
136     struct got_reflist_head *include_refs,
137     struct got_reflist_head *exclude_refs, struct got_repository *repo,
138     int loose_obj_only, got_pack_progress_cb progress_cb, void *progress_arg,
139     got_cancel_cb cancel_cb, void *cancel_arg)
140 {
141 	const struct got_error *err = NULL;
142 	struct got_object_id **ours = NULL, **theirs = NULL;
143 	int nours = 0, ntheirs = 0, packfd = -1, i;
144 	char *tmpfile_path = NULL, *path = NULL, *packfile_path = NULL;
145 	char *sha1_str = NULL;
146 
147 	*packfile = NULL;
148 	*pack_hash = NULL;
149 
150 	if (asprintf(&path, "%s/%s/packing.pack",
151 	    got_repo_get_path_git_dir(repo), GOT_OBJECTS_PACK_DIR) == -1) {
152 		err = got_error_from_errno("asprintf");
153 		goto done;
154 	}
155 	err = got_opentemp_named_fd(&tmpfile_path, &packfd, path);
156 	if (err)
157 		goto done;
158 
159 	if (fchmod(packfd, GOT_DEFAULT_FILE_MODE) != 0) {
160 		err = got_error_from_errno2("fchmod", tmpfile_path);
161 		goto done;
162 	}
163 
164 	*packfile = fdopen(packfd, "w");
165 	if (*packfile == NULL) {
166 		err = got_error_from_errno2("fdopen", tmpfile_path);
167 		goto done;
168 	}
169 	packfd = -1;
170 
171 	err = get_reflist_object_ids(&ours, &nours,
172 	   (1 << GOT_OBJ_TYPE_COMMIT) | (1 << GOT_OBJ_TYPE_TAG),
173 	   include_refs, repo, cancel_cb, cancel_arg);
174 	if (err)
175 		goto done;
176 
177 	if (nours == 0) {
178 		err = got_error(GOT_ERR_CANNOT_PACK);
179 		goto done;
180 	}
181 
182 	if (!TAILQ_EMPTY(exclude_refs)) {
183 		err = get_reflist_object_ids(&theirs, &ntheirs,
184 		   (1 << GOT_OBJ_TYPE_COMMIT) | (1 << GOT_OBJ_TYPE_TAG),
185 		   exclude_refs, repo,
186 		   cancel_cb, cancel_arg);
187 		if (err)
188 			goto done;
189 	}
190 
191 	*pack_hash = calloc(1, sizeof(**pack_hash));
192 	if (*pack_hash == NULL) {
193 		err = got_error_from_errno("calloc");
194 		goto done;
195 	}
196 
197 	err = got_pack_create((*pack_hash)->sha1, *packfile, theirs, ntheirs,
198 	    ours, nours, repo, loose_obj_only, progress_cb, progress_arg,
199 	    cancel_cb, cancel_arg);
200 	if (err)
201 		goto done;
202 
203 	err = got_object_id_str(&sha1_str, *pack_hash);
204 	if (err)
205 		goto done;
206 	if (asprintf(&packfile_path, "%s/%s/pack-%s.pack",
207 	    got_repo_get_path_git_dir(repo), GOT_OBJECTS_PACK_DIR,
208 	    sha1_str) == -1) {
209 		err = got_error_from_errno("asprintf");
210 		goto done;
211 	}
212 
213 	if (fflush(*packfile) == -1) {
214 		err = got_error_from_errno("fflush");
215 		goto done;
216 	}
217 	if (fseek(*packfile, 0L, SEEK_SET) == -1) {
218 		err = got_error_from_errno("fseek");
219 		goto done;
220 	}
221 	if (rename(tmpfile_path, packfile_path) == -1) {
222 		err = got_error_from_errno3("rename", tmpfile_path,
223 		    packfile_path);
224 		goto done;
225 	}
226 	free(tmpfile_path);
227 	tmpfile_path = NULL;
228 done:
229 	for (i = 0; i < nours; i++)
230 		free(ours[i]);
231 	free(ours);
232 	for (i = 0; i < ntheirs; i++)
233 		free(theirs[i]);
234 	free(theirs);
235 	if (packfd != -1 && close(packfd) == -1 && err == NULL)
236 		err = got_error_from_errno2("close", packfile_path);
237 	if (tmpfile_path && unlink(tmpfile_path) == -1 && err == NULL)
238 		err = got_error_from_errno2("unlink", tmpfile_path);
239 	free(tmpfile_path);
240 	free(packfile_path);
241 	free(sha1_str);
242 	free(path);
243 	if (err) {
244 		free(*pack_hash);
245 		*pack_hash = NULL;
246 		if (*packfile)
247 			fclose(*packfile);
248 		*packfile = NULL;
249 	}
250 	return err;
251 }
252 
253 const struct got_error *
254 got_repo_index_pack(FILE *packfile, struct got_object_id *pack_hash,
255     struct got_repository *repo,
256     got_pack_index_progress_cb progress_cb, void *progress_arg,
257     got_cancel_cb cancel_cb, void *cancel_arg)
258 {
259 	size_t i;
260 	char *path;
261 	int imsg_idxfds[2];
262 	int npackfd = -1, idxfd = -1, nidxfd = -1;
263 	int tmpfds[3];
264 	int idxstatus, done = 0;
265 	const struct got_error *err;
266 	struct imsgbuf idxibuf;
267 	pid_t idxpid;
268 	char *tmpidxpath = NULL;
269 	char *packfile_path = NULL, *idxpath = NULL, *id_str = NULL;
270 	const char *repo_path = got_repo_get_path_git_dir(repo);
271 	struct stat sb;
272 
273 	for (i = 0; i < nitems(tmpfds); i++)
274 		tmpfds[i] = -1;
275 
276 	if (asprintf(&path, "%s/%s/indexing.idx",
277 	    repo_path, GOT_OBJECTS_PACK_DIR) == -1) {
278 		err = got_error_from_errno("asprintf");
279 		goto done;
280 	}
281 	err = got_opentemp_named_fd(&tmpidxpath, &idxfd, path);
282 	free(path);
283 	if (err)
284 		goto done;
285 	if (fchmod(idxfd, GOT_DEFAULT_FILE_MODE) != 0) {
286 		err = got_error_from_errno2("fchmod", tmpidxpath);
287 		goto done;
288 	}
289 
290 	nidxfd = dup(idxfd);
291 	if (nidxfd == -1) {
292 		err = got_error_from_errno("dup");
293 		goto done;
294 	}
295 
296 	for (i = 0; i < nitems(tmpfds); i++) {
297 		tmpfds[i] = got_opentempfd();
298 		if (tmpfds[i] == -1) {
299 			err = got_error_from_errno("got_opentempfd");
300 			goto done;
301 		}
302 	}
303 
304 	err = got_object_id_str(&id_str, pack_hash);
305 	if (err)
306 		goto done;
307 
308 	if (asprintf(&packfile_path, "%s/%s/pack-%s.pack",
309 	    repo_path, GOT_OBJECTS_PACK_DIR, id_str) == -1) {
310 		err = got_error_from_errno("asprintf");
311 		goto done;
312 	}
313 
314 	if (fstat(fileno(packfile), &sb) == -1) {
315 		err = got_error_from_errno2("fstat", packfile_path);
316 		goto done;
317 	}
318 
319 	if (asprintf(&idxpath, "%s/%s/pack-%s.idx",
320 	    repo_path, GOT_OBJECTS_PACK_DIR, id_str) == -1) {
321 		err = got_error_from_errno("asprintf");
322 		goto done;
323 	}
324 
325 	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_idxfds) == -1) {
326 		err = got_error_from_errno("socketpair");
327 		goto done;
328 	}
329 	idxpid = fork();
330 	if (idxpid == -1) {
331 		err= got_error_from_errno("fork");
332 		goto done;
333 	} else if (idxpid == 0)
334 		got_privsep_exec_child(imsg_idxfds,
335 		    GOT_PATH_PROG_INDEX_PACK, packfile_path);
336 	if (close(imsg_idxfds[1]) == -1) {
337 		err = got_error_from_errno("close");
338 		goto done;
339 	}
340 	imsg_init(&idxibuf, imsg_idxfds[0]);
341 
342 	npackfd = dup(fileno(packfile));
343 	if (npackfd == -1) {
344 		err = got_error_from_errno("dup");
345 		goto done;
346 	}
347 	err = got_privsep_send_index_pack_req(&idxibuf, pack_hash->sha1,
348 	    npackfd);
349 	if (err != NULL)
350 		goto done;
351 	npackfd = -1;
352 	err = got_privsep_send_index_pack_outfd(&idxibuf, nidxfd);
353 	if (err != NULL)
354 		goto done;
355 	nidxfd = -1;
356 	for (i = 0; i < nitems(tmpfds); i++) {
357 		err = got_privsep_send_tmpfd(&idxibuf, tmpfds[i]);
358 		if (err != NULL)
359 			goto done;
360 		tmpfds[i] = -1;
361 	}
362 	done = 0;
363 	while (!done) {
364 		int nobj_total, nobj_indexed, nobj_loose, nobj_resolved;
365 
366 		if (cancel_cb) {
367 			err = cancel_cb(cancel_arg);
368 			if (err)
369 				goto done;
370 		}
371 
372 		err = got_privsep_recv_index_progress(&done, &nobj_total,
373 		    &nobj_indexed, &nobj_loose, &nobj_resolved,
374 		    &idxibuf);
375 		if (err != NULL)
376 			goto done;
377 		if (nobj_indexed != 0) {
378 			err = progress_cb(progress_arg, sb.st_size,
379 			    nobj_total, nobj_indexed, nobj_loose,
380 			    nobj_resolved);
381 			if (err)
382 				break;
383 		}
384 		imsg_clear(&idxibuf);
385 	}
386 	if (close(imsg_idxfds[0]) == -1) {
387 		err = got_error_from_errno("close");
388 		goto done;
389 	}
390 	if (waitpid(idxpid, &idxstatus, 0) == -1) {
391 		err = got_error_from_errno("waitpid");
392 		goto done;
393 	}
394 
395 	if (rename(tmpidxpath, idxpath) == -1) {
396 		err = got_error_from_errno3("rename", tmpidxpath, idxpath);
397 		goto done;
398 	}
399 	free(tmpidxpath);
400 	tmpidxpath = NULL;
401 
402 done:
403 	if (tmpidxpath && unlink(tmpidxpath) == -1 && err == NULL)
404 		err = got_error_from_errno2("unlink", tmpidxpath);
405 	if (npackfd != -1 && close(npackfd) == -1 && err == NULL)
406 		err = got_error_from_errno("close");
407 	if (idxfd != -1 && close(idxfd) == -1 && err == NULL)
408 		err = got_error_from_errno("close");
409 	for (i = 0; i < nitems(tmpfds); i++) {
410 		if (tmpfds[i] != -1 && close(tmpfds[i]) == -1 && err == NULL)
411 			err = got_error_from_errno("close");
412 	}
413 	free(tmpidxpath);
414 	free(idxpath);
415 	free(packfile_path);
416 	return err;
417 }
418 
419 const struct got_error *
420 got_repo_find_pack(FILE **packfile, struct got_object_id **pack_hash,
421     struct got_repository *repo, const char *packfile_path)
422 {
423 	const struct got_error *err = NULL;
424 	const char *packdir_path = NULL;
425 	char *packfile_name = NULL, *p, *dot;
426 	struct got_object_id id;
427 	int packfd = -1;
428 
429 	*packfile = NULL;
430 	*pack_hash = NULL;
431 
432 	packdir_path = got_repo_get_path_objects_pack(repo);
433 	if (packdir_path == NULL)
434 		return got_error_from_errno("got_repo_get_path_objects_pack");
435 
436 	if (!got_path_is_child(packfile_path, packdir_path,
437 	    strlen(packdir_path))) {
438 		err = got_error_path(packfile_path, GOT_ERR_BAD_PATH);
439 		goto done;
440 
441 	}
442 
443 	err = got_path_basename(&packfile_name, packfile_path);
444 	if (err)
445 		goto done;
446 	p = packfile_name;
447 
448 	if (strncmp(p, "pack-", 5) != 0) {
449 		err = got_error_fmt(GOT_ERR_BAD_PATH,
450 		   "'%s' is not a valid pack file name",
451 		   packfile_name);
452 		goto done;
453 	}
454 	p += 5;
455 	dot = strchr(p, '.');
456 	if (dot == NULL) {
457 		err = got_error_fmt(GOT_ERR_BAD_PATH,
458 		   "'%s' is not a valid pack file name",
459 		   packfile_name);
460 		goto done;
461 	}
462 	if (strcmp(dot + 1, "pack") != 0) {
463 		err = got_error_fmt(GOT_ERR_BAD_PATH,
464 		   "'%s' is not a valid pack file name",
465 		   packfile_name);
466 		goto done;
467 	}
468 	*dot = '\0';
469 	if (!got_parse_sha1_digest(id.sha1, p)) {
470 		err = got_error_fmt(GOT_ERR_BAD_PATH,
471 		   "'%s' is not a valid pack file name",
472 		   packfile_name);
473 		goto done;
474 	}
475 
476 	*pack_hash = got_object_id_dup(&id);
477 	if (*pack_hash == NULL) {
478 		err = got_error_from_errno("got_object_id_dup");
479 		goto done;
480 	}
481 
482 	packfd = open(packfile_path, O_RDONLY | O_NOFOLLOW);
483 	if (packfd == -1) {
484 		err = got_error_from_errno2("open", packfile_path);
485 		goto done;
486 	}
487 
488 	*packfile = fdopen(packfd, "r");
489 	if (*packfile == NULL) {
490 		err = got_error_from_errno2("fdopen", packfile_path);
491 		goto done;
492 	}
493 	packfd = -1;
494 done:
495 	if (packfd != -1 && close(packfd) == -1 && err == NULL)
496 		err = got_error_from_errno2("close", packfile_path);
497 	free(packfile_name);
498 	if (err) {
499 		free(*pack_hash);
500 		*pack_hash = NULL;
501 	}
502 	return err;
503 }
504 
505 const struct got_error *
506 got_repo_list_pack(FILE *packfile, struct got_object_id *pack_hash,
507     struct got_repository *repo, got_pack_list_cb list_cb, void *list_arg,
508     got_cancel_cb cancel_cb, void *cancel_arg)
509 {
510 	const struct got_error *err = NULL;
511 	char *id_str = NULL, *idxpath = NULL, *packpath = NULL;
512 	struct got_packidx *packidx = NULL;
513 	struct got_pack *pack = NULL;
514 	uint32_t nobj, i;
515 
516 	err = got_object_id_str(&id_str, pack_hash);
517 	if (err)
518 		goto done;
519 
520 	if (asprintf(&packpath, "%s/pack-%s.pack",
521 	    GOT_OBJECTS_PACK_DIR, id_str) == -1) {
522 		err = got_error_from_errno("asprintf");
523 		goto done;
524 	}
525 	if (asprintf(&idxpath, "%s/pack-%s.idx",
526 	    GOT_OBJECTS_PACK_DIR, id_str) == -1) {
527 		err = got_error_from_errno("asprintf");
528 		goto done;
529 	}
530 
531 	err = got_packidx_open(&packidx, got_repo_get_fd(repo), idxpath, 1);
532 	if (err)
533 		goto done;
534 
535 	err = got_repo_cache_pack(&pack, repo, packpath, packidx);
536 	if (err)
537 		goto done;
538 
539 	nobj = be32toh(packidx->hdr.fanout_table[0xff]);
540 	for (i = 0; i < nobj; i++) {
541 		struct got_packidx_object_id *oid;
542 		struct got_object_id id, base_id;
543 		off_t offset, base_offset = 0;
544 		uint8_t type;
545 		uint64_t size;
546 		size_t tslen, len;
547 
548 		if (cancel_cb) {
549 			err = cancel_cb(cancel_arg);
550 			if (err)
551 				break;
552 		}
553 		oid = &packidx->hdr.sorted_ids[i];
554 		memcpy(id.sha1, oid->sha1, SHA1_DIGEST_LENGTH);
555 
556 		offset = got_packidx_get_object_offset(packidx, i);
557 		if (offset == -1) {
558 			err = got_error(GOT_ERR_BAD_PACKIDX);
559 			goto done;
560 		}
561 
562 		err = got_pack_parse_object_type_and_size(&type, &size, &tslen,
563 		    pack, offset);
564 		if (err)
565 			goto done;
566 
567 		switch (type) {
568 		case GOT_OBJ_TYPE_OFFSET_DELTA:
569 			err = got_pack_parse_offset_delta(&base_offset, &len,
570 			    pack, offset, tslen);
571 			if (err)
572 				goto done;
573 			break;
574 		case GOT_OBJ_TYPE_REF_DELTA:
575 			err = got_pack_parse_ref_delta(&base_id,
576 			    pack, offset, tslen);
577 			if (err)
578 				goto done;
579 			break;
580 		}
581 		err = (*list_cb)(list_arg, &id, type, offset, size,
582 		    base_offset, &base_id);
583 		if (err)
584 			goto done;
585 	}
586 
587 done:
588 	free(id_str);
589 	free(idxpath);
590 	free(packpath);
591 	if (packidx)
592 		got_packidx_close(packidx);
593 	return err;
594 }