Blame


1 93658fb9 2020-03-18 stsp /*
2 93658fb9 2020-03-18 stsp * Copyright (c) 2019 Ori Bernstein <ori@openbsd.org>
3 668a20f6 2020-03-18 stsp * Copyright (c) 2020 Stefan Sperling <stsp@openbsd.org>
4 93658fb9 2020-03-18 stsp *
5 93658fb9 2020-03-18 stsp * Permission to use, copy, modify, and distribute this software for any
6 93658fb9 2020-03-18 stsp * purpose with or without fee is hereby granted, provided that the above
7 93658fb9 2020-03-18 stsp * copyright notice and this permission notice appear in all copies.
8 93658fb9 2020-03-18 stsp *
9 93658fb9 2020-03-18 stsp * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 93658fb9 2020-03-18 stsp * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 93658fb9 2020-03-18 stsp * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 93658fb9 2020-03-18 stsp * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 93658fb9 2020-03-18 stsp * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 93658fb9 2020-03-18 stsp * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 93658fb9 2020-03-18 stsp * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16 93658fb9 2020-03-18 stsp */
17 93658fb9 2020-03-18 stsp
18 93658fb9 2020-03-18 stsp #include <sys/types.h>
19 0136599f 2022-10-20 stsp #include <sys/queue.h>
20 2e5a6fad 2020-03-18 stsp #include <sys/mman.h>
21 0136599f 2022-10-20 stsp #include <sys/uio.h>
22 93658fb9 2020-03-18 stsp
23 0136599f 2022-10-20 stsp #include <sha1.h>
24 5822e79e 2023-02-23 op #include <sha2.h>
25 93658fb9 2020-03-18 stsp #include <stdint.h>
26 93658fb9 2020-03-18 stsp #include <stdio.h>
27 93658fb9 2020-03-18 stsp #include <stdlib.h>
28 93658fb9 2020-03-18 stsp #include <string.h>
29 0136599f 2022-10-20 stsp #include <imsg.h>
30 0136599f 2022-10-20 stsp #include <limits.h>
31 0136599f 2022-10-20 stsp #include <time.h>
32 81a12da5 2020-09-09 naddy #include <unistd.h>
33 93658fb9 2020-03-18 stsp
34 93658fb9 2020-03-18 stsp #include "got_error.h"
35 93658fb9 2020-03-18 stsp #include "got_object.h"
36 93658fb9 2020-03-18 stsp
37 93658fb9 2020-03-18 stsp #include "got_lib_delta.h"
38 0136599f 2022-10-20 stsp #include "got_lib_delta_cache.h"
39 ae25a666 2023-02-23 op #include "got_lib_hash.h"
40 93658fb9 2020-03-18 stsp #include "got_lib_object.h"
41 93658fb9 2020-03-18 stsp #include "got_lib_privsep.h"
42 d0f1e2f1 2022-02-23 stsp #include "got_lib_ratelimit.h"
43 0136599f 2022-10-20 stsp #include "got_lib_pack.h"
44 0136599f 2022-10-20 stsp #include "got_lib_pack_index.h"
45 93658fb9 2020-03-18 stsp
46 d582f26c 2020-03-18 stsp #ifndef nitems
47 d582f26c 2020-03-18 stsp #define nitems(_a) (sizeof((_a)) / sizeof((_a)[0]))
48 d582f26c 2020-03-18 stsp #endif
49 d582f26c 2020-03-18 stsp
50 668a20f6 2020-03-18 stsp static const struct got_error *
51 9316cc27 2022-10-20 stsp send_index_pack_progress(void *arg, uint32_t nobj_total, uint32_t nobj_indexed,
52 9316cc27 2022-10-20 stsp uint32_t nobj_loose, uint32_t nobj_resolved)
53 93658fb9 2020-03-18 stsp {
54 0136599f 2022-10-20 stsp struct imsgbuf *ibuf = arg;
55 e70bf110 2020-03-22 stsp struct got_imsg_index_pack_progress iprogress;
56 d0f1e2f1 2022-02-23 stsp
57 e70bf110 2020-03-22 stsp iprogress.nobj_total = nobj_total;
58 e70bf110 2020-03-22 stsp iprogress.nobj_indexed = nobj_indexed;
59 e70bf110 2020-03-22 stsp iprogress.nobj_loose = nobj_loose;
60 e70bf110 2020-03-22 stsp iprogress.nobj_resolved = nobj_resolved;
61 e70bf110 2020-03-22 stsp
62 e70bf110 2020-03-22 stsp if (imsg_compose(ibuf, GOT_IMSG_IDXPACK_PROGRESS, 0, 0, -1,
63 e70bf110 2020-03-22 stsp &iprogress, sizeof(iprogress)) == -1)
64 e70bf110 2020-03-22 stsp return got_error_from_errno("imsg_compose IDXPACK_PROGRESS");
65 e70bf110 2020-03-22 stsp
66 e70bf110 2020-03-22 stsp return got_privsep_flush_imsg(ibuf);
67 e70bf110 2020-03-22 stsp }
68 e70bf110 2020-03-22 stsp
69 e70bf110 2020-03-22 stsp static const struct got_error *
70 e70bf110 2020-03-22 stsp send_index_pack_done(struct imsgbuf *ibuf)
71 e70bf110 2020-03-22 stsp {
72 e70bf110 2020-03-22 stsp if (imsg_compose(ibuf, GOT_IMSG_IDXPACK_DONE, 0, 0, -1, NULL, 0) == -1)
73 e70bf110 2020-03-22 stsp return got_error_from_errno("imsg_compose FETCH");
74 e70bf110 2020-03-22 stsp return got_privsep_flush_imsg(ibuf);
75 950de2cd 2020-03-18 stsp }
76 950de2cd 2020-03-18 stsp
77 e70bf110 2020-03-22 stsp
78 93658fb9 2020-03-18 stsp int
79 93658fb9 2020-03-18 stsp main(int argc, char **argv)
80 93658fb9 2020-03-18 stsp {
81 668a20f6 2020-03-18 stsp const struct got_error *err = NULL, *close_err;
82 93658fb9 2020-03-18 stsp struct imsgbuf ibuf;
83 93658fb9 2020-03-18 stsp struct imsg imsg;
84 6059809a 2020-12-17 stsp size_t i;
85 6059809a 2020-12-17 stsp int idxfd = -1, tmpfd = -1;
86 d582f26c 2020-03-18 stsp FILE *tmpfiles[3];
87 668a20f6 2020-03-18 stsp struct got_pack pack;
88 668a20f6 2020-03-18 stsp uint8_t pack_hash[SHA1_DIGEST_LENGTH];
89 668a20f6 2020-03-18 stsp off_t packfile_size;
90 713d6e11 2022-10-20 stsp struct got_ratelimit rl;
91 668a20f6 2020-03-18 stsp #if 0
92 668a20f6 2020-03-18 stsp static int attached;
93 668a20f6 2020-03-18 stsp while (!attached)
94 668a20f6 2020-03-18 stsp sleep(1);
95 668a20f6 2020-03-18 stsp #endif
96 93658fb9 2020-03-18 stsp
97 713d6e11 2022-10-20 stsp got_ratelimit_init(&rl, 0, 500);
98 713d6e11 2022-10-20 stsp
99 d582f26c 2020-03-18 stsp for (i = 0; i < nitems(tmpfiles); i++)
100 d582f26c 2020-03-18 stsp tmpfiles[i] = NULL;
101 d582f26c 2020-03-18 stsp
102 668a20f6 2020-03-18 stsp memset(&pack, 0, sizeof(pack));
103 668a20f6 2020-03-18 stsp pack.fd = -1;
104 dac5c75e 2022-06-04 stsp err = got_delta_cache_alloc(&pack.delta_cache);
105 dac5c75e 2022-06-04 stsp if (err)
106 93658fb9 2020-03-18 stsp goto done;
107 668a20f6 2020-03-18 stsp
108 668a20f6 2020-03-18 stsp imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
109 861f3006 2020-03-18 stsp #ifndef PROFILE
110 861f3006 2020-03-18 stsp /* revoke access to most system calls */
111 861f3006 2020-03-18 stsp if (pledge("stdio recvfd", NULL) == -1) {
112 861f3006 2020-03-18 stsp err = got_error_from_errno("pledge");
113 861f3006 2020-03-18 stsp got_privsep_send_error(&ibuf, err);
114 861f3006 2020-03-18 stsp return 1;
115 861f3006 2020-03-18 stsp }
116 861f3006 2020-03-18 stsp #endif
117 668a20f6 2020-03-18 stsp err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
118 668a20f6 2020-03-18 stsp if (err)
119 668a20f6 2020-03-18 stsp goto done;
120 93658fb9 2020-03-18 stsp if (imsg.hdr.type == GOT_IMSG_STOP)
121 93658fb9 2020-03-18 stsp goto done;
122 93658fb9 2020-03-18 stsp if (imsg.hdr.type != GOT_IMSG_IDXPACK_REQUEST) {
123 93658fb9 2020-03-18 stsp err = got_error(GOT_ERR_PRIVSEP_MSG);
124 93658fb9 2020-03-18 stsp goto done;
125 93658fb9 2020-03-18 stsp }
126 668a20f6 2020-03-18 stsp if (imsg.hdr.len - IMSG_HEADER_SIZE != sizeof(pack_hash)) {
127 93658fb9 2020-03-18 stsp err = got_error(GOT_ERR_PRIVSEP_LEN);
128 93658fb9 2020-03-18 stsp goto done;
129 93658fb9 2020-03-18 stsp }
130 668a20f6 2020-03-18 stsp memcpy(pack_hash, imsg.data, sizeof(pack_hash));
131 668a20f6 2020-03-18 stsp pack.fd = imsg.fd;
132 93658fb9 2020-03-18 stsp
133 668a20f6 2020-03-18 stsp err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
134 668a20f6 2020-03-18 stsp if (err)
135 93658fb9 2020-03-18 stsp goto done;
136 93658fb9 2020-03-18 stsp if (imsg.hdr.type == GOT_IMSG_STOP)
137 93658fb9 2020-03-18 stsp goto done;
138 73ab1060 2020-03-18 stsp if (imsg.hdr.type != GOT_IMSG_IDXPACK_OUTFD) {
139 93658fb9 2020-03-18 stsp err = got_error(GOT_ERR_PRIVSEP_MSG);
140 93658fb9 2020-03-18 stsp goto done;
141 93658fb9 2020-03-18 stsp }
142 93658fb9 2020-03-18 stsp if (imsg.hdr.len - IMSG_HEADER_SIZE != 0) {
143 93658fb9 2020-03-18 stsp err = got_error(GOT_ERR_PRIVSEP_LEN);
144 93658fb9 2020-03-18 stsp goto done;
145 93658fb9 2020-03-18 stsp }
146 93658fb9 2020-03-18 stsp idxfd = imsg.fd;
147 4788f1ce 2020-03-18 stsp
148 d582f26c 2020-03-18 stsp for (i = 0; i < nitems(tmpfiles); i++) {
149 d582f26c 2020-03-18 stsp err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
150 d582f26c 2020-03-18 stsp if (err)
151 d582f26c 2020-03-18 stsp goto done;
152 d582f26c 2020-03-18 stsp if (imsg.hdr.type == GOT_IMSG_STOP)
153 d582f26c 2020-03-18 stsp goto done;
154 d582f26c 2020-03-18 stsp if (imsg.hdr.type != GOT_IMSG_TMPFD) {
155 d582f26c 2020-03-18 stsp err = got_error(GOT_ERR_PRIVSEP_MSG);
156 d582f26c 2020-03-18 stsp goto done;
157 d582f26c 2020-03-18 stsp }
158 d582f26c 2020-03-18 stsp if (imsg.hdr.len - IMSG_HEADER_SIZE != 0) {
159 d582f26c 2020-03-18 stsp err = got_error(GOT_ERR_PRIVSEP_LEN);
160 d582f26c 2020-03-18 stsp goto done;
161 d582f26c 2020-03-18 stsp }
162 d582f26c 2020-03-18 stsp tmpfd = imsg.fd;
163 d582f26c 2020-03-18 stsp tmpfiles[i] = fdopen(tmpfd, "w+");
164 d582f26c 2020-03-18 stsp if (tmpfiles[i] == NULL) {
165 d582f26c 2020-03-18 stsp err = got_error_from_errno("fdopen");
166 d582f26c 2020-03-18 stsp goto done;
167 d582f26c 2020-03-18 stsp }
168 d582f26c 2020-03-18 stsp tmpfd = -1;
169 4788f1ce 2020-03-18 stsp }
170 93658fb9 2020-03-18 stsp
171 668a20f6 2020-03-18 stsp if (lseek(pack.fd, 0, SEEK_END) == -1) {
172 668a20f6 2020-03-18 stsp err = got_error_from_errno("lseek");
173 668a20f6 2020-03-18 stsp goto done;
174 668a20f6 2020-03-18 stsp }
175 668a20f6 2020-03-18 stsp packfile_size = lseek(pack.fd, 0, SEEK_CUR);
176 668a20f6 2020-03-18 stsp if (packfile_size == -1) {
177 668a20f6 2020-03-18 stsp err = got_error_from_errno("lseek");
178 668a20f6 2020-03-18 stsp goto done;
179 668a20f6 2020-03-18 stsp }
180 ad4cc361 2022-10-27 op pack.filesize = packfile_size;
181 668a20f6 2020-03-18 stsp
182 668a20f6 2020-03-18 stsp if (lseek(pack.fd, 0, SEEK_SET) == -1) {
183 668a20f6 2020-03-18 stsp err = got_error_from_errno("lseek");
184 668a20f6 2020-03-18 stsp goto done;
185 668a20f6 2020-03-18 stsp }
186 668a20f6 2020-03-18 stsp
187 2e5a6fad 2020-03-18 stsp #ifndef GOT_PACK_NO_MMAP
188 1c28a361 2022-10-25 op if (pack.filesize > 0 && pack.filesize <= SIZE_MAX) {
189 1c28a361 2022-10-25 op pack.map = mmap(NULL, pack.filesize, PROT_READ, MAP_PRIVATE,
190 1c28a361 2022-10-25 op pack.fd, 0);
191 1c28a361 2022-10-25 op if (pack.map == MAP_FAILED)
192 1c28a361 2022-10-25 op pack.map = NULL; /* fall back to read(2) */
193 1c28a361 2022-10-25 op }
194 2e5a6fad 2020-03-18 stsp #endif
195 713d6e11 2022-10-20 stsp err = got_pack_index(&pack, idxfd, tmpfiles[0], tmpfiles[1],
196 713d6e11 2022-10-20 stsp tmpfiles[2], pack_hash, send_index_pack_progress, &ibuf, &rl);
197 93658fb9 2020-03-18 stsp done:
198 668a20f6 2020-03-18 stsp close_err = got_pack_close(&pack);
199 668a20f6 2020-03-18 stsp if (close_err && err == NULL)
200 668a20f6 2020-03-18 stsp err = close_err;
201 668a20f6 2020-03-18 stsp if (idxfd != -1 && close(idxfd) == -1 && err == NULL)
202 668a20f6 2020-03-18 stsp err = got_error_from_errno("close");
203 4788f1ce 2020-03-18 stsp if (tmpfd != -1 && close(tmpfd) == -1 && err == NULL)
204 4788f1ce 2020-03-18 stsp err = got_error_from_errno("close");
205 d582f26c 2020-03-18 stsp for (i = 0; i < nitems(tmpfiles); i++) {
206 d582f26c 2020-03-18 stsp if (tmpfiles[i] != NULL && fclose(tmpfiles[i]) == EOF &&
207 d582f26c 2020-03-18 stsp err == NULL)
208 07965a0d 2021-10-15 stsp err = got_error_from_errno("fclose");
209 d582f26c 2020-03-18 stsp }
210 668a20f6 2020-03-18 stsp
211 668a20f6 2020-03-18 stsp if (err == NULL)
212 e70bf110 2020-03-22 stsp err = send_index_pack_done(&ibuf);
213 668a20f6 2020-03-18 stsp if (err) {
214 668a20f6 2020-03-18 stsp got_privsep_send_error(&ibuf, err);
215 93658fb9 2020-03-18 stsp fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
216 93658fb9 2020-03-18 stsp got_privsep_send_error(&ibuf, err);
217 668a20f6 2020-03-18 stsp exit(1);
218 93658fb9 2020-03-18 stsp }
219 93658fb9 2020-03-18 stsp
220 93658fb9 2020-03-18 stsp exit(0);
221 93658fb9 2020-03-18 stsp }