Blob


1 /*
2 * Copyright (c) 2019 Ori Bernstein <ori@openbsd.org>
3 *
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 */
17 #include <sys/types.h>
18 #include <sys/queue.h>
19 #include <sys/uio.h>
20 #include <sys/time.h>
21 #include <sys/stat.h>
22 #include <sys/syslimits.h>
24 #include <stdint.h>
25 #include <errno.h>
26 #include <imsg.h>
27 #include <limits.h>
28 #include <signal.h>
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <ctype.h>
33 #include <sha1.h>
34 #include <fcntl.h>
35 #include <zlib.h>
36 #include <err.h>
38 #include "got_error.h"
39 #include "got_object.h"
40 #include "got_path.h"
41 #include "got_version.h"
42 #include "got_fetch.h"
43 #include "got_reference.h"
45 #include "got_lib_sha1.h"
46 #include "got_lib_delta.h"
47 #include "got_lib_object.h"
48 #include "got_lib_object_parse.h"
49 #include "got_lib_privsep.h"
50 #include "got_lib_pack.h"
52 #ifndef nitems
53 #define nitems(_a) (sizeof((_a)) / sizeof((_a)[0]))
54 #endif
56 struct got_object *indexed;
57 static int chattygot;
58 static struct got_object_id zhash = {.sha1={0}};
60 static const struct got_error *
61 readn(ssize_t *off, int fd, void *buf, size_t n)
62 {
63 ssize_t r;
65 *off = 0;
66 while (*off != n) {
67 r = read(fd, buf + *off, n - *off);
68 if (r == -1)
69 return got_error_from_errno("read");
70 if (r == 0)
71 return NULL;
72 *off += r;
73 }
74 return NULL;
75 }
77 static const struct got_error *
78 flushpkt(int fd)
79 {
80 ssize_t w;
82 if (chattygot > 1)
83 fprintf(stderr, "%s: writepkt: 0000\n", getprogname());
85 w = write(fd, "0000", 4);
86 if (w == -1)
87 return got_error_from_errno("write");
88 if (w != 4)
89 return got_error(GOT_ERR_IO);
90 return NULL;
91 }
93 /*
94 * Packet header contains a 4-byte hexstring which specifies the length
95 * of data which follows.
96 */
97 static const struct got_error *
98 read_pkthdr(int *datalen, int fd)
99 {
100 static const struct got_error *err = NULL;
101 char lenstr[5];
102 long len;
103 char *e;
104 int n, i;
105 ssize_t r;
107 *datalen = 0;
109 err = readn(&r, fd, lenstr, 4);
110 if (err)
111 return err;
112 if (r == 0) {
113 /* implicit "0000" */
114 if (chattygot > 1)
115 fprintf(stderr, "%s: readpkt: 0000\n", getprogname());
116 return NULL;
118 if (r != 4)
119 return got_error_msg(GOT_ERR_BAD_PACKET,
120 "wrong packet header length");
122 lenstr[4] = '\0';
123 for (i = 0; i < 4; i++) {
124 if (!isprint((unsigned char)lenstr[i]))
125 return got_error_msg(GOT_ERR_BAD_PACKET,
126 "unprintable character in packet length field");
128 for (i = 0; i < 4; i++) {
129 if (!isxdigit((unsigned char)lenstr[i])) {
130 if (chattygot)
131 fprintf(stderr, "%s: bad length: '%s'\n",
132 getprogname(), lenstr);
133 return got_error_msg(GOT_ERR_BAD_PACKET,
134 "packet length not specified in hex");
137 errno = 0;
138 len = strtol(lenstr, &e, 16);
139 if (lenstr[0] == '\0' || *e != '\0')
140 return got_error(GOT_ERR_BAD_PACKET);
141 if (errno == ERANGE && (len == LONG_MAX || len == LONG_MIN))
142 return got_error_msg(GOT_ERR_BAD_PACKET, "bad packet length");
143 if (len > INT_MAX || len < INT_MIN)
144 return got_error_msg(GOT_ERR_BAD_PACKET, "bad packet length");
145 n = len;
146 if (n == 0)
147 return NULL;
148 if (n <= 4)
149 return got_error_msg(GOT_ERR_BAD_PACKET, "packet too short");
150 n -= 4;
152 *datalen = n;
153 return NULL;
156 static const struct got_error *
157 readpkt(int *outlen, int fd, char *buf, int buflen)
159 const struct got_error *err = NULL;
160 int datalen, i;
161 ssize_t n;
163 err = read_pkthdr(&datalen, fd);
164 if (err)
165 return err;
167 if (datalen > buflen)
168 return got_error(GOT_ERR_NO_SPACE);
170 err = readn(&n, fd, buf, datalen);
171 if (err)
172 return err;
173 if (n != datalen)
174 return got_error_msg(GOT_ERR_BAD_PACKET, "short packet");
176 if (chattygot > 1) {
177 fprintf(stderr, "%s: readpkt: %zd:\t", getprogname(), n);
178 for (i = 0; i < n; i++) {
179 if (isprint(buf[i]))
180 fputc(buf[i], stderr);
181 else
182 fprintf(stderr, "[0x%.2x]", buf[i]);
184 fputc('\n', stderr);
187 *outlen = n;
188 return NULL;
191 static const struct got_error *
192 writepkt(int fd, char *buf, int nbuf)
194 char len[5];
195 int i;
196 ssize_t w;
198 if (snprintf(len, sizeof(len), "%04x", nbuf + 4) >= sizeof(len))
199 return got_error(GOT_ERR_NO_SPACE);
200 w = write(fd, len, 4);
201 if (w == -1)
202 return got_error_from_errno("write");
203 if (w != 4)
204 return got_error(GOT_ERR_IO);
205 w = write(fd, buf, nbuf);
206 if (w == -1)
207 return got_error_from_errno("write");
208 if (w != nbuf)
209 return got_error(GOT_ERR_IO);
210 if (chattygot > 1) {
211 fprintf(stderr, "%s: writepkt: %s:\t", getprogname(), len);
212 for (i = 0; i < nbuf; i++) {
213 if (isprint(buf[i]))
214 fputc(buf[i], stderr);
215 else
216 fprintf(stderr, "[0x%.2x]", buf[i]);
218 fputc('\n', stderr);
220 return NULL;
223 static void
224 match_remote_ref(struct got_pathlist_head *have_refs,
225 struct got_object_id *my_id, char *refname)
227 struct got_pathlist_entry *pe;
229 /* XXX zero-hash signifies we don't have this ref;
230 * we should use a flag instead */
231 memset(my_id, 0, sizeof(*my_id));
233 TAILQ_FOREACH(pe, have_refs, entry) {
234 struct got_object_id *id = pe->data;
235 if (strcmp(pe->path, refname) == 0) {
236 memcpy(my_id, id, sizeof(*my_id));
237 break;
242 static int
243 match_branch(const char *branch, const char *wanted_branch)
245 if (strncmp(branch, "refs/heads/", 11) != 0)
246 return 0;
248 if (strncmp(wanted_branch, "refs/heads/", 11) == 0)
249 wanted_branch += 11;
251 return (strcmp(branch + 11, wanted_branch) == 0);
254 static int
255 match_wanted_ref(const char *refname, const char *wanted_ref)
257 if (strncmp(refname, "refs/", 5) != 0)
258 return 0;
259 refname += 5;
261 /*
262 * Prevent fetching of references that won't make any
263 * sense outside of the remote repository's context.
264 */
265 if (strncmp(refname, "got/", 4) == 0)
266 return 0;
267 if (strncmp(refname, "remotes/", 8) == 0)
268 return 0;
270 if (strncmp(wanted_ref, "refs/", 5) == 0)
271 wanted_ref += 5;
273 /* Allow prefix match. */
274 if (got_path_is_child(refname, wanted_ref, strlen(wanted_ref)))
275 return 1;
277 /* Allow exact match. */
278 return (strcmp(refname, wanted_ref) == 0);
281 static const struct got_error *
282 tokenize_refline(char **tokens, char *line, int len, int maxtokens)
284 const struct got_error *err = NULL;
285 char *p;
286 size_t i, n = 0;
288 for (i = 0; i < maxtokens; i++)
289 tokens[i] = NULL;
291 for (i = 0; n < len && i < maxtokens; i++) {
292 while (isspace(*line)) {
293 line++;
294 n++;
296 p = line;
297 while (*line != '\0' &&
298 (!isspace(*line) || i == maxtokens - 1)) {
299 line++;
300 n++;
302 tokens[i] = strndup(p, line - p);
303 if (tokens[i] == NULL) {
304 err = got_error_from_errno("strndup");
305 goto done;
307 /* Skip \0 field-delimiter at end of token. */
308 while (line[0] == '\0' && n < len) {
309 line++;
310 n++;
313 if (i <= 2)
314 err = got_error(GOT_ERR_NOT_REF);
315 done:
316 if (err) {
317 int j;
318 for (j = 0; j < i; j++)
319 free(tokens[j]);
320 tokens[j] = NULL;
322 return err;
325 static const struct got_error *
326 parse_refline(char **id_str, char **refname, char **server_capabilities,
327 char *line, int len)
329 const struct got_error *err = NULL;
330 char *tokens[3];
332 err = tokenize_refline(tokens, line, len, nitems(tokens));
333 if (err)
334 return err;
336 if (tokens[0])
337 *id_str = tokens[0];
338 if (tokens[1])
339 *refname = tokens[1];
340 if (tokens[2]) {
341 char *p;
342 *server_capabilities = tokens[2];
343 p = strrchr(*server_capabilities, '\n');
344 if (p)
345 *p = '\0';
348 return NULL;
351 #define GOT_CAPA_AGENT "agent"
352 #define GOT_CAPA_OFS_DELTA "ofs-delta"
353 #define GOT_CAPA_SIDE_BAND_64K "side-band-64k"
355 #define GOT_SIDEBAND_PACKFILE_DATA 1
356 #define GOT_SIDEBAND_PROGRESS_INFO 2
357 #define GOT_SIDEBAND_ERROR_INFO 3
360 struct got_capability {
361 const char *key;
362 const char *value;
363 };
364 static const struct got_capability got_capabilities[] = {
365 { GOT_CAPA_AGENT, "got/" GOT_VERSION_STR },
366 { GOT_CAPA_OFS_DELTA, NULL },
367 { GOT_CAPA_SIDE_BAND_64K, NULL },
368 };
370 static const struct got_error *
371 match_capability(char **my_capabilities, const char *capa,
372 const struct got_capability *mycapa)
374 char *equalsign;
375 char *s;
377 equalsign = strchr(capa, '=');
378 if (equalsign) {
379 if (strncmp(capa, mycapa->key, equalsign - capa) != 0)
380 return NULL;
381 } else {
382 if (strcmp(capa, mycapa->key) != 0)
383 return NULL;
386 if (asprintf(&s, "%s %s%s%s",
387 *my_capabilities != NULL ? *my_capabilities : "",
388 mycapa->key,
389 mycapa->value != NULL ? "=" : "",
390 mycapa->value != NULL? mycapa->value : "") == -1)
391 return got_error_from_errno("asprintf");
393 free(*my_capabilities);
394 *my_capabilities = s;
395 return NULL;
398 static const struct got_error *
399 add_symref(struct got_pathlist_head *symrefs, char *capa)
401 const struct got_error *err = NULL;
402 char *colon, *name = NULL, *target = NULL;
404 /* Need at least "A:B" */
405 if (strlen(capa) < 3)
406 return NULL;
408 colon = strchr(capa, ':');
409 if (colon == NULL)
410 return NULL;
412 *colon = '\0';
413 name = strdup(capa);
414 if (name == NULL)
415 return got_error_from_errno("strdup");
417 target = strdup(colon + 1);
418 if (target == NULL) {
419 err = got_error_from_errno("strdup");
420 goto done;
423 /* We can't validate the ref itself here. The main process will. */
424 err = got_pathlist_append(symrefs, name, target);
425 done:
426 if (err) {
427 free(name);
428 free(target);
430 return err;
433 static const struct got_error *
434 match_capabilities(char **my_capabilities, struct got_pathlist_head *symrefs,
435 char *server_capabilities)
437 const struct got_error *err = NULL;
438 char *capa, *equalsign;
439 int i;
441 *my_capabilities = NULL;
442 do {
443 capa = strsep(&server_capabilities, " ");
444 if (capa == NULL)
445 return NULL;
447 equalsign = strchr(capa, '=');
448 if (equalsign != NULL &&
449 strncmp(capa, "symref", equalsign - capa) == 0) {
450 err = add_symref(symrefs, equalsign + 1);
451 if (err)
452 break;
453 continue;
456 for (i = 0; i < nitems(got_capabilities); i++) {
457 err = match_capability(my_capabilities,
458 capa, &got_capabilities[i]);
459 if (err)
460 break;
462 } while (capa);
464 if (*my_capabilities == NULL) {
465 *my_capabilities = strdup("");
466 if (*my_capabilities == NULL)
467 err = got_error_from_errno("strdup");
469 return err;
472 static const struct got_error *
473 send_fetch_server_progress(struct imsgbuf *ibuf, const char *msg, size_t msglen)
475 if (msglen > MAX_IMSGSIZE - IMSG_HEADER_SIZE)
476 return got_error(GOT_ERR_NO_SPACE);
478 if (msglen == 0)
479 return NULL;
481 if (imsg_compose(ibuf, GOT_IMSG_FETCH_SERVER_PROGRESS, 0, 0, -1,
482 msg, msglen) == -1)
483 return got_error_from_errno(
484 "imsg_compose FETCH_SERVER_PROGRESS");
486 return got_privsep_flush_imsg(ibuf);
489 static const struct got_error *
490 send_fetch_download_progress(struct imsgbuf *ibuf, off_t bytes)
492 if (imsg_compose(ibuf, GOT_IMSG_FETCH_DOWNLOAD_PROGRESS, 0, 0, -1,
493 &bytes, sizeof(bytes)) == -1)
494 return got_error_from_errno(
495 "imsg_compose FETCH_DOWNLOAD_PROGRESS");
497 return got_privsep_flush_imsg(ibuf);
500 static const struct got_error *
501 send_fetch_done(struct imsgbuf *ibuf, uint8_t *pack_sha1)
503 if (imsg_compose(ibuf, GOT_IMSG_FETCH_DONE, 0, 0, -1,
504 pack_sha1, SHA1_DIGEST_LENGTH) == -1)
505 return got_error_from_errno("imsg_compose FETCH");
506 return got_privsep_flush_imsg(ibuf);
511 static const struct got_error *
512 fetch_progress(struct imsgbuf *ibuf, const char *buf, size_t len)
514 int i;
516 if (len == 0)
517 return NULL;
519 /*
520 * Truncate messages which exceed the maximum imsg payload size.
521 * Server may send up to 64k.
522 */
523 if (len > MAX_IMSGSIZE - IMSG_HEADER_SIZE)
524 len = MAX_IMSGSIZE - IMSG_HEADER_SIZE;
526 /* Only allow printable ASCII. */
527 for (i = 0; i < len; i++) {
528 if (isprint((unsigned char)buf[i]) ||
529 isspace((unsigned char)buf[i]))
530 continue;
531 return got_error_msg(GOT_ERR_BAD_PACKET,
532 "non-printable progress message received from server");
535 return send_fetch_server_progress(ibuf, buf, len);
538 static const struct got_error *
539 fetch_error(const char *buf, size_t len)
541 static char msg[1024];
542 int i;
544 for (i = 0; i < len && i < sizeof(msg) - 1; i++) {
545 if (!isprint(buf[i]))
546 return got_error_msg(GOT_ERR_BAD_PACKET,
547 "non-printable error message received from server");
548 msg[i] = buf[i];
550 msg[i] = '\0';
551 return got_error_msg(GOT_ERR_FETCH_FAILED, msg);
554 static const struct got_error *
555 send_fetch_symrefs(struct imsgbuf *ibuf, struct got_pathlist_head *symrefs)
557 const struct got_error *err = NULL;
558 struct ibuf *wbuf;
559 size_t len, nsymrefs = 0;
560 struct got_pathlist_entry *pe;
562 len = sizeof(struct got_imsg_fetch_symrefs);
563 TAILQ_FOREACH(pe, symrefs, entry) {
564 const char *target = pe->data;
565 len += sizeof(struct got_imsg_fetch_symref) +
566 pe->path_len + strlen(target);
567 nsymrefs++;
570 if (len >= MAX_IMSGSIZE - IMSG_HEADER_SIZE)
571 return got_error(GOT_ERR_NO_SPACE);
573 wbuf = imsg_create(ibuf, GOT_IMSG_FETCH_SYMREFS, 0, 0, len);
574 if (wbuf == NULL)
575 return got_error_from_errno("imsg_create FETCH_SYMREFS");
577 /* Keep in sync with struct got_imsg_fetch_symrefs definition! */
578 if (imsg_add(wbuf, &nsymrefs, sizeof(nsymrefs)) == -1) {
579 err = got_error_from_errno("imsg_add FETCH_SYMREFS");
580 ibuf_free(wbuf);
581 return err;
584 TAILQ_FOREACH(pe, symrefs, entry) {
585 const char *name = pe->path;
586 size_t name_len = pe->path_len;
587 const char *target = pe->data;
588 size_t target_len = strlen(target);
590 /* Keep in sync with struct got_imsg_fetch_symref definition! */
591 if (imsg_add(wbuf, &name_len, sizeof(name_len)) == -1) {
592 err = got_error_from_errno("imsg_add FETCH_SYMREFS");
593 ibuf_free(wbuf);
594 return err;
596 if (imsg_add(wbuf, &target_len, sizeof(target_len)) == -1) {
597 err = got_error_from_errno("imsg_add FETCH_SYMREFS");
598 ibuf_free(wbuf);
599 return err;
601 if (imsg_add(wbuf, name, name_len) == -1) {
602 err = got_error_from_errno("imsg_add FETCH_SYMREFS");
603 ibuf_free(wbuf);
604 return err;
606 if (imsg_add(wbuf, target, target_len) == -1) {
607 err = got_error_from_errno("imsg_add FETCH_SYMREFS");
608 ibuf_free(wbuf);
609 return err;
613 wbuf->fd = -1;
614 imsg_close(ibuf, wbuf);
615 return got_privsep_flush_imsg(ibuf);
618 static const struct got_error *
619 send_fetch_ref(struct imsgbuf *ibuf, struct got_object_id *refid,
620 const char *refname)
622 const struct got_error *err = NULL;
623 struct ibuf *wbuf;
624 size_t len, reflen = strlen(refname);
626 len = sizeof(struct got_imsg_fetch_ref) + reflen;
627 if (len >= MAX_IMSGSIZE - IMSG_HEADER_SIZE)
628 return got_error(GOT_ERR_NO_SPACE);
630 wbuf = imsg_create(ibuf, GOT_IMSG_FETCH_REF, 0, 0, len);
631 if (wbuf == NULL)
632 return got_error_from_errno("imsg_create FETCH_REF");
634 /* Keep in sync with struct got_imsg_fetch_ref definition! */
635 if (imsg_add(wbuf, refid->sha1, SHA1_DIGEST_LENGTH) == -1) {
636 err = got_error_from_errno("imsg_add FETCH_REF");
637 ibuf_free(wbuf);
638 return err;
640 if (imsg_add(wbuf, refname, reflen) == -1) {
641 err = got_error_from_errno("imsg_add FETCH_REF");
642 ibuf_free(wbuf);
643 return err;
646 wbuf->fd = -1;
647 imsg_close(ibuf, wbuf);
648 return got_privsep_flush_imsg(ibuf);
651 static const struct got_error *
652 fetch_pack(int fd, int packfd, uint8_t *pack_sha1,
653 struct got_pathlist_head *have_refs, int fetch_all_branches,
654 struct got_pathlist_head *wanted_branches,
655 struct got_pathlist_head *wanted_refs, int list_refs_only,
656 struct imsgbuf *ibuf)
658 const struct got_error *err = NULL;
659 char buf[GOT_FETCH_PKTMAX];
660 char hashstr[SHA1_DIGEST_STRING_LENGTH];
661 struct got_object_id *have, *want;
662 int is_firstpkt = 1, nref = 0, refsz = 16;
663 int i, n, nwant = 0, nhave = 0, acked = 0;
664 off_t packsz = 0, last_reported_packsz = 0;
665 char *id_str = NULL, *refname = NULL;
666 char *server_capabilities = NULL, *my_capabilities = NULL;
667 const char *default_branch = NULL;
668 struct got_pathlist_head symrefs;
669 struct got_pathlist_entry *pe;
670 int sent_my_capabilites = 0, have_sidebands = 0;
671 int found_branch = 0;
672 SHA1_CTX sha1_ctx;
673 uint8_t sha1_buf[SHA1_DIGEST_LENGTH];
674 size_t sha1_buf_len = 0;
675 ssize_t w;
677 TAILQ_INIT(&symrefs);
678 SHA1Init(&sha1_ctx);
680 have = malloc(refsz * sizeof(have[0]));
681 if (have == NULL)
682 return got_error_from_errno("malloc");
683 want = malloc(refsz * sizeof(want[0]));
684 if (want == NULL) {
685 err = got_error_from_errno("malloc");
686 goto done;
688 while (1) {
689 err = readpkt(&n, fd, buf, sizeof(buf));
690 if (err)
691 goto done;
692 if (n == 0)
693 break;
694 if (n >= 4 && strncmp(buf, "ERR ", 4) == 0) {
695 err = fetch_error(&buf[4], n - 4);
696 goto done;
698 err = parse_refline(&id_str, &refname, &server_capabilities,
699 buf, n);
700 if (err)
701 goto done;
702 if (is_firstpkt) {
703 if (chattygot && server_capabilities[0] != '\0')
704 fprintf(stderr, "%s: server capabilities: %s\n",
705 getprogname(), server_capabilities);
706 err = match_capabilities(&my_capabilities, &symrefs,
707 server_capabilities);
708 if (err)
709 goto done;
710 if (chattygot)
711 fprintf(stderr, "%s: my capabilities:%s\n",
712 getprogname(), my_capabilities);
713 err = send_fetch_symrefs(ibuf, &symrefs);
714 if (err)
715 goto done;
716 is_firstpkt = 0;
717 if (!fetch_all_branches) {
718 TAILQ_FOREACH(pe, &symrefs, entry) {
719 const char *name = pe->path;
720 const char *symref_target = pe->data;
721 if (strcmp(name, GOT_REF_HEAD) != 0)
722 continue;
723 default_branch = symref_target;
724 break;
727 continue;
729 if (strstr(refname, "^{}")) {
730 if (chattygot) {
731 fprintf(stderr, "%s: ignoring %s\n",
732 getprogname(), refname);
734 continue;
737 if (strncmp(refname, "refs/heads/", 11) == 0) {
738 if (fetch_all_branches || list_refs_only) {
739 found_branch = 1;
740 } else if (!TAILQ_EMPTY(wanted_branches)) {
741 TAILQ_FOREACH(pe, wanted_branches, entry) {
742 if (match_branch(refname, pe->path))
743 break;
745 if (pe == NULL) {
746 if (chattygot) {
747 fprintf(stderr,
748 "%s: ignoring %s\n",
749 getprogname(), refname);
751 continue;
753 found_branch = 1;
754 } else if (default_branch != NULL) {
755 if (!match_branch(refname, default_branch)) {
756 if (chattygot) {
757 fprintf(stderr,
758 "%s: ignoring %s\n",
759 getprogname(), refname);
761 continue;
763 found_branch = 1;
765 } else if (strncmp(refname, "refs/tags/", 10) != 0) {
766 if (!TAILQ_EMPTY(wanted_refs)) {
767 TAILQ_FOREACH(pe, wanted_refs, entry) {
768 if (match_wanted_ref(refname, pe->path))
769 break;
771 if (pe == NULL) {
772 if (chattygot) {
773 fprintf(stderr,
774 "%s: ignoring %s\n",
775 getprogname(), refname);
777 continue;
779 found_branch = 1;
780 } else if (!list_refs_only) {
781 if (chattygot) {
782 fprintf(stderr, "%s: ignoring %s\n",
783 getprogname(), refname);
785 continue;
789 if (refsz == nref + 1) {
790 refsz *= 2;
791 have = reallocarray(have, refsz, sizeof(have[0]));
792 if (have == NULL) {
793 err = got_error_from_errno("reallocarray");
794 goto done;
796 want = reallocarray(want, refsz, sizeof(want[0]));
797 if (want == NULL) {
798 err = got_error_from_errno("reallocarray");
799 goto done;
802 if (!got_parse_sha1_digest(want[nref].sha1, id_str)) {
803 err = got_error(GOT_ERR_BAD_OBJ_ID_STR);
804 goto done;
806 match_remote_ref(have_refs, &have[nref], refname);
807 err = send_fetch_ref(ibuf, &want[nref], refname);
808 if (err)
809 goto done;
811 if (chattygot)
812 fprintf(stderr, "%s: %s will be fetched\n",
813 getprogname(), refname);
814 if (chattygot > 1) {
815 char *theirs, *mine;
816 err = got_object_id_str(&theirs, &want[nref]);
817 if (err)
818 goto done;
819 err = got_object_id_str(&mine, &have[nref]);
820 if (err) {
821 free(theirs);
822 goto done;
824 fprintf(stderr, "%s: remote: %s\n%s: local: %s\n",
825 getprogname(), theirs, getprogname(), mine);
826 free(theirs);
827 free(mine);
829 nref++;
832 if (list_refs_only)
833 goto done;
835 /* Abort if we haven't found any branch to fetch. */
836 if (!found_branch) {
837 err = got_error(GOT_ERR_FETCH_NO_BRANCH);
838 goto done;
841 for (i = 0; i < nref; i++) {
842 if (got_object_id_cmp(&have[i], &want[i]) == 0)
843 continue;
844 got_sha1_digest_to_str(want[i].sha1, hashstr, sizeof(hashstr));
845 n = snprintf(buf, sizeof(buf), "want %s%s\n", hashstr,
846 sent_my_capabilites ? "" : my_capabilities);
847 if (n >= sizeof(buf)) {
848 err = got_error(GOT_ERR_NO_SPACE);
849 goto done;
851 err = writepkt(fd, buf, n);
852 if (err)
853 goto done;
854 sent_my_capabilites = 1;
855 nwant++;
857 err = flushpkt(fd);
858 if (err)
859 goto done;
861 if (nwant == 0)
862 goto done;
864 for (i = 0; i < nref; i++) {
865 if (got_object_id_cmp(&have[i], &zhash) == 0)
866 continue;
867 got_sha1_digest_to_str(have[i].sha1, hashstr, sizeof(hashstr));
868 n = snprintf(buf, sizeof(buf), "have %s\n", hashstr);
869 if (n >= sizeof(buf)) {
870 err = got_error(GOT_ERR_NO_SPACE);
871 goto done;
873 err = writepkt(fd, buf, n);
874 if (err)
875 goto done;
876 nhave++;
879 while (nhave > 0 && !acked) {
880 struct got_object_id common_id;
882 /* The server should ACK the object IDs we need. */
883 err = readpkt(&n, fd, buf, sizeof(buf));
884 if (err)
885 goto done;
886 if (n >= 4 && strncmp(buf, "ERR ", 4) == 0) {
887 err = fetch_error(&buf[4], n - 4);
888 goto done;
890 if (n >= 4 && strncmp(buf, "NAK\n", 4) == 0) {
891 /* Server has not located our objects yet. */
892 continue;
894 if (n < 4 + SHA1_DIGEST_STRING_LENGTH ||
895 strncmp(buf, "ACK ", 4) != 0) {
896 err = got_error_msg(GOT_ERR_BAD_PACKET,
897 "unexpected message from server");
898 goto done;
900 if (!got_parse_sha1_digest(common_id.sha1, buf + 4)) {
901 err = got_error_msg(GOT_ERR_BAD_PACKET,
902 "bad object ID in ACK packet from server");
903 goto done;
905 acked++;
908 n = snprintf(buf, sizeof(buf), "done\n");
909 err = writepkt(fd, buf, n);
910 if (err)
911 goto done;
913 if (nhave == 0) {
914 err = readpkt(&n, fd, buf, sizeof(buf));
915 if (err)
916 goto done;
917 if (n != 4 || strncmp(buf, "NAK\n", n) != 0) {
918 err = got_error_msg(GOT_ERR_BAD_PACKET,
919 "unexpected message from server");
920 goto done;
924 if (chattygot)
925 fprintf(stderr, "%s: fetching...\n", getprogname());
927 if (my_capabilities != NULL &&
928 strstr(my_capabilities, GOT_CAPA_SIDE_BAND_64K) != NULL)
929 have_sidebands = 1;
931 while (1) {
932 ssize_t r = 0;
933 int datalen = -1;
935 if (have_sidebands) {
936 err = read_pkthdr(&datalen, fd);
937 if (err)
938 goto done;
939 if (datalen <= 0)
940 break;
942 /* Read sideband channel ID (one byte). */
943 r = read(fd, buf, 1);
944 if (r == -1) {
945 err = got_error_from_errno("read");
946 goto done;
948 if (r != 1) {
949 err = got_error_msg(GOT_ERR_BAD_PACKET,
950 "short packet");
951 goto done;
953 if (datalen > sizeof(buf) - 5) {
954 err = got_error_msg(GOT_ERR_BAD_PACKET,
955 "bad packet length");
956 goto done;
958 datalen--; /* sideband ID has been read */
959 if (buf[0] == GOT_SIDEBAND_PACKFILE_DATA) {
960 /* Read packfile data. */
961 err = readn(&r, fd, buf, datalen);
962 if (err)
963 goto done;
964 if (r != datalen) {
965 err = got_error_msg(GOT_ERR_BAD_PACKET,
966 "packet too short");
967 goto done;
969 } else if (buf[0] == GOT_SIDEBAND_PROGRESS_INFO) {
970 err = readn(&r, fd, buf, datalen);
971 if (err)
972 goto done;
973 if (r != datalen) {
974 err = got_error_msg(GOT_ERR_BAD_PACKET,
975 "packet too short");
976 goto done;
978 err = fetch_progress(ibuf, buf, r);
979 if (err)
980 goto done;
981 continue;
982 } else if (buf[0] == GOT_SIDEBAND_ERROR_INFO) {
983 err = readn(&r, fd, buf, datalen);
984 if (err)
985 goto done;
986 if (r != datalen) {
987 err = got_error_msg(GOT_ERR_BAD_PACKET,
988 "packet too short");
989 goto done;
991 err = fetch_error(buf, r);
992 goto done;
993 } else {
994 err = got_error_msg(GOT_ERR_BAD_PACKET,
995 "unknown side-band received from server");
996 goto done;
998 } else {
999 /* No sideband channel. Every byte is packfile data. */
1000 err = readn(&r, fd, buf, sizeof buf);
1001 if (err)
1002 goto done;
1003 if (r <= 0)
1004 break;
1008 * An expected SHA1 checksum sits at the end of the pack file.
1009 * Since we don't know the file size ahead of time we have to
1010 * keep SHA1_DIGEST_LENGTH bytes buffered and avoid mixing
1011 * those bytes into our SHA1 checksum computation until we
1012 * know for sure that additional pack file data bytes follow.
1014 * We can assume r > 0 since otherwise the loop would exit.
1016 if (r < SHA1_DIGEST_LENGTH) {
1017 if (sha1_buf_len < SHA1_DIGEST_LENGTH) {
1019 * If there's enough buffered + read data to
1020 * fill up the buffer then shift a sufficient
1021 * amount of bytes out at the front to make
1022 * room, mixing those bytes into the checksum.
1024 while (sha1_buf_len > 0 &&
1025 sha1_buf_len + r > SHA1_DIGEST_LENGTH) {
1026 SHA1Update(&sha1_ctx, sha1_buf, 1);
1027 memmove(sha1_buf, sha1_buf + 1, 1);
1028 sha1_buf_len--;
1031 /* Buffer potential checksum bytes. */
1032 memcpy(sha1_buf + sha1_buf_len, buf, r);
1033 sha1_buf_len += r;
1034 } else {
1036 * Mix in previously buffered bytes which
1037 * are not part of the checksum after all.
1039 SHA1Update(&sha1_ctx, sha1_buf, r);
1041 /* Update potential checksum buffer. */
1042 memmove(sha1_buf, sha1_buf + r,
1043 sha1_buf_len - r);
1044 memcpy(sha1_buf + sha1_buf_len - r, buf, r);
1046 } else {
1047 /* Mix in any previously buffered bytes. */
1048 SHA1Update(&sha1_ctx, sha1_buf, sha1_buf_len);
1050 /* Mix in bytes read minus potential checksum bytes. */
1051 SHA1Update(&sha1_ctx, buf, r - SHA1_DIGEST_LENGTH);
1053 /* Buffer potential checksum bytes. */
1054 memcpy(sha1_buf, buf + r - SHA1_DIGEST_LENGTH,
1055 SHA1_DIGEST_LENGTH);
1056 sha1_buf_len = SHA1_DIGEST_LENGTH;
1059 /* Write packfile data to temporary pack file. */
1060 w = write(packfd, buf, r);
1061 if (w == -1) {
1062 err = got_error_from_errno("write");
1063 goto done;
1065 if (w != r) {
1066 err = got_error(GOT_ERR_IO);
1067 goto done;
1069 packsz += w;
1071 /* Don't send too many progress privsep messages. */
1072 if (packsz > last_reported_packsz + 1024) {
1073 err = send_fetch_download_progress(ibuf, packsz);
1074 if (err)
1075 goto done;
1076 last_reported_packsz = packsz;
1079 err = send_fetch_download_progress(ibuf, packsz);
1080 if (err)
1081 goto done;
1083 SHA1Final(pack_sha1, &sha1_ctx);
1084 if (sha1_buf_len != SHA1_DIGEST_LENGTH ||
1085 memcmp(pack_sha1, sha1_buf, sha1_buf_len) != 0) {
1086 err = got_error_msg(GOT_ERR_BAD_PACKFILE,
1087 "pack file checksum mismatch");
1089 done:
1090 TAILQ_FOREACH(pe, &symrefs, entry) {
1091 free((void *)pe->path);
1092 free(pe->data);
1094 got_pathlist_free(&symrefs);
1095 free(have);
1096 free(want);
1097 free(id_str);
1098 free(refname);
1099 free(server_capabilities);
1100 return err;
1104 int
1105 main(int argc, char **argv)
1107 const struct got_error *err = NULL;
1108 int fetchfd, packfd = -1, i;
1109 uint8_t pack_sha1[SHA1_DIGEST_LENGTH];
1110 struct imsgbuf ibuf;
1111 struct imsg imsg;
1112 struct got_pathlist_head have_refs;
1113 struct got_pathlist_head wanted_branches;
1114 struct got_pathlist_head wanted_refs;
1115 struct got_pathlist_entry *pe;
1116 struct got_imsg_fetch_request fetch_req;
1117 struct got_imsg_fetch_have_ref href;
1118 struct got_imsg_fetch_wanted_branch wbranch;
1119 struct got_imsg_fetch_wanted_ref wref;
1120 size_t datalen;
1121 #if 0
1122 static int attached;
1123 while (!attached)
1124 sleep (1);
1125 #endif
1127 TAILQ_INIT(&have_refs);
1128 TAILQ_INIT(&wanted_branches);
1129 TAILQ_INIT(&wanted_refs);
1131 imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
1132 #ifndef PROFILE
1133 /* revoke access to most system calls */
1134 if (pledge("stdio recvfd", NULL) == -1) {
1135 err = got_error_from_errno("pledge");
1136 got_privsep_send_error(&ibuf, err);
1137 return 1;
1139 #endif
1140 if ((err = got_privsep_recv_imsg(&imsg, &ibuf, 0)) != 0) {
1141 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1142 err = NULL;
1143 goto done;
1145 if (imsg.hdr.type == GOT_IMSG_STOP)
1146 goto done;
1147 if (imsg.hdr.type != GOT_IMSG_FETCH_REQUEST) {
1148 err = got_error(GOT_ERR_PRIVSEP_MSG);
1149 goto done;
1151 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1152 if (datalen < sizeof(fetch_req)) {
1153 err = got_error(GOT_ERR_PRIVSEP_LEN);
1154 goto done;
1156 memcpy(&fetch_req, imsg.data, sizeof(fetch_req));
1157 fetchfd = imsg.fd;
1158 imsg_free(&imsg);
1160 if (fetch_req.verbosity > 0)
1161 chattygot += fetch_req.verbosity;
1163 for (i = 0; i < fetch_req.n_have_refs; i++) {
1164 struct got_object_id *id;
1165 char *refname;
1167 if ((err = got_privsep_recv_imsg(&imsg, &ibuf, 0)) != 0) {
1168 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1169 err = NULL;
1170 goto done;
1172 if (imsg.hdr.type == GOT_IMSG_STOP)
1173 goto done;
1174 if (imsg.hdr.type != GOT_IMSG_FETCH_HAVE_REF) {
1175 err = got_error(GOT_ERR_PRIVSEP_MSG);
1176 goto done;
1178 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1179 if (datalen < sizeof(href)) {
1180 err = got_error(GOT_ERR_PRIVSEP_LEN);
1181 goto done;
1183 memcpy(&href, imsg.data, sizeof(href));
1184 if (datalen - sizeof(href) < href.name_len) {
1185 err = got_error(GOT_ERR_PRIVSEP_LEN);
1186 goto done;
1188 refname = malloc(href.name_len + 1);
1189 if (refname == NULL) {
1190 err = got_error_from_errno("malloc");
1191 goto done;
1193 memcpy(refname, imsg.data + sizeof(href), href.name_len);
1194 refname[href.name_len] = '\0';
1196 id = malloc(sizeof(*id));
1197 if (id == NULL) {
1198 free(refname);
1199 err = got_error_from_errno("malloc");
1200 goto done;
1202 memcpy(id->sha1, href.id, SHA1_DIGEST_LENGTH);
1203 err = got_pathlist_append(&have_refs, refname, id);
1204 if (err) {
1205 free(refname);
1206 free(id);
1207 goto done;
1210 imsg_free(&imsg);
1213 for (i = 0; i < fetch_req.n_wanted_branches; i++) {
1214 char *refname;
1216 if ((err = got_privsep_recv_imsg(&imsg, &ibuf, 0)) != 0) {
1217 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1218 err = NULL;
1219 goto done;
1221 if (imsg.hdr.type == GOT_IMSG_STOP)
1222 goto done;
1223 if (imsg.hdr.type != GOT_IMSG_FETCH_WANTED_BRANCH) {
1224 err = got_error(GOT_ERR_PRIVSEP_MSG);
1225 goto done;
1227 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1228 if (datalen < sizeof(wbranch)) {
1229 err = got_error(GOT_ERR_PRIVSEP_LEN);
1230 goto done;
1232 memcpy(&wbranch, imsg.data, sizeof(wbranch));
1233 if (datalen - sizeof(wbranch) < wbranch.name_len) {
1234 err = got_error(GOT_ERR_PRIVSEP_LEN);
1235 goto done;
1237 refname = malloc(wbranch.name_len + 1);
1238 if (refname == NULL) {
1239 err = got_error_from_errno("malloc");
1240 goto done;
1242 memcpy(refname, imsg.data + sizeof(wbranch), wbranch.name_len);
1243 refname[wbranch.name_len] = '\0';
1245 err = got_pathlist_append(&wanted_branches, refname, NULL);
1246 if (err) {
1247 free(refname);
1248 goto done;
1251 imsg_free(&imsg);
1254 for (i = 0; i < fetch_req.n_wanted_refs; i++) {
1255 char *refname;
1257 if ((err = got_privsep_recv_imsg(&imsg, &ibuf, 0)) != 0) {
1258 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1259 err = NULL;
1260 goto done;
1262 if (imsg.hdr.type == GOT_IMSG_STOP)
1263 goto done;
1264 if (imsg.hdr.type != GOT_IMSG_FETCH_WANTED_REF) {
1265 err = got_error(GOT_ERR_PRIVSEP_MSG);
1266 goto done;
1268 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1269 if (datalen < sizeof(wref)) {
1270 err = got_error(GOT_ERR_PRIVSEP_LEN);
1271 goto done;
1273 memcpy(&wref, imsg.data, sizeof(wref));
1274 if (datalen - sizeof(wref) < wref.name_len) {
1275 err = got_error(GOT_ERR_PRIVSEP_LEN);
1276 goto done;
1278 refname = malloc(wref.name_len + 1);
1279 if (refname == NULL) {
1280 err = got_error_from_errno("malloc");
1281 goto done;
1283 memcpy(refname, imsg.data + sizeof(wref), wref.name_len);
1284 refname[wref.name_len] = '\0';
1286 err = got_pathlist_append(&wanted_refs, refname, NULL);
1287 if (err) {
1288 free(refname);
1289 goto done;
1292 imsg_free(&imsg);
1295 if ((err = got_privsep_recv_imsg(&imsg, &ibuf, 0)) != 0) {
1296 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1297 err = NULL;
1298 goto done;
1300 if (imsg.hdr.type == GOT_IMSG_STOP)
1301 goto done;
1302 if (imsg.hdr.type != GOT_IMSG_FETCH_OUTFD) {
1303 err = got_error(GOT_ERR_PRIVSEP_MSG);
1304 goto done;
1306 if (imsg.hdr.len - IMSG_HEADER_SIZE != 0) {
1307 err = got_error(GOT_ERR_PRIVSEP_LEN);
1308 goto done;
1310 packfd = imsg.fd;
1312 err = fetch_pack(fetchfd, packfd, pack_sha1, &have_refs,
1313 fetch_req.fetch_all_branches, &wanted_branches,
1314 &wanted_refs, fetch_req.list_refs_only, &ibuf);
1315 done:
1316 TAILQ_FOREACH(pe, &have_refs, entry) {
1317 free((char *)pe->path);
1318 free(pe->data);
1320 got_pathlist_free(&have_refs);
1321 TAILQ_FOREACH(pe, &wanted_branches, entry)
1322 free((char *)pe->path);
1323 got_pathlist_free(&wanted_branches);
1324 if (fetchfd != -1 && close(fetchfd) == -1 && err == NULL)
1325 err = got_error_from_errno("close");
1326 if (packfd != -1 && close(packfd) == -1 && err == NULL)
1327 err = got_error_from_errno("close");
1328 if (err != NULL)
1329 got_privsep_send_error(&ibuf, err);
1330 else
1331 err = send_fetch_done(&ibuf, pack_sha1);
1332 if (err != NULL) {
1333 fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
1334 got_privsep_send_error(&ibuf, err);
1337 exit(0);