Blob


1 /*
2 * Copyright (c) 2019 Ori Bernstein <ori@openbsd.org>
3 *
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 */
17 #include <sys/types.h>
18 #include <sys/queue.h>
19 #include <sys/uio.h>
20 #include <sys/time.h>
21 #include <sys/stat.h>
23 #include <stdint.h>
24 #include <errno.h>
25 #include <imsg.h>
26 #include <limits.h>
27 #include <signal.h>
28 #include <stdio.h>
29 #include <stdlib.h>
30 #include <string.h>
31 #include <ctype.h>
32 #include <sha1.h>
33 #include <fcntl.h>
34 #include <unistd.h>
35 #include <zlib.h>
36 #include <err.h>
38 #include "got_error.h"
39 #include "got_object.h"
40 #include "got_path.h"
41 #include "got_version.h"
42 #include "got_fetch.h"
43 #include "got_reference.h"
45 #include "got_lib_sha1.h"
46 #include "got_lib_delta.h"
47 #include "got_lib_object.h"
48 #include "got_lib_object_parse.h"
49 #include "got_lib_privsep.h"
50 #include "got_lib_pack.h"
52 #ifndef nitems
53 #define nitems(_a) (sizeof((_a)) / sizeof((_a)[0]))
54 #endif
56 struct got_object *indexed;
57 static int chattygot;
58 static struct got_object_id zhash = {.sha1={0}};
60 static const struct got_error *
61 readn(ssize_t *off, int fd, void *buf, size_t n)
62 {
63 ssize_t r;
65 *off = 0;
66 while (*off != n) {
67 r = read(fd, buf + *off, n - *off);
68 if (r == -1)
69 return got_error_from_errno("read");
70 if (r == 0)
71 return NULL;
72 *off += r;
73 }
74 return NULL;
75 }
77 static const struct got_error *
78 flushpkt(int fd)
79 {
80 ssize_t w;
82 if (chattygot > 1)
83 fprintf(stderr, "%s: writepkt: 0000\n", getprogname());
85 w = write(fd, "0000", 4);
86 if (w == -1)
87 return got_error_from_errno("write");
88 if (w != 4)
89 return got_error(GOT_ERR_IO);
90 return NULL;
91 }
93 /*
94 * Packet header contains a 4-byte hexstring which specifies the length
95 * of data which follows.
96 */
97 static const struct got_error *
98 read_pkthdr(int *datalen, int fd)
99 {
100 static const struct got_error *err = NULL;
101 char lenstr[5];
102 long len;
103 char *e;
104 int n, i;
105 ssize_t r;
107 *datalen = 0;
109 err = readn(&r, fd, lenstr, 4);
110 if (err)
111 return err;
112 if (r == 0) {
113 /* implicit "0000" */
114 if (chattygot > 1)
115 fprintf(stderr, "%s: readpkt: 0000\n", getprogname());
116 return NULL;
118 if (r != 4)
119 return got_error_msg(GOT_ERR_BAD_PACKET,
120 "wrong packet header length");
122 lenstr[4] = '\0';
123 for (i = 0; i < 4; i++) {
124 if (!isprint((unsigned char)lenstr[i]))
125 return got_error_msg(GOT_ERR_BAD_PACKET,
126 "unprintable character in packet length field");
128 for (i = 0; i < 4; i++) {
129 if (!isxdigit((unsigned char)lenstr[i])) {
130 if (chattygot)
131 fprintf(stderr, "%s: bad length: '%s'\n",
132 getprogname(), lenstr);
133 return got_error_msg(GOT_ERR_BAD_PACKET,
134 "packet length not specified in hex");
137 errno = 0;
138 len = strtol(lenstr, &e, 16);
139 if (lenstr[0] == '\0' || *e != '\0')
140 return got_error(GOT_ERR_BAD_PACKET);
141 if (errno == ERANGE && (len == LONG_MAX || len == LONG_MIN))
142 return got_error_msg(GOT_ERR_BAD_PACKET, "bad packet length");
143 if (len > INT_MAX || len < INT_MIN)
144 return got_error_msg(GOT_ERR_BAD_PACKET, "bad packet length");
145 n = len;
146 if (n == 0)
147 return NULL;
148 if (n <= 4)
149 return got_error_msg(GOT_ERR_BAD_PACKET, "packet too short");
150 n -= 4;
152 *datalen = n;
153 return NULL;
156 static const struct got_error *
157 readpkt(int *outlen, int fd, char *buf, int buflen)
159 const struct got_error *err = NULL;
160 int datalen, i;
161 ssize_t n;
163 err = read_pkthdr(&datalen, fd);
164 if (err)
165 return err;
167 if (datalen > buflen)
168 return got_error(GOT_ERR_NO_SPACE);
170 err = readn(&n, fd, buf, datalen);
171 if (err)
172 return err;
173 if (n != datalen)
174 return got_error_msg(GOT_ERR_BAD_PACKET, "short packet");
176 if (chattygot > 1) {
177 fprintf(stderr, "%s: readpkt: %zd:\t", getprogname(), n);
178 for (i = 0; i < n; i++) {
179 if (isprint(buf[i]))
180 fputc(buf[i], stderr);
181 else
182 fprintf(stderr, "[0x%.2x]", buf[i]);
184 fputc('\n', stderr);
187 *outlen = n;
188 return NULL;
191 static const struct got_error *
192 writepkt(int fd, char *buf, int nbuf)
194 char len[5];
195 int i;
196 ssize_t w;
198 if (snprintf(len, sizeof(len), "%04x", nbuf + 4) >= sizeof(len))
199 return got_error(GOT_ERR_NO_SPACE);
200 w = write(fd, len, 4);
201 if (w == -1)
202 return got_error_from_errno("write");
203 if (w != 4)
204 return got_error(GOT_ERR_IO);
205 w = write(fd, buf, nbuf);
206 if (w == -1)
207 return got_error_from_errno("write");
208 if (w != nbuf)
209 return got_error(GOT_ERR_IO);
210 if (chattygot > 1) {
211 fprintf(stderr, "%s: writepkt: %s:\t", getprogname(), len);
212 for (i = 0; i < nbuf; i++) {
213 if (isprint(buf[i]))
214 fputc(buf[i], stderr);
215 else
216 fprintf(stderr, "[0x%.2x]", buf[i]);
218 fputc('\n', stderr);
220 return NULL;
223 static void
224 match_remote_ref(struct got_pathlist_head *have_refs,
225 struct got_object_id *my_id, char *refname)
227 struct got_pathlist_entry *pe;
229 /* XXX zero-hash signifies we don't have this ref;
230 * we should use a flag instead */
231 memset(my_id, 0, sizeof(*my_id));
233 TAILQ_FOREACH(pe, have_refs, entry) {
234 struct got_object_id *id = pe->data;
235 if (strcmp(pe->path, refname) == 0) {
236 memcpy(my_id, id, sizeof(*my_id));
237 break;
242 static int
243 match_branch(const char *branch, const char *wanted_branch)
245 if (strncmp(branch, "refs/heads/", 11) != 0)
246 return 0;
248 if (strncmp(wanted_branch, "refs/heads/", 11) == 0)
249 wanted_branch += 11;
251 return (strcmp(branch + 11, wanted_branch) == 0);
254 static int
255 match_wanted_ref(const char *refname, const char *wanted_ref)
257 if (strncmp(refname, "refs/", 5) != 0)
258 return 0;
259 refname += 5;
261 /*
262 * Prevent fetching of references that won't make any
263 * sense outside of the remote repository's context.
264 */
265 if (strncmp(refname, "got/", 4) == 0)
266 return 0;
267 if (strncmp(refname, "remotes/", 8) == 0)
268 return 0;
270 if (strncmp(wanted_ref, "refs/", 5) == 0)
271 wanted_ref += 5;
273 /* Allow prefix match. */
274 if (got_path_is_child(refname, wanted_ref, strlen(wanted_ref)))
275 return 1;
277 /* Allow exact match. */
278 return (strcmp(refname, wanted_ref) == 0);
281 static const struct got_error *
282 tokenize_refline(char **tokens, char *line, int len, int maxtokens)
284 const struct got_error *err = NULL;
285 char *p;
286 size_t i, n = 0;
288 for (i = 0; i < maxtokens; i++)
289 tokens[i] = NULL;
291 for (i = 0; n < len && i < maxtokens; i++) {
292 while (isspace(*line)) {
293 line++;
294 n++;
296 p = line;
297 while (*line != '\0' &&
298 (!isspace(*line) || i == maxtokens - 1)) {
299 line++;
300 n++;
302 tokens[i] = strndup(p, line - p);
303 if (tokens[i] == NULL) {
304 err = got_error_from_errno("strndup");
305 goto done;
307 /* Skip \0 field-delimiter at end of token. */
308 while (line[0] == '\0' && n < len) {
309 line++;
310 n++;
313 if (i <= 2)
314 err = got_error(GOT_ERR_NOT_REF);
315 done:
316 if (err) {
317 int j;
318 for (j = 0; j < i; j++) {
319 free(tokens[j]);
320 tokens[j] = NULL;
323 return err;
326 static const struct got_error *
327 parse_refline(char **id_str, char **refname, char **server_capabilities,
328 char *line, int len)
330 const struct got_error *err = NULL;
331 char *tokens[3];
333 err = tokenize_refline(tokens, line, len, nitems(tokens));
334 if (err)
335 return err;
337 if (tokens[0])
338 *id_str = tokens[0];
339 if (tokens[1])
340 *refname = tokens[1];
341 if (tokens[2]) {
342 char *p;
343 *server_capabilities = tokens[2];
344 p = strrchr(*server_capabilities, '\n');
345 if (p)
346 *p = '\0';
349 return NULL;
352 #define GOT_CAPA_AGENT "agent"
353 #define GOT_CAPA_OFS_DELTA "ofs-delta"
354 #define GOT_CAPA_SIDE_BAND_64K "side-band-64k"
356 #define GOT_SIDEBAND_PACKFILE_DATA 1
357 #define GOT_SIDEBAND_PROGRESS_INFO 2
358 #define GOT_SIDEBAND_ERROR_INFO 3
361 struct got_capability {
362 const char *key;
363 const char *value;
364 };
365 static const struct got_capability got_capabilities[] = {
366 { GOT_CAPA_AGENT, "got/" GOT_VERSION_STR },
367 { GOT_CAPA_OFS_DELTA, NULL },
368 { GOT_CAPA_SIDE_BAND_64K, NULL },
369 };
371 static const struct got_error *
372 match_capability(char **my_capabilities, const char *capa,
373 const struct got_capability *mycapa)
375 char *equalsign;
376 char *s;
378 equalsign = strchr(capa, '=');
379 if (equalsign) {
380 if (strncmp(capa, mycapa->key, equalsign - capa) != 0)
381 return NULL;
382 } else {
383 if (strcmp(capa, mycapa->key) != 0)
384 return NULL;
387 if (asprintf(&s, "%s %s%s%s",
388 *my_capabilities != NULL ? *my_capabilities : "",
389 mycapa->key,
390 mycapa->value != NULL ? "=" : "",
391 mycapa->value != NULL? mycapa->value : "") == -1)
392 return got_error_from_errno("asprintf");
394 free(*my_capabilities);
395 *my_capabilities = s;
396 return NULL;
399 static const struct got_error *
400 add_symref(struct got_pathlist_head *symrefs, char *capa)
402 const struct got_error *err = NULL;
403 char *colon, *name = NULL, *target = NULL;
405 /* Need at least "A:B" */
406 if (strlen(capa) < 3)
407 return NULL;
409 colon = strchr(capa, ':');
410 if (colon == NULL)
411 return NULL;
413 *colon = '\0';
414 name = strdup(capa);
415 if (name == NULL)
416 return got_error_from_errno("strdup");
418 target = strdup(colon + 1);
419 if (target == NULL) {
420 err = got_error_from_errno("strdup");
421 goto done;
424 /* We can't validate the ref itself here. The main process will. */
425 err = got_pathlist_append(symrefs, name, target);
426 done:
427 if (err) {
428 free(name);
429 free(target);
431 return err;
434 static const struct got_error *
435 match_capabilities(char **my_capabilities, struct got_pathlist_head *symrefs,
436 char *server_capabilities)
438 const struct got_error *err = NULL;
439 char *capa, *equalsign;
440 int i;
442 *my_capabilities = NULL;
443 do {
444 capa = strsep(&server_capabilities, " ");
445 if (capa == NULL)
446 return NULL;
448 equalsign = strchr(capa, '=');
449 if (equalsign != NULL &&
450 strncmp(capa, "symref", equalsign - capa) == 0) {
451 err = add_symref(symrefs, equalsign + 1);
452 if (err)
453 break;
454 continue;
457 for (i = 0; i < nitems(got_capabilities); i++) {
458 err = match_capability(my_capabilities,
459 capa, &got_capabilities[i]);
460 if (err)
461 break;
463 } while (capa);
465 if (*my_capabilities == NULL) {
466 *my_capabilities = strdup("");
467 if (*my_capabilities == NULL)
468 err = got_error_from_errno("strdup");
470 return err;
473 static const struct got_error *
474 send_fetch_server_progress(struct imsgbuf *ibuf, const char *msg, size_t msglen)
476 if (msglen > MAX_IMSGSIZE - IMSG_HEADER_SIZE)
477 return got_error(GOT_ERR_NO_SPACE);
479 if (msglen == 0)
480 return NULL;
482 if (imsg_compose(ibuf, GOT_IMSG_FETCH_SERVER_PROGRESS, 0, 0, -1,
483 msg, msglen) == -1)
484 return got_error_from_errno(
485 "imsg_compose FETCH_SERVER_PROGRESS");
487 return got_privsep_flush_imsg(ibuf);
490 static const struct got_error *
491 send_fetch_download_progress(struct imsgbuf *ibuf, off_t bytes)
493 if (imsg_compose(ibuf, GOT_IMSG_FETCH_DOWNLOAD_PROGRESS, 0, 0, -1,
494 &bytes, sizeof(bytes)) == -1)
495 return got_error_from_errno(
496 "imsg_compose FETCH_DOWNLOAD_PROGRESS");
498 return got_privsep_flush_imsg(ibuf);
501 static const struct got_error *
502 send_fetch_done(struct imsgbuf *ibuf, uint8_t *pack_sha1)
504 if (imsg_compose(ibuf, GOT_IMSG_FETCH_DONE, 0, 0, -1,
505 pack_sha1, SHA1_DIGEST_LENGTH) == -1)
506 return got_error_from_errno("imsg_compose FETCH");
507 return got_privsep_flush_imsg(ibuf);
512 static const struct got_error *
513 fetch_progress(struct imsgbuf *ibuf, const char *buf, size_t len)
515 int i;
517 if (len == 0)
518 return NULL;
520 /*
521 * Truncate messages which exceed the maximum imsg payload size.
522 * Server may send up to 64k.
523 */
524 if (len > MAX_IMSGSIZE - IMSG_HEADER_SIZE)
525 len = MAX_IMSGSIZE - IMSG_HEADER_SIZE;
527 /* Only allow printable ASCII. */
528 for (i = 0; i < len; i++) {
529 if (isprint((unsigned char)buf[i]) ||
530 isspace((unsigned char)buf[i]))
531 continue;
532 return got_error_msg(GOT_ERR_BAD_PACKET,
533 "non-printable progress message received from server");
536 return send_fetch_server_progress(ibuf, buf, len);
539 static const struct got_error *
540 fetch_error(const char *buf, size_t len)
542 static char msg[1024];
543 int i;
545 for (i = 0; i < len && i < sizeof(msg) - 1; i++) {
546 if (!isprint(buf[i]))
547 return got_error_msg(GOT_ERR_BAD_PACKET,
548 "non-printable error message received from server");
549 msg[i] = buf[i];
551 msg[i] = '\0';
552 return got_error_msg(GOT_ERR_FETCH_FAILED, msg);
555 static const struct got_error *
556 send_fetch_symrefs(struct imsgbuf *ibuf, struct got_pathlist_head *symrefs)
558 const struct got_error *err = NULL;
559 struct ibuf *wbuf;
560 size_t len, nsymrefs = 0;
561 struct got_pathlist_entry *pe;
563 len = sizeof(struct got_imsg_fetch_symrefs);
564 TAILQ_FOREACH(pe, symrefs, entry) {
565 const char *target = pe->data;
566 len += sizeof(struct got_imsg_fetch_symref) +
567 pe->path_len + strlen(target);
568 nsymrefs++;
571 if (len >= MAX_IMSGSIZE - IMSG_HEADER_SIZE)
572 return got_error(GOT_ERR_NO_SPACE);
574 wbuf = imsg_create(ibuf, GOT_IMSG_FETCH_SYMREFS, 0, 0, len);
575 if (wbuf == NULL)
576 return got_error_from_errno("imsg_create FETCH_SYMREFS");
578 /* Keep in sync with struct got_imsg_fetch_symrefs definition! */
579 if (imsg_add(wbuf, &nsymrefs, sizeof(nsymrefs)) == -1) {
580 err = got_error_from_errno("imsg_add FETCH_SYMREFS");
581 ibuf_free(wbuf);
582 return err;
585 TAILQ_FOREACH(pe, symrefs, entry) {
586 const char *name = pe->path;
587 size_t name_len = pe->path_len;
588 const char *target = pe->data;
589 size_t target_len = strlen(target);
591 /* Keep in sync with struct got_imsg_fetch_symref definition! */
592 if (imsg_add(wbuf, &name_len, sizeof(name_len)) == -1) {
593 err = got_error_from_errno("imsg_add FETCH_SYMREFS");
594 ibuf_free(wbuf);
595 return err;
597 if (imsg_add(wbuf, &target_len, sizeof(target_len)) == -1) {
598 err = got_error_from_errno("imsg_add FETCH_SYMREFS");
599 ibuf_free(wbuf);
600 return err;
602 if (imsg_add(wbuf, name, name_len) == -1) {
603 err = got_error_from_errno("imsg_add FETCH_SYMREFS");
604 ibuf_free(wbuf);
605 return err;
607 if (imsg_add(wbuf, target, target_len) == -1) {
608 err = got_error_from_errno("imsg_add FETCH_SYMREFS");
609 ibuf_free(wbuf);
610 return err;
614 wbuf->fd = -1;
615 imsg_close(ibuf, wbuf);
616 return got_privsep_flush_imsg(ibuf);
619 static const struct got_error *
620 send_fetch_ref(struct imsgbuf *ibuf, struct got_object_id *refid,
621 const char *refname)
623 const struct got_error *err = NULL;
624 struct ibuf *wbuf;
625 size_t len, reflen = strlen(refname);
627 len = sizeof(struct got_imsg_fetch_ref) + reflen;
628 if (len >= MAX_IMSGSIZE - IMSG_HEADER_SIZE)
629 return got_error(GOT_ERR_NO_SPACE);
631 wbuf = imsg_create(ibuf, GOT_IMSG_FETCH_REF, 0, 0, len);
632 if (wbuf == NULL)
633 return got_error_from_errno("imsg_create FETCH_REF");
635 /* Keep in sync with struct got_imsg_fetch_ref definition! */
636 if (imsg_add(wbuf, refid->sha1, SHA1_DIGEST_LENGTH) == -1) {
637 err = got_error_from_errno("imsg_add FETCH_REF");
638 ibuf_free(wbuf);
639 return err;
641 if (imsg_add(wbuf, refname, reflen) == -1) {
642 err = got_error_from_errno("imsg_add FETCH_REF");
643 ibuf_free(wbuf);
644 return err;
647 wbuf->fd = -1;
648 imsg_close(ibuf, wbuf);
649 return got_privsep_flush_imsg(ibuf);
652 static const struct got_error *
653 fetch_pack(int fd, int packfd, uint8_t *pack_sha1,
654 struct got_pathlist_head *have_refs, int fetch_all_branches,
655 struct got_pathlist_head *wanted_branches,
656 struct got_pathlist_head *wanted_refs, int list_refs_only,
657 struct imsgbuf *ibuf)
659 const struct got_error *err = NULL;
660 char buf[GOT_FETCH_PKTMAX];
661 char hashstr[SHA1_DIGEST_STRING_LENGTH];
662 struct got_object_id *have, *want;
663 int is_firstpkt = 1, nref = 0, refsz = 16;
664 int i, n, nwant = 0, nhave = 0, acked = 0;
665 off_t packsz = 0, last_reported_packsz = 0;
666 char *id_str = NULL, *refname = NULL;
667 char *server_capabilities = NULL, *my_capabilities = NULL;
668 const char *default_branch = NULL;
669 struct got_pathlist_head symrefs;
670 struct got_pathlist_entry *pe;
671 int sent_my_capabilites = 0, have_sidebands = 0;
672 int found_branch = 0;
673 SHA1_CTX sha1_ctx;
674 uint8_t sha1_buf[SHA1_DIGEST_LENGTH];
675 size_t sha1_buf_len = 0;
676 ssize_t w;
678 TAILQ_INIT(&symrefs);
679 SHA1Init(&sha1_ctx);
681 have = malloc(refsz * sizeof(have[0]));
682 if (have == NULL)
683 return got_error_from_errno("malloc");
684 want = malloc(refsz * sizeof(want[0]));
685 if (want == NULL) {
686 err = got_error_from_errno("malloc");
687 goto done;
689 while (1) {
690 err = readpkt(&n, fd, buf, sizeof(buf));
691 if (err)
692 goto done;
693 if (n == 0)
694 break;
695 if (n >= 4 && strncmp(buf, "ERR ", 4) == 0) {
696 err = fetch_error(&buf[4], n - 4);
697 goto done;
699 err = parse_refline(&id_str, &refname, &server_capabilities,
700 buf, n);
701 if (err)
702 goto done;
703 if (is_firstpkt) {
704 if (chattygot && server_capabilities[0] != '\0')
705 fprintf(stderr, "%s: server capabilities: %s\n",
706 getprogname(), server_capabilities);
707 err = match_capabilities(&my_capabilities, &symrefs,
708 server_capabilities);
709 if (err)
710 goto done;
711 if (chattygot)
712 fprintf(stderr, "%s: my capabilities:%s\n",
713 getprogname(), my_capabilities);
714 err = send_fetch_symrefs(ibuf, &symrefs);
715 if (err)
716 goto done;
717 is_firstpkt = 0;
718 if (!fetch_all_branches) {
719 TAILQ_FOREACH(pe, &symrefs, entry) {
720 const char *name = pe->path;
721 const char *symref_target = pe->data;
722 if (strcmp(name, GOT_REF_HEAD) != 0)
723 continue;
724 default_branch = symref_target;
725 break;
728 continue;
730 if (strstr(refname, "^{}")) {
731 if (chattygot) {
732 fprintf(stderr, "%s: ignoring %s\n",
733 getprogname(), refname);
735 continue;
738 if (strncmp(refname, "refs/heads/", 11) == 0) {
739 if (fetch_all_branches || list_refs_only) {
740 found_branch = 1;
741 } else if (!TAILQ_EMPTY(wanted_branches)) {
742 TAILQ_FOREACH(pe, wanted_branches, entry) {
743 if (match_branch(refname, pe->path))
744 break;
746 if (pe == NULL) {
747 if (chattygot) {
748 fprintf(stderr,
749 "%s: ignoring %s\n",
750 getprogname(), refname);
752 continue;
754 found_branch = 1;
755 } else if (default_branch != NULL) {
756 if (!match_branch(refname, default_branch)) {
757 if (chattygot) {
758 fprintf(stderr,
759 "%s: ignoring %s\n",
760 getprogname(), refname);
762 continue;
764 found_branch = 1;
766 } else if (strncmp(refname, "refs/tags/", 10) != 0) {
767 if (!TAILQ_EMPTY(wanted_refs)) {
768 TAILQ_FOREACH(pe, wanted_refs, entry) {
769 if (match_wanted_ref(refname, pe->path))
770 break;
772 if (pe == NULL) {
773 if (chattygot) {
774 fprintf(stderr,
775 "%s: ignoring %s\n",
776 getprogname(), refname);
778 continue;
780 found_branch = 1;
781 } else if (!list_refs_only) {
782 if (chattygot) {
783 fprintf(stderr, "%s: ignoring %s\n",
784 getprogname(), refname);
786 continue;
790 if (refsz == nref + 1) {
791 refsz *= 2;
792 have = reallocarray(have, refsz, sizeof(have[0]));
793 if (have == NULL) {
794 err = got_error_from_errno("reallocarray");
795 goto done;
797 want = reallocarray(want, refsz, sizeof(want[0]));
798 if (want == NULL) {
799 err = got_error_from_errno("reallocarray");
800 goto done;
803 if (!got_parse_sha1_digest(want[nref].sha1, id_str)) {
804 err = got_error(GOT_ERR_BAD_OBJ_ID_STR);
805 goto done;
807 match_remote_ref(have_refs, &have[nref], refname);
808 err = send_fetch_ref(ibuf, &want[nref], refname);
809 if (err)
810 goto done;
812 if (chattygot)
813 fprintf(stderr, "%s: %s will be fetched\n",
814 getprogname(), refname);
815 if (chattygot > 1) {
816 char *theirs, *mine;
817 err = got_object_id_str(&theirs, &want[nref]);
818 if (err)
819 goto done;
820 err = got_object_id_str(&mine, &have[nref]);
821 if (err) {
822 free(theirs);
823 goto done;
825 fprintf(stderr, "%s: remote: %s\n%s: local: %s\n",
826 getprogname(), theirs, getprogname(), mine);
827 free(theirs);
828 free(mine);
830 nref++;
833 if (list_refs_only)
834 goto done;
836 /* Abort if we haven't found any branch to fetch. */
837 if (!found_branch) {
838 err = got_error(GOT_ERR_FETCH_NO_BRANCH);
839 goto done;
842 for (i = 0; i < nref; i++) {
843 if (got_object_id_cmp(&have[i], &want[i]) == 0)
844 continue;
845 got_sha1_digest_to_str(want[i].sha1, hashstr, sizeof(hashstr));
846 n = snprintf(buf, sizeof(buf), "want %s%s\n", hashstr,
847 sent_my_capabilites ? "" : my_capabilities);
848 if (n >= sizeof(buf)) {
849 err = got_error(GOT_ERR_NO_SPACE);
850 goto done;
852 err = writepkt(fd, buf, n);
853 if (err)
854 goto done;
855 sent_my_capabilites = 1;
856 nwant++;
858 err = flushpkt(fd);
859 if (err)
860 goto done;
862 if (nwant == 0)
863 goto done;
865 for (i = 0; i < nref; i++) {
866 if (got_object_id_cmp(&have[i], &zhash) == 0)
867 continue;
868 got_sha1_digest_to_str(have[i].sha1, hashstr, sizeof(hashstr));
869 n = snprintf(buf, sizeof(buf), "have %s\n", hashstr);
870 if (n >= sizeof(buf)) {
871 err = got_error(GOT_ERR_NO_SPACE);
872 goto done;
874 err = writepkt(fd, buf, n);
875 if (err)
876 goto done;
877 nhave++;
880 while (nhave > 0 && !acked) {
881 struct got_object_id common_id;
883 /* The server should ACK the object IDs we need. */
884 err = readpkt(&n, fd, buf, sizeof(buf));
885 if (err)
886 goto done;
887 if (n >= 4 && strncmp(buf, "ERR ", 4) == 0) {
888 err = fetch_error(&buf[4], n - 4);
889 goto done;
891 if (n >= 4 && strncmp(buf, "NAK\n", 4) == 0) {
892 /* Server has not located our objects yet. */
893 continue;
895 if (n < 4 + SHA1_DIGEST_STRING_LENGTH ||
896 strncmp(buf, "ACK ", 4) != 0) {
897 err = got_error_msg(GOT_ERR_BAD_PACKET,
898 "unexpected message from server");
899 goto done;
901 if (!got_parse_sha1_digest(common_id.sha1, buf + 4)) {
902 err = got_error_msg(GOT_ERR_BAD_PACKET,
903 "bad object ID in ACK packet from server");
904 goto done;
906 acked++;
909 n = snprintf(buf, sizeof(buf), "done\n");
910 err = writepkt(fd, buf, n);
911 if (err)
912 goto done;
914 if (nhave == 0) {
915 err = readpkt(&n, fd, buf, sizeof(buf));
916 if (err)
917 goto done;
918 if (n != 4 || strncmp(buf, "NAK\n", n) != 0) {
919 err = got_error_msg(GOT_ERR_BAD_PACKET,
920 "unexpected message from server");
921 goto done;
925 if (chattygot)
926 fprintf(stderr, "%s: fetching...\n", getprogname());
928 if (my_capabilities != NULL &&
929 strstr(my_capabilities, GOT_CAPA_SIDE_BAND_64K) != NULL)
930 have_sidebands = 1;
932 while (1) {
933 ssize_t r = 0;
934 int datalen = -1;
936 if (have_sidebands) {
937 err = read_pkthdr(&datalen, fd);
938 if (err)
939 goto done;
940 if (datalen <= 0)
941 break;
943 /* Read sideband channel ID (one byte). */
944 r = read(fd, buf, 1);
945 if (r == -1) {
946 err = got_error_from_errno("read");
947 goto done;
949 if (r != 1) {
950 err = got_error_msg(GOT_ERR_BAD_PACKET,
951 "short packet");
952 goto done;
954 if (datalen > sizeof(buf) - 5) {
955 err = got_error_msg(GOT_ERR_BAD_PACKET,
956 "bad packet length");
957 goto done;
959 datalen--; /* sideband ID has been read */
960 if (buf[0] == GOT_SIDEBAND_PACKFILE_DATA) {
961 /* Read packfile data. */
962 err = readn(&r, fd, buf, datalen);
963 if (err)
964 goto done;
965 if (r != datalen) {
966 err = got_error_msg(GOT_ERR_BAD_PACKET,
967 "packet too short");
968 goto done;
970 } else if (buf[0] == GOT_SIDEBAND_PROGRESS_INFO) {
971 err = readn(&r, fd, buf, datalen);
972 if (err)
973 goto done;
974 if (r != datalen) {
975 err = got_error_msg(GOT_ERR_BAD_PACKET,
976 "packet too short");
977 goto done;
979 err = fetch_progress(ibuf, buf, r);
980 if (err)
981 goto done;
982 continue;
983 } else if (buf[0] == GOT_SIDEBAND_ERROR_INFO) {
984 err = readn(&r, fd, buf, datalen);
985 if (err)
986 goto done;
987 if (r != datalen) {
988 err = got_error_msg(GOT_ERR_BAD_PACKET,
989 "packet too short");
990 goto done;
992 err = fetch_error(buf, r);
993 goto done;
994 } else {
995 err = got_error_msg(GOT_ERR_BAD_PACKET,
996 "unknown side-band received from server");
997 goto done;
999 } else {
1000 /* No sideband channel. Every byte is packfile data. */
1001 err = readn(&r, fd, buf, sizeof buf);
1002 if (err)
1003 goto done;
1004 if (r <= 0)
1005 break;
1009 * An expected SHA1 checksum sits at the end of the pack file.
1010 * Since we don't know the file size ahead of time we have to
1011 * keep SHA1_DIGEST_LENGTH bytes buffered and avoid mixing
1012 * those bytes into our SHA1 checksum computation until we
1013 * know for sure that additional pack file data bytes follow.
1015 * We can assume r > 0 since otherwise the loop would exit.
1017 if (r < SHA1_DIGEST_LENGTH) {
1018 if (sha1_buf_len < SHA1_DIGEST_LENGTH) {
1020 * If there's enough buffered + read data to
1021 * fill up the buffer then shift a sufficient
1022 * amount of bytes out at the front to make
1023 * room, mixing those bytes into the checksum.
1025 while (sha1_buf_len > 0 &&
1026 sha1_buf_len + r > SHA1_DIGEST_LENGTH) {
1027 SHA1Update(&sha1_ctx, sha1_buf, 1);
1028 memmove(sha1_buf, sha1_buf + 1, 1);
1029 sha1_buf_len--;
1032 /* Buffer potential checksum bytes. */
1033 memcpy(sha1_buf + sha1_buf_len, buf, r);
1034 sha1_buf_len += r;
1035 } else {
1037 * Mix in previously buffered bytes which
1038 * are not part of the checksum after all.
1040 SHA1Update(&sha1_ctx, sha1_buf, r);
1042 /* Update potential checksum buffer. */
1043 memmove(sha1_buf, sha1_buf + r,
1044 sha1_buf_len - r);
1045 memcpy(sha1_buf + sha1_buf_len - r, buf, r);
1047 } else {
1048 /* Mix in any previously buffered bytes. */
1049 SHA1Update(&sha1_ctx, sha1_buf, sha1_buf_len);
1051 /* Mix in bytes read minus potential checksum bytes. */
1052 SHA1Update(&sha1_ctx, buf, r - SHA1_DIGEST_LENGTH);
1054 /* Buffer potential checksum bytes. */
1055 memcpy(sha1_buf, buf + r - SHA1_DIGEST_LENGTH,
1056 SHA1_DIGEST_LENGTH);
1057 sha1_buf_len = SHA1_DIGEST_LENGTH;
1060 /* Write packfile data to temporary pack file. */
1061 w = write(packfd, buf, r);
1062 if (w == -1) {
1063 err = got_error_from_errno("write");
1064 goto done;
1066 if (w != r) {
1067 err = got_error(GOT_ERR_IO);
1068 goto done;
1070 packsz += w;
1072 /* Don't send too many progress privsep messages. */
1073 if (packsz > last_reported_packsz + 1024) {
1074 err = send_fetch_download_progress(ibuf, packsz);
1075 if (err)
1076 goto done;
1077 last_reported_packsz = packsz;
1080 err = send_fetch_download_progress(ibuf, packsz);
1081 if (err)
1082 goto done;
1084 SHA1Final(pack_sha1, &sha1_ctx);
1085 if (sha1_buf_len != SHA1_DIGEST_LENGTH ||
1086 memcmp(pack_sha1, sha1_buf, sha1_buf_len) != 0) {
1087 err = got_error_msg(GOT_ERR_BAD_PACKFILE,
1088 "pack file checksum mismatch");
1090 done:
1091 TAILQ_FOREACH(pe, &symrefs, entry) {
1092 free((void *)pe->path);
1093 free(pe->data);
1095 got_pathlist_free(&symrefs);
1096 free(have);
1097 free(want);
1098 free(id_str);
1099 free(refname);
1100 free(server_capabilities);
1101 return err;
1105 int
1106 main(int argc, char **argv)
1108 const struct got_error *err = NULL;
1109 int fetchfd, packfd = -1, i;
1110 uint8_t pack_sha1[SHA1_DIGEST_LENGTH];
1111 struct imsgbuf ibuf;
1112 struct imsg imsg;
1113 struct got_pathlist_head have_refs;
1114 struct got_pathlist_head wanted_branches;
1115 struct got_pathlist_head wanted_refs;
1116 struct got_pathlist_entry *pe;
1117 struct got_imsg_fetch_request fetch_req;
1118 struct got_imsg_fetch_have_ref href;
1119 struct got_imsg_fetch_wanted_branch wbranch;
1120 struct got_imsg_fetch_wanted_ref wref;
1121 size_t datalen;
1122 #if 0
1123 static int attached;
1124 while (!attached)
1125 sleep (1);
1126 #endif
1128 TAILQ_INIT(&have_refs);
1129 TAILQ_INIT(&wanted_branches);
1130 TAILQ_INIT(&wanted_refs);
1132 imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
1133 #ifndef PROFILE
1134 /* revoke access to most system calls */
1135 if (pledge("stdio recvfd", NULL) == -1) {
1136 err = got_error_from_errno("pledge");
1137 got_privsep_send_error(&ibuf, err);
1138 return 1;
1140 #endif
1141 if ((err = got_privsep_recv_imsg(&imsg, &ibuf, 0)) != 0) {
1142 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1143 err = NULL;
1144 goto done;
1146 if (imsg.hdr.type == GOT_IMSG_STOP)
1147 goto done;
1148 if (imsg.hdr.type != GOT_IMSG_FETCH_REQUEST) {
1149 err = got_error(GOT_ERR_PRIVSEP_MSG);
1150 goto done;
1152 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1153 if (datalen < sizeof(fetch_req)) {
1154 err = got_error(GOT_ERR_PRIVSEP_LEN);
1155 goto done;
1157 memcpy(&fetch_req, imsg.data, sizeof(fetch_req));
1158 fetchfd = imsg.fd;
1159 imsg_free(&imsg);
1161 if (fetch_req.verbosity > 0)
1162 chattygot += fetch_req.verbosity;
1164 for (i = 0; i < fetch_req.n_have_refs; i++) {
1165 struct got_object_id *id;
1166 char *refname;
1168 if ((err = got_privsep_recv_imsg(&imsg, &ibuf, 0)) != 0) {
1169 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1170 err = NULL;
1171 goto done;
1173 if (imsg.hdr.type == GOT_IMSG_STOP)
1174 goto done;
1175 if (imsg.hdr.type != GOT_IMSG_FETCH_HAVE_REF) {
1176 err = got_error(GOT_ERR_PRIVSEP_MSG);
1177 goto done;
1179 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1180 if (datalen < sizeof(href)) {
1181 err = got_error(GOT_ERR_PRIVSEP_LEN);
1182 goto done;
1184 memcpy(&href, imsg.data, sizeof(href));
1185 if (datalen - sizeof(href) < href.name_len) {
1186 err = got_error(GOT_ERR_PRIVSEP_LEN);
1187 goto done;
1189 refname = malloc(href.name_len + 1);
1190 if (refname == NULL) {
1191 err = got_error_from_errno("malloc");
1192 goto done;
1194 memcpy(refname, imsg.data + sizeof(href), href.name_len);
1195 refname[href.name_len] = '\0';
1197 id = malloc(sizeof(*id));
1198 if (id == NULL) {
1199 free(refname);
1200 err = got_error_from_errno("malloc");
1201 goto done;
1203 memcpy(id->sha1, href.id, SHA1_DIGEST_LENGTH);
1204 err = got_pathlist_append(&have_refs, refname, id);
1205 if (err) {
1206 free(refname);
1207 free(id);
1208 goto done;
1211 imsg_free(&imsg);
1214 for (i = 0; i < fetch_req.n_wanted_branches; i++) {
1215 char *refname;
1217 if ((err = got_privsep_recv_imsg(&imsg, &ibuf, 0)) != 0) {
1218 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1219 err = NULL;
1220 goto done;
1222 if (imsg.hdr.type == GOT_IMSG_STOP)
1223 goto done;
1224 if (imsg.hdr.type != GOT_IMSG_FETCH_WANTED_BRANCH) {
1225 err = got_error(GOT_ERR_PRIVSEP_MSG);
1226 goto done;
1228 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1229 if (datalen < sizeof(wbranch)) {
1230 err = got_error(GOT_ERR_PRIVSEP_LEN);
1231 goto done;
1233 memcpy(&wbranch, imsg.data, sizeof(wbranch));
1234 if (datalen - sizeof(wbranch) < wbranch.name_len) {
1235 err = got_error(GOT_ERR_PRIVSEP_LEN);
1236 goto done;
1238 refname = malloc(wbranch.name_len + 1);
1239 if (refname == NULL) {
1240 err = got_error_from_errno("malloc");
1241 goto done;
1243 memcpy(refname, imsg.data + sizeof(wbranch), wbranch.name_len);
1244 refname[wbranch.name_len] = '\0';
1246 err = got_pathlist_append(&wanted_branches, refname, NULL);
1247 if (err) {
1248 free(refname);
1249 goto done;
1252 imsg_free(&imsg);
1255 for (i = 0; i < fetch_req.n_wanted_refs; i++) {
1256 char *refname;
1258 if ((err = got_privsep_recv_imsg(&imsg, &ibuf, 0)) != 0) {
1259 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1260 err = NULL;
1261 goto done;
1263 if (imsg.hdr.type == GOT_IMSG_STOP)
1264 goto done;
1265 if (imsg.hdr.type != GOT_IMSG_FETCH_WANTED_REF) {
1266 err = got_error(GOT_ERR_PRIVSEP_MSG);
1267 goto done;
1269 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1270 if (datalen < sizeof(wref)) {
1271 err = got_error(GOT_ERR_PRIVSEP_LEN);
1272 goto done;
1274 memcpy(&wref, imsg.data, sizeof(wref));
1275 if (datalen - sizeof(wref) < wref.name_len) {
1276 err = got_error(GOT_ERR_PRIVSEP_LEN);
1277 goto done;
1279 refname = malloc(wref.name_len + 1);
1280 if (refname == NULL) {
1281 err = got_error_from_errno("malloc");
1282 goto done;
1284 memcpy(refname, imsg.data + sizeof(wref), wref.name_len);
1285 refname[wref.name_len] = '\0';
1287 err = got_pathlist_append(&wanted_refs, refname, NULL);
1288 if (err) {
1289 free(refname);
1290 goto done;
1293 imsg_free(&imsg);
1296 if ((err = got_privsep_recv_imsg(&imsg, &ibuf, 0)) != 0) {
1297 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1298 err = NULL;
1299 goto done;
1301 if (imsg.hdr.type == GOT_IMSG_STOP)
1302 goto done;
1303 if (imsg.hdr.type != GOT_IMSG_FETCH_OUTFD) {
1304 err = got_error(GOT_ERR_PRIVSEP_MSG);
1305 goto done;
1307 if (imsg.hdr.len - IMSG_HEADER_SIZE != 0) {
1308 err = got_error(GOT_ERR_PRIVSEP_LEN);
1309 goto done;
1311 packfd = imsg.fd;
1313 err = fetch_pack(fetchfd, packfd, pack_sha1, &have_refs,
1314 fetch_req.fetch_all_branches, &wanted_branches,
1315 &wanted_refs, fetch_req.list_refs_only, &ibuf);
1316 done:
1317 TAILQ_FOREACH(pe, &have_refs, entry) {
1318 free((char *)pe->path);
1319 free(pe->data);
1321 got_pathlist_free(&have_refs);
1322 TAILQ_FOREACH(pe, &wanted_branches, entry)
1323 free((char *)pe->path);
1324 got_pathlist_free(&wanted_branches);
1325 if (fetchfd != -1 && close(fetchfd) == -1 && err == NULL)
1326 err = got_error_from_errno("close");
1327 if (packfd != -1 && close(packfd) == -1 && err == NULL)
1328 err = got_error_from_errno("close");
1329 if (err != NULL)
1330 got_privsep_send_error(&ibuf, err);
1331 else
1332 err = send_fetch_done(&ibuf, pack_sha1);
1333 if (err != NULL) {
1334 fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
1335 got_privsep_send_error(&ibuf, err);
1338 exit(0);