commit 55da3778cb79bda6ee8eda5f117a838b4fd50a83 from: Stefan Sperling date: Mon Sep 10 16:39:31 2018 UTC read packed blobs with privsep commit - e78854052d44df6460677e042f83e2bae4cb06e0 commit + 55da3778cb79bda6ee8eda5f117a838b4fd50a83 blob - b427544ee597fb40d4802b61166507af2c7f124e blob + d77cc53056e356b8d086100eaa0cabc75ebe746e --- lib/got_lib_object_parse.h +++ lib/got_lib_object_parse.h @@ -44,3 +44,5 @@ const struct got_error *got_object_read_packed_commit_ struct got_commit_object **, struct got_object *, struct got_pack *); const struct got_error *got_object_read_packed_tree_privsep( struct got_tree_object **, struct got_object *, struct got_pack *); +const struct got_error *got_object_read_packed_blob_privsep(size_t *, int, + struct got_object *, struct got_pack *); blob - 7a9837d0906b4e613b5d2adf36530508388b9473 blob + c36311dd363087ed4b79603a54bd1cf61fe22145 --- lib/got_lib_privsep.h +++ lib/got_lib_privsep.h @@ -210,7 +210,8 @@ const struct got_error *got_privsep_recv_imsg(struct i void got_privsep_send_error(struct imsgbuf *, const struct got_error *); const struct got_error *got_privsep_send_obj_req(struct imsgbuf *, int, struct got_object *); -const struct got_error *got_privsep_send_blob_req(struct imsgbuf *, int, int); +const struct got_error *got_privsep_send_blob_req(struct imsgbuf *, int); +const struct got_error *got_privsep_send_blob_outfd(struct imsgbuf *, int); const struct got_error *got_privsep_send_obj(struct imsgbuf *, struct got_object *); const struct got_error *got_privsep_get_imsg_obj(struct got_object **, blob - dd2760739e7a9945475e8be34c37f8273d9865da blob + da1f8b58653dd67775c857702048d3d2aab690f8 --- lib/object.c +++ lib/object.c @@ -440,53 +440,15 @@ got_object_tree_get_entries(struct got_tree_object *tr { return &tree->entries; } - -static const struct got_error * -extract_packed_object(FILE **f, struct got_object *obj, - struct got_repository *repo) -{ - const struct got_error *err = NULL; - struct got_pack *pack; - int fd; - - if ((obj->flags & GOT_OBJ_FLAG_PACKED) == 0) - return got_error(GOT_ERR_OBJ_NOT_PACKED); - - fd = got_opentempfd(); - if (fd == -1) - return got_error_from_errno(); - - *f = fdopen(fd, "w+"); - if (*f == NULL) { - err = got_error_from_errno(); - goto done; - } - pack = got_repo_get_cached_pack(repo, obj->path_packfile); - if (pack == NULL) { - err = got_repo_cache_pack(&pack, repo, - obj->path_packfile, NULL); - if (err) - goto done; - } - - err = got_packfile_extract_object(pack, obj, *f); -done: - if (err) { - if (*f == NULL) - close(fd); - else - fclose(*f); - *f = NULL; - } - return err; -} - const struct got_error * got_object_blob_open(struct got_blob_object **blob, struct got_repository *repo, struct got_object *obj, size_t blocksize) { const struct got_error *err = NULL; + int outfd; + size_t size; + struct stat sb; if (obj->type != GOT_OBJ_TYPE_BLOB) return got_error(GOT_ERR_OBJ_TYPE); @@ -498,32 +460,36 @@ got_object_blob_open(struct got_blob_object **blob, if (*blob == NULL) return got_error_from_errno(); + outfd = got_opentempfd(); + if (outfd == -1) + return got_error_from_errno(); + (*blob)->read_buf = malloc(blocksize); if ((*blob)->read_buf == NULL) { err = got_error_from_errno(); goto done; } if (obj->flags & GOT_OBJ_FLAG_PACKED) { - err = extract_packed_object(&((*blob)->f), obj, repo); + struct got_pack *pack; + pack = got_repo_get_cached_pack(repo, obj->path_packfile); + if (pack == NULL) { + err = got_repo_cache_pack(&pack, repo, + obj->path_packfile, NULL); + if (err) + goto done; + } + err = got_object_read_packed_blob_privsep(&size, outfd, + obj, pack); if (err) goto done; + obj->size = size; } else { - int infd, outfd; - size_t size; - struct stat sb; + int infd; err = open_loose_object(&infd, obj, repo); if (err) goto done; - - outfd = got_opentempfd(); - if (outfd == -1) { - err = got_error_from_errno(); - close(infd); - goto done; - } - err = got_object_read_blob_privsep(&size, outfd, infd, repo); close(infd); if (err) @@ -531,28 +497,25 @@ got_object_blob_open(struct got_blob_object **blob, if (size != obj->hdrlen + obj->size) { err = got_error(GOT_ERR_PRIVSEP_LEN); - close(outfd); goto done; } + } - if (fstat(outfd, &sb) == -1) { - err = got_error_from_errno(); - close(outfd); - goto done; - } + if (fstat(outfd, &sb) == -1) { + err = got_error_from_errno(); + goto done; + } - if (sb.st_size != size) { - err = got_error(GOT_ERR_PRIVSEP_LEN); - close(outfd); - goto done; - } + if (sb.st_size != obj->hdrlen + obj->size) { + err = got_error(GOT_ERR_PRIVSEP_LEN); + goto done; + } - (*blob)->f = fdopen(outfd, "rb"); - if ((*blob)->f == NULL) { - err = got_error_from_errno(); - close(outfd); - goto done; - } + (*blob)->f = fdopen(outfd, "rb"); + if ((*blob)->f == NULL) { + err = got_error_from_errno(); + close(outfd); + goto done; } (*blob)->hdrlen = obj->hdrlen; @@ -560,12 +523,15 @@ got_object_blob_open(struct got_blob_object **blob, memcpy(&(*blob)->id.sha1, obj->id.sha1, SHA1_DIGEST_LENGTH); done: - if (err && *blob) { - if ((*blob)->f) - fclose((*blob)->f); - free((*blob)->read_buf); - free(*blob); - *blob = NULL; + if (err) { + if (*blob) { + if ((*blob)->f) + fclose((*blob)->f); + free((*blob)->read_buf); + free(*blob); + *blob = NULL; + } else if (outfd != -1) + close(outfd); } return err; } blob - ae91d511f1877e2f80758d82825de45907f3aa09 blob + fc364fdac12e464e94cb9080000d585d44d3d3b1 --- lib/object_parse.c +++ lib/object_parse.c @@ -837,23 +837,58 @@ got_object_read_packed_tree_privsep(struct got_tree_ob return got_privsep_recv_tree(tree, pack->privsep_child->ibuf); } -static const struct got_error * -request_blob(size_t *size, int outfd, int infd, struct got_repository *repo) +const struct got_error * +got_object_read_packed_blob_privsep(size_t *size, int outfd, + struct got_object *obj, struct got_pack *pack) { const struct got_error *err = NULL; int outfd_child; - struct imsgbuf *ibuf; outfd_child = dup(outfd); if (outfd_child == -1) return got_error_from_errno(); - ibuf = repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].ibuf; + err = got_privsep_send_obj_req(pack->privsep_child->ibuf, -1, obj); + if (err) + return err; - err = got_privsep_send_blob_req(ibuf, outfd_child, infd); + err = got_privsep_send_blob_outfd(pack->privsep_child->ibuf, + outfd_child); + if (err) { + close(outfd_child); + return err; + } + + err = got_privsep_recv_blob(size, pack->privsep_child->ibuf); if (err) return err; + if (lseek(outfd, SEEK_SET, 0) == -1) + err = got_error_from_errno(); + + return err; +} + +static const struct got_error * +request_blob(size_t *size, int outfd, int infd, struct imsgbuf *ibuf) +{ + const struct got_error *err = NULL; + int outfd_child; + + outfd_child = dup(outfd); + if (outfd_child == -1) + return got_error_from_errno(); + + err = got_privsep_send_blob_req(ibuf, infd); + if (err) + return err; + + err = got_privsep_send_blob_outfd(ibuf, outfd_child); + if (err) { + close(outfd_child); + return err; + } + err = got_privsep_recv_blob(size, ibuf); if (err) return err; @@ -872,8 +907,10 @@ got_object_read_blob_privsep(size_t *size, int outfd, pid_t pid; struct imsgbuf *ibuf; - if (repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].imsg_fd != -1) - return request_blob(size, outfd, infd, repo); + if (repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].imsg_fd != -1) { + ibuf = repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].ibuf; + return request_blob(size, outfd, infd, ibuf); + } ibuf = calloc(1, sizeof(*ibuf)); if (ibuf == NULL) @@ -898,5 +935,5 @@ got_object_read_blob_privsep(size_t *size, int outfd, imsg_init(ibuf, imsg_fds[0]); repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].ibuf = ibuf; - return request_blob(size, outfd, infd, repo); + return request_blob(size, outfd, infd, ibuf); } blob - faa53d4db1d1e837fd01b682d269781bc0303fca blob + b69b1fe537b7a8f35cb059168674e185ad81e1c7 --- lib/privsep.c +++ lib/privsep.c @@ -269,7 +269,9 @@ got_privsep_send_obj_req(struct imsgbuf *ibuf, int fd, case GOT_OBJ_TYPE_COMMIT: imsg_code = GOT_IMSG_COMMIT_REQUEST; break; - /* Blobs are handled in got_privsep_send_blob_req(). */ + case GOT_OBJ_TYPE_BLOB: + imsg_code = GOT_IMSG_BLOB_REQUEST; + break; default: return got_error(GOT_ERR_OBJ_TYPE); } @@ -306,28 +308,21 @@ got_privsep_send_obj_req(struct imsgbuf *ibuf, int fd, } const struct got_error * -got_privsep_send_blob_req(struct imsgbuf *ibuf, int outfd, int infd) -{ - const struct got_error *err = NULL; - +got_privsep_send_blob_req(struct imsgbuf *ibuf, int infd) +{ if (imsg_compose(ibuf, GOT_IMSG_BLOB_REQUEST, 0, 0, infd, NULL, 0) - == -1) { - close(infd); - close(outfd); + == -1) return got_error_from_errno(); - } - err = flush_imsg(ibuf); - if (err) { - close(outfd); - return err; - } + return flush_imsg(ibuf); +} +const struct got_error * +got_privsep_send_blob_outfd(struct imsgbuf *ibuf, int outfd) +{ if (imsg_compose(ibuf, GOT_IMSG_BLOB_OUTFD, 0, 0, outfd, NULL, 0) - == -1) { - close(outfd); + == -1) return got_error_from_errno(); - } return flush_imsg(ibuf); } blob - eaf315f5317e307d53c4178c251da9213dd36241 blob + 24a35777d623a89c76f44ac36125a7289527bec2 --- libexec/got-read-blob/got-read-blob.c +++ libexec/got-read-blob/got-read-blob.c @@ -59,7 +59,9 @@ main(int argc, char *argv[]) FILE *f = NULL; size_t size; + memset(&imsg, 0, sizeof(imsg)); imsg.fd = -1; + memset(&imsg_outfd, 0, sizeof(imsg_outfd)); imsg_outfd.fd = -1; err = got_privsep_recv_imsg(&imsg, &ibuf, 0); blob - 8110b5f191c21cb946f29466fcbef12db68721f5 blob + c6634c689a2cfb7fe806ec3e4eedb5bc3cd70cd5 --- libexec/got-read-pack/got-read-pack.c +++ libexec/got-read-pack/got-read-pack.c @@ -146,7 +146,70 @@ static const struct got_error * blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack, struct got_packidx *packidx) { - return got_error(GOT_ERR_NOT_IMPL); + const struct got_error *err = NULL; + struct got_object *obj = NULL; + FILE *f = NULL; + struct imsg imsg_outfd; + size_t datalen; + + memset(&imsg_outfd, 0, sizeof(imsg_outfd)); + imsg_outfd.fd = -1; + + err = got_privsep_get_imsg_obj(&obj, imsg, ibuf); + if (err) + return err; + + if (obj->type != GOT_OBJ_TYPE_BLOB) + return got_error(GOT_ERR_OBJ_TYPE); + + if ((obj->flags & GOT_OBJ_FLAG_PACKED) == 0) + return got_error(GOT_ERR_OBJ_NOT_PACKED); + + err = got_privsep_recv_imsg(&imsg_outfd, ibuf, 0); + if (err) + return err; + + if (imsg_outfd.hdr.type != GOT_IMSG_BLOB_OUTFD) { + err = got_error(GOT_ERR_PRIVSEP_MSG); + goto done; + } + + datalen = imsg_outfd.hdr.len - IMSG_HEADER_SIZE; + if (datalen != 0) { + err = got_error(GOT_ERR_PRIVSEP_LEN); + goto done; + } + if (imsg_outfd.fd == -1) { + err = got_error(GOT_ERR_PRIVSEP_NO_FD); + goto done; + } + + f = fdopen(imsg_outfd.fd, "w+"); + if (f == NULL) { + err = got_error_from_errno(); + goto done; + } + + err = got_packfile_extract_object(pack, obj, f); + if (err) + goto done; + + err = got_privsep_send_blob(ibuf, obj->size); +done: + if (f) + fclose(f); + else if (imsg_outfd.fd != -1) + close(imsg_outfd.fd); + imsg_free(&imsg_outfd); + + if (err) { + if (err->code == GOT_ERR_PRIVSEP_PIPE) + err = NULL; + else + got_privsep_send_error(ibuf, err); + } + + return err; } static const struct got_error *