commit ac544f8c6ffbe7126fa9ae6c7dd09c6661d1ef2d from: Stefan Sperling date: Sun Jan 13 14:58:39 2019 UTC pass smallish blobs directly via imsg buffer commit - c65fcef2fb5ed060705587a71de3fc3eff245082 commit + ac544f8c6ffbe7126fa9ae6c7dd09c6661d1ef2d blob - 14bd628bae7ca8d8048fca27f37f546c5aca6701 blob + 628c56137d7ed88aea67d80ba7b95c6be67bdecf --- lib/got_lib_object.h +++ lib/got_lib_object.h @@ -57,6 +57,7 @@ struct got_tree_object { struct got_blob_object { FILE *f; + uint8_t *data; struct got_zstream_buf zb; size_t hdrlen; size_t blocksize; blob - 63c033d6f774a39c9e2058276608a89c1daa1342 blob + 827635172297ba1c08d85c6c7e6d1ce7a2fcc7bb --- lib/got_lib_pack.h +++ lib/got_lib_pack.h @@ -164,6 +164,8 @@ int got_packidx_get_object_idx(struct got_packidx *, s const struct got_error *got_packfile_open_object(struct got_object **, struct got_pack *, struct got_packidx *, int, struct got_object_id *); +const struct got_error *got_pack_get_object_size(uint64_t *, + struct got_object *); const struct got_error *got_packfile_extract_object(struct got_pack *, struct got_object *, FILE *, FILE *, FILE *); const struct got_error *got_packfile_extract_object_to_mem(uint8_t **, size_t *, blob - 265fdf3b2b2444e83f6f86acd875fa01bb1da614 blob + 9d915bb9d69bfcf9dac03d5d3595636bfecfebec --- lib/got_lib_privsep.h +++ lib/got_lib_privsep.h @@ -166,8 +166,17 @@ struct got_imsg_tree_object { struct got_imsg_blob { size_t size; size_t hdrlen; + + /* + * If size <= GOT_PRIVSEP_INLINE_BLOB_DATA_MAX, blob data follows + * in the imsg buffer. Otherwise, blob data has been written to a + * file descriptor passed via the GOT_IMSG_BLOB_OUTFD imsg. + */ +#define GOT_PRIVSEP_INLINE_BLOB_DATA_MAX \ + (MAX_IMSGSIZE - IMSG_HEADER_SIZE - sizeof(struct got_imsg_blob)) }; + /* Structure for GOT_IMSG_TAG data. */ struct got_imsg_tag_object { uint8_t id[SHA1_DIGEST_LENGTH]; @@ -242,8 +251,9 @@ const struct got_error *got_privsep_recv_tree(struct g struct imsgbuf *); const struct got_error *got_privsep_send_tree(struct imsgbuf *, struct got_tree_object *); -const struct got_error *got_privsep_send_blob(struct imsgbuf *, size_t, size_t); -const struct got_error *got_privsep_recv_blob(size_t *, size_t *, +const struct got_error *got_privsep_send_blob(struct imsgbuf *, size_t, size_t, + const uint8_t *); +const struct got_error *got_privsep_recv_blob(uint8_t **, size_t *, size_t *, struct imsgbuf *); const struct got_error *got_privsep_send_tag(struct imsgbuf *, struct got_tag_object *); blob - daab8a5a6c149bb4f48f9aa3f6c7e0b89d1bb1cb blob + 2e2c89ad38123377ed31f13eb51696332766d8dd --- lib/object.c +++ lib/object.c @@ -815,7 +815,7 @@ got_object_tree_get_entries(struct got_tree_object *tr } static const struct got_error * -request_packed_blob(size_t *size, size_t *hdrlen, int outfd, +request_packed_blob(uint8_t **outbuf, size_t *size, size_t *hdrlen, int outfd, struct got_pack *pack, struct got_packidx *packidx, int idx, struct got_object_id *id) { @@ -861,7 +861,7 @@ request_packed_blob(size_t *size, size_t *hdrlen, int return err; } - err = got_privsep_recv_blob(size, hdrlen, + err = got_privsep_recv_blob(outbuf, size, hdrlen, pack->privsep_child->ibuf); if (err) return err; @@ -873,8 +873,8 @@ request_packed_blob(size_t *size, size_t *hdrlen, int } static const struct got_error * -read_packed_blob_privsep(size_t *size, size_t *hdrlen, int outfd, - struct got_pack *pack, struct got_packidx *packidx, int idx, +read_packed_blob_privsep(uint8_t **outbuf, size_t *size, size_t *hdrlen, + int outfd, struct got_pack *pack, struct got_packidx *packidx, int idx, struct got_object_id *id) { const struct got_error *err = NULL; @@ -885,12 +885,13 @@ read_packed_blob_privsep(size_t *size, size_t *hdrlen, return err; } - return request_packed_blob(size, hdrlen, outfd, pack, packidx, idx, id); + return request_packed_blob(outbuf, size, hdrlen, outfd, pack, packidx, + idx, id); } static const struct got_error * -request_blob(size_t *size, size_t *hdrlen, int outfd, int infd, - struct imsgbuf *ibuf) +request_blob(uint8_t **outbuf, size_t *size, size_t *hdrlen, int outfd, + int infd, struct imsgbuf *ibuf) { const struct got_error *err = NULL; int outfd_child; @@ -909,7 +910,7 @@ request_blob(size_t *size, size_t *hdrlen, int outfd, return err; } - err = got_privsep_recv_blob(size, hdrlen, ibuf); + err = got_privsep_recv_blob(outbuf, size, hdrlen, ibuf); if (err) return err; @@ -920,7 +921,7 @@ request_blob(size_t *size, size_t *hdrlen, int outfd, } static const struct got_error * -read_blob_privsep(size_t *size, size_t *hdrlen, +read_blob_privsep(uint8_t **outbuf, size_t *size, size_t *hdrlen, int outfd, int infd, struct got_repository *repo) { int imsg_fds[2]; @@ -929,7 +930,7 @@ read_blob_privsep(size_t *size, size_t *hdrlen, if (repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].imsg_fd != -1) { ibuf = repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].ibuf; - return request_blob(size, hdrlen, outfd, infd, ibuf); + return request_blob(outbuf, size, hdrlen, outfd, infd, ibuf); } ibuf = calloc(1, sizeof(*ibuf)); @@ -955,7 +956,7 @@ read_blob_privsep(size_t *size, size_t *hdrlen, imsg_init(ibuf, imsg_fds[0]); repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].ibuf = ibuf; - return request_blob(size, hdrlen, outfd, infd, ibuf); + return request_blob(outbuf, size, hdrlen, outfd, infd, ibuf); } static const struct got_error * @@ -966,6 +967,7 @@ open_blob(struct got_blob_object **blob, struct got_re struct got_packidx *packidx = NULL; int idx; char *path_packfile; + uint8_t *outbuf; int outfd; size_t size, hdrlen; struct stat sb; @@ -999,40 +1001,54 @@ open_blob(struct got_blob_object **blob, struct got_re if (err) goto done; } - err = read_packed_blob_privsep(&size, &hdrlen, outfd, pack, - packidx, idx, id); + err = read_packed_blob_privsep(&outbuf, &size, &hdrlen, outfd, + pack, packidx, idx, id); } else if (err->code == GOT_ERR_NO_OBJ) { int infd; err = open_loose_object(&infd, id, repo); if (err) goto done; - err = read_blob_privsep(&size, &hdrlen, outfd, infd, repo); + err = read_blob_privsep(&outbuf, &size, &hdrlen, outfd, infd, + repo); close(infd); } if (err) - goto done; - - if (fstat(outfd, &sb) == -1) { - err = got_error_from_errno(); - goto done; - } - - if (sb.st_size != size) { - err = got_error(GOT_ERR_PRIVSEP_LEN); goto done; - } if (hdrlen > size) { err = got_error(GOT_ERR_BAD_OBJ_HDR); goto done; } - (*blob)->f = fdopen(outfd, "rb"); - if ((*blob)->f == NULL) { - err = got_error_from_errno(); + if (outbuf) { close(outfd); - goto done; + outfd = -1; + (*blob)->f = fmemopen(outbuf, size, "rb"); + if ((*blob)->f == NULL) { + err = got_error_from_errno(); + free(outbuf); + goto done; + } + (*blob)->data = outbuf; + } else { + if (fstat(outfd, &sb) == -1) { + err = got_error_from_errno(); + goto done; + } + + if (sb.st_size != size) { + err = got_error(GOT_ERR_PRIVSEP_LEN); + goto done; + } + + (*blob)->f = fdopen(outfd, "rb"); + if ((*blob)->f == NULL) { + err = got_error_from_errno(); + close(outfd); + outfd = -1; + goto done; + } } (*blob)->hdrlen = hdrlen; @@ -1045,6 +1061,7 @@ done: if ((*blob)->f) fclose((*blob)->f); free((*blob)->read_buf); + free((*blob)->data); free(*blob); *blob = NULL; } else if (outfd != -1) @@ -1073,6 +1090,7 @@ got_object_blob_close(struct got_blob_object *blob) { free(blob->read_buf); fclose(blob->f); + free(blob->data); free(blob); } blob - 73190458ebaf7433d7caacf33a385c026139d5dd blob + 9a2d1c22cb22cb80e0474babc8eb35cd717132e2 --- lib/pack.c +++ lib/pack.c @@ -984,6 +984,12 @@ get_delta_chain_max_size(uint64_t *max_size, struct go } return NULL; +} + +const struct got_error * +got_pack_get_object_size(uint64_t *size, struct got_object *obj) +{ + return get_delta_chain_max_size(size, &obj->deltas); } static const struct got_error * blob - 923a10bad969053dcbbfd987f29c08ae6547123e blob + 2f9a594c755105e26826e0f439de56d36ce7923f --- lib/privsep.c +++ lib/privsep.c @@ -832,29 +832,54 @@ done: } const struct got_error * -got_privsep_send_blob(struct imsgbuf *ibuf, size_t size, size_t hdrlen) +got_privsep_send_blob(struct imsgbuf *ibuf, size_t size, size_t hdrlen, + const uint8_t *data) { struct got_imsg_blob iblob; iblob.size = size; iblob.hdrlen = hdrlen; - /* Data has already been written to file descriptor. */ - if (imsg_compose(ibuf, GOT_IMSG_BLOB, 0, 0, -1, &iblob, sizeof(iblob)) - == -1) - return got_error_from_errno(); + if (data) { + uint8_t *buf; + + if (size > GOT_PRIVSEP_INLINE_BLOB_DATA_MAX) + return got_error(GOT_ERR_NO_SPACE); + + buf = malloc(sizeof(iblob) + size); + if (buf == NULL) + return got_error_from_errno(); + memcpy(buf, &iblob, sizeof(iblob)); + memcpy(buf + sizeof(iblob), data, size); + if (imsg_compose(ibuf, GOT_IMSG_BLOB, 0, 0, -1, buf, + sizeof(iblob) + size) == -1) { + free(buf); + return got_error_from_errno(); + } + free(buf); + } else { + /* Data has already been written to file descriptor. */ + if (imsg_compose(ibuf, GOT_IMSG_BLOB, 0, 0, -1, &iblob, + sizeof(iblob)) == -1) + return got_error_from_errno(); + } + + return flush_imsg(ibuf); } const struct got_error * -got_privsep_recv_blob(size_t *size, size_t *hdrlen, struct imsgbuf *ibuf) +got_privsep_recv_blob(uint8_t **outbuf, size_t *size, size_t *hdrlen, + struct imsgbuf *ibuf) { const struct got_error *err = NULL; struct imsg imsg; struct got_imsg_blob *iblob; size_t datalen; + *outbuf = NULL; + err = got_privsep_recv_imsg(&imsg, ibuf, 0); if (err) return err; @@ -863,14 +888,30 @@ got_privsep_recv_blob(size_t *size, size_t *hdrlen, st switch (imsg.hdr.type) { case GOT_IMSG_BLOB: - if (datalen != sizeof(*iblob)) { + if (datalen < sizeof(*iblob)) { err = got_error(GOT_ERR_PRIVSEP_LEN); break; } iblob = imsg.data; *size = iblob->size; *hdrlen = iblob->hdrlen; - /* Data has been written to file descriptor. */ + + if (datalen == sizeof(*iblob)) { + /* Data has been written to file descriptor. */ + break; + } + + if (*size > GOT_PRIVSEP_INLINE_BLOB_DATA_MAX) { + err = got_error(GOT_ERR_PRIVSEP_LEN); + break; + } + + *outbuf = malloc(*size); + if (*outbuf == NULL) { + err = got_error_from_errno(); + break; + } + memcpy(*outbuf, imsg.data + sizeof(*iblob), *size); break; default: err = got_error(GOT_ERR_PRIVSEP_MSG); blob - d795af670c71cbef29833b04357bb80f33c64241 blob + bbb68a823b8ddb245f3587fcb016c803b4c35fa9 --- libexec/got-read-blob/got-read-blob.c +++ libexec/got-read-blob/got-read-blob.c @@ -72,6 +72,7 @@ main(int argc, char *argv[]) FILE *f = NULL; size_t size; struct got_object *obj = NULL; + uint8_t *buf = NULL; memset(&imsg, 0, sizeof(imsg)); imsg.fd = -1; @@ -143,16 +144,22 @@ main(int argc, char *argv[]) goto done; } - err = got_inflate_to_fd(&size, f, imsg_outfd.fd); - if (err) - goto done; + if (obj->size <= GOT_PRIVSEP_INLINE_BLOB_DATA_MAX) { + err = got_inflate_to_mem(&buf, &size, f); + if (err) + goto done; + } else { + err = got_inflate_to_fd(&size, f, imsg_outfd.fd); + if (err) + goto done; + } if (size < obj->hdrlen) { err = got_error(GOT_ERR_BAD_OBJ_HDR); goto done; } - err = got_privsep_send_blob(&ibuf, size, obj->hdrlen); + err = got_privsep_send_blob(&ibuf, size, obj->hdrlen, buf); done: if (f) fclose(f); blob - eec0b58eb918e3e41527914b3675a5b9a7b62032 blob + 1d422ec4629d227ba9bd55fb9a302f3ac9ef4c27 --- libexec/got-read-pack/got-read-pack.c +++ libexec/got-read-pack/got-read-pack.c @@ -222,6 +222,8 @@ blob_request(struct imsg *imsg, struct imsgbuf *ibuf, FILE *outfile = NULL, *basefile = NULL, *accumfile = NULL; struct got_object_id id; size_t datalen; + uint64_t blob_size; + uint8_t *buf = NULL; datalen = imsg->hdr.len - IMSG_HEADER_SIZE; if (datalen != sizeof(iobj)) @@ -235,21 +237,33 @@ blob_request(struct imsg *imsg, struct imsgbuf *ibuf, err = receive_file(&outfile, ibuf, GOT_IMSG_BLOB_OUTFD); if (err) - return err; + goto done; err = receive_file(&basefile, ibuf, GOT_IMSG_TMPFD); if (err) - return err; + goto done; err = receive_file(&accumfile, ibuf, GOT_IMSG_TMPFD); if (err) - return err; + goto done; - err = got_packfile_extract_object(pack, obj, outfile, basefile, - accumfile); + if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) { + err = got_pack_get_object_size(&blob_size, obj); + if (err) + goto done; + } else + blob_size = obj->size; + + if (blob_size <= GOT_PRIVSEP_INLINE_BLOB_DATA_MAX) + err = got_packfile_extract_object_to_mem(&buf, &obj->size, + obj, pack); + else + err = got_packfile_extract_object(pack, obj, outfile, basefile, + accumfile); if (err) goto done; - err = got_privsep_send_blob(ibuf, obj->size, obj->hdrlen); + err = got_privsep_send_blob(ibuf, obj->size, obj->hdrlen, buf); done: + free(buf); if (outfile) fclose(outfile); if (basefile)