commit db6960214d9d6517c59f10744fd5a8e62b2ba46a from: Stefan Sperling date: Tue Jan 04 16:11:52 2022 UTC avoid the creation of new temporary files whenever a packed object is read This speeds up the creation of pack files by about 30%. commit - 404bde06df84f6c5521c237cd1f57373e69fe5b7 commit + db6960214d9d6517c59f10744fd5a8e62b2ba46a blob - dc97d615eecefa57ba072dfa818660e2362700e1 blob + 2a9c135628c6fed4aeef4fb17093f23e0b5df9ae --- lib/got_lib_pack.h +++ lib/got_lib_pack.h @@ -21,6 +21,7 @@ struct got_pack { uint8_t *map; size_t filesize; struct got_privsep_child *privsep_child; + int child_has_tempfiles; struct got_delta_cache *delta_cache; }; blob - a4cf395476aee75f57a0281a2184b91a09e360e3 blob + fd5a9771d547a2696e75c7663d977c2e02cb367f --- lib/object.c +++ lib/object.c @@ -166,58 +166,65 @@ request_packed_object(struct got_object **obj, struct return NULL; } +/* Create temporary files used during delta application. */ static const struct got_error * -request_packed_object_raw(uint8_t **outbuf, off_t *size, size_t *hdrlen, - int outfd, struct got_pack *pack, int idx, struct got_object_id *id) +pack_child_send_tempfiles(struct imsgbuf *ibuf, struct got_pack *pack) { - const struct got_error *err = NULL; - struct imsgbuf *ibuf = pack->privsep_child->ibuf; - int outfd_child; - int basefd, accumfd; /* temporary files for delta application */ + const struct got_error *err; + int basefd, accumfd; + /* + * For performance reasons, the child will keep reusing the + * same temporary files during every object request. + * Opening and closing new files for every object request is + * too expensive during operations such as 'gotadmin pack'. + */ + if (pack->child_has_tempfiles) + return NULL; + basefd = got_opentempfd(); if (basefd == -1) return got_error_from_errno("got_opentempfd"); + err = got_privsep_send_tmpfd(ibuf, basefd); + if (err) + return err; + accumfd = got_opentempfd(); - if (accumfd == -1) { - close(basefd); + if (accumfd == -1) return got_error_from_errno("got_opentempfd"); - } - outfd_child = dup(outfd); - if (outfd_child == -1) { - err = got_error_from_errno("dup"); - close(basefd); - close(accumfd); + err = got_privsep_send_tmpfd(ibuf, accumfd); + if (err) return err; - } + pack->child_has_tempfiles = 1; + return NULL; +} + +static const struct got_error * +request_packed_object_raw(uint8_t **outbuf, off_t *size, size_t *hdrlen, + int outfd, struct got_pack *pack, int idx, struct got_object_id *id) +{ + const struct got_error *err = NULL; + struct imsgbuf *ibuf = pack->privsep_child->ibuf; + int outfd_child; + + err = pack_child_send_tempfiles(ibuf, pack); + if (err) + return err; + + outfd_child = dup(outfd); + if (outfd_child == -1) + return got_error_from_errno("dup"); + err = got_privsep_send_packed_raw_obj_req(ibuf, idx, id); if (err) { - close(basefd); - close(accumfd); close(outfd_child); return err; } err = got_privsep_send_raw_obj_outfd(ibuf, outfd_child); - if (err) { - close(basefd); - close(accumfd); - return err; - } - - - err = got_privsep_send_tmpfd(pack->privsep_child->ibuf, - basefd); - if (err) { - close(accumfd); - return err; - } - - err = got_privsep_send_tmpfd(pack->privsep_child->ibuf, - accumfd); if (err) return err; @@ -258,6 +265,7 @@ start_pack_privsep_child(struct got_pack *pack, struct free(ibuf); return err; } + pack->child_has_tempfiles = 0; if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_fds) == -1) { err = got_error_from_errno("socketpair"); @@ -1191,15 +1199,12 @@ request_packed_blob(uint8_t **outbuf, size_t *size, si struct got_object_id *id) { const struct got_error *err = NULL; - int outfd_child; - int basefd, accumfd; /* temporary files for delta application */ + struct imsgbuf *ibuf = pack->privsep_child->ibuf; + int outfd_child; - basefd = got_opentempfd(); - if (basefd == -1) - return got_error_from_errno("got_opentempfd"); - accumfd = got_opentempfd(); - if (accumfd == -1) - return got_error_from_errno("got_opentempfd"); + err = pack_child_send_tempfiles(ibuf, pack); + if (err) + return err; outfd_child = dup(outfd); if (outfd_child == -1) @@ -1211,24 +1216,10 @@ request_packed_blob(uint8_t **outbuf, size_t *size, si err = got_privsep_send_blob_outfd(pack->privsep_child->ibuf, outfd_child); - if (err) { - close(basefd); - close(accumfd); - return err; - } - - err = got_privsep_send_tmpfd(pack->privsep_child->ibuf, - basefd); if (err) { - close(accumfd); return err; } - err = got_privsep_send_tmpfd(pack->privsep_child->ibuf, - accumfd); - if (err) - return err; - err = got_privsep_recv_blob(outbuf, size, hdrlen, pack->privsep_child->ibuf); if (err) blob - 8bbfe87a70dfd1f3c936dd572b1ab0dcf33aff61 blob + 1ea0d617c52732faf35724fe601e28a24b03992f --- libexec/got-read-pack/got-read-pack.c +++ libexec/got-read-pack/got-read-pack.c @@ -286,16 +286,46 @@ receive_file(FILE **f, struct imsgbuf *ibuf, uint32_t done: imsg_free(&imsg); return err; +} + +static const struct got_error * +receive_tempfile(FILE **basefile, FILE **accumfile, struct imsg *imsg, + struct imsgbuf *ibuf) +{ + size_t datalen; + FILE **f; + + datalen = imsg->hdr.len - IMSG_HEADER_SIZE; + if (datalen != 0) + return got_error(GOT_ERR_PRIVSEP_LEN); + + if (imsg->fd == -1) + return got_error(GOT_ERR_PRIVSEP_NO_FD); + + if (*basefile == NULL) + f = basefile; + else if (*accumfile == NULL) + f = accumfile; + else + return got_error(GOT_ERR_PRIVSEP_MSG); + + *f = fdopen(imsg->fd, "w+"); + if (*f == NULL) + return got_error_from_errno("fdopen"); + imsg->fd = -1; + + return NULL; } static const struct got_error * blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack, - struct got_packidx *packidx, struct got_object_cache *objcache) + struct got_packidx *packidx, struct got_object_cache *objcache, + FILE *basefile, FILE *accumfile) { const struct got_error *err = NULL; struct got_imsg_packed_object iobj; struct got_object *obj = NULL; - FILE *outfile = NULL, *basefile = NULL, *accumfile = NULL; + FILE *outfile = NULL; struct got_object_id id; size_t datalen; uint64_t blob_size; @@ -320,12 +350,6 @@ blob_request(struct imsg *imsg, struct imsgbuf *ibuf, err = receive_file(&outfile, ibuf, GOT_IMSG_BLOB_OUTFD); if (err) goto done; - err = receive_file(&basefile, ibuf, GOT_IMSG_TMPFD); - if (err) - goto done; - err = receive_file(&accumfile, ibuf, GOT_IMSG_TMPFD); - if (err) - goto done; if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) { err = got_pack_get_max_delta_object_size(&blob_size, obj, pack); @@ -348,10 +372,6 @@ done: free(buf); if (outfile && fclose(outfile) == EOF && err == NULL) err = got_error_from_errno("fclose"); - if (basefile && fclose(basefile) == EOF && err == NULL) - err = got_error_from_errno("fclose"); - if (accumfile && fclose(accumfile) == EOF && err == NULL) - err = got_error_from_errno("fclose"); got_object_close(obj); if (err && err->code != GOT_ERR_PRIVSEP_PIPE) got_privsep_send_error(ibuf, err); @@ -775,12 +795,12 @@ done: static const struct got_error * raw_object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack, struct got_packidx *packidx, - struct got_object_cache *objcache) + struct got_object_cache *objcache, FILE *basefile, FILE *accumfile) { const struct got_error *err = NULL; uint8_t *buf = NULL; uint64_t size = 0; - FILE *outfile = NULL, *basefile = NULL, *accumfile = NULL; + FILE *outfile = NULL; struct got_imsg_packed_object iobj; struct got_object *obj; struct got_object_id id; @@ -805,12 +825,6 @@ raw_object_request(struct imsg *imsg, struct imsgbuf * err = receive_file(&outfile, ibuf, GOT_IMSG_RAW_OBJECT_OUTFD); if (err) return err; - err = receive_file(&basefile, ibuf, GOT_IMSG_TMPFD); - if (err) - goto done; - err = receive_file(&accumfile, ibuf, GOT_IMSG_TMPFD); - if (err) - goto done; if (obj->flags & GOT_OBJ_FLAG_DELTIFIED) { err = got_pack_get_max_delta_object_size(&size, obj, pack); @@ -833,10 +847,6 @@ done: free(buf); if (outfile && fclose(outfile) == EOF && err == NULL) err = got_error_from_errno("fclose"); - if (basefile && fclose(basefile) == EOF && err == NULL) - err = got_error_from_errno("fclose"); - if (accumfile && fclose(accumfile) == EOF && err == NULL) - err = got_error_from_errno("fclose"); got_object_close(obj); if (err && err->code != GOT_ERR_PRIVSEP_PIPE) got_privsep_send_error(ibuf, err); @@ -999,6 +1009,7 @@ main(int argc, char *argv[]) struct got_packidx *packidx = NULL; struct got_pack *pack = NULL; struct got_object_cache objcache; + FILE *basefile = NULL, *accumfile = NULL; //static int attached; //while (!attached) sleep(1); @@ -1054,13 +1065,21 @@ main(int argc, char *argv[]) break; switch (imsg.hdr.type) { + case GOT_IMSG_TMPFD: + err = receive_tempfile(&basefile, &accumfile, + &imsg, &ibuf); + break; case GOT_IMSG_PACKED_OBJECT_REQUEST: err = object_request(&imsg, &ibuf, pack, packidx, &objcache); break; case GOT_IMSG_PACKED_RAW_OBJECT_REQUEST: + if (basefile == NULL || accumfile == NULL) { + err = got_error(GOT_ERR_PRIVSEP_MSG); + break; + } err = raw_object_request(&imsg, &ibuf, pack, packidx, - &objcache); + &objcache, basefile, accumfile); break; case GOT_IMSG_COMMIT_REQUEST: err = commit_request(&imsg, &ibuf, pack, packidx, @@ -1071,8 +1090,12 @@ main(int argc, char *argv[]) &objcache); break; case GOT_IMSG_BLOB_REQUEST: + if (basefile == NULL || accumfile == NULL) { + err = got_error(GOT_ERR_PRIVSEP_MSG); + break; + } err = blob_request(&imsg, &ibuf, pack, packidx, - &objcache); + &objcache, basefile, accumfile); break; case GOT_IMSG_TAG_REQUEST: err = tag_request(&imsg, &ibuf, pack, packidx, @@ -1100,6 +1123,10 @@ main(int argc, char *argv[]) got_pack_close(pack); got_object_cache_close(&objcache); imsg_clear(&ibuf); + if (basefile && fclose(basefile) == EOF && err == NULL) + err = got_error_from_errno("fclose"); + if (accumfile && fclose(accumfile) == EOF && err == NULL) + err = got_error_from_errno("fclose"); if (err) { if (!sigint_received && err->code != GOT_ERR_PRIVSEP_PIPE) { fprintf(stderr, "%s: %s\n", getprogname(), err->msg);