Commit Diff

Commit:
ebc55e2dfc648063c434cc7a708ac24d266efbdb
From:
Stefan Sperling <stsp@stsp.name>
Date:
Message:
eliminate got_object_open() round-trip when opening blobs
Actions:
Patch | Tree
commit - 09de383e6e63acb41f89665120f8f9746b637605
commit + ebc55e2dfc648063c434cc7a708ac24d266efbdb
blob - 115b8ff9f9d012b61bcb855ef9c636acee91a445
blob + 9667c50953279012dfc4ed411a7bfeca643cf176
--- lib/got_lib_object_parse.h
+++ lib/got_lib_object_parse.h
@@ -17,10 +17,12 @@
 const struct got_error *got_object_qid_alloc_partial(struct got_object_qid **);
 struct got_commit_object *got_object_commit_alloc_partial(void);
 struct got_tree_entry *got_alloc_tree_entry_partial(void);
+
+/* XXX these declarations don't belong here */
 const struct got_error *got_object_read_header_privsep(struct got_object**,
     struct got_repository *repo, int);
-const struct got_error *got_object_read_blob_privsep(size_t *, int, int,
-    struct got_repository *repo);
+const struct got_error *got_object_read_blob_privsep(size_t *, size_t *, int,
+    int, struct got_repository *repo);
 const struct got_error *got_object_read_commit_privsep(
     struct got_commit_object **, int, struct got_repository *);
 const struct got_error *got_object_read_tree_privsep(struct got_tree_object **,
blob - 74f300e5d407ceb34a8c02eb0bbb4911b3afc6e4
blob + dc25646a1852be26f74f2c2fc27edc56e9c5dc0a
--- lib/got_lib_privsep.h
+++ lib/got_lib_privsep.h
@@ -165,6 +165,7 @@ struct got_imsg_tree_object {
 /* Structure for GOT_IMSG_BLOB. */
 struct got_imsg_blob {
 	size_t size;
+	size_t hdrlen;
 };
 
 /* Structure for GOT_IMSG_TAG data. */
@@ -224,7 +225,8 @@ const struct got_error *got_privsep_send_tree_req(stru
     struct got_object_id *, int);
 const struct got_error *got_privsep_send_tag_req(struct imsgbuf *, int,
     struct got_object_id *, int);
-const struct got_error *got_privsep_send_blob_req(struct imsgbuf *, int);
+const struct got_error *got_privsep_send_blob_req(struct imsgbuf *, int,
+    struct got_object_id *, int);
 const struct got_error *got_privsep_send_blob_outfd(struct imsgbuf *, int);
 const struct got_error *got_privsep_send_tmpfd(struct imsgbuf *, int);
 const struct got_error *got_privsep_send_obj(struct imsgbuf *,
@@ -241,8 +243,9 @@ const struct got_error *got_privsep_recv_tree(struct g
     struct imsgbuf *);
 const struct got_error *got_privsep_send_tree(struct imsgbuf *,
     struct got_tree_object *);
-const struct got_error *got_privsep_send_blob(struct imsgbuf *, size_t);
-const struct got_error *got_privsep_recv_blob(size_t *, struct imsgbuf *);
+const struct got_error *got_privsep_send_blob(struct imsgbuf *, size_t, size_t);
+const struct got_error *got_privsep_recv_blob(size_t *, size_t *,
+    struct imsgbuf *);
 const struct got_error *got_privsep_send_tag(struct imsgbuf *,
     struct got_tag_object *);
 const struct got_error *got_privsep_recv_tag(struct got_tag_object **,
blob - 11a2f7424fefddd1c5a32be9dd1207fb3f48a881
blob + 40fe11e5a4707452c6f3ffd2f243f1cc391e9f72
--- lib/object.c
+++ lib/object.c
@@ -653,8 +653,9 @@ got_object_tree_get_entries(struct got_tree_object *tr
 }
 
 static const struct got_error *
-read_packed_blob_privsep(size_t *size, int outfd, struct got_object *obj,
-    struct got_pack *pack)
+request_packed_blob(size_t *size, size_t *hdrlen, int outfd,
+    struct got_pack *pack, struct got_packidx *packidx, int idx,
+    struct got_object_id *id)
 {
 	const struct got_error *err = NULL;
 	int outfd_child;
@@ -671,7 +672,7 @@ read_packed_blob_privsep(size_t *size, int outfd, stru
 	if (outfd_child == -1)
 		return got_error_from_errno();
 
-	err = got_privsep_send_obj_req(pack->privsep_child->ibuf, -1, obj);
+	err = got_privsep_send_blob_req(pack->privsep_child->ibuf, -1, id, idx);
 	if (err)
 		return err;
 
@@ -698,7 +699,8 @@ read_packed_blob_privsep(size_t *size, int outfd, stru
 		return err;
 	}
 
-	err = got_privsep_recv_blob(size, pack->privsep_child->ibuf);
+	err = got_privsep_recv_blob(size, hdrlen,
+	    pack->privsep_child->ibuf);
 	if (err)
 		return err;
 
@@ -708,21 +710,34 @@ read_packed_blob_privsep(size_t *size, int outfd, stru
 	return err;
 }
 
-const struct got_error *
-got_object_blob_open(struct got_blob_object **blob,
-    struct got_repository *repo, struct got_object *obj, size_t blocksize)
+static const struct got_error *
+read_packed_blob_privsep(size_t *size, size_t *hdrlen, int outfd,
+    struct got_pack *pack, struct got_packidx *packidx, int idx,
+    struct got_object_id *id)
 {
 	const struct got_error *err = NULL;
-	int outfd;
-	size_t size;
-	struct stat sb;
+
+	if (pack->privsep_child == NULL) {
+		err = start_pack_privsep_child(pack, packidx);
+		if (err)
+			return err;
+	}
 
-	if (obj->type != GOT_OBJ_TYPE_BLOB)
-		return got_error(GOT_ERR_OBJ_TYPE);
+	return request_packed_blob(size, hdrlen, outfd, pack, packidx, idx, id);
+}
 
-	if (blocksize < obj->hdrlen)
-		return got_error(GOT_ERR_NO_SPACE);
-
+static const struct got_error *
+open_blob(struct got_blob_object **blob, struct got_repository *repo,
+    struct got_object_id *id, size_t blocksize)
+{
+	const struct got_error *err = NULL;
+	struct got_packidx *packidx = NULL;
+	int idx;
+	char *path_packfile;
+	int outfd;
+	size_t size, hdrlen;
+	struct stat sb;
+
 	*blob = calloc(1, sizeof(**blob));
 	if (*blob == NULL)
 		return got_error_from_errno();
@@ -736,47 +751,52 @@ got_object_blob_open(struct got_blob_object **blob,
 		err = got_error_from_errno();
 		goto done;
 	}
-	if (obj->flags & GOT_OBJ_FLAG_PACKED) {
-		struct got_pack *pack;
-		pack = got_repo_get_cached_pack(repo, obj->path_packfile);
+
+	err = got_repo_search_packidx(&packidx, &idx, repo, id);
+	if (err == NULL) {
+		struct got_pack *pack = NULL;
+
+		err = get_packfile_path(&path_packfile, packidx);
+		if (err)
+			goto done;
+
+		pack = got_repo_get_cached_pack(repo, path_packfile);
 		if (pack == NULL) {
-			err = got_repo_cache_pack(&pack, repo,
-			    obj->path_packfile, NULL);
+			err = got_repo_cache_pack(&pack, repo, path_packfile,
+			    packidx);
 			if (err)
 				goto done;
 		}
-		err = read_packed_blob_privsep(&size, outfd, obj, pack);
-		if (err)
-			goto done;
-		obj->size = size;
-	} else {
+		err = read_packed_blob_privsep(&size, &hdrlen, outfd, pack,
+		    packidx, idx, id);
+	} else if (err->code == GOT_ERR_NO_OBJ) {
 		int infd;
 
-		err = open_loose_object(&infd, got_object_get_id(obj), repo);
-		if (err)
-			goto done;
-
-		err = got_object_read_blob_privsep(&size, outfd, infd, repo);
-		close(infd);
+		err = open_loose_object(&infd, id, repo);
 		if (err)
 			goto done;
-
-		if (size != obj->hdrlen + obj->size) {
-			err = got_error(GOT_ERR_PRIVSEP_LEN);
-			goto done;
-		}
+		err = got_object_read_blob_privsep(&size, &hdrlen, outfd,
+		    infd, repo);
+		close(infd);
 	}
+	if (err)
+		goto done;
 
 	if (fstat(outfd, &sb) == -1) {
 		err = got_error_from_errno();
 		goto done;
 	}
 
-	if (sb.st_size != obj->hdrlen + obj->size) {
+	if (sb.st_size != size) {
 		err = got_error(GOT_ERR_PRIVSEP_LEN);
 		goto done;
 	}
 
+	if (hdrlen >= size) {
+		err = got_error(GOT_ERR_BAD_OBJ_HDR);
+		goto done;
+	}
+
 	(*blob)->f = fdopen(outfd, "rb");
 	if ((*blob)->f == NULL) {
 		err = got_error_from_errno();
@@ -784,9 +804,9 @@ got_object_blob_open(struct got_blob_object **blob,
 		goto done;
 	}
 
-	(*blob)->hdrlen = obj->hdrlen;
+	(*blob)->hdrlen = hdrlen;
 	(*blob)->blocksize = blocksize;
-	memcpy(&(*blob)->id.sha1, obj->id.sha1, SHA1_DIGEST_LENGTH);
+	memcpy(&(*blob)->id.sha1, id->sha1, SHA1_DIGEST_LENGTH);
 
 done:
 	if (err) {
@@ -807,23 +827,14 @@ got_object_open_as_blob(struct got_blob_object **blob,
     struct got_repository *repo, struct got_object_id *id,
     size_t blocksize)
 {
-	const struct got_error *err;
-	struct got_object *obj;
+	return open_blob(blob, repo, id, blocksize);
+}
 
-	*blob = NULL;
-
-	err = got_object_open(&obj, repo, id);
-	if (err)
-		return err;
-	if (obj->type != GOT_OBJ_TYPE_BLOB) {
-		err = got_error(GOT_ERR_OBJ_TYPE);
-		goto done;
-	}
-
-	err = got_object_blob_open(blob, repo, obj, blocksize);
-done:
-	got_object_close(obj);
-	return err;
+const struct got_error *
+got_object_blob_open(struct got_blob_object **blob,
+    struct got_repository *repo, struct got_object *obj, size_t blocksize)
+{
+	return open_blob(blob, repo, got_object_get_id(obj), blocksize);
 }
 
 void
@@ -1368,7 +1379,8 @@ got_object_read_tree_privsep(struct got_tree_object **
 }
 
 static const struct got_error *
-request_blob(size_t *size, int outfd, int infd, struct imsgbuf *ibuf)
+request_blob(size_t *size, size_t *hdrlen, int outfd, int infd,
+    struct imsgbuf *ibuf)
 {
 	const struct got_error *err = NULL;
 	int outfd_child;
@@ -1377,7 +1389,7 @@ request_blob(size_t *size, int outfd, int infd, struct
 	if (outfd_child == -1)
 		return got_error_from_errno();
 
-	err = got_privsep_send_blob_req(ibuf, infd);
+	err = got_privsep_send_blob_req(ibuf, infd, NULL, -1);
 	if (err)
 		return err;
 
@@ -1387,7 +1399,7 @@ request_blob(size_t *size, int outfd, int infd, struct
 		return err;
 	}
 
-	err = got_privsep_recv_blob(size, ibuf);
+	err = got_privsep_recv_blob(size, hdrlen, ibuf);
 	if (err)
 		return err;
 
@@ -1398,8 +1410,8 @@ request_blob(size_t *size, int outfd, int infd, struct
 }
 
 const struct got_error *
-got_object_read_blob_privsep(size_t *size, int outfd, int infd,
-    struct got_repository *repo)
+got_object_read_blob_privsep(size_t *size, size_t *hdrlen,
+    int outfd, int infd, struct got_repository *repo)
 {
 	int imsg_fds[2];
 	pid_t pid;
@@ -1407,7 +1419,7 @@ got_object_read_blob_privsep(size_t *size, int outfd, 
 
 	if (repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].imsg_fd != -1) {
 		ibuf = repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].ibuf;
-		return request_blob(size, outfd, infd, ibuf);
+		return request_blob(size, hdrlen, outfd, infd, ibuf);
 	}
 
 	ibuf = calloc(1, sizeof(*ibuf));
@@ -1433,7 +1445,7 @@ got_object_read_blob_privsep(size_t *size, int outfd, 
 	imsg_init(ibuf, imsg_fds[0]);
 	repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_BLOB].ibuf = ibuf;
 
-	return request_blob(size, outfd, infd, ibuf);
+	return request_blob(size, hdrlen, outfd, infd, ibuf);
 }
 
 static const struct got_error *
blob - 3c8114ca4f5d5b4026d90702aa47ffb3fc77cf2c
blob + 0ae91bf1e8abddd5bbc2ed48b4324d3e02b023f3
--- lib/privsep.c
+++ lib/privsep.c
@@ -338,9 +338,23 @@ got_privsep_send_tag_req(struct imsgbuf *ibuf, int fd,
 }
 
 const struct got_error *
-got_privsep_send_blob_req(struct imsgbuf *ibuf, int infd)
+got_privsep_send_blob_req(struct imsgbuf *ibuf, int infd,
+    struct got_object_id *id, int pack_idx)
 {
-	if (imsg_compose(ibuf, GOT_IMSG_BLOB_REQUEST, 0, 0, infd, NULL, 0)
+	struct got_imsg_packed_object iobj, *iobjp;
+	size_t len;
+
+	if (id) { /* blob is packed */
+		iobj.idx = pack_idx;
+		memcpy(iobj.id, id->sha1, sizeof(iobj.id));
+		iobjp = &iobj;
+		len = sizeof(iobj);
+	} else {
+		iobjp = NULL;
+		len = 0;
+	}
+
+	if (imsg_compose(ibuf, GOT_IMSG_BLOB_REQUEST, 0, 0, infd, iobjp, len)
 	    == -1)
 		return got_error_from_errno();
 
@@ -852,11 +866,12 @@ done:
 }
 
 const struct got_error *
-got_privsep_send_blob(struct imsgbuf *ibuf, size_t size)
+got_privsep_send_blob(struct imsgbuf *ibuf, size_t size, size_t hdrlen)
 {
 	struct got_imsg_blob iblob;
 
 	iblob.size = size;
+	iblob.hdrlen = hdrlen;
 	/* Data has already been written to file descriptor. */
 
 	if (imsg_compose(ibuf, GOT_IMSG_BLOB, 0, 0, -1, &iblob, sizeof(iblob))
@@ -867,7 +882,7 @@ got_privsep_send_blob(struct imsgbuf *ibuf, size_t siz
 }
 
 const struct got_error *
-got_privsep_recv_blob(size_t *size, struct imsgbuf *ibuf)
+got_privsep_recv_blob(size_t *size, size_t *hdrlen, struct imsgbuf *ibuf)
 {
 	const struct got_error *err = NULL;
 	struct imsg imsg;
@@ -888,6 +903,7 @@ got_privsep_recv_blob(size_t *size, struct imsgbuf *ib
 		}
 		iblob = imsg.data;
 		*size = iblob->size;
+		*hdrlen = iblob->hdrlen;
 		/* Data has been written to file descriptor. */
 		break;
 	default:
blob - d45817fbc3bc2a3022fa504990abbe4ad1fc9bdc
blob + bcd6bb4ba2b60ed90712b29a2e72129204bee3cb
--- libexec/got-read-blob/got-read-blob.c
+++ libexec/got-read-blob/got-read-blob.c
@@ -71,6 +71,7 @@ main(int argc, char *argv[])
 		struct imsg imsg, imsg_outfd;
 		FILE *f = NULL;
 		size_t size;
+		struct got_object *obj = NULL;
 	
 		memset(&imsg, 0, sizeof(imsg));
 		imsg.fd = -1;
@@ -97,11 +98,6 @@ main(int argc, char *argv[])
 			goto done;
 		}
 
-		datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
-		if (datalen != 0) {
-			err = got_error(GOT_ERR_PRIVSEP_LEN);
-			goto done;
-		}
 		if (imsg.fd == -1) {
 			err = got_error(GOT_ERR_PRIVSEP_NO_FD);
 			goto done;
@@ -132,6 +128,15 @@ main(int argc, char *argv[])
 			goto done;
 		}
 
+		err = got_object_read_header(&obj, imsg.fd);
+		if (err)
+			goto done;
+
+		if (lseek(imsg.fd, SEEK_SET, 0) == -1) {
+			err = got_error_from_errno();
+			goto done;
+		}
+
 		f = fdopen(imsg.fd, "rb");
 		if (f == NULL) {
 			err = got_error_from_errno();
@@ -142,7 +147,7 @@ main(int argc, char *argv[])
 		if (err)
 			goto done;
 
-		err = got_privsep_send_blob(&ibuf, size);
+		err = got_privsep_send_blob(&ibuf, size, obj->hdrlen);
 done:
 		if (f)
 			fclose(f);
@@ -152,6 +157,8 @@ done:
 			close(imsg_outfd.fd);
 		imsg_free(&imsg);
 		imsg_free(&imsg_outfd);
+		if (obj)
+			got_object_close(obj);
 		if (err)
 			break;
 	}
blob - a195548c98a53ac04849c666f664141ec7834607
blob + eec0b58eb918e3e41527914b3675a5b9a7b62032
--- libexec/got-read-pack/got-read-pack.c
+++ libexec/got-read-pack/got-read-pack.c
@@ -84,39 +84,6 @@ done:
 }
 
 static const struct got_error *
-get_object(struct got_object **obj, struct imsg *imsg, struct imsgbuf *ibuf,
-    struct got_pack *pack, struct got_packidx *packidx,
-    struct got_object_cache *objcache, int type)
-{
-	const struct got_error *err = NULL;
-	struct got_object *iobj;
-
-	err = got_privsep_get_imsg_obj(&iobj, imsg, ibuf);
-	if (err)
-		return err;
-
-	if (iobj->type != type) {
-		err = got_error(GOT_ERR_OBJ_TYPE);
-		goto done;
-	}
-
-	if ((iobj->flags & GOT_OBJ_FLAG_PACKED) == 0)
-		return got_error(GOT_ERR_OBJ_NOT_PACKED);
-
-	*obj = got_object_cache_get(objcache, &iobj->id);
-	if (*obj == NULL) {
-		err = got_packfile_open_object(obj, pack, packidx,
-		    iobj->pack_idx, &iobj->id);
-		if (err)
-			goto done;
-	}
-	(*obj)->refcnt++;
-done:
-	got_object_close(iobj);
-	return err;
-}
-
-static const struct got_error *
 commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
     struct got_packidx *packidx, struct got_object_cache *objcache)
 {
@@ -250,11 +217,19 @@ blob_request(struct imsg *imsg, struct imsgbuf *ibuf, 
     struct got_packidx *packidx, struct got_object_cache *objcache)
 {
 	const struct got_error *err = NULL;
+	struct got_imsg_packed_object iobj;
 	struct got_object *obj = NULL;
 	FILE *outfile = NULL, *basefile = NULL, *accumfile = NULL;
+	struct got_object_id id;
+	size_t datalen;
 
-	err = get_object(&obj, imsg, ibuf, pack, packidx, objcache,
-	    GOT_OBJ_TYPE_BLOB);
+	datalen = imsg->hdr.len - IMSG_HEADER_SIZE;
+	if (datalen != sizeof(iobj))
+		return got_error(GOT_ERR_PRIVSEP_LEN);
+	memcpy(&iobj, imsg->data, sizeof(iobj));
+	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
+
+	err = got_packfile_open_object(&obj, pack, packidx, iobj.idx, &id);
 	if (err)
 		return err;
 
@@ -273,7 +248,7 @@ blob_request(struct imsg *imsg, struct imsgbuf *ibuf, 
 	if (err)
 		goto done;
 
-	err = got_privsep_send_blob(ibuf, obj->size);
+	err = got_privsep_send_blob(ibuf, obj->size, obj->hdrlen);
 done:
 	if (outfile)
 		fclose(outfile);