commit 91b40e30e0dbff0c8a1546a02fb784fa8007a91b
from: Stefan Sperling <stsp@stsp.name>
date: Fri May 21 20:20:28 2021 UTC

add checksum support to got_deflate_to_file()

This will eventually be used by 'gotadmin pack'.
Checksum init and finalization will need to be done by the caller
since many objects will be written out in compressed form while
we are computing checksums across the entire pack file.

ok millert, naddy

commit - af8a5c4afbbf686b427349764d9a2315d49f99ad
commit + 91b40e30e0dbff0c8a1546a02fb784fa8007a91b
blob - c7771b4a68393727bf41bbc946aa8bf453fc98b8
blob + a9a36a64dd1519966e9aa8699e80bd93c895bdbf
--- lib/deflate.c
+++ lib/deflate.c
@@ -35,7 +35,8 @@
 #endif
 
 const struct got_error *
-got_deflate_init(struct got_deflate_buf *zb, uint8_t *outbuf, size_t bufsize)
+got_deflate_init(struct got_deflate_buf *zb, uint8_t *outbuf, size_t bufsize,
+    struct got_deflate_checksum *csum)
 {
 	const struct got_error *err = NULL;
 	int zerr;
@@ -74,12 +75,23 @@ got_deflate_init(struct got_deflate_buf *zb, uint8_t *
 	} else
 		zb->outbuf = outbuf;
 
+	zb->csum = csum;
 done:
 	if (err)
 		got_deflate_end(zb);
 	return err;
 }
 
+static void
+csum_output(struct got_deflate_checksum *csum, const char *buf, size_t len)
+{
+	if (csum->output_crc)
+		*csum->output_crc = crc32(*csum->output_crc, buf, len);
+
+	if (csum->output_sha1)
+		SHA1Update(csum->output_sha1, buf, len);
+}
+
 const struct got_error *
 got_deflate_read(struct got_deflate_buf *zb, FILE *f, size_t *outlenp)
 {
@@ -92,6 +104,9 @@ got_deflate_read(struct got_deflate_buf *zb, FILE *f, 
 
 	*outlenp = 0;
 	do {
+		char *csum_out = NULL;
+		size_t csum_avail = 0;
+
 		if (z->avail_in == 0) {
 			size_t n = fread(zb->inbuf, 1, zb->inlen, f);
 			if (n == 0) {
@@ -104,7 +119,15 @@ got_deflate_read(struct got_deflate_buf *zb, FILE *f, 
 			z->next_in = zb->inbuf;
 			z->avail_in = n;
 		}
+		if (zb->csum) {
+			csum_out = z->next_out;
+			csum_avail = z->avail_out;
+		}
 		ret = deflate(z, Z_NO_FLUSH);
+		if (zb->csum) {
+			csum_output(zb->csum, csum_out,
+			   csum_avail - z->avail_out);
+		}
 	} while (ret == Z_OK && z->avail_out > 0);
 
 	if (ret == Z_OK) {
@@ -129,13 +152,14 @@ got_deflate_end(struct got_deflate_buf *zb)
 }
 
 const struct got_error *
-got_deflate_to_file(size_t *outlen, FILE *infile, FILE *outfile)
+got_deflate_to_file(size_t *outlen, FILE *infile, FILE *outfile,
+    struct got_deflate_checksum *csum)
 {
 	const struct got_error *err;
 	size_t avail;
 	struct got_deflate_buf zb;
 
-	err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
+	err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE, csum);
 	if (err)
 		goto done;
 
@@ -157,8 +181,6 @@ got_deflate_to_file(size_t *outlen, FILE *infile, FILE
 	} while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
 
 done:
-	if (err == NULL)
-		rewind(outfile);
 	got_deflate_end(&zb);
 	return err;
 }
blob - 1777eab636e44eb4f1da022aa831a352eac52d5e
blob + c359d7c4cfbe99e2e20a5f7798a73ce6a55257c3
--- lib/got_lib_deflate.h
+++ lib/got_lib_deflate.h
@@ -14,6 +14,14 @@
  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  */
 
+struct got_deflate_checksum {
+	/* If not NULL, mix output bytes into this CRC checksum. */
+	uint32_t *output_crc;
+
+	/* If not NULL, mix output bytes into this SHA1 context. */
+	SHA1_CTX *output_sha1;
+};
+
 struct got_deflate_buf {
 	z_stream z;
 	char *inbuf;
@@ -23,13 +31,15 @@ struct got_deflate_buf {
 	int flags;
 #define GOT_DEFLATE_F_HAVE_MORE		0x01
 #define GOT_DEFLATE_F_OWN_OUTBUF	0x02
+	struct got_deflate_checksum *csum;
 };
 
 #define GOT_DEFLATE_BUFSIZE		8192
 
 const struct got_error *got_deflate_init(struct got_deflate_buf *, uint8_t *,
-    size_t);
+    size_t, struct got_deflate_checksum *);
 const struct got_error *got_deflate_read(struct got_deflate_buf *, FILE *,
     size_t *);
 void got_deflate_end(struct got_deflate_buf *);
-const struct got_error *got_deflate_to_file(size_t *, FILE *, FILE *);
+const struct got_error *got_deflate_to_file(size_t *, FILE *, FILE *,
+    struct got_deflate_checksum *);
blob - 5b8443903a0b59be4df01a675d1d07201d2df656
blob + 15982183420ae0e8f9de354180b64dc52689b58b
--- lib/object_create.c
+++ lib/object_create.c
@@ -83,7 +83,7 @@ create_object_file(struct got_object_id *id, FILE *con
 		goto done;
 	}
 
-	err = got_deflate_to_file(&tmplen, content, tmpfile);
+	err = got_deflate_to_file(&tmplen, content, tmpfile, NULL);
 	if (err)
 		goto done;