op public repos

Blame

Date:: Fri May 21 20:20:28 2021 UTC
Message:: add checksum support to got_deflate_to_file() This will eventually be used by 'gotadmin pack'. Checksum init and finalization will need to be done by the caller since many objects will be written out in compressed form while we are computing checksums across the entire pack file. ok millert, naddy
Actions:: History | Blob | Raw File
  1 2181e0c8 2019-03-19 stsp /*
  2 2181e0c8 2019-03-19 stsp  * Copyright (c) 2019 Stefan Sperling <stsp@openbsd.org>
  3 2181e0c8 2019-03-19 stsp  *
  4 2181e0c8 2019-03-19 stsp  * Permission to use, copy, modify, and distribute this software for any
  5 2181e0c8 2019-03-19 stsp  * purpose with or without fee is hereby granted, provided that the above
  6 2181e0c8 2019-03-19 stsp  * copyright notice and this permission notice appear in all copies.
  7 2181e0c8 2019-03-19 stsp  *
  8 2181e0c8 2019-03-19 stsp  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  9 2181e0c8 2019-03-19 stsp  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 10 2181e0c8 2019-03-19 stsp  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 11 2181e0c8 2019-03-19 stsp  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 12 2181e0c8 2019-03-19 stsp  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 13 2181e0c8 2019-03-19 stsp  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 14 2181e0c8 2019-03-19 stsp  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 15 2181e0c8 2019-03-19 stsp  */
 16 2181e0c8 2019-03-19 stsp 
 17 2181e0c8 2019-03-19 stsp #include <sys/queue.h>
 18 2181e0c8 2019-03-19 stsp 
 19 2181e0c8 2019-03-19 stsp #include <errno.h>
 20 2181e0c8 2019-03-19 stsp #include <stdio.h>
 21 2181e0c8 2019-03-19 stsp #include <stdlib.h>
 22 2181e0c8 2019-03-19 stsp #include <string.h>
 23 2181e0c8 2019-03-19 stsp #include <sha1.h>
 24 2181e0c8 2019-03-19 stsp #include <zlib.h>
 25 2181e0c8 2019-03-19 stsp #include <time.h>
 26 2181e0c8 2019-03-19 stsp 
 27 2181e0c8 2019-03-19 stsp #include "got_error.h"
 28 2181e0c8 2019-03-19 stsp #include "got_object.h"
 29 324d37e7 2019-05-11 stsp #include "got_path.h"
 30 2181e0c8 2019-03-19 stsp 
 31 2181e0c8 2019-03-19 stsp #include "got_lib_deflate.h"
 32 2181e0c8 2019-03-19 stsp 
 33 2181e0c8 2019-03-19 stsp #ifndef MIN
 34 2181e0c8 2019-03-19 stsp #define	MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
 35 2181e0c8 2019-03-19 stsp #endif
 36 2181e0c8 2019-03-19 stsp 
 37 2181e0c8 2019-03-19 stsp const struct got_error *
 38 91b40e30 2021-05-21 stsp got_deflate_init(struct got_deflate_buf *zb, uint8_t *outbuf, size_t bufsize,
 39 91b40e30 2021-05-21 stsp     struct got_deflate_checksum *csum)
 40 2181e0c8 2019-03-19 stsp {
 41 2181e0c8 2019-03-19 stsp 	const struct got_error *err = NULL;
 42 2181e0c8 2019-03-19 stsp 	int zerr;
 43 2181e0c8 2019-03-19 stsp 
 44 2181e0c8 2019-03-19 stsp 	memset(&zb->z, 0, sizeof(zb->z));
 45 2181e0c8 2019-03-19 stsp 
 46 2181e0c8 2019-03-19 stsp 	zb->z.zalloc = Z_NULL;
 47 2181e0c8 2019-03-19 stsp 	zb->z.zfree = Z_NULL;
 48 2181e0c8 2019-03-19 stsp 	zerr = deflateInit(&zb->z, Z_DEFAULT_COMPRESSION);
 49 2181e0c8 2019-03-19 stsp 	if (zerr != Z_OK) {
 50 2181e0c8 2019-03-19 stsp 		if  (zerr == Z_ERRNO)
 51 638f9024 2019-05-13 stsp 			return got_error_from_errno("deflateInit");
 52 2181e0c8 2019-03-19 stsp 		if  (zerr == Z_MEM_ERROR) {
 53 2181e0c8 2019-03-19 stsp 			errno = ENOMEM;
 54 638f9024 2019-05-13 stsp 			return got_error_from_errno("deflateInit");
 55 2181e0c8 2019-03-19 stsp 		}
 56 2181e0c8 2019-03-19 stsp 		return got_error(GOT_ERR_COMPRESSION);
 57 2181e0c8 2019-03-19 stsp 	}
 58 2181e0c8 2019-03-19 stsp 
 59 2181e0c8 2019-03-19 stsp 	zb->inlen = zb->outlen = bufsize;
 60 2181e0c8 2019-03-19 stsp 
 61 2181e0c8 2019-03-19 stsp 	zb->inbuf = calloc(1, zb->inlen);
 62 2181e0c8 2019-03-19 stsp 	if (zb->inbuf == NULL) {
 63 638f9024 2019-05-13 stsp 		err = got_error_from_errno("calloc");
 64 2181e0c8 2019-03-19 stsp 		goto done;
 65 2181e0c8 2019-03-19 stsp 	}
 66 2181e0c8 2019-03-19 stsp 
 67 2181e0c8 2019-03-19 stsp 	zb->flags = 0;
 68 2181e0c8 2019-03-19 stsp 	if (outbuf == NULL) {
 69 2181e0c8 2019-03-19 stsp 		zb->outbuf = calloc(1, zb->outlen);
 70 2181e0c8 2019-03-19 stsp 		if (zb->outbuf == NULL) {
 71 638f9024 2019-05-13 stsp 			err = got_error_from_errno("calloc");
 72 2181e0c8 2019-03-19 stsp 			goto done;
 73 2181e0c8 2019-03-19 stsp 		}
 74 2181e0c8 2019-03-19 stsp 		zb->flags |= GOT_DEFLATE_F_OWN_OUTBUF;
 75 2181e0c8 2019-03-19 stsp 	} else
 76 2181e0c8 2019-03-19 stsp 		zb->outbuf = outbuf;
 77 2181e0c8 2019-03-19 stsp 
 78 91b40e30 2021-05-21 stsp 	zb->csum = csum;
 79 2181e0c8 2019-03-19 stsp done:
 80 2181e0c8 2019-03-19 stsp 	if (err)
 81 2181e0c8 2019-03-19 stsp 		got_deflate_end(zb);
 82 2181e0c8 2019-03-19 stsp 	return err;
 83 2181e0c8 2019-03-19 stsp }
 84 2181e0c8 2019-03-19 stsp 
 85 91b40e30 2021-05-21 stsp static void
 86 91b40e30 2021-05-21 stsp csum_output(struct got_deflate_checksum *csum, const char *buf, size_t len)
 87 91b40e30 2021-05-21 stsp {
 88 91b40e30 2021-05-21 stsp 	if (csum->output_crc)
 89 91b40e30 2021-05-21 stsp 		*csum->output_crc = crc32(*csum->output_crc, buf, len);
 90 91b40e30 2021-05-21 stsp 
 91 91b40e30 2021-05-21 stsp 	if (csum->output_sha1)
 92 91b40e30 2021-05-21 stsp 		SHA1Update(csum->output_sha1, buf, len);
 93 91b40e30 2021-05-21 stsp }
 94 91b40e30 2021-05-21 stsp 
 95 2181e0c8 2019-03-19 stsp const struct got_error *
 96 2181e0c8 2019-03-19 stsp got_deflate_read(struct got_deflate_buf *zb, FILE *f, size_t *outlenp)
 97 2181e0c8 2019-03-19 stsp {
 98 2181e0c8 2019-03-19 stsp 	size_t last_total_out = zb->z.total_out;
 99 2181e0c8 2019-03-19 stsp 	z_stream *z = &zb->z;
100 2181e0c8 2019-03-19 stsp 	int ret = Z_ERRNO;
101 2181e0c8 2019-03-19 stsp 
102 2181e0c8 2019-03-19 stsp 	z->next_out = zb->outbuf;
103 2181e0c8 2019-03-19 stsp 	z->avail_out = zb->outlen;
104 2181e0c8 2019-03-19 stsp 
105 2181e0c8 2019-03-19 stsp 	*outlenp = 0;
106 2181e0c8 2019-03-19 stsp 	do {
107 91b40e30 2021-05-21 stsp 		char *csum_out = NULL;
108 91b40e30 2021-05-21 stsp 		size_t csum_avail = 0;
109 91b40e30 2021-05-21 stsp 
110 2181e0c8 2019-03-19 stsp 		if (z->avail_in == 0) {
111 2181e0c8 2019-03-19 stsp 			size_t n = fread(zb->inbuf, 1, zb->inlen, f);
112 2181e0c8 2019-03-19 stsp 			if (n == 0) {
113 2181e0c8 2019-03-19 stsp 				if (ferror(f))
114 2181e0c8 2019-03-19 stsp 					return got_ferror(f, GOT_ERR_IO);
115 2181e0c8 2019-03-19 stsp 				/* EOF */
116 4e4a7005 2019-04-13 stsp 				ret = deflate(z, Z_FINISH);
117 2181e0c8 2019-03-19 stsp 				break;
118 2181e0c8 2019-03-19 stsp 			}
119 2181e0c8 2019-03-19 stsp 			z->next_in = zb->inbuf;
120 2181e0c8 2019-03-19 stsp 			z->avail_in = n;
121 2181e0c8 2019-03-19 stsp 		}
122 91b40e30 2021-05-21 stsp 		if (zb->csum) {
123 91b40e30 2021-05-21 stsp 			csum_out = z->next_out;
124 91b40e30 2021-05-21 stsp 			csum_avail = z->avail_out;
125 91b40e30 2021-05-21 stsp 		}
126 2181e0c8 2019-03-19 stsp 		ret = deflate(z, Z_NO_FLUSH);
127 91b40e30 2021-05-21 stsp 		if (zb->csum) {
128 91b40e30 2021-05-21 stsp 			csum_output(zb->csum, csum_out,
129 91b40e30 2021-05-21 stsp 			   csum_avail - z->avail_out);
130 91b40e30 2021-05-21 stsp 		}
131 2181e0c8 2019-03-19 stsp 	} while (ret == Z_OK && z->avail_out > 0);
132 2181e0c8 2019-03-19 stsp 
133 2181e0c8 2019-03-19 stsp 	if (ret == Z_OK) {
134 2181e0c8 2019-03-19 stsp 		zb->flags |= GOT_DEFLATE_F_HAVE_MORE;
135 2181e0c8 2019-03-19 stsp 	} else {
136 2181e0c8 2019-03-19 stsp 		if (ret != Z_STREAM_END)
137 2181e0c8 2019-03-19 stsp 			return got_error(GOT_ERR_COMPRESSION);
138 2181e0c8 2019-03-19 stsp 		zb->flags &= ~GOT_DEFLATE_F_HAVE_MORE;
139 2181e0c8 2019-03-19 stsp 	}
140 2181e0c8 2019-03-19 stsp 
141 2181e0c8 2019-03-19 stsp 	*outlenp = z->total_out - last_total_out;
142 2181e0c8 2019-03-19 stsp 	return NULL;
143 2181e0c8 2019-03-19 stsp }
144 2181e0c8 2019-03-19 stsp 
145 2181e0c8 2019-03-19 stsp void
146 2181e0c8 2019-03-19 stsp got_deflate_end(struct got_deflate_buf *zb)
147 2181e0c8 2019-03-19 stsp {
148 2181e0c8 2019-03-19 stsp 	free(zb->inbuf);
149 2181e0c8 2019-03-19 stsp 	if (zb->flags & GOT_DEFLATE_F_OWN_OUTBUF)
150 2181e0c8 2019-03-19 stsp 		free(zb->outbuf);
151 2181e0c8 2019-03-19 stsp 	deflateEnd(&zb->z);
152 2181e0c8 2019-03-19 stsp }
153 2181e0c8 2019-03-19 stsp 
154 2181e0c8 2019-03-19 stsp const struct got_error *
155 91b40e30 2021-05-21 stsp got_deflate_to_file(size_t *outlen, FILE *infile, FILE *outfile,
156 91b40e30 2021-05-21 stsp     struct got_deflate_checksum *csum)
157 2181e0c8 2019-03-19 stsp {
158 2181e0c8 2019-03-19 stsp 	const struct got_error *err;
159 2181e0c8 2019-03-19 stsp 	size_t avail;
160 2181e0c8 2019-03-19 stsp 	struct got_deflate_buf zb;
161 2181e0c8 2019-03-19 stsp 
162 91b40e30 2021-05-21 stsp 	err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE, csum);
163 2181e0c8 2019-03-19 stsp 	if (err)
164 2181e0c8 2019-03-19 stsp 		goto done;
165 2181e0c8 2019-03-19 stsp 
166 2181e0c8 2019-03-19 stsp 	*outlen = 0;
167 2181e0c8 2019-03-19 stsp 
168 2181e0c8 2019-03-19 stsp 	do {
169 2181e0c8 2019-03-19 stsp 		err = got_deflate_read(&zb, infile, &avail);
170 2181e0c8 2019-03-19 stsp 		if (err)
171 2181e0c8 2019-03-19 stsp 			goto done;
172 2181e0c8 2019-03-19 stsp 		if (avail > 0) {
173 2181e0c8 2019-03-19 stsp 			size_t n;
174 2181e0c8 2019-03-19 stsp 			n = fwrite(zb.outbuf, avail, 1, outfile);
175 2181e0c8 2019-03-19 stsp 			if (n != 1) {
176 2181e0c8 2019-03-19 stsp 				err = got_ferror(outfile, GOT_ERR_IO);
177 2181e0c8 2019-03-19 stsp 				goto done;
178 2181e0c8 2019-03-19 stsp 			}
179 2181e0c8 2019-03-19 stsp 			*outlen += avail;
180 2181e0c8 2019-03-19 stsp 		}
181 2181e0c8 2019-03-19 stsp 	} while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
182 2181e0c8 2019-03-19 stsp 
183 2181e0c8 2019-03-19 stsp done:
184 2181e0c8 2019-03-19 stsp 	got_deflate_end(&zb);
185 2181e0c8 2019-03-19 stsp 	return err;
186 2181e0c8 2019-03-19 stsp }