op public repos

Blob

Date:: Tue Jan 28 13:52:27 2020 UTC
Message:: kill left-over debug code snippet from a46b9f33fb3018765180eb67cc954d863a5cd525
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018, 2019 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/queue.h>
18 #include <sys/tree.h>
19 #include <sys/stat.h>
20 
21 #include <errno.h>
22 #include <dirent.h>
23 #include <fcntl.h>
24 #include <stdio.h>
25 #include <stdlib.h>
26 #include <string.h>
27 #include <sha1.h>
28 #include <endian.h>
29 #include <limits.h>
30 #include <unistd.h>
31 #include <uuid.h>
32 
33 #include "got_error.h"
34 #include "got_object.h"
35 #include "got_path.h"
36 
37 #include "got_lib_fileindex.h"
38 #include "got_lib_worktree.h"
39 
40 /* got_fileindex_entry flags */
41 #define GOT_FILEIDX_F_PATH_LEN		0x00000fff
42 #define GOT_FILEIDX_F_STAGE		0x0000f000
43 #define GOT_FILEIDX_F_STAGE_SHIFT	12
44 #define GOT_FILEIDX_F_NOT_FLUSHED	0x00010000
45 #define GOT_FILEIDX_F_NO_BLOB		0x00020000
46 #define GOT_FILEIDX_F_NO_COMMIT		0x00040000
47 #define GOT_FILEIDX_F_NO_FILE_ON_DISK	0x00080000
48 #define GOT_FILEIDX_F_REMOVE_ON_FLUSH	0x00100000
49 
50 struct got_fileindex {
51 	struct got_fileindex_tree entries;
52 	int nentries; /* Does not include entries marked for removal. */
53 #define GOT_FILEIDX_MAX_ENTRIES INT_MAX
54 };
55 
56 uint16_t
57 got_fileindex_perms_from_st(struct stat *sb)
58 {
59 	uint16_t perms = (sb->st_mode & (S_IRWXU | S_IRWXG | S_IRWXO));
60 	return (perms << GOT_FILEIDX_MODE_PERMS_SHIFT);
61 }
62 
63 mode_t
64 got_fileindex_perms_to_st(struct got_fileindex_entry *ie)
65 {
66 	mode_t perms = (ie->mode >> GOT_FILEIDX_MODE_PERMS_SHIFT);
67 	return (S_IFREG | (perms & (S_IRWXU | S_IRWXG | S_IRWXO)));
68 }
69 
70 const struct got_error *
71 got_fileindex_entry_update(struct got_fileindex_entry *ie,
72     const char *ondisk_path, uint8_t *blob_sha1, uint8_t *commit_sha1,
73     int update_timestamps)
74 {
75 	struct stat sb;
76 
77 	if (lstat(ondisk_path, &sb) != 0) {
78 		if (!((ie->flags & GOT_FILEIDX_F_NO_FILE_ON_DISK) &&
79 		    errno == ENOENT))
80 			return got_error_from_errno2("lstat", ondisk_path);
81 	} else {
82 		if (sb.st_mode & S_IFDIR)
83 			return got_error_set_errno(EISDIR, ondisk_path);
84 		ie->flags &= ~GOT_FILEIDX_F_NO_FILE_ON_DISK;
85 	}
86 
87 
88 	if ((ie->flags & GOT_FILEIDX_F_NO_FILE_ON_DISK) == 0) {
89 		if (update_timestamps) {
90 			ie->ctime_sec = sb.st_ctime;
91 			ie->ctime_nsec = sb.st_ctimensec;
92 			ie->mtime_sec = sb.st_mtime;
93 			ie->mtime_nsec = sb.st_mtimensec;
94 		}
95 		ie->uid = sb.st_uid;
96 		ie->gid = sb.st_gid;
97 		ie->size = (sb.st_size & 0xffffffff);
98 		if (sb.st_mode & S_IFLNK)
99 			ie->mode = GOT_FILEIDX_MODE_SYMLINK;
100 		else
101 			ie->mode = GOT_FILEIDX_MODE_REGULAR_FILE;
102 		ie->mode |= got_fileindex_perms_from_st(&sb);
103 	}
104 
105 	if (blob_sha1) {
106 		memcpy(ie->blob_sha1, blob_sha1, SHA1_DIGEST_LENGTH);
107 		ie->flags &= ~GOT_FILEIDX_F_NO_BLOB;
108 	} else
109 		ie->flags |= GOT_FILEIDX_F_NO_BLOB;
110 
111 	if (commit_sha1) {
112 		memcpy(ie->commit_sha1, commit_sha1, SHA1_DIGEST_LENGTH);
113 		ie->flags &= ~GOT_FILEIDX_F_NO_COMMIT;
114 	} else
115 		ie->flags |= GOT_FILEIDX_F_NO_COMMIT;
116 
117 	return NULL;
118 }
119 
120 void
121 got_fileindex_entry_mark_deleted_from_disk(struct got_fileindex_entry *ie)
122 {
123 	ie->flags |= GOT_FILEIDX_F_NO_FILE_ON_DISK;
124 }
125 
126 const struct got_error *
127 got_fileindex_entry_alloc(struct got_fileindex_entry **ie,
128     const char *ondisk_path, const char *relpath, uint8_t *blob_sha1,
129     uint8_t *commit_sha1)
130 {
131 	size_t len;
132 
133 	*ie = calloc(1, sizeof(**ie));
134 	if (*ie == NULL)
135 		return got_error_from_errno("calloc");
136 
137 	(*ie)->path = strdup(relpath);
138 	if ((*ie)->path == NULL) {
139 		const struct got_error *err = got_error_from_errno("strdup");
140 		free(*ie);
141 		*ie = NULL;
142 		return err;
143 	}
144 
145 	len = strlen(relpath);
146 	if (len > GOT_FILEIDX_F_PATH_LEN)
147 		len = GOT_FILEIDX_F_PATH_LEN;
148 	(*ie)->flags |= len;
149 
150 	return got_fileindex_entry_update(*ie, ondisk_path, blob_sha1,
151 	    commit_sha1, 1);
152 }
153 
154 void
155 got_fileindex_entry_free(struct got_fileindex_entry *ie)
156 {
157 	free(ie->path);
158 	free(ie);
159 }
160 
161 size_t
162 got_fileindex_entry_path_len(const struct got_fileindex_entry *ie)
163 {
164 	return (size_t)(ie->flags & GOT_FILEIDX_F_PATH_LEN);
165 }
166 
167 uint32_t
168 got_fileindex_entry_stage_get(const struct got_fileindex_entry *ie)
169 {
170 	return ((ie->flags & GOT_FILEIDX_F_STAGE) >> GOT_FILEIDX_F_STAGE_SHIFT);
171 }
172 
173 void
174 got_fileindex_entry_stage_set(struct got_fileindex_entry *ie, uint32_t stage)
175 {
176 	ie->flags &= ~GOT_FILEIDX_F_STAGE;
177 	ie->flags |= ((stage << GOT_FILEIDX_F_STAGE_SHIFT) &
178 	    GOT_FILEIDX_F_STAGE);
179 }
180 
181 int
182 got_fileindex_entry_has_blob(struct got_fileindex_entry *ie)
183 {
184 	return (ie->flags & GOT_FILEIDX_F_NO_BLOB) == 0;
185 }
186 
187 int
188 got_fileindex_entry_has_commit(struct got_fileindex_entry *ie)
189 {
190 	return (ie->flags & GOT_FILEIDX_F_NO_COMMIT) == 0;
191 }
192 
193 int
194 got_fileindex_entry_has_file_on_disk(struct got_fileindex_entry *ie)
195 {
196 	return (ie->flags & GOT_FILEIDX_F_NO_FILE_ON_DISK) == 0;
197 }
198 
199 static const struct got_error *
200 add_entry(struct got_fileindex *fileindex, struct got_fileindex_entry *ie)
201 {
202 	if (fileindex->nentries >= GOT_FILEIDX_MAX_ENTRIES)
203 		return got_error(GOT_ERR_NO_SPACE);
204 
205 	RB_INSERT(got_fileindex_tree, &fileindex->entries, ie);
206 	fileindex->nentries++;
207 	return NULL;
208 }
209 
210 const struct got_error *
211 got_fileindex_entry_add(struct got_fileindex *fileindex,
212     struct got_fileindex_entry *ie)
213 {
214 	/* Flag this entry until it gets written out to disk. */
215 	ie->flags |= GOT_FILEIDX_F_NOT_FLUSHED;
216 
217 	return add_entry(fileindex, ie);
218 }
219 
220 void
221 got_fileindex_entry_remove(struct got_fileindex *fileindex,
222     struct got_fileindex_entry *ie)
223 {
224 	/*
225 	 * Removing an entry from the RB tree immediately breaks
226 	 * in-progress iterations over file index entries.
227 	 * So flag this entry for removal and skip it once the index
228 	 * is written out to disk, and pretend this entry no longer
229 	 * exists if we get queried for it again before then.
230 	 */
231 	ie->flags |= GOT_FILEIDX_F_REMOVE_ON_FLUSH;
232 	fileindex->nentries--;
233 }
234 
235 struct got_fileindex_entry *
236 got_fileindex_entry_get(struct got_fileindex *fileindex, const char *path,
237     size_t path_len)
238 {
239 	struct got_fileindex_entry *ie;
240 	struct got_fileindex_entry key;
241 	memset(&key, 0, sizeof(key));
242 	key.path = (char *)path;
243 	key.flags = (path_len & GOT_FILEIDX_F_PATH_LEN);
244 	ie = RB_FIND(got_fileindex_tree, &fileindex->entries, &key);
245 	if (ie && (ie->flags & GOT_FILEIDX_F_REMOVE_ON_FLUSH))
246 		return NULL;
247 	return ie;
248 }
249 
250 const struct got_error *
251 got_fileindex_for_each_entry_safe(struct got_fileindex *fileindex,
252     got_fileindex_cb cb, void *cb_arg)
253 {
254 	const struct got_error *err;
255 	struct got_fileindex_entry *ie, *tmp;
256 
257 	RB_FOREACH_SAFE(ie, got_fileindex_tree, &fileindex->entries, tmp) {
258 		if (ie->flags & GOT_FILEIDX_F_REMOVE_ON_FLUSH)
259 			continue;
260 		err = (*cb)(cb_arg, ie);
261 		if (err)
262 			return err;
263 	}
264 	return NULL;
265 }
266 
267 struct got_fileindex *
268 got_fileindex_alloc(void)
269 {
270 	struct got_fileindex *fileindex;
271 
272 	fileindex = calloc(1, sizeof(*fileindex));
273 	if (fileindex == NULL)
274 		return NULL;
275 
276 	RB_INIT(&fileindex->entries);
277 	return fileindex;
278 }
279 
280 void
281 got_fileindex_free(struct got_fileindex *fileindex)
282 {
283 	struct got_fileindex_entry *ie;
284 
285 	while ((ie = RB_MIN(got_fileindex_tree, &fileindex->entries))) {
286 		RB_REMOVE(got_fileindex_tree, &fileindex->entries, ie);
287 		got_fileindex_entry_free(ie);
288 	}
289 	free(fileindex);
290 }
291 
292 static const struct got_error *
293 write_fileindex_val64(SHA1_CTX *ctx, uint64_t val, FILE *outfile)
294 {
295 	size_t n;
296 
297 	val = htobe64(val);
298 	SHA1Update(ctx, (uint8_t *)&val, sizeof(val));
299 	n = fwrite(&val, 1, sizeof(val), outfile);
300 	if (n != sizeof(val))
301 		return got_ferror(outfile, GOT_ERR_IO);
302 	return NULL;
303 }
304 
305 static const struct got_error *
306 write_fileindex_val32(SHA1_CTX *ctx, uint32_t val, FILE *outfile)
307 {
308 	size_t n;
309 
310 	val = htobe32(val);
311 	SHA1Update(ctx, (uint8_t *)&val, sizeof(val));
312 	n = fwrite(&val, 1, sizeof(val), outfile);
313 	if (n != sizeof(val))
314 		return got_ferror(outfile, GOT_ERR_IO);
315 	return NULL;
316 }
317 
318 static const struct got_error *
319 write_fileindex_val16(SHA1_CTX *ctx, uint16_t val, FILE *outfile)
320 {
321 	size_t n;
322 
323 	val = htobe16(val);
324 	SHA1Update(ctx, (uint8_t *)&val, sizeof(val));
325 	n = fwrite(&val, 1, sizeof(val), outfile);
326 	if (n != sizeof(val))
327 		return got_ferror(outfile, GOT_ERR_IO);
328 	return NULL;
329 }
330 
331 static const struct got_error *
332 write_fileindex_path(SHA1_CTX *ctx, const char *path, FILE *outfile)
333 {
334 	size_t n, len, pad = 0;
335 	static const uint8_t zero[8] = { 0 };
336 
337 	len = strlen(path);
338 	while ((len + pad) % 8 != 0)
339 		pad++;
340 	if (pad == 0)
341 		pad = 8; /* NUL-terminate */
342 
343 	SHA1Update(ctx, path, len);
344 	n = fwrite(path, 1, len, outfile);
345 	if (n != len)
346 		return got_ferror(outfile, GOT_ERR_IO);
347 	SHA1Update(ctx, zero, pad);
348 	n = fwrite(zero, 1, pad, outfile);
349 	if (n != pad)
350 		return got_ferror(outfile, GOT_ERR_IO);
351 	return NULL;
352 }
353 
354 static const struct got_error *
355 write_fileindex_entry(SHA1_CTX *ctx, struct got_fileindex_entry *ie,
356     FILE *outfile)
357 {
358 	const struct got_error *err;
359 	size_t n;
360 	uint32_t stage;
361 
362 	err = write_fileindex_val64(ctx, ie->ctime_sec, outfile);
363 	if (err)
364 		return err;
365 	err = write_fileindex_val64(ctx, ie->ctime_nsec, outfile);
366 	if (err)
367 		return err;
368 	err = write_fileindex_val64(ctx, ie->mtime_sec, outfile);
369 	if (err)
370 		return err;
371 	err = write_fileindex_val64(ctx, ie->mtime_nsec, outfile);
372 	if (err)
373 		return err;
374 
375 	err = write_fileindex_val32(ctx, ie->uid, outfile);
376 	if (err)
377 		return err;
378 	err = write_fileindex_val32(ctx, ie->gid, outfile);
379 	if (err)
380 		return err;
381 	err = write_fileindex_val32(ctx, ie->size, outfile);
382 	if (err)
383 		return err;
384 
385 	err = write_fileindex_val16(ctx, ie->mode, outfile);
386 	if (err)
387 		return err;
388 
389 	SHA1Update(ctx, ie->blob_sha1, SHA1_DIGEST_LENGTH);
390 	n = fwrite(ie->blob_sha1, 1, SHA1_DIGEST_LENGTH, outfile);
391 	if (n != SHA1_DIGEST_LENGTH)
392 		return got_ferror(outfile, GOT_ERR_IO);
393 
394 	SHA1Update(ctx, ie->commit_sha1, SHA1_DIGEST_LENGTH);
395 	n = fwrite(ie->commit_sha1, 1, SHA1_DIGEST_LENGTH, outfile);
396 	if (n != SHA1_DIGEST_LENGTH)
397 		return got_ferror(outfile, GOT_ERR_IO);
398 
399 	err = write_fileindex_val32(ctx, ie->flags, outfile);
400 	if (err)
401 		return err;
402 
403 	err = write_fileindex_path(ctx, ie->path, outfile);
404 	if (err)
405 		return err;
406 
407 	stage = got_fileindex_entry_stage_get(ie);
408 	if (stage == GOT_FILEIDX_STAGE_MODIFY ||
409 	    stage == GOT_FILEIDX_STAGE_ADD) {
410 		SHA1Update(ctx, ie->staged_blob_sha1, SHA1_DIGEST_LENGTH);
411 		n = fwrite(ie->staged_blob_sha1, 1, SHA1_DIGEST_LENGTH,
412 		    outfile);
413 		if (n != SHA1_DIGEST_LENGTH)
414 			return got_ferror(outfile, GOT_ERR_IO);
415 	}
416 
417 	return NULL;
418 }
419 
420 const struct got_error *
421 got_fileindex_write(struct got_fileindex *fileindex, FILE *outfile)
422 {
423 	const struct got_error *err = NULL;
424 	struct got_fileindex_hdr hdr;
425 	SHA1_CTX ctx;
426 	uint8_t sha1[SHA1_DIGEST_LENGTH];
427 	size_t n;
428 	struct got_fileindex_entry *ie;
429 
430 	SHA1Init(&ctx);
431 
432 	hdr.signature = htobe32(GOT_FILE_INDEX_SIGNATURE);
433 	hdr.version = htobe32(GOT_FILE_INDEX_VERSION);
434 	hdr.nentries = htobe32(fileindex->nentries);
435 
436 	SHA1Update(&ctx, (uint8_t *)&hdr.signature, sizeof(hdr.signature));
437 	SHA1Update(&ctx, (uint8_t *)&hdr.version, sizeof(hdr.version));
438 	SHA1Update(&ctx, (uint8_t *)&hdr.nentries, sizeof(hdr.nentries));
439 	n = fwrite(&hdr.signature, 1, sizeof(hdr.signature), outfile);
440 	if (n != sizeof(hdr.signature))
441 		return got_ferror(outfile, GOT_ERR_IO);
442 	n = fwrite(&hdr.version, 1, sizeof(hdr.version), outfile);
443 	if (n != sizeof(hdr.version))
444 		return got_ferror(outfile, GOT_ERR_IO);
445 	n = fwrite(&hdr.nentries, 1, sizeof(hdr.nentries), outfile);
446 	if (n != sizeof(hdr.nentries))
447 		return got_ferror(outfile, GOT_ERR_IO);
448 
449 	RB_FOREACH(ie, got_fileindex_tree, &fileindex->entries) {
450 		ie->flags &= ~GOT_FILEIDX_F_NOT_FLUSHED;
451 		if (ie->flags & GOT_FILEIDX_F_REMOVE_ON_FLUSH)
452 			continue;
453 		err = write_fileindex_entry(&ctx, ie, outfile);
454 		if (err)
455 			return err;
456 	}
457 
458 	SHA1Final(sha1, &ctx);
459 	n = fwrite(sha1, 1, sizeof(sha1), outfile);
460 	if (n != sizeof(sha1))
461 		return got_ferror(outfile, GOT_ERR_IO);
462 
463 	if (fflush(outfile) != 0)
464 		return got_error_from_errno("fflush");
465 
466 	return NULL;
467 }
468 
469 static const struct got_error *
470 read_fileindex_val64(uint64_t *val, SHA1_CTX *ctx, FILE *infile)
471 {
472 	size_t n;
473 
474 	n = fread(val, 1, sizeof(*val), infile);
475 	if (n != sizeof(*val))
476 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
477 	SHA1Update(ctx, (uint8_t *)val, sizeof(*val));
478 	*val = be64toh(*val);
479 	return NULL;
480 }
481 
482 static const struct got_error *
483 read_fileindex_val32(uint32_t *val, SHA1_CTX *ctx, FILE *infile)
484 {
485 	size_t n;
486 
487 	n = fread(val, 1, sizeof(*val), infile);
488 	if (n != sizeof(*val))
489 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
490 	SHA1Update(ctx, (uint8_t *)val, sizeof(*val));
491 	*val = be32toh(*val);
492 	return NULL;
493 }
494 
495 static const struct got_error *
496 read_fileindex_val16(uint16_t *val, SHA1_CTX *ctx, FILE *infile)
497 {
498 	size_t n;
499 
500 	n = fread(val, 1, sizeof(*val), infile);
501 	if (n != sizeof(*val))
502 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
503 	SHA1Update(ctx, (uint8_t *)val, sizeof(*val));
504 	*val = be16toh(*val);
505 	return NULL;
506 }
507 
508 static const struct got_error *
509 read_fileindex_path(char **path, SHA1_CTX *ctx, FILE *infile)
510 {
511 	const struct got_error *err = NULL;
512 	const size_t chunk_size = 8;
513 	size_t n, len = 0, totlen = chunk_size;
514 
515 	*path = malloc(totlen);
516 	if (*path == NULL)
517 		return got_error_from_errno("malloc");
518 
519 	do {
520 		if (len + chunk_size > totlen) {
521 			char *p = reallocarray(*path, totlen + chunk_size, 1);
522 			if (p == NULL) {
523 				err = got_error_from_errno("reallocarray");
524 				break;
525 			}
526 			totlen += chunk_size;
527 			*path = p;
528 		}
529 		n = fread(*path + len, 1, chunk_size, infile);
530 		if (n != chunk_size) {
531 			err = got_ferror(infile, GOT_ERR_FILEIDX_BAD);
532 			break;
533 		}
534 		SHA1Update(ctx, *path + len, chunk_size);
535 		len += chunk_size;
536 	} while (memchr(*path + len - chunk_size, '\0', chunk_size) == NULL);
537 
538 	if (err) {
539 		free(*path);
540 		*path = NULL;
541 	}
542 	return err;
543 }
544 
545 static const struct got_error *
546 read_fileindex_entry(struct got_fileindex_entry **iep, SHA1_CTX *ctx,
547     FILE *infile, uint32_t version)
548 {
549 	const struct got_error *err;
550 	struct got_fileindex_entry *ie;
551 	size_t n;
552 
553 	*iep = NULL;
554 
555 	ie = calloc(1, sizeof(*ie));
556 	if (ie == NULL)
557 		return got_error_from_errno("calloc");
558 
559 	err = read_fileindex_val64(&ie->ctime_sec, ctx, infile);
560 	if (err)
561 		goto done;
562 	err = read_fileindex_val64(&ie->ctime_nsec, ctx, infile);
563 	if (err)
564 		goto done;
565 	err = read_fileindex_val64(&ie->mtime_sec, ctx, infile);
566 	if (err)
567 		goto done;
568 	err = read_fileindex_val64(&ie->mtime_nsec, ctx, infile);
569 	if (err)
570 		goto done;
571 
572 	err = read_fileindex_val32(&ie->uid, ctx, infile);
573 	if (err)
574 		goto done;
575 	err = read_fileindex_val32(&ie->gid, ctx, infile);
576 	if (err)
577 		goto done;
578 	err = read_fileindex_val32(&ie->size, ctx, infile);
579 	if (err)
580 		goto done;
581 
582 	err = read_fileindex_val16(&ie->mode, ctx, infile);
583 	if (err)
584 		goto done;
585 
586 	n = fread(ie->blob_sha1, 1, SHA1_DIGEST_LENGTH, infile);
587 	if (n != SHA1_DIGEST_LENGTH) {
588 		err = got_ferror(infile, GOT_ERR_FILEIDX_BAD);
589 		goto done;
590 	}
591 	SHA1Update(ctx, ie->blob_sha1, SHA1_DIGEST_LENGTH);
592 
593 	n = fread(ie->commit_sha1, 1, SHA1_DIGEST_LENGTH, infile);
594 	if (n != SHA1_DIGEST_LENGTH) {
595 		err = got_ferror(infile, GOT_ERR_FILEIDX_BAD);
596 		goto done;
597 	}
598 	SHA1Update(ctx, ie->commit_sha1, SHA1_DIGEST_LENGTH);
599 
600 	err = read_fileindex_val32(&ie->flags, ctx, infile);
601 	if (err)
602 		goto done;
603 
604 	err = read_fileindex_path(&ie->path, ctx, infile);
605 	if (err)
606 		goto done;
607 
608 	if (version >= 2) {
609 		uint32_t stage = got_fileindex_entry_stage_get(ie);
610 		if (stage == GOT_FILEIDX_STAGE_MODIFY ||
611 		    stage == GOT_FILEIDX_STAGE_ADD) {
612 			n = fread(ie->staged_blob_sha1, 1, SHA1_DIGEST_LENGTH,
613 			    infile);
614 			if (n != SHA1_DIGEST_LENGTH) {
615 				err = got_ferror(infile, GOT_ERR_FILEIDX_BAD);
616 				goto done;
617 			}
618 			SHA1Update(ctx, ie->staged_blob_sha1, SHA1_DIGEST_LENGTH);
619 		}
620 	} else {
621 		/* GOT_FILE_INDEX_VERSION 1 does not support staging. */
622 		ie->flags &= ~GOT_FILEIDX_F_STAGE;
623 	}
624 
625 done:
626 	if (err)
627 		got_fileindex_entry_free(ie);
628 	else
629 		*iep = ie;
630 	return err;
631 }
632 
633 const struct got_error *
634 got_fileindex_read(struct got_fileindex *fileindex, FILE *infile)
635 {
636 	const struct got_error *err = NULL;
637 	struct got_fileindex_hdr hdr;
638 	SHA1_CTX ctx;
639 	struct got_fileindex_entry *ie;
640 	uint8_t sha1_expected[SHA1_DIGEST_LENGTH];
641 	uint8_t sha1[SHA1_DIGEST_LENGTH];
642 	size_t n;
643 	int i;
644 
645 	SHA1Init(&ctx);
646 
647 	n = fread(&hdr.signature, 1, sizeof(hdr.signature), infile);
648 	if (n != sizeof(hdr.signature)) {
649 		if (n == 0) /* EOF */
650 			return NULL;
651 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
652 	}
653 	n = fread(&hdr.version, 1, sizeof(hdr.version), infile);
654 	if (n != sizeof(hdr.version)) {
655 		if (n == 0) /* EOF */
656 			return NULL;
657 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
658 	}
659 	n = fread(&hdr.nentries, 1, sizeof(hdr.nentries), infile);
660 	if (n != sizeof(hdr.nentries)) {
661 		if (n == 0) /* EOF */
662 			return NULL;
663 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
664 	}
665 
666 	SHA1Update(&ctx, (uint8_t *)&hdr.signature, sizeof(hdr.signature));
667 	SHA1Update(&ctx, (uint8_t *)&hdr.version, sizeof(hdr.version));
668 	SHA1Update(&ctx, (uint8_t *)&hdr.nentries, sizeof(hdr.nentries));
669 
670 	hdr.signature = be32toh(hdr.signature);
671 	hdr.version = be32toh(hdr.version);
672 	hdr.nentries = be32toh(hdr.nentries);
673 
674 	if (hdr.signature != GOT_FILE_INDEX_SIGNATURE)
675 		return got_error(GOT_ERR_FILEIDX_SIG);
676 	if (hdr.version > GOT_FILE_INDEX_VERSION)
677 		return got_error(GOT_ERR_FILEIDX_VER);
678 
679 	for (i = 0; i < hdr.nentries; i++) {
680 		err = read_fileindex_entry(&ie, &ctx, infile, hdr.version);
681 		if (err)
682 			return err;
683 		err = add_entry(fileindex, ie);
684 		if (err)
685 			return err;
686 	}
687 
688 	n = fread(sha1_expected, 1, sizeof(sha1_expected), infile);
689 	if (n != sizeof(sha1_expected))
690 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
691 	SHA1Final(sha1, &ctx);
692 	if (memcmp(sha1, sha1_expected, SHA1_DIGEST_LENGTH) != 0)
693 		return got_error(GOT_ERR_FILEIDX_CSUM);
694 
695 	return NULL;
696 }
697 
698 static struct got_fileindex_entry *
699 walk_fileindex(struct got_fileindex *fileindex, struct got_fileindex_entry *ie)
700 {
701 	struct got_fileindex_entry *next;
702 
703 	next = RB_NEXT(got_fileindex_tree, &fileindex->entries, ie);
704 
705 	/* Skip entries which were added or removed by diff callbacks. */
706 	while (next && (next->flags & (GOT_FILEIDX_F_NOT_FLUSHED |
707 	    GOT_FILEIDX_F_REMOVE_ON_FLUSH)))
708 		next = RB_NEXT(got_fileindex_tree, &fileindex->entries, next);
709 
710 	return next;
711 }
712 
713 static const struct got_error *
714 diff_fileindex_tree(struct got_fileindex *, struct got_fileindex_entry **ie,
715     struct got_tree_object *tree, const char *, const char *,
716     struct got_repository *, struct got_fileindex_diff_tree_cb *, void *);
717 
718 static const struct got_error *
719 walk_tree(struct got_tree_entry **next, struct got_fileindex *fileindex,
720     struct got_fileindex_entry **ie, struct got_tree_object *tree, int *tidx,
721     const char *path, const char *entry_name, struct got_repository *repo,
722     struct got_fileindex_diff_tree_cb *cb, void *cb_arg)
723 {
724 	const struct got_error *err = NULL;
725 	struct got_tree_entry *te = got_object_tree_get_entry(tree, *tidx);
726 
727 	if (!got_object_tree_entry_is_submodule(te) &&
728 	    S_ISDIR(got_tree_entry_get_mode(te))) {
729 		char *subpath;
730 		struct got_tree_object *subtree;
731 
732 		if (asprintf(&subpath, "%s%s%s", path,
733 		    path[0] == '\0' ? "" : "/",
734 		    got_tree_entry_get_name(te)) == -1)
735 			return got_error_from_errno("asprintf");
736 
737 		err = got_object_open_as_tree(&subtree, repo,
738 		    got_tree_entry_get_id(te));
739 		if (err) {
740 			free(subpath);
741 			return err;
742 		}
743 
744 		err = diff_fileindex_tree(fileindex, ie, subtree, subpath,
745 		    entry_name, repo, cb, cb_arg);
746 		free(subpath);
747 		got_object_tree_close(subtree);
748 		if (err)
749 			return err;
750 	}
751 
752 	(*tidx)++;
753 	*next = got_object_tree_get_entry(tree, *tidx);
754 	return NULL;
755 }
756 
757 static const struct got_error *
758 diff_fileindex_tree(struct got_fileindex *fileindex,
759     struct got_fileindex_entry **ie, struct got_tree_object *tree,
760     const char *path, const char *entry_name, struct got_repository *repo,
761     struct got_fileindex_diff_tree_cb *cb, void *cb_arg)
762 {
763 	const struct got_error *err = NULL;
764 	struct got_tree_entry *te = NULL;
765 	size_t path_len = strlen(path);
766 	struct got_fileindex_entry *next;
767 	int tidx = 0;
768 
769 	te = got_object_tree_get_entry(tree, tidx);
770 	while ((*ie && got_path_is_child((*ie)->path, path, path_len)) || te) {
771 		if (te && *ie) {
772 			char *te_path;
773 			const char *te_name = got_tree_entry_get_name(te);
774 			int cmp;
775 			if (asprintf(&te_path, "%s/%s", path, te_name) == -1) {
776 				err = got_error_from_errno("asprintf");
777 				break;
778 			}
779 			cmp = got_path_cmp((*ie)->path, te_path,
780 			    got_fileindex_entry_path_len(*ie), strlen(te_path));
781 			free(te_path);
782 			if (cmp == 0) {
783 				if (got_path_is_child((*ie)->path, path,
784 				    path_len) &&
785 				    !got_object_tree_entry_is_submodule(te) &&
786 				    (entry_name == NULL ||
787 				    strcmp(te_name, entry_name) == 0)) {
788 					err = cb->diff_old_new(cb_arg, *ie, te,
789 					    path);
790 					if (err || entry_name)
791 						break;
792 				}
793 				*ie = walk_fileindex(fileindex, *ie);
794 				err = walk_tree(&te, fileindex, ie, tree, &tidx,
795 				    path, entry_name, repo, cb, cb_arg);
796 			} else if (cmp < 0) {
797 				next = walk_fileindex(fileindex, *ie);
798 				if (got_path_is_child((*ie)->path, path,
799 				    path_len) && (entry_name == NULL ||
800 				    strcmp(te_name, entry_name) == 0)) {
801 					err = cb->diff_old(cb_arg, *ie, path);
802 					if (err || entry_name)
803 						break;
804 				}
805 				*ie = next;
806 			} else {
807 				if ((entry_name == NULL ||
808 				    strcmp(te_name, entry_name) == 0)) {
809 					err = cb->diff_new(cb_arg, te, path);
810 					if (err || entry_name)
811 						break;
812 				}
813 				err = walk_tree(&te, fileindex, ie, tree, &tidx,
814 				    path, entry_name, repo, cb, cb_arg);
815 			}
816 			if (err)
817 				break;
818 		} else if (*ie) {
819 			next = walk_fileindex(fileindex, *ie);
820 			if (got_path_is_child((*ie)->path, path, path_len) &&
821 			    (entry_name == NULL ||
822 			    (te && strcmp(got_tree_entry_get_name(te),
823 			    entry_name) == 0))) {
824 				err = cb->diff_old(cb_arg, *ie, path);
825 				if (err || entry_name)
826 					break;
827 			}
828 			*ie = next;
829 		} else if (te) {
830 			if (!got_object_tree_entry_is_submodule(te) &&
831 			    (entry_name == NULL ||
832 			    strcmp(got_tree_entry_get_name(te), entry_name)
833 			    == 0)) {
834 				err = cb->diff_new(cb_arg, te, path);
835 				if (err || entry_name)
836 					break;
837 			}
838 			err = walk_tree(&te, fileindex, ie, tree, &tidx, path,
839 			    entry_name, repo, cb, cb_arg);
840 			if (err)
841 				break;
842 		}
843 	}
844 
845 	return err;
846 }
847 
848 const struct got_error *
849 got_fileindex_diff_tree(struct got_fileindex *fileindex,
850     struct got_tree_object *tree, const char *path, const char *entry_name,
851     struct got_repository *repo,
852     struct got_fileindex_diff_tree_cb *cb, void *cb_arg)
853 {
854 	struct got_fileindex_entry *ie;
855 	ie = RB_MIN(got_fileindex_tree, &fileindex->entries);
856 	while (ie && !got_path_is_child(ie->path, path, strlen(path)))
857 		ie = walk_fileindex(fileindex, ie);
858 	return diff_fileindex_tree(fileindex, &ie, tree, path, entry_name, repo,
859 	    cb, cb_arg);
860 }
861 
862 static const struct got_error *
863 diff_fileindex_dir(struct got_fileindex *, struct got_fileindex_entry **,
864     struct got_pathlist_head *, int, const char *, const char *,
865     struct got_repository *, struct got_fileindex_diff_dir_cb *, void *);
866 
867 static const struct got_error *
868 read_dirlist(struct got_pathlist_head *dirlist, DIR *dir, const char *path)
869 {
870 	const struct got_error *err = NULL;
871 	struct got_pathlist_entry *new = NULL;
872 	struct dirent *dep = NULL;
873 	struct dirent *de = NULL;
874 
875 	for (;;) {
876 		de = malloc(sizeof(struct dirent) + NAME_MAX + 1);
877 		if (de == NULL) {
878 			err = got_error_from_errno("malloc");
879 			break;
880 		}
881 
882 		if (readdir_r(dir, de, &dep) != 0) {
883 			err = got_error_from_errno("readdir_r");
884 			free(de);
885 			break;
886 		}
887 		if (dep == NULL) {
888 			free(de);
889 			break;
890 		}
891 
892 		if (strcmp(de->d_name, ".") == 0 ||
893 		    strcmp(de->d_name, "..") == 0 ||
894 		    (path[0] == '\0' &&
895 		    strcmp(de->d_name, GOT_WORKTREE_GOT_DIR) == 0)) {
896 			free(de);
897 			continue;
898 		}
899 
900 		err = got_pathlist_insert(&new, dirlist, de->d_name, de);
901 		if (err) {
902 			free(de);
903 			break;
904 		}
905 		if (new == NULL) {
906 			err = got_error(GOT_ERR_DIR_DUP_ENTRY);
907 			free(de);
908 			break;
909 		}
910 	}
911 
912 	return err;
913 }
914 
915 void
916 free_dirlist(struct got_pathlist_head *dirlist)
917 {
918 	struct got_pathlist_entry *dle;
919 
920 	TAILQ_FOREACH(dle, dirlist, entry)
921 		free(dle->data);
922 	got_pathlist_free(dirlist);
923 }
924 
925 static const struct got_error *
926 walk_dir(struct got_pathlist_entry **next, struct got_fileindex *fileindex,
927     struct got_fileindex_entry **ie, struct got_pathlist_entry *dle, int fd,
928     const char *path, const char *rootpath, struct got_repository *repo,
929     struct got_fileindex_diff_dir_cb *cb, void *cb_arg)
930 {
931 	const struct got_error *err = NULL;
932 	struct dirent *de = dle->data;
933 	DIR *subdir = NULL;
934 	int subdirfd = -1;
935 
936 	*next = NULL;
937 
938 	if (de->d_type == DT_DIR) {
939 		char *subpath;
940 		char *subdirpath;
941 		struct got_pathlist_head subdirlist;
942 
943 		TAILQ_INIT(&subdirlist);
944 
945 		if (asprintf(&subpath, "%s%s%s", path,
946 		    path[0] == '\0' ? "" : "/", de->d_name) == -1)
947 			return got_error_from_errno("asprintf");
948 
949 		if (asprintf(&subdirpath, "%s/%s", rootpath, subpath) == -1) {
950 			free(subpath);
951 			return got_error_from_errno("asprintf");
952 		}
953 
954 		subdirfd = openat(fd, de->d_name,
955 		    O_RDONLY | O_NOFOLLOW | O_DIRECTORY);
956 		if (subdirfd == -1) {
957 			if (errno == EACCES) {
958 				*next = TAILQ_NEXT(dle, entry);
959 				return NULL;
960 			}
961 			err = got_error_from_errno2("openat", subdirpath);
962 			free(subpath);
963 			free(subdirpath);
964 			return err;
965 		}
966 
967 		subdir = fdopendir(subdirfd);
968 		if (subdir == NULL)
969 			return got_error_from_errno2("fdopendir", path);
970 		subdirfd = -1;
971 		err = read_dirlist(&subdirlist, subdir, subdirpath);
972 		if (err) {
973 			free(subpath);
974 			free(subdirpath);
975 			closedir(subdir);
976 			return err;
977 		}
978 		err = diff_fileindex_dir(fileindex, ie, &subdirlist,
979 		    dirfd(subdir), rootpath, subpath, repo, cb, cb_arg);
980 		if (subdir && closedir(subdir) == -1 && err == NULL)
981 			err = got_error_from_errno2("closedir", subdirpath);
982 		free(subpath);
983 		free(subdirpath);
984 		free_dirlist(&subdirlist);
985 		if (err)
986 			return err;
987 	}
988 
989 	*next = TAILQ_NEXT(dle, entry);
990 	return NULL;
991 }
992 
993 static const struct got_error *
994 diff_fileindex_dir(struct got_fileindex *fileindex,
995     struct got_fileindex_entry **ie, struct got_pathlist_head *dirlist,
996     int dirfd, const char *rootpath, const char *path,
997     struct got_repository *repo,
998     struct got_fileindex_diff_dir_cb *cb, void *cb_arg)
999 {
1000 	const struct got_error *err = NULL;
1001 	struct dirent *de = NULL;
1002 	size_t path_len = strlen(path);
1003 	struct got_pathlist_entry *dle;
1004 
1005 	dle = TAILQ_FIRST(dirlist);
1006 	while ((*ie && got_path_is_child((*ie)->path, path, path_len)) || dle) {
1007 		if (dle && *ie) {
1008 			char *de_path;
1009 			int cmp;
1010 			de = dle->data;
1011 			if (asprintf(&de_path, "%s/%s", path,
1012 			    de->d_name) == -1) {
1013 				err = got_error_from_errno("asprintf");
1014 				break;
1015 			}
1016 			cmp = got_path_cmp((*ie)->path, de_path,
1017 			    got_fileindex_entry_path_len(*ie),
1018 			    strlen(path) + 1 + de->d_namlen);
1019 			free(de_path);
1020 			if (cmp == 0) {
1021 				err = cb->diff_old_new(cb_arg, *ie, de, path,
1022 				    dirfd);
1023 				if (err)
1024 					break;
1025 				*ie = walk_fileindex(fileindex, *ie);
1026 				err = walk_dir(&dle, fileindex, ie, dle, dirfd,
1027 				    path, rootpath, repo, cb, cb_arg);
1028 			} else if (cmp < 0 ) {
1029 				err = cb->diff_old(cb_arg, *ie, path);
1030 				if (err)
1031 					break;
1032 				*ie = walk_fileindex(fileindex, *ie);
1033 			} else {
1034 				err = cb->diff_new(cb_arg, de, path, dirfd);
1035 				if (err)
1036 					break;
1037 				err = walk_dir(&dle, fileindex, ie, dle, dirfd,
1038 				    path, rootpath, repo, cb, cb_arg);
1039 			}
1040 			if (err)
1041 				break;
1042 		} else if (*ie) {
1043 			err = cb->diff_old(cb_arg, *ie, path);
1044 			if (err)
1045 				break;
1046 			*ie = walk_fileindex(fileindex, *ie);
1047 		} else if (dle) {
1048 			de = dle->data;
1049 			err = cb->diff_new(cb_arg, de, path, dirfd);
1050 			if (err)
1051 				break;
1052 			err = walk_dir(&dle, fileindex, ie, dle, dirfd, path,
1053 			    rootpath, repo, cb, cb_arg);
1054 			if (err)
1055 				break;
1056 		}
1057 	}
1058 
1059 	return err;
1060 }
1061 
1062 const struct got_error *
1063 got_fileindex_diff_dir(struct got_fileindex *fileindex, int fd,
1064     const char *rootpath, const char *path, struct got_repository *repo,
1065     struct got_fileindex_diff_dir_cb *cb, void *cb_arg)
1066 {
1067 	const struct got_error *err;
1068 	struct got_fileindex_entry *ie;
1069 	struct got_pathlist_head dirlist;
1070 	int fd2;
1071 	DIR *dir;
1072 
1073 	TAILQ_INIT(&dirlist);
1074 
1075 	/* 
1076 	 * Duplicate the file descriptor so we can call closedir() below
1077 	 * without closing the file descriptor passed in by our caller.
1078 	 */
1079 	fd2 = dup(fd);
1080 	if (fd2 == -1)
1081 		return got_error_from_errno2("dup", path);
1082 	if (lseek(fd2, 0, SEEK_SET) == -1) {
1083 		err = got_error_from_errno2("lseek", path);
1084 		close(fd2);
1085 		return err;
1086 	}
1087 	dir = fdopendir(fd2);
1088 	if (dir == NULL) {
1089 		err = got_error_from_errno2("fdopendir", path);
1090 		close(fd2);
1091 		return err;
1092 	}
1093 	err = read_dirlist(&dirlist, dir, path);
1094 	if (err) {
1095 		closedir(dir);
1096 		return err;
1097 	}
1098 
1099 	ie = RB_MIN(got_fileindex_tree, &fileindex->entries);
1100 	while (ie && !got_path_is_child(ie->path, path, strlen(path)))
1101 		ie = walk_fileindex(fileindex, ie);
1102 	err = diff_fileindex_dir(fileindex, &ie, &dirlist, dirfd(dir),
1103 	    rootpath, path, repo, cb, cb_arg);
1104 
1105 	if (closedir(dir) == -1 && err == NULL)
1106 		err = got_error_from_errno2("closedir", path);
1107 	free_dirlist(&dirlist);
1108 	return err;
1109 }
1110 
1111 RB_GENERATE(got_fileindex_tree, got_fileindex_entry, entry, got_fileindex_cmp);