op public repos

Blob

Date:: Sun Jun 7 09:25:08 2020 UTC
Message:: fix symlink check in got_fileindex_entry_update()
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018, 2019 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/queue.h>
18 #include <sys/tree.h>
19 #include <sys/stat.h>
20 
21 #include <errno.h>
22 #include <dirent.h>
23 #include <fcntl.h>
24 #include <stdio.h>
25 #include <stdlib.h>
26 #include <string.h>
27 #include <sha1.h>
28 #include <endian.h>
29 #include <limits.h>
30 #include <unistd.h>
31 #include <uuid.h>
32 
33 #include "got_error.h"
34 #include "got_object.h"
35 #include "got_path.h"
36 
37 #include "got_lib_fileindex.h"
38 #include "got_lib_worktree.h"
39 
40 /* got_fileindex_entry flags */
41 #define GOT_FILEIDX_F_PATH_LEN		0x00000fff
42 #define GOT_FILEIDX_F_STAGE		0x0000f000
43 #define GOT_FILEIDX_F_STAGE_SHIFT	12
44 #define GOT_FILEIDX_F_NOT_FLUSHED	0x00010000
45 #define GOT_FILEIDX_F_NO_BLOB		0x00020000
46 #define GOT_FILEIDX_F_NO_COMMIT		0x00040000
47 #define GOT_FILEIDX_F_NO_FILE_ON_DISK	0x00080000
48 #define GOT_FILEIDX_F_REMOVE_ON_FLUSH	0x00100000
49 
50 struct got_fileindex {
51 	struct got_fileindex_tree entries;
52 	int nentries; /* Does not include entries marked for removal. */
53 #define GOT_FILEIDX_MAX_ENTRIES INT_MAX
54 };
55 
56 uint16_t
57 got_fileindex_perms_from_st(struct stat *sb)
58 {
59 	uint16_t perms = (sb->st_mode & (S_IRWXU | S_IRWXG | S_IRWXO));
60 	return (perms << GOT_FILEIDX_MODE_PERMS_SHIFT);
61 }
62 
63 mode_t
64 got_fileindex_perms_to_st(struct got_fileindex_entry *ie)
65 {
66 	mode_t perms = (ie->mode >> GOT_FILEIDX_MODE_PERMS_SHIFT);
67 	return (S_IFREG | (perms & (S_IRWXU | S_IRWXG | S_IRWXO)));
68 }
69 
70 const struct got_error *
71 got_fileindex_entry_update(struct got_fileindex_entry *ie,
72     const char *ondisk_path, uint8_t *blob_sha1, uint8_t *commit_sha1,
73     int update_timestamps)
74 {
75 	struct stat sb;
76 
77 	if (lstat(ondisk_path, &sb) != 0) {
78 		if (!((ie->flags & GOT_FILEIDX_F_NO_FILE_ON_DISK) &&
79 		    errno == ENOENT))
80 			return got_error_from_errno2("lstat", ondisk_path);
81 	} else {
82 		if (sb.st_mode & S_IFDIR)
83 			return got_error_set_errno(EISDIR, ondisk_path);
84 		ie->flags &= ~GOT_FILEIDX_F_NO_FILE_ON_DISK;
85 	}
86 
87 
88 	if ((ie->flags & GOT_FILEIDX_F_NO_FILE_ON_DISK) == 0) {
89 		if (update_timestamps) {
90 			ie->ctime_sec = sb.st_ctime;
91 			ie->ctime_nsec = sb.st_ctimensec;
92 			ie->mtime_sec = sb.st_mtime;
93 			ie->mtime_nsec = sb.st_mtimensec;
94 		}
95 		ie->uid = sb.st_uid;
96 		ie->gid = sb.st_gid;
97 		ie->size = (sb.st_size & 0xffffffff);
98 		if (S_ISLNK(sb.st_mode))
99 			ie->mode = GOT_FILEIDX_MODE_SYMLINK;
100 		else
101 			ie->mode = GOT_FILEIDX_MODE_REGULAR_FILE;
102 		ie->mode |= got_fileindex_perms_from_st(&sb);
103 	}
104 
105 	if (blob_sha1) {
106 		memcpy(ie->blob_sha1, blob_sha1, SHA1_DIGEST_LENGTH);
107 		ie->flags &= ~GOT_FILEIDX_F_NO_BLOB;
108 	} else
109 		ie->flags |= GOT_FILEIDX_F_NO_BLOB;
110 
111 	if (commit_sha1) {
112 		memcpy(ie->commit_sha1, commit_sha1, SHA1_DIGEST_LENGTH);
113 		ie->flags &= ~GOT_FILEIDX_F_NO_COMMIT;
114 	} else
115 		ie->flags |= GOT_FILEIDX_F_NO_COMMIT;
116 
117 	return NULL;
118 }
119 
120 void
121 got_fileindex_entry_mark_deleted_from_disk(struct got_fileindex_entry *ie)
122 {
123 	ie->flags |= GOT_FILEIDX_F_NO_FILE_ON_DISK;
124 }
125 
126 const struct got_error *
127 got_fileindex_entry_alloc(struct got_fileindex_entry **ie,
128     const char *relpath)
129 {
130 	size_t len;
131 
132 	*ie = calloc(1, sizeof(**ie));
133 	if (*ie == NULL)
134 		return got_error_from_errno("calloc");
135 
136 	(*ie)->path = strdup(relpath);
137 	if ((*ie)->path == NULL) {
138 		const struct got_error *err = got_error_from_errno("strdup");
139 		free(*ie);
140 		*ie = NULL;
141 		return err;
142 	}
143 
144 	len = strlen(relpath);
145 	if (len > GOT_FILEIDX_F_PATH_LEN)
146 		len = GOT_FILEIDX_F_PATH_LEN;
147 	(*ie)->flags |= len;
148 
149 	return NULL;
150 }
151 
152 void
153 got_fileindex_entry_free(struct got_fileindex_entry *ie)
154 {
155 	free(ie->path);
156 	free(ie);
157 }
158 
159 size_t
160 got_fileindex_entry_path_len(const struct got_fileindex_entry *ie)
161 {
162 	return (size_t)(ie->flags & GOT_FILEIDX_F_PATH_LEN);
163 }
164 
165 uint32_t
166 got_fileindex_entry_stage_get(const struct got_fileindex_entry *ie)
167 {
168 	return ((ie->flags & GOT_FILEIDX_F_STAGE) >> GOT_FILEIDX_F_STAGE_SHIFT);
169 }
170 
171 void
172 got_fileindex_entry_stage_set(struct got_fileindex_entry *ie, uint32_t stage)
173 {
174 	ie->flags &= ~GOT_FILEIDX_F_STAGE;
175 	ie->flags |= ((stage << GOT_FILEIDX_F_STAGE_SHIFT) &
176 	    GOT_FILEIDX_F_STAGE);
177 }
178 
179 int
180 got_fileindex_entry_has_blob(struct got_fileindex_entry *ie)
181 {
182 	return (ie->flags & GOT_FILEIDX_F_NO_BLOB) == 0;
183 }
184 
185 int
186 got_fileindex_entry_has_commit(struct got_fileindex_entry *ie)
187 {
188 	return (ie->flags & GOT_FILEIDX_F_NO_COMMIT) == 0;
189 }
190 
191 int
192 got_fileindex_entry_has_file_on_disk(struct got_fileindex_entry *ie)
193 {
194 	return (ie->flags & GOT_FILEIDX_F_NO_FILE_ON_DISK) == 0;
195 }
196 
197 static const struct got_error *
198 add_entry(struct got_fileindex *fileindex, struct got_fileindex_entry *ie)
199 {
200 	if (fileindex->nentries >= GOT_FILEIDX_MAX_ENTRIES)
201 		return got_error(GOT_ERR_NO_SPACE);
202 
203 	RB_INSERT(got_fileindex_tree, &fileindex->entries, ie);
204 	fileindex->nentries++;
205 	return NULL;
206 }
207 
208 const struct got_error *
209 got_fileindex_entry_add(struct got_fileindex *fileindex,
210     struct got_fileindex_entry *ie)
211 {
212 	/* Flag this entry until it gets written out to disk. */
213 	ie->flags |= GOT_FILEIDX_F_NOT_FLUSHED;
214 
215 	return add_entry(fileindex, ie);
216 }
217 
218 void
219 got_fileindex_entry_remove(struct got_fileindex *fileindex,
220     struct got_fileindex_entry *ie)
221 {
222 	/*
223 	 * Removing an entry from the RB tree immediately breaks
224 	 * in-progress iterations over file index entries.
225 	 * So flag this entry for removal and remove it once the index
226 	 * is written out to disk. Meanwhile, pretend this entry no longer
227 	 * exists if we get queried for it again before then.
228 	 */
229 	ie->flags |= GOT_FILEIDX_F_REMOVE_ON_FLUSH;
230 	fileindex->nentries--;
231 }
232 
233 struct got_fileindex_entry *
234 got_fileindex_entry_get(struct got_fileindex *fileindex, const char *path,
235     size_t path_len)
236 {
237 	struct got_fileindex_entry *ie;
238 	struct got_fileindex_entry key;
239 	memset(&key, 0, sizeof(key));
240 	key.path = (char *)path;
241 	key.flags = (path_len & GOT_FILEIDX_F_PATH_LEN);
242 	ie = RB_FIND(got_fileindex_tree, &fileindex->entries, &key);
243 	if (ie && (ie->flags & GOT_FILEIDX_F_REMOVE_ON_FLUSH))
244 		return NULL;
245 	return ie;
246 }
247 
248 const struct got_error *
249 got_fileindex_for_each_entry_safe(struct got_fileindex *fileindex,
250     got_fileindex_cb cb, void *cb_arg)
251 {
252 	const struct got_error *err;
253 	struct got_fileindex_entry *ie, *tmp;
254 
255 	RB_FOREACH_SAFE(ie, got_fileindex_tree, &fileindex->entries, tmp) {
256 		if (ie->flags & GOT_FILEIDX_F_REMOVE_ON_FLUSH)
257 			continue;
258 		err = (*cb)(cb_arg, ie);
259 		if (err)
260 			return err;
261 	}
262 	return NULL;
263 }
264 
265 struct got_fileindex *
266 got_fileindex_alloc(void)
267 {
268 	struct got_fileindex *fileindex;
269 
270 	fileindex = calloc(1, sizeof(*fileindex));
271 	if (fileindex == NULL)
272 		return NULL;
273 
274 	RB_INIT(&fileindex->entries);
275 	return fileindex;
276 }
277 
278 void
279 got_fileindex_free(struct got_fileindex *fileindex)
280 {
281 	struct got_fileindex_entry *ie;
282 
283 	while ((ie = RB_MIN(got_fileindex_tree, &fileindex->entries))) {
284 		RB_REMOVE(got_fileindex_tree, &fileindex->entries, ie);
285 		got_fileindex_entry_free(ie);
286 	}
287 	free(fileindex);
288 }
289 
290 static const struct got_error *
291 write_fileindex_val64(SHA1_CTX *ctx, uint64_t val, FILE *outfile)
292 {
293 	size_t n;
294 
295 	val = htobe64(val);
296 	SHA1Update(ctx, (uint8_t *)&val, sizeof(val));
297 	n = fwrite(&val, 1, sizeof(val), outfile);
298 	if (n != sizeof(val))
299 		return got_ferror(outfile, GOT_ERR_IO);
300 	return NULL;
301 }
302 
303 static const struct got_error *
304 write_fileindex_val32(SHA1_CTX *ctx, uint32_t val, FILE *outfile)
305 {
306 	size_t n;
307 
308 	val = htobe32(val);
309 	SHA1Update(ctx, (uint8_t *)&val, sizeof(val));
310 	n = fwrite(&val, 1, sizeof(val), outfile);
311 	if (n != sizeof(val))
312 		return got_ferror(outfile, GOT_ERR_IO);
313 	return NULL;
314 }
315 
316 static const struct got_error *
317 write_fileindex_val16(SHA1_CTX *ctx, uint16_t val, FILE *outfile)
318 {
319 	size_t n;
320 
321 	val = htobe16(val);
322 	SHA1Update(ctx, (uint8_t *)&val, sizeof(val));
323 	n = fwrite(&val, 1, sizeof(val), outfile);
324 	if (n != sizeof(val))
325 		return got_ferror(outfile, GOT_ERR_IO);
326 	return NULL;
327 }
328 
329 static const struct got_error *
330 write_fileindex_path(SHA1_CTX *ctx, const char *path, FILE *outfile)
331 {
332 	size_t n, len, pad = 0;
333 	static const uint8_t zero[8] = { 0 };
334 
335 	len = strlen(path);
336 	while ((len + pad) % 8 != 0)
337 		pad++;
338 	if (pad == 0)
339 		pad = 8; /* NUL-terminate */
340 
341 	SHA1Update(ctx, path, len);
342 	n = fwrite(path, 1, len, outfile);
343 	if (n != len)
344 		return got_ferror(outfile, GOT_ERR_IO);
345 	SHA1Update(ctx, zero, pad);
346 	n = fwrite(zero, 1, pad, outfile);
347 	if (n != pad)
348 		return got_ferror(outfile, GOT_ERR_IO);
349 	return NULL;
350 }
351 
352 static const struct got_error *
353 write_fileindex_entry(SHA1_CTX *ctx, struct got_fileindex_entry *ie,
354     FILE *outfile)
355 {
356 	const struct got_error *err;
357 	size_t n;
358 	uint32_t stage;
359 
360 	err = write_fileindex_val64(ctx, ie->ctime_sec, outfile);
361 	if (err)
362 		return err;
363 	err = write_fileindex_val64(ctx, ie->ctime_nsec, outfile);
364 	if (err)
365 		return err;
366 	err = write_fileindex_val64(ctx, ie->mtime_sec, outfile);
367 	if (err)
368 		return err;
369 	err = write_fileindex_val64(ctx, ie->mtime_nsec, outfile);
370 	if (err)
371 		return err;
372 
373 	err = write_fileindex_val32(ctx, ie->uid, outfile);
374 	if (err)
375 		return err;
376 	err = write_fileindex_val32(ctx, ie->gid, outfile);
377 	if (err)
378 		return err;
379 	err = write_fileindex_val32(ctx, ie->size, outfile);
380 	if (err)
381 		return err;
382 
383 	err = write_fileindex_val16(ctx, ie->mode, outfile);
384 	if (err)
385 		return err;
386 
387 	SHA1Update(ctx, ie->blob_sha1, SHA1_DIGEST_LENGTH);
388 	n = fwrite(ie->blob_sha1, 1, SHA1_DIGEST_LENGTH, outfile);
389 	if (n != SHA1_DIGEST_LENGTH)
390 		return got_ferror(outfile, GOT_ERR_IO);
391 
392 	SHA1Update(ctx, ie->commit_sha1, SHA1_DIGEST_LENGTH);
393 	n = fwrite(ie->commit_sha1, 1, SHA1_DIGEST_LENGTH, outfile);
394 	if (n != SHA1_DIGEST_LENGTH)
395 		return got_ferror(outfile, GOT_ERR_IO);
396 
397 	err = write_fileindex_val32(ctx, ie->flags, outfile);
398 	if (err)
399 		return err;
400 
401 	err = write_fileindex_path(ctx, ie->path, outfile);
402 	if (err)
403 		return err;
404 
405 	stage = got_fileindex_entry_stage_get(ie);
406 	if (stage == GOT_FILEIDX_STAGE_MODIFY ||
407 	    stage == GOT_FILEIDX_STAGE_ADD) {
408 		SHA1Update(ctx, ie->staged_blob_sha1, SHA1_DIGEST_LENGTH);
409 		n = fwrite(ie->staged_blob_sha1, 1, SHA1_DIGEST_LENGTH,
410 		    outfile);
411 		if (n != SHA1_DIGEST_LENGTH)
412 			return got_ferror(outfile, GOT_ERR_IO);
413 	}
414 
415 	return NULL;
416 }
417 
418 const struct got_error *
419 got_fileindex_write(struct got_fileindex *fileindex, FILE *outfile)
420 {
421 	const struct got_error *err = NULL;
422 	struct got_fileindex_hdr hdr;
423 	SHA1_CTX ctx;
424 	uint8_t sha1[SHA1_DIGEST_LENGTH];
425 	size_t n;
426 	struct got_fileindex_entry *ie, *tmp;
427 
428 	SHA1Init(&ctx);
429 
430 	hdr.signature = htobe32(GOT_FILE_INDEX_SIGNATURE);
431 	hdr.version = htobe32(GOT_FILE_INDEX_VERSION);
432 	hdr.nentries = htobe32(fileindex->nentries);
433 
434 	SHA1Update(&ctx, (uint8_t *)&hdr.signature, sizeof(hdr.signature));
435 	SHA1Update(&ctx, (uint8_t *)&hdr.version, sizeof(hdr.version));
436 	SHA1Update(&ctx, (uint8_t *)&hdr.nentries, sizeof(hdr.nentries));
437 	n = fwrite(&hdr.signature, 1, sizeof(hdr.signature), outfile);
438 	if (n != sizeof(hdr.signature))
439 		return got_ferror(outfile, GOT_ERR_IO);
440 	n = fwrite(&hdr.version, 1, sizeof(hdr.version), outfile);
441 	if (n != sizeof(hdr.version))
442 		return got_ferror(outfile, GOT_ERR_IO);
443 	n = fwrite(&hdr.nentries, 1, sizeof(hdr.nentries), outfile);
444 	if (n != sizeof(hdr.nentries))
445 		return got_ferror(outfile, GOT_ERR_IO);
446 
447 	RB_FOREACH_SAFE(ie, got_fileindex_tree, &fileindex->entries, tmp) {
448 		ie->flags &= ~GOT_FILEIDX_F_NOT_FLUSHED;
449 		if (ie->flags & GOT_FILEIDX_F_REMOVE_ON_FLUSH) {
450 			RB_REMOVE(got_fileindex_tree, &fileindex->entries, ie);
451 			got_fileindex_entry_free(ie);
452 			continue;
453 		}
454 		err = write_fileindex_entry(&ctx, ie, outfile);
455 		if (err)
456 			return err;
457 	}
458 
459 	SHA1Final(sha1, &ctx);
460 	n = fwrite(sha1, 1, sizeof(sha1), outfile);
461 	if (n != sizeof(sha1))
462 		return got_ferror(outfile, GOT_ERR_IO);
463 
464 	if (fflush(outfile) != 0)
465 		return got_error_from_errno("fflush");
466 
467 	return NULL;
468 }
469 
470 static const struct got_error *
471 read_fileindex_val64(uint64_t *val, SHA1_CTX *ctx, FILE *infile)
472 {
473 	size_t n;
474 
475 	n = fread(val, 1, sizeof(*val), infile);
476 	if (n != sizeof(*val))
477 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
478 	SHA1Update(ctx, (uint8_t *)val, sizeof(*val));
479 	*val = be64toh(*val);
480 	return NULL;
481 }
482 
483 static const struct got_error *
484 read_fileindex_val32(uint32_t *val, SHA1_CTX *ctx, FILE *infile)
485 {
486 	size_t n;
487 
488 	n = fread(val, 1, sizeof(*val), infile);
489 	if (n != sizeof(*val))
490 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
491 	SHA1Update(ctx, (uint8_t *)val, sizeof(*val));
492 	*val = be32toh(*val);
493 	return NULL;
494 }
495 
496 static const struct got_error *
497 read_fileindex_val16(uint16_t *val, SHA1_CTX *ctx, FILE *infile)
498 {
499 	size_t n;
500 
501 	n = fread(val, 1, sizeof(*val), infile);
502 	if (n != sizeof(*val))
503 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
504 	SHA1Update(ctx, (uint8_t *)val, sizeof(*val));
505 	*val = be16toh(*val);
506 	return NULL;
507 }
508 
509 static const struct got_error *
510 read_fileindex_path(char **path, SHA1_CTX *ctx, FILE *infile)
511 {
512 	const struct got_error *err = NULL;
513 	const size_t chunk_size = 8;
514 	size_t n, len = 0, totlen = chunk_size;
515 
516 	*path = malloc(totlen);
517 	if (*path == NULL)
518 		return got_error_from_errno("malloc");
519 
520 	do {
521 		if (len + chunk_size > totlen) {
522 			char *p = reallocarray(*path, totlen + chunk_size, 1);
523 			if (p == NULL) {
524 				err = got_error_from_errno("reallocarray");
525 				break;
526 			}
527 			totlen += chunk_size;
528 			*path = p;
529 		}
530 		n = fread(*path + len, 1, chunk_size, infile);
531 		if (n != chunk_size) {
532 			err = got_ferror(infile, GOT_ERR_FILEIDX_BAD);
533 			break;
534 		}
535 		SHA1Update(ctx, *path + len, chunk_size);
536 		len += chunk_size;
537 	} while (memchr(*path + len - chunk_size, '\0', chunk_size) == NULL);
538 
539 	if (err) {
540 		free(*path);
541 		*path = NULL;
542 	}
543 	return err;
544 }
545 
546 static const struct got_error *
547 read_fileindex_entry(struct got_fileindex_entry **iep, SHA1_CTX *ctx,
548     FILE *infile, uint32_t version)
549 {
550 	const struct got_error *err;
551 	struct got_fileindex_entry *ie;
552 	size_t n;
553 
554 	*iep = NULL;
555 
556 	ie = calloc(1, sizeof(*ie));
557 	if (ie == NULL)
558 		return got_error_from_errno("calloc");
559 
560 	err = read_fileindex_val64(&ie->ctime_sec, ctx, infile);
561 	if (err)
562 		goto done;
563 	err = read_fileindex_val64(&ie->ctime_nsec, ctx, infile);
564 	if (err)
565 		goto done;
566 	err = read_fileindex_val64(&ie->mtime_sec, ctx, infile);
567 	if (err)
568 		goto done;
569 	err = read_fileindex_val64(&ie->mtime_nsec, ctx, infile);
570 	if (err)
571 		goto done;
572 
573 	err = read_fileindex_val32(&ie->uid, ctx, infile);
574 	if (err)
575 		goto done;
576 	err = read_fileindex_val32(&ie->gid, ctx, infile);
577 	if (err)
578 		goto done;
579 	err = read_fileindex_val32(&ie->size, ctx, infile);
580 	if (err)
581 		goto done;
582 
583 	err = read_fileindex_val16(&ie->mode, ctx, infile);
584 	if (err)
585 		goto done;
586 
587 	n = fread(ie->blob_sha1, 1, SHA1_DIGEST_LENGTH, infile);
588 	if (n != SHA1_DIGEST_LENGTH) {
589 		err = got_ferror(infile, GOT_ERR_FILEIDX_BAD);
590 		goto done;
591 	}
592 	SHA1Update(ctx, ie->blob_sha1, SHA1_DIGEST_LENGTH);
593 
594 	n = fread(ie->commit_sha1, 1, SHA1_DIGEST_LENGTH, infile);
595 	if (n != SHA1_DIGEST_LENGTH) {
596 		err = got_ferror(infile, GOT_ERR_FILEIDX_BAD);
597 		goto done;
598 	}
599 	SHA1Update(ctx, ie->commit_sha1, SHA1_DIGEST_LENGTH);
600 
601 	err = read_fileindex_val32(&ie->flags, ctx, infile);
602 	if (err)
603 		goto done;
604 
605 	err = read_fileindex_path(&ie->path, ctx, infile);
606 	if (err)
607 		goto done;
608 
609 	if (version >= 2) {
610 		uint32_t stage = got_fileindex_entry_stage_get(ie);
611 		if (stage == GOT_FILEIDX_STAGE_MODIFY ||
612 		    stage == GOT_FILEIDX_STAGE_ADD) {
613 			n = fread(ie->staged_blob_sha1, 1, SHA1_DIGEST_LENGTH,
614 			    infile);
615 			if (n != SHA1_DIGEST_LENGTH) {
616 				err = got_ferror(infile, GOT_ERR_FILEIDX_BAD);
617 				goto done;
618 			}
619 			SHA1Update(ctx, ie->staged_blob_sha1, SHA1_DIGEST_LENGTH);
620 		}
621 	} else {
622 		/* GOT_FILE_INDEX_VERSION 1 does not support staging. */
623 		ie->flags &= ~GOT_FILEIDX_F_STAGE;
624 	}
625 
626 done:
627 	if (err)
628 		got_fileindex_entry_free(ie);
629 	else
630 		*iep = ie;
631 	return err;
632 }
633 
634 const struct got_error *
635 got_fileindex_read(struct got_fileindex *fileindex, FILE *infile)
636 {
637 	const struct got_error *err = NULL;
638 	struct got_fileindex_hdr hdr;
639 	SHA1_CTX ctx;
640 	struct got_fileindex_entry *ie;
641 	uint8_t sha1_expected[SHA1_DIGEST_LENGTH];
642 	uint8_t sha1[SHA1_DIGEST_LENGTH];
643 	size_t n;
644 	int i;
645 
646 	SHA1Init(&ctx);
647 
648 	n = fread(&hdr.signature, 1, sizeof(hdr.signature), infile);
649 	if (n != sizeof(hdr.signature)) {
650 		if (n == 0) /* EOF */
651 			return NULL;
652 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
653 	}
654 	n = fread(&hdr.version, 1, sizeof(hdr.version), infile);
655 	if (n != sizeof(hdr.version)) {
656 		if (n == 0) /* EOF */
657 			return NULL;
658 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
659 	}
660 	n = fread(&hdr.nentries, 1, sizeof(hdr.nentries), infile);
661 	if (n != sizeof(hdr.nentries)) {
662 		if (n == 0) /* EOF */
663 			return NULL;
664 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
665 	}
666 
667 	SHA1Update(&ctx, (uint8_t *)&hdr.signature, sizeof(hdr.signature));
668 	SHA1Update(&ctx, (uint8_t *)&hdr.version, sizeof(hdr.version));
669 	SHA1Update(&ctx, (uint8_t *)&hdr.nentries, sizeof(hdr.nentries));
670 
671 	hdr.signature = be32toh(hdr.signature);
672 	hdr.version = be32toh(hdr.version);
673 	hdr.nentries = be32toh(hdr.nentries);
674 
675 	if (hdr.signature != GOT_FILE_INDEX_SIGNATURE)
676 		return got_error(GOT_ERR_FILEIDX_SIG);
677 	if (hdr.version > GOT_FILE_INDEX_VERSION)
678 		return got_error(GOT_ERR_FILEIDX_VER);
679 
680 	for (i = 0; i < hdr.nentries; i++) {
681 		err = read_fileindex_entry(&ie, &ctx, infile, hdr.version);
682 		if (err)
683 			return err;
684 		err = add_entry(fileindex, ie);
685 		if (err)
686 			return err;
687 	}
688 
689 	n = fread(sha1_expected, 1, sizeof(sha1_expected), infile);
690 	if (n != sizeof(sha1_expected))
691 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
692 	SHA1Final(sha1, &ctx);
693 	if (memcmp(sha1, sha1_expected, SHA1_DIGEST_LENGTH) != 0)
694 		return got_error(GOT_ERR_FILEIDX_CSUM);
695 
696 	return NULL;
697 }
698 
699 static struct got_fileindex_entry *
700 walk_fileindex(struct got_fileindex *fileindex, struct got_fileindex_entry *ie)
701 {
702 	struct got_fileindex_entry *next;
703 
704 	next = RB_NEXT(got_fileindex_tree, &fileindex->entries, ie);
705 
706 	/* Skip entries which were added or removed by diff callbacks. */
707 	while (next && (next->flags & (GOT_FILEIDX_F_NOT_FLUSHED |
708 	    GOT_FILEIDX_F_REMOVE_ON_FLUSH)))
709 		next = RB_NEXT(got_fileindex_tree, &fileindex->entries, next);
710 
711 	return next;
712 }
713 
714 static const struct got_error *
715 diff_fileindex_tree(struct got_fileindex *, struct got_fileindex_entry **ie,
716     struct got_tree_object *tree, const char *, const char *,
717     struct got_repository *, struct got_fileindex_diff_tree_cb *, void *);
718 
719 static const struct got_error *
720 walk_tree(struct got_tree_entry **next, struct got_fileindex *fileindex,
721     struct got_fileindex_entry **ie, struct got_tree_object *tree, int *tidx,
722     const char *path, const char *entry_name, struct got_repository *repo,
723     struct got_fileindex_diff_tree_cb *cb, void *cb_arg)
724 {
725 	const struct got_error *err = NULL;
726 	struct got_tree_entry *te = got_object_tree_get_entry(tree, *tidx);
727 
728 	if (!got_object_tree_entry_is_submodule(te) &&
729 	    S_ISDIR(got_tree_entry_get_mode(te))) {
730 		char *subpath;
731 		struct got_tree_object *subtree;
732 
733 		if (asprintf(&subpath, "%s%s%s", path,
734 		    path[0] == '\0' ? "" : "/",
735 		    got_tree_entry_get_name(te)) == -1)
736 			return got_error_from_errno("asprintf");
737 
738 		err = got_object_open_as_tree(&subtree, repo,
739 		    got_tree_entry_get_id(te));
740 		if (err) {
741 			free(subpath);
742 			return err;
743 		}
744 
745 		err = diff_fileindex_tree(fileindex, ie, subtree, subpath,
746 		    entry_name, repo, cb, cb_arg);
747 		free(subpath);
748 		got_object_tree_close(subtree);
749 		if (err)
750 			return err;
751 	}
752 
753 	(*tidx)++;
754 	*next = got_object_tree_get_entry(tree, *tidx);
755 	return NULL;
756 }
757 
758 static const struct got_error *
759 diff_fileindex_tree(struct got_fileindex *fileindex,
760     struct got_fileindex_entry **ie, struct got_tree_object *tree,
761     const char *path, const char *entry_name, struct got_repository *repo,
762     struct got_fileindex_diff_tree_cb *cb, void *cb_arg)
763 {
764 	const struct got_error *err = NULL;
765 	struct got_tree_entry *te = NULL;
766 	size_t path_len = strlen(path);
767 	struct got_fileindex_entry *next;
768 	int tidx = 0;
769 
770 	te = got_object_tree_get_entry(tree, tidx);
771 	while ((*ie && got_path_is_child((*ie)->path, path, path_len)) || te) {
772 		if (te && *ie) {
773 			char *te_path;
774 			const char *te_name = got_tree_entry_get_name(te);
775 			int cmp;
776 			if (asprintf(&te_path, "%s/%s", path, te_name) == -1) {
777 				err = got_error_from_errno("asprintf");
778 				break;
779 			}
780 			cmp = got_path_cmp((*ie)->path, te_path,
781 			    got_fileindex_entry_path_len(*ie), strlen(te_path));
782 			free(te_path);
783 			if (cmp == 0) {
784 				if (got_path_is_child((*ie)->path, path,
785 				    path_len) &&
786 				    !got_object_tree_entry_is_submodule(te) &&
787 				    (entry_name == NULL ||
788 				    strcmp(te_name, entry_name) == 0)) {
789 					err = cb->diff_old_new(cb_arg, *ie, te,
790 					    path);
791 					if (err || entry_name)
792 						break;
793 				}
794 				*ie = walk_fileindex(fileindex, *ie);
795 				err = walk_tree(&te, fileindex, ie, tree, &tidx,
796 				    path, entry_name, repo, cb, cb_arg);
797 			} else if (cmp < 0) {
798 				next = walk_fileindex(fileindex, *ie);
799 				if (got_path_is_child((*ie)->path, path,
800 				    path_len) && (entry_name == NULL ||
801 				    strcmp(te_name, entry_name) == 0)) {
802 					err = cb->diff_old(cb_arg, *ie, path);
803 					if (err || entry_name)
804 						break;
805 				}
806 				*ie = next;
807 			} else {
808 				if ((entry_name == NULL ||
809 				    strcmp(te_name, entry_name) == 0)) {
810 					err = cb->diff_new(cb_arg, te, path);
811 					if (err || entry_name)
812 						break;
813 				}
814 				err = walk_tree(&te, fileindex, ie, tree, &tidx,
815 				    path, entry_name, repo, cb, cb_arg);
816 			}
817 			if (err)
818 				break;
819 		} else if (*ie) {
820 			next = walk_fileindex(fileindex, *ie);
821 			if (got_path_is_child((*ie)->path, path, path_len) &&
822 			    (entry_name == NULL ||
823 			    (te && strcmp(got_tree_entry_get_name(te),
824 			    entry_name) == 0))) {
825 				err = cb->diff_old(cb_arg, *ie, path);
826 				if (err || entry_name)
827 					break;
828 			}
829 			*ie = next;
830 		} else if (te) {
831 			if (!got_object_tree_entry_is_submodule(te) &&
832 			    (entry_name == NULL ||
833 			    strcmp(got_tree_entry_get_name(te), entry_name)
834 			    == 0)) {
835 				err = cb->diff_new(cb_arg, te, path);
836 				if (err || entry_name)
837 					break;
838 			}
839 			err = walk_tree(&te, fileindex, ie, tree, &tidx, path,
840 			    entry_name, repo, cb, cb_arg);
841 			if (err)
842 				break;
843 		}
844 	}
845 
846 	return err;
847 }
848 
849 const struct got_error *
850 got_fileindex_diff_tree(struct got_fileindex *fileindex,
851     struct got_tree_object *tree, const char *path, const char *entry_name,
852     struct got_repository *repo,
853     struct got_fileindex_diff_tree_cb *cb, void *cb_arg)
854 {
855 	struct got_fileindex_entry *ie;
856 	ie = RB_MIN(got_fileindex_tree, &fileindex->entries);
857 	while (ie && !got_path_is_child(ie->path, path, strlen(path)))
858 		ie = walk_fileindex(fileindex, ie);
859 	return diff_fileindex_tree(fileindex, &ie, tree, path, entry_name, repo,
860 	    cb, cb_arg);
861 }
862 
863 static const struct got_error *
864 diff_fileindex_dir(struct got_fileindex *, struct got_fileindex_entry **,
865     struct got_pathlist_head *, int, const char *, const char *,
866     struct got_repository *, struct got_fileindex_diff_dir_cb *, void *);
867 
868 static const struct got_error *
869 read_dirlist(struct got_pathlist_head *dirlist, DIR *dir, const char *path)
870 {
871 	const struct got_error *err = NULL;
872 	struct got_pathlist_entry *new = NULL;
873 	struct dirent *dep = NULL;
874 	struct dirent *de = NULL;
875 
876 	for (;;) {
877 		de = malloc(sizeof(struct dirent) + NAME_MAX + 1);
878 		if (de == NULL) {
879 			err = got_error_from_errno("malloc");
880 			break;
881 		}
882 
883 		if (readdir_r(dir, de, &dep) != 0) {
884 			err = got_error_from_errno("readdir_r");
885 			free(de);
886 			break;
887 		}
888 		if (dep == NULL) {
889 			free(de);
890 			break;
891 		}
892 
893 		if (strcmp(de->d_name, ".") == 0 ||
894 		    strcmp(de->d_name, "..") == 0 ||
895 		    (path[0] == '\0' &&
896 		    strcmp(de->d_name, GOT_WORKTREE_GOT_DIR) == 0)) {
897 			free(de);
898 			continue;
899 		}
900 
901 		err = got_pathlist_insert(&new, dirlist, de->d_name, de);
902 		if (err) {
903 			free(de);
904 			break;
905 		}
906 		if (new == NULL) {
907 			err = got_error(GOT_ERR_DIR_DUP_ENTRY);
908 			free(de);
909 			break;
910 		}
911 	}
912 
913 	return err;
914 }
915 
916 void
917 free_dirlist(struct got_pathlist_head *dirlist)
918 {
919 	struct got_pathlist_entry *dle;
920 
921 	TAILQ_FOREACH(dle, dirlist, entry)
922 		free(dle->data);
923 	got_pathlist_free(dirlist);
924 }
925 
926 static const struct got_error *
927 walk_dir(struct got_pathlist_entry **next, struct got_fileindex *fileindex,
928     struct got_fileindex_entry **ie, struct got_pathlist_entry *dle, int fd,
929     const char *path, const char *rootpath, struct got_repository *repo,
930     struct got_fileindex_diff_dir_cb *cb, void *cb_arg)
931 {
932 	const struct got_error *err = NULL;
933 	struct dirent *de = dle->data;
934 	DIR *subdir = NULL;
935 	int subdirfd = -1;
936 
937 	*next = NULL;
938 
939 	if (de->d_type == DT_DIR) {
940 		char *subpath;
941 		char *subdirpath;
942 		struct got_pathlist_head subdirlist;
943 
944 		TAILQ_INIT(&subdirlist);
945 
946 		if (asprintf(&subpath, "%s%s%s", path,
947 		    path[0] == '\0' ? "" : "/", de->d_name) == -1)
948 			return got_error_from_errno("asprintf");
949 
950 		if (asprintf(&subdirpath, "%s/%s", rootpath, subpath) == -1) {
951 			free(subpath);
952 			return got_error_from_errno("asprintf");
953 		}
954 
955 		subdirfd = openat(fd, de->d_name,
956 		    O_RDONLY | O_NOFOLLOW | O_DIRECTORY);
957 		if (subdirfd == -1) {
958 			if (errno == EACCES) {
959 				*next = TAILQ_NEXT(dle, entry);
960 				return NULL;
961 			}
962 			err = got_error_from_errno2("openat", subdirpath);
963 			free(subpath);
964 			free(subdirpath);
965 			return err;
966 		}
967 
968 		subdir = fdopendir(subdirfd);
969 		if (subdir == NULL)
970 			return got_error_from_errno2("fdopendir", path);
971 		subdirfd = -1;
972 		err = read_dirlist(&subdirlist, subdir, subdirpath);
973 		if (err) {
974 			free(subpath);
975 			free(subdirpath);
976 			closedir(subdir);
977 			return err;
978 		}
979 		err = diff_fileindex_dir(fileindex, ie, &subdirlist,
980 		    dirfd(subdir), rootpath, subpath, repo, cb, cb_arg);
981 		if (subdir && closedir(subdir) == -1 && err == NULL)
982 			err = got_error_from_errno2("closedir", subdirpath);
983 		free(subpath);
984 		free(subdirpath);
985 		free_dirlist(&subdirlist);
986 		if (err)
987 			return err;
988 	}
989 
990 	*next = TAILQ_NEXT(dle, entry);
991 	return NULL;
992 }
993 
994 static const struct got_error *
995 diff_fileindex_dir(struct got_fileindex *fileindex,
996     struct got_fileindex_entry **ie, struct got_pathlist_head *dirlist,
997     int dirfd, const char *rootpath, const char *path,
998     struct got_repository *repo,
999     struct got_fileindex_diff_dir_cb *cb, void *cb_arg)
1000 {
1001 	const struct got_error *err = NULL;
1002 	struct dirent *de = NULL;
1003 	size_t path_len = strlen(path);
1004 	struct got_pathlist_entry *dle;
1005 
1006 	dle = TAILQ_FIRST(dirlist);
1007 	while ((*ie && got_path_is_child((*ie)->path, path, path_len)) || dle) {
1008 		if (dle && *ie) {
1009 			char *de_path;
1010 			int cmp;
1011 			de = dle->data;
1012 			if (asprintf(&de_path, "%s/%s", path,
1013 			    de->d_name) == -1) {
1014 				err = got_error_from_errno("asprintf");
1015 				break;
1016 			}
1017 			cmp = got_path_cmp((*ie)->path, de_path,
1018 			    got_fileindex_entry_path_len(*ie),
1019 			    strlen(path) + 1 + de->d_namlen);
1020 			free(de_path);
1021 			if (cmp == 0) {
1022 				err = cb->diff_old_new(cb_arg, *ie, de, path,
1023 				    dirfd);
1024 				if (err)
1025 					break;
1026 				*ie = walk_fileindex(fileindex, *ie);
1027 				err = walk_dir(&dle, fileindex, ie, dle, dirfd,
1028 				    path, rootpath, repo, cb, cb_arg);
1029 			} else if (cmp < 0 ) {
1030 				err = cb->diff_old(cb_arg, *ie, path);
1031 				if (err)
1032 					break;
1033 				*ie = walk_fileindex(fileindex, *ie);
1034 			} else {
1035 				err = cb->diff_new(cb_arg, de, path, dirfd);
1036 				if (err)
1037 					break;
1038 				err = walk_dir(&dle, fileindex, ie, dle, dirfd,
1039 				    path, rootpath, repo, cb, cb_arg);
1040 			}
1041 			if (err)
1042 				break;
1043 		} else if (*ie) {
1044 			err = cb->diff_old(cb_arg, *ie, path);
1045 			if (err)
1046 				break;
1047 			*ie = walk_fileindex(fileindex, *ie);
1048 		} else if (dle) {
1049 			de = dle->data;
1050 			err = cb->diff_new(cb_arg, de, path, dirfd);
1051 			if (err)
1052 				break;
1053 			err = walk_dir(&dle, fileindex, ie, dle, dirfd, path,
1054 			    rootpath, repo, cb, cb_arg);
1055 			if (err)
1056 				break;
1057 		}
1058 	}
1059 
1060 	return err;
1061 }
1062 
1063 const struct got_error *
1064 got_fileindex_diff_dir(struct got_fileindex *fileindex, int fd,
1065     const char *rootpath, const char *path, struct got_repository *repo,
1066     struct got_fileindex_diff_dir_cb *cb, void *cb_arg)
1067 {
1068 	const struct got_error *err;
1069 	struct got_fileindex_entry *ie;
1070 	struct got_pathlist_head dirlist;
1071 	int fd2;
1072 	DIR *dir;
1073 
1074 	TAILQ_INIT(&dirlist);
1075 
1076 	/* 
1077 	 * Duplicate the file descriptor so we can call closedir() below
1078 	 * without closing the file descriptor passed in by our caller.
1079 	 */
1080 	fd2 = dup(fd);
1081 	if (fd2 == -1)
1082 		return got_error_from_errno2("dup", path);
1083 	if (lseek(fd2, 0, SEEK_SET) == -1) {
1084 		err = got_error_from_errno2("lseek", path);
1085 		close(fd2);
1086 		return err;
1087 	}
1088 	dir = fdopendir(fd2);
1089 	if (dir == NULL) {
1090 		err = got_error_from_errno2("fdopendir", path);
1091 		close(fd2);
1092 		return err;
1093 	}
1094 	err = read_dirlist(&dirlist, dir, path);
1095 	if (err) {
1096 		closedir(dir);
1097 		return err;
1098 	}
1099 
1100 	ie = RB_MIN(got_fileindex_tree, &fileindex->entries);
1101 	while (ie && !got_path_is_child(ie->path, path, strlen(path)))
1102 		ie = walk_fileindex(fileindex, ie);
1103 	err = diff_fileindex_dir(fileindex, &ie, &dirlist, dirfd(dir),
1104 	    rootpath, path, repo, cb, cb_arg);
1105 
1106 	if (closedir(dir) == -1 && err == NULL)
1107 		err = got_error_from_errno2("closedir", path);
1108 	free_dirlist(&dirlist);
1109 	return err;
1110 }
1111 
1112 RB_GENERATE(got_fileindex_tree, got_fileindex_entry, entry, got_fileindex_cmp);