op public repos

Blob

Date:: Fri Oct 1 13:58:31 2021 UTC
Message:: skip ignored directories during 'got status' disk crawl
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018, 2019 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/queue.h>
18 #include <sys/tree.h>
19 #include <sys/stat.h>
20 
21 #include <errno.h>
22 #include <dirent.h>
23 #include <fcntl.h>
24 #include <stdio.h>
25 #include <stdlib.h>
26 #include <string.h>
27 #include <sha1.h>
28 #include <endian.h>
29 #include <limits.h>
30 #include <unistd.h>
31 #include <uuid.h>
32 
33 #include "got_error.h"
34 #include "got_object.h"
35 #include "got_path.h"
36 
37 #include "got_lib_fileindex.h"
38 #include "got_lib_worktree.h"
39 
40 /* got_fileindex_entry flags */
41 #define GOT_FILEIDX_F_PATH_LEN		0x00000fff
42 #define GOT_FILEIDX_F_STAGE		0x0000f000
43 #define GOT_FILEIDX_F_STAGE_SHIFT	12
44 #define GOT_FILEIDX_F_NOT_FLUSHED	0x00010000
45 #define GOT_FILEIDX_F_NO_BLOB		0x00020000
46 #define GOT_FILEIDX_F_NO_COMMIT		0x00040000
47 #define GOT_FILEIDX_F_NO_FILE_ON_DISK	0x00080000
48 #define GOT_FILEIDX_F_REMOVE_ON_FLUSH	0x00100000
49 #define GOT_FILEIDX_F_SKIPPED		0x00200000
50 
51 struct got_fileindex {
52 	struct got_fileindex_tree entries;
53 	int nentries; /* Does not include entries marked for removal. */
54 #define GOT_FILEIDX_MAX_ENTRIES INT_MAX
55 };
56 
57 mode_t
58 got_fileindex_entry_perms_get(struct got_fileindex_entry *ie)
59 {
60 	return ((ie->mode & GOT_FILEIDX_MODE_PERMS) >>
61 	    GOT_FILEIDX_MODE_PERMS_SHIFT);
62 }
63 
64 static void
65 fileindex_entry_perms_set(struct got_fileindex_entry *ie, mode_t mode)
66 {
67 	ie->mode &= ~GOT_FILEIDX_MODE_PERMS;
68 	ie->mode |= ((mode << GOT_FILEIDX_MODE_PERMS_SHIFT) &
69 	    GOT_FILEIDX_MODE_PERMS);
70 }
71 
72 mode_t
73 got_fileindex_perms_to_st(struct got_fileindex_entry *ie)
74 {
75 	mode_t perms = got_fileindex_entry_perms_get(ie);
76 	int type = got_fileindex_entry_filetype_get(ie);
77 	uint32_t ftype;
78 
79 	if (type == GOT_FILEIDX_MODE_REGULAR_FILE ||
80 	    type == GOT_FILEIDX_MODE_BAD_SYMLINK)
81 		ftype = S_IFREG;
82 	else
83 		ftype = S_IFLNK;
84 
85 	return (ftype | (perms & (S_IRWXU | S_IRWXG | S_IRWXO)));
86 }
87 
88 const struct got_error *
89 got_fileindex_entry_update(struct got_fileindex_entry *ie,
90     int wt_fd, const char *ondisk_path, uint8_t *blob_sha1,
91     uint8_t *commit_sha1, int update_timestamps)
92 {
93 	struct stat sb;
94 
95 	if (fstatat(wt_fd, ondisk_path, &sb, AT_SYMLINK_NOFOLLOW) != 0) {
96 		if (!((ie->flags & GOT_FILEIDX_F_NO_FILE_ON_DISK) &&
97 		    errno == ENOENT))
98 			return got_error_from_errno2("fstatat", ondisk_path);
99 		sb.st_mode = GOT_DEFAULT_FILE_MODE;
100 	} else {
101 		if (sb.st_mode & S_IFDIR)
102 			return got_error_set_errno(EISDIR, ondisk_path);
103 		ie->flags &= ~GOT_FILEIDX_F_NO_FILE_ON_DISK;
104 	}
105 
106 
107 	if ((ie->flags & GOT_FILEIDX_F_NO_FILE_ON_DISK) == 0) {
108 		if (update_timestamps) {
109 			ie->ctime_sec = sb.st_ctim.tv_sec;
110 			ie->ctime_nsec = sb.st_ctim.tv_nsec;
111 			ie->mtime_sec = sb.st_mtim.tv_sec;
112 			ie->mtime_nsec = sb.st_mtim.tv_nsec;
113 		}
114 		ie->uid = sb.st_uid;
115 		ie->gid = sb.st_gid;
116 		ie->size = (sb.st_size & 0xffffffff);
117 		if (S_ISLNK(sb.st_mode)) {
118 			got_fileindex_entry_filetype_set(ie,
119 			    GOT_FILEIDX_MODE_SYMLINK);
120 			fileindex_entry_perms_set(ie, 0);
121 		} else {
122 			got_fileindex_entry_filetype_set(ie,
123 			    GOT_FILEIDX_MODE_REGULAR_FILE);
124 			fileindex_entry_perms_set(ie,
125 			    sb.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO));
126 		}
127 	}
128 
129 	if (blob_sha1) {
130 		memcpy(ie->blob_sha1, blob_sha1, SHA1_DIGEST_LENGTH);
131 		ie->flags &= ~GOT_FILEIDX_F_NO_BLOB;
132 	} else
133 		ie->flags |= GOT_FILEIDX_F_NO_BLOB;
134 
135 	if (commit_sha1) {
136 		memcpy(ie->commit_sha1, commit_sha1, SHA1_DIGEST_LENGTH);
137 		ie->flags &= ~GOT_FILEIDX_F_NO_COMMIT;
138 	} else
139 		ie->flags |= GOT_FILEIDX_F_NO_COMMIT;
140 
141 	return NULL;
142 }
143 
144 void
145 got_fileindex_entry_mark_deleted_from_disk(struct got_fileindex_entry *ie)
146 {
147 	ie->flags |= GOT_FILEIDX_F_NO_FILE_ON_DISK;
148 }
149 
150 void
151 got_fileindex_entry_mark_skipped(struct got_fileindex_entry *ie)
152 {
153 	ie->flags |= GOT_FILEIDX_F_SKIPPED;
154 }
155 
156 const struct got_error *
157 got_fileindex_entry_alloc(struct got_fileindex_entry **ie,
158     const char *relpath)
159 {
160 	size_t len;
161 
162 	*ie = calloc(1, sizeof(**ie));
163 	if (*ie == NULL)
164 		return got_error_from_errno("calloc");
165 
166 	(*ie)->path = strdup(relpath);
167 	if ((*ie)->path == NULL) {
168 		const struct got_error *err = got_error_from_errno("strdup");
169 		free(*ie);
170 		*ie = NULL;
171 		return err;
172 	}
173 
174 	len = strlen(relpath);
175 	if (len > GOT_FILEIDX_F_PATH_LEN)
176 		len = GOT_FILEIDX_F_PATH_LEN;
177 	(*ie)->flags |= len;
178 
179 	return NULL;
180 }
181 
182 void
183 got_fileindex_entry_free(struct got_fileindex_entry *ie)
184 {
185 	free(ie->path);
186 	free(ie);
187 }
188 
189 size_t
190 got_fileindex_entry_path_len(const struct got_fileindex_entry *ie)
191 {
192 	return (size_t)(ie->flags & GOT_FILEIDX_F_PATH_LEN);
193 }
194 
195 uint32_t
196 got_fileindex_entry_stage_get(const struct got_fileindex_entry *ie)
197 {
198 	return ((ie->flags & GOT_FILEIDX_F_STAGE) >> GOT_FILEIDX_F_STAGE_SHIFT);
199 }
200 
201 void
202 got_fileindex_entry_stage_set(struct got_fileindex_entry *ie, uint32_t stage)
203 {
204 	ie->flags &= ~GOT_FILEIDX_F_STAGE;
205 	ie->flags |= ((stage << GOT_FILEIDX_F_STAGE_SHIFT) &
206 	    GOT_FILEIDX_F_STAGE);
207 }
208 
209 int
210 got_fileindex_entry_filetype_get(struct got_fileindex_entry *ie)
211 {
212 	return (ie->mode & GOT_FILEIDX_MODE_FILE_TYPE_ONDISK);
213 }
214 
215 void
216 got_fileindex_entry_filetype_set(struct got_fileindex_entry *ie, int type)
217 {
218 	ie->mode &= ~GOT_FILEIDX_MODE_FILE_TYPE_ONDISK;
219 	ie->mode |= (type & GOT_FILEIDX_MODE_FILE_TYPE_ONDISK);
220 }
221 
222 void
223 got_fileindex_entry_staged_filetype_set(struct got_fileindex_entry *ie, int type)
224 {
225 	ie->mode &= ~GOT_FILEIDX_MODE_FILE_TYPE_STAGED;
226 	ie->mode |= ((type << GOT_FILEIDX_MODE_FILE_TYPE_STAGED_SHIFT) &
227 	    GOT_FILEIDX_MODE_FILE_TYPE_STAGED);
228 }
229 
230 int
231 got_fileindex_entry_staged_filetype_get(struct got_fileindex_entry *ie)
232 {
233 	return (ie->mode & GOT_FILEIDX_MODE_FILE_TYPE_STAGED) >>
234 	    GOT_FILEIDX_MODE_FILE_TYPE_STAGED_SHIFT;
235 }
236 
237 int
238 got_fileindex_entry_has_blob(struct got_fileindex_entry *ie)
239 {
240 	return (ie->flags & GOT_FILEIDX_F_NO_BLOB) == 0;
241 }
242 
243 int
244 got_fileindex_entry_has_commit(struct got_fileindex_entry *ie)
245 {
246 	return (ie->flags & GOT_FILEIDX_F_NO_COMMIT) == 0;
247 }
248 
249 int
250 got_fileindex_entry_has_file_on_disk(struct got_fileindex_entry *ie)
251 {
252 	return (ie->flags & GOT_FILEIDX_F_NO_FILE_ON_DISK) == 0;
253 }
254 
255 int
256 got_fileindex_entry_was_skipped(struct got_fileindex_entry *ie)
257 {
258 	return (ie->flags & GOT_FILEIDX_F_SKIPPED) != 0;
259 }
260 
261 static const struct got_error *
262 add_entry(struct got_fileindex *fileindex, struct got_fileindex_entry *ie)
263 {
264 	if (fileindex->nentries >= GOT_FILEIDX_MAX_ENTRIES)
265 		return got_error(GOT_ERR_NO_SPACE);
266 
267 	RB_INSERT(got_fileindex_tree, &fileindex->entries, ie);
268 	fileindex->nentries++;
269 	return NULL;
270 }
271 
272 const struct got_error *
273 got_fileindex_entry_add(struct got_fileindex *fileindex,
274     struct got_fileindex_entry *ie)
275 {
276 	/* Flag this entry until it gets written out to disk. */
277 	ie->flags |= GOT_FILEIDX_F_NOT_FLUSHED;
278 
279 	return add_entry(fileindex, ie);
280 }
281 
282 void
283 got_fileindex_entry_remove(struct got_fileindex *fileindex,
284     struct got_fileindex_entry *ie)
285 {
286 	/*
287 	 * Removing an entry from the RB tree immediately breaks
288 	 * in-progress iterations over file index entries.
289 	 * So flag this entry for removal and remove it once the index
290 	 * is written out to disk. Meanwhile, pretend this entry no longer
291 	 * exists if we get queried for it again before then.
292 	 */
293 	ie->flags |= GOT_FILEIDX_F_REMOVE_ON_FLUSH;
294 	fileindex->nentries--;
295 }
296 
297 struct got_fileindex_entry *
298 got_fileindex_entry_get(struct got_fileindex *fileindex, const char *path,
299     size_t path_len)
300 {
301 	struct got_fileindex_entry *ie;
302 	struct got_fileindex_entry key;
303 	memset(&key, 0, sizeof(key));
304 	key.path = (char *)path;
305 	key.flags = (path_len & GOT_FILEIDX_F_PATH_LEN);
306 	ie = RB_FIND(got_fileindex_tree, &fileindex->entries, &key);
307 	if (ie && (ie->flags & GOT_FILEIDX_F_REMOVE_ON_FLUSH))
308 		return NULL;
309 	return ie;
310 }
311 
312 const struct got_error *
313 got_fileindex_for_each_entry_safe(struct got_fileindex *fileindex,
314     got_fileindex_cb cb, void *cb_arg)
315 {
316 	const struct got_error *err;
317 	struct got_fileindex_entry *ie, *tmp;
318 
319 	RB_FOREACH_SAFE(ie, got_fileindex_tree, &fileindex->entries, tmp) {
320 		if (ie->flags & GOT_FILEIDX_F_REMOVE_ON_FLUSH)
321 			continue;
322 		err = (*cb)(cb_arg, ie);
323 		if (err)
324 			return err;
325 	}
326 	return NULL;
327 }
328 
329 struct got_fileindex *
330 got_fileindex_alloc(void)
331 {
332 	struct got_fileindex *fileindex;
333 
334 	fileindex = calloc(1, sizeof(*fileindex));
335 	if (fileindex == NULL)
336 		return NULL;
337 
338 	RB_INIT(&fileindex->entries);
339 	return fileindex;
340 }
341 
342 void
343 got_fileindex_free(struct got_fileindex *fileindex)
344 {
345 	struct got_fileindex_entry *ie;
346 
347 	while ((ie = RB_MIN(got_fileindex_tree, &fileindex->entries))) {
348 		RB_REMOVE(got_fileindex_tree, &fileindex->entries, ie);
349 		got_fileindex_entry_free(ie);
350 	}
351 	free(fileindex);
352 }
353 
354 static const struct got_error *
355 write_fileindex_val64(SHA1_CTX *ctx, uint64_t val, FILE *outfile)
356 {
357 	size_t n;
358 
359 	val = htobe64(val);
360 	SHA1Update(ctx, (uint8_t *)&val, sizeof(val));
361 	n = fwrite(&val, 1, sizeof(val), outfile);
362 	if (n != sizeof(val))
363 		return got_ferror(outfile, GOT_ERR_IO);
364 	return NULL;
365 }
366 
367 static const struct got_error *
368 write_fileindex_val32(SHA1_CTX *ctx, uint32_t val, FILE *outfile)
369 {
370 	size_t n;
371 
372 	val = htobe32(val);
373 	SHA1Update(ctx, (uint8_t *)&val, sizeof(val));
374 	n = fwrite(&val, 1, sizeof(val), outfile);
375 	if (n != sizeof(val))
376 		return got_ferror(outfile, GOT_ERR_IO);
377 	return NULL;
378 }
379 
380 static const struct got_error *
381 write_fileindex_val16(SHA1_CTX *ctx, uint16_t val, FILE *outfile)
382 {
383 	size_t n;
384 
385 	val = htobe16(val);
386 	SHA1Update(ctx, (uint8_t *)&val, sizeof(val));
387 	n = fwrite(&val, 1, sizeof(val), outfile);
388 	if (n != sizeof(val))
389 		return got_ferror(outfile, GOT_ERR_IO);
390 	return NULL;
391 }
392 
393 static const struct got_error *
394 write_fileindex_path(SHA1_CTX *ctx, const char *path, FILE *outfile)
395 {
396 	size_t n, len, pad = 0;
397 	static const uint8_t zero[8] = { 0 };
398 
399 	len = strlen(path);
400 	while ((len + pad) % 8 != 0)
401 		pad++;
402 	if (pad == 0)
403 		pad = 8; /* NUL-terminate */
404 
405 	SHA1Update(ctx, path, len);
406 	n = fwrite(path, 1, len, outfile);
407 	if (n != len)
408 		return got_ferror(outfile, GOT_ERR_IO);
409 	SHA1Update(ctx, zero, pad);
410 	n = fwrite(zero, 1, pad, outfile);
411 	if (n != pad)
412 		return got_ferror(outfile, GOT_ERR_IO);
413 	return NULL;
414 }
415 
416 static const struct got_error *
417 write_fileindex_entry(SHA1_CTX *ctx, struct got_fileindex_entry *ie,
418     FILE *outfile)
419 {
420 	const struct got_error *err;
421 	size_t n;
422 	uint32_t stage;
423 
424 	err = write_fileindex_val64(ctx, ie->ctime_sec, outfile);
425 	if (err)
426 		return err;
427 	err = write_fileindex_val64(ctx, ie->ctime_nsec, outfile);
428 	if (err)
429 		return err;
430 	err = write_fileindex_val64(ctx, ie->mtime_sec, outfile);
431 	if (err)
432 		return err;
433 	err = write_fileindex_val64(ctx, ie->mtime_nsec, outfile);
434 	if (err)
435 		return err;
436 
437 	err = write_fileindex_val32(ctx, ie->uid, outfile);
438 	if (err)
439 		return err;
440 	err = write_fileindex_val32(ctx, ie->gid, outfile);
441 	if (err)
442 		return err;
443 	err = write_fileindex_val32(ctx, ie->size, outfile);
444 	if (err)
445 		return err;
446 
447 	err = write_fileindex_val16(ctx, ie->mode, outfile);
448 	if (err)
449 		return err;
450 
451 	SHA1Update(ctx, ie->blob_sha1, SHA1_DIGEST_LENGTH);
452 	n = fwrite(ie->blob_sha1, 1, SHA1_DIGEST_LENGTH, outfile);
453 	if (n != SHA1_DIGEST_LENGTH)
454 		return got_ferror(outfile, GOT_ERR_IO);
455 
456 	SHA1Update(ctx, ie->commit_sha1, SHA1_DIGEST_LENGTH);
457 	n = fwrite(ie->commit_sha1, 1, SHA1_DIGEST_LENGTH, outfile);
458 	if (n != SHA1_DIGEST_LENGTH)
459 		return got_ferror(outfile, GOT_ERR_IO);
460 
461 	err = write_fileindex_val32(ctx, ie->flags, outfile);
462 	if (err)
463 		return err;
464 
465 	err = write_fileindex_path(ctx, ie->path, outfile);
466 	if (err)
467 		return err;
468 
469 	stage = got_fileindex_entry_stage_get(ie);
470 	if (stage == GOT_FILEIDX_STAGE_MODIFY ||
471 	    stage == GOT_FILEIDX_STAGE_ADD) {
472 		SHA1Update(ctx, ie->staged_blob_sha1, SHA1_DIGEST_LENGTH);
473 		n = fwrite(ie->staged_blob_sha1, 1, SHA1_DIGEST_LENGTH,
474 		    outfile);
475 		if (n != SHA1_DIGEST_LENGTH)
476 			return got_ferror(outfile, GOT_ERR_IO);
477 	}
478 
479 	return NULL;
480 }
481 
482 const struct got_error *
483 got_fileindex_write(struct got_fileindex *fileindex, FILE *outfile)
484 {
485 	const struct got_error *err = NULL;
486 	struct got_fileindex_hdr hdr;
487 	SHA1_CTX ctx;
488 	uint8_t sha1[SHA1_DIGEST_LENGTH];
489 	size_t n;
490 	struct got_fileindex_entry *ie, *tmp;
491 
492 	SHA1Init(&ctx);
493 
494 	hdr.signature = htobe32(GOT_FILE_INDEX_SIGNATURE);
495 	hdr.version = htobe32(GOT_FILE_INDEX_VERSION);
496 	hdr.nentries = htobe32(fileindex->nentries);
497 
498 	SHA1Update(&ctx, (uint8_t *)&hdr.signature, sizeof(hdr.signature));
499 	SHA1Update(&ctx, (uint8_t *)&hdr.version, sizeof(hdr.version));
500 	SHA1Update(&ctx, (uint8_t *)&hdr.nentries, sizeof(hdr.nentries));
501 	n = fwrite(&hdr.signature, 1, sizeof(hdr.signature), outfile);
502 	if (n != sizeof(hdr.signature))
503 		return got_ferror(outfile, GOT_ERR_IO);
504 	n = fwrite(&hdr.version, 1, sizeof(hdr.version), outfile);
505 	if (n != sizeof(hdr.version))
506 		return got_ferror(outfile, GOT_ERR_IO);
507 	n = fwrite(&hdr.nentries, 1, sizeof(hdr.nentries), outfile);
508 	if (n != sizeof(hdr.nentries))
509 		return got_ferror(outfile, GOT_ERR_IO);
510 
511 	RB_FOREACH_SAFE(ie, got_fileindex_tree, &fileindex->entries, tmp) {
512 		ie->flags &= ~GOT_FILEIDX_F_NOT_FLUSHED;
513 		ie->flags &= ~GOT_FILEIDX_F_SKIPPED;
514 		if (ie->flags & GOT_FILEIDX_F_REMOVE_ON_FLUSH) {
515 			RB_REMOVE(got_fileindex_tree, &fileindex->entries, ie);
516 			got_fileindex_entry_free(ie);
517 			continue;
518 		}
519 		err = write_fileindex_entry(&ctx, ie, outfile);
520 		if (err)
521 			return err;
522 	}
523 
524 	SHA1Final(sha1, &ctx);
525 	n = fwrite(sha1, 1, sizeof(sha1), outfile);
526 	if (n != sizeof(sha1))
527 		return got_ferror(outfile, GOT_ERR_IO);
528 
529 	if (fflush(outfile) != 0)
530 		return got_error_from_errno("fflush");
531 
532 	return NULL;
533 }
534 
535 static const struct got_error *
536 read_fileindex_val64(uint64_t *val, SHA1_CTX *ctx, FILE *infile)
537 {
538 	size_t n;
539 
540 	n = fread(val, 1, sizeof(*val), infile);
541 	if (n != sizeof(*val))
542 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
543 	SHA1Update(ctx, (uint8_t *)val, sizeof(*val));
544 	*val = be64toh(*val);
545 	return NULL;
546 }
547 
548 static const struct got_error *
549 read_fileindex_val32(uint32_t *val, SHA1_CTX *ctx, FILE *infile)
550 {
551 	size_t n;
552 
553 	n = fread(val, 1, sizeof(*val), infile);
554 	if (n != sizeof(*val))
555 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
556 	SHA1Update(ctx, (uint8_t *)val, sizeof(*val));
557 	*val = be32toh(*val);
558 	return NULL;
559 }
560 
561 static const struct got_error *
562 read_fileindex_val16(uint16_t *val, SHA1_CTX *ctx, FILE *infile)
563 {
564 	size_t n;
565 
566 	n = fread(val, 1, sizeof(*val), infile);
567 	if (n != sizeof(*val))
568 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
569 	SHA1Update(ctx, (uint8_t *)val, sizeof(*val));
570 	*val = be16toh(*val);
571 	return NULL;
572 }
573 
574 static const struct got_error *
575 read_fileindex_path(char **path, SHA1_CTX *ctx, FILE *infile)
576 {
577 	const struct got_error *err = NULL;
578 	const size_t chunk_size = 8;
579 	size_t n, len = 0, totlen = chunk_size;
580 
581 	*path = malloc(totlen);
582 	if (*path == NULL)
583 		return got_error_from_errno("malloc");
584 
585 	do {
586 		if (len + chunk_size > totlen) {
587 			char *p = reallocarray(*path, totlen + chunk_size, 1);
588 			if (p == NULL) {
589 				err = got_error_from_errno("reallocarray");
590 				break;
591 			}
592 			totlen += chunk_size;
593 			*path = p;
594 		}
595 		n = fread(*path + len, 1, chunk_size, infile);
596 		if (n != chunk_size) {
597 			err = got_ferror(infile, GOT_ERR_FILEIDX_BAD);
598 			break;
599 		}
600 		SHA1Update(ctx, *path + len, chunk_size);
601 		len += chunk_size;
602 	} while (memchr(*path + len - chunk_size, '\0', chunk_size) == NULL);
603 
604 	if (err) {
605 		free(*path);
606 		*path = NULL;
607 	}
608 	return err;
609 }
610 
611 static const struct got_error *
612 read_fileindex_entry(struct got_fileindex_entry **iep, SHA1_CTX *ctx,
613     FILE *infile, uint32_t version)
614 {
615 	const struct got_error *err;
616 	struct got_fileindex_entry *ie;
617 	size_t n;
618 
619 	*iep = NULL;
620 
621 	ie = calloc(1, sizeof(*ie));
622 	if (ie == NULL)
623 		return got_error_from_errno("calloc");
624 
625 	err = read_fileindex_val64(&ie->ctime_sec, ctx, infile);
626 	if (err)
627 		goto done;
628 	err = read_fileindex_val64(&ie->ctime_nsec, ctx, infile);
629 	if (err)
630 		goto done;
631 	err = read_fileindex_val64(&ie->mtime_sec, ctx, infile);
632 	if (err)
633 		goto done;
634 	err = read_fileindex_val64(&ie->mtime_nsec, ctx, infile);
635 	if (err)
636 		goto done;
637 
638 	err = read_fileindex_val32(&ie->uid, ctx, infile);
639 	if (err)
640 		goto done;
641 	err = read_fileindex_val32(&ie->gid, ctx, infile);
642 	if (err)
643 		goto done;
644 	err = read_fileindex_val32(&ie->size, ctx, infile);
645 	if (err)
646 		goto done;
647 
648 	err = read_fileindex_val16(&ie->mode, ctx, infile);
649 	if (err)
650 		goto done;
651 
652 	n = fread(ie->blob_sha1, 1, SHA1_DIGEST_LENGTH, infile);
653 	if (n != SHA1_DIGEST_LENGTH) {
654 		err = got_ferror(infile, GOT_ERR_FILEIDX_BAD);
655 		goto done;
656 	}
657 	SHA1Update(ctx, ie->blob_sha1, SHA1_DIGEST_LENGTH);
658 
659 	n = fread(ie->commit_sha1, 1, SHA1_DIGEST_LENGTH, infile);
660 	if (n != SHA1_DIGEST_LENGTH) {
661 		err = got_ferror(infile, GOT_ERR_FILEIDX_BAD);
662 		goto done;
663 	}
664 	SHA1Update(ctx, ie->commit_sha1, SHA1_DIGEST_LENGTH);
665 
666 	err = read_fileindex_val32(&ie->flags, ctx, infile);
667 	if (err)
668 		goto done;
669 
670 	err = read_fileindex_path(&ie->path, ctx, infile);
671 	if (err)
672 		goto done;
673 
674 	if (version >= 2) {
675 		uint32_t stage = got_fileindex_entry_stage_get(ie);
676 		if (stage == GOT_FILEIDX_STAGE_MODIFY ||
677 		    stage == GOT_FILEIDX_STAGE_ADD) {
678 			n = fread(ie->staged_blob_sha1, 1, SHA1_DIGEST_LENGTH,
679 			    infile);
680 			if (n != SHA1_DIGEST_LENGTH) {
681 				err = got_ferror(infile, GOT_ERR_FILEIDX_BAD);
682 				goto done;
683 			}
684 			SHA1Update(ctx, ie->staged_blob_sha1, SHA1_DIGEST_LENGTH);
685 		}
686 	} else {
687 		/* GOT_FILE_INDEX_VERSION 1 does not support staging. */
688 		ie->flags &= ~GOT_FILEIDX_F_STAGE;
689 	}
690 
691 done:
692 	if (err)
693 		got_fileindex_entry_free(ie);
694 	else
695 		*iep = ie;
696 	return err;
697 }
698 
699 const struct got_error *
700 got_fileindex_read(struct got_fileindex *fileindex, FILE *infile)
701 {
702 	const struct got_error *err = NULL;
703 	struct got_fileindex_hdr hdr;
704 	SHA1_CTX ctx;
705 	struct got_fileindex_entry *ie;
706 	uint8_t sha1_expected[SHA1_DIGEST_LENGTH];
707 	uint8_t sha1[SHA1_DIGEST_LENGTH];
708 	size_t n;
709 	int i;
710 
711 	SHA1Init(&ctx);
712 
713 	n = fread(&hdr.signature, 1, sizeof(hdr.signature), infile);
714 	if (n != sizeof(hdr.signature)) {
715 		if (n == 0) /* EOF */
716 			return NULL;
717 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
718 	}
719 	n = fread(&hdr.version, 1, sizeof(hdr.version), infile);
720 	if (n != sizeof(hdr.version)) {
721 		if (n == 0) /* EOF */
722 			return NULL;
723 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
724 	}
725 	n = fread(&hdr.nentries, 1, sizeof(hdr.nentries), infile);
726 	if (n != sizeof(hdr.nentries)) {
727 		if (n == 0) /* EOF */
728 			return NULL;
729 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
730 	}
731 
732 	SHA1Update(&ctx, (uint8_t *)&hdr.signature, sizeof(hdr.signature));
733 	SHA1Update(&ctx, (uint8_t *)&hdr.version, sizeof(hdr.version));
734 	SHA1Update(&ctx, (uint8_t *)&hdr.nentries, sizeof(hdr.nentries));
735 
736 	hdr.signature = be32toh(hdr.signature);
737 	hdr.version = be32toh(hdr.version);
738 	hdr.nentries = be32toh(hdr.nentries);
739 
740 	if (hdr.signature != GOT_FILE_INDEX_SIGNATURE)
741 		return got_error(GOT_ERR_FILEIDX_SIG);
742 	if (hdr.version > GOT_FILE_INDEX_VERSION)
743 		return got_error(GOT_ERR_FILEIDX_VER);
744 
745 	for (i = 0; i < hdr.nentries; i++) {
746 		err = read_fileindex_entry(&ie, &ctx, infile, hdr.version);
747 		if (err)
748 			return err;
749 		err = add_entry(fileindex, ie);
750 		if (err)
751 			return err;
752 	}
753 
754 	n = fread(sha1_expected, 1, sizeof(sha1_expected), infile);
755 	if (n != sizeof(sha1_expected))
756 		return got_ferror(infile, GOT_ERR_FILEIDX_BAD);
757 	SHA1Final(sha1, &ctx);
758 	if (memcmp(sha1, sha1_expected, SHA1_DIGEST_LENGTH) != 0)
759 		return got_error(GOT_ERR_FILEIDX_CSUM);
760 
761 	return NULL;
762 }
763 
764 static struct got_fileindex_entry *
765 walk_fileindex(struct got_fileindex *fileindex, struct got_fileindex_entry *ie)
766 {
767 	struct got_fileindex_entry *next;
768 
769 	next = RB_NEXT(got_fileindex_tree, &fileindex->entries, ie);
770 
771 	/* Skip entries which were added or removed by diff callbacks. */
772 	while (next && (next->flags & (GOT_FILEIDX_F_NOT_FLUSHED |
773 	    GOT_FILEIDX_F_REMOVE_ON_FLUSH)))
774 		next = RB_NEXT(got_fileindex_tree, &fileindex->entries, next);
775 
776 	return next;
777 }
778 
779 static const struct got_error *
780 diff_fileindex_tree(struct got_fileindex *, struct got_fileindex_entry **ie,
781     struct got_tree_object *tree, const char *, const char *,
782     struct got_repository *, struct got_fileindex_diff_tree_cb *, void *);
783 
784 static const struct got_error *
785 walk_tree(struct got_tree_entry **next, struct got_fileindex *fileindex,
786     struct got_fileindex_entry **ie, struct got_tree_object *tree, int *tidx,
787     const char *path, const char *entry_name, struct got_repository *repo,
788     struct got_fileindex_diff_tree_cb *cb, void *cb_arg)
789 {
790 	const struct got_error *err = NULL;
791 	struct got_tree_entry *te = got_object_tree_get_entry(tree, *tidx);
792 
793 	if (!got_object_tree_entry_is_submodule(te) &&
794 	    S_ISDIR(got_tree_entry_get_mode(te))) {
795 		char *subpath;
796 		struct got_tree_object *subtree;
797 
798 		if (asprintf(&subpath, "%s%s%s", path,
799 		    path[0] == '\0' ? "" : "/",
800 		    got_tree_entry_get_name(te)) == -1)
801 			return got_error_from_errno("asprintf");
802 
803 		err = got_object_open_as_tree(&subtree, repo,
804 		    got_tree_entry_get_id(te));
805 		if (err) {
806 			free(subpath);
807 			return err;
808 		}
809 
810 		err = diff_fileindex_tree(fileindex, ie, subtree, subpath,
811 		    entry_name, repo, cb, cb_arg);
812 		free(subpath);
813 		got_object_tree_close(subtree);
814 		if (err)
815 			return err;
816 	}
817 
818 	(*tidx)++;
819 	*next = got_object_tree_get_entry(tree, *tidx);
820 	return NULL;
821 }
822 
823 static const struct got_error *
824 diff_fileindex_tree(struct got_fileindex *fileindex,
825     struct got_fileindex_entry **ie, struct got_tree_object *tree,
826     const char *path, const char *entry_name, struct got_repository *repo,
827     struct got_fileindex_diff_tree_cb *cb, void *cb_arg)
828 {
829 	const struct got_error *err = NULL;
830 	struct got_tree_entry *te = NULL;
831 	size_t path_len = strlen(path);
832 	struct got_fileindex_entry *next;
833 	int tidx = 0;
834 
835 	te = got_object_tree_get_entry(tree, tidx);
836 	while ((*ie && got_path_is_child((*ie)->path, path, path_len)) || te) {
837 		if (te && *ie) {
838 			char *te_path;
839 			const char *te_name = got_tree_entry_get_name(te);
840 			int cmp;
841 			if (asprintf(&te_path, "%s/%s", path, te_name) == -1) {
842 				err = got_error_from_errno("asprintf");
843 				break;
844 			}
845 			cmp = got_path_cmp((*ie)->path, te_path,
846 			    got_fileindex_entry_path_len(*ie), strlen(te_path));
847 			free(te_path);
848 			if (cmp == 0) {
849 				if (got_path_is_child((*ie)->path, path,
850 				    path_len) &&
851 				    !got_object_tree_entry_is_submodule(te) &&
852 				    (entry_name == NULL ||
853 				    strcmp(te_name, entry_name) == 0)) {
854 					err = cb->diff_old_new(cb_arg, *ie, te,
855 					    path);
856 					if (err || entry_name)
857 						break;
858 				}
859 				*ie = walk_fileindex(fileindex, *ie);
860 				err = walk_tree(&te, fileindex, ie, tree, &tidx,
861 				    path, entry_name, repo, cb, cb_arg);
862 			} else if (cmp < 0) {
863 				next = walk_fileindex(fileindex, *ie);
864 				if (got_path_is_child((*ie)->path, path,
865 				    path_len) && entry_name == NULL) {
866 					err = cb->diff_old(cb_arg, *ie, path);
867 					if (err || entry_name)
868 						break;
869 				}
870 				*ie = next;
871 			} else {
872 				if ((entry_name == NULL ||
873 				    strcmp(te_name, entry_name) == 0)) {
874 					err = cb->diff_new(cb_arg, te, path);
875 					if (err || entry_name)
876 						break;
877 				}
878 				err = walk_tree(&te, fileindex, ie, tree, &tidx,
879 				    path, entry_name, repo, cb, cb_arg);
880 			}
881 			if (err)
882 				break;
883 		} else if (*ie) {
884 			next = walk_fileindex(fileindex, *ie);
885 			if (got_path_is_child((*ie)->path, path, path_len) &&
886 			    (entry_name == NULL ||
887 			    (te && strcmp(got_tree_entry_get_name(te),
888 			    entry_name) == 0))) {
889 				err = cb->diff_old(cb_arg, *ie, path);
890 				if (err || entry_name)
891 					break;
892 			}
893 			*ie = next;
894 		} else if (te) {
895 			if (!got_object_tree_entry_is_submodule(te) &&
896 			    (entry_name == NULL ||
897 			    strcmp(got_tree_entry_get_name(te), entry_name)
898 			    == 0)) {
899 				err = cb->diff_new(cb_arg, te, path);
900 				if (err || entry_name)
901 					break;
902 			}
903 			err = walk_tree(&te, fileindex, ie, tree, &tidx, path,
904 			    entry_name, repo, cb, cb_arg);
905 			if (err)
906 				break;
907 		}
908 	}
909 
910 	return err;
911 }
912 
913 const struct got_error *
914 got_fileindex_diff_tree(struct got_fileindex *fileindex,
915     struct got_tree_object *tree, const char *path, const char *entry_name,
916     struct got_repository *repo,
917     struct got_fileindex_diff_tree_cb *cb, void *cb_arg)
918 {
919 	struct got_fileindex_entry *ie;
920 	ie = RB_MIN(got_fileindex_tree, &fileindex->entries);
921 	while (ie && !got_path_is_child(ie->path, path, strlen(path)))
922 		ie = walk_fileindex(fileindex, ie);
923 	return diff_fileindex_tree(fileindex, &ie, tree, path, entry_name, repo,
924 	    cb, cb_arg);
925 }
926 
927 static const struct got_error *
928 diff_fileindex_dir(struct got_fileindex *, struct got_fileindex_entry **,
929     struct got_pathlist_head *, int, const char *, const char *,
930     struct got_repository *, struct got_fileindex_diff_dir_cb *, void *);
931 
932 static const struct got_error *
933 read_dirlist(struct got_pathlist_head *dirlist, DIR *dir, const char *path)
934 {
935 	const struct got_error *err = NULL;
936 	struct got_pathlist_entry *new = NULL;
937 	struct dirent *dep = NULL;
938 	struct dirent *de = NULL;
939 
940 	for (;;) {
941 		de = malloc(sizeof(struct dirent) + NAME_MAX + 1);
942 		if (de == NULL) {
943 			err = got_error_from_errno("malloc");
944 			break;
945 		}
946 
947 		if (readdir_r(dir, de, &dep) != 0) {
948 			err = got_error_from_errno("readdir_r");
949 			free(de);
950 			break;
951 		}
952 		if (dep == NULL) {
953 			free(de);
954 			break;
955 		}
956 
957 		if (strcmp(de->d_name, ".") == 0 ||
958 		    strcmp(de->d_name, "..") == 0 ||
959 		    (path[0] == '\0' &&
960 		    strcmp(de->d_name, GOT_WORKTREE_GOT_DIR) == 0)) {
961 			free(de);
962 			continue;
963 		}
964 
965 		err = got_pathlist_insert(&new, dirlist, de->d_name, de);
966 		if (err) {
967 			free(de);
968 			break;
969 		}
970 		if (new == NULL) {
971 			err = got_error(GOT_ERR_DIR_DUP_ENTRY);
972 			free(de);
973 			break;
974 		}
975 	}
976 
977 	return err;
978 }
979 
980 void
981 free_dirlist(struct got_pathlist_head *dirlist)
982 {
983 	struct got_pathlist_entry *dle;
984 
985 	TAILQ_FOREACH(dle, dirlist, entry)
986 		free(dle->data);
987 	got_pathlist_free(dirlist);
988 }
989 
990 static const struct got_error *
991 walk_dir(struct got_pathlist_entry **next, struct got_fileindex *fileindex,
992     struct got_fileindex_entry **ie, struct got_pathlist_entry *dle, int fd,
993     const char *path, const char *rootpath, struct got_repository *repo,
994     int ignore, struct got_fileindex_diff_dir_cb *cb, void *cb_arg)
995 {
996 	const struct got_error *err = NULL;
997 	struct dirent *de = dle->data;
998 	DIR *subdir = NULL;
999 	int subdirfd = -1;
1000 	int type;
1001 
1002 	*next = NULL;
1003 
1004 	if (de->d_type == DT_UNKNOWN) {
1005 		/* Occurs on NFS mounts without "readdir plus" RPC. */
1006 		char *dir_path;
1007 		if (asprintf(&dir_path, "%s/%s", rootpath, path) == -1)
1008 			return got_error_from_errno("asprintf");
1009 		err = got_path_dirent_type(&type, dir_path, de);
1010 		free(dir_path);
1011 		if (err)
1012 			return err;
1013 	} else
1014 		type = de->d_type;
1015 
1016 	if (type == DT_DIR && !ignore) {
1017 		char *subpath;
1018 		char *subdirpath;
1019 		struct got_pathlist_head subdirlist;
1020 
1021 		TAILQ_INIT(&subdirlist);
1022 
1023 		if (asprintf(&subpath, "%s%s%s", path,
1024 		    path[0] == '\0' ? "" : "/", de->d_name) == -1)
1025 			return got_error_from_errno("asprintf");
1026 
1027 		if (asprintf(&subdirpath, "%s/%s", rootpath, subpath) == -1) {
1028 			free(subpath);
1029 			return got_error_from_errno("asprintf");
1030 		}
1031 
1032 		subdirfd = openat(fd, de->d_name,
1033 		    O_RDONLY | O_NOFOLLOW | O_DIRECTORY);
1034 		if (subdirfd == -1) {
1035 			if (errno == EACCES) {
1036 				*next = TAILQ_NEXT(dle, entry);
1037 				return NULL;
1038 			}
1039 			err = got_error_from_errno2("openat", subdirpath);
1040 			free(subpath);
1041 			free(subdirpath);
1042 			return err;
1043 		}
1044 
1045 		subdir = fdopendir(subdirfd);
1046 		if (subdir == NULL)
1047 			return got_error_from_errno2("fdopendir", path);
1048 		subdirfd = -1;
1049 		err = read_dirlist(&subdirlist, subdir, subdirpath);
1050 		if (err) {
1051 			free(subpath);
1052 			free(subdirpath);
1053 			closedir(subdir);
1054 			return err;
1055 		}
1056 		err = diff_fileindex_dir(fileindex, ie, &subdirlist,
1057 		    dirfd(subdir), rootpath, subpath, repo, cb, cb_arg);
1058 		if (subdir && closedir(subdir) == -1 && err == NULL)
1059 			err = got_error_from_errno2("closedir", subdirpath);
1060 		free(subpath);
1061 		free(subdirpath);
1062 		free_dirlist(&subdirlist);
1063 		if (err)
1064 			return err;
1065 	}
1066 
1067 	*next = TAILQ_NEXT(dle, entry);
1068 	return NULL;
1069 }
1070 
1071 static const struct got_error *
1072 diff_fileindex_dir(struct got_fileindex *fileindex,
1073     struct got_fileindex_entry **ie, struct got_pathlist_head *dirlist,
1074     int dirfd, const char *rootpath, const char *path,
1075     struct got_repository *repo,
1076     struct got_fileindex_diff_dir_cb *cb, void *cb_arg)
1077 {
1078 	const struct got_error *err = NULL;
1079 	struct dirent *de = NULL;
1080 	size_t path_len = strlen(path);
1081 	struct got_pathlist_entry *dle;
1082 	int ignore;
1083 
1084 	if (cb->diff_traverse) {
1085 		err = cb->diff_traverse(cb_arg, path, dirfd);
1086 		if (err)
1087 			return err;
1088 	}
1089 
1090 	dle = TAILQ_FIRST(dirlist);
1091 	while ((*ie && got_path_is_child((*ie)->path, path, path_len)) || dle) {
1092 		if (dle && *ie) {
1093 			char *de_path;
1094 			int cmp;
1095 			de = dle->data;
1096 			if (asprintf(&de_path, "%s/%s", path,
1097 			    de->d_name) == -1) {
1098 				err = got_error_from_errno("asprintf");
1099 				break;
1100 			}
1101 			cmp = got_path_cmp((*ie)->path, de_path,
1102 			    got_fileindex_entry_path_len(*ie),
1103 			    strlen(path) + 1 + de->d_namlen);
1104 			free(de_path);
1105 			if (cmp == 0) {
1106 				err = cb->diff_old_new(cb_arg, *ie, de, path,
1107 				    dirfd);
1108 				if (err)
1109 					break;
1110 				*ie = walk_fileindex(fileindex, *ie);
1111 				err = walk_dir(&dle, fileindex, ie, dle, dirfd,
1112 				    path, rootpath, repo, 0, cb, cb_arg);
1113 			} else if (cmp < 0 ) {
1114 				err = cb->diff_old(cb_arg, *ie, path);
1115 				if (err)
1116 					break;
1117 				*ie = walk_fileindex(fileindex, *ie);
1118 			} else {
1119 				err = cb->diff_new(&ignore, cb_arg, de, path,
1120 				    dirfd);
1121 				if (err)
1122 					break;
1123 				err = walk_dir(&dle, fileindex, ie, dle, dirfd,
1124 				    path, rootpath, repo, ignore, cb, cb_arg);
1125 			}
1126 			if (err)
1127 				break;
1128 		} else if (*ie) {
1129 			err = cb->diff_old(cb_arg, *ie, path);
1130 			if (err)
1131 				break;
1132 			*ie = walk_fileindex(fileindex, *ie);
1133 		} else if (dle) {
1134 			de = dle->data;
1135 			err = cb->diff_new(&ignore, cb_arg, de, path, dirfd);
1136 			if (err)
1137 				break;
1138 			err = walk_dir(&dle, fileindex, ie, dle, dirfd, path,
1139 			    rootpath, repo, ignore, cb, cb_arg);
1140 			if (err)
1141 				break;
1142 		}
1143 	}
1144 
1145 	return err;
1146 }
1147 
1148 const struct got_error *
1149 got_fileindex_diff_dir(struct got_fileindex *fileindex, int fd,
1150     const char *rootpath, const char *path, struct got_repository *repo,
1151     struct got_fileindex_diff_dir_cb *cb, void *cb_arg)
1152 {
1153 	const struct got_error *err;
1154 	struct got_fileindex_entry *ie;
1155 	struct got_pathlist_head dirlist;
1156 	int fd2;
1157 	DIR *dir;
1158 
1159 	TAILQ_INIT(&dirlist);
1160 
1161 	/* 
1162 	 * Duplicate the file descriptor so we can call closedir() below
1163 	 * without closing the file descriptor passed in by our caller.
1164 	 */
1165 	fd2 = dup(fd);
1166 	if (fd2 == -1)
1167 		return got_error_from_errno2("dup", path);
1168 	if (lseek(fd2, 0, SEEK_SET) == -1) {
1169 		err = got_error_from_errno2("lseek", path);
1170 		close(fd2);
1171 		return err;
1172 	}
1173 	dir = fdopendir(fd2);
1174 	if (dir == NULL) {
1175 		err = got_error_from_errno2("fdopendir", path);
1176 		close(fd2);
1177 		return err;
1178 	}
1179 	err = read_dirlist(&dirlist, dir, path);
1180 	if (err) {
1181 		closedir(dir);
1182 		return err;
1183 	}
1184 
1185 	ie = RB_MIN(got_fileindex_tree, &fileindex->entries);
1186 	while (ie && !got_path_is_child(ie->path, path, strlen(path)))
1187 		ie = walk_fileindex(fileindex, ie);
1188 	err = diff_fileindex_dir(fileindex, &ie, &dirlist, dirfd(dir),
1189 	    rootpath, path, repo, cb, cb_arg);
1190 
1191 	if (closedir(dir) == -1 && err == NULL)
1192 		err = got_error_from_errno2("closedir", path);
1193 	free_dirlist(&dirlist);
1194 	return err;
1195 }
1196 
1197 RB_GENERATE(got_fileindex_tree, got_fileindex_entry, entry, got_fileindex_cmp);