op public repos

Blob

Date:: Sun Feb 10 13:03:49 2019 UTC
Message:: eliminate redundant stat() calls during status crawl
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018, 2019 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/stat.h>
18 #include <sys/limits.h>
19 #include <sys/queue.h>
20 #include <sys/tree.h>
21 
22 #include <dirent.h>
23 #include <stddef.h>
24 #include <string.h>
25 #include <stdio.h>
26 #include <stdlib.h>
27 #include <fcntl.h>
28 #include <errno.h>
29 #include <unistd.h>
30 #include <sha1.h>
31 #include <zlib.h>
32 #include <fnmatch.h>
33 #include <libgen.h>
34 
35 #include "got_error.h"
36 #include "got_repository.h"
37 #include "got_reference.h"
38 #include "got_object.h"
39 #include "got_worktree.h"
40 #include "got_opentemp.h"
41 
42 #include "got_lib_worktree.h"
43 #include "got_lib_path.h"
44 #include "got_lib_sha1.h"
45 #include "got_lib_fileindex.h"
46 #include "got_lib_inflate.h"
47 #include "got_lib_delta.h"
48 #include "got_lib_object.h"
49 #include "got_lib_diff.h"
50 
51 #ifndef MIN
52 #define	MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
53 #endif
54 
55 static const struct got_error *
56 create_meta_file(const char *path_got, const char *name, const char *content)
57 {
58 	const struct got_error *err = NULL;
59 	char *path;
60 	int fd = -1;
61 
62 	if (asprintf(&path, "%s/%s", path_got, name) == -1) {
63 		err = got_error_from_errno();
64 		path = NULL;
65 		goto done;
66 	}
67 
68 	fd = open(path, O_RDWR | O_CREAT | O_EXCL | O_NOFOLLOW,
69 	    GOT_DEFAULT_FILE_MODE);
70 	if (fd == -1) {
71 		err = got_error_from_errno();
72 		goto done;
73 	}
74 
75 	if (content) {
76 		int len = dprintf(fd, "%s\n", content);
77 		if (len != strlen(content) + 1) {
78 			err = got_error_from_errno();
79 			goto done;
80 		}
81 	}
82 
83 done:
84 	if (fd != -1 && close(fd) == -1 && err == NULL)
85 		err = got_error_from_errno();
86 	free(path);
87 	return err;
88 }
89 
90 static const struct got_error *
91 update_meta_file(const char *path_got, const char *name, const char *content)
92 {
93 	const struct got_error *err = NULL;
94 	FILE *tmpfile = NULL;
95 	char *tmppath = NULL;
96 	char *path = NULL;
97 
98 	if (asprintf(&path, "%s/%s", path_got, name) == -1) {
99 		err = got_error_from_errno();
100 		path = NULL;
101 		goto done;
102 	}
103 
104 	err = got_opentemp_named(&tmppath, &tmpfile, path);
105 	if (err)
106 		goto done;
107 
108 	if (content) {
109 		int len = fprintf(tmpfile, "%s\n", content);
110 		if (len != strlen(content) + 1) {
111 			err = got_error_from_errno();
112 			goto done;
113 		}
114 	}
115 
116 	if (rename(tmppath, path) != 0) {
117 		err = got_error_from_errno();
118 		goto done;
119 	}
120 
121 done:
122 	free(tmppath);
123 	fclose(tmpfile);
124 	return err;
125 }
126 
127 static const struct got_error *
128 read_meta_file(char **content, const char *path_got, const char *name)
129 {
130 	const struct got_error *err = NULL;
131 	char *path;
132 	int fd = -1;
133 	ssize_t n;
134 	struct stat sb;
135 
136 	*content = NULL;
137 
138 	if (asprintf(&path, "%s/%s", path_got, name) == -1) {
139 		err = got_error_from_errno();
140 		path = NULL;
141 		goto done;
142 	}
143 
144 	fd = open(path, O_RDONLY | O_NOFOLLOW);
145 	if (fd == -1) {
146 		err = got_error_from_errno();
147 		goto done;
148 	}
149 	if (flock(fd, LOCK_SH | LOCK_NB) == -1) {
150 		err = (errno == EWOULDBLOCK ? got_error(GOT_ERR_WORKTREE_BUSY)
151 		    : got_error_from_errno());
152 		goto done;
153 	}
154 
155 	stat(path, &sb);
156 	*content = calloc(1, sb.st_size);
157 	if (*content == NULL) {
158 		err = got_error_from_errno();
159 		goto done;
160 	}
161 
162 	n = read(fd, *content, sb.st_size);
163 	if (n != sb.st_size) {
164 		err = (n == -1 ? got_error_from_errno() :
165 		    got_error(GOT_ERR_WORKTREE_META));
166 		goto done;
167 	}
168 	if ((*content)[sb.st_size - 1] != '\n') {
169 		err = got_error(GOT_ERR_WORKTREE_META);
170 		goto done;
171 	}
172 	(*content)[sb.st_size - 1] = '\0';
173 
174 done:
175 	if (fd != -1 && close(fd) == -1 && err == NULL)
176 		err = got_error_from_errno();
177 	free(path);
178 	if (err) {
179 		free(*content);
180 		*content = NULL;
181 	}
182 	return err;
183 }
184 
185 const struct got_error *
186 got_worktree_init(const char *path, struct got_reference *head_ref,
187     const char *prefix, struct got_repository *repo)
188 {
189 	const struct got_error *err = NULL;
190 	struct got_object_id *commit_id = NULL;
191 	int obj_type;
192 	char *path_got = NULL;
193 	char *refstr = NULL;
194 	char *formatstr = NULL;
195 	char *absprefix = NULL;
196 	char *basestr = NULL;
197 
198 	err = got_ref_resolve(&commit_id, repo, head_ref);
199 	if (err)
200 		return err;
201 	err = got_object_get_type(&obj_type, repo, commit_id);
202 	if (err)
203 		return err;
204 	if (obj_type != GOT_OBJ_TYPE_COMMIT)
205 		return got_error(GOT_ERR_OBJ_TYPE);
206 
207 	if (!got_path_is_absolute(prefix)) {
208 		if (asprintf(&absprefix, "/%s", prefix) == -1)
209 			return got_error_from_errno();
210 	}
211 
212 	/* Create top-level directory (may already exist). */
213 	if (mkdir(path, GOT_DEFAULT_DIR_MODE) == -1 && errno != EEXIST) {
214 		err = got_error_from_errno();
215 		goto done;
216 	}
217 
218 	/* Create .got directory (may already exist). */
219 	if (asprintf(&path_got, "%s/%s", path, GOT_WORKTREE_GOT_DIR) == -1) {
220 		err = got_error_from_errno();
221 		goto done;
222 	}
223 	if (mkdir(path_got, GOT_DEFAULT_DIR_MODE) == -1 && errno != EEXIST) {
224 		err = got_error_from_errno();
225 		goto done;
226 	}
227 
228 	/* Create an empty lock file. */
229 	err = create_meta_file(path_got, GOT_WORKTREE_LOCK, NULL);
230 	if (err)
231 		goto done;
232 
233 	/* Create an empty file index. */
234 	err = create_meta_file(path_got, GOT_WORKTREE_FILE_INDEX, NULL);
235 	if (err)
236 		goto done;
237 
238 	/* Write the HEAD reference. */
239 	refstr = got_ref_to_str(head_ref);
240 	if (refstr == NULL) {
241 		err = got_error_from_errno();
242 		goto done;
243 	}
244 	err = create_meta_file(path_got, GOT_WORKTREE_HEAD_REF, refstr);
245 	if (err)
246 		goto done;
247 
248 	/* Record our base commit. */
249 	err = got_object_id_str(&basestr, commit_id);
250 	if (err)
251 		goto done;
252 	err = create_meta_file(path_got, GOT_WORKTREE_BASE_COMMIT, basestr);
253 	if (err)
254 		goto done;
255 
256 	/* Store path to repository. */
257 	err = create_meta_file(path_got, GOT_WORKTREE_REPOSITORY,
258 	    got_repo_get_path(repo));
259 	if (err)
260 		goto done;
261 
262 	/* Store in-repository path prefix. */
263 	err = create_meta_file(path_got, GOT_WORKTREE_PATH_PREFIX,
264 	    absprefix ? absprefix : prefix);
265 	if (err)
266 		goto done;
267 
268 	/* Stamp work tree with format file. */
269 	if (asprintf(&formatstr, "%d", GOT_WORKTREE_FORMAT_VERSION) == -1) {
270 		err = got_error_from_errno();
271 		goto done;
272 	}
273 	err = create_meta_file(path_got, GOT_WORKTREE_FORMAT, formatstr);
274 	if (err)
275 		goto done;
276 
277 done:
278 	free(commit_id);
279 	free(path_got);
280 	free(formatstr);
281 	free(refstr);
282 	free(absprefix);
283 	free(basestr);
284 	return err;
285 }
286 
287 static const struct got_error *
288 open_worktree(struct got_worktree **worktree, const char *path)
289 {
290 	const struct got_error *err = NULL;
291 	char *path_got;
292 	char *formatstr = NULL;
293 	char *path_lock = NULL;
294 	char *base_commit_id_str = NULL;
295 	char *head_ref_str = NULL;
296 	int version, fd = -1;
297 	const char *errstr;
298 	struct got_repository *repo = NULL;
299 
300 	*worktree = NULL;
301 
302 	if (asprintf(&path_got, "%s/%s", path, GOT_WORKTREE_GOT_DIR) == -1) {
303 		err = got_error_from_errno();
304 		path_got = NULL;
305 		goto done;
306 	}
307 
308 	if (asprintf(&path_lock, "%s/%s", path_got, GOT_WORKTREE_LOCK) == -1) {
309 		err = got_error_from_errno();
310 		path_lock = NULL;
311 		goto done;
312 	}
313 
314 	fd = open(path_lock, O_RDWR | O_EXLOCK | O_NONBLOCK);
315 	if (fd == -1) {
316 		err = (errno == EWOULDBLOCK ? got_error(GOT_ERR_WORKTREE_BUSY)
317 		    : got_error_from_errno());
318 		goto done;
319 	}
320 
321 	err = read_meta_file(&formatstr, path_got, GOT_WORKTREE_FORMAT);
322 	if (err)
323 		goto done;
324 
325 	version = strtonum(formatstr, 1, INT_MAX, &errstr);
326 	if (errstr) {
327 		err = got_error(GOT_ERR_WORKTREE_META);
328 		goto done;
329 	}
330 	if (version != GOT_WORKTREE_FORMAT_VERSION) {
331 		err = got_error(GOT_ERR_WORKTREE_VERS);
332 		goto done;
333 	}
334 
335 	*worktree = calloc(1, sizeof(**worktree));
336 	if (*worktree == NULL) {
337 		err = got_error_from_errno();
338 		goto done;
339 	}
340 	(*worktree)->lockfd = -1;
341 
342 	(*worktree)->root_path = strdup(path);
343 	if ((*worktree)->root_path == NULL) {
344 		err = got_error_from_errno();
345 		goto done;
346 	}
347 	err = read_meta_file(&(*worktree)->repo_path, path_got,
348 	    GOT_WORKTREE_REPOSITORY);
349 	if (err)
350 		goto done;
351 
352 	err = read_meta_file(&(*worktree)->path_prefix, path_got,
353 	    GOT_WORKTREE_PATH_PREFIX);
354 	if (err)
355 		goto done;
356 
357 	err = read_meta_file(&base_commit_id_str, path_got,
358 	    GOT_WORKTREE_BASE_COMMIT);
359 	if (err)
360 		goto done;
361 
362 	err = got_repo_open(&repo, (*worktree)->repo_path);
363 	if (err)
364 		goto done;
365 
366 	err = got_object_resolve_id_str(&(*worktree)->base_commit_id, repo,
367 	    base_commit_id_str);
368 	if (err)
369 		goto done;
370 
371 	err = read_meta_file(&head_ref_str, path_got, GOT_WORKTREE_HEAD_REF);
372 	if (err)
373 		goto done;
374 
375 	err = got_ref_open(&(*worktree)->head_ref, repo, head_ref_str);
376 done:
377 	if (repo)
378 		got_repo_close(repo);
379 	free(path_got);
380 	free(path_lock);
381 	free(head_ref_str);
382 	free(base_commit_id_str);
383 	if (err) {
384 		if (fd != -1)
385 			close(fd);
386 		if (*worktree != NULL)
387 			got_worktree_close(*worktree);
388 		*worktree = NULL;
389 	} else
390 		(*worktree)->lockfd = fd;
391 
392 	return err;
393 }
394 
395 const struct got_error *
396 got_worktree_open(struct got_worktree **worktree, const char *path)
397 {
398 	const struct got_error *err = NULL;
399 
400 	do {
401 		err = open_worktree(worktree, path);
402 		if (err && (err->code != GOT_ERR_ERRNO && errno != ENOENT))
403 			return err;
404 		if (*worktree)
405 			return NULL;
406 		path = dirname(path);
407 		if (path == NULL)
408 			return got_error_from_errno();
409 	} while (!((path[0] == '.' || path[0] == '/') && path[1] == '\0'));
410 
411 	return got_error(GOT_ERR_NOT_WORKTREE);
412 }
413 
414 void
415 got_worktree_close(struct got_worktree *worktree)
416 {
417 	free(worktree->root_path);
418 	free(worktree->repo_path);
419 	free(worktree->path_prefix);
420 	free(worktree->base_commit_id);
421 	if (worktree->head_ref)
422 		got_ref_close(worktree->head_ref);
423 	if (worktree->lockfd != -1)
424 		close(worktree->lockfd);
425 	free(worktree);
426 }
427 
428 const char *
429 got_worktree_get_root_path(struct got_worktree *worktree)
430 {
431 	return worktree->root_path;
432 }
433 
434 const char *
435 got_worktree_get_repo_path(struct got_worktree *worktree)
436 {
437 	return worktree->repo_path;
438 }
439 
440 const char *
441 got_worktree_get_path_prefix(struct got_worktree *worktree)
442 {
443 	return worktree->path_prefix;
444 }
445 
446 const struct got_error *
447 got_worktree_match_path_prefix(int *match, struct got_worktree *worktree,
448     const char *path_prefix)
449 {
450 	char *absprefix = NULL;
451 
452 	if (!got_path_is_absolute(path_prefix)) {
453 		if (asprintf(&absprefix, "/%s", path_prefix) == -1)
454 			return got_error_from_errno();
455 	}
456 	*match = (strcmp(absprefix ? absprefix : path_prefix,
457 	    worktree->path_prefix) == 0);
458 	free(absprefix);
459 	return NULL;
460 }
461 
462 char *
463 got_worktree_get_head_ref_name(struct got_worktree *worktree)
464 {
465 	return got_ref_to_str(worktree->head_ref);
466 }
467 
468 struct got_reference *
469 got_worktree_get_head_ref(struct got_worktree *worktree)
470 {
471 	return got_ref_dup(worktree->head_ref);
472 }
473 
474 struct got_object_id *
475 got_worktree_get_base_commit_id(struct got_worktree *worktree)
476 {
477 	return worktree->base_commit_id;
478 }
479 
480 const struct got_error *
481 got_worktree_set_base_commit_id(struct got_worktree *worktree,
482     struct got_repository *repo, struct got_object_id *commit_id)
483 {
484 	const struct got_error *err;
485 	struct got_object *obj = NULL;
486 	char *id_str = NULL;
487 	char *path_got = NULL;
488 
489 	if (asprintf(&path_got, "%s/%s", worktree->root_path,
490 	    GOT_WORKTREE_GOT_DIR) == -1) {
491 		err = got_error_from_errno();
492 		path_got = NULL;
493 		goto done;
494 	}
495 
496 	err = got_object_open(&obj, repo, commit_id);
497 	if (err)
498 		return err;
499 
500 	if (obj->type != GOT_OBJ_TYPE_COMMIT) {
501 		err = got_error(GOT_ERR_OBJ_TYPE);
502 		goto done;
503 	}
504 
505 	/* Record our base commit. */
506 	err = got_object_id_str(&id_str, commit_id);
507 	if (err)
508 		goto done;
509 	err = update_meta_file(path_got, GOT_WORKTREE_BASE_COMMIT, id_str);
510 	if (err)
511 		goto done;
512 
513 	free(worktree->base_commit_id);
514 	worktree->base_commit_id = got_object_id_dup(commit_id);
515 	if (worktree->base_commit_id == NULL) {
516 		err = got_error_from_errno();
517 		goto done;
518 	}
519 done:
520 	if (obj)
521 		got_object_close(obj);
522 	free(id_str);
523 	free(path_got);
524 	return err;
525 }
526 
527 static const struct got_error *
528 lock_worktree(struct got_worktree *worktree, int operation)
529 {
530 	if (flock(worktree->lockfd, operation | LOCK_NB) == -1)
531 		return (errno == EWOULDBLOCK ? got_error(GOT_ERR_WORKTREE_BUSY)
532 		    : got_error_from_errno());
533 	return NULL;
534 }
535 
536 static const struct got_error *
537 make_parent_dirs(const char *abspath)
538 {
539 	const struct got_error *err = NULL;
540 
541 	char *parent = dirname(abspath);
542 	if (parent == NULL)
543 		return NULL;
544 
545 	if (mkdir(parent, GOT_DEFAULT_DIR_MODE) == -1) {
546 		if (errno == ENOENT) {
547 			err = make_parent_dirs(parent);
548 			if (err)
549 				return err;
550 			if (mkdir(parent, GOT_DEFAULT_DIR_MODE) == -1)
551 				return got_error_from_errno();
552 		} else
553 			err = got_error_from_errno();
554 	}
555 
556 	return err;
557 }
558 
559 static const struct got_error *
560 add_dir_on_disk(struct got_worktree *worktree, const char *path)
561 {
562 	const struct got_error *err = NULL;
563 	char *abspath;
564 
565 	if (asprintf(&abspath, "%s/%s", worktree->root_path, path) == -1)
566 		return got_error_from_errno();
567 
568 	/* XXX queue work rather than editing disk directly? */
569 	if (mkdir(abspath, GOT_DEFAULT_DIR_MODE) == -1) {
570 		struct stat sb;
571 
572 		if (errno == EEXIST) {
573 			if (lstat(abspath, &sb) == -1) {
574 				err = got_error_from_errno();
575 				goto done;
576 			}
577 
578 			if (!S_ISDIR(sb.st_mode)) {
579 				/* TODO directory is obstructed; do something */
580 				err = got_error(GOT_ERR_FILE_OBSTRUCTED);
581 				goto done;
582 			}
583 
584 			return NULL;
585 		} else if (errno == ENOENT) {
586 			err = make_parent_dirs(abspath);
587 			if (err)
588 				goto done;
589 			if (mkdir(abspath, GOT_DEFAULT_DIR_MODE) == -1)
590 				err = got_error_from_errno();
591 		} else
592 			err = got_error_from_errno();
593 	}
594 
595 done:
596 	free(abspath);
597 	return err;
598 }
599 
600 /*
601  * Perform a 3-way merge where the file's version in the file index (blob2)
602  * acts as the common ancestor, the incoming blob (blob1) acts as the first
603  * derived version, and the file on disk acts as the second derived version.
604  */
605 static const struct got_error *
606 merge_blob(struct got_worktree *worktree, struct got_fileindex *fileindex,
607    struct got_fileindex_entry *ie, const char *ondisk_path, const char *path,
608    uint16_t te_mode, uint16_t st_mode, struct got_blob_object *blob1,
609    struct got_repository *repo,
610    got_worktree_checkout_cb progress_cb, void *progress_arg)
611 {
612 	const struct got_error *err = NULL;
613 	int merged_fd = -1;
614 	struct got_blob_object *blob2 = NULL;
615 	FILE *f1 = NULL, *f2 = NULL;
616 	char *blob1_path = NULL, *blob2_path = NULL;
617 	char *merged_path = NULL;
618 	struct got_object_id id2;
619 	char *id_str = NULL;
620 	char *label1 = NULL;
621 	int overlapcnt = 0;
622 
623 	err = got_opentemp_named_fd(&merged_path, &merged_fd, "/tmp/got-merged");
624 	if (err)
625 		return err;
626 	err = got_opentemp_named(&blob1_path, &f1, "/tmp/got-merge-blob1");
627 	if (err)
628 		goto done;
629 	err = got_object_blob_dump_to_file(NULL, NULL, f1, blob1);
630 	if (err)
631 		goto done;
632 
633 	err = got_opentemp_named(&blob2_path, &f2, "/tmp/got-merge-blob2");
634 	if (err)
635 		goto done;
636 
637 	memcpy(id2.sha1, ie->blob_sha1, SHA1_DIGEST_LENGTH);
638 	err = got_object_open_as_blob(&blob2, repo, &id2, 8192);
639 	if (err)
640 		goto done;
641 	err = got_object_blob_dump_to_file(NULL, NULL, f2, blob2);
642 	if (err)
643 		goto done;
644 
645 	err = got_object_id_str(&id_str, worktree->base_commit_id);
646 	if (err)
647 		goto done;
648 	if (asprintf(&label1, "commit %s", id_str) == -1) {
649 		err = got_error_from_errno();
650 		goto done;
651 	}
652 
653 	err = got_merge_diff3(&overlapcnt, merged_fd, blob1_path,
654 	    blob2_path, ondisk_path, label1, path);
655 	if (err)
656 		goto done;
657 
658 	(*progress_cb)(progress_arg,
659 	    overlapcnt > 0 ? GOT_STATUS_CONFLICT : GOT_STATUS_MERGE, path);
660 
661 
662 	fsync(merged_fd);
663 
664 	if (rename(merged_path, ondisk_path) != 0) {
665 		err = got_error_from_errno();
666 		goto done;
667 	}
668 
669 	/*
670 	 * Do not update timestamps of already modified files. Otherwise,
671 	 * the status walk would treat them as unmodified files again.
672 	 */
673 	err = got_fileindex_entry_update(ie, ondisk_path,
674 	    blob1->id.sha1, worktree->base_commit_id->sha1, 0);
675 done:
676 	if (merged_fd != -1)
677 		close(merged_fd);
678 	if (f1)
679 		fclose(f1);
680 	if (f2)
681 		fclose(f2);
682 	if (blob2)
683 		got_object_blob_close(blob2);
684 	free(merged_path);
685 	if (blob1_path) {
686 		unlink(blob1_path);
687 		free(blob1_path);
688 	}
689 	if (blob2_path) {
690 		unlink(blob2_path);
691 		free(blob2_path);
692 	}
693 	free(id_str);
694 	free(label1);
695 	return err;
696 }
697 
698 static const struct got_error *
699 install_blob(struct got_worktree *worktree, struct got_fileindex *fileindex,
700    struct got_fileindex_entry *entry, const char *ondisk_path, const char *path,
701    uint16_t te_mode, uint16_t st_mode, struct got_blob_object *blob,
702    int restoring_missing_file, struct got_repository *repo,
703    got_worktree_checkout_cb progress_cb, void *progress_arg)
704 {
705 	const struct got_error *err = NULL;
706 	int fd = -1;
707 	size_t len, hdrlen;
708 	int update = 0;
709 	char *tmppath = NULL;
710 
711 	fd = open(ondisk_path, O_RDWR | O_CREAT | O_EXCL | O_NOFOLLOW,
712 	    GOT_DEFAULT_FILE_MODE);
713 	if (fd == -1) {
714 		if (errno == ENOENT) {
715 			char *parent = dirname(path);
716 			if (parent == NULL)
717 				return got_error_from_errno();
718 			err = add_dir_on_disk(worktree, parent);
719 			if (err)
720 				return err;
721 			fd = open(ondisk_path,
722 			    O_RDWR | O_CREAT | O_EXCL | O_NOFOLLOW,
723 			    GOT_DEFAULT_FILE_MODE);
724 			if (fd == -1)
725 				return got_error_from_errno();
726 		} else if (errno == EEXIST) {
727 			if (!S_ISREG(st_mode)) {
728 				/* TODO file is obstructed; do something */
729 				err = got_error(GOT_ERR_FILE_OBSTRUCTED);
730 				goto done;
731 			} else {
732 				err = got_opentemp_named_fd(&tmppath, &fd,
733 				    ondisk_path);
734 				if (err)
735 					goto done;
736 				update = 1;
737 			}
738 		} else
739 			return got_error_from_errno();
740 	}
741 
742 	if (restoring_missing_file)
743 		(*progress_cb)(progress_arg, GOT_STATUS_MISSING, path);
744 	else
745 		(*progress_cb)(progress_arg,
746 		    update ? GOT_STATUS_UPDATE : GOT_STATUS_ADD, path);
747 
748 	hdrlen = got_object_blob_get_hdrlen(blob);
749 	do {
750 		const uint8_t *buf = got_object_blob_get_read_buf(blob);
751 		err = got_object_blob_read_block(&len, blob);
752 		if (err)
753 			break;
754 		if (len > 0) {
755 			/* Skip blob object header first time around. */
756 			ssize_t outlen = write(fd, buf + hdrlen, len - hdrlen);
757 			if (outlen == -1) {
758 				err = got_error_from_errno();
759 				goto done;
760 			} else if (outlen != len - hdrlen) {
761 				err = got_error(GOT_ERR_IO);
762 				goto done;
763 			}
764 			hdrlen = 0;
765 		}
766 	} while (len != 0);
767 
768 	fsync(fd);
769 
770 	if (update) {
771 		if (rename(tmppath, ondisk_path) != 0) {
772 			err = got_error_from_errno();
773 			goto done;
774 		}
775 	}
776 
777 	if (te_mode & S_IXUSR) {
778 		if (chmod(ondisk_path, st_mode | S_IXUSR) == -1) {
779 			err = got_error_from_errno();
780 			goto done;
781 		}
782 	} else {
783 		if (chmod(ondisk_path, st_mode & ~S_IXUSR) == -1) {
784 			err = got_error_from_errno();
785 			goto done;
786 		}
787 	}
788 
789 	if (entry == NULL)
790 		entry = got_fileindex_entry_get(fileindex, path);
791 	if (entry)
792 		err = got_fileindex_entry_update(entry, ondisk_path,
793 		    blob->id.sha1, worktree->base_commit_id->sha1, 1);
794 	else {
795 		err = got_fileindex_entry_alloc(&entry, ondisk_path,
796 		    path, blob->id.sha1, worktree->base_commit_id->sha1);
797 		if (err)
798 			goto done;
799 		err = got_fileindex_entry_add(fileindex, entry);
800 	}
801 done:
802 	if (fd != -1)
803 		close(fd);
804 	free(tmppath);
805 	return err;
806 }
807 
808 static const struct got_error *
809 get_file_status(unsigned char *status, struct stat *sb,
810     struct got_fileindex_entry *ie, const char *abspath,
811     struct got_repository *repo)
812 {
813 	const struct got_error *err = NULL;
814 	struct got_object_id id;
815 	size_t hdrlen;
816 	FILE *f = NULL;
817 	uint8_t fbuf[8192];
818 	struct got_blob_object *blob = NULL;
819 	size_t flen, blen;
820 
821 	*status = GOT_STATUS_NO_CHANGE;
822 
823 	if (lstat(abspath, sb) == -1) {
824 		if (errno == ENOENT) {
825 			if (ie) {
826 				*status = GOT_STATUS_MISSING;
827 				sb->st_mode =
828 				    ((ie->mode >> GOT_FILEIDX_MODE_PERMS_SHIFT)
829 				    & (S_IRWXU | S_IRWXG | S_IRWXO));
830 			} else
831 				sb->st_mode = GOT_DEFAULT_FILE_MODE;
832 			return NULL;
833 		}
834 		return got_error_from_errno();
835 	}
836 
837 	if (!S_ISREG(sb->st_mode)) {
838 		*status = GOT_STATUS_OBSTRUCTED;
839 		return NULL;
840 	}
841 
842 	if (ie == NULL)
843 		return NULL;
844 
845 	if (ie->ctime_sec == sb->st_ctime &&
846 	    ie->ctime_nsec == sb->st_ctimensec &&
847 	    ie->mtime_sec == sb->st_mtime &&
848 	    ie->mtime_sec == sb->st_mtime &&
849 	    ie->mtime_nsec == sb->st_mtimensec &&
850 	    ie->size == (sb->st_size & 0xffffffff))
851 		return NULL;
852 
853 	memcpy(id.sha1, ie->blob_sha1, sizeof(id.sha1));
854 	err = got_object_open_as_blob(&blob, repo, &id, sizeof(fbuf));
855 	if (err)
856 		return err;
857 
858 	f = fopen(abspath, "r");
859 	if (f == NULL) {
860 		err = got_error_from_errno();
861 		goto done;
862 	}
863 	hdrlen = got_object_blob_get_hdrlen(blob);
864 	while (1) {
865 		const uint8_t *bbuf = got_object_blob_get_read_buf(blob);
866 		err = got_object_blob_read_block(&blen, blob);
867 		if (err)
868 			break;
869 		flen = fread(fbuf, 1, sizeof(fbuf), f);
870 		if (blen == 0) {
871 			if (flen != 0)
872 				*status = GOT_STATUS_MODIFY;
873 			break;
874 		} else if (flen == 0) {
875 			if (blen != 0)
876 				*status = GOT_STATUS_MODIFY;
877 			break;
878 		} else if (blen - hdrlen == flen) {
879 			/* Skip blob object header first time around. */
880 			if (memcmp(bbuf + hdrlen, fbuf, flen) != 0) {
881 				*status = GOT_STATUS_MODIFY;
882 				break;
883 			}
884 		} else {
885 			*status = GOT_STATUS_MODIFY;
886 			break;
887 		}
888 		hdrlen = 0;
889 	}
890 done:
891 	if (blob)
892 		got_object_blob_close(blob);
893 	if (f)
894 		fclose(f);
895 	return err;
896 }
897 
898 static const struct got_error *
899 update_blob(struct got_worktree *worktree,
900     struct got_fileindex *fileindex, struct got_fileindex_entry *ie,
901     struct got_tree_entry *te, const char *path,
902     struct got_repository *repo, got_worktree_checkout_cb progress_cb,
903     void *progress_arg, got_worktree_cancel_cb cancel_cb, void *cancel_arg)
904 {
905 	const struct got_error *err = NULL;
906 	struct got_blob_object *blob = NULL;
907 	char *ondisk_path;
908 	unsigned char status = GOT_STATUS_NO_CHANGE;
909 	struct stat sb;
910 
911 	if (asprintf(&ondisk_path, "%s/%s", worktree->root_path, path) == -1)
912 		return got_error_from_errno();
913 
914 	err = get_file_status(&status, &sb, ie, ondisk_path, repo);
915 	if (err)
916 		goto done;
917 
918 	if (status == GOT_STATUS_OBSTRUCTED) {
919 		(*progress_cb)(progress_arg, status, path);
920 		goto done;
921 	}
922 
923 	if (ie && status != GOT_STATUS_MISSING) {
924 		if (memcmp(ie->commit_sha1, worktree->base_commit_id->sha1,
925 		    SHA1_DIGEST_LENGTH) == 0) {
926 			(*progress_cb)(progress_arg, GOT_STATUS_EXISTS,
927 			    path);
928 			goto done;
929 		}
930 		if (memcmp(ie->blob_sha1,
931 		    te->id->sha1, SHA1_DIGEST_LENGTH) == 0)
932 			goto done;
933 	}
934 
935 	err = got_object_open_as_blob(&blob, repo, te->id, 8192);
936 	if (err)
937 		goto done;
938 
939 	if (status == GOT_STATUS_MODIFY)
940 		err = merge_blob(worktree, fileindex, ie, ondisk_path, path,
941 		    te->mode, sb.st_mode, blob, repo, progress_cb,
942 		    progress_arg);
943 	else
944 		err = install_blob(worktree, fileindex, ie, ondisk_path, path,
945 		    te->mode, sb.st_mode, blob, status == GOT_STATUS_MISSING,
946 		    repo, progress_cb, progress_arg);
947 
948 	got_object_blob_close(blob);
949 done:
950 	free(ondisk_path);
951 	return err;
952 }
953 
954 static const struct got_error *
955 remove_ondisk_file(const char *root_path, const char *path)
956 {
957 	const struct got_error *err = NULL;
958 	char *ondisk_path = NULL;
959 
960 	if (asprintf(&ondisk_path, "%s/%s", root_path, path) == -1)
961 		return got_error_from_errno();
962 
963 	if (unlink(ondisk_path) == -1) {
964 		if (errno != ENOENT)
965 			err = got_error_from_errno();
966 	} else {
967 		char *parent = dirname(ondisk_path);
968 		while (parent && strcmp(parent, root_path) != 0) {
969 			if (rmdir(parent) == -1) {
970 				if (errno != ENOTEMPTY)
971 					err = got_error_from_errno();
972 				break;
973 			}
974 			parent = dirname(parent);
975 		}
976 	}
977 	free(ondisk_path);
978 	return err;
979 }
980 
981 struct diff_cb_arg {
982     struct got_fileindex *fileindex;
983     struct got_worktree *worktree;
984     struct got_repository *repo;
985     got_worktree_checkout_cb progress_cb;
986     void *progress_arg;
987     got_worktree_cancel_cb cancel_cb;
988     void *cancel_arg;
989 };
990 
991 static const struct got_error *
992 diff_old_new(void *arg, struct got_fileindex_entry *ie,
993     struct got_tree_entry *te, const char *parent_path)
994 {
995 	struct diff_cb_arg *a = arg;
996 
997 	return update_blob(a->worktree, a->fileindex, ie, te,
998 	    ie->path, a->repo, a->progress_cb, a->progress_arg,
999 	    a->cancel_cb, a->cancel_arg);
1000 }
1001 
1002 static const struct got_error *
1003 diff_old(void *arg, struct got_fileindex_entry *ie, const char *parent_path)
1004 {
1005 	const struct got_error *err;
1006 	struct diff_cb_arg *a = arg;
1007 
1008 	(*a->progress_cb)(a->progress_arg, GOT_STATUS_DELETE, ie->path);
1009 
1010 	err = remove_ondisk_file(a->worktree->root_path, ie->path);
1011 	if (err)
1012 		return err;
1013 	got_fileindex_entry_remove(a->fileindex, ie);
1014 	return NULL;
1015 }
1016 
1017 static const struct got_error *
1018 diff_new(void *arg, struct got_tree_entry *te, const char *parent_path)
1019 {
1020 	struct diff_cb_arg *a = arg;
1021 	const struct got_error *err;
1022 	char *path;
1023 
1024 	if (asprintf(&path, "%s%s%s", parent_path,
1025 	    parent_path[0] ? "/" : "", te->name)
1026 	    == -1)
1027 		return got_error_from_errno();
1028 
1029 	if (S_ISDIR(te->mode))
1030 		err = add_dir_on_disk(a->worktree, path);
1031 	else
1032 		err = update_blob(a->worktree, a->fileindex, NULL, te, path,
1033 		    a->repo, a->progress_cb, a->progress_arg,
1034 		    a->cancel_cb, a->cancel_arg);
1035 
1036 	free(path);
1037 	return err;
1038 }
1039 
1040 const struct got_error *
1041 got_worktree_checkout_files(struct got_worktree *worktree,
1042     struct got_repository *repo, got_worktree_checkout_cb progress_cb,
1043     void *progress_arg, got_worktree_cancel_cb cancel_cb, void *cancel_arg)
1044 {
1045 	const struct got_error *err = NULL, *unlockerr, *checkout_err = NULL;
1046 	struct got_commit_object *commit = NULL;
1047 	struct got_object_id *tree_id = NULL;
1048 	struct got_tree_object *tree = NULL;
1049 	char *fileindex_path = NULL, *new_fileindex_path = NULL;
1050 	struct got_fileindex *fileindex = NULL;
1051 	FILE *index = NULL, *new_index = NULL;
1052 	struct got_fileindex_diff_tree_cb diff_cb;
1053 	struct diff_cb_arg arg;
1054 
1055 	err = lock_worktree(worktree, LOCK_EX);
1056 	if (err)
1057 		return err;
1058 
1059 	fileindex = got_fileindex_alloc();
1060 	if (fileindex == NULL) {
1061 		err = got_error_from_errno();
1062 		goto done;
1063 	}
1064 
1065 	if (asprintf(&fileindex_path, "%s/%s/%s", worktree->root_path,
1066 	    GOT_WORKTREE_GOT_DIR, GOT_WORKTREE_FILE_INDEX) == -1) {
1067 		err = got_error_from_errno();
1068 		fileindex_path = NULL;
1069 		goto done;
1070 	}
1071 
1072 	/*
1073 	 * Read the file index.
1074 	 * Checking out files is supposed to be an idempotent operation.
1075 	 * If the on-disk file index is incomplete we will try to complete it.
1076 	 */
1077 	index = fopen(fileindex_path, "rb");
1078 	if (index == NULL) {
1079 		if (errno != ENOENT) {
1080 			err = got_error_from_errno();
1081 			goto done;
1082 		}
1083 	} else {
1084 		err = got_fileindex_read(fileindex, index);
1085 		fclose(index);
1086 		if (err)
1087 			goto done;
1088 	}
1089 
1090 	err = got_opentemp_named(&new_fileindex_path, &new_index,
1091 	    fileindex_path);
1092 	if (err)
1093 		goto done;
1094 
1095 	err = got_object_open_as_commit(&commit, repo,
1096 	   worktree->base_commit_id);
1097 	if (err)
1098 		goto done;
1099 
1100 	err = got_object_id_by_path(&tree_id, repo,
1101 	    worktree->base_commit_id, worktree->path_prefix);
1102 	if (err)
1103 		goto done;
1104 
1105 	err = got_object_open_as_tree(&tree, repo, tree_id);
1106 	if (err)
1107 		goto done;
1108 
1109 	diff_cb.diff_old_new = diff_old_new;
1110 	diff_cb.diff_old = diff_old;
1111 	diff_cb.diff_new = diff_new;
1112 	arg.fileindex = fileindex;
1113 	arg.worktree = worktree;
1114 	arg.repo = repo;
1115 	arg.progress_cb = progress_cb;
1116 	arg.progress_arg = progress_arg;
1117 	arg.cancel_cb = cancel_cb;
1118 	arg.cancel_arg = cancel_arg;
1119 	checkout_err = got_fileindex_diff_tree(fileindex, tree, repo,
1120 	    &diff_cb, &arg);
1121 
1122 	/* Try to sync the fileindex back to disk in any case. */
1123 	err = got_fileindex_write(fileindex, new_index);
1124 	if (err)
1125 		goto done;
1126 
1127 	if (rename(new_fileindex_path, fileindex_path) != 0) {
1128 		err = got_error_from_errno();
1129 		goto done;
1130 	}
1131 
1132 	free(new_fileindex_path);
1133 	new_fileindex_path = NULL;
1134 
1135 done:
1136 	if (tree)
1137 		got_object_tree_close(tree);
1138 	if (commit)
1139 		got_object_commit_close(commit);
1140 	if (new_fileindex_path)
1141 		unlink(new_fileindex_path);
1142 	if (new_index)
1143 		fclose(new_index);
1144 	free(new_fileindex_path);
1145 	free(fileindex_path);
1146 	got_fileindex_free(fileindex);
1147 	if (checkout_err)
1148 		err = checkout_err;
1149 	unlockerr = lock_worktree(worktree, LOCK_SH);
1150 	if (unlockerr && err == NULL)
1151 		err = unlockerr;
1152 	return err;
1153 }
1154 
1155 struct diff_dir_cb_arg {
1156     struct got_fileindex *fileindex;
1157     struct got_worktree *worktree;
1158     struct got_repository *repo;
1159     got_worktree_status_cb status_cb;
1160     void *status_arg;
1161     got_worktree_cancel_cb cancel_cb;
1162     void *cancel_arg;
1163 };
1164 
1165 static const struct got_error *
1166 status_old_new(void *arg, struct got_fileindex_entry *ie,
1167     struct dirent *de, const char *parent_path)
1168 {
1169 	const struct got_error *err = NULL;
1170 	struct diff_dir_cb_arg *a = arg;
1171 	char *abspath;
1172 	unsigned char status = GOT_STATUS_NO_CHANGE;
1173 	struct stat sb;
1174 
1175 	if (parent_path[0]) {
1176 		if (asprintf(&abspath, "%s/%s/%s", a->worktree->root_path,
1177 		    parent_path, de->d_name) == -1)
1178 			return got_error_from_errno();
1179 	} else {
1180 		if (asprintf(&abspath, "%s/%s", a->worktree->root_path,
1181 		    de->d_name) == -1)
1182 			return got_error_from_errno();
1183 	}
1184 
1185 	err = get_file_status(&status, &sb, ie, abspath, a->repo);
1186 	if (err == NULL && status != GOT_STATUS_NO_CHANGE) {
1187 		struct got_object_id id;
1188 		memcpy(id.sha1, ie->blob_sha1, SHA1_DIGEST_LENGTH);
1189 		err = (*a->status_cb)(a->status_arg, status, ie->path, &id);
1190 	}
1191 	free(abspath);
1192 	return err;
1193 }
1194 
1195 static const struct got_error *
1196 status_old(void *arg, struct got_fileindex_entry *ie, const char *parent_path)
1197 {
1198 	struct diff_dir_cb_arg *a = arg;
1199 	struct got_object_id id;
1200 	memcpy(id.sha1, ie->blob_sha1, SHA1_DIGEST_LENGTH);
1201 	return (*a->status_cb)(a->status_arg, GOT_STATUS_MISSING, ie->path,
1202 	    &id);
1203 }
1204 
1205 static const struct got_error *
1206 status_new(void *arg, struct dirent *de, const char *parent_path)
1207 {
1208 	const struct got_error *err = NULL;
1209 	struct diff_dir_cb_arg *a = arg;
1210 	char *path = NULL;
1211 
1212 	if (de->d_type == DT_DIR)
1213 		return NULL;
1214 
1215 	if (parent_path[0]) {
1216 		if (asprintf(&path, "%s/%s", parent_path, de->d_name) == -1)
1217 			return got_error_from_errno();
1218 	} else {
1219 		path = de->d_name;
1220 	}
1221 
1222 	err = (*a->status_cb)(a->status_arg, GOT_STATUS_UNVERSIONED, path,
1223 	    NULL);
1224 	if (parent_path[0])
1225 		free(path);
1226 	return err;
1227 }
1228 
1229 const struct got_error *
1230 got_worktree_status(struct got_worktree *worktree,
1231     struct got_repository *repo, got_worktree_status_cb status_cb,
1232     void *status_arg, got_worktree_cancel_cb cancel_cb, void *cancel_arg)
1233 {
1234 	const struct got_error *err = NULL;
1235 	DIR *workdir = NULL;
1236 	char *fileindex_path = NULL;
1237 	struct got_fileindex *fileindex = NULL;
1238 	FILE *index = NULL;
1239 	struct got_fileindex_diff_dir_cb fdiff_cb;
1240 	struct diff_dir_cb_arg arg;
1241 
1242 	fileindex = got_fileindex_alloc();
1243 	if (fileindex == NULL) {
1244 		err = got_error_from_errno();
1245 		goto done;
1246 	}
1247 
1248 	if (asprintf(&fileindex_path, "%s/%s/%s", worktree->root_path,
1249 	    GOT_WORKTREE_GOT_DIR, GOT_WORKTREE_FILE_INDEX) == -1) {
1250 		err = got_error_from_errno();
1251 		fileindex_path = NULL;
1252 		goto done;
1253 	}
1254 
1255 	index = fopen(fileindex_path, "rb");
1256 	if (index == NULL) {
1257 		if (errno != ENOENT) {
1258 			err = got_error_from_errno();
1259 			goto done;
1260 		}
1261 	} else {
1262 		err = got_fileindex_read(fileindex, index);
1263 		fclose(index);
1264 		if (err)
1265 			goto done;
1266 	}
1267 
1268 	workdir = opendir(worktree->root_path);
1269 	if (workdir == NULL) {
1270 		err = got_error_from_errno();
1271 		goto done;
1272 	}
1273 	fdiff_cb.diff_old_new = status_old_new;
1274 	fdiff_cb.diff_old = status_old;
1275 	fdiff_cb.diff_new = status_new;
1276 	arg.fileindex = fileindex;
1277 	arg.worktree = worktree;
1278 	arg.repo = repo;
1279 	arg.status_cb = status_cb;
1280 	arg.status_arg = status_arg;
1281 	arg.cancel_cb = cancel_cb;
1282 	arg.cancel_arg = cancel_arg;
1283 	err = got_fileindex_diff_dir(fileindex, workdir, worktree->root_path,
1284 	    repo, &fdiff_cb, &arg);
1285 done:
1286 	if (workdir)
1287 		closedir(workdir);
1288 	free(fileindex_path);
1289 	got_fileindex_free(fileindex);
1290 	return err;
1291 }