Game of Trees Repos

Blob

Date:: Mon Feb 11 11:32:53 2019 UTC
Message:: check for errors from close(2)
Actions:: History | Blame | Raw File
1 /*
2  * Copyright (c) 2018, 2019 Stefan Sperling <stsp@openbsd.org>
3  *
4  * Permission to use, copy, modify, and distribute this software for any
5  * purpose with or without fee is hereby granted, provided that the above
6  * copyright notice and this permission notice appear in all copies.
7  *
8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15  */
16 
17 #include <sys/stat.h>
18 #include <sys/limits.h>
19 #include <sys/queue.h>
20 #include <sys/tree.h>
21 
22 #include <dirent.h>
23 #include <stddef.h>
24 #include <string.h>
25 #include <stdio.h>
26 #include <stdlib.h>
27 #include <fcntl.h>
28 #include <errno.h>
29 #include <unistd.h>
30 #include <sha1.h>
31 #include <zlib.h>
32 #include <fnmatch.h>
33 #include <libgen.h>
34 
35 #include "got_error.h"
36 #include "got_repository.h"
37 #include "got_reference.h"
38 #include "got_object.h"
39 #include "got_worktree.h"
40 #include "got_opentemp.h"
41 
42 #include "got_lib_worktree.h"
43 #include "got_lib_path.h"
44 #include "got_lib_sha1.h"
45 #include "got_lib_fileindex.h"
46 #include "got_lib_inflate.h"
47 #include "got_lib_delta.h"
48 #include "got_lib_object.h"
49 #include "got_lib_diff.h"
50 
51 #ifndef MIN
52 #define	MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
53 #endif
54 
55 static const struct got_error *
56 create_meta_file(const char *path_got, const char *name, const char *content)
57 {
58 	const struct got_error *err = NULL;
59 	char *path;
60 	int fd = -1;
61 
62 	if (asprintf(&path, "%s/%s", path_got, name) == -1) {
63 		err = got_error_from_errno();
64 		path = NULL;
65 		goto done;
66 	}
67 
68 	fd = open(path, O_RDWR | O_CREAT | O_EXCL | O_NOFOLLOW,
69 	    GOT_DEFAULT_FILE_MODE);
70 	if (fd == -1) {
71 		err = got_error_from_errno();
72 		goto done;
73 	}
74 
75 	if (content) {
76 		int len = dprintf(fd, "%s\n", content);
77 		if (len != strlen(content) + 1) {
78 			err = got_error_from_errno();
79 			goto done;
80 		}
81 	}
82 
83 done:
84 	if (fd != -1 && close(fd) == -1 && err == NULL)
85 		err = got_error_from_errno();
86 	free(path);
87 	return err;
88 }
89 
90 static const struct got_error *
91 update_meta_file(const char *path_got, const char *name, const char *content)
92 {
93 	const struct got_error *err = NULL;
94 	FILE *tmpfile = NULL;
95 	char *tmppath = NULL;
96 	char *path = NULL;
97 
98 	if (asprintf(&path, "%s/%s", path_got, name) == -1) {
99 		err = got_error_from_errno();
100 		path = NULL;
101 		goto done;
102 	}
103 
104 	err = got_opentemp_named(&tmppath, &tmpfile, path);
105 	if (err)
106 		goto done;
107 
108 	if (content) {
109 		int len = fprintf(tmpfile, "%s\n", content);
110 		if (len != strlen(content) + 1) {
111 			err = got_error_from_errno();
112 			goto done;
113 		}
114 	}
115 
116 	if (rename(tmppath, path) != 0) {
117 		err = got_error_from_errno();
118 		goto done;
119 	}
120 
121 done:
122 	free(tmppath);
123 	if (fclose(tmpfile) != 0 && err == NULL)
124 		err = got_error_from_errno();
125 	return err;
126 }
127 
128 static const struct got_error *
129 read_meta_file(char **content, const char *path_got, const char *name)
130 {
131 	const struct got_error *err = NULL;
132 	char *path;
133 	int fd = -1;
134 	ssize_t n;
135 	struct stat sb;
136 
137 	*content = NULL;
138 
139 	if (asprintf(&path, "%s/%s", path_got, name) == -1) {
140 		err = got_error_from_errno();
141 		path = NULL;
142 		goto done;
143 	}
144 
145 	fd = open(path, O_RDONLY | O_NOFOLLOW);
146 	if (fd == -1) {
147 		err = got_error_from_errno();
148 		goto done;
149 	}
150 	if (flock(fd, LOCK_SH | LOCK_NB) == -1) {
151 		err = (errno == EWOULDBLOCK ? got_error(GOT_ERR_WORKTREE_BUSY)
152 		    : got_error_from_errno());
153 		goto done;
154 	}
155 
156 	stat(path, &sb);
157 	*content = calloc(1, sb.st_size);
158 	if (*content == NULL) {
159 		err = got_error_from_errno();
160 		goto done;
161 	}
162 
163 	n = read(fd, *content, sb.st_size);
164 	if (n != sb.st_size) {
165 		err = (n == -1 ? got_error_from_errno() :
166 		    got_error(GOT_ERR_WORKTREE_META));
167 		goto done;
168 	}
169 	if ((*content)[sb.st_size - 1] != '\n') {
170 		err = got_error(GOT_ERR_WORKTREE_META);
171 		goto done;
172 	}
173 	(*content)[sb.st_size - 1] = '\0';
174 
175 done:
176 	if (fd != -1 && close(fd) == -1 && err == NULL)
177 		err = got_error_from_errno();
178 	free(path);
179 	if (err) {
180 		free(*content);
181 		*content = NULL;
182 	}
183 	return err;
184 }
185 
186 const struct got_error *
187 got_worktree_init(const char *path, struct got_reference *head_ref,
188     const char *prefix, struct got_repository *repo)
189 {
190 	const struct got_error *err = NULL;
191 	struct got_object_id *commit_id = NULL;
192 	int obj_type;
193 	char *path_got = NULL;
194 	char *refstr = NULL;
195 	char *formatstr = NULL;
196 	char *absprefix = NULL;
197 	char *basestr = NULL;
198 
199 	err = got_ref_resolve(&commit_id, repo, head_ref);
200 	if (err)
201 		return err;
202 	err = got_object_get_type(&obj_type, repo, commit_id);
203 	if (err)
204 		return err;
205 	if (obj_type != GOT_OBJ_TYPE_COMMIT)
206 		return got_error(GOT_ERR_OBJ_TYPE);
207 
208 	if (!got_path_is_absolute(prefix)) {
209 		if (asprintf(&absprefix, "/%s", prefix) == -1)
210 			return got_error_from_errno();
211 	}
212 
213 	/* Create top-level directory (may already exist). */
214 	if (mkdir(path, GOT_DEFAULT_DIR_MODE) == -1 && errno != EEXIST) {
215 		err = got_error_from_errno();
216 		goto done;
217 	}
218 
219 	/* Create .got directory (may already exist). */
220 	if (asprintf(&path_got, "%s/%s", path, GOT_WORKTREE_GOT_DIR) == -1) {
221 		err = got_error_from_errno();
222 		goto done;
223 	}
224 	if (mkdir(path_got, GOT_DEFAULT_DIR_MODE) == -1 && errno != EEXIST) {
225 		err = got_error_from_errno();
226 		goto done;
227 	}
228 
229 	/* Create an empty lock file. */
230 	err = create_meta_file(path_got, GOT_WORKTREE_LOCK, NULL);
231 	if (err)
232 		goto done;
233 
234 	/* Create an empty file index. */
235 	err = create_meta_file(path_got, GOT_WORKTREE_FILE_INDEX, NULL);
236 	if (err)
237 		goto done;
238 
239 	/* Write the HEAD reference. */
240 	refstr = got_ref_to_str(head_ref);
241 	if (refstr == NULL) {
242 		err = got_error_from_errno();
243 		goto done;
244 	}
245 	err = create_meta_file(path_got, GOT_WORKTREE_HEAD_REF, refstr);
246 	if (err)
247 		goto done;
248 
249 	/* Record our base commit. */
250 	err = got_object_id_str(&basestr, commit_id);
251 	if (err)
252 		goto done;
253 	err = create_meta_file(path_got, GOT_WORKTREE_BASE_COMMIT, basestr);
254 	if (err)
255 		goto done;
256 
257 	/* Store path to repository. */
258 	err = create_meta_file(path_got, GOT_WORKTREE_REPOSITORY,
259 	    got_repo_get_path(repo));
260 	if (err)
261 		goto done;
262 
263 	/* Store in-repository path prefix. */
264 	err = create_meta_file(path_got, GOT_WORKTREE_PATH_PREFIX,
265 	    absprefix ? absprefix : prefix);
266 	if (err)
267 		goto done;
268 
269 	/* Stamp work tree with format file. */
270 	if (asprintf(&formatstr, "%d", GOT_WORKTREE_FORMAT_VERSION) == -1) {
271 		err = got_error_from_errno();
272 		goto done;
273 	}
274 	err = create_meta_file(path_got, GOT_WORKTREE_FORMAT, formatstr);
275 	if (err)
276 		goto done;
277 
278 done:
279 	free(commit_id);
280 	free(path_got);
281 	free(formatstr);
282 	free(refstr);
283 	free(absprefix);
284 	free(basestr);
285 	return err;
286 }
287 
288 static const struct got_error *
289 open_worktree(struct got_worktree **worktree, const char *path)
290 {
291 	const struct got_error *err = NULL;
292 	char *path_got;
293 	char *formatstr = NULL;
294 	char *path_lock = NULL;
295 	char *base_commit_id_str = NULL;
296 	char *head_ref_str = NULL;
297 	int version, fd = -1;
298 	const char *errstr;
299 	struct got_repository *repo = NULL;
300 
301 	*worktree = NULL;
302 
303 	if (asprintf(&path_got, "%s/%s", path, GOT_WORKTREE_GOT_DIR) == -1) {
304 		err = got_error_from_errno();
305 		path_got = NULL;
306 		goto done;
307 	}
308 
309 	if (asprintf(&path_lock, "%s/%s", path_got, GOT_WORKTREE_LOCK) == -1) {
310 		err = got_error_from_errno();
311 		path_lock = NULL;
312 		goto done;
313 	}
314 
315 	fd = open(path_lock, O_RDWR | O_EXLOCK | O_NONBLOCK);
316 	if (fd == -1) {
317 		err = (errno == EWOULDBLOCK ? got_error(GOT_ERR_WORKTREE_BUSY)
318 		    : got_error_from_errno());
319 		goto done;
320 	}
321 
322 	err = read_meta_file(&formatstr, path_got, GOT_WORKTREE_FORMAT);
323 	if (err)
324 		goto done;
325 
326 	version = strtonum(formatstr, 1, INT_MAX, &errstr);
327 	if (errstr) {
328 		err = got_error(GOT_ERR_WORKTREE_META);
329 		goto done;
330 	}
331 	if (version != GOT_WORKTREE_FORMAT_VERSION) {
332 		err = got_error(GOT_ERR_WORKTREE_VERS);
333 		goto done;
334 	}
335 
336 	*worktree = calloc(1, sizeof(**worktree));
337 	if (*worktree == NULL) {
338 		err = got_error_from_errno();
339 		goto done;
340 	}
341 	(*worktree)->lockfd = -1;
342 
343 	(*worktree)->root_path = strdup(path);
344 	if ((*worktree)->root_path == NULL) {
345 		err = got_error_from_errno();
346 		goto done;
347 	}
348 	err = read_meta_file(&(*worktree)->repo_path, path_got,
349 	    GOT_WORKTREE_REPOSITORY);
350 	if (err)
351 		goto done;
352 
353 	err = read_meta_file(&(*worktree)->path_prefix, path_got,
354 	    GOT_WORKTREE_PATH_PREFIX);
355 	if (err)
356 		goto done;
357 
358 	err = read_meta_file(&base_commit_id_str, path_got,
359 	    GOT_WORKTREE_BASE_COMMIT);
360 	if (err)
361 		goto done;
362 
363 	err = got_repo_open(&repo, (*worktree)->repo_path);
364 	if (err)
365 		goto done;
366 
367 	err = got_object_resolve_id_str(&(*worktree)->base_commit_id, repo,
368 	    base_commit_id_str);
369 	if (err)
370 		goto done;
371 
372 	err = read_meta_file(&head_ref_str, path_got, GOT_WORKTREE_HEAD_REF);
373 	if (err)
374 		goto done;
375 
376 	err = got_ref_open(&(*worktree)->head_ref, repo, head_ref_str);
377 done:
378 	if (repo)
379 		got_repo_close(repo);
380 	free(path_got);
381 	free(path_lock);
382 	free(head_ref_str);
383 	free(base_commit_id_str);
384 	if (err) {
385 		if (fd != -1)
386 			close(fd);
387 		if (*worktree != NULL)
388 			got_worktree_close(*worktree);
389 		*worktree = NULL;
390 	} else
391 		(*worktree)->lockfd = fd;
392 
393 	return err;
394 }
395 
396 const struct got_error *
397 got_worktree_open(struct got_worktree **worktree, const char *path)
398 {
399 	const struct got_error *err = NULL;
400 
401 	do {
402 		err = open_worktree(worktree, path);
403 		if (err && (err->code != GOT_ERR_ERRNO && errno != ENOENT))
404 			return err;
405 		if (*worktree)
406 			return NULL;
407 		path = dirname(path);
408 		if (path == NULL)
409 			return got_error_from_errno();
410 	} while (!((path[0] == '.' || path[0] == '/') && path[1] == '\0'));
411 
412 	return got_error(GOT_ERR_NOT_WORKTREE);
413 }
414 
415 const struct got_error *
416 got_worktree_close(struct got_worktree *worktree)
417 {
418 	const struct got_error *err = NULL;
419 	free(worktree->root_path);
420 	free(worktree->repo_path);
421 	free(worktree->path_prefix);
422 	free(worktree->base_commit_id);
423 	if (worktree->head_ref)
424 		got_ref_close(worktree->head_ref);
425 	if (worktree->lockfd != -1)
426 		if (close(worktree->lockfd) != 0)
427 			err = got_error_from_errno();
428 	free(worktree);
429 	return err;
430 }
431 
432 const char *
433 got_worktree_get_root_path(struct got_worktree *worktree)
434 {
435 	return worktree->root_path;
436 }
437 
438 const char *
439 got_worktree_get_repo_path(struct got_worktree *worktree)
440 {
441 	return worktree->repo_path;
442 }
443 
444 const char *
445 got_worktree_get_path_prefix(struct got_worktree *worktree)
446 {
447 	return worktree->path_prefix;
448 }
449 
450 const struct got_error *
451 got_worktree_match_path_prefix(int *match, struct got_worktree *worktree,
452     const char *path_prefix)
453 {
454 	char *absprefix = NULL;
455 
456 	if (!got_path_is_absolute(path_prefix)) {
457 		if (asprintf(&absprefix, "/%s", path_prefix) == -1)
458 			return got_error_from_errno();
459 	}
460 	*match = (strcmp(absprefix ? absprefix : path_prefix,
461 	    worktree->path_prefix) == 0);
462 	free(absprefix);
463 	return NULL;
464 }
465 
466 char *
467 got_worktree_get_head_ref_name(struct got_worktree *worktree)
468 {
469 	return got_ref_to_str(worktree->head_ref);
470 }
471 
472 struct got_reference *
473 got_worktree_get_head_ref(struct got_worktree *worktree)
474 {
475 	return got_ref_dup(worktree->head_ref);
476 }
477 
478 struct got_object_id *
479 got_worktree_get_base_commit_id(struct got_worktree *worktree)
480 {
481 	return worktree->base_commit_id;
482 }
483 
484 const struct got_error *
485 got_worktree_set_base_commit_id(struct got_worktree *worktree,
486     struct got_repository *repo, struct got_object_id *commit_id)
487 {
488 	const struct got_error *err;
489 	struct got_object *obj = NULL;
490 	char *id_str = NULL;
491 	char *path_got = NULL;
492 
493 	if (asprintf(&path_got, "%s/%s", worktree->root_path,
494 	    GOT_WORKTREE_GOT_DIR) == -1) {
495 		err = got_error_from_errno();
496 		path_got = NULL;
497 		goto done;
498 	}
499 
500 	err = got_object_open(&obj, repo, commit_id);
501 	if (err)
502 		return err;
503 
504 	if (obj->type != GOT_OBJ_TYPE_COMMIT) {
505 		err = got_error(GOT_ERR_OBJ_TYPE);
506 		goto done;
507 	}
508 
509 	/* Record our base commit. */
510 	err = got_object_id_str(&id_str, commit_id);
511 	if (err)
512 		goto done;
513 	err = update_meta_file(path_got, GOT_WORKTREE_BASE_COMMIT, id_str);
514 	if (err)
515 		goto done;
516 
517 	free(worktree->base_commit_id);
518 	worktree->base_commit_id = got_object_id_dup(commit_id);
519 	if (worktree->base_commit_id == NULL) {
520 		err = got_error_from_errno();
521 		goto done;
522 	}
523 done:
524 	if (obj)
525 		got_object_close(obj);
526 	free(id_str);
527 	free(path_got);
528 	return err;
529 }
530 
531 static const struct got_error *
532 lock_worktree(struct got_worktree *worktree, int operation)
533 {
534 	if (flock(worktree->lockfd, operation | LOCK_NB) == -1)
535 		return (errno == EWOULDBLOCK ? got_error(GOT_ERR_WORKTREE_BUSY)
536 		    : got_error_from_errno());
537 	return NULL;
538 }
539 
540 static const struct got_error *
541 make_parent_dirs(const char *abspath)
542 {
543 	const struct got_error *err = NULL;
544 
545 	char *parent = dirname(abspath);
546 	if (parent == NULL)
547 		return NULL;
548 
549 	if (mkdir(parent, GOT_DEFAULT_DIR_MODE) == -1) {
550 		if (errno == ENOENT) {
551 			err = make_parent_dirs(parent);
552 			if (err)
553 				return err;
554 			if (mkdir(parent, GOT_DEFAULT_DIR_MODE) == -1)
555 				return got_error_from_errno();
556 		} else
557 			err = got_error_from_errno();
558 	}
559 
560 	return err;
561 }
562 
563 static const struct got_error *
564 add_dir_on_disk(struct got_worktree *worktree, const char *path)
565 {
566 	const struct got_error *err = NULL;
567 	char *abspath;
568 
569 	if (asprintf(&abspath, "%s/%s", worktree->root_path, path) == -1)
570 		return got_error_from_errno();
571 
572 	/* XXX queue work rather than editing disk directly? */
573 	if (mkdir(abspath, GOT_DEFAULT_DIR_MODE) == -1) {
574 		struct stat sb;
575 
576 		if (errno == EEXIST) {
577 			if (lstat(abspath, &sb) == -1) {
578 				err = got_error_from_errno();
579 				goto done;
580 			}
581 
582 			if (!S_ISDIR(sb.st_mode)) {
583 				/* TODO directory is obstructed; do something */
584 				err = got_error(GOT_ERR_FILE_OBSTRUCTED);
585 				goto done;
586 			}
587 
588 			return NULL;
589 		} else if (errno == ENOENT) {
590 			err = make_parent_dirs(abspath);
591 			if (err)
592 				goto done;
593 			if (mkdir(abspath, GOT_DEFAULT_DIR_MODE) == -1)
594 				err = got_error_from_errno();
595 		} else
596 			err = got_error_from_errno();
597 	}
598 
599 done:
600 	free(abspath);
601 	return err;
602 }
603 
604 /*
605  * Perform a 3-way merge where the file's version in the file index (blob2)
606  * acts as the common ancestor, the incoming blob (blob1) acts as the first
607  * derived version, and the file on disk acts as the second derived version.
608  */
609 static const struct got_error *
610 merge_blob(struct got_worktree *worktree, struct got_fileindex *fileindex,
611    struct got_fileindex_entry *ie, const char *ondisk_path, const char *path,
612    uint16_t te_mode, uint16_t st_mode, struct got_blob_object *blob1,
613    struct got_repository *repo,
614    got_worktree_checkout_cb progress_cb, void *progress_arg)
615 {
616 	const struct got_error *err = NULL;
617 	int merged_fd = -1;
618 	struct got_blob_object *blob2 = NULL;
619 	FILE *f1 = NULL, *f2 = NULL;
620 	char *blob1_path = NULL, *blob2_path = NULL;
621 	char *merged_path = NULL;
622 	struct got_object_id id2;
623 	char *id_str = NULL;
624 	char *label1 = NULL;
625 	int overlapcnt = 0;
626 
627 	err = got_opentemp_named_fd(&merged_path, &merged_fd, "/tmp/got-merged");
628 	if (err)
629 		return err;
630 	err = got_opentemp_named(&blob1_path, &f1, "/tmp/got-merge-blob1");
631 	if (err)
632 		goto done;
633 	err = got_object_blob_dump_to_file(NULL, NULL, f1, blob1);
634 	if (err)
635 		goto done;
636 
637 	err = got_opentemp_named(&blob2_path, &f2, "/tmp/got-merge-blob2");
638 	if (err)
639 		goto done;
640 
641 	memcpy(id2.sha1, ie->blob_sha1, SHA1_DIGEST_LENGTH);
642 	err = got_object_open_as_blob(&blob2, repo, &id2, 8192);
643 	if (err)
644 		goto done;
645 	err = got_object_blob_dump_to_file(NULL, NULL, f2, blob2);
646 	if (err)
647 		goto done;
648 
649 	err = got_object_id_str(&id_str, worktree->base_commit_id);
650 	if (err)
651 		goto done;
652 	if (asprintf(&label1, "commit %s", id_str) == -1) {
653 		err = got_error_from_errno();
654 		goto done;
655 	}
656 
657 	err = got_merge_diff3(&overlapcnt, merged_fd, blob1_path,
658 	    blob2_path, ondisk_path, label1, path);
659 	if (err)
660 		goto done;
661 
662 	(*progress_cb)(progress_arg,
663 	    overlapcnt > 0 ? GOT_STATUS_CONFLICT : GOT_STATUS_MERGE, path);
664 
665 
666 	fsync(merged_fd);
667 
668 	if (rename(merged_path, ondisk_path) != 0) {
669 		err = got_error_from_errno();
670 		goto done;
671 	}
672 
673 	/*
674 	 * Do not update timestamps of already modified files. Otherwise,
675 	 * the status walk would treat them as unmodified files again.
676 	 */
677 	err = got_fileindex_entry_update(ie, ondisk_path,
678 	    blob1->id.sha1, worktree->base_commit_id->sha1, 0);
679 done:
680 	if (merged_fd != -1 && close(merged_fd) != 0 && err == NULL)
681 		err = got_error_from_errno();
682 	if (f1 && fclose(f1) != 0 && err == NULL)
683 		err = got_error_from_errno();
684 	if (f2 && fclose(f2) != 0 && err == NULL)
685 		err = got_error_from_errno();
686 	if (blob2)
687 		got_object_blob_close(blob2);
688 	free(merged_path);
689 	if (blob1_path) {
690 		unlink(blob1_path);
691 		free(blob1_path);
692 	}
693 	if (blob2_path) {
694 		unlink(blob2_path);
695 		free(blob2_path);
696 	}
697 	free(id_str);
698 	free(label1);
699 	return err;
700 }
701 
702 static const struct got_error *
703 install_blob(struct got_worktree *worktree, struct got_fileindex *fileindex,
704    struct got_fileindex_entry *entry, const char *ondisk_path, const char *path,
705    uint16_t te_mode, uint16_t st_mode, struct got_blob_object *blob,
706    int restoring_missing_file, struct got_repository *repo,
707    got_worktree_checkout_cb progress_cb, void *progress_arg)
708 {
709 	const struct got_error *err = NULL;
710 	int fd = -1;
711 	size_t len, hdrlen;
712 	int update = 0;
713 	char *tmppath = NULL;
714 
715 	fd = open(ondisk_path, O_RDWR | O_CREAT | O_EXCL | O_NOFOLLOW,
716 	    GOT_DEFAULT_FILE_MODE);
717 	if (fd == -1) {
718 		if (errno == ENOENT) {
719 			char *parent = dirname(path);
720 			if (parent == NULL)
721 				return got_error_from_errno();
722 			err = add_dir_on_disk(worktree, parent);
723 			if (err)
724 				return err;
725 			fd = open(ondisk_path,
726 			    O_RDWR | O_CREAT | O_EXCL | O_NOFOLLOW,
727 			    GOT_DEFAULT_FILE_MODE);
728 			if (fd == -1)
729 				return got_error_from_errno();
730 		} else if (errno == EEXIST) {
731 			if (!S_ISREG(st_mode)) {
732 				/* TODO file is obstructed; do something */
733 				err = got_error(GOT_ERR_FILE_OBSTRUCTED);
734 				goto done;
735 			} else {
736 				err = got_opentemp_named_fd(&tmppath, &fd,
737 				    ondisk_path);
738 				if (err)
739 					goto done;
740 				update = 1;
741 			}
742 		} else
743 			return got_error_from_errno();
744 	}
745 
746 	if (restoring_missing_file)
747 		(*progress_cb)(progress_arg, GOT_STATUS_MISSING, path);
748 	else
749 		(*progress_cb)(progress_arg,
750 		    update ? GOT_STATUS_UPDATE : GOT_STATUS_ADD, path);
751 
752 	hdrlen = got_object_blob_get_hdrlen(blob);
753 	do {
754 		const uint8_t *buf = got_object_blob_get_read_buf(blob);
755 		err = got_object_blob_read_block(&len, blob);
756 		if (err)
757 			break;
758 		if (len > 0) {
759 			/* Skip blob object header first time around. */
760 			ssize_t outlen = write(fd, buf + hdrlen, len - hdrlen);
761 			if (outlen == -1) {
762 				err = got_error_from_errno();
763 				goto done;
764 			} else if (outlen != len - hdrlen) {
765 				err = got_error(GOT_ERR_IO);
766 				goto done;
767 			}
768 			hdrlen = 0;
769 		}
770 	} while (len != 0);
771 
772 	fsync(fd);
773 
774 	if (update) {
775 		if (rename(tmppath, ondisk_path) != 0) {
776 			err = got_error_from_errno();
777 			goto done;
778 		}
779 	}
780 
781 	if (te_mode & S_IXUSR) {
782 		if (chmod(ondisk_path, st_mode | S_IXUSR) == -1) {
783 			err = got_error_from_errno();
784 			goto done;
785 		}
786 	} else {
787 		if (chmod(ondisk_path, st_mode & ~S_IXUSR) == -1) {
788 			err = got_error_from_errno();
789 			goto done;
790 		}
791 	}
792 
793 	if (entry == NULL)
794 		entry = got_fileindex_entry_get(fileindex, path);
795 	if (entry)
796 		err = got_fileindex_entry_update(entry, ondisk_path,
797 		    blob->id.sha1, worktree->base_commit_id->sha1, 1);
798 	else {
799 		err = got_fileindex_entry_alloc(&entry, ondisk_path,
800 		    path, blob->id.sha1, worktree->base_commit_id->sha1);
801 		if (err)
802 			goto done;
803 		err = got_fileindex_entry_add(fileindex, entry);
804 	}
805 done:
806 	if (fd != -1 && close(fd) != 0 && err == NULL)
807 		err = got_error_from_errno();
808 	free(tmppath);
809 	return err;
810 }
811 
812 static const struct got_error *
813 get_file_status(unsigned char *status, struct stat *sb,
814     struct got_fileindex_entry *ie, const char *abspath,
815     struct got_repository *repo)
816 {
817 	const struct got_error *err = NULL;
818 	struct got_object_id id;
819 	size_t hdrlen;
820 	FILE *f = NULL;
821 	uint8_t fbuf[8192];
822 	struct got_blob_object *blob = NULL;
823 	size_t flen, blen;
824 
825 	*status = GOT_STATUS_NO_CHANGE;
826 
827 	if (lstat(abspath, sb) == -1) {
828 		if (errno == ENOENT) {
829 			if (ie) {
830 				*status = GOT_STATUS_MISSING;
831 				sb->st_mode =
832 				    ((ie->mode >> GOT_FILEIDX_MODE_PERMS_SHIFT)
833 				    & (S_IRWXU | S_IRWXG | S_IRWXO));
834 			} else
835 				sb->st_mode = GOT_DEFAULT_FILE_MODE;
836 			return NULL;
837 		}
838 		return got_error_from_errno();
839 	}
840 
841 	if (!S_ISREG(sb->st_mode)) {
842 		*status = GOT_STATUS_OBSTRUCTED;
843 		return NULL;
844 	}
845 
846 	if (ie == NULL)
847 		return NULL;
848 
849 	if (ie->ctime_sec == sb->st_ctime &&
850 	    ie->ctime_nsec == sb->st_ctimensec &&
851 	    ie->mtime_sec == sb->st_mtime &&
852 	    ie->mtime_sec == sb->st_mtime &&
853 	    ie->mtime_nsec == sb->st_mtimensec &&
854 	    ie->size == (sb->st_size & 0xffffffff))
855 		return NULL;
856 
857 	memcpy(id.sha1, ie->blob_sha1, sizeof(id.sha1));
858 	err = got_object_open_as_blob(&blob, repo, &id, sizeof(fbuf));
859 	if (err)
860 		return err;
861 
862 	f = fopen(abspath, "r");
863 	if (f == NULL) {
864 		err = got_error_from_errno();
865 		goto done;
866 	}
867 	hdrlen = got_object_blob_get_hdrlen(blob);
868 	while (1) {
869 		const uint8_t *bbuf = got_object_blob_get_read_buf(blob);
870 		err = got_object_blob_read_block(&blen, blob);
871 		if (err)
872 			break;
873 		flen = fread(fbuf, 1, sizeof(fbuf), f);
874 		if (blen == 0) {
875 			if (flen != 0)
876 				*status = GOT_STATUS_MODIFY;
877 			break;
878 		} else if (flen == 0) {
879 			if (blen != 0)
880 				*status = GOT_STATUS_MODIFY;
881 			break;
882 		} else if (blen - hdrlen == flen) {
883 			/* Skip blob object header first time around. */
884 			if (memcmp(bbuf + hdrlen, fbuf, flen) != 0) {
885 				*status = GOT_STATUS_MODIFY;
886 				break;
887 			}
888 		} else {
889 			*status = GOT_STATUS_MODIFY;
890 			break;
891 		}
892 		hdrlen = 0;
893 	}
894 done:
895 	if (blob)
896 		got_object_blob_close(blob);
897 	if (f)
898 		fclose(f);
899 	return err;
900 }
901 
902 static const struct got_error *
903 update_blob(struct got_worktree *worktree,
904     struct got_fileindex *fileindex, struct got_fileindex_entry *ie,
905     struct got_tree_entry *te, const char *path,
906     struct got_repository *repo, got_worktree_checkout_cb progress_cb,
907     void *progress_arg, got_worktree_cancel_cb cancel_cb, void *cancel_arg)
908 {
909 	const struct got_error *err = NULL;
910 	struct got_blob_object *blob = NULL;
911 	char *ondisk_path;
912 	unsigned char status = GOT_STATUS_NO_CHANGE;
913 	struct stat sb;
914 
915 	if (asprintf(&ondisk_path, "%s/%s", worktree->root_path, path) == -1)
916 		return got_error_from_errno();
917 
918 	err = get_file_status(&status, &sb, ie, ondisk_path, repo);
919 	if (err)
920 		goto done;
921 
922 	if (status == GOT_STATUS_OBSTRUCTED) {
923 		(*progress_cb)(progress_arg, status, path);
924 		goto done;
925 	}
926 
927 	if (ie && status != GOT_STATUS_MISSING) {
928 		if (memcmp(ie->commit_sha1, worktree->base_commit_id->sha1,
929 		    SHA1_DIGEST_LENGTH) == 0) {
930 			(*progress_cb)(progress_arg, GOT_STATUS_EXISTS,
931 			    path);
932 			goto done;
933 		}
934 		if (memcmp(ie->blob_sha1,
935 		    te->id->sha1, SHA1_DIGEST_LENGTH) == 0)
936 			goto done;
937 	}
938 
939 	err = got_object_open_as_blob(&blob, repo, te->id, 8192);
940 	if (err)
941 		goto done;
942 
943 	if (status == GOT_STATUS_MODIFY)
944 		err = merge_blob(worktree, fileindex, ie, ondisk_path, path,
945 		    te->mode, sb.st_mode, blob, repo, progress_cb,
946 		    progress_arg);
947 	else
948 		err = install_blob(worktree, fileindex, ie, ondisk_path, path,
949 		    te->mode, sb.st_mode, blob, status == GOT_STATUS_MISSING,
950 		    repo, progress_cb, progress_arg);
951 
952 	got_object_blob_close(blob);
953 done:
954 	free(ondisk_path);
955 	return err;
956 }
957 
958 static const struct got_error *
959 remove_ondisk_file(const char *root_path, const char *path)
960 {
961 	const struct got_error *err = NULL;
962 	char *ondisk_path = NULL;
963 
964 	if (asprintf(&ondisk_path, "%s/%s", root_path, path) == -1)
965 		return got_error_from_errno();
966 
967 	if (unlink(ondisk_path) == -1) {
968 		if (errno != ENOENT)
969 			err = got_error_from_errno();
970 	} else {
971 		char *parent = dirname(ondisk_path);
972 		while (parent && strcmp(parent, root_path) != 0) {
973 			if (rmdir(parent) == -1) {
974 				if (errno != ENOTEMPTY)
975 					err = got_error_from_errno();
976 				break;
977 			}
978 			parent = dirname(parent);
979 		}
980 	}
981 	free(ondisk_path);
982 	return err;
983 }
984 
985 struct diff_cb_arg {
986     struct got_fileindex *fileindex;
987     struct got_worktree *worktree;
988     struct got_repository *repo;
989     got_worktree_checkout_cb progress_cb;
990     void *progress_arg;
991     got_worktree_cancel_cb cancel_cb;
992     void *cancel_arg;
993 };
994 
995 static const struct got_error *
996 diff_old_new(void *arg, struct got_fileindex_entry *ie,
997     struct got_tree_entry *te, const char *parent_path)
998 {
999 	struct diff_cb_arg *a = arg;
1000 
1001 	return update_blob(a->worktree, a->fileindex, ie, te,
1002 	    ie->path, a->repo, a->progress_cb, a->progress_arg,
1003 	    a->cancel_cb, a->cancel_arg);
1004 }
1005 
1006 static const struct got_error *
1007 diff_old(void *arg, struct got_fileindex_entry *ie, const char *parent_path)
1008 {
1009 	const struct got_error *err;
1010 	struct diff_cb_arg *a = arg;
1011 
1012 	(*a->progress_cb)(a->progress_arg, GOT_STATUS_DELETE, ie->path);
1013 
1014 	err = remove_ondisk_file(a->worktree->root_path, ie->path);
1015 	if (err)
1016 		return err;
1017 	got_fileindex_entry_remove(a->fileindex, ie);
1018 	return NULL;
1019 }
1020 
1021 static const struct got_error *
1022 diff_new(void *arg, struct got_tree_entry *te, const char *parent_path)
1023 {
1024 	struct diff_cb_arg *a = arg;
1025 	const struct got_error *err;
1026 	char *path;
1027 
1028 	if (asprintf(&path, "%s%s%s", parent_path,
1029 	    parent_path[0] ? "/" : "", te->name)
1030 	    == -1)
1031 		return got_error_from_errno();
1032 
1033 	if (S_ISDIR(te->mode))
1034 		err = add_dir_on_disk(a->worktree, path);
1035 	else
1036 		err = update_blob(a->worktree, a->fileindex, NULL, te, path,
1037 		    a->repo, a->progress_cb, a->progress_arg,
1038 		    a->cancel_cb, a->cancel_arg);
1039 
1040 	free(path);
1041 	return err;
1042 }
1043 
1044 const struct got_error *
1045 got_worktree_checkout_files(struct got_worktree *worktree,
1046     struct got_repository *repo, got_worktree_checkout_cb progress_cb,
1047     void *progress_arg, got_worktree_cancel_cb cancel_cb, void *cancel_arg)
1048 {
1049 	const struct got_error *err = NULL, *unlockerr, *checkout_err = NULL;
1050 	struct got_commit_object *commit = NULL;
1051 	struct got_object_id *tree_id = NULL;
1052 	struct got_tree_object *tree = NULL;
1053 	char *fileindex_path = NULL, *new_fileindex_path = NULL;
1054 	struct got_fileindex *fileindex = NULL;
1055 	FILE *index = NULL, *new_index = NULL;
1056 	struct got_fileindex_diff_tree_cb diff_cb;
1057 	struct diff_cb_arg arg;
1058 
1059 	err = lock_worktree(worktree, LOCK_EX);
1060 	if (err)
1061 		return err;
1062 
1063 	fileindex = got_fileindex_alloc();
1064 	if (fileindex == NULL) {
1065 		err = got_error_from_errno();
1066 		goto done;
1067 	}
1068 
1069 	if (asprintf(&fileindex_path, "%s/%s/%s", worktree->root_path,
1070 	    GOT_WORKTREE_GOT_DIR, GOT_WORKTREE_FILE_INDEX) == -1) {
1071 		err = got_error_from_errno();
1072 		fileindex_path = NULL;
1073 		goto done;
1074 	}
1075 
1076 	/*
1077 	 * Read the file index.
1078 	 * Checking out files is supposed to be an idempotent operation.
1079 	 * If the on-disk file index is incomplete we will try to complete it.
1080 	 */
1081 	index = fopen(fileindex_path, "rb");
1082 	if (index == NULL) {
1083 		if (errno != ENOENT) {
1084 			err = got_error_from_errno();
1085 			goto done;
1086 		}
1087 	} else {
1088 		err = got_fileindex_read(fileindex, index);
1089 		fclose(index);
1090 		if (err)
1091 			goto done;
1092 	}
1093 
1094 	err = got_opentemp_named(&new_fileindex_path, &new_index,
1095 	    fileindex_path);
1096 	if (err)
1097 		goto done;
1098 
1099 	err = got_object_open_as_commit(&commit, repo,
1100 	   worktree->base_commit_id);
1101 	if (err)
1102 		goto done;
1103 
1104 	err = got_object_id_by_path(&tree_id, repo,
1105 	    worktree->base_commit_id, worktree->path_prefix);
1106 	if (err)
1107 		goto done;
1108 
1109 	err = got_object_open_as_tree(&tree, repo, tree_id);
1110 	if (err)
1111 		goto done;
1112 
1113 	diff_cb.diff_old_new = diff_old_new;
1114 	diff_cb.diff_old = diff_old;
1115 	diff_cb.diff_new = diff_new;
1116 	arg.fileindex = fileindex;
1117 	arg.worktree = worktree;
1118 	arg.repo = repo;
1119 	arg.progress_cb = progress_cb;
1120 	arg.progress_arg = progress_arg;
1121 	arg.cancel_cb = cancel_cb;
1122 	arg.cancel_arg = cancel_arg;
1123 	checkout_err = got_fileindex_diff_tree(fileindex, tree, repo,
1124 	    &diff_cb, &arg);
1125 
1126 	/* Try to sync the fileindex back to disk in any case. */
1127 	err = got_fileindex_write(fileindex, new_index);
1128 	if (err)
1129 		goto done;
1130 
1131 	if (rename(new_fileindex_path, fileindex_path) != 0) {
1132 		err = got_error_from_errno();
1133 		goto done;
1134 	}
1135 
1136 	free(new_fileindex_path);
1137 	new_fileindex_path = NULL;
1138 
1139 done:
1140 	if (tree)
1141 		got_object_tree_close(tree);
1142 	if (commit)
1143 		got_object_commit_close(commit);
1144 	if (new_fileindex_path)
1145 		unlink(new_fileindex_path);
1146 	if (new_index)
1147 		fclose(new_index);
1148 	free(new_fileindex_path);
1149 	free(fileindex_path);
1150 	got_fileindex_free(fileindex);
1151 	if (checkout_err)
1152 		err = checkout_err;
1153 	unlockerr = lock_worktree(worktree, LOCK_SH);
1154 	if (unlockerr && err == NULL)
1155 		err = unlockerr;
1156 	return err;
1157 }
1158 
1159 struct diff_dir_cb_arg {
1160     struct got_fileindex *fileindex;
1161     struct got_worktree *worktree;
1162     const char *status_path;
1163     size_t status_path_len;
1164     struct got_repository *repo;
1165     got_worktree_status_cb status_cb;
1166     void *status_arg;
1167     got_worktree_cancel_cb cancel_cb;
1168     void *cancel_arg;
1169 };
1170 
1171 static const struct got_error *
1172 report_file_status(struct got_fileindex_entry *ie, const char *abspath,
1173     got_worktree_status_cb status_cb, void *status_arg,
1174     struct got_repository *repo)
1175 {
1176 	const struct got_error *err = NULL;
1177 	unsigned char status = GOT_STATUS_NO_CHANGE;
1178 	struct stat sb;
1179 	struct got_object_id id;
1180 
1181 	err = get_file_status(&status, &sb, ie, abspath, repo);
1182 	if (err == NULL && status != GOT_STATUS_NO_CHANGE) {
1183 		memcpy(id.sha1, ie->blob_sha1, SHA1_DIGEST_LENGTH);
1184 		err = (*status_cb)(status_arg, status, ie->path, &id);
1185 	}
1186 	return err;
1187 }
1188 
1189 static const struct got_error *
1190 status_old_new(void *arg, struct got_fileindex_entry *ie,
1191     struct dirent *de, const char *parent_path)
1192 {
1193 	const struct got_error *err = NULL;
1194 	struct diff_dir_cb_arg *a = arg;
1195 	char *abspath;
1196 
1197 	if (got_path_cmp(parent_path, a->status_path) != 0 &&
1198 	    !got_path_is_child(parent_path, a->status_path, a->status_path_len))
1199 		return NULL;
1200 
1201 	if (parent_path[0]) {
1202 		if (asprintf(&abspath, "%s/%s/%s", a->worktree->root_path,
1203 		    parent_path, de->d_name) == -1)
1204 			return got_error_from_errno();
1205 	} else {
1206 		if (asprintf(&abspath, "%s/%s", a->worktree->root_path,
1207 		    de->d_name) == -1)
1208 			return got_error_from_errno();
1209 	}
1210 
1211 	err = report_file_status(ie, abspath, a->status_cb, a->status_arg,
1212 	    a->repo);
1213 	free(abspath);
1214 	return err;
1215 }
1216 
1217 static const struct got_error *
1218 status_old(void *arg, struct got_fileindex_entry *ie, const char *parent_path)
1219 {
1220 	struct diff_dir_cb_arg *a = arg;
1221 	struct got_object_id id;
1222 
1223 	if (!got_path_is_child(parent_path, a->status_path, a->status_path_len))
1224 		return NULL;
1225 
1226 	memcpy(id.sha1, ie->blob_sha1, SHA1_DIGEST_LENGTH);
1227 	return (*a->status_cb)(a->status_arg, GOT_STATUS_MISSING, ie->path,
1228 	    &id);
1229 }
1230 
1231 static const struct got_error *
1232 status_new(void *arg, struct dirent *de, const char *parent_path)
1233 {
1234 	const struct got_error *err = NULL;
1235 	struct diff_dir_cb_arg *a = arg;
1236 	char *path = NULL;
1237 
1238 	if (de->d_type == DT_DIR)
1239 		return NULL;
1240 
1241 	/* XXX ignore symlinks for now */
1242 	if (de->d_type == DT_LNK)
1243 		return NULL;
1244 
1245 	if (!got_path_is_child(parent_path, a->status_path, a->status_path_len))
1246 		return NULL;
1247 
1248 	if (parent_path[0]) {
1249 		if (asprintf(&path, "%s/%s", parent_path, de->d_name) == -1)
1250 			return got_error_from_errno();
1251 	} else {
1252 		path = de->d_name;
1253 	}
1254 
1255 	err = (*a->status_cb)(a->status_arg, GOT_STATUS_UNVERSIONED, path,
1256 	    NULL);
1257 	if (parent_path[0])
1258 		free(path);
1259 	return err;
1260 }
1261 
1262 const struct got_error *
1263 got_worktree_status(struct got_worktree *worktree, const char *path,
1264     struct got_repository *repo, got_worktree_status_cb status_cb,
1265     void *status_arg, got_worktree_cancel_cb cancel_cb, void *cancel_arg)
1266 {
1267 	const struct got_error *err = NULL;
1268 	DIR *workdir = NULL;
1269 	char *fileindex_path = NULL;
1270 	struct got_fileindex *fileindex = NULL;
1271 	FILE *index = NULL;
1272 	struct got_fileindex_diff_dir_cb fdiff_cb;
1273 	struct diff_dir_cb_arg arg;
1274 	char *ondisk_path = NULL;
1275 
1276 	fileindex = got_fileindex_alloc();
1277 	if (fileindex == NULL) {
1278 		err = got_error_from_errno();
1279 		goto done;
1280 	}
1281 
1282 	if (asprintf(&fileindex_path, "%s/%s/%s", worktree->root_path,
1283 	    GOT_WORKTREE_GOT_DIR, GOT_WORKTREE_FILE_INDEX) == -1) {
1284 		err = got_error_from_errno();
1285 		fileindex_path = NULL;
1286 		goto done;
1287 	}
1288 
1289 	index = fopen(fileindex_path, "rb");
1290 	if (index == NULL) {
1291 		if (errno != ENOENT) {
1292 			err = got_error_from_errno();
1293 			goto done;
1294 		}
1295 	} else {
1296 		err = got_fileindex_read(fileindex, index);
1297 		fclose(index);
1298 		if (err)
1299 			goto done;
1300 	}
1301 
1302 	if (asprintf(&ondisk_path, "%s%s%s",
1303 	    worktree->root_path, path[0] ? "/" : "", path) == -1) {
1304 		err = got_error_from_errno();
1305 		goto done;
1306 	}
1307 	workdir = opendir(ondisk_path);
1308 	if (workdir == NULL) {
1309 		if (errno == ENOTDIR) {
1310 			struct got_fileindex_entry *ie;
1311 			ie = got_fileindex_entry_get(fileindex, path);
1312 			if (ie == NULL) {
1313 				err = got_error(GOT_ERR_BAD_PATH);
1314 				goto done;
1315 			}
1316 			err = report_file_status(ie, ondisk_path,
1317 			    status_cb, status_arg, repo);
1318 			goto done;
1319 		} else {
1320 			err = got_error_from_errno();
1321 			goto done;
1322 		}
1323 	}
1324 	fdiff_cb.diff_old_new = status_old_new;
1325 	fdiff_cb.diff_old = status_old;
1326 	fdiff_cb.diff_new = status_new;
1327 	arg.fileindex = fileindex;
1328 	arg.worktree = worktree;
1329 	arg.status_path = path;
1330 	arg.status_path_len = strlen(path);
1331 	arg.repo = repo;
1332 	arg.status_cb = status_cb;
1333 	arg.status_arg = status_arg;
1334 	arg.cancel_cb = cancel_cb;
1335 	arg.cancel_arg = cancel_arg;
1336 	err = got_fileindex_diff_dir(fileindex, workdir, worktree->root_path,
1337 	    path, repo, &fdiff_cb, &arg);
1338 done:
1339 	if (workdir)
1340 		closedir(workdir);
1341 	free(ondisk_path);
1342 	free(fileindex_path);
1343 	got_fileindex_free(fileindex);
1344 	return err;
1345 }