Commit Diff

Commit:
de47d040c7458d8bddac77ba76a8a5e61aa8eb44
From:
Stefan Sperling <stsp@stsp.name>
Via:
Thomas Adam <thomas@xteddy.org>
Date:
Message:
cache a list of known pack index files when the repository is opened Avoids overhead due to readdir calls while searching a pack index. ok op@
Actions:
Patch | Tree
commit - eaf99875e8a296a7a1128d804339ed1935f8d2a0
commit + de47d040c7458d8bddac77ba76a8a5e61aa8eb44
blob - 798762e89557feb475f650e62e549abe41eb43c4
blob + 5fb86bb46e7a4329db5383980128ed5c1b627aec
--- lib/got_lib_repository.h
+++ lib/got_lib_repository.h
@@ -51,6 +51,8 @@ struct got_repository {
 	char *path_git_dir;
 	int gitdir_fd;
 
+	struct got_pathlist_head packidx_paths;
+
 	/* The pack index cache speeds up search for packed objects. */
 	struct got_packidx *packidx_cache[GOT_PACK_CACHE_SIZE];
 
blob - 138499bf96babfa8fcd49411e9ae49826a51ae96
blob + 96b95856b52b6f229cbee1d31aba3cdde0b726b3
--- lib/pack_create.c
+++ lib/pack_create.c
@@ -487,20 +487,14 @@ static const struct got_error *
 find_pack_for_reuse(struct got_packidx **best_packidx,
     struct got_repository *repo)
 {
-	const struct got_error *err;
-	struct got_pathlist_head packidx_paths;
+	const struct got_error *err = NULL;
 	struct got_pathlist_entry *pe;
 	const char *best_packidx_path = NULL;
 	int nobj_max = 0;
 
-	TAILQ_INIT(&packidx_paths);
 	*best_packidx = NULL;
 
-	err = got_repo_list_packidx(&packidx_paths, repo);
-	if (err)
-		return err;
-
-	TAILQ_FOREACH(pe, &packidx_paths, entry) {
+	TAILQ_FOREACH(pe, &repo->packidx_paths, entry) {
 		const char *path_packidx = pe->path;
 		struct got_packidx *packidx;
 		int nobj;
@@ -521,9 +515,6 @@ find_pack_for_reuse(struct got_packidx **best_packidx,
 		    repo);
 	}
 
-	TAILQ_FOREACH(pe, &packidx_paths, entry)
-		free((void *)pe->path);
-	got_pathlist_free(&packidx_paths);
 	return err;
 }
 
blob - a7c2e0f246cc3c3568df8293679707567e5ec8e1
blob + dc7435908d610d55270466c158a0dea7033d6b19
--- lib/repository.c
+++ lib/repository.c
@@ -664,6 +664,7 @@ got_repo_open(struct got_repository **repop, const cha
 	}
 
 	RB_INIT(&repo->packidx_bloom_filters);
+	TAILQ_INIT(&repo->packidx_paths);
 
 	for (i = 0; i < nitems(repo->privsep_children); i++) {
 		memset(&repo->privsep_children[i], 0,
@@ -744,6 +745,8 @@ got_repo_open(struct got_repository **repop, const cha
 			goto done;
 		}
 	}
+
+	err = got_repo_list_packidx(&repo->packidx_paths, repo);
 done:
 	if (err)
 		got_repo_close(repo);
@@ -758,6 +761,7 @@ got_repo_close(struct got_repository *repo)
 {
 	const struct got_error *err = NULL, *child_err;
 	struct got_packidx_bloom_filter *bf;
+	struct got_pathlist_entry *pe;
 	size_t i;
 
 	for (i = 0; i < repo->pack_cache_size; i++) {
@@ -818,6 +822,10 @@ got_repo_close(struct got_repository *repo)
 	for (i = 0; i < repo->nextensions; i++)
 		free(repo->extensions[i]);
 	free(repo->extensions);
+
+	TAILQ_FOREACH(pe, &repo->packidx_paths, entry)
+		free((void *)pe->path);
+	got_pathlist_free(&repo->packidx_paths);
 	free(repo);
 
 	return err;
@@ -1086,11 +1094,8 @@ got_repo_search_packidx(struct got_packidx **packidx, 
     struct got_repository *repo, struct got_object_id *id)
 {
 	const struct got_error *err;
-	DIR *packdir = NULL;
-	struct dirent *dent;
-	char *path_packidx;
+	struct got_pathlist_entry *pe;
 	size_t i;
-	int packdir_fd;
 
 	/* Search pack index cache. */
 	for (i = 0; i < repo->pack_cache_size; i++) {
@@ -1117,42 +1122,13 @@ got_repo_search_packidx(struct got_packidx **packidx, 
 	}
 	/* No luck. Search the filesystem. */
 
-	packdir_fd = openat(got_repo_get_fd(repo),
-	    GOT_OBJECTS_PACK_DIR, O_DIRECTORY | O_CLOEXEC);
-	if (packdir_fd == -1) {
-		if (errno == ENOENT)
-			err = got_error_no_obj(id);
-		else
-			err = got_error_from_errno_fmt("openat: %s/%s",
-			    got_repo_get_path_git_dir(repo),
-			    GOT_OBJECTS_PACK_DIR);
-		goto done;
-	}
-
-	packdir = fdopendir(packdir_fd);
-	if (packdir == NULL) {
-		err = got_error_from_errno("fdopendir");
-		goto done;
-	}
-
-	while ((dent = readdir(packdir)) != NULL) {
+	TAILQ_FOREACH(pe, &repo->packidx_paths, entry) {
+		const char *path_packidx = pe->path;
 		int is_cached = 0;
 
-		if (!got_repo_is_packidx_filename(dent->d_name,
-		    strlen(dent->d_name)))
-			continue;
-
-		if (asprintf(&path_packidx, "%s/%s", GOT_OBJECTS_PACK_DIR,
-		    dent->d_name) == -1) {
-			err = got_error_from_errno("asprintf");
-			goto done;
-		}
-
 		if (!got_repo_check_packidx_bloom_filter(repo,
-		    path_packidx, id)) {
-			free(path_packidx);
+		    pe->path, id))
 			continue; /* object will not be found in this index */
-		}
 
 		for (i = 0; i < repo->pack_cache_size; i++) {
 			if (repo->packidx_cache[i] == NULL)
@@ -1163,26 +1139,19 @@ got_repo_search_packidx(struct got_packidx **packidx, 
 				break;
 			}
 		}
-		if (is_cached) {
-			free(path_packidx);
+		if (is_cached)
 			continue; /* already searched */
-		}
 
 		err = got_packidx_open(packidx, got_repo_get_fd(repo),
 		    path_packidx, 0);
-		if (err) {
-			free(path_packidx);
+		if (err)
 			goto done;
-		}
 
 		err = add_packidx_bloom_filter(repo, *packidx, path_packidx);
-		if (err) {
-			free(path_packidx);
+		if (err)
 			goto done;
-		}
 
 		err = cache_packidx(repo, *packidx, path_packidx);
-		free(path_packidx);
 		if (err)
 			goto done;
 
@@ -1195,8 +1164,6 @@ got_repo_search_packidx(struct got_packidx **packidx, 
 
 	err = got_error_no_obj(id);
 done:
-	if (packdir && closedir(packdir) != 0 && err == NULL)
-		err = got_error_from_errno("closedir");
 	return err;
 }
 
@@ -1481,47 +1448,18 @@ match_packed_object(struct got_object_id **unique_id,
     struct got_repository *repo, const char *id_str_prefix, int obj_type)
 {
 	const struct got_error *err = NULL;
-	DIR *packdir = NULL;
-	struct dirent *dent;
-	char *path_packidx;
-	struct got_object_id_queue matched_ids;
-	int packdir_fd;
+	struct got_object_id_queue matched_ids;
+	struct got_pathlist_entry *pe;
 
 	STAILQ_INIT(&matched_ids);
 
-	packdir_fd = openat(got_repo_get_fd(repo),
-	    GOT_OBJECTS_PACK_DIR, O_DIRECTORY | O_CLOEXEC);
-	if (packdir_fd == -1) {
-		if (errno != ENOENT) {
-			err = got_error_from_errno2("openat",
-			    GOT_OBJECTS_PACK_DIR);
-		}
-		goto done;
-	}
-
-	packdir = fdopendir(packdir_fd);
-	if (packdir == NULL) {
-		err = got_error_from_errno("fdopendir");
-		goto done;
-	}
-
-	while ((dent = readdir(packdir)) != NULL) {
+	TAILQ_FOREACH(pe, &repo->packidx_paths, entry) {
+		const char *path_packidx = pe->path;
 		struct got_packidx *packidx;
 		struct got_object_qid *qid;
-
-		if (!got_repo_is_packidx_filename(dent->d_name,
-		    strlen(dent->d_name)))
-			continue;
 
-		if (asprintf(&path_packidx, "%s/%s", GOT_OBJECTS_PACK_DIR,
-		    dent->d_name) == -1) {
-			err = got_error_from_errno("strdup");
-			break;
-		}
-
 		err = got_packidx_open(&packidx, got_repo_get_fd(repo),
 		    path_packidx, 0);
-		free(path_packidx);
 		if (err)
 			break;
 
@@ -1561,8 +1499,6 @@ match_packed_object(struct got_object_id **unique_id,
 	}
 done:
 	got_object_id_queue_free(&matched_ids);
-	if (packdir && closedir(packdir) != 0 && err == NULL)
-		err = got_error_from_errno("closedir");
 	if (err) {
 		free(*unique_id);
 		*unique_id = NULL;