commit 6c41426143b93f57eebf1aaac35dcee4fa3fc282
from: Stefan Sperling <stsp@stsp.name>
date: Tue Mar 30 01:26:42 2021 UTC

cap pack file cache size at 1/8 of the current open file desciptor limit

commit - 61303e6e0b9649166ad58054f5fe7b4e8ccaa136
commit + 6c41426143b93f57eebf1aaac35dcee4fa3fc282
blob - 926534be5a780a41e69bd370a8511f00d7fcc19a
blob + e26649f47637e11dd9477fd50158d147bac28aad
--- lib/got_lib_repository.h
+++ lib/got_lib_repository.h
@@ -28,8 +28,7 @@
 #define GOT_OBJECTS_PACK_DIR	"objects/pack"
 #define GOT_PACKED_REFS_FILE	"packed-refs"
 
-#define GOT_PACKIDX_CACHE_SIZE	64
-#define GOT_PACK_CACHE_SIZE	GOT_PACKIDX_CACHE_SIZE
+#define GOT_PACK_CACHE_SIZE	64
 
 struct got_repository {
 	char *path;
@@ -37,11 +36,17 @@ struct got_repository {
 	int gitdir_fd;
 
 	/* The pack index cache speeds up search for packed objects. */
-	struct got_packidx *packidx_cache[GOT_PACKIDX_CACHE_SIZE];
+	struct got_packidx *packidx_cache[GOT_PACK_CACHE_SIZE];
 
 	/* Open file handles for pack files. */
 	struct got_pack packs[GOT_PACK_CACHE_SIZE];
 
+	/*
+	 * The cache size limit may be lower than GOT_PACK_CACHE_SIZE,
+	 * depending on resource limits.
+	 */
+	int pack_cache_size;
+
 	/* Handles to child processes for reading loose objects. */
 	 struct got_privsep_child privsep_children[5];
 #define GOT_REPO_PRIVSEP_CHILD_OBJECT	0
blob - 16a7584017b72e56816c5302da3591842f6b009e
blob + 56e526ad2a7c56e76b3d397516bbac2fb58a0910
--- lib/repository.c
+++ lib/repository.c
@@ -20,6 +20,7 @@
 #include <sys/socket.h>
 #include <sys/stat.h>
 #include <sys/mman.h>
+#include <sys/resource.h>
 
 #include <ctype.h>
 #include <endian.h>
@@ -615,8 +616,12 @@ got_repo_open(struct got_repository **repop, const cha
 	const struct got_error *err = NULL;
 	char *repo_path = NULL;
 	size_t i;
+	struct rlimit rl;
 
 	*repop = NULL;
+
+	if (getrlimit(RLIMIT_NOFILE, &rl) == -1)
+		return got_error_from_errno("getrlimit");
 
 	repo = calloc(1, sizeof(*repo));
 	if (repo == NULL) {
@@ -647,6 +652,10 @@ got_repo_open(struct got_repository **repop, const cha
 	if (err)
 		goto done;
 
+	repo->pack_cache_size = GOT_PACK_CACHE_SIZE;
+	if (repo->pack_cache_size > rl.rlim_cur / 8)
+		repo->pack_cache_size = rl.rlim_cur / 8;
+
 	repo_path = realpath(path, NULL);
 	if (repo_path == NULL) {
 		err = got_error_from_errno2("realpath", path);
@@ -710,13 +719,13 @@ got_repo_close(struct got_repository *repo)
 	const struct got_error *err = NULL, *child_err;
 	size_t i;
 
-	for (i = 0; i < nitems(repo->packidx_cache); i++) {
+	for (i = 0; i < repo->pack_cache_size; i++) {
 		if (repo->packidx_cache[i] == NULL)
 			break;
 		got_packidx_close(repo->packidx_cache[i]);
 	}
 
-	for (i = 0; i < nitems(repo->packs); i++) {
+	for (i = 0; i < repo->pack_cache_size; i++) {
 		if (repo->packs[i].path_packfile == NULL)
 			break;
 		got_pack_close(&repo->packs[i]);
@@ -893,7 +902,7 @@ cache_packidx(struct got_repository *repo, struct got_
 	const struct got_error *err = NULL;
 	size_t i;
 
-	for (i = 0; i < nitems(repo->packidx_cache); i++) {
+	for (i = 0; i < repo->pack_cache_size; i++) {
 		if (repo->packidx_cache[i] == NULL)
 			break;
 		if (strcmp(repo->packidx_cache[i]->path_packidx,
@@ -901,7 +910,7 @@ cache_packidx(struct got_repository *repo, struct got_
 			return got_error(GOT_ERR_CACHE_DUP_ENTRY);
 		}
 	}
-	if (i == nitems(repo->packidx_cache)) {
+	if (i == repo->pack_cache_size) {
 		err = got_packidx_close(repo->packidx_cache[i - 1]);
 		if (err)
 			return err;
@@ -947,7 +956,7 @@ got_repo_search_packidx(struct got_packidx **packidx, 
 	int packdir_fd;
 
 	/* Search pack index cache. */
-	for (i = 0; i < nitems(repo->packidx_cache); i++) {
+	for (i = 0; i < repo->pack_cache_size; i++) {
 		if (repo->packidx_cache[i] == NULL)
 			break;
 		*idx = got_packidx_get_object_idx(repo->packidx_cache[i], id);
@@ -998,7 +1007,7 @@ got_repo_search_packidx(struct got_packidx **packidx, 
 			goto done;
 		}
 
-		for (i = 0; i < nitems(repo->packidx_cache); i++) {
+		for (i = 0; i < repo->pack_cache_size; i++) {
 			if (repo->packidx_cache[i] == NULL)
 				break;
 			if (strcmp(repo->packidx_cache[i]->path_packidx,
@@ -1094,7 +1103,7 @@ got_repo_cache_pack(struct got_pack **packp, struct go
 	if (packp)
 		*packp = NULL;
 
-	for (i = 0; i < nitems(repo->packs); i++) {
+	for (i = 0; i < repo->pack_cache_size; i++) {
 		pack = &repo->packs[i];
 		if (pack->path_packfile == NULL)
 			break;
@@ -1102,7 +1111,7 @@ got_repo_cache_pack(struct got_pack **packp, struct go
 			return got_error(GOT_ERR_CACHE_DUP_ENTRY);
 	}
 
-	if (i == nitems(repo->packs)) {
+	if (i == repo->pack_cache_size) {
 		err = got_pack_close(&repo->packs[i - 1]);
 		if (err)
 			return err;
@@ -1159,7 +1168,7 @@ got_repo_get_cached_pack(struct got_repository *repo, 
 	struct got_pack *pack = NULL;
 	size_t i;
 
-	for (i = 0; i < nitems(repo->packs); i++) {
+	for (i = 0; i < repo->pack_cache_size; i++) {
 		pack = &repo->packs[i];
 		if (pack->path_packfile == NULL)
 			break;