commit c77e00b3da8bc9349b4512c0b9905c7c9f3f52ef
from: Stefan Sperling <stsp@stsp.name>
via: Thomas Adam <thomas@xteddy.org>
date: Tue Oct 18 22:12:22 2022 UTC

allow got_object_parse_tree to reuse entries buffer allocations for speed

ok millert@

commit - abd468944be5280ec9e2019af467e1602492eea7
commit + c77e00b3da8bc9349b4512c0b9905c7c9f3f52ef
blob - 79b6e83d9872c4ee03472131af53beb3186ab6be
blob + 723508cf635f39153b1b47082dca8bf6b3bb09ca
--- lib/got_lib_object_parse.h
+++ lib/got_lib_object_parse.h
@@ -30,7 +30,7 @@ struct got_parsed_tree_entry {
 	uint8_t *id; /* Points to ID in parsed tree buffer. */
 };
 const struct got_error *got_object_parse_tree(struct got_parsed_tree_entry **,
-    int *, uint8_t *, size_t);
+    size_t *, size_t *, uint8_t *, size_t);
 
 const struct got_error *got_object_parse_tag(struct got_tag_object **,
     uint8_t *, size_t);
blob - d4be09b540ddfefaf18c0b29e61182be66f0a692
blob + af8e733b643b04550f7c172a1ca2ddf85333358e
--- lib/object_parse.c
+++ lib/object_parse.c
@@ -712,11 +712,11 @@ pte_cmp(const void *pa, const void *pb)
 }
 
 const struct got_error *
-got_object_parse_tree(struct got_parsed_tree_entry **entries, int *nentries,
-    uint8_t *buf, size_t len)
+got_object_parse_tree(struct got_parsed_tree_entry **entries, size_t *nentries,
+    size_t *nentries_alloc, uint8_t *buf, size_t len)
 {
 	const struct got_error *err = NULL;
-	size_t remain = len, totalloc;
+	size_t remain = len;
 	const size_t nalloc = 16;
 	struct got_parsed_tree_entry *pte;
 	int i;
@@ -725,23 +725,18 @@ got_object_parse_tree(struct got_parsed_tree_entry **e
 	if (remain == 0)
 		return NULL; /* tree is empty */
 
-	*entries = calloc(nalloc, sizeof(**entries));
-	if (*entries == NULL)
-		return got_error_from_errno("calloc");
-	totalloc = nalloc;
-
 	while (remain > 0) {
 		size_t elen;
 
-		if (*nentries >= totalloc) {
-			pte = recallocarray(*entries, totalloc,
-			    totalloc + nalloc, sizeof(**entries));
+		if (*nentries >= *nentries_alloc) {
+			pte = recallocarray(*entries, *nentries_alloc,
+			    *nentries_alloc + nalloc, sizeof(**entries));
 			if (pte == NULL) {
 				err = got_error_from_errno("recallocarray");
 				goto done;
 			}
 			*entries = pte;
-			totalloc += nalloc;
+			*nentries_alloc += nalloc;
 		}
 
 		pte = &(*entries)[*nentries];
@@ -772,11 +767,8 @@ got_object_parse_tree(struct got_parsed_tree_entry **e
 		}
 	}
 done:
-	if (err) {
-		free(*entries);
-		*entries = NULL;
+	if (err)
 		*nentries = 0;
-	}
 	return err;
 }
 
blob - eff5ccf4f78170ab2a5144c001c046c3ad1fbd2e
blob + 0ef5492a6774b94872b024b057f3dc9e562b9550
--- libexec/got-read-pack/got-read-pack.c
+++ libexec/got-read-pack/got-read-pack.c
@@ -183,9 +183,9 @@ done:
 }
 
 static const struct got_error *
-open_tree(uint8_t **buf, struct got_parsed_tree_entry **entries, int *nentries,
-    struct got_pack *pack, struct got_packidx *packidx, int obj_idx,
-    struct got_object_id *id, struct got_object_cache *objcache)
+open_tree(uint8_t **buf, struct got_parsed_tree_entry **entries, size_t *nentries,
+    size_t *nentries_alloc, struct got_pack *pack, struct got_packidx *packidx,
+    int obj_idx, struct got_object_id *id, struct got_object_cache *objcache)
 {
 	const struct got_error *err = NULL;
 	struct got_object *obj = NULL;
@@ -210,7 +210,8 @@ open_tree(uint8_t **buf, struct got_parsed_tree_entry 
 
 	obj->size = len;
 
-	err = got_object_parse_tree(entries, nentries, *buf, len);
+	err = got_object_parse_tree(entries, nentries, nentries_alloc,
+	    *buf, len);
 done:
 	got_object_close(obj);
 	if (err) {
@@ -222,12 +223,12 @@ done:
 
 static const struct got_error *
 tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
-    struct got_packidx *packidx, struct got_object_cache *objcache)
+    struct got_packidx *packidx, struct got_object_cache *objcache,
+    struct got_parsed_tree_entry **entries, size_t *nentries,
+    size_t *nentries_alloc)
 {
 	const struct got_error *err = NULL;
 	struct got_imsg_packed_object iobj;
-	struct got_parsed_tree_entry *entries = NULL;
-	int nentries = 0;
 	uint8_t *buf = NULL;
 	struct got_object_id id;
 	size_t datalen;
@@ -238,13 +239,12 @@ tree_request(struct imsg *imsg, struct imsgbuf *ibuf, 
 	memcpy(&iobj, imsg->data, sizeof(iobj));
 	memcpy(id.sha1, iobj.id, SHA1_DIGEST_LENGTH);
 
-	err = open_tree(&buf, &entries, &nentries, pack, packidx, iobj.idx,
-	    &id, objcache);
+	err = open_tree(&buf, entries, nentries, nentries_alloc,
+	    pack, packidx, iobj.idx, &id, objcache);
 	if (err)
 		return err;
 
-	err = got_privsep_send_tree(ibuf, entries, nentries);
-	free(entries);
+	err = got_privsep_send_tree(ibuf, *entries, *nentries);
 	free(buf);
 	if (err) {
 		if (err->code == GOT_ERR_PRIVSEP_PIPE)
@@ -453,8 +453,10 @@ find_entry_by_name(struct got_parsed_tree_entry *entri
 
 static const struct got_error *
 tree_path_changed(int *changed, uint8_t **buf1, uint8_t **buf2,
-    struct got_parsed_tree_entry **entries1, int *nentries1,
-    struct got_parsed_tree_entry **entries2, int *nentries2,
+    struct got_parsed_tree_entry **entries1, size_t *nentries1,
+    size_t *nentries_alloc1,
+    struct got_parsed_tree_entry **entries2, size_t *nentries2,
+    size_t *nentries_alloc2,
     const char *path, struct got_pack *pack, struct got_packidx *packidx,
     struct imsgbuf *ibuf, struct got_object_cache *objcache)
 {
@@ -522,12 +524,12 @@ tree_path_changed(int *changed, uint8_t **buf1, uint8_
 				err = got_error_no_obj(&id1);
 				break;
 			}
-			free(*entries1);
 			*nentries1 = 0;
 			free(*buf1);
 			*buf1 = NULL;
-			err = open_tree(buf1, entries1, nentries1, pack,
-			    packidx, idx, &id1, objcache);
+			err = open_tree(buf1, entries1, nentries1,
+			    nentries_alloc1, pack, packidx, idx, &id1,
+			    objcache);
 			pte1 = NULL;
 			if (err)
 				break;
@@ -538,12 +540,11 @@ tree_path_changed(int *changed, uint8_t **buf1, uint8_
 				err = got_error_no_obj(&id2);
 				break;
 			}
-			free(*entries2);
 			*nentries2 = 0;
 			free(*buf2);
 			*buf2 = NULL;
-			err = open_tree(buf2, entries2, nentries2, pack,
-			    packidx, idx, &id2, objcache);
+			err = open_tree(buf2, entries2, nentries2,
+			nentries_alloc2, pack, packidx, idx, &id2, objcache);
 			pte2 = NULL;
 			if (err)
 				break;
@@ -603,7 +604,8 @@ commit_traversal_request(struct imsg *imsg, struct ims
 	struct got_object_qid *pid;
 	struct got_commit_object *commit = NULL, *pcommit = NULL;
 	struct got_parsed_tree_entry *entries = NULL, *pentries = NULL;
-	int nentries = 0, pnentries = 0;
+	size_t nentries = 0, nentries_alloc = 0;
+	size_t pnentries = 0, pnentries_alloc = 0;
 	struct got_object_id id;
 	size_t datalen, path_len;
 	char *path = NULL;
@@ -713,27 +715,26 @@ commit_traversal_request(struct imsg *imsg, struct ims
 			if (pidx == -1)
 				break;
 
-			err = open_tree(&buf, &entries, &nentries, pack,
-			    packidx, idx, commit->tree_id, objcache);
+			err = open_tree(&buf, &entries, &nentries,
+			    &nentries_alloc, pack, packidx, idx,
+			    commit->tree_id, objcache);
 			if (err)
 				goto done;
-			err = open_tree(&pbuf, &pentries, &pnentries, pack,
-			    packidx, pidx, pcommit->tree_id, objcache);
+			err = open_tree(&pbuf, &pentries, &pnentries,
+			    &pnentries_alloc, pack, packidx, pidx,
+			    pcommit->tree_id, objcache);
 			if (err) {
 				free(buf);
 				goto done;
 			}
 
 			err = tree_path_changed(&changed, &buf, &pbuf,
-			    &entries, &nentries, &pentries, &pnentries, path,
-			    pack, packidx, ibuf, objcache);
+			    &entries, &nentries, &nentries_alloc,
+			    &pentries, &pnentries, &pnentries_alloc,
+			    path, pack, packidx, ibuf, objcache);
 
-			free(entries);
-			entries = NULL;
 			nentries = 0;
 			free(buf);
-			free(pentries);
-			pentries = NULL;
 			pnentries = 0;
 			free(pbuf);
 			if (err) {
@@ -1191,7 +1192,7 @@ enumerate_tree(int *have_all_entries, struct imsgbuf *
 	struct got_object_qid *qid;
 	uint8_t *buf = NULL;
 	struct got_parsed_tree_entry *entries = NULL;
-	int nentries = 0, i;
+	size_t nentries = 0, nentries_alloc = 0, i;
 	struct enumerated_tree *tree;
 
 	*ntrees = 0;
@@ -1230,7 +1231,7 @@ enumerate_tree(int *have_all_entries, struct imsgbuf *
 			break;
 		}
 
-		err = open_tree(&buf, &entries, &nentries,
+		err = open_tree(&buf, &entries, &nentries, &nentries_alloc,
 		    pack, packidx, idx, &qid->id, objcache);
 		if (err) {
 			if (err->code != GOT_ERR_NO_OBJ)
@@ -1289,7 +1290,9 @@ enumerate_tree(int *have_all_entries, struct imsgbuf *
 		buf = NULL;
 		tree->entries = entries;
 		entries = NULL;
+		nentries_alloc = 0;
 		tree->nentries = nentries;
+		nentries = 0;
 
 		got_object_qid_free(qid);
 		qid = NULL;
@@ -1898,6 +1901,8 @@ main(int argc, char *argv[])
 	struct got_object_cache objcache;
 	FILE *basefile = NULL, *accumfile = NULL, *delta_outfile = NULL;
 	struct got_object_idset *keep = NULL, *drop = NULL, *skip = NULL;
+	struct got_parsed_tree_entry *entries = NULL;
+	size_t nentries = 0, nentries_alloc = 0;
 
 	//static int attached;
 	//while (!attached) sleep(1);
@@ -2017,7 +2022,7 @@ main(int argc, char *argv[])
 			break;
 		case GOT_IMSG_TREE_REQUEST:
 			err = tree_request(&imsg, &ibuf, pack, packidx,
-			    &objcache);
+			    &objcache, &entries, &nentries, &nentries_alloc);
 			break;
 		case GOT_IMSG_BLOB_REQUEST:
 			if (basefile == NULL || accumfile == NULL) {
@@ -2066,6 +2071,7 @@ main(int argc, char *argv[])
 			break;
 	}
 
+	free(entries);
 	commit_painting_free(&keep, &drop, &skip);
 	if (packidx)
 		got_packidx_close(packidx);
blob - 25c2f2bc4b6753de4a44414334391ac4c7ab1716
blob + 935d89ec4a39d35fd3daf91aea3d997156158b97
--- libexec/got-read-tree/got-read-tree.c
+++ libexec/got-read-tree/got-read-tree.c
@@ -50,8 +50,8 @@ catch_sigint(int signo)
 }
 
 static const struct got_error *
-read_tree_object(struct got_parsed_tree_entry **entries, int *nentries,
-    uint8_t **p, FILE *f, struct got_object_id *expected_id)
+read_tree_object(struct got_parsed_tree_entry **entries, size_t *nentries,
+    size_t *nentries_alloc, uint8_t **p, FILE *f, struct got_object_id *expected_id)
 {
 	const struct got_error *err = NULL;
 	struct got_object *obj = NULL;
@@ -89,7 +89,8 @@ read_tree_object(struct got_parsed_tree_entry **entrie
 
 	/* Skip object header. */
 	len -= obj->hdrlen;
-	err = got_object_parse_tree(entries, nentries, *p + obj->hdrlen, len);
+	err = got_object_parse_tree(entries, nentries, nentries_alloc,
+	    *p + obj->hdrlen, len);
 done:
 	if (obj)
 		got_object_close(obj);
@@ -102,6 +103,8 @@ main(int argc, char *argv[])
 	const struct got_error *err = NULL;
 	struct imsgbuf ibuf;
 	size_t datalen;
+	struct got_parsed_tree_entry *entries = NULL;
+	size_t nentries = 0, nentries_alloc = 0;
 
 	signal(SIGINT, catch_sigint);
 
@@ -131,8 +134,6 @@ main(int argc, char *argv[])
 	for (;;) {
 		struct imsg imsg;
 		FILE *f = NULL;
-		struct got_parsed_tree_entry *entries = NULL;
-		int nentries = 0;
 		uint8_t *buf = NULL;
 		struct got_object_id expected_id;
 
@@ -175,14 +176,13 @@ main(int argc, char *argv[])
 			goto done;
 		}
 
-		err = read_tree_object(&entries, &nentries, &buf, f,
-		    &expected_id);
+		err = read_tree_object(&entries, &nentries, &nentries_alloc,
+		    &buf, f, &expected_id);
 		if (err)
 			goto done;
 
 		err = got_privsep_send_tree(&ibuf, entries, nentries);
 done:
-		free(entries);
 		free(buf);
 		if (f) {
 			if (fclose(f) == EOF && err == NULL)
@@ -196,6 +196,7 @@ done:
 			break;
 	}
 
+	free(entries);
 	imsg_clear(&ibuf);
 	if (err) {
 		if (!sigint_received && err->code != GOT_ERR_PRIVSEP_PIPE) {