commit c565dfd37a157bab9556aceac96ff27d64525fc9
from: Stefan Sperling <stsp@stsp.name>
via: Thomas Adam <thomas@xteddy.org>
date: Sat Oct 16 11:09:49 2021 UTC

cache raw objects in order to speed up gotadmin pack

commit - cfe4112187cdecbd49844c80bb337c1671a2d7ab
commit + c565dfd37a157bab9556aceac96ff27d64525fc9
blob - 0115e7b78c440326e9e91c010a967b702ff8432f
blob + 0379a4f0e0a95fc21aceea4e5fa496a9073e2043
--- lib/got_lib_object.h
+++ lib/got_lib_object.h
@@ -40,6 +40,8 @@ struct got_raw_object {
 	uint8_t *data;
 	off_t size;
 	size_t hdrlen;
+	size_t blocksize;
+	uint8_t *read_buf;
 	int refcnt;		/* > 0 if open and/or cached */
 };
 
@@ -106,7 +108,12 @@ const struct got_error *got_object_read_header_privsep
 const struct got_error *got_object_open(struct got_object **,
     struct got_repository *, struct got_object_id *);
 const struct got_error *got_object_raw_open(struct got_raw_object **, int *,
-    struct got_repository *, struct got_object_id *);
+    struct got_repository *, struct got_object_id *, size_t);
+void got_object_raw_rewind(struct got_raw_object *);
+size_t got_object_raw_get_hdrlen(struct got_raw_object *);
+const uint8_t *got_object_raw_get_read_buf(struct got_raw_object *);
+const struct got_error * got_object_raw_read_block(size_t *,
+    struct got_raw_object *);
 const struct got_error *got_object_raw_close(struct got_raw_object *);
 const struct got_error *got_object_open_by_id_str(struct got_object **,
     struct got_repository *, const char *);
blob - 29ea30e10bd9622eac9d105e0f8999912a726c9d
blob + 0addced236c0baf4175fa5076c8cf2ba3f3d9265
--- lib/object.c
+++ lib/object.c
@@ -531,7 +531,7 @@ got_object_open(struct got_object **obj, struct got_re
 /* *outfd must be initialized to -1 by caller */
 const struct got_error *
 got_object_raw_open(struct got_raw_object **obj, int *outfd,
-    struct got_repository *repo, struct got_object_id *id)
+    struct got_repository *repo, struct got_object_id *id, size_t blocksize)
 {
 	const struct got_error *err = NULL;
 	struct got_packidx *packidx = NULL;
@@ -620,6 +620,7 @@ got_object_raw_open(struct got_raw_object **obj, int *
 	}
 	(*obj)->hdrlen = hdrlen;
 	(*obj)->size = size;
+	(*obj)->blocksize = blocksize;
 	err = got_repo_cache_raw_object(repo, id, *obj);
 done:
 	free(path_packfile);
@@ -632,6 +633,37 @@ done:
 	} else
 		(*obj)->refcnt++;
 	return err;
+}
+
+void
+got_object_raw_rewind(struct got_raw_object *obj)
+{
+	if (obj->f)
+		rewind(obj->f);
+}
+
+size_t
+got_object_raw_get_hdrlen(struct got_raw_object *obj)
+{
+	return obj->hdrlen;
+}
+
+const uint8_t *
+got_object_raw_get_read_buf(struct got_raw_object *obj)
+{
+	return obj->read_buf;
+}
+
+const struct got_error *
+got_object_raw_read_block(size_t *outlenp, struct got_raw_object *obj)
+{
+	size_t n;
+
+	n = fread(obj->read_buf, 1, obj->blocksize, obj->f);
+	if (n == 0 && ferror(obj->f))
+		return got_ferror(obj->f, GOT_ERR_IO);
+	*outlenp = n;
+	return NULL;
 }
 
 const struct got_error *
blob - 39e9558bbc6cd4d7e28daadb2d034a781f141463
blob + e3bb199653594105098b9b1a03c6e5a58bed641d
--- lib/object_cache.c
+++ lib/object_cache.c
@@ -15,6 +15,7 @@
  */
 
 #include <sys/time.h>
+#include <sys/queue.h>
 #include <sys/resource.h>
 
 #include <stdio.h>
blob - b78b7ba3c8fbd3036ee95c115be0ea37323e9496
blob + c1a34dc79d27cbb721f95e952200e227cb3b1725
--- lib/object_parse.c
+++ lib/object_parse.c
@@ -144,6 +144,7 @@ got_object_raw_close(struct got_raw_object *obj)
 			return NULL;
 	}
 
+	free(obj->read_buf);
 	if (obj->f != NULL && fclose(obj->f) == EOF && err == NULL)
 		err = got_error_from_errno("fclose");
 	free(obj->data);
blob - 6ad71896f6f904435d6699beaa3bc9dbcf7d2147
blob + e5a82eb84a82270063d1e13b3f0aecd3ee6f5a86
--- lib/pack_create.c
+++ lib/pack_create.c
@@ -284,7 +284,7 @@ pick_deltas(struct got_pack_meta **meta, int nmeta, in
 		    m->obj_type == GOT_OBJ_TYPE_TAG)
 			continue;
 
-		err = got_object_raw_open(&raw, &outfd, repo, &m->id);
+		err = got_object_raw_open(&raw, &outfd, repo, &m->id, 8192);
 		if (err)
 			goto done;
 		m->size = raw->size;
@@ -316,7 +316,7 @@ pick_deltas(struct got_pack_meta **meta, int nmeta, in
 				continue;
 
 			err = got_object_raw_open(&base_raw, &outfd, repo,
-			    &base->id);
+			    &base->id, 8192);
 			if (err)
 				goto done;
 			err = got_deltify(&deltas, &ndeltas,
@@ -1177,10 +1177,10 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt
 		}
 		m = meta[i];
 		m->off = ftello(packfile);
-		err = got_object_raw_open(&raw, &outfd, repo, &m->id);
+		err = got_object_raw_open(&raw, &outfd, repo, &m->id, 8192);
 		if (err)
 			goto done;
-		if (m->delta_len == 0) {
+		if (m->deltas == NULL) {
 			err = packhdr(&nh, buf, sizeof(buf),
 			    m->obj_type, raw->size);
 			if (err)