commit e93fb944fe7b32d96fafd4726043a28062563b54
from: Stefan Sperling <stsp@stsp.name>
date: Tue May 10 11:34:16 2022 UTC

map delta cache file into memory if possible while writing a pack file

with a fix from + ok op@

commit - dc3fe1bf10950c200037f0198f2445ea176f8cc3
commit + e93fb944fe7b32d96fafd4726043a28062563b54
blob - 65a01855e246721cbb578dbccf44ebc5691837fc
blob + 71ea8bfe444c0b08f54ccf2700e8d86efef2cd63
--- lib/pack_create.c
+++ lib/pack_create.c
@@ -21,8 +21,10 @@
 #include <sys/uio.h>
 #include <sys/stat.h>
 #include <sys/time.h>
+#include <sys/mman.h>
 
 #include <endian.h>
+#include <errno.h>
 #include <stdint.h>
 #include <imsg.h>
 #include <stdio.h>
@@ -1553,7 +1555,24 @@ hcopy(FILE *fsrc, FILE *fdst, off_t len, SHA1_CTX *ctx
 
 	return NULL;
 }
+
+const struct got_error *
+hcopy_mmap(uint8_t *src, off_t src_offset, size_t src_size,
+    FILE *fdst, off_t len, SHA1_CTX *ctx)
+{
+	size_t n;
 
+	if (src_offset + len > src_size)
+		return got_error(GOT_ERR_RANGE);
+
+	SHA1Update(ctx, src + src_offset, len);
+	n = fwrite(src + src_offset, 1, len, fdst);
+	if (n != len)
+		return got_ferror(fdst, GOT_ERR_IO);
+
+	return NULL;
+}
+
 static void
 putbe32(char *b, uint32_t n)
 {
@@ -1683,8 +1702,9 @@ deltahdr(off_t *packfile_size, SHA1_CTX *ctx, FILE *pa
 
 static const struct got_error *
 write_packed_object(off_t *packfile_size, FILE *packfile,
-    FILE *delta_cache, struct got_pack_meta *m, int *outfd,
-    SHA1_CTX *ctx, struct got_repository *repo)
+    FILE *delta_cache, uint8_t *delta_cache_map, size_t delta_cache_size,
+    struct got_pack_meta *m, int *outfd, SHA1_CTX *ctx,
+    struct got_repository *repo)
 {
 	const struct got_error *err = NULL;
 	struct got_deflate_checksum csum;
@@ -1740,6 +1760,16 @@ write_packed_object(off_t *packfile_size, FILE *packfi
 		*packfile_size += m->delta_compressed_len;
 		free(m->delta_buf);
 		m->delta_buf = NULL;
+	} else if (delta_cache_map) {
+		err = deltahdr(packfile_size, ctx, packfile, m);
+		if (err)
+			goto done;
+		err = hcopy_mmap(delta_cache_map, m->delta_offset,
+		    delta_cache_size, packfile, m->delta_compressed_len,
+		    ctx);
+		if (err)
+			goto done;
+		*packfile_size += m->delta_compressed_len;
 	} else {
 		if (fseeko(delta_cache, m->delta_offset, SEEK_SET)
 		    == -1) {
@@ -1779,12 +1809,37 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt
 	size_t n;
 	off_t packfile_size = 0;
 	int outfd = -1;
+	int delta_cache_fd = -1;
+	uint8_t *delta_cache_map = NULL;
+	size_t delta_cache_size = 0;
 
 	SHA1Init(&ctx);
 
+#ifndef GOT_PACK_NO_MMAP
+	delta_cache_fd = dup(fileno(delta_cache));
+	if (delta_cache_fd != -1) {
+		struct stat sb;
+		if (fstat(delta_cache_fd, &sb) == -1) {
+			err = got_error_from_errno("fstat");
+			goto done;
+		}
+		if (sb.st_size > 0 && sb.st_size <= SIZE_MAX) {
+			delta_cache_map = mmap(NULL, sb.st_size,
+			    PROT_READ, MAP_PRIVATE, delta_cache_fd, 0);
+			if (delta_cache_map == MAP_FAILED) {
+				if (errno != ENOMEM) {
+					err = got_error_from_errno("mmap");
+					goto done;
+				}
+				delta_cache_map = NULL; /* fallback on stdio */
+			} else
+				delta_cache_size = (size_t)sb.st_size;
+		}
+	}
+#endif
 	err = hwrite(packfile, "PACK", 4, &ctx);
 	if (err)
-		return err;
+		goto done;
 	putbe32(buf, GOT_PACKFILE_VERSION);
 	err = hwrite(packfile, buf, 4, &ctx);
 	if (err)
@@ -1804,7 +1859,8 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt
 			goto done;
 		m = deltify[i];
 		err = write_packed_object(&packfile_size, packfile,
-		    delta_cache, m, &outfd, &ctx, repo);
+		    delta_cache, delta_cache_map, delta_cache_size,
+		    m, &outfd, &ctx, repo);
 		if (err)
 			goto done;
 	}
@@ -1819,7 +1875,8 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt
 			goto done;
 		m = reuse[i];
 		err = write_packed_object(&packfile_size, packfile,
-		    delta_cache, m, &outfd, &ctx, repo);
+		    delta_cache, delta_cache_map, delta_cache_size,
+		    m, &outfd, &ctx, repo);
 		if (err)
 			goto done;
 	}
@@ -1840,6 +1897,10 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt
 done:
 	if (outfd != -1 && close(outfd) == -1 && err == NULL)
 		err = got_error_from_errno("close");
+	if (delta_cache_map && munmap(delta_cache_map, delta_cache_size) == -1)
+		err = got_error_from_errno("munmap");
+	if (delta_cache_fd != -1 && close(delta_cache_fd) == -1 && err == NULL)
+		err = got_error_from_errno("close");
 	return err;
 }