Game of Trees Repos

Commit Diff

Commit:: 894e4711ffca9e21b24e567a610d387bd0c31817
From:: Stefan Sperling <stsp@stsp.name>
Date:: Sat Oct 15 18:39:28 2022 UTC
Message:: make got_pack_create() write to a file descriptor instead of a stdio FILE The old code required a seekable output file. This conflicts with requirements of future gotd(8), which will write pack file data to network sockets. ok op@
Actions:: Patch | Tree
commit - 9cda65e55e3f6d565e83377712e1b81177639cda
commit + 894e4711ffca9e21b24e567a610d387bd0c31817
blob - d4180f6aedf54820d7406dc3029ad2247929e580
blob + bed54dcd68f16ffd5582f7f477c963c86ca93fc6
--- lib/deflate.c
+++ lib/deflate.c
@@ -232,6 +232,88 @@ got_deflate_end(struct got_deflate_buf *zb)
 	if (zb->flags & GOT_DEFLATE_F_OWN_OUTBUF)
 		free(zb->outbuf);
 	deflateEnd(&zb->z);
+}
+
+const struct got_error *
+got_deflate_to_fd(off_t *outlen, FILE *infile, off_t len, int outfd,
+    struct got_deflate_checksum *csum)
+{
+	const struct got_error *err;
+	size_t avail;
+	off_t consumed;
+	struct got_deflate_buf zb;
+
+	err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
+	if (err)
+		goto done;
+
+	*outlen = 0;
+
+	do {
+		err = got_deflate_read(&zb, infile, len, &avail, &consumed);
+		if (err)
+			goto done;
+		len -= consumed;
+		if (avail > 0) {
+			ssize_t w;
+			w = write(outfd, zb.outbuf, avail);
+			if (w == -1) {
+				err = got_error_from_errno("write");
+				goto done;
+			} else if (w != avail) {
+				err = got_error(GOT_ERR_IO);
+				goto done;
+			}
+			if (csum)
+				csum_output(csum, zb.outbuf, avail);
+			*outlen += avail;
+		}
+	} while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
+
+done:
+	got_deflate_end(&zb);
+	return err;
+}
+
+const struct got_error *
+got_deflate_to_fd_mmap(off_t *outlen, uint8_t *map, size_t offset,
+    size_t len, int outfd, struct got_deflate_checksum *csum)
+{
+	const struct got_error *err;
+	size_t avail, consumed;
+	struct got_deflate_buf zb;
+
+	err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
+	if (err)
+		goto done;
+
+	*outlen = 0;
+	do {
+		err = got_deflate_read_mmap(&zb, map, offset, len, &avail,
+		    &consumed);
+		if (err)
+			goto done;
+		offset += consumed;
+		len -= consumed;
+		if (avail > 0) {
+			ssize_t w;
+			w = write(outfd, zb.outbuf, avail);
+			if (w == -1) {
+				err = got_error_from_errno("write");
+				goto done;
+			} else if (w != avail) {
+				err = got_error(GOT_ERR_IO);
+				goto done;
+			}
+			if (csum)
+				csum_output(csum, zb.outbuf, avail);
+			*outlen += avail;
+		}
+	} while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
+
+done:
+	got_deflate_end(&zb);
+	return err;
 }
 
 const struct got_error *
blob - 09a8755cf062db2ffd1f7e2c26deef470dbba250
blob + 2575eb63d3f9bfe0cd35903ae30824124fa07ca8
--- lib/got_lib_deflate.h
+++ lib/got_lib_deflate.h
@@ -42,6 +42,10 @@ const struct got_error *got_deflate_read(struct got_de
 const struct got_error *got_deflate_read_mmap(struct got_deflate_buf *,
     uint8_t *, size_t, size_t, size_t *, size_t *);
 void got_deflate_end(struct got_deflate_buf *);
+const struct got_error *got_deflate_to_fd(off_t *, FILE *, off_t, int,
+    struct got_deflate_checksum *);
+const struct got_error *got_deflate_to_fd_mmap(off_t *, uint8_t *,
+    size_t, size_t, int, struct got_deflate_checksum *);
 const struct got_error *got_deflate_to_file(off_t *, FILE *, off_t, FILE *,
     struct got_deflate_checksum *);
 const struct got_error *got_deflate_to_file_mmap(off_t *, uint8_t *,
blob - 8a84ff3488e6ae227869888581b991d1ca4203c2
blob + 699f13578180bc1e991c1ced4b845dda7fa3a633
--- lib/got_lib_pack_create.h
+++ lib/got_lib_pack_create.h
@@ -21,7 +21,7 @@
  * Return the SHA1 digest of the resulting pack file in pack_sha1 which must
  * be pre-allocated by the caller with at least SHA1_DIGEST_LENGTH bytes.
  */
-const struct got_error *got_pack_create(uint8_t *pack_sha1, FILE *packfile,
+const struct got_error *got_pack_create(uint8_t *pack_sha1, int packfd,
     struct got_object_id **theirs, int ntheirs,
     struct got_object_id **ours, int nours,
     struct got_repository *repo, int loose_obj_only, int allow_empty,
blob - e1c54a2cfa511cc98322cf15497e14590c8dd6f3
blob + 4b2c3294522e2a180c7438c962c007f6b988e399
--- lib/pack_create.c
+++ lib/pack_create.c
@@ -2106,23 +2106,26 @@ done:
 }
 
 static const struct got_error *
-hwrite(FILE *f, const void *buf, off_t len, SHA1_CTX *ctx)
+hwrite(int fd, const void *buf, off_t len, SHA1_CTX *ctx)
 {
-	size_t n;
+	ssize_t w;
 
 	SHA1Update(ctx, buf, len);
-	n = fwrite(buf, 1, len, f);
-	if (n != len)
-		return got_ferror(f, GOT_ERR_IO);
+	w = write(fd, buf, len);
+	if (w == -1)
+		return got_error_from_errno("write");
+	else if (w != len)
+		return got_error(GOT_ERR_IO);
 	return NULL;
 }
 
 static const struct got_error *
-hcopy(FILE *fsrc, FILE *fdst, off_t len, SHA1_CTX *ctx)
+hcopy(FILE *fsrc, int fd_dst, off_t len, SHA1_CTX *ctx)
 {
 	unsigned char buf[65536];
 	off_t remain = len;
 	size_t n;
+	ssize_t w;
 
 	while (remain > 0) {
 		size_t copylen = MIN(sizeof(buf), remain);
@@ -2130,9 +2133,11 @@ hcopy(FILE *fsrc, FILE *fdst, off_t len, SHA1_CTX *ctx
 		if (n != copylen)
 			return got_ferror(fsrc, GOT_ERR_IO);
 		SHA1Update(ctx, buf, copylen);
-		n = fwrite(buf, 1, copylen, fdst);
-		if (n != copylen)
-			return got_ferror(fdst, GOT_ERR_IO);
+		w = write(fd_dst, buf, copylen);
+		if (w == -1)
+			return got_error_from_errno("write");
+		else if (w != copylen)
+			return got_error(GOT_ERR_IO);
 		remain -= copylen;
 	}
 
@@ -2141,18 +2146,19 @@ hcopy(FILE *fsrc, FILE *fdst, off_t len, SHA1_CTX *ctx
 
 static const struct got_error *
 hcopy_mmap(uint8_t *src, off_t src_offset, size_t src_size,
-    FILE *fdst, off_t len, SHA1_CTX *ctx)
+    int fd, off_t len, SHA1_CTX *ctx)
 {
-	size_t n;
+	ssize_t w;
 
 	if (src_offset + len > src_size)
 		return got_error(GOT_ERR_RANGE);
 
 	SHA1Update(ctx, src + src_offset, len);
-	n = fwrite(src + src_offset, 1, len, fdst);
-	if (n != len)
-		return got_ferror(fdst, GOT_ERR_IO);
-
+	w = write(fd, src + src_offset, len);
+	if (w == -1)
+		return got_error_from_errno("write");
+	else if (w != len)
+		return got_error(GOT_ERR_IO);
 	return NULL;
 }
 
@@ -2247,7 +2253,7 @@ packoff(char *hdr, off_t off)
 }
 
 static const struct got_error *
-deltahdr(off_t *packfile_size, SHA1_CTX *ctx, FILE *packfile,
+deltahdr(off_t *packfile_size, SHA1_CTX *ctx, int packfd,
     struct got_pack_meta *m)
 {
 	const struct got_error *err;
@@ -2260,7 +2266,7 @@ deltahdr(off_t *packfile_size, SHA1_CTX *ctx, FILE *pa
 		if (err)
 			return err;
 		nh += packoff(buf + nh, m->off - m->prev->off);
-		err = hwrite(packfile, buf, nh, ctx);
+		err = hwrite(packfd, buf, nh, ctx);
 		if (err)
 			return err;
 		*packfile_size += nh;
@@ -2269,11 +2275,11 @@ deltahdr(off_t *packfile_size, SHA1_CTX *ctx, FILE *pa
 		    GOT_OBJ_TYPE_REF_DELTA, m->delta_len);
 		if (err)
 			return err;
-		err = hwrite(packfile, buf, nh, ctx);
+		err = hwrite(packfd, buf, nh, ctx);
 		if (err)
 			return err;
 		*packfile_size += nh;
-		err = hwrite(packfile, m->prev->id.sha1,
+		err = hwrite(packfd, m->prev->id.sha1,
 		    sizeof(m->prev->id.sha1), ctx);
 		if (err)
 			return err;
@@ -2284,7 +2290,7 @@ deltahdr(off_t *packfile_size, SHA1_CTX *ctx, FILE *pa
 }
 
 static const struct got_error *
-write_packed_object(off_t *packfile_size, FILE *packfile,
+write_packed_object(off_t *packfile_size, int packfd,
     FILE *delta_cache, uint8_t *delta_cache_map, size_t delta_cache_size,
     struct got_pack_meta *m, int *outfd, SHA1_CTX *ctx,
     struct got_repository *repo)
@@ -2299,7 +2305,7 @@ write_packed_object(off_t *packfile_size, FILE *packfi
 	csum.output_sha1 = ctx;
 	csum.output_crc = NULL;
 
-	m->off = ftello(packfile);
+	m->off = *packfile_size;
 	if (m->delta_len == 0) {
 		err = got_object_raw_open(&raw, outfd, repo, &m->id);
 		if (err)
@@ -2308,14 +2314,14 @@ write_packed_object(off_t *packfile_size, FILE *packfi
 		    m->obj_type, raw->size);
 		if (err)
 			goto done;
-		err = hwrite(packfile, buf, nh, ctx);
+		err = hwrite(packfd, buf, nh, ctx);
 		if (err)
 			goto done;
 		*packfile_size += nh;
 		if (raw->f == NULL) {
-			err = got_deflate_to_file_mmap(&outlen,
+			err = got_deflate_to_fd_mmap(&outlen,
 			    raw->data + raw->hdrlen, 0, raw->size,
-			    packfile, &csum);
+			    packfd, &csum);
 			if (err)
 				goto done;
 		} else {
@@ -2324,8 +2330,8 @@ write_packed_object(off_t *packfile_size, FILE *packfi
 				err = got_error_from_errno("fseeko");
 				goto done;
 			}
-			err = got_deflate_to_file(&outlen, raw->f,
-			    raw->size, packfile, &csum);
+			err = got_deflate_to_fd(&outlen, raw->f,
+			    raw->size, packfd, &csum);
 			if (err)
 				goto done;
 		}
@@ -2333,10 +2339,10 @@ write_packed_object(off_t *packfile_size, FILE *packfi
 		got_object_raw_close(raw);
 		raw = NULL;
 	} else if (m->delta_buf) {
-		err = deltahdr(packfile_size, ctx, packfile, m);
+		err = deltahdr(packfile_size, ctx, packfd, m);
 		if (err)
 			goto done;
-		err = hwrite(packfile, m->delta_buf,
+		err = hwrite(packfd, m->delta_buf,
 		    m->delta_compressed_len, ctx);
 		if (err)
 			goto done;
@@ -2344,11 +2350,11 @@ write_packed_object(off_t *packfile_size, FILE *packfi
 		free(m->delta_buf);
 		m->delta_buf = NULL;
 	} else if (delta_cache_map) {
-		err = deltahdr(packfile_size, ctx, packfile, m);
+		err = deltahdr(packfile_size, ctx, packfd, m);
 		if (err)
 			goto done;
 		err = hcopy_mmap(delta_cache_map, m->delta_offset,
-		    delta_cache_size, packfile, m->delta_compressed_len,
+		    delta_cache_size, packfd, m->delta_compressed_len,
 		    ctx);
 		if (err)
 			goto done;
@@ -2359,10 +2365,10 @@ write_packed_object(off_t *packfile_size, FILE *packfi
 			err = got_error_from_errno("fseeko");
 			goto done;
 		}
-		err = deltahdr(packfile_size, ctx, packfile, m);
+		err = deltahdr(packfile_size, ctx, packfd, m);
 		if (err)
 			goto done;
-		err = hcopy(delta_cache, packfile,
+		err = hcopy(delta_cache, packfd,
 		    m->delta_compressed_len, ctx);
 		if (err)
 			goto done;
@@ -2375,7 +2381,7 @@ done:
 }
 
 static const struct got_error *
-genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delta_cache,
+genpack(uint8_t *pack_sha1, int packfd, FILE *delta_cache,
     struct got_pack_meta **deltify, int ndeltify,
     struct got_pack_meta **reuse, int nreuse,
     int ncolored, int nfound, int ntrees, int nours,
@@ -2389,7 +2395,7 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt
 	SHA1_CTX ctx;
 	struct got_pack_meta *m;
 	char buf[32];
-	size_t n;
+	ssize_t w;
 	off_t packfile_size = 0;
 	int outfd = -1;
 	int delta_cache_fd = -1;
@@ -2420,15 +2426,15 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt
 		}
 	}
 #endif
-	err = hwrite(packfile, "PACK", 4, &ctx);
+	err = hwrite(packfd, "PACK", 4, &ctx);
 	if (err)
 		goto done;
 	putbe32(buf, GOT_PACKFILE_VERSION);
-	err = hwrite(packfile, buf, 4, &ctx);
+	err = hwrite(packfd, buf, 4, &ctx);
 	if (err)
 		goto done;
 	putbe32(buf, ndeltify + nreuse);
-	err = hwrite(packfile, buf, 4, &ctx);
+	err = hwrite(packfd, buf, 4, &ctx);
 	if (err)
 		goto done;
 
@@ -2441,7 +2447,7 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt
 		if (err)
 			goto done;
 		m = deltify[i];
-		err = write_packed_object(&packfile_size, packfile,
+		err = write_packed_object(&packfile_size, packfd,
 		    delta_cache, delta_cache_map, delta_cache_size,
 		    m, &outfd, &ctx, repo);
 		if (err)
@@ -2457,7 +2463,7 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt
 		if (err)
 			goto done;
 		m = reuse[i];
-		err = write_packed_object(&packfile_size, packfile,
+		err = write_packed_object(&packfile_size, packfd,
 		    delta_cache, delta_cache_map, delta_cache_size,
 		    m, &outfd, &ctx, repo);
 		if (err)
@@ -2465,9 +2471,13 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt
 	}
 
 	SHA1Final(pack_sha1, &ctx);
-	n = fwrite(pack_sha1, 1, SHA1_DIGEST_LENGTH, packfile);
-	if (n != SHA1_DIGEST_LENGTH)
-		err = got_ferror(packfile, GOT_ERR_IO);
+	w = write(packfd, pack_sha1, SHA1_DIGEST_LENGTH);
+	if (w == -1)
+		err = got_error_from_errno("write");
+	else if (w != SHA1_DIGEST_LENGTH)
+		err = got_error(GOT_ERR_IO);
+	if (err)
+		goto done;
 	packfile_size += SHA1_DIGEST_LENGTH;
 	packfile_size += sizeof(struct got_packfile_hdr);
 	if (progress_cb) {
@@ -2500,7 +2510,7 @@ add_meta_idset_cb(struct got_object_id *id, void *data
 }
 
 const struct got_error *
-got_pack_create(uint8_t *packsha1, FILE *packfile,
+got_pack_create(uint8_t *packsha1, int packfd,
     struct got_object_id **theirs, int ntheirs,
     struct got_object_id **ours, int nours,
     struct got_repository *repo, int loose_obj_only, int allow_empty,
@@ -2605,7 +2615,7 @@ got_pack_create(uint8_t *packsha1, FILE *packfile,
 		err = got_error_from_errno("fflush");
 		goto done;
 	}
-	err = genpack(packsha1, packfile, delta_cache, deltify.meta,
+	err = genpack(packsha1, packfd, delta_cache, deltify.meta,
 	    deltify.nmeta, reuse.meta, reuse.nmeta, ncolored, nfound, ntrees,
 	    nours, repo, progress_cb, progress_arg, &rl,
 	    cancel_cb, cancel_arg);
blob - 2cf33ed80132762b274635d2ca08a08e5e5a5f22
blob + 77fcb6b22fcd226378437eda19e4fe0c96478ada
--- lib/repository_admin.c
+++ lib/repository_admin.c
@@ -172,13 +172,6 @@ got_repo_pack_objects(FILE **packfile, struct got_obje
 		goto done;
 	}
 
-	*packfile = fdopen(packfd, "w");
-	if (*packfile == NULL) {
-		err = got_error_from_errno2("fdopen", tmpfile_path);
-		goto done;
-	}
-	packfd = -1;
-
 	err = get_reflist_object_ids(&ours, &nours,
 	    (1 << GOT_OBJ_TYPE_COMMIT) | (1 << GOT_OBJ_TYPE_TAG),
 	    include_refs, repo, cancel_cb, cancel_arg);
@@ -205,7 +198,7 @@ got_repo_pack_objects(FILE **packfile, struct got_obje
 		goto done;
 	}
 
-	err = got_pack_create((*pack_hash)->sha1, *packfile, theirs, ntheirs,
+	err = got_pack_create((*pack_hash)->sha1, packfd, theirs, ntheirs,
 	    ours, nours, repo, loose_obj_only, 0, progress_cb, progress_arg,
 	    cancel_cb, cancel_arg);
 	if (err)
@@ -221,12 +214,8 @@ got_repo_pack_objects(FILE **packfile, struct got_obje
 		goto done;
 	}
 
-	if (fflush(*packfile) == -1) {
-		err = got_error_from_errno("fflush");
-		goto done;
-	}
-	if (fseek(*packfile, 0L, SEEK_SET) == -1) {
-		err = got_error_from_errno("fseek");
+	if (lseek(packfd, 0L, SEEK_SET) == -1) {
+		err = got_error_from_errno("lseek");
 		goto done;
 	}
 	if (rename(tmpfile_path, packfile_path) == -1) {
@@ -236,6 +225,13 @@ got_repo_pack_objects(FILE **packfile, struct got_obje
 	}
 	free(tmpfile_path);
 	tmpfile_path = NULL;
+
+	*packfile = fdopen(packfd, "w");
+	if (*packfile == NULL) {
+		err = got_error_from_errno2("fdopen", tmpfile_path);
+		goto done;
+	}
+	packfd = -1;
 done:
 	for (i = 0; i < nours; i++)
 		free(ours[i]);
blob - 69f2244f86117ab0b9aa9b8ad44c2f28052f2f26
blob + 4393fbc3fe2bb713de61b06db047c3fcff6de871
--- lib/send.c
+++ lib/send.c
@@ -345,7 +345,7 @@ got_send_pack(const char *remote_name, struct got_path
 	off_t bytes_sent = 0, bytes_sent_cur = 0;
 	struct pack_progress_arg ppa;
 	uint8_t packsha1[SHA1_DIGEST_LENGTH];
-	FILE *packfile = NULL;
+	int packfd = -1;
 
 	TAILQ_INIT(&refs);
 	TAILQ_INIT(&have_refs);
@@ -436,9 +436,9 @@ got_send_pack(const char *remote_name, struct got_path
 		}
 	}
 
-	packfile = got_opentemp();
-	if (packfile == NULL) {
-		err = got_error_from_errno("got_opentemp");
+	packfd = got_opentempfd();
+	if (packfd == -1) {
+		err = got_error_from_errno("got_opentempfd");
 		goto done;
 	}
 
@@ -636,18 +636,13 @@ got_send_pack(const char *remote_name, struct got_path
 		memset(&ppa, 0, sizeof(ppa));
 		ppa.progress_cb = progress_cb;
 		ppa.progress_arg = progress_arg;
-		err = got_pack_create(packsha1, packfile, their_ids, ntheirs,
+		err = got_pack_create(packsha1, packfd, their_ids, ntheirs,
 		    our_ids, nours, repo, 0, 1, pack_progress, &ppa,
 		    cancel_cb, cancel_arg);
 		if (err)
 			goto done;
 
-		if (fflush(packfile) == -1) {
-			err = got_error_from_errno("fflush");
-			goto done;
-		}
-
-		npackfd = dup(fileno(packfile));
+		npackfd = dup(packfd);
 		if (npackfd == -1) {
 			err = got_error_from_errno("dup");
 			goto done;
@@ -711,8 +706,8 @@ done:
 		if (waitpid(sendpid, &sendstatus, 0) == -1 && err == NULL)
 			err = got_error_from_errno("waitpid");
 	}
-	if (packfile && fclose(packfile) == EOF && err == NULL)
-		err = got_error_from_errno("fclose");
+	if (packfd != -1 && close(packfd) == -1 && err == NULL)
+		err = got_error_from_errno("close");
 	if (nsendfd != -1 && close(nsendfd) == -1 && err == NULL)
 		err = got_error_from_errno("close");
 	if (npackfd != -1 && close(npackfd) == -1 && err == NULL)