commit 894e4711ffca9e21b24e567a610d387bd0c31817 from: Stefan Sperling date: Sat Oct 15 18:39:28 2022 UTC make got_pack_create() write to a file descriptor instead of a stdio FILE The old code required a seekable output file. This conflicts with requirements of future gotd(8), which will write pack file data to network sockets. ok op@ commit - 9cda65e55e3f6d565e83377712e1b81177639cda commit + 894e4711ffca9e21b24e567a610d387bd0c31817 blob - d4180f6aedf54820d7406dc3029ad2247929e580 blob + bed54dcd68f16ffd5582f7f477c963c86ca93fc6 --- lib/deflate.c +++ lib/deflate.c @@ -232,6 +232,88 @@ got_deflate_end(struct got_deflate_buf *zb) if (zb->flags & GOT_DEFLATE_F_OWN_OUTBUF) free(zb->outbuf); deflateEnd(&zb->z); +} + +const struct got_error * +got_deflate_to_fd(off_t *outlen, FILE *infile, off_t len, int outfd, + struct got_deflate_checksum *csum) +{ + const struct got_error *err; + size_t avail; + off_t consumed; + struct got_deflate_buf zb; + + err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE); + if (err) + goto done; + + *outlen = 0; + + do { + err = got_deflate_read(&zb, infile, len, &avail, &consumed); + if (err) + goto done; + len -= consumed; + if (avail > 0) { + ssize_t w; + w = write(outfd, zb.outbuf, avail); + if (w == -1) { + err = got_error_from_errno("write"); + goto done; + } else if (w != avail) { + err = got_error(GOT_ERR_IO); + goto done; + } + if (csum) + csum_output(csum, zb.outbuf, avail); + *outlen += avail; + } + } while (zb.flags & GOT_DEFLATE_F_HAVE_MORE); + +done: + got_deflate_end(&zb); + return err; +} + +const struct got_error * +got_deflate_to_fd_mmap(off_t *outlen, uint8_t *map, size_t offset, + size_t len, int outfd, struct got_deflate_checksum *csum) +{ + const struct got_error *err; + size_t avail, consumed; + struct got_deflate_buf zb; + + err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE); + if (err) + goto done; + + *outlen = 0; + do { + err = got_deflate_read_mmap(&zb, map, offset, len, &avail, + &consumed); + if (err) + goto done; + offset += consumed; + len -= consumed; + if (avail > 0) { + ssize_t w; + w = write(outfd, zb.outbuf, avail); + if (w == -1) { + err = got_error_from_errno("write"); + goto done; + } else if (w != avail) { + err = got_error(GOT_ERR_IO); + goto done; + } + if (csum) + csum_output(csum, zb.outbuf, avail); + *outlen += avail; + } + } while (zb.flags & GOT_DEFLATE_F_HAVE_MORE); + +done: + got_deflate_end(&zb); + return err; } const struct got_error * blob - 09a8755cf062db2ffd1f7e2c26deef470dbba250 blob + 2575eb63d3f9bfe0cd35903ae30824124fa07ca8 --- lib/got_lib_deflate.h +++ lib/got_lib_deflate.h @@ -42,6 +42,10 @@ const struct got_error *got_deflate_read(struct got_de const struct got_error *got_deflate_read_mmap(struct got_deflate_buf *, uint8_t *, size_t, size_t, size_t *, size_t *); void got_deflate_end(struct got_deflate_buf *); +const struct got_error *got_deflate_to_fd(off_t *, FILE *, off_t, int, + struct got_deflate_checksum *); +const struct got_error *got_deflate_to_fd_mmap(off_t *, uint8_t *, + size_t, size_t, int, struct got_deflate_checksum *); const struct got_error *got_deflate_to_file(off_t *, FILE *, off_t, FILE *, struct got_deflate_checksum *); const struct got_error *got_deflate_to_file_mmap(off_t *, uint8_t *, blob - 8a84ff3488e6ae227869888581b991d1ca4203c2 blob + 699f13578180bc1e991c1ced4b845dda7fa3a633 --- lib/got_lib_pack_create.h +++ lib/got_lib_pack_create.h @@ -21,7 +21,7 @@ * Return the SHA1 digest of the resulting pack file in pack_sha1 which must * be pre-allocated by the caller with at least SHA1_DIGEST_LENGTH bytes. */ -const struct got_error *got_pack_create(uint8_t *pack_sha1, FILE *packfile, +const struct got_error *got_pack_create(uint8_t *pack_sha1, int packfd, struct got_object_id **theirs, int ntheirs, struct got_object_id **ours, int nours, struct got_repository *repo, int loose_obj_only, int allow_empty, blob - e1c54a2cfa511cc98322cf15497e14590c8dd6f3 blob + 4b2c3294522e2a180c7438c962c007f6b988e399 --- lib/pack_create.c +++ lib/pack_create.c @@ -2106,23 +2106,26 @@ done: } static const struct got_error * -hwrite(FILE *f, const void *buf, off_t len, SHA1_CTX *ctx) +hwrite(int fd, const void *buf, off_t len, SHA1_CTX *ctx) { - size_t n; + ssize_t w; SHA1Update(ctx, buf, len); - n = fwrite(buf, 1, len, f); - if (n != len) - return got_ferror(f, GOT_ERR_IO); + w = write(fd, buf, len); + if (w == -1) + return got_error_from_errno("write"); + else if (w != len) + return got_error(GOT_ERR_IO); return NULL; } static const struct got_error * -hcopy(FILE *fsrc, FILE *fdst, off_t len, SHA1_CTX *ctx) +hcopy(FILE *fsrc, int fd_dst, off_t len, SHA1_CTX *ctx) { unsigned char buf[65536]; off_t remain = len; size_t n; + ssize_t w; while (remain > 0) { size_t copylen = MIN(sizeof(buf), remain); @@ -2130,9 +2133,11 @@ hcopy(FILE *fsrc, FILE *fdst, off_t len, SHA1_CTX *ctx if (n != copylen) return got_ferror(fsrc, GOT_ERR_IO); SHA1Update(ctx, buf, copylen); - n = fwrite(buf, 1, copylen, fdst); - if (n != copylen) - return got_ferror(fdst, GOT_ERR_IO); + w = write(fd_dst, buf, copylen); + if (w == -1) + return got_error_from_errno("write"); + else if (w != copylen) + return got_error(GOT_ERR_IO); remain -= copylen; } @@ -2141,18 +2146,19 @@ hcopy(FILE *fsrc, FILE *fdst, off_t len, SHA1_CTX *ctx static const struct got_error * hcopy_mmap(uint8_t *src, off_t src_offset, size_t src_size, - FILE *fdst, off_t len, SHA1_CTX *ctx) + int fd, off_t len, SHA1_CTX *ctx) { - size_t n; + ssize_t w; if (src_offset + len > src_size) return got_error(GOT_ERR_RANGE); SHA1Update(ctx, src + src_offset, len); - n = fwrite(src + src_offset, 1, len, fdst); - if (n != len) - return got_ferror(fdst, GOT_ERR_IO); - + w = write(fd, src + src_offset, len); + if (w == -1) + return got_error_from_errno("write"); + else if (w != len) + return got_error(GOT_ERR_IO); return NULL; } @@ -2247,7 +2253,7 @@ packoff(char *hdr, off_t off) } static const struct got_error * -deltahdr(off_t *packfile_size, SHA1_CTX *ctx, FILE *packfile, +deltahdr(off_t *packfile_size, SHA1_CTX *ctx, int packfd, struct got_pack_meta *m) { const struct got_error *err; @@ -2260,7 +2266,7 @@ deltahdr(off_t *packfile_size, SHA1_CTX *ctx, FILE *pa if (err) return err; nh += packoff(buf + nh, m->off - m->prev->off); - err = hwrite(packfile, buf, nh, ctx); + err = hwrite(packfd, buf, nh, ctx); if (err) return err; *packfile_size += nh; @@ -2269,11 +2275,11 @@ deltahdr(off_t *packfile_size, SHA1_CTX *ctx, FILE *pa GOT_OBJ_TYPE_REF_DELTA, m->delta_len); if (err) return err; - err = hwrite(packfile, buf, nh, ctx); + err = hwrite(packfd, buf, nh, ctx); if (err) return err; *packfile_size += nh; - err = hwrite(packfile, m->prev->id.sha1, + err = hwrite(packfd, m->prev->id.sha1, sizeof(m->prev->id.sha1), ctx); if (err) return err; @@ -2284,7 +2290,7 @@ deltahdr(off_t *packfile_size, SHA1_CTX *ctx, FILE *pa } static const struct got_error * -write_packed_object(off_t *packfile_size, FILE *packfile, +write_packed_object(off_t *packfile_size, int packfd, FILE *delta_cache, uint8_t *delta_cache_map, size_t delta_cache_size, struct got_pack_meta *m, int *outfd, SHA1_CTX *ctx, struct got_repository *repo) @@ -2299,7 +2305,7 @@ write_packed_object(off_t *packfile_size, FILE *packfi csum.output_sha1 = ctx; csum.output_crc = NULL; - m->off = ftello(packfile); + m->off = *packfile_size; if (m->delta_len == 0) { err = got_object_raw_open(&raw, outfd, repo, &m->id); if (err) @@ -2308,14 +2314,14 @@ write_packed_object(off_t *packfile_size, FILE *packfi m->obj_type, raw->size); if (err) goto done; - err = hwrite(packfile, buf, nh, ctx); + err = hwrite(packfd, buf, nh, ctx); if (err) goto done; *packfile_size += nh; if (raw->f == NULL) { - err = got_deflate_to_file_mmap(&outlen, + err = got_deflate_to_fd_mmap(&outlen, raw->data + raw->hdrlen, 0, raw->size, - packfile, &csum); + packfd, &csum); if (err) goto done; } else { @@ -2324,8 +2330,8 @@ write_packed_object(off_t *packfile_size, FILE *packfi err = got_error_from_errno("fseeko"); goto done; } - err = got_deflate_to_file(&outlen, raw->f, - raw->size, packfile, &csum); + err = got_deflate_to_fd(&outlen, raw->f, + raw->size, packfd, &csum); if (err) goto done; } @@ -2333,10 +2339,10 @@ write_packed_object(off_t *packfile_size, FILE *packfi got_object_raw_close(raw); raw = NULL; } else if (m->delta_buf) { - err = deltahdr(packfile_size, ctx, packfile, m); + err = deltahdr(packfile_size, ctx, packfd, m); if (err) goto done; - err = hwrite(packfile, m->delta_buf, + err = hwrite(packfd, m->delta_buf, m->delta_compressed_len, ctx); if (err) goto done; @@ -2344,11 +2350,11 @@ write_packed_object(off_t *packfile_size, FILE *packfi free(m->delta_buf); m->delta_buf = NULL; } else if (delta_cache_map) { - err = deltahdr(packfile_size, ctx, packfile, m); + err = deltahdr(packfile_size, ctx, packfd, m); if (err) goto done; err = hcopy_mmap(delta_cache_map, m->delta_offset, - delta_cache_size, packfile, m->delta_compressed_len, + delta_cache_size, packfd, m->delta_compressed_len, ctx); if (err) goto done; @@ -2359,10 +2365,10 @@ write_packed_object(off_t *packfile_size, FILE *packfi err = got_error_from_errno("fseeko"); goto done; } - err = deltahdr(packfile_size, ctx, packfile, m); + err = deltahdr(packfile_size, ctx, packfd, m); if (err) goto done; - err = hcopy(delta_cache, packfile, + err = hcopy(delta_cache, packfd, m->delta_compressed_len, ctx); if (err) goto done; @@ -2375,7 +2381,7 @@ done: } static const struct got_error * -genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delta_cache, +genpack(uint8_t *pack_sha1, int packfd, FILE *delta_cache, struct got_pack_meta **deltify, int ndeltify, struct got_pack_meta **reuse, int nreuse, int ncolored, int nfound, int ntrees, int nours, @@ -2389,7 +2395,7 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt SHA1_CTX ctx; struct got_pack_meta *m; char buf[32]; - size_t n; + ssize_t w; off_t packfile_size = 0; int outfd = -1; int delta_cache_fd = -1; @@ -2420,15 +2426,15 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt } } #endif - err = hwrite(packfile, "PACK", 4, &ctx); + err = hwrite(packfd, "PACK", 4, &ctx); if (err) goto done; putbe32(buf, GOT_PACKFILE_VERSION); - err = hwrite(packfile, buf, 4, &ctx); + err = hwrite(packfd, buf, 4, &ctx); if (err) goto done; putbe32(buf, ndeltify + nreuse); - err = hwrite(packfile, buf, 4, &ctx); + err = hwrite(packfd, buf, 4, &ctx); if (err) goto done; @@ -2441,7 +2447,7 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt if (err) goto done; m = deltify[i]; - err = write_packed_object(&packfile_size, packfile, + err = write_packed_object(&packfile_size, packfd, delta_cache, delta_cache_map, delta_cache_size, m, &outfd, &ctx, repo); if (err) @@ -2457,7 +2463,7 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt if (err) goto done; m = reuse[i]; - err = write_packed_object(&packfile_size, packfile, + err = write_packed_object(&packfile_size, packfd, delta_cache, delta_cache_map, delta_cache_size, m, &outfd, &ctx, repo); if (err) @@ -2465,9 +2471,13 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt } SHA1Final(pack_sha1, &ctx); - n = fwrite(pack_sha1, 1, SHA1_DIGEST_LENGTH, packfile); - if (n != SHA1_DIGEST_LENGTH) - err = got_ferror(packfile, GOT_ERR_IO); + w = write(packfd, pack_sha1, SHA1_DIGEST_LENGTH); + if (w == -1) + err = got_error_from_errno("write"); + else if (w != SHA1_DIGEST_LENGTH) + err = got_error(GOT_ERR_IO); + if (err) + goto done; packfile_size += SHA1_DIGEST_LENGTH; packfile_size += sizeof(struct got_packfile_hdr); if (progress_cb) { @@ -2500,7 +2510,7 @@ add_meta_idset_cb(struct got_object_id *id, void *data } const struct got_error * -got_pack_create(uint8_t *packsha1, FILE *packfile, +got_pack_create(uint8_t *packsha1, int packfd, struct got_object_id **theirs, int ntheirs, struct got_object_id **ours, int nours, struct got_repository *repo, int loose_obj_only, int allow_empty, @@ -2605,7 +2615,7 @@ got_pack_create(uint8_t *packsha1, FILE *packfile, err = got_error_from_errno("fflush"); goto done; } - err = genpack(packsha1, packfile, delta_cache, deltify.meta, + err = genpack(packsha1, packfd, delta_cache, deltify.meta, deltify.nmeta, reuse.meta, reuse.nmeta, ncolored, nfound, ntrees, nours, repo, progress_cb, progress_arg, &rl, cancel_cb, cancel_arg); blob - 2cf33ed80132762b274635d2ca08a08e5e5a5f22 blob + 77fcb6b22fcd226378437eda19e4fe0c96478ada --- lib/repository_admin.c +++ lib/repository_admin.c @@ -172,13 +172,6 @@ got_repo_pack_objects(FILE **packfile, struct got_obje goto done; } - *packfile = fdopen(packfd, "w"); - if (*packfile == NULL) { - err = got_error_from_errno2("fdopen", tmpfile_path); - goto done; - } - packfd = -1; - err = get_reflist_object_ids(&ours, &nours, (1 << GOT_OBJ_TYPE_COMMIT) | (1 << GOT_OBJ_TYPE_TAG), include_refs, repo, cancel_cb, cancel_arg); @@ -205,7 +198,7 @@ got_repo_pack_objects(FILE **packfile, struct got_obje goto done; } - err = got_pack_create((*pack_hash)->sha1, *packfile, theirs, ntheirs, + err = got_pack_create((*pack_hash)->sha1, packfd, theirs, ntheirs, ours, nours, repo, loose_obj_only, 0, progress_cb, progress_arg, cancel_cb, cancel_arg); if (err) @@ -221,12 +214,8 @@ got_repo_pack_objects(FILE **packfile, struct got_obje goto done; } - if (fflush(*packfile) == -1) { - err = got_error_from_errno("fflush"); - goto done; - } - if (fseek(*packfile, 0L, SEEK_SET) == -1) { - err = got_error_from_errno("fseek"); + if (lseek(packfd, 0L, SEEK_SET) == -1) { + err = got_error_from_errno("lseek"); goto done; } if (rename(tmpfile_path, packfile_path) == -1) { @@ -236,6 +225,13 @@ got_repo_pack_objects(FILE **packfile, struct got_obje } free(tmpfile_path); tmpfile_path = NULL; + + *packfile = fdopen(packfd, "w"); + if (*packfile == NULL) { + err = got_error_from_errno2("fdopen", tmpfile_path); + goto done; + } + packfd = -1; done: for (i = 0; i < nours; i++) free(ours[i]); blob - 69f2244f86117ab0b9aa9b8ad44c2f28052f2f26 blob + 4393fbc3fe2bb713de61b06db047c3fcff6de871 --- lib/send.c +++ lib/send.c @@ -345,7 +345,7 @@ got_send_pack(const char *remote_name, struct got_path off_t bytes_sent = 0, bytes_sent_cur = 0; struct pack_progress_arg ppa; uint8_t packsha1[SHA1_DIGEST_LENGTH]; - FILE *packfile = NULL; + int packfd = -1; TAILQ_INIT(&refs); TAILQ_INIT(&have_refs); @@ -436,9 +436,9 @@ got_send_pack(const char *remote_name, struct got_path } } - packfile = got_opentemp(); - if (packfile == NULL) { - err = got_error_from_errno("got_opentemp"); + packfd = got_opentempfd(); + if (packfd == -1) { + err = got_error_from_errno("got_opentempfd"); goto done; } @@ -636,18 +636,13 @@ got_send_pack(const char *remote_name, struct got_path memset(&ppa, 0, sizeof(ppa)); ppa.progress_cb = progress_cb; ppa.progress_arg = progress_arg; - err = got_pack_create(packsha1, packfile, their_ids, ntheirs, + err = got_pack_create(packsha1, packfd, their_ids, ntheirs, our_ids, nours, repo, 0, 1, pack_progress, &ppa, cancel_cb, cancel_arg); if (err) goto done; - if (fflush(packfile) == -1) { - err = got_error_from_errno("fflush"); - goto done; - } - - npackfd = dup(fileno(packfile)); + npackfd = dup(packfd); if (npackfd == -1) { err = got_error_from_errno("dup"); goto done; @@ -711,8 +706,8 @@ done: if (waitpid(sendpid, &sendstatus, 0) == -1 && err == NULL) err = got_error_from_errno("waitpid"); } - if (packfile && fclose(packfile) == EOF && err == NULL) - err = got_error_from_errno("fclose"); + if (packfd != -1 && close(packfd) == -1 && err == NULL) + err = got_error_from_errno("close"); if (nsendfd != -1 && close(nsendfd) == -1 && err == NULL) err = got_error_from_errno("close"); if (npackfd != -1 && close(npackfd) == -1 && err == NULL)