commit 876c234bd6598fe15505cb4befb6e0809752c450 from: Stefan Sperling date: Mon Sep 10 10:30:47 2018 UTC start reading pack files with privsep; still WIP commit - 40cf7208131e42d6d484d93ea5e4fd69daf3de04 commit + 876c234bd6598fe15505cb4befb6e0809752c450 blob - b1d1830b01b36ea576c29ee80aba577fc0adf4ae blob + f37d59860962d9ba6610ce564206b13bddd1f756 --- lib/got_lib_object_parse.h +++ lib/got_lib_object_parse.h @@ -33,3 +33,10 @@ const struct got_error *got_object_parse_tree(struct g const struct got_error *got_read_file_to_mem(uint8_t **, size_t *, FILE *); void got_object_tree_entry_close(struct got_tree_entry *); + +struct got_pack; +struct got_packidx; + +const struct got_error *got_object_packed_read_privsep(struct got_object **, + struct got_repository *, struct got_pack *, struct got_packidx *, int, + struct got_object_id *); blob - 4de7dfb978353377b576337a4fb4bea2052821a4 blob + c36360e3371cbea006d4f4cffe3cad2e726123d4 --- lib/got_lib_pack.h +++ lib/got_lib_pack.h @@ -20,8 +20,10 @@ struct got_pack { int fd; uint8_t *map; size_t filesize; + struct got_privsep_child *privsep_child; }; +const struct got_error *got_pack_stop_privsep_child(struct got_pack *); const struct got_error *got_pack_close(struct got_pack *); #define GOT_PACK_PREFIX "pack-" blob - 1932648fb838ffb3fe750b46ae9dd0e9adebae65 blob + bd788e1cf8afe72c07db5ba44934534ace650175 --- lib/got_lib_privsep.h +++ lib/got_lib_privsep.h @@ -39,6 +39,7 @@ #define GOT_PROG_READ_TREE got-read-tree #define GOT_PROG_READ_COMMIT got-read-commit #define GOT_PROG_READ_BLOB got-read-blob +#define GOT_PROG_READ_PACK got-read-pack #define GOT_STRINGIFY(x) #x #define GOT_STRINGVAL(x) GOT_STRINGIFY(x) @@ -52,7 +53,15 @@ GOT_STRINGVAL(GOT_LIBEXECDIR) "/" GOT_STRINGVAL(GOT_PROG_READ_COMMIT) #define GOT_PATH_PROG_READ_BLOB \ GOT_STRINGVAL(GOT_LIBEXECDIR) "/" GOT_STRINGVAL(GOT_PROG_READ_BLOB) +#define GOT_PATH_PROG_READ_PACK \ + GOT_STRINGVAL(GOT_LIBEXECDIR) "/" GOT_STRINGVAL(GOT_PROG_READ_PACK) +struct got_privsep_child { + int imsg_fd; + pid_t pid; + struct imsgbuf *ibuf; +}; + enum got_imsg_type { /* An error occured while processing a request. */ GOT_IMSG_ERROR, @@ -78,6 +87,12 @@ enum got_imsg_type { GOT_IMSG_BLOB_REQUEST, GOT_IMSG_BLOB_OUTFD, GOT_IMSG_BLOB, + + /* Messages related to pack files. */ + GOT_IMSG_PACKIDX, + GOT_IMSG_PACK, + GOT_IMSG_PACKED_OBJECT_REQUEST, + /* Messages for transmitting deltas and associated delta streams: */ GOT_IMSG_DELTA, GOT_IMSG_DELTA_STREAM, @@ -100,10 +115,8 @@ struct got_imsg_delta { size_t delta_len; /* - * Followed by delta stream in remaining bytes of imsg buffer. - * If delta_len exceeds imsg buffer length, followed by one or - * more DELTA_STREAM messages until delta_len bytes of delta - * stream have been transmitted. + * Followed by one or more DELTA_STREAM messages until delta_len + * bytes of delta stream have been transmitted. */ }; @@ -126,6 +139,7 @@ struct got_imsg_object { size_t hdrlen; size_t size; + off_t pack_offset; int ndeltas; /* this many GOT_IMSG_DELTA messages follow */ }; @@ -166,6 +180,30 @@ struct got_imsg_blob { size_t size; }; +/* Structure for GOT_IMSG_PACKIDX. */ +struct got_imsg_packidx { + size_t len; + /* Additionally, a file desciptor is passed via imsg. */ +}; + +/* Structure for GOT_IMSG_PACK. */ +struct got_imsg_pack { + char path_packfile[PATH_MAX]; + size_t filesize; + /* Additionally, a file desciptor is passed via imsg. */ +}; + +/* + * Structure for GOT_IMSG_PACKED_OBJECT_REQUEST data. + */ +struct got_imsg_packed_object { + int idx; +}; + +struct got_pack; +struct got_packidx; + +const struct got_error *got_privsep_wait_for_child(pid_t); const struct got_error *got_privsep_send_stop(int); const struct got_error *got_privsep_recv_imsg(struct imsg *, struct imsgbuf *, size_t); @@ -174,7 +212,7 @@ const struct got_error *got_privsep_send_obj_req(struc struct got_object *); const struct got_error *got_privsep_send_blob_req(struct imsgbuf *, int, int); const struct got_error *got_privsep_send_obj(struct imsgbuf *, - struct got_object *, int); + struct got_object *); const struct got_error *got_privsep_recv_obj(struct got_object **, struct imsgbuf *); const struct got_error *got_privsep_send_commit(struct imsgbuf *, @@ -187,3 +225,7 @@ const struct got_error *got_privsep_send_tree(struct i struct got_tree_object *); const struct got_error *got_privsep_send_blob(struct imsgbuf *, size_t); const struct got_error *got_privsep_recv_blob(size_t *, struct imsgbuf *); +const struct got_error *got_privsep_init_pack_child(struct imsgbuf *, + struct got_pack *, struct got_packidx *); +const struct got_error *got_privsep_send_packed_obj_req(struct imsgbuf *, int); +const struct got_error *got_privsep_send_pack_child_ready(struct imsgbuf *); blob - 4c18ac2b1f6e9d6d8c7eaf5b8fb1b21cf92d2813 blob + d663456eb11bc81aa37a87e77507c574b311dcc7 --- lib/got_lib_repository.h +++ lib/got_lib_repository.h @@ -44,12 +44,6 @@ struct got_object_cache { int cache_miss; }; -struct got_privsep_child { - int imsg_fd; - pid_t pid; - struct imsgbuf *ibuf; -}; - struct got_repository { char *path; char *path_git_dir; @@ -60,9 +54,6 @@ struct got_repository { /* Open file handles for pack files. */ struct got_pack packs[GOT_PACK_CACHE_SIZE]; - /* Handles to child processes for reading pack files. */ - struct got_privsep_child pack_privsep_children[GOT_PACK_CACHE_SIZE]; - /* Handles to child processes for reading loose objects. */ struct got_privsep_child privsep_children[4]; #define GOT_REPO_PRIVSEP_CHILD_OBJECT 0 blob - 876b0f74524ae19d1d229b5b36fdf463a0b76678 blob + a7327c30c72cf05bcb5f66bd7533321a2618c7f6 --- lib/object.c +++ lib/object.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -214,7 +215,7 @@ open_packed_object(struct got_object **obj, struct got goto done; } - err = got_packfile_open_object(obj, pack, packidx, idx, id); + err = got_object_packed_read_privsep(obj, repo, pack, packidx, idx, id); if (err) goto done; blob - ae0f79da9b119071a9e68b2b7e4dc9471dc0fbbe blob + e65fda042d9ca8b020b2446672505c315fe8f146 --- lib/object_parse.c +++ lib/object_parse.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -164,6 +165,100 @@ got_object_read_header_privsep(struct got_object **obj return request_object(obj, repo, obj_fd); } +static const struct got_error * +request_packed_object(struct got_object **obj, struct got_pack *pack, int idx, + struct got_object_id *id) +{ + const struct got_error *err = NULL; + struct imsgbuf *ibuf = pack->privsep_child->ibuf; + + err = got_privsep_send_packed_obj_req(ibuf, idx); + if (err) + return err; + + err = got_privsep_recv_obj(obj, ibuf); + if (err) + return err; + + (*obj)->path_packfile = strdup(pack->path_packfile); + if ((*obj)->path_packfile == NULL) { + err = got_error_from_errno(); + return err; + } + memcpy(&(*obj)->id, id, sizeof((*obj)->id)); + + return NULL; +} + +const struct got_error * +got_object_packed_read_privsep(struct got_object **obj, + struct got_repository *repo, struct got_pack *pack, + struct got_packidx *packidx, int idx, struct got_object_id *id) +{ + const struct got_error *err = NULL; + int imsg_fds[2]; + pid_t pid; + struct imsgbuf *ibuf; + + if (pack->privsep_child) + return request_packed_object(obj, pack, idx, id); + + ibuf = calloc(1, sizeof(*ibuf)); + if (ibuf == NULL) + return got_error_from_errno(); + + pack->privsep_child = calloc(1, sizeof(*pack->privsep_child)); + if (pack->privsep_child == NULL) { + err = got_error_from_errno(); + free(ibuf); + return err; + } + + if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_fds) == -1) { + err = got_error_from_errno(); + goto done; + } + + pid = fork(); + if (pid == -1) { + err = got_error_from_errno(); + goto done; + } else if (pid == 0) { + exec_privsep_child(imsg_fds, GOT_PATH_PROG_READ_PACK, + pack->path_packfile); + /* not reached */ + } + + close(imsg_fds[1]); + pack->privsep_child->imsg_fd = imsg_fds[0]; + pack->privsep_child->pid = pid; + imsg_init(ibuf, imsg_fds[0]); + pack->privsep_child->ibuf = ibuf; + + err = got_privsep_init_pack_child(ibuf, pack, packidx); + if (err) { + const struct got_error *child_err; + err = got_privsep_send_stop(pack->privsep_child->imsg_fd); + child_err = got_privsep_wait_for_child( + pack->privsep_child->pid); + if (child_err && err == NULL) + err = child_err; + free(ibuf); + free(pack->privsep_child); + pack->privsep_child = NULL; + return err; + } + +done: + if (err) { + free(ibuf); + free(pack->privsep_child); + pack->privsep_child = NULL; + } else + err = request_packed_object(obj, pack, idx, id); + return err; +} + struct got_commit_object * got_object_commit_alloc_partial(void) { blob - d0a4ae18c3cecf4dc659772b0c5f005b5d15119b blob + 00d5a158a66d76ebd4b9210a50d5bbabccef3c7b --- lib/pack.c +++ lib/pack.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -29,6 +30,7 @@ #include #include #include +#include #include "got_error.h" #include "got_object.h" @@ -40,6 +42,7 @@ #include "got_lib_delta.h" #include "got_lib_inflate.h" #include "got_lib_object.h" +#include "got_lib_privsep.h" #ifndef nitems #define nitems(_a) (sizeof(_a) / sizeof((_a)[0])) @@ -457,6 +460,24 @@ got_packidx_get_object_idx(struct got_packidx *packidx } return -1; +} + +const struct got_error * +got_pack_stop_privsep_child(struct got_pack *pack) +{ + const struct got_error *err = NULL, *child_err = NULL; + + if (pack->privsep_child == NULL) + return NULL; + + err = got_privsep_send_stop(pack->privsep_child->imsg_fd); + child_err = got_privsep_wait_for_child( + pack->privsep_child->pid); + if (child_err && err == NULL) + err = child_err; + free(pack->privsep_child); + pack->privsep_child = NULL; + return err; } const struct got_error * @@ -464,7 +485,8 @@ got_pack_close(struct got_pack *pack) { const struct got_error *err = NULL; - if (pack->map && munmap(pack->map, pack->filesize) == -1) + err = got_pack_stop_privsep_child(pack); + if (pack->map && munmap(pack->map, pack->filesize) == -1 && !err) err = got_error_from_errno(); close(pack->fd); pack->fd = -1; @@ -550,7 +572,8 @@ open_plain_object(struct got_object **obj, const char (*obj)->flags = GOT_OBJ_FLAG_PACKED; (*obj)->hdrlen = 0; (*obj)->size = size; - memcpy(&(*obj)->id, id, sizeof((*obj)->id)); + if (id) + memcpy(&(*obj)->id, id, sizeof((*obj)->id)); (*obj)->pack_offset = offset; return NULL; @@ -847,7 +870,8 @@ open_delta_object(struct got_object **obj, struct got_ (*obj)->flags = 0; (*obj)->hdrlen = 0; (*obj)->size = 0; /* Not known because deltas aren't applied yet. */ - memcpy(&(*obj)->id, id, sizeof((*obj)->id)); + if (id) + memcpy(&(*obj)->id, id, sizeof((*obj)->id)); (*obj)->pack_offset = offset + tslen; (*obj)->path_packfile = strdup(pack->path_packfile); blob - e01100b024d9a4e2277b55413469962af808b8c4 blob + fbd2510c80004cc514a078c8e4d6c7a2c2b89fa3 --- lib/privsep.c +++ lib/privsep.c @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include #include @@ -38,6 +40,7 @@ #include "got_lib_object.h" #include "got_lib_object_parse.h" #include "got_lib_privsep.h" +#include "got_lib_pack.h" #ifndef MIN #define MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b)) @@ -88,18 +91,37 @@ read_imsg(struct imsgbuf *ibuf) } const struct got_error * +got_privsep_wait_for_child(pid_t pid) +{ + int child_status; + + waitpid(pid, &child_status, 0); + + if (!WIFEXITED(child_status)) + return got_error(GOT_ERR_PRIVSEP_DIED); + + if (WEXITSTATUS(child_status) != 0) + return got_error(GOT_ERR_PRIVSEP_EXIT); + + return NULL; +} + +const struct got_error * got_privsep_recv_imsg(struct imsg *imsg, struct imsgbuf *ibuf, size_t min_datalen) { const struct got_error *err; ssize_t n; - err = read_imsg(ibuf); - if (err) - return err; - n = imsg_get(ibuf, imsg); - if (n == 0) - return got_error(GOT_ERR_PRIVSEP_READ); + if (n == -1) + return got_error_from_errno(); + + while (n == 0) { + err = read_imsg(ibuf); + if (err) + return err; + n = imsg_get(ibuf, imsg); + } if (imsg->hdr.len < IMSG_HEADER_SIZE + min_datalen) return got_error(GOT_ERR_PRIVSEP_LEN); @@ -216,6 +238,8 @@ got_privsep_send_obj_req(struct imsgbuf *ibuf, int fd, iobj.hdrlen = obj->hdrlen; iobj.size = obj->size; iobj.ndeltas = 0; + if (iobj.flags & GOT_OBJ_FLAG_PACKED) + iobj.pack_offset = obj->pack_offset; iobjp = &iobj; iobj_size = sizeof(iobj); @@ -254,28 +278,161 @@ got_privsep_send_blob_req(struct imsgbuf *ibuf, int ou return flush_imsg(ibuf); } +static const struct got_error * +send_delta(struct got_delta *delta, struct imsgbuf *ibuf) +{ + struct got_imsg_delta idelta; + size_t offset, remain; + + idelta.offset = delta->offset; + idelta.tslen = delta->tslen; + idelta.type = delta->type; + idelta.size = delta->size; + idelta.data_offset = delta->data_offset; + idelta.delta_len = delta->delta_len; + + if (imsg_compose(ibuf, GOT_IMSG_DELTA, 0, 0, -1, + &idelta, sizeof(idelta)) == -1) + return got_error_from_errno(); + + if (imsg_flush(ibuf) == -1) + return got_error_from_errno(); + + offset = 0; + remain = delta->delta_len; + while (remain > 0) { + size_t n = MIN(MAX_IMSGSIZE - IMSG_HEADER_SIZE, remain); + + if (imsg_compose(ibuf, GOT_IMSG_DELTA_STREAM, 0, 0, -1, + delta->delta_buf + offset, n) == -1) + return got_error_from_errno(); + + if (imsg_flush(ibuf) == -1) + return got_error_from_errno(); + + offset += n; + remain -= n; + } + + return NULL; +} + const struct got_error * -got_privsep_send_obj(struct imsgbuf *ibuf, struct got_object *obj, int ndeltas) +got_privsep_send_obj(struct imsgbuf *ibuf, struct got_object *obj) { + const struct got_error *err = NULL; struct got_imsg_object iobj; + struct got_delta *delta; iobj.type = obj->type; iobj.flags = obj->flags; iobj.hdrlen = obj->hdrlen; iobj.size = obj->size; - iobj.ndeltas = ndeltas; + iobj.ndeltas = obj->deltas.nentries; + if (iobj.flags & GOT_OBJ_FLAG_PACKED) + iobj.pack_offset = obj->pack_offset; - if (ndeltas > 0) { - /* TODO: Handle deltas */ - } - if (imsg_compose(ibuf, GOT_IMSG_OBJECT, 0, 0, -1, &iobj, sizeof(iobj)) == -1) return got_error_from_errno(); - return flush_imsg(ibuf); + err = flush_imsg(ibuf); + if (err) + return err; + + SIMPLEQ_FOREACH(delta, &obj->deltas.entries, entry) { + err = send_delta(delta, ibuf); + if (err) + break; + } + + return err; } +static const struct got_error * +receive_delta(struct got_delta **delta, struct imsgbuf *ibuf) +{ + const struct got_error *err = NULL; + struct imsg imsg; + struct got_imsg_delta idelta; + uint8_t *delta_buf = NULL; + const size_t min_datalen = + MIN(sizeof(struct got_imsg_error), sizeof(struct got_imsg_delta)); + size_t datalen, offset, remain; + + err = got_privsep_recv_imsg(&imsg, ibuf, min_datalen); + if (err) + return err; + + datalen = imsg.hdr.len - IMSG_HEADER_SIZE; + if (imsg.hdr.type == GOT_IMSG_ERROR) + return recv_imsg_error(&imsg, datalen); + + if (imsg.hdr.type != GOT_IMSG_DELTA) + return got_error(GOT_ERR_PRIVSEP_MSG); + if (datalen != sizeof(idelta)) + return got_error(GOT_ERR_PRIVSEP_LEN); + + memcpy(&idelta, imsg.data, sizeof(idelta)); + imsg_free(&imsg); + + switch (idelta.type) { + case GOT_OBJ_TYPE_OFFSET_DELTA: + case GOT_OBJ_TYPE_REF_DELTA: + if (idelta.delta_len < GOT_DELTA_STREAM_LENGTH_MIN) + return got_error(GOT_ERR_BAD_DELTA); + break; + default: + if (idelta.delta_len != 0) + return got_error(GOT_ERR_BAD_DELTA); + break; + } + + if (idelta.delta_len > 0) { + delta_buf = calloc(1, idelta.delta_len); + if (delta_buf == NULL) + return got_error_from_errno(); + + offset = 0; + remain = idelta.delta_len; + while (remain > 0) { + size_t n = MIN(MAX_IMSGSIZE - IMSG_HEADER_SIZE, remain); + + err = got_privsep_recv_imsg(&imsg, ibuf, n); + if (err) + return err; + + if (imsg.hdr.type == GOT_IMSG_ERROR) + return recv_imsg_error(&imsg, datalen); + + if (imsg.hdr.type == GOT_IMSG_STOP) + break; + + if (imsg.hdr.type != GOT_IMSG_DELTA_STREAM) + return got_error(GOT_ERR_PRIVSEP_MSG); + + datalen = imsg.hdr.len - IMSG_HEADER_SIZE; + if (datalen != n) + return got_error(GOT_ERR_PRIVSEP_LEN); + + memcpy(delta_buf + offset, imsg.data, n); + imsg_free(&imsg); + + offset += n; + remain -= n; + } + } + + *delta = got_delta_open(idelta.offset, idelta.tslen, idelta.type, + idelta.size, idelta.data_offset, delta_buf, idelta.delta_len); + if (*delta == NULL) { + err = got_error_from_errno(); + free(delta_buf); + } + + return err; +} + const struct got_error * got_privsep_recv_obj(struct got_object **obj, struct imsgbuf *ibuf) { @@ -311,6 +468,11 @@ got_privsep_recv_obj(struct got_object **obj, struct i err = got_error(GOT_ERR_PRIVSEP_LEN); break; } + if (iobj.ndeltas > 0 && + (iobj.flags & GOT_OBJ_FLAG_DELTIFIED) == 0) { + err = got_error(GOT_ERR_BAD_OBJ_DATA); + break; + } *obj = calloc(1, sizeof(**obj)); if (*obj == NULL) { @@ -319,10 +481,20 @@ got_privsep_recv_obj(struct got_object **obj, struct i } (*obj)->type = iobj.type; + (*obj)->flags = iobj.flags; (*obj)->hdrlen = iobj.hdrlen; (*obj)->size = iobj.size; + /* id and path_packfile might be copied in by caller */ + (*obj)->pack_offset = iobj.pack_offset; + SIMPLEQ_INIT(&(*obj)->deltas.entries); for (i = 0; i < iobj.ndeltas; i++) { - /* TODO: Handle deltas */ + struct got_delta *delta; + err = receive_delta(&delta, ibuf); + if (err) + break; + (*obj)->deltas.nentries++; + SIMPLEQ_INSERT_TAIL(&(*obj)->deltas.entries, delta, + entry); } break; default: @@ -745,3 +917,50 @@ got_privsep_recv_blob(size_t *size, struct imsgbuf *ib return err; } + +const struct got_error * +got_privsep_init_pack_child(struct imsgbuf *ibuf, struct got_pack *pack, + struct got_packidx *packidx) +{ + struct got_imsg_packidx ipackidx; + struct got_imsg_pack ipack; + int fd; + + ipackidx.len = packidx->len; + fd = dup(packidx->fd); + if (fd == -1) + return got_error_from_errno(); + + if (imsg_compose(ibuf, GOT_IMSG_PACKIDX, 0, 0, fd, &ipackidx, + sizeof(ipackidx)) == -1) + return got_error_from_errno(); + + if (strlcpy(ipack.path_packfile, pack->path_packfile, + sizeof(ipack.path_packfile)) >= sizeof(ipack.path_packfile)) + return got_error(GOT_ERR_NO_SPACE); + ipack.filesize = pack->filesize; + + fd = dup(pack->fd); + if (fd == -1) + return got_error_from_errno(); + + if (imsg_compose(ibuf, GOT_IMSG_PACK, 0, 0, fd, &ipack, sizeof(ipack)) + == -1) + return got_error_from_errno(); + + return flush_imsg(ibuf); +} + +const struct got_error * +got_privsep_send_packed_obj_req(struct imsgbuf *ibuf, int idx) +{ + struct got_imsg_packed_object iobj; + + iobj.idx = idx; + + if (imsg_compose(ibuf, GOT_IMSG_PACKED_OBJECT_REQUEST, 0, 0, -1, + &iobj, sizeof(iobj)) == -1) + return got_error_from_errno(); + + return flush_imsg(ibuf); +} blob - d7db2f58c13d7070176e00eb3563b5a567520170 blob + f0fb85579ae6052de9bd7214f47e96826c6d1577 --- lib/repository.c +++ lib/repository.c @@ -18,8 +18,8 @@ #include #include #include -#include #include +#include #include #include @@ -45,10 +45,10 @@ #include "got_lib_inflate.h" #include "got_lib_object.h" #include "got_lib_pack.h" +#include "got_lib_privsep.h" #include "got_lib_repository.h" #include "got_lib_worktree.h" #include "got_lib_object_idcache.h" -#include "got_lib_privsep.h" #ifndef nitems #define nitems(_a) (sizeof(_a) / sizeof((_a)[0])) @@ -531,22 +531,6 @@ void check_refcount(struct got_object_id *id, void *da } #endif -static const struct got_error * -wait_for_child(pid_t pid) -{ - int child_status; - - waitpid(pid, &child_status, 0); - - if (!WIFEXITED(child_status)) - return got_error(GOT_ERR_PRIVSEP_DIED); - - if (WEXITSTATUS(child_status) != 0) - return got_error(GOT_ERR_PRIVSEP_EXIT); - - return NULL; -} - const struct got_error * got_repo_close(struct got_repository *repo) { @@ -593,7 +577,8 @@ got_repo_close(struct got_repository *repo) imsg_clear(repo->privsep_children[i].ibuf); free(repo->privsep_children[i].ibuf); err = got_privsep_send_stop(repo->privsep_children[i].imsg_fd); - child_err = wait_for_child(repo->privsep_children[i].pid); + child_err = got_privsep_wait_for_child( + repo->privsep_children[i].pid); if (child_err && err == NULL) err = child_err; close(repo->privsep_children[i].imsg_fd); blob - 33f79980386c63a8da9a85071f4a800d04fd8d20 blob + ca4265c6c9cffaefb598a20e1a7d751b1b8d97aa --- libexec/Makefile +++ libexec/Makefile @@ -1,3 +1,4 @@ -SUBDIR = got-read-blob got-read-commit got-read-object got-read-tree +SUBDIR = got-read-blob got-read-commit got-read-object got-read-tree \ + got-read-pack .include blob - 2302e711eaaf23075b4319ef22ba5cc36caefdff blob + eaf315f5317e307d53c4178c251da9213dd36241 --- libexec/got-read-blob/got-read-blob.c +++ libexec/got-read-blob/got-read-blob.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -63,7 +64,7 @@ main(int argc, char *argv[]) err = got_privsep_recv_imsg(&imsg, &ibuf, 0); if (err) { - if (imsg.hdr.len == 0) + if (err->code == GOT_ERR_PRIVSEP_PIPE) err = NULL; break; } @@ -132,7 +133,10 @@ done: imsg_free(&imsg); imsg_free(&imsg_outfd); if (err) { - got_privsep_send_error(&ibuf, err); + if (err->code == GOT_ERR_PRIVSEP_PIPE) + err = NULL; + else + got_privsep_send_error(&ibuf, err); break; } } blob - 178dd961e87c303cc67120aaa8d9878f9fb741b2 blob + 7936f1a29d699fa3b1c6a3252f558c313a59f205 --- libexec/got-read-commit/got-read-commit.c +++ libexec/got-read-commit/got-read-commit.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -90,7 +91,7 @@ main(int argc, char *argv[]) err = got_privsep_recv_imsg(&imsg, &ibuf, 0); if (err) { - if (imsg.hdr.len == 0) + if (err->code == GOT_ERR_PRIVSEP_PIPE) err = NULL; break; } @@ -150,7 +151,10 @@ done: if (obj) got_object_close(obj); if (err) { - got_privsep_send_error(&ibuf, err); + if (err->code == GOT_ERR_PRIVSEP_PIPE) + err = NULL; + else + got_privsep_send_error(&ibuf, err); break; } } blob - 0911e80092f384fa5eec6a65e28f5b24b5849066 blob + 8a787460e3108b743e5595fb7ee1bb1a2da55afb --- libexec/got-read-object/got-read-object.c +++ libexec/got-read-object/got-read-object.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -165,7 +166,7 @@ main(int argc, char *argv[]) while (1) { err = got_privsep_recv_imsg(&imsg, &ibuf, 0); if (err) { - if (imsg.hdr.len == 0) + if (err->code == GOT_ERR_PRIVSEP_PIPE) err = NULL; break; } @@ -188,14 +189,17 @@ main(int argc, char *argv[]) if (err) goto done; - err = got_privsep_send_obj(&ibuf, obj, 0); + err = got_privsep_send_obj(&ibuf, obj); done: close(imsg.fd); imsg_free(&imsg); if (obj) got_object_close(obj); if (err) { - got_privsep_send_error(&ibuf, err); + if (err->code == GOT_ERR_PRIVSEP_PIPE) + err = NULL; + else + got_privsep_send_error(&ibuf, err); break; } } blob - 43696a70aa71949a1566e43e2e43d3f9b0076481 blob + cb003bb2a8f4e554783c2a0c22796481435ee1f3 --- libexec/got-read-tree/got-read-tree.c +++ libexec/got-read-tree/got-read-tree.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -89,7 +90,7 @@ main(int argc, char *argv[]) err = got_privsep_recv_imsg(&imsg, &ibuf, 0); if (err) { - if (imsg.hdr.len == 0) + if (err->code == GOT_ERR_PRIVSEP_PIPE) err = NULL; break; } @@ -149,7 +150,10 @@ done: if (obj) got_object_close(obj); if (err) { - got_privsep_send_error(&ibuf, err); + if (err->code == GOT_ERR_PRIVSEP_PIPE) + err = NULL; + else + got_privsep_send_error(&ibuf, err); break; } } blob - /dev/null blob + 688107f4d8b2a5ce11d51c5dab085e2ec0da4d97 (mode 644) --- /dev/null +++ libexec/got-read-pack/Makefile @@ -0,0 +1,20 @@ +.PATH:${.CURDIR}/../../lib + +PROG= got-read-pack +SRCS= got-read-pack.c delta.c error.c inflate.c object_parse.c \ + opentemp.c pack.c path.c privsep.c sha1.c + +CPPFLAGS = -I${.CURDIR}/../../include -I${.CURDIR}/../../lib +LDADD = -lutil -lz +DPADD = ${LIBZ} ${LIBUTIL} + +# For now, default to installing binary in ~/bin +GROUP!=id -g -n +install: + ${INSTALL} ${INSTALL_COPY} -o ${USER} -g ${GROUP} \ + -m ${BINMODE} ${PROG} ${GOT_LIBEXECDIR}/${PROG} + +# Don't install man pages yet +NOMAN = Yes + +.include blob - /dev/null blob + f243749639d228ffa0c9ee34ec70c83ce07e3b5d (mode 644) --- /dev/null +++ libexec/got-read-pack/got-read-pack.c @@ -0,0 +1,299 @@ +/* + * Copyright (c) 2018 Stefan Sperling + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "got_error.h" +#include "got_object.h" + +#include "got_lib_delta.h" +#include "got_lib_inflate.h" +#include "got_lib_object.h" +#include "got_lib_object_parse.h" +#include "got_lib_privsep.h" +#include "got_lib_pack.h" + +static const struct got_error * +object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack, + struct got_packidx *packidx) +{ + const struct got_error *err = NULL; + struct got_imsg_packed_object iobj; + struct got_object *obj; + size_t datalen; + + datalen = imsg->hdr.len - IMSG_HEADER_SIZE; + if (datalen != sizeof(iobj)) + return got_error(GOT_ERR_PRIVSEP_LEN); + memcpy(&iobj, imsg->data, sizeof(iobj)); + + err = got_packfile_open_object(&obj, pack, packidx, iobj.idx, NULL); + if (err) + return err; + + err = got_privsep_send_obj(ibuf, obj); + got_object_close(obj); + return err; +} + +static const struct got_error * +commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack, + struct got_packidx *packidx) +{ + return got_error(GOT_ERR_NOT_IMPL); +} + +static const struct got_error * +tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack, + struct got_packidx *packidx) +{ + return got_error(GOT_ERR_NOT_IMPL); +} + +static const struct got_error * +blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack, + struct got_packidx *packidx) +{ + return got_error(GOT_ERR_NOT_IMPL); +} + +static const struct got_error * +receive_packidx(struct got_packidx **packidx, struct imsgbuf *ibuf) +{ + const struct got_error *err = NULL; + struct imsg imsg; + struct got_imsg_packidx ipackidx; + size_t datalen; + struct got_packidx *p; + + *packidx = NULL; + + err = got_privsep_recv_imsg(&imsg, ibuf, 0); + if (err) + return err; + + p = calloc(1, sizeof(*p)); + if (p == NULL) { + err = got_error_from_errno(); + goto done; + } + + if (imsg.hdr.type != GOT_IMSG_PACKIDX) { + err = got_error(GOT_ERR_PRIVSEP_MSG); + goto done; + } + + if (imsg.fd == -1) { + err = got_error(GOT_ERR_PRIVSEP_NO_FD); + goto done; + } + + datalen = imsg.hdr.len - IMSG_HEADER_SIZE; + if (datalen != sizeof(ipackidx)) { + err = got_error(GOT_ERR_PRIVSEP_LEN); + goto done; + } + memcpy(&ipackidx, imsg.data, sizeof(ipackidx)); + + p->len = ipackidx.len; + p->fd = dup(imsg.fd); + if (p->fd == -1) { + err = got_error_from_errno(); + goto done; + } + +#ifndef GOT_PACK_NO_MMAP + p->map = mmap(NULL, p->len, PROT_READ, MAP_PRIVATE, p->fd, 0); + if (p->map == MAP_FAILED) + p->map = NULL; /* fall back to read(2) */ +#endif + err = got_packidx_init_hdr(p, 1); +done: + if (err) { + if (imsg.fd != -1) + close(imsg.fd); + got_packidx_close(p); + } else + *packidx = p; + imsg_free(&imsg); + return err; +} + +static const struct got_error * +receive_pack(struct got_pack **packp, struct imsgbuf *ibuf) +{ + const struct got_error *err = NULL; + struct imsg imsg; + struct got_imsg_pack ipack; + size_t datalen; + struct got_pack *pack; + + *packp = NULL; + + err = got_privsep_recv_imsg(&imsg, ibuf, 0); + if (err) + return err; + + pack = calloc(1, sizeof(*pack)); + if (pack == NULL) { + err = got_error_from_errno(); + goto done; + } + + if (imsg.hdr.type != GOT_IMSG_PACK) { + err = got_error(GOT_ERR_PRIVSEP_MSG); + goto done; + } + + if (imsg.fd == -1) { + err = got_error(GOT_ERR_PRIVSEP_NO_FD); + goto done; + } + + datalen = imsg.hdr.len - IMSG_HEADER_SIZE; + if (datalen != sizeof(ipack)) { + err = got_error(GOT_ERR_PRIVSEP_LEN); + goto done; + } + memcpy(&ipack, imsg.data, sizeof(ipack)); + + pack->filesize = ipack.filesize; + pack->fd = dup(imsg.fd); + if (pack->fd == -1) { + err = got_error_from_errno(); + goto done; + } + pack->path_packfile = strdup(ipack.path_packfile); + if (pack->path_packfile == NULL) { + err = got_error_from_errno(); + goto done; + } + +#ifndef GOT_PACK_NO_MMAP + pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE, + pack->fd, 0); + if (pack->map == MAP_FAILED) + pack->map = NULL; /* fall back to read(2) */ +#endif +done: + if (err) { + if (imsg.fd != -1) + close(imsg.fd); + free(pack); + } else + *packp = pack; + imsg_free(&imsg); + return err; +} + +int +main(int argc, char *argv[]) +{ + const struct got_error *err = NULL; + struct imsgbuf ibuf; + struct imsg imsg; + struct got_packidx *packidx; + struct got_pack *pack; + + //static int attached; + //while (!attached) sleep(1); + + imsg_init(&ibuf, GOT_IMSG_FD_CHILD); + + /* revoke access to most system calls */ + if (pledge("stdio recvfd", NULL) == -1) { + err = got_error_from_errno(); + got_privsep_send_error(&ibuf, err); + return 1; + } + + err = receive_packidx(&packidx, &ibuf); + if (err) { + got_privsep_send_error(&ibuf, err); + return 1; + } + + err = receive_pack(&pack, &ibuf); + if (err) { + got_privsep_send_error(&ibuf, err); + return 1; + } + + while (1) { + imsg.fd = -1; + + err = got_privsep_recv_imsg(&imsg, &ibuf, 0); + if (err) { + if (err->code == GOT_ERR_PRIVSEP_PIPE) + err = NULL; + break; + } + + if (imsg.hdr.type == GOT_IMSG_STOP) + break; + + switch (imsg.hdr.type) { + case GOT_IMSG_PACKED_OBJECT_REQUEST: + err = object_request(&imsg, &ibuf, pack, packidx); + break; + case GOT_IMSG_COMMIT_REQUEST: + err = commit_request(&imsg, &ibuf, pack, packidx); + break; + case GOT_IMSG_TREE_REQUEST: + err = tree_request(&imsg, &ibuf, pack, packidx); + break; + case GOT_IMSG_BLOB_REQUEST: + err = blob_request(&imsg, &ibuf, pack, packidx); + break; + default: + err = got_error(GOT_ERR_PRIVSEP_MSG); + break; + } + + if (imsg.fd != -1) + close(imsg.fd); + imsg_free(&imsg); + if (err) { + if (err->code == GOT_ERR_PRIVSEP_PIPE) + err = NULL; + else + got_privsep_send_error(&ibuf, err); + break; + } + } + + got_pack_close(pack); + imsg_clear(&ibuf); + if (err) + fprintf(stderr, "%s: %s\n", getprogname(), err->msg); + close(GOT_IMSG_FD_CHILD); + return err ? 1 : 0; +}