commit 710f3f4e5168164687c5586641920d82960b0a3a from: Stefan Sperling date: Mon Nov 05 14:26:18 2018 UTC make got-read-pack pre-seed the main process mini commit cache commit - 05e1230b0e68bb81cdae380edccc969dd16215dc commit + 710f3f4e5168164687c5586641920d82960b0a3a blob - 0be7fadfe9c2f58693e847bba52c229ef5fc6ca7 blob + 45ef92bdc8abd72d2d49020aa475062d8ba4047c --- lib/got_lib_object_parse.h +++ lib/got_lib_object_parse.h @@ -52,6 +52,7 @@ const struct got_error *got_object_packed_read_privsep const struct got_error *got_object_read_packed_commit_privsep( struct got_commit_object **, struct got_object *, struct got_pack *); const struct got_error *got_object_read_packed_mini_commit_privsep( - struct got_mini_commit_object **, struct got_object *, struct got_pack *); + struct got_mini_commit_object **, struct got_object *, struct got_pack *, + struct got_repository *); const struct got_error *got_object_read_packed_tree_privsep( struct got_tree_object **, struct got_object *, struct got_pack *); blob - bd18a5bd6340e39b7060420602783f883176a779 blob + 4a84a701e68e04e761b6bbb494ea7edad584cfa2 --- lib/got_lib_privsep.h +++ lib/got_lib_privsep.h @@ -150,6 +150,13 @@ struct got_imsg_commit_object_mini { struct tm tm_committer; int nparents; + /* + * Set if this commit is a parent of the requested commit. + * Allows got-read-pack to seed the main process mini-commit cache. + */ + int is_parent; + uint8_t parent_id[SHA1_DIGEST_LENGTH]; + /* Followed by 'nparents' SHA1_DIGEST_LENGTH length strings */ } __attribute__((__packed__)); @@ -213,11 +220,12 @@ const struct got_error *got_privsep_recv_obj(struct go const struct got_error *got_privsep_send_commit(struct imsgbuf *, struct got_commit_object *); const struct got_error *got_privsep_send_mini_commit(struct imsgbuf *, - struct got_mini_commit_object *); + struct got_mini_commit_object *, struct got_object_id *); const struct got_error *got_privsep_recv_commit(struct got_commit_object **, struct imsgbuf *); const struct got_error *got_privsep_recv_mini_commit( - struct got_mini_commit_object **, struct imsgbuf *); + struct got_mini_commit_object **, struct got_object_id **, + struct imsgbuf *); const struct got_error *got_privsep_recv_tree(struct got_tree_object **, struct imsgbuf *); const struct got_error *got_privsep_send_tree(struct imsgbuf *, blob - fff88ff5430dd2dbee093f31aaf74496bb5e164f blob + 959246bfdf1644ebd7609f3be3417a60a5e723ed --- lib/object.c +++ lib/object.c @@ -378,7 +378,7 @@ open_mini_commit(struct got_mini_commit_object **commi return err; } err = got_object_read_packed_mini_commit_privsep(commit, obj, - pack); + pack, repo); } else { int fd; err = open_loose_object(&fd, obj, repo); @@ -1179,12 +1179,11 @@ request_mini_commit(struct got_mini_commit_object **co struct imsgbuf *ibuf; ibuf = repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_COMMIT].ibuf; - err = got_privsep_send_mini_commit_req(ibuf, fd, obj); if (err) return err; - return got_privsep_recv_mini_commit(commit, ibuf); + return got_privsep_recv_mini_commit(commit, NULL, ibuf); } const struct got_error * @@ -1201,17 +1200,32 @@ got_object_read_packed_commit_privsep(struct got_commi } const struct got_error * -got_object_read_packed_mini_commit_privsep(struct got_mini_commit_object **commit, - struct got_object *obj, struct got_pack *pack) +got_object_read_packed_mini_commit_privsep( + struct got_mini_commit_object **commit, struct got_object *obj, + struct got_pack *pack, struct got_repository *repo) { const struct got_error *err = NULL; + struct got_object_id *pid = NULL; err = got_privsep_send_mini_commit_req(pack->privsep_child->ibuf, -1, obj); if (err) return err; - return got_privsep_recv_mini_commit(commit, pack->privsep_child->ibuf); + while (1) { + err = got_privsep_recv_mini_commit(commit, &pid, + pack->privsep_child->ibuf); + if (err || pid == NULL) + break; + + /* got-read-pack has sent a parent commit; cache it. */ + err = got_repo_cache_mini_commit(repo, pid, *commit); + free(pid); + if (err) + break; + } + + return err; } const struct got_error * blob - 6d49541d49c07af58c34459694fab0d5ba313cb4 blob + 112f328bc9dd80c6cd7d73c762fbc323cc7a493c --- lib/object_cache.c +++ lib/object_cache.c @@ -35,7 +35,7 @@ #define GOT_OBJECT_CACHE_SIZE_OBJ 1024 #define GOT_OBJECT_CACHE_SIZE_TREE 2048 #define GOT_OBJECT_CACHE_SIZE_COMMIT 512 -#define GOT_OBJECT_CACHE_SIZE_MINI_COMMIT 512 +#define GOT_OBJECT_CACHE_SIZE_MINI_COMMIT 32 const struct got_error * got_object_cache_init(struct got_object_cache *cache, blob - d5e0f72f96ef3ae2c31d5e9c9e24f74e106f881b blob + ff2cfa6aea905cc11db48b3346b4e168e9765425 --- lib/privsep.c +++ lib/privsep.c @@ -39,6 +39,7 @@ #include "got_lib_inflate.h" #include "got_lib_object.h" #include "got_lib_object_parse.h" +#include "got_lib_object_cache.h" #include "got_lib_privsep.h" #include "got_lib_pack.h" @@ -488,7 +489,7 @@ done: const struct got_error * got_privsep_send_mini_commit(struct imsgbuf *ibuf, - struct got_mini_commit_object *commit) + struct got_mini_commit_object *commit, struct got_object_id *parent_id) { const struct got_error *err = NULL; struct got_imsg_commit_object_mini icommit; @@ -501,6 +502,13 @@ got_privsep_send_mini_commit(struct imsgbuf *ibuf, sizeof(icommit.tm_committer)); icommit.nparents = commit->nparents; + icommit.is_parent = parent_id ? 1 : 0; + if (parent_id) + memcpy(icommit.parent_id, parent_id->sha1, + sizeof(icommit.parent_id)); + else + memset(&icommit.parent_id, 0, sizeof(icommit.parent_id)); + total = sizeof(icommit) + icommit.nparents * SHA1_DIGEST_LENGTH; buf = malloc(total); @@ -523,7 +531,8 @@ got_privsep_send_mini_commit(struct imsgbuf *ibuf, free(buf); if (err) return err; - return flush_imsg(ibuf); + + return parent_id ? NULL : flush_imsg(ibuf); } const struct got_error * @@ -679,16 +688,48 @@ got_privsep_recv_commit(struct got_commit_object **com return err; } + +static const struct got_error * +get_mini_commit(struct got_mini_commit_object **commit, + struct got_imsg_commit_object_mini *icommit, uint8_t *data) +{ + const struct got_error *err = NULL; + int i; + + *commit = got_object_mini_commit_alloc_partial(); + if (*commit == NULL) + return got_error_from_errno(); + memcpy((*commit)->tree_id->sha1, icommit->tree_id, + SHA1_DIGEST_LENGTH); + memcpy(&(*commit)->tm_committer, &icommit->tm_committer, + sizeof((*commit)->tm_committer)); + + for (i = 0; i < icommit->nparents; i++) { + struct got_object_qid *qid; + + err = got_object_qid_alloc_partial(&qid); + if (err) + break; + + memcpy(qid->id, + data + sizeof(*icommit) + (i * SHA1_DIGEST_LENGTH), + sizeof(*qid->id)); + SIMPLEQ_INSERT_TAIL(&(*commit)->parent_ids, qid, entry); + (*commit)->nparents++; + } + + return err; +} + const struct got_error * got_privsep_recv_mini_commit(struct got_mini_commit_object **commit, - struct imsgbuf *ibuf) + struct got_object_id **parent_id, struct imsgbuf *ibuf) { const struct got_error *err = NULL; struct imsg imsg; struct got_imsg_commit_object_mini icommit; - size_t len, datalen; - int i; + size_t datalen; const size_t min_datalen = MIN(sizeof(struct got_imsg_error), sizeof(struct got_imsg_commit_object_mini)); @@ -702,7 +743,6 @@ got_privsep_recv_mini_commit(struct got_mini_commit_ob data = imsg.data; datalen = imsg.hdr.len - IMSG_HEADER_SIZE; - len = 0; switch (imsg.hdr.type) { case GOT_IMSG_ERROR: @@ -724,31 +764,18 @@ got_privsep_recv_mini_commit(struct got_mini_commit_ob err = got_error(GOT_ERR_PRIVSEP_LEN); break; } - len += sizeof(icommit); - - *commit = got_object_mini_commit_alloc_partial(); - if (*commit == NULL) { - err = got_error_from_errno(); - break; - } - - memcpy((*commit)->tree_id->sha1, icommit.tree_id, - SHA1_DIGEST_LENGTH); - memcpy(&(*commit)->tm_committer, &icommit.tm_committer, - sizeof((*commit)->tm_committer)); - - for (i = 0; i < icommit.nparents; i++) { - struct got_object_qid *qid; - - err = got_object_qid_alloc_partial(&qid); - if (err) + /* got-read-pack might send us parent commits for caching */ + if (icommit.is_parent) { + if (parent_id == NULL) { + err = got_error(GOT_ERR_PRIVSEP_MSG); break; - - memcpy(qid->id, data + len + i * SHA1_DIGEST_LENGTH, - sizeof(*qid->id)); - SIMPLEQ_INSERT_TAIL(&(*commit)->parent_ids, qid, entry); - (*commit)->nparents++; - } + } + *parent_id = malloc(sizeof(**parent_id)); + memcpy((*parent_id)->sha1, &icommit.parent_id, + sizeof(*(*parent_id)->sha1)); + } else if (parent_id) + *parent_id = NULL; + err = get_mini_commit(commit, &icommit, data); break; default: err = got_error(GOT_ERR_PRIVSEP_MSG); blob - b662f7b17edeb01dcdfee90cc68577e9eac499a8 blob + 8ae1f392af49cd613c2b255b1716dc9310d037e2 --- libexec/got-read-commit/got-read-commit.c +++ libexec/got-read-commit/got-read-commit.c @@ -181,7 +181,7 @@ main(int argc, char *argv[]) err = read_commit_object_mini(&commit, obj, f); if (err) goto done; - err = got_privsep_send_mini_commit(&ibuf, commit); + err = got_privsep_send_mini_commit(&ibuf, commit, NULL); got_object_mini_commit_close(commit); } else { struct got_commit_object *commit; blob - f9e29703a891eb79b40e9960be5268d9fa038865 blob + f1be1b62756ea356027a45e6568e41e7b3765f43 --- libexec/got-read-pack/got-read-pack.c +++ libexec/got-read-pack/got-read-pack.c @@ -145,6 +145,60 @@ commit_request(struct imsg *imsg, struct imsgbuf *ibuf } static const struct got_error * +send_parent_commit(size_t *totlen, struct imsgbuf *ibuf, + struct got_object_id *pid, struct got_pack *pack, + struct got_packidx *packidx, int recurse) +{ + const struct got_error *err = NULL; + struct got_object *pobj; + struct got_mini_commit_object *pcommit = NULL; + uint8_t *buf; + size_t len, plen; + int idx; + + idx = got_packidx_get_object_idx(packidx, pid); + if (idx == -1) /* parent commit not in same pack file */ + return NULL; + + err = got_packfile_open_object(&pobj, pack, packidx, idx, pid); + if (err) + return err; + + err = got_packfile_extract_object_to_mem(&buf, &len, pobj, + pack); + if (err) { + got_object_close(pobj); + return err; + } + + pobj->size = len; + err = got_object_parse_mini_commit(&pcommit, buf, len); + free(buf); + + plen = sizeof(*pcommit) + (pcommit->nparents * SHA1_DIGEST_LENGTH); + if (*totlen + plen >= MAX_IMSGSIZE - IMSG_HEADER_SIZE) + goto done; + *totlen += plen; + + err = got_privsep_send_mini_commit(ibuf, pcommit, pid); + if (err) + goto done; + + /* Send the first grandparent along as well if there is room. */ + if (recurse > 0) { + struct got_object_qid *qid; + qid = SIMPLEQ_FIRST(&pcommit->parent_ids); + if (qid) + err = send_parent_commit(totlen, ibuf, qid->id, pack, + packidx, recurse - 1); + } +done: + got_object_close(pobj); + got_object_mini_commit_close(pcommit); + return err; +} + +static const struct got_error * mini_commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack, struct got_packidx *packidx, struct got_object_cache *objcache) @@ -153,7 +207,8 @@ mini_commit_request(struct imsg *imsg, struct imsgbuf struct got_object *obj = NULL; struct got_mini_commit_object *commit = NULL; uint8_t *buf; - size_t len; + size_t len, totlen; + struct got_object_qid *qid; err = get_object(&obj, imsg, ibuf, pack, packidx, objcache, GOT_OBJ_TYPE_COMMIT); @@ -168,9 +223,25 @@ mini_commit_request(struct imsg *imsg, struct imsgbuf err = got_object_parse_mini_commit(&commit, buf, len); free(buf); - err = got_privsep_send_mini_commit(ibuf, commit); - if (obj) - got_object_close(obj); + /* + * Try to pre-seed the main process mini-commit cache with parent + * commits from this pack file. This makes more efficient use of + * imsg pipe buffers per system call. + */ + totlen = sizeof(struct got_imsg_commit_object_mini) + + (commit->nparents * sizeof(SHA1_DIGEST_LENGTH)); + SIMPLEQ_FOREACH(qid, &commit->parent_ids, entry) { + if (totlen >= MAX_IMSGSIZE - IMSG_HEADER_SIZE) + break; + err = send_parent_commit(&totlen, ibuf, qid->id, + pack, packidx, 1); + if (err) + goto done; + } + + err = got_privsep_send_mini_commit(ibuf, commit, NULL); +done: + got_object_close(obj); got_object_mini_commit_close(commit); if (err) { if (err->code == GOT_ERR_PRIVSEP_PIPE)