commit c59b334653ceb49c2009ed64fe3a16fa89e0e7f4 from: Stefan Sperling date: Tue Sep 11 10:29:46 2018 UTC stop sending deltas in imsg; cache deltas in got-read-pack commit - edfa7d7fafb6ce603f9463e5184d2586be36234e commit + c59b334653ceb49c2009ed64fe3a16fa89e0e7f4 blob - fab42876b12d54898f6e2d271a7566f85276b99e blob + fc8b2eef010a4a07760e360d7e9102d16791ddbd --- lib/got_lib_object.h +++ lib/got_lib_object.h @@ -29,6 +29,7 @@ struct got_object { struct got_object_id id; char *path_packfile; /* if packed */ + int pack_idx; /* if packed */ off_t pack_offset; /* if packed */ struct got_delta_chain deltas; /* if deltified */ int refcnt; /* > 0 if open and/or cached */ blob - c36311dd363087ed4b79603a54bd1cf61fe22145 blob + 38a83477cfa367f9bd2c029371bbe1fbcf2f0598 --- lib/got_lib_privsep.h +++ lib/got_lib_privsep.h @@ -92,10 +92,6 @@ enum got_imsg_type { GOT_IMSG_PACKIDX, GOT_IMSG_PACK, GOT_IMSG_PACKED_OBJECT_REQUEST, - - /* Messages for transmitting deltas and associated delta streams: */ - GOT_IMSG_DELTA, - GOT_IMSG_DELTA_STREAM, }; /* Structure for GOT_IMSG_ERROR. */ @@ -104,43 +100,20 @@ struct got_imsg_error { int errno_code; /* in case code equals GOT_ERR_ERRNO */ }; -/* Structure for GOT_IMSG_DELTA data. */ -struct got_imsg_delta { - /* These fields are the same as in struct got_delta. */ - off_t offset; - size_t tslen; - int type; - size_t size; - off_t data_offset; - size_t delta_len; - - /* - * Followed by one or more DELTA_STREAM messages until delta_len - * bytes of delta stream have been transmitted. - */ -}; - -/* Structure for GOT_IMSG_DELTA_STREAM data. */ -struct got_imsg_delta_stream { - /* - * Empty since the following is implied: - * Read additional delta stream data from imsg buffer. - */ -}; - /* * Structure for GOT_IMSG_TREE_REQUEST, GOT_IMSG_COMMIT_REQUEST, * and GOT_IMSG_OBJECT data. */ struct got_imsg_object { + uint8_t id[SHA1_DIGEST_LENGTH]; + /* These fields are the same as in struct got_object. */ int type; int flags; size_t hdrlen; size_t size; - off_t pack_offset; - int ndeltas; /* this many GOT_IMSG_DELTA messages follow */ + int pack_idx; }; /* Structure for GOT_IMSG_COMMIT data. */ blob - adef4e516db7b524740a5a8192cc6d199d1d645a blob + c10be9b25f2c42ae51559018bee66d6d981c67f9 --- lib/object_idcache.c +++ lib/object_idcache.c @@ -105,7 +105,7 @@ got_object_idcache_get(struct got_object_idcache *cach struct got_object_idcache_element *entry; TAILQ_FOREACH(entry, &cache->entries, entry) { - if (got_object_id_cmp(&entry->id, id) != 0) + if (memcmp(&entry->id.sha1, id->sha1, SHA1_DIGEST_LENGTH) != 0) continue; if (entry != TAILQ_FIRST(&cache->entries)) { TAILQ_REMOVE(&cache->entries, entry, entry); @@ -144,7 +144,7 @@ got_object_idcache_contains(struct got_object_idcache struct got_object_idcache_element *entry; TAILQ_FOREACH(entry, &cache->entries, entry) { - if (got_object_id_cmp(&entry->id, id) == 0) + if (memcmp(&entry->id.sha1, id->sha1, SHA1_DIGEST_LENGTH) == 0) return 1; } blob - 00d5a158a66d76ebd4b9210a50d5bbabccef3c7b blob + 3560e5c6fecdfae43a2c4fb617448f293a1db529 --- lib/pack.c +++ lib/pack.c @@ -554,7 +554,7 @@ parse_object_type_and_size(uint8_t *type, uint64_t *si static const struct got_error * open_plain_object(struct got_object **obj, const char *path_packfile, - struct got_object_id *id, uint8_t type, off_t offset, size_t size) + struct got_object_id *id, uint8_t type, off_t offset, size_t size, int idx) { *obj = calloc(1, sizeof(**obj)); if (*obj == NULL) @@ -570,6 +570,7 @@ open_plain_object(struct got_object **obj, const char (*obj)->type = type; (*obj)->flags = GOT_OBJ_FLAG_PACKED; + (*obj)->pack_idx = idx; (*obj)->hdrlen = 0; (*obj)->size = size; if (id) @@ -858,7 +859,7 @@ resolve_delta_chain(struct got_delta_chain *deltas, st static const struct got_error * open_delta_object(struct got_object **obj, struct got_packidx *packidx, struct got_pack *pack, struct got_object_id *id, off_t offset, - size_t tslen, int delta_type, size_t delta_size) + size_t tslen, int delta_type, size_t delta_size, int idx) { const struct got_error *err = NULL; int resolved_type; @@ -880,6 +881,7 @@ open_delta_object(struct got_object **obj, struct got_ goto done; } (*obj)->flags |= GOT_OBJ_FLAG_PACKED; + (*obj)->pack_idx = idx; SIMPLEQ_INIT(&(*obj)->deltas.entries); (*obj)->flags |= GOT_OBJ_FLAG_DELTIFIED; @@ -893,7 +895,6 @@ open_delta_object(struct got_object **obj, struct got_ if (err) goto done; (*obj)->type = resolved_type; - done: if (err) { got_object_close(*obj); @@ -928,12 +929,12 @@ got_packfile_open_object(struct got_object **obj, stru case GOT_OBJ_TYPE_BLOB: case GOT_OBJ_TYPE_TAG: err = open_plain_object(obj, pack->path_packfile, id, type, - offset + tslen, size); + offset + tslen, size, idx); break; case GOT_OBJ_TYPE_OFFSET_DELTA: case GOT_OBJ_TYPE_REF_DELTA: err = open_delta_object(obj, packidx, pack, id, offset, - tslen, type, size); + tslen, type, size, idx); break; default: err = got_error(GOT_ERR_OBJ_TYPE); blob - b69b1fe537b7a8f35cb059168674e185ad81e1c7 blob + b101ac47cc0a29ad631f01913e378cc90cba9014 --- lib/privsep.c +++ lib/privsep.c @@ -212,51 +212,11 @@ got_privsep_send_stop(int fd) err = flush_imsg(&ibuf); imsg_clear(&ibuf); return err; -} - -static const struct got_error * -send_delta(struct got_delta *delta, struct imsgbuf *ibuf) -{ - struct got_imsg_delta idelta; - size_t offset, remain; - - idelta.offset = delta->offset; - idelta.tslen = delta->tslen; - idelta.type = delta->type; - idelta.size = delta->size; - idelta.data_offset = delta->data_offset; - idelta.delta_len = delta->delta_len; - - if (imsg_compose(ibuf, GOT_IMSG_DELTA, 0, 0, -1, - &idelta, sizeof(idelta)) == -1) - return got_error_from_errno(); - - if (imsg_flush(ibuf) == -1) - return got_error_from_errno(); - - offset = 0; - remain = delta->delta_len; - while (remain > 0) { - size_t n = MIN(MAX_IMSGSIZE - IMSG_HEADER_SIZE, remain); - - if (imsg_compose(ibuf, GOT_IMSG_DELTA_STREAM, 0, 0, -1, - delta->delta_buf + offset, n) == -1) - return got_error_from_errno(); - - if (imsg_flush(ibuf) == -1) - return got_error_from_errno(); - - offset += n; - remain -= n; - } - - return NULL; } const struct got_error * got_privsep_send_obj_req(struct imsgbuf *ibuf, int fd, struct got_object *obj) { - const struct got_error *err = NULL; struct got_imsg_object iobj, *iobjp = NULL; size_t iobj_size = 0; int imsg_code = GOT_IMSG_OBJECT_REQUEST; @@ -276,13 +236,15 @@ got_privsep_send_obj_req(struct imsgbuf *ibuf, int fd, return got_error(GOT_ERR_OBJ_TYPE); } + memcpy(iobj.id, obj->id.sha1, sizeof(iobj.id)); iobj.type = obj->type; iobj.flags = obj->flags; iobj.hdrlen = obj->hdrlen; iobj.size = obj->size; - iobj.ndeltas = obj->deltas.nentries; - if (iobj.flags & GOT_OBJ_FLAG_PACKED) + if (iobj.flags & GOT_OBJ_FLAG_PACKED) { iobj.pack_offset = obj->pack_offset; + iobj.pack_idx = obj->pack_idx; + } iobjp = &iobj; iobj_size = sizeof(iobj); @@ -290,21 +252,8 @@ got_privsep_send_obj_req(struct imsgbuf *ibuf, int fd, if (imsg_compose(ibuf, imsg_code, 0, 0, fd, iobjp, iobj_size) == -1) return got_error_from_errno(); - - err = flush_imsg(ibuf); - if (err) - return err; - if (obj && obj->flags & GOT_OBJ_FLAG_DELTIFIED) { - struct got_delta *delta; - SIMPLEQ_FOREACH(delta, &obj->deltas.entries, entry) { - err = send_delta(delta, ibuf); - if (err) - break; - } - } - - return err; + return flush_imsg(ibuf); } const struct got_error * @@ -330,117 +279,23 @@ got_privsep_send_blob_outfd(struct imsgbuf *ibuf, int const struct got_error * got_privsep_send_obj(struct imsgbuf *ibuf, struct got_object *obj) { - const struct got_error *err = NULL; struct got_imsg_object iobj; - struct got_delta *delta; + memcpy(iobj.id, obj->id.sha1, sizeof(iobj.id)); iobj.type = obj->type; iobj.flags = obj->flags; iobj.hdrlen = obj->hdrlen; iobj.size = obj->size; - iobj.ndeltas = obj->deltas.nentries; - if (iobj.flags & GOT_OBJ_FLAG_PACKED) + if (iobj.flags & GOT_OBJ_FLAG_PACKED) { iobj.pack_offset = obj->pack_offset; + iobj.pack_idx = obj->pack_idx; + } if (imsg_compose(ibuf, GOT_IMSG_OBJECT, 0, 0, -1, &iobj, sizeof(iobj)) == -1) return got_error_from_errno(); - err = flush_imsg(ibuf); - if (err) - return err; - - SIMPLEQ_FOREACH(delta, &obj->deltas.entries, entry) { - err = send_delta(delta, ibuf); - if (err) - break; - } - - return err; -} - -static const struct got_error * -receive_delta(struct got_delta **delta, struct imsgbuf *ibuf) -{ - const struct got_error *err = NULL; - struct imsg imsg; - struct got_imsg_delta idelta; - uint8_t *delta_buf = NULL; - const size_t min_datalen = - MIN(sizeof(struct got_imsg_error), sizeof(struct got_imsg_delta)); - size_t datalen, offset, remain; - - err = got_privsep_recv_imsg(&imsg, ibuf, min_datalen); - if (err) - return err; - - datalen = imsg.hdr.len - IMSG_HEADER_SIZE; - if (imsg.hdr.type == GOT_IMSG_ERROR) - return recv_imsg_error(&imsg, datalen); - - if (imsg.hdr.type != GOT_IMSG_DELTA) - return got_error(GOT_ERR_PRIVSEP_MSG); - if (datalen != sizeof(idelta)) - return got_error(GOT_ERR_PRIVSEP_LEN); - - memcpy(&idelta, imsg.data, sizeof(idelta)); - imsg_free(&imsg); - - switch (idelta.type) { - case GOT_OBJ_TYPE_OFFSET_DELTA: - case GOT_OBJ_TYPE_REF_DELTA: - if (idelta.delta_len < GOT_DELTA_STREAM_LENGTH_MIN) - return got_error(GOT_ERR_BAD_DELTA); - break; - default: - if (idelta.delta_len != 0) - return got_error(GOT_ERR_BAD_DELTA); - break; - } - - if (idelta.delta_len > 0) { - delta_buf = calloc(1, idelta.delta_len); - if (delta_buf == NULL) - return got_error_from_errno(); - - offset = 0; - remain = idelta.delta_len; - while (remain > 0) { - size_t n = MIN(MAX_IMSGSIZE - IMSG_HEADER_SIZE, remain); - - err = got_privsep_recv_imsg(&imsg, ibuf, n); - if (err) - return err; - - if (imsg.hdr.type == GOT_IMSG_ERROR) - return recv_imsg_error(&imsg, datalen); - - if (imsg.hdr.type == GOT_IMSG_STOP) - break; - - if (imsg.hdr.type != GOT_IMSG_DELTA_STREAM) - return got_error(GOT_ERR_PRIVSEP_MSG); - - datalen = imsg.hdr.len - IMSG_HEADER_SIZE; - if (datalen != n) - return got_error(GOT_ERR_PRIVSEP_LEN); - - memcpy(delta_buf + offset, imsg.data, n); - imsg_free(&imsg); - - offset += n; - remain -= n; - } - } - - *delta = got_delta_open(idelta.offset, idelta.tslen, idelta.type, - idelta.size, idelta.data_offset, delta_buf, idelta.delta_len); - if (*delta == NULL) { - err = got_error_from_errno(); - free(delta_buf); - } - - return err; + return flush_imsg(ibuf); } const struct got_error * @@ -450,39 +305,25 @@ got_privsep_get_imsg_obj(struct got_object **obj, stru const struct got_error *err = NULL; struct got_imsg_object iobj; size_t datalen = imsg->hdr.len - IMSG_HEADER_SIZE; - int i; if (datalen != sizeof(iobj)) return got_error(GOT_ERR_PRIVSEP_LEN); memcpy(&iobj, imsg->data, sizeof(iobj)); - if (iobj.ndeltas < 0 || - iobj.ndeltas > GOT_DELTA_CHAIN_RECURSION_MAX) - return got_error(GOT_ERR_PRIVSEP_LEN); - if (iobj.ndeltas > 0 && - (iobj.flags & GOT_OBJ_FLAG_DELTIFIED) == 0) - return got_error(GOT_ERR_BAD_OBJ_DATA); - *obj = calloc(1, sizeof(**obj)); if (*obj == NULL) return got_error_from_errno(); + memcpy((*obj)->id.sha1, iobj.id, SHA1_DIGEST_LENGTH); (*obj)->type = iobj.type; (*obj)->flags = iobj.flags; (*obj)->hdrlen = iobj.hdrlen; (*obj)->size = iobj.size; - /* id and path_packfile might be copied in by caller */ - (*obj)->pack_offset = iobj.pack_offset; - SIMPLEQ_INIT(&(*obj)->deltas.entries); - for (i = 0; i < iobj.ndeltas; i++) { - struct got_delta *delta; - err = receive_delta(&delta, ibuf); - if (err) - break; - (*obj)->deltas.nentries++; - SIMPLEQ_INSERT_TAIL(&(*obj)->deltas.entries, delta, - entry); + /* path_packfile is handled by caller */ + if (iobj.flags & GOT_OBJ_FLAG_PACKED) { + (*obj)->pack_offset = iobj.pack_offset; + (*obj)->pack_idx = iobj.pack_idx; } return err; blob - 682e940922ba11be438043238488405228f75f38 blob + ee42ef15b6041ff85391515838d089565cc8ebdb --- lib/repository.c +++ lib/repository.c @@ -218,7 +218,6 @@ got_repo_cache_commit(struct got_repository *repo, str err = got_object_cache_add(&repo->commitcache, id, commit); if (err) return err; - commit->refcnt++; #endif return NULL; blob - 688107f4d8b2a5ce11d51c5dab085e2ec0da4d97 blob + 1d11b5ccacd8a00511e57a311e6be23aa1eb2c64 --- libexec/got-read-pack/Makefile +++ libexec/got-read-pack/Makefile @@ -1,8 +1,9 @@ .PATH:${.CURDIR}/../../lib PROG= got-read-pack -SRCS= got-read-pack.c delta.c error.c inflate.c object_parse.c \ - opentemp.c pack.c path.c privsep.c sha1.c +SRCS= got-read-pack.c delta.c error.c inflate.c object_cache.c \ + object_idcache.c object_parse.c opentemp.c pack.c path.c \ + privsep.c sha1.c CPPFLAGS = -I${.CURDIR}/../../include -I${.CURDIR}/../../lib LDADD = -lutil -lz blob - c6634c689a2cfb7fe806ec3e4eedb5bc3cd70cd5 blob + 001b16f5d436b8d2a68d194127cb3fde8cac7d97 --- libexec/got-read-pack/got-read-pack.c +++ libexec/got-read-pack/got-read-pack.c @@ -37,13 +37,14 @@ #include "got_lib_delta.h" #include "got_lib_inflate.h" #include "got_lib_object.h" +#include "got_lib_object_cache.h" #include "got_lib_object_parse.h" #include "got_lib_privsep.h" #include "got_lib_pack.h" static const struct got_error * object_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack, - struct got_packidx *packidx) + struct got_packidx *packidx, struct got_object_cache *objcache) { const struct got_error *err = NULL; struct got_imsg_packed_object iobj; @@ -58,15 +59,55 @@ object_request(struct imsg *imsg, struct imsgbuf *ibuf err = got_packfile_open_object(&obj, pack, packidx, iobj.idx, NULL); if (err) return err; + obj->refcnt++; + err = got_object_cache_add(objcache, &obj->id, obj); + if (err) + goto done; + obj->refcnt++; + err = got_privsep_send_obj(ibuf, obj); +done: got_object_close(obj); return err; } static const struct got_error * +get_object(struct got_object **obj, struct imsg *imsg, struct imsgbuf *ibuf, + struct got_pack *pack, struct got_packidx *packidx, + struct got_object_cache *objcache, int type) +{ + const struct got_error *err = NULL; + struct got_object *iobj; + + err = got_privsep_get_imsg_obj(&iobj, imsg, ibuf); + if (err) + return err; + + if (iobj->type != type) { + err = got_error(GOT_ERR_OBJ_TYPE); + goto done; + } + + if ((iobj->flags & GOT_OBJ_FLAG_PACKED) == 0) + return got_error(GOT_ERR_OBJ_NOT_PACKED); + + *obj = got_object_cache_get(objcache, &iobj->id); + if (*obj == NULL) { + err = got_packfile_open_object(obj, pack, packidx, + iobj->pack_idx, &iobj->id); + if (err) + goto done; + } + (*obj)->refcnt++; +done: + got_object_close(iobj); + return err; +} + +static const struct got_error * commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack, - struct got_packidx *packidx) + struct got_packidx *packidx, struct got_object_cache *objcache) { const struct got_error *err = NULL; struct got_object *obj = NULL; @@ -74,12 +115,10 @@ commit_request(struct imsg *imsg, struct imsgbuf *ibuf uint8_t *buf; size_t len; - err = got_privsep_get_imsg_obj(&obj, imsg, ibuf); + err = get_object(&obj, imsg, ibuf, pack, packidx, objcache, + GOT_OBJ_TYPE_COMMIT); if (err) return err; - - if (obj->type != GOT_OBJ_TYPE_COMMIT) - return got_error(GOT_ERR_OBJ_TYPE); err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack); if (err) @@ -105,7 +144,7 @@ commit_request(struct imsg *imsg, struct imsgbuf *ibuf static const struct got_error * tree_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack, - struct got_packidx *packidx) + struct got_packidx *packidx, struct got_object_cache *objcache) { const struct got_error *err = NULL; struct got_object *obj = NULL; @@ -113,13 +152,11 @@ tree_request(struct imsg *imsg, struct imsgbuf *ibuf, uint8_t *buf; size_t len; - err = got_privsep_get_imsg_obj(&obj, imsg, ibuf); + err = get_object(&obj, imsg, ibuf, pack, packidx, objcache, + GOT_OBJ_TYPE_TREE); if (err) return err; - if (obj->type != GOT_OBJ_TYPE_TREE) - return got_error(GOT_ERR_OBJ_TYPE); - err = got_packfile_extract_object_to_mem(&buf, &len, obj, pack); if (err) return err; @@ -144,7 +181,7 @@ tree_request(struct imsg *imsg, struct imsgbuf *ibuf, static const struct got_error * blob_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack, - struct got_packidx *packidx) + struct got_packidx *packidx, struct got_object_cache *objcache) { const struct got_error *err = NULL; struct got_object *obj = NULL; @@ -155,16 +192,11 @@ blob_request(struct imsg *imsg, struct imsgbuf *ibuf, memset(&imsg_outfd, 0, sizeof(imsg_outfd)); imsg_outfd.fd = -1; - err = got_privsep_get_imsg_obj(&obj, imsg, ibuf); + err = get_object(&obj, imsg, ibuf, pack, packidx, objcache, + GOT_OBJ_TYPE_BLOB); if (err) return err; - if (obj->type != GOT_OBJ_TYPE_BLOB) - return got_error(GOT_ERR_OBJ_TYPE); - - if ((obj->flags & GOT_OBJ_FLAG_PACKED) == 0) - return got_error(GOT_ERR_OBJ_NOT_PACKED); - err = got_privsep_recv_imsg(&imsg_outfd, ibuf, 0); if (err) return err; @@ -201,7 +233,8 @@ done: else if (imsg_outfd.fd != -1) close(imsg_outfd.fd); imsg_free(&imsg_outfd); - + if (obj) + got_object_close(obj); if (err) { if (err->code == GOT_ERR_PRIVSEP_PIPE) err = NULL; @@ -347,14 +380,22 @@ main(int argc, char *argv[]) const struct got_error *err = NULL; struct imsgbuf ibuf; struct imsg imsg; - struct got_packidx *packidx; - struct got_pack *pack; + struct got_packidx *packidx = NULL; + struct got_pack *pack = NULL; + struct got_object_cache objcache; //static int attached; //while (!attached) sleep(1); imsg_init(&ibuf, GOT_IMSG_FD_CHILD); + err = got_object_cache_init(&objcache, GOT_OBJECT_CACHE_TYPE_OBJ); + if (err) { + err = got_error_from_errno(); + got_privsep_send_error(&ibuf, err); + return 1; + } + /* revoke access to most system calls */ if (pledge("stdio recvfd", NULL) == -1) { err = got_error_from_errno(); @@ -389,16 +430,20 @@ main(int argc, char *argv[]) switch (imsg.hdr.type) { case GOT_IMSG_PACKED_OBJECT_REQUEST: - err = object_request(&imsg, &ibuf, pack, packidx); + err = object_request(&imsg, &ibuf, pack, packidx, + &objcache); break; case GOT_IMSG_COMMIT_REQUEST: - err = commit_request(&imsg, &ibuf, pack, packidx); + err = commit_request(&imsg, &ibuf, pack, packidx, + &objcache); break; case GOT_IMSG_TREE_REQUEST: - err = tree_request(&imsg, &ibuf, pack, packidx); + err = tree_request(&imsg, &ibuf, pack, packidx, + &objcache); break; case GOT_IMSG_BLOB_REQUEST: - err = blob_request(&imsg, &ibuf, pack, packidx); + err = blob_request(&imsg, &ibuf, pack, packidx, + &objcache); break; default: err = got_error(GOT_ERR_PRIVSEP_MSG); @@ -417,7 +462,10 @@ main(int argc, char *argv[]) } } - got_pack_close(pack); + if (packidx) + got_packidx_close(packidx); + if (pack) + got_pack_close(pack); imsg_clear(&ibuf); if (err) fprintf(stderr, "%s: %s\n", getprogname(), err->msg);