commit e44d939152693c16e95d2855b539ad6b30e81b15 from: Stefan Sperling date: Tue Jun 07 19:20:01 2022 UTC revert object enumeration in got-read-pack for now; needs more work This implementation marked commits and trees as enumerated before all trees which they depend on were enumerated. This behaviour leads to incomplete pack files when a tree is only partially packed and got-read-pack hits a missing tree entry as a result. The algorithm must be reworked such that packed leave nodes are marked enumerated first, then bubble-up. Found by op@ commit - 0e90588fc653d9ef0965c9420e8c39f615393b0d commit + e44d939152693c16e95d2855b539ad6b30e81b15 blob - 1f4430dcde3e6550f20a42cee4f1955e7ea675ba blob + 709eec07d085ee013da4bb4ebb657e8f97fa16b4 --- lib/got_lib_object.h +++ lib/got_lib_object.h @@ -133,14 +133,3 @@ const struct got_error *got_object_tree_entry_dup(stru const struct got_error *got_traverse_packed_commits( struct got_object_id_queue *, struct got_object_id *, const char *, struct got_repository *); - -typedef const struct got_error *(*got_object_enumerate_commit_cb)(void *, - time_t, struct got_object_id *, struct got_repository *); -typedef const struct got_error *(*got_object_enumerate_tree_cb)(void *, - struct got_tree_object *, time_t, struct got_object_id *, const char *, - struct got_repository *); - -const struct got_error *got_object_enumerate(got_object_enumerate_commit_cb, - got_object_enumerate_tree_cb, void *, struct got_object_id **, int, - struct got_object_id **, int, struct got_packidx *, - struct got_repository *); blob - c6c29a099c6a9b6dc6883641ef3ef96ce7f47156 blob + e719a95bde6bbe971668bb22ea3a72e2990ad927 --- lib/got_lib_privsep.h +++ lib/got_lib_privsep.h @@ -145,11 +145,6 @@ enum got_imsg_type { GOT_IMSG_COMMIT_TRAVERSAL_REQUEST, GOT_IMSG_TRAVERSED_COMMITS, GOT_IMSG_COMMIT_TRAVERSAL_DONE, - GOT_IMSG_OBJECT_ENUMERATION_REQUEST, - GOT_IMSG_ENUMERATED_COMMIT, - GOT_IMSG_ENUMERATED_TREE, - GOT_IMSG_TREE_ENUMERATION_DONE, - GOT_IMSG_OBJECT_ENUMERATION_DONE, /* Message sending file descriptor to a temporary file. */ GOT_IMSG_TMPFD, @@ -559,24 +554,8 @@ struct got_imsg_commit_traversal_request { struct got_imsg_traversed_commits { size_t ncommits; /* Followed by ncommit IDs of SHA1_DIGEST_LENGTH each */ -} __attribute__((__packed__)); - -/* Structure for GOT_IMSG_ENUMERATED_COMMIT */ -struct got_imsg_enumerated_commit { - uint8_t id[SHA1_DIGEST_LENGTH]; - time_t mtime; } __attribute__((__packed__)); -/* Structure for GOT_IMSG_ENUMERATED_TREE */ -struct got_imsg_enumerated_tree { - uint8_t id[SHA1_DIGEST_LENGTH]; /* tree ID */ - int nentries; /* number of tree entries */ - - /* Followed by tree's path in remaining data of imsg buffer. */ - - /* Followed by nentries * GOT_IMSG_TREE_ENTRY messages. */ -} __attribute__((__packed__)); - /* * Structure for GOT_IMSG_GOTCONFIG_REMOTE and * GOT_IMSG_GOTCONFIG_REMOTE data. @@ -742,18 +721,6 @@ const struct got_error *got_privsep_send_commit_traver const struct got_error *got_privsep_recv_traversed_commits( struct got_commit_object **, struct got_object_id **, struct got_object_id_queue *, struct imsgbuf *); -const struct got_error *got_privsep_send_enumerated_tree(size_t *, - struct imsgbuf *, struct got_object_id *, const char *, - struct got_parsed_tree_entry *, int); -const struct got_error *got_privsep_send_object_enumeration_request( - struct imsgbuf *); -const struct got_error *got_privsep_send_object_enumeration_done( - struct imsgbuf *); -const struct got_error *got_privsep_send_enumerated_commit(struct imsgbuf *, - struct got_object_id *, time_t); -const struct got_error *got_privsep_recv_enumerated_objects(struct imsgbuf *, - got_object_enumerate_commit_cb, got_object_enumerate_tree_cb, void *, - struct got_repository *); const struct got_error *got_privsep_send_raw_delta_req(struct imsgbuf *, int, struct got_object_id *); blob - eb06a1be95981661f8eed6fcf981bbe32a5b6119 blob + 47175236af75523bec4c9972e40679dcdd6d15f0 --- lib/object.c +++ lib/object.c @@ -60,10 +60,6 @@ #define MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b)) #endif -#ifndef nitems -#define nitems(_a) (sizeof((_a)) / sizeof((_a)[0])) -#endif - struct got_object_id * got_object_get_id(struct got_object *obj) { @@ -2400,60 +2396,3 @@ done: free(changed_commit_id); return err; } - -const struct got_error * -got_object_enumerate(got_object_enumerate_commit_cb cb_commit, - got_object_enumerate_tree_cb cb_tree, void *cb_arg, - struct got_object_id **ours, int nours, - struct got_object_id **theirs, int ntheirs, - struct got_packidx *packidx, struct got_repository *repo) -{ - const struct got_error *err = NULL; - struct got_pack *pack; - char *path_packfile = NULL; - - err = got_packidx_get_packfile_path(&path_packfile, - packidx->path_packidx); - if (err) - return err; - - pack = got_repo_get_cached_pack(repo, path_packfile); - if (pack == NULL) { - err = got_repo_cache_pack(&pack, repo, path_packfile, packidx); - if (err) - goto done; - } - - if (pack->privsep_child == NULL) { - err = start_pack_privsep_child(pack, packidx); - if (err) - goto done; - } - - err = got_privsep_send_object_enumeration_request( - pack->privsep_child->ibuf); - if (err) - goto done; - - err = got_privsep_send_object_idlist(pack->privsep_child->ibuf, - ours, nours); - if (err) - goto done; - err = got_privsep_send_object_idlist_done(pack->privsep_child->ibuf); - if (err) - goto done; - - err = got_privsep_send_object_idlist(pack->privsep_child->ibuf, - theirs, ntheirs); - if (err) - goto done; - err = got_privsep_send_object_idlist_done(pack->privsep_child->ibuf); - if (err) - goto done; - - err = got_privsep_recv_enumerated_objects(pack->privsep_child->ibuf, - cb_commit, cb_tree, cb_arg, repo); -done: - free(path_packfile); - return err; -} blob - fff58b083aa8eadb970ebd09ce593455d223fcc9 blob + 16709d206a932e5931f9b6441439d71eace4473d --- lib/pack_create.c +++ lib/pack_create.c @@ -903,15 +903,20 @@ add_object(int want_meta, struct got_object_idset *ids static const struct got_error * load_tree_entries(struct got_object_id_queue *ids, int want_meta, struct got_object_idset *idset, struct got_object_idset *idset_exclude, - struct got_tree_object *tree, + struct got_object_id *tree_id, const char *dpath, time_t mtime, uint32_t seed, struct got_repository *repo, int loose_obj_only, int *ncolored, int *nfound, int *ntrees, got_pack_progress_cb progress_cb, void *progress_arg, struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg) { const struct got_error *err; + struct got_tree_object *tree; char *p = NULL; int i; + + err = got_object_open_as_tree(&tree, repo, tree_id); + if (err) + return err; (*ntrees)++; err = report_progress(progress_cb, progress_arg, rl, @@ -933,17 +938,9 @@ load_tree_entries(struct got_object_id_queue *ids, int if (got_object_tree_entry_is_submodule(e) || got_object_idset_contains(idset, id) || got_object_idset_contains(idset_exclude, id)) - continue; - - /* - * If got-read-pack is crawling trees for us then - * we are only here to collect blob IDs. - */ - if (ids == NULL && S_ISDIR(mode)) continue; - - if (asprintf(&p, "%s%s%s", dpath, - got_path_is_root_dir(dpath) ? "" : "/", + + if (asprintf(&p, "%s%s%s", dpath, dpath[0] != '\0' ? "/" : "", got_tree_entry_get_name(e)) == -1) { err = got_error_from_errno("asprintf"); break; @@ -973,6 +970,7 @@ load_tree_entries(struct got_object_id_queue *ids, int } } + got_object_tree_close(tree); free(p); return err; } @@ -989,7 +987,6 @@ load_tree(int want_meta, struct got_object_idset *idse const struct got_error *err = NULL; struct got_object_id_queue tree_ids; struct got_object_qid *qid; - struct got_tree_object *tree = NULL; if (got_object_idset_contains(idset, tree_id) || got_object_idset_contains(idset_exclude, tree_id)) @@ -1037,32 +1034,20 @@ load_tree(int want_meta, struct got_object_idset *idse break; } - err = got_object_open_as_tree(&tree, repo, &qid->id); - if (err) { - free(qid->data); - got_object_qid_free(qid); - break; - } - err = load_tree_entries(&tree_ids, want_meta, idset, - idset_exclude, tree, path, mtime, seed, repo, - loose_obj_only, ncolored, nfound, ntrees, - progress_cb, progress_arg, rl, + idset_exclude, &qid->id, + path, mtime, seed, repo, loose_obj_only, ncolored, nfound, + ntrees, progress_cb, progress_arg, rl, cancel_cb, cancel_arg); free(qid->data); got_object_qid_free(qid); if (err) break; - - got_object_tree_close(tree); - tree = NULL; } STAILQ_FOREACH(qid, &tree_ids, entry) free(qid->data); got_object_id_queue_free(&tree_ids); - if (tree) - got_object_tree_close(tree); return err; } @@ -1461,210 +1446,10 @@ done: if (skip) got_object_idset_free(skip); got_object_id_queue_free(&ids); - return err; -} - -struct load_packed_obj_arg { - /* output parameters: */ - struct got_object_id *id; - char *dpath; - time_t mtime; - - /* input parameters: */ - uint32_t seed; - int want_meta; - struct got_object_idset *idset; - struct got_object_idset *idset_exclude; - int loose_obj_only; - int *ncolored; - int *nfound; - int *ntrees; - got_pack_progress_cb progress_cb; - void *progress_arg; - struct got_ratelimit *rl; - got_cancel_cb cancel_cb; - void *cancel_arg; -}; - -static const struct got_error * -load_packed_commit_id(void *arg, time_t mtime, struct got_object_id *id, - struct got_repository *repo) -{ - struct load_packed_obj_arg *a = arg; - - if (got_object_idset_contains(a->idset, id) || - got_object_idset_contains(a->idset_exclude, id)) - return NULL; - - return add_object(a->want_meta, - a->want_meta ? a->idset : a->idset_exclude, - id, "", GOT_OBJ_TYPE_COMMIT, mtime, a->seed, a->loose_obj_only, - repo, a->ncolored, a->nfound, a->ntrees, - a->progress_cb, a->progress_arg, a->rl); -} - -static const struct got_error * -load_packed_tree_ids(void *arg, struct got_tree_object *tree, time_t mtime, - struct got_object_id *id, const char *dpath, struct got_repository *repo) -{ - const struct got_error *err; - struct load_packed_obj_arg *a = arg; - const char *relpath; - - /* - * When we receive a tree's ID and path but not the tree itself, - * this tree object was not found in the pack file. This is the - * last time we are being called for this optimized traversal. - * Return from here and switch to loading objects the slow way. - */ - if (tree == NULL) { - free(a->id); - a->id = got_object_id_dup(id); - if (a->id == NULL) { - err = got_error_from_errno("got_object_id_dup"); - free(a->dpath); - a->dpath = NULL; - return err; - } - - free(a->dpath); - a->dpath = strdup(dpath); - if (a->dpath == NULL) { - err = got_error_from_errno("strdup"); - free(a->id); - a->id = NULL; - return err; - } - - a->mtime = mtime; - return NULL; - } - - if (got_object_idset_contains(a->idset, id) || - got_object_idset_contains(a->idset_exclude, id)) - return NULL; - - relpath = dpath; - while (relpath[0] == '/') - relpath++; - - err = add_object(a->want_meta, - a->want_meta ? a->idset : a->idset_exclude, - id, relpath, GOT_OBJ_TYPE_TREE, mtime, a->seed, - a->loose_obj_only, repo, a->ncolored, a->nfound, a->ntrees, - a->progress_cb, a->progress_arg, a->rl); - if (err) - return err; - - return load_tree_entries(NULL, a->want_meta, a->idset, - a->idset_exclude, tree, dpath, mtime, a->seed, repo, - a->loose_obj_only, a->ncolored, a->nfound, a->ntrees, - a->progress_cb, a->progress_arg, a->rl, - a->cancel_cb, a->cancel_arg); -} - -static const struct got_error * -load_packed_object_ids(struct got_object_id **ours, int nours, - struct got_object_id **theirs, int ntheirs, - int want_meta, uint32_t seed, struct got_object_idset *idset, - struct got_object_idset *idset_exclude, int loose_obj_only, - struct got_repository *repo, struct got_packidx *packidx, - int *ncolored, int *nfound, int *ntrees, - got_pack_progress_cb progress_cb, void *progress_arg, - struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg) -{ - const struct got_error *err = NULL; - struct load_packed_obj_arg lpa; - - memset(&lpa, 0, sizeof(lpa)); - lpa.seed = seed; - lpa.want_meta = want_meta; - lpa.idset = idset; - lpa.idset_exclude = idset_exclude; - lpa.loose_obj_only = loose_obj_only; - lpa.ncolored = ncolored; - lpa.nfound = nfound; - lpa.ntrees = ntrees; - lpa.progress_cb = progress_cb; - lpa.progress_arg = progress_arg; - lpa.rl = rl; - lpa.cancel_cb = cancel_cb; - lpa.cancel_arg = cancel_arg; - - /* Attempt to load objects via got-read-pack, as far as possible. */ - err = got_object_enumerate(load_packed_commit_id, - load_packed_tree_ids, &lpa, ours, nours, theirs, ntheirs, - packidx, repo); - if (err) - return err; - - if (lpa.id == NULL) - return NULL; - - /* - * An incomplete tree hierarchy was present in the pack file - * and caused loading to be aborted midway through a commit. - * Continue loading trees the slow way. - */ - err = load_tree(want_meta, idset, idset_exclude, - lpa.id, lpa.dpath, lpa.mtime, seed, repo, loose_obj_only, - ncolored, nfound, ntrees, progress_cb, progress_arg, rl, - cancel_cb, cancel_arg); - free(lpa.id); - free(lpa.dpath); return err; } static const struct got_error * -find_pack_for_enumeration(struct got_packidx **best_packidx, - struct got_object_id **ids, int nids, struct got_repository *repo) -{ - const struct got_error *err = NULL; - struct got_pathlist_entry *pe; - const char *best_packidx_path = NULL; - int nobj_max = 0; - int ncommits_max = 0; - - *best_packidx = NULL; - - /* - * Find the largest pack which contains at least some of the - * commits and tags we are interested in. - */ - TAILQ_FOREACH(pe, &repo->packidx_paths, entry) { - const char *path_packidx = pe->path; - struct got_packidx *packidx; - int nobj, i, idx, ncommits = 0; - - err = got_repo_get_packidx(&packidx, path_packidx, repo); - if (err) - break; - - nobj = be32toh(packidx->hdr.fanout_table[0xff]); - if (nobj <= nobj_max) - continue; - - for (i = 0; i < nids; i++) { - idx = got_packidx_get_object_idx(packidx, ids[i]); - if (idx != -1) - ncommits++; - } - if (ncommits > ncommits_max) { - best_packidx_path = path_packidx; - nobj_max = nobj; - ncommits_max = ncommits; - } - } - - if (best_packidx_path) { - err = got_repo_get_packidx(best_packidx, best_packidx_path, - repo); - } - - return err; -} - -static const struct got_error * load_object_ids(int *ncolored, int *nfound, int *ntrees, struct got_object_idset *idset, struct got_object_id **theirs, int ntheirs, struct got_object_id **ours, int nours, struct got_repository *repo, @@ -1674,7 +1459,6 @@ load_object_ids(int *ncolored, int *nfound, int *ntree { const struct got_error *err = NULL; struct got_object_id **ids = NULL; - struct got_packidx *packidx = NULL; int i, nobj = 0, obj_type; struct got_object_idset *idset_exclude; @@ -1690,18 +1474,6 @@ load_object_ids(int *ncolored, int *nfound, int *ntree repo, progress_cb, progress_arg, rl, cancel_cb, cancel_arg); if (err) goto done; - - err = find_pack_for_enumeration(&packidx, theirs, ntheirs, repo); - if (err) - goto done; - if (packidx) { - err = load_packed_object_ids(theirs, ntheirs, NULL, 0, 0, - seed, idset, idset_exclude, loose_obj_only, repo, packidx, - ncolored, nfound, ntrees, progress_cb, progress_arg, rl, - cancel_cb, cancel_arg); - if (err) - goto done; - } for (i = 0; i < ntheirs; i++) { struct got_object_id *id = theirs[i]; @@ -1727,18 +1499,6 @@ load_object_ids(int *ncolored, int *nfound, int *ntree } } - err = find_pack_for_enumeration(&packidx, ids, nobj, repo); - if (err) - goto done; - if (packidx) { - err = load_packed_object_ids(ids, nobj, theirs, ntheirs, 1, - seed, idset, idset_exclude, loose_obj_only, repo, packidx, - ncolored, nfound, ntrees, - progress_cb, progress_arg, rl, cancel_cb, cancel_arg); - if (err) - goto done; - } - for (i = 0; i < nobj; i++) { err = load_commit(1, idset, idset_exclude, ids[i], repo, seed, loose_obj_only, ncolored, nfound, ntrees, blob - 00c7956115117e87787354c1733a69adbe78cc93 blob + 782f94ad26528e54d59fa5855bd1a0a4f674588b --- lib/privsep.c +++ lib/privsep.c @@ -1443,8 +1443,8 @@ got_privsep_recv_commit(struct got_commit_object **com } static const struct got_error * -send_tree_entries_batch(struct imsgbuf *ibuf, - struct got_parsed_tree_entry *entries, int idx0, int idxN, size_t len) +send_tree_entries(struct imsgbuf *ibuf, struct got_parsed_tree_entry *entries, + int idx0, int idxN, size_t len) { struct ibuf *wbuf; struct got_imsg_tree_entries ientries; @@ -1479,14 +1479,21 @@ send_tree_entries_batch(struct imsgbuf *ibuf, return NULL; } -static const struct got_error * -send_tree_entries(struct imsgbuf *ibuf, struct got_parsed_tree_entry *entries, - int nentries) +const struct got_error * +got_privsep_send_tree(struct imsgbuf *ibuf, + struct got_parsed_tree_entry *entries, int nentries) { const struct got_error *err = NULL; + struct got_imsg_tree_object itree; + size_t entries_len; int i, j; - size_t entries_len = sizeof(struct got_imsg_tree_entries); + + itree.nentries = nentries; + if (imsg_compose(ibuf, GOT_IMSG_TREE, 0, 0, -1, &itree, sizeof(itree)) + == -1) + return got_error_from_errno("imsg_compose TREE"); + entries_len = sizeof(struct got_imsg_tree_entries); i = 0; for (j = 0; j < nentries; j++) { struct got_parsed_tree_entry *pte = &entries[j]; @@ -1494,7 +1501,7 @@ send_tree_entries(struct imsgbuf *ibuf, struct got_par if (j > 0 && entries_len + len > MAX_IMSGSIZE - IMSG_HEADER_SIZE) { - err = send_tree_entries_batch(ibuf, entries, + err = send_tree_entries(ibuf, entries, i, j - 1, entries_len); if (err) return err; @@ -1506,95 +1513,12 @@ send_tree_entries(struct imsgbuf *ibuf, struct got_par } if (j > 0) { - err = send_tree_entries_batch(ibuf, entries, i, j - 1, - entries_len); + err = send_tree_entries(ibuf, entries, i, j - 1, entries_len); if (err) return err; } - return NULL; -} - -const struct got_error * -got_privsep_send_tree(struct imsgbuf *ibuf, - struct got_parsed_tree_entry *entries, int nentries) -{ - const struct got_error *err = NULL; - struct got_imsg_tree_object itree; - - itree.nentries = nentries; - if (imsg_compose(ibuf, GOT_IMSG_TREE, 0, 0, -1, &itree, sizeof(itree)) - == -1) - return got_error_from_errno("imsg_compose TREE"); - - err = send_tree_entries(ibuf, entries, nentries); - if (err) - return err; - return flush_imsg(ibuf); -} - - -static const struct got_error * -recv_tree_entries(void *data, size_t datalen, struct got_tree_object *tree, - int *nentries) -{ - const struct got_error *err = NULL; - struct got_imsg_tree_entries *ientries; - struct got_tree_entry *te; - size_t te_offset; - size_t i; - - if (datalen <= sizeof(*ientries) || - datalen > MAX_IMSGSIZE - IMSG_HEADER_SIZE) - return got_error(GOT_ERR_PRIVSEP_LEN); - - ientries = (struct got_imsg_tree_entries *)data; - if (ientries->nentries > INT_MAX) { - return got_error_msg(GOT_ERR_NO_SPACE, - "too many tree entries"); - } - - te_offset = sizeof(*ientries); - for (i = 0; i < ientries->nentries; i++) { - struct got_imsg_tree_entry ite; - const char *te_name; - uint8_t *buf = (uint8_t *)data + te_offset; - - if (te_offset >= datalen) { - err = got_error(GOT_ERR_PRIVSEP_LEN); - break; - } - - /* Might not be aligned, size is ~32 bytes. */ - memcpy(&ite, buf, sizeof(ite)); - - if (ite.namelen >= sizeof(te->name)) { - err = got_error(GOT_ERR_PRIVSEP_LEN); - break; - } - if (te_offset + sizeof(ite) + ite.namelen > datalen) { - err = got_error(GOT_ERR_PRIVSEP_LEN); - break; - } - - if (*nentries >= tree->nentries) { - err = got_error(GOT_ERR_PRIVSEP_LEN); - break; - } - te = &tree->entries[*nentries]; - te_name = buf + sizeof(ite); - memcpy(te->name, te_name, ite.namelen); - te->name[ite.namelen] = '\0'; - memcpy(te->id.sha1, ite.id, SHA1_DIGEST_LENGTH); - te->mode = ite.mode; - te->idx = *nentries; - (*nentries)++; - - te_offset += sizeof(ite) + ite.namelen; - } - - return err; } const struct got_error * @@ -1605,6 +1529,7 @@ got_privsep_recv_tree(struct got_tree_object **tree, s MIN(sizeof(struct got_imsg_error), sizeof(struct got_imsg_tree_object)); struct got_imsg_tree_object *itree; + size_t i; int nentries = 0; *tree = NULL; @@ -1617,6 +1542,9 @@ got_privsep_recv_tree(struct got_tree_object **tree, s struct imsg imsg; size_t n; size_t datalen; + struct got_imsg_tree_entries *ientries; + struct got_tree_entry *te = NULL; + size_t te_offset; n = imsg_get(ibuf, &imsg); if (n == 0) { @@ -1683,8 +1611,56 @@ got_privsep_recv_tree(struct got_tree_object **tree, s err = got_error(GOT_ERR_PRIVSEP_MSG); break; } - err = recv_tree_entries(imsg.data, datalen, - *tree, &nentries); + if (datalen <= sizeof(*ientries) || + datalen > MAX_IMSGSIZE - IMSG_HEADER_SIZE) { + err = got_error(GOT_ERR_PRIVSEP_LEN); + break; + } + + ientries = imsg.data; + if (ientries->nentries > INT_MAX) { + err = got_error_msg(GOT_ERR_NO_SPACE, + "too many tree entries"); + break; + } + te_offset = sizeof(*ientries); + for (i = 0; i < ientries->nentries; i++) { + struct got_imsg_tree_entry ite; + const char *te_name; + uint8_t *buf = imsg.data + te_offset; + + if (te_offset >= datalen) { + err = got_error(GOT_ERR_PRIVSEP_LEN); + break; + } + + /* Might not be aligned, size is ~32 bytes. */ + memcpy(&ite, buf, sizeof(ite)); + + if (ite.namelen >= sizeof(te->name)) { + err = got_error(GOT_ERR_PRIVSEP_LEN); + break; + } + if (te_offset + sizeof(ite) + ite.namelen > + datalen) { + err = got_error(GOT_ERR_PRIVSEP_LEN); + break; + } + if (nentries >= (*tree)->nentries) { + err = got_error(GOT_ERR_PRIVSEP_LEN); + break; + } + te = &(*tree)->entries[nentries]; + te_name = buf + sizeof(ite); + memcpy(te->name, te_name, ite.namelen); + te->name[ite.namelen] = '\0'; + memcpy(te->id.sha1, ite.id, SHA1_DIGEST_LENGTH); + te->mode = ite.mode; + te->idx = nentries; + nentries++; + + te_offset += sizeof(ite) + ite.namelen; + } break; default: err = got_error(GOT_ERR_PRIVSEP_MSG); @@ -2751,269 +2727,6 @@ got_privsep_recv_traversed_commits(struct got_commit_o if (err) got_object_id_queue_free(commit_ids); - return err; -} - -const struct got_error * -got_privsep_send_enumerated_tree(size_t *totlen, struct imsgbuf *ibuf, - struct got_object_id *tree_id, const char *path, - struct got_parsed_tree_entry *entries, int nentries) -{ - const struct got_error *err = NULL; - struct ibuf *wbuf; - size_t path_len = strlen(path); - size_t msglen; - - msglen = sizeof(struct got_imsg_enumerated_tree) + path_len; - wbuf = imsg_create(ibuf, GOT_IMSG_ENUMERATED_TREE, 0, 0, msglen); - if (wbuf == NULL) - return got_error_from_errno("imsg_create ENUMERATED_TREE"); - - if (imsg_add(wbuf, tree_id->sha1, SHA1_DIGEST_LENGTH) == -1) - return got_error_from_errno("imsg_add ENUMERATED_TREE"); - if (imsg_add(wbuf, &nentries, sizeof(nentries)) == -1) - return got_error_from_errno("imsg_add ENUMERATED_TREE"); - if (imsg_add(wbuf, path, path_len) == -1) - return got_error_from_errno("imsg_add ENUMERATED_TREE"); - - wbuf->fd = -1; - imsg_close(ibuf, wbuf); - - if (entries) { - err = send_tree_entries(ibuf, entries, nentries); - if (err) - return err; - } - - return flush_imsg(ibuf); -} - -const struct got_error * -got_privsep_send_object_enumeration_request(struct imsgbuf *ibuf) -{ - if (imsg_compose(ibuf, GOT_IMSG_OBJECT_ENUMERATION_REQUEST, - 0, 0, -1, NULL, 0) == -1) - return got_error_from_errno("imsg_compose " - "OBJECT_ENUMERATION_REQUEST"); - - return flush_imsg(ibuf); -} - -const struct got_error * -got_privsep_send_object_enumeration_done(struct imsgbuf *ibuf) -{ - if (imsg_compose(ibuf, GOT_IMSG_OBJECT_ENUMERATION_DONE, - 0, 0, -1, NULL, 0) == -1) - return got_error_from_errno("imsg_compose " - "OBJECT_ENUMERATION_DONE"); - - return flush_imsg(ibuf); -} - -const struct got_error * -got_privsep_send_enumerated_commit(struct imsgbuf *ibuf, - struct got_object_id *id, time_t mtime) -{ - struct ibuf *wbuf; - - wbuf = imsg_create(ibuf, GOT_IMSG_ENUMERATED_COMMIT, 0, 0, - sizeof(struct got_imsg_enumerated_commit) + SHA1_DIGEST_LENGTH); - if (wbuf == NULL) - return got_error_from_errno("imsg_create ENUMERATED_COMMIT"); - - /* Keep in sync with struct got_imsg_enumerated_commit! */ - if (imsg_add(wbuf, id, SHA1_DIGEST_LENGTH) == -1) - return got_error_from_errno("imsg_add ENUMERATED_COMMIT"); - if (imsg_add(wbuf, &mtime, sizeof(mtime)) == -1) - return got_error_from_errno("imsg_add ENUMERATED_COMMIT"); - - wbuf->fd = -1; - imsg_close(ibuf, wbuf); - /* Don't flush yet, tree entries or ENUMERATION_DONE will follow. */ - return NULL; -} - -const struct got_error * -got_privsep_recv_enumerated_objects(struct imsgbuf *ibuf, - got_object_enumerate_commit_cb cb_commit, - got_object_enumerate_tree_cb cb_tree, void *cb_arg, - struct got_repository *repo) -{ - const struct got_error *err = NULL; - struct imsg imsg; - struct got_imsg_enumerated_commit *icommit = NULL; - struct got_object_id commit_id; - int have_commit = 0; - time_t mtime = 0; - struct got_tree_object tree; - struct got_imsg_enumerated_tree *itree; - struct got_object_id tree_id; - char *path = NULL, *canon_path = NULL; - size_t datalen, path_len; - int nentries = -1; - int done = 0; - - memset(&tree, 0, sizeof(tree)); - - while (!done) { - err = got_privsep_recv_imsg(&imsg, ibuf, 0); - if (err) - break; - - datalen = imsg.hdr.len - IMSG_HEADER_SIZE; - switch (imsg.hdr.type) { - case GOT_IMSG_ENUMERATED_COMMIT: - if (have_commit && nentries != -1) { - err = got_error(GOT_ERR_PRIVSEP_MSG); - break; - } - if (datalen != sizeof(*icommit)) { - err = got_error(GOT_ERR_PRIVSEP_LEN); - break; - } - icommit = (struct got_imsg_enumerated_commit *)imsg.data; - memcpy(commit_id.sha1, icommit->id, SHA1_DIGEST_LENGTH); - mtime = icommit->mtime; - err = cb_commit(cb_arg, mtime, &commit_id, repo); - if (err) - break; - have_commit = 1; - break; - case GOT_IMSG_ENUMERATED_TREE: - /* Should be preceeded by GOT_IMSG_ENUMERATED_COMMIT. */ - if (!have_commit) { - err = got_error(GOT_ERR_PRIVSEP_MSG); - break; - } - if (datalen < sizeof(*itree)) { - err = got_error(GOT_ERR_PRIVSEP_LEN); - break; - } - itree = imsg.data; - path_len = datalen - sizeof(*itree); - if (path_len == 0) { - err = got_error(GOT_ERR_PRIVSEP_LEN); - break; - } - memcpy(tree_id.sha1, itree->id, sizeof(tree_id.sha1)); - free(path); - path = malloc(path_len + 1); - if (path == NULL) { - err = got_error_from_errno("malloc"); - break; - } - free(canon_path); - canon_path = malloc(path_len + 1); - if (canon_path == NULL) { - err = got_error_from_errno("malloc"); - break; - } - memcpy(path, (uint8_t *)imsg.data + sizeof(*itree), - path_len); - path[path_len] = '\0'; - if (!got_path_is_absolute(path)) { - err = got_error(GOT_ERR_BAD_PATH); - break; - } - if (got_path_is_root_dir(path)) { - /* XXX check what got_canonpath() does wrong */ - canon_path[0] = '/'; - canon_path[1] = '\0'; - } else { - err = got_canonpath(path, canon_path, - path_len + 1); - if (err) - break; - } - if (strcmp(path, canon_path) != 0) { - err = got_error(GOT_ERR_BAD_PATH); - break; - } - if (nentries != -1) { - err = got_error(GOT_ERR_PRIVSEP_MSG); - break; - } - if (itree->nentries < -1) { - err = got_error(GOT_ERR_PRIVSEP_MSG); - break; - } - if (itree->nentries == -1) { - /* Tree was not found in pack file. */ - err = cb_tree(cb_arg, NULL, mtime, &tree_id, - path, repo); - break; - } - if (itree->nentries > INT_MAX) { - err = got_error(GOT_ERR_PRIVSEP_LEN); - break; - } - tree.entries = calloc(itree->nentries, - sizeof(struct got_tree_entry)); - if (tree.entries == NULL) { - err = got_error_from_errno("calloc"); - break; - } - if (itree->nentries == 0) { - err = cb_tree(cb_arg, &tree, mtime, &tree_id, - path, repo); - if (err) - break; - - /* Prepare for next tree. */ - free(tree.entries); - memset(&tree, 0, sizeof(tree)); - nentries = -1; - } else { - tree.nentries = itree->nentries; - nentries = 0; - } - break; - case GOT_IMSG_TREE_ENTRIES: - /* Should be preceeded by GOT_IMSG_ENUMERATED_TREE. */ - if (nentries <= -1) { - err = got_error(GOT_ERR_PRIVSEP_MSG); - break; - } - err = recv_tree_entries(imsg.data, datalen, - &tree, &nentries); - if (err) - break; - if (tree.nentries == nentries) { - err = cb_tree(cb_arg, &tree, mtime, &tree_id, - path, repo); - if (err) - break; - - /* Prepare for next tree. */ - free(tree.entries); - memset(&tree, 0, sizeof(tree)); - nentries = -1; - } - break; - case GOT_IMSG_TREE_ENUMERATION_DONE: - /* All trees have been found and traversed. */ - if (path == NULL || nentries != -1) { - err = got_error(GOT_ERR_PRIVSEP_MSG); - break; - } - have_commit = 0; - break; - case GOT_IMSG_OBJECT_ENUMERATION_DONE: - done = 1; - break; - default: - err = got_error(GOT_ERR_PRIVSEP_MSG); - break; - } - - imsg_free(&imsg); - if (err) - break; - } - - free(path); - free(canon_path); - free(tree.entries); return err; } @@ -3121,8 +2834,9 @@ got_privsep_recv_raw_delta(uint64_t *base_size, uint64 return err; } -static const struct got_error * -send_idlist(struct imsgbuf *ibuf, struct got_object_id **ids, size_t nids) +const struct got_error * +got_privsep_send_object_idlist(struct imsgbuf *ibuf, + struct got_object_id **ids, size_t nids) { const struct got_error *err = NULL; struct got_imsg_object_idlist idlist; @@ -3156,34 +2870,6 @@ send_idlist(struct imsgbuf *ibuf, struct got_object_id } const struct got_error * -got_privsep_send_object_idlist(struct imsgbuf *ibuf, - struct got_object_id **ids, size_t nids) -{ - const struct got_error *err = NULL; - struct got_object_id *idlist[GOT_IMSG_OBJ_ID_LIST_MAX_NIDS]; - int i, j = 0; - - for (i = 0; i < nids; i++) { - j = i % nitems(idlist); - idlist[j] = ids[i]; - if (j >= nitems(idlist) - 1) { - err = send_idlist(ibuf, idlist, j + 1); - if (err) - return err; - j = 0; - } - } - - if (j > 0) { - err = send_idlist(ibuf, idlist, j + 1); - if (err) - return err; - } - - return NULL; -} - -const struct got_error * got_privsep_send_object_idlist_done(struct imsgbuf *ibuf) { if (imsg_compose(ibuf, GOT_IMSG_OBJ_ID_LIST_DONE, 0, 0, -1, NULL, 0) blob - 07b64ca6e1b938a8bcab156f09eb337405c2977a blob + 63bd0d3fe2ed9ec49659d7fcd00aa9f1412aacab --- libexec/got-read-pack/got-read-pack.c +++ libexec/got-read-pack/got-read-pack.c @@ -14,7 +14,6 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -#include #include #include #include @@ -587,6 +586,7 @@ send_commit_traversal_done(struct imsgbuf *ibuf) return got_privsep_flush_imsg(ibuf); } + static const struct got_error * commit_traversal_request(struct imsg *imsg, struct imsgbuf *ibuf, @@ -1022,31 +1022,6 @@ recv_object_ids(struct got_object_idset *idset, struct } static const struct got_error * -recv_object_id_queue(struct got_object_id_queue *queue, struct imsgbuf *ibuf) -{ - const struct got_error *err = NULL; - int done = 0; - struct got_object_qid *qid; - struct got_object_id *ids; - size_t nids, i; - - for (;;) { - err = got_privsep_recv_object_idlist(&done, &ids, &nids, ibuf); - if (err || done) - break; - for (i = 0; i < nids; i++) { - err = got_object_qid_alloc_partial(&qid); - if (err) - return err; - memcpy(&qid->id, &ids[i], sizeof(qid->id)); - STAILQ_INSERT_TAIL(queue, qid, entry); - } - } - - return err; -} - -static const struct got_error * delta_reuse_request(struct imsg *imsg, struct imsgbuf *ibuf, FILE *delta_outfile, struct got_pack *pack, struct got_packidx *packidx) { @@ -1157,288 +1132,6 @@ done: } static const struct got_error * -send_tree_enumeration_done(struct imsgbuf *ibuf) -{ - if (imsg_compose(ibuf, GOT_IMSG_TREE_ENUMERATION_DONE, 0, 0, -1, - NULL, 0) == -1) - return got_error_from_errno("imsg_compose TREE_ENUMERATION_DONE"); - - return got_privsep_flush_imsg(ibuf); -} - -static const struct got_error * -enumerate_tree(struct imsgbuf *ibuf, size_t *totlen, - struct got_object_id *tree_id, - const char *path, struct got_pack *pack, struct got_packidx *packidx, - struct got_object_cache *objcache, struct got_object_idset *idset) -{ - const struct got_error *err = NULL; - struct got_object_id_queue ids; - struct got_object_qid *qid; - uint8_t *buf = NULL; - struct got_parsed_tree_entry *entries = NULL; - - STAILQ_INIT(&ids); - - err = got_object_qid_alloc_partial(&qid); - if (err) - return err; - memcpy(&qid->id.sha1, tree_id, SHA1_DIGEST_LENGTH); - qid->data = strdup(path); - if (qid->data == NULL) { - err = got_error_from_errno("strdup"); - goto done; - } - STAILQ_INSERT_TAIL(&ids, qid, entry); - qid = NULL; - - do { - const char *path; - int idx, nentries, i; - - if (sigint_received) { - err = got_error(GOT_ERR_CANCELLED); - goto done; - } - - qid = STAILQ_FIRST(&ids); - STAILQ_REMOVE_HEAD(&ids, entry); - path = qid->data; - - idx = got_packidx_get_object_idx(packidx, &qid->id); - if (idx == -1) { - err = got_privsep_send_enumerated_tree(totlen, ibuf, - &qid->id, path, NULL, -1); - break; - } - - err = open_tree(&buf, &entries, &nentries, - pack, packidx, idx, &qid->id, objcache); - if (err) { - if (err->code != GOT_ERR_NO_OBJ) - goto done; - } - - err = got_privsep_send_enumerated_tree(totlen, - ibuf, &qid->id, path, entries, nentries); - if (err) - goto done; - - err = got_object_idset_add(idset, &qid->id, NULL); - if (err) - goto done; - - for (i = 0; i < nentries; i++) { - struct got_object_qid *eqid = NULL; - struct got_parsed_tree_entry *pte = &entries[i]; - char *p; - - if (!S_ISDIR(pte->mode)) - continue; - - err = got_object_qid_alloc_partial(&eqid); - if (err) - goto done; - memcpy(eqid->id.sha1, pte->id, sizeof(eqid->id.sha1)); - - if (got_object_idset_contains(idset, &eqid->id)) { - got_object_qid_free(eqid); - continue; - } - - if (asprintf(&p, "%s%s%s", path, - got_path_is_root_dir(path) ? "" : "/", - pte->name) == -1) { - err = got_error_from_errno("asprintf"); - got_object_qid_free(eqid); - goto done; - } - eqid->data = p; - STAILQ_INSERT_TAIL(&ids, eqid, entry); - idx = got_packidx_get_object_idx(packidx, &eqid->id); - if (idx == -1) - break; - } - - free(qid->data); - got_object_qid_free(qid); - qid = NULL; - - free(entries); - entries = NULL; - free(buf); - buf = NULL; - } while (!STAILQ_EMPTY(&ids)); - - err = send_tree_enumeration_done(ibuf); -done: - free(buf); - if (qid) - free(qid->data); - got_object_qid_free(qid); - got_object_id_queue_free(&ids); - free(entries); - if (err) { - if (err->code == GOT_ERR_PRIVSEP_PIPE) - err = NULL; - else - got_privsep_send_error(ibuf, err); - } - - return err; -} - -static const struct got_error * -enumeration_request(struct imsg *imsg, struct imsgbuf *ibuf, - struct got_pack *pack, struct got_packidx *packidx, - struct got_object_cache *objcache) -{ - const struct got_error *err = NULL; - struct got_object_id_queue commit_ids; - const struct got_object_id_queue *parents = NULL; - struct got_object_qid *qid = NULL; - struct got_object *obj = NULL; - struct got_commit_object *commit = NULL; - struct got_object_id *tree_id = NULL; - size_t totlen = 0; - struct got_object_idset *idset; - int idx; - - STAILQ_INIT(&commit_ids); - - idset = got_object_idset_alloc(); - if (idset == NULL) - return got_error_from_errno("got_object_idset_alloc"); - - err = recv_object_id_queue(&commit_ids, ibuf); - if (err) - goto done; - - err = recv_object_ids(idset, ibuf); - if (err) - goto done; - - while (!STAILQ_EMPTY(&commit_ids)) { - if (sigint_received) { - err = got_error(GOT_ERR_CANCELLED); - goto done; - } - - qid = STAILQ_FIRST(&commit_ids); - STAILQ_REMOVE_HEAD(&commit_ids, entry); - - if (got_object_idset_contains(idset, &qid->id)) { - got_object_qid_free(qid); - qid = NULL; - continue; - } - - idx = got_packidx_get_object_idx(packidx, &qid->id); - if (idx == -1) - break; - - err = open_object(&obj, pack, packidx, idx, &qid->id, - objcache); - if (err) - goto done; - if (obj->type == GOT_OBJ_TYPE_TAG) { - struct got_tag_object *tag; - uint8_t *buf; - size_t len; - err = got_packfile_extract_object_to_mem(&buf, - &len, obj, pack); - if (err) - goto done; - obj->size = len; - err = got_object_parse_tag(&tag, buf, len); - if (err) { - free(buf); - goto done; - } - idx = got_packidx_get_object_idx(packidx, &tag->id); - if (idx == -1) - break; - err = open_commit(&commit, pack, packidx, idx, - &tag->id, objcache); - got_object_tag_close(tag); - free(buf); - if (err) - goto done; - } else if (obj->type == GOT_OBJ_TYPE_COMMIT) { - err = open_commit(&commit, pack, packidx, idx, - &qid->id, objcache); - if (err) - goto done; - } else { - err = got_error(GOT_ERR_OBJ_TYPE); - goto done; - } - got_object_close(obj); - obj = NULL; - - err = got_privsep_send_enumerated_commit(ibuf, &qid->id, - got_object_commit_get_committer_time(commit)); - if (err) - goto done; - - tree_id = got_object_commit_get_tree_id(commit); - idx = got_packidx_get_object_idx(packidx, tree_id); - if (idx == -1) { - err = got_privsep_send_enumerated_tree(&totlen, ibuf, - tree_id, "", NULL, -1); - if (err) - goto done; - break; - } - - if (got_object_idset_contains(idset, tree_id)) { - got_object_qid_free(qid); - qid = NULL; - continue; - } - - err = enumerate_tree(ibuf, &totlen, tree_id, "/", - pack, packidx, objcache, idset); - if (err) - goto done; - - got_object_qid_free(qid); - qid = NULL; - - parents = got_object_commit_get_parent_ids(commit); - if (parents) { - struct got_object_qid *pid; - STAILQ_FOREACH(pid, parents, entry) { - if (got_object_idset_contains(idset, &pid->id)) - continue; - err = got_object_qid_alloc_partial(&qid); - if (err) - goto done; - memcpy(&qid->id, &pid->id, sizeof(qid->id)); - STAILQ_INSERT_TAIL(&commit_ids, qid, entry); - qid = NULL; - } - } - - got_object_commit_close(commit); - commit = NULL; - } - - err = got_privsep_send_object_enumeration_done(ibuf); - if (err) - goto done; -done: - if (obj) - got_object_close(obj); - if (commit) - got_object_commit_close(commit); - got_object_qid_free(qid); - got_object_id_queue_free(&commit_ids); - got_object_idset_free(idset); - return err; -} - -static const struct got_error * receive_pack(struct got_pack **packp, struct imsgbuf *ibuf) { const struct got_error *err = NULL; @@ -1648,10 +1341,6 @@ main(int argc, char *argv[]) err = commit_traversal_request(&imsg, &ibuf, pack, packidx, &objcache); break; - case GOT_IMSG_OBJECT_ENUMERATION_REQUEST: - err = enumeration_request(&imsg, &ibuf, pack, - packidx, &objcache); - break; default: err = got_error(GOT_ERR_PRIVSEP_MSG); break;