commit 05118f5ae5dbf7f5e714baec9417e4192659d06a from: Stefan Sperling date: Tue Jun 22 19:37:20 2021 UTC implement gotadmin pack, indexpack, and listpack commands commit - e6bcace54ccc965bd93cf6769c7119f21cc1311e commit + 05118f5ae5dbf7f5e714baec9417e4192659d06a blob - 920c70117d0c5d2d028f95b8082678537cbf4f45 blob + fa31072bce1bf22d8516963a6d472bf2f45cfea1 --- gotadmin/Makefile +++ gotadmin/Makefile @@ -6,8 +6,9 @@ PROG= gotadmin SRCS= gotadmin.c \ deflate.c delta.c delta_cache.c deltify.c error.c gotconfig.c \ inflate.c lockfile.c object.c object_cache.c object_create.c \ - object_idset.c object_parse.c opentemp.c pack.c \ - path.c privsep.c reference.c repository.c sha1.c + object_idset.c object_parse.c opentemp.c pack.c pack_create.c \ + path.c privsep.c reference.c repository.c repository_admin.c \ + sha1.c MAN = ${PROG}.1 CPPFLAGS = -I${.CURDIR}/../include -I${.CURDIR}/../lib blob - 61c214f5d923cc2ae9198a585bfa364076f830eb blob + b87613ae1d71ee82a80a2d3c9bdbf4f980c22963 --- gotadmin/gotadmin.1 +++ gotadmin/gotadmin.1 @@ -69,7 +69,119 @@ Use the repository at the specified path. If not specified, assume the repository is located at or above the current working directory. .El +.It Cm pack Oo Fl a Oc Oo Fl r Ar repository-path Oc Oo Fl x Ar reference Oc Op Ar reference ... +Generate a new pack file and a corresponding pack file index. +By default, add any loose objects which are reachable via any references +to the generated pack file. +.Pp +If one or more +.Ar reference +arguments is specified, only add objects which are reachable via the specified +references. +Each +.Ar reference +argument may either specify a specific reference or a reference namespace, +in which case all references within this namespace will be used. +.Pp +.Cm gotadmin pack +always ignores references in the +.Pa refs/got/ +namespace, effectively treating such references as if they did not refer +to any objects. +.Pp +The options for +.Cm gotadmin pack +are as follows: +.Bl -tag -width Ds +.It Fl a +Add objects to the generated pack file even if they are already packed +in a different pack file. +Unless this option is specified, only loose objects will be added. +.It Fl r Ar repository-path +Use the repository at the specified path. +If not specified, assume the repository is located at or above the current +working directory. +.It Fl x Ar reference +Exclude objects reachable via the specified +.Ar reference +from the pack file. +The +.Ar reference +argument may either specify a specific reference or a reference namespace, +in which case all references within this namespace will be excluded. +The +.Fl x +option may be specified multiple times to build a list of references to exclude. +.Pp +Exclusion takes precedence over inclusion. +If a reference appears in both the included and excluded lists, it will +be excluded. .El +.It Cm indexpack Ar packfile-path +Create a pack index for the pack file at +.Ar packfile-path . +.Pp +A pack index is required for using the corresponding pack file with +.Xr got 1 . +Usually, a pack index will be created by commands such as +.Cm gotadmin pack +or +.Cm got fetch +as part of regular operation. +The +.Cm gotadmin indexpack +command may be used to recover from a corrupt or missing index. +A given pack file will always yield the same bit-identical index. +.Pp +The provided +.Ar packfile-path +must be located within the +.Pa objects/pack/ +directory of the repository and should end in +.Pa .pack . +The filename of the corresponding pack index is equivalent, except +that it ends in +.Pa .idx . +.Pp +.It Cm ix +Short alias for +.Cm indexpack . +.It Cm listpack Oo Fl h Oc Oo Fl s Oc Ar packfile-path +List the contents of the pack file at +.Ar packfile-path . +.Pp +Each object contained in the pack file will be displayed on a single line. +The information shown includes the object ID, object type, object offset, +and object size. +.Pp +If a packed object is deltified against another object the delta base +will be shown as well. +For offset deltas, the delta base is identified via an offset into the +pack file. +For reference deltas, the delta base is identified via an object ID. +.Pp +The provided +.Ar packfile-path +must be located within the +.Pa objects/pack/ +directory of the repository and should end in +.Pa .pack . +.Pp +The options for +.Cm gotadmin listpack +are as follows: +.Bl -tag -width Ds +.It Fl h +Show object sizes in human-readable form. +.It Fl s +Display statistics about the pack file after listing objects. +This includes the total number of objects stored in the pack file +and a break-down of the number of objects per object type. +.El +.It Cm ls +Short alias for +.Cm listpack . +.El .Sh EXIT STATUS .Ex -std gotadmin .Sh SEE ALSO @@ -79,3 +191,4 @@ working directory. .Xr got.conf 5 .Sh AUTHORS .An Stefan Sperling Aq Mt stsp@openbsd.org +.An Ori Bernstein Aq Mt ori@openbsd.org blob - 87e781ed58930ff07563b90ffc8f130eda1d0cfc blob + 090d85e840f61e0caff5c3956bfd96897183cdc0 --- gotadmin/gotadmin.c +++ gotadmin/gotadmin.c @@ -15,11 +15,15 @@ */ #include +#include +#include #include #include #include #include +#include +#include #include #include #include @@ -31,10 +35,11 @@ #include "got_error.h" #include "got_object.h" #include "got_reference.h" +#include "got_cancel.h" #include "got_repository.h" +#include "got_repository_admin.h" #include "got_gotconfig.h" #include "got_path.h" -#include "got_cancel.h" #include "got_privsep.h" #include "got_opentemp.h" @@ -57,6 +62,13 @@ catch_sigpipe(int signo) sigpipe_received = 1; } +static const struct got_error * +check_cancelled(void *arg) +{ + if (sigint_received || sigpipe_received) + return got_error(GOT_ERR_CANCELLED); + return NULL; +} struct gotadmin_cmd { const char *cmd_name; @@ -67,11 +79,20 @@ struct gotadmin_cmd { __dead static void usage(int, int); __dead static void usage_info(void); +__dead static void usage_pack(void); +__dead static void usage_indexpack(void); +__dead static void usage_listpack(void); static const struct got_error* cmd_info(int, char *[]); +static const struct got_error* cmd_pack(int, char *[]); +static const struct got_error* cmd_indexpack(int, char *[]); +static const struct got_error* cmd_listpack(int, char *[]); static struct gotadmin_cmd gotadmin_commands[] = { { "info", cmd_info, usage_info, "" }, + { "pack", cmd_pack, usage_pack, "" }, + { "indexpack", cmd_indexpack, usage_indexpack,"ix" }, + { "listpack", cmd_listpack, usage_listpack, "ls" }, }; static void @@ -302,4 +323,570 @@ done: got_repo_close(repo); free(cwd); return error; +} + +__dead static void +usage_pack(void) +{ + fprintf(stderr, "usage: %s pack [-a] [-r repository-path] " + "[-x reference] [reference ...]\n", + getprogname()); + exit(1); +} + +struct got_pack_progress_arg { + char last_scaled_size[FMT_SCALED_STRSIZE]; + int last_ncommits; + int last_nobj_total; + int last_p_deltify; + int last_p_written; + int last_p_indexed; + int last_p_resolved; + int verbosity; + int printed_something; +}; + +static const struct got_error * +pack_progress(void *arg, off_t packfile_size, int ncommits, + int nobj_total, int nobj_deltify, int nobj_written) +{ + struct got_pack_progress_arg *a = arg; + char scaled_size[FMT_SCALED_STRSIZE]; + int p_deltify, p_written; + int print_searching = 0, print_total = 0; + int print_deltify = 0, print_written = 0; + + if (a->verbosity < 0) + return NULL; + + if (fmt_scaled(packfile_size, scaled_size) == -1) + return got_error_from_errno("fmt_scaled"); + + if (a->last_ncommits != ncommits) { + print_searching = 1; + a->last_ncommits = ncommits; + } + + if (a->last_nobj_total != nobj_total) { + print_searching = 1; + print_total = 1; + a->last_nobj_total = nobj_total; + } + + if (packfile_size > 0 && (a->last_scaled_size[0] == '\0' || + strcmp(scaled_size, a->last_scaled_size)) != 0) { + if (strlcpy(a->last_scaled_size, scaled_size, + FMT_SCALED_STRSIZE) >= FMT_SCALED_STRSIZE) + return got_error(GOT_ERR_NO_SPACE); + } + + if (nobj_deltify > 0 || nobj_written > 0) { + if (nobj_deltify > 0) { + p_deltify = (nobj_deltify * 100) / nobj_total; + if (p_deltify != a->last_p_deltify) { + a->last_p_deltify = p_deltify; + print_searching = 1; + print_total = 1; + print_deltify = 1; + } + } + if (nobj_written > 0) { + p_written = (nobj_written * 100) / nobj_total; + if (p_written != a->last_p_written) { + a->last_p_written = p_written; + print_searching = 1; + print_total = 1; + print_deltify = 1; + print_written = 1; + } + } + } + + if (print_searching || print_total || print_deltify || print_written) + printf("\r"); + if (print_searching) + printf("packing %d reference%s", ncommits, + ncommits == 1 ? "" : "s"); + if (print_total) + printf("; %d object%s", nobj_total, + nobj_total == 1 ? "" : "s"); + if (print_deltify) + printf("; deltify: %d%%", p_deltify); + if (print_written) + printf("; writing pack: %*s %d%%", FMT_SCALED_STRSIZE, + scaled_size, p_written); + if (print_searching || print_total || print_deltify || + print_written) { + a->printed_something = 1; + fflush(stdout); + } + return NULL; +} + +static const struct got_error * +pack_index_progress(void *arg, off_t packfile_size, int nobj_total, + int nobj_indexed, int nobj_loose, int nobj_resolved) +{ + struct got_pack_progress_arg *a = arg; + char scaled_size[FMT_SCALED_STRSIZE]; + int p_indexed, p_resolved; + int print_size = 0, print_indexed = 0, print_resolved = 0; + + if (a->verbosity < 0) + return NULL; + + if (packfile_size > 0 || nobj_indexed > 0) { + if (fmt_scaled(packfile_size, scaled_size) == 0 && + (a->last_scaled_size[0] == '\0' || + strcmp(scaled_size, a->last_scaled_size)) != 0) { + print_size = 1; + if (strlcpy(a->last_scaled_size, scaled_size, + FMT_SCALED_STRSIZE) >= FMT_SCALED_STRSIZE) + return got_error(GOT_ERR_NO_SPACE); + } + if (nobj_indexed > 0) { + p_indexed = (nobj_indexed * 100) / nobj_total; + if (p_indexed != a->last_p_indexed) { + a->last_p_indexed = p_indexed; + print_indexed = 1; + print_size = 1; + } + } + if (nobj_resolved > 0) { + p_resolved = (nobj_resolved * 100) / + (nobj_total - nobj_loose); + if (p_resolved != a->last_p_resolved) { + a->last_p_resolved = p_resolved; + print_resolved = 1; + print_indexed = 1; + print_size = 1; + } + } + + } + if (print_size || print_indexed || print_resolved) + printf("\r"); + if (print_size) + printf("%*s packed", FMT_SCALED_STRSIZE, scaled_size); + if (print_indexed) + printf("; indexing %d%%", p_indexed); + if (print_resolved) + printf("; resolving deltas %d%%", p_resolved); + if (print_size || print_indexed || print_resolved) + fflush(stdout); + + return NULL; } + +static const struct got_error * +add_ref(struct got_reflist_entry **new, struct got_reflist_head *refs, + const char *refname, struct got_repository *repo) +{ + const struct got_error *err; + struct got_reference *ref; + + *new = NULL; + + err = got_ref_open(&ref, repo, refname, 0); + if (err) { + if (err->code != GOT_ERR_NOT_REF) + return err; + + /* Treat argument as a reference prefix. */ + err = got_ref_list(refs, repo, refname, + got_ref_cmp_by_name, NULL); + } else { + err = got_reflist_insert(new, refs, ref, repo, + got_ref_cmp_by_name, NULL); + if (err || *new == NULL /* duplicate */) + got_ref_close(ref); + } + + return err; +} + +static const struct got_error * +cmd_pack(int argc, char *argv[]) +{ + const struct got_error *error = NULL; + char *cwd = NULL, *repo_path = NULL; + struct got_repository *repo = NULL; + int ch, i, loose_obj_only = 1; + struct got_object_id *pack_hash = NULL; + char *id_str = NULL; + struct got_pack_progress_arg ppa; + FILE *packfile = NULL; + struct got_pathlist_head exclude_args; + struct got_pathlist_entry *pe; + struct got_reflist_head exclude_refs; + struct got_reflist_head include_refs; + struct got_reflist_entry *re, *new; + + TAILQ_INIT(&exclude_args); + TAILQ_INIT(&exclude_refs); + TAILQ_INIT(&include_refs); + + while ((ch = getopt(argc, argv, "ar:x:")) != -1) { + switch (ch) { + case 'a': + loose_obj_only = 0; + break; + case 'r': + repo_path = realpath(optarg, NULL); + if (repo_path == NULL) + return got_error_from_errno2("realpath", + optarg); + got_path_strip_trailing_slashes(repo_path); + break; + case 'x': + got_path_strip_trailing_slashes(optarg); + error = got_pathlist_append(&exclude_args, + optarg, NULL); + if (error) + return error; + break; + default: + usage_pack(); + /* NOTREACHED */ + } + } + + argc -= optind; + argv += optind; + +#ifndef PROFILE + if (pledge("stdio rpath wpath cpath fattr flock proc exec sendfd unveil", + NULL) == -1) + err(1, "pledge"); +#endif + cwd = getcwd(NULL, 0); + if (cwd == NULL) { + error = got_error_from_errno("getcwd"); + goto done; + } + + error = got_repo_open(&repo, repo_path ? repo_path : cwd, NULL); + if (error) + goto done; + + error = apply_unveil(got_repo_get_path_git_dir(repo), 1); + if (error) + goto done; + + TAILQ_FOREACH(pe, &exclude_args, entry) { + const char *refname = pe->path; + error = add_ref(&new, &exclude_refs, refname, repo); + if (error) + goto done; + + } + + if (argc == 0) { + error = got_ref_list(&include_refs, repo, "", + got_ref_cmp_by_name, NULL); + if (error) + goto done; + } else { + for (i = 0; i < argc; i++) { + const char *refname; + got_path_strip_trailing_slashes(argv[i]); + refname = argv[i]; + error = add_ref(&new, &include_refs, refname, repo); + if (error) + goto done; + } + } + + /* Ignore references in the refs/got/ namespace. */ + TAILQ_FOREACH_SAFE(re, &include_refs, entry, new) { + const char *refname = got_ref_get_name(re->ref); + if (strncmp("refs/got/", refname, 9) != 0) + continue; + TAILQ_REMOVE(&include_refs, re, entry); + got_ref_close(re->ref); + free(re); + } + + memset(&ppa, 0, sizeof(ppa)); + ppa.last_scaled_size[0] = '\0'; + ppa.last_p_indexed = -1; + ppa.last_p_resolved = -1; + + error = got_repo_pack_objects(&packfile, &pack_hash, + &include_refs, &exclude_refs, repo, loose_obj_only, + pack_progress, &ppa, check_cancelled, NULL); + if (error) { + if (ppa.printed_something) + printf("\n"); + goto done; + } + + error = got_object_id_str(&id_str, pack_hash); + if (error) + goto done; + printf("\nWrote %s.pack\n", id_str); + + error = got_repo_index_pack(packfile, pack_hash, repo, + pack_index_progress, &ppa, check_cancelled, NULL); + if (error) + goto done; + printf("\nIndexed %s.pack\n", id_str); +done: + got_pathlist_free(&exclude_args); + got_ref_list_free(&exclude_refs); + got_ref_list_free(&include_refs); + free(id_str); + free(pack_hash); + free(cwd); + return error; +} + +__dead static void +usage_indexpack(void) +{ + fprintf(stderr, "usage: %s indexpack packfile-path\n", + getprogname()); + exit(1); +} + +static const struct got_error * +cmd_indexpack(int argc, char *argv[]) +{ + const struct got_error *error = NULL; + struct got_repository *repo = NULL; + int ch; + struct got_object_id *pack_hash = NULL; + char *packfile_path = NULL; + char *id_str = NULL; + struct got_pack_progress_arg ppa; + FILE *packfile = NULL; + + while ((ch = getopt(argc, argv, "")) != -1) { + switch (ch) { + default: + usage_indexpack(); + /* NOTREACHED */ + } + } + + argc -= optind; + argv += optind; + + if (argc != 1) + usage_indexpack(); + + packfile_path = realpath(argv[0], NULL); + if (packfile_path == NULL) + return got_error_from_errno2("realpath", argv[0]); + +#ifndef PROFILE + if (pledge("stdio rpath wpath cpath fattr flock proc exec sendfd unveil", + NULL) == -1) + err(1, "pledge"); +#endif + + error = got_repo_open(&repo, packfile_path, NULL); + if (error) + goto done; + + error = apply_unveil(got_repo_get_path_git_dir(repo), 1); + if (error) + goto done; + + memset(&ppa, 0, sizeof(ppa)); + ppa.last_scaled_size[0] = '\0'; + ppa.last_p_indexed = -1; + ppa.last_p_resolved = -1; + + error = got_repo_find_pack(&packfile, &pack_hash, repo, + packfile_path); + if (error) + goto done; + + error = got_object_id_str(&id_str, pack_hash); + if (error) + goto done; + + error = got_repo_index_pack(packfile, pack_hash, repo, + pack_index_progress, &ppa, check_cancelled, NULL); + if (error) + goto done; + printf("\nIndexed %s.pack\n", id_str); +done: + free(id_str); + free(pack_hash); + return error; +} + +__dead static void +usage_listpack(void) +{ + fprintf(stderr, "usage: %s listpack [-h] [-s] packfile-path\n", + getprogname()); + exit(1); +} + +struct gotadmin_list_pack_cb_args { + int nblobs; + int ntrees; + int ncommits; + int ntags; + int noffdeltas; + int nrefdeltas; + int human_readable; +}; + +static const struct got_error * +list_pack_cb(void *arg, struct got_object_id *id, int type, off_t offset, + off_t size, off_t base_offset, struct got_object_id *base_id) +{ + const struct got_error *err; + struct gotadmin_list_pack_cb_args *a = arg; + char *id_str, *delta_str = NULL, *base_id_str = NULL; + const char *type_str; + + err = got_object_id_str(&id_str, id); + if (err) + return err; + + switch (type) { + case GOT_OBJ_TYPE_BLOB: + type_str = GOT_OBJ_LABEL_BLOB; + a->nblobs++; + break; + case GOT_OBJ_TYPE_TREE: + type_str = GOT_OBJ_LABEL_TREE; + a->ntrees++; + break; + case GOT_OBJ_TYPE_COMMIT: + type_str = GOT_OBJ_LABEL_COMMIT; + a->ncommits++; + break; + case GOT_OBJ_TYPE_TAG: + type_str = GOT_OBJ_LABEL_TAG; + a->ntags++; + break; + case GOT_OBJ_TYPE_OFFSET_DELTA: + type_str = "offset-delta"; + if (asprintf(&delta_str, " base-offset %llu", + base_offset) == -1) { + err = got_error_from_errno("asprintf"); + goto done; + } + a->noffdeltas++; + break; + case GOT_OBJ_TYPE_REF_DELTA: + type_str = "ref-delta"; + err = got_object_id_str(&base_id_str, base_id); + if (err) + goto done; + if (asprintf(&delta_str, " base-id %s", base_id_str) == -1) { + err = got_error_from_errno("asprintf"); + goto done; + } + a->nrefdeltas++; + break; + default: + err = got_error(GOT_ERR_OBJ_TYPE); + goto done; + } + if (a->human_readable) { + char scaled[FMT_SCALED_STRSIZE]; + char *s;; + if (fmt_scaled(size, scaled) == -1) { + err = got_error_from_errno("fmt_scaled"); + goto done; + } + s = scaled; + while (isspace((unsigned char)*s)) + s++; + printf("%s %s at %llu size %s%s\n", id_str, type_str, offset, + s, delta_str ? delta_str : ""); + } else { + printf("%s %s at %llu size %llu%s\n", id_str, type_str, offset, + size, delta_str ? delta_str : ""); + } +done: + free(id_str); + free(base_id_str); + free(delta_str); + return err; +} + +static const struct got_error * +cmd_listpack(int argc, char *argv[]) +{ + const struct got_error *error = NULL; + struct got_repository *repo = NULL; + int ch; + struct got_object_id *pack_hash = NULL; + char *packfile_path = NULL; + char *id_str = NULL; + struct gotadmin_list_pack_cb_args lpa; + FILE *packfile = NULL; + int show_stats = 0, human_readable = 0; + + while ((ch = getopt(argc, argv, "hs")) != -1) { + switch (ch) { + case 'h': + human_readable = 1; + break; + case 's': + show_stats = 1; + break; + default: + usage_listpack(); + /* NOTREACHED */ + } + } + + argc -= optind; + argv += optind; + + if (argc != 1) + usage_listpack(); + packfile_path = realpath(argv[0], NULL); + if (packfile_path == NULL) + return got_error_from_errno2("realpath", argv[0]); + +#ifndef PROFILE + if (pledge("stdio rpath wpath flock proc exec sendfd unveil", + NULL) == -1) + err(1, "pledge"); +#endif + error = got_repo_open(&repo, packfile_path, NULL); + if (error) + goto done; + + error = apply_unveil(got_repo_get_path_git_dir(repo), 1); + if (error) + goto done; + + error = got_repo_find_pack(&packfile, &pack_hash, repo, + packfile_path); + if (error) + goto done; + error = got_object_id_str(&id_str, pack_hash); + if (error) + goto done; + + memset(&lpa, 0, sizeof(lpa)); + lpa.human_readable = human_readable; + error = got_repo_list_pack(packfile, pack_hash, repo, + list_pack_cb, &lpa, check_cancelled, NULL); + if (error) + goto done; + if (show_stats) { + printf("objects: %d\n blobs: %d\n trees: %d\n commits: %d\n" + " tags: %d\n offset-deltas: %d\n ref-deltas: %d\n", + lpa.nblobs + lpa.ntrees + lpa.ncommits + lpa.ntags + + lpa.noffdeltas + lpa.nrefdeltas, + lpa.nblobs, lpa.ntrees, lpa.ncommits, lpa.ntags, + lpa.noffdeltas, lpa.nrefdeltas); + } +done: + free(id_str); + free(pack_hash); + free(packfile_path); + return error; +} blob - 9244c2c00f8a56e7ef54417632a95c0f7aa36ddd blob + 851d4ad4128c5bcac0cd5616088fb5063844a83f --- include/got_error.h +++ include/got_error.h @@ -145,6 +145,7 @@ #define GOT_ERR_NO_CONFIG_FILE 128 #define GOT_ERR_BAD_SYMLINK 129 #define GOT_ERR_GIT_REPO_EXT 130 +#define GOT_ERR_CANNOT_PACK 131 static const struct got_error { int code; @@ -297,6 +298,7 @@ static const struct got_error { { GOT_ERR_BAD_SYMLINK, "symbolic link points outside of paths under " "version control" }, { GOT_ERR_GIT_REPO_EXT, "unsupported repository format extension" }, + { GOT_ERR_CANNOT_PACK, "not enough objects to pack" }, }; /* blob - /dev/null blob + 6c27cd2e90cece2a2487a786135e4cb6b269f1ad (mode 644) --- /dev/null +++ include/got_repository_admin.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2021 Stefan Sperling + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +/* A callback function which gets invoked with progress information to print. */ +typedef const struct got_error *(*got_pack_progress_cb)(void *arg, + off_t packfile_size, int ncommits, int nobj_total, int obj_deltify, + int nobj_written); + +/* + * Attempt to pack objects reachable via 'include_refs' into a new packfile. + * If 'excluded_refs' is not an empty list, do not pack any objects + * reachable from the listed references. + * If loose_obj_only is zero, pack reachable objects even if they are + * already packed in another packfile. Otherwise, add only loose + * objects to the new pack file. + * Return an open file handle for the generated pack file. + * Return the SHA1 digest of the resulting pack file in pack_hash which + * must freed by the caller when done. + */ +const struct got_error * +got_repo_pack_objects(FILE **packfile, struct got_object_id **pack_hash, + struct got_reflist_head *include_refs, + struct got_reflist_head *exclude_refs, struct got_repository *repo, + int loose_obj_only, + got_pack_progress_cb progress_cb, void *progress_arg, + got_cancel_cb cancel_cb, void *cancel_arg); + +/* + * Attempt to open a pack file at the specified path. Return an open + * file handle and the expected hash of pack file contents. + */ +const struct got_error * +got_repo_find_pack(FILE **packfile, struct got_object_id **pack_hash, + struct got_repository *repo, const char *packfile_path); + +/* A callback function which gets invoked with progress information to print. */ +typedef const struct got_error *(*got_pack_index_progress_cb)(void *arg, + off_t packfile_size, int nobj_total, int nobj_indexed, + int nobj_loose, int nobj_resolved); + +/* (Re-)Index the pack file identified by the given hash. */ +const struct got_error * +got_repo_index_pack(FILE *packfile, struct got_object_id *pack_hash, + struct got_repository *repo, + got_pack_index_progress_cb progress_cb, void *progress_arg, + got_cancel_cb cancel_cb, void *cancel_arg); + +typedef const struct got_error *(*got_pack_list_cb)(void *arg, + struct got_object_id *id, int type, off_t offset, off_t size, + off_t base_offset, struct got_object_id *base_id); + +/* List the pack file identified by the given hash. */ +const struct got_error * +got_repo_list_pack(FILE *packfile, struct got_object_id *pack_hash, + struct got_repository *repo, got_pack_list_cb list_cb, void *list_arg, + got_cancel_cb cancel_cb, void *cancel_arg); blob - cddff333b1f0292034312d89f789fef6b2dcf720 blob + 180adaeec95923b909b3ce9e754eb670b166792d --- lib/got_lib_pack_create.h +++ lib/got_lib_pack_create.h @@ -24,4 +24,6 @@ const struct got_error *got_pack_create(uint8_t *pack_sha1, FILE *packfile, struct got_object_id **theirs, int ntheirs, struct got_object_id **ours, int nours, - struct got_repository *repo, got_cancel_cb cancel_cb, void *cancel_arg); + struct got_repository *repo, int loose_obj_only, + got_pack_progress_cb progress_cb, void *progress_arg, + got_cancel_cb cancel_cb, void *cancel_arg); blob - 5d3b9a7d345565cc8c91eb8cf92b750344d2998c blob + 687269a4a6ab8472e6491b4ac5485ba24c54934c --- lib/pack_create.c +++ lib/pack_create.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -29,13 +30,18 @@ #include "got_error.h" #include "got_cancel.h" #include "got_object.h" +#include "got_reference.h" +#include "got_repository_admin.h" #include "got_lib_deltify.h" #include "got_lib_delta.h" #include "got_lib_object.h" #include "got_lib_object_idset.h" +#include "got_lib_object_cache.h" #include "got_lib_deflate.h" #include "got_lib_pack.h" +#include "got_lib_privsep.h" +#include "got_lib_repository.h" #ifndef MAX #define MAX(_a,_b) ((_a) > (_b) ? (_a) : (_b)) @@ -136,16 +142,6 @@ delta_order_cmp(const void *pa, const void *pb) } static int -showprogress(int x, int pct) -{ - if(x > pct){ - pct = x; - fprintf(stderr, "\b\b\b\b%3d%%", pct); - } - return pct; -} - -static int delta_size(struct got_delta_instruction *deltas, int ndeltas) { int i, size = 32; @@ -160,19 +156,18 @@ delta_size(struct got_delta_instruction *deltas, int n static const struct got_error * -pick_deltas(struct got_pack_meta **meta, int nmeta, struct got_repository *repo, +pick_deltas(struct got_pack_meta **meta, int nmeta, int nours, + struct got_repository *repo, + got_pack_progress_cb progress_cb, void *progress_arg, got_cancel_cb cancel_cb, void *cancel_arg) { const struct got_error *err = NULL; struct got_pack_meta *m = NULL, *base = NULL; struct got_raw_object *raw = NULL, *base_raw = NULL; struct got_delta_instruction *deltas; - int i, j, size, ndeltas, pct, best; + int i, j, size, ndeltas, best; const int max_base_candidates = 10; - pct = 0; - fprintf(stderr, "picking deltas\n"); - fprintf(stderr, "deltifying %d objects: 0%%", nmeta); qsort(meta, nmeta, sizeof(struct got_pack_meta *), delta_order_cmp); for (i = 0; i < nmeta; i++) { if (cancel_cb) { @@ -180,9 +175,12 @@ pick_deltas(struct got_pack_meta **meta, int nmeta, st if (err) break; } - + if (progress_cb) { + err = progress_cb(progress_arg, 0L, nours, nmeta, i, 0); + if (err) + goto done; + } m = meta[i]; - pct = showprogress((i*100) / nmeta, pct); m->deltas = NULL; m->ndeltas = 0; @@ -257,8 +255,6 @@ pick_deltas(struct got_pack_meta **meta, int nmeta, st got_object_raw_close(raw); raw = NULL; } - - fprintf(stderr, "\b\b\b\b100%%\n"); done: for (i = MAX(0, nmeta - max_base_candidates); i < nmeta; i++) { got_deltify_free(meta[i]->dtab); @@ -272,13 +268,40 @@ done: } static const struct got_error * +search_packidx(int *found, struct got_object_id *id, + struct got_repository *repo) +{ + const struct got_error *err = NULL; + struct got_packidx *packidx = NULL; + int idx; + + *found = 0; + + err = got_repo_search_packidx(&packidx, &idx, repo, id); + if (err == NULL) + *found = 1; /* object is already packed */ + else if (err->code == GOT_ERR_NO_OBJ) + err = NULL; + return err; +} + +static const struct got_error * add_meta(struct got_pack_metavec *v, struct got_object_idset *idset, struct got_object_id *id, const char *path, int obj_type, - time_t mtime) + time_t mtime, int loose_obj_only, struct got_repository *repo) { const struct got_error *err; struct got_pack_meta *m; + if (loose_obj_only) { + int is_packed; + err = search_packidx(&is_packed, id, repo); + if (err) + return err; + if (is_packed) + return NULL; + } + err = got_object_idset_add(idset, id, NULL); if (err) return err; @@ -315,7 +338,7 @@ static const struct got_error * load_tree_entries(struct got_object_id_queue *ids, struct got_pack_metavec *v, struct got_object_idset *idset, struct got_object_id *tree_id, const char *dpath, time_t mtime, struct got_repository *repo, - got_cancel_cb cancel_cb, void *cancel_arg) + int loose_obj_only, got_cancel_cb cancel_cb, void *cancel_arg) { const struct got_error *err; struct got_tree_object *tree; @@ -356,7 +379,7 @@ load_tree_entries(struct got_object_id_queue *ids, str SIMPLEQ_INSERT_TAIL(ids, qid, entry); } else if (S_ISREG(mode)) { err = add_meta(v, idset, id, p, GOT_OBJ_TYPE_BLOB, - mtime); + mtime, loose_obj_only, repo); if (err) break; } @@ -372,7 +395,8 @@ load_tree_entries(struct got_object_id_queue *ids, str static const struct got_error * load_tree(struct got_pack_metavec *v, struct got_object_idset *idset, struct got_object_id *tree_id, const char *dpath, time_t mtime, - struct got_repository *repo, got_cancel_cb cancel_cb, void *cancel_arg) + int loose_obj_only, struct got_repository *repo, + got_cancel_cb cancel_cb, void *cancel_arg) { const struct got_error *err = NULL; struct got_object_id_queue tree_ids; @@ -404,14 +428,14 @@ load_tree(struct got_pack_metavec *v, struct got_objec } err = add_meta(v, idset, qid->id, dpath, GOT_OBJ_TYPE_TREE, - mtime); + mtime, loose_obj_only, repo); if (err) { got_object_qid_free(qid); break; } err = load_tree_entries(&tree_ids, v, idset, qid->id, dpath, - mtime, repo, cancel_cb, cancel_arg); + mtime, repo, loose_obj_only, cancel_cb, cancel_arg); got_object_qid_free(qid); if (err) break; @@ -423,7 +447,7 @@ load_tree(struct got_pack_metavec *v, struct got_objec static const struct got_error * load_commit(struct got_pack_metavec *v, struct got_object_idset *idset, - struct got_object_id *id, struct got_repository *repo, + struct got_object_id *id, struct got_repository *repo, int loose_obj_only, got_cancel_cb cancel_cb, void *cancel_arg) { const struct got_error *err; @@ -431,24 +455,84 @@ load_commit(struct got_pack_metavec *v, struct got_obj if (got_object_idset_contains(idset, id)) return NULL; + + if (loose_obj_only) { + int is_packed; + err = search_packidx(&is_packed, id, repo); + if (err) + return err; + if (is_packed) + return NULL; + } err = got_object_open_as_commit(&commit, repo, id); if (err) return err; err = add_meta(v, idset, id, "", GOT_OBJ_TYPE_COMMIT, - got_object_commit_get_committer_time(commit)); + got_object_commit_get_committer_time(commit), + loose_obj_only, repo); if (err) goto done; err = load_tree(v, idset, got_object_commit_get_tree_id(commit), - "", got_object_commit_get_committer_time(commit), repo, - cancel_cb, cancel_arg); + "", got_object_commit_get_committer_time(commit), + loose_obj_only, repo, cancel_cb, cancel_arg); done: got_object_commit_close(commit); return err; } +static const struct got_error * +load_tag(struct got_pack_metavec *v, struct got_object_idset *idset, + struct got_object_id *id, struct got_repository *repo, int loose_obj_only, + got_cancel_cb cancel_cb, void *cancel_arg) +{ + const struct got_error *err; + struct got_tag_object *tag = NULL; + + if (got_object_idset_contains(idset, id)) + return NULL; + + if (loose_obj_only) { + int is_packed; + err = search_packidx(&is_packed, id, repo); + if (err) + return err; + if (is_packed) + return NULL; + } + + err = got_object_open_as_tag(&tag, repo, id); + if (err) + return err; + + err = add_meta(v, idset, id, "", GOT_OBJ_TYPE_TAG, + got_object_tag_get_tagger_time(tag), + loose_obj_only, repo); + if (err) + goto done; + + switch (got_object_tag_get_object_type(tag)) { + case GOT_OBJ_TYPE_COMMIT: + err = load_commit(NULL, idset, + got_object_tag_get_object_id(tag), repo, + loose_obj_only, cancel_cb, cancel_arg); + break; + case GOT_OBJ_TYPE_TREE: + err = load_tree(v, idset, got_object_tag_get_object_id(tag), + "", got_object_tag_get_tagger_time(tag), + loose_obj_only, repo, cancel_cb, cancel_arg); + break; + default: + break; + } + +done: + got_object_tag_close(tag); + return err; +} + enum findtwixt_color { COLOR_KEEP = 0, COLOR_DROP, @@ -466,22 +550,7 @@ queue_commit_id(struct got_object_id_queue *ids, struc { const struct got_error *err; struct got_object_qid *qid; - char *id_str; - int obj_type; - err = got_object_get_type(&obj_type, repo, id); - if (err) - return err; - - if (obj_type != GOT_OBJ_TYPE_COMMIT) { - err = got_object_id_str(&id_str, id); - if (err) - return err; - err = got_error_fmt(GOT_ERR_OBJ_TYPE, - "%s is not a commit", id_str); - free(id_str); - return err; - } err = got_object_qid_alloc(&qid, id); if (err) return err; @@ -584,7 +653,7 @@ findtwixt(struct got_object_id ***res, int *nres, struct got_object_id_queue ids; struct got_object_idset *keep, *drop; struct got_object_qid *qid; - int i, ncolor, nkeep; + int i, ncolor, nkeep, obj_type; SIMPLEQ_INIT(&ids); *res = NULL; @@ -600,19 +669,31 @@ findtwixt(struct got_object_id ***res, int *nres, goto done; } - for (i = 0; i < nhead; i++){ - if (head[i]) { - err = queue_commit_id(&ids, head[i], COLOR_KEEP, repo); - if (err) - goto done; - } + for (i = 0; i < nhead; i++) { + struct got_object_id *id = head[i]; + if (id == NULL) + continue; + err = got_object_get_type(&obj_type, repo, id); + if (err) + return err; + if (obj_type != GOT_OBJ_TYPE_COMMIT) + continue; + err = queue_commit_id(&ids, id, COLOR_KEEP, repo); + if (err) + goto done; } - for (i = 0; i < ntail; i++){ - if (tail[i]) { - err = queue_commit_id(&ids, tail[i], COLOR_DROP, repo); - if (err) - goto done; - } + for (i = 0; i < ntail; i++) { + struct got_object_id *id = tail[i]; + if (id == NULL) + continue; + err = got_object_get_type(&obj_type, repo, id); + if (err) + return err; + if (obj_type != GOT_OBJ_TYPE_COMMIT) + continue; + err = queue_commit_id(&ids, id, COLOR_DROP, repo); + if (err) + goto done; } while (!SIMPLEQ_EMPTY(&ids)) { @@ -716,12 +797,13 @@ static const struct got_error * read_meta(struct got_pack_meta ***meta, int *nmeta, struct got_object_id **theirs, int ntheirs, struct got_object_id **ours, int nours, struct got_repository *repo, + int loose_obj_only, got_pack_progress_cb progress_cb, void *progress_arg, got_cancel_cb cancel_cb, void *cancel_arg) { const struct got_error *err = NULL; struct got_object_id **ids = NULL; struct got_object_idset *idset; - int i, nobj = 0; + int i, nobj = 0, obj_type; struct got_pack_metavec v; *meta = NULL; @@ -745,9 +827,21 @@ read_meta(struct got_pack_meta ***meta, int *nmeta, goto done; for (i = 0; i < ntheirs; i++) { - if (theirs[i] != NULL) { - err = load_commit(NULL, idset, theirs[i], repo, - cancel_cb, cancel_arg); + struct got_object_id *id = theirs[i]; + if (id == NULL) + continue; + err = got_object_get_type(&obj_type, repo, id); + if (err) + return err; + if (obj_type != GOT_OBJ_TYPE_COMMIT) + continue; + err = load_commit(NULL, idset, id, repo, + loose_obj_only, cancel_cb, cancel_arg); + if (err) + goto done; + if (progress_cb) { + err = progress_cb(progress_arg, 0L, nours, + v.nmeta, 0, 0); if (err) goto done; } @@ -755,10 +849,59 @@ read_meta(struct got_pack_meta ***meta, int *nmeta, for (i = 0; i < nobj; i++) { err = load_commit(&v, idset, ids[i], repo, - cancel_cb, cancel_arg); + loose_obj_only, cancel_cb, cancel_arg); + if (err) + goto done; + if (progress_cb) { + err = progress_cb(progress_arg, 0L, nours, + v.nmeta, 0, 0); + if (err) + goto done; + } + } + + for (i = 0; i < ntheirs; i++) { + struct got_object_id *id = ours[i]; + if (id == NULL) + continue; + err = got_object_get_type(&obj_type, repo, id); + if (err) + return err; + if (obj_type != GOT_OBJ_TYPE_TAG) + continue; + err = load_tag(NULL, idset, id, repo, + loose_obj_only, cancel_cb, cancel_arg); + if (err) + goto done; + if (progress_cb) { + err = progress_cb(progress_arg, 0L, nours, + v.nmeta, 0, 0); + if (err) + goto done; + } + } + + for (i = 0; i < nours; i++) { + struct got_object_id *id = ours[i]; + if (id == NULL) + continue; + err = got_object_get_type(&obj_type, repo, id); if (err) + return err; + if (obj_type != GOT_OBJ_TYPE_TAG) + continue; + err = load_tag(&v, idset, id, repo, + loose_obj_only, cancel_cb, cancel_arg); + if (err) goto done; + if (progress_cb) { + err = progress_cb(progress_arg, 0L, nours, + v.nmeta, 0, 0); + if (err) + goto done; + } } + done: for (i = 0; i < nobj; i++) { free(ids[i]); @@ -970,25 +1113,25 @@ packoff(char *hdr, off_t off) static const struct got_error * genpack(uint8_t *pack_sha1, FILE *packfile, - struct got_pack_meta **meta, int nmeta, int use_offset_deltas, - struct got_repository *repo) + struct got_pack_meta **meta, int nmeta, int nours, + int use_offset_deltas, struct got_repository *repo, + got_pack_progress_cb progress_cb, void *progress_arg, + got_cancel_cb cancel_cb, void *cancel_arg) { const struct got_error *err = NULL; - int i, nh, nd, pct; + int i, nh, nd; SHA1_CTX ctx; struct got_pack_meta *m; struct got_raw_object *raw; char *p = NULL, buf[32]; size_t outlen, n; struct got_deflate_checksum csum; + off_t packfile_size = 0; SHA1Init(&ctx); csum.output_sha1 = &ctx; csum.output_crc = NULL; - pct = 0; - fprintf(stderr, "generating pack\n"); - err = hwrite(packfile, "PACK", 4, &ctx); if (err) return err; @@ -1001,9 +1144,13 @@ genpack(uint8_t *pack_sha1, FILE *packfile, if (err) goto done; qsort(meta, nmeta, sizeof(struct got_pack_meta *), write_order_cmp); - fprintf(stderr, "writing %d objects: 0%%", nmeta); - for (i = 0; i < nmeta; i++) { - pct = showprogress((i*100)/nmeta, pct); + for (i = 0; i < nmeta; i++) { + if (progress_cb) { + err = progress_cb(progress_arg, packfile_size, nours, + nmeta, nmeta, i); + if (err) + goto done; + } m = meta[i]; m->off = ftello(packfile); err = got_object_raw_open(&raw, repo, &m->id, 8192); @@ -1017,6 +1164,7 @@ genpack(uint8_t *pack_sha1, FILE *packfile, err = hwrite(packfile, buf, nh, &ctx); if (err) goto done; + packfile_size += nh; if (fseeko(raw->f, raw->hdrlen, SEEK_SET) == -1) { err = got_error_from_errno("fseeko"); goto done; @@ -1025,6 +1173,7 @@ genpack(uint8_t *pack_sha1, FILE *packfile, &csum); if (err) goto done; + packfile_size += outlen; } else { FILE *delta_file; struct got_raw_object *base_raw; @@ -1046,14 +1195,17 @@ genpack(uint8_t *pack_sha1, FILE *packfile, err = hwrite(packfile, buf, nh, &ctx); if (err) goto done; + packfile_size += nh; } else { err = packhdr(&nh, buf, sizeof(buf), GOT_OBJ_TYPE_REF_DELTA, nd); err = hwrite(packfile, buf, nh, &ctx); if (err) goto done; + packfile_size += nh; err = hwrite(packfile, m->prev->id.sha1, sizeof(m->prev->id.sha1), &ctx); + packfile_size += sizeof(m->prev->id.sha1); if (err) goto done; } @@ -1068,17 +1220,23 @@ genpack(uint8_t *pack_sha1, FILE *packfile, fclose(delta_file); if (err) goto done; + packfile_size += outlen; free(p); p = NULL; } got_object_raw_close(raw); raw = NULL; } - fprintf(stderr, "\b\b\b\b100%%\n"); SHA1Final(pack_sha1, &ctx); n = fwrite(pack_sha1, 1, SHA1_DIGEST_LENGTH, packfile); if (n != SHA1_DIGEST_LENGTH) err = got_ferror(packfile, GOT_ERR_IO); + packfile_size += SHA1_DIGEST_LENGTH; + packfile_size += 16; /* pack file header */ + err = progress_cb(progress_arg, packfile_size, nours, + nmeta, nmeta, nmeta); + if (err) + goto done; done: free(p); return err; @@ -1088,22 +1246,31 @@ const struct got_error * got_pack_create(uint8_t *packsha1, FILE *packfile, struct got_object_id **theirs, int ntheirs, struct got_object_id **ours, int nours, - struct got_repository *repo, got_cancel_cb cancel_cb, void *cancel_arg) + struct got_repository *repo, int loose_obj_only, + got_pack_progress_cb progress_cb, void *progress_arg, + got_cancel_cb cancel_cb, void *cancel_arg) { const struct got_error *err; struct got_pack_meta **meta; int nmeta; err = read_meta(&meta, &nmeta, theirs, ntheirs, ours, nours, repo, - cancel_cb, cancel_arg); + loose_obj_only, progress_cb, progress_arg, cancel_cb, cancel_arg); if (err) return err; - err = pick_deltas(meta, nmeta, repo, cancel_cb, cancel_arg); + if (nmeta == 0) { + err = got_error(GOT_ERR_CANNOT_PACK); + goto done; + } + + err = pick_deltas(meta, nmeta, nours, repo, + progress_cb, progress_arg, cancel_cb, cancel_arg); if (err) goto done; - err = genpack(packsha1, packfile, meta, nmeta, 1, repo); + err = genpack(packsha1, packfile, meta, nmeta, nours, 1, repo, + progress_cb, progress_arg, cancel_cb, cancel_arg); if (err) goto done; done: blob - /dev/null blob + 95151516f8146f021c5f8f3a919b810499c016e3 (mode 644) --- /dev/null +++ lib/repository_admin.c @@ -0,0 +1,594 @@ +/* + * Copyright (c) 2020 Stefan Sperling + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "got_error.h" +#include "got_cancel.h" +#include "got_object.h" +#include "got_reference.h" +#include "got_repository.h" +#include "got_repository_admin.h" +#include "got_opentemp.h" +#include "got_path.h" + +#include "got_lib_delta.h" +#include "got_lib_object.h" +#include "got_lib_object_cache.h" +#include "got_lib_pack.h" +#include "got_lib_privsep.h" +#include "got_lib_repository.h" +#include "got_lib_pack_create.h" +#include "got_lib_sha1.h" + +#ifndef nitems +#define nitems(_a) (sizeof((_a)) / sizeof((_a)[0])) +#endif + +static const struct got_error * +get_reflist_object_ids(struct got_object_id ***ids, int *nobjects, + unsigned int wanted_obj_type_mask, struct got_reflist_head *refs, + struct got_repository *repo, + got_cancel_cb cancel_cb, void *cancel_arg) +{ + const struct got_error *err = NULL; + const size_t alloc_chunksz = 256; + size_t nalloc; + struct got_reflist_entry *re; + int i; + + *ids = NULL; + *nobjects = 0; + + *ids = reallocarray(NULL, alloc_chunksz, sizeof(struct got_object_id *)); + if (*ids == NULL) + return got_error_from_errno("reallocarray"); + nalloc = alloc_chunksz; + + TAILQ_FOREACH(re, refs, entry) { + struct got_object_id *id; + + if (cancel_cb) { + err = cancel_cb(cancel_arg); + if (err) + goto done; + } + + err = got_ref_resolve(&id, repo, re->ref); + if (err) + goto done; + + if (wanted_obj_type_mask != GOT_OBJ_TYPE_ANY) { + int obj_type; + err = got_object_get_type(&obj_type, repo, id); + if (err) + goto done; + if ((wanted_obj_type_mask & (1 << obj_type)) == 0) { + free(id); + id = NULL; + continue; + } + } + + if (nalloc >= *nobjects) { + struct got_object_id **new; + new = recallocarray(*ids, nalloc, + nalloc + alloc_chunksz, + sizeof(struct got_object_id *)); + if (new == NULL) { + err = got_error_from_errno( + "recallocarray"); + goto done; + } + *ids = new; + nalloc += alloc_chunksz; + } + (*ids)[*nobjects] = id; + if ((*ids)[*nobjects] == NULL) { + err = got_error_from_errno("got_object_id_dup"); + goto done; + } + (*nobjects)++; + } +done: + if (err) { + for (i = 0; i < *nobjects; i++) + free((*ids)[i]); + free(*ids); + *ids = NULL; + *nobjects = 0; + } + return err; +} + +const struct got_error * +got_repo_pack_objects(FILE **packfile, struct got_object_id **pack_hash, + struct got_reflist_head *include_refs, + struct got_reflist_head *exclude_refs, struct got_repository *repo, + int loose_obj_only, got_pack_progress_cb progress_cb, void *progress_arg, + got_cancel_cb cancel_cb, void *cancel_arg) +{ + const struct got_error *err = NULL; + struct got_object_id **ours = NULL, **theirs = NULL; + int nours = 0, ntheirs = 0, packfd = -1, i; + char *tmpfile_path = NULL, *path = NULL, *packfile_path = NULL; + char *sha1_str = NULL; + + *packfile = NULL; + *pack_hash = NULL; + + if (asprintf(&path, "%s/%s/packing.pack", + got_repo_get_path_git_dir(repo), GOT_OBJECTS_PACK_DIR) == -1) { + err = got_error_from_errno("asprintf"); + goto done; + } + err = got_opentemp_named_fd(&tmpfile_path, &packfd, path); + if (err) + goto done; + + if (fchmod(packfd, GOT_DEFAULT_FILE_MODE) != 0) { + err = got_error_from_errno2("fchmod", tmpfile_path); + goto done; + } + + *packfile = fdopen(packfd, "w"); + if (*packfile == NULL) { + err = got_error_from_errno2("fdopen", tmpfile_path); + goto done; + } + packfd = -1; + + err = get_reflist_object_ids(&ours, &nours, + (1 << GOT_OBJ_TYPE_COMMIT) | (1 << GOT_OBJ_TYPE_TAG), + include_refs, repo, cancel_cb, cancel_arg); + if (err) + goto done; + + if (nours == 0) { + err = got_error(GOT_ERR_CANNOT_PACK); + goto done; + } + + if (!TAILQ_EMPTY(exclude_refs)) { + err = get_reflist_object_ids(&theirs, &ntheirs, + (1 << GOT_OBJ_TYPE_COMMIT) | (1 << GOT_OBJ_TYPE_TAG), + exclude_refs, repo, + cancel_cb, cancel_arg); + if (err) + goto done; + } + + *pack_hash = calloc(1, sizeof(**pack_hash)); + if (*pack_hash == NULL) { + err = got_error_from_errno("calloc"); + goto done; + } + + err = got_pack_create((*pack_hash)->sha1, *packfile, theirs, ntheirs, + ours, nours, repo, loose_obj_only, progress_cb, progress_arg, + cancel_cb, cancel_arg); + if (err) + goto done; + + err = got_object_id_str(&sha1_str, *pack_hash); + if (err) + goto done; + if (asprintf(&packfile_path, "%s/%s/pack-%s.pack", + got_repo_get_path_git_dir(repo), GOT_OBJECTS_PACK_DIR, + sha1_str) == -1) { + err = got_error_from_errno("asprintf"); + goto done; + } + + if (fflush(*packfile) == -1) { + err = got_error_from_errno("fflush"); + goto done; + } + if (fseek(*packfile, 0L, SEEK_SET) == -1) { + err = got_error_from_errno("fseek"); + goto done; + } + if (rename(tmpfile_path, packfile_path) == -1) { + err = got_error_from_errno3("rename", tmpfile_path, + packfile_path); + goto done; + } + free(tmpfile_path); + tmpfile_path = NULL; +done: + for (i = 0; i < nours; i++) + free(ours[i]); + free(ours); + for (i = 0; i < ntheirs; i++) + free(theirs[i]); + free(theirs); + if (packfd != -1 && close(packfd) == -1 && err == NULL) + err = got_error_from_errno2("close", packfile_path); + if (tmpfile_path && unlink(tmpfile_path) == -1 && err == NULL) + err = got_error_from_errno2("unlink", tmpfile_path); + free(tmpfile_path); + free(packfile_path); + free(sha1_str); + free(path); + if (err) { + free(*pack_hash); + *pack_hash = NULL; + if (*packfile) + fclose(*packfile); + *packfile = NULL; + } + return err; +} + +const struct got_error * +got_repo_index_pack(FILE *packfile, struct got_object_id *pack_hash, + struct got_repository *repo, + got_pack_index_progress_cb progress_cb, void *progress_arg, + got_cancel_cb cancel_cb, void *cancel_arg) +{ + size_t i; + char *path; + int imsg_idxfds[2]; + int npackfd = -1, idxfd = -1, nidxfd = -1; + int tmpfds[3]; + int idxstatus, done = 0; + const struct got_error *err; + struct imsgbuf idxibuf; + pid_t idxpid; + char *tmpidxpath = NULL; + char *packfile_path = NULL, *idxpath = NULL, *id_str = NULL; + const char *repo_path = got_repo_get_path_git_dir(repo); + struct stat sb; + + for (i = 0; i < nitems(tmpfds); i++) + tmpfds[i] = -1; + + if (asprintf(&path, "%s/%s/indexing.idx", + repo_path, GOT_OBJECTS_PACK_DIR) == -1) { + err = got_error_from_errno("asprintf"); + goto done; + } + err = got_opentemp_named_fd(&tmpidxpath, &idxfd, path); + free(path); + if (err) + goto done; + if (fchmod(idxfd, GOT_DEFAULT_FILE_MODE) != 0) { + err = got_error_from_errno2("fchmod", tmpidxpath); + goto done; + } + + nidxfd = dup(idxfd); + if (nidxfd == -1) { + err = got_error_from_errno("dup"); + goto done; + } + + for (i = 0; i < nitems(tmpfds); i++) { + tmpfds[i] = got_opentempfd(); + if (tmpfds[i] == -1) { + err = got_error_from_errno("got_opentempfd"); + goto done; + } + } + + err = got_object_id_str(&id_str, pack_hash); + if (err) + goto done; + + if (asprintf(&packfile_path, "%s/%s/pack-%s.pack", + repo_path, GOT_OBJECTS_PACK_DIR, id_str) == -1) { + err = got_error_from_errno("asprintf"); + goto done; + } + + if (fstat(fileno(packfile), &sb) == -1) { + err = got_error_from_errno2("fstat", packfile_path); + goto done; + } + + if (asprintf(&idxpath, "%s/%s/pack-%s.idx", + repo_path, GOT_OBJECTS_PACK_DIR, id_str) == -1) { + err = got_error_from_errno("asprintf"); + goto done; + } + + if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_idxfds) == -1) { + err = got_error_from_errno("socketpair"); + goto done; + } + idxpid = fork(); + if (idxpid == -1) { + err= got_error_from_errno("fork"); + goto done; + } else if (idxpid == 0) + got_privsep_exec_child(imsg_idxfds, + GOT_PATH_PROG_INDEX_PACK, packfile_path); + if (close(imsg_idxfds[1]) == -1) { + err = got_error_from_errno("close"); + goto done; + } + imsg_init(&idxibuf, imsg_idxfds[0]); + + npackfd = dup(fileno(packfile)); + if (npackfd == -1) { + err = got_error_from_errno("dup"); + goto done; + } + err = got_privsep_send_index_pack_req(&idxibuf, pack_hash->sha1, + npackfd); + if (err != NULL) + goto done; + npackfd = -1; + err = got_privsep_send_index_pack_outfd(&idxibuf, nidxfd); + if (err != NULL) + goto done; + nidxfd = -1; + for (i = 0; i < nitems(tmpfds); i++) { + err = got_privsep_send_tmpfd(&idxibuf, tmpfds[i]); + if (err != NULL) + goto done; + tmpfds[i] = -1; + } + done = 0; + while (!done) { + int nobj_total, nobj_indexed, nobj_loose, nobj_resolved; + + if (cancel_cb) { + err = cancel_cb(cancel_arg); + if (err) + goto done; + } + + err = got_privsep_recv_index_progress(&done, &nobj_total, + &nobj_indexed, &nobj_loose, &nobj_resolved, + &idxibuf); + if (err != NULL) + goto done; + if (nobj_indexed != 0) { + err = progress_cb(progress_arg, sb.st_size, + nobj_total, nobj_indexed, nobj_loose, + nobj_resolved); + if (err) + break; + } + imsg_clear(&idxibuf); + } + if (close(imsg_idxfds[0]) == -1) { + err = got_error_from_errno("close"); + goto done; + } + if (waitpid(idxpid, &idxstatus, 0) == -1) { + err = got_error_from_errno("waitpid"); + goto done; + } + + if (rename(tmpidxpath, idxpath) == -1) { + err = got_error_from_errno3("rename", tmpidxpath, idxpath); + goto done; + } + free(tmpidxpath); + tmpidxpath = NULL; + +done: + if (tmpidxpath && unlink(tmpidxpath) == -1 && err == NULL) + err = got_error_from_errno2("unlink", tmpidxpath); + if (npackfd != -1 && close(npackfd) == -1 && err == NULL) + err = got_error_from_errno("close"); + if (idxfd != -1 && close(idxfd) == -1 && err == NULL) + err = got_error_from_errno("close"); + for (i = 0; i < nitems(tmpfds); i++) { + if (tmpfds[i] != -1 && close(tmpfds[i]) == -1 && err == NULL) + err = got_error_from_errno("close"); + } + free(tmpidxpath); + free(idxpath); + free(packfile_path); + return err; +} + +const struct got_error * +got_repo_find_pack(FILE **packfile, struct got_object_id **pack_hash, + struct got_repository *repo, const char *packfile_path) +{ + const struct got_error *err = NULL; + const char *packdir_path = NULL; + char *packfile_name = NULL, *p, *dot; + struct got_object_id id; + int packfd = -1; + + *packfile = NULL; + *pack_hash = NULL; + + packdir_path = got_repo_get_path_objects_pack(repo); + if (packdir_path == NULL) + return got_error_from_errno("got_repo_get_path_objects_pack"); + + if (!got_path_is_child(packfile_path, packdir_path, + strlen(packdir_path))) { + err = got_error_path(packfile_path, GOT_ERR_BAD_PATH); + goto done; + + } + + err = got_path_basename(&packfile_name, packfile_path); + if (err) + goto done; + p = packfile_name; + + if (strncmp(p, "pack-", 5) != 0) { + err = got_error_fmt(GOT_ERR_BAD_PATH, + "'%s' is not a valid pack file name", + packfile_name); + goto done; + } + p += 5; + dot = strchr(p, '.'); + if (dot == NULL) { + err = got_error_fmt(GOT_ERR_BAD_PATH, + "'%s' is not a valid pack file name", + packfile_name); + goto done; + } + if (strcmp(dot + 1, "pack") != 0) { + err = got_error_fmt(GOT_ERR_BAD_PATH, + "'%s' is not a valid pack file name", + packfile_name); + goto done; + } + *dot = '\0'; + if (!got_parse_sha1_digest(id.sha1, p)) { + err = got_error_fmt(GOT_ERR_BAD_PATH, + "'%s' is not a valid pack file name", + packfile_name); + goto done; + } + + *pack_hash = got_object_id_dup(&id); + if (*pack_hash == NULL) { + err = got_error_from_errno("got_object_id_dup"); + goto done; + } + + packfd = open(packfile_path, O_RDONLY | O_NOFOLLOW); + if (packfd == -1) { + err = got_error_from_errno2("open", packfile_path); + goto done; + } + + *packfile = fdopen(packfd, "r"); + if (*packfile == NULL) { + err = got_error_from_errno2("fdopen", packfile_path); + goto done; + } + packfd = -1; +done: + if (packfd != -1 && close(packfd) == -1 && err == NULL) + err = got_error_from_errno2("close", packfile_path); + free(packfile_name); + if (err) { + free(*pack_hash); + *pack_hash = NULL; + } + return err; +} + +const struct got_error * +got_repo_list_pack(FILE *packfile, struct got_object_id *pack_hash, + struct got_repository *repo, got_pack_list_cb list_cb, void *list_arg, + got_cancel_cb cancel_cb, void *cancel_arg) +{ + const struct got_error *err = NULL; + char *id_str = NULL, *idxpath = NULL, *packpath = NULL; + struct got_packidx *packidx = NULL; + struct got_pack *pack = NULL; + uint32_t nobj, i; + + err = got_object_id_str(&id_str, pack_hash); + if (err) + goto done; + + if (asprintf(&packpath, "%s/pack-%s.pack", + GOT_OBJECTS_PACK_DIR, id_str) == -1) { + err = got_error_from_errno("asprintf"); + goto done; + } + if (asprintf(&idxpath, "%s/pack-%s.idx", + GOT_OBJECTS_PACK_DIR, id_str) == -1) { + err = got_error_from_errno("asprintf"); + goto done; + } + + err = got_packidx_open(&packidx, got_repo_get_fd(repo), idxpath, 1); + if (err) + goto done; + + err = got_repo_cache_pack(&pack, repo, packpath, packidx); + if (err) + goto done; + + nobj = be32toh(packidx->hdr.fanout_table[0xff]); + for (i = 0; i < nobj; i++) { + struct got_packidx_object_id *oid; + struct got_object_id id, base_id; + off_t offset, base_offset = 0; + uint8_t type; + uint64_t size; + size_t tslen, len; + + if (cancel_cb) { + err = cancel_cb(cancel_arg); + if (err) + break; + } + oid = &packidx->hdr.sorted_ids[i]; + memcpy(id.sha1, oid->sha1, SHA1_DIGEST_LENGTH); + + offset = got_packidx_get_object_offset(packidx, i); + if (offset == -1) { + err = got_error(GOT_ERR_BAD_PACKIDX); + goto done; + } + + err = got_pack_parse_object_type_and_size(&type, &size, &tslen, + pack, offset); + if (err) + goto done; + + switch (type) { + case GOT_OBJ_TYPE_OFFSET_DELTA: + err = got_pack_parse_offset_delta(&base_offset, &len, + pack, offset, tslen); + if (err) + goto done; + break; + case GOT_OBJ_TYPE_REF_DELTA: + err = got_pack_parse_ref_delta(&base_id, + pack, offset, tslen); + if (err) + goto done; + break; + } + err = (*list_cb)(list_arg, &id, type, offset, size, + base_offset, &base_id); + if (err) + goto done; + } + +done: + free(id_str); + free(idxpath); + free(packpath); + if (packidx) + got_packidx_close(packidx); + return err; +} blob - e32b5a142455da777776713187383ad1166a01d4 blob + 994b3c728968b83c996b8eb7fd40f01354a38383 --- regress/cmdline/Makefile +++ regress/cmdline/Makefile @@ -1,6 +1,6 @@ REGRESS_TARGETS=checkout update status log add rm diff blame branch tag \ ref commit revert cherrypick backout rebase import histedit \ - integrate stage unstage cat clone fetch tree + integrate stage unstage cat clone fetch tree pack NOOBJ=Yes GOT_TEST_ROOT=/tmp @@ -80,4 +80,7 @@ fetch: tree: ./tree.sh -q -r "$(GOT_TEST_ROOT)" +pack: + ./pack.sh -q -r "$(GOT_TEST_ROOT)" + .include blob - /dev/null blob + b6d15ae7a39b4f755d66d6ea89e268ee45b22005 (mode 755) --- /dev/null +++ regress/cmdline/pack.sh @@ -0,0 +1,589 @@ +#!/bin/sh +# +# Copyright (c) 2021 Stefan Sperling +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +. ./common.sh + +# disable automatic packing for these tests +export GOT_TEST_PACK="" + +test_pack_all_loose_objects() { + local testroot=`test_init pack_all_loose_objects` + + # tags should also be packed + got tag -r $testroot/repo -m 1.0 1.0 >/dev/null + + # no pack files should exist yet + ls $testroot/repo/.git/objects/pack/ > $testroot/stdout + ret="$?" + if [ "$ret" != "0" ]; then + test_done "$testroot" "$ret" + return 1 + fi + echo -n > $testroot/stdout.expected + cmp -s $testroot/stdout.expected $testroot/stdout + ret="$?" + if [ "$ret" != "0" ]; then + diff -u $testroot/stdout.expected $testroot/stdout + test_done "$testroot" "$ret" + return 1 + fi + + gotadmin pack -r $testroot/repo > $testroot/stdout + ret="$?" + if [ "$ret" != "0" ]; then + echo "gotadmin pack failed unexpectedly" >&2 + test_done "$testroot" "$ret" + return 1 + fi + packname=`grep ^Wrote $testroot/stdout | cut -d ' ' -f2` + gotadmin listpack $testroot/repo/.git/objects/pack/pack-$packname \ + > $testroot/stdout + + for d in $testroot/repo/.git/objects/[0-9a-f][0-9a-f]; do + id0=`basename $d` + ret=0 + for e in `ls $d`; do + obj_id=${id0}${e} + if grep -q ^$obj_id $testroot/stdout; then + continue + fi + echo "loose object $obj_id was not packed" >&2 + ret=1 + break + done + if [ "$ret" == "1" ]; then + break + fi + done + + test_done "$testroot" "$ret" +} + +test_pack_exclude() { + local testroot=`test_init pack_exclude` + local commit0=`git_show_head $testroot/repo` + + # no pack files should exist yet + ls $testroot/repo/.git/objects/pack/ > $testroot/stdout + ret="$?" + if [ "$ret" != "0" ]; then + test_done "$testroot" "$ret" + return 1 + fi + echo -n > $testroot/stdout.expected + cmp -s $testroot/stdout.expected $testroot/stdout + ret="$?" + if [ "$ret" != "0" ]; then + diff -u $testroot/stdout.expected $testroot/stdout + test_done "$testroot" "$ret" + return 1 + fi + + got branch -r $testroot/repo mybranch + ret="$?" + if [ "$ret" != "0" ]; then + test_done "$testroot" "$ret" + return 1 + fi + + got checkout -b mybranch $testroot/repo $testroot/wt > /dev/null + ret="$?" + if [ "$ret" != "0" ]; then + test_done "$testroot" "$ret" + return 1 + fi + + echo a new line >> $testroot/wt/alpha + (cd $testroot/wt && got commit -m "edit alpha" >/dev/null) + + gotadmin pack -r $testroot/repo -x master > $testroot/stdout + ret="$?" + if [ "$ret" != "0" ]; then + echo "gotadmin pack failed unexpectedly" >&2 + test_done "$testroot" "$ret" + return 1 + fi + packname=`grep ^Wrote $testroot/stdout | cut -d ' ' -f2` + gotadmin listpack $testroot/repo/.git/objects/pack/pack-$packname \ + > $testroot/stdout + + tree0=`got cat -r $testroot/repo $commit0 | grep ^tree | \ + cut -d ' ' -f2` + excluded_ids=`got tree -r $testroot/repo -c $commit0 -R -i | \ + cut -d ' ' -f 1` + excluded_ids="$excluded_ids $commit0 $tree0" + for id in $excluded_ids; do + ret=0 + if grep -q ^$id $testroot/stdout; then + echo "found excluded object $id in pack file" >&2 + ret=1 + fi + if [ "$ret" == "1" ]; then + break + fi + done + if [ "$ret" == "1" ]; then + test_done "$testroot" "$ret" + return 1 + fi + + for d in $testroot/repo/.git/objects/[0-9a-f][0-9a-f]; do + id0=`basename $d` + ret=0 + for e in `ls $d`; do + obj_id=${id0}${e} + excluded=0 + for id in $excluded_ids; do + if [ "$obj_id" == "$id" ]; then + excluded=1 + break + fi + done + if [ "$excluded" == "1" ]; then + continue + fi + if grep -q ^$obj_id $testroot/stdout; then + continue + fi + echo "loose object $obj_id was not packed" >&2 + ret=1 + break + done + if [ "$ret" == "1" ]; then + break + fi + done + + test_done "$testroot" "$ret" +} + +test_pack_include() { + local testroot=`test_init pack_include` + local commit0=`git_show_head $testroot/repo` + + # no pack files should exist yet + ls $testroot/repo/.git/objects/pack/ > $testroot/stdout + ret="$?" + if [ "$ret" != "0" ]; then + test_done "$testroot" "$ret" + return 1 + fi + echo -n > $testroot/stdout.expected + cmp -s $testroot/stdout.expected $testroot/stdout + ret="$?" + if [ "$ret" != "0" ]; then + diff -u $testroot/stdout.expected $testroot/stdout + test_done "$testroot" "$ret" + return 1 + fi + + got branch -r $testroot/repo mybranch + ret="$?" + if [ "$ret" != "0" ]; then + test_done "$testroot" "$ret" + return 1 + fi + + got checkout -b mybranch $testroot/repo $testroot/wt > /dev/null + ret="$?" + if [ "$ret" != "0" ]; then + test_done "$testroot" "$ret" + return 1 + fi + + echo a new line >> $testroot/wt/alpha + (cd $testroot/wt && got commit -m "edit alpha" >/dev/null) + local commit1=`git_show_branch_head $testroot/repo mybranch` + + gotadmin pack -r $testroot/repo master > $testroot/stdout + ret="$?" + if [ "$ret" != "0" ]; then + echo "gotadmin pack failed unexpectedly" >&2 + test_done "$testroot" "$ret" + return 1 + fi + packname=`grep ^Wrote $testroot/stdout | cut -d ' ' -f2` + gotadmin listpack $testroot/repo/.git/objects/pack/pack-$packname \ + > $testroot/stdout + + tree1=`got cat -r $testroot/repo $commit1 | grep ^tree | \ + cut -d ' ' -f2` + alpha1=`got tree -r $testroot/repo -i -c $commit1 | \ + grep "[0-9a-f] alpha$" | cut -d' ' -f 1` + excluded_ids="$alpha1 $commit1 $tree1" + for id in $excluded_ids; do + ret=0 + if grep -q ^$id $testroot/stdout; then + echo "found excluded object $id in pack file" >&2 + ret=1 + fi + if [ "$ret" == "1" ]; then + break + fi + done + if [ "$ret" == "1" ]; then + test_done "$testroot" "$ret" + return 1 + fi + + tree0=`got cat -r $testroot/repo $commit0 | grep ^tree | \ + cut -d ' ' -f2` + included_ids=`got tree -r $testroot/repo -c $commit0 -R -i | \ + cut -d ' ' -f 1` + included_ids="$included_ids $commit0 $tree0" + for obj_id in $included_ids; do + for id in $excluded_ids; do + if [ "$obj_id" == "$id" ]; then + excluded=1 + break + fi + done + if [ "$excluded" == "1" ]; then + continue + fi + if grep -q ^$obj_id $testroot/stdout; then + continue + fi + echo "included object $obj_id was not packed" >&2 + ret=1 + break + done + + test_done "$testroot" "$ret" +} + +test_pack_ambiguous_arg() { + local testroot=`test_init pack_ambiguous_arg` + local commit0=`git_show_head $testroot/repo` + + # no pack files should exist yet + ls $testroot/repo/.git/objects/pack/ > $testroot/stdout + ret="$?" + if [ "$ret" != "0" ]; then + test_done "$testroot" "$ret" + return 1 + fi + echo -n > $testroot/stdout.expected + cmp -s $testroot/stdout.expected $testroot/stdout + ret="$?" + if [ "$ret" != "0" ]; then + diff -u $testroot/stdout.expected $testroot/stdout + test_done "$testroot" "$ret" + return 1 + fi + + got branch -r $testroot/repo mybranch + ret="$?" + if [ "$ret" != "0" ]; then + test_done "$testroot" "$ret" + return 1 + fi + + got checkout -b mybranch $testroot/repo $testroot/wt > /dev/null + ret="$?" + if [ "$ret" != "0" ]; then + test_done "$testroot" "$ret" + return 1 + fi + + echo a new line >> $testroot/wt/alpha + (cd $testroot/wt && got commit -m "edit alpha" >/dev/null) + local commit1=`git_show_branch_head $testroot/repo mybranch` + + gotadmin pack -r $testroot/repo -x master master \ + > $testroot/stdout 2> $testroot/stderr + ret="$?" + if [ "$ret" == "0" ]; then + echo "gotadmin pack succeeded unexpectedly" >&2 + test_done "$testroot" "1" + return 1 + fi + + printf "\rpacking 1 reference\n" > $testroot/stdout.expected + cmp -s $testroot/stdout.expected $testroot/stdout + ret="$?" + if [ "$ret" != "0" ]; then + diff -u $testroot/stdout.expected $testroot/stdout + test_done "$testroot" "$ret" + return 1 + fi + + echo "gotadmin: not enough objects to pack" > $testroot/stderr.expected + cmp -s $testroot/stderr.expected $testroot/stderr + ret="$?" + if [ "$ret" != "0" ]; then + diff -u $testroot/stderr.expected $testroot/stderr + fi + test_done "$testroot" "$ret" +} + +test_pack_loose_only() { + local testroot=`test_init pack_loose_only` + local commit0=`git_show_head $testroot/repo` + + # no pack files should exist yet + ls $testroot/repo/.git/objects/pack/ > $testroot/stdout + ret="$?" + if [ "$ret" != "0" ]; then + test_done "$testroot" "$ret" + return 1 + fi + echo -n > $testroot/stdout.expected + cmp -s $testroot/stdout.expected $testroot/stdout + ret="$?" + if [ "$ret" != "0" ]; then + diff -u $testroot/stdout.expected $testroot/stdout + test_done "$testroot" "$ret" + return 1 + fi + + got branch -r $testroot/repo mybranch + ret="$?" + if [ "$ret" != "0" ]; then + test_done "$testroot" "$ret" + return 1 + fi + + got checkout -b mybranch $testroot/repo $testroot/wt > /dev/null + ret="$?" + if [ "$ret" != "0" ]; then + test_done "$testroot" "$ret" + return 1 + fi + + echo a new line >> $testroot/wt/alpha + (cd $testroot/wt && got commit -m "edit alpha" >/dev/null) + + # pack objects belonging to the 'master' branch; its objects + # should then be excluded while packing 'mybranch' since they + # are already packed + gotadmin pack -r $testroot/repo master > /dev/null + ret="$?" + if [ "$ret" != "0" ]; then + echo "gotadmin pack failed unexpectedly" >&2 + test_done "$testroot" "$ret" + return 1 + fi + + gotadmin pack -r $testroot/repo mybranch > $testroot/stdout + ret="$?" + if [ "$ret" != "0" ]; then + echo "gotadmin pack failed unexpectedly" >&2 + test_done "$testroot" "$ret" + return 1 + fi + packname=`grep ^Wrote $testroot/stdout | cut -d ' ' -f2` + gotadmin listpack $testroot/repo/.git/objects/pack/pack-$packname \ + > $testroot/stdout + + tree0=`got cat -r $testroot/repo $commit0 | grep ^tree | \ + cut -d ' ' -f2` + excluded_ids=`got tree -r $testroot/repo -c $commit0 -R -i | \ + cut -d ' ' -f 1` + excluded_ids="$excluded_ids $commit0 $tree0" + for id in $excluded_ids; do + ret=0 + if grep -q ^$id $testroot/stdout; then + echo "found excluded object $id in pack file" >&2 + ret=1 + fi + if [ "$ret" == "1" ]; then + break + fi + done + if [ "$ret" == "1" ]; then + test_done "$testroot" "$ret" + return 1 + fi + + for d in $testroot/repo/.git/objects/[0-9a-f][0-9a-f]; do + id0=`basename $d` + ret=0 + for e in `ls $d`; do + obj_id=${id0}${e} + excluded=0 + for id in $excluded_ids; do + if [ "$obj_id" == "$id" ]; then + excluded=1 + break + fi + done + if [ "$excluded" == "1" ]; then + continue + fi + if grep -q ^$obj_id $testroot/stdout; then + continue + fi + echo "loose object $obj_id was not packed" >&2 + ret=1 + break + done + if [ "$ret" == "1" ]; then + break + fi + done + + test_done "$testroot" "$ret" +} + +test_pack_all_objects() { + local testroot=`test_init pack_all_objects` + local commit0=`git_show_head $testroot/repo` + + # no pack files should exist yet + ls $testroot/repo/.git/objects/pack/ > $testroot/stdout + ret="$?" + if [ "$ret" != "0" ]; then + test_done "$testroot" "$ret" + return 1 + fi + echo -n > $testroot/stdout.expected + cmp -s $testroot/stdout.expected $testroot/stdout + ret="$?" + if [ "$ret" != "0" ]; then + diff -u $testroot/stdout.expected $testroot/stdout + test_done "$testroot" "$ret" + return 1 + fi + + got branch -r $testroot/repo mybranch + ret="$?" + if [ "$ret" != "0" ]; then + test_done "$testroot" "$ret" + return 1 + fi + + got checkout -b mybranch $testroot/repo $testroot/wt > /dev/null + ret="$?" + if [ "$ret" != "0" ]; then + test_done "$testroot" "$ret" + return 1 + fi + + echo a new line >> $testroot/wt/alpha + (cd $testroot/wt && got commit -m "edit alpha" >/dev/null) + + # pack objects belonging to the 'master' branch + gotadmin pack -r $testroot/repo master > /dev/null + ret="$?" + if [ "$ret" != "0" ]; then + echo "gotadmin pack failed unexpectedly" >&2 + test_done "$testroot" "$ret" + return 1 + fi + + # pack mybranch, including already packed objects on the + # 'master' branch which are reachable from mybranch + gotadmin pack -r $testroot/repo -a mybranch > $testroot/stdout + ret="$?" + if [ "$ret" != "0" ]; then + echo "gotadmin pack failed unexpectedly" >&2 + test_done "$testroot" "$ret" + return 1 + fi + packname=`grep ^Wrote $testroot/stdout | cut -d ' ' -f2` + gotadmin listpack $testroot/repo/.git/objects/pack/pack-$packname \ + > $testroot/stdout + + for d in $testroot/repo/.git/objects/[0-9a-f][0-9a-f]; do + id0=`basename $d` + ret=0 + for e in `ls $d`; do + obj_id=${id0}${e} + if grep -q ^$obj_id $testroot/stdout; then + continue + fi + echo "loose object $obj_id was not packed" >&2 + ret=1 + break + done + if [ "$ret" == "1" ]; then + break + fi + done + + test_done "$testroot" "$ret" +} + +test_pack_bad_ref() { + local testroot=`test_init pack_bad_ref` + local commit0=`git_show_head $testroot/repo` + + # no pack files should exist yet + ls $testroot/repo/.git/objects/pack/ > $testroot/stdout + ret="$?" + if [ "$ret" != "0" ]; then + test_done "$testroot" "$ret" + return 1 + fi + echo -n > $testroot/stdout.expected + cmp -s $testroot/stdout.expected $testroot/stdout + ret="$?" + if [ "$ret" != "0" ]; then + diff -u $testroot/stdout.expected $testroot/stdout + test_done "$testroot" "$ret" + return 1 + fi + + got branch -r $testroot/repo mybranch + ret="$?" + if [ "$ret" != "0" ]; then + test_done "$testroot" "$ret" + return 1 + fi + + got checkout -b mybranch $testroot/repo $testroot/wt > /dev/null + ret="$?" + if [ "$ret" != "0" ]; then + test_done "$testroot" "$ret" + return 1 + fi + + gotadmin pack -r $testroot/repo refs/got/worktree/ \ + > $testroot/stdout 2> $testroot/stderr + ret="$?" + if [ "$ret" == "0" ]; then + echo "gotadmin pack succeeded unexpectedly" >&2 + test_done "$testroot" "1" + return 1 + fi + + echo -n > $testroot/stdout.expected + cmp -s $testroot/stdout.expected $testroot/stdout + ret="$?" + if [ "$ret" != "0" ]; then + diff -u $testroot/stdout.expected $testroot/stdout + test_done "$testroot" "$ret" + return 1 + fi + + echo "gotadmin: not enough objects to pack" > $testroot/stderr.expected + cmp -s $testroot/stderr.expected $testroot/stderr + ret="$?" + if [ "$ret" != "0" ]; then + diff -u $testroot/stderr.expected $testroot/stderr + fi + test_done "$testroot" "$ret" +} + +test_parseargs "$@" +run_test test_pack_all_loose_objects +run_test test_pack_exclude +run_test test_pack_include +run_test test_pack_ambiguous_arg +run_test test_pack_loose_only +run_test test_pack_all_objects +run_test test_pack_bad_ref