commit 659e7fbd6952b401014702a1e181642a9bba17df from: Stefan Sperling date: Fri Mar 20 15:55:08 2020 UTC fetch only HEAD branch and tags by default; add -a option to fetch everything commit - 09f630849f5a9cc540b1289cdd7a034b171fa7d4 commit + 659e7fbd6952b401014702a1e181642a9bba17df blob - 2d0fb7825a027ea6c5a10e684045f5035dd50ecc blob + 2b7971b0b0d9defaddcc26bc2ff5f92b69db18f7 --- got/got.1 +++ got/got.1 @@ -136,7 +136,7 @@ follows the globbing rules documented in .It Cm im Short alias for .Cm import . -.It Cm clone Oo Fl q Oc Oo Fl v Oc Ar repository-URL Op Ar target-directory +.It Cm clone Oo Fl a Oc Oo Fl q Oc Oo Fl v Oc Ar repository-URL Op Ar target-directory Clone a Git repository at the specified .Ar repository-URL into the specified @@ -191,6 +191,10 @@ The options for .Cm got clone are as follows: .Bl -tag -width Ds +.It Fl a +Fetch all branches from the remote repository. +If this option is not specified, a branch resolved via the repository's HEAD +reference will be fetched. .It Fl m Create the cloned repository as a mirror of the original repository. This is useful if the cloned repository will not be used to store @@ -236,7 +240,7 @@ The maximum is 3. .It Cm cl Short alias for .Cm clone . -.It Cm fetch Oo Fl r Ar repository-path Oc Oo Fl q Oc Oo Fl v Oc Op Ar remote-repository-name +.It Cm fetch Oo Fl a Oc Oo Fl r Ar repository-path Oc Oo Fl q Oc Oo Fl v Oc Op Ar remote-repository-name Fetch new changes from a remote repository. If no .Ar remote-repository-name @@ -275,6 +279,10 @@ The options for .Cm got fetch are as follows: .Bl -tag -width Ds +.It Fl a +Fetch all branches from the remote repository. +If this option is not specified, a branch resolved via the repository's HEAD +reference will be fetched. .It Fl r Ar repository-path Use the repository at the specified path. If not specified, assume the repository is located at or above the current blob - 74366142a8b460dd0ad22df86bdd774bea7281a0 blob + 35b58f047d1d5baf09f6eccf06313afe8392cf1d --- got/got.c +++ got/got.c @@ -811,7 +811,7 @@ done: __dead static void usage_clone(void) { - fprintf(stderr, "usage: %s clone [-m] [-q] [-v] repository-url " + fprintf(stderr, "usage: %s clone [-a] [-m] [-q] [-v] repository-url " "[target-directory]\n", getprogname()); exit(1); } @@ -903,13 +903,17 @@ cmd_clone(int argc, char *argv[]) char *gitconfig = NULL; FILE *gitconfig_file = NULL; ssize_t n; - int verbosity = 0, mirror_references = 0; + int verbosity = 0, fetch_all_branches = 0, mirror_references = 0; + struct got_reference *head_symref = NULL; TAILQ_INIT(&refs); TAILQ_INIT(&symrefs); - while ((ch = getopt(argc, argv, "mvq")) != -1) { + while ((ch = getopt(argc, argv, "amvq")) != -1) { switch (ch) { + case 'a': + fetch_all_branches = 1; + break; case 'm': mirror_references = 1; break; @@ -1012,43 +1016,6 @@ cmd_clone(int argc, char *argv[]) if (verbosity >= 0) printf("Connected to %s%s%s\n", host, port ? ":" : "", port ? port : ""); - - /* Create a config file git-fetch(1) can understand. */ - gitconfig_path = got_repo_get_path_gitconfig(repo); - if (gitconfig_path == NULL) { - error = got_error_from_errno("got_repo_get_path_gitconfig"); - goto done; - } - gitconfig_file = fopen(gitconfig_path, "a"); - if (gitconfig_file == NULL) { - error = got_error_from_errno2("fopen", gitconfig_path); - goto done; - } - if (mirror_references) { - if (asprintf(&gitconfig, - "[remote \"%s\"]\n" - "\turl = %s\n" - "\tmirror = true\n", - GOT_FETCH_DEFAULT_REMOTE_NAME, git_url) == -1) { - error = got_error_from_errno("asprintf"); - goto done; - } - } else { - if (asprintf(&gitconfig, - "[remote \"%s\"]\n" - "\turl = %s\n" - "\tfetch = +refs/heads/*:refs/remotes/%s/*\n", - GOT_FETCH_DEFAULT_REMOTE_NAME, git_url, - GOT_FETCH_DEFAULT_REMOTE_NAME) == -1) { - error = got_error_from_errno("asprintf"); - goto done; - } - } - n = fwrite(gitconfig, 1, strlen(gitconfig), gitconfig_file); - if (n != strlen(gitconfig)) { - error = got_ferror(gitconfig_file, GOT_ERR_IO); - goto done; - } fpa.last_scaled_size[0] = '\0'; fpa.last_p_indexed = -1; @@ -1056,7 +1023,7 @@ cmd_clone(int argc, char *argv[]) fpa.verbosity = verbosity; error = got_fetch_pack(&pack_hash, &refs, &symrefs, GOT_FETCH_DEFAULT_REMOTE_NAME, mirror_references, - fetchfd, repo, fetch_progress, &fpa); + fetch_all_branches, fetchfd, repo, fetch_progress, &fpa); if (error) goto done; @@ -1105,7 +1072,7 @@ cmd_clone(int argc, char *argv[]) /* Set the HEAD reference if the server provided one. */ TAILQ_FOREACH(pe, &symrefs, entry) { - struct got_reference *symref, *target_ref; + struct got_reference *target_ref; const char *refname = pe->path; const char *target = pe->data; @@ -1119,20 +1086,81 @@ cmd_clone(int argc, char *argv[]) goto done; } - error = got_ref_alloc_symref(&symref, GOT_REF_HEAD, target_ref); + error = got_ref_alloc_symref(&head_symref, + GOT_REF_HEAD, target_ref); got_ref_close(target_ref); if (error) goto done; if (verbosity >= 0) printf("Setting %s to %s\n", GOT_REF_HEAD, - got_ref_get_symref_target(symref)); + got_ref_get_symref_target(head_symref)); - error = got_ref_write(symref, repo); - got_ref_close(symref); + error = got_ref_write(head_symref, repo); break; + } + + /* Create a config file git-fetch(1) can understand. */ + gitconfig_path = got_repo_get_path_gitconfig(repo); + if (gitconfig_path == NULL) { + error = got_error_from_errno("got_repo_get_path_gitconfig"); + goto done; + } + gitconfig_file = fopen(gitconfig_path, "a"); + if (gitconfig_file == NULL) { + error = got_error_from_errno2("fopen", gitconfig_path); + goto done; } + if (mirror_references) { + if (asprintf(&gitconfig, + "[remote \"%s\"]\n" + "\turl = %s\n" + "\tmirror = true\n", + GOT_FETCH_DEFAULT_REMOTE_NAME, git_url) == -1) { + error = got_error_from_errno("asprintf"); + goto done; + } + } else if (fetch_all_branches) { + if (asprintf(&gitconfig, + "[remote \"%s\"]\n" + "\turl = %s\n" + "\tfetch = +refs/heads/*:refs/remotes/%s/*\n", + GOT_FETCH_DEFAULT_REMOTE_NAME, git_url, + GOT_FETCH_DEFAULT_REMOTE_NAME) == -1) { + error = got_error_from_errno("asprintf"); + goto done; + } + } else { + const char *branchname; + /* + * If the server specified a default branch, use just that one. + * Otherwise fall back to fetching all branches on next fetch. + */ + if (head_symref) { + branchname = got_ref_get_symref_target(head_symref); + if (strncmp(branchname, "refs/heads/", 11) == 0) + branchname += 11; + } else + branchname = "*"; /* fall back to all branches */ + if (asprintf(&gitconfig, + "[remote \"%s\"]\n" + "\turl = %s\n" + "\tfetch = +refs/heads/%s:refs/remotes/%s/%s\n", + GOT_FETCH_DEFAULT_REMOTE_NAME, git_url, + branchname, GOT_FETCH_DEFAULT_REMOTE_NAME, + branchname) == -1) { + error = got_error_from_errno("asprintf"); + goto done; + } + } + n = fwrite(gitconfig, 1, strlen(gitconfig), gitconfig_file); + if (n != strlen(gitconfig)) { + error = got_ferror(gitconfig_file, GOT_ERR_IO); + goto done; + } + + if (verbosity >= 0) printf("Created %s repository '%s'\n", mirror_references ? "mirrored" : "cloned", repo_path); @@ -1143,6 +1171,8 @@ done: error = got_error_from_errno("fclose"); if (repo) got_repo_close(repo); + if (head_symref) + got_ref_close(head_symref); TAILQ_FOREACH(pe, &refs, entry) { free((void *)pe->path); free(pe->data); @@ -1234,7 +1264,7 @@ done: __dead static void usage_fetch(void) { - fprintf(stderr, "usage: %s fetch [-r repository-path] [-q] [-v] " + fprintf(stderr, "usage: %s fetch [-a] [-r repository-path] [-q] [-v] " "[remote-repository-name]\n", getprogname()); exit(1); } @@ -1257,13 +1287,16 @@ cmd_fetch(int argc, char *argv[]) struct got_object_id *pack_hash = NULL; int i, ch, fetchfd = -1; struct got_fetch_progress_arg fpa; - int verbosity = 0; + int verbosity = 0, fetch_all_branches = 0; TAILQ_INIT(&refs); TAILQ_INIT(&symrefs); - while ((ch = getopt(argc, argv, "r:vq")) != -1) { + while ((ch = getopt(argc, argv, "ar:vq")) != -1) { switch (ch) { + case 'a': + fetch_all_branches = 1; + break; case 'r': repo_path = realpath(optarg, NULL); if (repo_path == NULL) @@ -1390,7 +1423,8 @@ cmd_fetch(int argc, char *argv[]) fpa.last_p_resolved = -1; fpa.verbosity = verbosity; error = got_fetch_pack(&pack_hash, &refs, &symrefs, remote->name, - remote->mirror_references, fetchfd, repo, fetch_progress, &fpa); + remote->mirror_references, fetch_all_branches, fetchfd, repo, + fetch_progress, &fpa); if (error) goto done; blob - d29011ab619ea8abc514f16e7e1fa5a63b015a55 blob + 7d8cd87f9386f24c86d96e95cac445f281b1b452 --- include/got_error.h +++ include/got_error.h @@ -138,6 +138,7 @@ #define GOT_ERR_ADDRINFO 121 #define GOT_ERR_BAD_PACKET 122 #define GOT_ERR_NO_REMOTE 123 +#define GOT_ERR_FETCH_NO_BRANCH 124 static const struct got_error { int code; @@ -282,6 +283,7 @@ static const struct got_error { { GOT_ERR_ADDRINFO, "getaddrinfo failed" }, { GOT_ERR_BAD_PACKET, "bad packet received" }, { GOT_ERR_NO_REMOTE, "remote repository not found" }, + { GOT_ERR_FETCH_NO_BRANCH, "could not find any branches to fetch" }, }; /* blob - 9df27fa28b21dbc3fd4faab5673e0efdd0eaa24c blob + 14f29c4e58a7a424ab01ef02f0e6205fbdbf016a --- include/got_fetch.h +++ include/got_fetch.h @@ -61,4 +61,4 @@ typedef const struct got_error *(*got_fetch_progress_c */ const struct got_error *got_fetch_pack(struct got_object_id **, struct got_pathlist_head *, struct got_pathlist_head *, const char *, - int, int, struct got_repository *, got_fetch_progress_cb, void *); + int, int, int, struct got_repository *, got_fetch_progress_cb, void *); blob - b5b03f3f8b3c2b8923bf764d7b8d82f18181c5eb blob + e36d5b846fbbe85dbec5aaf078f073bde3b2ef11 --- lib/fetch.c +++ lib/fetch.c @@ -387,7 +387,8 @@ check_pack_hash(int fd, size_t sz, uint8_t *hcomp) const struct got_error* got_fetch_pack(struct got_object_id **pack_hash, struct got_pathlist_head *refs, struct got_pathlist_head *symrefs, const char *remote_name, - int mirror_references, int fetchfd, struct got_repository *repo, + int mirror_references, int fetch_all_branches, int fetchfd, + struct got_repository *repo, got_fetch_progress_cb progress_cb, void *progress_arg) { int imsg_fetchfds[2], imsg_idxfds[2]; @@ -552,7 +553,8 @@ got_fetch_pack(struct got_object_id **pack_hash, struc err = got_error_from_errno("dup"); goto done; } - err = got_privsep_send_fetch_req(&fetchibuf, nfetchfd, &have_refs); + err = got_privsep_send_fetch_req(&fetchibuf, nfetchfd, &have_refs, + fetch_all_branches); if (err != NULL) goto done; nfetchfd = -1; blob - 5d6fc07ab4644eda2d4e44d188b854367410fd15 blob + 3df98dcefa961785e3259c8141d5647f3648ccee --- lib/got_lib_privsep.h +++ lib/got_lib_privsep.h @@ -246,10 +246,11 @@ struct got_imsg_fetch_have_ref { /* Followed by name_len data bytes. */ } __attribute__((__packed__)); -struct got_imsg_fetch_have_refs { +struct got_imsg_fetch_request { + int fetch_all_branches; size_t n_have_refs; /* Followed by n_have_refs times of got_imsg_fetch_have_ref data. */ -}; +} __attribute__((__packed__)); /* Structures for GOT_IMSG_FETCH_SYMREFS data. */ struct got_imsg_fetch_symref { @@ -389,7 +390,7 @@ const struct got_error *got_privsep_send_index_pack_do const struct got_error *got_privsep_recv_index_progress(int *, int *, int *, int *, int *, struct imsgbuf *ibuf); const struct got_error *got_privsep_send_fetch_req(struct imsgbuf *, int, - struct got_pathlist_head *); + struct got_pathlist_head *, int); const struct got_error *got_privsep_send_fetch_outfd(struct imsgbuf *, int); const struct got_error *got_privsep_send_fetch_symrefs(struct imsgbuf *, struct got_pathlist_head *); blob - b1b5821766da5649ffd832d79e95bde75f3bd7de blob + e658bf0b714ea249a3acb3a4b4e9db34503a26d6 --- lib/privsep.c +++ lib/privsep.c @@ -412,14 +412,14 @@ got_privsep_send_obj(struct imsgbuf *ibuf, struct got_ const struct got_error * got_privsep_send_fetch_req(struct imsgbuf *ibuf, int fd, - struct got_pathlist_head *have_refs) + struct got_pathlist_head *have_refs, int fetch_all_branches) { const struct got_error *err = NULL; struct ibuf *wbuf; size_t len, n_have_refs = 0; struct got_pathlist_entry *pe; - len = sizeof(struct got_imsg_fetch_have_refs); + len = sizeof(struct got_imsg_fetch_request); TAILQ_FOREACH(pe, have_refs, entry) { len += sizeof(struct got_imsg_fetch_have_ref) + pe->path_len; n_have_refs++; @@ -435,7 +435,14 @@ got_privsep_send_fetch_req(struct imsgbuf *ibuf, int f return got_error_from_errno("imsg_create FETCH_REQUEST"); } - /* Keep in sync with struct got_imsg_fetch_have_refs definition! */ + /* Keep in sync with struct got_imsg_fetch_request definition! */ + if (imsg_add(wbuf, &fetch_all_branches, sizeof(fetch_all_branches)) + == -1) { + err = got_error_from_errno("imsg_add FETCH_REQUEST"); + ibuf_free(wbuf); + close(fd); + return err; + } if (imsg_add(wbuf, &n_have_refs, sizeof(n_have_refs)) == -1) { err = got_error_from_errno("imsg_add FETCH_REQUEST"); ibuf_free(wbuf); blob - 751733ae970b5645970f9846119c7e60367bed65 blob + 35e6b7739ac8c1477f7ec61d9e2524ce6c3f7f7b --- libexec/got-fetch-pack/got-fetch-pack.c +++ libexec/got-fetch-pack/got-fetch-pack.c @@ -40,6 +40,7 @@ #include "got_path.h" #include "got_version.h" #include "got_fetch.h" +#include "got_reference.h" #include "got_lib_sha1.h" #include "got_lib_delta.h" @@ -54,7 +55,6 @@ struct got_object *indexed; static int chattygot; -static char *fetchbranch; static struct got_object_id zhash = {.sha1={0}}; static const struct got_error * @@ -240,23 +240,15 @@ match_remote_ref(struct got_pathlist_head *have_refs, } static int -match_branch(char *br, char *pat) +match_branch(const char *branch, const char *wanted_branch) { - char name[128]; + if (strncmp(branch, "refs/heads/", 11) != 0) + return 0; - if (strstr(pat, "refs/heads") == pat) { - if (snprintf(name, sizeof(name), "%s", pat) >= sizeof(name)) - return -1; - } else if (strstr(pat, "heads")) { - if (snprintf(name, sizeof(name), "refs/%s", pat) - >= sizeof(name)) - return -1; - } else { - if (snprintf(name, sizeof(name), "refs/heads/%s", pat) - >= sizeof(name)) - return -1; - } - return strcmp(br, name) == 0; + if (strncmp(wanted_branch, "refs/heads/", 11) == 0) + wanted_branch += 11; + + return (strcmp(branch + 11, wanted_branch) == 0); } static const struct got_error * @@ -490,7 +482,8 @@ fetch_error(const char *buf, size_t len) static const struct got_error * fetch_pack(int fd, int packfd, struct got_object_id *packid, - struct got_pathlist_head *have_refs, struct imsgbuf *ibuf) + struct got_pathlist_head *have_refs, int fetch_all_branches, + struct imsgbuf *ibuf) { const struct got_error *err = NULL; char buf[GOT_FETCH_PKTMAX]; @@ -501,9 +494,11 @@ fetch_pack(int fd, int packfd, struct got_object_id *p off_t packsz = 0, last_reported_packsz = 0; char *id_str = NULL, *refname = NULL; char *server_capabilities = NULL, *my_capabilities = NULL; + const char *default_branch = NULL; struct got_pathlist_head symrefs; struct got_pathlist_entry *pe; int sent_my_capabilites = 0, have_sidebands = 0; + int found_branch = 0; TAILQ_INIT(&symrefs); @@ -546,12 +541,39 @@ fetch_pack(int fd, int packfd, struct got_object_id *p if (err) goto done; is_firstpkt = 0; + if (!fetch_all_branches) { + TAILQ_FOREACH(pe, &symrefs, entry) { + const char *name = pe->path; + const char *symref_target = pe->data; + if (strcmp(name, GOT_REF_HEAD) != 0) + continue; + default_branch = symref_target; + break; + } + } continue; } if (strstr(refname, "^{}")) continue; - if (fetchbranch && !match_branch(refname, fetchbranch)) + + if (chattygot) + fprintf(stderr, "%s: discovered remote ref %s\n", + getprogname(), refname); + + if (strncmp(refname, "refs/heads/", 11) == 0) { + if (default_branch != NULL && + !match_branch(refname, default_branch)) + continue; + found_branch = 1; + } else if (strncmp(refname, "refs/tags/", 10) != 0) { + if (chattygot) { + fprintf(stderr, "%s: ignoring '%s' which is " + "neither a branch nor a tag\n", + getprogname(), refname); + } continue; + } + if (refsz == nref + 1) { refsz *= 2; have = reallocarray(have, refsz, sizeof(have[0])); @@ -585,16 +607,22 @@ fetch_pack(int fd, int packfd, struct got_object_id *p free(theirs); goto done; } - fprintf(stderr, "%s: discovered remote ref %s\n", + fprintf(stderr, "%s: %s will be fetched\n", getprogname(), refname); fprintf(stderr, "%s: theirs=%s\n%s: mine=%s\n", - getprogname(), theirs, getprogname(), mine); + getprogname(), theirs, getprogname(), mine); free(theirs); free(mine); } nref++; } + /* Abort if we haven't found any branch to fetch. */ + if (!found_branch) { + err = got_error(GOT_ERR_FETCH_NO_BRANCH); + goto done; + } + for (i = 0; i < nref; i++) { if (got_object_id_cmp(&have[i], &want[i]) == 0) continue; @@ -810,9 +838,10 @@ main(int argc, char **argv) struct imsg imsg; struct got_pathlist_head have_refs; struct got_pathlist_entry *pe; - struct got_imsg_fetch_have_refs *fetch_have_refs = NULL; - struct got_imsg_fetch_have_ref *href = NULL; - size_t datalen; + struct got_imsg_fetch_request *fetch_req = NULL; + struct got_imsg_fetch_have_ref href; + size_t datalen, remain; + size_t offset; #if 0 static int attached; while (!attached) @@ -847,50 +876,58 @@ main(int argc, char **argv) goto done; } datalen = imsg.hdr.len - IMSG_HEADER_SIZE; - if (datalen < sizeof(struct got_imsg_fetch_have_refs)) { + if (datalen < sizeof(struct got_imsg_fetch_request)) { err = got_error(GOT_ERR_PRIVSEP_LEN); goto done; } - fetch_have_refs = (struct got_imsg_fetch_have_refs *)imsg.data; - if (datalen < sizeof(struct got_imsg_fetch_have_refs) + + fetch_req = (struct got_imsg_fetch_request *)imsg.data; + if (datalen < sizeof(*fetch_req) + sizeof(struct got_imsg_fetch_have_ref) * - fetch_have_refs->n_have_refs) { + fetch_req->n_have_refs) { err = got_error(GOT_ERR_PRIVSEP_LEN); goto done; } - href = (struct got_imsg_fetch_have_ref *)( - (uint8_t *)fetch_have_refs + sizeof(fetch_have_refs->n_have_refs)); - for (i = 0; i < fetch_have_refs->n_have_refs; i++) { + offset = sizeof(*fetch_req); + remain = datalen; + for (i = 0; i < fetch_req->n_have_refs; i++) { struct got_object_id *id; char *refname; - if (datalen < sizeof(*href) + href->name_len) { + if (remain < sizeof(href) || offset > datalen) { + err = got_error(GOT_ERR_PRIVSEP_LEN); + goto done; + } + memcpy(&href, imsg.data + offset, sizeof(href)); + remain -= sizeof(href); + if (remain < href.name_len) { err = got_error(GOT_ERR_PRIVSEP_LEN); goto done; } - datalen -= sizeof(*href) + href->name_len; - refname = strndup((uint8_t *)href + sizeof(href->id) + - sizeof(href->name_len), href->name_len); + remain -= href.name_len; + refname = malloc(href.name_len + 1); if (refname == NULL) { - err = got_error_from_errno("strndump"); + err = got_error_from_errno("malloc"); goto done; } + offset += sizeof(href); + memcpy(refname, imsg.data + offset, href.name_len); + refname[href.name_len] = '\0'; + offset += href.name_len; + id = malloc(sizeof(*id)); if (id == NULL) { free(refname); err = got_error_from_errno("malloc"); goto done; } - memcpy(id->sha1, href->id, SHA1_DIGEST_LENGTH); + memcpy(id->sha1, href.id, SHA1_DIGEST_LENGTH); err = got_pathlist_append(&have_refs, refname, id); if (err) { free(refname); free(id); goto done; } - href = (struct got_imsg_fetch_have_ref *)( - (uint8_t *)href + sizeof(*href) + href->name_len); - } + } fetchfd = imsg.fd; if ((err = got_privsep_recv_imsg(&imsg, &ibuf, 0)) != 0) { @@ -910,7 +947,8 @@ main(int argc, char **argv) } packfd = imsg.fd; - err = fetch_pack(fetchfd, packfd, &packid, &have_refs, &ibuf); + err = fetch_pack(fetchfd, packfd, &packid, &have_refs, + fetch_req->fetch_all_branches, &ibuf); done: TAILQ_FOREACH(pe, &have_refs, entry) { free((char *)pe->path);