commit 531c39852ff6a7454ce0e618bacb7b7e20f93523 from: Stefan Sperling date: Wed Mar 18 16:11:32 2020 UTC add support for git protocol sidebands and display server progress commit - 52915720b2da188a660677add4b597550d443e77 commit + 531c39852ff6a7454ce0e618bacb7b7e20f93523 blob - 741cccfe85b2fe967ee397f3df73d260df9a4c47 blob + a3cf2526d4ee1971e59ae8f61e78f60326046405 --- got/got.c +++ got/got.c @@ -969,6 +969,15 @@ done: } static const struct got_error * +fetch_progress(void *arg, const char *message) +{ + char *servername = arg; + printf("\rserver %s: %s", servername, message); + fflush(stdout); + return NULL; +} + +static const struct got_error * cmd_clone(int argc, char *argv[]) { const struct got_error *err = NULL; @@ -1036,7 +1045,7 @@ cmd_clone(int argc, char *argv[]) goto done; err = got_fetch_pack(&pack_hash, &refs, &symrefs, fetchfd, - repo); + repo, fetch_progress, host); if (err) goto done; blob - 36345e86940fffeffb539c6aa44c66858f3a4bd9 blob + e3e0144058ef5d5342ddef6dc9a0edd935474201 --- include/got_fetch.h +++ include/got_fetch.h @@ -38,6 +38,10 @@ const struct got_error *got_fetch_parse_uri(char **, c const struct got_error *got_fetch_connect(int *, const char *, const char *, const char *, const char *); +/* A callback function which gets invoked with progress information to print. */ +typedef const struct got_error *(*got_fetch_progress_cb)(void *, + const char *); + /* * Attempt to fetch a packfile from a server. This pack file will contain * objects which that are not yet contained in the provided repository. @@ -46,4 +50,4 @@ const struct got_error *got_fetch_connect(int *, const */ const struct got_error *got_fetch_pack(struct got_object_id **, struct got_pathlist_head *, struct got_pathlist_head *, int, - struct got_repository *); + struct got_repository *, got_fetch_progress_cb, void *); blob - 4016f649d26812603b825e73b9cf0839b6ca3d8c blob + 15ecb2be5c719d2c618a30756def530640a9dad7 --- lib/fetch.c +++ lib/fetch.c @@ -299,7 +299,8 @@ done: const struct got_error* got_fetch_pack(struct got_object_id **pack_hash, struct got_pathlist_head *refs, - struct got_pathlist_head *symrefs, int fetchfd, struct got_repository *repo) + struct got_pathlist_head *symrefs, int fetchfd, struct got_repository *repo, + got_fetch_progress_cb progress_cb, void *progress_arg) { int imsg_fetchfds[2], imsg_idxfds[2]; int packfd = -1, npackfd = -1, idxfd = -1, nidxfd = -1, nfetchfd = -1; @@ -387,9 +388,10 @@ got_fetch_pack(struct got_object_id **pack_hash, struc while (!done) { struct got_object_id *id = NULL; char *refname = NULL; + char *server_progress = NULL; err = got_privsep_recv_fetch_progress(&done, - &id, &refname, symrefs, &ibuf); + &id, &refname, symrefs, &server_progress, &ibuf); if (err != NULL) goto done; if (done) @@ -398,14 +400,25 @@ got_fetch_pack(struct got_object_id **pack_hash, struc err = got_pathlist_append(refs, refname, id); if (err) goto done; + } else if (server_progress) { + char *s, *s0 = server_progress; + while ((s = strsep(&s0, "\r")) != NULL) { + if (*s == '\0') + continue; + err = progress_cb(progress_arg, s); + if (err) + break; + } + free(server_progress); + if (err) + goto done; } - /* TODO remote status / download progress callback */ } if (waitpid(pid, &status, 0) == -1) { err = got_error_from_errno("waitpid"); goto done; } - + if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_idxfds) == -1) { err = got_error_from_errno("socketpair"); goto done; blob - 05435e3e27eea7688d03567a8c5ec462c53976ac blob + 9b2faf40409610d2dc3c08c75caabcf5b9d0f60b --- lib/got_lib_privsep.h +++ lib/got_lib_privsep.h @@ -112,6 +112,7 @@ enum got_imsg_type { GOT_IMSG_FETCH_REQUEST, GOT_IMSG_FETCH_SYMREFS, GOT_IMSG_FETCH_PROGRESS, + GOT_IMSG_FETCH_SERVER_PROGRESS, GOT_IMSG_FETCH_DONE, GOT_IMSG_IDXPACK_REQUEST, GOT_IMSG_IDXPACK_DONE, @@ -356,9 +357,11 @@ const struct got_error *got_privsep_send_fetch_symrefs struct got_pathlist_head *); const struct got_error *got_privsep_send_fetch_progress(struct imsgbuf *, struct got_object_id *, const char *); +const struct got_error *got_privsep_send_fetch_server_progress(struct imsgbuf *, + const char *, size_t); const struct got_error *got_privsep_recv_fetch_progress(int *, struct got_object_id **, char **, struct got_pathlist_head *, - struct imsgbuf *); + char **, struct imsgbuf *); const struct got_error *got_privsep_send_fetch_done(struct imsgbuf *, struct got_object_id); const struct got_error *got_privsep_get_imsg_obj(struct got_object **, blob - 6bd3dc90e688d5c8b9f5684562599a13ede9032c blob + bbd1527cce0abf93067edaea6b101c04a7c0e8d8 --- lib/privsep.c +++ lib/privsep.c @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -565,6 +566,24 @@ got_privsep_send_fetch_progress(struct imsgbuf *ibuf, wbuf->fd = -1; imsg_close(ibuf, wbuf); + return flush_imsg(ibuf); +} + +const struct got_error * +got_privsep_send_fetch_server_progress(struct imsgbuf *ibuf, const char *msg, + size_t msglen) +{ + if (msglen > MAX_IMSGSIZE - IMSG_HEADER_SIZE) + return got_error(GOT_ERR_NO_SPACE); + + if (msglen == 0) + return NULL; + + if (imsg_compose(ibuf, GOT_IMSG_FETCH_SERVER_PROGRESS, 0, 0, -1, + msg, msglen) == -1) + return got_error_from_errno( + "imsg_compose FETCH_SERVER_PROGRESS"); + return flush_imsg(ibuf); } @@ -580,30 +599,33 @@ got_privsep_send_fetch_done(struct imsgbuf *ibuf, stru const struct got_error * got_privsep_recv_fetch_progress(int *done, struct got_object_id **id, - char **refname, struct got_pathlist_head *symrefs, struct imsgbuf *ibuf) + char **refname, struct got_pathlist_head *symrefs, char **server_progress, + struct imsgbuf *ibuf) { const struct got_error *err = NULL; struct imsg imsg; size_t datalen; - const size_t min_datalen = - MIN(MIN(sizeof(struct got_imsg_error), - sizeof(struct got_imsg_fetch_progress)), - sizeof(struct got_imsg_fetch_symrefs)); struct got_imsg_fetch_symrefs *isymrefs = NULL; size_t n, remain; off_t off; + int i; *done = 0; *id = NULL; *refname = NULL; + *server_progress = NULL; - err = got_privsep_recv_imsg(&imsg, ibuf, min_datalen); + err = got_privsep_recv_imsg(&imsg, ibuf, 0); if (err) return err; datalen = imsg.hdr.len - IMSG_HEADER_SIZE; switch (imsg.hdr.type) { case GOT_IMSG_ERROR: + if (datalen < sizeof(struct got_imsg_error)) { + err = got_error(GOT_ERR_PRIVSEP_LEN); + break; + } err = recv_imsg_error(&imsg, datalen); break; case GOT_IMSG_FETCH_SYMREFS: @@ -661,16 +683,16 @@ got_privsep_recv_fetch_progress(int *done, struct got_ } break; case GOT_IMSG_FETCH_PROGRESS: + if (datalen <= SHA1_DIGEST_LENGTH) { + err = got_error(GOT_ERR_PRIVSEP_MSG); + break; + } *id = malloc(sizeof(**id)); if (*id == NULL) { err = got_error_from_errno("malloc"); break; } memcpy((*id)->sha1, imsg.data, SHA1_DIGEST_LENGTH); - if (datalen <= SHA1_DIGEST_LENGTH) { - err = got_error(GOT_ERR_PRIVSEP_MSG); - break; - } *refname = strndup(imsg.data + SHA1_DIGEST_LENGTH, datalen - SHA1_DIGEST_LENGTH); if (*refname == NULL) { @@ -678,6 +700,26 @@ got_privsep_recv_fetch_progress(int *done, struct got_ break; } break; + case GOT_IMSG_FETCH_SERVER_PROGRESS: + if (datalen == 0) { + err = got_error(GOT_ERR_PRIVSEP_LEN); + break; + } + *server_progress = strndup(imsg.data, datalen); + if (*server_progress == NULL) { + err = got_error_from_errno("strndup"); + break; + } + for (i = 0; i < datalen; i++) { + if (!isprint((unsigned char)(*server_progress)[i]) && + !isspace((unsigned char)(*server_progress)[i])) { + err = got_error(GOT_ERR_PRIVSEP_MSG); + free(*server_progress); + *server_progress = NULL; + goto done; + } + } + break; case GOT_IMSG_FETCH_DONE: *id = malloc(sizeof(**id)); if (*id == NULL) { blob - d74463fe8160f2ddcb0f6db0a19abf3e104084bb blob + b1b439e4c1fd3286a870a660a24649e1f1ddbada --- libexec/got-fetch-pack/got-fetch-pack.c +++ libexec/got-fetch-pack/got-fetch-pack.c @@ -91,58 +91,75 @@ flushpkt(int fd) return NULL; } - +/* + * Packet header contains a 4-byte hexstring which specifies the length + * of data which follows. + */ static const struct got_error * -readpkt(int *outlen, int fd, char *buf, int nbuf) +read_pkthdr(int *datalen, int fd) { - const struct got_error *err = NULL; + static const struct got_error *err = NULL; char lenstr[5]; long len; char *e; int n, i; ssize_t r; - *outlen = 0; + *datalen = 0; err = readn(&r, fd, lenstr, 4); if (err) return err; + if (r == 0) /* implicit "0000" */ + return NULL; if (r != 4) - return got_error(GOT_ERR_IO); + return got_error_msg(GOT_ERR_BAD_PACKET, + "wrong packet header length"); lenstr[4] = '\0'; for (i = 0; i < 4; i++) { if (!isxdigit(lenstr[i])) - return got_error(GOT_ERR_BAD_PACKET); + return got_error_msg(GOT_ERR_BAD_PACKET, + "packet length not specified in hex"); } errno = 0; len = strtol(lenstr, &e, 16); if (lenstr[0] == '\0' || *e != '\0') return got_error(GOT_ERR_BAD_PACKET); if (errno == ERANGE && (len == LONG_MAX || len == LONG_MIN)) - return got_error(GOT_ERR_BAD_PACKET); + return got_error_msg(GOT_ERR_BAD_PACKET, "bad packet length"); if (len > INT_MAX || len < INT_MIN) - return got_error(GOT_ERR_BAD_PACKET); + return got_error_msg(GOT_ERR_BAD_PACKET, "bad packet length"); n = len; - if (n == 0) { - if (chattygit) - fprintf(stderr, "readpkt: 0000\n"); + if (n == 0) return NULL; - } if (n <= 4) - return got_error(GOT_ERR_BAD_PACKET); + return got_error_msg(GOT_ERR_BAD_PACKET, "packet too short"); n -= 4; - if (n >= nbuf) + + *datalen = n; + return NULL; +} + +static const struct got_error * +readpkt(int *outlen, int fd, char *buf, int buflen) +{ + const struct got_error *err = NULL; + int datalen; + ssize_t n; + + err = read_pkthdr(&datalen, fd); + if (err) + return err; + + if (datalen > buflen) return got_error(GOT_ERR_NO_SPACE); - err = readn(&r, fd, buf, n); + err = readn(&n, fd, buf, datalen); if (err) return err; - if (r != n) - return got_error(GOT_ERR_BAD_PACKET); - buf[n] = 0; - if (chattygit) - fprintf(stderr, "readpkt: %s:\t%.*s\n", lenstr, nbuf, buf); + if (n != datalen) + return got_error_msg(GOT_ERR_BAD_PACKET, "short packet"); *outlen = n; return NULL; @@ -320,13 +337,23 @@ parse_refline(char **id_str, char **refname, char **se return NULL; } +#define GOT_CAPA_AGENT "agent" +#define GOT_CAPA_OFS_DELTA "ofs-delta" +#define GOT_CAPA_SIDE_BAND_64K "side-band-64k" + +#define GOT_SIDEBAND_PACKFILE_DATA 1 +#define GOT_SIDEBAND_PROGRESS_INFO 2 +#define GOT_SIDEBAND_ERROR_INFO 3 + + struct got_capability { const char *key; const char *value; }; static const struct got_capability got_capabilities[] = { - { "ofs-delta", NULL }, - { "agent", "got/" GOT_VERSION_STR }, + { GOT_CAPA_AGENT, "got/" GOT_VERSION_STR }, + { GOT_CAPA_OFS_DELTA, NULL }, + { GOT_CAPA_SIDE_BAND_64K, NULL }, }; static const struct got_error * @@ -425,9 +452,53 @@ match_capabilities(char **my_capabilities, struct got_ } while (capa); return err; +} + +static const struct got_error * +fetch_progress(struct imsgbuf *ibuf, const char *buf, size_t len) +{ + int i; + + + if (len == 0) + return NULL; + + /* + * Truncate messages which exceed the maximum imsg payload size. + * Server may send up to 64k. + */ + if (len > MAX_IMSGSIZE - IMSG_HEADER_SIZE) + len = MAX_IMSGSIZE - IMSG_HEADER_SIZE; + + /* Only allow printable ASCII. */ + for (i = 0; i < len; i++) { + if (isprint((unsigned char)buf[i]) || + isspace((unsigned char)buf[i])) + continue; + return got_error_msg(GOT_ERR_BAD_PACKET, + "non-printable progress message received from server"); + } + + return got_privsep_send_fetch_server_progress(ibuf, buf, len); } static const struct got_error * +fetch_error(const char *buf, size_t len) +{ + static char msg[1024]; + int i; + + for (i = 0; i < len && i < sizeof(msg) - 1; i++) { + if (!isprint(buf[i])) + return got_error_msg(GOT_ERR_BAD_PACKET, + "non-printable error message received from server"); + msg[i] = buf[i]; + } + msg[i] = '\0'; + return got_error_msg(GOT_ERR_FETCH_FAILED, msg); +} + +static const struct got_error * fetch_pack(int fd, int packfd, struct got_object_id *packid, struct got_pathlist_head *have_refs, struct imsgbuf *ibuf) { @@ -442,6 +513,7 @@ fetch_pack(int fd, int packfd, struct got_object_id *p char *server_capabilities = NULL, *my_capabilities = NULL; struct got_pathlist_head symrefs; struct got_pathlist_entry *pe; + int have_sidebands = 0; TAILQ_INIT(&symrefs); @@ -462,16 +534,7 @@ fetch_pack(int fd, int packfd, struct got_object_id *p if (n == 0) break; if (n >= 4 && strncmp(buf, "ERR ", 4) == 0) { - static char msg[1024]; - for (i = 0; i < n && i < sizeof(msg) - 1; i++) { - if (!isprint(buf[i])) { - err = got_error(GOT_ERR_FETCH_FAILED); - goto done; - } - msg[i] = buf[i]; - } - msg[i] = '\0'; - err = got_error_msg(GOT_ERR_FETCH_FAILED, msg); + err = fetch_error(&buf[4], n - 4); goto done; } err = parse_refline(&id_str, &refname, &server_capabilities, @@ -584,20 +647,96 @@ fetch_pack(int fd, int packfd, struct got_object_id *p * will now send a "NAK" (meaning no common objects were found). */ if (n != 4 || strncmp(buf, "NAK\n", n) != 0) { - err = got_error(GOT_ERR_BAD_PACKET); + err = got_error_msg(GOT_ERR_BAD_PACKET, + "unexpected message from server"); goto done; } if (chattygit) fprintf(stderr, "fetching...\n"); + + if (my_capabilities != NULL && + strstr(my_capabilities, GOT_CAPA_SIDE_BAND_64K) != NULL) + have_sidebands = 1; + packsz = 0; while (1) { - ssize_t r, w; - err = readn(&r, fd, buf, sizeof buf); - if (err) - goto done; - if (r == 0) - break; + ssize_t r = 0, w; + int datalen = -1; + + if (have_sidebands) { + err = read_pkthdr(&datalen, fd); + if (err) + goto done; + if (datalen <= 0) + break; + + /* Read sideband channel ID (one byte). */ + r = read(fd, buf, 1); + if (r == -1) { + err = got_error_from_errno("read"); + goto done; + } + if (r != 1) { + err = got_error_msg(GOT_ERR_BAD_PACKET, + "short packet"); + goto done; + } + if (datalen > sizeof(buf) - 5) { + err = got_error_msg(GOT_ERR_BAD_PACKET, + "bad packet length"); + goto done; + } + datalen--; /* sideband ID has been read */ + if (buf[0] == GOT_SIDEBAND_PACKFILE_DATA) { + /* Read packfile data. */ + err = readn(&r, fd, buf, datalen); + if (err) + goto done; + if (r != datalen) { + err = got_error_msg(GOT_ERR_BAD_PACKET, + "packet too short"); + goto done; + } + } else if (buf[0] == GOT_SIDEBAND_PROGRESS_INFO) { + err = readn(&r, fd, buf, datalen); + if (err) + goto done; + if (r != datalen) { + err = got_error_msg(GOT_ERR_BAD_PACKET, + "packet too short"); + goto done; + } + err = fetch_progress(ibuf, buf, r); + if (err) + goto done; + continue; + } else if (buf[0] == GOT_SIDEBAND_ERROR_INFO) { + err = readn(&r, fd, buf, datalen); + if (err) + goto done; + if (r != datalen) { + err = got_error_msg(GOT_ERR_BAD_PACKET, + "packet too short"); + goto done; + } + err = fetch_error(buf, r); + goto done; + } else { + err = got_error_msg(GOT_ERR_BAD_PACKET, + "unknown side-band received from server"); + goto done; + } + } else { + /* No sideband channel. Every byte is packfile data. */ + err = readn(&r, fd, buf, sizeof buf); + if (err) + goto done; + if (r <= 0) + break; + } + + /* Write packfile data to temporary pack file. */ w = write(packfd, buf, r); if (w == -1) { err = got_error_from_errno("write"); @@ -607,7 +746,7 @@ fetch_pack(int fd, int packfd, struct got_object_id *p err = got_error(GOT_ERR_IO); goto done; } - packsz += r; + packsz += w; } if (lseek(packfd, 0, SEEK_SET) == -1) { err = got_error_from_errno("lseek");