commit - 52915720b2da188a660677add4b597550d443e77
commit + 531c39852ff6a7454ce0e618bacb7b7e20f93523
blob - 741cccfe85b2fe967ee397f3df73d260df9a4c47
blob + a3cf2526d4ee1971e59ae8f61e78f60326046405
--- got/got.c
+++ got/got.c
}
static const struct got_error *
+fetch_progress(void *arg, const char *message)
+{
+ char *servername = arg;
+ printf("\rserver %s: %s", servername, message);
+ fflush(stdout);
+ return NULL;
+}
+
+static const struct got_error *
cmd_clone(int argc, char *argv[])
{
const struct got_error *err = NULL;
goto done;
err = got_fetch_pack(&pack_hash, &refs, &symrefs, fetchfd,
- repo);
+ repo, fetch_progress, host);
if (err)
goto done;
blob - 36345e86940fffeffb539c6aa44c66858f3a4bd9
blob + e3e0144058ef5d5342ddef6dc9a0edd935474201
--- include/got_fetch.h
+++ include/got_fetch.h
const struct got_error *got_fetch_connect(int *, const char *, const char *,
const char *, const char *);
+/* A callback function which gets invoked with progress information to print. */
+typedef const struct got_error *(*got_fetch_progress_cb)(void *,
+ const char *);
+
/*
* Attempt to fetch a packfile from a server. This pack file will contain
* objects which that are not yet contained in the provided repository.
*/
const struct got_error *got_fetch_pack(struct got_object_id **,
struct got_pathlist_head *, struct got_pathlist_head *, int,
- struct got_repository *);
+ struct got_repository *, got_fetch_progress_cb, void *);
blob - 4016f649d26812603b825e73b9cf0839b6ca3d8c
blob + 15ecb2be5c719d2c618a30756def530640a9dad7
--- lib/fetch.c
+++ lib/fetch.c
const struct got_error*
got_fetch_pack(struct got_object_id **pack_hash, struct got_pathlist_head *refs,
- struct got_pathlist_head *symrefs, int fetchfd, struct got_repository *repo)
+ struct got_pathlist_head *symrefs, int fetchfd, struct got_repository *repo,
+ got_fetch_progress_cb progress_cb, void *progress_arg)
{
int imsg_fetchfds[2], imsg_idxfds[2];
int packfd = -1, npackfd = -1, idxfd = -1, nidxfd = -1, nfetchfd = -1;
while (!done) {
struct got_object_id *id = NULL;
char *refname = NULL;
+ char *server_progress = NULL;
err = got_privsep_recv_fetch_progress(&done,
- &id, &refname, symrefs, &ibuf);
+ &id, &refname, symrefs, &server_progress, &ibuf);
if (err != NULL)
goto done;
if (done)
err = got_pathlist_append(refs, refname, id);
if (err)
goto done;
+ } else if (server_progress) {
+ char *s, *s0 = server_progress;
+ while ((s = strsep(&s0, "\r")) != NULL) {
+ if (*s == '\0')
+ continue;
+ err = progress_cb(progress_arg, s);
+ if (err)
+ break;
+ }
+ free(server_progress);
+ if (err)
+ goto done;
}
- /* TODO remote status / download progress callback */
}
if (waitpid(pid, &status, 0) == -1) {
err = got_error_from_errno("waitpid");
goto done;
}
-
+
if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_idxfds) == -1) {
err = got_error_from_errno("socketpair");
goto done;
blob - 05435e3e27eea7688d03567a8c5ec462c53976ac
blob + 9b2faf40409610d2dc3c08c75caabcf5b9d0f60b
--- lib/got_lib_privsep.h
+++ lib/got_lib_privsep.h
GOT_IMSG_FETCH_REQUEST,
GOT_IMSG_FETCH_SYMREFS,
GOT_IMSG_FETCH_PROGRESS,
+ GOT_IMSG_FETCH_SERVER_PROGRESS,
GOT_IMSG_FETCH_DONE,
GOT_IMSG_IDXPACK_REQUEST,
GOT_IMSG_IDXPACK_DONE,
struct got_pathlist_head *);
const struct got_error *got_privsep_send_fetch_progress(struct imsgbuf *,
struct got_object_id *, const char *);
+const struct got_error *got_privsep_send_fetch_server_progress(struct imsgbuf *,
+ const char *, size_t);
const struct got_error *got_privsep_recv_fetch_progress(int *,
struct got_object_id **, char **, struct got_pathlist_head *,
- struct imsgbuf *);
+ char **, struct imsgbuf *);
const struct got_error *got_privsep_send_fetch_done(struct imsgbuf *,
struct got_object_id);
const struct got_error *got_privsep_get_imsg_obj(struct got_object **,
blob - 6bd3dc90e688d5c8b9f5684562599a13ede9032c
blob + bbd1527cce0abf93067edaea6b101c04a7c0e8d8
--- lib/privsep.c
+++ lib/privsep.c
#include <sys/syslimits.h>
#include <sys/wait.h>
+#include <ctype.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
wbuf->fd = -1;
imsg_close(ibuf, wbuf);
+ return flush_imsg(ibuf);
+}
+
+const struct got_error *
+got_privsep_send_fetch_server_progress(struct imsgbuf *ibuf, const char *msg,
+ size_t msglen)
+{
+ if (msglen > MAX_IMSGSIZE - IMSG_HEADER_SIZE)
+ return got_error(GOT_ERR_NO_SPACE);
+
+ if (msglen == 0)
+ return NULL;
+
+ if (imsg_compose(ibuf, GOT_IMSG_FETCH_SERVER_PROGRESS, 0, 0, -1,
+ msg, msglen) == -1)
+ return got_error_from_errno(
+ "imsg_compose FETCH_SERVER_PROGRESS");
+
return flush_imsg(ibuf);
}
const struct got_error *
got_privsep_recv_fetch_progress(int *done, struct got_object_id **id,
- char **refname, struct got_pathlist_head *symrefs, struct imsgbuf *ibuf)
+ char **refname, struct got_pathlist_head *symrefs, char **server_progress,
+ struct imsgbuf *ibuf)
{
const struct got_error *err = NULL;
struct imsg imsg;
size_t datalen;
- const size_t min_datalen =
- MIN(MIN(sizeof(struct got_imsg_error),
- sizeof(struct got_imsg_fetch_progress)),
- sizeof(struct got_imsg_fetch_symrefs));
struct got_imsg_fetch_symrefs *isymrefs = NULL;
size_t n, remain;
off_t off;
+ int i;
*done = 0;
*id = NULL;
*refname = NULL;
+ *server_progress = NULL;
- err = got_privsep_recv_imsg(&imsg, ibuf, min_datalen);
+ err = got_privsep_recv_imsg(&imsg, ibuf, 0);
if (err)
return err;
datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
switch (imsg.hdr.type) {
case GOT_IMSG_ERROR:
+ if (datalen < sizeof(struct got_imsg_error)) {
+ err = got_error(GOT_ERR_PRIVSEP_LEN);
+ break;
+ }
err = recv_imsg_error(&imsg, datalen);
break;
case GOT_IMSG_FETCH_SYMREFS:
}
break;
case GOT_IMSG_FETCH_PROGRESS:
+ if (datalen <= SHA1_DIGEST_LENGTH) {
+ err = got_error(GOT_ERR_PRIVSEP_MSG);
+ break;
+ }
*id = malloc(sizeof(**id));
if (*id == NULL) {
err = got_error_from_errno("malloc");
break;
}
memcpy((*id)->sha1, imsg.data, SHA1_DIGEST_LENGTH);
- if (datalen <= SHA1_DIGEST_LENGTH) {
- err = got_error(GOT_ERR_PRIVSEP_MSG);
- break;
- }
*refname = strndup(imsg.data + SHA1_DIGEST_LENGTH,
datalen - SHA1_DIGEST_LENGTH);
if (*refname == NULL) {
break;
}
break;
+ case GOT_IMSG_FETCH_SERVER_PROGRESS:
+ if (datalen == 0) {
+ err = got_error(GOT_ERR_PRIVSEP_LEN);
+ break;
+ }
+ *server_progress = strndup(imsg.data, datalen);
+ if (*server_progress == NULL) {
+ err = got_error_from_errno("strndup");
+ break;
+ }
+ for (i = 0; i < datalen; i++) {
+ if (!isprint((unsigned char)(*server_progress)[i]) &&
+ !isspace((unsigned char)(*server_progress)[i])) {
+ err = got_error(GOT_ERR_PRIVSEP_MSG);
+ free(*server_progress);
+ *server_progress = NULL;
+ goto done;
+ }
+ }
+ break;
case GOT_IMSG_FETCH_DONE:
*id = malloc(sizeof(**id));
if (*id == NULL) {
blob - d74463fe8160f2ddcb0f6db0a19abf3e104084bb
blob + b1b439e4c1fd3286a870a660a24649e1f1ddbada
--- libexec/got-fetch-pack/got-fetch-pack.c
+++ libexec/got-fetch-pack/got-fetch-pack.c
return NULL;
}
-
+/*
+ * Packet header contains a 4-byte hexstring which specifies the length
+ * of data which follows.
+ */
static const struct got_error *
-readpkt(int *outlen, int fd, char *buf, int nbuf)
+read_pkthdr(int *datalen, int fd)
{
- const struct got_error *err = NULL;
+ static const struct got_error *err = NULL;
char lenstr[5];
long len;
char *e;
int n, i;
ssize_t r;
- *outlen = 0;
+ *datalen = 0;
err = readn(&r, fd, lenstr, 4);
if (err)
return err;
+ if (r == 0) /* implicit "0000" */
+ return NULL;
if (r != 4)
- return got_error(GOT_ERR_IO);
+ return got_error_msg(GOT_ERR_BAD_PACKET,
+ "wrong packet header length");
lenstr[4] = '\0';
for (i = 0; i < 4; i++) {
if (!isxdigit(lenstr[i]))
- return got_error(GOT_ERR_BAD_PACKET);
+ return got_error_msg(GOT_ERR_BAD_PACKET,
+ "packet length not specified in hex");
}
errno = 0;
len = strtol(lenstr, &e, 16);
if (lenstr[0] == '\0' || *e != '\0')
return got_error(GOT_ERR_BAD_PACKET);
if (errno == ERANGE && (len == LONG_MAX || len == LONG_MIN))
- return got_error(GOT_ERR_BAD_PACKET);
+ return got_error_msg(GOT_ERR_BAD_PACKET, "bad packet length");
if (len > INT_MAX || len < INT_MIN)
- return got_error(GOT_ERR_BAD_PACKET);
+ return got_error_msg(GOT_ERR_BAD_PACKET, "bad packet length");
n = len;
- if (n == 0) {
- if (chattygit)
- fprintf(stderr, "readpkt: 0000\n");
+ if (n == 0)
return NULL;
- }
if (n <= 4)
- return got_error(GOT_ERR_BAD_PACKET);
+ return got_error_msg(GOT_ERR_BAD_PACKET, "packet too short");
n -= 4;
- if (n >= nbuf)
+
+ *datalen = n;
+ return NULL;
+}
+
+static const struct got_error *
+readpkt(int *outlen, int fd, char *buf, int buflen)
+{
+ const struct got_error *err = NULL;
+ int datalen;
+ ssize_t n;
+
+ err = read_pkthdr(&datalen, fd);
+ if (err)
+ return err;
+
+ if (datalen > buflen)
return got_error(GOT_ERR_NO_SPACE);
- err = readn(&r, fd, buf, n);
+ err = readn(&n, fd, buf, datalen);
if (err)
return err;
- if (r != n)
- return got_error(GOT_ERR_BAD_PACKET);
- buf[n] = 0;
- if (chattygit)
- fprintf(stderr, "readpkt: %s:\t%.*s\n", lenstr, nbuf, buf);
+ if (n != datalen)
+ return got_error_msg(GOT_ERR_BAD_PACKET, "short packet");
*outlen = n;
return NULL;
return NULL;
}
+#define GOT_CAPA_AGENT "agent"
+#define GOT_CAPA_OFS_DELTA "ofs-delta"
+#define GOT_CAPA_SIDE_BAND_64K "side-band-64k"
+
+#define GOT_SIDEBAND_PACKFILE_DATA 1
+#define GOT_SIDEBAND_PROGRESS_INFO 2
+#define GOT_SIDEBAND_ERROR_INFO 3
+
+
struct got_capability {
const char *key;
const char *value;
};
static const struct got_capability got_capabilities[] = {
- { "ofs-delta", NULL },
- { "agent", "got/" GOT_VERSION_STR },
+ { GOT_CAPA_AGENT, "got/" GOT_VERSION_STR },
+ { GOT_CAPA_OFS_DELTA, NULL },
+ { GOT_CAPA_SIDE_BAND_64K, NULL },
};
static const struct got_error *
} while (capa);
return err;
+}
+
+static const struct got_error *
+fetch_progress(struct imsgbuf *ibuf, const char *buf, size_t len)
+{
+ int i;
+
+
+ if (len == 0)
+ return NULL;
+
+ /*
+ * Truncate messages which exceed the maximum imsg payload size.
+ * Server may send up to 64k.
+ */
+ if (len > MAX_IMSGSIZE - IMSG_HEADER_SIZE)
+ len = MAX_IMSGSIZE - IMSG_HEADER_SIZE;
+
+ /* Only allow printable ASCII. */
+ for (i = 0; i < len; i++) {
+ if (isprint((unsigned char)buf[i]) ||
+ isspace((unsigned char)buf[i]))
+ continue;
+ return got_error_msg(GOT_ERR_BAD_PACKET,
+ "non-printable progress message received from server");
+ }
+
+ return got_privsep_send_fetch_server_progress(ibuf, buf, len);
}
static const struct got_error *
+fetch_error(const char *buf, size_t len)
+{
+ static char msg[1024];
+ int i;
+
+ for (i = 0; i < len && i < sizeof(msg) - 1; i++) {
+ if (!isprint(buf[i]))
+ return got_error_msg(GOT_ERR_BAD_PACKET,
+ "non-printable error message received from server");
+ msg[i] = buf[i];
+ }
+ msg[i] = '\0';
+ return got_error_msg(GOT_ERR_FETCH_FAILED, msg);
+}
+
+static const struct got_error *
fetch_pack(int fd, int packfd, struct got_object_id *packid,
struct got_pathlist_head *have_refs, struct imsgbuf *ibuf)
{
char *server_capabilities = NULL, *my_capabilities = NULL;
struct got_pathlist_head symrefs;
struct got_pathlist_entry *pe;
+ int have_sidebands = 0;
TAILQ_INIT(&symrefs);
if (n == 0)
break;
if (n >= 4 && strncmp(buf, "ERR ", 4) == 0) {
- static char msg[1024];
- for (i = 0; i < n && i < sizeof(msg) - 1; i++) {
- if (!isprint(buf[i])) {
- err = got_error(GOT_ERR_FETCH_FAILED);
- goto done;
- }
- msg[i] = buf[i];
- }
- msg[i] = '\0';
- err = got_error_msg(GOT_ERR_FETCH_FAILED, msg);
+ err = fetch_error(&buf[4], n - 4);
goto done;
}
err = parse_refline(&id_str, &refname, &server_capabilities,
* will now send a "NAK" (meaning no common objects were found).
*/
if (n != 4 || strncmp(buf, "NAK\n", n) != 0) {
- err = got_error(GOT_ERR_BAD_PACKET);
+ err = got_error_msg(GOT_ERR_BAD_PACKET,
+ "unexpected message from server");
goto done;
}
if (chattygit)
fprintf(stderr, "fetching...\n");
+
+ if (my_capabilities != NULL &&
+ strstr(my_capabilities, GOT_CAPA_SIDE_BAND_64K) != NULL)
+ have_sidebands = 1;
+
packsz = 0;
while (1) {
- ssize_t r, w;
- err = readn(&r, fd, buf, sizeof buf);
- if (err)
- goto done;
- if (r == 0)
- break;
+ ssize_t r = 0, w;
+ int datalen = -1;
+
+ if (have_sidebands) {
+ err = read_pkthdr(&datalen, fd);
+ if (err)
+ goto done;
+ if (datalen <= 0)
+ break;
+
+ /* Read sideband channel ID (one byte). */
+ r = read(fd, buf, 1);
+ if (r == -1) {
+ err = got_error_from_errno("read");
+ goto done;
+ }
+ if (r != 1) {
+ err = got_error_msg(GOT_ERR_BAD_PACKET,
+ "short packet");
+ goto done;
+ }
+ if (datalen > sizeof(buf) - 5) {
+ err = got_error_msg(GOT_ERR_BAD_PACKET,
+ "bad packet length");
+ goto done;
+ }
+ datalen--; /* sideband ID has been read */
+ if (buf[0] == GOT_SIDEBAND_PACKFILE_DATA) {
+ /* Read packfile data. */
+ err = readn(&r, fd, buf, datalen);
+ if (err)
+ goto done;
+ if (r != datalen) {
+ err = got_error_msg(GOT_ERR_BAD_PACKET,
+ "packet too short");
+ goto done;
+ }
+ } else if (buf[0] == GOT_SIDEBAND_PROGRESS_INFO) {
+ err = readn(&r, fd, buf, datalen);
+ if (err)
+ goto done;
+ if (r != datalen) {
+ err = got_error_msg(GOT_ERR_BAD_PACKET,
+ "packet too short");
+ goto done;
+ }
+ err = fetch_progress(ibuf, buf, r);
+ if (err)
+ goto done;
+ continue;
+ } else if (buf[0] == GOT_SIDEBAND_ERROR_INFO) {
+ err = readn(&r, fd, buf, datalen);
+ if (err)
+ goto done;
+ if (r != datalen) {
+ err = got_error_msg(GOT_ERR_BAD_PACKET,
+ "packet too short");
+ goto done;
+ }
+ err = fetch_error(buf, r);
+ goto done;
+ } else {
+ err = got_error_msg(GOT_ERR_BAD_PACKET,
+ "unknown side-band received from server");
+ goto done;
+ }
+ } else {
+ /* No sideband channel. Every byte is packfile data. */
+ err = readn(&r, fd, buf, sizeof buf);
+ if (err)
+ goto done;
+ if (r <= 0)
+ break;
+ }
+
+ /* Write packfile data to temporary pack file. */
w = write(packfd, buf, r);
if (w == -1) {
err = got_error_from_errno("write");
err = got_error(GOT_ERR_IO);
goto done;
}
- packsz += r;
+ packsz += w;
}
if (lseek(packfd, 0, SEEK_SET) == -1) {
err = got_error_from_errno("lseek");
