commit baa9fea074326a73792e66efbca9746a1bccc596
from: Stefan Sperling <stsp@stsp.name>
date: Wed Mar 18 16:11:33 2020 UTC

send indexing progress via an imsg

commit - 85e8591f64a845fcf84d74d5158c9905dec81c4e
commit + baa9fea074326a73792e66efbca9746a1bccc596
blob - e4bbfb0c494e0df6e355a7bcc31c0af2d86a4b07
blob + 9fce345c5049ce75a157248987a26b165a993c82
--- got/got.c
+++ got/got.c
@@ -970,7 +970,8 @@ done:
 }
 
 static const struct got_error *
-fetch_progress(void *arg, const char *message, off_t packfile_size)
+fetch_progress(void *arg, const char *message, off_t packfile_size,
+    int nobjects_total, int nobjects_indexed)
 {
 	int *did_something = arg;
 	char scaled[FMT_SCALED_STRSIZE];
@@ -978,8 +979,13 @@ fetch_progress(void *arg, const char *message, off_t p
 	if (message) {
 		printf("\rserver: %s", message);
 		*did_something = 1;
-	} else if (packfile_size > 0 && fmt_scaled(packfile_size, scaled) == 0) {
-		printf("\rfetching... %*s", FMT_SCALED_STRSIZE, scaled);
+	} else if (packfile_size > 0 || nobjects_indexed > 0) {
+		printf("\rfetching...");
+		if (fmt_scaled(packfile_size, scaled) == 0)
+			printf(" %*s", FMT_SCALED_STRSIZE, scaled);
+		if (nobjects_indexed > 0)
+			printf(" indexed %d/%d objects", nobjects_indexed,
+			    nobjects_total);
 		*did_something = 1;
 	}
 	fflush(stdout);
blob - 773e689afb160f0428740fef229371814d293eb0
blob + 5c84e0ca2a0a85854b24b9704a1dfeafb4994c52
--- include/got_fetch.h
+++ include/got_fetch.h
@@ -40,7 +40,7 @@ const struct got_error *got_fetch_connect(int *, const
 
 /* A callback function which gets invoked with progress information to print. */
 typedef const struct got_error *(*got_fetch_progress_cb)(void *,
-    const char *, off_t);
+    const char *, off_t, int, int);
 
 /*
  * Attempt to fetch a packfile from a server. This pack file will contain
blob - cb016271f82e01a23936226d46a6fda2b45f5dd7
blob + 734df058a9220e2d98f2e29f7605822aa1c9c4cc
--- lib/fetch.c
+++ lib/fetch.c
@@ -451,7 +451,7 @@ got_fetch_pack(struct got_object_id **pack_hash, struc
 			while ((s = strsep(&s0, "\r")) != NULL) {
 				if (*s == '\0')
 					continue;
-				err = progress_cb(progress_arg, s, 0);
+				err = progress_cb(progress_arg, s, 0, 0, 0);
 				if (err)
 					break;
 			}
@@ -460,7 +460,7 @@ got_fetch_pack(struct got_object_id **pack_hash, struc
 				goto done;
 		} else if (packfile_size_cur != packfile_size) {
 			err = progress_cb(progress_arg, NULL,
-			    packfile_size_cur);
+			    packfile_size_cur, 0, 0);
 			if (err)
 				break;
 			packfile_size = packfile_size_cur;
@@ -504,10 +504,22 @@ got_fetch_pack(struct got_object_id **pack_hash, struc
 	if (err != NULL)
 		goto done;
 	nidxfd = -1;
-	err = got_privsep_wait_index_pack_done(&idxibuf);
-	if (err != NULL)
-		goto done;
-	imsg_clear(&idxibuf);
+	done = 0;
+	while (!done) {
+		int nobjects_total, nobjects_indexed;
+		err = got_privsep_recv_index_progress(&done, &nobjects_total,
+		    &nobjects_indexed, &idxibuf);
+		if (err != NULL)
+			goto done;
+		if (nobjects_indexed != 0) {
+			err = progress_cb(progress_arg, NULL,
+			    packfile_size, nobjects_total,
+			    nobjects_indexed);
+			if (err)
+				break;
+		}
+		imsg_clear(&idxibuf);
+	}
 	if (close(imsg_idxfds[0]) == -1) {
 		err = got_error_from_errno("close");
 		goto done;
blob - c557473418f6733044e922f4df08164429fe995f
blob + 20e77442a08f16547d7bddcb93d3bf617d9afdd9
--- lib/got_lib_privsep.h
+++ lib/got_lib_privsep.h
@@ -116,6 +116,7 @@ enum got_imsg_type {
 	GOT_IMSG_FETCH_DOWNLOAD_PROGRESS,
 	GOT_IMSG_FETCH_DONE,
 	GOT_IMSG_IDXPACK_REQUEST,
+	GOT_IMSG_IDXPACK_PROGRESS,
 	GOT_IMSG_IDXPACK_DONE,
 
 	/* Messages related to pack files. */
@@ -274,6 +275,14 @@ struct got_imsg_fetch_ref {
 struct got_imsg_fetch_download_progress {
 	/* Number of packfile data bytes downloaded so far. */
 	off_t packfile_bytes;
+};
+
+/* Structure for GOT_IMSG_IDXPACK_PROGRESS data. */
+struct got_imsg_index_pack_progress {
+	/* Total number of objects in pack file. */
+	int nobjects_total;
+	/* Number of objects indexed so far. */
+	int nobjects_indexed;
 };
 
 /* Structure for GOT_IMSG_PACKIDX. */
@@ -356,8 +365,11 @@ const struct got_error *got_privsep_send_obj(struct im
     struct got_object *);
 const struct got_error *got_privsep_send_index_pack_req(struct imsgbuf *, int,
     struct got_object_id *);
+const struct got_error *got_privsep_send_index_pack_progress(struct imsgbuf *,
+    int, int);
 const struct got_error *got_privsep_send_index_pack_done(struct imsgbuf *);
-const struct got_error *got_privsep_wait_index_pack_done(struct imsgbuf *);
+const struct got_error *got_privsep_recv_index_progress(int *, int *, int *,
+    struct imsgbuf *ibuf);
 const struct got_error *got_privsep_send_fetch_req(struct imsgbuf *, int,
     struct got_pathlist_head *);
 const struct got_error *got_privsep_send_fetch_symrefs(struct imsgbuf *,
blob - 96c8759b3613fda441384c283df161cf5059f8fa
blob + 986fccc8b38434f6702bead30a7eddae76b24dd5
--- lib/privsep.c
+++ lib/privsep.c
@@ -777,6 +777,22 @@ got_privsep_send_index_pack_req(struct imsgbuf *ibuf, 
 		close(fd);
 		return err;
 	}
+	return flush_imsg(ibuf);
+}
+
+const struct got_error *
+got_privsep_send_index_pack_progress(struct imsgbuf *ibuf, int nobjects_total,
+    int nobjects_indexed)
+{
+	struct got_imsg_index_pack_progress iprogress;
+
+	iprogress.nobjects_total = nobjects_total;
+	iprogress.nobjects_indexed = nobjects_indexed;
+
+	if (imsg_compose(ibuf, GOT_IMSG_IDXPACK_PROGRESS, 0, 0, -1,
+	    &iprogress, sizeof(iprogress)) == -1)
+		return got_error_from_errno("imsg_compose IDXPACK_PROGRESS");
+
 	return flush_imsg(ibuf);
 }
 
@@ -789,19 +805,54 @@ got_privsep_send_index_pack_done(struct imsgbuf *ibuf)
 }
 
 const struct got_error *
-got_privsep_wait_index_pack_done(struct imsgbuf *ibuf)
+got_privsep_recv_index_progress(int *done, int *nobjects_total,
+    int *nobjects_indexed, struct imsgbuf *ibuf)
 {
 	const struct got_error *err = NULL;
 	struct imsg imsg;
+	struct got_imsg_index_pack_progress *iprogress;
+	size_t datalen;
 
+	*done = 0;
+	*nobjects_total = 0;
+	*nobjects_indexed = 0;
+
 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
 	if (err)
 		return err;
-	if (imsg.hdr.type == GOT_IMSG_IDXPACK_DONE)
-		return NULL;
-	else
-		return got_error(GOT_ERR_PRIVSEP_MSG);
+
+	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
+	switch (imsg.hdr.type) {
+	case GOT_IMSG_ERROR:
+		if (datalen < sizeof(struct got_imsg_error)) {
+			err = got_error(GOT_ERR_PRIVSEP_LEN);
+			break;
+		}
+		err = recv_imsg_error(&imsg, datalen);
+		break;
+	case GOT_IMSG_IDXPACK_PROGRESS:
+		if (datalen < sizeof(*iprogress)) {
+			err = got_error(GOT_ERR_PRIVSEP_LEN);
+			break;
+		}
+		iprogress = (struct got_imsg_index_pack_progress *)imsg.data;
+		*nobjects_total = iprogress->nobjects_total;
+		*nobjects_indexed = iprogress->nobjects_indexed;
+		break;
+	case GOT_IMSG_IDXPACK_DONE:
+		if (datalen != 0) {
+			err = got_error(GOT_ERR_PRIVSEP_LEN);
+			break;
+		}
+		*done = 1;
+		break;
+	default:
+		err = got_error(GOT_ERR_PRIVSEP_MSG);
+		break;
+	}
+
 	imsg_free(&imsg);
+	return err;
 }
 
 const struct got_error *
blob - 1e044637ec38d03deda6d442ef53cad1853a8456
blob + 4247c3edb9fba18b593047110c5cd3c2330efdfc
--- libexec/got-index-pack/got-index-pack.c
+++ libexec/got-index-pack/got-index-pack.c
@@ -1085,7 +1085,8 @@ objectcrc(FILE *f, Object *o)
 }
 
 int
-indexpack(int packfd, int idxfd, struct got_object_id *packhash)
+indexpack(int packfd, int idxfd, struct got_object_id *packhash,
+    struct imsgbuf *ibuf)
 {
 	char hdr[4*3], buf[8];
 	int nobj, nvalid, nbig, n, i, step;
@@ -1117,15 +1118,13 @@ indexpack(int packfd, int idxfd, struct got_object_id 
 	if(!step)
 		step++;
 	while (nvalid != nobj) {
-		fprintf(stderr, "indexing (%d/%d):", nvalid, nobj);
+		got_privsep_send_index_pack_progress(ibuf, nobj, nvalid);
 		n = 0;
 		for (i = 0; i < nobj; i++) {
 			if (valid[i]) {
 				n++;
 				continue;
 			}
-			if (i % step == 0)
-				fprintf(stderr, ".");
 			if (!objects[i]) {
 				o = emalloc(sizeof(Object));
 				o->off = ftello(f);
@@ -1144,7 +1143,6 @@ indexpack(int packfd, int idxfd, struct got_object_id 
 			if(objectcrc(f, o) == -1)
 				return -1;
 		}
-		fprintf(stderr, "\n");
 		if (n == nvalid) {
 			errx(1, "fix point reached too early: %d/%d", nvalid, nobj);
 			goto error;
@@ -1254,7 +1252,7 @@ main(int argc, char **argv)
 	}
 	idxfd = imsg.fd;
 
-	indexpack(packfd, idxfd, &packhash);
+	indexpack(packfd, idxfd, &packhash, &ibuf);
 done:
 	if(err != NULL)
 		got_privsep_send_error(&ibuf, err);