commit 31ba223621c960bfec6da52df2ab65f2b0791d68 from: Stefan Sperling via: Thomas Adam date: Wed Jan 05 20:45:09 2022 UTC use time-based rate-limiting for gotadmin progress output Suggested by naddy some time ago. ok tracey commit - bc1f382f7aba640a7b6949938af27d41c1cabacf commit + 31ba223621c960bfec6da52df2ab65f2b0791d68 blob - /dev/null blob + 9b95dcd8f24e8a9a1ad8900a16ece71c03e03a94 (mode 644) --- /dev/null +++ lib/got_lib_ratelimit.h @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2022 Stefan Sperling + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +struct got_ratelimit { + struct timespec last; + struct timespec interval; +}; + +void got_ratelimit_init(struct got_ratelimit *, time_t, unsigned int); +const struct got_error *got_ratelimit_check(int *, struct got_ratelimit *); blob - d0f59316e2f326a98b4f4e4a666136791edc48b1 blob + ff8f487b05d2d9af496f1b846a580ce369fd95af --- lib/pack_create.c +++ lib/pack_create.c @@ -18,11 +18,14 @@ #include #include #include +#include #include #include #include #include +#include +#include #include #include @@ -47,6 +50,7 @@ #include "got_lib_pack.h" #include "got_lib_privsep.h" #include "got_lib_repository.h" +#include "got_lib_ratelimit.h" #ifndef MIN #define MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b)) @@ -248,13 +252,31 @@ encode_delta(struct got_pack_meta *m, struct got_raw_o return NULL; } + +static const struct got_error * +report_progress(got_pack_progress_cb progress_cb, void *progress_arg, + struct got_ratelimit *rl, off_t packfile_size, int ncommits, + int nobj_total, int obj_deltify, int nobj_written) +{ + const struct got_error *err; + int elapsed; + + if (progress_cb == NULL) + return NULL; + err = got_ratelimit_check(&elapsed, rl); + if (err || !elapsed) + return err; + return progress_cb(progress_arg, packfile_size, ncommits, + nobj_total, obj_deltify, nobj_written); +} + static const struct got_error * pick_deltas(struct got_pack_meta **meta, int nmeta, int nours, FILE *delta_cache, struct got_repository *repo, got_pack_progress_cb progress_cb, void *progress_arg, - got_cancel_cb cancel_cb, void *cancel_arg) + struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg) { const struct got_error *err = NULL; struct got_pack_meta *m = NULL, *base = NULL; @@ -271,11 +293,10 @@ pick_deltas(struct got_pack_meta **meta, int nmeta, in if (err) break; } - if (progress_cb) { - err = progress_cb(progress_arg, 0L, nours, nmeta, i, 0); - if (err) - goto done; - } + err = report_progress(progress_cb, progress_arg, rl, + 0L, nours, nmeta, i, 0); + if (err) + goto done; m = meta[i]; if (m->obj_type == GOT_OBJ_TYPE_COMMIT || @@ -923,7 +944,7 @@ read_meta(struct got_pack_meta ***meta, int *nmeta, struct got_object_id **theirs, int ntheirs, struct got_object_id **ours, int nours, struct got_repository *repo, int loose_obj_only, got_pack_progress_cb progress_cb, void *progress_arg, - got_cancel_cb cancel_cb, void *cancel_arg) + struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg) { const struct got_error *err = NULL; struct got_object_id **ids = NULL; @@ -964,12 +985,10 @@ read_meta(struct got_pack_meta ***meta, int *nmeta, loose_obj_only, cancel_cb, cancel_arg); if (err) goto done; - if (progress_cb) { - err = progress_cb(progress_arg, 0L, nours, - v.nmeta, 0, 0); - if (err) - goto done; - } + err = report_progress(progress_cb, progress_arg, rl, + 0L, nours, v.nmeta, 0, 0); + if (err) + goto done; } for (i = 0; i < ntheirs; i++) { @@ -990,12 +1009,10 @@ read_meta(struct got_pack_meta ***meta, int *nmeta, loose_obj_only, cancel_cb, cancel_arg); if (err) goto done; - if (progress_cb) { - err = progress_cb(progress_arg, 0L, nours, - v.nmeta, 0, 0); - if (err) - goto done; - } + err = report_progress(progress_cb, progress_arg, rl, + 0L, nours, v.nmeta, 0, 0); + if (err) + goto done; } for (i = 0; i < nobj; i++) { @@ -1003,12 +1020,12 @@ read_meta(struct got_pack_meta ***meta, int *nmeta, loose_obj_only, cancel_cb, cancel_arg); if (err) goto done; - if (progress_cb) { - err = progress_cb(progress_arg, 0L, nours, - v.nmeta, 0, 0); - if (err) - goto done; - } + if (err) + goto done; + err = report_progress(progress_cb, progress_arg, rl, + 0L, nours, v.nmeta, 0, 0); + if (err) + goto done; } for (i = 0; i < nours; i++) { @@ -1029,14 +1046,17 @@ read_meta(struct got_pack_meta ***meta, int *nmeta, loose_obj_only, cancel_cb, cancel_arg); if (err) goto done; - if (progress_cb) { - err = progress_cb(progress_arg, 0L, nours, - v.nmeta, 0, 0); - if (err) - goto done; - } + err = report_progress(progress_cb, progress_arg, rl, + 0L, nours, v.nmeta, 0, 0); + if (err) + goto done; } + if (progress_cb) { + err = progress_cb(progress_arg, 0L, nours, v.nmeta, 0, 0); + if (err) + goto done; + } done: for (i = 0; i < nobj; i++) { free(ids[i]); @@ -1136,6 +1156,7 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt struct got_pack_meta **meta, int nmeta, int nours, int use_offset_deltas, struct got_repository *repo, got_pack_progress_cb progress_cb, void *progress_arg, + struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg) { const struct got_error *err = NULL; @@ -1167,12 +1188,10 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt goto done; qsort(meta, nmeta, sizeof(struct got_pack_meta *), write_order_cmp); for (i = 0; i < nmeta; i++) { - if (progress_cb) { - err = progress_cb(progress_arg, packfile_size, nours, - nmeta, nmeta, i); - if (err) - goto done; - } + err = report_progress(progress_cb, progress_arg, rl, + packfile_size, nours, nmeta, nmeta, i); + if (err) + goto done; m = meta[i]; m->off = ftello(packfile); err = got_object_raw_open(&raw, &outfd, repo, &m->id); @@ -1281,10 +1300,12 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt err = got_ferror(packfile, GOT_ERR_IO); packfile_size += SHA1_DIGEST_LENGTH; packfile_size += sizeof(struct got_packfile_hdr); - err = progress_cb(progress_arg, packfile_size, nours, - nmeta, nmeta, nmeta); - if (err) - goto done; + if (progress_cb) { + err = progress_cb(progress_arg, packfile_size, nours, + nmeta, nmeta, nmeta); + if (err) + goto done; + } done: if (delta_file && fclose(delta_file) == EOF && err == NULL) err = got_error_from_errno("fclose"); @@ -1307,9 +1328,13 @@ got_pack_create(uint8_t *packsha1, FILE *packfile, struct got_pack_meta **meta; int nmeta; FILE *delta_cache = NULL; + struct got_ratelimit rl; + got_ratelimit_init(&rl, 0, 500); + err = read_meta(&meta, &nmeta, theirs, ntheirs, ours, nours, repo, - loose_obj_only, progress_cb, progress_arg, cancel_cb, cancel_arg); + loose_obj_only, progress_cb, progress_arg, &rl, + cancel_cb, cancel_arg); if (err) return err; @@ -1326,7 +1351,7 @@ got_pack_create(uint8_t *packsha1, FILE *packfile, if (nmeta > 0) { err = pick_deltas(meta, nmeta, nours, delta_cache, repo, - progress_cb, progress_arg, cancel_cb, cancel_arg); + progress_cb, progress_arg, &rl, cancel_cb, cancel_arg); if (err) goto done; if (fseeko(delta_cache, 0L, SEEK_SET) == -1) { @@ -1336,7 +1361,7 @@ got_pack_create(uint8_t *packsha1, FILE *packfile, } err = genpack(packsha1, packfile, delta_cache, meta, nmeta, nours, 1, - repo, progress_cb, progress_arg, cancel_cb, cancel_arg); + repo, progress_cb, progress_arg, &rl, cancel_cb, cancel_arg); if (err) goto done; done: blob - /dev/null blob + 219e7905e4765ae964a3ab7a1e8145d1ce344c2f (mode 644) --- /dev/null +++ lib/ratelimit.c @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2022 Stefan Sperling + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include + +#include +#include +#include +#include + +#include "got_lib_ratelimit.h" + +#include "got_error.h" + +void +got_ratelimit_init(struct got_ratelimit *rl, time_t interval_sec, + unsigned int interval_msec) +{ + memset(rl, 0, sizeof(*rl)); + rl->interval.tv_sec = interval_sec; + rl->interval.tv_nsec = interval_msec * 1000000UL; +} + +const struct got_error * +got_ratelimit_check(int *elapsed, struct got_ratelimit *rl) +{ + struct timespec now, delta; + + if (clock_gettime(CLOCK_MONOTONIC, &now) == -1) + return got_error_from_errno("clock_gettime"); + + if (timespecisset(&rl->last)) { + timespecsub(&now, &rl->last, &delta); + *elapsed = timespeccmp(&delta, &rl->interval, >=) ? 1 : 0; + } else + *elapsed = 1; + + if (*elapsed) + rl->last = now; + + return NULL; +} blob - 739ff3d01b9b761e9af99eed1c1ef1141465eb90 blob + 831a3bcbe7d86084e4f2106af90f970ab8dcbc8f --- lib/repository_admin.c +++ lib/repository_admin.c @@ -49,6 +49,7 @@ #include "got_lib_pack_create.h" #include "got_lib_sha1.h" #include "got_lib_lockfile.h" +#include "got_lib_ratelimit.h" #ifndef nitems #define nitems(_a) (sizeof((_a)) / sizeof((_a)[0])) @@ -593,9 +594,27 @@ done: } static const struct got_error * +report_cleanup_progress(got_cleanup_progress_cb progress_cb, + void *progress_arg, struct got_ratelimit *rl, + int nloose, int ncommits, int npurged) +{ + const struct got_error *err; + int elapsed; + + if (progress_cb == NULL) + return NULL; + + err = got_ratelimit_check(&elapsed, rl); + if (err || !elapsed) + return err; + + return progress_cb(progress_arg, nloose, ncommits, npurged); +} + +static const struct got_error * get_loose_object_ids(struct got_object_idset **loose_ids, off_t *ondisk_size, got_cleanup_progress_cb progress_cb, void *progress_arg, - struct got_repository *repo) + struct got_ratelimit *rl, struct got_repository *repo) { const struct got_error *err = NULL; char *path_objects = NULL, *path = NULL; @@ -683,13 +702,11 @@ get_loose_object_ids(struct got_object_idset **loose_i err = got_object_idset_add(*loose_ids, &id, NULL); if (err) goto done; - if (progress_cb) { - err = progress_cb(progress_arg, - got_object_idset_num_elements(*loose_ids), - -1, -1); - if (err) - goto done; - } + err = report_cleanup_progress(progress_cb, + progress_arg, rl, + got_object_idset_num_elements(*loose_ids), -1, -1); + if (err) + goto done; } if (closedir(dir) != 0) { @@ -873,7 +890,8 @@ static const struct got_error * load_commit_or_tag(struct got_object_idset *loose_ids, int *ncommits, int *npacked, struct got_object_idset *traversed_ids, struct got_object_id *id, struct got_repository *repo, - got_cleanup_progress_cb progress_cb, void *progress_arg, int nloose, + got_cleanup_progress_cb progress_cb, void *progress_arg, + struct got_ratelimit *rl, int nloose, got_cancel_cb cancel_cb, void *cancel_arg) { const struct got_error *err; @@ -981,11 +999,10 @@ load_commit_or_tag(struct got_object_idset *loose_ids, if (commit || tag) (*ncommits)++; /* scanned tags are counted as commits */ - if (progress_cb) { - err = progress_cb(progress_arg, nloose, *ncommits, -1); - if (err) - break; - } + err = report_cleanup_progress(progress_cb, progress_arg, rl, + nloose, *ncommits, -1); + if (err) + break; if (commit) { /* Find parent commits to scan. */ @@ -1019,6 +1036,7 @@ struct purge_loose_object_arg { struct got_repository *repo; got_cleanup_progress_cb progress_cb; void *progress_arg; + struct got_ratelimit *rl; int nloose; int ncommits; int npurged; @@ -1070,10 +1088,10 @@ purge_loose_object(struct got_object_id *id, void *dat a->npurged++; a->size_purged += sb.st_size; - if (a->progress_cb) { - err = a->progress_cb(a->progress_arg, a->nloose, - a->ncommits, a->npurged); - } + err = report_cleanup_progress(a->progress_cb, a->progress_arg, + a->rl, a->nloose, a->ncommits, a->npurged); + if (err) + goto done; } done: if (fd != -1 && close(fd) == -1 && err == NULL) @@ -1099,15 +1117,17 @@ got_repo_purge_unreferenced_loose_objects(struct got_r struct got_reflist_entry *re; struct purge_loose_object_arg arg; time_t max_mtime = 0; + struct got_ratelimit rl; TAILQ_INIT(&refs); + got_ratelimit_init(&rl, 0, 500); *size_before = 0; *size_after = 0; *npacked = 0; err = get_loose_object_ids(&loose_ids, size_before, - progress_cb, progress_arg, repo); + progress_cb, progress_arg, &rl, repo); if (err) return err; nloose = got_object_idset_num_elements(loose_ids); @@ -1148,23 +1168,17 @@ got_repo_purge_unreferenced_loose_objects(struct got_r for (i = 0; i < nreferenced; i++) { struct got_object_id *id = referenced_ids[i]; err = load_commit_or_tag(loose_ids, &ncommits, npacked, - traversed_ids, id, repo, progress_cb, progress_arg, nloose, - cancel_cb, cancel_arg); + traversed_ids, id, repo, progress_cb, progress_arg, &rl, + nloose, cancel_cb, cancel_arg); if (err) goto done; } - /* Produce a final progress report in case no objects can be purged. */ - if (got_object_idset_num_elements(loose_ids) == 0 && progress_cb) { - err = progress_cb(progress_arg, nloose, ncommits, 0); - if (err) - goto done; - } - /* Any remaining loose objects are unreferenced and can be purged. */ arg.repo = repo; arg.progress_arg = progress_arg; arg.progress_cb = progress_cb; + arg.rl = &rl; arg.nloose = nloose; arg.npurged = 0; arg.size_purged = 0; @@ -1176,6 +1190,14 @@ got_repo_purge_unreferenced_loose_objects(struct got_r if (err) goto done; *size_after = *size_before - arg.size_purged; + + /* Produce a final progress report. */ + if (progress_cb) { + err = progress_cb(progress_arg, nloose, ncommits, + got_object_idset_num_elements(loose_ids)); + if (err) + goto done; + } done: got_object_idset_free(loose_ids); got_object_idset_free(traversed_ids);