commit 828ad4330c114d7c18c2d02c0f18675aeff78cbfc59578675fa48fbd79161e7c from: Stefan Sperling date: Sun Nov 15 15:09:25 2026 UTC speed up gotadmin cleanup object enumeration The gotadmin cleanup main process was enumerating objects one-by-one, before creating a pack file, by opening every object in the repository with a round-trip to libexec helpers like got-read-pack, got-read-blob, etc. This enumeration loop was a remnant of an early version of gotadmin cleanup which only supported cleaning up loose objects. We can remove this loop, and rely on the pack file creation code instead where we benefit from offloading of object enumeration to got-read-pack with much improved speed. As a side-effect, fix some issues where some objects where not added to generated pack files: directly referenced trees and blobs, and tags which are only referenced via other tags. These issues were uncovered by the regression test suite of gotadmin cleanup. commit - 1ba8f4363f62dd04b504e3d42107f0cad7807f2c2e33090f619752615ceaaaf1 commit + 828ad4330c114d7c18c2d02c0f18675aeff78cbfc59578675fa48fbd79161e7c blob - 167876f05813be7e7d49f6cc6a730916223efa6b82ecbfee9711a49db0f54db6 blob + 400f6a5ca6eba8fb35252c2991fdc8522c0259a26dafd874aee95da5264dcfd4 --- lib/pack_create.c +++ lib/pack_create.c @@ -1106,8 +1106,10 @@ load_tag(int want_meta, struct got_object_idset *idset got_pack_progress_cb progress_cb, void *progress_arg, struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg) { - const struct got_error *err; + const struct got_error *err = NULL; struct got_tag_object *tag = NULL; + struct got_object_id *tag_id = NULL; + int obj_type = GOT_OBJ_TYPE_TAG; if (got_object_idset_contains(idset, id) || got_object_idset_contains(idset_exclude, id)) @@ -1122,36 +1124,61 @@ load_tag(int want_meta, struct got_object_idset *idset return NULL; } - err = got_object_open_as_tag(&tag, repo, id); - if (err) - return err; + tag_id = got_object_id_dup(id); + if (tag_id == NULL) + return got_error_from_errno("got_object_id_dup"); - err = got_pack_add_object(want_meta, - want_meta ? idset : idset_exclude, id, 0, GOT_OBJ_TYPE_TAG, - got_object_tag_get_tagger_time(tag), loose_obj_only, repo, - ncolored, nfound, ntrees, progress_cb, progress_arg, rl); - if (err) - goto done; + while (err == NULL && obj_type == GOT_OBJ_TYPE_TAG) { + err = got_object_open_as_tag(&tag, repo, tag_id); + if (err) + goto done; - switch (got_object_tag_get_object_type(tag)) { - case GOT_OBJ_TYPE_COMMIT: - err = load_commit(want_meta, idset, idset_exclude, - got_object_tag_get_object_id(tag), repo, seed, - loose_obj_only, ncolored, nfound, ntrees, - progress_cb, progress_arg, rl, cancel_cb, cancel_arg); - break; - case GOT_OBJ_TYPE_TREE: - err = got_pack_load_tree(want_meta, idset, idset_exclude, - got_object_tag_get_object_id(tag), "", - got_object_tag_get_tagger_time(tag), seed, repo, - loose_obj_only, ncolored, nfound, ntrees, - progress_cb, progress_arg, rl, cancel_cb, cancel_arg); - break; - default: - break; - } + err = got_pack_add_object(want_meta, + want_meta ? idset : idset_exclude, tag_id, 0, obj_type, + got_object_tag_get_tagger_time(tag), loose_obj_only, repo, + ncolored, nfound, ntrees, progress_cb, progress_arg, rl); + if (err) + goto done; + obj_type = got_object_tag_get_object_type(tag); + switch (obj_type) { + case GOT_OBJ_TYPE_COMMIT: + err = load_commit(want_meta, idset, idset_exclude, + got_object_tag_get_object_id(tag), repo, seed, + loose_obj_only, ncolored, nfound, ntrees, + progress_cb, progress_arg, rl, + cancel_cb, cancel_arg); + break; + case GOT_OBJ_TYPE_TREE: + err = got_pack_load_tree(want_meta, + idset, idset_exclude, + got_object_tag_get_object_id(tag), "", + got_object_tag_get_tagger_time(tag), seed, repo, + loose_obj_only, ncolored, nfound, ntrees, + progress_cb, progress_arg, rl, + cancel_cb, cancel_arg); + break; + case GOT_OBJ_TYPE_BLOB: + err = got_pack_add_object(0, idset_exclude, + got_object_tag_get_object_id(tag), 0, + GOT_OBJ_TYPE_BLOB, + got_object_tag_get_tagger_time(tag), + loose_obj_only, repo, ncolored, nfound, ntrees, + progress_cb, progress_arg, rl); + break; + case GOT_OBJ_TYPE_TAG: + free(tag_id); + tag_id = got_object_id_dup( + got_object_tag_get_object_id(tag)); + if (tag_id == NULL) + err = got_error_from_errno("got_object_id_dup"); + break; + default: + break; + } + } done: + free(tag_id); got_object_tag_close(tag); return err; } @@ -1515,6 +1542,21 @@ load_object_ids(int *ncolored, int *nfound, int *ntree cancel_cb, cancel_arg); if (err) goto done; + } else if (obj_type == GOT_OBJ_TYPE_TREE) { + err = got_pack_load_tree(0, idset, idset_exclude, + id, "", 0, seed, repo, loose_obj_only, + ncolored, nfound, ntrees, + progress_cb, progress_arg, rl, + cancel_cb, cancel_arg); + if (err) + goto done; + } else if (obj_type == GOT_OBJ_TYPE_BLOB) { + err = got_pack_add_object(0, idset_exclude, id, 0, + GOT_OBJ_TYPE_BLOB, 0, loose_obj_only, + repo, ncolored, nfound, ntrees, + progress_cb, progress_arg, rl); + if (err) + goto done; } } @@ -1554,13 +1596,30 @@ load_object_ids(int *ncolored, int *nfound, int *ntree goto done; } else obj_type = m->obj_type; - if (obj_type != GOT_OBJ_TYPE_TAG) - continue; - err = load_tag(1, idset, idset_exclude, id, repo, - seed, loose_obj_only, ncolored, nfound, ntrees, - progress_cb, progress_arg, rl, cancel_cb, cancel_arg); - if (err) - goto done; + + if (obj_type == GOT_OBJ_TYPE_TAG) { + err = load_tag(1, idset, idset_exclude, id, repo, + seed, loose_obj_only, ncolored, nfound, ntrees, + progress_cb, progress_arg, rl, + cancel_cb, cancel_arg); + if (err) + goto done; + } else if (obj_type == GOT_OBJ_TYPE_TREE) { + err = got_pack_load_tree(1, idset, idset_exclude, id, + "", 0, seed, repo, loose_obj_only, + ncolored, nfound, ntrees, + progress_cb, progress_arg, rl, + cancel_cb, cancel_arg); + if (err) + goto done; + } else if (obj_type == GOT_OBJ_TYPE_BLOB) { + err = got_pack_add_object(1, idset, id, 0, + GOT_OBJ_TYPE_BLOB, 0, loose_obj_only, + repo, ncolored, nfound, ntrees, + progress_cb, progress_arg, rl); + if (err) + goto done; + } } done: for (i = 0; i < nobj; i++) { blob - baeb0642dfc71e6ba108c4bf907f51ba1543916da17a0dde3233b6b82dbacdf2 blob + 0277f75d51de4b88c4261310b5ada34f0139819d04dbdd241030c929bd08bd2e --- lib/repository_admin.c +++ lib/repository_admin.c @@ -840,305 +840,6 @@ done: } static const struct got_error * -load_tree_entries(struct got_object_id_queue *ids, - struct got_object_idset *traversed_ids, struct got_object_id *tree_id, - const char *dpath, struct got_repository *repo, - got_cancel_cb cancel_cb, void *cancel_arg) -{ - const struct got_error *err; - struct got_tree_object *tree; - char *p = NULL; - int i; - - err = got_object_open_as_tree(&tree, repo, tree_id); - if (err) - return err; - - for (i = 0; i < got_object_tree_get_nentries(tree); i++) { - struct got_tree_entry *e; - struct got_object_id *id; - mode_t mode; - - e = got_object_tree_get_entry(tree, i); - if (e == NULL) { - char buf[GOT_HASH_DIGEST_STRING_MAXLEN]; - char *id_str; - - id_str = got_hash_digest_to_str(tree_id->hash, buf, - sizeof(buf), tree_id->algo); - err = got_error_fmt(GOT_ERR_BAD_OBJ_DATA, - "cannot get entry %d of %d in tree %s", i + 1, - got_object_tree_get_nentries(tree), id_str); - break; - } - - id = got_tree_entry_get_id(e); - mode = got_tree_entry_get_mode(e); - - if (cancel_cb) { - err = (*cancel_cb)(cancel_arg); - if (err) - break; - } - - if (got_object_tree_entry_is_submodule(e) || - got_object_idset_contains(traversed_ids, id)) - continue; - - if (asprintf(&p, "%s%s%s", dpath, dpath[0] != '\0' ? "/" : "", - got_tree_entry_get_name(e)) == -1) { - err = got_error_from_errno("asprintf"); - break; - } - - if (S_ISDIR(mode)) { - struct got_object_qid *qid; - err = got_object_qid_alloc(&qid, id); - if (err) - break; - STAILQ_INSERT_TAIL(ids, qid, entry); - } else if (S_ISREG(mode) || S_ISLNK(mode)) { - /* This blob is referenced. */ - err = got_object_idset_add(traversed_ids, id, NULL); - if (err) - break; - } - free(p); - p = NULL; - } - - got_object_tree_close(tree); - free(p); - return err; -} - -static const struct got_error * -load_tree(struct got_object_idset *traversed_ids, - struct got_object_id *tree_id, - const char *dpath, struct got_repository *repo, - got_cancel_cb cancel_cb, void *cancel_arg) -{ - const struct got_error *err = NULL; - struct got_object_id_queue tree_ids; - struct got_object_qid *qid; - - err = got_object_qid_alloc(&qid, tree_id); - if (err) - return err; - - STAILQ_INIT(&tree_ids); - STAILQ_INSERT_TAIL(&tree_ids, qid, entry); - - while (!STAILQ_EMPTY(&tree_ids)) { - if (cancel_cb) { - err = (*cancel_cb)(cancel_arg); - if (err) - break; - } - - qid = STAILQ_FIRST(&tree_ids); - STAILQ_REMOVE_HEAD(&tree_ids, entry); - - if (got_object_idset_contains(traversed_ids, &qid->id)) { - got_object_qid_free(qid); - continue; - } - - err = got_object_idset_add(traversed_ids, &qid->id, NULL); - if (err) { - got_object_qid_free(qid); - break; - } - - err = load_tree_entries(&tree_ids, traversed_ids, - &qid->id, dpath, repo, cancel_cb, cancel_arg); - got_object_qid_free(qid); - if (err) - break; - } - - got_object_id_queue_free(&tree_ids); - return err; -} - -static const struct got_error * -load_commit_or_tag(int *ncommits, struct got_object_idset *traversed_ids, - struct got_object_id *id, struct got_repository *repo, - got_cleanup_progress_cb progress_cb, void *progress_arg, - struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg) -{ - const struct got_error *err; - struct got_commit_object *commit = NULL; - struct got_tag_object *tag = NULL; - struct got_object_id *tree_id = NULL; - struct got_object_id_queue ids; - struct got_object_qid *qid; - int obj_type; - - err = got_object_qid_alloc(&qid, id); - if (err) - return err; - - STAILQ_INIT(&ids); - STAILQ_INSERT_TAIL(&ids, qid, entry); - - while (!STAILQ_EMPTY(&ids)) { - if (cancel_cb) { - err = (*cancel_cb)(cancel_arg); - if (err) - break; - } - - qid = STAILQ_FIRST(&ids); - STAILQ_REMOVE_HEAD(&ids, entry); - - if (got_object_idset_contains(traversed_ids, &qid->id)) { - got_object_qid_free(qid); - qid = NULL; - continue; - } - - err = got_object_idset_add(traversed_ids, &qid->id, NULL); - if (err) - break; - - err = got_object_get_type(&obj_type, repo, &qid->id); - if (err) - break; - switch (obj_type) { - case GOT_OBJ_TYPE_COMMIT: - err = got_object_open_as_commit(&commit, repo, - &qid->id); - if (err) - goto done; - tree_id = got_object_commit_get_tree_id(commit); - break; - case GOT_OBJ_TYPE_TAG: - err = got_object_open_as_tag(&tag, repo, &qid->id); - if (err) - goto done; - /* tree_id will be set below */ - break; - case GOT_OBJ_TYPE_TREE: - tree_id = &qid->id; - break; - case GOT_OBJ_TYPE_BLOB: - tree_id = NULL; - break; - default: - /* should not happen */ - err = got_error(GOT_ERR_OBJ_TYPE); - goto done; - } - - if (tag) { - struct got_object_id *id; - - obj_type = got_object_tag_get_object_type(tag); - while (obj_type == GOT_OBJ_TYPE_TAG) { - struct got_tag_object *next_tag; - - id = got_object_tag_get_object_id(tag); - if (!got_object_idset_contains(traversed_ids, - id)) { - err = got_object_idset_add( - traversed_ids, id, NULL); - if (err) - goto done; - } - - err = got_object_open_as_tag(&next_tag, repo, - id); - if (err) - goto done; - - got_object_tag_close(tag); - tag = next_tag; - obj_type = got_object_tag_get_object_type(tag); - } - id = got_object_tag_get_object_id(tag); - switch (obj_type) { - case GOT_OBJ_TYPE_COMMIT: - if (got_object_idset_contains(traversed_ids, - id)) - break; - err = got_object_idset_add(traversed_ids, id, - NULL); - if (err) - goto done; - err = got_object_open_as_commit(&commit, repo, - id); - if (err) - goto done; - tree_id = got_object_commit_get_tree_id(commit); - break; - case GOT_OBJ_TYPE_TREE: - tree_id = id; - break; - case GOT_OBJ_TYPE_BLOB: - if (got_object_idset_contains(traversed_ids, - id)) - break; - err = got_object_idset_add(traversed_ids, id, - NULL); - if (err) - goto done; - break; - default: - /* should not happen */ - err = got_error(GOT_ERR_OBJ_TYPE); - goto done; - } - } else if (tree_id == NULL) { - /* Blob which has already been marked as traversed. */ - continue; - } - - if (tree_id) { - err = load_tree(traversed_ids, tree_id, "", - repo, cancel_cb, cancel_arg); - if (err) - break; - tree_id = NULL; - } - - if (commit || tag) - (*ncommits)++; /* scanned tags are counted as commits */ - - err = report_cleanup_progress(progress_cb, progress_arg, rl, - *ncommits, -1, -1, -1); - if (err) - break; - - if (commit) { - /* Find parent commits to scan. */ - const struct got_object_id_queue *parent_ids; - parent_ids = got_object_commit_get_parent_ids(commit); - err = got_object_id_queue_copy(parent_ids, &ids); - if (err) - break; - got_object_commit_close(commit); - commit = NULL; - } - if (tag) { - got_object_tag_close(tag); - tag = NULL; - } - got_object_qid_free(qid); - qid = NULL; - } -done: - if (qid) - got_object_qid_free(qid); - if (commit) - got_object_commit_close(commit); - if (tag) - got_object_tag_close(tag); - got_object_id_queue_free(&ids); - return err; -} - -static const struct got_error * is_object_packed(int *packed, struct got_repository *repo, struct got_object_id *id) { @@ -1572,7 +1273,7 @@ got_repo_cleanup(struct got_repository *repo, struct got_object_idset *traversed_ids = NULL; struct got_reflist_entry *re; struct got_object_id **referenced_ids; - int i, nreferenced; + int nreferenced; int npurged = 0, packfd = -1; char *tmpfile_path = NULL, *packfile_path = NULL, *idxpath = NULL; FILE *delta_cache = NULL, *packfile = NULL; @@ -1633,15 +1334,6 @@ got_repo_cleanup(struct got_repository *repo, if (err) goto done; - for (i = 0; i < nreferenced; i++) { - struct got_object_id *id = referenced_ids[i]; - err = load_commit_or_tag(ncommits, traversed_ids, - id, repo, progress_cb, progress_arg, &rl, - cancel_cb, cancel_arg); - if (err) - goto done; - } - err = got_pack_create(&pack_hash, packfd, delta_cache, NULL, 0, referenced_ids, nreferenced, repo, 0, 0, 0, pack_progress_cb, pack_progress_arg,