commit ad52156f9e918051844eb400a4e6e2cbc36772ca2dfa37d33f37a8854bc9b78a from: Stefan Sperling date: Tue Jan 21 15:03:27 2025 UTC make gotadmin cleanup pack the repository before cleaning Our cleanup implementation is only safe to use after everything referenced has been packed into a single pack file. Otherwise, the algorithm we use for checking pack redundancy might remove small packs which contain objects that other objects depend on. The easy fix for this issue is to have 'gotadmin cleanup' create the required pack file before cleaning up, making cleanup safe by default. This happens to be what 'git gc' does as well. ok jamsek commit - 9a47c4d14cf416a9acdc96d46cc26daf80cfa34fdd5a63692022d2eb04df435d commit + ad52156f9e918051844eb400a4e6e2cbc36772ca2dfa37d33f37a8854bc9b78a blob - 8f580521cde4b4b7f013462b66ebceb0a110ab691161e9ca8598984d2031f192 blob + 481490b9d333ae01bcb99484bbd977e63b5bd157e634ad7b5da72ee798f4981a --- gotadmin/gotadmin.c +++ gotadmin/gotadmin.c @@ -520,7 +520,7 @@ print_load_info(FILE *out, int print_colored, int prin static const struct got_error * pack_progress(void *arg, int ncolored, int nfound, int ntrees, off_t packfile_size, int ncommits, int nobj_total, int nobj_deltify, - int nobj_written) + int nobj_written, int pack_done) { struct got_pack_progress_arg *a = arg; char scaled_size[FMT_SCALED_STRSIZE]; @@ -621,17 +621,18 @@ pack_progress(void *arg, int ncolored, int nfound, int if (print_written) fprintf(a->out, "; writing pack: %*s %d%%", FMT_SCALED_STRSIZE - 2, scaled_size, p_written); - if (print_searching || print_total || print_deltify || - print_written) { + if (print_searching || print_total || print_deltify || print_written) { a->printed_something = 1; fflush(a->out); } + if (pack_done) + fprintf(a->out, "\n"); return NULL; } static const struct got_error * pack_index_progress(void *arg, off_t packfile_size, int nobj_total, - int nobj_indexed, int nobj_loose, int nobj_resolved) + int nobj_indexed, int nobj_loose, int nobj_resolved, int indexing_done) { struct got_pack_progress_arg *a = arg; char scaled_size[FMT_SCALED_STRSIZE]; @@ -678,7 +679,9 @@ pack_index_progress(void *arg, off_t packfile_size, in printf("; indexing %d%%", p_indexed); if (print_resolved) printf("; resolving deltas %d%%", p_resolved); - if (print_size || print_indexed || print_resolved) + if (indexing_done) + printf("\n"); + if (print_size || print_indexed || print_resolved || indexing_done) fflush(stdout); return NULL; @@ -719,7 +722,7 @@ cmd_pack(int argc, char *argv[]) struct got_repository *repo = NULL; int ch, i, loose_obj_only = 1, force_refdelta = 0, verbosity = 0; struct got_object_id *pack_hash = NULL; - char *id_str = NULL; + char *id_str = NULL, *idxpath = NULL; struct got_pack_progress_arg ppa; FILE *packfile = NULL; struct got_pathlist_head exclude_args; @@ -844,7 +847,7 @@ cmd_pack(int argc, char *argv[]) if (verbosity >= 0) printf("\nWrote %s.pack\n", id_str); - error = got_repo_index_pack(packfile, pack_hash, repo, + error = got_repo_index_pack(&idxpath, packfile, pack_hash, repo, pack_index_progress, &ppa, check_cancelled, NULL); if (error) goto done; @@ -865,6 +868,7 @@ done: free(id_str); free(pack_hash); free(repo_path); + free(idxpath); return error; } @@ -883,7 +887,7 @@ cmd_indexpack(int argc, char *argv[]) struct got_repository *repo = NULL; int ch; struct got_object_id *pack_hash = NULL; - char *packfile_path = NULL; + char *packfile_path = NULL, *idxpath = NULL; char *id_str = NULL; struct got_pack_progress_arg ppa; FILE *packfile = NULL; @@ -939,7 +943,7 @@ cmd_indexpack(int argc, char *argv[]) if (error) goto done; - error = got_repo_index_pack(packfile, pack_hash, repo, + error = got_repo_index_pack(&idxpath, packfile, pack_hash, repo, pack_index_progress, &ppa, check_cancelled, NULL); if (error) goto done; @@ -955,6 +959,7 @@ done: } free(id_str); free(pack_hash); + free(idxpath); return error; } @@ -1266,6 +1271,7 @@ cmd_cleanup(int argc, char *argv[]) int ch, dry_run = 0, verbosity = 0; int ncommits = 0, nloose = 0, npacked = 0; int remove_lonely_packidx = 0, ignore_mtime = 0; + struct got_pack_progress_arg ppa; struct got_cleanup_progress_arg cpa; struct got_lonely_packidx_progress_arg lpa; off_t loose_before, loose_after; @@ -1279,7 +1285,7 @@ cmd_cleanup(int argc, char *argv[]) int *pack_fds = NULL; #ifndef PROFILE - if (pledge("stdio rpath wpath cpath flock proc exec sendfd unveil", + if (pledge("stdio rpath wpath cpath fattr flock proc exec sendfd unveil", NULL) == -1) err(1, "pledge"); #endif @@ -1353,11 +1359,16 @@ cmd_cleanup(int argc, char *argv[]) cpa.dry_run = dry_run; cpa.verbosity = verbosity; + memset(&ppa, 0, sizeof(ppa)); + ppa.out = stdout; + ppa.verbosity = verbosity; + error = got_repo_cleanup(repo, &loose_before, &loose_after, &pack_before, &pack_after, &ncommits, &nloose, &npacked, dry_run, ignore_mtime, cleanup_progress, &cpa, + pack_progress, &ppa, pack_index_progress, &ppa, check_cancelled, NULL); - if (cpa.printed_something) + if (ppa.printed_something || cpa.printed_something) printf("\n"); if (error) goto done; @@ -1562,7 +1573,7 @@ load_progress(void *arg, off_t packfile_size, int nobj int nobj_indexed, int nobj_loose, int nobj_resolved) { return pack_index_progress(arg, packfile_size, nobj_total, - nobj_indexed, nobj_loose, nobj_resolved); + nobj_indexed, nobj_loose, nobj_resolved, 0); } static int blob - d455d3e82cd359252bb374a3abe62d0d2bf17d34c1a6bfcc4a73b35f1d84107f blob + 832fdc8961798948d0e3ece2a5d9308b06dd8c47596d2c8a9b4ecbc6c41d7352 --- gotd/repo_read.c +++ gotd/repo_read.c @@ -510,7 +510,7 @@ struct repo_read_pack_progress_arg { static const struct got_error * pack_progress(void *arg, int ncolored, int nfound, int ntrees, off_t packfile_size, int ncommits, int nobj_total, int nobj_deltify, - int nobj_written) + int nobj_written, int pack_done) { struct repo_read_pack_progress_arg *a = arg; struct gotd_imsg_packfile_progress iprog; blob - 77d619cc10cc7aa7be1445d31d983d7f954efbff2db70119a0cff51a6cfb66e0 blob + 1f1aa1805ae17f62d874299ee7221a753add3951ac9aeb75bf6e4fdbed612f5d --- include/got_repository_admin.h +++ include/got_repository_admin.h @@ -17,7 +17,7 @@ /* A callback function which gets invoked with progress information to print. */ typedef const struct got_error *(*got_pack_progress_cb)(void *arg, int ncolored, int nfound, int ntrees, off_t packfile_size, int ncommits, - int nobj_total, int obj_deltify, int nobj_written); + int nobj_total, int obj_deltify, int nobj_written, int pack_done); /* * Attempt to pack objects reachable via 'include_refs' into a new packfile. @@ -49,12 +49,12 @@ got_repo_find_pack(FILE **packfile, struct got_object_ /* A callback function which gets invoked with progress information to print. */ typedef const struct got_error *(*got_pack_index_progress_cb)(void *arg, off_t packfile_size, int nobj_total, int nobj_indexed, - int nobj_loose, int nobj_resolved); + int nobj_loose, int nobj_resolved, int indexing_done); /* (Re-)Index the pack file identified by the given hash. */ const struct got_error * -got_repo_index_pack(FILE *packfile, struct got_object_id *pack_hash, - struct got_repository *repo, +got_repo_index_pack(char **idxpath, FILE *packfile, + struct got_object_id *pack_hash, struct got_repository *repo, got_pack_index_progress_cb progress_cb, void *progress_arg, got_cancel_cb cancel_cb, void *cancel_arg); @@ -73,9 +73,10 @@ typedef const struct got_error *(*got_cleanup_progress int ncommits, int nloose, int npurged, int nredundant); /* - * Walk objects reachable via references to determine whether any loose - * objects can be removed from disk. Do remove such objects from disk - * unless the dry_run parameter is set. + * Walk objects reachable via references and, unless the dry-run parameter + * is set, pack all referenced objects into a single pack. + * Then determine whether any loose objects can be removed from disk. + * Do remove such objects from disk unless the dry_run parameter is set. * Do not remove objects with a modification timestamp above an * implementation-defined timestamp threshold, unless ignore_mtime is set. * Remove packfiles which objects are either unreachable or provided @@ -91,6 +92,8 @@ got_repo_cleanup(struct got_repository *repo, int *ncommits, int *nloose, int *npacked, int dry_run, int ignore_mtime, got_cleanup_progress_cb progress_cb, void *progress_arg, + got_pack_progress_cb pack_progress_cb, void *pack_progress_arg, + got_pack_index_progress_cb index_progress_cb, void *index_progress_arg, got_cancel_cb cancel_cb, void *cancel_arg); /* A callback function which gets invoked with cleanup information to print. */ blob - 791b923758e3761cefc96469ee90bdd06a26120b9644b3d3297eba1be4c1fc74 blob + af9d941d599beff2661fbe07ddcf04419ddca5cbb81958f551bb36e349231ac5 --- lib/got_lib_pack_create.h +++ lib/got_lib_pack_create.h @@ -106,7 +106,7 @@ const struct got_error * got_pack_report_progress(got_pack_progress_cb progress_cb, void *progress_arg, struct got_ratelimit *rl, int ncolored, int nfound, int ntrees, off_t packfile_size, int ncommits, int nobj_total, int obj_deltify, - int nobj_written); + int nobj_written, int pack_done); const struct got_error * got_pack_load_packed_object_ids(int *found_all_objects, blob - e26e54ae451057c928c835e2484441b2d2aca884dbe88f7cab6fa16cec206318 blob + 4e3cfc6b8f88b856e566a5a2f00c4e3231b29d7d8f86b1b489653afa5a7dfc90 --- lib/pack_create.c +++ lib/pack_create.c @@ -445,7 +445,7 @@ const struct got_error * got_pack_report_progress(got_pack_progress_cb progress_cb, void *progress_arg, struct got_ratelimit *rl, int ncolored, int nfound, int ntrees, off_t packfile_size, int ncommits, int nobj_total, int obj_deltify, - int nobj_written) + int nobj_written, int pack_done) { const struct got_error *err; int elapsed; @@ -458,7 +458,8 @@ got_pack_report_progress(got_pack_progress_cb progress return err; return progress_cb(progress_arg, ncolored, nfound, ntrees, - packfile_size, ncommits, nobj_total, obj_deltify, nobj_written); + packfile_size, ncommits, nobj_total, obj_deltify, nobj_written, + pack_done); } const struct got_error * @@ -566,7 +567,7 @@ pick_deltas(struct got_pack_meta **meta, int nmeta, in } err = got_pack_report_progress(progress_cb, progress_arg, rl, ncolored, nfound, ntrees, 0L, ncommits, nreused + nmeta, - nreused + i, 0); + nreused + i, 0, 0); if (err) goto done; m = meta[i]; @@ -750,7 +751,7 @@ got_pack_add_object(int want_meta, struct got_object_i (*nfound)++; err = got_pack_report_progress(progress_cb, progress_arg, rl, - *ncolored, *nfound, *ntrees, 0L, 0, 0, 0, 0); + *ncolored, *nfound, *ntrees, 0L, 0, 0, 0, 0, 0); if (err) { clear_meta(m); free(m); @@ -781,7 +782,7 @@ got_pack_load_tree_entries(struct got_object_id_queue (*ntrees)++; err = got_pack_report_progress(progress_cb, progress_arg, rl, - *ncolored, *nfound, *ntrees, 0L, 0, 0, 0, 0); + *ncolored, *nfound, *ntrees, 0L, 0, 0, 0, 0, 0); if (err) return err; @@ -1764,7 +1765,7 @@ genpack(struct got_object_id *pack_hash, int packfd, for (i = 0; i < ndeltify; i++) { err = got_pack_report_progress(progress_cb, progress_arg, rl, ncolored, nfound, ntrees, packfile_size, nours, - ndeltify + nreuse, ndeltify + nreuse, i); + ndeltify + nreuse, ndeltify + nreuse, i, 0); if (err) goto done; m = deltify[i]; @@ -1793,7 +1794,7 @@ genpack(struct got_object_id *pack_hash, int packfd, for (i = 0; i < nreuse; i++) { err = got_pack_report_progress(progress_cb, progress_arg, rl, ncolored, nfound, ntrees, packfile_size, nours, - ndeltify + nreuse, ndeltify + nreuse, ndeltify + i); + ndeltify + nreuse, ndeltify + nreuse, ndeltify + i, 0); if (err) goto done; m = reuse[i]; @@ -1813,7 +1814,7 @@ genpack(struct got_object_id *pack_hash, int packfd, if (progress_cb) { err = progress_cb(progress_arg, ncolored, nfound, ntrees, packfile_size, nours, ndeltify + nreuse, - ndeltify + nreuse, ndeltify + nreuse); + ndeltify + nreuse, ndeltify + nreuse, 1); if (err) goto done; } @@ -1875,7 +1876,7 @@ got_pack_create(struct got_object_id *packhash, int pa if (progress_cb) { err = progress_cb(progress_arg, ncolored, nfound, ntrees, - 0L, nours, got_object_idset_num_elements(idset), 0, 0); + 0L, nours, got_object_idset_num_elements(idset), 0, 0, 0); if (err) goto done; } @@ -1946,7 +1947,7 @@ got_pack_create(struct got_object_id *packhash, int pa err = progress_cb(progress_arg, ncolored, nfound, ntrees, 1 /* packfile_size */, nours, got_object_idset_num_elements(idset), - deltify.nmeta + reuse.nmeta, 0); + deltify.nmeta + reuse.nmeta, 0, 0); if (err) goto done; } blob - bcaebb44a765c02ee6f3ca914166f2c9447410e566aca4449908d00d58f7c68a blob + f1030ff9a47cd74ca33627e4e80821c3ad5b67aef58c053581ef96235454d7df --- lib/pack_create_io.c +++ lib/pack_create_io.c @@ -156,7 +156,7 @@ search_delta_for_object(struct got_object_id *id, void err = got_pack_report_progress(a->progress_cb, a->progress_arg, a->rl, a->ncolored, a->nfound, a->ntrees, 0L, a->ncommits, - got_object_idset_num_elements(a->idset), a->v->nmeta, 0); + got_object_idset_num_elements(a->idset), a->v->nmeta, 0, 0); if (err) goto done; } @@ -312,7 +312,7 @@ got_pack_paint_commits(int *ncolored, struct got_objec } err = got_pack_report_progress(progress_cb, progress_arg, rl, - *ncolored, 0, 0, 0L, 0, 0, 0, 0); + *ncolored, 0, 0, 0L, 0, 0, 0, 0, 0); if (err) break; blob - 6b9d3f0050a0ad5235bd077d16fef02267a8518965dd9f1d82e5bdb3199c5afa blob + 01b674dc63b1714a739ff40927c2dad3518eddf37221bebce8252520aee56c32 --- lib/pack_create_privsep.c +++ lib/pack_create_privsep.c @@ -190,7 +190,7 @@ got_pack_search_deltas(struct got_packidx **packidx, s err = got_pack_report_progress(progress_cb, progress_arg, rl, ncolored, nfound, ntrees, 0L, ncommits, - got_object_idset_num_elements(idset), v->nmeta, 0); + got_object_idset_num_elements(idset), v->nmeta, 0, 0); if (err) break; } @@ -263,7 +263,7 @@ recv_painted_commit(void *arg, struct got_object_id *i } return got_pack_report_progress(a->progress_cb, a->progress_arg, a->rl, - *a->ncolored, 0, 0, 0L, 0, 0, 0, 0); + *a->ncolored, 0, 0, 0L, 0, 0, 0, 0, 0); } static const struct got_error * @@ -447,7 +447,7 @@ got_pack_paint_commits(int *ncolored, struct got_objec } err = got_pack_report_progress(progress_cb, progress_arg, rl, - *ncolored, 0, 0, 0L, 0, 0, 0, 0); + *ncolored, 0, 0, 0L, 0, 0, 0, 0, 0); if (err) break; blob - af0d17d4eca8a2e0905047c95a1314728e9e6c21f741bd50753c8be07785dda5 blob + f109076b9e1535aed6d69c591fe0359f793d01dd8e999d8d9fa6ee909a225d6a --- lib/repository_admin.c +++ lib/repository_admin.c @@ -142,6 +142,79 @@ done: return err; } +static const struct got_error * +create_temp_packfile(int *packfd, char **tmpfile_path, + struct got_repository *repo) +{ + const struct got_error *err = NULL; + char *path; + + *packfd = -1; + + if (asprintf(&path, "%s/%s/packing.pack", + got_repo_get_path_git_dir(repo), GOT_OBJECTS_PACK_DIR) == -1) + return got_error_from_errno("asprintf"); + + err = got_opentemp_named_fd(tmpfile_path, packfd, path, ""); + if (err) + goto done; + + if (fchmod(*packfd, GOT_DEFAULT_PACK_MODE) == -1) + err = got_error_from_errno2("fchmod", *tmpfile_path); +done: + if (err) { + close(*packfd); + *packfd = -1; + free(*tmpfile_path); + *tmpfile_path = NULL; + } + return err; +} + +static const struct got_error * +install_packfile(FILE **packfile, int *packfd, char **packfile_path, + char **tmpfile_path, struct got_object_id *pack_hash, + struct got_repository *repo) +{ + const struct got_error *err; + char *hash_str; + + err = got_object_id_str(&hash_str, pack_hash); + if (err) + return err; + + if (asprintf(packfile_path, "%s/%s/pack-%s.pack", + got_repo_get_path_git_dir(repo), GOT_OBJECTS_PACK_DIR, + hash_str) == -1) { + err = got_error_from_errno("asprintf"); + goto done; + } + + if (lseek(*packfd, 0L, SEEK_SET) == -1) { + err = got_error_from_errno("lseek"); + goto done; + } + + if (rename(*tmpfile_path, *packfile_path) == -1) { + err = got_error_from_errno3("rename", *tmpfile_path, + *packfile_path); + goto done; + } + + free(*tmpfile_path); + *tmpfile_path = NULL; + + *packfile = fdopen(*packfd, "w"); + if (*packfile == NULL) { + err = got_error_from_errno2("fdopen", *packfile_path); + goto done; + } + *packfd = -1; +done: + free(hash_str); + return err; +} + const struct got_error * got_repo_pack_objects(FILE **packfile, struct got_object_id **pack_hash, struct got_reflist_head *include_refs, @@ -153,8 +226,7 @@ got_repo_pack_objects(FILE **packfile, struct got_obje const struct got_error *err = NULL; struct got_object_id **ours = NULL, **theirs = NULL; int nours = 0, ntheirs = 0, packfd = -1, i; - char *tmpfile_path = NULL, *path = NULL, *packfile_path = NULL; - char *hash_str = NULL; + char *tmpfile_path = NULL, *packfile_path = NULL; FILE *delta_cache = NULL; struct got_ratelimit rl; @@ -163,20 +235,10 @@ got_repo_pack_objects(FILE **packfile, struct got_obje got_ratelimit_init(&rl, 0, 500); - if (asprintf(&path, "%s/%s/packing.pack", - got_repo_get_path_git_dir(repo), GOT_OBJECTS_PACK_DIR) == -1) { - err = got_error_from_errno("asprintf"); - goto done; - } - err = got_opentemp_named_fd(&tmpfile_path, &packfd, path, ""); + err = create_temp_packfile(&packfd, &tmpfile_path, repo); if (err) - goto done; + return err; - if (fchmod(packfd, GOT_DEFAULT_PACK_MODE) == -1) { - err = got_error_from_errno2("fchmod", tmpfile_path); - goto done; - } - delta_cache = got_opentemp(); if (delta_cache == NULL) { err = got_error_from_errno("got_opentemp"); @@ -215,34 +277,8 @@ got_repo_pack_objects(FILE **packfile, struct got_obje if (err) goto done; - err = got_object_id_str(&hash_str, *pack_hash); - if (err) - goto done; - if (asprintf(&packfile_path, "%s/%s/pack-%s.pack", - got_repo_get_path_git_dir(repo), GOT_OBJECTS_PACK_DIR, - hash_str) == -1) { - err = got_error_from_errno("asprintf"); - goto done; - } - - if (lseek(packfd, 0L, SEEK_SET) == -1) { - err = got_error_from_errno("lseek"); - goto done; - } - if (rename(tmpfile_path, packfile_path) == -1) { - err = got_error_from_errno3("rename", tmpfile_path, - packfile_path); - goto done; - } - free(tmpfile_path); - tmpfile_path = NULL; - - *packfile = fdopen(packfd, "w"); - if (*packfile == NULL) { - err = got_error_from_errno2("fdopen", tmpfile_path); - goto done; - } - packfd = -1; + err = install_packfile(packfile, &packfd, &packfile_path, + &tmpfile_path, *pack_hash, repo); done: for (i = 0; i < nours; i++) free(ours[i]); @@ -258,8 +294,6 @@ done: err = got_error_from_errno2("unlink", tmpfile_path); free(tmpfile_path); free(packfile_path); - free(hash_str); - free(path); if (err) { free(*pack_hash); *pack_hash = NULL; @@ -271,8 +305,8 @@ done: } const struct got_error * -got_repo_index_pack(FILE *packfile, struct got_object_id *pack_hash, - struct got_repository *repo, +got_repo_index_pack(char **idxpath, FILE *packfile, + struct got_object_id *pack_hash, struct got_repository *repo, got_pack_index_progress_cb progress_cb, void *progress_arg, got_cancel_cb cancel_cb, void *cancel_arg) { @@ -282,14 +316,16 @@ got_repo_index_pack(FILE *packfile, struct got_object_ int npackfd = -1, idxfd = -1, nidxfd = -1; int tmpfds[3]; int idxstatus, done = 0; + int nobj_total = 0, nobj_indexed = 0, nobj_loose = 0, nobj_resolved = 0; const struct got_error *err; struct imsgbuf idxibuf; pid_t idxpid; char *tmpidxpath = NULL; - char *packfile_path = NULL, *idxpath = NULL, *id_str = NULL; + char *packfile_path = NULL, *id_str = NULL; const char *repo_path = got_repo_get_path_git_dir(repo); struct stat sb; + *idxpath = NULL; memset(&idxibuf, 0, sizeof(idxibuf)); for (i = 0; i < nitems(tmpfds); i++) @@ -338,7 +374,7 @@ got_repo_index_pack(FILE *packfile, struct got_object_ goto done; } - if (asprintf(&idxpath, "%s/%s/pack-%s.idx", + if (asprintf(idxpath, "%s/%s/pack-%s.idx", repo_path, GOT_OBJECTS_PACK_DIR, id_str) == -1) { err = got_error_from_errno("asprintf"); goto done; @@ -386,8 +422,6 @@ got_repo_index_pack(FILE *packfile, struct got_object_ } done = 0; while (!done) { - int nobj_total, nobj_indexed, nobj_loose, nobj_resolved; - if (cancel_cb) { err = cancel_cb(cancel_arg); if (err) @@ -402,11 +436,18 @@ got_repo_index_pack(FILE *packfile, struct got_object_ if (nobj_indexed != 0) { err = progress_cb(progress_arg, sb.st_size, nobj_total, nobj_indexed, nobj_loose, - nobj_resolved); + nobj_resolved, 0); if (err) break; } } + if (done) { + err = progress_cb(progress_arg, sb.st_size, + nobj_total, nobj_indexed, nobj_loose, + nobj_resolved, done); + if (err) + goto done; + } if (close(imsg_idxfds[0]) == -1) { err = got_error_from_errno("close"); goto done; @@ -416,8 +457,8 @@ got_repo_index_pack(FILE *packfile, struct got_object_ goto done; } - if (rename(tmpidxpath, idxpath) == -1) { - err = got_error_from_errno3("rename", tmpidxpath, idxpath); + if (rename(tmpidxpath, *idxpath) == -1) { + err = got_error_from_errno3("rename", tmpidxpath, *idxpath); goto done; } free(tmpidxpath); @@ -437,7 +478,6 @@ done: err = got_error_from_errno("close"); } free(tmpidxpath); - free(idxpath); free(packfile_path); return err; } @@ -1420,6 +1460,8 @@ got_repo_cleanup(struct got_repository *repo, off_t *pack_before, off_t *pack_after, int *ncommits, int *nloose, int *npacked, int dry_run, int ignore_mtime, got_cleanup_progress_cb progress_cb, void *progress_arg, + got_pack_progress_cb pack_progress_cb, void *pack_progress_arg, + got_pack_index_progress_cb index_progress_cb, void *index_progress_arg, got_cancel_cb cancel_cb, void *cancel_arg) { const struct got_error *unlock_err, *err = NULL; @@ -1430,11 +1472,15 @@ got_repo_cleanup(struct got_repository *repo, struct got_reflist_entry *re; struct got_object_id **referenced_ids; int i, nreferenced; - int npurged = 0; + int npurged = 0, packfd = -1; + char *tmpfile_path = NULL, *packfile_path = NULL, *idxpath = NULL; + FILE *delta_cache = NULL, *packfile = NULL; + struct got_object_id pack_hash; time_t max_mtime = 0; TAILQ_INIT(&refs); got_ratelimit_init(&rl, 0, 500); + memset(&pack_hash, 0, sizeof(pack_hash)); *loose_before = 0; *loose_after = 0; @@ -1448,6 +1494,16 @@ got_repo_cleanup(struct got_repository *repo, if (err) return err; + err = create_temp_packfile(&packfd, &tmpfile_path, repo); + if (err) + goto done; + + delta_cache = got_opentemp(); + if (delta_cache == NULL) { + err = got_error_from_errno("got_opentemp"); + goto done; + } + traversed_ids = got_object_idset_alloc(); if (traversed_ids == NULL) { err = got_error_from_errno("got_object_idset_alloc"); @@ -1486,6 +1542,28 @@ got_repo_cleanup(struct got_repository *repo, goto done; } + err = got_pack_create(&pack_hash, packfd, delta_cache, + NULL, 0, referenced_ids, nreferenced, repo, 0, + 0, 0, pack_progress_cb, pack_progress_arg, + &rl, cancel_cb, cancel_arg); + if (err) + goto done; + + err = install_packfile(&packfile, &packfd, &packfile_path, + &tmpfile_path, &pack_hash, repo); + if (err) + goto done; + + err = got_repo_index_pack(&idxpath, packfile, &pack_hash, repo, + index_progress_cb, index_progress_arg, + cancel_cb, cancel_arg); + if (err) + goto done; + + err = got_repo_list_packidx(&repo->packidx_paths, repo); + if (err) + goto done; + err = repo_purge_unreferenced_loose_objects(repo, traversed_ids, loose_before, loose_after, *ncommits, nloose, npacked, &npurged, dry_run, ignore_mtime, max_mtime, &rl, progress_cb, progress_arg, @@ -1500,6 +1578,12 @@ got_repo_cleanup(struct got_repository *repo, if (err) goto done; + if (dry_run) { + if (idxpath && unlink(idxpath) == -1) + err = got_error_from_errno2("unlink", idxpath); + if (packfile_path && unlink(packfile_path) == -1 && err == NULL) + err = got_error_from_errno2("unlink", packfile_path); + } done: if (lk) { unlock_err = got_lockfile_unlock(lk, got_repo_get_fd(repo)); @@ -1508,6 +1592,15 @@ got_repo_cleanup(struct got_repository *repo, } if (traversed_ids) got_object_idset_free(traversed_ids); + if (packfd != -1 && close(packfd) == -1 && err == NULL) + err = got_error_from_errno2("close", packfile_path); + if (delta_cache && fclose(delta_cache) == EOF && err == NULL) + err = got_error_from_errno("fclose"); + if (tmpfile_path && unlink(tmpfile_path) == -1 && err == NULL) + err = got_error_from_errno2("unlink", tmpfile_path); + free(tmpfile_path); + free(packfile_path); + free(idxpath); return err; } blob - 88dc9f7aa445e30b844c79453bb3b7373a560aee8413e6050ba856ce70f67919 blob + e5c7c1355d4bb1e862f6fbbb5819d9049796c2a05c01ad1b8a6d7cd550cf6a09 --- lib/send.c +++ lib/send.c @@ -125,7 +125,7 @@ struct pack_progress_arg { static const struct got_error * pack_progress(void *arg, int ncolored, int nfound, int ntrees, off_t packfile_size, int ncommits, int nobj_total, int nobj_deltify, - int nobj_written) + int nobj_written, int pack_done) { const struct got_error *err; struct pack_progress_arg *a = arg; blob - 1c7d27e960b9ab0426d0204be02f565c64a10479112ecab3a432d1d84f7ede7b blob + 57400ae20d6bdf9bc5ece96af1c23769f9db287d4ded87b5ea2dd73194f90034 --- regress/cmdline/cleanup.sh +++ regress/cmdline/cleanup.sh @@ -90,7 +90,7 @@ test_cleanup_unreferenced_loose_objects() { return 1 fi - # cleanup should remove loose objects that belonged to the branch + # cleanup should remove all loose objects gotadmin cleanup -a -q -r $testroot/repo > $testroot/stdout ret=$? if [ $ret -ne 0 ]; then @@ -109,7 +109,7 @@ test_cleanup_unreferenced_loose_objects() { nloose2=`gotadmin info -r $testroot/repo | grep '^loose objects:' | \ cut -d ':' -f 2 | tr -d ' '` - if [ "$nloose2" != "$nloose0" ]; then + if [ "$nloose2" != "0" ]; then echo "unexpected number of loose objects: $nloose2" >&2 test_done "$testroot" "1" return 1 @@ -304,8 +304,8 @@ test_cleanup_redundant_pack_files() { gotadmin cleanup -a -q -r "$testroot/repo" n=$(gotadmin info -r "$testroot/repo" | awk '/^pack files/{print $3}') - if [ "$n" -ne 3 ]; then - echo "expected 3 pack files left, $n found instead" >&2 + if [ "$n" -ne 2 ]; then + echo "expected 2 pack files left, $n found instead" >&2 test_done "$testroot" 1 return 1 fi @@ -326,8 +326,8 @@ test_cleanup_redundant_pack_files() { gotadmin cleanup -a -q -r "$testroot/repo" n=$(gotadmin info -r "$testroot/repo" | awk '/^pack files/{print $3}') - if [ "$n" -ne 3 ]; then - echo "expected 3 pack files left, $n found instead" >&2 + if [ "$n" -ne 1 ]; then + echo "expected 1 pack files left, $n found instead" >&2 test_done "$testroot" 1 return 1 fi blob - 26caf7828e2a03f9b81940f2c0510c7ed4a8468db40d1ccb3cc8afd54a1b237a blob + 6ff032a29e680fff673e6434bf588750e3d9c8fea06db508b50dc2554bf53ca4 --- regress/cmdline/common.sh +++ regress/cmdline/common.sh @@ -156,7 +156,7 @@ git_fsck() local testroot="$1" local repo="$2" - git -C $repo fsck --strict \ + git -C $repo fsck --strict --no-reflogs \ > $testroot/fsck.stdout 2> $testroot/fsck.stderr ret=$? if [ $ret -ne 0 ]; then