gc: add `--expire-to` option

From: ZheNing Hu <[email protected]>

This commit extends the functionality of git gc
by adding a new option, --expire-to=<dir>. Previously, this feature was implemented in git repack (see 91badeb), allowing users to specify a directory where unreachable and expired cruft packs are stored during garbage collection. However, users had to run git repack --cruft --expire-to=<dir> followed by git prune to achieve similar results within git gc.

By introducing --expire-to=<dir> directly into git gc, we simplify the process for users who wish to manage their repository's cleanup more efficiently. This change involves passing the --expire-to=<dir> parameter through to git repack, making it easier for users to set up a backup location for cruft packs that will be pruned.

Signed-off-by ZheNing Hu <[email protected]>

---
 Documentation/git-gc.txt | 6 ++++++
 builtin/gc.c             | 6 +++++-
 t/t6500-gc.sh            | 6 ++++++
 3 files changed, 17 insertions(+), 1 deletion(-)

Documentation/git-gc.txt +6 -0

index 370e22faaeb..b4c0cf02972 100644
@@ -69,6 +69,12 @@ be performed as well.
        the `--max-cruft-size` option of linkgit:git-repack[1] for
        more.
 
+--expire-to=<dir>::
+       When packing unreachable objects into a cruft pack, write a cruft
+       pack containing pruned objects (if any) to the directory `<dir>`.
+       See the `--expire-to` option of linkgit:git-repack[1] for
+       more.
+
 --prune=<date>::
        Prune loose objects older than date (default is 2 weeks ago,
        overridable by the config variable `gc.pruneExpire`).

builtin/gc.c +5 -1

index d52735354c9..77904694c9f 100644
@@ -136,6 +136,7 @@ struct gc_config {
        char *prune_worktrees_expire;
        char *repack_filter;
        char *repack_filter_to;
+       char *repack_expire_to;
        unsigned long big_pack_threshold;
        unsigned long max_delta_cache_size;
 };

@@ -441,6 +442,8 @@ static void add_repack_all_option(struct gc_config *cfg,
                if (cfg->max_cruft_size)
                        strvec_pushf(&repack, "--max-cruft-size=%lu",
                                     cfg->max_cruft_size);
+               if (cfg->repack_expire_to)
+                       strvec_pushf(&repack, "--expire-to=%s", cfg->repack_expire_to);
        } else {
                strvec_push(&repack, "-A");
                if (cfg->prune_expire)

@@ -675,7 +678,6 @@ struct repository *repo UNUSED)
        const char *prune_expire_sentinel = "sentinel";
        const char *prune_expire_arg = prune_expire_sentinel;
        int ret;
-
        struct option builtin_gc_options[] = {
                OPT__QUIET(&quiet, N_("suppress progress reporting")),
                { OPTION_STRING, 0, "prune", &prune_expire_arg, N_("date"),

@@ -694,6 +696,8 @@ struct repository *repo UNUSED)
                           PARSE_OPT_NOCOMPLETE),
                OPT_BOOL(0, "keep-largest-pack", &keep_largest_pack,
                         N_("repack all other packs except the largest pack")),
+               OPT_STRING(0, "expire-to", &cfg.repack_expire_to, N_("dir"),
+                          N_("pack prefix to store a pack containing pruned objects")),
                OPT_END()
        };
 

t/t6500-gc.sh +6 -1

index ee074b99b70..d4b0653a9b7 100755
@@ -339,6 +339,12 @@ test_expect_success 'gc.maxCruftSize sets appropriate repack options' '
        test_subcommand $cruft_max_size_opts --max-cruft-size=3145728 <trace2.txt
 '
 
+test_expect_success '--expire-to sets appropriate repack options' '
+       mkdir expired &&
+       GIT_TRACE2_EVENT=$(pwd)/trace2.txt git -C cruft--max-size gc --cruft --expire-to=./expired/pack &&
+       test_subcommand $cruft_max_size_opts --expire-to=./expired/pack <trace2.txt
+'
+
 run_and_wait_for_gc () {
        # We read stdout from gc for the side effect of waiting until the
        # background gc process exits, closing its fd 9.  Furthermore, the
--

gitgitgadget

From: ZheNing Hu <[email protected]>

This commit extends the functionality of git gc
by adding a new option, --expire-to=<dir>. Previously, this feature was implemented in git repack (see 91badeb), allowing users to specify a directory where unreachable and expired cruft packs are stored during garbage collection. However, users had to run git repack --cruft --expire-to=<dir> followed by git prune to achieve similar results within git gc.

By introducing --expire-to=<dir> directly into git gc, we simplify the process for users who wish to manage their repository's cleanup more efficiently. This change involves passing the --expire-to=<dir> parameter through to git repack, making it easier for users to set up a backup location for cruft packs that will be pruned.

Signed-off-by ZheNing Hu <[email protected]>

---
 Documentation/git-gc.txt | 6 ++++++
 builtin/gc.c             | 6 +++++-
 t/t6500-gc.sh            | 6 ++++++
 3 files changed, 17 insertions(+), 1 deletion(-)

Documentation/git-gc.txt +6 -0

index 370e22faaeb..b4c0cf02972 100644
@@ -69,6 +69,12 @@ be performed as well.
        the `--max-cruft-size` option of linkgit:git-repack[1] for
        more.
 
+--expire-to=<dir>::
+       When packing unreachable objects into a cruft pack, write a cruft
+       pack containing pruned objects (if any) to the directory `<dir>`.
+       See the `--expire-to` option of linkgit:git-repack[1] for
+       more.
+
 --prune=<date>::
        Prune loose objects older than date (default is 2 weeks ago,
        overridable by the config variable `gc.pruneExpire`).

builtin/gc.c +5 -1

index d52735354c9..77904694c9f 100644
@@ -136,6 +136,7 @@ struct gc_config {
        char *prune_worktrees_expire;
        char *repack_filter;
        char *repack_filter_to;
+       char *repack_expire_to;
        unsigned long big_pack_threshold;
        unsigned long max_delta_cache_size;
 };

@@ -441,6 +442,8 @@ static void add_repack_all_option(struct gc_config *cfg,
                if (cfg->max_cruft_size)
                        strvec_pushf(&repack, "--max-cruft-size=%lu",
                                     cfg->max_cruft_size);
+               if (cfg->repack_expire_to)
+                       strvec_pushf(&repack, "--expire-to=%s", cfg->repack_expire_to);
        } else {
                strvec_push(&repack, "-A");
                if (cfg->prune_expire)

@@ -675,7 +678,6 @@ struct repository *repo UNUSED)
        const char *prune_expire_sentinel = "sentinel";
        const char *prune_expire_arg = prune_expire_sentinel;
        int ret;
-
        struct option builtin_gc_options[] = {
                OPT__QUIET(&quiet, N_("suppress progress reporting")),
                { OPTION_STRING, 0, "prune", &prune_expire_arg, N_("date"),

@@ -694,6 +696,8 @@ struct repository *repo UNUSED)
                           PARSE_OPT_NOCOMPLETE),
                OPT_BOOL(0, "keep-largest-pack", &keep_largest_pack,
                         N_("repack all other packs except the largest pack")),
+               OPT_STRING(0, "expire-to", &cfg.repack_expire_to, N_("dir"),
+                          N_("pack prefix to store a pack containing pruned objects")),
                OPT_END()
        };
 

t/t6500-gc.sh +6 -1

index ee074b99b70..d4b0653a9b7 100755
@@ -339,6 +339,12 @@ test_expect_success 'gc.maxCruftSize sets appropriate repack options' '
        test_subcommand $cruft_max_size_opts --max-cruft-size=3145728 <trace2.txt
 '
 
+test_expect_success '--expire-to sets appropriate repack options' '
+       mkdir expired &&
+       GIT_TRACE2_EVENT=$(pwd)/trace2.txt git -C cruft--max-size gc --cruft --expire-to=./expired/pack &&
+       test_subcommand $cruft_max_size_opts --expire-to=./expired/pack <trace2.txt
+'
+
 run_and_wait_for_gc () {
        # We read stdout from gc for the side effect of waiting until the
        # background gc process exits, closing its fd 9.  Furthermore, the
--

gitgitgadget

builtin/gc.c static int keep_one_pack(struct string_list_item *item, void *data UNUSED) · line 438

Quoted 12 lines · ZheNing Hu via GitGitGadget

builtin/gc.c +2 -1

index 77904694c9f..8656e1caff0 100644
@@ -433,7 +433,8 @@ static int keep_one_pack(struct string_list_item *item, void *data UNUSED)
 static void add_repack_all_option(struct gc_config *cfg,
                                struct string_list *keep_pack)
 {
-     if (cfg->prune_expire && !strcmp(cfg->prune_expire, "now"))
+     if (cfg->prune_expire && !strcmp(cfg->prune_expire, "now")
+             && !(cfg->cruft_packs && cfg->repack_expire_to))
              strvec_push(&repack, "-a");

I expected to see a mention of repack_expire_to here, but not cfg->cruft_packs. These two are AND-ed together so we are only disabling "repack -a" when both options ("--expire-to" and "--cruft") are passed. Can we --expire-to without cruft? I.e., what should happen with:

  git gc --expire-to=some-path --prune=now --no-cruft

Looking at the underlying git-repack, it seems that we only respect --expire-to at all when used with "--cruft", and don't otherwise consider it. Which is what the manpage says ("Only useful with --cruft -d").

But if we look at this proposed patch for example:

https://lore.kernel.org/git/48438876fb42a889110e100a6c42ca84e93aac49.1733011259.git.me@ttaylorr.com/

then it is expanding how --expire-to is used during the pruning step. OTOH, I think the way your patch 1 is structured means that we'd always pass --expire-to to git-repack anyway, and I _think_ even with the patch linked above that "repack -a -d --expire-to=whatever" would do the right thing.

In which case the problem really is the combination of cruft packs and expire-to. Just cruft packs by themselves do not need to override using "-a" for "--prune=now" because we know that any such cruft pack would be empty.

So I think this logic is correct. Taylor might have more thoughts, though (and ideas on whether he intends to revisit that earlier patch).

I do think this change should probably be done as part of patch 1, rather than introducing a buggy state and then fixing it in patch 2.

-Peff

From: ZheNing Hu <[email protected]>

This commit extends the functionality of git gc
by adding a new option, --expire-to=<dir>. Previously, this feature was implemented in git repack (see 91badeb), allowing users to specify a directory where unreachable and expired cruft packs are stored during garbage collection. However, users had to run git repack --cruft --expire-to=<dir> followed by git prune to achieve similar results within git gc.

By introducing --expire-to=<dir> directly into git gc, we simplify the process for users who wish to manage their repository's cleanup more efficiently. This change involves passing the --expire-to=<dir> parameter through to git repack, making it easier for users to set up a backup location for cruft packs that will be pruned.

Note: When git-gc is used with both --cruft and --expire-to, it does not pass -a to git-repack to delete all unreachable objects as git gc --prune=now originally did. Instead, it generates a cruft pack in the directory specified by expire-to.

builtin/gc.c +1 -0

index d52735354c9..8656e1caff0 100644
@@ -136,6 +136,7 @@ struct gc_config {
 	char *prune_worktrees_expire;
 	char *repack_filter;
 	char *repack_filter_to;
+	char *repack_expire_to;
 	unsigned long big_pack_threshold;
 	unsigned long max_delta_cache_size;
 };

Quoted 10 lines · ZheNing Hu via GitGitGadget

From: ZheNing Hu <[email protected]>

This commit extends the functionality of git gc
by adding a new option, --expire-to=<dir>. Previously, this feature was implemented in git repack (see 91badeb), allowing users to specify a directory where unreachable and expired cruft packs are stored during garbage collection. However, users had to run git repack --cruft --expire-to=<dir> followed by git prune to achieve similar results within git gc.

Quoted 6 lines

By introducing --expire-to=<dir> directly into git gc, we simplify the process for users who wish to manage their repository's cleanup more efficiently. This change involves passing the --expire-to=<dir> parameter through to git repack, making it easier for users to set up a backup location for cruft packs that will be pruned.

Today I do not have enough time to do my usual commit log message critique. Please use "git show -s --format=reference" when referring to an earlier commit.

Quoted 4 lines

Note: When git-gc is used with both --cruft and --expire-to, it does not pass -a to git-repack to delete all unreachable objects as git gc --prune=now originally did. Instead, it generates a cruft pack in the directory specified by expire-to.

Is this less important than "we added --expire-to to gc that is passed down to underlying repack" in the previous paragraph?

Not removing the unreachables too early with "repack -a" is an essential part of the design of this new feature to allow us not to lose the cruft objects, so I was a bit surprised that this was described as a "Note:".

Quoted 8 lines

Documentation/git-gc.txt +0 -0

index 370e22faaeb..b4c0cf02972 100644
@@ -69,6 +69,12 @@ be performed as well.
      the `--max-cruft-size` option of linkgit:git-repack[1] for
      more.
 

Documentation/git-gc.txt be performed as well. · line 76

Quoted 5 lines

Documentation/git-gc.txt +5 -0

+--expire-to=<dir>::
+     When packing unreachable objects into a cruft pack, write a cruft
+     pack containing pruned objects (if any) to the directory `<dir>`.
+     See the `--expire-to` option of linkgit:git-repack[1] for
+     more.

Does "When packing unreachable objects into a cruft pack" mean that this option is only meaningful with "--cruft"? As "--cruft" is on by default, is it an error to pass "--no-cruft" when you use this option?

"for more" -> "for more information" or something?

Quoted 13 lines

builtin/gc.c +1 -0

index d52735354c9..8656e1caff0 100644
@@ -136,6 +136,7 @@ struct gc_config {
      char *prune_worktrees_expire;
      char *repack_filter;
      char *repack_filter_to;
+     char *repack_expire_to;
      unsigned long big_pack_threshold;
      unsigned long max_delta_cache_size;
 };

builtin/gc.c static int keep_one_pack(struct string_list_item *item, void *data UNUSED) · line 438

Quoted 7 lines

builtin/gc.c +2 -1

@@ -432,7 +433,8 @@ static int keep_one_pack(struct string_list_item *item, void *data UNUSED)
 static void add_repack_all_option(struct gc_config *cfg,
                                struct string_list *keep_pack)
 {
-     if (cfg->prune_expire && !strcmp(cfg->prune_expire, "now"))
+     if (cfg->prune_expire && !strcmp(cfg->prune_expire, "now")
+             && !(cfg->cruft_packs && cfg->repack_expire_to))
              strvec_push(&repack, "-a");

Hmph. When "--expire-to=<there>" is given, we are dropping these unreachable objects right away, but we said "--no-cruft", then we say "repack -a". If we have both "--cruft" and "--expire-to=<there>", then ...

builtin/gc.c static void add_repack_all_option(struct gc_config *cfg, · line 447

Quoted 8 lines

builtin/gc.c +2 -0

      else if (cfg->cruft_packs) {
              strvec_push(&repack, "--cruft");

@@ -441,6 +443,8 @@ static void add_repack_all_option(struct gc_config *cfg,
              if (cfg->max_cruft_size)
                      strvec_pushf(&repack, "--max-cruft-size=%lu",
                                   cfg->max_cruft_size);
+             if (cfg->repack_expire_to)
+                     strvec_pushf(&repack, "--expire-to=%s", cfg->repack_expire_to);

... we do the usual "repack --cruft --expire-to=<there>" in the next block.

builtin/gc.c struct repository *repo UNUSED) · line 684

Quoted 8 lines

builtin/gc.c +0 -1

@@ -675,7 +679,6 @@ struct repository *repo UNUSED)
      const char *prune_expire_sentinel = "sentinel";
      const char *prune_expire_arg = prune_expire_sentinel;
      int ret;
-
      struct option builtin_gc_options[] = {
              OPT__QUIET(&quiet, N_("suppress progress reporting")),
              { OPTION_STRING, 0, "prune", &prune_expire_arg, N_("date"),

OK.

builtin/gc.c struct repository *repo UNUSED) · line 703

Quoted 8 lines

builtin/gc.c +2 -0

@@ -694,6 +697,8 @@ struct repository *repo UNUSED)
                         PARSE_OPT_NOCOMPLETE),
              OPT_BOOL(0, "keep-largest-pack", &keep_largest_pack,
                       N_("repack all other packs except the largest pack")),
+             OPT_STRING(0, "expire-to", &cfg.repack_expire_to, N_("dir"),
+                        N_("pack prefix to store a pack containing pruned objects")),
              OPT_END()
      };

OK.

t/t6500-gc.sh test_expect_success 'gc.maxCruftSize sets appropriate repack options' ' · line 346

Quoted 13 lines

t/t6500-gc.sh +5 -0

index ee074b99b70..d4b0653a9b7 100755
@@ -339,6 +339,12 @@ test_expect_success 'gc.maxCruftSize sets appropriate repack options' '
      test_subcommand $cruft_max_size_opts --max-cruft-size=3145728 <trace2.txt
 '
 
+test_expect_success '--expire-to sets appropriate repack options' '
+     mkdir expired &&
+     GIT_TRACE2_EVENT=$(pwd)/trace2.txt git -C cruft--max-size gc --cruft --expire-to=./expired/pack &&
+     test_subcommand $cruft_max_size_opts --expire-to=./expired/pack <trace2.txt
+'

As "--cruft" is on by default, the command line does not have to have it, but being explicit is good.

Should we also see what happens when "--no-cruft" is given?

Thanks.

From: ZheNing Hu <[email protected]>

This commit extends the functionality of git gc
by adding a new option, --expire-to=<dir>. Previously, this feature was implemented in 91badeba32 (builtin/repack.c:
implement --expire-to for storing pruned objects, 2022-10-24), which allowing users to specify a directory where unreachable and expired cruft packs are stored during garbage collection. However, users had to run git repack --cruft --expire-to=<dir> followed by git prune to achieve similar results within git gc.

By introducing --expire-to=<dir> directly into git gc, we simplify the process for users who wish to manage their repository's cleanup more efficiently. This change involves passing the --expire-to=<dir> parameter through to git repack, making it easier for users to set up a backup location for cruft packs that will be pruned.

Due to the original git gc --prune=now deleting all unreachable objects by passing the -a parameter to git repack. With the addition of the --cruft and --expire-to options, it is necessary to modify this default behavior: instead of deleting these unreachable objects, they should be merged into a cruft pack and collected in a specified directory. Therefore, we do not pass -a to the repack command but instead pass --cruft, --expire-to, and --cruft-expiration=now to repack.

Signed-off-by ZheNing Hu <[email protected]>

---

 Documentation/git-gc.txt |  7 +++++++
 builtin/gc.c             |  9 +++++++--
 t/t6500-gc.sh            | 33 +++++++++++++++++++++++++++++++++
 3 files changed, 47 insertions(+), 2 deletions(-)

Documentation/git-gc.txt +7 -0

index 370e22faaeb..0eac8e85f08 100644
@@ -69,6 +69,13 @@ be performed as well.
 	the `--max-cruft-size` option of linkgit:git-repack[1] for
 	more.
 
+--expire-to=<dir>::
+	When packing unreachable objects into a cruft pack, write a cruft
+	pack containing pruned objects (if any) to the directory `<dir>`.
+	This option only has an effect when used together with `--cruft`.
+	See the `--expire-to` option of linkgit:git-repack[1] for
+	more information.
+
 --prune=<date>::
 	Prune loose objects older than date (default is 2 weeks ago,
 	overridable by the config variable `gc.pruneExpire`).

builtin/gc.c +7 -2

index d52735354c9..8656e1caff0 100644
@@ -136,6 +136,7 @@ struct gc_config {
 	char *prune_worktrees_expire;
 	char *repack_filter;
 	char *repack_filter_to;
+	char *repack_expire_to;
 	unsigned long big_pack_threshold;
 	unsigned long max_delta_cache_size;
 };

@@ -432,7 +433,8 @@ static int keep_one_pack(struct string_list_item *item, void *data UNUSED)
 static void add_repack_all_option(struct gc_config *cfg,
 				  struct string_list *keep_pack)
 {
-	if (cfg->prune_expire && !strcmp(cfg->prune_expire, "now"))
+	if (cfg->prune_expire && !strcmp(cfg->prune_expire, "now")
+		&& !(cfg->cruft_packs && cfg->repack_expire_to))
 		strvec_push(&repack, "-a");
 	else if (cfg->cruft_packs) {
 		strvec_push(&repack, "--cruft");

@@ -441,6 +443,8 @@ static void add_repack_all_option(struct gc_config *cfg,
 		if (cfg->max_cruft_size)
 			strvec_pushf(&repack, "--max-cruft-size=%lu",
 				     cfg->max_cruft_size);
+		if (cfg->repack_expire_to)
+			strvec_pushf(&repack, "--expire-to=%s", cfg->repack_expire_to);
 	} else {
 		strvec_push(&repack, "-A");
 		if (cfg->prune_expire)

@@ -675,7 +679,6 @@ struct repository *repo UNUSED)
 	const char *prune_expire_sentinel = "sentinel";
 	const char *prune_expire_arg = prune_expire_sentinel;
 	int ret;
-
 	struct option builtin_gc_options[] = {
 		OPT__QUIET(&quiet, N_("suppress progress reporting")),
 		{ OPTION_STRING, 0, "prune", &prune_expire_arg, N_("date"),

@@ -694,6 +697,8 @@ struct repository *repo UNUSED)
 			   PARSE_OPT_NOCOMPLETE),
 		OPT_BOOL(0, "keep-largest-pack", &keep_largest_pack,
 			 N_("repack all other packs except the largest pack")),
+		OPT_STRING(0, "expire-to", &cfg.repack_expire_to, N_("dir"),
+			   N_("pack prefix to store a pack containing pruned objects")),
 		OPT_END()
 	};
 

t/t6500-gc.sh +33 -0

index ee074b99b70..74f7bd09046 100755
@@ -339,6 +339,39 @@ test_expect_success 'gc.maxCruftSize sets appropriate repack options' '
 	test_subcommand $cruft_max_size_opts --max-cruft-size=3145728 <trace2.txt
 '
 
+test_expect_success '--expire-to sets repack --expire-to' '
+	rm -rf expired &&
+	mkdir expired &&
+	expire_to="$(pwd)/expired/pack" &&
+	GIT_TRACE2_EVENT=$(pwd)/trace2.txt git -C cruft--max-size gc --cruft --expire-to="$expire_to" &&
+	test_subcommand $cruft_max_size_opts --expire-to="$expire_to" <trace2.txt
+'
+
+test_expect_success '--expire-to with --prune=now sets repack --expire-to' '
+	rm -rf expired &&
+	mkdir expired &&
+	expire_to="$(pwd)/expired/pack" &&
+	GIT_TRACE2_EVENT=$(pwd)/trace2.txt git -C cruft--max-size gc --cruft --prune=now --expire-to="$expire_to" &&
+	test_subcommand git repack -d -l --cruft --cruft-expiration=now --expire-to="$expire_to" <trace2.txt
+'
+
+
+test_expect_success '--expire-to with --no-cruft sets repack -A' '
+	rm -rf expired &&
+	mkdir expired &&
+	expire_to="$(pwd)/expired/pack" &&
+	GIT_TRACE2_EVENT=$(pwd)/trace2.txt git -C cruft--max-size gc --no-cruft --expire-to="$expire_to" &&
+	test_subcommand git repack -d -l -A --unpack-unreachable=2.weeks.ago <trace2.txt
+'
+
+test_expect_success '--expire-to with --no-cruft sets repack -a' '
+	rm -rf expired &&
+	mkdir expired &&
+	expire_to="$(pwd)/expired/pack" &&
+	GIT_TRACE2_EVENT=$(pwd)/trace2.txt git -C cruft--max-size gc --no-cruft --prune=now --expire-to="$expire_to" &&
+	test_subcommand git repack -d -l -a <trace2.txt
+'
+
 run_and_wait_for_gc () {
 	# We read stdout from gc for the side effect of waiting until the
 	# background gc process exits, closing its fd 9.  Furthermore, the

base-commit: 92999a42db1c5f43f330e4f2bca4026b5b81576f