From ad3955c0fa44b31c5e4bd59dfbf948c30fb09f54 Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Sat, 19 Feb 2022 16:38:16 -0800 Subject: [PATCH] Add --git-gc flag to control GC on each sync Values: * "auto" - run `git gc --auto` (default, respects git gc.* configs) * "always" - run `git gc` * "aggressive" - run `git gc --aggressive` (may require a longer timeout) * "off" - do not run `git gc` on each sync (good for --one-time use) --- cmd/git-sync/main.go | 60 ++++++++++++++++++++++++++----- test_e2e.sh | 84 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 9 deletions(-) diff --git a/cmd/git-sync/main.go b/cmd/git-sync/main.go index 56283dc..12211b9 100644 --- a/cmd/git-sync/main.go +++ b/cmd/git-sync/main.go @@ -126,6 +126,8 @@ var flGitCmd = flag.String("git", envString("GIT_SYNC_GIT", "git"), "the git command to run (subject to PATH search, mostly for testing)") var flGitConfig = flag.String("git-config", envString("GIT_SYNC_GIT_CONFIG", ""), "additional git config options in 'key1:val1,key2:val2' format") +var flGitGC = flag.String("git-gc", envString("GIT_SYNC_GIT_GC", "auto"), + "git garbage collection behavior: one of 'auto', 'always', 'aggressive', or 'off'") var flHTTPBind = flag.String("http-bind", envString("GIT_SYNC_HTTP_BIND", ""), "the bind address (including port) for git-sync's HTTP endpoint") @@ -169,6 +171,11 @@ const ( submodulesRecursive = "recursive" submodulesShallow = "shallow" submodulesOff = "off" + + gcAuto = "auto" + gcAlways = "always" + gcAggressive = "aggressive" + gcOff = "off" ) func init() { @@ -278,6 +285,12 @@ func main() { handleError(true, "ERROR: --submodules must be one of %q, %q, or %q", submodulesRecursive, submodulesShallow, submodulesOff) } + switch *flGitGC { + case gcAuto, gcAlways, gcAggressive, gcOff: + default: + handleError(true, "ERROR: --git-gc must be one of %q, %q, %q, or %q", gcAuto, gcAlways, gcAggressive, gcOff) + } + if *flRoot == "" { handleError(true, "ERROR: --root must be specified") } @@ -761,11 +774,6 @@ func addWorktreeAndSwap(ctx context.Context, gitRoot, dest, branch, rev string, return err } - // Run GC if needed. - if _, err := cmdRunner.Run(ctx, gitRoot, nil, *flGitCmd, "gc", "--auto"); err != nil { - return err - } - // The .git file in the worktree directory holds a reference to // /git/.git/worktrees/. Replace it with a reference // using relative paths, so that other containers can use a different volume @@ -859,19 +867,53 @@ func addWorktreeAndSwap(ctx context.Context, gitRoot, dest, branch, rev string, setRepoReady() // From here on we have to save errors until the end. + var cleanupErrs multiError // Clean up previous worktree(s). - var cleanupErr error if oldWorktree != "" { - cleanupErr = cleanupWorkTree(ctx, gitRoot, oldWorktree) + if err := cleanupWorkTree(ctx, gitRoot, oldWorktree); err != nil { + cleanupErrs = append(cleanupErrs, err) + } } - if cleanupErr != nil { - return cleanupErr + // Run GC if needed. + if *flGitGC != gcOff { + args := []string{"gc"} + switch *flGitGC { + case gcAuto: + args = append(args, "--auto") + case gcAlways: + // no extra flags + case gcAggressive: + args = append(args, "--aggressive") + } + if _, err := cmdRunner.Run(ctx, gitRoot, nil, *flGitCmd, args...); err != nil { + cleanupErrs = append(cleanupErrs, err) + } + } + + if len(cleanupErrs) > 0 { + return cleanupErrs } return nil } +type multiError []error + +func (m multiError) Error() string { + if len(m) == 0 { + return "" + } + if len(m) == 1 { + return m[0].Error() + } + strs := make([]string, 0, len(m)) + for _, e := range m { + strs = append(strs, e.Error()) + } + return strings.Join(strs, "; ") +} + func cloneRepo(ctx context.Context, repo, branch, rev string, depth int, gitRoot string) error { args := []string{"clone", "-v", "--no-checkout", "-b", branch} if depth != 0 { diff --git a/test_e2e.sh b/test_e2e.sh index d656da4..ac93739 100755 --- a/test_e2e.sh +++ b/test_e2e.sh @@ -1729,6 +1729,90 @@ function e2e::github_https() { assert_file_exists "$ROOT"/link/LICENSE } +############################################## +# Test git-gc=auto +############################################## +function e2e::gc_auto() { + echo "$FUNCNAME" > "$REPO"/file + git -C "$REPO" commit -qam "$FUNCNAME" + + GIT_SYNC \ + --one-time \ + --repo="file://$REPO" \ + --branch="$MAIN_BRANCH" \ + --rev=HEAD \ + --root="$ROOT" \ + --dest="link" \ + --git-gc="auto" \ + >> "$1" 2>&1 + assert_link_exists "$ROOT"/link + assert_file_exists "$ROOT"/link/file + assert_file_eq "$ROOT"/link/file "$FUNCNAME" +} + +############################################## +# Test git-gc=always +############################################## +function e2e::gc_always() { + echo "$FUNCNAME" > "$REPO"/file + git -C "$REPO" commit -qam "$FUNCNAME" + + GIT_SYNC \ + --one-time \ + --repo="file://$REPO" \ + --branch="$MAIN_BRANCH" \ + --rev=HEAD \ + --root="$ROOT" \ + --dest="link" \ + --git-gc="always" \ + >> "$1" 2>&1 + assert_link_exists "$ROOT"/link + assert_file_exists "$ROOT"/link/file + assert_file_eq "$ROOT"/link/file "$FUNCNAME" +} + +############################################## +# Test git-gc=aggressive +############################################## +function e2e::gc_aggressive() { + echo "$FUNCNAME" > "$REPO"/file + git -C "$REPO" commit -qam "$FUNCNAME" + + GIT_SYNC \ + --one-time \ + --repo="file://$REPO" \ + --branch="$MAIN_BRANCH" \ + --rev=HEAD \ + --root="$ROOT" \ + --dest="link" \ + --git-gc="aggressive" \ + >> "$1" 2>&1 + assert_link_exists "$ROOT"/link + assert_file_exists "$ROOT"/link/file + assert_file_eq "$ROOT"/link/file "$FUNCNAME" +} + +############################################## +# Test git-gc=off +############################################## +function e2e::gc_off() { + echo "$FUNCNAME" > "$REPO"/file + git -C "$REPO" commit -qam "$FUNCNAME" + + GIT_SYNC \ + --one-time \ + --repo="file://$REPO" \ + --branch="$MAIN_BRANCH" \ + --rev=HEAD \ + --root="$ROOT" \ + --dest="link" \ + --git-gc="off" \ + >> "$1" 2>&1 + assert_link_exists "$ROOT"/link + assert_file_exists "$ROOT"/link/file + assert_file_eq "$ROOT"/link/file "$FUNCNAME" +} + function list_tests() { ( shopt -s extdebug