diff --git a/README.md b/README.md index a3c125f..7d59ef4 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ docker run -d \ | GIT_SYNC_BRANCH | `--branch` | the git branch to check out | "master" | | GIT_SYNC_REV | `--rev` | the git revision (tag or hash) to check out | "HEAD" | | GIT_SYNC_DEPTH | `--depth` | use a shallow clone with a history truncated to the specified number of commits | 0 | +| GIT_SYNC_SUBMODULES | `--submodules` | submodule clone option. One of recursive, shallow (does not use --recursive flag), or off (no submodule cloning) | recursive | | GIT_SYNC_ROOT | `--root` | the root directory for git-sync operations, under which --dest will be created | "$HOME/git" | | GIT_SYNC_DEST | `--dest` | the name of (a symlink to) a directory in which to check-out files under --root (defaults to the leaf dir of --repo) | "" | | GIT_SYNC_WAIT | `--wait` | the number of seconds between syncs | 0 | diff --git a/cmd/git-sync/main.go b/cmd/git-sync/main.go index 6c8648d..464c43a 100644 --- a/cmd/git-sync/main.go +++ b/cmd/git-sync/main.go @@ -54,6 +54,8 @@ var flRev = flag.String("rev", envString("GIT_SYNC_REV", "HEAD"), "the git revision (tag or hash) to check out") var flDepth = flag.Int("depth", envInt("GIT_SYNC_DEPTH", 0), "use a shallow clone with a history truncated to the specified number of commits") +var flSubmodules = flag.String("submodules", envString("GIT_SYNC_SUBMODULES", "recursive"), + "Configure submodule behavior - options are 'recursive', 'shallow' and 'off'.") var flRoot = flag.String("root", envString("GIT_SYNC_ROOT", envString("HOME", "")+"/git"), "the root directory for git-sync operations, under which --dest will be created") @@ -132,6 +134,12 @@ var ( // initTimeout is a timeout for initialization, like git credentials setup. const initTimeout = time.Second * 30 +const ( + SubmoduleModeRecursive = "recursive" + SubmoduleModeOff = "off" + SubmoduleModeShallow = "shallow" +) + func init() { prometheus.MustRegister(syncDuration) prometheus.MustRegister(syncCount) @@ -250,6 +258,13 @@ func main() { } } + switch *flSubmodules { + case SubmoduleModeRecursive, SubmoduleModeShallow, SubmoduleModeOff: + default: + fmt.Fprintf(os.Stderr, "ERROR: --submodules must be one of %s, %s or %s, but recieved %s", SubmoduleModeRecursive, SubmoduleModeOff, SubmoduleModeShallow, *flSubmodules) + os.Exit(1) + } + // This context is used only for git credentials initialization. There are no long-running operations like // `git clone`, so initTimeout set to 30 seconds should be enough. ctx, cancel := context.WithTimeout(context.Background(), initTimeout) @@ -339,7 +354,7 @@ func main() { for { start := time.Now() ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(*flSyncTimeout)) - if changed, hash, err := syncRepo(ctx, *flRepo, *flBranch, *flRev, *flDepth, *flRoot, *flDest, *flAskPassURL); err != nil { + if changed, hash, err := syncRepo(ctx, *flRepo, *flBranch, *flRev, *flDepth, *flRoot, *flDest, *flAskPassURL, *flSubmodules); err != nil { syncDuration.WithLabelValues("error").Observe(time.Since(start).Seconds()) syncCount.WithLabelValues("error").Inc() if *flMaxSyncFailures != -1 && failCount >= *flMaxSyncFailures { @@ -476,7 +491,7 @@ func setRepoReady() { } // addWorktreeAndSwap creates a new worktree and calls updateSymlink to swap the symlink to point to the new worktree -func addWorktreeAndSwap(ctx context.Context, gitRoot, dest, branch, rev string, depth int, hash string) error { +func addWorktreeAndSwap(ctx context.Context, gitRoot, dest, branch, rev string, depth int, hash string, submoduleMode string) error { log.V(0).Info("syncing git", "rev", rev, "hash", hash) args := []string{"fetch", "-f", "--tags"} @@ -525,14 +540,19 @@ func addWorktreeAndSwap(ctx context.Context, gitRoot, dest, branch, rev string, // Update submodules // NOTE: this works for repo with or without submodules. - log.V(0).Info("updating submodules") - submodulesArgs := []string{"submodule", "update", "--init", "--recursive"} - if depth != 0 { - submodulesArgs = append(submodulesArgs, "--depth", strconv.Itoa(depth)) - } - _, err = runCommand(ctx, worktreePath, *flGitCmd, submodulesArgs...) - if err != nil { - return err + if submoduleMode != SubmoduleModeOff { + log.V(0).Info("updating submodules") + submodulesArgs := []string{"submodule", "update", "--init"} + if submoduleMode == SubmoduleModeRecursive { + submodulesArgs = append(submodulesArgs, "--recursive") + } + if depth != 0 { + submodulesArgs = append(submodulesArgs, "--depth", strconv.Itoa(depth)) + } + _, err = runCommand(ctx, worktreePath, *flGitCmd, submodulesArgs...) + if err != nil { + return err + } } // Change the file permissions, if requested. @@ -637,7 +657,7 @@ func revIsHash(ctx context.Context, rev, gitRoot string) (bool, error) { // syncRepo syncs the branch of a given repository to the destination at the given rev. // returns (1) whether a change occured, (2) the new hash, and (3) an error if one happened -func syncRepo(ctx context.Context, repo, branch, rev string, depth int, gitRoot, dest string, authUrl string) (bool, string, error) { +func syncRepo(ctx context.Context, repo, branch, rev string, depth int, gitRoot, dest string, authUrl string, submoduleMode string) (bool, string, error) { if authUrl != "" { // For ASKPASS Callback URL, the credentials behind is dynamic, it needs to be // re-fetched each time. @@ -677,7 +697,7 @@ func syncRepo(ctx context.Context, repo, branch, rev string, depth int, gitRoot, hash = remote } - return true, hash, addWorktreeAndSwap(ctx, gitRoot, dest, branch, rev, depth, hash) + return true, hash, addWorktreeAndSwap(ctx, gitRoot, dest, branch, rev, depth, hash, submoduleMode) } // getRevs returns the local and upstream hashes for rev. @@ -780,7 +800,6 @@ func setupGitSSH(setupKnownHosts bool) error { if err != nil { return fmt.Errorf("error: could not access SSH known_hosts file: %v", err) } - err = os.Setenv("GIT_SSH_COMMAND", fmt.Sprintf("ssh -q -o UserKnownHostsFile=%s -i %s", pathToSSHKnownHosts, pathToSSHSecret)) } else { err = os.Setenv("GIT_SSH_COMMAND", fmt.Sprintf("ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s", pathToSSHSecret)) diff --git a/test_e2e.sh b/test_e2e.sh index 0a7db3c..28cbd08 100755 --- a/test_e2e.sh +++ b/test_e2e.sh @@ -982,6 +982,87 @@ fi rm -rf $SUBMODULE pass +############################################## +# Test submodules off +############################################## +testcase "submodule-sync-off" + +# Init submodule repo +SUBMODULE_REPO_NAME="sub" +SUBMODULE="$DIR/$SUBMODULE_REPO_NAME" +mkdir "$SUBMODULE" + +git -C "$SUBMODULE" init -q +echo "submodule" > "$SUBMODULE"/submodule +git -C "$SUBMODULE" add submodule +git -C "$SUBMODULE" commit -aqm "init submodule file" + +# Add submodule +git -C "$REPO" submodule add -q file://$SUBMODULE +git -C "$REPO" commit -aqm "add submodule" + +GIT_SYNC \ + --logtostderr \ + --v=5 \ + --submodules=off \ + --wait=0.1 \ + --repo="file://$REPO" \ + --root="$ROOT" \ + --dest="link" \ + > "$DIR"/log."$TESTCASE" 2>&1 & +sleep 3 +assert_file_absent "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule +rm -rf $SUBMODULE +pass + +############################################## +# Test submodules shallow +############################################## +testcase "submodule-sync-shallow" + +# Init submodule repo +SUBMODULE_REPO_NAME="sub" +SUBMODULE="$DIR/$SUBMODULE_REPO_NAME" +mkdir "$SUBMODULE" + +git -C "$SUBMODULE" init -q +echo "submodule" > "$SUBMODULE"/submodule +git -C "$SUBMODULE" add submodule +git -C "$SUBMODULE" commit -aqm "init submodule file" +# Init nested submodule repo +NESTED_SUBMODULE_REPO_NAME="nested-sub" +NESTED_SUBMODULE="$DIR/$NESTED_SUBMODULE_REPO_NAME" +mkdir "$NESTED_SUBMODULE" + +git -C "$NESTED_SUBMODULE" init -q +echo "nested-submodule" > "$NESTED_SUBMODULE"/nested-submodule +git -C "$NESTED_SUBMODULE" add nested-submodule +git -C "$NESTED_SUBMODULE" commit -aqm "init nested-submodule file" +git -C "$SUBMODULE" submodule add -q file://$NESTED_SUBMODULE +git -C "$SUBMODULE" commit -aqm "add nested submodule" + +# Add submodule +git -C "$REPO" submodule add -q file://$SUBMODULE +git -C "$REPO" commit -aqm "add submodule" + +GIT_SYNC \ + --logtostderr \ + --v=5 \ + --submodules=shallow \ + --wait=0.1 \ + --repo="file://$REPO" \ + --root="$ROOT" \ + --dest="link" \ + > "$DIR"/log."$TESTCASE" 2>&1 & +sleep 3 +assert_link_exists "$ROOT"/link +assert_file_exists "$ROOT"/link/file +assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule +assert_file_absent "$ROOT"/link/$SUBMODULE_REPO_NAME/$NESTED_SUBMODULE_REPO_NAME/nested-submodule +rm -rf $SUBMODULE +rm -rf $NESTED_SUBMODULE +pass + ############################################## # Test SSH ##############################################