add option to NOT recursively clone submodules (#246)

* add option to NOT recursively clone submodules

* Add "shallow" and "off" modes to submodule clone

* update readme to reflect new flag

* wording and such feedback

README.md

@@ -84,6 +84,7 @@ docker run -d \
 | GIT_SYNC_BRANCH                 | `--branch`                 | the git branch to check out                                                                                                            | "master"                      |
 | GIT_SYNC_REV                    | `--rev`                    | the git revision (tag or hash) to check out                                                                                            | "HEAD"                        |
 | GIT_SYNC_DEPTH                  | `--depth`                  | use a shallow clone with a history truncated to the specified number of commits                                                        | 0                             |
+| GIT_SYNC_SUBMODULES             | `--submodules`             | submodule clone option. One of recursive, shallow (does not use --recursive flag), or off (no submodule cloning)                       | recursive                     |
 | GIT_SYNC_ROOT                   | `--root`                   | the root directory for git-sync operations, under which --dest will be created                                                         | "$HOME/git"                   |
 | GIT_SYNC_DEST                   | `--dest`                   | the name of (a symlink to) a directory in which to check-out files under --root (defaults to the leaf dir of --repo)                   | ""                            |
 | GIT_SYNC_WAIT                   | `--wait`                   | the number of seconds between syncs                                                                                                    | 0                             |

cmd/git-sync/main.go

@@ -54,6 +54,8 @@ var flRev = flag.String("rev", envString("GIT_SYNC_REV", "HEAD"),
 	"the git revision (tag or hash) to check out")
 var flDepth = flag.Int("depth", envInt("GIT_SYNC_DEPTH", 0),
 	"use a shallow clone with a history truncated to the specified number of commits")
+var flSubmodules = flag.String("submodules", envString("GIT_SYNC_SUBMODULES", "recursive"),
+	"Configure submodule behavior - options are 'recursive', 'shallow' and 'off'.")
 
 var flRoot = flag.String("root", envString("GIT_SYNC_ROOT", envString("HOME", "")+"/git"),
 	"the root directory for git-sync operations, under which --dest will be created")
@@ -132,6 +134,12 @@ var (
 // initTimeout is a timeout for initialization, like git credentials setup.
 const initTimeout = time.Second * 30
 
+const (
+	SubmoduleModeRecursive = "recursive"
+	SubmoduleModeOff       = "off"
+	SubmoduleModeShallow   = "shallow"
+)
+
 func init() {
 	prometheus.MustRegister(syncDuration)
 	prometheus.MustRegister(syncCount)
@@ -250,6 +258,13 @@ func main() {
 		}
 	}
 
+	switch *flSubmodules {
+	case SubmoduleModeRecursive, SubmoduleModeShallow, SubmoduleModeOff:
+	default:
+		fmt.Fprintf(os.Stderr, "ERROR: --submodules must be one of %s, %s or %s, but recieved %s", SubmoduleModeRecursive, SubmoduleModeOff, SubmoduleModeShallow, *flSubmodules)
+		os.Exit(1)
+	}
+
 	// This context is used only for git credentials initialization. There are no long-running operations like
 	// `git clone`, so initTimeout set to 30 seconds should be enough.
 	ctx, cancel := context.WithTimeout(context.Background(), initTimeout)
@@ -339,7 +354,7 @@ func main() {
 	for {
 		start := time.Now()
 		ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(*flSyncTimeout))
-		if changed, hash, err := syncRepo(ctx, *flRepo, *flBranch, *flRev, *flDepth, *flRoot, *flDest, *flAskPassURL); err != nil {
+		if changed, hash, err := syncRepo(ctx, *flRepo, *flBranch, *flRev, *flDepth, *flRoot, *flDest, *flAskPassURL, *flSubmodules); err != nil {
 			syncDuration.WithLabelValues("error").Observe(time.Since(start).Seconds())
 			syncCount.WithLabelValues("error").Inc()
 			if *flMaxSyncFailures != -1 && failCount >= *flMaxSyncFailures {
@@ -476,7 +491,7 @@ func setRepoReady() {
 }
 
 // addWorktreeAndSwap creates a new worktree and calls updateSymlink to swap the symlink to point to the new worktree
-func addWorktreeAndSwap(ctx context.Context, gitRoot, dest, branch, rev string, depth int, hash string) error {
+func addWorktreeAndSwap(ctx context.Context, gitRoot, dest, branch, rev string, depth int, hash string, submoduleMode string) error {
 	log.V(0).Info("syncing git", "rev", rev, "hash", hash)
 
 	args := []string{"fetch", "-f", "--tags"}
@@ -525,8 +540,12 @@ func addWorktreeAndSwap(ctx context.Context, gitRoot, dest, branch, rev string,
 
 	// Update submodules
 	// NOTE: this works for repo with or without submodules.
+	if submoduleMode != SubmoduleModeOff {
 		log.V(0).Info("updating submodules")
-	submodulesArgs := []string{"submodule", "update", "--init", "--recursive"}
+		submodulesArgs := []string{"submodule", "update", "--init"}
+		if submoduleMode == SubmoduleModeRecursive {
+			submodulesArgs = append(submodulesArgs, "--recursive")
+		}
 		if depth != 0 {
 			submodulesArgs = append(submodulesArgs, "--depth", strconv.Itoa(depth))
 		}
@@ -534,6 +553,7 @@ func addWorktreeAndSwap(ctx context.Context, gitRoot, dest, branch, rev string,
 		if err != nil {
 			return err
 		}
+	}
 
 	// Change the file permissions, if requested.
 	if *flChmod != 0 {
@@ -637,7 +657,7 @@ func revIsHash(ctx context.Context, rev, gitRoot string) (bool, error) {
 
 // syncRepo syncs the branch of a given repository to the destination at the given rev.
 // returns (1) whether a change occured, (2) the new hash, and (3) an error if one happened
-func syncRepo(ctx context.Context, repo, branch, rev string, depth int, gitRoot, dest string, authUrl string) (bool, string, error) {
+func syncRepo(ctx context.Context, repo, branch, rev string, depth int, gitRoot, dest string, authUrl string, submoduleMode string) (bool, string, error) {
 	if authUrl != "" {
 		// For ASKPASS Callback URL, the credentials behind is dynamic, it needs to be
 		// re-fetched each time.
@@ -677,7 +697,7 @@ func syncRepo(ctx context.Context, repo, branch, rev string, depth int, gitRoot,
 		hash = remote
 	}
 
-	return true, hash, addWorktreeAndSwap(ctx, gitRoot, dest, branch, rev, depth, hash)
+	return true, hash, addWorktreeAndSwap(ctx, gitRoot, dest, branch, rev, depth, hash, submoduleMode)
 }
 
 // getRevs returns the local and upstream hashes for rev.
@@ -780,7 +800,6 @@ func setupGitSSH(setupKnownHosts bool) error {
 		if err != nil {
 			return fmt.Errorf("error: could not access SSH known_hosts file: %v", err)
 		}
-
 		err = os.Setenv("GIT_SSH_COMMAND", fmt.Sprintf("ssh -q -o UserKnownHostsFile=%s -i %s", pathToSSHKnownHosts, pathToSSHSecret))
 	} else {
 		err = os.Setenv("GIT_SSH_COMMAND", fmt.Sprintf("ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s", pathToSSHSecret))

test_e2e.sh

@@ -982,6 +982,87 @@ fi
 rm -rf $SUBMODULE
 pass
 
+##############################################
+# Test submodules off
+##############################################
+testcase "submodule-sync-off"
+
+# Init submodule repo
+SUBMODULE_REPO_NAME="sub"
+SUBMODULE="$DIR/$SUBMODULE_REPO_NAME"
+mkdir "$SUBMODULE"
+
+git -C "$SUBMODULE" init -q
+echo "submodule" > "$SUBMODULE"/submodule
+git -C "$SUBMODULE" add submodule
+git -C "$SUBMODULE" commit -aqm "init submodule file"
+
+# Add submodule
+git -C "$REPO" submodule add -q file://$SUBMODULE
+git -C "$REPO" commit -aqm "add submodule"
+
+GIT_SYNC \
+    --logtostderr \
+    --v=5 \
+    --submodules=off \
+    --wait=0.1 \
+    --repo="file://$REPO" \
+    --root="$ROOT" \
+    --dest="link" \
+    > "$DIR"/log."$TESTCASE" 2>&1 &
+sleep 3
+assert_file_absent "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
+rm -rf $SUBMODULE
+pass
+
+##############################################
+# Test submodules shallow
+##############################################
+testcase "submodule-sync-shallow"
+
+# Init submodule repo
+SUBMODULE_REPO_NAME="sub"
+SUBMODULE="$DIR/$SUBMODULE_REPO_NAME"
+mkdir "$SUBMODULE"
+
+git -C "$SUBMODULE" init -q
+echo "submodule" > "$SUBMODULE"/submodule
+git -C "$SUBMODULE" add submodule
+git -C "$SUBMODULE" commit -aqm "init submodule file"
+# Init nested submodule repo
+NESTED_SUBMODULE_REPO_NAME="nested-sub"
+NESTED_SUBMODULE="$DIR/$NESTED_SUBMODULE_REPO_NAME"
+mkdir "$NESTED_SUBMODULE"
+
+git -C "$NESTED_SUBMODULE" init -q
+echo "nested-submodule" > "$NESTED_SUBMODULE"/nested-submodule
+git -C "$NESTED_SUBMODULE" add nested-submodule
+git -C "$NESTED_SUBMODULE" commit -aqm "init nested-submodule file"
+git -C "$SUBMODULE" submodule add -q file://$NESTED_SUBMODULE
+git -C "$SUBMODULE" commit -aqm "add nested submodule"
+
+# Add submodule
+git -C "$REPO" submodule add -q file://$SUBMODULE
+git -C "$REPO" commit -aqm "add submodule"
+
+GIT_SYNC \
+    --logtostderr \
+    --v=5 \
+    --submodules=shallow \
+    --wait=0.1 \
+    --repo="file://$REPO" \
+    --root="$ROOT" \
+    --dest="link" \
+    > "$DIR"/log."$TESTCASE" 2>&1 &
+sleep 3
+assert_link_exists "$ROOT"/link
+assert_file_exists "$ROOT"/link/file
+assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
+assert_file_absent "$ROOT"/link/$SUBMODULE_REPO_NAME/$NESTED_SUBMODULE_REPO_NAME/nested-submodule
+rm -rf $SUBMODULE
+rm -rf $NESTED_SUBMODULE
+pass
+
 ##############################################
 # Test SSH
 ##############################################