add option to NOT recursively clone submodules (#246)

* add option to NOT recursively clone submodules * Add "shallow" and "off" modes to submodule clone * update readme to reflect new flag * wording and such feedback
2020-07-31 00:29:07 -04:00 · 2020-07-31 00:29:07 -04:00 · dcf169c10a
parent c17c10638d
commit dcf169c10a
3 changed files with 114 additions and 13 deletions
--- a/README.md
+++ b/README.md
@ -84,6 +84,7 @@ docker run -d \
 | GIT_SYNC_BRANCH                 | `--branch`                 | the git branch to check out                                                                                                            | "master"                      |
 | GIT_SYNC_REV                    | `--rev`                    | the git revision (tag or hash) to check out                                                                                            | "HEAD"                        |
 | GIT_SYNC_DEPTH                  | `--depth`                  | use a shallow clone with a history truncated to the specified number of commits                                                        | 0                             |
+| GIT_SYNC_SUBMODULES             | `--submodules`             | submodule clone option. One of recursive, shallow (does not use --recursive flag), or off (no submodule cloning)                       | recursive                     |
 | GIT_SYNC_ROOT                   | `--root`                   | the root directory for git-sync operations, under which --dest will be created                                                         | "$HOME/git"                   |
 | GIT_SYNC_DEST                   | `--dest`                   | the name of (a symlink to) a directory in which to check-out files under --root (defaults to the leaf dir of --repo)                   | ""                            |
 | GIT_SYNC_WAIT                   | `--wait`                   | the number of seconds between syncs                                                                                                    | 0                             |
--- a/cmd/git-sync/main.go
+++ b/cmd/git-sync/main.go
@ -54,6 +54,8 @@ var flRev = flag.String("rev", envString("GIT_SYNC_REV", "HEAD"),
 	"the git revision (tag or hash) to check out")
 var flDepth = flag.Int("depth", envInt("GIT_SYNC_DEPTH", 0),
 	"use a shallow clone with a history truncated to the specified number of commits")
+var flSubmodules = flag.String("submodules", envString("GIT_SYNC_SUBMODULES", "recursive"),
+	"Configure submodule behavior - options are 'recursive', 'shallow' and 'off'.")

 var flRoot = flag.String("root", envString("GIT_SYNC_ROOT", envString("HOME", "")+"/git"),
 	"the root directory for git-sync operations, under which --dest will be created")
@ -132,6 +134,12 @@ var (
 // initTimeout is a timeout for initialization, like git credentials setup.
 const initTimeout = time.Second * 30

+const (
+	SubmoduleModeRecursive = "recursive"
+	SubmoduleModeOff       = "off"
+	SubmoduleModeShallow   = "shallow"
+)
+
 func init() {
 	prometheus.MustRegister(syncDuration)
 	prometheus.MustRegister(syncCount)
@ -250,6 +258,13 @@ func main() {
 		}
 	}

+	switch *flSubmodules {
+	case SubmoduleModeRecursive, SubmoduleModeShallow, SubmoduleModeOff:
+	default:
+		fmt.Fprintf(os.Stderr, "ERROR: --submodules must be one of %s, %s or %s, but recieved %s", SubmoduleModeRecursive, SubmoduleModeOff, SubmoduleModeShallow, *flSubmodules)
+		os.Exit(1)
+	}
+
 	// This context is used only for git credentials initialization. There are no long-running operations like
 	// `git clone`, so initTimeout set to 30 seconds should be enough.
 	ctx, cancel := context.WithTimeout(context.Background(), initTimeout)
@ -339,7 +354,7 @@ func main() {
 	for {
 		start := time.Now()
 		ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(*flSyncTimeout))
-		if changed, hash, err := syncRepo(ctx, *flRepo, *flBranch, *flRev, *flDepth, *flRoot, *flDest, *flAskPassURL); err != nil {
+		if changed, hash, err := syncRepo(ctx, *flRepo, *flBranch, *flRev, *flDepth, *flRoot, *flDest, *flAskPassURL, *flSubmodules); err != nil {
 			syncDuration.WithLabelValues("error").Observe(time.Since(start).Seconds())
 			syncCount.WithLabelValues("error").Inc()
 			if *flMaxSyncFailures != -1 && failCount >= *flMaxSyncFailures {
@ -476,7 +491,7 @@ func setRepoReady() {
 }

 // addWorktreeAndSwap creates a new worktree and calls updateSymlink to swap the symlink to point to the new worktree
-func addWorktreeAndSwap(ctx context.Context, gitRoot, dest, branch, rev string, depth int, hash string) error {
+func addWorktreeAndSwap(ctx context.Context, gitRoot, dest, branch, rev string, depth int, hash string, submoduleMode string) error {
 	log.V(0).Info("syncing git", "rev", rev, "hash", hash)

 	args := []string{"fetch", "-f", "--tags"}
@ -525,14 +540,19 @@ func addWorktreeAndSwap(ctx context.Context, gitRoot, dest, branch, rev string,

 	// Update submodules
 	// NOTE: this works for repo with or without submodules.
-	log.V(0).Info("updating submodules")
-	submodulesArgs := []string{"submodule", "update", "--init", "--recursive"}
-	if depth != 0 {
-		submodulesArgs = append(submodulesArgs, "--depth", strconv.Itoa(depth))
-	}
-	_, err = runCommand(ctx, worktreePath, *flGitCmd, submodulesArgs...)
-	if err != nil {
-		return err
+	if submoduleMode != SubmoduleModeOff {
+		log.V(0).Info("updating submodules")
+		submodulesArgs := []string{"submodule", "update", "--init"}
+		if submoduleMode == SubmoduleModeRecursive {
+			submodulesArgs = append(submodulesArgs, "--recursive")
+		}
+		if depth != 0 {
+			submodulesArgs = append(submodulesArgs, "--depth", strconv.Itoa(depth))
+		}
+		_, err = runCommand(ctx, worktreePath, *flGitCmd, submodulesArgs...)
+		if err != nil {
+			return err
+		}
 	}

 	// Change the file permissions, if requested.
@ -637,7 +657,7 @@ func revIsHash(ctx context.Context, rev, gitRoot string) (bool, error) {

 // syncRepo syncs the branch of a given repository to the destination at the given rev.
 // returns (1) whether a change occured, (2) the new hash, and (3) an error if one happened
-func syncRepo(ctx context.Context, repo, branch, rev string, depth int, gitRoot, dest string, authUrl string) (bool, string, error) {
+func syncRepo(ctx context.Context, repo, branch, rev string, depth int, gitRoot, dest string, authUrl string, submoduleMode string) (bool, string, error) {
 	if authUrl != "" {
 		// For ASKPASS Callback URL, the credentials behind is dynamic, it needs to be
 		// re-fetched each time.
@ -677,7 +697,7 @@ func syncRepo(ctx context.Context, repo, branch, rev string, depth int, gitRoot,
 		hash = remote
 	}

-	return true, hash, addWorktreeAndSwap(ctx, gitRoot, dest, branch, rev, depth, hash)
+	return true, hash, addWorktreeAndSwap(ctx, gitRoot, dest, branch, rev, depth, hash, submoduleMode)
 }

 // getRevs returns the local and upstream hashes for rev.
@ -780,7 +800,6 @@ func setupGitSSH(setupKnownHosts bool) error {
 		if err != nil {
 			return fmt.Errorf("error: could not access SSH known_hosts file: %v", err)
 		}
-
 		err = os.Setenv("GIT_SSH_COMMAND", fmt.Sprintf("ssh -q -o UserKnownHostsFile=%s -i %s", pathToSSHKnownHosts, pathToSSHSecret))
 	} else {
 		err = os.Setenv("GIT_SSH_COMMAND", fmt.Sprintf("ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s", pathToSSHSecret))
--- a/test_e2e.sh
+++ b/test_e2e.sh
@ -982,6 +982,87 @@ fi
 rm -rf $SUBMODULE
 pass

+##############################################
+# Test submodules off
+##############################################
+testcase "submodule-sync-off"
+
+# Init submodule repo
+SUBMODULE_REPO_NAME="sub"
+SUBMODULE="$DIR/$SUBMODULE_REPO_NAME"
+mkdir "$SUBMODULE"
+
+git -C "$SUBMODULE" init -q
+echo "submodule" > "$SUBMODULE"/submodule
+git -C "$SUBMODULE" add submodule
+git -C "$SUBMODULE" commit -aqm "init submodule file"
+
+# Add submodule
+git -C "$REPO" submodule add -q file://$SUBMODULE
+git -C "$REPO" commit -aqm "add submodule"
+
+GIT_SYNC \
+    --logtostderr \
+    --v=5 \
+    --submodules=off \
+    --wait=0.1 \
+    --repo="file://$REPO" \
+    --root="$ROOT" \
+    --dest="link" \
+    > "$DIR"/log."$TESTCASE" 2>&1 &
+sleep 3
+assert_file_absent "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
+rm -rf $SUBMODULE
+pass
+
+##############################################
+# Test submodules shallow
+##############################################
+testcase "submodule-sync-shallow"
+
+# Init submodule repo
+SUBMODULE_REPO_NAME="sub"
+SUBMODULE="$DIR/$SUBMODULE_REPO_NAME"
+mkdir "$SUBMODULE"
+
+git -C "$SUBMODULE" init -q
+echo "submodule" > "$SUBMODULE"/submodule
+git -C "$SUBMODULE" add submodule
+git -C "$SUBMODULE" commit -aqm "init submodule file"
+# Init nested submodule repo
+NESTED_SUBMODULE_REPO_NAME="nested-sub"
+NESTED_SUBMODULE="$DIR/$NESTED_SUBMODULE_REPO_NAME"
+mkdir "$NESTED_SUBMODULE"
+
+git -C "$NESTED_SUBMODULE" init -q
+echo "nested-submodule" > "$NESTED_SUBMODULE"/nested-submodule
+git -C "$NESTED_SUBMODULE" add nested-submodule
+git -C "$NESTED_SUBMODULE" commit -aqm "init nested-submodule file"
+git -C "$SUBMODULE" submodule add -q file://$NESTED_SUBMODULE
+git -C "$SUBMODULE" commit -aqm "add nested submodule"
+
+# Add submodule
+git -C "$REPO" submodule add -q file://$SUBMODULE
+git -C "$REPO" commit -aqm "add submodule"
+
+GIT_SYNC \
+    --logtostderr \
+    --v=5 \
+    --submodules=shallow \
+    --wait=0.1 \
+    --repo="file://$REPO" \
+    --root="$ROOT" \
+    --dest="link" \
+    > "$DIR"/log."$TESTCASE" 2>&1 &
+sleep 3
+assert_link_exists "$ROOT"/link
+assert_file_exists "$ROOT"/link/file
+assert_file_exists "$ROOT"/link/$SUBMODULE_REPO_NAME/submodule
+assert_file_absent "$ROOT"/link/$SUBMODULE_REPO_NAME/$NESTED_SUBMODULE_REPO_NAME/nested-submodule
+rm -rf $SUBMODULE
+rm -rf $NESTED_SUBMODULE
+pass
+
 ##############################################
 # Test SSH
 ##############################################