Add support for Git submodules with go-git

Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>

api/v1beta1/gitrepository_types.go

@@ -81,6 +81,12 @@ type GitRepositorySpec struct {
 	// +kubebuilder:default:=go-git
 	// +optional
 	GitImplementation string `json:"gitImplementation,omitempty"`
+
+	// When enabled, after the clone is created, initializes all submodules within,
+	// using their default settings.
+	// This option is available only when using the 'go-git' GitImplementation.
+	// +optional
+	RecurseSubmodules bool `json:"recurseSubmodules,omitempty"`
 }
 
 // GitRepositoryRef defines the Git ref used for pull and checkout operations.

config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml

@@ -66,6 +66,11 @@ spec:
               interval:
                 description: The interval at which to check for repository updates.
                 type: string
+              recurseSubmodules:
+                description: When enabled, after the clone is created, initializes
+                  all submodules within, using their default settings. This option
+                  is available only when using the 'go-git' GitImplementation.
+                type: boolean
               ref:
                 description: The Git reference to checkout and monitor for changes,
                   defaults to master branch.

controllers/gitrepository_controller.go

@@ -183,7 +183,12 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, repository sour
 	// determine auth method
 	auth := &git.Auth{}
 	if repository.Spec.SecretRef != nil {
-		authStrategy, err := strategy.AuthSecretStrategyForURL(repository.Spec.URL, repository.Spec.GitImplementation)
+		authStrategy, err := strategy.AuthSecretStrategyForURL(
+			repository.Spec.URL,
+			git.CheckoutOptions{
+				GitImplementation: repository.Spec.GitImplementation,
+				RecurseSubmodules: repository.Spec.RecurseSubmodules,
+			})
 		if err != nil {
 			return sourcev1.GitRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err
 		}
@@ -207,7 +212,12 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, repository sour
 		}
 	}
 
-	checkoutStrategy, err := strategy.CheckoutStrategyForRef(repository.Spec.Reference, repository.Spec.GitImplementation)
+	checkoutStrategy, err := strategy.CheckoutStrategyForRef(
+		repository.Spec.Reference,
+		git.CheckoutOptions{
+			GitImplementation: repository.Spec.GitImplementation,
+			RecurseSubmodules: repository.Spec.RecurseSubmodules,
+		})
 	if err != nil {
 		return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
 	}

docs/api/source.md

@@ -400,6 +400,20 @@ string
 Defaults to go-git, valid values are (&lsquo;go-git&rsquo;, &lsquo;libgit2&rsquo;).</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>recurseSubmodules</code><br>
+<em>
+bool
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>When enabled, after the clone is created, initializes all submodules within,
+using their default settings.
+This option is available only when using the &lsquo;go-git&rsquo; GitImplementation.</p>
+</td>
+</tr>
 </table>
 </td>
 </tr>
@@ -1246,6 +1260,20 @@ string
 Defaults to go-git, valid values are (&lsquo;go-git&rsquo;, &lsquo;libgit2&rsquo;).</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>recurseSubmodules</code><br>
+<em>
+bool
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>When enabled, after the clone is created, initializes all submodules within,
+using their default settings.
+This option is available only when using the &lsquo;go-git&rsquo; GitImplementation.</p>
+</td>
+</tr>
 </tbody>
 </table>
 </div>

docs/spec/v1beta1/gitrepositories.md

@@ -57,6 +57,11 @@ type GitRepositorySpec struct {
 	// +kubebuilder:default:=go-git
 	// +optional
 	GitImplementation string `json:"gitImplementation,omitempty"`
+	
+	// When enabled, after the clone is created, initializes all submodules within.
+	// This option is available only when using the 'go-git' GitImplementation.
+	// +optional
+	RecurseSubmodules bool `json:"recurseSubmodules,omitempty"`
 }
 ```
 
@@ -434,6 +439,42 @@ kubectl create secret generic pgp-public-keys \
     --from-file=author2.asc
 ```
 
+### Git submodules
+
+With `spec.recurseSubmodules` you can configure the controller to
+clone a specific branch including its Git submodules:
+
+```yaml
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: GitRepository
+metadata:
+  name: repo-with-submodules
+  namespace: default
+spec:
+  interval: 1m
+  url: https://github.com/<organization>/<repository>
+  secretRef:
+    name: https-credentials
+  ref:
+    branch: main
+  recurseSubmodules: true
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: https-credentials
+  namespace: default
+type: Opaque
+data:
+  username: <GitHub Username>
+  password: <GitHub Token>
+```
+
+Note that deploy keys can't be used to pull submodules from private repositories
+as GitHub and GitLab doesn't allow a deploy key to be reused across repositories.
+You have to use either HTTPS token-based authentication, or an SSH key belonging
+to a user that has access to the main repository and all its submodules.
+
 ## Status examples
 
 Successful sync:

pkg/git/git.go

@@ -40,6 +40,11 @@ type CheckoutStrategy interface {
 	Checkout(ctx context.Context, path, url string, auth *Auth) (Commit, string, error)
 }
 
+type CheckoutOptions struct {
+	GitImplementation string
+	RecurseSubmodules bool
+}
+
 // TODO(hidde): candidate for refactoring, so that we do not directly
 //  depend on implementation specifics here.
 type Auth struct {

pkg/git/gogit/checkout.go

@@ -33,29 +33,30 @@ import (
 	"github.com/fluxcd/source-controller/pkg/git"
 )
 
-func CheckoutStrategyForRef(ref *sourcev1.GitRepositoryRef) git.CheckoutStrategy {
+func CheckoutStrategyForRef(ref *sourcev1.GitRepositoryRef, opt git.CheckoutOptions) git.CheckoutStrategy {
 	switch {
 	case ref == nil:
 		return &CheckoutBranch{branch: git.DefaultBranch}
 	case ref.SemVer != "":
-		return &CheckoutSemVer{semVer: ref.SemVer}
+		return &CheckoutSemVer{semVer: ref.SemVer, recurseSubmodules: opt.RecurseSubmodules}
 	case ref.Tag != "":
-		return &CheckoutTag{tag: ref.Tag}
+		return &CheckoutTag{tag: ref.Tag, recurseSubmodules: opt.RecurseSubmodules}
 	case ref.Commit != "":
-		strategy := &CheckoutCommit{branch: ref.Branch, commit: ref.Commit}
+		strategy := &CheckoutCommit{branch: ref.Branch, commit: ref.Commit, recurseSubmodules: opt.RecurseSubmodules}
 		if strategy.branch == "" {
 			strategy.branch = git.DefaultBranch
 		}
 		return strategy
 	case ref.Branch != "":
-		return &CheckoutBranch{branch: ref.Branch}
+		return &CheckoutBranch{branch: ref.Branch, recurseSubmodules: opt.RecurseSubmodules}
 	default:
 		return &CheckoutBranch{branch: git.DefaultBranch}
 	}
 }
 
 type CheckoutBranch struct {
-	branch string
+	branch            string
+	recurseSubmodules bool
 }
 
 func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, auth *git.Auth) (git.Commit, string, error) {
@@ -67,7 +68,7 @@ func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, auth *g
 		SingleBranch:      true,
 		NoCheckout:        false,
 		Depth:             1,
-		RecurseSubmodules: extgogit.DefaultSubmoduleRecursionDepth,
+		RecurseSubmodules: recurseSubmodules(c.recurseSubmodules),
 		Progress:          nil,
 		Tags:              extgogit.NoTags,
 		CABundle:          auth.CABundle,
@@ -87,7 +88,8 @@ func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, auth *g
 }
 
 type CheckoutTag struct {
-	tag string
+	tag               string
+	recurseSubmodules bool
 }
 
 func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, auth *git.Auth) (git.Commit, string, error) {
@@ -99,7 +101,7 @@ func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, auth *git.
 		SingleBranch:      true,
 		NoCheckout:        false,
 		Depth:             1,
-		RecurseSubmodules: extgogit.DefaultSubmoduleRecursionDepth,
+		RecurseSubmodules: recurseSubmodules(c.recurseSubmodules),
 		Progress:          nil,
 		Tags:              extgogit.NoTags,
 		CABundle:          auth.CABundle,
@@ -119,8 +121,9 @@ func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, auth *git.
 }
 
 type CheckoutCommit struct {
-	branch string
+	branch            string
-	commit string
+	commit            string
+	recurseSubmodules bool
 }
 
 func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, auth *git.Auth) (git.Commit, string, error) {
@@ -131,7 +134,7 @@ func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, auth *g
 		ReferenceName:     plumbing.NewBranchReferenceName(c.branch),
 		SingleBranch:      true,
 		NoCheckout:        false,
-		RecurseSubmodules: extgogit.DefaultSubmoduleRecursionDepth,
+		RecurseSubmodules: recurseSubmodules(c.recurseSubmodules),
 		Progress:          nil,
 		Tags:              extgogit.NoTags,
 		CABundle:          auth.CABundle,
@@ -158,7 +161,8 @@ func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, auth *g
 }
 
 type CheckoutSemVer struct {
-	semVer string
+	semVer            string
+	recurseSubmodules bool
 }
 
 func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, auth *git.Auth) (git.Commit, string, error) {
@@ -173,7 +177,7 @@ func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, auth *g
 		RemoteName:        git.DefaultOrigin,
 		NoCheckout:        false,
 		Depth:             1,
-		RecurseSubmodules: extgogit.DefaultSubmoduleRecursionDepth,
+		RecurseSubmodules: recurseSubmodules(c.recurseSubmodules),
 		Progress:          nil,
 		Tags:              extgogit.AllTags,
 		CABundle:          auth.CABundle,
@@ -262,3 +266,10 @@ func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, auth *g
 
 	return &Commit{commit}, fmt.Sprintf("%s/%s", t, head.Hash().String()), nil
 }
+
+func recurseSubmodules(recurse bool) extgogit.SubmoduleRescursivity {
+	if recurse {
+		return extgogit.DefaultSubmoduleRecursionDepth
+	}
+	return extgogit.NoRecurseSubmodules
+}

pkg/git/libgit2/checkout.go

@@ -29,7 +29,7 @@ import (
 	"github.com/fluxcd/source-controller/pkg/git"
 )
 
-func CheckoutStrategyForRef(ref *sourcev1.GitRepositoryRef) git.CheckoutStrategy {
+func CheckoutStrategyForRef(ref *sourcev1.GitRepositoryRef, opt git.CheckoutOptions) git.CheckoutStrategy {
 	switch {
 	case ref == nil:
 		return &CheckoutBranch{branch: git.DefaultBranch}

pkg/git/strategy/strategy.go

@@ -25,24 +25,24 @@ import (
 	"github.com/fluxcd/source-controller/pkg/git/libgit2"
 )
 
-func CheckoutStrategyForRef(ref *sourcev1.GitRepositoryRef, gitImplementation string) (git.CheckoutStrategy, error) {
+func CheckoutStrategyForRef(ref *sourcev1.GitRepositoryRef, opt git.CheckoutOptions) (git.CheckoutStrategy, error) {
-	switch gitImplementation {
+	switch opt.GitImplementation {
 	case sourcev1.GoGitImplementation:
-		return gogit.CheckoutStrategyForRef(ref), nil
+		return gogit.CheckoutStrategyForRef(ref, opt), nil
 	case sourcev1.LibGit2Implementation:
-		return libgit2.CheckoutStrategyForRef(ref), nil
+		return libgit2.CheckoutStrategyForRef(ref, opt), nil
 	default:
-		return nil, fmt.Errorf("invalid git implementation %s", gitImplementation)
+		return nil, fmt.Errorf("invalid Git implementation %s", opt.GitImplementation)
 	}
 }
 
-func AuthSecretStrategyForURL(url string, gitImplementation string) (git.AuthSecretStrategy, error) {
+func AuthSecretStrategyForURL(url string, opt git.CheckoutOptions) (git.AuthSecretStrategy, error) {
-	switch gitImplementation {
+	switch opt.GitImplementation {
 	case sourcev1.GoGitImplementation:
 		return gogit.AuthSecretStrategyForURL(url)
 	case sourcev1.LibGit2Implementation:
 		return libgit2.AuthSecretStrategyForURL(url)
 	default:
-		return nil, fmt.Errorf("invalid git implementation %s", gitImplementation)
+		return nil, fmt.Errorf("invalid Git implementation %s", opt.GitImplementation)
 	}
 }