Merge pull request #327 from fluxcd/fetch-submodules

Add support for Git submodules with go-git

api/v1beta1/gitrepository_types.go

@@ -81,6 +81,12 @@ type GitRepositorySpec struct {
 	// +kubebuilder:default:=go-git
 	// +optional
 	GitImplementation string `json:"gitImplementation,omitempty"`
+
+	// When enabled, after the clone is created, initializes all submodules within,
+	// using their default settings.
+	// This option is available only when using the 'go-git' GitImplementation.
+	// +optional
+	RecurseSubmodules bool `json:"recurseSubmodules,omitempty"`
 }
 
 // GitRepositoryRef defines the Git ref used for pull and checkout operations.

config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml

@@ -66,6 +66,11 @@ spec:
               interval:
                 description: The interval at which to check for repository updates.
                 type: string
+              recurseSubmodules:
+                description: When enabled, after the clone is created, initializes
+                  all submodules within, using their default settings. This option
+                  is available only when using the 'go-git' GitImplementation.
+                type: boolean
               ref:
                 description: The Git reference to checkout and monitor for changes,
                   defaults to master branch.

controllers/gitrepository_controller.go

@@ -183,7 +183,12 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, repository sour
 	// determine auth method
 	auth := &git.Auth{}
 	if repository.Spec.SecretRef != nil {
-		authStrategy, err := strategy.AuthSecretStrategyForURL(repository.Spec.URL, repository.Spec.GitImplementation)
+		authStrategy, err := strategy.AuthSecretStrategyForURL(
+			repository.Spec.URL,
+			git.CheckoutOptions{
+				GitImplementation: repository.Spec.GitImplementation,
+				RecurseSubmodules: repository.Spec.RecurseSubmodules,
+			})
 		if err != nil {
 			return sourcev1.GitRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err
 		}
@@ -207,7 +212,12 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, repository sour
 		}
 	}
 
-	checkoutStrategy, err := strategy.CheckoutStrategyForRef(repository.Spec.Reference, repository.Spec.GitImplementation)
+	checkoutStrategy, err := strategy.CheckoutStrategyForRef(
+		repository.Spec.Reference,
+		git.CheckoutOptions{
+			GitImplementation: repository.Spec.GitImplementation,
+			RecurseSubmodules: repository.Spec.RecurseSubmodules,
+		})
 	if err != nil {
 		return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
 	}

controllers/gitrepository_controller_test.go

@@ -20,10 +20,13 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
+	"os/exec"
 	"path"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -42,9 +45,10 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 
-	"github.com/fluxcd/pkg/gittestserver"
-
 	"github.com/fluxcd/pkg/apis/meta"
+	"github.com/fluxcd/pkg/gittestserver"
+	"github.com/fluxcd/pkg/untar"
+
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
 )
 
@@ -136,8 +140,6 @@ var _ = Describe("GitRepositoryReconciler", func() {
 			}})
 			Expect(err).NotTo(HaveOccurred())
 
-			gitrepo.Worktree()
-
 			for _, ref := range t.createRefs {
 				hRef := plumbing.NewHashReference(plumbing.ReferenceName(ref), commit)
 				err = gitrepo.Storer.SetReference(hRef)
@@ -410,5 +412,137 @@ var _ = Describe("GitRepositoryReconciler", func() {
 				gitImplementation: sourcev1.GoGitImplementation,
 			}),
 		)
+
+		Context("recurse submodules", func() {
+			It("downloads submodules when asked", func() {
+				Expect(gitServer.StartHTTP()).To(Succeed())
+				defer gitServer.StopHTTP()
+
+				u, err := url.Parse(gitServer.HTTPAddress())
+				Expect(err).NotTo(HaveOccurred())
+
+				subRepoURL := *u
+				subRepoURL.Path = path.Join(u.Path, fmt.Sprintf("subrepository-%s.git", randStringRunes(5)))
+
+				// create the git repo to use as a submodule
+				fs := memfs.New()
+				subRepo, err := git.Init(memory.NewStorage(), fs)
+				Expect(err).NotTo(HaveOccurred())
+
+				wt, err := subRepo.Worktree()
+				Expect(err).NotTo(HaveOccurred())
+
+				ff, _ := fs.Create("fixture")
+				_ = ff.Close()
+				_, err = wt.Add(fs.Join("fixture"))
+				Expect(err).NotTo(HaveOccurred())
+
+				_, err = wt.Commit("Sample", &git.CommitOptions{Author: &object.Signature{
+					Name:  "John Doe",
+					Email: "john@example.com",
+					When:  time.Now(),
+				}})
+				Expect(err).NotTo(HaveOccurred())
+
+				remote, err := subRepo.CreateRemote(&config.RemoteConfig{
+					Name: "origin",
+					URLs: []string{subRepoURL.String()},
+				})
+				Expect(err).NotTo(HaveOccurred())
+
+				err = remote.Push(&git.PushOptions{
+					RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"},
+				})
+				Expect(err).NotTo(HaveOccurred())
+
+				// this one is linked to a real directory, so that I can
+				// exec `git submodule add` later
+				tmp, err := ioutil.TempDir("", "flux-test")
+				Expect(err).NotTo(HaveOccurred())
+				defer os.RemoveAll(tmp)
+
+				repoDir := filepath.Join(tmp, "git")
+				repo, err := git.PlainInit(repoDir, false)
+				Expect(err).NotTo(HaveOccurred())
+
+				wt, err = repo.Worktree()
+				Expect(err).NotTo(HaveOccurred())
+				_, err = wt.Commit("Initial revision", &git.CommitOptions{
+					Author: &object.Signature{
+						Name:  "John Doe",
+						Email: "john@example.com",
+						When:  time.Now(),
+					}})
+				Expect(err).NotTo(HaveOccurred())
+
+				submodAdd := exec.Command("git", "submodule", "add", "-b", "master", subRepoURL.String(), "sub")
+				submodAdd.Dir = repoDir
+				out, err := submodAdd.CombinedOutput()
+				os.Stdout.Write(out)
+				Expect(err).NotTo(HaveOccurred())
+
+				_, err = wt.Commit("Add submodule", &git.CommitOptions{
+					Author: &object.Signature{
+						Name:  "John Doe",
+						Email: "john@example.com",
+						When:  time.Now(),
+					}})
+				Expect(err).NotTo(HaveOccurred())
+
+				mainRepoURL := *u
+				mainRepoURL.Path = path.Join(u.Path, fmt.Sprintf("repository-%s.git", randStringRunes(5)))
+				remote, err = repo.CreateRemote(&config.RemoteConfig{
+					Name: "origin",
+					URLs: []string{mainRepoURL.String()},
+				})
+				Expect(err).NotTo(HaveOccurred())
+
+				err = remote.Push(&git.PushOptions{
+					RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"},
+				})
+				Expect(err).NotTo(HaveOccurred())
+
+				key := types.NamespacedName{
+					Name:      fmt.Sprintf("git-ref-test-%s", randStringRunes(5)),
+					Namespace: namespace.Name,
+				}
+				created := &sourcev1.GitRepository{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      key.Name,
+						Namespace: key.Namespace,
+					},
+					Spec: sourcev1.GitRepositorySpec{
+						URL:               mainRepoURL.String(),
+						Interval:          metav1.Duration{Duration: indexInterval},
+						Reference:         &sourcev1.GitRepositoryRef{Branch: "master"},
+						GitImplementation: sourcev1.GoGitImplementation, // only works with go-git
+						RecurseSubmodules: true,
+					},
+				}
+				Expect(k8sClient.Create(context.Background(), created)).Should(Succeed())
+				defer k8sClient.Delete(context.Background(), created)
+
+				got := &sourcev1.GitRepository{}
+				Eventually(func() bool {
+					_ = k8sClient.Get(context.Background(), key, got)
+					for _, c := range got.Status.Conditions {
+						if c.Reason == sourcev1.GitOperationSucceedReason {
+							return true
+						}
+					}
+					return false
+				}, timeout, interval).Should(BeTrue())
+
+				// check that the downloaded artifact includes the
+				// file from the submodule
+				res, err := http.Get(got.Status.URL)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(res.StatusCode).To(Equal(http.StatusOK))
+
+				_, err = untar.Untar(res.Body, filepath.Join(tmp, "tar"))
+				Expect(err).NotTo(HaveOccurred())
+				Expect(filepath.Join(tmp, "tar", "sub", "fixture")).To(BeAnExistingFile())
+			})
+		})
 	})
 })

controllers/suite_test.go

@@ -19,6 +19,7 @@ package controllers
 import (
 	"io/ioutil"
 	"math/rand"
+	"net/http"
 	"os"
 	"path/filepath"
 	"testing"
@@ -99,8 +100,12 @@ var _ = BeforeSuite(func(done Done) {
 	tmpStoragePath, err := ioutil.TempDir("", "source-controller-storage-")
 	Expect(err).NotTo(HaveOccurred(), "failed to create tmp storage dir")
 
-	storage, err = NewStorage(tmpStoragePath, "localhost", time.Second*30)
+	storage, err = NewStorage(tmpStoragePath, "localhost:5050", time.Second*30)
 	Expect(err).NotTo(HaveOccurred(), "failed to create tmp storage")
+	// serve artifacts from the filesystem, as done in main.go
+	fs := http.FileServer(http.Dir(tmpStoragePath))
+	http.Handle("/", fs)
+	go http.ListenAndServe(":5050", nil)
 
 	k8sManager, err = ctrl.NewManager(cfg, ctrl.Options{
 		Scheme: scheme.Scheme,

docs/api/source.md

@@ -400,6 +400,20 @@ string
 Defaults to go-git, valid values are (&lsquo;go-git&rsquo;, &lsquo;libgit2&rsquo;).</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>recurseSubmodules</code><br>
+<em>
+bool
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>When enabled, after the clone is created, initializes all submodules within,
+using their default settings.
+This option is available only when using the &lsquo;go-git&rsquo; GitImplementation.</p>
+</td>
+</tr>
 </table>
 </td>
 </tr>
@@ -1246,6 +1260,20 @@ string
 Defaults to go-git, valid values are (&lsquo;go-git&rsquo;, &lsquo;libgit2&rsquo;).</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>recurseSubmodules</code><br>
+<em>
+bool
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>When enabled, after the clone is created, initializes all submodules within,
+using their default settings.
+This option is available only when using the &lsquo;go-git&rsquo; GitImplementation.</p>
+</td>
+</tr>
 </tbody>
 </table>
 </div>

docs/spec/v1beta1/gitrepositories.md

@@ -57,6 +57,11 @@ type GitRepositorySpec struct {
 	// +kubebuilder:default:=go-git
 	// +optional
 	GitImplementation string `json:"gitImplementation,omitempty"`
+	
+	// When enabled, after the clone is created, initializes all submodules within.
+	// This option is available only when using the 'go-git' GitImplementation.
+	// +optional
+	RecurseSubmodules bool `json:"recurseSubmodules,omitempty"`
 }
 ```
 
@@ -434,6 +439,42 @@ kubectl create secret generic pgp-public-keys \
     --from-file=author2.asc
 ```
 
+### Git submodules
+
+With `spec.recurseSubmodules` you can configure the controller to
+clone a specific branch including its Git submodules:
+
+```yaml
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: GitRepository
+metadata:
+  name: repo-with-submodules
+  namespace: default
+spec:
+  interval: 1m
+  url: https://github.com/<organization>/<repository>
+  secretRef:
+    name: https-credentials
+  ref:
+    branch: main
+  recurseSubmodules: true
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: https-credentials
+  namespace: default
+type: Opaque
+data:
+  username: <GitHub Username>
+  password: <GitHub Token>
+```
+
+Note that deploy keys can't be used to pull submodules from private repositories
+as GitHub and GitLab doesn't allow a deploy key to be reused across repositories.
+You have to use either HTTPS token-based authentication, or an SSH key belonging
+to a user that has access to the main repository and all its submodules.
+
 ## Status examples
 
 Successful sync:

pkg/git/git.go

@@ -40,6 +40,11 @@ type CheckoutStrategy interface {
 	Checkout(ctx context.Context, path, url string, auth *Auth) (Commit, string, error)
 }
 
+type CheckoutOptions struct {
+	GitImplementation string
+	RecurseSubmodules bool
+}
+
 // TODO(hidde): candidate for refactoring, so that we do not directly
 //  depend on implementation specifics here.
 type Auth struct {

pkg/git/gogit/checkout.go

@@ -33,29 +33,30 @@ import (
 	"github.com/fluxcd/source-controller/pkg/git"
 )
 
-func CheckoutStrategyForRef(ref *sourcev1.GitRepositoryRef) git.CheckoutStrategy {
+func CheckoutStrategyForRef(ref *sourcev1.GitRepositoryRef, opt git.CheckoutOptions) git.CheckoutStrategy {
 	switch {
 	case ref == nil:
 		return &CheckoutBranch{branch: git.DefaultBranch}
 	case ref.SemVer != "":
-		return &CheckoutSemVer{semVer: ref.SemVer}
+		return &CheckoutSemVer{semVer: ref.SemVer, recurseSubmodules: opt.RecurseSubmodules}
 	case ref.Tag != "":
-		return &CheckoutTag{tag: ref.Tag}
+		return &CheckoutTag{tag: ref.Tag, recurseSubmodules: opt.RecurseSubmodules}
 	case ref.Commit != "":
-		strategy := &CheckoutCommit{branch: ref.Branch, commit: ref.Commit}
+		strategy := &CheckoutCommit{branch: ref.Branch, commit: ref.Commit, recurseSubmodules: opt.RecurseSubmodules}
 		if strategy.branch == "" {
 			strategy.branch = git.DefaultBranch
 		}
 		return strategy
 	case ref.Branch != "":
-		return &CheckoutBranch{branch: ref.Branch}
+		return &CheckoutBranch{branch: ref.Branch, recurseSubmodules: opt.RecurseSubmodules}
 	default:
 		return &CheckoutBranch{branch: git.DefaultBranch}
 	}
 }
 
 type CheckoutBranch struct {
-	branch string
+	branch            string
+	recurseSubmodules bool
 }
 
 func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, auth *git.Auth) (git.Commit, string, error) {
@@ -67,7 +68,7 @@ func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, auth *g
 		SingleBranch:      true,
 		NoCheckout:        false,
 		Depth:             1,
-		RecurseSubmodules: extgogit.DefaultSubmoduleRecursionDepth,
+		RecurseSubmodules: recurseSubmodules(c.recurseSubmodules),
 		Progress:          nil,
 		Tags:              extgogit.NoTags,
 		CABundle:          auth.CABundle,
@@ -87,7 +88,8 @@ func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, auth *g
 }
 
 type CheckoutTag struct {
-	tag string
+	tag               string
+	recurseSubmodules bool
 }
 
 func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, auth *git.Auth) (git.Commit, string, error) {
@@ -99,7 +101,7 @@ func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, auth *git.
 		SingleBranch:      true,
 		NoCheckout:        false,
 		Depth:             1,
-		RecurseSubmodules: extgogit.DefaultSubmoduleRecursionDepth,
+		RecurseSubmodules: recurseSubmodules(c.recurseSubmodules),
 		Progress:          nil,
 		Tags:              extgogit.NoTags,
 		CABundle:          auth.CABundle,
@@ -119,8 +121,9 @@ func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, auth *git.
 }
 
 type CheckoutCommit struct {
-	branch string
+	branch            string
-	commit string
+	commit            string
+	recurseSubmodules bool
 }
 
 func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, auth *git.Auth) (git.Commit, string, error) {
@@ -131,7 +134,7 @@ func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, auth *g
 		ReferenceName:     plumbing.NewBranchReferenceName(c.branch),
 		SingleBranch:      true,
 		NoCheckout:        false,
-		RecurseSubmodules: extgogit.DefaultSubmoduleRecursionDepth,
+		RecurseSubmodules: recurseSubmodules(c.recurseSubmodules),
 		Progress:          nil,
 		Tags:              extgogit.NoTags,
 		CABundle:          auth.CABundle,
@@ -158,7 +161,8 @@ func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, auth *g
 }
 
 type CheckoutSemVer struct {
-	semVer string
+	semVer            string
+	recurseSubmodules bool
 }
 
 func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, auth *git.Auth) (git.Commit, string, error) {
@@ -173,7 +177,7 @@ func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, auth *g
 		RemoteName:        git.DefaultOrigin,
 		NoCheckout:        false,
 		Depth:             1,
-		RecurseSubmodules: extgogit.DefaultSubmoduleRecursionDepth,
+		RecurseSubmodules: recurseSubmodules(c.recurseSubmodules),
 		Progress:          nil,
 		Tags:              extgogit.AllTags,
 		CABundle:          auth.CABundle,
@@ -262,3 +266,10 @@ func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, auth *g
 
 	return &Commit{commit}, fmt.Sprintf("%s/%s", t, head.Hash().String()), nil
 }
+
+func recurseSubmodules(recurse bool) extgogit.SubmoduleRescursivity {
+	if recurse {
+		return extgogit.DefaultSubmoduleRecursionDepth
+	}
+	return extgogit.NoRecurseSubmodules
+}

pkg/git/libgit2/checkout.go

@@ -29,7 +29,7 @@ import (
 	"github.com/fluxcd/source-controller/pkg/git"
 )
 
-func CheckoutStrategyForRef(ref *sourcev1.GitRepositoryRef) git.CheckoutStrategy {
+func CheckoutStrategyForRef(ref *sourcev1.GitRepositoryRef, opt git.CheckoutOptions) git.CheckoutStrategy {
 	switch {
 	case ref == nil:
 		return &CheckoutBranch{branch: git.DefaultBranch}

pkg/git/strategy/strategy.go

@@ -25,24 +25,24 @@ import (
 	"github.com/fluxcd/source-controller/pkg/git/libgit2"
 )
 
-func CheckoutStrategyForRef(ref *sourcev1.GitRepositoryRef, gitImplementation string) (git.CheckoutStrategy, error) {
+func CheckoutStrategyForRef(ref *sourcev1.GitRepositoryRef, opt git.CheckoutOptions) (git.CheckoutStrategy, error) {
-	switch gitImplementation {
+	switch opt.GitImplementation {
 	case sourcev1.GoGitImplementation:
-		return gogit.CheckoutStrategyForRef(ref), nil
+		return gogit.CheckoutStrategyForRef(ref, opt), nil
 	case sourcev1.LibGit2Implementation:
-		return libgit2.CheckoutStrategyForRef(ref), nil
+		return libgit2.CheckoutStrategyForRef(ref, opt), nil
 	default:
-		return nil, fmt.Errorf("invalid git implementation %s", gitImplementation)
+		return nil, fmt.Errorf("invalid Git implementation %s", opt.GitImplementation)
 	}
 }
 
-func AuthSecretStrategyForURL(url string, gitImplementation string) (git.AuthSecretStrategy, error) {
+func AuthSecretStrategyForURL(url string, opt git.CheckoutOptions) (git.AuthSecretStrategy, error) {
-	switch gitImplementation {
+	switch opt.GitImplementation {
 	case sourcev1.GoGitImplementation:
 		return gogit.AuthSecretStrategyForURL(url)
 	case sourcev1.LibGit2Implementation:
 		return libgit2.AuthSecretStrategyForURL(url)
 	default:
-		return nil, fmt.Errorf("invalid git implementation %s", gitImplementation)
+		return nil, fmt.Errorf("invalid Git implementation %s", opt.GitImplementation)
 	}
 }