From 664a5688222460a16fe7a19de8936ed546e9b2bc Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Mon, 29 Mar 2021 19:58:19 +0300 Subject: [PATCH 1/2] Add support for Git submodules with go-git Signed-off-by: Stefan Prodan --- api/v1beta1/gitrepository_types.go | 6 +++ ...rce.toolkit.fluxcd.io_gitrepositories.yaml | 5 +++ controllers/gitrepository_controller.go | 14 ++++++- docs/api/source.md | 28 +++++++++++++ docs/spec/v1beta1/gitrepositories.md | 41 +++++++++++++++++++ pkg/git/git.go | 5 +++ pkg/git/gogit/checkout.go | 39 +++++++++++------- pkg/git/libgit2/checkout.go | 2 +- pkg/git/strategy/strategy.go | 16 ++++---- 9 files changed, 131 insertions(+), 25 deletions(-) diff --git a/api/v1beta1/gitrepository_types.go b/api/v1beta1/gitrepository_types.go index 1b0689cb..594c3c66 100644 --- a/api/v1beta1/gitrepository_types.go +++ b/api/v1beta1/gitrepository_types.go @@ -81,6 +81,12 @@ type GitRepositorySpec struct { // +kubebuilder:default:=go-git // +optional GitImplementation string `json:"gitImplementation,omitempty"` + + // When enabled, after the clone is created, initializes all submodules within, + // using their default settings. + // This option is available only when using the 'go-git' GitImplementation. + // +optional + RecurseSubmodules bool `json:"recurseSubmodules,omitempty"` } // GitRepositoryRef defines the Git ref used for pull and checkout operations. diff --git a/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml b/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml index 995bfbd5..75df32da 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml @@ -66,6 +66,11 @@ spec: interval: description: The interval at which to check for repository updates. type: string + recurseSubmodules: + description: When enabled, after the clone is created, initializes + all submodules within, using their default settings. This option + is available only when using the 'go-git' GitImplementation. + type: boolean ref: description: The Git reference to checkout and monitor for changes, defaults to master branch. diff --git a/controllers/gitrepository_controller.go b/controllers/gitrepository_controller.go index 8aea8f77..00986aee 100644 --- a/controllers/gitrepository_controller.go +++ b/controllers/gitrepository_controller.go @@ -183,7 +183,12 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, repository sour // determine auth method auth := &git.Auth{} if repository.Spec.SecretRef != nil { - authStrategy, err := strategy.AuthSecretStrategyForURL(repository.Spec.URL, repository.Spec.GitImplementation) + authStrategy, err := strategy.AuthSecretStrategyForURL( + repository.Spec.URL, + git.CheckoutOptions{ + GitImplementation: repository.Spec.GitImplementation, + RecurseSubmodules: repository.Spec.RecurseSubmodules, + }) if err != nil { return sourcev1.GitRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err } @@ -207,7 +212,12 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, repository sour } } - checkoutStrategy, err := strategy.CheckoutStrategyForRef(repository.Spec.Reference, repository.Spec.GitImplementation) + checkoutStrategy, err := strategy.CheckoutStrategyForRef( + repository.Spec.Reference, + git.CheckoutOptions{ + GitImplementation: repository.Spec.GitImplementation, + RecurseSubmodules: repository.Spec.RecurseSubmodules, + }) if err != nil { return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err } diff --git a/docs/api/source.md b/docs/api/source.md index c0de25e9..bb994354 100644 --- a/docs/api/source.md +++ b/docs/api/source.md @@ -400,6 +400,20 @@ string Defaults to go-git, valid values are (‘go-git’, ‘libgit2’).

+ + +recurseSubmodules
+ +bool + + + +(Optional) +

When enabled, after the clone is created, initializes all submodules within, +using their default settings. +This option is available only when using the ‘go-git’ GitImplementation.

+ + @@ -1246,6 +1260,20 @@ string Defaults to go-git, valid values are (‘go-git’, ‘libgit2’).

+ + +recurseSubmodules
+ +bool + + + +(Optional) +

When enabled, after the clone is created, initializes all submodules within, +using their default settings. +This option is available only when using the ‘go-git’ GitImplementation.

+ + diff --git a/docs/spec/v1beta1/gitrepositories.md b/docs/spec/v1beta1/gitrepositories.md index 9f3b9375..99bf41ce 100644 --- a/docs/spec/v1beta1/gitrepositories.md +++ b/docs/spec/v1beta1/gitrepositories.md @@ -57,6 +57,11 @@ type GitRepositorySpec struct { // +kubebuilder:default:=go-git // +optional GitImplementation string `json:"gitImplementation,omitempty"` + + // When enabled, after the clone is created, initializes all submodules within. + // This option is available only when using the 'go-git' GitImplementation. + // +optional + RecurseSubmodules bool `json:"recurseSubmodules,omitempty"` } ``` @@ -434,6 +439,42 @@ kubectl create secret generic pgp-public-keys \ --from-file=author2.asc ``` +### Git submodules + +With `spec.recurseSubmodules` you can configure the controller to +clone a specific branch including its Git submodules: + +```yaml +apiVersion: source.toolkit.fluxcd.io/v1beta1 +kind: GitRepository +metadata: + name: repo-with-submodules + namespace: default +spec: + interval: 1m + url: https://github.com// + secretRef: + name: https-credentials + ref: + branch: main + recurseSubmodules: true +--- +apiVersion: v1 +kind: Secret +metadata: + name: https-credentials + namespace: default +type: Opaque +data: + username: + password: +``` + +Note that deploy keys can't be used to pull submodules from private repositories +as GitHub and GitLab doesn't allow a deploy key to be reused across repositories. +You have to use either HTTPS token-based authentication, or an SSH key belonging +to a user that has access to the main repository and all its submodules. + ## Status examples Successful sync: diff --git a/pkg/git/git.go b/pkg/git/git.go index 55fd5d7a..6ec7257a 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -40,6 +40,11 @@ type CheckoutStrategy interface { Checkout(ctx context.Context, path, url string, auth *Auth) (Commit, string, error) } +type CheckoutOptions struct { + GitImplementation string + RecurseSubmodules bool +} + // TODO(hidde): candidate for refactoring, so that we do not directly // depend on implementation specifics here. type Auth struct { diff --git a/pkg/git/gogit/checkout.go b/pkg/git/gogit/checkout.go index d76cddc4..dfcde849 100644 --- a/pkg/git/gogit/checkout.go +++ b/pkg/git/gogit/checkout.go @@ -33,29 +33,30 @@ import ( "github.com/fluxcd/source-controller/pkg/git" ) -func CheckoutStrategyForRef(ref *sourcev1.GitRepositoryRef) git.CheckoutStrategy { +func CheckoutStrategyForRef(ref *sourcev1.GitRepositoryRef, opt git.CheckoutOptions) git.CheckoutStrategy { switch { case ref == nil: return &CheckoutBranch{branch: git.DefaultBranch} case ref.SemVer != "": - return &CheckoutSemVer{semVer: ref.SemVer} + return &CheckoutSemVer{semVer: ref.SemVer, recurseSubmodules: opt.RecurseSubmodules} case ref.Tag != "": - return &CheckoutTag{tag: ref.Tag} + return &CheckoutTag{tag: ref.Tag, recurseSubmodules: opt.RecurseSubmodules} case ref.Commit != "": - strategy := &CheckoutCommit{branch: ref.Branch, commit: ref.Commit} + strategy := &CheckoutCommit{branch: ref.Branch, commit: ref.Commit, recurseSubmodules: opt.RecurseSubmodules} if strategy.branch == "" { strategy.branch = git.DefaultBranch } return strategy case ref.Branch != "": - return &CheckoutBranch{branch: ref.Branch} + return &CheckoutBranch{branch: ref.Branch, recurseSubmodules: opt.RecurseSubmodules} default: return &CheckoutBranch{branch: git.DefaultBranch} } } type CheckoutBranch struct { - branch string + branch string + recurseSubmodules bool } func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, auth *git.Auth) (git.Commit, string, error) { @@ -67,7 +68,7 @@ func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, auth *g SingleBranch: true, NoCheckout: false, Depth: 1, - RecurseSubmodules: extgogit.DefaultSubmoduleRecursionDepth, + RecurseSubmodules: recurseSubmodules(c.recurseSubmodules), Progress: nil, Tags: extgogit.NoTags, CABundle: auth.CABundle, @@ -87,7 +88,8 @@ func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, auth *g } type CheckoutTag struct { - tag string + tag string + recurseSubmodules bool } func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, auth *git.Auth) (git.Commit, string, error) { @@ -99,7 +101,7 @@ func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, auth *git. SingleBranch: true, NoCheckout: false, Depth: 1, - RecurseSubmodules: extgogit.DefaultSubmoduleRecursionDepth, + RecurseSubmodules: recurseSubmodules(c.recurseSubmodules), Progress: nil, Tags: extgogit.NoTags, CABundle: auth.CABundle, @@ -119,8 +121,9 @@ func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, auth *git. } type CheckoutCommit struct { - branch string - commit string + branch string + commit string + recurseSubmodules bool } func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, auth *git.Auth) (git.Commit, string, error) { @@ -131,7 +134,7 @@ func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, auth *g ReferenceName: plumbing.NewBranchReferenceName(c.branch), SingleBranch: true, NoCheckout: false, - RecurseSubmodules: extgogit.DefaultSubmoduleRecursionDepth, + RecurseSubmodules: recurseSubmodules(c.recurseSubmodules), Progress: nil, Tags: extgogit.NoTags, CABundle: auth.CABundle, @@ -158,7 +161,8 @@ func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, auth *g } type CheckoutSemVer struct { - semVer string + semVer string + recurseSubmodules bool } func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, auth *git.Auth) (git.Commit, string, error) { @@ -173,7 +177,7 @@ func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, auth *g RemoteName: git.DefaultOrigin, NoCheckout: false, Depth: 1, - RecurseSubmodules: extgogit.DefaultSubmoduleRecursionDepth, + RecurseSubmodules: recurseSubmodules(c.recurseSubmodules), Progress: nil, Tags: extgogit.AllTags, CABundle: auth.CABundle, @@ -262,3 +266,10 @@ func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, auth *g return &Commit{commit}, fmt.Sprintf("%s/%s", t, head.Hash().String()), nil } + +func recurseSubmodules(recurse bool) extgogit.SubmoduleRescursivity { + if recurse { + return extgogit.DefaultSubmoduleRecursionDepth + } + return extgogit.NoRecurseSubmodules +} diff --git a/pkg/git/libgit2/checkout.go b/pkg/git/libgit2/checkout.go index f5254016..a5007b70 100644 --- a/pkg/git/libgit2/checkout.go +++ b/pkg/git/libgit2/checkout.go @@ -29,7 +29,7 @@ import ( "github.com/fluxcd/source-controller/pkg/git" ) -func CheckoutStrategyForRef(ref *sourcev1.GitRepositoryRef) git.CheckoutStrategy { +func CheckoutStrategyForRef(ref *sourcev1.GitRepositoryRef, opt git.CheckoutOptions) git.CheckoutStrategy { switch { case ref == nil: return &CheckoutBranch{branch: git.DefaultBranch} diff --git a/pkg/git/strategy/strategy.go b/pkg/git/strategy/strategy.go index 106a5129..6b3ea266 100644 --- a/pkg/git/strategy/strategy.go +++ b/pkg/git/strategy/strategy.go @@ -25,24 +25,24 @@ import ( "github.com/fluxcd/source-controller/pkg/git/libgit2" ) -func CheckoutStrategyForRef(ref *sourcev1.GitRepositoryRef, gitImplementation string) (git.CheckoutStrategy, error) { - switch gitImplementation { +func CheckoutStrategyForRef(ref *sourcev1.GitRepositoryRef, opt git.CheckoutOptions) (git.CheckoutStrategy, error) { + switch opt.GitImplementation { case sourcev1.GoGitImplementation: - return gogit.CheckoutStrategyForRef(ref), nil + return gogit.CheckoutStrategyForRef(ref, opt), nil case sourcev1.LibGit2Implementation: - return libgit2.CheckoutStrategyForRef(ref), nil + return libgit2.CheckoutStrategyForRef(ref, opt), nil default: - return nil, fmt.Errorf("invalid git implementation %s", gitImplementation) + return nil, fmt.Errorf("invalid Git implementation %s", opt.GitImplementation) } } -func AuthSecretStrategyForURL(url string, gitImplementation string) (git.AuthSecretStrategy, error) { - switch gitImplementation { +func AuthSecretStrategyForURL(url string, opt git.CheckoutOptions) (git.AuthSecretStrategy, error) { + switch opt.GitImplementation { case sourcev1.GoGitImplementation: return gogit.AuthSecretStrategyForURL(url) case sourcev1.LibGit2Implementation: return libgit2.AuthSecretStrategyForURL(url) default: - return nil, fmt.Errorf("invalid git implementation %s", gitImplementation) + return nil, fmt.Errorf("invalid Git implementation %s", opt.GitImplementation) } } From 681ddd5db0d60ccdca768492141d8a34ca978675 Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Wed, 31 Mar 2021 09:41:28 +0100 Subject: [PATCH 2/2] Test RecurseSubmodules This commit adds a test specifically for RecurseSubmodules. It takes a bit more preparation, since it needs a repo using submodules to start with. go-git doesn't appear to support adding submodules programmatically, so the preparation is done in part by execing `git`. Signed-off-by: Michael Bridgen --- controllers/gitrepository_controller_test.go | 142 ++++++++++++++++++- controllers/suite_test.go | 7 +- 2 files changed, 144 insertions(+), 5 deletions(-) diff --git a/controllers/gitrepository_controller_test.go b/controllers/gitrepository_controller_test.go index 38343e62..411a2bb2 100644 --- a/controllers/gitrepository_controller_test.go +++ b/controllers/gitrepository_controller_test.go @@ -20,10 +20,13 @@ import ( "context" "crypto/tls" "fmt" + "io/ioutil" "net/http" "net/url" "os" + "os/exec" "path" + "path/filepath" "strings" "time" @@ -42,9 +45,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "github.com/fluxcd/pkg/gittestserver" - "github.com/fluxcd/pkg/apis/meta" + "github.com/fluxcd/pkg/gittestserver" + "github.com/fluxcd/pkg/untar" + sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" ) @@ -136,8 +140,6 @@ var _ = Describe("GitRepositoryReconciler", func() { }}) Expect(err).NotTo(HaveOccurred()) - gitrepo.Worktree() - for _, ref := range t.createRefs { hRef := plumbing.NewHashReference(plumbing.ReferenceName(ref), commit) err = gitrepo.Storer.SetReference(hRef) @@ -410,5 +412,137 @@ var _ = Describe("GitRepositoryReconciler", func() { gitImplementation: sourcev1.GoGitImplementation, }), ) + + Context("recurse submodules", func() { + It("downloads submodules when asked", func() { + Expect(gitServer.StartHTTP()).To(Succeed()) + defer gitServer.StopHTTP() + + u, err := url.Parse(gitServer.HTTPAddress()) + Expect(err).NotTo(HaveOccurred()) + + subRepoURL := *u + subRepoURL.Path = path.Join(u.Path, fmt.Sprintf("subrepository-%s.git", randStringRunes(5))) + + // create the git repo to use as a submodule + fs := memfs.New() + subRepo, err := git.Init(memory.NewStorage(), fs) + Expect(err).NotTo(HaveOccurred()) + + wt, err := subRepo.Worktree() + Expect(err).NotTo(HaveOccurred()) + + ff, _ := fs.Create("fixture") + _ = ff.Close() + _, err = wt.Add(fs.Join("fixture")) + Expect(err).NotTo(HaveOccurred()) + + _, err = wt.Commit("Sample", &git.CommitOptions{Author: &object.Signature{ + Name: "John Doe", + Email: "john@example.com", + When: time.Now(), + }}) + Expect(err).NotTo(HaveOccurred()) + + remote, err := subRepo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{subRepoURL.String()}, + }) + Expect(err).NotTo(HaveOccurred()) + + err = remote.Push(&git.PushOptions{ + RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"}, + }) + Expect(err).NotTo(HaveOccurred()) + + // this one is linked to a real directory, so that I can + // exec `git submodule add` later + tmp, err := ioutil.TempDir("", "flux-test") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(tmp) + + repoDir := filepath.Join(tmp, "git") + repo, err := git.PlainInit(repoDir, false) + Expect(err).NotTo(HaveOccurred()) + + wt, err = repo.Worktree() + Expect(err).NotTo(HaveOccurred()) + _, err = wt.Commit("Initial revision", &git.CommitOptions{ + Author: &object.Signature{ + Name: "John Doe", + Email: "john@example.com", + When: time.Now(), + }}) + Expect(err).NotTo(HaveOccurred()) + + submodAdd := exec.Command("git", "submodule", "add", "-b", "master", subRepoURL.String(), "sub") + submodAdd.Dir = repoDir + out, err := submodAdd.CombinedOutput() + os.Stdout.Write(out) + Expect(err).NotTo(HaveOccurred()) + + _, err = wt.Commit("Add submodule", &git.CommitOptions{ + Author: &object.Signature{ + Name: "John Doe", + Email: "john@example.com", + When: time.Now(), + }}) + Expect(err).NotTo(HaveOccurred()) + + mainRepoURL := *u + mainRepoURL.Path = path.Join(u.Path, fmt.Sprintf("repository-%s.git", randStringRunes(5))) + remote, err = repo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{mainRepoURL.String()}, + }) + Expect(err).NotTo(HaveOccurred()) + + err = remote.Push(&git.PushOptions{ + RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"}, + }) + Expect(err).NotTo(HaveOccurred()) + + key := types.NamespacedName{ + Name: fmt.Sprintf("git-ref-test-%s", randStringRunes(5)), + Namespace: namespace.Name, + } + created := &sourcev1.GitRepository{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + }, + Spec: sourcev1.GitRepositorySpec{ + URL: mainRepoURL.String(), + Interval: metav1.Duration{Duration: indexInterval}, + Reference: &sourcev1.GitRepositoryRef{Branch: "master"}, + GitImplementation: sourcev1.GoGitImplementation, // only works with go-git + RecurseSubmodules: true, + }, + } + Expect(k8sClient.Create(context.Background(), created)).Should(Succeed()) + defer k8sClient.Delete(context.Background(), created) + + got := &sourcev1.GitRepository{} + Eventually(func() bool { + _ = k8sClient.Get(context.Background(), key, got) + for _, c := range got.Status.Conditions { + if c.Reason == sourcev1.GitOperationSucceedReason { + return true + } + } + return false + }, timeout, interval).Should(BeTrue()) + + // check that the downloaded artifact includes the + // file from the submodule + res, err := http.Get(got.Status.URL) + Expect(err).NotTo(HaveOccurred()) + Expect(res.StatusCode).To(Equal(http.StatusOK)) + + _, err = untar.Untar(res.Body, filepath.Join(tmp, "tar")) + Expect(err).NotTo(HaveOccurred()) + Expect(filepath.Join(tmp, "tar", "sub", "fixture")).To(BeAnExistingFile()) + }) + }) }) }) diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 8bba7889..0dd4351a 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -19,6 +19,7 @@ package controllers import ( "io/ioutil" "math/rand" + "net/http" "os" "path/filepath" "testing" @@ -99,8 +100,12 @@ var _ = BeforeSuite(func(done Done) { tmpStoragePath, err := ioutil.TempDir("", "source-controller-storage-") Expect(err).NotTo(HaveOccurred(), "failed to create tmp storage dir") - storage, err = NewStorage(tmpStoragePath, "localhost", time.Second*30) + storage, err = NewStorage(tmpStoragePath, "localhost:5050", time.Second*30) Expect(err).NotTo(HaveOccurred(), "failed to create tmp storage") + // serve artifacts from the filesystem, as done in main.go + fs := http.FileServer(http.Dir(tmpStoragePath)) + http.Handle("/", fs) + go http.ListenAndServe(":5050", nil) k8sManager, err = ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme.Scheme,