fluxcd
/
source-controller
mirror of https://github.com/fluxcd/source-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #665 from pjbgf/optimise-clone 

Optimise clone operations
This commit is contained in:
Paulo Gomes 2022-05-11 16:59:56 +01:00 committed by GitHub
parent 5b4750b87d 7f315f9807
commit 9c1bbc45eb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 471 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
controllers/gitrepository_controller.go
View File

@ -48,6 +48,7 @@ import (
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
serror "github.com/fluxcd/source-controller/internal/error"
"github.com/fluxcd/source-controller/internal/features"
sreconcile "github.com/fluxcd/source-controller/internal/reconcile"
"github.com/fluxcd/source-controller/internal/reconcile/summarize"
"github.com/fluxcd/source-controller/internal/util"
@ -311,8 +312,9 @@ func (r *GitRepositoryReconciler) notify(oldObj, newObj *sourcev1.GitRepository,
// reconcileStorage ensures the current state of the storage matches the
// desired and previously observed state.
//
// All Artifacts for the object except for the current one in the Status are
// garbage collected from the Storage.
// The garbage collection is executed based on the flag based settings and
// may remove files that are beyond their TTL or the maximum number of files
// to survive a collection cycle.
// If the Artifact in the Status of the object disappeared from the Storage,
// it is removed from the object.
// If the object does not have an Artifact in its Status, a Reconciling
@ -411,6 +413,13 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
checkoutOpts.Tag = ref.Tag
checkoutOpts.SemVer = ref.SemVer
}
if oc, _ := features.Enabled(features.OptimizedGitClones); oc {
if artifact := obj.GetArtifact(); artifact != nil {
checkoutOpts.LastRevision = artifact.Revision
}
}
checkoutStrategy, err := strategy.CheckoutStrategyForImplementation(ctx,
git.Implementation(obj.Spec.GitImplementation), checkoutOpts)
if err != nil {
@ -455,6 +464,12 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
defer cancel()
c, err := checkoutStrategy.Checkout(gitCtx, dir, repositoryURL, authOpts)
if err != nil {
var v git.NoChangesError
if errors.As(err, &v) {
return sreconcile.ResultSuccess,
&serror.Waiting{Err: v, Reason: v.Message, RequeueAfter: obj.GetRequeueAfter()}
}
e := &serror.Event{
Err: fmt.Errorf("failed to checkout and determine revision: %w", err),
Reason: sourcev1.GitOperationFailedReason,
@ -495,6 +510,7 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
// object are set, and the symlink in the Storage is updated to its path.
func (r *GitRepositoryReconciler) reconcileArtifact(ctx context.Context,
obj *sourcev1.GitRepository, commit *git.Commit, includes *artifactSet, dir string) (sreconcile.Result, error) {
// Create potential new artifact with current available metadata
artifact := r.Storage.NewArtifactFor(obj.Kind, obj.GetObjectMeta(), commit.String(), fmt.Sprintf("%s.tar.gz", commit.Hash.String()))

2
controllers/gitrepository_controller_test.go
View File

@ -359,7 +359,7 @@ func TestGitRepositoryReconciler_reconcileSource_authStrategy(t *testing.T) {
},
wantErr: true,
assertConditions: []metav1.Condition{
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.GitOperationFailedReason, "failed to checkout and determine revision: unable to clone '<url>': PEM CA bundle could not be appended to x509 certificate pool"),
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.GitOperationFailedReason, "failed to checkout and determine revision: unable to fetch-connect to remote '<url>': PEM CA bundle could not be appended to x509 certificate pool"),
},
},
{

16
docs/spec/v1beta2/gitrepositories.md
View File

@ -399,6 +399,22 @@ transport being handled by the controller, instead of `libgit2`.
This may lead to an increased number of timeout messages in the logs, however
it will fix the bug in which Git operations make the controllers hang indefinitely.
#### Optimized Git clones
Optimized Git clones decreases resource utilization for GitRepository
reconciliations. It supports both `go-git` and `libgit2` implementations
when cloning repositories using branches or tags.
When enabled, avoids full clone operations by first checking whether
the last revision is still the same at the target repository,
and if that is so, skips the reconciliation.
This feature is enabled by default. It can be disabled by starting the
controller with the argument `--feature-gates=OptimizedGitClones=false`.
NB: GitRepository objects configured for SemVer or Commit clones are
not affected by this functionality.
#### Proxy support
When a proxy is configured in the source-controller Pod through the appropriate

4
go.mod
View File

@ -25,7 +25,7 @@ require (
github.com/fluxcd/pkg/gitutil v0.1.0
github.com/fluxcd/pkg/helmtestserver v0.7.2
github.com/fluxcd/pkg/lockedfile v0.1.0
github.com/fluxcd/pkg/runtime v0.14.2
github.com/fluxcd/pkg/runtime v0.15.1
github.com/fluxcd/pkg/ssh v0.3.3
github.com/fluxcd/pkg/testserver v0.2.0
github.com/fluxcd/pkg/untar v0.1.0
@ -185,7 +185,7 @@ require (
github.com/shopspring/decimal v1.2.0 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/cobra v1.3.0 // indirect
github.com/spf13/cobra v1.4.0 // indirect
github.com/stretchr/testify v1.7.1 // indirect
github.com/xanzy/ssh-agent v0.3.1 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect

7
go.sum
View File

@ -364,8 +364,8 @@ github.com/fluxcd/pkg/helmtestserver v0.7.2/go.mod h1:WtUXBrfpJdwK54LX1Tqd8PpLJY
github.com/fluxcd/pkg/lockedfile v0.1.0 h1:YsYFAkd6wawMCcD74ikadAKXA4s2sukdxrn7w8RB5eo=
github.com/fluxcd/pkg/lockedfile v0.1.0/go.mod h1:EJLan8t9MiOcgTs8+puDjbE6I/KAfHbdvIy9VUgIjm8=
github.com/fluxcd/pkg/runtime v0.13.0-rc.6/go.mod h1:4oKUO19TeudXrnCRnxCfMSS7EQTYpYlgfXwlQuDJ/Eg=
github.com/fluxcd/pkg/runtime v0.14.2 h1:ktyUjcX4pHoC8DRoBmhEP6eMHbmR6+/MYoARe4YulZY=
github.com/fluxcd/pkg/runtime v0.14.2/go.mod h1:NZr3PRK7xX2M1bl0LdtugvQyWkOmu2NcW3NrZH6U0is=
github.com/fluxcd/pkg/runtime v0.15.1 h1:PKooYqlZM+KLhnNz10sQnBH0AHllS40PIDHtiRH/BGU=
github.com/fluxcd/pkg/runtime v0.15.1/go.mod h1:TPAoOEgUFG60FXBA4ID41uaPldxuXCEI4jt3qfd5i5Q=
github.com/fluxcd/pkg/ssh v0.3.3 h1:/tc7W7LO1VoVUI5jB+p9ZHCA+iQaXTkaSCDZJsxcZ9k=
github.com/fluxcd/pkg/ssh v0.3.3/go.mod h1:+bKhuv0/pJy3HZwkK54Shz68sNv1uf5aI6wtPaEHaYk=
github.com/fluxcd/pkg/testserver v0.2.0 h1:Mj0TapmKaywI6Fi5wvt1LAZpakUHmtzWQpJNKQ0Krt4=
@ -990,8 +990,9 @@ github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHN
github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk=
github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0=
github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4=
github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q=
github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=

54
internal/features/features.go Normal file
View File

@ -0,0 +1,54 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package features sets the feature gates that
// source-controller supports, and their default
// states.
package features
import feathelper "github.com/fluxcd/pkg/runtime/features"
const (
// OptimizedGitClones decreases resource utilization for GitRepository
// reconciliations. It supports both go-git and libgit2 implementations
// when cloning repositories using branches or tags.
//
// When enabled, avoids full clone operations by first checking whether
// the last revision is still the same at the target repository,
// and if that is so, skips the reconciliation.
OptimizedGitClones = "OptimizedGitClones"
)
var features = map[string]bool{
// OptimizedGitClones
// opt-out from v0.25
OptimizedGitClones: true,
}
// DefaultFeatureGates contains a list of all supported feature gates and
// their default values.
func FeatureGates() map[string]bool {
return features
}
// Enabled verifies whether the feature is enabled or not.
//
// This is only a wrapper around the Enabled func in
// pkg/runtime/features, so callers won't need to import
// both packages for checking whether a feature is enabled.
func Enabled(feature string) (bool, error) {
return feathelper.Enabled(feature)
}

12
main.go
View File

@ -36,10 +36,12 @@ import (
"github.com/fluxcd/pkg/runtime/client"
helper "github.com/fluxcd/pkg/runtime/controller"
"github.com/fluxcd/pkg/runtime/events"
feathelper "github.com/fluxcd/pkg/runtime/features"
"github.com/fluxcd/pkg/runtime/leaderelection"
"github.com/fluxcd/pkg/runtime/logger"
"github.com/fluxcd/pkg/runtime/pprof"
"github.com/fluxcd/pkg/runtime/probes"
"github.com/fluxcd/source-controller/internal/features"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
"github.com/fluxcd/source-controller/controllers"
@ -88,6 +90,7 @@ func main() {
logOptions logger.Options
leaderElectionOptions leaderelection.Options
rateLimiterOptions helper.RateLimiterOptions
featureGates feathelper.FeatureGates
helmCacheMaxSize int
helmCacheTTL string
helmCachePurgeInterval string
@ -136,11 +139,20 @@ func main() {
logOptions.BindFlags(flag.CommandLine)
leaderElectionOptions.BindFlags(flag.CommandLine)
rateLimiterOptions.BindFlags(flag.CommandLine)
featureGates.BindFlags(flag.CommandLine)
flag.Parse()
ctrl.SetLogger(logger.NewLogger(logOptions))
err := featureGates.WithLogger(setupLog).
SupportedFeatures(features.FeatureGates())
if err != nil {
setupLog.Error(err, "unable to load feature gates")
os.Exit(1)
}
// Set upper bound file size limits Helm
helm.MaxIndexSize = helmIndexLimit
helm.MaxChartSize = helmChartLimit

12
pkg/git/git.go
View File

@ -106,3 +106,15 @@ func (c *Commit) ShortMessage() string {
type CheckoutStrategy interface {
Checkout(ctx context.Context, path, url string, config *AuthOptions) (*Commit, error)
}
// NoChangesError represents the case in which a Git clone operation
// is attempted, but cancelled as the revision is still the same as
// the one observed on the last successful reconciliation.
type NoChangesError struct {
Message string
ObservedRevision string
}
func (e NoChangesError) Error() string {
return fmt.Sprintf("%s: observed revision '%s'", e.Message, e.ObservedRevision)
}

70
pkg/git/gogit/checkout.go
View File

@ -26,8 +26,11 @@ import (
"github.com/Masterminds/semver/v3"
extgogit "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/plumbing/transport"
"github.com/go-git/go-git/v5/storage/memory"
"github.com/fluxcd/pkg/gitutil"
"github.com/fluxcd/pkg/version"
@ -44,19 +47,20 @@ func CheckoutStrategyForOptions(_ context.Context, opts git.CheckoutOptions) git
case opts.SemVer != "":
return &CheckoutSemVer{SemVer: opts.SemVer, RecurseSubmodules: opts.RecurseSubmodules}
case opts.Tag != "":
return &CheckoutTag{Tag: opts.Tag, RecurseSubmodules: opts.RecurseSubmodules}
return &CheckoutTag{Tag: opts.Tag, RecurseSubmodules: opts.RecurseSubmodules, LastRevision: opts.LastRevision}
default:
branch := opts.Branch
if branch == "" {
branch = git.DefaultBranch
}
return &CheckoutBranch{Branch: branch, RecurseSubmodules: opts.RecurseSubmodules}
return &CheckoutBranch{Branch: branch, RecurseSubmodules: opts.RecurseSubmodules, LastRevision: opts.LastRevision}
}
}
type CheckoutBranch struct {
Branch string
RecurseSubmodules bool
LastRevision string
}
func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (*git.Commit, error) {
@ -64,7 +68,23 @@ func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *g
if err != nil {
return nil, fmt.Errorf("failed to construct auth method with options: %w", err)
}
ref := plumbing.NewBranchReferenceName(c.Branch)
// check if previous revision has changed before attempting to clone
if c.LastRevision != "" {
currentRevision, err := getLastRevision(ctx, url, ref, opts, authMethod)
if err != nil {
return nil, err
}
if currentRevision != "" && currentRevision == c.LastRevision {
return nil, git.NoChangesError{
Message: "no changes since last reconcilation",
ObservedRevision: currentRevision,
}
}
}
repo, err := extgogit.PlainCloneContext(ctx, path, false, &extgogit.CloneOptions{
URL: url,
Auth: authMethod,
@ -92,9 +112,31 @@ func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *g
return buildCommitWithRef(cc, ref)
}
func getLastRevision(ctx context.Context, url string, ref plumbing.ReferenceName, opts *git.AuthOptions, authMethod transport.AuthMethod) (string, error) {
config := &config.RemoteConfig{
Name: git.DefaultOrigin,
URLs: []string{url},
}
rem := extgogit.NewRemote(memory.NewStorage(), config)
listOpts := &extgogit.ListOptions{
Auth: authMethod,
}
if opts != nil && opts.CAFile != nil {
listOpts.CABundle = opts.CAFile
}
refs, err := rem.ListContext(ctx, listOpts)
if err != nil {
return "", fmt.Errorf("unable to list remote for '%s': %w", url, err)
}
currentRevision := filterRefs(refs, ref)
return currentRevision, nil
}
type CheckoutTag struct {
Tag string
RecurseSubmodules bool
LastRevision string
}
func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (*git.Commit, error) {
@ -103,6 +145,20 @@ func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, opts *git.
return nil, fmt.Errorf("failed to construct auth method with options: %w", err)
}
ref := plumbing.NewTagReferenceName(c.Tag)
// check if previous revision has changed before attempting to clone
if c.LastRevision != "" {
currentRevision, err := getLastRevision(ctx, url, ref, opts, authMethod)
if err != nil {
return nil, err
}
if currentRevision != "" && currentRevision == c.LastRevision {
return nil, git.NoChangesError{
Message: "no changes since last reconcilation",
ObservedRevision: currentRevision,
}
}
}
repo, err := extgogit.PlainCloneContext(ctx, path, false, &extgogit.CloneOptions{
URL: url,
Auth: authMethod,
@ -333,3 +389,13 @@ func recurseSubmodules(recurse bool) extgogit.SubmoduleRescursivity {
}
return extgogit.NoRecurseSubmodules
}
func filterRefs(refs []*plumbing.Reference, currentRef plumbing.ReferenceName) string {
for _, ref := range refs {
if ref.Name().String() == currentRef.String() {
return fmt.Sprintf("%s/%s", currentRef.Short(), ref.Hash().String())
}
}
return ""
}

41
pkg/git/gogit/checkout_test.go
View File

@ -72,6 +72,7 @@ func TestCheckoutBranch_Checkout(t *testing.T) {
filesCreated map[string]string
expectedCommit string
expectedErr string
lastRevision string
}{
{
name: "Default branch",
@ -80,10 +81,18 @@ func TestCheckoutBranch_Checkout(t *testing.T) {
expectedCommit: firstCommit.String(),
},
{
name: "Other branch",
name: "skip clone if LastRevision hasn't changed",
branch: "master",
filesCreated: map[string]string{"branch": "init"},
expectedErr: fmt.Sprintf("no changes since last reconcilation: observed revision 'master/%s'", firstCommit.String()),
lastRevision: fmt.Sprintf("master/%s", firstCommit.String()),
},
{
name: "Other branch - revision has changed",
branch: "test",
filesCreated: map[string]string{"branch": "second"},
expectedCommit: secondCommit.String(),
lastRevision: fmt.Sprintf("master/%s", firstCommit.String()),
},
{
name: "Non existing branch",
@ -97,7 +106,8 @@ func TestCheckoutBranch_Checkout(t *testing.T) {
g := NewWithT(t)
branch := CheckoutBranch{
Branch: tt.branch,
Branch: tt.branch,
LastRevision: tt.lastRevision,
}
tmpDir := t.TempDir()
@ -127,6 +137,8 @@ func TestCheckoutTag_Checkout(t *testing.T) {
checkoutTag string
expectTag string
expectErr string
lastRev string
setLastRev bool
}{
{
name: "Tag",
@ -134,6 +146,20 @@ func TestCheckoutTag_Checkout(t *testing.T) {
checkoutTag: "tag-1",
expectTag: "tag-1",
},
{
name: "Skip Tag if last revision hasn't changed",
tag: "tag-2",
checkoutTag: "tag-2",
setLastRev: true,
expectErr: "no changes since last reconcilation",
},
{
name: "Last revision changed",
tag: "tag-3",
checkoutTag: "tag-3",
expectTag: "tag-3",
lastRev: "tag-3/<fake-hash>",
},
{
name: "Annotated",
tag: "annotated",
@ -158,12 +184,13 @@ func TestCheckoutTag_Checkout(t *testing.T) {
}
var h plumbing.Hash
var tagHash *plumbing.Reference
if tt.tag != "" {
h, err = commitFile(repo, "tag", tt.tag, time.Now())
if err != nil {
t.Fatal(err)
}
_, err = tag(repo, h, !tt.annotated, tt.tag, time.Now())
tagHash, err = tag(repo, h, !tt.annotated, tt.tag, time.Now())
if err != nil {
t.Fatal(err)
}
@ -172,10 +199,18 @@ func TestCheckoutTag_Checkout(t *testing.T) {
tag := CheckoutTag{
Tag: tt.checkoutTag,
}
if tt.setLastRev {
tag.LastRevision = fmt.Sprintf("%s/%s", tt.tag, tagHash.Hash().String())
}
if tt.lastRev != "" {
tag.LastRevision = tt.lastRev
}
tmpDir := t.TempDir()
cc, err := tag.Checkout(context.TODO(), tmpDir, path, nil)
if tt.expectErr != "" {
g.Expect(err).ToNot(BeNil())
g.Expect(err.Error()).To(ContainSubstring(tt.expectErr))
g.Expect(cc).To(BeNil())
return

189
pkg/git/libgit2/checkout.go
View File

@ -52,59 +52,149 @@ func CheckoutStrategyForOptions(ctx context.Context, opt git.CheckoutOptions) gi
if branch == "" {
branch = git.DefaultBranch
}
return &CheckoutBranch{Branch: branch}
return &CheckoutBranch{
Branch: branch,
LastRevision: opt.LastRevision,
}
}
}
type CheckoutBranch struct {
Branch string
Branch string
LastRevision string
}
func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (*git.Commit, error) {
repo, err := safeClone(url, path, &git2go.CloneOptions{
FetchOptions: git2go.FetchOptions{
func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) {
defer recoverPanic(&err)
repo, remote, free, err := getBlankRepoAndRemote(ctx, path, url, opts)
if err != nil {
return nil, err
}
defer free()
// When the last observed revision is set, check whether it is still
// the same at the remote branch. If so, short-circuit the clone operation here.
if c.LastRevision != "" {
heads, err := remote.Ls(c.Branch)
if err != nil {
return nil, fmt.Errorf("unable to remote ls for '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
if len(heads) > 0 {
currentRevision := fmt.Sprintf("%s/%s", c.Branch, heads[0].Id.String())
if currentRevision == c.LastRevision {
return nil, git.NoChangesError{
Message: "no changes since last reconciliation",
ObservedRevision: currentRevision,
}
}
}
}
// Limit the fetch operation to the specific branch, to decrease network usage.
err = remote.Fetch([]string{c.Branch},
&git2go.FetchOptions{
DownloadTags: git2go.DownloadTagsNone,
RemoteCallbacks: RemoteCallbacks(ctx, opts),
ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
},
CheckoutOptions: git2go.CheckoutOptions{
Strategy: git2go.CheckoutForce,
},
CheckoutBranch: c.Branch,
"")
if err != nil {
return nil, fmt.Errorf("unable to fetch remote '%s': %w",
managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
branch, err := repo.References.Lookup(fmt.Sprintf("refs/remotes/origin/%s", c.Branch))
if err != nil {
return nil, fmt.Errorf("unable to lookup branch '%s' for '%s': %w",
c.Branch, managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
defer branch.Free()
upstreamCommit, err := repo.LookupCommit(branch.Target())
if err != nil {
return nil, fmt.Errorf("unable to lookup commit '%s' for '%s': %w",
c.Branch, managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
defer upstreamCommit.Free()
// Once the index has been updated with Fetch, and we know the tip commit,
// a hard reset can be used to align the local worktree with the remote branch's.
err = repo.ResetToCommit(upstreamCommit, git2go.ResetHard, &git2go.CheckoutOptions{
Strategy: git2go.CheckoutForce,
})
if err != nil {
return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
return nil, fmt.Errorf("unable to hard reset to commit for '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
defer repo.Free()
// Use the current worktree's head as reference for the commit to be returned.
head, err := repo.Head()
if err != nil {
return nil, fmt.Errorf("git resolve HEAD error: %w", err)
}
defer head.Free()
cc, err := repo.LookupCommit(head.Target())
if err != nil {
return nil, fmt.Errorf("failed to lookup HEAD commit '%s' for branch '%s': %w", head.Target(), c.Branch, err)
}
defer cc.Free()
return buildCommit(cc, "refs/heads/"+c.Branch), nil
}
type CheckoutTag struct {
Tag string
Tag string
LastRevision string
}
func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (*git.Commit, error) {
repo, err := safeClone(url, path, &git2go.CloneOptions{
FetchOptions: git2go.FetchOptions{
DownloadTags: git2go.DownloadTagsAll,
func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) {
defer recoverPanic(&err)
repo, remote, free, err := getBlankRepoAndRemote(ctx, path, url, opts)
if err != nil {
return nil, err
}
defer free()
if c.LastRevision != "" {
heads, err := remote.Ls(c.Tag)
if err != nil {
return nil, fmt.Errorf("unable to remote ls for '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
if len(heads) > 0 {
currentRevision := fmt.Sprintf("%s/%s", c.Tag, heads[0].Id.String())
var same bool
if currentRevision == c.LastRevision {
same = true
} else if len(heads) > 1 {
currentAnnotatedRevision := fmt.Sprintf("%s/%s", c.Tag, heads[1].Id.String())
if currentAnnotatedRevision == c.LastRevision {
same = true
}
}
if same {
return nil, git.NoChangesError{
Message: "no changes since last reconciliation",
ObservedRevision: currentRevision,
}
}
}
}
err = remote.Fetch([]string{c.Tag},
&git2go.FetchOptions{
DownloadTags: git2go.DownloadTagsAuto,
RemoteCallbacks: RemoteCallbacks(ctx, opts),
ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
},
})
"")
if err != nil {
return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
return nil, fmt.Errorf("unable to fetch remote '%s': %w",
managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
defer repo.Free()
cc, err := checkoutDetachedDwim(repo, c.Tag)
if err != nil {
return nil, err
@ -117,8 +207,10 @@ type CheckoutCommit struct {
Commit string
}
func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (*git.Commit, error) {
repo, err := safeClone(url, path, &git2go.CloneOptions{
func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) {
defer recoverPanic(&err)
repo, err := git2go.Clone(url, path, &git2go.CloneOptions{
FetchOptions: git2go.FetchOptions{
DownloadTags: git2go.DownloadTagsNone,
RemoteCallbacks: RemoteCallbacks(ctx, opts),
@ -144,13 +236,15 @@ type CheckoutSemVer struct {
SemVer string
}
func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (*git.Commit, error) {
func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) {
defer recoverPanic(&err)
verConstraint, err := semver.NewConstraint(c.SemVer)
if err != nil {
return nil, fmt.Errorf("semver parse error: %w", err)
}
repo, err := safeClone(url, path, &git2go.CloneOptions{
repo, err := git2go.Clone(url, path, &git2go.CloneOptions{
FetchOptions: git2go.FetchOptions{
DownloadTags: git2go.DownloadTagsAll,
RemoteCallbacks: RemoteCallbacks(ctx, opts),
@ -239,19 +333,6 @@ func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, opts *g
return buildCommit(cc, "refs/tags/"+t), nil
}
// safeClone wraps git2go calls with panic recovering logic, ensuring
// a predictable execution path for callers.
func safeClone(url, path string, cloneOpts *git2go.CloneOptions) (repo *git2go.Repository, err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("recovered from git2go panic: %v", r)
}
}()
repo, err = git2go.Clone(url, path, cloneOpts)
return
}
// checkoutDetachedDwim attempts to perform a detached HEAD checkout by first DWIMing the short name
// to get a concrete reference, and then calling checkoutDetachedHEAD.
func checkoutDetachedDwim(repo *git2go.Repository, name string) (*git2go.Commit, error) {
@ -326,3 +407,39 @@ func buildSignature(s *git2go.Signature) git.Signature {
When: s.When,
}
}
// getBlankRepoAndRemote returns a newly initialized repository, and a remote connected to the provided url.
// Callers must call the returning function to free all git2go objects.
func getBlankRepoAndRemote(ctx context.Context, path, url string, opts *git.AuthOptions) (*git2go.Repository, *git2go.Remote, func(), error) {
repo, err := git2go.InitRepository(path, false)
if err != nil {
return nil, nil, nil, fmt.Errorf("unable to init repository for '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
remote, err := repo.Remotes.Create("origin", url)
if err != nil {
repo.Free()
return nil, nil, nil, fmt.Errorf("unable to create remote for '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
callBacks := RemoteCallbacks(ctx, opts)
err = remote.ConnectFetch(&callBacks, &git2go.ProxyOptions{Type: git2go.ProxyTypeAuto}, nil)
if err != nil {
remote.Free()
repo.Free()
return nil, nil, nil, fmt.Errorf("unable to fetch-connect to remote '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
free := func() {
remote.Disconnect()
remote.Free()
repo.Free()
}
return repo, remote, free, nil
}
func recoverPanic(err *error) {
if r := recover(); r != nil {
*err = fmt.Errorf("recovered from git2go panic: %v", r)
}
}

140
pkg/git/libgit2/checkout_test.go
View File

@ -51,8 +51,19 @@ func TestCheckoutBranch_Checkout(t *testing.T) {
// ignores the error here because it can be defaulted
// https://github.blog/2020-07-27-highlights-from-git-2-28/#introducing-init-defaultbranch
defaultBranch := "master"
if v, err := cfg.LookupString("init.defaultBranch"); err != nil && v != "" {
defaultBranch = v
iter, err := cfg.NewIterator()
if err != nil {
t.Fatal(err)
}
for {
val, e := iter.Next()
if e != nil {
break
}
if val.Name == "init.defaultbranch" {
defaultBranch = val.Value
break
}
}
firstCommit, err := commitFile(repo, "branch", "init", time.Now())
@ -77,6 +88,7 @@ func TestCheckoutBranch_Checkout(t *testing.T) {
filesCreated map[string]string
expectedCommit string
expectedErr string
lastRevision string
}{
{
name: "Default branch",
@ -95,6 +107,21 @@ func TestCheckoutBranch_Checkout(t *testing.T) {
branch: "invalid",
expectedErr: "reference 'refs/remotes/origin/invalid' not found",
},
{
name: "skip clone - lastRevision hasn't changed",
branch: defaultBranch,
filesCreated: map[string]string{"branch": "second"},
expectedCommit: secondCommit.String(),
lastRevision: fmt.Sprintf("%s/%s", defaultBranch, secondCommit.String()),
expectedErr: fmt.Sprintf("no changes since last reconciliation: observed revision '%s/%s'", defaultBranch, secondCommit.String()),
},
{
name: "lastRevision is different",
branch: defaultBranch,
filesCreated: map[string]string{"branch": "second"},
expectedCommit: secondCommit.String(),
lastRevision: fmt.Sprintf("%s/%s", defaultBranch, firstCommit.String()),
},
}
for _, tt := range tests {
@ -102,7 +129,8 @@ func TestCheckoutBranch_Checkout(t *testing.T) {
g := NewWithT(t)
branch := CheckoutBranch{
Branch: tt.branch,
Branch: tt.branch,
LastRevision: tt.lastRevision,
}
tmpDir := t.TempDir()
@ -126,12 +154,13 @@ func TestCheckoutBranch_Checkout(t *testing.T) {
func TestCheckoutTag_Checkout(t *testing.T) {
tests := []struct {
name string
tag string
annotated bool
checkoutTag string
expectTag string
expectErr string
name string
tag string
annotated bool
checkoutTag string
expectTag string
expectErr string
lastRevision bool
}{
{
name: "Tag",
@ -151,6 +180,21 @@ func TestCheckoutTag_Checkout(t *testing.T) {
checkoutTag: "invalid",
expectErr: "unable to find 'invalid': no reference found for shorthand 'invalid'",
},
{
name: "skip clone - last revision is unchanged",
tag: "tag-1",
checkoutTag: "tag-1",
expectTag: "tag-1",
lastRevision: true,
expectErr: "no changes since last reconciliation",
},
{
name: "last revision changed",
tag: "tag-1",
checkoutTag: "tag-1",
expectTag: "tag-2",
lastRevision: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@ -171,29 +215,60 @@ func TestCheckoutTag_Checkout(t *testing.T) {
if commit, err = repo.LookupCommit(c); err != nil {
t.Fatal(err)
}
_, err = tag(repo, c, !tt.annotated, tt.tag, time.Now())
_, err = tag(repo, commit.Id(), !tt.annotated, tt.tag, time.Now())
if err != nil {
t.Fatal(err)
}
}
tag := CheckoutTag{
checkoutTag := CheckoutTag{
Tag: tt.checkoutTag,
}
tmpDir := t.TempDir()
cc, err := tag.Checkout(context.TODO(), tmpDir, repo.Path(), nil)
cc, err := checkoutTag.Checkout(context.TODO(), tmpDir, repo.Path(), nil)
if tt.expectErr != "" {
if tt.lastRevision {
tmpDir, _ = os.MkdirTemp("", "test")
defer os.RemoveAll(tmpDir)
checkoutTag.LastRevision = cc.String()
cc, err = checkoutTag.Checkout(context.TODO(), tmpDir, repo.Path(), nil)
}
g.Expect(err).To(HaveOccurred())
g.Expect(err.Error()).To(ContainSubstring(tt.expectErr))
g.Expect(cc).To(BeNil())
return
}
if tt.lastRevision {
checkoutTag.LastRevision = fmt.Sprintf("%s/%s", tt.tag, commit.Id().String())
checkoutTag.Tag = tt.expectTag
if tt.tag != "" {
c, err := commitFile(repo, "tag", "changed tag", time.Now())
if err != nil {
t.Fatal(err)
}
if commit, err = repo.LookupCommit(c); err != nil {
t.Fatal(err)
}
_, err = tag(repo, commit.Id(), !tt.annotated, tt.expectTag, time.Now())
if err != nil {
t.Fatal(err)
}
tmpDir, _ = os.MkdirTemp("", "test")
defer os.RemoveAll(tmpDir)
cc, err = checkoutTag.Checkout(context.TODO(), tmpDir, repo.Path(), nil)
}
}
g.Expect(err).ToNot(HaveOccurred())
g.Expect(cc.String()).To(Equal(tt.expectTag + "/" + commit.Id().String()))
g.Expect(filepath.Join(tmpDir, "tag")).To(BeARegularFile())
g.Expect(os.ReadFile(filepath.Join(tmpDir, "tag"))).To(BeEquivalentTo(tt.tag))
if tt.lastRevision {
g.Expect(os.ReadFile(filepath.Join(tmpDir, "tag"))).To(BeEquivalentTo("changed tag"))
} else {
g.Expect(os.ReadFile(filepath.Join(tmpDir, "tag"))).To(BeEquivalentTo(tt.tag))
}
})
}
}
@ -506,42 +581,3 @@ func TestCheckout_ED25519(t *testing.T) {
_, err = branchCheckoutStrat.Checkout(ctx, tmpDir, repoURL, authOpts)
g.Expect(err).ToNot(HaveOccurred())
}
func TestSafeClone(t *testing.T) {
g := NewWithT(t)
// Create a git test server.
server, err := gittestserver.NewTempGitServer()
g.Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(server.Root())
server.Auth("test-user", "test-pswd")
server.AutoCreate()
server.KeyDir(filepath.Join(server.Root(), "keys"))
g.Expect(server.ListenSSH()).To(Succeed())
go func() {
server.StartSSH()
}()
defer server.StopSSH()
sshURL := server.SSHAddress()
repoURL := sshURL + "/test.git"
u, err := url.Parse(sshURL)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(u.Host).ToNot(BeEmpty())
repo, err := safeClone(repoURL, t.TempDir(), &git2go.CloneOptions{
FetchOptions: git2go.FetchOptions{
RemoteCallbacks: git2go.RemoteCallbacks{
CertificateCheckCallback: func(cert *git2go.Certificate, valid bool, hostname string) error {
panic("Oops!")
},
},
}})
g.Expect(repo).To(BeNil())
g.Expect(err).To(HaveOccurred())
g.Expect(err.Error()).Should(ContainSubstring("recovered from git2go panic"))
}

5
pkg/git/options.go
View File

@ -48,6 +48,11 @@ type CheckoutOptions struct {
// RecurseSubmodules defines if submodules should be checked out,
// not supported by all Implementations.
RecurseSubmodules bool
// LastRevision holds the revision observed on the last successful
// reconciliation.
// It is used to skip clone operations when no changes were detected.
LastRevision string
}
type TransportType string