Merge pull request #665 from pjbgf/optimise-clone

Optimise clone operations

controllers/gitrepository_controller.go

@@ -48,6 +48,7 @@ import (
 
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
 	serror "github.com/fluxcd/source-controller/internal/error"
+	"github.com/fluxcd/source-controller/internal/features"
 	sreconcile "github.com/fluxcd/source-controller/internal/reconcile"
 	"github.com/fluxcd/source-controller/internal/reconcile/summarize"
 	"github.com/fluxcd/source-controller/internal/util"
@@ -311,8 +312,9 @@ func (r *GitRepositoryReconciler) notify(oldObj, newObj *sourcev1.GitRepository,
 // reconcileStorage ensures the current state of the storage matches the
 // desired and previously observed state.
 //
-// All Artifacts for the object except for the current one in the Status are
+// The garbage collection is executed based on the flag based settings and
-// garbage collected from the Storage.
+// may remove files that are beyond their TTL or the maximum number of files
+// to survive a collection cycle.
 // If the Artifact in the Status of the object disappeared from the Storage,
 // it is removed from the object.
 // If the object does not have an Artifact in its Status, a Reconciling
@@ -411,6 +413,13 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
 		checkoutOpts.Tag = ref.Tag
 		checkoutOpts.SemVer = ref.SemVer
 	}
+
+	if oc, _ := features.Enabled(features.OptimizedGitClones); oc {
+		if artifact := obj.GetArtifact(); artifact != nil {
+			checkoutOpts.LastRevision = artifact.Revision
+		}
+	}
+
 	checkoutStrategy, err := strategy.CheckoutStrategyForImplementation(ctx,
 		git.Implementation(obj.Spec.GitImplementation), checkoutOpts)
 	if err != nil {
@@ -455,6 +464,12 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
 	defer cancel()
 	c, err := checkoutStrategy.Checkout(gitCtx, dir, repositoryURL, authOpts)
 	if err != nil {
+		var v git.NoChangesError
+		if errors.As(err, &v) {
+			return sreconcile.ResultSuccess,
+				&serror.Waiting{Err: v, Reason: v.Message, RequeueAfter: obj.GetRequeueAfter()}
+		}
+
 		e := &serror.Event{
 			Err:    fmt.Errorf("failed to checkout and determine revision: %w", err),
 			Reason: sourcev1.GitOperationFailedReason,
@@ -495,6 +510,7 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
 // object are set, and the symlink in the Storage is updated to its path.
 func (r *GitRepositoryReconciler) reconcileArtifact(ctx context.Context,
 	obj *sourcev1.GitRepository, commit *git.Commit, includes *artifactSet, dir string) (sreconcile.Result, error) {
+
 	// Create potential new artifact with current available metadata
 	artifact := r.Storage.NewArtifactFor(obj.Kind, obj.GetObjectMeta(), commit.String(), fmt.Sprintf("%s.tar.gz", commit.Hash.String()))
 

controllers/gitrepository_controller_test.go

@@ -359,7 +359,7 @@ func TestGitRepositoryReconciler_reconcileSource_authStrategy(t *testing.T) {
 			},
 			wantErr: true,
 			assertConditions: []metav1.Condition{
-				*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.GitOperationFailedReason, "failed to checkout and determine revision: unable to clone '<url>': PEM CA bundle could not be appended to x509 certificate pool"),
+				*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.GitOperationFailedReason, "failed to checkout and determine revision: unable to fetch-connect to remote '<url>': PEM CA bundle could not be appended to x509 certificate pool"),
 			},
 		},
 		{

docs/spec/v1beta2/gitrepositories.md

@@ -399,6 +399,22 @@ transport being handled by the controller, instead of `libgit2`.
 This may lead to an increased number of timeout messages in the logs, however
 it will fix the bug in which Git operations make the controllers hang indefinitely.
 
+#### Optimized Git clones
+
+Optimized Git clones decreases resource utilization for GitRepository
+reconciliations. It supports both `go-git` and `libgit2` implementations
+when cloning repositories using branches or tags.
+
+When enabled, avoids full clone operations by first checking whether
+the last revision is still the same at the target repository,
+and if that is so, skips the reconciliation.
+
+This feature is enabled by default. It can be disabled by starting the
+controller with the argument `--feature-gates=OptimizedGitClones=false`.
+
+NB: GitRepository objects configured for SemVer or Commit clones are
+not affected by this functionality.
+
 #### Proxy support
 
 When a proxy is configured in the source-controller Pod through the appropriate

go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/fluxcd/pkg/gitutil v0.1.0
 	github.com/fluxcd/pkg/helmtestserver v0.7.2
 	github.com/fluxcd/pkg/lockedfile v0.1.0
-	github.com/fluxcd/pkg/runtime v0.14.2
+	github.com/fluxcd/pkg/runtime v0.15.1
 	github.com/fluxcd/pkg/ssh v0.3.3
 	github.com/fluxcd/pkg/testserver v0.2.0
 	github.com/fluxcd/pkg/untar v0.1.0
@@ -185,7 +185,7 @@ require (
 	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
-	github.com/spf13/cobra v1.3.0 // indirect
+	github.com/spf13/cobra v1.4.0 // indirect
 	github.com/stretchr/testify v1.7.1 // indirect
 	github.com/xanzy/ssh-agent v0.3.1 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect

go.sum

@@ -364,8 +364,8 @@ github.com/fluxcd/pkg/helmtestserver v0.7.2/go.mod h1:WtUXBrfpJdwK54LX1Tqd8PpLJY
 github.com/fluxcd/pkg/lockedfile v0.1.0 h1:YsYFAkd6wawMCcD74ikadAKXA4s2sukdxrn7w8RB5eo=
 github.com/fluxcd/pkg/lockedfile v0.1.0/go.mod h1:EJLan8t9MiOcgTs8+puDjbE6I/KAfHbdvIy9VUgIjm8=
 github.com/fluxcd/pkg/runtime v0.13.0-rc.6/go.mod h1:4oKUO19TeudXrnCRnxCfMSS7EQTYpYlgfXwlQuDJ/Eg=
-github.com/fluxcd/pkg/runtime v0.14.2 h1:ktyUjcX4pHoC8DRoBmhEP6eMHbmR6+/MYoARe4YulZY=
+github.com/fluxcd/pkg/runtime v0.15.1 h1:PKooYqlZM+KLhnNz10sQnBH0AHllS40PIDHtiRH/BGU=
-github.com/fluxcd/pkg/runtime v0.14.2/go.mod h1:NZr3PRK7xX2M1bl0LdtugvQyWkOmu2NcW3NrZH6U0is=
+github.com/fluxcd/pkg/runtime v0.15.1/go.mod h1:TPAoOEgUFG60FXBA4ID41uaPldxuXCEI4jt3qfd5i5Q=
 github.com/fluxcd/pkg/ssh v0.3.3 h1:/tc7W7LO1VoVUI5jB+p9ZHCA+iQaXTkaSCDZJsxcZ9k=
 github.com/fluxcd/pkg/ssh v0.3.3/go.mod h1:+bKhuv0/pJy3HZwkK54Shz68sNv1uf5aI6wtPaEHaYk=
 github.com/fluxcd/pkg/testserver v0.2.0 h1:Mj0TapmKaywI6Fi5wvt1LAZpakUHmtzWQpJNKQ0Krt4=
@@ -990,8 +990,9 @@ github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHN
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
 github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk=
-github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0=
 github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4=
+github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q=
+github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=

internal/features/features.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package features sets the feature gates that
+// source-controller supports, and their default
+// states.
+package features
+
+import feathelper "github.com/fluxcd/pkg/runtime/features"
+
+const (
+	// OptimizedGitClones decreases resource utilization for GitRepository
+	// reconciliations. It supports both go-git and libgit2 implementations
+	// when cloning repositories using branches or tags.
+	//
+	// When enabled, avoids full clone operations by first checking whether
+	// the last revision is still the same at the target repository,
+	// and if that is so, skips the reconciliation.
+	OptimizedGitClones = "OptimizedGitClones"
+)
+
+var features = map[string]bool{
+	// OptimizedGitClones
+	// opt-out from v0.25
+	OptimizedGitClones: true,
+}
+
+// DefaultFeatureGates contains a list of all supported feature gates and
+// their default values.
+func FeatureGates() map[string]bool {
+	return features
+}
+
+// Enabled verifies whether the feature is enabled or not.
+//
+// This is only a wrapper around the Enabled func in
+// pkg/runtime/features, so callers won't need to import
+// both packages for checking whether a feature is enabled.
+func Enabled(feature string) (bool, error) {
+	return feathelper.Enabled(feature)
+}

main.go

@@ -36,10 +36,12 @@ import (
 	"github.com/fluxcd/pkg/runtime/client"
 	helper "github.com/fluxcd/pkg/runtime/controller"
 	"github.com/fluxcd/pkg/runtime/events"
+	feathelper "github.com/fluxcd/pkg/runtime/features"
 	"github.com/fluxcd/pkg/runtime/leaderelection"
 	"github.com/fluxcd/pkg/runtime/logger"
 	"github.com/fluxcd/pkg/runtime/pprof"
 	"github.com/fluxcd/pkg/runtime/probes"
+	"github.com/fluxcd/source-controller/internal/features"
 
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
 	"github.com/fluxcd/source-controller/controllers"
@@ -88,6 +90,7 @@ func main() {
 		logOptions               logger.Options
 		leaderElectionOptions    leaderelection.Options
 		rateLimiterOptions       helper.RateLimiterOptions
+		featureGates             feathelper.FeatureGates
 		helmCacheMaxSize         int
 		helmCacheTTL             string
 		helmCachePurgeInterval   string
@@ -136,11 +139,20 @@ func main() {
 	logOptions.BindFlags(flag.CommandLine)
 	leaderElectionOptions.BindFlags(flag.CommandLine)
 	rateLimiterOptions.BindFlags(flag.CommandLine)
+	featureGates.BindFlags(flag.CommandLine)
 
 	flag.Parse()
 
 	ctrl.SetLogger(logger.NewLogger(logOptions))
 
+	err := featureGates.WithLogger(setupLog).
+		SupportedFeatures(features.FeatureGates())
+
+	if err != nil {
+		setupLog.Error(err, "unable to load feature gates")
+		os.Exit(1)
+	}
+
 	// Set upper bound file size limits Helm
 	helm.MaxIndexSize = helmIndexLimit
 	helm.MaxChartSize = helmChartLimit

pkg/git/git.go

@@ -106,3 +106,15 @@ func (c *Commit) ShortMessage() string {
 type CheckoutStrategy interface {
 	Checkout(ctx context.Context, path, url string, config *AuthOptions) (*Commit, error)
 }
+
+// NoChangesError represents the case in which a Git clone operation
+// is attempted, but cancelled as the revision is still the same as
+// the one observed on the last successful reconciliation.
+type NoChangesError struct {
+	Message          string
+	ObservedRevision string
+}
+
+func (e NoChangesError) Error() string {
+	return fmt.Sprintf("%s: observed revision '%s'", e.Message, e.ObservedRevision)
+}

pkg/git/gogit/checkout.go

@@ -26,8 +26,11 @@ import (
 
 	"github.com/Masterminds/semver/v3"
 	extgogit "github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/object"
+	"github.com/go-git/go-git/v5/plumbing/transport"
+	"github.com/go-git/go-git/v5/storage/memory"
 
 	"github.com/fluxcd/pkg/gitutil"
 	"github.com/fluxcd/pkg/version"
@@ -44,19 +47,20 @@ func CheckoutStrategyForOptions(_ context.Context, opts git.CheckoutOptions) git
 	case opts.SemVer != "":
 		return &CheckoutSemVer{SemVer: opts.SemVer, RecurseSubmodules: opts.RecurseSubmodules}
 	case opts.Tag != "":
-		return &CheckoutTag{Tag: opts.Tag, RecurseSubmodules: opts.RecurseSubmodules}
+		return &CheckoutTag{Tag: opts.Tag, RecurseSubmodules: opts.RecurseSubmodules, LastRevision: opts.LastRevision}
 	default:
 		branch := opts.Branch
 		if branch == "" {
 			branch = git.DefaultBranch
 		}
-		return &CheckoutBranch{Branch: branch, RecurseSubmodules: opts.RecurseSubmodules}
+		return &CheckoutBranch{Branch: branch, RecurseSubmodules: opts.RecurseSubmodules, LastRevision: opts.LastRevision}
 	}
 }
 
 type CheckoutBranch struct {
 	Branch            string
 	RecurseSubmodules bool
+	LastRevision      string
 }
 
 func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (*git.Commit, error) {
@@ -64,7 +68,23 @@ func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *g
 	if err != nil {
 		return nil, fmt.Errorf("failed to construct auth method with options: %w", err)
 	}
+
 	ref := plumbing.NewBranchReferenceName(c.Branch)
+	// check if previous revision has changed before attempting to clone
+	if c.LastRevision != "" {
+		currentRevision, err := getLastRevision(ctx, url, ref, opts, authMethod)
+		if err != nil {
+			return nil, err
+		}
+
+		if currentRevision != "" && currentRevision == c.LastRevision {
+			return nil, git.NoChangesError{
+				Message:          "no changes since last reconcilation",
+				ObservedRevision: currentRevision,
+			}
+		}
+	}
+
 	repo, err := extgogit.PlainCloneContext(ctx, path, false, &extgogit.CloneOptions{
 		URL:               url,
 		Auth:              authMethod,
@@ -92,9 +112,31 @@ func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *g
 	return buildCommitWithRef(cc, ref)
 }
 
+func getLastRevision(ctx context.Context, url string, ref plumbing.ReferenceName, opts *git.AuthOptions, authMethod transport.AuthMethod) (string, error) {
+	config := &config.RemoteConfig{
+		Name: git.DefaultOrigin,
+		URLs: []string{url},
+	}
+	rem := extgogit.NewRemote(memory.NewStorage(), config)
+	listOpts := &extgogit.ListOptions{
+		Auth: authMethod,
+	}
+	if opts != nil && opts.CAFile != nil {
+		listOpts.CABundle = opts.CAFile
+	}
+	refs, err := rem.ListContext(ctx, listOpts)
+	if err != nil {
+		return "", fmt.Errorf("unable to list remote for '%s': %w", url, err)
+	}
+
+	currentRevision := filterRefs(refs, ref)
+	return currentRevision, nil
+}
+
 type CheckoutTag struct {
 	Tag               string
 	RecurseSubmodules bool
+	LastRevision      string
 }
 
 func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (*git.Commit, error) {
@@ -103,6 +145,20 @@ func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, opts *git.
 		return nil, fmt.Errorf("failed to construct auth method with options: %w", err)
 	}
 	ref := plumbing.NewTagReferenceName(c.Tag)
+	// check if previous revision has changed before attempting to clone
+	if c.LastRevision != "" {
+		currentRevision, err := getLastRevision(ctx, url, ref, opts, authMethod)
+		if err != nil {
+			return nil, err
+		}
+
+		if currentRevision != "" && currentRevision == c.LastRevision {
+			return nil, git.NoChangesError{
+				Message:          "no changes since last reconcilation",
+				ObservedRevision: currentRevision,
+			}
+		}
+	}
 	repo, err := extgogit.PlainCloneContext(ctx, path, false, &extgogit.CloneOptions{
 		URL:               url,
 		Auth:              authMethod,
@@ -333,3 +389,13 @@ func recurseSubmodules(recurse bool) extgogit.SubmoduleRescursivity {
 	}
 	return extgogit.NoRecurseSubmodules
 }
+
+func filterRefs(refs []*plumbing.Reference, currentRef plumbing.ReferenceName) string {
+	for _, ref := range refs {
+		if ref.Name().String() == currentRef.String() {
+			return fmt.Sprintf("%s/%s", currentRef.Short(), ref.Hash().String())
+		}
+	}
+
+	return ""
+}

pkg/git/gogit/checkout_test.go

@@ -72,6 +72,7 @@ func TestCheckoutBranch_Checkout(t *testing.T) {
 		filesCreated   map[string]string
 		expectedCommit string
 		expectedErr    string
+		lastRevision   string
 	}{
 		{
 			name:           "Default branch",
@@ -80,10 +81,18 @@ func TestCheckoutBranch_Checkout(t *testing.T) {
 			expectedCommit: firstCommit.String(),
 		},
 		{
-			name:           "Other branch",
+			name:         "skip clone if LastRevision hasn't changed",
+			branch:       "master",
+			filesCreated: map[string]string{"branch": "init"},
+			expectedErr:  fmt.Sprintf("no changes since last reconcilation: observed revision 'master/%s'", firstCommit.String()),
+			lastRevision: fmt.Sprintf("master/%s", firstCommit.String()),
+		},
+		{
+			name:           "Other branch - revision has changed",
 			branch:         "test",
 			filesCreated:   map[string]string{"branch": "second"},
 			expectedCommit: secondCommit.String(),
+			lastRevision:   fmt.Sprintf("master/%s", firstCommit.String()),
 		},
 		{
 			name:        "Non existing branch",
@@ -97,7 +106,8 @@ func TestCheckoutBranch_Checkout(t *testing.T) {
 			g := NewWithT(t)
 
 			branch := CheckoutBranch{
-				Branch: tt.branch,
+				Branch:       tt.branch,
+				LastRevision: tt.lastRevision,
 			}
 			tmpDir := t.TempDir()
 
@@ -127,6 +137,8 @@ func TestCheckoutTag_Checkout(t *testing.T) {
 		checkoutTag string
 		expectTag   string
 		expectErr   string
+		lastRev     string
+		setLastRev  bool
 	}{
 		{
 			name:        "Tag",
@@ -134,6 +146,20 @@ func TestCheckoutTag_Checkout(t *testing.T) {
 			checkoutTag: "tag-1",
 			expectTag:   "tag-1",
 		},
+		{
+			name:        "Skip Tag if last revision hasn't changed",
+			tag:         "tag-2",
+			checkoutTag: "tag-2",
+			setLastRev:  true,
+			expectErr:   "no changes since last reconcilation",
+		},
+		{
+			name:        "Last revision changed",
+			tag:         "tag-3",
+			checkoutTag: "tag-3",
+			expectTag:   "tag-3",
+			lastRev:     "tag-3/<fake-hash>",
+		},
 		{
 			name:        "Annotated",
 			tag:         "annotated",
@@ -158,12 +184,13 @@ func TestCheckoutTag_Checkout(t *testing.T) {
 			}
 
 			var h plumbing.Hash
+			var tagHash *plumbing.Reference
 			if tt.tag != "" {
 				h, err = commitFile(repo, "tag", tt.tag, time.Now())
 				if err != nil {
 					t.Fatal(err)
 				}
-				_, err = tag(repo, h, !tt.annotated, tt.tag, time.Now())
+				tagHash, err = tag(repo, h, !tt.annotated, tt.tag, time.Now())
 				if err != nil {
 					t.Fatal(err)
 				}
@@ -172,10 +199,18 @@ func TestCheckoutTag_Checkout(t *testing.T) {
 			tag := CheckoutTag{
 				Tag: tt.checkoutTag,
 			}
+			if tt.setLastRev {
+				tag.LastRevision = fmt.Sprintf("%s/%s", tt.tag, tagHash.Hash().String())
+			}
+
+			if tt.lastRev != "" {
+				tag.LastRevision = tt.lastRev
+			}
 			tmpDir := t.TempDir()
 
 			cc, err := tag.Checkout(context.TODO(), tmpDir, path, nil)
 			if tt.expectErr != "" {
+				g.Expect(err).ToNot(BeNil())
 				g.Expect(err.Error()).To(ContainSubstring(tt.expectErr))
 				g.Expect(cc).To(BeNil())
 				return

pkg/git/libgit2/checkout.go

@@ -52,59 +52,149 @@ func CheckoutStrategyForOptions(ctx context.Context, opt git.CheckoutOptions) gi
 		if branch == "" {
 			branch = git.DefaultBranch
 		}
-		return &CheckoutBranch{Branch: branch}
+		return &CheckoutBranch{
+			Branch:       branch,
+			LastRevision: opt.LastRevision,
+		}
 	}
 }
 
 type CheckoutBranch struct {
-	Branch string
+	Branch       string
+	LastRevision string
 }
 
-func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (*git.Commit, error) {
+func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) {
-	repo, err := safeClone(url, path, &git2go.CloneOptions{
+	defer recoverPanic(&err)
-		FetchOptions: git2go.FetchOptions{
+
+	repo, remote, free, err := getBlankRepoAndRemote(ctx, path, url, opts)
+	if err != nil {
+		return nil, err
+	}
+	defer free()
+
+	// When the last observed revision is set, check whether it is still
+	// the same at the remote branch. If so, short-circuit the clone operation here.
+	if c.LastRevision != "" {
+		heads, err := remote.Ls(c.Branch)
+		if err != nil {
+			return nil, fmt.Errorf("unable to remote ls for '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
+		}
+		if len(heads) > 0 {
+			currentRevision := fmt.Sprintf("%s/%s", c.Branch, heads[0].Id.String())
+			if currentRevision == c.LastRevision {
+				return nil, git.NoChangesError{
+					Message:          "no changes since last reconciliation",
+					ObservedRevision: currentRevision,
+				}
+			}
+		}
+	}
+
+	// Limit the fetch operation to the specific branch, to decrease network usage.
+	err = remote.Fetch([]string{c.Branch},
+		&git2go.FetchOptions{
 			DownloadTags:    git2go.DownloadTagsNone,
 			RemoteCallbacks: RemoteCallbacks(ctx, opts),
 			ProxyOptions:    git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
 		},
-		CheckoutOptions: git2go.CheckoutOptions{
+		"")
-			Strategy: git2go.CheckoutForce,
+	if err != nil {
-		},
+		return nil, fmt.Errorf("unable to fetch remote '%s': %w",
-		CheckoutBranch: c.Branch,
+			managed.EffectiveURL(url), gitutil.LibGit2Error(err))
+	}
+
+	branch, err := repo.References.Lookup(fmt.Sprintf("refs/remotes/origin/%s", c.Branch))
+	if err != nil {
+		return nil, fmt.Errorf("unable to lookup branch '%s' for '%s': %w",
+			c.Branch, managed.EffectiveURL(url), gitutil.LibGit2Error(err))
+	}
+	defer branch.Free()
+
+	upstreamCommit, err := repo.LookupCommit(branch.Target())
+	if err != nil {
+		return nil, fmt.Errorf("unable to lookup commit '%s' for '%s': %w",
+			c.Branch, managed.EffectiveURL(url), gitutil.LibGit2Error(err))
+	}
+	defer upstreamCommit.Free()
+
+	// Once the index has been updated with Fetch, and we know the tip commit,
+	// a hard reset can be used to align the local worktree with the remote branch's.
+	err = repo.ResetToCommit(upstreamCommit, git2go.ResetHard, &git2go.CheckoutOptions{
+		Strategy: git2go.CheckoutForce,
 	})
 	if err != nil {
-		return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
+		return nil, fmt.Errorf("unable to hard reset to commit for '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
 	}
-	defer repo.Free()
+
+	// Use the current worktree's head as reference for the commit to be returned.
 	head, err := repo.Head()
 	if err != nil {
 		return nil, fmt.Errorf("git resolve HEAD error: %w", err)
 	}
 	defer head.Free()
+
 	cc, err := repo.LookupCommit(head.Target())
 	if err != nil {
 		return nil, fmt.Errorf("failed to lookup HEAD commit '%s' for branch '%s': %w", head.Target(), c.Branch, err)
 	}
 	defer cc.Free()
+
 	return buildCommit(cc, "refs/heads/"+c.Branch), nil
 }
 
 type CheckoutTag struct {
-	Tag string
+	Tag          string
+	LastRevision string
 }
 
-func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (*git.Commit, error) {
+func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) {
-	repo, err := safeClone(url, path, &git2go.CloneOptions{
+	defer recoverPanic(&err)
-		FetchOptions: git2go.FetchOptions{
+
-			DownloadTags:    git2go.DownloadTagsAll,
+	repo, remote, free, err := getBlankRepoAndRemote(ctx, path, url, opts)
+	if err != nil {
+		return nil, err
+	}
+	defer free()
+
+	if c.LastRevision != "" {
+		heads, err := remote.Ls(c.Tag)
+		if err != nil {
+			return nil, fmt.Errorf("unable to remote ls for '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
+		}
+		if len(heads) > 0 {
+			currentRevision := fmt.Sprintf("%s/%s", c.Tag, heads[0].Id.String())
+			var same bool
+			if currentRevision == c.LastRevision {
+				same = true
+			} else if len(heads) > 1 {
+				currentAnnotatedRevision := fmt.Sprintf("%s/%s", c.Tag, heads[1].Id.String())
+				if currentAnnotatedRevision == c.LastRevision {
+					same = true
+				}
+			}
+			if same {
+				return nil, git.NoChangesError{
+					Message:          "no changes since last reconciliation",
+					ObservedRevision: currentRevision,
+				}
+			}
+		}
+	}
+
+	err = remote.Fetch([]string{c.Tag},
+		&git2go.FetchOptions{
+			DownloadTags:    git2go.DownloadTagsAuto,
 			RemoteCallbacks: RemoteCallbacks(ctx, opts),
 			ProxyOptions:    git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
 		},
-	})
+		"")
+
 	if err != nil {
-		return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
+		return nil, fmt.Errorf("unable to fetch remote '%s': %w",
+			managed.EffectiveURL(url), gitutil.LibGit2Error(err))
 	}
-	defer repo.Free()
+
 	cc, err := checkoutDetachedDwim(repo, c.Tag)
 	if err != nil {
 		return nil, err
@@ -117,8 +207,10 @@ type CheckoutCommit struct {
 	Commit string
 }
 
-func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (*git.Commit, error) {
+func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) {
-	repo, err := safeClone(url, path, &git2go.CloneOptions{
+	defer recoverPanic(&err)
+
+	repo, err := git2go.Clone(url, path, &git2go.CloneOptions{
 		FetchOptions: git2go.FetchOptions{
 			DownloadTags:    git2go.DownloadTagsNone,
 			RemoteCallbacks: RemoteCallbacks(ctx, opts),
@@ -144,13 +236,15 @@ type CheckoutSemVer struct {
 	SemVer string
 }
 
-func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (*git.Commit, error) {
+func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) {
+	defer recoverPanic(&err)
+
 	verConstraint, err := semver.NewConstraint(c.SemVer)
 	if err != nil {
 		return nil, fmt.Errorf("semver parse error: %w", err)
 	}
 
-	repo, err := safeClone(url, path, &git2go.CloneOptions{
+	repo, err := git2go.Clone(url, path, &git2go.CloneOptions{
 		FetchOptions: git2go.FetchOptions{
 			DownloadTags:    git2go.DownloadTagsAll,
 			RemoteCallbacks: RemoteCallbacks(ctx, opts),
@@ -239,19 +333,6 @@ func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, opts *g
 	return buildCommit(cc, "refs/tags/"+t), nil
 }
 
-// safeClone wraps git2go calls with panic recovering logic, ensuring
-// a predictable execution path for callers.
-func safeClone(url, path string, cloneOpts *git2go.CloneOptions) (repo *git2go.Repository, err error) {
-	defer func() {
-		if r := recover(); r != nil {
-			err = fmt.Errorf("recovered from git2go panic: %v", r)
-		}
-	}()
-
-	repo, err = git2go.Clone(url, path, cloneOpts)
-	return
-}
-
 // checkoutDetachedDwim attempts to perform a detached HEAD checkout by first DWIMing the short name
 // to get a concrete reference, and then calling checkoutDetachedHEAD.
 func checkoutDetachedDwim(repo *git2go.Repository, name string) (*git2go.Commit, error) {
@@ -326,3 +407,39 @@ func buildSignature(s *git2go.Signature) git.Signature {
 		When:  s.When,
 	}
 }
+
+// getBlankRepoAndRemote returns a newly initialized repository, and a remote connected to the provided url.
+// Callers must call the returning function to free all git2go objects.
+func getBlankRepoAndRemote(ctx context.Context, path, url string, opts *git.AuthOptions) (*git2go.Repository, *git2go.Remote, func(), error) {
+	repo, err := git2go.InitRepository(path, false)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("unable to init repository for '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
+	}
+
+	remote, err := repo.Remotes.Create("origin", url)
+	if err != nil {
+		repo.Free()
+		return nil, nil, nil, fmt.Errorf("unable to create remote for '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
+	}
+
+	callBacks := RemoteCallbacks(ctx, opts)
+	err = remote.ConnectFetch(&callBacks, &git2go.ProxyOptions{Type: git2go.ProxyTypeAuto}, nil)
+	if err != nil {
+		remote.Free()
+		repo.Free()
+		return nil, nil, nil, fmt.Errorf("unable to fetch-connect to remote '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
+	}
+
+	free := func() {
+		remote.Disconnect()
+		remote.Free()
+		repo.Free()
+	}
+	return repo, remote, free, nil
+}
+
+func recoverPanic(err *error) {
+	if r := recover(); r != nil {
+		*err = fmt.Errorf("recovered from git2go panic: %v", r)
+	}
+}

pkg/git/libgit2/checkout_test.go

@@ -51,8 +51,19 @@ func TestCheckoutBranch_Checkout(t *testing.T) {
 	// ignores the error here because it can be defaulted
 	// https://github.blog/2020-07-27-highlights-from-git-2-28/#introducing-init-defaultbranch
 	defaultBranch := "master"
-	if v, err := cfg.LookupString("init.defaultBranch"); err != nil && v != "" {
+	iter, err := cfg.NewIterator()
-		defaultBranch = v
+	if err != nil {
+		t.Fatal(err)
+	}
+	for {
+		val, e := iter.Next()
+		if e != nil {
+			break
+		}
+		if val.Name == "init.defaultbranch" {
+			defaultBranch = val.Value
+			break
+		}
 	}
 
 	firstCommit, err := commitFile(repo, "branch", "init", time.Now())
@@ -77,6 +88,7 @@ func TestCheckoutBranch_Checkout(t *testing.T) {
 		filesCreated   map[string]string
 		expectedCommit string
 		expectedErr    string
+		lastRevision   string
 	}{
 		{
 			name:           "Default branch",
@@ -95,6 +107,21 @@ func TestCheckoutBranch_Checkout(t *testing.T) {
 			branch:      "invalid",
 			expectedErr: "reference 'refs/remotes/origin/invalid' not found",
 		},
+		{
+			name:           "skip clone - lastRevision hasn't changed",
+			branch:         defaultBranch,
+			filesCreated:   map[string]string{"branch": "second"},
+			expectedCommit: secondCommit.String(),
+			lastRevision:   fmt.Sprintf("%s/%s", defaultBranch, secondCommit.String()),
+			expectedErr:    fmt.Sprintf("no changes since last reconciliation: observed revision '%s/%s'", defaultBranch, secondCommit.String()),
+		},
+		{
+			name:           "lastRevision is different",
+			branch:         defaultBranch,
+			filesCreated:   map[string]string{"branch": "second"},
+			expectedCommit: secondCommit.String(),
+			lastRevision:   fmt.Sprintf("%s/%s", defaultBranch, firstCommit.String()),
+		},
 	}
 
 	for _, tt := range tests {
@@ -102,7 +129,8 @@ func TestCheckoutBranch_Checkout(t *testing.T) {
 			g := NewWithT(t)
 
 			branch := CheckoutBranch{
-				Branch: tt.branch,
+				Branch:       tt.branch,
+				LastRevision: tt.lastRevision,
 			}
 			tmpDir := t.TempDir()
 
@@ -126,12 +154,13 @@ func TestCheckoutBranch_Checkout(t *testing.T) {
 
 func TestCheckoutTag_Checkout(t *testing.T) {
 	tests := []struct {
-		name        string
+		name         string
-		tag         string
+		tag          string
-		annotated   bool
+		annotated    bool
-		checkoutTag string
+		checkoutTag  string
-		expectTag   string
+		expectTag    string
-		expectErr   string
+		expectErr    string
+		lastRevision bool
 	}{
 		{
 			name:        "Tag",
@@ -151,6 +180,21 @@ func TestCheckoutTag_Checkout(t *testing.T) {
 			checkoutTag: "invalid",
 			expectErr:   "unable to find 'invalid': no reference found for shorthand 'invalid'",
 		},
+		{
+			name:         "skip clone - last revision is unchanged",
+			tag:          "tag-1",
+			checkoutTag:  "tag-1",
+			expectTag:    "tag-1",
+			lastRevision: true,
+			expectErr:    "no changes since last reconciliation",
+		},
+		{
+			name:         "last revision changed",
+			tag:          "tag-1",
+			checkoutTag:  "tag-1",
+			expectTag:    "tag-2",
+			lastRevision: true,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -171,29 +215,60 @@ func TestCheckoutTag_Checkout(t *testing.T) {
 				if commit, err = repo.LookupCommit(c); err != nil {
 					t.Fatal(err)
 				}
-				_, err = tag(repo, c, !tt.annotated, tt.tag, time.Now())
+				_, err = tag(repo, commit.Id(), !tt.annotated, tt.tag, time.Now())
 				if err != nil {
 					t.Fatal(err)
 				}
 			}
 
-			tag := CheckoutTag{
+			checkoutTag := CheckoutTag{
 				Tag: tt.checkoutTag,
 			}
 			tmpDir := t.TempDir()
 
-			cc, err := tag.Checkout(context.TODO(), tmpDir, repo.Path(), nil)
+			cc, err := checkoutTag.Checkout(context.TODO(), tmpDir, repo.Path(), nil)
+
 			if tt.expectErr != "" {
+				if tt.lastRevision {
+					tmpDir, _ = os.MkdirTemp("", "test")
+					defer os.RemoveAll(tmpDir)
+					checkoutTag.LastRevision = cc.String()
+					cc, err = checkoutTag.Checkout(context.TODO(), tmpDir, repo.Path(), nil)
+				}
 				g.Expect(err).To(HaveOccurred())
 				g.Expect(err.Error()).To(ContainSubstring(tt.expectErr))
 				g.Expect(cc).To(BeNil())
 				return
 			}
+			if tt.lastRevision {
+				checkoutTag.LastRevision = fmt.Sprintf("%s/%s", tt.tag, commit.Id().String())
+				checkoutTag.Tag = tt.expectTag
+				if tt.tag != "" {
+					c, err := commitFile(repo, "tag", "changed tag", time.Now())
+					if err != nil {
+						t.Fatal(err)
+					}
+					if commit, err = repo.LookupCommit(c); err != nil {
+						t.Fatal(err)
+					}
+					_, err = tag(repo, commit.Id(), !tt.annotated, tt.expectTag, time.Now())
+					if err != nil {
+						t.Fatal(err)
+					}
+					tmpDir, _ = os.MkdirTemp("", "test")
+					defer os.RemoveAll(tmpDir)
+					cc, err = checkoutTag.Checkout(context.TODO(), tmpDir, repo.Path(), nil)
+				}
+			}
 
 			g.Expect(err).ToNot(HaveOccurred())
 			g.Expect(cc.String()).To(Equal(tt.expectTag + "/" + commit.Id().String()))
 			g.Expect(filepath.Join(tmpDir, "tag")).To(BeARegularFile())
-			g.Expect(os.ReadFile(filepath.Join(tmpDir, "tag"))).To(BeEquivalentTo(tt.tag))
+			if tt.lastRevision {
+				g.Expect(os.ReadFile(filepath.Join(tmpDir, "tag"))).To(BeEquivalentTo("changed tag"))
+			} else {
+				g.Expect(os.ReadFile(filepath.Join(tmpDir, "tag"))).To(BeEquivalentTo(tt.tag))
+			}
 		})
 	}
 }
@@ -506,42 +581,3 @@ func TestCheckout_ED25519(t *testing.T) {
 	_, err = branchCheckoutStrat.Checkout(ctx, tmpDir, repoURL, authOpts)
 	g.Expect(err).ToNot(HaveOccurred())
 }
-
-func TestSafeClone(t *testing.T) {
-	g := NewWithT(t)
-
-	// Create a git test server.
-	server, err := gittestserver.NewTempGitServer()
-	g.Expect(err).ToNot(HaveOccurred())
-	defer os.RemoveAll(server.Root())
-	server.Auth("test-user", "test-pswd")
-	server.AutoCreate()
-
-	server.KeyDir(filepath.Join(server.Root(), "keys"))
-	g.Expect(server.ListenSSH()).To(Succeed())
-
-	go func() {
-		server.StartSSH()
-	}()
-	defer server.StopSSH()
-
-	sshURL := server.SSHAddress()
-	repoURL := sshURL + "/test.git"
-
-	u, err := url.Parse(sshURL)
-	g.Expect(err).NotTo(HaveOccurred())
-	g.Expect(u.Host).ToNot(BeEmpty())
-
-	repo, err := safeClone(repoURL, t.TempDir(), &git2go.CloneOptions{
-		FetchOptions: git2go.FetchOptions{
-			RemoteCallbacks: git2go.RemoteCallbacks{
-				CertificateCheckCallback: func(cert *git2go.Certificate, valid bool, hostname string) error {
-					panic("Oops!")
-				},
-			},
-		}})
-
-	g.Expect(repo).To(BeNil())
-	g.Expect(err).To(HaveOccurred())
-	g.Expect(err.Error()).Should(ContainSubstring("recovered from git2go panic"))
-}

pkg/git/options.go

@@ -48,6 +48,11 @@ type CheckoutOptions struct {
 	// RecurseSubmodules defines if submodules should be checked out,
 	// not supported by all Implementations.
 	RecurseSubmodules bool
+
+	// LastRevision holds the revision observed on the last successful
+	// reconciliation.
+	// It is used to skip clone operations when no changes were detected.
+	LastRevision string
 }
 
 type TransportType string