diff --git a/controllers/gitrepository_controller.go b/controllers/gitrepository_controller.go index 98dd98c5..1a9c7895 100644 --- a/controllers/gitrepository_controller.go +++ b/controllers/gitrepository_controller.go @@ -236,6 +236,7 @@ func (r *GitRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reques r.reconcileInclude, r.reconcileArtifact, } + recResult, retErr = r.reconcile(ctx, obj, reconcilers) return } @@ -428,6 +429,13 @@ func (r *GitRepositoryReconciler) reconcileStorage(ctx context.Context, // change, it short-circuits the whole reconciliation with an early return. func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, obj *sourcev1.GitRepository, commit *git.Commit, includes *artifactSet, dir string) (sreconcile.Result, error) { + // Exit early, if we need to use libgit2 AND managed transport hasn't been intialized. + if !managed.Enabled() && obj.Spec.GitImplementation == sourcev1.LibGit2Implementation { + fmt.Println(managed.Enabled()) + return sreconcile.ResultEmpty, serror.NewStalling( + errors.New("libgit2 managed transport not initialized"), "Libgit2TransportNotEnabled", + ) + } // Configure authentication strategy to access the source var authOpts *git.AuthOptions var err error @@ -745,8 +753,8 @@ func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context, return nil, e } - // managed GIT transport only affects the libgit2 implementation - if managed.Enabled() && obj.Spec.GitImplementation == sourcev1.LibGit2Implementation { + // this is needed only for libgit2, due to managed transport. + if obj.Spec.GitImplementation == sourcev1.LibGit2Implementation { // We set the TransportOptionsURL of this set of authentication options here by constructing // a unique URL that won't clash in a multi tenant environment. This unique URL is used by // libgit2 managed transports. This enables us to bypass the inbuilt credentials callback in diff --git a/controllers/suite_test.go b/controllers/suite_test.go index a633bbcd..43053e2d 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -37,7 +37,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "github.com/fluxcd/pkg/runtime/controller" - feathelper "github.com/fluxcd/pkg/runtime/features" "github.com/fluxcd/pkg/runtime/testenv" "github.com/fluxcd/pkg/testserver" "github.com/phayes/freeport" @@ -206,8 +205,6 @@ func TestMain(m *testing.M) { panic(fmt.Sprintf("Failed to create a test registry server: %v", err)) } - fg := feathelper.FeatureGates{} - fg.SupportedFeatures(features.FeatureGates()) managed.InitManagedTransport() if err := (&GitRepositoryReconciler{ diff --git a/internal/features/features.go b/internal/features/features.go index 9cc2cfd1..0449cf41 100644 --- a/internal/features/features.go +++ b/internal/features/features.go @@ -30,25 +30,12 @@ const ( // the last revision is still the same at the target repository, // and if that is so, skips the reconciliation. OptimizedGitClones = "OptimizedGitClones" - - // GitManagedTransport implements a managed transport for GitRepository - // objects that use the libgit2 implementation. - // - // When enabled, improves the reliability of libgit2 reconciliations, - // by enforcing timeouts and ensuring libgit2 cannot hijack the process - // and hang it indefinitely. - GitManagedTransport = "GitManagedTransport" ) var features = map[string]bool{ // OptimizedGitClones // opt-out from v0.25 OptimizedGitClones: true, - - // GitManagedTransport - // opt-in from v0.22 (via environment variable) - // opt-out from v0.25 - GitManagedTransport: true, } // DefaultFeatureGates contains a list of all supported feature gates and diff --git a/main.go b/main.go index e19f1ddb..1f209d20 100644 --- a/main.go +++ b/main.go @@ -310,15 +310,9 @@ func main() { startFileServer(storage.BasePath, storageAddr, setupLog) }() - if enabled, _ := features.Enabled(features.GitManagedTransport); enabled { - managed.InitManagedTransport() - } else { - if optimize, _ := feathelper.Enabled(features.OptimizedGitClones); optimize { - features.Disable(features.OptimizedGitClones) - setupLog.Info( - "disabling optimized git clones; git clones can only be optimized when using managed transport", - ) - } + if err = managed.InitManagedTransport(); err != nil { + // Log the error, but don't exit so as to not block reconcilers that are healthy. + setupLog.Error(err, "unable to initialize libgit2 managed transport") } setupLog.Info("starting manager") diff --git a/pkg/git/libgit2/checkout.go b/pkg/git/libgit2/checkout.go index 056dc0b1..6fdd8a28 100644 --- a/pkg/git/libgit2/checkout.go +++ b/pkg/git/libgit2/checkout.go @@ -18,6 +18,7 @@ package libgit2 import ( "context" + "errors" "fmt" "sort" "strings" @@ -72,177 +73,122 @@ type CheckoutBranch struct { func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) { defer recoverPanic(&err) - // This branching is temporary, to address the transient panics observed when using unmanaged transport. - // The panics probably happen because we perform multiple fetch ops (introduced as a part of optimizing git clones). - // The branching lets us establish a clear code path to help us be certain of the expected behaviour. - // When we get rid of unmanaged transports, we can get rid of this branching as well. - if managed.Enabled() { - // We store the target URL and auth options mapped to a unique ID. We overwrite the target URL - // with the TransportOptionsURL, because managed transports don't provide a way for any kind of - // dependency injection. This lets us have a way of doing interop between application level code - // and transport level code. - // Performing all fetch operations with the TransportOptionsURL as the URL, lets the managed - // transport action use it to fetch the registered transport options which contains the - // _actual_ target URL and the correct credentials to use. - if opts == nil { - return nil, fmt.Errorf("can't use managed transport with an empty set of auth options") - } - if opts.TransportOptionsURL == "" { - return nil, fmt.Errorf("can't use managed transport without a valid transport auth id.") - } - managed.AddTransportOptions(opts.TransportOptionsURL, managed.TransportOptions{ - TargetURL: url, - AuthOpts: opts, - ProxyOptions: &git2go.ProxyOptions{Type: git2go.ProxyTypeAuto}, - Context: ctx, - }) - url = opts.TransportOptionsURL - remoteCallBacks := managed.RemoteCallbacks() - defer managed.RemoveTransportOptions(opts.TransportOptionsURL) - - repo, remote, err := initializeRepoWithRemote(ctx, path, url, opts) - if err != nil { - return nil, err - } - // Open remote connection. - err = remote.ConnectFetch(&remoteCallBacks, nil, nil) - if err != nil { - remote.Free() - repo.Free() - return nil, fmt.Errorf("unable to fetch-connect to remote '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err)) - } - defer func() { - remote.Disconnect() - remote.Free() - repo.Free() - }() - - // When the last observed revision is set, check whether it is still the - // same at the remote branch. If so, short-circuit the clone operation here. - if c.LastRevision != "" { - heads, err := remote.Ls(c.Branch) - if err != nil { - return nil, fmt.Errorf("unable to remote ls for '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err)) - } - if len(heads) > 0 { - hash := heads[0].Id.String() - currentRevision := fmt.Sprintf("%s/%s", c.Branch, hash) - if currentRevision == c.LastRevision { - // Construct a partial commit with the existing information. - c := &git.Commit{ - Hash: git.Hash(hash), - Reference: "refs/heads/" + c.Branch, - } - return c, nil - } - } - } - - // Limit the fetch operation to the specific branch, to decrease network usage. - err = remote.Fetch([]string{c.Branch}, - &git2go.FetchOptions{ - DownloadTags: git2go.DownloadTagsNone, - RemoteCallbacks: remoteCallBacks, - }, - "") - if err != nil { - return nil, fmt.Errorf("unable to fetch remote '%s': %w", - managed.EffectiveURL(url), gitutil.LibGit2Error(err)) - } - - branch, err := repo.References.Lookup(fmt.Sprintf("refs/remotes/origin/%s", c.Branch)) - if err != nil { - return nil, fmt.Errorf("unable to lookup branch '%s' for '%s': %w", - c.Branch, managed.EffectiveURL(url), gitutil.LibGit2Error(err)) - } - defer branch.Free() - - upstreamCommit, err := repo.LookupCommit(branch.Target()) - if err != nil { - return nil, fmt.Errorf("unable to lookup commit '%s' for '%s': %w", - c.Branch, managed.EffectiveURL(url), gitutil.LibGit2Error(err)) - } - defer upstreamCommit.Free() - - // We try to lookup the branch (and create it if it doesn't exist), so that we can - // switch the repo to the specified branch. This is done so that users of this api - // can expect the repo to be at the desired branch, when cloned. - localBranch, err := repo.LookupBranch(c.Branch, git2go.BranchLocal) - if git2go.IsErrorCode(err, git2go.ErrorCodeNotFound) { - localBranch, err = repo.CreateBranch(c.Branch, upstreamCommit, false) - if err != nil { - return nil, fmt.Errorf("unable to create local branch '%s': %w", c.Branch, err) - } - } else if err != nil { - return nil, fmt.Errorf("unable to lookup branch '%s': %w", c.Branch, err) - } - defer localBranch.Free() - - tree, err := repo.LookupTree(upstreamCommit.TreeId()) - if err != nil { - return nil, fmt.Errorf("unable to lookup tree for branch '%s': %w", c.Branch, err) - } - defer tree.Free() - - err = repo.CheckoutTree(tree, &git2go.CheckoutOpts{ - // the remote branch should take precedence if it exists at this point in time. - Strategy: git2go.CheckoutForce, - }) - if err != nil { - return nil, fmt.Errorf("unable to checkout tree for branch '%s': %w", c.Branch, err) - } - - // Set the current head to point to the requested branch. - err = repo.SetHead("refs/heads/" + c.Branch) - if err != nil { - return nil, fmt.Errorf("unable to set HEAD to branch '%s':%w", c.Branch, err) - } - - // Use the current worktree's head as reference for the commit to be returned. - head, err := repo.Head() - if err != nil { - return nil, fmt.Errorf("unable to resolve HEAD: %w", err) - } - defer head.Free() - - cc, err := repo.LookupCommit(head.Target()) - if err != nil { - return nil, fmt.Errorf("unable to lookup HEAD commit '%s' for branch '%s': %w", head.Target(), c.Branch, err) - } - defer cc.Free() - - return buildCommit(cc, "refs/heads/"+c.Branch), nil - } else { - return c.checkoutUnmanaged(ctx, path, url, opts) + err = registerManagedTransportOptions(ctx, url, opts) + if err != nil { + return nil, err } -} + transportOptsURL := opts.TransportOptionsURL + remoteCallBacks := managed.RemoteCallbacks() + defer managed.RemoveTransportOptions(transportOptsURL) -func (c *CheckoutBranch) checkoutUnmanaged(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) { - repo, err := git2go.Clone(url, path, &git2go.CloneOptions{ - FetchOptions: git2go.FetchOptions{ + repo, remote, err := initializeRepoWithRemote(ctx, path, url, opts) + if err != nil { + return nil, err + } + // Open remote connection. + err = remote.ConnectFetch(&remoteCallBacks, nil, nil) + if err != nil { + remote.Free() + repo.Free() + return nil, fmt.Errorf("unable to fetch-connect to remote '%s': %w", url, gitutil.LibGit2Error(err)) + } + defer func() { + remote.Disconnect() + remote.Free() + repo.Free() + }() + + // When the last observed revision is set, check whether it is still the + // same at the remote branch. If so, short-circuit the clone operation here. + if c.LastRevision != "" { + heads, err := remote.Ls(c.Branch) + if err != nil { + return nil, fmt.Errorf("unable to remote ls for '%s': %w", url, gitutil.LibGit2Error(err)) + } + if len(heads) > 0 { + hash := heads[0].Id.String() + currentRevision := fmt.Sprintf("%s/%s", c.Branch, hash) + if currentRevision == c.LastRevision { + // Construct a partial commit with the existing information. + c := &git.Commit{ + Hash: git.Hash(hash), + Reference: "refs/heads/" + c.Branch, + } + return c, nil + } + } + } + + // Limit the fetch operation to the specific branch, to decrease network usage. + err = remote.Fetch([]string{c.Branch}, + &git2go.FetchOptions{ DownloadTags: git2go.DownloadTagsNone, - RemoteCallbacks: RemoteCallbacks(ctx, opts), - ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto}, + RemoteCallbacks: remoteCallBacks, }, - CheckoutOptions: git2go.CheckoutOptions{ - Strategy: git2go.CheckoutForce, - }, - CheckoutBranch: c.Branch, + "") + if err != nil { + return nil, fmt.Errorf("unable to fetch remote '%s': %w", url, gitutil.LibGit2Error(err)) + } + + branch, err := repo.References.Lookup(fmt.Sprintf("refs/remotes/origin/%s", c.Branch)) + if err != nil { + return nil, fmt.Errorf("unable to lookup branch '%s' for '%s': %w", c.Branch, url, gitutil.LibGit2Error(err)) + } + defer branch.Free() + + upstreamCommit, err := repo.LookupCommit(branch.Target()) + if err != nil { + return nil, fmt.Errorf("unable to lookup commit '%s' for '%s': %w", c.Branch, url, gitutil.LibGit2Error(err)) + } + defer upstreamCommit.Free() + + // We try to lookup the branch (and create it if it doesn't exist), so that we can + // switch the repo to the specified branch. This is done so that users of this api + // can expect the repo to be at the desired branch, when cloned. + localBranch, err := repo.LookupBranch(c.Branch, git2go.BranchLocal) + if git2go.IsErrorCode(err, git2go.ErrorCodeNotFound) { + localBranch, err = repo.CreateBranch(c.Branch, upstreamCommit, false) + if err != nil { + return nil, fmt.Errorf("unable to create local branch '%s': %w", c.Branch, err) + } + } else if err != nil { + return nil, fmt.Errorf("unable to lookup branch '%s': %w", c.Branch, err) + } + defer localBranch.Free() + + tree, err := repo.LookupTree(upstreamCommit.TreeId()) + if err != nil { + return nil, fmt.Errorf("unable to lookup tree for branch '%s': %w", c.Branch, err) + } + defer tree.Free() + + err = repo.CheckoutTree(tree, &git2go.CheckoutOpts{ + // the remote branch should take precedence if it exists at this point in time. + Strategy: git2go.CheckoutForce, }) if err != nil { - return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err)) + return nil, fmt.Errorf("unable to checkout tree for branch '%s': %w", c.Branch, err) } - defer repo.Free() + + // Set the current head to point to the requested branch. + err = repo.SetHead("refs/heads/" + c.Branch) + if err != nil { + return nil, fmt.Errorf("unable to set HEAD to branch '%s':%w", c.Branch, err) + } + + // Use the current worktree's head as reference for the commit to be returned. head, err := repo.Head() if err != nil { - return nil, fmt.Errorf("git resolve HEAD error: %w", err) + return nil, fmt.Errorf("unable to resolve HEAD: %w", err) } defer head.Free() + cc, err := repo.LookupCommit(head.Target()) if err != nil { - return nil, fmt.Errorf("failed to lookup HEAD commit '%s' for branch '%s': %w", head.Target(), c.Branch, err) + return nil, fmt.Errorf("unable to lookup HEAD commit '%s' for branch '%s': %w", head.Target(), c.Branch, err) } defer cc.Free() + return buildCommit(cc, "refs/heads/"+c.Branch), nil } @@ -254,107 +200,73 @@ type CheckoutTag struct { func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) { defer recoverPanic(&err) - // This branching is temporary, to address the transient panics observed when using unmanaged transport. - // The panics probably happen because we perform multiple fetch ops (introduced as a part of optimizing git clones). - // The branching lets us establish a clear code path to help us be certain of the expected behaviour. - // When we get rid of unmanaged transports, we can get rid of this branching as well. - if managed.Enabled() { - if opts.TransportOptionsURL == "" { - return nil, fmt.Errorf("can't use managed transport without a valid transport auth id.") - } - managed.AddTransportOptions(opts.TransportOptionsURL, managed.TransportOptions{ - TargetURL: url, - AuthOpts: opts, - ProxyOptions: &git2go.ProxyOptions{Type: git2go.ProxyTypeAuto}, - Context: ctx, - }) - url = opts.TransportOptionsURL - remoteCallBacks := managed.RemoteCallbacks() - defer managed.RemoveTransportOptions(opts.TransportOptionsURL) - - repo, remote, err := initializeRepoWithRemote(ctx, path, url, opts) - if err != nil { - return nil, err - } - // Open remote connection. - err = remote.ConnectFetch(&remoteCallBacks, nil, nil) - if err != nil { - remote.Free() - repo.Free() - return nil, fmt.Errorf("unable to fetch-connect to remote '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err)) - } - defer func() { - remote.Disconnect() - remote.Free() - repo.Free() - }() - - // When the last observed revision is set, check whether it is still the - // same at the remote branch. If so, short-circuit the clone operation here. - if c.LastRevision != "" { - heads, err := remote.Ls(c.Tag) - if err != nil { - return nil, fmt.Errorf("unable to remote ls for '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err)) - } - if len(heads) > 0 { - hash := heads[0].Id.String() - currentRevision := fmt.Sprintf("%s/%s", c.Tag, hash) - var same bool - if currentRevision == c.LastRevision { - same = true - } else if len(heads) > 1 { - hash = heads[1].Id.String() - currentAnnotatedRevision := fmt.Sprintf("%s/%s", c.Tag, hash) - if currentAnnotatedRevision == c.LastRevision { - same = true - } - } - if same { - // Construct a partial commit with the existing information. - c := &git.Commit{ - Hash: git.Hash(hash), - Reference: "refs/tags/" + c.Tag, - } - return c, nil - } - } - } - - err = remote.Fetch([]string{c.Tag}, - &git2go.FetchOptions{ - DownloadTags: git2go.DownloadTagsAuto, - RemoteCallbacks: remoteCallBacks, - }, - "") - - if err != nil { - return nil, fmt.Errorf("unable to fetch remote '%s': %w", - managed.EffectiveURL(url), gitutil.LibGit2Error(err)) - } - - cc, err := checkoutDetachedDwim(repo, c.Tag) - if err != nil { - return nil, err - } - defer cc.Free() - return buildCommit(cc, "refs/tags/"+c.Tag), nil - } else { - return c.checkoutUnmanaged(ctx, path, url, opts) - } -} - -func (c *CheckoutTag) checkoutUnmanaged(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) { - repo, err := git2go.Clone(url, path, &git2go.CloneOptions{ - FetchOptions: git2go.FetchOptions{ - DownloadTags: git2go.DownloadTagsAll, - RemoteCallbacks: RemoteCallbacks(ctx, opts), - ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto}, - }, - }) + err = registerManagedTransportOptions(ctx, url, opts) if err != nil { - return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err)) + return nil, err } - defer repo.Free() + transportOptsURL := opts.TransportOptionsURL + remoteCallBacks := managed.RemoteCallbacks() + defer managed.RemoveTransportOptions(transportOptsURL) + + repo, remote, err := initializeRepoWithRemote(ctx, path, url, opts) + if err != nil { + return nil, err + } + // Open remote connection. + err = remote.ConnectFetch(&remoteCallBacks, nil, nil) + if err != nil { + remote.Free() + repo.Free() + return nil, fmt.Errorf("unable to fetch-connect to remote '%s': %w", url, gitutil.LibGit2Error(err)) + } + defer func() { + remote.Disconnect() + remote.Free() + repo.Free() + }() + + // When the last observed revision is set, check whether it is still the + // same at the remote branch. If so, short-circuit the clone operation here. + if c.LastRevision != "" { + heads, err := remote.Ls(c.Tag) + if err != nil { + return nil, fmt.Errorf("unable to remote ls for '%s': %w", url, gitutil.LibGit2Error(err)) + } + if len(heads) > 0 { + hash := heads[0].Id.String() + currentRevision := fmt.Sprintf("%s/%s", c.Tag, hash) + var same bool + if currentRevision == c.LastRevision { + same = true + } else if len(heads) > 1 { + hash = heads[1].Id.String() + currentAnnotatedRevision := fmt.Sprintf("%s/%s", c.Tag, hash) + if currentAnnotatedRevision == c.LastRevision { + same = true + } + } + if same { + // Construct a partial commit with the existing information. + c := &git.Commit{ + Hash: git.Hash(hash), + Reference: "refs/tags/" + c.Tag, + } + return c, nil + } + } + } + + err = remote.Fetch([]string{c.Tag}, + &git2go.FetchOptions{ + DownloadTags: git2go.DownloadTagsAuto, + RemoteCallbacks: remoteCallBacks, + }, + "") + + if err != nil { + return nil, fmt.Errorf("unable to fetch remote '%s': %w", url, gitutil.LibGit2Error(err)) + } + cc, err := checkoutDetachedDwim(repo, c.Tag) if err != nil { return nil, err @@ -370,31 +282,21 @@ type CheckoutCommit struct { func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) { defer recoverPanic(&err) - remoteCallBacks := RemoteCallbacks(ctx, opts) - - if managed.Enabled() { - if opts.TransportOptionsURL == "" { - return nil, fmt.Errorf("can't use managed transport without a valid transport auth id.") - } - managed.AddTransportOptions(opts.TransportOptionsURL, managed.TransportOptions{ - TargetURL: url, - AuthOpts: opts, - ProxyOptions: &git2go.ProxyOptions{Type: git2go.ProxyTypeAuto}, - Context: ctx, - }) - url = opts.TransportOptionsURL - remoteCallBacks = managed.RemoteCallbacks() - defer managed.RemoveTransportOptions(opts.TransportOptionsURL) + err = registerManagedTransportOptions(ctx, url, opts) + if err != nil { + return nil, err } + transportOptsURL := opts.TransportOptionsURL + defer managed.RemoveTransportOptions(transportOptsURL) - repo, err := git2go.Clone(url, path, &git2go.CloneOptions{ + repo, err := git2go.Clone(transportOptsURL, path, &git2go.CloneOptions{ FetchOptions: git2go.FetchOptions{ DownloadTags: git2go.DownloadTagsNone, - RemoteCallbacks: remoteCallBacks, + RemoteCallbacks: managed.RemoteCallbacks(), }, }) if err != nil { - return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err)) + return nil, fmt.Errorf("unable to clone '%s': %w", url, gitutil.LibGit2Error(err)) } defer repo.Free() oid, err := git2go.NewOid(c.Commit) @@ -415,36 +317,26 @@ type CheckoutSemVer struct { func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) { defer recoverPanic(&err) - remoteCallBacks := RemoteCallbacks(ctx, opts) - - if managed.Enabled() { - if opts.TransportOptionsURL == "" { - return nil, fmt.Errorf("can't use managed transport without a valid transport auth id.") - } - managed.AddTransportOptions(opts.TransportOptionsURL, managed.TransportOptions{ - TargetURL: url, - AuthOpts: opts, - ProxyOptions: &git2go.ProxyOptions{Type: git2go.ProxyTypeAuto}, - Context: ctx, - }) - url = opts.TransportOptionsURL - remoteCallBacks = managed.RemoteCallbacks() - defer managed.RemoveTransportOptions(opts.TransportOptionsURL) + err = registerManagedTransportOptions(ctx, url, opts) + if err != nil { + return nil, err } + transportOptsURL := opts.TransportOptionsURL + defer managed.RemoveTransportOptions(transportOptsURL) verConstraint, err := semver.NewConstraint(c.SemVer) if err != nil { return nil, fmt.Errorf("semver parse error: %w", err) } - repo, err := git2go.Clone(url, path, &git2go.CloneOptions{ + repo, err := git2go.Clone(transportOptsURL, path, &git2go.CloneOptions{ FetchOptions: git2go.FetchOptions{ DownloadTags: git2go.DownloadTagsAll, - RemoteCallbacks: remoteCallBacks, + RemoteCallbacks: managed.RemoteCallbacks(), }, }) if err != nil { - return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err)) + return nil, fmt.Errorf("unable to clone '%s': %w", url, gitutil.LibGit2Error(err)) } defer repo.Free() @@ -630,6 +522,29 @@ func initializeRepoWithRemote(ctx context.Context, path, url string, opts *git.A return repo, remote, nil } +// registerManagedTransportOptions registers the given url and it's transport options. +// Callers must make sure to call `managed.RemoveTransportOptions()` to avoid increase in +// memory consumption. +// We store the target URL, auth options, etc. mapped to TransporOptsURL because managed transports +// don't provide a way for any kind of dependency injection. +// This lets us have a way of doing interop between application level code and transport level code +// which enables us to fetch the required credentials, context, etc. at the transport level. +func registerManagedTransportOptions(ctx context.Context, url string, authOpts *git.AuthOptions) error { + if authOpts == nil { + return errors.New("can't checkout using libgit2 with an empty set of auth options") + } + if authOpts.TransportOptionsURL == "" { + return errors.New("can't checkout using libgit2 without a valid transport auth id") + } + managed.AddTransportOptions(authOpts.TransportOptionsURL, managed.TransportOptions{ + TargetURL: url, + AuthOpts: authOpts, + ProxyOptions: &git2go.ProxyOptions{Type: git2go.ProxyTypeAuto}, + Context: ctx, + }) + return nil +} + func recoverPanic(err *error) { if r := recover(); r != nil { *err = fmt.Errorf("recovered from git2go panic: %v", r) diff --git a/pkg/git/libgit2/managed_test.go b/pkg/git/libgit2/checkout_ssh_test.go similarity index 92% rename from pkg/git/libgit2/managed_test.go rename to pkg/git/libgit2/checkout_ssh_test.go index a99fe906..3eb5b34c 100644 --- a/pkg/git/libgit2/managed_test.go +++ b/pkg/git/libgit2/checkout_ssh_test.go @@ -19,7 +19,6 @@ package libgit2 import ( "context" "fmt" - "math/rand" "net/url" "os" "path/filepath" @@ -31,22 +30,17 @@ import ( "github.com/fluxcd/pkg/gittestserver" "github.com/fluxcd/pkg/ssh" - feathelper "github.com/fluxcd/pkg/runtime/features" . "github.com/onsi/gomega" cryptossh "golang.org/x/crypto/ssh" - "github.com/fluxcd/source-controller/internal/features" "github.com/fluxcd/source-controller/pkg/git" - "github.com/fluxcd/source-controller/pkg/git/libgit2/managed" ) const testRepositoryPath = "../testdata/git/repo" -// Test_managedSSH_KeyTypes assures support for the different +// Test_ssh_keyTypes assures support for the different // types of keys for SSH Authentication supported by Flux. -func Test_managedSSH_KeyTypes(t *testing.T) { - enableManagedTransport() - +func Test_ssh_keyTypes(t *testing.T) { tests := []struct { name string keyType ssh.KeyPairType @@ -171,11 +165,9 @@ func Test_managedSSH_KeyTypes(t *testing.T) { } } -// Test_managedSSH_KeyExchangeAlgos assures support for the different +// Test_ssh_keyExchangeAlgos assures support for the different // types of SSH key exchange algorithms supported by Flux. -func Test_managedSSH_KeyExchangeAlgos(t *testing.T) { - enableManagedTransport() - +func Test_ssh_keyExchangeAlgos(t *testing.T) { tests := []struct { name string ClientKex []string @@ -294,11 +286,9 @@ func Test_managedSSH_KeyExchangeAlgos(t *testing.T) { } } -// Test_managedSSH_HostKeyAlgos assures support for the different +// Test_ssh_hostKeyAlgos assures support for the different // types of SSH Host Key algorithms supported by Flux. -func Test_managedSSH_HostKeyAlgos(t *testing.T) { - enableManagedTransport() - +func Test_ssh_hostKeyAlgos(t *testing.T) { tests := []struct { name string keyType ssh.KeyPairType @@ -457,18 +447,3 @@ func Test_managedSSH_HostKeyAlgos(t *testing.T) { }) } } - -func getTransportOptionsURL(transport git.TransportType) string { - letterRunes := []rune("abcdefghijklmnopqrstuvwxyz1234567890") - b := make([]rune, 10) - for i := range b { - b[i] = letterRunes[rand.Intn(len(letterRunes))] - } - return string(transport) + "://" + string(b) -} - -func enableManagedTransport() { - fg := feathelper.FeatureGates{} - fg.SupportedFeatures(features.FeatureGates()) - managed.InitManagedTransport() -} diff --git a/pkg/git/libgit2/checkout_test.go b/pkg/git/libgit2/checkout_test.go index 07bc46b2..0f9bb316 100644 --- a/pkg/git/libgit2/checkout_test.go +++ b/pkg/git/libgit2/checkout_test.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "math/rand" "os" "path/filepath" "testing" @@ -30,17 +31,19 @@ import ( . "github.com/onsi/gomega" "github.com/fluxcd/source-controller/pkg/git" - - mt "github.com/fluxcd/source-controller/pkg/git/libgit2/managed" + "github.com/fluxcd/source-controller/pkg/git/libgit2/managed" ) -func TestCheckoutBranch_unmanaged(t *testing.T) { - checkoutBranch(t, false) +func TestMain(m *testing.M) { + err := managed.InitManagedTransport() + if err != nil { + panic(fmt.Sprintf("failed to initialize libgit2 managed transport: %s", err)) + } + code := m.Run() + os.Exit(code) } -// checkoutBranch is a test helper function which runs the tests for checking out -// via CheckoutBranch. -func checkoutBranch(t *testing.T, managed bool) { +func TestCheckoutBranch_Checkout(t *testing.T) { // we use a HTTP Git server instead of a bare repo (for all tests in this // package), because our managed transports don't support the file protocol, // so we wouldn't actually be using our custom transports, if we used a bare @@ -138,7 +141,6 @@ func checkoutBranch(t *testing.T, managed bool) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { g := NewWithT(t) - g.Expect(mt.Enabled()).To(Equal(managed)) branch := CheckoutBranch{ Branch: tt.branch, @@ -159,9 +161,7 @@ func checkoutBranch(t *testing.T, managed bool) { } g.Expect(err).ToNot(HaveOccurred()) g.Expect(cc.String()).To(Equal(tt.branch + "/" + tt.expectedCommit)) - if managed { - g.Expect(git.IsConcreteCommit(*cc)).To(Equal(tt.expectedConcreteCommit)) - } + g.Expect(git.IsConcreteCommit(*cc)).To(Equal(tt.expectedConcreteCommit)) if tt.expectedConcreteCommit { for k, v := range tt.filesCreated { @@ -173,13 +173,7 @@ func checkoutBranch(t *testing.T, managed bool) { } } -func TestCheckoutTag_unmanaged(t *testing.T) { - checkoutTag(t, false) -} - -// checkoutTag is a test helper function which runs the tests for checking out -// via CheckoutTag. -func checkoutTag(t *testing.T, managed bool) { +func TestCheckoutTag_Checkout(t *testing.T) { type testTag struct { name string annotated bool @@ -229,7 +223,6 @@ func checkoutTag(t *testing.T, managed bool) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { g := NewWithT(t) - g.Expect(mt.Enabled()).To(Equal(managed)) server, err := gittestserver.NewTempGitServer() g.Expect(err).ToNot(HaveOccurred()) @@ -297,10 +290,7 @@ func checkoutTag(t *testing.T, managed bool) { targetTagCommit := tagCommits[tt.checkoutTag] g.Expect(err).ToNot(HaveOccurred()) g.Expect(cc.String()).To(Equal(tt.checkoutTag + "/" + targetTagCommit.Id().String())) - if managed { - g.Expect(git.IsConcreteCommit(*cc)).To(Equal(tt.expectConcreteCommit)) - - } + g.Expect(git.IsConcreteCommit(*cc)).To(Equal(tt.expectConcreteCommit)) // Check file content only when there's an actual checkout. if tt.lastRevTag != tt.checkoutTag { @@ -311,15 +301,8 @@ func checkoutTag(t *testing.T, managed bool) { } } -func TestCheckoutCommit_unmanaged(t *testing.T) { - checkoutCommit(t, false) -} - -// checkoutCommit is a test helper function which runs the tests for checking out -// via CheckoutCommit. -func checkoutCommit(t *testing.T, managed bool) { +func TestCheckoutCommit_Checkout(t *testing.T) { g := NewWithT(t) - g.Expect(mt.Enabled()).To(Equal(managed)) server, err := gittestserver.NewTempGitServer() if err != nil { @@ -380,13 +363,7 @@ func checkoutCommit(t *testing.T, managed bool) { g.Expect(cc).To(BeNil()) } -func TestCheckoutTagSemVer_unmanaged(t *testing.T) { - checkoutSemVer(t, false) -} - -// checkoutSemVer is a test helper function which runs the tests for checking out -// via CheckoutSemVer. -func checkoutSemVer(t *testing.T, managed bool) { +func TestCheckoutSemVer_Checkout(t *testing.T) { g := NewWithT(t) now := time.Now() @@ -498,7 +475,6 @@ func checkoutSemVer(t *testing.T, managed bool) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { g := NewWithT(t) - g.Expect(mt.Enabled()).To(Equal(managed)) semVer := CheckoutSemVer{ SemVer: tt.constraint, @@ -524,6 +500,112 @@ func checkoutSemVer(t *testing.T, managed bool) { } } +func Test_initializeRepoWithRemote(t *testing.T) { + g := NewWithT(t) + + tmp := t.TempDir() + ctx := context.TODO() + testRepoURL := "https://example.com/foo/bar" + testRepoURL2 := "https://example.com/foo/baz" + authOpts, err := git.AuthOptionsWithoutSecret(testRepoURL) + g.Expect(err).ToNot(HaveOccurred()) + authOpts.TransportOptionsURL = "https://bar123" + authOpts2, err := git.AuthOptionsWithoutSecret(testRepoURL2) + g.Expect(err).ToNot(HaveOccurred()) + authOpts2.TransportOptionsURL = "https://baz789" + + // Fresh initialization. + repo, remote, err := initializeRepoWithRemote(ctx, tmp, testRepoURL, authOpts) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(repo.IsBare()).To(BeFalse()) + g.Expect(remote.Name()).To(Equal(defaultRemoteName)) + g.Expect(remote.Url()).To(Equal(authOpts.TransportOptionsURL)) + remote.Free() + repo.Free() + + // Reinitialize to ensure it reuses the existing origin. + repo, remote, err = initializeRepoWithRemote(ctx, tmp, testRepoURL, authOpts) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(repo.IsBare()).To(BeFalse()) + g.Expect(remote.Name()).To(Equal(defaultRemoteName)) + g.Expect(remote.Url()).To(Equal(authOpts.TransportOptionsURL)) + remote.Free() + repo.Free() + + // Reinitialize with a different remote URL for existing origin. + repo, remote, err = initializeRepoWithRemote(ctx, tmp, testRepoURL2, authOpts2) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(repo.IsBare()).To(BeFalse()) + g.Expect(remote.Name()).To(Equal(defaultRemoteName)) + g.Expect(remote.Url()).To(Equal(authOpts2.TransportOptionsURL)) + remote.Free() + repo.Free() +} + +func TestCheckoutStrategyForOptions(t *testing.T) { + tests := []struct { + name string + opts git.CheckoutOptions + expectedStrat git.CheckoutStrategy + }{ + { + name: "commit works", + opts: git.CheckoutOptions{ + Commit: "commit", + }, + expectedStrat: &CheckoutCommit{ + Commit: "commit", + }, + }, + { + name: "semver works", + opts: git.CheckoutOptions{ + SemVer: ">= 1.0.0", + }, + expectedStrat: &CheckoutSemVer{ + SemVer: ">= 1.0.0", + }, + }, + { + name: "tag with latest revision works", + opts: git.CheckoutOptions{ + Tag: "v0.1.0", + LastRevision: "ar34oi2njrngjrng", + }, + expectedStrat: &CheckoutTag{ + Tag: "v0.1.0", + LastRevision: "ar34oi2njrngjrng", + }, + }, + { + name: "branch with latest revision works", + opts: git.CheckoutOptions{ + Branch: "main", + LastRevision: "rrgij20mkmrg", + }, + expectedStrat: &CheckoutBranch{ + Branch: "main", + LastRevision: "rrgij20mkmrg", + }, + }, + { + name: "empty branch falls back to default", + opts: git.CheckoutOptions{}, + expectedStrat: &CheckoutBranch{ + Branch: git.DefaultBranch, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + strat := CheckoutStrategyForOptions(context.TODO(), tt.opts) + g.Expect(strat).To(Equal(tt.expectedStrat)) + }) + } +} + func initBareRepo(t *testing.T) (*git2go.Repository, error) { tmpDir := t.TempDir() repo, err := git2go.InitRepository(tmpDir, true) @@ -615,102 +697,11 @@ func mockSignature(time time.Time) *git2go.Signature { } } -func TestInitializeRepoWithRemote(t *testing.T) { - g := NewWithT(t) - - g.Expect(mt.Enabled()).To(BeFalse()) - tmp := t.TempDir() - ctx := context.TODO() - testRepoURL := "https://example.com/foo/bar" - testRepoURL2 := "https://example.com/foo/baz" - authOpts, err := git.AuthOptionsWithoutSecret(testRepoURL) - g.Expect(err).ToNot(HaveOccurred()) - authOpts2, err := git.AuthOptionsWithoutSecret(testRepoURL2) - g.Expect(err).ToNot(HaveOccurred()) - - // Fresh initialization. - repo, remote, err := initializeRepoWithRemote(ctx, tmp, testRepoURL, authOpts) - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(repo.IsBare()).To(BeFalse()) - g.Expect(remote.Name()).To(Equal(defaultRemoteName)) - g.Expect(remote.Url()).To(Equal(testRepoURL)) - remote.Free() - repo.Free() - - // Reinitialize to ensure it reuses the existing origin. - repo, remote, err = initializeRepoWithRemote(ctx, tmp, testRepoURL, authOpts) - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(repo.IsBare()).To(BeFalse()) - g.Expect(remote.Name()).To(Equal(defaultRemoteName)) - g.Expect(remote.Url()).To(Equal(testRepoURL)) - remote.Free() - repo.Free() - - // Reinitialize with a different remote URL for existing origin. - _, _, err = initializeRepoWithRemote(ctx, tmp, testRepoURL2, authOpts2) - g.Expect(err).To(HaveOccurred()) -} - -func TestCheckoutStrategyForOptions(t *testing.T) { - tests := []struct { - name string - opts git.CheckoutOptions - expectedStrat git.CheckoutStrategy - }{ - { - name: "commit works", - opts: git.CheckoutOptions{ - Commit: "commit", - }, - expectedStrat: &CheckoutCommit{ - Commit: "commit", - }, - }, - { - name: "semver works", - opts: git.CheckoutOptions{ - SemVer: ">= 1.0.0", - }, - expectedStrat: &CheckoutSemVer{ - SemVer: ">= 1.0.0", - }, - }, - { - name: "tag with latest revision works", - opts: git.CheckoutOptions{ - Tag: "v0.1.0", - LastRevision: "ar34oi2njrngjrng", - }, - expectedStrat: &CheckoutTag{ - Tag: "v0.1.0", - LastRevision: "ar34oi2njrngjrng", - }, - }, - { - name: "branch with latest revision works", - opts: git.CheckoutOptions{ - Branch: "main", - LastRevision: "rrgij20mkmrg", - }, - expectedStrat: &CheckoutBranch{ - Branch: "main", - LastRevision: "rrgij20mkmrg", - }, - }, - { - name: "empty branch falls back to default", - opts: git.CheckoutOptions{}, - expectedStrat: &CheckoutBranch{ - Branch: git.DefaultBranch, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - g := NewWithT(t) - strat := CheckoutStrategyForOptions(context.TODO(), tt.opts) - g.Expect(strat).To(Equal(tt.expectedStrat)) - }) +func getTransportOptionsURL(transport git.TransportType) string { + letterRunes := []rune("abcdefghijklmnopqrstuvwxyz1234567890") + b := make([]rune, 10) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] } + return string(transport) + "://" + string(b) } diff --git a/pkg/git/libgit2/managed/http.go b/pkg/git/libgit2/managed/http.go index 7ae23db9..d9c3d01a 100644 --- a/pkg/git/libgit2/managed/http.go +++ b/pkg/git/libgit2/managed/http.go @@ -419,6 +419,7 @@ func (self *httpSmartSubtransportStream) sendRequest() error { URL: self.req.URL, Header: self.req.Header, } + if req.Method == "POST" { if len(content) == 0 { // a copy of the request body needs to be saved so diff --git a/pkg/git/libgit2/managed/http_test.go b/pkg/git/libgit2/managed/http_test.go index 3907e443..fc957cbe 100644 --- a/pkg/git/libgit2/managed/http_test.go +++ b/pkg/git/libgit2/managed/http_test.go @@ -30,6 +30,15 @@ import ( git2go "github.com/libgit2/git2go/v33" ) +func TestMain(m *testing.M) { + err := InitManagedTransport() + if err != nil { + panic(fmt.Sprintf("failed to initialize libgit2 managed transport: %s", err)) + } + code := m.Run() + os.Exit(code) +} + func TestHttpAction_CreateClientRequest(t *testing.T) { authOpts := git.AuthOptions{ Username: "user", @@ -56,8 +65,8 @@ func TestHttpAction_CreateClientRequest(t *testing.T) { g.Expect(req.URL.String()).To(Equal("https://final-target/abc/git-upload-pack")) g.Expect(req.Method).To(Equal("POST")) g.Expect(req.Header).To(BeEquivalentTo(map[string][]string{ - "User-Agent": []string{"git/2.0 (flux-libgit2)"}, - "Content-Type": []string{"application/x-git-upload-pack-request"}, + "User-Agent": {"git/2.0 (flux-libgit2)"}, + "Content-Type": {"application/x-git-upload-pack-request"}, })) }, wantedErr: nil, @@ -70,7 +79,7 @@ func TestHttpAction_CreateClientRequest(t *testing.T) { g.Expect(req.URL.String()).To(Equal("https://final-target/abc/info/refs?service=git-upload-pack")) g.Expect(req.Method).To(Equal("GET")) g.Expect(req.Header).To(BeEquivalentTo(map[string][]string{ - "User-Agent": []string{"git/2.0 (flux-libgit2)"}, + "User-Agent": {"git/2.0 (flux-libgit2)"}, })) }, wantedErr: nil, @@ -86,8 +95,8 @@ func TestHttpAction_CreateClientRequest(t *testing.T) { g.Expect(req.URL.String()).To(Equal("https://final-target/abc/git-receive-pack")) g.Expect(req.Method).To(Equal("POST")) g.Expect(req.Header).To(BeEquivalentTo(map[string][]string{ - "Content-Type": []string{"application/x-git-receive-pack-request"}, - "User-Agent": []string{"git/2.0 (flux-libgit2)"}, + "Content-Type": {"application/x-git-receive-pack-request"}, + "User-Agent": {"git/2.0 (flux-libgit2)"}, })) }, wantedErr: nil, @@ -100,7 +109,7 @@ func TestHttpAction_CreateClientRequest(t *testing.T) { g.Expect(req.URL.String()).To(Equal("https://final-target/abc/info/refs?service=git-receive-pack")) g.Expect(req.Method).To(Equal("GET")) g.Expect(req.Header).To(BeEquivalentTo(map[string][]string{ - "User-Agent": []string{"git/2.0 (flux-libgit2)"}, + "User-Agent": {"git/2.0 (flux-libgit2)"}, })) }, wantedErr: nil, @@ -162,7 +171,7 @@ func TestHttpAction_CreateClientRequest(t *testing.T) { } } -func TestHTTPManagedTransport_E2E(t *testing.T) { +func TestHTTP_E2E(t *testing.T) { g := NewWithT(t) server, err := gittestserver.NewTempGitServer() @@ -178,9 +187,6 @@ func TestHTTPManagedTransport_E2E(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) defer server.StopHTTP() - // Force managed transport to be enabled - InitManagedTransport() - repoPath := "test.git" err = server.InitRepo("../../testdata/git/repo", git.DefaultBranch, repoPath) g.Expect(err).ToNot(HaveOccurred()) @@ -252,7 +258,7 @@ func TestTrimActionSuffix(t *testing.T) { } } -func TestHTTPManagedTransport_HandleRedirect(t *testing.T) { +func TestHTTP_HandleRedirect(t *testing.T) { tests := []struct { name string repoURL string @@ -261,9 +267,6 @@ func TestHTTPManagedTransport_HandleRedirect(t *testing.T) { {name: "handle gitlab redirect", repoURL: "https://gitlab.com/stefanprodan/podinfo"}, } - // Force managed transport to be enabled - InitManagedTransport() - for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { g := NewWithT(t) diff --git a/pkg/git/libgit2/managed/ssh_test.go b/pkg/git/libgit2/managed/ssh_test.go index a6e0fd4c..4e575f4e 100644 --- a/pkg/git/libgit2/managed/ssh_test.go +++ b/pkg/git/libgit2/managed/ssh_test.go @@ -74,7 +74,7 @@ func TestSSHAction_clientConfig(t *testing.T) { } } -func TestSSHManagedTransport_E2E(t *testing.T) { +func TestSSH_E2E(t *testing.T) { g := NewWithT(t) server, err := gittestserver.NewTempGitServer() @@ -90,7 +90,6 @@ func TestSSHManagedTransport_E2E(t *testing.T) { server.StartSSH() }() defer server.StopSSH() - InitManagedTransport() kp, err := ssh.NewEd25519Generator().Generate() g.Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/git/libgit2/managed_checkout_test.go b/pkg/git/libgit2/managed_checkout_test.go deleted file mode 100644 index 4c3bb42d..00000000 --- a/pkg/git/libgit2/managed_checkout_test.go +++ /dev/null @@ -1,46 +0,0 @@ -/* -Copyright 2022 The Flux authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This file is named `managed_checkout_test.go` on purpose to make sure that -// tests needing to use unmanaged transports run before the tests that use managed -// transports do, since the the former are present in `checkout_test.go`. `checkout_test.go` -// comes first in this package (alphabetically speaking), which makes golang run the tests -// in that file first. -package libgit2 - -import ( - "testing" -) - -func TestCheckoutBranch_CheckoutManaged(t *testing.T) { - enableManagedTransport() - checkoutBranch(t, true) -} - -func TestCheckoutTag_CheckoutManaged(t *testing.T) { - enableManagedTransport() - checkoutTag(t, true) -} - -func TestCheckoutCommit_CheckoutManaged(t *testing.T) { - enableManagedTransport() - checkoutCommit(t, true) -} - -func TestCheckoutTagSemVer_CheckoutManaged(t *testing.T) { - enableManagedTransport() - checkoutSemVer(t, true) -} diff --git a/pkg/git/libgit2/transport.go b/pkg/git/libgit2/transport.go deleted file mode 100644 index f9aeefe2..00000000 --- a/pkg/git/libgit2/transport.go +++ /dev/null @@ -1,165 +0,0 @@ -/* -Copyright 2020 The Flux authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package libgit2 - -import ( - "context" - "crypto/x509" - "fmt" - "time" - - git2go "github.com/libgit2/git2go/v33" - "golang.org/x/crypto/ssh" - - "github.com/fluxcd/source-controller/pkg/git" - "github.com/fluxcd/source-controller/pkg/git/libgit2/managed" -) - -var ( - now = time.Now -) - -// RemoteCallbacks constructs RemoteCallbacks with credentialsCallback and -// certificateCallback, and the given options if the given opts is not nil. -func RemoteCallbacks(ctx context.Context, opts *git.AuthOptions) git2go.RemoteCallbacks { - if opts != nil { - return git2go.RemoteCallbacks{ - SidebandProgressCallback: transportMessageCallback(ctx), - TransferProgressCallback: transferProgressCallback(ctx), - PushTransferProgressCallback: pushTransferProgressCallback(ctx), - CredentialsCallback: credentialsCallback(opts), - CertificateCheckCallback: certificateCallback(opts), - } - } - return git2go.RemoteCallbacks{} -} - -// transferProgressCallback constructs TransferProgressCallbacks which signals -// libgit2 it should stop the transfer when the given context is closed (due to -// e.g. a timeout). -func transferProgressCallback(ctx context.Context) git2go.TransferProgressCallback { - return func(p git2go.TransferProgress) error { - // Early return if all the objects have been received. - if p.ReceivedObjects == p.TotalObjects { - return nil - } - select { - case <-ctx.Done(): - return fmt.Errorf("transport close (potentially due to a timeout)") - default: - return nil - } - } -} - -// transportMessageCallback constructs TransportMessageCallback which signals -// libgit2 it should cancel the network operation when the given context is -// closed. -func transportMessageCallback(ctx context.Context) git2go.TransportMessageCallback { - return func(_ string) error { - select { - case <-ctx.Done(): - return fmt.Errorf("transport closed") - default: - return nil - } - } -} - -// pushTransferProgressCallback constructs PushTransferProgressCallback which -// signals libgit2 it should stop the push transfer when the given context is -// closed (due to e.g. a timeout). -func pushTransferProgressCallback(ctx context.Context) git2go.PushTransferProgressCallback { - return func(current, total uint32, _ uint) error { - // Early return if current equals total. - if current == total { - return nil - } - select { - case <-ctx.Done(): - return fmt.Errorf("transport close (potentially due to a timeout)") - default: - return nil - } - } -} - -// credentialsCallback constructs CredentialsCallbacks with the given options -// for git.Transport, and returns the result. -func credentialsCallback(opts *git.AuthOptions) git2go.CredentialsCallback { - return func(url string, username string, allowedTypes git2go.CredentialType) (*git2go.Credential, error) { - if allowedTypes&(git2go.CredentialTypeSSHKey|git2go.CredentialTypeSSHCustom|git2go.CredentialTypeSSHMemory) != 0 { - var ( - signer ssh.Signer - err error - ) - if opts.Password != "" { - signer, err = ssh.ParsePrivateKeyWithPassphrase(opts.Identity, []byte(opts.Password)) - } else { - signer, err = ssh.ParsePrivateKey(opts.Identity) - } - if err != nil { - return nil, err - } - return git2go.NewCredentialSSHKeyFromSigner(opts.Username, signer) - } - if (allowedTypes & git2go.CredentialTypeUserpassPlaintext) != 0 { - return git2go.NewCredentialUserpassPlaintext(opts.Username, opts.Password) - } - if (allowedTypes & git2go.CredentialTypeUsername) != 0 { - return git2go.NewCredentialUsername(opts.Username) - } - return nil, fmt.Errorf("unknown credential type %+v", allowedTypes) - } -} - -// certificateCallback constructs CertificateCallback with the given options -// for git.Transport if the given opts is not nil, and returns the result. -func certificateCallback(opts *git.AuthOptions) git2go.CertificateCheckCallback { - switch opts.Transport { - case git.HTTPS: - if len(opts.CAFile) > 0 { - return x509Callback(opts.CAFile) - } - case git.SSH: - if len(opts.KnownHosts) > 0 && opts.Host != "" { - return managed.KnownHostsCallback(opts.Host, opts.KnownHosts) - } - } - return nil -} - -// x509Callback returns a CertificateCheckCallback that verifies the -// certificate against the given caBundle for git.HTTPS Transports. -func x509Callback(caBundle []byte) git2go.CertificateCheckCallback { - return func(cert *git2go.Certificate, valid bool, hostname string) error { - roots := x509.NewCertPool() - if ok := roots.AppendCertsFromPEM(caBundle); !ok { - return fmt.Errorf("PEM CA bundle could not be appended to x509 certificate pool") - } - - opts := x509.VerifyOptions{ - Roots: roots, - DNSName: hostname, - CurrentTime: now(), - } - if _, err := cert.X509.Verify(opts); err != nil { - return fmt.Errorf("verification failed: %w", err) - } - return nil - } -} diff --git a/pkg/git/libgit2/transport_test.go b/pkg/git/libgit2/transport_test.go deleted file mode 100644 index 2e0c57d1..00000000 --- a/pkg/git/libgit2/transport_test.go +++ /dev/null @@ -1,377 +0,0 @@ -/* -Copyright 2020 The Flux authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package libgit2 - -import ( - "bytes" - "context" - "crypto/x509" - "encoding/pem" - "errors" - "fmt" - "testing" - "time" - - git2go "github.com/libgit2/git2go/v33" - . "github.com/onsi/gomega" -) - -const ( - geoTrustRootFixture = `-----BEGIN CERTIFICATE----- -MIIDVDCCAjygAwIBAgIDAjRWMA0GCSqGSIb3DQEBBQUAMEIxCzAJBgNVBAYTAlVT -MRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMRswGQYDVQQDExJHZW9UcnVzdCBHbG9i -YWwgQ0EwHhcNMDIwNTIxMDQwMDAwWhcNMjIwNTIxMDQwMDAwWjBCMQswCQYDVQQG -EwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEbMBkGA1UEAxMSR2VvVHJ1c3Qg -R2xvYmFsIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2swYYzD9 -9BcjGlZ+W988bDjkcbd4kdS8odhM+KhDtgPpTSEHCIjaWC9mOSm9BXiLnTjoBbdq -fnGk5sRgprDvgOSJKA+eJdbtg/OtppHHmMlCGDUUna2YRpIuT8rxh0PBFpVXLVDv -iS2Aelet8u5fa9IAjbkU+BQVNdnARqN7csiRv8lVK83Qlz6cJmTM386DGXHKTubU -1XupGc1V3sjs0l44U+VcT4wt/lAjNvxm5suOpDkZALeVAjmRCw7+OC7RHQWa9k0+ -bw8HHa8sHo9gOeL6NlMTOdReJivbPagUvTLrGAMoUgRx5aszPeE4uwc2hGKceeoW -MPRfwCvocWvk+QIDAQABo1MwUTAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTA -ephojYn7qwVkDBF9qn1luMrMTjAfBgNVHSMEGDAWgBTAephojYn7qwVkDBF9qn1l -uMrMTjANBgkqhkiG9w0BAQUFAAOCAQEANeMpauUvXVSOKVCUn5kaFOSPeCpilKIn -Z57QzxpeR+nBsqTP3UEaBU6bS+5Kb1VSsyShNwrrZHYqLizz/Tt1kL/6cdjHPTfS -tQWVYrmm3ok9Nns4d0iXrKYgjy6myQzCsplFAMfOEVEiIuCl6rYVSAlk6l5PdPcF -PseKUgzbFbS9bZvlxrFUaKnjaZC2mqUPuLk/IH2uSrW4nOQdtqvmlKXBx4Ot2/Un -hw4EbNX/3aBd7YdStysVAq45pmp06drE57xNNB6pXE0zX5IJL4hmXXeXxx12E6nV -5fEWCRE11azbJHFwLJhWC9kXtNHjUStedejV0NxPNO3CBWaAocvmMw== ------END CERTIFICATE-----` - - giag2IntermediateFixture = `-----BEGIN CERTIFICATE----- -MIIEBDCCAuygAwIBAgIDAjppMA0GCSqGSIb3DQEBBQUAMEIxCzAJBgNVBAYTAlVT -MRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMRswGQYDVQQDExJHZW9UcnVzdCBHbG9i -YWwgQ0EwHhcNMTMwNDA1MTUxNTU1WhcNMTUwNDA0MTUxNTU1WjBJMQswCQYDVQQG -EwJVUzETMBEGA1UEChMKR29vZ2xlIEluYzElMCMGA1UEAxMcR29vZ2xlIEludGVy -bmV0IEF1dGhvcml0eSBHMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB -AJwqBHdc2FCROgajguDYUEi8iT/xGXAaiEZ+4I/F8YnOIe5a/mENtzJEiaB0C1NP -VaTOgmKV7utZX8bhBYASxF6UP7xbSDj0U/ck5vuR6RXEz/RTDfRK/J9U3n2+oGtv -h8DQUB8oMANA2ghzUWx//zo8pzcGjr1LEQTrfSTe5vn8MXH7lNVg8y5Kr0LSy+rE -ahqyzFPdFUuLH8gZYR/Nnag+YyuENWllhMgZxUYi+FOVvuOAShDGKuy6lyARxzmZ -EASg8GF6lSWMTlJ14rbtCMoU/M4iarNOz0YDl5cDfsCx3nuvRTPPuj5xt970JSXC -DTWJnZ37DhF5iR43xa+OcmkCAwEAAaOB+zCB+DAfBgNVHSMEGDAWgBTAephojYn7 -qwVkDBF9qn1luMrMTjAdBgNVHQ4EFgQUSt0GFhu89mi1dvWBtrtiGrpagS8wEgYD -VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAQYwOgYDVR0fBDMwMTAvoC2g -K4YpaHR0cDovL2NybC5nZW90cnVzdC5jb20vY3Jscy9ndGdsb2JhbC5jcmwwPQYI -KwYBBQUHAQEEMTAvMC0GCCsGAQUFBzABhiFodHRwOi8vZ3RnbG9iYWwtb2NzcC5n -ZW90cnVzdC5jb20wFwYDVR0gBBAwDjAMBgorBgEEAdZ5AgUBMA0GCSqGSIb3DQEB -BQUAA4IBAQA21waAESetKhSbOHezI6B1WLuxfoNCunLaHtiONgaX4PCVOzf9G0JY -/iLIa704XtE7JW4S615ndkZAkNoUyHgN7ZVm2o6Gb4ChulYylYbc3GrKBIxbf/a/ -zG+FA1jDaFETzf3I93k9mTXwVqO94FntT0QJo544evZG0R0SnU++0ED8Vf4GXjza -HFa9llF7b1cq26KqltyMdMKVvvBulRP/F/A8rLIQjcxz++iPAsbw+zOzlTvjwsto -WHPbqCRiOwY1nQ2pM714A5AuTHhdUDqB1O6gyHA43LL5Z/qHQF1hwFGPa4NrzQU6 -yuGnBXj8ytqU0CwIPX4WecigUCAkVDNx ------END CERTIFICATE-----` - - googleLeafFixture = `-----BEGIN CERTIFICATE----- -MIIEdjCCA16gAwIBAgIIcR5k4dkoe04wDQYJKoZIhvcNAQEFBQAwSTELMAkGA1UE -BhMCVVMxEzARBgNVBAoTCkdvb2dsZSBJbmMxJTAjBgNVBAMTHEdvb2dsZSBJbnRl -cm5ldCBBdXRob3JpdHkgRzIwHhcNMTQwMzEyMDkzODMwWhcNMTQwNjEwMDAwMDAw -WjBoMQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwN -TW91bnRhaW4gVmlldzETMBEGA1UECgwKR29vZ2xlIEluYzEXMBUGA1UEAwwOd3d3 -Lmdvb2dsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC4zYCe -m0oUBhwE0EwBr65eBOcgcQO2PaSIAB2dEP/c1EMX2tOy0ov8rk83ePhJ+MWdT1z6 -jge9X4zQQI8ZyA9qIiwrKBZOi8DNUvrqNZC7fJAVRrb9aX/99uYOJCypIbpmWG1q -fhbHjJewhwf8xYPj71eU4rLG80a+DapWmphtfq3h52lDQIBzLVf1yYbyrTaELaz4 -NXF7HXb5YkId/gxIsSzM0aFUVu2o8sJcLYAsJqwfFKBKOMxUcn545nlspf0mTcWZ -0APlbwsKznNs4/xCDwIxxWjjqgHrYAFl6y07i1gzbAOqdNEyR24p+3JWI8WZBlBI -dk2KGj0W1fIfsvyxAgMBAAGjggFBMIIBPTAdBgNVHSUEFjAUBggrBgEFBQcDAQYI -KwYBBQUHAwIwGQYDVR0RBBIwEIIOd3d3Lmdvb2dsZS5jb20waAYIKwYBBQUHAQEE -XDBaMCsGCCsGAQUFBzAChh9odHRwOi8vcGtpLmdvb2dsZS5jb20vR0lBRzIuY3J0 -MCsGCCsGAQUFBzABhh9odHRwOi8vY2xpZW50czEuZ29vZ2xlLmNvbS9vY3NwMB0G -A1UdDgQWBBTXD5Bx6iqT+dmEhbFL4OUoHyZn8zAMBgNVHRMBAf8EAjAAMB8GA1Ud -IwQYMBaAFErdBhYbvPZotXb1gba7Yhq6WoEvMBcGA1UdIAQQMA4wDAYKKwYBBAHW -eQIFATAwBgNVHR8EKTAnMCWgI6Ahhh9odHRwOi8vcGtpLmdvb2dsZS5jb20vR0lB -RzIuY3JsMA0GCSqGSIb3DQEBBQUAA4IBAQCR3RJtHzgDh33b/MI1ugiki+nl8Ikj -5larbJRE/rcA5oite+QJyAr6SU1gJJ/rRrK3ItVEHr9L621BCM7GSdoNMjB9MMcf -tJAW0kYGJ+wqKm53wG/JaOADTnnq2Mt/j6F2uvjgN/ouns1nRHufIvd370N0LeH+ -orKqTuAPzXK7imQk6+OycYABbqCtC/9qmwRd8wwn7sF97DtYfK8WuNHtFalCAwyi -8LxJJYJCLWoMhZ+V8GZm+FOex5qkQAjnZrtNlbQJ8ro4r+rpKXtmMFFhfa+7L+PA -Kom08eUK8skxAzfDDijZPh10VtJ66uBoiDPdT+uCBehcBIcmSTrKjFGX ------END CERTIFICATE-----` - - // googleLeafWithInvalidHashFixture is the same as googleLeafFixture, but the signature - // algorithm in the certificate contains a nonsense OID. - googleLeafWithInvalidHashFixture = `-----BEGIN CERTIFICATE----- -MIIEdjCCA16gAwIBAgIIcR5k4dkoe04wDQYJKoZIhvcNAWAFBQAwSTELMAkGA1UE -BhMCVVMxEzARBgNVBAoTCkdvb2dsZSBJbmMxJTAjBgNVBAMTHEdvb2dsZSBJbnRl -cm5ldCBBdXRob3JpdHkgRzIwHhcNMTQwMzEyMDkzODMwWhcNMTQwNjEwMDAwMDAw -WjBoMQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwN -TW91bnRhaW4gVmlldzETMBEGA1UECgwKR29vZ2xlIEluYzEXMBUGA1UEAwwOd3d3 -Lmdvb2dsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC4zYCe -m0oUBhwE0EwBr65eBOcgcQO2PaSIAB2dEP/c1EMX2tOy0ov8rk83ePhJ+MWdT1z6 -jge9X4zQQI8ZyA9qIiwrKBZOi8DNUvrqNZC7fJAVRrb9aX/99uYOJCypIbpmWG1q -fhbHjJewhwf8xYPj71eU4rLG80a+DapWmphtfq3h52lDQIBzLVf1yYbyrTaELaz4 -NXF7HXb5YkId/gxIsSzM0aFUVu2o8sJcLYAsJqwfFKBKOMxUcn545nlspf0mTcWZ -0APlbwsKznNs4/xCDwIxxWjjqgHrYAFl6y07i1gzbAOqdNEyR24p+3JWI8WZBlBI -dk2KGj0W1fIfsvyxAgMBAAGjggFBMIIBPTAdBgNVHSUEFjAUBggrBgEFBQcDAQYI -KwYBBQUHAwIwGQYDVR0RBBIwEIIOd3d3Lmdvb2dsZS5jb20waAYIKwYBBQUHAQEE -XDBaMCsGCCsGAQUFBzAChh9odHRwOi8vcGtpLmdvb2dsZS5jb20vR0lBRzIuY3J0 -MCsGCCsGAQUFBzABhh9odHRwOi8vY2xpZW50czEuZ29vZ2xlLmNvbS9vY3NwMB0G -A1UdDgQWBBTXD5Bx6iqT+dmEhbFL4OUoHyZn8zAMBgNVHRMBAf8EAjAAMB8GA1Ud -IwQYMBaAFErdBhYbvPZotXb1gba7Yhq6WoEvMBcGA1UdIAQQMA4wDAYKKwYBBAHW -eQIFATAwBgNVHR8EKTAnMCWgI6Ahhh9odHRwOi8vcGtpLmdvb2dsZS5jb20vR0lB -RzIuY3JsMA0GCSqGSIb3DQFgBQUAA4IBAQCR3RJtHzgDh33b/MI1ugiki+nl8Ikj -5larbJRE/rcA5oite+QJyAr6SU1gJJ/rRrK3ItVEHr9L621BCM7GSdoNMjB9MMcf -tJAW0kYGJ+wqKm53wG/JaOADTnnq2Mt/j6F2uvjgN/ouns1nRHufIvd370N0LeH+ -orKqTuAPzXK7imQk6+OycYABbqCtC/9qmwRd8wwn7sF97DtYfK8WuNHtFalCAwyi -8LxJJYJCLWoMhZ+V8GZm+FOex5qkQAjnZrtNlbQJ8ro4r+rpKXtmMFFhfa+7L+PA -Kom08eUK8skxAzfDDijZPh10VtJ66uBoiDPdT+uCBehcBIcmSTrKjFGX ------END CERTIFICATE-----` - - knownHostsFixture string = `github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==` -) - -func Test_x509Callback(t *testing.T) { - now = func() time.Time { return time.Unix(1395785200, 0) } - - tests := []struct { - name string - certificate string - host string - caBundle []byte - want error - }{ - { - name: "Valid certificate authority bundle", - certificate: googleLeafFixture, - host: "www.google.com", - caBundle: []byte(giag2IntermediateFixture + "\n" + geoTrustRootFixture), - want: nil, - }, - { - name: "Invalid certificate", - certificate: googleLeafWithInvalidHashFixture, - host: "www.google.com", - caBundle: []byte(giag2IntermediateFixture + "\n" + geoTrustRootFixture), - want: fmt.Errorf(`verification failed: x509: certificate signed by unknown authority (possibly because of "x509: cannot verify signature: algorithm unimplemented" while trying to verify candidate authority certificate "Google Internet Authority G2")`), - }, - { - name: "Invalid certificate authority bundle", - certificate: googleLeafFixture, - host: "www.google.com", - caBundle: bytes.Trim([]byte(giag2IntermediateFixture+"\n"+geoTrustRootFixture), "-"), - want: fmt.Errorf("PEM CA bundle could not be appended to x509 certificate pool"), - }, - { - name: "Missing intermediate in bundle", - certificate: googleLeafFixture, - host: "www.google.com", - caBundle: []byte(geoTrustRootFixture), - want: fmt.Errorf("verification failed: x509: certificate signed by unknown authority"), - }, - { - name: "Invalid host", - certificate: googleLeafFixture, - host: "www.google.co", - caBundle: []byte(giag2IntermediateFixture + "\n" + geoTrustRootFixture), - want: fmt.Errorf("verification failed: x509: certificate is valid for www.google.com, not www.google.co"), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - g := NewWithT(t) - - cert := &git2go.Certificate{} - if tt.certificate != "" { - x509Cert, err := certificateFromPEM(tt.certificate) - g.Expect(err).ToNot(HaveOccurred()) - cert.X509 = x509Cert - } - - callback := x509Callback(tt.caBundle) - result := callback(cert, false, tt.host) - if tt.want == nil { - g.Expect(result).To(BeNil()) - } else { - g.Expect(result.Error()).To(Equal(tt.want.Error())) - } - }) - } -} - -func Test_transferProgressCallback(t *testing.T) { - tests := []struct { - name string - progress git2go.TransferProgress - cancelFunc func(context.CancelFunc) - wantErr error - }{ - { - name: "ok - in progress", - progress: git2go.TransferProgress{ - TotalObjects: 30, - ReceivedObjects: 21, - }, - cancelFunc: func(cf context.CancelFunc) {}, - wantErr: nil, - }, - { - name: "ok - transfer complete", - progress: git2go.TransferProgress{ - TotalObjects: 30, - ReceivedObjects: 30, - }, - cancelFunc: func(cf context.CancelFunc) {}, - wantErr: nil, - }, - { - name: "ok - transfer complete, context cancelled", - progress: git2go.TransferProgress{ - TotalObjects: 30, - ReceivedObjects: 30, - }, - cancelFunc: func(cf context.CancelFunc) { cf() }, - wantErr: nil, - }, - { - name: "error - context cancelled", - progress: git2go.TransferProgress{ - TotalObjects: 30, - ReceivedObjects: 21, - }, - cancelFunc: func(cf context.CancelFunc) { cf() }, - wantErr: fmt.Errorf("transport close (potentially due to a timeout)"), - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - g := NewWithT(t) - - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() - - tpcb := transferProgressCallback(ctx) - - tt.cancelFunc(cancel) - - result := g.Expect(tpcb(tt.progress)) - if tt.wantErr == nil { - result.To(BeNil()) - } else { - result.To(Equal(tt.wantErr)) - } - }) - } -} - -func Test_transportMessageCallback(t *testing.T) { - tests := []struct { - name string - cancelFunc func(context.CancelFunc) - wantErr error - }{ - { - name: "ok - transport open", - cancelFunc: func(cf context.CancelFunc) {}, - wantErr: nil, - }, - { - name: "error - transport closed", - cancelFunc: func(cf context.CancelFunc) { cf() }, - wantErr: fmt.Errorf("transport closed"), - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - g := NewWithT(t) - - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() - - tmcb := transportMessageCallback(ctx) - - tt.cancelFunc(cancel) - - result := g.Expect(tmcb("")) - if tt.wantErr == nil { - result.To(BeNil()) - } else { - result.To(Equal(tt.wantErr)) - } - }) - } -} - -func Test_pushTransferProgressCallback(t *testing.T) { - type pushProgress struct { - current uint32 - total uint32 - bytes uint - } - tests := []struct { - name string - progress pushProgress - cancelFunc func(context.CancelFunc) - wantErr error - }{ - { - name: "ok - in progress", - progress: pushProgress{current: 20, total: 25}, - cancelFunc: func(cf context.CancelFunc) {}, - wantErr: nil, - }, - { - name: "ok - transfer complete", - progress: pushProgress{current: 25, total: 25}, - cancelFunc: func(cf context.CancelFunc) {}, - wantErr: nil, - }, - { - name: "ok - transfer complete, context cancelled", - progress: pushProgress{current: 25, total: 25}, - cancelFunc: func(cf context.CancelFunc) { cf() }, - wantErr: nil, - }, - { - name: "error - context cancelled", - progress: pushProgress{current: 20, total: 25}, - cancelFunc: func(cf context.CancelFunc) { cf() }, - wantErr: fmt.Errorf("transport close (potentially due to a timeout)"), - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - g := NewWithT(t) - - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() - - ptpcb := pushTransferProgressCallback(ctx) - - tt.cancelFunc(cancel) - - result := g.Expect(ptpcb(tt.progress.current, tt.progress.total, tt.progress.bytes)) - if tt.wantErr == nil { - result.To(BeNil()) - } else { - result.To(Equal(tt.wantErr)) - } - }) - } -} - -func certificateFromPEM(pemBytes string) (*x509.Certificate, error) { - block, _ := pem.Decode([]byte(pemBytes)) - if block == nil { - return nil, errors.New("failed to decode PEM") - } - return x509.ParseCertificate(block.Bytes) -} diff --git a/pkg/git/strategy/proxy/strategy_proxy_test.go b/pkg/git/strategy/proxy/strategy_proxy_test.go index dc06ab18..0006e118 100644 --- a/pkg/git/strategy/proxy/strategy_proxy_test.go +++ b/pkg/git/strategy/proxy/strategy_proxy_test.go @@ -30,10 +30,8 @@ import ( "github.com/elazarl/goproxy" "github.com/fluxcd/pkg/gittestserver" - feathelper "github.com/fluxcd/pkg/runtime/features" . "github.com/onsi/gomega" - "github.com/fluxcd/source-controller/internal/features" "github.com/fluxcd/source-controller/pkg/git" "github.com/fluxcd/source-controller/pkg/git/gogit" "github.com/fluxcd/source-controller/pkg/git/libgit2" @@ -44,12 +42,6 @@ import ( // These tests are run in a different _test.go file because go-git uses the ProxyFromEnvironment function of the net/http package // which caches the Proxy settings, hence not including other tests in the same file ensures a clean proxy setup for the tests to run. func TestCheckoutStrategyForImplementation_Proxied(t *testing.T) { - // for libgit2 we are only testing for managed transport, - // as unmanaged is sunsetting. - // Unmanaged transport does not support HTTP_PROXY. - fg := feathelper.FeatureGates{} - fg.SupportedFeatures(features.FeatureGates()) - managed.InitManagedTransport() type cleanupFunc func() diff --git a/pkg/git/strategy/strategy_test.go b/pkg/git/strategy/strategy_test.go index acee0afa..2aee3a22 100644 --- a/pkg/git/strategy/strategy_test.go +++ b/pkg/git/strategy/strategy_test.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "math/rand" "net/http" "net/url" "os" @@ -39,8 +40,18 @@ import ( "github.com/fluxcd/source-controller/pkg/git" "github.com/fluxcd/source-controller/pkg/git/gogit" "github.com/fluxcd/source-controller/pkg/git/libgit2" + "github.com/fluxcd/source-controller/pkg/git/libgit2/managed" ) +func TestMain(m *testing.M) { + err := managed.InitManagedTransport() + if err != nil { + panic(fmt.Sprintf("failed to initialize libgit2 managed transport: %s", err)) + } + code := m.Run() + os.Exit(code) +} + func TestCheckoutStrategyForImplementation_Auth(t *testing.T) { gitImpls := []git.Implementation{gogit.Implementation, libgit2.Implementation} @@ -61,9 +72,10 @@ func TestCheckoutStrategyForImplementation_Auth(t *testing.T) { }, authOptsFunc: func(g *WithT, u *url.URL, user string, pswd string, ca []byte) *git.AuthOptions { return &git.AuthOptions{ - Transport: git.HTTP, - Username: user, - Password: pswd, + Transport: git.HTTP, + Username: user, + Password: pswd, + TransportOptionsURL: getTransportOptionsURL(git.HTTP), } }, wantFunc: func(g *WithT, cs git.CheckoutStrategy, dir string, repoURL string, authOpts *git.AuthOptions) { @@ -79,10 +91,11 @@ func TestCheckoutStrategyForImplementation_Auth(t *testing.T) { }, authOptsFunc: func(g *WithT, u *url.URL, user, pswd string, ca []byte) *git.AuthOptions { return &git.AuthOptions{ - Transport: git.HTTPS, - Username: user, - Password: pswd, - CAFile: ca, + Transport: git.HTTPS, + Username: user, + Password: pswd, + CAFile: ca, + TransportOptionsURL: getTransportOptionsURL(git.HTTPS), } }, wantFunc: func(g *WithT, cs git.CheckoutStrategy, dir, repoURL string, authOpts *git.AuthOptions) { @@ -105,11 +118,12 @@ func TestCheckoutStrategyForImplementation_Auth(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) return &git.AuthOptions{ - Host: u.Host, // Without this libgit2 returns error "user cancelled hostkey check". - Transport: git.SSH, - Username: "git", // Without this libgit2 returns error "username does not match previous request". - Identity: pair.PrivateKey, - KnownHosts: knownhosts, + Host: u.Host, // Without this libgit2 returns error "user cancelled hostkey check". + Transport: git.SSH, + Username: "git", // Without this libgit2 returns error "username does not match previous request". + Identity: pair.PrivateKey, + KnownHosts: knownhosts, + TransportOptionsURL: getTransportOptionsURL(git.SSH), } }, wantFunc: func(g *WithT, cs git.CheckoutStrategy, dir, repoURL string, authOpts *git.AuthOptions) { @@ -225,9 +239,10 @@ func TestCheckoutStrategyForImplementation_SemVerCheckout(t *testing.T) { repoURL := gitServer.HTTPAddressWithCredentials() + "/" + repoPath authOpts := &git.AuthOptions{ - Transport: git.HTTP, - Username: username, - Password: password, + Transport: git.HTTP, + Username: username, + Password: password, + TransportOptionsURL: getTransportOptionsURL(git.HTTP), } // Create test tags in the repo. @@ -408,9 +423,10 @@ func TestCheckoutStrategyForImplementation_WithCtxTimeout(t *testing.T) { repoURL := gitServer.HTTPAddressWithCredentials() + "/" + repoPath authOpts := &git.AuthOptions{ - Transport: git.HTTP, - Username: username, - Password: password, + Transport: git.HTTP, + Username: username, + Password: password, + TransportOptionsURL: getTransportOptionsURL(git.HTTP), } checkoutOpts := git.CheckoutOptions{ @@ -486,3 +502,12 @@ func mockSignature(time time.Time) *object.Signature { When: time, } } + +func getTransportOptionsURL(transport git.TransportType) string { + letterRunes := []rune("abcdefghijklmnopqrstuvwxyz1234567890") + b := make([]rune, 10) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return string(transport) + "://" + string(b) +}