fluxcd
/
source-controller
mirror of https://github.com/fluxcd/source-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

libgit2: decommission unmanaged transport 

Decommission libgit2 unmanaged transport and remove the related feature
gate, making managed transport the default.

Signed-off-by: Sanskar Jaiswal <jaiswalsanskar078@gmail.com>
This commit is contained in:
Sanskar Jaiswal 2022-07-07 12:18:18 +05:30
parent 8f3cf1276e
commit f5ada743d5
15 changed files with 414 additions and 1115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
controllers/gitrepository_controller.go
View File

@ -236,6 +236,7 @@ func (r *GitRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reques
r.reconcileInclude,
r.reconcileArtifact,
}
recResult, retErr = r.reconcile(ctx, obj, reconcilers)
return
}
@ -428,6 +429,13 @@ func (r *GitRepositoryReconciler) reconcileStorage(ctx context.Context,
// change, it short-circuits the whole reconciliation with an early return.
func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
obj *sourcev1.GitRepository, commit *git.Commit, includes *artifactSet, dir string) (sreconcile.Result, error) {
// Exit early, if we need to use libgit2 AND managed transport hasn't been intialized.
if !managed.Enabled() && obj.Spec.GitImplementation == sourcev1.LibGit2Implementation {
fmt.Println(managed.Enabled())
return sreconcile.ResultEmpty, serror.NewStalling(
errors.New("libgit2 managed transport not initialized"), "Libgit2TransportNotEnabled",
)
}
// Configure authentication strategy to access the source
var authOpts *git.AuthOptions
var err error
@ -745,8 +753,8 @@ func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context,
return nil, e
}
// managed GIT transport only affects the libgit2 implementation
if managed.Enabled() && obj.Spec.GitImplementation == sourcev1.LibGit2Implementation {
// this is needed only for libgit2, due to managed transport.
if obj.Spec.GitImplementation == sourcev1.LibGit2Implementation {
// We set the TransportOptionsURL of this set of authentication options here by constructing
// a unique URL that won't clash in a multi tenant environment. This unique URL is used by
// libgit2 managed transports. This enables us to bypass the inbuilt credentials callback in

3
controllers/suite_test.go
View File

@ -37,7 +37,6 @@ import (
ctrl "sigs.k8s.io/controller-runtime"
"github.com/fluxcd/pkg/runtime/controller"
feathelper "github.com/fluxcd/pkg/runtime/features"
"github.com/fluxcd/pkg/runtime/testenv"
"github.com/fluxcd/pkg/testserver"
"github.com/phayes/freeport"
@ -206,8 +205,6 @@ func TestMain(m *testing.M) {
panic(fmt.Sprintf("Failed to create a test registry server: %v", err))
}
fg := feathelper.FeatureGates{}
fg.SupportedFeatures(features.FeatureGates())
managed.InitManagedTransport()
if err := (&GitRepositoryReconciler{

13
internal/features/features.go
View File

@ -30,25 +30,12 @@ const (
// the last revision is still the same at the target repository,
// and if that is so, skips the reconciliation.
OptimizedGitClones = "OptimizedGitClones"
// GitManagedTransport implements a managed transport for GitRepository
// objects that use the libgit2 implementation.
//
// When enabled, improves the reliability of libgit2 reconciliations,
// by enforcing timeouts and ensuring libgit2 cannot hijack the process
// and hang it indefinitely.
GitManagedTransport = "GitManagedTransport"
)
var features = map[string]bool{
// OptimizedGitClones
// opt-out from v0.25
OptimizedGitClones: true,
// GitManagedTransport
// opt-in from v0.22 (via environment variable)
// opt-out from v0.25
GitManagedTransport: true,
}
// DefaultFeatureGates contains a list of all supported feature gates and

12
main.go
View File

@ -310,15 +310,9 @@ func main() {
startFileServer(storage.BasePath, storageAddr, setupLog)
}()
if enabled, _ := features.Enabled(features.GitManagedTransport); enabled {
managed.InitManagedTransport()
} else {
if optimize, _ := feathelper.Enabled(features.OptimizedGitClones); optimize {
features.Disable(features.OptimizedGitClones)
setupLog.Info(
"disabling optimized git clones; git clones can only be optimized when using managed transport",
)
}
if err = managed.InitManagedTransport(); err != nil {
// Log the error, but don't exit so as to not block reconcilers that are healthy.
setupLog.Error(err, "unable to initialize libgit2 managed transport")
}
setupLog.Info("starting manager")

201
pkg/git/libgit2/checkout.go
View File

@ -18,6 +18,7 @@ package libgit2
import (
"context"
"errors"
"fmt"
"sort"
"strings"
@ -72,33 +73,13 @@ type CheckoutBranch struct {
func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) {
defer recoverPanic(&err)
// This branching is temporary, to address the transient panics observed when using unmanaged transport.
// The panics probably happen because we perform multiple fetch ops (introduced as a part of optimizing git clones).
// The branching lets us establish a clear code path to help us be certain of the expected behaviour.
// When we get rid of unmanaged transports, we can get rid of this branching as well.
if managed.Enabled() {
// We store the target URL and auth options mapped to a unique ID. We overwrite the target URL
// with the TransportOptionsURL, because managed transports don't provide a way for any kind of
// dependency injection. This lets us have a way of doing interop between application level code
// and transport level code.
// Performing all fetch operations with the TransportOptionsURL as the URL, lets the managed
// transport action use it to fetch the registered transport options which contains the
// _actual_ target URL and the correct credentials to use.
if opts == nil {
return nil, fmt.Errorf("can't use managed transport with an empty set of auth options")
err = registerManagedTransportOptions(ctx, url, opts)
if err != nil {
return nil, err
}
if opts.TransportOptionsURL == "" {
return nil, fmt.Errorf("can't use managed transport without a valid transport auth id.")
}
managed.AddTransportOptions(opts.TransportOptionsURL, managed.TransportOptions{
TargetURL: url,
AuthOpts: opts,
ProxyOptions: &git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
Context: ctx,
})
url = opts.TransportOptionsURL
transportOptsURL := opts.TransportOptionsURL
remoteCallBacks := managed.RemoteCallbacks()
defer managed.RemoveTransportOptions(opts.TransportOptionsURL)
defer managed.RemoveTransportOptions(transportOptsURL)
repo, remote, err := initializeRepoWithRemote(ctx, path, url, opts)
if err != nil {
@ -109,7 +90,7 @@ func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *g
if err != nil {
remote.Free()
repo.Free()
return nil, fmt.Errorf("unable to fetch-connect to remote '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
return nil, fmt.Errorf("unable to fetch-connect to remote '%s': %w", url, gitutil.LibGit2Error(err))
}
defer func() {
remote.Disconnect()
@ -122,7 +103,7 @@ func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *g
if c.LastRevision != "" {
heads, err := remote.Ls(c.Branch)
if err != nil {
return nil, fmt.Errorf("unable to remote ls for '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
return nil, fmt.Errorf("unable to remote ls for '%s': %w", url, gitutil.LibGit2Error(err))
}
if len(heads) > 0 {
hash := heads[0].Id.String()
@ -146,21 +127,18 @@ func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *g
},
"")
if err != nil {
return nil, fmt.Errorf("unable to fetch remote '%s': %w",
managed.EffectiveURL(url), gitutil.LibGit2Error(err))
return nil, fmt.Errorf("unable to fetch remote '%s': %w", url, gitutil.LibGit2Error(err))
}
branch, err := repo.References.Lookup(fmt.Sprintf("refs/remotes/origin/%s", c.Branch))
if err != nil {
return nil, fmt.Errorf("unable to lookup branch '%s' for '%s': %w",
c.Branch, managed.EffectiveURL(url), gitutil.LibGit2Error(err))
return nil, fmt.Errorf("unable to lookup branch '%s' for '%s': %w", c.Branch, url, gitutil.LibGit2Error(err))
}
defer branch.Free()
upstreamCommit, err := repo.LookupCommit(branch.Target())
if err != nil {
return nil, fmt.Errorf("unable to lookup commit '%s' for '%s': %w",
c.Branch, managed.EffectiveURL(url), gitutil.LibGit2Error(err))
return nil, fmt.Errorf("unable to lookup commit '%s' for '%s': %w", c.Branch, url, gitutil.LibGit2Error(err))
}
defer upstreamCommit.Free()
@ -211,38 +189,6 @@ func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *g
}
defer cc.Free()
return buildCommit(cc, "refs/heads/"+c.Branch), nil
} else {
return c.checkoutUnmanaged(ctx, path, url, opts)
}
}
func (c *CheckoutBranch) checkoutUnmanaged(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) {
repo, err := git2go.Clone(url, path, &git2go.CloneOptions{
FetchOptions: git2go.FetchOptions{
DownloadTags: git2go.DownloadTagsNone,
RemoteCallbacks: RemoteCallbacks(ctx, opts),
ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
},
CheckoutOptions: git2go.CheckoutOptions{
Strategy: git2go.CheckoutForce,
},
CheckoutBranch: c.Branch,
})
if err != nil {
return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
defer repo.Free()
head, err := repo.Head()
if err != nil {
return nil, fmt.Errorf("git resolve HEAD error: %w", err)
}
defer head.Free()
cc, err := repo.LookupCommit(head.Target())
if err != nil {
return nil, fmt.Errorf("failed to lookup HEAD commit '%s' for branch '%s': %w", head.Target(), c.Branch, err)
}
defer cc.Free()
return buildCommit(cc, "refs/heads/"+c.Branch), nil
}
@ -254,23 +200,13 @@ type CheckoutTag struct {
func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) {
defer recoverPanic(&err)
// This branching is temporary, to address the transient panics observed when using unmanaged transport.
// The panics probably happen because we perform multiple fetch ops (introduced as a part of optimizing git clones).
// The branching lets us establish a clear code path to help us be certain of the expected behaviour.
// When we get rid of unmanaged transports, we can get rid of this branching as well.
if managed.Enabled() {
if opts.TransportOptionsURL == "" {
return nil, fmt.Errorf("can't use managed transport without a valid transport auth id.")
err = registerManagedTransportOptions(ctx, url, opts)
if err != nil {
return nil, err
}
managed.AddTransportOptions(opts.TransportOptionsURL, managed.TransportOptions{
TargetURL: url,
AuthOpts: opts,
ProxyOptions: &git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
Context: ctx,
})
url = opts.TransportOptionsURL
transportOptsURL := opts.TransportOptionsURL
remoteCallBacks := managed.RemoteCallbacks()
defer managed.RemoveTransportOptions(opts.TransportOptionsURL)
defer managed.RemoveTransportOptions(transportOptsURL)
repo, remote, err := initializeRepoWithRemote(ctx, path, url, opts)
if err != nil {
@ -281,7 +217,7 @@ func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, opts *git.
if err != nil {
remote.Free()
repo.Free()
return nil, fmt.Errorf("unable to fetch-connect to remote '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
return nil, fmt.Errorf("unable to fetch-connect to remote '%s': %w", url, gitutil.LibGit2Error(err))
}
defer func() {
remote.Disconnect()
@ -294,7 +230,7 @@ func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, opts *git.
if c.LastRevision != "" {
heads, err := remote.Ls(c.Tag)
if err != nil {
return nil, fmt.Errorf("unable to remote ls for '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
return nil, fmt.Errorf("unable to remote ls for '%s': %w", url, gitutil.LibGit2Error(err))
}
if len(heads) > 0 {
hash := heads[0].Id.String()
@ -328,33 +264,9 @@ func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, opts *git.
"")
if err != nil {
return nil, fmt.Errorf("unable to fetch remote '%s': %w",
managed.EffectiveURL(url), gitutil.LibGit2Error(err))
return nil, fmt.Errorf("unable to fetch remote '%s': %w", url, gitutil.LibGit2Error(err))
}
cc, err := checkoutDetachedDwim(repo, c.Tag)
if err != nil {
return nil, err
}
defer cc.Free()
return buildCommit(cc, "refs/tags/"+c.Tag), nil
} else {
return c.checkoutUnmanaged(ctx, path, url, opts)
}
}
func (c *CheckoutTag) checkoutUnmanaged(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) {
repo, err := git2go.Clone(url, path, &git2go.CloneOptions{
FetchOptions: git2go.FetchOptions{
DownloadTags: git2go.DownloadTagsAll,
RemoteCallbacks: RemoteCallbacks(ctx, opts),
ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
},
})
if err != nil {
return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
defer repo.Free()
cc, err := checkoutDetachedDwim(repo, c.Tag)
if err != nil {
return nil, err
@ -370,31 +282,21 @@ type CheckoutCommit struct {
func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) {
defer recoverPanic(&err)
remoteCallBacks := RemoteCallbacks(ctx, opts)
if managed.Enabled() {
if opts.TransportOptionsURL == "" {
return nil, fmt.Errorf("can't use managed transport without a valid transport auth id.")
}
managed.AddTransportOptions(opts.TransportOptionsURL, managed.TransportOptions{
TargetURL: url,
AuthOpts: opts,
ProxyOptions: &git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
Context: ctx,
})
url = opts.TransportOptionsURL
remoteCallBacks = managed.RemoteCallbacks()
defer managed.RemoveTransportOptions(opts.TransportOptionsURL)
err = registerManagedTransportOptions(ctx, url, opts)
if err != nil {
return nil, err
}
transportOptsURL := opts.TransportOptionsURL
defer managed.RemoveTransportOptions(transportOptsURL)
repo, err := git2go.Clone(url, path, &git2go.CloneOptions{
repo, err := git2go.Clone(transportOptsURL, path, &git2go.CloneOptions{
FetchOptions: git2go.FetchOptions{
DownloadTags: git2go.DownloadTagsNone,
RemoteCallbacks: remoteCallBacks,
RemoteCallbacks: managed.RemoteCallbacks(),
},
})
if err != nil {
return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
return nil, fmt.Errorf("unable to clone '%s': %w", url, gitutil.LibGit2Error(err))
}
defer repo.Free()
oid, err := git2go.NewOid(c.Commit)
@ -415,36 +317,26 @@ type CheckoutSemVer struct {
func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) {
defer recoverPanic(&err)
remoteCallBacks := RemoteCallbacks(ctx, opts)
if managed.Enabled() {
if opts.TransportOptionsURL == "" {
return nil, fmt.Errorf("can't use managed transport without a valid transport auth id.")
}
managed.AddTransportOptions(opts.TransportOptionsURL, managed.TransportOptions{
TargetURL: url,
AuthOpts: opts,
ProxyOptions: &git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
Context: ctx,
})
url = opts.TransportOptionsURL
remoteCallBacks = managed.RemoteCallbacks()
defer managed.RemoveTransportOptions(opts.TransportOptionsURL)
err = registerManagedTransportOptions(ctx, url, opts)
if err != nil {
return nil, err
}
transportOptsURL := opts.TransportOptionsURL
defer managed.RemoveTransportOptions(transportOptsURL)
verConstraint, err := semver.NewConstraint(c.SemVer)
if err != nil {
return nil, fmt.Errorf("semver parse error: %w", err)
}
repo, err := git2go.Clone(url, path, &git2go.CloneOptions{
repo, err := git2go.Clone(transportOptsURL, path, &git2go.CloneOptions{
FetchOptions: git2go.FetchOptions{
DownloadTags: git2go.DownloadTagsAll,
RemoteCallbacks: remoteCallBacks,
RemoteCallbacks: managed.RemoteCallbacks(),
},
})
if err != nil {
return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
return nil, fmt.Errorf("unable to clone '%s': %w", url, gitutil.LibGit2Error(err))
}
defer repo.Free()
@ -630,6 +522,29 @@ func initializeRepoWithRemote(ctx context.Context, path, url string, opts *git.A
return repo, remote, nil
}
// registerManagedTransportOptions registers the given url and it's transport options.
// Callers must make sure to call `managed.RemoveTransportOptions()` to avoid increase in
// memory consumption.
// We store the target URL, auth options, etc. mapped to TransporOptsURL because managed transports
// don't provide a way for any kind of dependency injection.
// This lets us have a way of doing interop between application level code and transport level code
// which enables us to fetch the required credentials, context, etc. at the transport level.
func registerManagedTransportOptions(ctx context.Context, url string, authOpts *git.AuthOptions) error {
if authOpts == nil {
return errors.New("can't checkout using libgit2 with an empty set of auth options")
}
if authOpts.TransportOptionsURL == "" {
return errors.New("can't checkout using libgit2 without a valid transport auth id")
}
managed.AddTransportOptions(authOpts.TransportOptionsURL, managed.TransportOptions{
TargetURL: url,
AuthOpts: authOpts,
ProxyOptions: &git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
Context: ctx,
})
return nil
}
func recoverPanic(err *error) {
if r := recover(); r != nil {
*err = fmt.Errorf("recovered from git2go panic: %v", r)

37
pkg/git/libgit2/managed_test.go → pkg/git/libgit2/checkout_ssh_test.go
View File

@ -19,7 +19,6 @@ package libgit2
import (
"context"
"fmt"
"math/rand"
"net/url"
"os"
"path/filepath"
@ -31,22 +30,17 @@ import (
"github.com/fluxcd/pkg/gittestserver"
"github.com/fluxcd/pkg/ssh"
feathelper "github.com/fluxcd/pkg/runtime/features"
. "github.com/onsi/gomega"
cryptossh "golang.org/x/crypto/ssh"
"github.com/fluxcd/source-controller/internal/features"
"github.com/fluxcd/source-controller/pkg/git"
"github.com/fluxcd/source-controller/pkg/git/libgit2/managed"
)
const testRepositoryPath = "../testdata/git/repo"
// Test_managedSSH_KeyTypes assures support for the different
// Test_ssh_keyTypes assures support for the different
// types of keys for SSH Authentication supported by Flux.
func Test_managedSSH_KeyTypes(t *testing.T) {
enableManagedTransport()
func Test_ssh_keyTypes(t *testing.T) {
tests := []struct {
name string
keyType ssh.KeyPairType
@ -171,11 +165,9 @@ func Test_managedSSH_KeyTypes(t *testing.T) {
}
}
// Test_managedSSH_KeyExchangeAlgos assures support for the different
// Test_ssh_keyExchangeAlgos assures support for the different
// types of SSH key exchange algorithms supported by Flux.
func Test_managedSSH_KeyExchangeAlgos(t *testing.T) {
enableManagedTransport()
func Test_ssh_keyExchangeAlgos(t *testing.T) {
tests := []struct {
name string
ClientKex []string
@ -294,11 +286,9 @@ func Test_managedSSH_KeyExchangeAlgos(t *testing.T) {
}
}
// Test_managedSSH_HostKeyAlgos assures support for the different
// Test_ssh_hostKeyAlgos assures support for the different
// types of SSH Host Key algorithms supported by Flux.
func Test_managedSSH_HostKeyAlgos(t *testing.T) {
enableManagedTransport()
func Test_ssh_hostKeyAlgos(t *testing.T) {
tests := []struct {
name string
keyType ssh.KeyPairType
@ -457,18 +447,3 @@ func Test_managedSSH_HostKeyAlgos(t *testing.T) {
})
}
}
func getTransportOptionsURL(transport git.TransportType) string {
letterRunes := []rune("abcdefghijklmnopqrstuvwxyz1234567890")
b := make([]rune, 10)
for i := range b {
b[i] = letterRunes[rand.Intn(len(letterRunes))]
}
return string(transport) + "://" + string(b)
}
func enableManagedTransport() {
fg := feathelper.FeatureGates{}
fg.SupportedFeatures(features.FeatureGates())
managed.InitManagedTransport()
}

259
pkg/git/libgit2/checkout_test.go
View File

@ -20,6 +20,7 @@ import (
"context"
"errors"
"fmt"
"math/rand"
"os"
"path/filepath"
"testing"
@ -30,17 +31,19 @@ import (
. "github.com/onsi/gomega"
"github.com/fluxcd/source-controller/pkg/git"
mt "github.com/fluxcd/source-controller/pkg/git/libgit2/managed"
"github.com/fluxcd/source-controller/pkg/git/libgit2/managed"
)
func TestCheckoutBranch_unmanaged(t *testing.T) {
checkoutBranch(t, false)
func TestMain(m *testing.M) {
err := managed.InitManagedTransport()
if err != nil {
panic(fmt.Sprintf("failed to initialize libgit2 managed transport: %s", err))
}
code := m.Run()
os.Exit(code)
}
// checkoutBranch is a test helper function which runs the tests for checking out
// via CheckoutBranch.
func checkoutBranch(t *testing.T, managed bool) {
func TestCheckoutBranch_Checkout(t *testing.T) {
// we use a HTTP Git server instead of a bare repo (for all tests in this
// package), because our managed transports don't support the file protocol,
// so we wouldn't actually be using our custom transports, if we used a bare
@ -138,7 +141,6 @@ func checkoutBranch(t *testing.T, managed bool) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
g.Expect(mt.Enabled()).To(Equal(managed))
branch := CheckoutBranch{
Branch: tt.branch,
@ -159,9 +161,7 @@ func checkoutBranch(t *testing.T, managed bool) {
}
g.Expect(err).ToNot(HaveOccurred())
g.Expect(cc.String()).To(Equal(tt.branch + "/" + tt.expectedCommit))
if managed {
g.Expect(git.IsConcreteCommit(*cc)).To(Equal(tt.expectedConcreteCommit))
}
if tt.expectedConcreteCommit {
for k, v := range tt.filesCreated {
@ -173,13 +173,7 @@ func checkoutBranch(t *testing.T, managed bool) {
}
}
func TestCheckoutTag_unmanaged(t *testing.T) {
checkoutTag(t, false)
}
// checkoutTag is a test helper function which runs the tests for checking out
// via CheckoutTag.
func checkoutTag(t *testing.T, managed bool) {
func TestCheckoutTag_Checkout(t *testing.T) {
type testTag struct {
name string
annotated bool
@ -229,7 +223,6 @@ func checkoutTag(t *testing.T, managed bool) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
g.Expect(mt.Enabled()).To(Equal(managed))
server, err := gittestserver.NewTempGitServer()
g.Expect(err).ToNot(HaveOccurred())
@ -297,11 +290,8 @@ func checkoutTag(t *testing.T, managed bool) {
targetTagCommit := tagCommits[tt.checkoutTag]
g.Expect(err).ToNot(HaveOccurred())
g.Expect(cc.String()).To(Equal(tt.checkoutTag + "/" + targetTagCommit.Id().String()))
if managed {
g.Expect(git.IsConcreteCommit(*cc)).To(Equal(tt.expectConcreteCommit))
}
// Check file content only when there's an actual checkout.
if tt.lastRevTag != tt.checkoutTag {
g.Expect(filepath.Join(tmpDir, "tag")).To(BeARegularFile())
@ -311,15 +301,8 @@ func checkoutTag(t *testing.T, managed bool) {
}
}
func TestCheckoutCommit_unmanaged(t *testing.T) {
checkoutCommit(t, false)
}
// checkoutCommit is a test helper function which runs the tests for checking out
// via CheckoutCommit.
func checkoutCommit(t *testing.T, managed bool) {
func TestCheckoutCommit_Checkout(t *testing.T) {
g := NewWithT(t)
g.Expect(mt.Enabled()).To(Equal(managed))
server, err := gittestserver.NewTempGitServer()
if err != nil {
@ -380,13 +363,7 @@ func checkoutCommit(t *testing.T, managed bool) {
g.Expect(cc).To(BeNil())
}
func TestCheckoutTagSemVer_unmanaged(t *testing.T) {
checkoutSemVer(t, false)
}
// checkoutSemVer is a test helper function which runs the tests for checking out
// via CheckoutSemVer.
func checkoutSemVer(t *testing.T, managed bool) {
func TestCheckoutSemVer_Checkout(t *testing.T) {
g := NewWithT(t)
now := time.Now()
@ -498,7 +475,6 @@ func checkoutSemVer(t *testing.T, managed bool) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
g.Expect(mt.Enabled()).To(Equal(managed))
semVer := CheckoutSemVer{
SemVer: tt.constraint,
@ -524,6 +500,112 @@ func checkoutSemVer(t *testing.T, managed bool) {
}
}
func Test_initializeRepoWithRemote(t *testing.T) {
g := NewWithT(t)
tmp := t.TempDir()
ctx := context.TODO()
testRepoURL := "https://example.com/foo/bar"
testRepoURL2 := "https://example.com/foo/baz"
authOpts, err := git.AuthOptionsWithoutSecret(testRepoURL)
g.Expect(err).ToNot(HaveOccurred())
authOpts.TransportOptionsURL = "https://bar123"
authOpts2, err := git.AuthOptionsWithoutSecret(testRepoURL2)
g.Expect(err).ToNot(HaveOccurred())
authOpts2.TransportOptionsURL = "https://baz789"
// Fresh initialization.
repo, remote, err := initializeRepoWithRemote(ctx, tmp, testRepoURL, authOpts)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(repo.IsBare()).To(BeFalse())
g.Expect(remote.Name()).To(Equal(defaultRemoteName))
g.Expect(remote.Url()).To(Equal(authOpts.TransportOptionsURL))
remote.Free()
repo.Free()
// Reinitialize to ensure it reuses the existing origin.
repo, remote, err = initializeRepoWithRemote(ctx, tmp, testRepoURL, authOpts)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(repo.IsBare()).To(BeFalse())
g.Expect(remote.Name()).To(Equal(defaultRemoteName))
g.Expect(remote.Url()).To(Equal(authOpts.TransportOptionsURL))
remote.Free()
repo.Free()
// Reinitialize with a different remote URL for existing origin.
repo, remote, err = initializeRepoWithRemote(ctx, tmp, testRepoURL2, authOpts2)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(repo.IsBare()).To(BeFalse())
g.Expect(remote.Name()).To(Equal(defaultRemoteName))
g.Expect(remote.Url()).To(Equal(authOpts2.TransportOptionsURL))
remote.Free()
repo.Free()
}
func TestCheckoutStrategyForOptions(t *testing.T) {
tests := []struct {
name string
opts git.CheckoutOptions
expectedStrat git.CheckoutStrategy
}{
{
name: "commit works",
opts: git.CheckoutOptions{
Commit: "commit",
},
expectedStrat: &CheckoutCommit{
Commit: "commit",
},
},
{
name: "semver works",
opts: git.CheckoutOptions{
SemVer: ">= 1.0.0",
},
expectedStrat: &CheckoutSemVer{
SemVer: ">= 1.0.0",
},
},
{
name: "tag with latest revision works",
opts: git.CheckoutOptions{
Tag: "v0.1.0",
LastRevision: "ar34oi2njrngjrng",
},
expectedStrat: &CheckoutTag{
Tag: "v0.1.0",
LastRevision: "ar34oi2njrngjrng",
},
},
{
name: "branch with latest revision works",
opts: git.CheckoutOptions{
Branch: "main",
LastRevision: "rrgij20mkmrg",
},
expectedStrat: &CheckoutBranch{
Branch: "main",
LastRevision: "rrgij20mkmrg",
},
},
{
name: "empty branch falls back to default",
opts: git.CheckoutOptions{},
expectedStrat: &CheckoutBranch{
Branch: git.DefaultBranch,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
strat := CheckoutStrategyForOptions(context.TODO(), tt.opts)
g.Expect(strat).To(Equal(tt.expectedStrat))
})
}
}
func initBareRepo(t *testing.T) (*git2go.Repository, error) {
tmpDir := t.TempDir()
repo, err := git2go.InitRepository(tmpDir, true)
@ -615,102 +697,11 @@ func mockSignature(time time.Time) *git2go.Signature {
}
}
func TestInitializeRepoWithRemote(t *testing.T) {
g := NewWithT(t)
g.Expect(mt.Enabled()).To(BeFalse())
tmp := t.TempDir()
ctx := context.TODO()
testRepoURL := "https://example.com/foo/bar"
testRepoURL2 := "https://example.com/foo/baz"
authOpts, err := git.AuthOptionsWithoutSecret(testRepoURL)
g.Expect(err).ToNot(HaveOccurred())
authOpts2, err := git.AuthOptionsWithoutSecret(testRepoURL2)
g.Expect(err).ToNot(HaveOccurred())
// Fresh initialization.
repo, remote, err := initializeRepoWithRemote(ctx, tmp, testRepoURL, authOpts)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(repo.IsBare()).To(BeFalse())
g.Expect(remote.Name()).To(Equal(defaultRemoteName))
g.Expect(remote.Url()).To(Equal(testRepoURL))
remote.Free()
repo.Free()
// Reinitialize to ensure it reuses the existing origin.
repo, remote, err = initializeRepoWithRemote(ctx, tmp, testRepoURL, authOpts)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(repo.IsBare()).To(BeFalse())
g.Expect(remote.Name()).To(Equal(defaultRemoteName))
g.Expect(remote.Url()).To(Equal(testRepoURL))
remote.Free()
repo.Free()
// Reinitialize with a different remote URL for existing origin.
_, _, err = initializeRepoWithRemote(ctx, tmp, testRepoURL2, authOpts2)
g.Expect(err).To(HaveOccurred())
}
func TestCheckoutStrategyForOptions(t *testing.T) {
tests := []struct {
name string
opts git.CheckoutOptions
expectedStrat git.CheckoutStrategy
}{
{
name: "commit works",
opts: git.CheckoutOptions{
Commit: "commit",
},
expectedStrat: &CheckoutCommit{
Commit: "commit",
},
},
{
name: "semver works",
opts: git.CheckoutOptions{
SemVer: ">= 1.0.0",
},
expectedStrat: &CheckoutSemVer{
SemVer: ">= 1.0.0",
},
},
{
name: "tag with latest revision works",
opts: git.CheckoutOptions{
Tag: "v0.1.0",
LastRevision: "ar34oi2njrngjrng",
},
expectedStrat: &CheckoutTag{
Tag: "v0.1.0",
LastRevision: "ar34oi2njrngjrng",
},
},
{
name: "branch with latest revision works",
opts: git.CheckoutOptions{
Branch: "main",
LastRevision: "rrgij20mkmrg",
},
expectedStrat: &CheckoutBranch{
Branch: "main",
LastRevision: "rrgij20mkmrg",
},
},
{
name: "empty branch falls back to default",
opts: git.CheckoutOptions{},
expectedStrat: &CheckoutBranch{
Branch: git.DefaultBranch,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
strat := CheckoutStrategyForOptions(context.TODO(), tt.opts)
g.Expect(strat).To(Equal(tt.expectedStrat))
})
func getTransportOptionsURL(transport git.TransportType) string {
letterRunes := []rune("abcdefghijklmnopqrstuvwxyz1234567890")
b := make([]rune, 10)
for i := range b {
b[i] = letterRunes[rand.Intn(len(letterRunes))]
}
return string(transport) + "://" + string(b)
}

1
pkg/git/libgit2/managed/http.go
View File

@ -419,6 +419,7 @@ func (self *httpSmartSubtransportStream) sendRequest() error {
URL: self.req.URL,
Header: self.req.Header,
}
if req.Method == "POST" {
if len(content) == 0 {
// a copy of the request body needs to be saved so

31
pkg/git/libgit2/managed/http_test.go
View File

@ -30,6 +30,15 @@ import (
git2go "github.com/libgit2/git2go/v33"
)
func TestMain(m *testing.M) {
err := InitManagedTransport()
if err != nil {
panic(fmt.Sprintf("failed to initialize libgit2 managed transport: %s", err))
}
code := m.Run()
os.Exit(code)
}
func TestHttpAction_CreateClientRequest(t *testing.T) {
authOpts := git.AuthOptions{
Username: "user",
@ -56,8 +65,8 @@ func TestHttpAction_CreateClientRequest(t *testing.T) {
g.Expect(req.URL.String()).To(Equal("https://final-target/abc/git-upload-pack"))
g.Expect(req.Method).To(Equal("POST"))
g.Expect(req.Header).To(BeEquivalentTo(map[string][]string{
"User-Agent": []string{"git/2.0 (flux-libgit2)"},
"Content-Type": []string{"application/x-git-upload-pack-request"},
"User-Agent": {"git/2.0 (flux-libgit2)"},
"Content-Type": {"application/x-git-upload-pack-request"},
}))
},
wantedErr: nil,
@ -70,7 +79,7 @@ func TestHttpAction_CreateClientRequest(t *testing.T) {
g.Expect(req.URL.String()).To(Equal("https://final-target/abc/info/refs?service=git-upload-pack"))
g.Expect(req.Method).To(Equal("GET"))
g.Expect(req.Header).To(BeEquivalentTo(map[string][]string{
"User-Agent": []string{"git/2.0 (flux-libgit2)"},
"User-Agent": {"git/2.0 (flux-libgit2)"},
}))
},
wantedErr: nil,
@ -86,8 +95,8 @@ func TestHttpAction_CreateClientRequest(t *testing.T) {
g.Expect(req.URL.String()).To(Equal("https://final-target/abc/git-receive-pack"))
g.Expect(req.Method).To(Equal("POST"))
g.Expect(req.Header).To(BeEquivalentTo(map[string][]string{
"Content-Type": []string{"application/x-git-receive-pack-request"},
"User-Agent": []string{"git/2.0 (flux-libgit2)"},
"Content-Type": {"application/x-git-receive-pack-request"},
"User-Agent": {"git/2.0 (flux-libgit2)"},
}))
},
wantedErr: nil,
@ -100,7 +109,7 @@ func TestHttpAction_CreateClientRequest(t *testing.T) {
g.Expect(req.URL.String()).To(Equal("https://final-target/abc/info/refs?service=git-receive-pack"))
g.Expect(req.Method).To(Equal("GET"))
g.Expect(req.Header).To(BeEquivalentTo(map[string][]string{
"User-Agent": []string{"git/2.0 (flux-libgit2)"},
"User-Agent": {"git/2.0 (flux-libgit2)"},
}))
},
wantedErr: nil,
@ -162,7 +171,7 @@ func TestHttpAction_CreateClientRequest(t *testing.T) {
}
}
func TestHTTPManagedTransport_E2E(t *testing.T) {
func TestHTTP_E2E(t *testing.T) {
g := NewWithT(t)
server, err := gittestserver.NewTempGitServer()
@ -178,9 +187,6 @@ func TestHTTPManagedTransport_E2E(t *testing.T) {
g.Expect(err).ToNot(HaveOccurred())
defer server.StopHTTP()
// Force managed transport to be enabled
InitManagedTransport()
repoPath := "test.git"
err = server.InitRepo("../../testdata/git/repo", git.DefaultBranch, repoPath)
g.Expect(err).ToNot(HaveOccurred())
@ -252,7 +258,7 @@ func TestTrimActionSuffix(t *testing.T) {
}
}
func TestHTTPManagedTransport_HandleRedirect(t *testing.T) {
func TestHTTP_HandleRedirect(t *testing.T) {
tests := []struct {
name string
repoURL string
@ -261,9 +267,6 @@ func TestHTTPManagedTransport_HandleRedirect(t *testing.T) {
{name: "handle gitlab redirect", repoURL: "https://gitlab.com/stefanprodan/podinfo"},
}
// Force managed transport to be enabled
InitManagedTransport()
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)

3
pkg/git/libgit2/managed/ssh_test.go
View File

@ -74,7 +74,7 @@ func TestSSHAction_clientConfig(t *testing.T) {
}
}
func TestSSHManagedTransport_E2E(t *testing.T) {
func TestSSH_E2E(t *testing.T) {
g := NewWithT(t)
server, err := gittestserver.NewTempGitServer()
@ -90,7 +90,6 @@ func TestSSHManagedTransport_E2E(t *testing.T) {
server.StartSSH()
}()
defer server.StopSSH()
InitManagedTransport()
kp, err := ssh.NewEd25519Generator().Generate()
g.Expect(err).ToNot(HaveOccurred())

46
pkg/git/libgit2/managed_checkout_test.go
View File

@ -1,46 +0,0 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// This file is named `managed_checkout_test.go` on purpose to make sure that
// tests needing to use unmanaged transports run before the tests that use managed
// transports do, since the the former are present in `checkout_test.go`. `checkout_test.go`
// comes first in this package (alphabetically speaking), which makes golang run the tests
// in that file first.
package libgit2
import (
"testing"
)
func TestCheckoutBranch_CheckoutManaged(t *testing.T) {
enableManagedTransport()
checkoutBranch(t, true)
}
func TestCheckoutTag_CheckoutManaged(t *testing.T) {
enableManagedTransport()
checkoutTag(t, true)
}
func TestCheckoutCommit_CheckoutManaged(t *testing.T) {
enableManagedTransport()
checkoutCommit(t, true)
}
func TestCheckoutTagSemVer_CheckoutManaged(t *testing.T) {
enableManagedTransport()
checkoutSemVer(t, true)
}

165
pkg/git/libgit2/transport.go
View File

@ -1,165 +0,0 @@
/*
Copyright 2020 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package libgit2
import (
"context"
"crypto/x509"
"fmt"
"time"
git2go "github.com/libgit2/git2go/v33"
"golang.org/x/crypto/ssh"
"github.com/fluxcd/source-controller/pkg/git"
"github.com/fluxcd/source-controller/pkg/git/libgit2/managed"
)
var (
now = time.Now
)
// RemoteCallbacks constructs RemoteCallbacks with credentialsCallback and
// certificateCallback, and the given options if the given opts is not nil.
func RemoteCallbacks(ctx context.Context, opts *git.AuthOptions) git2go.RemoteCallbacks {
if opts != nil {
return git2go.RemoteCallbacks{
SidebandProgressCallback: transportMessageCallback(ctx),
TransferProgressCallback: transferProgressCallback(ctx),
PushTransferProgressCallback: pushTransferProgressCallback(ctx),
CredentialsCallback: credentialsCallback(opts),
CertificateCheckCallback: certificateCallback(opts),
}
}
return git2go.RemoteCallbacks{}
}
// transferProgressCallback constructs TransferProgressCallbacks which signals
// libgit2 it should stop the transfer when the given context is closed (due to
// e.g. a timeout).
func transferProgressCallback(ctx context.Context) git2go.TransferProgressCallback {
return func(p git2go.TransferProgress) error {
// Early return if all the objects have been received.
if p.ReceivedObjects == p.TotalObjects {
return nil
}
select {
case <-ctx.Done():
return fmt.Errorf("transport close (potentially due to a timeout)")
default:
return nil
}
}
}
// transportMessageCallback constructs TransportMessageCallback which signals
// libgit2 it should cancel the network operation when the given context is
// closed.
func transportMessageCallback(ctx context.Context) git2go.TransportMessageCallback {
return func(_ string) error {
select {
case <-ctx.Done():
return fmt.Errorf("transport closed")
default:
return nil
}
}
}
// pushTransferProgressCallback constructs PushTransferProgressCallback which
// signals libgit2 it should stop the push transfer when the given context is
// closed (due to e.g. a timeout).
func pushTransferProgressCallback(ctx context.Context) git2go.PushTransferProgressCallback {
return func(current, total uint32, _ uint) error {
// Early return if current equals total.
if current == total {
return nil
}
select {
case <-ctx.Done():
return fmt.Errorf("transport close (potentially due to a timeout)")
default:
return nil
}
}
}
// credentialsCallback constructs CredentialsCallbacks with the given options
// for git.Transport, and returns the result.
func credentialsCallback(opts *git.AuthOptions) git2go.CredentialsCallback {
return func(url string, username string, allowedTypes git2go.CredentialType) (*git2go.Credential, error) {
if allowedTypes&(git2go.CredentialTypeSSHKey|git2go.CredentialTypeSSHCustom|git2go.CredentialTypeSSHMemory) != 0 {
var (
signer ssh.Signer
err error
)
if opts.Password != "" {
signer, err = ssh.ParsePrivateKeyWithPassphrase(opts.Identity, []byte(opts.Password))
} else {
signer, err = ssh.ParsePrivateKey(opts.Identity)
}
if err != nil {
return nil, err
}
return git2go.NewCredentialSSHKeyFromSigner(opts.Username, signer)
}
if (allowedTypes & git2go.CredentialTypeUserpassPlaintext) != 0 {
return git2go.NewCredentialUserpassPlaintext(opts.Username, opts.Password)
}
if (allowedTypes & git2go.CredentialTypeUsername) != 0 {
return git2go.NewCredentialUsername(opts.Username)
}
return nil, fmt.Errorf("unknown credential type %+v", allowedTypes)
}
}
// certificateCallback constructs CertificateCallback with the given options
// for git.Transport if the given opts is not nil, and returns the result.
func certificateCallback(opts *git.AuthOptions) git2go.CertificateCheckCallback {
switch opts.Transport {
case git.HTTPS:
if len(opts.CAFile) > 0 {
return x509Callback(opts.CAFile)
}
case git.SSH:
if len(opts.KnownHosts) > 0 && opts.Host != "" {
return managed.KnownHostsCallback(opts.Host, opts.KnownHosts)
}
}
return nil
}
// x509Callback returns a CertificateCheckCallback that verifies the
// certificate against the given caBundle for git.HTTPS Transports.
func x509Callback(caBundle []byte) git2go.CertificateCheckCallback {
return func(cert *git2go.Certificate, valid bool, hostname string) error {
roots := x509.NewCertPool()
if ok := roots.AppendCertsFromPEM(caBundle); !ok {
return fmt.Errorf("PEM CA bundle could not be appended to x509 certificate pool")
}
opts := x509.VerifyOptions{
Roots: roots,
DNSName: hostname,
CurrentTime: now(),
}
if _, err := cert.X509.Verify(opts); err != nil {
return fmt.Errorf("verification failed: %w", err)
}
return nil
}
}

377
pkg/git/libgit2/transport_test.go
View File

@ -1,377 +0,0 @@
/*
Copyright 2020 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package libgit2
import (
"bytes"
"context"
"crypto/x509"
"encoding/pem"
"errors"
"fmt"
"testing"
"time"
git2go "github.com/libgit2/git2go/v33"
. "github.com/onsi/gomega"
)
const (
geoTrustRootFixture = `-----BEGIN CERTIFICATE-----
MIIDVDCCAjygAwIBAgIDAjRWMA0GCSqGSIb3DQEBBQUAMEIxCzAJBgNVBAYTAlVT
MRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMRswGQYDVQQDExJHZW9UcnVzdCBHbG9i
YWwgQ0EwHhcNMDIwNTIxMDQwMDAwWhcNMjIwNTIxMDQwMDAwWjBCMQswCQYDVQQG
EwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEbMBkGA1UEAxMSR2VvVHJ1c3Qg
R2xvYmFsIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2swYYzD9
9BcjGlZ+W988bDjkcbd4kdS8odhM+KhDtgPpTSEHCIjaWC9mOSm9BXiLnTjoBbdq
fnGk5sRgprDvgOSJKA+eJdbtg/OtppHHmMlCGDUUna2YRpIuT8rxh0PBFpVXLVDv
iS2Aelet8u5fa9IAjbkU+BQVNdnARqN7csiRv8lVK83Qlz6cJmTM386DGXHKTubU
1XupGc1V3sjs0l44U+VcT4wt/lAjNvxm5suOpDkZALeVAjmRCw7+OC7RHQWa9k0+
bw8HHa8sHo9gOeL6NlMTOdReJivbPagUvTLrGAMoUgRx5aszPeE4uwc2hGKceeoW
MPRfwCvocWvk+QIDAQABo1MwUTAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTA
ephojYn7qwVkDBF9qn1luMrMTjAfBgNVHSMEGDAWgBTAephojYn7qwVkDBF9qn1l
uMrMTjANBgkqhkiG9w0BAQUFAAOCAQEANeMpauUvXVSOKVCUn5kaFOSPeCpilKIn
Z57QzxpeR+nBsqTP3UEaBU6bS+5Kb1VSsyShNwrrZHYqLizz/Tt1kL/6cdjHPTfS
tQWVYrmm3ok9Nns4d0iXrKYgjy6myQzCsplFAMfOEVEiIuCl6rYVSAlk6l5PdPcF
PseKUgzbFbS9bZvlxrFUaKnjaZC2mqUPuLk/IH2uSrW4nOQdtqvmlKXBx4Ot2/Un
hw4EbNX/3aBd7YdStysVAq45pmp06drE57xNNB6pXE0zX5IJL4hmXXeXxx12E6nV
5fEWCRE11azbJHFwLJhWC9kXtNHjUStedejV0NxPNO3CBWaAocvmMw==
-----END CERTIFICATE-----`
giag2IntermediateFixture = `-----BEGIN CERTIFICATE-----
MIIEBDCCAuygAwIBAgIDAjppMA0GCSqGSIb3DQEBBQUAMEIxCzAJBgNVBAYTAlVT
MRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMRswGQYDVQQDExJHZW9UcnVzdCBHbG9i
YWwgQ0EwHhcNMTMwNDA1MTUxNTU1WhcNMTUwNDA0MTUxNTU1WjBJMQswCQYDVQQG
EwJVUzETMBEGA1UEChMKR29vZ2xlIEluYzElMCMGA1UEAxMcR29vZ2xlIEludGVy
bmV0IEF1dGhvcml0eSBHMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
AJwqBHdc2FCROgajguDYUEi8iT/xGXAaiEZ+4I/F8YnOIe5a/mENtzJEiaB0C1NP
VaTOgmKV7utZX8bhBYASxF6UP7xbSDj0U/ck5vuR6RXEz/RTDfRK/J9U3n2+oGtv
h8DQUB8oMANA2ghzUWx//zo8pzcGjr1LEQTrfSTe5vn8MXH7lNVg8y5Kr0LSy+rE
ahqyzFPdFUuLH8gZYR/Nnag+YyuENWllhMgZxUYi+FOVvuOAShDGKuy6lyARxzmZ
EASg8GF6lSWMTlJ14rbtCMoU/M4iarNOz0YDl5cDfsCx3nuvRTPPuj5xt970JSXC
DTWJnZ37DhF5iR43xa+OcmkCAwEAAaOB+zCB+DAfBgNVHSMEGDAWgBTAephojYn7
qwVkDBF9qn1luMrMTjAdBgNVHQ4EFgQUSt0GFhu89mi1dvWBtrtiGrpagS8wEgYD
VR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAQYwOgYDVR0fBDMwMTAvoC2g
K4YpaHR0cDovL2NybC5nZW90cnVzdC5jb20vY3Jscy9ndGdsb2JhbC5jcmwwPQYI
KwYBBQUHAQEEMTAvMC0GCCsGAQUFBzABhiFodHRwOi8vZ3RnbG9iYWwtb2NzcC5n
ZW90cnVzdC5jb20wFwYDVR0gBBAwDjAMBgorBgEEAdZ5AgUBMA0GCSqGSIb3DQEB
BQUAA4IBAQA21waAESetKhSbOHezI6B1WLuxfoNCunLaHtiONgaX4PCVOzf9G0JY
/iLIa704XtE7JW4S615ndkZAkNoUyHgN7ZVm2o6Gb4ChulYylYbc3GrKBIxbf/a/
zG+FA1jDaFETzf3I93k9mTXwVqO94FntT0QJo544evZG0R0SnU++0ED8Vf4GXjza
HFa9llF7b1cq26KqltyMdMKVvvBulRP/F/A8rLIQjcxz++iPAsbw+zOzlTvjwsto
WHPbqCRiOwY1nQ2pM714A5AuTHhdUDqB1O6gyHA43LL5Z/qHQF1hwFGPa4NrzQU6
yuGnBXj8ytqU0CwIPX4WecigUCAkVDNx
-----END CERTIFICATE-----`
googleLeafFixture = `-----BEGIN CERTIFICATE-----
MIIEdjCCA16gAwIBAgIIcR5k4dkoe04wDQYJKoZIhvcNAQEFBQAwSTELMAkGA1UE
BhMCVVMxEzARBgNVBAoTCkdvb2dsZSBJbmMxJTAjBgNVBAMTHEdvb2dsZSBJbnRl
cm5ldCBBdXRob3JpdHkgRzIwHhcNMTQwMzEyMDkzODMwWhcNMTQwNjEwMDAwMDAw
WjBoMQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwN
TW91bnRhaW4gVmlldzETMBEGA1UECgwKR29vZ2xlIEluYzEXMBUGA1UEAwwOd3d3
Lmdvb2dsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC4zYCe
m0oUBhwE0EwBr65eBOcgcQO2PaSIAB2dEP/c1EMX2tOy0ov8rk83ePhJ+MWdT1z6
jge9X4zQQI8ZyA9qIiwrKBZOi8DNUvrqNZC7fJAVRrb9aX/99uYOJCypIbpmWG1q
fhbHjJewhwf8xYPj71eU4rLG80a+DapWmphtfq3h52lDQIBzLVf1yYbyrTaELaz4
NXF7HXb5YkId/gxIsSzM0aFUVu2o8sJcLYAsJqwfFKBKOMxUcn545nlspf0mTcWZ
0APlbwsKznNs4/xCDwIxxWjjqgHrYAFl6y07i1gzbAOqdNEyR24p+3JWI8WZBlBI
dk2KGj0W1fIfsvyxAgMBAAGjggFBMIIBPTAdBgNVHSUEFjAUBggrBgEFBQcDAQYI
KwYBBQUHAwIwGQYDVR0RBBIwEIIOd3d3Lmdvb2dsZS5jb20waAYIKwYBBQUHAQEE
XDBaMCsGCCsGAQUFBzAChh9odHRwOi8vcGtpLmdvb2dsZS5jb20vR0lBRzIuY3J0
MCsGCCsGAQUFBzABhh9odHRwOi8vY2xpZW50czEuZ29vZ2xlLmNvbS9vY3NwMB0G
A1UdDgQWBBTXD5Bx6iqT+dmEhbFL4OUoHyZn8zAMBgNVHRMBAf8EAjAAMB8GA1Ud
IwQYMBaAFErdBhYbvPZotXb1gba7Yhq6WoEvMBcGA1UdIAQQMA4wDAYKKwYBBAHW
eQIFATAwBgNVHR8EKTAnMCWgI6Ahhh9odHRwOi8vcGtpLmdvb2dsZS5jb20vR0lB
RzIuY3JsMA0GCSqGSIb3DQEBBQUAA4IBAQCR3RJtHzgDh33b/MI1ugiki+nl8Ikj
5larbJRE/rcA5oite+QJyAr6SU1gJJ/rRrK3ItVEHr9L621BCM7GSdoNMjB9MMcf
tJAW0kYGJ+wqKm53wG/JaOADTnnq2Mt/j6F2uvjgN/ouns1nRHufIvd370N0LeH+
orKqTuAPzXK7imQk6+OycYABbqCtC/9qmwRd8wwn7sF97DtYfK8WuNHtFalCAwyi
8LxJJYJCLWoMhZ+V8GZm+FOex5qkQAjnZrtNlbQJ8ro4r+rpKXtmMFFhfa+7L+PA
Kom08eUK8skxAzfDDijZPh10VtJ66uBoiDPdT+uCBehcBIcmSTrKjFGX
-----END CERTIFICATE-----`
// googleLeafWithInvalidHashFixture is the same as googleLeafFixture, but the signature
// algorithm in the certificate contains a nonsense OID.
googleLeafWithInvalidHashFixture = `-----BEGIN CERTIFICATE-----
MIIEdjCCA16gAwIBAgIIcR5k4dkoe04wDQYJKoZIhvcNAWAFBQAwSTELMAkGA1UE
BhMCVVMxEzARBgNVBAoTCkdvb2dsZSBJbmMxJTAjBgNVBAMTHEdvb2dsZSBJbnRl
cm5ldCBBdXRob3JpdHkgRzIwHhcNMTQwMzEyMDkzODMwWhcNMTQwNjEwMDAwMDAw
WjBoMQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwN
TW91bnRhaW4gVmlldzETMBEGA1UECgwKR29vZ2xlIEluYzEXMBUGA1UEAwwOd3d3
Lmdvb2dsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC4zYCe
m0oUBhwE0EwBr65eBOcgcQO2PaSIAB2dEP/c1EMX2tOy0ov8rk83ePhJ+MWdT1z6
jge9X4zQQI8ZyA9qIiwrKBZOi8DNUvrqNZC7fJAVRrb9aX/99uYOJCypIbpmWG1q
fhbHjJewhwf8xYPj71eU4rLG80a+DapWmphtfq3h52lDQIBzLVf1yYbyrTaELaz4
NXF7HXb5YkId/gxIsSzM0aFUVu2o8sJcLYAsJqwfFKBKOMxUcn545nlspf0mTcWZ
0APlbwsKznNs4/xCDwIxxWjjqgHrYAFl6y07i1gzbAOqdNEyR24p+3JWI8WZBlBI
dk2KGj0W1fIfsvyxAgMBAAGjggFBMIIBPTAdBgNVHSUEFjAUBggrBgEFBQcDAQYI
KwYBBQUHAwIwGQYDVR0RBBIwEIIOd3d3Lmdvb2dsZS5jb20waAYIKwYBBQUHAQEE
XDBaMCsGCCsGAQUFBzAChh9odHRwOi8vcGtpLmdvb2dsZS5jb20vR0lBRzIuY3J0
MCsGCCsGAQUFBzABhh9odHRwOi8vY2xpZW50czEuZ29vZ2xlLmNvbS9vY3NwMB0G
A1UdDgQWBBTXD5Bx6iqT+dmEhbFL4OUoHyZn8zAMBgNVHRMBAf8EAjAAMB8GA1Ud
IwQYMBaAFErdBhYbvPZotXb1gba7Yhq6WoEvMBcGA1UdIAQQMA4wDAYKKwYBBAHW
eQIFATAwBgNVHR8EKTAnMCWgI6Ahhh9odHRwOi8vcGtpLmdvb2dsZS5jb20vR0lB
RzIuY3JsMA0GCSqGSIb3DQFgBQUAA4IBAQCR3RJtHzgDh33b/MI1ugiki+nl8Ikj
5larbJRE/rcA5oite+QJyAr6SU1gJJ/rRrK3ItVEHr9L621BCM7GSdoNMjB9MMcf
tJAW0kYGJ+wqKm53wG/JaOADTnnq2Mt/j6F2uvjgN/ouns1nRHufIvd370N0LeH+
orKqTuAPzXK7imQk6+OycYABbqCtC/9qmwRd8wwn7sF97DtYfK8WuNHtFalCAwyi
8LxJJYJCLWoMhZ+V8GZm+FOex5qkQAjnZrtNlbQJ8ro4r+rpKXtmMFFhfa+7L+PA
Kom08eUK8skxAzfDDijZPh10VtJ66uBoiDPdT+uCBehcBIcmSTrKjFGX
-----END CERTIFICATE-----`
knownHostsFixture string = `github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==`
)
func Test_x509Callback(t *testing.T) {
now = func() time.Time { return time.Unix(1395785200, 0) }
tests := []struct {
name string
certificate string
host string
caBundle []byte
want error
}{
{
name: "Valid certificate authority bundle",
certificate: googleLeafFixture,
host: "www.google.com",
caBundle: []byte(giag2IntermediateFixture + "\n" + geoTrustRootFixture),
want: nil,
},
{
name: "Invalid certificate",
certificate: googleLeafWithInvalidHashFixture,
host: "www.google.com",
caBundle: []byte(giag2IntermediateFixture + "\n" + geoTrustRootFixture),
want: fmt.Errorf(`verification failed: x509: certificate signed by unknown authority (possibly because of "x509: cannot verify signature: algorithm unimplemented" while trying to verify candidate authority certificate "Google Internet Authority G2")`),
},
{
name: "Invalid certificate authority bundle",
certificate: googleLeafFixture,
host: "www.google.com",
caBundle: bytes.Trim([]byte(giag2IntermediateFixture+"\n"+geoTrustRootFixture), "-"),
want: fmt.Errorf("PEM CA bundle could not be appended to x509 certificate pool"),
},
{
name: "Missing intermediate in bundle",
certificate: googleLeafFixture,
host: "www.google.com",
caBundle: []byte(geoTrustRootFixture),
want: fmt.Errorf("verification failed: x509: certificate signed by unknown authority"),
},
{
name: "Invalid host",
certificate: googleLeafFixture,
host: "www.google.co",
caBundle: []byte(giag2IntermediateFixture + "\n" + geoTrustRootFixture),
want: fmt.Errorf("verification failed: x509: certificate is valid for www.google.com, not www.google.co"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
cert := &git2go.Certificate{}
if tt.certificate != "" {
x509Cert, err := certificateFromPEM(tt.certificate)
g.Expect(err).ToNot(HaveOccurred())
cert.X509 = x509Cert
}
callback := x509Callback(tt.caBundle)
result := callback(cert, false, tt.host)
if tt.want == nil {
g.Expect(result).To(BeNil())
} else {
g.Expect(result.Error()).To(Equal(tt.want.Error()))
}
})
}
}
func Test_transferProgressCallback(t *testing.T) {
tests := []struct {
name string
progress git2go.TransferProgress
cancelFunc func(context.CancelFunc)
wantErr error
}{
{
name: "ok - in progress",
progress: git2go.TransferProgress{
TotalObjects: 30,
ReceivedObjects: 21,
},
cancelFunc: func(cf context.CancelFunc) {},
wantErr: nil,
},
{
name: "ok - transfer complete",
progress: git2go.TransferProgress{
TotalObjects: 30,
ReceivedObjects: 30,
},
cancelFunc: func(cf context.CancelFunc) {},
wantErr: nil,
},
{
name: "ok - transfer complete, context cancelled",
progress: git2go.TransferProgress{
TotalObjects: 30,
ReceivedObjects: 30,
},
cancelFunc: func(cf context.CancelFunc) { cf() },
wantErr: nil,
},
{
name: "error - context cancelled",
progress: git2go.TransferProgress{
TotalObjects: 30,
ReceivedObjects: 21,
},
cancelFunc: func(cf context.CancelFunc) { cf() },
wantErr: fmt.Errorf("transport close (potentially due to a timeout)"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
tpcb := transferProgressCallback(ctx)
tt.cancelFunc(cancel)
result := g.Expect(tpcb(tt.progress))
if tt.wantErr == nil {
result.To(BeNil())
} else {
result.To(Equal(tt.wantErr))
}
})
}
}
func Test_transportMessageCallback(t *testing.T) {
tests := []struct {
name string
cancelFunc func(context.CancelFunc)
wantErr error
}{
{
name: "ok - transport open",
cancelFunc: func(cf context.CancelFunc) {},
wantErr: nil,
},
{
name: "error - transport closed",
cancelFunc: func(cf context.CancelFunc) { cf() },
wantErr: fmt.Errorf("transport closed"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
tmcb := transportMessageCallback(ctx)
tt.cancelFunc(cancel)
result := g.Expect(tmcb(""))
if tt.wantErr == nil {
result.To(BeNil())
} else {
result.To(Equal(tt.wantErr))
}
})
}
}
func Test_pushTransferProgressCallback(t *testing.T) {
type pushProgress struct {
current uint32
total uint32
bytes uint
}
tests := []struct {
name string
progress pushProgress
cancelFunc func(context.CancelFunc)
wantErr error
}{
{
name: "ok - in progress",
progress: pushProgress{current: 20, total: 25},
cancelFunc: func(cf context.CancelFunc) {},
wantErr: nil,
},
{
name: "ok - transfer complete",
progress: pushProgress{current: 25, total: 25},
cancelFunc: func(cf context.CancelFunc) {},
wantErr: nil,
},
{
name: "ok - transfer complete, context cancelled",
progress: pushProgress{current: 25, total: 25},
cancelFunc: func(cf context.CancelFunc) { cf() },
wantErr: nil,
},
{
name: "error - context cancelled",
progress: pushProgress{current: 20, total: 25},
cancelFunc: func(cf context.CancelFunc) { cf() },
wantErr: fmt.Errorf("transport close (potentially due to a timeout)"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
ptpcb := pushTransferProgressCallback(ctx)
tt.cancelFunc(cancel)
result := g.Expect(ptpcb(tt.progress.current, tt.progress.total, tt.progress.bytes))
if tt.wantErr == nil {
result.To(BeNil())
} else {
result.To(Equal(tt.wantErr))
}
})
}
}
func certificateFromPEM(pemBytes string) (*x509.Certificate, error) {
block, _ := pem.Decode([]byte(pemBytes))
if block == nil {
return nil, errors.New("failed to decode PEM")
}
return x509.ParseCertificate(block.Bytes)
}

8
pkg/git/strategy/proxy/strategy_proxy_test.go
View File

@ -30,10 +30,8 @@ import (
"github.com/elazarl/goproxy"
"github.com/fluxcd/pkg/gittestserver"
feathelper "github.com/fluxcd/pkg/runtime/features"
. "github.com/onsi/gomega"
"github.com/fluxcd/source-controller/internal/features"
"github.com/fluxcd/source-controller/pkg/git"
"github.com/fluxcd/source-controller/pkg/git/gogit"
"github.com/fluxcd/source-controller/pkg/git/libgit2"
@ -44,12 +42,6 @@ import (
// These tests are run in a different _test.go file because go-git uses the ProxyFromEnvironment function of the net/http package
// which caches the Proxy settings, hence not including other tests in the same file ensures a clean proxy setup for the tests to run.
func TestCheckoutStrategyForImplementation_Proxied(t *testing.T) {
// for libgit2 we are only testing for managed transport,
// as unmanaged is sunsetting.
// Unmanaged transport does not support HTTP_PROXY.
fg := feathelper.FeatureGates{}
fg.SupportedFeatures(features.FeatureGates())
managed.InitManagedTransport()
type cleanupFunc func()

25
pkg/git/strategy/strategy_test.go
View File

@ -20,6 +20,7 @@ import (
"context"
"errors"
"fmt"
"math/rand"
"net/http"
"net/url"
"os"
@ -39,8 +40,18 @@ import (
"github.com/fluxcd/source-controller/pkg/git"
"github.com/fluxcd/source-controller/pkg/git/gogit"
"github.com/fluxcd/source-controller/pkg/git/libgit2"
"github.com/fluxcd/source-controller/pkg/git/libgit2/managed"
)
func TestMain(m *testing.M) {
err := managed.InitManagedTransport()
if err != nil {
panic(fmt.Sprintf("failed to initialize libgit2 managed transport: %s", err))
}
code := m.Run()
os.Exit(code)
}
func TestCheckoutStrategyForImplementation_Auth(t *testing.T) {
gitImpls := []git.Implementation{gogit.Implementation, libgit2.Implementation}
@ -64,6 +75,7 @@ func TestCheckoutStrategyForImplementation_Auth(t *testing.T) {
Transport: git.HTTP,
Username: user,
Password: pswd,
TransportOptionsURL: getTransportOptionsURL(git.HTTP),
}
},
wantFunc: func(g *WithT, cs git.CheckoutStrategy, dir string, repoURL string, authOpts *git.AuthOptions) {
@ -83,6 +95,7 @@ func TestCheckoutStrategyForImplementation_Auth(t *testing.T) {
Username: user,
Password: pswd,
CAFile: ca,
TransportOptionsURL: getTransportOptionsURL(git.HTTPS),
}
},
wantFunc: func(g *WithT, cs git.CheckoutStrategy, dir, repoURL string, authOpts *git.AuthOptions) {
@ -110,6 +123,7 @@ func TestCheckoutStrategyForImplementation_Auth(t *testing.T) {
Username: "git", // Without this libgit2 returns error "username does not match previous request".
Identity: pair.PrivateKey,
KnownHosts: knownhosts,
TransportOptionsURL: getTransportOptionsURL(git.SSH),
}
},
wantFunc: func(g *WithT, cs git.CheckoutStrategy, dir, repoURL string, authOpts *git.AuthOptions) {
@ -228,6 +242,7 @@ func TestCheckoutStrategyForImplementation_SemVerCheckout(t *testing.T) {
Transport: git.HTTP,
Username: username,
Password: password,
TransportOptionsURL: getTransportOptionsURL(git.HTTP),
}
// Create test tags in the repo.
@ -411,6 +426,7 @@ func TestCheckoutStrategyForImplementation_WithCtxTimeout(t *testing.T) {
Transport: git.HTTP,
Username: username,
Password: password,
TransportOptionsURL: getTransportOptionsURL(git.HTTP),
}
checkoutOpts := git.CheckoutOptions{
@ -486,3 +502,12 @@ func mockSignature(time time.Time) *object.Signature {
When: time,
}
}
func getTransportOptionsURL(transport git.TransportType) string {
letterRunes := []rune("abcdefghijklmnopqrstuvwxyz1234567890")
b := make([]rune, 10)
for i := range b {
b[i] = letterRunes[rand.Intn(len(letterRunes))]
}
return string(transport) + "://" + string(b)
}