fluxcd
/
source-controller
mirror of https://github.com/fluxcd/source-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #727 from aryan9600/improve-managed 

Remove dependency on libgit2 credentials callback
This commit is contained in:
Paulo Gomes 2022-05-27 10:27:02 +01:00 committed by GitHub
parent 5c9a8443d1 972d1cac2a
commit 82cd05eb3d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 1426 additions and 1192 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
controllers/gitrepository_controller.go
View File

@ -455,33 +455,6 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
return sreconcile.ResultEmpty, e
}
repositoryURL := obj.Spec.URL
// managed GIT transport only affects the libgit2 implementation
if managed.Enabled() && obj.Spec.GitImplementation == sourcev1.LibGit2Implementation {
// At present only HTTP connections have the ability to define remote options.
// Although this can be easily extended by ensuring that the fake URL below uses the
// target ssh scheme, and the libgit2/managed/ssh.go pulls that information accordingly.
//
// This is due to the fact the key libgit2 remote callbacks do not take place for HTTP
// whilst most still work for SSH.
if strings.HasPrefix(repositoryURL, "http") {
// Due to the lack of the callback feature, a fake target URL is created to allow
// for the smart sub transport be able to pick the options specific for this
// GitRepository object.
// The URL should use unique information that do not collide in a multi tenant
// deployment.
repositoryURL = fmt.Sprintf("http://%s/%s/%d", obj.Name, obj.UID, obj.Generation)
managed.AddTransportOptions(repositoryURL,
managed.TransportOptions{
TargetURL: obj.Spec.URL,
CABundle: authOpts.CAFile,
})
// We remove the options from memory, to avoid accumulating unused options over time.
defer managed.RemoveTransportOptions(repositoryURL)
}
}
// Fetch the included artifact metadata.
artifacts, err := r.fetchIncludes(ctx, obj)
if err != nil {
@ -503,7 +476,7 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
optimizedClone = true
}
c, err := r.gitCheckout(ctx, obj, repositoryURL, authOpts, dir, optimizedClone)
c, err := r.gitCheckout(ctx, obj, authOpts, dir, optimizedClone)
if err != nil {
return sreconcile.ResultEmpty, err
}
@ -537,7 +510,7 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
// If we can't skip the reconciliation, checkout again without any
// optimization.
c, err := r.gitCheckout(ctx, obj, repositoryURL, authOpts, dir, false)
c, err := r.gitCheckout(ctx, obj, authOpts, dir, false)
if err != nil {
return sreconcile.ResultEmpty, err
}
@ -729,7 +702,7 @@ func (r *GitRepositoryReconciler) reconcileInclude(ctx context.Context,
// gitCheckout builds checkout options with the given configurations and
// performs a git checkout.
func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context,
obj *sourcev1.GitRepository, repoURL string, authOpts *git.AuthOptions, dir string, optimized bool) (*git.Commit, error) {
obj *sourcev1.GitRepository, authOpts *git.AuthOptions, dir string, optimized bool) (*git.Commit, error) {
// Configure checkout strategy.
checkoutOpts := git.CheckoutOptions{RecurseSubmodules: obj.Spec.RecurseSubmodules}
if ref := obj.Spec.Reference; ref != nil {
@ -755,15 +728,34 @@ func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context,
Err: fmt.Errorf("failed to configure checkout strategy for Git implementation '%s': %w", obj.Spec.GitImplementation, err),
Reason: sourcev1.GitOperationFailedReason,
}
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
// Do not return err as recovery without changes is impossible.
return nil, e
}
// managed GIT transport only affects the libgit2 implementation
if managed.Enabled() && obj.Spec.GitImplementation == sourcev1.LibGit2Implementation {
// We set the TransportOptionsURL of this set of authentication options here by constructing
// a unique URL that won't clash in a multi tenant environment. This unique URL is used by
// libgit2 managed transports. This enables us to bypass the inbuilt credentials callback in
// libgit2, which is inflexible and unstable.
if strings.HasPrefix(obj.Spec.URL, "http") {
authOpts.TransportOptionsURL = fmt.Sprintf("http://%s/%s/%d", obj.Name, obj.UID, obj.Generation)
} else if strings.HasPrefix(obj.Spec.URL, "ssh") {
authOpts.TransportOptionsURL = fmt.Sprintf("ssh://%s/%s/%d", obj.Name, obj.UID, obj.Generation)
} else {
e := &serror.Stalling{
Err: fmt.Errorf("git repository URL has invalid transport type: '%s'", obj.Spec.URL),
Reason: sourcev1.URLInvalidReason,
}
return nil, e
}
}
// Checkout HEAD of reference in object
gitCtx, cancel := context.WithTimeout(ctx, obj.Spec.Timeout.Duration)
defer cancel()
commit, err := checkoutStrategy.Checkout(gitCtx, dir, repoURL, authOpts)
commit, err := checkoutStrategy.Checkout(gitCtx, dir, obj.Spec.URL, authOpts)
if err != nil {
e := serror.NewGeneric(
fmt.Errorf("failed to checkout and determine revision: %w", err),

4
controllers/suite_test.go
View File

@ -39,6 +39,7 @@ import (
"github.com/fluxcd/pkg/runtime/controller"
"github.com/fluxcd/pkg/runtime/testenv"
"github.com/fluxcd/pkg/testserver"
"github.com/go-logr/logr"
"github.com/phayes/freeport"
"github.com/distribution/distribution/v3/configuration"
@ -50,6 +51,7 @@ import (
"github.com/fluxcd/source-controller/internal/cache"
"github.com/fluxcd/source-controller/internal/features"
"github.com/fluxcd/source-controller/internal/helm/registry"
"github.com/fluxcd/source-controller/pkg/git/libgit2/managed"
// +kubebuilder:scaffold:imports
)
@ -207,6 +209,8 @@ func TestMain(m *testing.M) {
panic(fmt.Sprintf("Failed to create OCI registry client"))
}
managed.InitManagedTransport(logr.Discard())
if err := (&GitRepositoryReconciler{
Client: testEnv,
EventRecorder: record.NewFakeRecorder(32),

2
go.mod
View File

@ -28,7 +28,7 @@ require (
github.com/fluxcd/pkg/helmtestserver v0.7.2
github.com/fluxcd/pkg/lockedfile v0.1.0
github.com/fluxcd/pkg/runtime v0.16.1
github.com/fluxcd/pkg/ssh v0.3.4
github.com/fluxcd/pkg/ssh v0.4.0
github.com/fluxcd/pkg/testserver v0.2.0
github.com/fluxcd/pkg/untar v0.1.0
github.com/fluxcd/pkg/version v0.1.0

4
go.sum
View File

@ -282,8 +282,8 @@ github.com/fluxcd/pkg/lockedfile v0.1.0 h1:YsYFAkd6wawMCcD74ikadAKXA4s2sukdxrn7w
github.com/fluxcd/pkg/lockedfile v0.1.0/go.mod h1:EJLan8t9MiOcgTs8+puDjbE6I/KAfHbdvIy9VUgIjm8=
github.com/fluxcd/pkg/runtime v0.16.1 h1:WU1vNZz4TAzmATQ/tl2zB/FX6GIUTgYeBn/G5RuTA2c=
github.com/fluxcd/pkg/runtime v0.16.1/go.mod h1:cgVJkOXCg9OmrIUGklf/0UtV28MNzkuoBJhaEQICT6E=
github.com/fluxcd/pkg/ssh v0.3.4 h1:Ko+MUNiiQG3evyoMO19iRk7d4X0VJ6w6+GEeVQ1jLC0=
github.com/fluxcd/pkg/ssh v0.3.4/go.mod h1:KGgOUOy1uI6RC6+qxIBLvP1AeOOs/nLB25Ca6TZMIXE=
github.com/fluxcd/pkg/ssh v0.4.0 h1:2HY88irZ5BCSMlzZExR6cnhRkjxCDsK/lTHHQqCJDJQ=
github.com/fluxcd/pkg/ssh v0.4.0/go.mod h1:KGgOUOy1uI6RC6+qxIBLvP1AeOOs/nLB25Ca6TZMIXE=
github.com/fluxcd/pkg/testserver v0.2.0 h1:Mj0TapmKaywI6Fi5wvt1LAZpakUHmtzWQpJNKQ0Krt4=
github.com/fluxcd/pkg/testserver v0.2.0/go.mod h1:bgjjydkXsZTeFzjz9Cr4heGANr41uTB1Aj1Q5qzuYVk=
github.com/fluxcd/pkg/untar v0.1.0 h1:k97V/xV5hFrAkIkVPuv5AVhyxh1ZzzAKba/lbDfGo6o=

8
internal/features/features.go
View File

@ -65,3 +65,11 @@ func FeatureGates() map[string]bool {
func Enabled(feature string) (bool, error) {
return feathelper.Enabled(feature)
}
// Disable disables the specified feature. If the feature is not
// present, it's a no-op.
func Disable(feature string) {
if _, ok := features[feature]; ok {
features[feature] = false
}
}

7
main.go
View File

@ -312,6 +312,13 @@ func main() {
if enabled, _ := features.Enabled(features.GitManagedTransport); enabled {
managed.InitManagedTransport(ctrl.Log.WithName("managed-transport"))
} else {
if optimize, _ := feathelper.Enabled(features.OptimizedGitClones); optimize {
features.Disable(features.OptimizedGitClones)
setupLog.Info(
"disabling optimized git clones; git clones can only be optimized when using managed transport",
)
}
}
setupLog.Info("starting manager")

371
pkg/git/libgit2/checkout.go
View File

@ -69,96 +69,151 @@ type CheckoutBranch struct {
func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) {
defer recoverPanic(&err)
remoteCallBacks := RemoteCallbacks(ctx, opts)
proxyOpts := &git2go.ProxyOptions{Type: git2go.ProxyTypeAuto}
repo, remote, err := initializeRepoWithRemote(ctx, path, url, opts)
if err != nil {
return nil, err
}
// Open remote connection.
err = remote.ConnectFetch(&remoteCallBacks, proxyOpts, nil)
if err != nil {
remote.Free()
repo.Free()
return nil, fmt.Errorf("unable to fetch-connect to remote '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
defer func() {
remote.Disconnect()
remote.Free()
repo.Free()
}()
// When the last observed revision is set, check whether it is still the
// same at the remote branch. If so, short-circuit the clone operation here.
if c.LastRevision != "" {
heads, err := remote.Ls(c.Branch)
if err != nil {
return nil, fmt.Errorf("unable to remote ls for '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
// This branching is temporary, to address the transient panics observed when using unmanaged transport.
// The panics probably happen because we perform multiple fetch ops (introduced as a part of optimizing git clones).
// The branching lets us establish a clear code path to help us be certain of the expected behaviour.
// When we get rid of unmanaged transports, we can get rid of this branching as well.
if managed.Enabled() {
// We store the target URL and auth options mapped to a unique ID. We overwrite the target URL
// with the TransportOptionsURL, because managed transports don't provide a way for any kind of
// dependency injection. This lets us have a way of doing interop between application level code
// and transport level code.
// Performing all fetch operations with the TransportOptionsURL as the URL, lets the managed
// transport action use it to fetch the registered transport options which contains the
// _actual_ target URL and the correct credentials to use.
if opts == nil {
return nil, fmt.Errorf("can't use managed transport with an empty set of auth options")
}
if len(heads) > 0 {
hash := heads[0].Id.String()
currentRevision := fmt.Sprintf("%s/%s", c.Branch, hash)
if currentRevision == c.LastRevision {
// Construct a partial commit with the existing information.
c := &git.Commit{
Hash: git.Hash(hash),
Reference: "refs/heads/" + c.Branch,
if opts.TransportOptionsURL == "" {
return nil, fmt.Errorf("can't use managed transport without a valid transport auth id.")
}
managed.AddTransportOptions(opts.TransportOptionsURL, managed.TransportOptions{
TargetURL: url,
AuthOpts: opts,
ProxyOptions: &git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
})
url = opts.TransportOptionsURL
remoteCallBacks := managed.RemoteCallbacks()
defer managed.RemoveTransportOptions(opts.TransportOptionsURL)
repo, remote, err := initializeRepoWithRemote(ctx, path, url, opts)
if err != nil {
return nil, err
}
// Open remote connection.
err = remote.ConnectFetch(&remoteCallBacks, nil, nil)
if err != nil {
remote.Free()
repo.Free()
return nil, fmt.Errorf("unable to fetch-connect to remote '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
defer func() {
remote.Disconnect()
remote.Free()
repo.Free()
}()
// When the last observed revision is set, check whether it is still the
// same at the remote branch. If so, short-circuit the clone operation here.
if c.LastRevision != "" {
heads, err := remote.Ls(c.Branch)
if err != nil {
return nil, fmt.Errorf("unable to remote ls for '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
if len(heads) > 0 {
hash := heads[0].Id.String()
currentRevision := fmt.Sprintf("%s/%s", c.Branch, hash)
if currentRevision == c.LastRevision {
// Construct a partial commit with the existing information.
c := &git.Commit{
Hash: git.Hash(hash),
Reference: "refs/heads/" + c.Branch,
}
return c, nil
}
return c, nil
}
}
}
// Limit the fetch operation to the specific branch, to decrease network usage.
err = remote.Fetch([]string{c.Branch},
&git2go.FetchOptions{
// Limit the fetch operation to the specific branch, to decrease network usage.
err = remote.Fetch([]string{c.Branch},
&git2go.FetchOptions{
DownloadTags: git2go.DownloadTagsNone,
RemoteCallbacks: remoteCallBacks,
},
"")
if err != nil {
return nil, fmt.Errorf("unable to fetch remote '%s': %w",
managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
branch, err := repo.References.Lookup(fmt.Sprintf("refs/remotes/origin/%s", c.Branch))
if err != nil {
return nil, fmt.Errorf("unable to lookup branch '%s' for '%s': %w",
c.Branch, managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
defer branch.Free()
upstreamCommit, err := repo.LookupCommit(branch.Target())
if err != nil {
return nil, fmt.Errorf("unable to lookup commit '%s' for '%s': %w",
c.Branch, managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
defer upstreamCommit.Free()
// Once the index has been updated with Fetch, and we know the tip commit,
// a hard reset can be used to align the local worktree with the remote branch's.
err = repo.ResetToCommit(upstreamCommit, git2go.ResetHard, &git2go.CheckoutOptions{
Strategy: git2go.CheckoutForce,
})
if err != nil {
return nil, fmt.Errorf("unable to hard reset to commit for '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
// Use the current worktree's head as reference for the commit to be returned.
head, err := repo.Head()
if err != nil {
return nil, fmt.Errorf("git resolve HEAD error: %w", err)
}
defer head.Free()
cc, err := repo.LookupCommit(head.Target())
if err != nil {
return nil, fmt.Errorf("failed to lookup HEAD commit '%s' for branch '%s': %w", head.Target(), c.Branch, err)
}
defer cc.Free()
return buildCommit(cc, "refs/heads/"+c.Branch), nil
} else {
return c.checkoutUnmanaged(ctx, path, url, opts)
}
}
func (c *CheckoutBranch) checkoutUnmanaged(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) {
repo, err := git2go.Clone(url, path, &git2go.CloneOptions{
FetchOptions: git2go.FetchOptions{
DownloadTags: git2go.DownloadTagsNone,
RemoteCallbacks: remoteCallBacks,
RemoteCallbacks: RemoteCallbacks(ctx, opts),
ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
},
"")
if err != nil {
return nil, fmt.Errorf("unable to fetch remote '%s': %w",
managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
branch, err := repo.References.Lookup(fmt.Sprintf("refs/remotes/origin/%s", c.Branch))
if err != nil {
return nil, fmt.Errorf("unable to lookup branch '%s' for '%s': %w",
c.Branch, managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
defer branch.Free()
upstreamCommit, err := repo.LookupCommit(branch.Target())
if err != nil {
return nil, fmt.Errorf("unable to lookup commit '%s' for '%s': %w",
c.Branch, managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
defer upstreamCommit.Free()
// Once the index has been updated with Fetch, and we know the tip commit,
// a hard reset can be used to align the local worktree with the remote branch's.
err = repo.ResetToCommit(upstreamCommit, git2go.ResetHard, &git2go.CheckoutOptions{
Strategy: git2go.CheckoutForce,
CheckoutOptions: git2go.CheckoutOptions{
Strategy: git2go.CheckoutForce,
},
CheckoutBranch: c.Branch,
})
if err != nil {
return nil, fmt.Errorf("unable to hard reset to commit for '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
// Use the current worktree's head as reference for the commit to be returned.
defer repo.Free()
head, err := repo.Head()
if err != nil {
return nil, fmt.Errorf("git resolve HEAD error: %w", err)
}
defer head.Free()
cc, err := repo.LookupCommit(head.Target())
if err != nil {
return nil, fmt.Errorf("failed to lookup HEAD commit '%s' for branch '%s': %w", head.Target(), c.Branch, err)
}
defer cc.Free()
return buildCommit(cc, "refs/heads/"+c.Branch), nil
}
@ -170,70 +225,106 @@ type CheckoutTag struct {
func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) {
defer recoverPanic(&err)
remoteCallBacks := RemoteCallbacks(ctx, opts)
proxyOpts := &git2go.ProxyOptions{Type: git2go.ProxyTypeAuto}
// This branching is temporary, to address the transient panics observed when using unmanaged transport.
// The panics probably happen because we perform multiple fetch ops (introduced as a part of optimizing git clones).
// The branching lets us establish a clear code path to help us be certain of the expected behaviour.
// When we get rid of unmanaged transports, we can get rid of this branching as well.
if managed.Enabled() {
if opts.TransportOptionsURL == "" {
return nil, fmt.Errorf("can't use managed transport without a valid transport auth id.")
}
managed.AddTransportOptions(opts.TransportOptionsURL, managed.TransportOptions{
TargetURL: url,
AuthOpts: opts,
ProxyOptions: &git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
})
url = opts.TransportOptionsURL
remoteCallBacks := managed.RemoteCallbacks()
defer managed.RemoveTransportOptions(opts.TransportOptionsURL)
repo, remote, err := initializeRepoWithRemote(ctx, path, url, opts)
if err != nil {
return nil, err
}
// Open remote connection.
err = remote.ConnectFetch(&remoteCallBacks, proxyOpts, nil)
if err != nil {
remote.Free()
repo.Free()
return nil, fmt.Errorf("unable to fetch-connect to remote '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
defer func() {
remote.Disconnect()
remote.Free()
repo.Free()
}()
// When the last observed revision is set, check whether it is still the
// same at the remote branch. If so, short-circuit the clone operation here.
if c.LastRevision != "" {
heads, err := remote.Ls(c.Tag)
repo, remote, err := initializeRepoWithRemote(ctx, path, url, opts)
if err != nil {
return nil, fmt.Errorf("unable to remote ls for '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
return nil, err
}
if len(heads) > 0 {
hash := heads[0].Id.String()
currentRevision := fmt.Sprintf("%s/%s", c.Tag, hash)
var same bool
if currentRevision == c.LastRevision {
same = true
} else if len(heads) > 1 {
hash = heads[1].Id.String()
currentAnnotatedRevision := fmt.Sprintf("%s/%s", c.Tag, hash)
if currentAnnotatedRevision == c.LastRevision {
// Open remote connection.
err = remote.ConnectFetch(&remoteCallBacks, nil, nil)
if err != nil {
remote.Free()
repo.Free()
return nil, fmt.Errorf("unable to fetch-connect to remote '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
defer func() {
remote.Disconnect()
remote.Free()
repo.Free()
}()
// When the last observed revision is set, check whether it is still the
// same at the remote branch. If so, short-circuit the clone operation here.
if c.LastRevision != "" {
heads, err := remote.Ls(c.Tag)
if err != nil {
return nil, fmt.Errorf("unable to remote ls for '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
if len(heads) > 0 {
hash := heads[0].Id.String()
currentRevision := fmt.Sprintf("%s/%s", c.Tag, hash)
var same bool
if currentRevision == c.LastRevision {
same = true
} else if len(heads) > 1 {
hash = heads[1].Id.String()
currentAnnotatedRevision := fmt.Sprintf("%s/%s", c.Tag, hash)
if currentAnnotatedRevision == c.LastRevision {
same = true
}
}
}
if same {
// Construct a partial commit with the existing information.
c := &git.Commit{
Hash: git.Hash(hash),
Reference: "refs/tags/" + c.Tag,
if same {
// Construct a partial commit with the existing information.
c := &git.Commit{
Hash: git.Hash(hash),
Reference: "refs/tags/" + c.Tag,
}
return c, nil
}
return c, nil
}
}
}
err = remote.Fetch([]string{c.Tag},
&git2go.FetchOptions{
DownloadTags: git2go.DownloadTagsAuto,
RemoteCallbacks: remoteCallBacks,
ProxyOptions: *proxyOpts,
err = remote.Fetch([]string{c.Tag},
&git2go.FetchOptions{
DownloadTags: git2go.DownloadTagsAuto,
RemoteCallbacks: remoteCallBacks,
},
"")
if err != nil {
return nil, fmt.Errorf("unable to fetch remote '%s': %w",
managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
cc, err := checkoutDetachedDwim(repo, c.Tag)
if err != nil {
return nil, err
}
defer cc.Free()
return buildCommit(cc, "refs/tags/"+c.Tag), nil
} else {
return c.checkoutUnmanaged(ctx, path, url, opts)
}
}
func (c *CheckoutTag) checkoutUnmanaged(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) {
repo, err := git2go.Clone(url, path, &git2go.CloneOptions{
FetchOptions: git2go.FetchOptions{
DownloadTags: git2go.DownloadTagsAll,
RemoteCallbacks: RemoteCallbacks(ctx, opts),
ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
},
"")
})
if err != nil {
return nil, fmt.Errorf("unable to fetch remote '%s': %w",
managed.EffectiveURL(url), gitutil.LibGit2Error(err))
return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
defer repo.Free()
cc, err := checkoutDetachedDwim(repo, c.Tag)
if err != nil {
return nil, err
@ -249,11 +340,26 @@ type CheckoutCommit struct {
func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) {
defer recoverPanic(&err)
remoteCallBacks := RemoteCallbacks(ctx, opts)
if managed.Enabled() {
if opts.TransportOptionsURL == "" {
return nil, fmt.Errorf("can't use managed transport without a valid transport auth id.")
}
managed.AddTransportOptions(opts.TransportOptionsURL, managed.TransportOptions{
TargetURL: url,
AuthOpts: opts,
ProxyOptions: &git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
})
url = opts.TransportOptionsURL
remoteCallBacks = managed.RemoteCallbacks()
defer managed.RemoveTransportOptions(opts.TransportOptionsURL)
}
repo, err := git2go.Clone(url, path, &git2go.CloneOptions{
FetchOptions: git2go.FetchOptions{
DownloadTags: git2go.DownloadTagsNone,
RemoteCallbacks: RemoteCallbacks(ctx, opts),
ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
RemoteCallbacks: remoteCallBacks,
},
})
if err != nil {
@ -278,6 +384,22 @@ type CheckoutSemVer struct {
func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (_ *git.Commit, err error) {
defer recoverPanic(&err)
remoteCallBacks := RemoteCallbacks(ctx, opts)
if managed.Enabled() {
if opts.TransportOptionsURL == "" {
return nil, fmt.Errorf("can't use managed transport without a valid transport auth id.")
}
managed.AddTransportOptions(opts.TransportOptionsURL, managed.TransportOptions{
TargetURL: url,
AuthOpts: opts,
ProxyOptions: &git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
})
url = opts.TransportOptionsURL
remoteCallBacks = managed.RemoteCallbacks()
defer managed.RemoveTransportOptions(opts.TransportOptionsURL)
}
verConstraint, err := semver.NewConstraint(c.SemVer)
if err != nil {
return nil, fmt.Errorf("semver parse error: %w", err)
@ -286,8 +408,7 @@ func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, opts *g
repo, err := git2go.Clone(url, path, &git2go.CloneOptions{
FetchOptions: git2go.FetchOptions{
DownloadTags: git2go.DownloadTagsAll,
RemoteCallbacks: RemoteCallbacks(ctx, opts),
ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto},
RemoteCallbacks: remoteCallBacks,
},
})
if err != nil {

120
pkg/git/libgit2/checkout_test.go
View File

@ -30,7 +30,7 @@ import (
. "github.com/onsi/gomega"
)
func TestCheckoutBranch_Checkout(t *testing.T) {
func TestCheckoutBranch_checkoutUnmanaged(t *testing.T) {
repo, err := initBareRepo(t)
if err != nil {
t.Fatal(err)
@ -77,49 +77,29 @@ func TestCheckoutBranch_Checkout(t *testing.T) {
}
tests := []struct {
name string
branch string
filesCreated map[string]string
lastRevision string
expectedCommit string
expectedConcreteCommit bool
expectedErr string
name string
branch string
filesCreated map[string]string
lastRevision string
expectedCommit string
expectedErr string
}{
{
name: "Default branch",
branch: defaultBranch,
filesCreated: map[string]string{"branch": "second"},
expectedCommit: secondCommit.String(),
expectedConcreteCommit: true,
name: "Default branch",
branch: defaultBranch,
filesCreated: map[string]string{"branch": "second"},
expectedCommit: secondCommit.String(),
},
{
name: "Other branch",
branch: "test",
filesCreated: map[string]string{"branch": "init"},
expectedCommit: firstCommit.String(),
expectedConcreteCommit: true,
name: "Other branch",
branch: "test",
filesCreated: map[string]string{"branch": "init"},
expectedCommit: firstCommit.String(),
},
{
name: "Non existing branch",
branch: "invalid",
expectedErr: "reference 'refs/remotes/origin/invalid' not found",
expectedConcreteCommit: true,
},
{
name: "skip clone - lastRevision hasn't changed",
branch: defaultBranch,
filesCreated: map[string]string{"branch": "second"},
lastRevision: fmt.Sprintf("%s/%s", defaultBranch, secondCommit.String()),
expectedCommit: secondCommit.String(),
expectedConcreteCommit: false,
},
{
name: "lastRevision is different",
branch: defaultBranch,
filesCreated: map[string]string{"branch": "second"},
lastRevision: fmt.Sprintf("%s/%s", defaultBranch, firstCommit.String()),
expectedCommit: secondCommit.String(),
expectedConcreteCommit: true,
name: "Non existing branch",
branch: "invalid",
expectedErr: "reference 'refs/remotes/origin/invalid' not found",
},
}
@ -142,43 +122,31 @@ func TestCheckoutBranch_Checkout(t *testing.T) {
}
g.Expect(err).ToNot(HaveOccurred())
g.Expect(cc.String()).To(Equal(tt.branch + "/" + tt.expectedCommit))
g.Expect(git.IsConcreteCommit(*cc)).To(Equal(tt.expectedConcreteCommit))
if tt.expectedConcreteCommit {
for k, v := range tt.filesCreated {
g.Expect(filepath.Join(tmpDir, k)).To(BeARegularFile())
g.Expect(os.ReadFile(filepath.Join(tmpDir, k))).To(BeEquivalentTo(v))
}
}
})
}
}
func TestCheckoutTag_Checkout(t *testing.T) {
func TestCheckoutTag_checkoutUnmanaged(t *testing.T) {
type testTag struct {
name string
annotated bool
}
tests := []struct {
name string
tagsInRepo []testTag
checkoutTag string
lastRevTag string
expectErr string
expectConcreteCommit bool
name string
tagsInRepo []testTag
checkoutTag string
expectErr string
}{
{
name: "Tag",
tagsInRepo: []testTag{{"tag-1", false}},
checkoutTag: "tag-1",
expectConcreteCommit: true,
name: "Tag",
tagsInRepo: []testTag{{"tag-1", false}},
checkoutTag: "tag-1",
},
{
name: "Annotated",
tagsInRepo: []testTag{{"annotated", true}},
checkoutTag: "annotated",
expectConcreteCommit: true,
name: "Annotated",
tagsInRepo: []testTag{{"annotated", true}},
checkoutTag: "annotated",
},
{
name: "Non existing tag",
@ -186,18 +154,14 @@ func TestCheckoutTag_Checkout(t *testing.T) {
expectErr: "unable to find 'invalid': no reference found for shorthand 'invalid'",
},
{
name: "Skip clone - last revision unchanged",
tagsInRepo: []testTag{{"tag-1", false}},
checkoutTag: "tag-1",
lastRevTag: "tag-1",
expectConcreteCommit: false,
name: "Skip clone - last revision unchanged",
tagsInRepo: []testTag{{"tag-1", false}},
checkoutTag: "tag-1",
},
{
name: "Last revision changed",
tagsInRepo: []testTag{{"tag-1", false}, {"tag-2", false}},
checkoutTag: "tag-2",
lastRevTag: "tag-1",
expectConcreteCommit: true,
name: "Last revision changed",
tagsInRepo: []testTag{{"tag-1", false}, {"tag-2", false}},
checkoutTag: "tag-2",
},
}
for _, tt := range tests {
@ -235,12 +199,6 @@ func TestCheckoutTag_Checkout(t *testing.T) {
checkoutTag := CheckoutTag{
Tag: tt.checkoutTag,
}
// If last revision is provided, configure it.
if tt.lastRevTag != "" {
lc := tagCommits[tt.lastRevTag]
checkoutTag.LastRevision = fmt.Sprintf("%s/%s", tt.lastRevTag, lc.Id().String())
}
tmpDir := t.TempDir()
cc, err := checkoutTag.Checkout(context.TODO(), tmpDir, repo.Path(), nil)
@ -252,16 +210,12 @@ func TestCheckoutTag_Checkout(t *testing.T) {
}
// Check successful checkout results.
g.Expect(git.IsConcreteCommit(*cc)).To(Equal(tt.expectConcreteCommit))
targetTagCommit := tagCommits[tt.checkoutTag]
g.Expect(err).ToNot(HaveOccurred())
g.Expect(cc.String()).To(Equal(tt.checkoutTag + "/" + targetTagCommit.Id().String()))
// Check file content only when there's an actual checkout.
if tt.lastRevTag != tt.checkoutTag {
g.Expect(filepath.Join(tmpDir, "tag")).To(BeARegularFile())
g.Expect(os.ReadFile(filepath.Join(tmpDir, "tag"))).To(BeEquivalentTo(tt.checkoutTag))
}
g.Expect(filepath.Join(tmpDir, "tag")).To(BeARegularFile())
g.Expect(os.ReadFile(filepath.Join(tmpDir, "tag"))).To(BeEquivalentTo(tt.checkoutTag))
})
}
}

140
pkg/git/libgit2/managed/http.go
View File

@ -55,6 +55,7 @@ import (
"sync"
pool "github.com/fluxcd/source-controller/internal/transport"
"github.com/fluxcd/source-controller/pkg/git"
git2go "github.com/libgit2/git2go/v33"
)
@ -86,30 +87,45 @@ type httpSmartSubtransport struct {
httpTransport *http.Transport
}
func (t *httpSmartSubtransport) Action(targetUrl string, action git2go.SmartServiceAction) (git2go.SmartSubtransportStream, error) {
func (t *httpSmartSubtransport) Action(transportOptionsURL string, action git2go.SmartServiceAction) (git2go.SmartSubtransportStream, error) {
opts, found := getTransportOptions(transportOptionsURL)
if !found {
return nil, fmt.Errorf("failed to create client: could not find transport options for the object: %s", transportOptionsURL)
}
targetURL := opts.TargetURL
if targetURL == "" {
return nil, fmt.Errorf("repository URL cannot be empty")
}
if len(targetURL) > URLMaxLength {
return nil, fmt.Errorf("URL exceeds the max length (%d)", URLMaxLength)
}
var proxyFn func(*http.Request) (*url.URL, error)
proxyOpts, err := t.transport.SmartProxyOptions()
if err != nil {
return nil, err
}
switch proxyOpts.Type {
case git2go.ProxyTypeNone:
proxyFn = nil
case git2go.ProxyTypeAuto:
proxyFn = http.ProxyFromEnvironment
case git2go.ProxyTypeSpecified:
parsedUrl, err := url.Parse(proxyOpts.Url)
if err != nil {
return nil, err
proxyOpts := opts.ProxyOptions
if proxyOpts != nil {
switch proxyOpts.Type {
case git2go.ProxyTypeNone:
proxyFn = nil
case git2go.ProxyTypeAuto:
proxyFn = http.ProxyFromEnvironment
case git2go.ProxyTypeSpecified:
parsedUrl, err := url.Parse(proxyOpts.Url)
if err != nil {
return nil, err
}
proxyFn = http.ProxyURL(parsedUrl)
}
proxyFn = http.ProxyURL(parsedUrl)
t.httpTransport.Proxy = proxyFn
t.httpTransport.ProxyConnectHeader = map[string][]string{}
} else {
t.httpTransport.Proxy = nil
}
t.httpTransport.Proxy = proxyFn
t.httpTransport.DisableCompression = false
client, req, err := createClientRequest(targetUrl, action, t.httpTransport)
client, req, err := createClientRequest(targetURL, action, t.httpTransport, opts.AuthOpts)
if err != nil {
return nil, err
}
@ -142,7 +158,8 @@ func (t *httpSmartSubtransport) Action(targetUrl string, action git2go.SmartServ
return stream, nil
}
func createClientRequest(targetUrl string, action git2go.SmartServiceAction, t *http.Transport) (*http.Client, *http.Request, error) {
func createClientRequest(targetURL string, action git2go.SmartServiceAction,
t *http.Transport, authOpts *git.AuthOptions) (*http.Client, *http.Request, error) {
var req *http.Request
var err error
@ -150,31 +167,6 @@ func createClientRequest(targetUrl string, action git2go.SmartServiceAction, t *
return nil, nil, fmt.Errorf("failed to create client: transport cannot be nil")
}
finalUrl := targetUrl
opts, found := transportOptions(targetUrl)
if found {
if opts.TargetURL != "" {
// override target URL only if options are found and a new targetURL
// is provided.
finalUrl = opts.TargetURL
}
// Add any provided certificate to the http transport.
if len(opts.CABundle) > 0 {
cap := x509.NewCertPool()
if ok := cap.AppendCertsFromPEM(opts.CABundle); !ok {
return nil, nil, fmt.Errorf("failed to use certificate from PEM")
}
t.TLSClientConfig = &tls.Config{
RootCAs: cap,
}
}
}
if len(finalUrl) > URLMaxLength {
return nil, nil, fmt.Errorf("URL exceeds the max length (%d)", URLMaxLength)
}
client := &http.Client{
Transport: t,
Timeout: fullHttpClientTimeOut,
@ -182,24 +174,30 @@ func createClientRequest(targetUrl string, action git2go.SmartServiceAction, t *
switch action {
case git2go.SmartServiceActionUploadpackLs:
req, err = http.NewRequest("GET", finalUrl+"/info/refs?service=git-upload-pack", nil)
req, err = http.NewRequest("GET", targetURL+"/info/refs?service=git-upload-pack", nil)
case git2go.SmartServiceActionUploadpack:
req, err = http.NewRequest("POST", finalUrl+"/git-upload-pack", nil)
req, err = http.NewRequest("POST", targetURL+"/git-upload-pack", nil)
if err != nil {
break
}
req.Header.Set("Content-Type", "application/x-git-upload-pack-request")
if t.Proxy != nil {
t.ProxyConnectHeader.Set("Content-Type", "application/x-git-upload-pack-request")
}
case git2go.SmartServiceActionReceivepackLs:
req, err = http.NewRequest("GET", finalUrl+"/info/refs?service=git-receive-pack", nil)
req, err = http.NewRequest("GET", targetURL+"/info/refs?service=git-receive-pack", nil)
case git2go.SmartServiceActionReceivepack:
req, err = http.NewRequest("POST", finalUrl+"/git-receive-pack", nil)
req, err = http.NewRequest("POST", targetURL+"/git-receive-pack", nil)
if err != nil {
break
}
req.Header.Set("Content-Type", "application/x-git-receive-pack-request")
if t.Proxy != nil {
t.ProxyConnectHeader.Set("Content-Type", "application/x-git-receive-pack-request")
}
default:
err = errors.New("unknown action")
@ -209,7 +207,26 @@ func createClientRequest(targetUrl string, action git2go.SmartServiceAction, t *
return nil, nil, err
}
// Apply authentication and TLS settings to the HTTP transport.
if authOpts != nil {
if len(authOpts.Username) > 0 {
req.SetBasicAuth(authOpts.Username, authOpts.Password)
}
if len(authOpts.CAFile) > 0 {
certPool := x509.NewCertPool()
if ok := certPool.AppendCertsFromPEM(authOpts.CAFile); !ok {
return nil, nil, fmt.Errorf("PEM CA bundle could not be appended to x509 certificate pool")
}
t.TLSClientConfig = &tls.Config{
RootCAs: certPool,
}
}
}
req.Header.Set("User-Agent", "git/2.0 (flux-libgit2)")
if t.Proxy != nil {
t.ProxyConnectHeader.Set("User-Agent", "git/2.0 (flux-libgit2)")
}
return client, req, nil
}
@ -239,7 +256,6 @@ type httpSmartSubtransportStream struct {
recvReply sync.WaitGroup
httpError error
m sync.RWMutex
targetURL string
}
func newManagedHttpStream(owner *httpSmartSubtransport, req *http.Request, client *http.Client) *httpSmartSubtransportStream {
@ -324,29 +340,8 @@ func (self *httpSmartSubtransportStream) sendRequest() error {
var resp *http.Response
var err error
var userName string
var password string
// Obtain the credentials and use them if available.
cred, err := self.owner.transport.SmartCredentials("", git2go.CredentialTypeUserpassPlaintext)
if err != nil {
// Passthrough error indicates that no credentials were provided.
// Continue without credentials.
if err.Error() != git2go.ErrorCodePassthrough.String() {
return err
}
}
if cred != nil {
defer cred.Free()
userName, password, err = cred.GetUserpassPlaintext()
if err != nil {
return err
}
}
var content []byte
for {
req := &http.Request{
Method: self.req.Method,
@ -365,7 +360,6 @@ func (self *httpSmartSubtransportStream) sendRequest() error {
req.ContentLength = -1
}
req.SetBasicAuth(userName, password)
traceLog.Info("[http]: new request", "method", req.Method, "URL", req.URL)
resp, err = self.client.Do(req)
if err != nil {

225
pkg/git/libgit2/managed/http_test.go Normal file
View File

@ -0,0 +1,225 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package managed
import (
"fmt"
"net/http"
"os"
"path/filepath"
"testing"
"github.com/fluxcd/pkg/gittestserver"
"github.com/fluxcd/source-controller/pkg/git"
"github.com/go-logr/logr"
. "github.com/onsi/gomega"
git2go "github.com/libgit2/git2go/v33"
)
func TestHttpAction_CreateClientRequest(t *testing.T) {
authOpts := git.AuthOptions{
Username: "user",
Password: "pwd",
}
url := "https://final-target/abc"
tests := []struct {
name string
assertFunc func(g *WithT, req *http.Request, client *http.Client)
action git2go.SmartServiceAction
authOpts git.AuthOptions
transport *http.Transport
wantedErr error
}{
{
name: "Uploadpack: URL, method and headers are correctly set",
action: git2go.SmartServiceActionUploadpack,
transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
ProxyConnectHeader: map[string][]string{},
},
assertFunc: func(g *WithT, req *http.Request, _ *http.Client) {
g.Expect(req.URL.String()).To(Equal("https://final-target/abc/git-upload-pack"))
g.Expect(req.Method).To(Equal("POST"))
g.Expect(req.Header).To(BeEquivalentTo(map[string][]string{
"User-Agent": []string{"git/2.0 (flux-libgit2)"},
"Content-Type": []string{"application/x-git-upload-pack-request"},
}))
},
wantedErr: nil,
},
{
name: "UploadpackLs: URL, method and headers are correctly set",
action: git2go.SmartServiceActionUploadpackLs,
transport: &http.Transport{},
assertFunc: func(g *WithT, req *http.Request, _ *http.Client) {
g.Expect(req.URL.String()).To(Equal("https://final-target/abc/info/refs?service=git-upload-pack"))
g.Expect(req.Method).To(Equal("GET"))
g.Expect(req.Header).To(BeEquivalentTo(map[string][]string{
"User-Agent": []string{"git/2.0 (flux-libgit2)"},
}))
},
wantedErr: nil,
},
{
name: "Receivepack: URL, method and headers are correctly set",
action: git2go.SmartServiceActionReceivepack,
transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
ProxyConnectHeader: map[string][]string{},
},
assertFunc: func(g *WithT, req *http.Request, _ *http.Client) {
g.Expect(req.URL.String()).To(Equal("https://final-target/abc/git-receive-pack"))
g.Expect(req.Method).To(Equal("POST"))
g.Expect(req.Header).To(BeEquivalentTo(map[string][]string{
"Content-Type": []string{"application/x-git-receive-pack-request"},
"User-Agent": []string{"git/2.0 (flux-libgit2)"},
}))
},
wantedErr: nil,
},
{
name: "ReceivepackLs: URL, method and headars are correctly set",
action: git2go.SmartServiceActionReceivepackLs,
transport: &http.Transport{},
assertFunc: func(g *WithT, req *http.Request, _ *http.Client) {
g.Expect(req.URL.String()).To(Equal("https://final-target/abc/info/refs?service=git-receive-pack"))
g.Expect(req.Method).To(Equal("GET"))
g.Expect(req.Header).To(BeEquivalentTo(map[string][]string{
"User-Agent": []string{"git/2.0 (flux-libgit2)"},
}))
},
wantedErr: nil,
},
{
name: "credentials are correctly configured",
action: git2go.SmartServiceActionUploadpack,
transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
ProxyConnectHeader: map[string][]string{},
},
authOpts: authOpts,
assertFunc: func(g *WithT, req *http.Request, client *http.Client) {
g.Expect(req.URL.String()).To(Equal("https://final-target/abc/git-upload-pack"))
g.Expect(req.Method).To(Equal("POST"))
username, pwd, ok := req.BasicAuth()
if !ok {
t.Errorf("could not find Authentication header in request.")
}
g.Expect(username).To(Equal("user"))
g.Expect(pwd).To(Equal("pwd"))
},
wantedErr: nil,
},
{
name: "error when no http.transport provided",
action: git2go.SmartServiceActionUploadpack,
transport: nil,
wantedErr: fmt.Errorf("failed to create client: transport cannot be nil"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
client, req, err := createClientRequest(url, tt.action, tt.transport, &tt.authOpts)
if err != nil {
t.Log(err)
}
if tt.wantedErr != nil {
g.Expect(err).To(Equal(tt.wantedErr))
} else {
tt.assertFunc(g, req, client)
}
})
}
}
func TestHTTPManagedTransport_E2E(t *testing.T) {
g := NewWithT(t)
server, err := gittestserver.NewTempGitServer()
g.Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(server.Root())
user := "test-user"
pwd := "test-pswd"
server.Auth(user, pwd)
server.KeyDir(filepath.Join(server.Root(), "keys"))
err = server.StartHTTP()
g.Expect(err).ToNot(HaveOccurred())
defer server.StopHTTP()
// Force managed transport to be enabled
InitManagedTransport(logr.Discard())
repoPath := "test.git"
err = server.InitRepo("../../testdata/git/repo", git.DefaultBranch, repoPath)
g.Expect(err).ToNot(HaveOccurred())
tmpDir := t.TempDir()
// Register the auth options and target url mapped to a unique url.
id := "http://obj-id"
AddTransportOptions(id, TransportOptions{
TargetURL: server.HTTPAddress() + "/" + repoPath,
AuthOpts: &git.AuthOptions{
Username: user,
Password: pwd,
},
})
// We call git2go.Clone with transportOptsURL instead of the actual URL,
// as the transport action will fetch the actual URL and the required
// credentials using the it as an identifier.
repo, err := git2go.Clone(id, tmpDir, &git2go.CloneOptions{
CheckoutOptions: git2go.CheckoutOptions{
Strategy: git2go.CheckoutForce,
},
})
g.Expect(err).ToNot(HaveOccurred())
repo.Free()
}
func TestHTTPManagedTransport_HandleRedirect(t *testing.T) {
g := NewWithT(t)
tmpDir := t.TempDir()
// Force managed transport to be enabled
InitManagedTransport(logr.Discard())
id := "http://obj-id"
AddTransportOptions(id, TransportOptions{
TargetURL: "http://github.com/stefanprodan/podinfo",
})
// GitHub will cause a 301 and redirect to https
repo, err := git2go.Clone(id, tmpDir, &git2go.CloneOptions{
CheckoutOptions: git2go.CheckoutOptions{
Strategy: git2go.CheckoutForce,
},
})
g.Expect(err).ToNot(HaveOccurred())
repo.Free()
}

303
pkg/git/libgit2/managed/managed_test.go
View File

@ -1,303 +0,0 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package managed
import (
"fmt"
"net/http"
"os"
"path/filepath"
"reflect"
"testing"
"github.com/fluxcd/pkg/gittestserver"
"github.com/fluxcd/pkg/ssh"
"github.com/fluxcd/source-controller/pkg/git"
"github.com/go-logr/logr"
git2go "github.com/libgit2/git2go/v33"
. "github.com/onsi/gomega"
"gotest.tools/assert"
)
func TestHttpAction_CreateClientRequest(t *testing.T) {
tests := []struct {
name string
url string
expectedUrl string
expectedMethod string
action git2go.SmartServiceAction
opts *TransportOptions
transport *http.Transport
wantedErr error
}{
{
name: "Uploadpack: no changes when no options found",
url: "https://sometarget/abc",
expectedUrl: "https://sometarget/abc/git-upload-pack",
expectedMethod: "POST",
action: git2go.SmartServiceActionUploadpack,
transport: &http.Transport{},
opts: nil,
wantedErr: nil,
},
{
name: "UploadpackLs: no changes when no options found",
url: "https://sometarget/abc",
expectedUrl: "https://sometarget/abc/info/refs?service=git-upload-pack",
expectedMethod: "GET",
action: git2go.SmartServiceActionUploadpackLs,
transport: &http.Transport{},
opts: nil,
wantedErr: nil,
},
{
name: "Receivepack: no changes when no options found",
url: "https://sometarget/abc",
expectedUrl: "https://sometarget/abc/git-receive-pack",
expectedMethod: "POST",
action: git2go.SmartServiceActionReceivepack,
transport: &http.Transport{},
opts: nil,
wantedErr: nil,
},
{
name: "ReceivepackLs: no changes when no options found",
url: "https://sometarget/abc",
expectedUrl: "https://sometarget/abc/info/refs?service=git-receive-pack",
expectedMethod: "GET",
action: git2go.SmartServiceActionReceivepackLs,
transport: &http.Transport{},
opts: nil,
wantedErr: nil,
},
{
name: "override URL via options",
url: "https://initial-target/abc",
expectedUrl: "https://final-target/git-upload-pack",
expectedMethod: "POST",
action: git2go.SmartServiceActionUploadpack,
transport: &http.Transport{},
opts: &TransportOptions{
TargetURL: "https://final-target",
},
wantedErr: nil,
},
{
name: "error when no http.transport provided",
url: "https://initial-target/abc",
expectedUrl: "",
expectedMethod: "",
action: git2go.SmartServiceActionUploadpack,
transport: nil,
opts: nil,
wantedErr: fmt.Errorf("failed to create client: transport cannot be nil"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.opts != nil {
AddTransportOptions(tt.url, *tt.opts)
}
_, req, err := createClientRequest(tt.url, tt.action, tt.transport)
if tt.wantedErr != nil {
if tt.wantedErr.Error() != err.Error() {
t.Errorf("wanted: %v got: %v", tt.wantedErr, err)
}
} else {
assert.Equal(t, req.URL.String(), tt.expectedUrl)
assert.Equal(t, req.Method, tt.expectedMethod)
}
if tt.opts != nil {
RemoveTransportOptions(tt.url)
}
})
}
}
func TestOptions(t *testing.T) {
tests := []struct {
name string
registerOpts bool
url string
opts TransportOptions
expectOpts bool
expectedOpts *TransportOptions
}{
{
name: "return registered option",
registerOpts: true,
url: "https://target/?123",
opts: TransportOptions{},
expectOpts: true,
expectedOpts: &TransportOptions{},
},
{
name: "match registered options",
registerOpts: true,
url: "https://target/?876",
opts: TransportOptions{
TargetURL: "https://new-target/321",
CABundle: []byte{123, 213, 132},
},
expectOpts: true,
expectedOpts: &TransportOptions{
TargetURL: "https://new-target/321",
CABundle: []byte{123, 213, 132},
},
},
{
name: "ignore when options not registered",
registerOpts: false,
url: "",
opts: TransportOptions{},
expectOpts: false,
expectedOpts: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.registerOpts {
AddTransportOptions(tt.url, tt.opts)
}
opts, found := transportOptions(tt.url)
if tt.expectOpts != found {
t.Errorf("%s: wanted %v got %v", tt.name, tt.expectOpts, found)
}
if tt.expectOpts {
if reflect.DeepEqual(opts, *tt.expectedOpts) {
t.Errorf("%s: wanted %v got %v", tt.name, *tt.expectedOpts, opts)
}
}
if tt.registerOpts {
RemoveTransportOptions(tt.url)
}
if _, found = transportOptions(tt.url); found {
t.Errorf("%s: option for %s was not removed", tt.name, tt.url)
}
})
}
}
func TestManagedTransport_E2E(t *testing.T) {
g := NewWithT(t)
server, err := gittestserver.NewTempGitServer()
g.Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(server.Root())
user := "test-user"
pasword := "test-pswd"
server.Auth(user, pasword)
server.KeyDir(filepath.Join(server.Root(), "keys"))
err = server.ListenSSH()
g.Expect(err).ToNot(HaveOccurred())
err = server.StartHTTP()
g.Expect(err).ToNot(HaveOccurred())
defer server.StopHTTP()
go func() {
server.StartSSH()
}()
defer server.StopSSH()
// Force managed transport to be enabled
InitManagedTransport(logr.Discard())
repoPath := "test.git"
err = server.InitRepo("../../testdata/git/repo", git.DefaultBranch, repoPath)
g.Expect(err).ToNot(HaveOccurred())
tmpDir := t.TempDir()
// Test HTTP transport
// Use a fake-url and force it to be overriden by the smart transport.
// This was the way found to ensure that the built-in transport was not used.
httpAddress := "http://fake-url"
AddTransportOptions(httpAddress, TransportOptions{
TargetURL: server.HTTPAddress() + "/" + repoPath,
})
repo, err := git2go.Clone(httpAddress, tmpDir, &git2go.CloneOptions{
FetchOptions: git2go.FetchOptions{
RemoteCallbacks: git2go.RemoteCallbacks{
CredentialsCallback: func(url, username_from_url string, allowed_types git2go.CredentialType) (*git2go.Credential, error) {
return git2go.NewCredentialUserpassPlaintext(user, pasword)
},
},
},
CheckoutOptions: git2go.CheckoutOptions{
Strategy: git2go.CheckoutForce,
},
})
g.Expect(err).ToNot(HaveOccurred())
repo.Free()
tmpDir2 := t.TempDir()
kp, err := ssh.NewEd25519Generator().Generate()
g.Expect(err).ToNot(HaveOccurred())
// Test SSH transport
sshAddress := server.SSHAddress() + "/" + repoPath
repo, err = git2go.Clone(sshAddress, tmpDir2, &git2go.CloneOptions{
FetchOptions: git2go.FetchOptions{
RemoteCallbacks: git2go.RemoteCallbacks{
CredentialsCallback: func(url, username_from_url string, allowed_types git2go.CredentialType) (*git2go.Credential, error) {
return git2go.NewCredentialSSHKeyFromMemory("git", "", string(kp.PrivateKey), "")
},
},
},
CheckoutOptions: git2go.CheckoutOptions{
Strategy: git2go.CheckoutForce,
},
})
g.Expect(err).ToNot(HaveOccurred())
repo.Free()
}
func TestManagedTransport_HandleRedirect(t *testing.T) {
g := NewWithT(t)
tmpDir := t.TempDir()
// Force managed transport to be enabled
InitManagedTransport(logr.Discard())
// GitHub will cause a 301 and redirect to https
repo, err := git2go.Clone("http://github.com/stefanprodan/podinfo", tmpDir, &git2go.CloneOptions{
FetchOptions: git2go.FetchOptions{},
CheckoutOptions: git2go.CheckoutOptions{
Strategy: git2go.CheckoutForce,
},
})
g.Expect(err).ToNot(HaveOccurred())
repo.Free()
}

36
pkg/git/libgit2/managed/options.go
View File

@ -18,35 +18,47 @@ package managed
import (
"sync"
"github.com/fluxcd/source-controller/pkg/git"
git2go "github.com/libgit2/git2go/v33"
)
// TransportOptions represents options to be applied at transport-level
// at request time.
type TransportOptions struct {
TargetURL string
CABundle []byte
TargetURL string
AuthOpts *git.AuthOptions
ProxyOptions *git2go.ProxyOptions
}
var (
// transportOpts maps a unique URL to a set of transport options.
transportOpts = make(map[string]TransportOptions, 0)
m sync.RWMutex
)
func AddTransportOptions(targetUrl string, opts TransportOptions) {
// AddTransportOptions registers a TransportOptions object mapped to the
// provided transportOptsURL, which must be a valid URL, i.e. prefixed with "http://"
// or "ssh://", as it is used as a dummy URL for all git operations and the managed
// transports will only be invoked for the protocols that they have been
// registered for.
func AddTransportOptions(transportOptsURL string, opts TransportOptions) {
m.Lock()
transportOpts[targetUrl] = opts
transportOpts[transportOptsURL] = opts
m.Unlock()
}
func RemoveTransportOptions(targetUrl string) {
// RemoveTransportOptions removes the registerd TransportOptions object
// mapped to the provided id.
func RemoveTransportOptions(transportOptsURL string) {
m.Lock()
delete(transportOpts, targetUrl)
delete(transportOpts, transportOptsURL)
m.Unlock()
}
func transportOptions(targetUrl string) (*TransportOptions, bool) {
func getTransportOptions(transportOptsURL string) (*TransportOptions, bool) {
m.RLock()
opts, found := transportOpts[targetUrl]
opts, found := transportOpts[transportOptsURL]
m.RUnlock()
if found {
@ -60,16 +72,16 @@ func transportOptions(targetUrl string) (*TransportOptions, bool) {
// Given that TransportOptions can allow for the target URL to be overriden
// this returns the same input if Managed Transport is disabled or if no TargetURL
// is set on TransportOptions.
func EffectiveURL(targetUrl string) string {
func EffectiveURL(transporOptsURL string) string {
if !Enabled() {
return targetUrl
return transporOptsURL
}
if opts, found := transportOptions(targetUrl); found {
if opts, found := getTransportOptions(transporOptsURL); found {
if opts.TargetURL != "" {
return opts.TargetURL
}
}
return targetUrl
return transporOptsURL
}

94
pkg/git/libgit2/managed/options_test.go Normal file
View File

@ -0,0 +1,94 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package managed
import (
"testing"
"github.com/fluxcd/source-controller/pkg/git"
. "github.com/onsi/gomega"
)
func TestTransportOptions(t *testing.T) {
tests := []struct {
name string
registerOpts bool
url string
opts TransportOptions
expectOpts bool
expectedOpts *TransportOptions
}{
{
name: "return registered option",
registerOpts: true,
url: "https://target/?123",
opts: TransportOptions{},
expectOpts: true,
expectedOpts: &TransportOptions{},
},
{
name: "match registered options",
registerOpts: true,
url: "https://target/?876",
opts: TransportOptions{
TargetURL: "https://new-target/321",
AuthOpts: &git.AuthOptions{
CAFile: []byte{123, 213, 132},
},
},
expectOpts: true,
expectedOpts: &TransportOptions{
TargetURL: "https://new-target/321",
AuthOpts: &git.AuthOptions{
CAFile: []byte{123, 213, 132},
},
},
},
{
name: "ignore when options not registered",
registerOpts: false,
url: "",
opts: TransportOptions{},
expectOpts: false,
expectedOpts: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
if tt.registerOpts {
AddTransportOptions(tt.url, tt.opts)
}
opts, found := getTransportOptions(tt.url)
g.Expect(found).To(Equal(found))
if tt.expectOpts {
g.Expect(tt.expectedOpts).To(Equal(opts))
}
if tt.registerOpts {
RemoveTransportOptions(tt.url)
}
_, found = getTransportOptions(tt.url)
g.Expect(found).To(BeFalse())
})
}
}

61
pkg/git/libgit2/managed/ssh.go
View File

@ -45,8 +45,6 @@ package managed
import (
"context"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"fmt"
"io"
@ -96,11 +94,16 @@ type sshSmartSubtransport struct {
connected bool
}
func (t *sshSmartSubtransport) Action(urlString string, action git2go.SmartServiceAction) (git2go.SmartSubtransportStream, error) {
func (t *sshSmartSubtransport) Action(transportOptionsURL string, action git2go.SmartServiceAction) (git2go.SmartSubtransportStream, error) {
runtime.LockOSThread()
defer runtime.UnlockOSThread()
u, err := url.Parse(urlString)
opts, found := getTransportOptions(transportOptionsURL)
if !found {
return nil, fmt.Errorf("could not find transport options for object: %s", transportOptionsURL)
}
u, err := url.Parse(opts.TargetURL)
if err != nil {
return nil, err
}
@ -146,19 +149,13 @@ func (t *sshSmartSubtransport) Action(urlString string, action git2go.SmartServi
_ = t.Close()
}
cred, err := t.transport.SmartCredentials("", git2go.CredentialTypeSSHMemory)
if err != nil {
return nil, err
}
defer cred.Free()
port := "22"
if u.Port() != "" {
port = u.Port()
}
t.addr = net.JoinHostPort(u.Hostname(), port)
sshConfig, err := clientConfig(t.addr, cred)
sshConfig, err := createClientConfig(opts.AuthOpts)
if err != nil {
return nil, err
}
@ -168,16 +165,17 @@ func (t *sshSmartSubtransport) Action(urlString string, action git2go.SmartServi
cert := &git2go.Certificate{
Kind: git2go.CertificateHostkey,
Hostkey: git2go.HostkeyCertificate{
Kind: git2go.HostkeySHA1 | git2go.HostkeyMD5 | git2go.HostkeySHA256 | git2go.HostkeyRaw,
HashMD5: md5.Sum(marshaledKey),
HashSHA1: sha1.Sum(marshaledKey),
Kind: git2go.HostkeySHA256 | git2go.HostkeyRaw,
HashSHA256: sha256.Sum256(marshaledKey),
Hostkey: marshaledKey,
SSHPublicKey: key,
},
}
return t.transport.SmartCertificateCheck(cert, true, hostname)
if len(opts.AuthOpts.KnownHosts) > 0 {
return KnownHostsCallback(hostname, opts.AuthOpts.KnownHosts)(cert, true, hostname)
}
return nil
}
err = t.createConn(t.addr, sshConfig)
@ -307,39 +305,28 @@ func (stream *sshSmartSubtransportStream) Free() {
traceLog.Info("[ssh]: sshSmartSubtransportStream.Free()")
}
func clientConfig(remoteAddress string, cred *git2go.Credential) (*ssh.ClientConfig, error) {
if cred == nil {
return nil, fmt.Errorf("cannot create ssh client config from a nil credential")
func createClientConfig(authOpts *git.AuthOptions) (*ssh.ClientConfig, error) {
if authOpts == nil {
return nil, fmt.Errorf("cannot create ssh client config from nil ssh auth options")
}
username, _, privatekey, passphrase, err := cred.GetSSHKey()
if err != nil {
return nil, err
}
var pemBytes []byte
if cred.Type() == git2go.CredentialTypeSSHMemory {
pemBytes = []byte(privatekey)
var signer ssh.Signer
var err error
if authOpts.Password != "" {
signer, err = ssh.ParsePrivateKeyWithPassphrase(authOpts.Identity, []byte(authOpts.Password))
} else {
return nil, fmt.Errorf("file based SSH credential is not supported")
signer, err = ssh.ParsePrivateKey(authOpts.Identity)
}
var key ssh.Signer
if passphrase != "" {
key, err = ssh.ParsePrivateKeyWithPassphrase(pemBytes, []byte(passphrase))
} else {
key, err = ssh.ParsePrivateKey(pemBytes)
}
if err != nil {
return nil, err
}
cfg := &ssh.ClientConfig{
User: username,
Auth: []ssh.AuthMethod{ssh.PublicKeys(key)},
User: authOpts.Username,
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
Timeout: sshConnectionTimeOut,
}
if len(git.KexAlgos) > 0 {
cfg.Config.KeyExchanges = git.KexAlgos
}

126
pkg/git/libgit2/managed/ssh_test.go Normal file
View File

@ -0,0 +1,126 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package managed
import (
"os"
"path/filepath"
"testing"
"github.com/fluxcd/pkg/ssh"
"github.com/fluxcd/source-controller/pkg/git"
"github.com/go-logr/logr"
. "github.com/onsi/gomega"
"github.com/fluxcd/pkg/gittestserver"
git2go "github.com/libgit2/git2go/v33"
)
func TestSSHAction_clientConfig(t *testing.T) {
kp, err := ssh.GenerateKeyPair(ssh.RSA_4096)
if err != nil {
t.Fatalf("could not generate keypair: %s", err)
}
tests := []struct {
name string
authOpts *git.AuthOptions
expectedUsername string
expectedAuthLen int
expectErr string
}{
{
name: "nil SSHTransportOptions returns an error",
authOpts: nil,
expectErr: "cannot create ssh client config from nil ssh auth options",
},
{
name: "valid SSHTransportOptions returns a valid SSHClientConfig",
authOpts: &git.AuthOptions{
Identity: kp.PrivateKey,
Username: "user",
},
expectedUsername: "user",
expectedAuthLen: 1,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
cfg, err := createClientConfig(tt.authOpts)
if tt.expectErr != "" {
g.Expect(tt.expectErr).To(Equal(err.Error()))
return
}
g.Expect(err).ToNot(HaveOccurred())
g.Expect(cfg.User).To(Equal(tt.expectedUsername))
g.Expect(len(cfg.Auth)).To(Equal(tt.expectedAuthLen))
})
}
}
func TestSSHManagedTransport_E2E(t *testing.T) {
g := NewWithT(t)
server, err := gittestserver.NewTempGitServer()
g.Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(server.Root())
server.KeyDir(filepath.Join(server.Root(), "keys"))
err = server.ListenSSH()
g.Expect(err).ToNot(HaveOccurred())
go func() {
server.StartSSH()
}()
defer server.StopSSH()
InitManagedTransport(logr.Discard())
kp, err := ssh.NewEd25519Generator().Generate()
g.Expect(err).ToNot(HaveOccurred())
repoPath := "test.git"
err = server.InitRepo("../../testdata/git/repo", git.DefaultBranch, repoPath)
g.Expect(err).ToNot(HaveOccurred())
transportOptsURL := "ssh://git@fake-url"
sshAddress := server.SSHAddress() + "/" + repoPath
AddTransportOptions(transportOptsURL, TransportOptions{
TargetURL: sshAddress,
AuthOpts: &git.AuthOptions{
Username: "user",
Identity: kp.PrivateKey,
},
})
tmpDir := t.TempDir()
// We call git2go.Clone with transportOptsURL, so that the managed ssh transport can
// fetch the correct set of credentials and the actual target url as well.
repo, err := git2go.Clone(transportOptsURL, tmpDir, &git2go.CloneOptions{
FetchOptions: git2go.FetchOptions{
RemoteCallbacks: RemoteCallbacks(),
},
CheckoutOptions: git2go.CheckoutOptions{
Strategy: git2go.CheckoutForce,
},
})
g.Expect(err).ToNot(HaveOccurred())
repo.Free()
}

93
pkg/git/libgit2/managed/transport.go Normal file
View File

@ -0,0 +1,93 @@
package managed
import (
"crypto/sha256"
"fmt"
"hash"
"net"
pkgkh "github.com/fluxcd/pkg/ssh/knownhosts"
git2go "github.com/libgit2/git2go/v33"
"golang.org/x/crypto/ssh/knownhosts"
)
// knownHostCallback returns a CertificateCheckCallback that verifies
// the key of Git server against the given host and known_hosts for
// git.SSH Transports.
func KnownHostsCallback(host string, knownHosts []byte) git2go.CertificateCheckCallback {
return func(cert *git2go.Certificate, valid bool, hostname string) error {
kh, err := pkgkh.ParseKnownHosts(string(knownHosts))
if err != nil {
return fmt.Errorf("failed to parse known_hosts: %w", err)
}
// First, attempt to split the configured host and port to validate
// the port-less hostname given to the callback.
hostWithoutPort, _, err := net.SplitHostPort(host)
if err != nil {
// SplitHostPort returns an error if the host is missing
// a port, assume the host has no port.
hostWithoutPort = host
}
// Different versions of libgit handle this differently.
// This fixes the case in which ports may be sent back.
hostnameWithoutPort, _, err := net.SplitHostPort(hostname)
if err != nil {
hostnameWithoutPort = hostname
}
if hostnameWithoutPort != hostWithoutPort {
return fmt.Errorf("host mismatch: %q %q", hostWithoutPort, hostnameWithoutPort)
}
var fingerprint []byte
var hasher hash.Hash
switch {
case cert.Hostkey.Kind&git2go.HostkeySHA256 > 0:
fingerprint = cert.Hostkey.HashSHA256[:]
hasher = sha256.New()
default:
return fmt.Errorf("invalid host key kind, expected to be of kind SHA256")
}
// We are now certain that the configured host and the hostname
// given to the callback match. Use the configured host (that
// includes the port), and normalize it, so we can check if there
// is an entry for the hostname _and_ port.
h := knownhosts.Normalize(host)
for _, k := range kh {
if k.Matches(h, fingerprint, hasher) {
return nil
}
}
return fmt.Errorf("hostkey could not be verified")
}
}
// RemoteCallbacks constructs git2go.RemoteCallbacks with dummy callbacks.
func RemoteCallbacks() git2go.RemoteCallbacks {
// This may not be fully removed as without some of the callbacks git2go
// gets anxious and panics.
return git2go.RemoteCallbacks{
CredentialsCallback: credentialsCallback(),
CertificateCheckCallback: certificateCallback(),
}
}
// credentialsCallback constructs a dummy CredentialsCallback.
func credentialsCallback() git2go.CredentialsCallback {
return func(url string, username string, allowedTypes git2go.CredentialType) (*git2go.Credential, error) {
// If credential is nil, panic will ensue. We fake it as managed transport does not
// require it.
return git2go.NewCredentialUserpassPlaintext("", "")
}
}
// certificateCallback constructs a dummy CertificateCallback.
func certificateCallback() git2go.CertificateCheckCallback {
// returning a nil func can cause git2go to panic.
return func(cert *git2go.Certificate, valid bool, hostname string) error {
return nil
}
}

93
pkg/git/libgit2/managed/transport_test.go Normal file
View File

@ -0,0 +1,93 @@
package managed
import (
"crypto/x509"
"encoding/base64"
"encoding/pem"
"errors"
"fmt"
"testing"
git2go "github.com/libgit2/git2go/v33"
. "github.com/onsi/gomega"
)
// knownHostsFixture is known_hosts fixture in the expected
// format.
var knownHostsFixture = `github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==`
func TestKnownHostsCallback(t *testing.T) {
tests := []struct {
name string
host string
expectedHost string
knownHosts []byte
hostkey git2go.HostkeyCertificate
want error
}{
{
name: "Match",
host: "github.com",
knownHosts: []byte(knownHostsFixture),
hostkey: git2go.HostkeyCertificate{Kind: git2go.HostkeySHA256, HashSHA256: sha256Fingerprint("nThbg6kXUpJWGl7E1IGOCspRomTxdCARLviKw6E5SY8")},
expectedHost: "github.com",
want: nil,
},
{
name: "Match with port",
host: "github.com",
knownHosts: []byte(knownHostsFixture),
hostkey: git2go.HostkeyCertificate{Kind: git2go.HostkeySHA256, HashSHA256: sha256Fingerprint("nThbg6kXUpJWGl7E1IGOCspRomTxdCARLviKw6E5SY8")},
expectedHost: "github.com:22",
want: nil,
},
{
name: "Hostname mismatch",
host: "github.com",
knownHosts: []byte(knownHostsFixture),
hostkey: git2go.HostkeyCertificate{Kind: git2go.HostkeySHA256, HashSHA256: sha256Fingerprint("nThbg6kXUpJWGl7E1IGOCspRomTxdCARLviKw6E5SY8")},
expectedHost: "example.com",
want: fmt.Errorf("host mismatch: %q %q", "example.com", "github.com"),
},
{
name: "Hostkey mismatch",
host: "github.com",
knownHosts: []byte(knownHostsFixture),
hostkey: git2go.HostkeyCertificate{Kind: git2go.HostkeySHA256, HashSHA256: sha256Fingerprint("ROQFvPThGrW4RuWLoL9tq9I9zJ42fK4XywyRtbOz/EQ")},
expectedHost: "github.com",
want: fmt.Errorf("hostkey could not be verified"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
cert := &git2go.Certificate{Hostkey: tt.hostkey}
callback := KnownHostsCallback(tt.expectedHost, tt.knownHosts)
result := g.Expect(callback(cert, false, tt.host))
if tt.want == nil {
result.To(BeNil())
} else {
result.To(Equal(tt.want))
}
})
}
}
func sha256Fingerprint(in string) [32]byte {
d, err := base64.RawStdEncoding.DecodeString(in)
if err != nil {
panic(err)
}
var out [32]byte
copy(out[:], d)
return out
}
func certificateFromPEM(pemBytes string) (*x509.Certificate, error) {
block, _ := pem.Decode([]byte(pemBytes))
if block == nil {
return nil, errors.New("failed to decode PEM")
}
return x509.ParseCertificate(block.Bytes)
}

237
pkg/git/libgit2/managed_test.go
View File

@ -19,6 +19,7 @@ package libgit2
import (
"context"
"fmt"
"math/rand"
"net/url"
"os"
"path/filepath"
@ -35,12 +36,16 @@ import (
"github.com/go-logr/logr"
. "github.com/onsi/gomega"
git2go "github.com/libgit2/git2go/v33"
cryptossh "golang.org/x/crypto/ssh"
corev1 "k8s.io/api/core/v1"
)
const testRepositoryPath = "../testdata/git/repo"
func TestMain(m *testing.M) {
managed.InitManagedTransport(logr.Discard())
}
// Test_ManagedSSH_KeyTypes assures support for the different
// types of keys for SSH Authentication supported by Flux.
func Test_ManagedSSH_KeyTypes(t *testing.T) {
@ -50,12 +55,36 @@ func Test_ManagedSSH_KeyTypes(t *testing.T) {
authorized bool
wantErr string
}{
{name: "RSA 4096", keyType: ssh.RSA_4096, authorized: true},
{name: "ECDSA P256", keyType: ssh.ECDSA_P256, authorized: true},
{name: "ECDSA P384", keyType: ssh.ECDSA_P384, authorized: true},
{name: "ECDSA P521", keyType: ssh.ECDSA_P521, authorized: true},
{name: "ED25519", keyType: ssh.ED25519, authorized: true},
{name: "unauthorized key", keyType: ssh.RSA_4096, wantErr: "Failed to retrieve list of SSH authentication methods"},
{
name: "RSA 4096",
keyType: ssh.RSA_4096,
authorized: true,
},
{
name: "ECDSA P256",
keyType: ssh.ECDSA_P256,
authorized: true,
},
{
name: "ECDSA P384",
keyType: ssh.ECDSA_P384,
authorized: true,
},
{
name: "ECDSA P521",
keyType: ssh.ECDSA_P521,
authorized: true,
},
{
name: "ED25519",
keyType: ssh.ED25519,
authorized: true,
},
{
name: "unauthorized key",
keyType: ssh.RSA_4096,
wantErr: "unable to authenticate, attempted methods [none publickey], no supported methods remain",
},
}
serverRootDir := t.TempDir()
@ -112,15 +141,11 @@ func Test_ManagedSSH_KeyTypes(t *testing.T) {
authorizedPublicKey = string(kp.PublicKey)
}
secret := corev1.Secret{
Data: map[string][]byte{
"identity": kp.PrivateKey,
"known_hosts": knownHosts,
},
authOpts := &git.AuthOptions{
Identity: kp.PrivateKey,
KnownHosts: knownHosts,
}
authOpts, err := git.AuthOptionsFromSecret(repoURL, &secret)
g.Expect(err).ToNot(HaveOccurred())
authOpts.TransportOptionsURL = getTransportOptionsURL(git.SSH)
// Prepare for checkout.
branchCheckoutStrat := &CheckoutBranch{Branch: git.DefaultBranch}
@ -223,8 +248,6 @@ func Test_ManagedSSH_KeyExchangeAlgos(t *testing.T) {
}()
defer server.StopSSH()
os.Setenv("EXPERIMENTAL_GIT_TRANSPORT", "true")
managed.InitManagedTransport(logr.Discard())
repoPath := "test.git"
err := server.InitRepo(testRepositoryPath, git.DefaultBranch, repoPath)
@ -246,15 +269,11 @@ func Test_ManagedSSH_KeyExchangeAlgos(t *testing.T) {
kp, err := ssh.GenerateKeyPair(ssh.ED25519)
g.Expect(err).ToNot(HaveOccurred())
secret := corev1.Secret{
Data: map[string][]byte{
"identity": kp.PrivateKey,
"known_hosts": knownHosts,
},
authOpts := &git.AuthOptions{
Identity: kp.PrivateKey,
KnownHosts: knownHosts,
}
authOpts, err := git.AuthOptionsFromSecret(repoURL, &secret)
g.Expect(err).ToNot(HaveOccurred())
authOpts.TransportOptionsURL = getTransportOptionsURL(git.SSH)
// Prepare for checkout.
branchCheckoutStrat := &CheckoutBranch{Branch: git.DefaultBranch}
@ -396,8 +415,6 @@ func Test_ManagedSSH_HostKeyAlgos(t *testing.T) {
}()
defer server.StopSSH()
os.Setenv("EXPERIMENTAL_GIT_TRANSPORT", "true")
managed.InitManagedTransport(logr.Discard())
repoPath := "test.git"
err = server.InitRepo(testRepositoryPath, git.DefaultBranch, repoPath)
@ -419,15 +436,11 @@ func Test_ManagedSSH_HostKeyAlgos(t *testing.T) {
kp, err := ssh.GenerateKeyPair(ssh.ED25519)
g.Expect(err).ToNot(HaveOccurred())
secret := corev1.Secret{
Data: map[string][]byte{
"identity": kp.PrivateKey,
"known_hosts": knownHosts,
},
authOpts := &git.AuthOptions{
Identity: kp.PrivateKey,
KnownHosts: knownHosts,
}
authOpts, err := git.AuthOptionsFromSecret(repoURL, &secret)
g.Expect(err).ToNot(HaveOccurred())
authOpts.TransportOptionsURL = getTransportOptionsURL(git.SSH)
// Prepare for checkout.
branchCheckoutStrat := &CheckoutBranch{Branch: git.DefaultBranch}
@ -442,3 +455,157 @@ func Test_ManagedSSH_HostKeyAlgos(t *testing.T) {
})
}
}
func Test_ManagedHTTPCheckout(t *testing.T) {
g := NewWithT(t)
timeout := 5 * time.Second
server, err := gittestserver.NewTempGitServer()
g.Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(server.Root())
user := "test-user"
pwd := "test-pswd"
server.Auth(user, pwd)
err = server.StartHTTP()
g.Expect(err).ToNot(HaveOccurred())
defer server.StopHTTP()
repoPath := "test.git"
err = server.InitRepo("../testdata/git/repo", git.DefaultBranch, repoPath)
g.Expect(err).ToNot(HaveOccurred())
authOpts := &git.AuthOptions{
Username: "test-user",
Password: "test-pswd",
}
authOpts.TransportOptionsURL = getTransportOptionsURL(git.HTTP)
// Prepare for checkout.
branchCheckoutStrat := &CheckoutBranch{Branch: git.DefaultBranch}
tmpDir := t.TempDir()
ctx, cancel := context.WithTimeout(context.TODO(), timeout)
defer cancel()
repoURL := server.HTTPAddress() + "/" + repoPath
// Checkout the repo.
_, err = branchCheckoutStrat.Checkout(ctx, tmpDir, repoURL, authOpts)
g.Expect(err).Error().ShouldNot(HaveOccurred())
}
func TestManagedCheckoutBranch_Checkout(t *testing.T) {
g := NewWithT(t)
timeout := 5 * time.Second
server, err := gittestserver.NewTempGitServer()
g.Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(server.Root())
err = server.StartHTTP()
g.Expect(err).ToNot(HaveOccurred())
defer server.StopHTTP()
repoPath := "test.git"
err = server.InitRepo("../testdata/git/repo", git.DefaultBranch, repoPath)
g.Expect(err).ToNot(HaveOccurred())
repo, err := git2go.OpenRepository(filepath.Join(server.Root(), repoPath))
g.Expect(err).ToNot(HaveOccurred())
branchRef, err := repo.References.Lookup(fmt.Sprintf("refs/heads/%s", git.DefaultBranch))
g.Expect(err).ToNot(HaveOccurred())
defer branchRef.Free()
commit, err := repo.LookupCommit(branchRef.Target())
g.Expect(err).ToNot(HaveOccurred())
authOpts := &git.AuthOptions{
TransportOptionsURL: getTransportOptionsURL(git.HTTP),
}
tmpDir := t.TempDir()
ctx, cancel := context.WithTimeout(context.TODO(), timeout)
defer cancel()
repoURL := server.HTTPAddress() + "/" + repoPath
branch := CheckoutBranch{
Branch: git.DefaultBranch,
// Set last revision to HEAD commit, to force a no-op clone.
LastRevision: fmt.Sprintf("%s/%s", git.DefaultBranch, commit.Id().String()),
}
cc, err := branch.Checkout(ctx, tmpDir, repoURL, authOpts)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(cc.String()).To(Equal(git.DefaultBranch + "/" + commit.Id().String()))
g.Expect(git.IsConcreteCommit(*cc)).To(Equal(false))
// Set last revision to a fake commit to force a full clone.
branch.LastRevision = fmt.Sprintf("%s/non-existent-commit", git.DefaultBranch)
cc, err = branch.Checkout(ctx, tmpDir, repoURL, authOpts)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(cc.String()).To(Equal(git.DefaultBranch + "/" + commit.Id().String()))
g.Expect(git.IsConcreteCommit(*cc)).To(Equal(true))
}
func TestManagedCheckoutTag_Checkout(t *testing.T) {
g := NewWithT(t)
timeout := 5 * time.Second
server, err := gittestserver.NewTempGitServer()
g.Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(server.Root())
err = server.StartHTTP()
g.Expect(err).ToNot(HaveOccurred())
defer server.StopHTTP()
repoPath := "test.git"
err = server.InitRepo("../testdata/git/repo", git.DefaultBranch, repoPath)
g.Expect(err).ToNot(HaveOccurred())
repo, err := git2go.OpenRepository(filepath.Join(server.Root(), repoPath))
g.Expect(err).ToNot(HaveOccurred())
branchRef, err := repo.References.Lookup(fmt.Sprintf("refs/heads/%s", git.DefaultBranch))
g.Expect(err).ToNot(HaveOccurred())
defer branchRef.Free()
commit, err := repo.LookupCommit(branchRef.Target())
g.Expect(err).ToNot(HaveOccurred())
_, err = tag(repo, commit.Id(), false, "tag-1", time.Now())
checkoutTag := CheckoutTag{
Tag: "tag-1",
}
authOpts := &git.AuthOptions{
TransportOptionsURL: getTransportOptionsURL(git.HTTP),
}
repoURL := server.HTTPAddress() + "/" + repoPath
tmpDir := t.TempDir()
ctx, cancel := context.WithTimeout(context.TODO(), timeout)
defer cancel()
cc, err := checkoutTag.Checkout(ctx, tmpDir, repoURL, authOpts)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(cc.String()).To(Equal("tag-1" + "/" + commit.Id().String()))
g.Expect(git.IsConcreteCommit(*cc)).To(Equal(true))
checkoutTag.LastRevision = "tag-1" + "/" + commit.Id().String()
cc, err = checkoutTag.Checkout(ctx, tmpDir, repoURL, authOpts)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(cc.String()).To(Equal("tag-1" + "/" + commit.Id().String()))
g.Expect(git.IsConcreteCommit(*cc)).To(Equal(false))
}
func getTransportOptionsURL(transport git.TransportType) string {
letterRunes := []rune("abcdefghijklmnopqrstuvwxyz1234567890")
b := make([]rune, 10)
for i := range b {
b[i] = letterRunes[rand.Intn(len(letterRunes))]
}
return string(transport) + "://" + string(b)
}

180
pkg/git/libgit2/transport.go
View File

@ -17,25 +17,13 @@ limitations under the License.
package libgit2
import (
"bufio"
"bytes"
"context"
"crypto/hmac"
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/x509"
"encoding/base64"
"fmt"
"hash"
"io"
"net"
"strings"
"time"
git2go "github.com/libgit2/git2go/v33"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/knownhosts"
"github.com/fluxcd/source-controller/pkg/git"
"github.com/fluxcd/source-controller/pkg/git/libgit2/managed"
@ -115,18 +103,6 @@ func pushTransferProgressCallback(ctx context.Context) git2go.PushTransferProgre
func credentialsCallback(opts *git.AuthOptions) git2go.CredentialsCallback {
return func(url string, username string, allowedTypes git2go.CredentialType) (*git2go.Credential, error) {
if allowedTypes&(git2go.CredentialTypeSSHKey|git2go.CredentialTypeSSHCustom|git2go.CredentialTypeSSHMemory) != 0 {
if managed.Enabled() {
// CredentialTypeSSHMemory requires libgit2 to be built using libssh2.
// When using managed transport (handled in go instead of libgit2),
// there may be ways to remove such requirement, thefore decreasing the
// need of libz, libssh2 and OpenSSL but further investigation is required
// once Managed Transport is no longer experimental.
//
// CredentialSSHKeyFromMemory is currently required for SSH key access
// when managed transport is enabled.
return git2go.NewCredentialSSHKeyFromMemory(opts.Username, "", string(opts.Identity), opts.Password)
}
var (
signer ssh.Signer
err error
@ -161,7 +137,7 @@ func certificateCallback(opts *git.AuthOptions) git2go.CertificateCheckCallback
}
case git.SSH:
if len(opts.KnownHosts) > 0 && opts.Host != "" {
return knownHostsCallback(opts.Host, opts.KnownHosts)
return managed.KnownHostsCallback(opts.Host, opts.KnownHosts)
}
}
return nil
@ -187,157 +163,3 @@ func x509Callback(caBundle []byte) git2go.CertificateCheckCallback {
return nil
}
}
// knownHostCallback returns a CertificateCheckCallback that verifies
// the key of Git server against the given host and known_hosts for
// git.SSH Transports.
func knownHostsCallback(host string, knownHosts []byte) git2go.CertificateCheckCallback {
return func(cert *git2go.Certificate, valid bool, hostname string) error {
kh, err := parseKnownHosts(string(knownHosts))
if err != nil {
return fmt.Errorf("failed to parse known_hosts: %w", err)
}
// First, attempt to split the configured host and port to validate
// the port-less hostname given to the callback.
hostWithoutPort, _, err := net.SplitHostPort(host)
if err != nil {
// SplitHostPort returns an error if the host is missing
// a port, assume the host has no port.
hostWithoutPort = host
}
// Different versions of libgit handle this differently.
// This fixes the case in which ports may be sent back.
hostnameWithoutPort, _, err := net.SplitHostPort(hostname)
if err != nil {
hostnameWithoutPort = hostname
}
if hostnameWithoutPort != hostWithoutPort {
return fmt.Errorf("host mismatch: %q %q", hostWithoutPort, hostnameWithoutPort)
}
// We are now certain that the configured host and the hostname
// given to the callback match. Use the configured host (that
// includes the port), and normalize it, so we can check if there
// is an entry for the hostname _and_ port.
h := knownhosts.Normalize(host)
for _, k := range kh {
if k.matches(h, cert.Hostkey) {
return nil
}
}
return fmt.Errorf("hostkey could not be verified")
}
}
type knownKey struct {
hosts []string
key ssh.PublicKey
}
func parseKnownHosts(s string) ([]knownKey, error) {
var knownHosts []knownKey
scanner := bufio.NewScanner(strings.NewReader(s))
for scanner.Scan() {
_, hosts, pubKey, _, _, err := ssh.ParseKnownHosts(scanner.Bytes())
if err != nil {
// Lines that aren't host public key result in EOF, like a comment
// line. Continue parsing the other lines.
if err == io.EOF {
continue
}
return []knownKey{}, err
}
knownHost := knownKey{
hosts: hosts,
key: pubKey,
}
knownHosts = append(knownHosts, knownHost)
}
if err := scanner.Err(); err != nil {
return []knownKey{}, err
}
return knownHosts, nil
}
func (k knownKey) matches(host string, hostkey git2go.HostkeyCertificate) bool {
if !containsHost(k.hosts, host) {
return false
}
var fingerprint []byte
var hasher hash.Hash
switch {
case hostkey.Kind&git2go.HostkeySHA256 > 0:
fingerprint = hostkey.HashSHA256[:]
hasher = sha256.New()
case hostkey.Kind&git2go.HostkeySHA1 > 0:
fingerprint = hostkey.HashSHA1[:]
hasher = sha1.New()
case hostkey.Kind&git2go.HostkeyMD5 > 0:
fingerprint = hostkey.HashMD5[:]
hasher = md5.New()
default:
return false
}
hasher.Write(k.key.Marshal())
return bytes.Equal(hasher.Sum(nil), fingerprint)
}
func containsHost(hosts []string, host string) bool {
for _, kh := range hosts {
// hashed host must start with a pipe
if kh[0] == '|' {
match, _ := MatchHashedHost(kh, host)
if match {
return true
}
} else if kh == host { // unhashed host check
return true
}
}
return false
}
// MatchHashedHost tries to match a hashed known host (kh) to
// host.
//
// Note that host is not hashed, but it is rather hashed during
// the matching process using the same salt used when hashing
// the known host.
func MatchHashedHost(kh, host string) (bool, error) {
if kh == "" || kh[0] != '|' {
return false, fmt.Errorf("hashed known host must begin with '|': '%s'", kh)
}
components := strings.Split(kh, "|")
if len(components) != 4 {
return false, fmt.Errorf("invalid format for hashed known host: '%s'", kh)
}
if components[1] != "1" {
return false, fmt.Errorf("unsupported hash type '%s'", components[1])
}
hkSalt, err := base64.StdEncoding.DecodeString(components[2])
if err != nil {
return false, fmt.Errorf("cannot decode hashed known host: '%w'", err)
}
hkHash, err := base64.StdEncoding.DecodeString(components[3])
if err != nil {
return false, fmt.Errorf("cannot decode hashed known host: '%w'", err)
}
mac := hmac.New(sha1.New, hkSalt)
mac.Write([]byte(host))
hostHash := mac.Sum(nil)
return bytes.Equal(hostHash, hkHash), nil
}

242
pkg/git/libgit2/transport_test.go
View File

@ -20,7 +20,6 @@ import (
"bytes"
"context"
"crypto/x509"
"encoding/base64"
"encoding/pem"
"errors"
"fmt"
@ -205,159 +204,6 @@ func Test_x509Callback(t *testing.T) {
}
}
func Test_knownHostsCallback(t *testing.T) {
tests := []struct {
name string
host string
expectedHost string
knownHosts []byte
hostkey git2go.HostkeyCertificate
want error
}{
{
name: "Match",
host: "github.com",
knownHosts: []byte(knownHostsFixture),
hostkey: git2go.HostkeyCertificate{Kind: git2go.HostkeySHA1 | git2go.HostkeyMD5, HashSHA1: sha1Fingerprint("v2toJdKXfFEaR1u++4iq1UqSrHM")},
expectedHost: "github.com",
want: nil,
},
{
name: "Match with port",
host: "github.com",
knownHosts: []byte(knownHostsFixture),
hostkey: git2go.HostkeyCertificate{Kind: git2go.HostkeySHA1 | git2go.HostkeyMD5, HashSHA1: sha1Fingerprint("v2toJdKXfFEaR1u++4iq1UqSrHM")},
expectedHost: "github.com:22",
want: nil,
},
{
name: "Hostname mismatch",
host: "github.com",
knownHosts: []byte(knownHostsFixture),
hostkey: git2go.HostkeyCertificate{Kind: git2go.HostkeySHA1 | git2go.HostkeyMD5, HashSHA1: sha1Fingerprint("v2toJdKXfFEaR1u++4iq1UqSrHM")},
expectedHost: "example.com",
want: fmt.Errorf("host mismatch: %q %q", "example.com", "github.com"),
},
{
name: "Hostkey mismatch",
host: "github.com",
knownHosts: []byte(knownHostsFixture),
hostkey: git2go.HostkeyCertificate{Kind: git2go.HostkeyMD5, HashMD5: md5Fingerprint("\xb6\x03\x0e\x39\x97\x9e\xd0\xe7\x24\xce\xa3\x77\x3e\x01\x42\x09")},
expectedHost: "github.com",
want: fmt.Errorf("hostkey could not be verified"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
cert := &git2go.Certificate{Hostkey: tt.hostkey}
callback := knownHostsCallback(tt.expectedHost, tt.knownHosts)
result := g.Expect(callback(cert, false, tt.host))
if tt.want == nil {
result.To(BeNil())
} else {
result.To(Equal(tt.want))
}
})
}
}
func Test_parseKnownHosts_matches(t *testing.T) {
tests := []struct {
name string
hostkey git2go.HostkeyCertificate
wantMatches bool
}{
{"good sha256 hostkey", git2go.HostkeyCertificate{Kind: git2go.HostkeySHA256 | git2go.HostkeySHA1 | git2go.HostkeyMD5, HashSHA256: sha256Fingerprint("nThbg6kXUpJWGl7E1IGOCspRomTxdCARLviKw6E5SY8")}, true},
{"bad sha256 hostkey", git2go.HostkeyCertificate{Kind: git2go.HostkeySHA256 | git2go.HostkeySHA1 | git2go.HostkeyMD5, HashSHA256: sha256Fingerprint("ROQFvPThGrW4RuWLoL9tq9I9zJ42fK4XywyRtbOz/EQ")}, false},
{"good sha1 hostkey", git2go.HostkeyCertificate{Kind: git2go.HostkeySHA1 | git2go.HostkeyMD5, HashSHA1: sha1Fingerprint("v2toJdKXfFEaR1u++4iq1UqSrHM")}, true},
{"bad sha1 hostkey", git2go.HostkeyCertificate{Kind: git2go.HostkeySHA1 | git2go.HostkeyMD5, HashSHA1: sha1Fingerprint("tfpLlQhDDFP3yGdewTvHNxWmAdk")}, false},
{"good md5 hostkey", git2go.HostkeyCertificate{Kind: git2go.HostkeyMD5, HashMD5: md5Fingerprint("\x16\x27\xac\xa5\x76\x28\x2d\x36\x63\x1b\x56\x4d\xeb\xdf\xa6\x48")}, true},
{"bad md5 hostkey", git2go.HostkeyCertificate{Kind: git2go.HostkeyMD5, HashMD5: md5Fingerprint("\xb6\x03\x0e\x39\x97\x9e\xd0\xe7\x24\xce\xa3\x77\x3e\x01\x42\x09")}, false},
{"invalid hostkey", git2go.HostkeyCertificate{}, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
knownKeys, err := parseKnownHosts(knownHostsFixture)
if err != nil {
t.Error(err)
return
}
matches := knownKeys[0].matches("github.com", tt.hostkey)
g.Expect(matches).To(Equal(tt.wantMatches))
})
}
}
func Test_parseKnownHosts(t *testing.T) {
tests := []struct {
name string
fixture string
wantErr bool
}{
{
name: "empty file",
fixture: "",
wantErr: false,
},
{
name: "single host",
fixture: `github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==`,
wantErr: false,
},
{
name: "single host with comment",
fixture: `# github.com
github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==`,
wantErr: false,
},
{
name: "multiple hosts with comments",
fixture: `# github.com
github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
# gitlab.com
gitlab.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf`,
},
{
name: "no host key, only comments",
fixture: `# example.com
#github.com
# gitlab.com`,
wantErr: false,
},
{
name: "invalid host entry",
fixture: `github.com ssh-rsa`,
wantErr: true,
},
{
name: "invalid content",
fixture: `some random text`,
wantErr: true,
},
{
name: "invalid line with valid host key",
fixture: `some random text
gitlab.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf`,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
_, err := parseKnownHosts(tt.fixture)
if tt.wantErr {
g.Expect(err).To(HaveOccurred())
} else {
g.Expect(err).ToNot(HaveOccurred())
}
})
}
}
func Test_transferProgressCallback(t *testing.T) {
tests := []struct {
name string
@ -522,94 +368,6 @@ func Test_pushTransferProgressCallback(t *testing.T) {
}
}
func TestMatchHashedHost(t *testing.T) {
tests := []struct {
name string
knownHost string
host string
match bool
wantErr string
}{
{
name: "match valid known host",
knownHost: "|1|vApZG0Ybr4rHfTb69+cjjFIGIv0=|M5sSXen14encOvQAy0gseRahnJw=",
host: "[127.0.0.1]:44167",
match: true,
},
{
name: "empty known host errors",
wantErr: "hashed known host must begin with '|'",
},
{
name: "unhashed known host errors",
knownHost: "[127.0.0.1]:44167",
wantErr: "hashed known host must begin with '|'",
},
{
name: "invalid known host format errors",
knownHost: "|1M5sSXen14encOvQAy0gseRahnJw=",
wantErr: "invalid format for hashed known host",
},
{
name: "invalid hash type errors",
knownHost: "|2|vApZG0Ybr4rHfTb69+cjjFIGIv0=|M5sSXen14encOvQAy0gseRahnJw=",
wantErr: "unsupported hash type",
},
{
name: "invalid base64 component[2] errors",
knownHost: "|1|azz|M5sSXen14encOvQAy0gseRahnJw=",
wantErr: "cannot decode hashed known host",
},
{
name: "invalid base64 component[3] errors",
knownHost: "|1|M5sSXen14encOvQAy0gseRahnJw=|azz",
wantErr: "cannot decode hashed known host",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
matched, err := MatchHashedHost(tt.knownHost, tt.host)
if tt.wantErr == "" {
g.Expect(err).NotTo(HaveOccurred())
g.Expect(matched).To(Equal(tt.match))
} else {
g.Expect(err).To(HaveOccurred())
g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
}
})
}
}
func md5Fingerprint(in string) [16]byte {
var out [16]byte
copy(out[:], in)
return out
}
func sha1Fingerprint(in string) [20]byte {
d, err := base64.RawStdEncoding.DecodeString(in)
if err != nil {
panic(err)
}
var out [20]byte
copy(out[:], d)
return out
}
func sha256Fingerprint(in string) [32]byte {
d, err := base64.RawStdEncoding.DecodeString(in)
if err != nil {
panic(err)
}
var out [32]byte
copy(out[:], d)
return out
}
func certificateFromPEM(pemBytes string) (*x509.Certificate, error) {
block, _ := pem.Decode([]byte(pemBytes))
if block == nil {

10
pkg/git/options.go
View File

@ -72,6 +72,16 @@ type AuthOptions struct {
Identity []byte
KnownHosts []byte
CAFile []byte
// TransportOptionsURL is a unique identifier for this set of authentication
// options. It's used by managed libgit2 transports to uniquely identify
// which credentials to use for a particular Git operation, and avoid misuse
// of credentials in a multi-tenant environment.
// It must be prefixed with a valid transport protocol ("ssh:// "or "http://") because
// of the way managed transports are registered and invoked.
// It's a field of AuthOptions despite not providing any kind of authentication
// info, as it's the only way to sneak it into git.Checkout, without polluting
// it's args and keeping it generic.
TransportOptionsURL string
}
// KexAlgos hosts the key exchange algorithms to be used for SSH connections.

206
pkg/git/strategy/proxy/strategy_proxy_test.go
View File

@ -29,17 +29,23 @@ import (
"github.com/elazarl/goproxy"
"github.com/fluxcd/pkg/gittestserver"
"github.com/go-logr/logr"
. "github.com/onsi/gomega"
"github.com/fluxcd/source-controller/pkg/git"
"github.com/fluxcd/source-controller/pkg/git/gogit"
"github.com/fluxcd/source-controller/pkg/git/libgit2"
"github.com/fluxcd/source-controller/pkg/git/libgit2/managed"
"github.com/fluxcd/source-controller/pkg/git/strategy"
)
// These tests are run in a different _test.go file because go-git uses the ProxyFromEnvironment function of the net/http package
// which caches the Proxy settings, hence not including other tests in the same file ensures a clean proxy setup for the tests to run.
func TestCheckoutStrategyForImplementation_Proxied(t *testing.T) {
// for libgit2 we are only testing for managed transport,
// as unmanaged is sunsetting.
// Unmanaged transport does not support HTTP_PROXY.
managed.InitManagedTransport(logr.Discard())
type cleanupFunc func()
@ -62,69 +68,7 @@ func TestCheckoutStrategyForImplementation_Proxied(t *testing.T) {
proxyAddr := fmt.Sprintf("localhost:%d", l.Addr().(*net.TCPAddr).Port)
g.Expect(l.Close()).ToNot(HaveOccurred())
// Note there is no libgit2 HTTP_PROXY test as libgit2 doesnt support proxied HTTP requests.
cases := []testCase{
{
name: "libgit2_HTTPS_PROXY",
gitImpl: libgit2.Implementation,
url: "https://example.com/bar/test-reponame",
branch: "main",
setupGitProxy: func(g *WithT, proxy *goproxy.ProxyHttpServer, proxyGotRequest *bool) (*git.AuthOptions, cleanupFunc) {
// Create the git server.
gitServer, err := gittestserver.NewTempGitServer()
g.Expect(err).ToNot(HaveOccurred())
username := "test-user"
password := "test-password"
gitServer.Auth(username, password)
gitServer.KeyDir(gitServer.Root())
// Start the HTTPS server.
examplePublicKey, err := os.ReadFile("../testdata/certs/server.pem")
g.Expect(err).ToNot(HaveOccurred())
examplePrivateKey, err := os.ReadFile("../testdata/certs/server-key.pem")
g.Expect(err).ToNot(HaveOccurred())
exampleCA, err := os.ReadFile("../testdata/certs/ca.pem")
g.Expect(err).ToNot(HaveOccurred())
err = gitServer.StartHTTPS(examplePublicKey, examplePrivateKey, exampleCA, "example.com")
g.Expect(err).ToNot(HaveOccurred())
// Initialize a git repo.
repoPath := "bar/test-reponame"
err = gitServer.InitRepo("../testdata/repo1", "main", repoPath)
g.Expect(err).ToNot(HaveOccurred())
u, err := url.Parse(gitServer.HTTPAddress())
g.Expect(err).ToNot(HaveOccurred())
// The request is being forwarded to the local test git server in this handler.
// The certificate used here is valid for both example.com and localhost.
var proxyHandler goproxy.FuncHttpsHandler = func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) {
// Check if the host matches with the git server address and the user-agent is the expected git client.
userAgent := ctx.Req.Header.Get("User-Agent")
if strings.Contains(host, "example.com") && strings.Contains(userAgent, "libgit2") {
*proxyGotRequest = true
return goproxy.OkConnect, u.Host
}
// Reject if it isn't our request.
return goproxy.RejectConnect, host
}
proxy.OnRequest().HandleConnect(proxyHandler)
return &git.AuthOptions{
Transport: git.HTTPS,
Username: username,
Password: password,
CAFile: exampleCA,
}, func() {
os.RemoveAll(gitServer.Root())
gitServer.StopHTTP()
}
},
shortTimeout: false,
wantUsedProxy: true,
wantError: false,
},
{
name: "gogit_HTTP_PROXY",
gitImpl: gogit.Implementation,
@ -222,9 +166,136 @@ func TestCheckoutStrategyForImplementation_Proxied(t *testing.T) {
wantUsedProxy: false,
wantError: true,
},
// TODO: Add a NO_PROXY test for libgit2 once the version of libgit2 used by the source controller is updated to a version that includes
// the NO_PROXY functionality
// This PR introduces the functionality in libgit2: https://github.com/libgit2/libgit2/pull/6026
{
name: "libgit2_HTTPS_PROXY",
gitImpl: libgit2.Implementation,
url: "https://example.com/bar/test-reponame",
branch: "main",
setupGitProxy: func(g *WithT, proxy *goproxy.ProxyHttpServer, proxyGotRequest *bool) (*git.AuthOptions, cleanupFunc) {
// Create the git server.
gitServer, err := gittestserver.NewTempGitServer()
g.Expect(err).ToNot(HaveOccurred())
username := "test-user"
password := "test-password"
gitServer.Auth(username, password)
gitServer.KeyDir(gitServer.Root())
// Start the HTTPS server.
examplePublicKey, err := os.ReadFile("../testdata/certs/server.pem")
g.Expect(err).ToNot(HaveOccurred())
examplePrivateKey, err := os.ReadFile("../testdata/certs/server-key.pem")
g.Expect(err).ToNot(HaveOccurred())
exampleCA, err := os.ReadFile("../testdata/certs/ca.pem")
g.Expect(err).ToNot(HaveOccurred())
err = gitServer.StartHTTPS(examplePublicKey, examplePrivateKey, exampleCA, "example.com")
g.Expect(err).ToNot(HaveOccurred())
// Initialize a git repo.
repoPath := "bar/test-reponame"
err = gitServer.InitRepo("../testdata/repo1", "main", repoPath)
g.Expect(err).ToNot(HaveOccurred())
u, err := url.Parse(gitServer.HTTPAddress())
g.Expect(err).ToNot(HaveOccurred())
// The request is being forwarded to the local test git server in this handler.
// The certificate used here is valid for both example.com and localhost.
var proxyHandler goproxy.FuncHttpsHandler = func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) {
defer managed.RemoveTransportOptions("https://example.com/bar/test-reponame")
// Check if the host matches with the git server address and the user-agent is the expected git client.
userAgent := ctx.Req.Header.Get("User-Agent")
if strings.Contains(host, "example.com") && strings.Contains(userAgent, "libgit2") {
*proxyGotRequest = true
return goproxy.OkConnect, u.Host
}
// Reject if it isn't our request.
return goproxy.RejectConnect, host
}
proxy.OnRequest().HandleConnect(proxyHandler)
return &git.AuthOptions{
Transport: git.HTTPS,
Username: username,
Password: password,
CAFile: exampleCA,
TransportOptionsURL: "https://proxy-test",
}, func() {
os.RemoveAll(gitServer.Root())
gitServer.StopHTTP()
}
},
shortTimeout: false,
wantUsedProxy: true,
wantError: false,
},
{
name: "libgit2_HTTP_PROXY",
gitImpl: libgit2.Implementation,
url: "http://example.com/bar/test-reponame",
branch: "main",
setupGitProxy: func(g *WithT, proxy *goproxy.ProxyHttpServer, proxyGotRequest *bool) (*git.AuthOptions, cleanupFunc) {
// Create the git server.
gitServer, err := gittestserver.NewTempGitServer()
g.Expect(err).ToNot(HaveOccurred())
err = gitServer.StartHTTP()
g.Expect(err).ToNot(HaveOccurred())
// Initialize a git repo.
repoPath := "bar/test-reponame"
err = gitServer.InitRepo("../testdata/repo1", "main", repoPath)
g.Expect(err).ToNot(HaveOccurred())
u, err := url.Parse(gitServer.HTTPAddress())
g.Expect(err).ToNot(HaveOccurred())
// The request is being forwarded to the local test git server in this handler.
// The certificate used here is valid for both example.com and localhost.
var proxyHandler goproxy.FuncReqHandler = func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
userAgent := req.Header.Get("User-Agent")
if strings.Contains(req.Host, "example.com") && strings.Contains(userAgent, "libgit2") {
*proxyGotRequest = true
req.Host = u.Host
req.URL.Host = req.Host
return req, nil
}
// Reject if it isnt our request.
return req, goproxy.NewResponse(req, goproxy.ContentTypeText, http.StatusForbidden, "")
}
proxy.OnRequest().Do(proxyHandler)
return &git.AuthOptions{
Transport: git.HTTP,
TransportOptionsURL: "http://proxy-test",
}, func() {
os.RemoveAll(gitServer.Root())
gitServer.StopHTTP()
}
},
shortTimeout: false,
wantUsedProxy: true,
wantError: false,
},
{
name: "libgit2_NO_PROXY",
gitImpl: libgit2.Implementation,
url: "https://192.0.2.1/bar/test-reponame",
branch: "main",
setupGitProxy: func(g *WithT, proxy *goproxy.ProxyHttpServer, proxyGotRequest *bool) (*git.AuthOptions, cleanupFunc) {
var proxyHandler goproxy.FuncHttpsHandler = func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) {
// We shouldn't hit the proxy so we just want to check for any interaction, then reject.
*proxyGotRequest = true
return goproxy.RejectConnect, host
}
proxy.OnRequest().HandleConnect(proxyHandler)
return nil, func() {}
},
shortTimeout: false,
wantUsedProxy: false,
wantError: true,
},
}
for _, tt := range cases {
@ -249,10 +320,10 @@ func TestCheckoutStrategyForImplementation_Proxied(t *testing.T) {
defer proxyServer.Close()
// Set the proxy env vars for both HTTP and HTTPS because go-git caches them.
os.Setenv("HTTPS_PROXY", fmt.Sprintf("http://%s", proxyAddr))
os.Setenv("HTTPS_PROXY", fmt.Sprintf("http://smth:else@%s", proxyAddr))
defer os.Unsetenv("HTTPS_PROXY")
os.Setenv("HTTP_PROXY", fmt.Sprintf("http://%s", proxyAddr))
os.Setenv("HTTP_PROXY", fmt.Sprintf("http://smth:else@%s", proxyAddr))
defer os.Unsetenv("HTTP_PROXY")
os.Setenv("NO_PROXY", "*.0.2.1")
@ -282,7 +353,6 @@ func TestCheckoutStrategyForImplementation_Proxied(t *testing.T) {
}
g.Expect(proxyGotRequest).To(Equal(tt.wantUsedProxy))
})
}
}