fluxcd
/
source-controller
mirror of https://github.com/fluxcd/source-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #606 from pjbgf/managed-transport-libgit130 

Experimental managed transport for libgit2 operations
This commit is contained in:
Sunny 2022-03-16 17:05:58 +05:30 committed by GitHub
parent 9bbcd095f0 115040e9ea
commit a3cbe6ee46
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 1207 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
controllers/gitrepository_controller.go
View File

@ -50,6 +50,7 @@ import (
"github.com/fluxcd/source-controller/internal/reconcile/summarize"
"github.com/fluxcd/source-controller/internal/util"
"github.com/fluxcd/source-controller/pkg/git"
"github.com/fluxcd/source-controller/pkg/git/libgit2/managed"
"github.com/fluxcd/source-controller/pkg/git/strategy"
"github.com/fluxcd/source-controller/pkg/sourceignore"
)
@ -369,10 +370,37 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
return sreconcile.ResultEmpty, e
}
repositoryURL := obj.Spec.URL
// managed GIT transport only affects the libgit2 implementation
if managed.Enabled() && obj.Spec.GitImplementation == sourcev1.LibGit2Implementation {
// At present only HTTP connections have the ability to define remote options.
// Although this can be easily extended by ensuring that the fake URL below uses the
// target ssh scheme, and the libgit2/managed/ssh.go pulls that information accordingly.
//
// This is due to the fact the key libgit2 remote callbacks do not take place for HTTP
// whilst most still work for SSH.
if strings.HasPrefix(repositoryURL, "http") {
// Due to the lack of the callback feature, a fake target URL is created to allow
// for the smart sub transport be able to pick the options specific for this
// GitRepository object.
// The URL should use unique information that do not collide in a multi tenant
// deployment.
repositoryURL = fmt.Sprintf("http://%s/%s/%d", obj.Name, obj.UID, obj.Generation)
managed.AddTransportOptions(repositoryURL,
managed.TransportOptions{
TargetURL: obj.Spec.URL,
CABundle: authOpts.CAFile,
})
// We remove the options from memory, to avoid accumulating unused options over time.
defer managed.RemoveTransportOptions(repositoryURL)
}
}
// Checkout HEAD of reference in object
gitCtx, cancel := context.WithTimeout(ctx, obj.Spec.Timeout.Duration)
defer cancel()
c, err := checkoutStrategy.Checkout(gitCtx, dir, obj.Spec.URL, authOpts)
c, err := checkoutStrategy.Checkout(gitCtx, dir, repositoryURL, authOpts)
if err != nil {
e := &serror.Event{
Err: fmt.Errorf("failed to checkout and determine revision: %w", err),

16
hack/ci/e2e.sh
View File

@ -139,3 +139,19 @@ echo "Run large Git repo tests"
kubectl -n source-system apply -f "${ROOT_DIR}/config/testdata/git/large-repo.yaml"
kubectl -n source-system wait gitrepository/large-repo-go-git --for=condition=ready --timeout=2m15s
kubectl -n source-system wait gitrepository/large-repo-libgit2 --for=condition=ready --timeout=2m15s
# Test experimental libgit2 transport. Any tests against the default transport must
# either run before this, or patch the deployment again to disable this, as once enabled
# only the managed transport will be used.
kubectl -n source-system patch deployment source-controller \
--patch '{"spec": {"template": {"spec": {"containers": [{"name": "manager","env": [{"name": "EXPERIMENTAL_GIT_TRANSPORT", "value": "true"}]}]}}}}'
# wait until the patch took effect and the new source-controller is running
sleep 20s
kubectl -n source-system wait --for=condition=ready --timeout=1m -l app=source-controller pod
echo "Re-run large libgit2 repo test with managed transport"
kubectl -n source-system wait gitrepository/large-repo-libgit2 --for=condition=ready --timeout=2m15s
kubectl -n source-system exec deploy/source-controller -- printenv | grep EXPERIMENTAL_GIT_TRANSPORT=true

5
main.go
View File

@ -45,6 +45,7 @@ import (
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
"github.com/fluxcd/source-controller/controllers"
"github.com/fluxcd/source-controller/internal/helm"
"github.com/fluxcd/source-controller/pkg/git/libgit2/managed"
// +kubebuilder:scaffold:imports
)
@ -226,6 +227,10 @@ func main() {
startFileServer(storage.BasePath, storageAddr, setupLog)
}()
if managed.Enabled() {
managed.InitManagedTransport()
}
setupLog.Info("starting manager")
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
setupLog.Error(err, "problem running manager")

9
pkg/git/libgit2/checkout.go
View File

@ -31,6 +31,7 @@ import (
"github.com/fluxcd/pkg/version"
"github.com/fluxcd/source-controller/pkg/git"
"github.com/fluxcd/source-controller/pkg/git/libgit2/managed"
)
// CheckoutStrategyForOptions returns the git.CheckoutStrategy for the given
@ -72,7 +73,7 @@ func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *g
CheckoutBranch: c.Branch,
})
if err != nil {
return nil, fmt.Errorf("unable to clone '%s': %w", url, gitutil.LibGit2Error(err))
return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
defer repo.Free()
head, err := repo.Head()
@ -101,7 +102,7 @@ func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, opts *git.
},
})
if err != nil {
return nil, fmt.Errorf("unable to clone '%s': %w", url, gitutil.LibGit2Error(err))
return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
defer repo.Free()
cc, err := checkoutDetachedDwim(repo, c.Tag)
@ -125,7 +126,7 @@ func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, opts *g
},
})
if err != nil {
return nil, fmt.Errorf("unable to clone '%s': %w", url, gitutil.LibGit2Error(err))
return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
defer repo.Free()
oid, err := git2go.NewOid(c.Commit)
@ -157,7 +158,7 @@ func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, opts *g
},
})
if err != nil {
return nil, fmt.Errorf("unable to clone '%s': %w", url, gitutil.LibGit2Error(err))
return nil, fmt.Errorf("unable to clone '%s': %w", managed.EffectiveURL(url), gitutil.LibGit2Error(err))
}
defer repo.Free()

34
pkg/git/libgit2/managed/flag.go Normal file
View File

@ -0,0 +1,34 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package managed
import (
"os"
"strings"
)
// Enabled defines whether the use of Managed Transport should be enabled.
// This is only affects git operations that uses libgit2 implementation.
//
// True is returned when the environment variable `EXPERIMENTAL_GIT_TRANSPORT`
// is detected with the value of `true` or `1`.
func Enabled() bool {
if v, ok := os.LookupEnv("EXPERIMENTAL_GIT_TRANSPORT"); ok {
return strings.ToLower(v) == "true" || v == "1"
}
return false
}

386
pkg/git/libgit2/managed/http.go Normal file
View File

@ -0,0 +1,386 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
/*
This was inspired and contains part of:
https://github.com/libgit2/git2go/blob/eae00773cce87d5282a8ac7c10b5c1961ee6f9cb/http.go
The MIT License
Copyright (c) 2013 The git2go contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package managed
import (
"bytes"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"sync"
"time"
git2go "github.com/libgit2/git2go/v33"
)
// registerManagedHTTP registers a Go-native implementation of an
// HTTP(S) transport that doesn't rely on any lower-level libraries
// such as OpenSSL.
func registerManagedHTTP() error {
for _, protocol := range []string{"http", "https"} {
_, err := git2go.NewRegisteredSmartTransport(protocol, true, httpSmartSubtransportFactory)
if err != nil {
return fmt.Errorf("failed to register transport for %q: %v", protocol, err)
}
}
return nil
}
func httpSmartSubtransportFactory(remote *git2go.Remote, transport *git2go.Transport) (git2go.SmartSubtransport, error) {
sst := &httpSmartSubtransport{
transport: transport,
}
return sst, nil
}
type httpSmartSubtransport struct {
transport *git2go.Transport
}
func (t *httpSmartSubtransport) Action(targetUrl string, action git2go.SmartServiceAction) (git2go.SmartSubtransportStream, error) {
var proxyFn func(*http.Request) (*url.URL, error)
proxyOpts, err := t.transport.SmartProxyOptions()
if err != nil {
return nil, err
}
switch proxyOpts.Type {
case git2go.ProxyTypeNone:
proxyFn = nil
case git2go.ProxyTypeAuto:
proxyFn = http.ProxyFromEnvironment
case git2go.ProxyTypeSpecified:
parsedUrl, err := url.Parse(proxyOpts.Url)
if err != nil {
return nil, err
}
proxyFn = http.ProxyURL(parsedUrl)
}
httpTransport := &http.Transport{
// Add the proxy to the http transport.
Proxy: proxyFn,
// Set reasonable timeouts to ensure connections are not
// left open in an idle state, nor they hang indefinitely.
//
// These are based on the official go http.DefaultTransport:
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}
client, req, err := createClientRequest(targetUrl, action, httpTransport)
if err != nil {
return nil, err
}
stream := newManagedHttpStream(t, req, client)
if req.Method == "POST" {
stream.recvReply.Add(1)
stream.sendRequestBackground()
}
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
if len(via) >= 3 {
return fmt.Errorf("too many redirects")
}
// golang will change POST to GET in case of redirects.
if len(via) >= 0 && req.Method != via[0].Method {
if via[0].URL.Scheme == "https" && req.URL.Scheme == "http" {
return fmt.Errorf("downgrade from https to http is not allowed: from %q to %q", via[0].URL.String(), req.URL.String())
}
if via[0].URL.Host != req.URL.Host {
return fmt.Errorf("cross hosts redirects are not allowed: from %s to %s", via[0].URL.Host, req.URL.Host)
}
return http.ErrUseLastResponse
}
return nil
}
return stream, nil
}
func createClientRequest(targetUrl string, action git2go.SmartServiceAction, t *http.Transport) (*http.Client, *http.Request, error) {
var req *http.Request
var err error
if t == nil {
return nil, nil, fmt.Errorf("failed to create client: transport cannot be nil")
}
finalUrl := targetUrl
opts, found := transportOptions(targetUrl)
if found {
if opts.TargetURL != "" {
// override target URL only if options are found and a new targetURL
// is provided.
finalUrl = opts.TargetURL
}
// Add any provided certificate to the http transport.
if len(opts.CABundle) > 0 {
cap := x509.NewCertPool()
if ok := cap.AppendCertsFromPEM(opts.CABundle); !ok {
return nil, nil, fmt.Errorf("failed to use certificate from PEM")
}
t.TLSClientConfig = &tls.Config{
RootCAs: cap,
}
}
}
client := &http.Client{
Transport: t,
Timeout: fullHttpClientTimeOut,
}
switch action {
case git2go.SmartServiceActionUploadpackLs:
req, err = http.NewRequest("GET", finalUrl+"/info/refs?service=git-upload-pack", nil)
case git2go.SmartServiceActionUploadpack:
req, err = http.NewRequest("POST", finalUrl+"/git-upload-pack", nil)
if err != nil {
break
}
req.Header.Set("Content-Type", "application/x-git-upload-pack-request")
case git2go.SmartServiceActionReceivepackLs:
req, err = http.NewRequest("GET", finalUrl+"/info/refs?service=git-receive-pack", nil)
case git2go.SmartServiceActionReceivepack:
req, err = http.NewRequest("POST", finalUrl+"/git-receive-pack", nil)
if err != nil {
break
}
req.Header.Set("Content-Type", "application/x-git-receive-pack-request")
default:
err = errors.New("unknown action")
}
if err != nil {
return nil, nil, err
}
req.Header.Set("User-Agent", "git/2.0 (flux-libgit2)")
return client, req, nil
}
func (t *httpSmartSubtransport) Close() error {
return nil
}
func (t *httpSmartSubtransport) Free() {
}
type httpSmartSubtransportStream struct {
owner *httpSmartSubtransport
client *http.Client
req *http.Request
resp *http.Response
reader *io.PipeReader
writer *io.PipeWriter
sentRequest bool
recvReply sync.WaitGroup
httpError error
m sync.RWMutex
targetURL string
}
func newManagedHttpStream(owner *httpSmartSubtransport, req *http.Request, client *http.Client) *httpSmartSubtransportStream {
r, w := io.Pipe()
return &httpSmartSubtransportStream{
owner: owner,
client: client,
req: req,
reader: r,
writer: w,
}
}
func (self *httpSmartSubtransportStream) Read(buf []byte) (int, error) {
if !self.sentRequest {
self.recvReply.Add(1)
if err := self.sendRequest(); err != nil {
return 0, err
}
}
if err := self.writer.Close(); err != nil {
return 0, err
}
self.recvReply.Wait()
self.m.RLock()
err := self.httpError
self.m.RUnlock()
if err != nil {
return 0, self.httpError
}
return self.resp.Body.Read(buf)
}
func (self *httpSmartSubtransportStream) Write(buf []byte) (int, error) {
self.m.RLock()
err := self.httpError
self.m.RUnlock()
if err != nil {
return 0, self.httpError
}
return self.writer.Write(buf)
}
func (self *httpSmartSubtransportStream) Free() {
if self.resp != nil {
self.resp.Body.Close()
}
}
func (self *httpSmartSubtransportStream) sendRequestBackground() {
go func() {
err := self.sendRequest()
self.m.Lock()
self.httpError = err
self.m.Unlock()
}()
self.sentRequest = true
}
func (self *httpSmartSubtransportStream) sendRequest() error {
defer self.recvReply.Done()
self.resp = nil
var resp *http.Response
var err error
var userName string
var password string
// Obtain the credentials and use them if available.
cred, err := self.owner.transport.SmartCredentials("", git2go.CredentialTypeUserpassPlaintext)
if err != nil {
// Passthrough error indicates that no credentials were provided.
// Continue without credentials.
if err.Error() != git2go.ErrorCodePassthrough.String() {
return err
}
}
if cred != nil {
defer cred.Free()
userName, password, err = cred.GetUserpassPlaintext()
if err != nil {
return err
}
}
var content []byte
for {
req := &http.Request{
Method: self.req.Method,
URL: self.req.URL,
Header: self.req.Header,
}
if req.Method == "POST" {
if len(content) == 0 {
// a copy of the request body needs to be saved so
// it can be reused in case of redirects.
if content, err = io.ReadAll(self.reader); err != nil {
return err
}
}
req.Body = io.NopCloser(bytes.NewReader(content))
req.ContentLength = -1
}
req.SetBasicAuth(userName, password)
resp, err = self.client.Do(req)
if err != nil {
return err
}
// GET requests will be automatically redirected.
// POST require the new destination, and also the body content.
if req.Method == "POST" && resp.StatusCode >= 301 && resp.StatusCode <= 308 {
// The next try will go against the new destination
self.req.URL, err = resp.Location()
if err != nil {
return err
}
continue
}
if resp.StatusCode == http.StatusOK {
break
}
io.Copy(io.Discard, resp.Body)
defer resp.Body.Close()
return fmt.Errorf("Unhandled HTTP error %s", resp.Status)
}
self.resp = resp
self.sentRequest = true
return nil
}

61
pkg/git/libgit2/managed/init.go Normal file
View File

@ -0,0 +1,61 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package managed
import (
"sync"
"time"
)
var (
once sync.Once
// sshConnectionTimeOut defines the timeout used for when
// creating ssh.ClientConfig, which translates in the timeout
// for stablishing the SSH TCP connections.
sshConnectionTimeOut time.Duration = 30 * time.Second
// fullHttpClientTimeOut defines the maximum amount of
// time a http client may take before timing out,
// regardless of the current operation (i.e. connection,
// handshake, put/get).
fullHttpClientTimeOut time.Duration = 10 * time.Minute
)
// InitManagedTransport initialises HTTP(S) and SSH managed transport
// for git2go, and therefore only impact git operations using the
// libgit2 implementation.
//
// This must run after git2go.init takes place, hence this is not executed
// within a init().
// Regardless of the state in libgit2/git2go, this will replace the
// built-in transports.
//
// This function will only register managed transports once, subsequent calls
// leads to no-op.
func InitManagedTransport() error {
var err error
once.Do(func() {
if err = registerManagedHTTP(); err != nil {
return
}
err = registerManagedSSH()
})
return err
}

327
pkg/git/libgit2/managed/managed_test.go Normal file
View File

@ -0,0 +1,327 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package managed
import (
"fmt"
"net/http"
"os"
"path/filepath"
"reflect"
"testing"
"github.com/fluxcd/pkg/gittestserver"
"github.com/fluxcd/pkg/ssh"
"github.com/fluxcd/source-controller/pkg/git"
git2go "github.com/libgit2/git2go/v33"
. "github.com/onsi/gomega"
"gotest.tools/assert"
)
func TestHttpAction_CreateClientRequest(t *testing.T) {
tests := []struct {
description string
url string
expectedUrl string
expectedMethod string
action git2go.SmartServiceAction
opts *TransportOptions
transport *http.Transport
wantedErr error
}{
{
description: "Uploadpack: no changes when no options found",
url: "https://sometarget/abc",
expectedUrl: "https://sometarget/abc/git-upload-pack",
expectedMethod: "POST",
action: git2go.SmartServiceActionUploadpack,
transport: &http.Transport{},
opts: nil,
wantedErr: nil,
},
{
description: "UploadpackLs: no changes when no options found",
url: "https://sometarget/abc",
expectedUrl: "https://sometarget/abc/info/refs?service=git-upload-pack",
expectedMethod: "GET",
action: git2go.SmartServiceActionUploadpackLs,
transport: &http.Transport{},
opts: nil,
wantedErr: nil,
},
{
description: "Receivepack: no changes when no options found",
url: "https://sometarget/abc",
expectedUrl: "https://sometarget/abc/git-receive-pack",
expectedMethod: "POST",
action: git2go.SmartServiceActionReceivepack,
transport: &http.Transport{},
opts: nil,
wantedErr: nil,
},
{
description: "ReceivepackLs: no changes when no options found",
url: "https://sometarget/abc",
expectedUrl: "https://sometarget/abc/info/refs?service=git-receive-pack",
expectedMethod: "GET",
action: git2go.SmartServiceActionReceivepackLs,
transport: &http.Transport{},
opts: nil,
wantedErr: nil,
},
{
description: "override URL via options",
url: "https://initial-target/abc",
expectedUrl: "https://final-target/git-upload-pack",
expectedMethod: "POST",
action: git2go.SmartServiceActionUploadpack,
transport: &http.Transport{},
opts: &TransportOptions{
TargetURL: "https://final-target",
},
wantedErr: nil,
},
{
description: "error when no http.transport provided",
url: "https://initial-target/abc",
expectedUrl: "",
expectedMethod: "",
action: git2go.SmartServiceActionUploadpack,
transport: nil,
opts: nil,
wantedErr: fmt.Errorf("failed to create client: transport cannot be nil"),
},
}
for _, tt := range tests {
if tt.opts != nil {
AddTransportOptions(tt.url, *tt.opts)
}
_, req, err := createClientRequest(tt.url, tt.action, tt.transport)
if tt.wantedErr != nil {
if tt.wantedErr.Error() != err.Error() {
t.Errorf("%s: wanted: %v got: %v", tt.description, tt.wantedErr, err)
}
} else {
assert.Equal(t, req.URL.String(), tt.expectedUrl)
assert.Equal(t, req.Method, tt.expectedMethod)
}
if tt.opts != nil {
RemoveTransportOptions(tt.url)
}
}
}
func TestOptions(t *testing.T) {
tests := []struct {
description string
registerOpts bool
url string
opts TransportOptions
expectOpts bool
expectedOpts *TransportOptions
}{
{
description: "return registered option",
registerOpts: true,
url: "https://target/?123",
opts: TransportOptions{},
expectOpts: true,
expectedOpts: &TransportOptions{},
},
{
description: "match registered options",
registerOpts: true,
url: "https://target/?876",
opts: TransportOptions{
TargetURL: "https://new-target/321",
CABundle: []byte{123, 213, 132},
},
expectOpts: true,
expectedOpts: &TransportOptions{
TargetURL: "https://new-target/321",
CABundle: []byte{123, 213, 132},
},
},
{
description: "ignore when options not registered",
registerOpts: false,
url: "",
opts: TransportOptions{},
expectOpts: false,
expectedOpts: nil,
},
}
for _, tt := range tests {
if tt.registerOpts {
AddTransportOptions(tt.url, tt.opts)
}
opts, found := transportOptions(tt.url)
if tt.expectOpts != found {
t.Errorf("%s: wanted %v got %v", tt.description, tt.expectOpts, found)
}
if tt.expectOpts {
if reflect.DeepEqual(opts, *tt.expectedOpts) {
t.Errorf("%s: wanted %v got %v", tt.description, *tt.expectedOpts, opts)
}
}
if tt.registerOpts {
RemoveTransportOptions(tt.url)
}
if _, found = transportOptions(tt.url); found {
t.Errorf("%s: option for %s was not removed", tt.description, tt.url)
}
}
}
func TestFlagStatus(t *testing.T) {
if Enabled() {
t.Errorf("experimental transport should not be enabled by default")
}
os.Setenv("EXPERIMENTAL_GIT_TRANSPORT", "true")
if !Enabled() {
t.Errorf("experimental transport should be enabled when env EXPERIMENTAL_GIT_TRANSPORT=true")
}
os.Setenv("EXPERIMENTAL_GIT_TRANSPORT", "1")
if !Enabled() {
t.Errorf("experimental transport should be enabled when env EXPERIMENTAL_GIT_TRANSPORT=1")
}
os.Setenv("EXPERIMENTAL_GIT_TRANSPORT", "somethingelse")
if Enabled() {
t.Errorf("experimental transport should be enabled only when env EXPERIMENTAL_GIT_TRANSPORT is 1 or true but was enabled for 'somethingelse'")
}
os.Unsetenv("EXPERIMENTAL_GIT_TRANSPORT")
if Enabled() {
t.Errorf("experimental transport should not be enabled when env EXPERIMENTAL_GIT_TRANSPORT is not present")
}
}
func TestManagedTransport_E2E(t *testing.T) {
g := NewWithT(t)
server, err := gittestserver.NewTempGitServer()
g.Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(server.Root())
user := "test-user"
pasword := "test-pswd"
server.Auth(user, pasword)
server.KeyDir(filepath.Join(server.Root(), "keys"))
err = server.ListenSSH()
g.Expect(err).ToNot(HaveOccurred())
err = server.StartHTTP()
g.Expect(err).ToNot(HaveOccurred())
defer server.StopHTTP()
go func() {
server.StartSSH()
}()
defer server.StopSSH()
// Force managed transport to be enabled
InitManagedTransport()
repoPath := "test.git"
err = server.InitRepo("../testdata/git/repo", git.DefaultBranch, repoPath)
g.Expect(err).ToNot(HaveOccurred())
tmpDir, _ := os.MkdirTemp("", "test")
defer os.RemoveAll(tmpDir)
// Test HTTP transport
// Use a fake-url and force it to be overriden by the smart transport.
// This was the way found to ensure that the built-in transport was not used.
httpAddress := "http://fake-url"
AddTransportOptions(httpAddress, TransportOptions{
TargetURL: server.HTTPAddress() + "/" + repoPath,
})
repo, err := git2go.Clone(httpAddress, tmpDir, &git2go.CloneOptions{
FetchOptions: git2go.FetchOptions{
RemoteCallbacks: git2go.RemoteCallbacks{
CredentialsCallback: func(url, username_from_url string, allowed_types git2go.CredentialType) (*git2go.Credential, error) {
return git2go.NewCredentialUserpassPlaintext(user, pasword)
},
},
},
CheckoutOptions: git2go.CheckoutOptions{
Strategy: git2go.CheckoutForce,
},
})
g.Expect(err).ToNot(HaveOccurred())
repo.Free()
tmpDir2, _ := os.MkdirTemp("", "test")
defer os.RemoveAll(tmpDir2)
kp, err := ssh.NewEd25519Generator().Generate()
g.Expect(err).ToNot(HaveOccurred())
// Test SSH transport
sshAddress := server.SSHAddress() + "/" + repoPath
repo, err = git2go.Clone(sshAddress, tmpDir2, &git2go.CloneOptions{
FetchOptions: git2go.FetchOptions{
RemoteCallbacks: git2go.RemoteCallbacks{
CredentialsCallback: func(url, username_from_url string, allowed_types git2go.CredentialType) (*git2go.Credential, error) {
return git2go.NewCredentialSSHKeyFromMemory("git", "", string(kp.PrivateKey), "")
},
},
},
CheckoutOptions: git2go.CheckoutOptions{
Strategy: git2go.CheckoutForce,
},
})
g.Expect(err).ToNot(HaveOccurred())
repo.Free()
}
func TestManagedTransport_HandleRedirect(t *testing.T) {
g := NewWithT(t)
tmpDir, _ := os.MkdirTemp("", "test")
defer os.RemoveAll(tmpDir)
// Force managed transport to be enabled
InitManagedTransport()
// GitHub will cause a 301 and redirect to https
repo, err := git2go.Clone("http://github.com/stefanprodan/podinfo", tmpDir, &git2go.CloneOptions{
FetchOptions: git2go.FetchOptions{},
CheckoutOptions: git2go.CheckoutOptions{
Strategy: git2go.CheckoutForce,
},
})
g.Expect(err).ToNot(HaveOccurred())
repo.Free()
}

75
pkg/git/libgit2/managed/options.go Normal file
View File

@ -0,0 +1,75 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package managed
import (
"sync"
)
// TransportOptions represents options to be applied at transport-level
// at request time.
type TransportOptions struct {
TargetURL string
CABundle []byte
}
var (
transportOpts = make(map[string]TransportOptions, 0)
m sync.RWMutex
)
func AddTransportOptions(targetUrl string, opts TransportOptions) {
m.Lock()
transportOpts[targetUrl] = opts
m.Unlock()
}
func RemoveTransportOptions(targetUrl string) {
m.Lock()
delete(transportOpts, targetUrl)
m.Unlock()
}
func transportOptions(targetUrl string) (*TransportOptions, bool) {
m.RLock()
opts, found := transportOpts[targetUrl]
m.RUnlock()
if found {
return &opts, true
}
return nil, false
}
// EffectiveURL returns the effective URL for requests.
//
// Given that TransportOptions can allow for the target URL to be overriden
// this returns the same input if Managed Transport is disabled or if no TargetURL
// is set on TransportOptions.
func EffectiveURL(targetUrl string) string {
if !Enabled() {
return targetUrl
}
if opts, found := transportOptions(targetUrl); found {
if opts.TargetURL != "" {
return opts.TargetURL
}
}
return targetUrl
}

256
pkg/git/libgit2/managed/ssh.go Normal file
View File

@ -0,0 +1,256 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
/*
This was inspired and contains part of:
https://github.com/libgit2/git2go/blob/eae00773cce87d5282a8ac7c10b5c1961ee6f9cb/ssh.go
The MIT License
Copyright (c) 2013 The git2go contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package managed
import (
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"fmt"
"io"
"net"
"net/url"
"runtime"
"strings"
"golang.org/x/crypto/ssh"
git2go "github.com/libgit2/git2go/v33"
)
// registerManagedSSH registers a Go-native implementation of
// SSH transport that doesn't rely on any lower-level libraries
// such as libssh2.
func registerManagedSSH() error {
for _, protocol := range []string{"ssh", "ssh+git", "git+ssh"} {
_, err := git2go.NewRegisteredSmartTransport(protocol, false, sshSmartSubtransportFactory)
if err != nil {
return fmt.Errorf("failed to register transport for %q: %v", protocol, err)
}
}
return nil
}
func sshSmartSubtransportFactory(remote *git2go.Remote, transport *git2go.Transport) (git2go.SmartSubtransport, error) {
return &sshSmartSubtransport{
transport: transport,
}, nil
}
type sshSmartSubtransport struct {
transport *git2go.Transport
lastAction git2go.SmartServiceAction
client *ssh.Client
session *ssh.Session
stdin io.WriteCloser
stdout io.Reader
currentStream *sshSmartSubtransportStream
}
func (t *sshSmartSubtransport) Action(urlString string, action git2go.SmartServiceAction) (git2go.SmartSubtransportStream, error) {
runtime.LockOSThread()
defer runtime.UnlockOSThread()
u, err := url.Parse(urlString)
if err != nil {
return nil, err
}
// Escape \ and '.
uPath := strings.Replace(u.Path, `\`, `\\`, -1)
uPath = strings.Replace(uPath, `'`, `\'`, -1)
// TODO: Add percentage decode similar to libgit2.
// Refer: https://github.com/libgit2/libgit2/blob/358a60e1b46000ea99ef10b4dd709e92f75ff74b/src/str.c#L455-L481
var cmd string
switch action {
case git2go.SmartServiceActionUploadpackLs, git2go.SmartServiceActionUploadpack:
if t.currentStream != nil {
if t.lastAction == git2go.SmartServiceActionUploadpackLs {
return t.currentStream, nil
}
t.Close()
}
cmd = fmt.Sprintf("git-upload-pack '%s'", uPath)
case git2go.SmartServiceActionReceivepackLs, git2go.SmartServiceActionReceivepack:
if t.currentStream != nil {
if t.lastAction == git2go.SmartServiceActionReceivepackLs {
return t.currentStream, nil
}
t.Close()
}
cmd = fmt.Sprintf("git-receive-pack '%s'", uPath)
default:
return nil, fmt.Errorf("unexpected action: %v", action)
}
cred, err := t.transport.SmartCredentials("", git2go.CredentialTypeSSHKey|git2go.CredentialTypeSSHMemory)
if err != nil {
return nil, err
}
defer cred.Free()
sshConfig, err := getSSHConfigFromCredential(cred)
if err != nil {
return nil, err
}
sshConfig.HostKeyCallback = func(hostname string, remote net.Addr, key ssh.PublicKey) error {
marshaledKey := key.Marshal()
cert := &git2go.Certificate{
Kind: git2go.CertificateHostkey,
Hostkey: git2go.HostkeyCertificate{
Kind: git2go.HostkeySHA1 | git2go.HostkeyMD5 | git2go.HostkeySHA256 | git2go.HostkeyRaw,
HashMD5: md5.Sum(marshaledKey),
HashSHA1: sha1.Sum(marshaledKey),
HashSHA256: sha256.Sum256(marshaledKey),
Hostkey: marshaledKey,
SSHPublicKey: key,
},
}
return t.transport.SmartCertificateCheck(cert, true, hostname)
}
var addr string
if u.Port() != "" {
addr = fmt.Sprintf("%s:%s", u.Hostname(), u.Port())
} else {
addr = fmt.Sprintf("%s:22", u.Hostname())
}
t.client, err = ssh.Dial("tcp", addr, sshConfig)
if err != nil {
return nil, err
}
t.session, err = t.client.NewSession()
if err != nil {
return nil, err
}
t.stdin, err = t.session.StdinPipe()
if err != nil {
return nil, err
}
t.stdout, err = t.session.StdoutPipe()
if err != nil {
return nil, err
}
if err := t.session.Start(cmd); err != nil {
return nil, err
}
t.lastAction = action
t.currentStream = &sshSmartSubtransportStream{
owner: t,
}
return t.currentStream, nil
}
func (t *sshSmartSubtransport) Close() error {
t.currentStream = nil
if t.client != nil {
t.stdin.Close()
t.session.Wait()
t.session.Close()
t.client = nil
}
return nil
}
func (t *sshSmartSubtransport) Free() {
}
type sshSmartSubtransportStream struct {
owner *sshSmartSubtransport
}
func (stream *sshSmartSubtransportStream) Read(buf []byte) (int, error) {
return stream.owner.stdout.Read(buf)
}
func (stream *sshSmartSubtransportStream) Write(buf []byte) (int, error) {
return stream.owner.stdin.Write(buf)
}
func (stream *sshSmartSubtransportStream) Free() {
}
func getSSHConfigFromCredential(cred *git2go.Credential) (*ssh.ClientConfig, error) {
username, _, privatekey, passphrase, err := cred.GetSSHKey()
if err != nil {
return nil, err
}
var pemBytes []byte
if cred.Type() == git2go.CredentialTypeSSHMemory {
pemBytes = []byte(privatekey)
} else {
return nil, fmt.Errorf("file based SSH credential is not supported")
}
var key ssh.Signer
if passphrase != "" {
key, err = ssh.ParsePrivateKeyWithPassphrase(pemBytes, []byte(passphrase))
} else {
key, err = ssh.ParsePrivateKey(pemBytes)
}
if err != nil {
return nil, err
}
return &ssh.ClientConfig{
User: username,
Auth: []ssh.AuthMethod{ssh.PublicKeys(key)},
Timeout: sshConnectionTimeOut,
}, nil
}

13
pkg/git/libgit2/transport.go
View File

@ -36,6 +36,7 @@ import (
"golang.org/x/crypto/ssh/knownhosts"
"github.com/fluxcd/source-controller/pkg/git"
"github.com/fluxcd/source-controller/pkg/git/libgit2/managed"
)
var (
@ -112,6 +113,18 @@ func pushTransferProgressCallback(ctx context.Context) git2go.PushTransferProgre
func credentialsCallback(opts *git.AuthOptions) git2go.CredentialsCallback {
return func(url string, username string, allowedTypes git2go.CredentialType) (*git2go.Credential, error) {
if allowedTypes&(git2go.CredentialTypeSSHKey|git2go.CredentialTypeSSHCustom|git2go.CredentialTypeSSHMemory) != 0 {
if managed.Enabled() {
// CredentialTypeSSHMemory requires libgit2 to be built using libssh2.
// When using managed transport (handled in go instead of libgit2),
// there may be ways to remove such requirement, thefore decreasing the
// need of libz, libssh2 and OpenSSL but further investigation is required
// once Managed Transport is no longer experimental.
//
// CredentialSSHKeyFromMemory is currently required for SSH key access
// when managed transport is enabled.
return git2go.NewCredentialSSHKeyFromMemory(opts.Username, "", string(opts.Identity), opts.Password)
}
var (
signer ssh.Signer
err error