Merge pull request #720 from pjbgf/git-tests

libgit2: Add support for hashed known_hosts

controllers/gitrepository_controller_test.go

@@ -478,7 +478,7 @@ func TestGitRepositoryReconciler_reconcileSource_authStrategy(t *testing.T) {
 					u, err := url.Parse(obj.Spec.URL)
 					g.Expect(err).NotTo(HaveOccurred())
 					g.Expect(u.Host).ToNot(BeEmpty())
-					knownHosts, err := ssh.ScanHostKey(u.Host, timeout, git.HostKeyAlgos)
+					knownHosts, err := ssh.ScanHostKey(u.Host, timeout, git.HostKeyAlgos, false)
 					g.Expect(err).NotTo(HaveOccurred())
 					secret.Data["known_hosts"] = knownHosts
 				}

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/fluxcd/pkg/helmtestserver v0.7.2
 	github.com/fluxcd/pkg/lockedfile v0.1.0
 	github.com/fluxcd/pkg/runtime v0.15.1
-	github.com/fluxcd/pkg/ssh v0.3.3
+	github.com/fluxcd/pkg/ssh v0.3.4
 	github.com/fluxcd/pkg/testserver v0.2.0
 	github.com/fluxcd/pkg/untar v0.1.0
 	github.com/fluxcd/pkg/version v0.1.0

go.sum

@@ -366,8 +366,8 @@ github.com/fluxcd/pkg/lockedfile v0.1.0/go.mod h1:EJLan8t9MiOcgTs8+puDjbE6I/KAfH
 github.com/fluxcd/pkg/runtime v0.13.0-rc.6/go.mod h1:4oKUO19TeudXrnCRnxCfMSS7EQTYpYlgfXwlQuDJ/Eg=
 github.com/fluxcd/pkg/runtime v0.15.1 h1:PKooYqlZM+KLhnNz10sQnBH0AHllS40PIDHtiRH/BGU=
 github.com/fluxcd/pkg/runtime v0.15.1/go.mod h1:TPAoOEgUFG60FXBA4ID41uaPldxuXCEI4jt3qfd5i5Q=
-github.com/fluxcd/pkg/ssh v0.3.3 h1:/tc7W7LO1VoVUI5jB+p9ZHCA+iQaXTkaSCDZJsxcZ9k=
+github.com/fluxcd/pkg/ssh v0.3.4 h1:Ko+MUNiiQG3evyoMO19iRk7d4X0VJ6w6+GEeVQ1jLC0=
-github.com/fluxcd/pkg/ssh v0.3.3/go.mod h1:+bKhuv0/pJy3HZwkK54Shz68sNv1uf5aI6wtPaEHaYk=
+github.com/fluxcd/pkg/ssh v0.3.4/go.mod h1:KGgOUOy1uI6RC6+qxIBLvP1AeOOs/nLB25Ca6TZMIXE=
 github.com/fluxcd/pkg/testserver v0.2.0 h1:Mj0TapmKaywI6Fi5wvt1LAZpakUHmtzWQpJNKQ0Krt4=
 github.com/fluxcd/pkg/testserver v0.2.0/go.mod h1:bgjjydkXsZTeFzjz9Cr4heGANr41uTB1Aj1Q5qzuYVk=
 github.com/fluxcd/pkg/untar v0.1.0 h1:k97V/xV5hFrAkIkVPuv5AVhyxh1ZzzAKba/lbDfGo6o=

pkg/git/gogit/checkout_test.go

@@ -461,7 +461,7 @@ func Test_KeyTypes(t *testing.T) {
 	g.Expect(err).NotTo(HaveOccurred())
 	g.Expect(u.Host).ToNot(BeEmpty())
 
-	knownHosts, err := ssh.ScanHostKey(u.Host, timeout, git.HostKeyAlgos)
+	knownHosts, err := ssh.ScanHostKey(u.Host, timeout, git.HostKeyAlgos, false)
 	g.Expect(err).ToNot(HaveOccurred())
 
 	for _, tt := range tests {
@@ -600,7 +600,7 @@ func Test_KeyExchangeAlgos(t *testing.T) {
 			g.Expect(err).NotTo(HaveOccurred())
 			g.Expect(u.Host).ToNot(BeEmpty())
 
-			knownHosts, err := ssh.ScanHostKey(u.Host, timeout, git.HostKeyAlgos)
+			knownHosts, err := ssh.ScanHostKey(u.Host, timeout, git.HostKeyAlgos, false)
 			g.Expect(err).ToNot(HaveOccurred())
 
 			// No authentication is required for this test, but it is
@@ -644,6 +644,7 @@ func TestHostKeyAlgos(t *testing.T) {
 		name               string
 		keyType            ssh.KeyPairType
 		ClientHostKeyAlgos []string
+		hashHostNames      bool
 	}{
 		{
 			name:               "support for hostkey: ssh-rsa",
@@ -680,6 +681,48 @@ func TestHostKeyAlgos(t *testing.T) {
 			keyType:            ssh.ED25519,
 			ClientHostKeyAlgos: []string{"ssh-ed25519"},
 		},
+		{
+			name:               "support for hostkey: ssh-rsa with hashed host names",
+			keyType:            ssh.RSA_4096,
+			ClientHostKeyAlgos: []string{"ssh-rsa"},
+			hashHostNames:      true,
+		},
+		{
+			name:               "support for hostkey: rsa-sha2-256 with hashed host names",
+			keyType:            ssh.RSA_4096,
+			ClientHostKeyAlgos: []string{"rsa-sha2-256"},
+			hashHostNames:      true,
+		},
+		{
+			name:               "support for hostkey: rsa-sha2-512 with hashed host names",
+			keyType:            ssh.RSA_4096,
+			ClientHostKeyAlgos: []string{"rsa-sha2-512"},
+			hashHostNames:      true,
+		},
+		{
+			name:               "support for hostkey: ecdsa-sha2-nistp256 with hashed host names",
+			keyType:            ssh.ECDSA_P256,
+			ClientHostKeyAlgos: []string{"ecdsa-sha2-nistp256"},
+			hashHostNames:      true,
+		},
+		{
+			name:               "support for hostkey: ecdsa-sha2-nistp384 with hashed host names",
+			keyType:            ssh.ECDSA_P384,
+			ClientHostKeyAlgos: []string{"ecdsa-sha2-nistp384"},
+			hashHostNames:      true,
+		},
+		{
+			name:               "support for hostkey: ecdsa-sha2-nistp521 with hashed host names",
+			keyType:            ssh.ECDSA_P521,
+			ClientHostKeyAlgos: []string{"ecdsa-sha2-nistp521"},
+			hashHostNames:      true,
+		},
+		{
+			name:               "support for hostkey: ssh-ed25519 with hashed host names",
+			keyType:            ssh.ED25519,
+			ClientHostKeyAlgos: []string{"ssh-ed25519"},
+			hashHostNames:      true,
+		},
 	}
 
 	for _, tt := range tests {
@@ -727,7 +770,7 @@ func TestHostKeyAlgos(t *testing.T) {
 			g.Expect(err).NotTo(HaveOccurred())
 			g.Expect(u.Host).ToNot(BeEmpty())
 
-			knownHosts, err := ssh.ScanHostKey(u.Host, timeout, git.HostKeyAlgos)
+			knownHosts, err := ssh.ScanHostKey(u.Host, timeout, git.HostKeyAlgos, tt.hashHostNames)
 			g.Expect(err).ToNot(HaveOccurred())
 
 			// No authentication is required for this test, but it is

pkg/git/libgit2/checkout_test.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"net/url"
 	"os"
 	"path/filepath"
 	"testing"
@@ -28,12 +27,6 @@ import (
 
 	git2go "github.com/libgit2/git2go/v33"
 	. "github.com/onsi/gomega"
-	corev1 "k8s.io/api/core/v1"
-
-	"github.com/fluxcd/pkg/gittestserver"
-	"github.com/fluxcd/pkg/ssh"
-
-	"github.com/fluxcd/source-controller/pkg/git"
 )
 
 func TestCheckoutBranch_Checkout(t *testing.T) {
@@ -517,67 +510,3 @@ func mockSignature(time time.Time) *git2go.Signature {
 		When:  time,
 	}
 }
-
-// This test is specifically to detect regression in libgit2's ED25519 key
-// support for client authentication.
-// Refer: https://github.com/fluxcd/source-controller/issues/399
-func TestCheckout_ED25519(t *testing.T) {
-	g := NewWithT(t)
-	timeout := 5 * time.Second
-
-	// Create a git test server.
-	server, err := gittestserver.NewTempGitServer()
-	g.Expect(err).ToNot(HaveOccurred())
-	defer os.RemoveAll(server.Root())
-	server.Auth("test-user", "test-pswd")
-	server.AutoCreate()
-
-	server.KeyDir(filepath.Join(server.Root(), "keys"))
-	g.Expect(server.ListenSSH()).To(Succeed())
-
-	go func() {
-		server.StartSSH()
-	}()
-	defer server.StopSSH()
-
-	repoPath := "test.git"
-
-	err = server.InitRepo(testRepositoryPath, git.DefaultBranch, repoPath)
-	g.Expect(err).NotTo(HaveOccurred())
-
-	sshURL := server.SSHAddress()
-	repoURL := sshURL + "/" + repoPath
-
-	// Fetch host key.
-	u, err := url.Parse(sshURL)
-	g.Expect(err).NotTo(HaveOccurred())
-	g.Expect(u.Host).ToNot(BeEmpty())
-	knownHosts, err := ssh.ScanHostKey(u.Host, timeout, git.HostKeyAlgos)
-	g.Expect(err).ToNot(HaveOccurred())
-
-	kp, err := ssh.NewEd25519Generator().Generate()
-	g.Expect(err).ToNot(HaveOccurred())
-
-	secret := corev1.Secret{
-		Data: map[string][]byte{
-			"identity":    kp.PrivateKey,
-			"known_hosts": knownHosts,
-		},
-	}
-
-	authOpts, err := git.AuthOptionsFromSecret(repoURL, &secret)
-	g.Expect(err).ToNot(HaveOccurred())
-
-	// Prepare for checkout.
-	branchCheckoutStrat := &CheckoutBranch{Branch: git.DefaultBranch}
-	tmpDir := t.TempDir()
-
-	ctx, cancel := context.WithTimeout(context.TODO(), timeout)
-	defer cancel()
-
-	// Checkout the repo.
-	// This should always fail because the generated key above isn't present in
-	// the git server.
-	_, err = branchCheckoutStrat.Checkout(ctx, tmpDir, repoURL, authOpts)
-	g.Expect(err).ToNot(HaveOccurred())
-}

pkg/git/libgit2/managed_test.go

@@ -96,7 +96,7 @@ func Test_ManagedSSH_KeyTypes(t *testing.T) {
 	g.Expect(err).NotTo(HaveOccurred())
 	g.Expect(u.Host).ToNot(BeEmpty())
 
-	knownHosts, err := ssh.ScanHostKey(u.Host, timeout, git.HostKeyAlgos)
+	knownHosts, err := ssh.ScanHostKey(u.Host, timeout, git.HostKeyAlgos, false)
 	g.Expect(err).ToNot(HaveOccurred())
 
 	for _, tt := range tests {
@@ -238,7 +238,7 @@ func Test_ManagedSSH_KeyExchangeAlgos(t *testing.T) {
 			g.Expect(err).NotTo(HaveOccurred())
 			g.Expect(u.Host).ToNot(BeEmpty())
 
-			knownHosts, err := ssh.ScanHostKey(u.Host, timeout, git.HostKeyAlgos)
+			knownHosts, err := ssh.ScanHostKey(u.Host, timeout, git.HostKeyAlgos, false)
 			g.Expect(err).ToNot(HaveOccurred())
 
 			// No authentication is required for this test, but it is
@@ -282,6 +282,7 @@ func Test_ManagedSSH_HostKeyAlgos(t *testing.T) {
 		name               string
 		keyType            ssh.KeyPairType
 		ClientHostKeyAlgos []string
+		hashHostNames      bool
 	}{
 		{
 			name:               "support for hostkey: ssh-rsa",
@@ -318,6 +319,48 @@ func Test_ManagedSSH_HostKeyAlgos(t *testing.T) {
 			keyType:            ssh.ED25519,
 			ClientHostKeyAlgos: []string{"ssh-ed25519"},
 		},
+		{
+			name:               "support for hostkey: ssh-rsa with hashed host names",
+			keyType:            ssh.RSA_4096,
+			ClientHostKeyAlgos: []string{"ssh-rsa"},
+			hashHostNames:      true,
+		},
+		{
+			name:               "support for hostkey: rsa-sha2-256 with hashed host names",
+			keyType:            ssh.RSA_4096,
+			ClientHostKeyAlgos: []string{"rsa-sha2-256"},
+			hashHostNames:      true,
+		},
+		{
+			name:               "support for hostkey: rsa-sha2-512 with hashed host names",
+			keyType:            ssh.RSA_4096,
+			ClientHostKeyAlgos: []string{"rsa-sha2-512"},
+			hashHostNames:      true,
+		},
+		{
+			name:               "support for hostkey: ecdsa-sha2-nistp256 with hashed host names",
+			keyType:            ssh.ECDSA_P256,
+			ClientHostKeyAlgos: []string{"ecdsa-sha2-nistp256"},
+			hashHostNames:      true,
+		},
+		{
+			name:               "support for hostkey: ecdsa-sha2-nistp384 with hashed host names",
+			keyType:            ssh.ECDSA_P384,
+			ClientHostKeyAlgos: []string{"ecdsa-sha2-nistp384"},
+			hashHostNames:      true,
+		},
+		{
+			name:               "support for hostkey: ecdsa-sha2-nistp521 with hashed host names",
+			keyType:            ssh.ECDSA_P521,
+			ClientHostKeyAlgos: []string{"ecdsa-sha2-nistp521"},
+			hashHostNames:      true,
+		},
+		{
+			name:               "support for hostkey: ssh-ed25519 with hashed host names",
+			keyType:            ssh.ED25519,
+			ClientHostKeyAlgos: []string{"ssh-ed25519"},
+			hashHostNames:      true,
+		},
 	}
 
 	for _, tt := range tests {
@@ -368,7 +411,7 @@ func Test_ManagedSSH_HostKeyAlgos(t *testing.T) {
 			g.Expect(err).NotTo(HaveOccurred())
 			g.Expect(u.Host).ToNot(BeEmpty())
 
-			knownHosts, err := ssh.ScanHostKey(u.Host, timeout, tt.ClientHostKeyAlgos)
+			knownHosts, err := ssh.ScanHostKey(u.Host, timeout, tt.ClientHostKeyAlgos, tt.hashHostNames)
 			g.Expect(err).ToNot(HaveOccurred())
 
 			// No authentication is required for this test, but it is

pkg/git/libgit2/transport.go

@@ -20,10 +20,12 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"crypto/hmac"
 	"crypto/md5"
 	"crypto/sha1"
 	"crypto/sha256"
 	"crypto/x509"
+	"encoding/base64"
 	"fmt"
 	"hash"
 	"io"
@@ -288,10 +290,54 @@ func (k knownKey) matches(host string, hostkey git2go.HostkeyCertificate) bool {
 }
 
 func containsHost(hosts []string, host string) bool {
-	for _, h := range hosts {
+	for _, kh := range hosts {
-		if h == host {
+		// hashed host must start with a pipe
+		if kh[0] == '|' {
+			match, _ := MatchHashedHost(kh, host)
+			if match {
+				return true
+			}
+
+		} else if kh == host { // unhashed host check
 			return true
 		}
 	}
 	return false
 }
+
+// MatchHashedHost tries to match a hashed known host (kh) to
+// host.
+//
+// Note that host is not hashed, but it is rather hashed during
+// the matching process using the same salt used when hashing
+// the known host.
+func MatchHashedHost(kh, host string) (bool, error) {
+	if kh == "" || kh[0] != '|' {
+		return false, fmt.Errorf("hashed known host must begin with '|': '%s'", kh)
+	}
+
+	components := strings.Split(kh, "|")
+	if len(components) != 4 {
+		return false, fmt.Errorf("invalid format for hashed known host: '%s'", kh)
+	}
+
+	if components[1] != "1" {
+		return false, fmt.Errorf("unsupported hash type '%s'", components[1])
+	}
+
+	hkSalt, err := base64.StdEncoding.DecodeString(components[2])
+	if err != nil {
+		return false, fmt.Errorf("cannot decode hashed known host: '%w'", err)
+	}
+
+	hkHash, err := base64.StdEncoding.DecodeString(components[3])
+	if err != nil {
+		return false, fmt.Errorf("cannot decode hashed known host: '%w'", err)
+	}
+
+	mac := hmac.New(sha1.New, hkSalt)
+	mac.Write([]byte(host))
+	hostHash := mac.Sum(nil)
+
+	return bytes.Equal(hostHash, hkHash), nil
+}

pkg/git/libgit2/transport_test.go

@@ -522,6 +522,68 @@ func Test_pushTransferProgressCallback(t *testing.T) {
 	}
 }
 
+func TestMatchHashedHost(t *testing.T) {
+	tests := []struct {
+		name      string
+		knownHost string
+		host      string
+		match     bool
+		wantErr   string
+	}{
+		{
+			name:      "match valid known host",
+			knownHost: "|1|vApZG0Ybr4rHfTb69+cjjFIGIv0=|M5sSXen14encOvQAy0gseRahnJw=",
+			host:      "[127.0.0.1]:44167",
+			match:     true,
+		},
+		{
+			name:    "empty known host errors",
+			wantErr: "hashed known host must begin with '|'",
+		},
+		{
+			name:      "unhashed known host errors",
+			knownHost: "[127.0.0.1]:44167",
+			wantErr:   "hashed known host must begin with '|'",
+		},
+		{
+			name:      "invalid known host format errors",
+			knownHost: "|1M5sSXen14encOvQAy0gseRahnJw=",
+			wantErr:   "invalid format for hashed known host",
+		},
+		{
+			name:      "invalid hash type errors",
+			knownHost: "|2|vApZG0Ybr4rHfTb69+cjjFIGIv0=|M5sSXen14encOvQAy0gseRahnJw=",
+			wantErr:   "unsupported hash type",
+		},
+		{
+			name:      "invalid base64 component[2] errors",
+			knownHost: "|1|azz|M5sSXen14encOvQAy0gseRahnJw=",
+			wantErr:   "cannot decode hashed known host",
+		},
+		{
+			name:      "invalid base64 component[3] errors",
+			knownHost: "|1|M5sSXen14encOvQAy0gseRahnJw=|azz",
+			wantErr:   "cannot decode hashed known host",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := NewWithT(t)
+
+			matched, err := MatchHashedHost(tt.knownHost, tt.host)
+
+			if tt.wantErr == "" {
+				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(matched).To(Equal(tt.match))
+			} else {
+				g.Expect(err).To(HaveOccurred())
+				g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
+			}
+		})
+	}
+}
+
 func md5Fingerprint(in string) [16]byte {
 	var out [16]byte
 	copy(out[:], in)

pkg/git/strategy/strategy_test.go

@@ -97,7 +97,7 @@ func TestCheckoutStrategyForImplementation_Auth(t *testing.T) {
 				return getSSHRepoURL(srv.SSHAddress(), repoPath)
 			},
 			authOptsFunc: func(g *WithT, u *url.URL, user, pswd string, ca []byte) *git.AuthOptions {
-				knownhosts, err := ssh.ScanHostKey(u.Host, 5*time.Second, git.HostKeyAlgos)
+				knownhosts, err := ssh.ScanHostKey(u.Host, 5*time.Second, git.HostKeyAlgos, false)
 				g.Expect(err).ToNot(HaveOccurred())
 
 				keygen := ssh.NewRSAGenerator(2048)