fluxcd
/
source-controller
mirror of https://github.com/fluxcd/source-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #52 from fluxcd/git-improvements

This commit is contained in:
Hidde Beydals 2020-05-03 23:38:13 +02:00 committed by GitHub
parent 4f1829914d 078d0057a8
commit 0354d8da39
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 1085 additions and 237 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -20,7 +20,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o source-c
FROM alpine:3.11
RUN apk add --no-cache openssh-client ca-certificates tar tini 'git>=2.12.0' socat curl bash
RUN apk add --no-cache ca-certificates tar tini 'git>=2.12.0' socat curl bash
COPY --from=builder /workspace/source-controller /usr/local/bin/

214
controllers/gitrepository_controller.go
View File

@ -22,9 +22,7 @@ import (
"io/ioutil"
"os"
"github.com/blang/semver"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/plumbing/transport"
"github.com/go-logr/logr"
corev1 "k8s.io/api/core/v1"
@ -121,34 +119,18 @@ func (r *GitRepositoryReconciler) SetupWithManagerAndOptions(mgr ctrl.Manager, o
}
func (r *GitRepositoryReconciler) sync(ctx context.Context, repository sourcev1.GitRepository) (sourcev1.GitRepository, error) {
// set defaults: master branch, no tags fetching, max two commits
branch := "master"
revision := ""
tagMode := git.NoTags
depth := 2
// determine ref
refName := plumbing.NewBranchReferenceName(branch)
if repository.Spec.Reference != nil {
if repository.Spec.Reference.Branch != "" {
branch = repository.Spec.Reference.Branch
refName = plumbing.NewBranchReferenceName(branch)
}
if repository.Spec.Reference.Commit != "" {
depth = 0
} else {
if repository.Spec.Reference.Tag != "" {
refName = plumbing.NewTagReferenceName(repository.Spec.Reference.Tag)
}
if repository.Spec.Reference.SemVer != "" {
tagMode = git.AllTags
}
}
// create tmp dir for the Git clone
tmpGit, err := ioutil.TempDir("", repository.Name)
if err != nil {
err = fmt.Errorf("tmp dir error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
}
defer os.RemoveAll(tmpGit)
// determine auth method
var auth transport.AuthMethod
if repository.Spec.SecretRef != nil {
authStrategy := intgit.AuthSecretStrategyForURL(repository.Spec.URL)
if repository.Spec.SecretRef != nil && authStrategy != nil {
name := types.NamespacedName{
Namespace: repository.GetNamespace(),
Name: repository.Spec.SecretRef.Name,
@ -161,173 +143,32 @@ func (r *GitRepositoryReconciler) sync(ctx context.Context, repository sourcev1.
return sourcev1.GitRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err
}
method, cleanup, err := intgit.AuthMethodFromSecret(repository.Spec.URL, secret)
auth, err = authStrategy.Method(secret)
if err != nil {
err = fmt.Errorf("auth error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err
}
if cleanup != nil {
defer cleanup()
}
auth = method
}
// create tmp dir for the Git clone
tmpGit, err := ioutil.TempDir("", repository.Name)
checkoutStrategy := intgit.CheckoutStrategyForRef(repository.Spec.Reference)
commit, revision, err := checkoutStrategy.Checkout(ctx, tmpGit, repository.Spec.URL, auth)
if err != nil {
err = fmt.Errorf("tmp dir error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
}
defer os.RemoveAll(tmpGit)
// clone to tmp
gitCtx, cancel := context.WithTimeout(ctx, repository.GetTimeout())
repo, err := git.PlainCloneContext(gitCtx, tmpGit, false, &git.CloneOptions{
URL: repository.Spec.URL,
Auth: auth,
RemoteName: "origin",
ReferenceName: refName,
SingleBranch: true,
NoCheckout: false,
Depth: depth,
RecurseSubmodules: 0,
Progress: nil,
Tags: tagMode,
})
cancel()
if err != nil {
err = fmt.Errorf("git clone error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
}
// checkout commit or tag
if repository.Spec.Reference != nil {
if commit := repository.Spec.Reference.Commit; commit != "" {
w, err := repo.Worktree()
if err != nil {
err = fmt.Errorf("git worktree error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
}
err = w.Checkout(&git.CheckoutOptions{
Hash: plumbing.NewHash(commit),
Force: true,
})
if err != nil {
err = fmt.Errorf("git checkout '%s' for '%s' error: %w", commit, branch, err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
}
} else if exp := repository.Spec.Reference.SemVer; exp != "" {
rng, err := semver.ParseRange(exp)
if err != nil {
err = fmt.Errorf("semver parse range error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
}
repoTags, err := repo.Tags()
if err != nil {
err = fmt.Errorf("git list tags error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
}
tags := make(map[string]string)
_ = repoTags.ForEach(func(t *plumbing.Reference) error {
tags[t.Name().Short()] = t.Strings()[1]
return nil
})
svTags := make(map[string]string)
var svers []semver.Version
for tag, _ := range tags {
v, _ := semver.ParseTolerant(tag)
if rng(v) {
svers = append(svers, v)
svTags[v.String()] = tag
}
}
if len(svers) > 0 {
semver.Sort(svers)
v := svers[len(svers)-1]
t := svTags[v.String()]
commit := tags[t]
revision = fmt.Sprintf("%s/%s", t, commit)
w, err := repo.Worktree()
if err != nil {
err = fmt.Errorf("git worktree error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
}
err = w.Checkout(&git.CheckoutOptions{
Hash: plumbing.NewHash(commit),
})
if err != nil {
err = fmt.Errorf("git checkout error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
}
} else {
err = fmt.Errorf("no match found for semver: %s", repository.Spec.Reference.SemVer)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
}
}
}
// read commit hash
ref, err := repo.Head()
if err != nil {
err = fmt.Errorf("git resolve HEAD error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
}
// verify PGP signature
if repository.Spec.Verification != nil {
commit, err := repo.CommitObject(ref.Hash())
if err != nil {
err = fmt.Errorf("git resolve HEAD error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
}
if commit.PGPSignature == "" {
err = fmt.Errorf("PGP signature not found for commit '%s'", ref.Hash())
return sourcev1.GitRepositoryNotReady(repository, sourcev1.VerificationFailedReason, err.Error()), err
}
name := types.NamespacedName{
Namespace: repository.GetNamespace(),
err := r.verify(ctx, types.NamespacedName{
Namespace: repository.Namespace,
Name: repository.Spec.Verification.SecretRef.Name,
}
var secret corev1.Secret
err = r.Client.Get(ctx, name, &secret)
}, commit)
if err != nil {
err = fmt.Errorf("PGP public keys secret error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.VerificationFailedReason, err.Error()), err
}
var verified bool
for _, bytes := range secret.Data {
if _, err := commit.Verify(string(bytes)); err == nil {
verified = true
break
}
}
if !verified {
err = fmt.Errorf("PGP signature of '%s' can't be verified", commit.Author)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.VerificationFailedReason, err.Error()), err
}
}
if revision == "" {
revision = fmt.Sprintf("%s/%s", branch, ref.Hash().String())
if repository.Spec.Reference != nil && repository.Spec.Reference.Tag != "" {
revision = fmt.Sprintf("%s/%s", repository.Spec.Reference.Tag, ref.Hash().String())
}
}
artifact := r.Storage.ArtifactFor(repository.Kind, repository.ObjectMeta.GetObjectMeta(),
fmt.Sprintf("%s.tar.gz", ref.Hash().String()), revision)
fmt.Sprintf("%s.tar.gz", commit.Hash.String()), revision)
// create artifact dir
err = r.Storage.MkdirAll(artifact)
@ -388,6 +229,29 @@ func (r *GitRepositoryReconciler) shouldResetStatus(repository sourcev1.GitRepos
}
}
func (r *GitRepositoryReconciler) verify(ctx context.Context, publicKeySecret types.NamespacedName, commit *object.Commit) error {
if commit.PGPSignature == "" {
return fmt.Errorf("no PGP signature found for commit: %s", commit.Hash)
}
var secret corev1.Secret
if err := r.Client.Get(ctx, publicKeySecret, &secret); err != nil {
return fmt.Errorf("PGP public keys secret error: %w", err)
}
var verified bool
for _, bytes := range secret.Data {
if _, err := commit.Verify(string(bytes)); err == nil {
verified = true
break
}
}
if !verified {
return fmt.Errorf("PGP signature '%s' of '%s' can't be verified", commit.PGPSignature, commit.Author)
}
return nil
}
// gc performs a garbage collection on all but current artifacts of
// the given repository.
func (r *GitRepositoryReconciler) gc(repository sourcev1.GitRepository) error {

2
go.mod
View File

@ -8,7 +8,7 @@ require (
github.com/go-logr/logr v0.1.0
github.com/onsi/ginkgo v1.11.0
github.com/onsi/gomega v1.8.1
github.com/pkg/errors v0.9.1
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073
helm.sh/helm/v3 v3.1.2
k8s.io/api v0.17.2
k8s.io/apimachinery v0.17.2

446
internal/crypto/ssh/knownhosts/knownhosts.go Normal file
View File

@ -0,0 +1,446 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Copyright 2020 The FluxCD contributors. All rights reserved.
// This package provides an in-memory known hosts database
// derived from the golang.org/x/crypto/ssh/knownhosts
// package.
// It has been slightly modified and adapted to work with
// in-memory host keys not related to any known_hosts files
// on disk, and the database can be initialized with just a
// known_hosts byte blob.
// https://pkg.go.dev/golang.org/x/crypto/ssh/knownhosts
package knownhosts
import (
"bufio"
"bytes"
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"errors"
"fmt"
"io"
"net"
"strings"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/knownhosts"
)
// See the sshd manpage
// (http://man.openbsd.org/sshd#SSH_KNOWN_HOSTS_FILE_FORMAT) for
// background.
type addr struct{ host, port string }
func (a *addr) String() string {
h := a.host
if strings.Contains(h, ":") {
h = "[" + h + "]"
}
return h + ":" + a.port
}
type matcher interface {
match(addr) bool
}
type hostPattern struct {
negate bool
addr addr
}
func (p *hostPattern) String() string {
n := ""
if p.negate {
n = "!"
}
return n + p.addr.String()
}
type hostPatterns []hostPattern
func (ps hostPatterns) match(a addr) bool {
matched := false
for _, p := range ps {
if !p.match(a) {
continue
}
if p.negate {
return false
}
matched = true
}
return matched
}
// See
// https://android.googlesource.com/platform/external/openssh/+/ab28f5495c85297e7a597c1ba62e996416da7c7e/addrmatch.c
// The matching of * has no regard for separators, unlike filesystem globs
func wildcardMatch(pat []byte, str []byte) bool {
for {
if len(pat) == 0 {
return len(str) == 0
}
if len(str) == 0 {
return false
}
if pat[0] == '*' {
if len(pat) == 1 {
return true
}
for j := range str {
if wildcardMatch(pat[1:], str[j:]) {
return true
}
}
return false
}
if pat[0] == '?' || pat[0] == str[0] {
pat = pat[1:]
str = str[1:]
} else {
return false
}
}
}
func (p *hostPattern) match(a addr) bool {
return wildcardMatch([]byte(p.addr.host), []byte(a.host)) && p.addr.port == a.port
}
type inMemoryHostKeyDB struct {
hostKeys []hostKey
revoked map[string]*ssh.PublicKey
}
func newInMemoryHostKeyDB() *inMemoryHostKeyDB {
db := &inMemoryHostKeyDB{
revoked: make(map[string]*ssh.PublicKey),
}
return db
}
func keyEq(a, b ssh.PublicKey) bool {
return bytes.Equal(a.Marshal(), b.Marshal())
}
type hostKey struct {
matcher matcher
cert bool
key ssh.PublicKey
}
func (l *hostKey) match(a addr) bool {
return l.matcher.match(a)
}
// IsAuthorityForHost can be used as a callback in ssh.CertChecker
func (db *inMemoryHostKeyDB) IsHostAuthority(remote ssh.PublicKey, address string) bool {
h, p, err := net.SplitHostPort(address)
if err != nil {
return false
}
a := addr{host: h, port: p}
for _, l := range db.hostKeys {
if l.cert && keyEq(l.key, remote) && l.match(a) {
return true
}
}
return false
}
// IsRevoked can be used as a callback in ssh.CertChecker
func (db *inMemoryHostKeyDB) IsRevoked(key *ssh.Certificate) bool {
_, ok := db.revoked[string(key.Marshal())]
return ok
}
const markerCert = "@cert-authority"
const markerRevoked = "@revoked"
func nextWord(line []byte) (string, []byte) {
i := bytes.IndexAny(line, "\t ")
if i == -1 {
return string(line), nil
}
return string(line[:i]), bytes.TrimSpace(line[i:])
}
func parseLine(line []byte) (marker, host string, key ssh.PublicKey, err error) {
if w, next := nextWord(line); w == markerCert || w == markerRevoked {
marker = w
line = next
}
host, line = nextWord(line)
if len(line) == 0 {
return "", "", nil, errors.New("knownhosts: missing host pattern")
}
// ignore the keytype as it's in the key blob anyway.
_, line = nextWord(line)
if len(line) == 0 {
return "", "", nil, errors.New("knownhosts: missing key type pattern")
}
keyBlob, _ := nextWord(line)
keyBytes, err := base64.StdEncoding.DecodeString(keyBlob)
if err != nil {
return "", "", nil, err
}
key, err = ssh.ParsePublicKey(keyBytes)
if err != nil {
return "", "", nil, err
}
return marker, host, key, nil
}
func (db *inMemoryHostKeyDB) parseLine(line []byte) error {
marker, pattern, key, err := parseLine(line)
if err != nil {
return err
}
if marker == markerRevoked {
db.revoked[string(key.Marshal())] = &key
return nil
}
entry := hostKey{
key: key,
cert: marker == markerCert,
}
if pattern[0] == '|' {
entry.matcher, err = newHashedHost(pattern)
} else {
entry.matcher, err = newHostnameMatcher(pattern)
}
if err != nil {
return err
}
db.hostKeys = append(db.hostKeys, entry)
return nil
}
func newHostnameMatcher(pattern string) (matcher, error) {
var hps hostPatterns
for _, p := range strings.Split(pattern, ",") {
if len(p) == 0 {
continue
}
var a addr
var negate bool
if p[0] == '!' {
negate = true
p = p[1:]
}
if len(p) == 0 {
return nil, errors.New("knownhosts: negation without following hostname")
}
var err error
if p[0] == '[' {
a.host, a.port, err = net.SplitHostPort(p)
if err != nil {
return nil, err
}
} else {
a.host, a.port, err = net.SplitHostPort(p)
if err != nil {
a.host = p
a.port = "22"
}
}
hps = append(hps, hostPattern{
negate: negate,
addr: a,
})
}
return hps, nil
}
// check checks a key against the host database. This should not be
// used for verifying certificates.
func (db *inMemoryHostKeyDB) check(address string, remote net.Addr, remoteKey ssh.PublicKey) error {
if revoked := db.revoked[string(remoteKey.Marshal())]; revoked != nil {
return &knownhosts.RevokedError{Revoked: knownhosts.KnownKey{Key: *revoked}}
}
host, port, err := net.SplitHostPort(remote.String())
if err != nil {
return fmt.Errorf("knownhosts: SplitHostPort(%s): %v", remote, err)
}
hostToCheck := addr{host, port}
if address != "" {
// Give preference to the hostname if available.
host, port, err := net.SplitHostPort(address)
if err != nil {
return fmt.Errorf("knownhosts: SplitHostPort(%s): %v", address, err)
}
hostToCheck = addr{host, port}
}
return db.checkAddr(hostToCheck, remoteKey)
}
// checkAddr checks if we can find the given public key for the
// given address. If we only find an entry for the IP address,
// or only the hostname, then this still succeeds.
func (db *inMemoryHostKeyDB) checkAddr(a addr, remoteKey ssh.PublicKey) error {
// TODO(hanwen): are these the right semantics? What if there
// is just a key for the IP address, but not for the
// hostname?
// Algorithm => key.
knownKeys := map[string]ssh.PublicKey{}
for _, l := range db.hostKeys {
if l.match(a) {
typ := l.key.Type()
if _, ok := knownKeys[typ]; !ok {
knownKeys[typ] = l.key
}
}
}
keyErr := &knownhosts.KeyError{}
for _, v := range knownKeys {
keyErr.Want = append(keyErr.Want, knownhosts.KnownKey{Key: v})
}
// Unknown remote host.
if len(knownKeys) == 0 {
return keyErr
}
// If the remote host starts using a different, unknown key type, we
// also interpret that as a mismatch.
if known, ok := knownKeys[remoteKey.Type()]; !ok || !keyEq(known, remoteKey) {
return keyErr
}
return nil
}
// The Read function parses file contents.
func (db *inMemoryHostKeyDB) Read(r io.Reader) error {
scanner := bufio.NewScanner(r)
lineNum := 0
for scanner.Scan() {
lineNum++
line := scanner.Bytes()
line = bytes.TrimSpace(line)
if len(line) == 0 || line[0] == '#' {
continue
}
if err := db.parseLine(line); err != nil {
return fmt.Errorf("knownhosts: %v", err)
}
}
return scanner.Err()
}
// New creates a host key callback from the given OpenSSH host key
// file bytes. The returned callback is for use in
// ssh.ClientConfig.HostKeyCallback. By preference, the key check
// operates on the hostname if available, i.e. if a server changes its
// IP address, the host key check will still succeed, even though a
// record of the new IP address is not available.
func New(b []byte) (ssh.HostKeyCallback, error) {
db := newInMemoryHostKeyDB()
r := bytes.NewReader(b)
if err := db.Read(r); err != nil {
return nil, err
}
var certChecker ssh.CertChecker
certChecker.IsHostAuthority = db.IsHostAuthority
certChecker.IsRevoked = db.IsRevoked
certChecker.HostKeyFallback = db.check
return certChecker.CheckHostKey, nil
}
func decodeHash(encoded string) (hashType string, salt, hash []byte, err error) {
if len(encoded) == 0 || encoded[0] != '|' {
err = errors.New("knownhosts: hashed host must start with '|'")
return
}
components := strings.Split(encoded, "|")
if len(components) != 4 {
err = fmt.Errorf("knownhosts: got %d components, want 3", len(components))
return
}
hashType = components[1]
if salt, err = base64.StdEncoding.DecodeString(components[2]); err != nil {
return
}
if hash, err = base64.StdEncoding.DecodeString(components[3]); err != nil {
return
}
return
}
func encodeHash(typ string, salt []byte, hash []byte) string {
return strings.Join([]string{"",
typ,
base64.StdEncoding.EncodeToString(salt),
base64.StdEncoding.EncodeToString(hash),
}, "|")
}
// See https://android.googlesource.com/platform/external/openssh/+/ab28f5495c85297e7a597c1ba62e996416da7c7e/hostfile.c#120
func hashHost(hostname string, salt []byte) []byte {
mac := hmac.New(sha1.New, salt)
mac.Write([]byte(hostname))
return mac.Sum(nil)
}
type hashedHost struct {
salt []byte
hash []byte
}
const sha1HashType = "1"
func newHashedHost(encoded string) (*hashedHost, error) {
typ, salt, hash, err := decodeHash(encoded)
if err != nil {
return nil, err
}
// The type field seems for future algorithm agility, but it's
// actually hardcoded in openssh currently, see
// https://android.googlesource.com/platform/external/openssh/+/ab28f5495c85297e7a597c1ba62e996416da7c7e/hostfile.c#120
if typ != sha1HashType {
return nil, fmt.Errorf("knownhosts: got hash type %s, must be '1'", typ)
}
return &hashedHost{salt: salt, hash: hash}, nil
}
func (h *hashedHost) match(a addr) bool {
return bytes.Equal(hashHost(knownhosts.Normalize(a.String()), h.salt), h.hash)
}

327
internal/crypto/ssh/knownhosts/knownhosts_test.go Normal file
View File

@ -0,0 +1,327 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Copyright 2020 The FluxCD contributors. All rights reserved.
// This package provides an in-memory known hosts database
// derived from the golang.org/x/crypto/ssh/knownhosts
// package.
// It has been slightly modified and adapted to work with
// in-memory host keys not related to any known_hosts files
// on disk, and the database can be initialized with just a
// known_hosts byte blob.
// https://pkg.go.dev/golang.org/x/crypto/ssh/knownhosts
package knownhosts
import (
"bytes"
"fmt"
"net"
"reflect"
"testing"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/knownhosts"
)
const edKeyStr = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGBAarftlLeoyf+v+nVchEZII/vna2PCV8FaX4vsF5BX"
const alternateEdKeyStr = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIXffBYeYL+WVzVru8npl5JHt2cjlr4ornFTWzoij9sx"
const ecKeyStr = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNLCu01+wpXe3xB5olXCN4SqU2rQu0qjSRKJO4Bg+JRCPU+ENcgdA5srTU8xYDz/GEa4dzK5ldPw4J/gZgSXCMs="
var ecKey, alternateEdKey, edKey ssh.PublicKey
var testAddr = &net.TCPAddr{
IP: net.IP{198, 41, 30, 196},
Port: 22,
}
var testAddr6 = &net.TCPAddr{
IP: net.IP{198, 41, 30, 196,
1, 2, 3, 4,
1, 2, 3, 4,
1, 2, 3, 4,
},
Port: 22,
}
func init() {
var err error
ecKey, _, _, _, err = ssh.ParseAuthorizedKey([]byte(ecKeyStr))
if err != nil {
panic(err)
}
edKey, _, _, _, err = ssh.ParseAuthorizedKey([]byte(edKeyStr))
if err != nil {
panic(err)
}
alternateEdKey, _, _, _, err = ssh.ParseAuthorizedKey([]byte(alternateEdKeyStr))
if err != nil {
panic(err)
}
}
func testDB(t *testing.T, s string) *inMemoryHostKeyDB {
db := newInMemoryHostKeyDB()
if err := db.Read(bytes.NewBufferString(s)); err != nil {
t.Fatalf("Read: %v", err)
}
return db
}
func TestRevoked(t *testing.T) {
db := testDB(t, "\n\n@revoked * "+edKeyStr+"\n")
want := &knownhosts.RevokedError{
Revoked: knownhosts.KnownKey{
Key: edKey,
},
}
if err := db.check("", &net.TCPAddr{
Port: 42,
}, edKey); err == nil {
t.Fatal("no error for revoked key")
} else if !reflect.DeepEqual(want, err) {
t.Fatalf("got %#v, want %#v", want, err)
}
}
func TestHostAuthority(t *testing.T) {
for _, m := range []struct {
authorityFor string
address string
good bool
}{
{authorityFor: "localhost", address: "localhost:22", good: true},
{authorityFor: "localhost", address: "localhost", good: false},
{authorityFor: "localhost", address: "localhost:1234", good: false},
{authorityFor: "[localhost]:1234", address: "localhost:1234", good: true},
{authorityFor: "[localhost]:1234", address: "localhost:22", good: false},
{authorityFor: "[localhost]:1234", address: "localhost", good: false},
} {
db := testDB(t, `@cert-authority `+m.authorityFor+` `+edKeyStr)
if ok := db.IsHostAuthority(db.hostKeys[0].key, m.address); ok != m.good {
t.Errorf("IsHostAuthority: authority %s, address %s, wanted good = %v, got good = %v",
m.authorityFor, m.address, m.good, ok)
}
}
}
func TestBracket(t *testing.T) {
db := testDB(t, `[git.eclipse.org]:29418,[198.41.30.196]:29418 `+edKeyStr)
if err := db.check("git.eclipse.org:29418", &net.TCPAddr{
IP: net.IP{198, 41, 30, 196},
Port: 29418,
}, edKey); err != nil {
t.Errorf("got error %v, want none", err)
}
if err := db.check("git.eclipse.org:29419", &net.TCPAddr{
Port: 42,
}, edKey); err == nil {
t.Fatalf("no error for unknown address")
} else if ke, ok := err.(*knownhosts.KeyError); !ok {
t.Fatalf("got type %T, want *KeyError", err)
} else if len(ke.Want) > 0 {
t.Fatalf("got Want %v, want []", ke.Want)
}
}
func TestNewKeyType(t *testing.T) {
str := fmt.Sprintf("%s %s", testAddr, edKeyStr)
db := testDB(t, str)
if err := db.check("", testAddr, ecKey); err == nil {
t.Fatalf("no error for unknown address")
} else if ke, ok := err.(*knownhosts.KeyError); !ok {
t.Fatalf("got type %T, want *KeyError", err)
} else if len(ke.Want) == 0 {
t.Fatalf("got empty KeyError.Want")
}
}
func TestSameKeyType(t *testing.T) {
str := fmt.Sprintf("%s %s", testAddr, edKeyStr)
db := testDB(t, str)
if err := db.check("", testAddr, alternateEdKey); err == nil {
t.Fatalf("no error for unknown address")
} else if ke, ok := err.(*knownhosts.KeyError); !ok {
t.Fatalf("got type %T, want *KeyError", err)
} else if len(ke.Want) == 0 {
t.Fatalf("got empty KeyError.Want")
} else if got, want := ke.Want[0].Key.Marshal(), edKey.Marshal(); !bytes.Equal(got, want) {
t.Fatalf("got key %q, want %q", got, want)
}
}
func TestIPAddress(t *testing.T) {
str := fmt.Sprintf("%s %s", testAddr, edKeyStr)
db := testDB(t, str)
if err := db.check("", testAddr, edKey); err != nil {
t.Errorf("got error %q, want none", err)
}
}
func TestIPv6Address(t *testing.T) {
str := fmt.Sprintf("%s %s", testAddr6, edKeyStr)
db := testDB(t, str)
if err := db.check("", testAddr6, edKey); err != nil {
t.Errorf("got error %q, want none", err)
}
}
func TestBasic(t *testing.T) {
str := fmt.Sprintf("#comment\n\nserver.org,%s %s\notherhost %s", testAddr, edKeyStr, ecKeyStr)
db := testDB(t, str)
if err := db.check("server.org:22", testAddr, edKey); err != nil {
t.Errorf("got error %v, want none", err)
}
want := knownhosts.KnownKey{
Key: edKey,
}
if err := db.check("server.org:22", testAddr, ecKey); err == nil {
t.Errorf("succeeded, want KeyError")
} else if ke, ok := err.(*knownhosts.KeyError); !ok {
t.Errorf("got %T, want *KeyError", err)
} else if len(ke.Want) != 1 {
t.Errorf("got %v, want 1 entry", ke)
} else if !reflect.DeepEqual(ke.Want[0], want) {
t.Errorf("got %v, want %v", ke.Want[0], want)
}
}
func TestHostNamePrecedence(t *testing.T) {
var evilAddr = &net.TCPAddr{
IP: net.IP{66, 66, 66, 66},
Port: 22,
}
str := fmt.Sprintf("server.org,%s %s\nevil.org,%s %s", testAddr, edKeyStr, evilAddr, ecKeyStr)
db := testDB(t, str)
if err := db.check("server.org:22", evilAddr, ecKey); err == nil {
t.Errorf("check succeeded")
} else if _, ok := err.(*knownhosts.KeyError); !ok {
t.Errorf("got %T, want *KeyError", err)
}
}
func TestDBOrderingPrecedenceKeyType(t *testing.T) {
str := fmt.Sprintf("server.org,%s %s\nserver.org,%s %s", testAddr, edKeyStr, testAddr, alternateEdKeyStr)
db := testDB(t, str)
if err := db.check("server.org:22", testAddr, alternateEdKey); err == nil {
t.Errorf("check succeeded")
} else if _, ok := err.(*knownhosts.KeyError); !ok {
t.Errorf("got %T, want *KeyError", err)
}
}
func TestNegate(t *testing.T) {
str := fmt.Sprintf("%s,!server.org %s", testAddr, edKeyStr)
db := testDB(t, str)
if err := db.check("server.org:22", testAddr, ecKey); err == nil {
t.Errorf("succeeded")
} else if ke, ok := err.(*knownhosts.KeyError); !ok {
t.Errorf("got error type %T, want *KeyError", err)
} else if len(ke.Want) != 0 {
t.Errorf("got expected keys %d (first of type %s), want []", len(ke.Want), ke.Want[0].Key.Type())
}
}
func TestWildcard(t *testing.T) {
str := fmt.Sprintf("server*.domain %s", edKeyStr)
db := testDB(t, str)
want := &knownhosts.KeyError{
Want: []knownhosts.KnownKey{{
Key: edKey,
}},
}
got := db.check("server.domain:22", &net.TCPAddr{}, ecKey)
if !reflect.DeepEqual(got, want) {
t.Errorf("got %s, want %s", got, want)
}
}
func TestWildcardMatch(t *testing.T) {
for _, c := range []struct {
pat, str string
want bool
}{
{"a?b", "abb", true},
{"ab", "abc", false},
{"abc", "ab", false},
{"a*b", "axxxb", true},
{"a*b", "axbxb", true},
{"a*b", "axbxbc", false},
{"a*?", "axbxc", true},
{"a*b*", "axxbxxxxxx", true},
{"a*b*c", "axxbxxxxxxc", true},
{"a*b*?", "axxbxxxxxxc", true},
{"a*b*z", "axxbxxbxxxz", true},
{"a*b*z", "axxbxxzxxxz", true},
{"a*b*z", "axxbxxzxxx", false},
} {
got := wildcardMatch([]byte(c.pat), []byte(c.str))
if got != c.want {
t.Errorf("wildcardMatch(%q, %q) = %v, want %v", c.pat, c.str, got, c.want)
}
}
}
// TODO(hanwen): test coverage for certificates.
const testHostname = "hostname"
// generated with keygen -H -f
const encodedTestHostnameHash = "|1|IHXZvQMvTcZTUU29+2vXFgx8Frs=|UGccIWfRVDwilMBnA3WJoRAC75Y="
func TestHostHash(t *testing.T) {
testHostHash(t, testHostname, encodedTestHostnameHash)
}
func TestHashList(t *testing.T) {
encoded := knownhosts.HashHostname(testHostname)
testHostHash(t, testHostname, encoded)
}
func testHostHash(t *testing.T, hostname, encoded string) {
typ, salt, hash, err := decodeHash(encoded)
if err != nil {
t.Fatalf("decodeHash: %v", err)
}
if got := encodeHash(typ, salt, hash); got != encoded {
t.Errorf("got encoding %s want %s", got, encoded)
}
if typ != sha1HashType {
t.Fatalf("got hash type %q, want %q", typ, sha1HashType)
}
got := hashHost(hostname, salt)
if !bytes.Equal(got, hash) {
t.Errorf("got hash %x want %x", got, hash)
}
}
func TestHashedHostkeyCheck(t *testing.T) {
str := fmt.Sprintf("%s %s", knownhosts.HashHostname(testHostname), edKeyStr)
db := testDB(t, str)
if err := db.check(testHostname+":22", testAddr, edKey); err != nil {
t.Errorf("check(%s): %v", testHostname, err)
}
want := &knownhosts.KeyError{
Want: []knownhosts.KnownKey{{
Key: edKey,
}},
}
if got := db.check(testHostname+":22", testAddr, alternateEdKey); !reflect.DeepEqual(got, want) {
t.Errorf("got error %v, want %v", got, want)
}
}

230
internal/git/checkout.go Normal file
View File

@ -0,0 +1,230 @@
/*
Copyright 2020 The Flux CD contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package git
import (
"context"
"fmt"
"github.com/blang/semver"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/plumbing/transport"
sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1"
)
const (
defaultOrigin = "origin"
defaultBranch = "master"
)
func CheckoutStrategyForRef(ref *sourcev1.GitRepositoryRef) CheckoutStrategy {
switch {
case ref == nil:
return &CheckoutBranch{branch: defaultBranch}
case ref.SemVer != "":
return &CheckoutSemVer{semVer: ref.SemVer}
case ref.Tag != "":
return &CheckoutTag{tag: ref.Tag}
case ref.Commit != "":
return &CheckoutCommit{branch: ref.Branch, commit: ref.Commit}
case ref.Branch != "":
return &CheckoutBranch{branch: ref.Branch}
default:
return &CheckoutBranch{branch: defaultBranch}
}
}
type CheckoutStrategy interface {
Checkout(ctx context.Context, path, url string, auth transport.AuthMethod) (*object.Commit, string, error)
}
type CheckoutBranch struct {
branch string
}
func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, auth transport.AuthMethod) (*object.Commit, string, error) {
repo, err := git.PlainCloneContext(ctx, path, false, &git.CloneOptions{
URL: url,
Auth: auth,
RemoteName: defaultOrigin,
ReferenceName: plumbing.NewBranchReferenceName(c.branch),
SingleBranch: true,
NoCheckout: false,
Depth: 1,
RecurseSubmodules: 0,
Progress: nil,
Tags: git.NoTags,
})
if err != nil {
return nil, "", fmt.Errorf("git clone error: %w", err)
}
head, err := repo.Head()
if err != nil {
return nil, "", fmt.Errorf("git resolve HEAD error: %w", err)
}
commit, err := repo.CommitObject(head.Hash())
if err != nil {
return nil, "", fmt.Errorf("git commit not found: %w", err)
}
return commit, fmt.Sprintf("%s/%s", c.branch, head.Hash().String()), nil
}
type CheckoutTag struct {
tag string
}
func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, auth transport.AuthMethod) (*object.Commit, string, error) {
repo, err := git.PlainCloneContext(ctx, path, false, &git.CloneOptions{
URL: url,
Auth: auth,
RemoteName: defaultOrigin,
ReferenceName: plumbing.NewTagReferenceName(c.tag),
SingleBranch: true,
NoCheckout: false,
Depth: 1,
RecurseSubmodules: 0,
Progress: nil,
Tags: git.NoTags,
})
if err != nil {
return nil, "", fmt.Errorf("git clone error: %w", err)
}
head, err := repo.Head()
if err != nil {
return nil, "", fmt.Errorf("git resolve HEAD error: %w", err)
}
commit, err := repo.CommitObject(head.Hash())
if err != nil {
return nil, "", fmt.Errorf("git commit not found: %w", err)
}
return commit, fmt.Sprintf("%s/%s", c.tag, head.Hash().String()), nil
}
type CheckoutCommit struct {
branch string
commit string
}
func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, auth transport.AuthMethod) (*object.Commit, string, error) {
repo, err := git.PlainCloneContext(ctx, path, false, &git.CloneOptions{
URL: url,
Auth: auth,
RemoteName: defaultOrigin,
ReferenceName: plumbing.NewBranchReferenceName(c.branch),
SingleBranch: true,
NoCheckout: false,
RecurseSubmodules: 0,
Progress: nil,
Tags: git.NoTags,
})
if err != nil {
return nil, "", fmt.Errorf("git clone error: %w", err)
}
w, err := repo.Worktree()
if err != nil {
return nil, "", fmt.Errorf("git worktree error: %w", err)
}
commit, err := repo.CommitObject(plumbing.NewHash(c.commit))
if err != nil {
return nil, "", fmt.Errorf("git commit not found: %w", err)
}
err = w.Checkout(&git.CheckoutOptions{
Hash: commit.Hash,
Force: true,
})
if err != nil {
return nil, "", fmt.Errorf("git checkout error: %w", err)
}
return commit, fmt.Sprintf("%s/%s", c.branch, commit.Hash.String()), nil
}
type CheckoutSemVer struct {
semVer string
}
func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, auth transport.AuthMethod) (*object.Commit, string, error) {
rng, err := semver.ParseRange(c.semVer)
if err != nil {
return nil, "", fmt.Errorf("semver parse range error: %w", err)
}
repo, err := git.PlainCloneContext(ctx, path, false, &git.CloneOptions{
URL: url,
Auth: auth,
RemoteName: defaultOrigin,
SingleBranch: true,
NoCheckout: false,
Depth: 1,
RecurseSubmodules: 0,
Progress: nil,
Tags: git.AllTags,
})
if err != nil {
return nil, "", fmt.Errorf("git clone error: %w", err)
}
repoTags, err := repo.Tags()
if err != nil {
return nil, "", fmt.Errorf("git list tags error: %w", err)
}
tags := make(map[string]string)
_ = repoTags.ForEach(func(t *plumbing.Reference) error {
tags[t.Name().Short()] = t.Strings()[1]
return nil
})
svTags := make(map[string]string)
var svers []semver.Version
for tag, _ := range tags {
v, _ := semver.ParseTolerant(tag)
if rng(v) {
svers = append(svers, v)
svTags[v.String()] = tag
}
}
if len(svers) == 0 {
return nil, "", fmt.Errorf("no match found for semver: %s", c.semVer)
}
semver.Sort(svers)
v := svers[len(svers)-1]
t := svTags[v.String()]
commitRef := tags[t]
w, err := repo.Worktree()
if err != nil {
return nil, "", fmt.Errorf("git worktree error: %w", err)
}
commit, err := repo.CommitObject(plumbing.NewHash(commitRef))
if err != nil {
return nil, "", fmt.Errorf("git commit not found: %w", err)
}
err = w.Checkout(&git.CheckoutOptions{
Hash: commit.Hash,
})
if err != nil {
return nil, "", fmt.Errorf("git checkout error: %w", err)
}
return commit, fmt.Sprintf("%s/%s", t, commitRef), nil
}

50
internal/git/transport.go
View File

@ -18,29 +18,33 @@ package git
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/go-git/go-git/v5/plumbing/transport"
"github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
corev1 "k8s.io/api/core/v1"
"github.com/fluxcd/source-controller/internal/crypto/ssh/knownhosts"
)
func AuthMethodFromSecret(url string, secret corev1.Secret) (transport.AuthMethod, func(), error) {
func AuthSecretStrategyForURL(url string) AuthSecretStrategy {
switch {
case strings.HasPrefix(url, "http"):
auth, err := BasicAuthFromSecret(secret)
return auth, nil, err
return &BasicAuth{}
case strings.HasPrefix(url, "ssh"):
return PublicKeysFromSecret(secret)
return &PublicKeyAuth{}
}
return nil, nil, nil
return nil
}
func BasicAuthFromSecret(secret corev1.Secret) (*http.BasicAuth, error) {
type AuthSecretStrategy interface {
Method(secret corev1.Secret) (transport.AuthMethod, error)
}
type BasicAuth struct{}
func (s *BasicAuth) Method(secret corev1.Secret) (transport.AuthMethod, error) {
auth := &http.BasicAuth{}
if username, ok := secret.Data["username"]; ok {
auth.Username = string(username)
@ -54,36 +58,24 @@ func BasicAuthFromSecret(secret corev1.Secret) (*http.BasicAuth, error) {
return auth, nil
}
func PublicKeysFromSecret(secret corev1.Secret) (*ssh.PublicKeys, func(), error) {
type PublicKeyAuth struct{}
func (s *PublicKeyAuth) Method(secret corev1.Secret) (transport.AuthMethod, error) {
identity := secret.Data["identity"]
knownHosts := secret.Data["known_hosts"]
if len(identity) == 0 || len(knownHosts) == 0 {
return nil, nil, fmt.Errorf("invalid '%s' secret data: required fields 'identity' and 'known_hosts'", secret.Name)
return nil, fmt.Errorf("invalid '%s' secret data: required fields 'identity' and 'known_hosts'", secret.Name)
}
pk, err := ssh.NewPublicKeys("git", identity, "")
if err != nil {
return nil, nil, err
return nil, err
}
// create tmp dir for known_hosts
tmp, err := ioutil.TempDir("", "ssh-"+secret.Name)
callback, err := knownhosts.New(knownHosts)
if err != nil {
return nil, nil, err
}
cleanup := func() { os.RemoveAll(tmp) }
knownHostsPath := filepath.Join(tmp, "known_hosts")
if err := ioutil.WriteFile(knownHostsPath, knownHosts, 0644); err != nil {
cleanup()
return nil, nil, err
}
callback, err := ssh.NewKnownHostsCallback(knownHostsPath)
if err != nil {
cleanup()
return nil, nil, err
return nil, err
}
pk.HostKeyCallback = callback
return pk, cleanup, nil
return pk, nil
}

51
internal/git/transport_test.go
View File

@ -22,7 +22,6 @@ import (
"github.com/go-git/go-git/v5/plumbing/transport"
"github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
corev1 "k8s.io/api/core/v1"
)
@ -65,42 +64,33 @@ var (
}
)
func TestAuthMethodFromSecret(t *testing.T) {
func TestAuthSecretStrategyForURL(t *testing.T) {
tests := []struct {
name string
url string
secret corev1.Secret
want transport.AuthMethod
wantErr bool
name string
url string
want AuthSecretStrategy
}{
{"HTTP", "http://git.example.com/org/repo.git", basicAuthSecretFixture, &http.BasicAuth{}, false},
{"HTTPS", "https://git.example.com/org/repo.git", basicAuthSecretFixture, &http.BasicAuth{}, false},
{"SSH", "ssh://git.example.com:2222/org/repo.git", privateKeySecretFixture, &ssh.PublicKeys{}, false},
{"unsupported", "protocol://git.example.com/org/repo.git", corev1.Secret{}, nil, false},
{"HTTP", "http://git.example.com/org/repo.git", &BasicAuth{}},
{"HTTPS", "https://git.example.com/org/repo.git", &BasicAuth{}},
{"SSH", "ssh://git.example.com:2222/org/repo.git", &PublicKeyAuth{}},
{"unsupported", "protocol://example.com", nil},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, cleanup, err := AuthMethodFromSecret(tt.url, tt.secret)
if cleanup != nil {
defer cleanup()
}
if (err != nil) != tt.wantErr {
t.Errorf("AuthMethodFromSecret() error = %v, wantErr %v", err, tt.wantErr)
return
}
got := AuthSecretStrategyForURL(tt.url)
if reflect.TypeOf(got) != reflect.TypeOf(tt.want) {
t.Errorf("AuthMethodFromSecret() got = %v, want %v", got, tt.want)
t.Errorf("AuthSecretStrategyForURL() got = %v, want %v", got, tt.want)
}
})
}
}
func TestBasicAuthFromSecret(t *testing.T) {
func TestBasicAuthStrategy_Method(t *testing.T) {
tests := []struct {
name string
secret corev1.Secret
modify func(secret *corev1.Secret)
want *http.BasicAuth
want transport.AuthMethod
wantErr bool
}{
{"username and password", basicAuthSecretFixture, nil, &http.BasicAuth{Username: "git", Password: "password"}, false},
@ -114,19 +104,20 @@ func TestBasicAuthFromSecret(t *testing.T) {
if tt.modify != nil {
tt.modify(secret)
}
got, err := BasicAuthFromSecret(*secret)
s := &BasicAuth{}
got, err := s.Method(*secret)
if (err != nil) != tt.wantErr {
t.Errorf("BasicAuthFromSecret() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("Method() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("BasicAuthFromSecret() got = %v, want %v", got, tt.want)
t.Errorf("Method() got = %v, want %v", got, tt.want)
}
})
}
}
func TestPublicKeysFromSecret(t *testing.T) {
func TestPublicKeyStrategy_Method(t *testing.T) {
tests := []struct {
name string
secret corev1.Secret
@ -146,12 +137,10 @@ func TestPublicKeysFromSecret(t *testing.T) {
if tt.modify != nil {
tt.modify(secret)
}
_, cleanup, err := PublicKeysFromSecret(*secret)
if cleanup != nil {
defer cleanup()
}
s := &PublicKeyAuth{}
_, err := s.Method(*secret)
if (err != nil) != tt.wantErr {
t.Errorf("PublicKeysFromSecret() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("Method() error = %v, wantErr %v", err, tt.wantErr)
return
}
})