kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #11219 from johngmyers/refactor-keypair 

Refactor keypair code in preparation for secret rotation
This commit is contained in:
Kubernetes Prow Robot 2021-06-12 14:25:00 -07:00 committed by GitHub
parent cfc93e5178 12465ac27c
commit b71ba1d566
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 733 additions and 607 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/kops-controller/pkg/server/keystore.go
View File

@ -35,12 +35,12 @@ type keystoreEntry struct {
var _ pki.Keystore = keystore{}
func (k keystore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) {
func (k keystore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) {
entry, ok := k.keys[name]
if !ok {
return nil, nil, false, fmt.Errorf("unknown CA %q", name)
return nil, nil, fmt.Errorf("unknown CA %q", name)
}
return entry.certificate, entry.key, false, nil
return entry.certificate, entry.key, nil
}
func newKeystore(basePath string, cas []string) (pki.Keystore, error) {

2
cmd/kops-controller/pkg/server/node_config.go
View File

@ -73,7 +73,7 @@ func (s *Server) getNodeConfig(ctx context.Context, req *nodeup.BootstrapRequest
// We populate some certificates that we know the node will need.
for _, name := range []string{"ca"} {
cert, _, _, err := s.keystore.FindKeypair(name)
cert, _, err := s.keystore.FindPrimaryKeypair(name)
if err != nil {
return nil, fmt.Errorf("error getting certificate %q: %w", name, err)
}

15
cmd/kops/create_secret_keypair_ca.go
View File

@ -133,7 +133,20 @@ func RunCreateSecretCaCert(ctx context.Context, f *util.Factory, out io.Writer,
return fmt.Errorf("error loading certificate %q: %v", options.CaCertPath, err)
}
err = keyStore.StoreKeypair(fi.CertificateIDCA, cert, privateKey)
serialString := cert.Certificate.SerialNumber.String()
ki := &fi.KeysetItem{
Id: serialString,
Certificate: cert,
PrivateKey: privateKey,
}
err = keyStore.StoreKeyset(fi.CertificateIDCA, &fi.Keyset{
LegacyFormat: false,
Items: map[string]*fi.KeysetItem{
serialString: ki,
},
Primary: ki,
})
if err != nil {
return fmt.Errorf("error storing user provided keys %q %q: %v", options.CaCertPath, options.CaPrivateKeyPath, err)
}

3
k8s/crds/kops.k8s.io_keysets.yaml
View File

@ -59,6 +59,9 @@ spec:
type: string
type: object
type: array
primaryId:
description: PrimaryId is the id of the key used to make new signatures.
type: string
type:
description: Type is the type of the Keyset (PKI keypair, or secret
token)

100
nodeup/pkg/model/fakes_test.go
View File

@ -18,6 +18,7 @@ package model
import (
"crypto/x509"
"fmt"
"testing"
"k8s.io/kops/pkg/apis/kops"
@ -26,40 +27,69 @@ import (
"k8s.io/kops/util/pkg/vfs"
)
// fakeKeyStore mocks out some of fi.KeyStore, for our tests.
type fakeKeyStore struct {
T *testing.T
}
var _ fi.Keystore = &fakeKeyStore{}
func (k fakeKeyStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) {
panic("fakeKeyStore does not implement FindKeypair")
}
func (k fakeKeyStore) CreateKeypair(signer string, name string, template *x509.Certificate, privateKey *pki.PrivateKey) (*pki.Certificate, error) {
panic("fakeKeyStore does not implement CreateKeypair")
}
func (k fakeKeyStore) StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error {
panic("fakeKeyStore does not implement StoreKeypair")
}
func (k fakeKeyStore) MirrorTo(basedir vfs.Path) error {
panic("fakeKeyStore does not implement MirrorTo")
}
// fakeCAStore mocks out some of fi.CAStore, for our tests.
// Although CAStore currently embeds KeyStore, we maintain the split here in the hope we can clean this up in future.
type fakeCAStore struct {
fakeKeyStore
privateKeys map[string]*pki.PrivateKey
certs map[string]*pki.Certificate
T *testing.T
privateKeysets map[string]*kops.Keyset
certs map[string]*pki.Certificate
}
var _ fi.CAStore = &fakeCAStore{}
func (k fakeCAStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) {
panic("fakeCAStore does not implement FindPrimaryKeypair")
}
func (k fakeCAStore) FindKeyset(name string) (*fi.Keyset, error) {
kopsKeyset := k.privateKeysets[name]
if kopsKeyset == nil {
return nil, nil
}
keyset := &fi.Keyset{
Items: make(map[string]*fi.KeysetItem),
}
for _, key := range kopsKeyset.Spec.Keys {
ki := &fi.KeysetItem{
Id: key.Id,
}
if len(key.PublicMaterial) != 0 {
cert, err := pki.ParsePEMCertificate(key.PublicMaterial)
if err != nil {
return nil, fmt.Errorf("error loading certificate %s/%s: %v", name, key.Id, err)
}
ki.Certificate = cert
}
if len(key.PrivateMaterial) != 0 {
privateKey, err := pki.ParsePEMPrivateKey(key.PrivateMaterial)
if err != nil {
return nil, fmt.Errorf("error loading private key %s/%s: %v", name, key.Id, err)
}
ki.PrivateKey = privateKey
}
keyset.Items[key.Id] = ki
}
keyset.Primary = keyset.Items[fi.FindPrimary(kopsKeyset).Id]
return keyset, nil
}
func (k fakeCAStore) CreateKeypair(signer string, name string, template *x509.Certificate, privateKey *pki.PrivateKey) (*pki.Certificate, error) {
panic("fakeCAStore does not implement CreateKeypair")
}
func (k fakeCAStore) StoreKeyset(name string, keyset *fi.Keyset) error {
panic("fakeCAStore does not implement StoreKeyset")
}
func (k fakeCAStore) MirrorTo(basedir vfs.Path) error {
panic("fakeCAStore does not implement MirrorTo")
}
func (k fakeCAStore) FindCertificatePool(name string) (*fi.CertificatePool, error) {
panic("fakeCAStore does not implement FindCertificatePool")
}
@ -69,11 +99,17 @@ func (k fakeCAStore) FindCertificateKeyset(name string) (*kops.Keyset, error) {
}
func (k fakeCAStore) FindPrivateKey(name string) (*pki.PrivateKey, error) {
return k.privateKeys[name], nil
primaryId := k.privateKeysets[name].Spec.PrimaryId
for _, item := range k.privateKeysets[name].Spec.Keys {
if item.Id == primaryId {
return pki.ParsePEMPrivateKey(item.PrivateMaterial)
}
}
return nil, nil
}
func (k fakeCAStore) FindPrivateKeyset(name string) (*kops.Keyset, error) {
panic("fakeCAStore does not implement FindPrivateKeyset")
return k.privateKeysets[name], nil
}
func (k fakeCAStore) FindCert(name string) (*pki.Certificate, error) {
@ -84,10 +120,6 @@ func (k fakeCAStore) ListKeysets() ([]*kops.Keyset, error) {
panic("fakeCAStore does not implement ListKeysets")
}
func (k fakeCAStore) AddCert(name string, cert *pki.Certificate) error {
panic("fakeCAStore does not implement AddCert")
}
func (k fakeCAStore) DeleteKeysetItem(item *kops.Keyset, id string) error {
panic("fakeCAStore does not implement DeleteKeysetItem")
}

25
nodeup/pkg/model/kube_apiserver.go
View File

@ -81,6 +81,28 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error {
}
}
}
{
keyset, err := b.KeyStore.FindKeyset("service-account")
if err != nil {
return err
}
if keyset == nil {
return fmt.Errorf("service-account keyset not found")
}
buf, err := keyset.ToPublicKeyBytes()
if err != nil {
return err
}
c.AddTask(&nodetasks.File{
Path: filepath.Join(b.PathSrvKubernetes(), "service-account.pub"),
Contents: fi.NewBytesResource(buf),
Type: nodetasks.FileType_File,
Mode: s("0600"),
})
}
{
pod, err := b.buildPod()
if err != nil {
@ -282,8 +304,7 @@ func (b *KubeAPIServerBuilder) writeAuthenticationConfig(c *fi.ModelBuilderConte
func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
kubeAPIServer := b.Cluster.Spec.KubeAPIServer
// TODO pass the public key instead. We would first need to segregate the secrets better.
kubeAPIServer.ServiceAccountKeyFile = append(kubeAPIServer.ServiceAccountKeyFile, filepath.Join(b.PathSrvKubernetes(), "service-account.key"))
kubeAPIServer.ServiceAccountKeyFile = append(kubeAPIServer.ServiceAccountKeyFile, filepath.Join(b.PathSrvKubernetes(), "service-account.pub"))
// Set the signing key if we're using Service Account Token VolumeProjection
if kubeAPIServer.ServiceAccountSigningKeyFile == nil {

59
nodeup/pkg/model/kubelet_test.go
View File

@ -269,13 +269,48 @@ func mockedPopulateClusterSpec(c *kops.Cluster, cloud fi.Cloud) (*kops.Cluster,
const dummyCertificate = "-----BEGIN CERTIFICATE-----\nMIIC2DCCAcCgAwIBAgIRALJXAkVj964tq67wMSI8oJQwDQYJKoZIhvcNAQELBQAw\nFTETMBEGA1UEAxMKa3ViZXJuZXRlczAeFw0xNzEyMjcyMzUyNDBaFw0yNzEyMjcy\nMzUyNDBaMBUxEzARBgNVBAMTCmt1YmVybmV0ZXMwggEiMA0GCSqGSIb3DQEBAQUA\nA4IBDwAwggEKAoIBAQDgnCkSmtnmfxEgS3qNPaUCH5QOBGDH/inHbWCODLBCK9gd\nXEcBl7FVv8T2kFr1DYb0HVDtMI7tixRVFDLgkwNlW34xwWdZXB7GeoFgU1xWOQSY\nOACC8JgYTQ/139HBEvgq4sej67p+/s/SNcw34Kk7HIuFhlk1rRk5kMexKIlJBKP1\nYYUYetsJ/QpUOkqJ5HW4GoetE76YtHnORfYvnybviSMrh2wGGaN6r/s4ChOaIbZC\nAn8/YiPKGIDaZGpj6GXnmXARRX/TIdgSQkLwt0aTDBnPZ4XvtpI8aaL8DYJIqAzA\nNPH2b4/uNylat5jDo0b0G54agMi97+2AUrC9UUXpAgMBAAGjIzAhMA4GA1UdDwEB\n/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBVGR2r\nhzXzRMU5wriPQAJScszNORvoBpXfZoZ09FIupudFxBVU3d4hV9StKnQgPSGA5XQO\nHE97+BxJDuA/rB5oBUsMBjc7y1cde/T6hmi3rLoEYBSnSudCOXJE4G9/0f8byAJe\nrN8+No1r2VgZvZh6p74TEkXv/l3HBPWM7IdUV0HO9JDhSgOVF1fyQKJxRuLJR8jt\nO6mPH2UX0vMwVa4jvwtkddqk2OAdYQvH9rbDjjbzaiW0KnmdueRo92KHAN7BsDZy\nVpXHpqo1Kzg7D3fpaXCf5si7lqqrdJVXH4JC72zxsPehqgi8eIuqOBkiDWmRxAxh\n8yGeRx9AbknHh4Ia\n-----END CERTIFICATE-----\n"
const dummyKey = "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA4JwpEprZ5n8RIEt6jT2lAh+UDgRgx/4px21gjgywQivYHVxH\nAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMDZVt+McFnWVwexnqBYFNcVjkEmDgA\ngvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+CpOxyLhYZZNa0ZOZDHsSiJSQSj9WGF\nGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m74kjK4dsBhmjeq/7OAoTmiG2QgJ/\nP2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdGkwwZz2eF77aSPGmi/A2CSKgMwDTx\n9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF6QIDAQABAoIBAA0ktjaTfyrAxsTI\nBezb7Zr5NBW55dvuII299cd6MJo+rI/TRYhvUv48kY8IFXp/hyUjzgeDLunxmIf9\n/Zgsoic9Ol44/g45mMduhcGYPzAAeCdcJ5OB9rR9VfDCXyjYLlN8H8iU0734tTqM\n0V13tQ9zdSqkGPZOIcq/kR/pylbOZaQMe97BTlsAnOMSMKDgnftY4122Lq3GYy+t\nvpr+bKVaQZwvkLoSU3rECCaKaghgwCyX7jft9aEkhdJv+KlwbsGY6WErvxOaLWHd\ncuMQjGapY1Fa/4UD00mvrA260NyKfzrp6+P46RrVMwEYRJMIQ8YBAk6N6Hh7dc0G\n8Z6i1m0CgYEA9HeCJR0TSwbIQ1bDXUrzpftHuidG5BnSBtax/ND9qIPhR/FBW5nj\n22nwLc48KkyirlfIULd0ae4qVXJn7wfYcuX/cJMLDmSVtlM5Dzmi/91xRiFgIzx1\nAsbBzaFjISP2HpSgL+e9FtSXaaqeZVrflitVhYKUpI/AKV31qGHf04sCgYEA6zTV\n99Sb49Wdlns5IgsfnXl6ToRttB18lfEKcVfjAM4frnkk06JpFAZeR+9GGKUXZHqs\nz2qcplw4d/moCC6p3rYPBMLXsrGNEUFZqBlgz72QA6BBq3X0Cg1Bc2ZbK5VIzwkg\nST2SSux6ccROfgULmN5ZiLOtdUKNEZpFF3i3qtsCgYADT/s7dYFlatobz3kmMnXK\nsfTu2MllHdRys0YGHu7Q8biDuQkhrJwhxPW0KS83g4JQym+0aEfzh36bWcl+u6R7\nKhKj+9oSf9pndgk345gJz35RbPJYh+EuAHNvzdgCAvK6x1jETWeKf6btj5pF1U1i\nQ4QNIw/QiwIXjWZeubTGsQKBgQCbduLu2rLnlyyAaJZM8DlHZyH2gAXbBZpxqU8T\nt9mtkJDUS/KRiEoYGFV9CqS0aXrayVMsDfXY6B/S/UuZjO5u7LtklDzqOf1aKG3Q\ndGXPKibknqqJYH+bnUNjuYYNerETV57lijMGHuSYCf8vwLn3oxBfERRX61M/DU8Z\nworz/QKBgQDCTJI2+jdXg26XuYUmM4XXfnocfzAXhXBULt1nENcogNf1fcptAVtu\nBAiz4/HipQKqoWVUYmxfgbbLRKKLK0s0lOWKbYdVjhEm/m2ZU8wtXTagNwkIGoyq\nY/C1Lox4f1ROJnCjc/hfcOjcxX5M8A8peecHWlVtUPKTJgxQ7oMKcw==\n-----END RSA PRIVATE KEY-----\n"
const previousCertificate = "-----BEGIN CERTIFICATE-----\nMIIBZzCCARGgAwIBAgIBAjANBgkqhkiG9w0BAQsFADAaMRgwFgYDVQQDEw9zZXJ2\naWNlLWFjY291bnQwHhcNMjEwNTAyMjAzMDA2WhcNMzEwNTAyMjAzMDA2WjAaMRgw\nFgYDVQQDEw9zZXJ2aWNlLWFjY291bnQwXDANBgkqhkiG9w0BAQEFAANLADBIAkEA\n2JbeF8dNwqfEKKD65aGlVs58fWkA0qZdVLKw8qATzRBJTi1nqbj2kAR4gyy/C8Mx\nouxva/om9d7Sq8Ka55T7+wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T\nAQH/BAUwAwEB/zAdBgNVHQ4EFgQUI5beFHueAGyT1pQ6UTOdbMfj3gQwDQYJKoZI\nhvcNAQELBQADQQBwPLO+Np8o6k3aNBGKE4JTCOs06X72OXNivkWWWP/9XGz6x4DI\nHPU65kbUn/pWXBUVVlpsKsdmWA2Bu8pd/vD+\n-----END CERTIFICATE-----\n"
const previousKey = "-----BEGIN RSA PRIVATE KEY-----\nMIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4\n9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R\n2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo\nxTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+\nZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr\nKl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh\nAOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY\n-----END RSA PRIVATE KEY-----"
const nextCertificate = "-----BEGIN CERTIFICATE-----\nMIIBZzCCARGgAwIBAgIBBDANBgkqhkiG9w0BAQsFADAaMRgwFgYDVQQDEw9zZXJ2\naWNlLWFjY291bnQwHhcNMjEwNTAyMjAzMjE3WhcNMzEwNTAyMjAzMjE3WjAaMRgw\nFgYDVQQDEw9zZXJ2aWNlLWFjY291bnQwXDANBgkqhkiG9w0BAQEFAANLADBIAkEA\no4Tridlsf4Yz3UAiup/scSTiG/OqxkUW3Fz7zGKvVcLeYj9GEIKuzoB1VFk1nboD\nq4cCuGLfdzaQdCQKPIsDuwIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T\nAQH/BAUwAwEB/zAdBgNVHQ4EFgQUhPbxEmUbwVOCa+fZgxreFhf67UEwDQYJKoZI\nhvcNAQELBQADQQALMsyK2Q7C/bk27eCvXyZKUfrLvor10hEjwGhv14zsKWDeTj/J\nA1LPYp7U9VtFfgFOkVbkLE9Rstc0ltNrPqxA\n-----END CERTIFICATE-----\n"
const nextKey = "-----BEGIN RSA PRIVATE KEY-----\nMIIBOgIBAAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCC\nrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQJAejInjmEzqmzQr0NxcIN4\nPukwK3FBKl+RAOZfqNIKcww14mfOn7Gc6lF2zEC4GnLiB3tthbSXoBGi54nkW4ki\nyQIhANZNne9UhQlwyjsd3WxDWWrl6OOZ3J8ppMOIQni9WRLlAiEAw1XEdxPOSOSO\nB6rucpTT1QivVvyEFIb/ukvPm769Mh8CIQDNQwKnHdlfNX0+KljPPaMD1LrAZbr/\naC+8aWLhqtsKUQIgF7gUcTkwdV17eabh6Xv09Qtm7zMefred2etWvFy+8JUCIECv\nFYOKQVWHX+Q7CHX2K1oTECVnZuW1UItdDYVlFYxQ\n-----END RSA PRIVATE KEY-----"
func mustParsePrivateKey(s string) *pki.PrivateKey {
k, err := pki.ParsePEMPrivateKey([]byte(s))
if err != nil {
klog.Fatalf("error parsing private key %v", err)
func simplePrivateKeyset(s string) *kops.Keyset {
return &kops.Keyset{
Spec: kops.KeysetSpec{
PrimaryId: "3",
Keys: []kops.KeysetItem{
{
Id: "3",
PrivateMaterial: []byte(s),
},
},
},
}
}
func rotatingPrivateKeyset() *kops.Keyset {
return &kops.Keyset{
Spec: kops.KeysetSpec{
PrimaryId: "3",
Keys: []kops.KeysetItem{
{
Id: "2",
PrivateMaterial: []byte(previousKey),
PublicMaterial: []byte(previousCertificate),
},
{
Id: "3",
PrivateMaterial: []byte(dummyKey),
PublicMaterial: []byte(dummyCertificate),
},
{
Id: "4",
PrivateMaterial: []byte(nextKey),
PublicMaterial: []byte(nextCertificate),
},
},
},
}
return k
}
func mustParseCertificate(s string) *pki.Certificate {
@ -303,13 +338,13 @@ func RunGoldenTest(t *testing.T, basedir string, key string, builder func(*Nodeu
keystore := &fakeCAStore{}
keystore.T = t
keystore.privateKeys = map[string]*pki.PrivateKey{
"ca": mustParsePrivateKey(dummyKey),
"apiserver-aggregator-ca": mustParsePrivateKey(dummyKey),
"kube-controller-manager": mustParsePrivateKey(dummyKey),
"kube-proxy": mustParsePrivateKey(dummyKey),
"kube-scheduler": mustParsePrivateKey(dummyKey),
"service-account": mustParsePrivateKey(dummyKey),
keystore.privateKeysets = map[string]*kops.Keyset{
"ca": simplePrivateKeyset(dummyKey),
"apiserver-aggregator-ca": simplePrivateKeyset(dummyKey),
"kube-controller-manager": simplePrivateKeyset(dummyKey),
"kube-proxy": simplePrivateKeyset(dummyKey),
"kube-scheduler": simplePrivateKeyset(dummyKey),
"service-account": rotatingPrivateKeyset(),
}
keystore.certs = map[string]*pki.Certificate{
"ca": mustParseCertificate(dummyCertificate),

24
nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml
View File

@ -62,7 +62,7 @@ contents: |
- --requestheader-group-headers=X-Remote-Group
- --requestheader-username-headers=X-Remote-User
- --secure-port=443
- --service-account-key-file=/srv/kubernetes/service-account.key
- --service-account-key-file=/srv/kubernetes/service-account.pub
- --service-cluster-ip-range=100.64.0.0/13
- --storage-backend=etcd3
- --tls-cert-file=/srv/kubernetes/server.crt
@ -248,6 +248,28 @@ mode: "0600"
path: /srv/kubernetes/kubelet-api.key
type: file
---
contents: |
-----BEGIN RSA PUBLIC KEY-----
MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm
XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ==
-----END RSA PUBLIC KEY-----
-----BEGIN RSA PUBLIC KEY-----
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l
Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD
ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp
OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m
74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG
kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF
6QIDAQAB
-----END RSA PUBLIC KEY-----
-----BEGIN RSA PUBLIC KEY-----
MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF
Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ==
-----END RSA PUBLIC KEY-----
mode: "0600"
path: /srv/kubernetes/service-account.pub
type: file
---
contents: ""
ifNotExists: true
mode: "0400"

24
nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml
View File

@ -40,7 +40,7 @@ contents: |
- --requestheader-group-headers=X-Remote-Group
- --requestheader-username-headers=X-Remote-User
- --secure-port=443
- --service-account-key-file=/srv/kubernetes/service-account.key
- --service-account-key-file=/srv/kubernetes/service-account.pub
- --service-cluster-ip-range=100.64.0.0/13
- --storage-backend=etcd3
- --tls-cert-file=/srv/kubernetes/server.crt
@ -188,6 +188,28 @@ mode: "0600"
path: /srv/kubernetes/kubelet-api.key
type: file
---
contents: |
-----BEGIN RSA PUBLIC KEY-----
MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm
XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ==
-----END RSA PUBLIC KEY-----
-----BEGIN RSA PUBLIC KEY-----
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l
Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD
ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp
OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m
74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG
kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF
6QIDAQAB
-----END RSA PUBLIC KEY-----
-----BEGIN RSA PUBLIC KEY-----
MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF
Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ==
-----END RSA PUBLIC KEY-----
mode: "0600"
path: /srv/kubernetes/service-account.pub
type: file
---
contents: ""
ifNotExists: true
mode: "0400"

24
nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml
View File

@ -40,7 +40,7 @@ contents: |
- --requestheader-group-headers=X-Remote-Group
- --requestheader-username-headers=X-Remote-User
- --secure-port=443
- --service-account-key-file=/srv/kubernetes/service-account.key
- --service-account-key-file=/srv/kubernetes/service-account.pub
- --service-cluster-ip-range=100.64.0.0/13
- --storage-backend=etcd3
- --tls-cert-file=/srv/kubernetes/server.crt
@ -188,6 +188,28 @@ mode: "0600"
path: /srv/kubernetes/kubelet-api.key
type: file
---
contents: |
-----BEGIN RSA PUBLIC KEY-----
MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm
XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ==
-----END RSA PUBLIC KEY-----
-----BEGIN RSA PUBLIC KEY-----
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l
Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD
ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp
OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m
74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG
kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF
6QIDAQAB
-----END RSA PUBLIC KEY-----
-----BEGIN RSA PUBLIC KEY-----
MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF
Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ==
-----END RSA PUBLIC KEY-----
mode: "0600"
path: /srv/kubernetes/service-account.pub
type: file
---
contents: ""
ifNotExists: true
mode: "0400"

24
nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml
View File

@ -40,7 +40,7 @@ contents: |
- --requestheader-group-headers=X-Remote-Group
- --requestheader-username-headers=X-Remote-User
- --secure-port=443
- --service-account-key-file=/srv/kubernetes/service-account.key
- --service-account-key-file=/srv/kubernetes/service-account.pub
- --service-cluster-ip-range=100.64.0.0/13
- --storage-backend=etcd3
- --tls-cert-file=/srv/kubernetes/server.crt
@ -188,6 +188,28 @@ mode: "0600"
path: /srv/kubernetes/kubelet-api.key
type: file
---
contents: |
-----BEGIN RSA PUBLIC KEY-----
MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm
XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ==
-----END RSA PUBLIC KEY-----
-----BEGIN RSA PUBLIC KEY-----
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l
Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD
ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp
OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m
74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG
kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF
6QIDAQAB
-----END RSA PUBLIC KEY-----
-----BEGIN RSA PUBLIC KEY-----
MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF
Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ==
-----END RSA PUBLIC KEY-----
mode: "0600"
path: /srv/kubernetes/service-account.pub
type: file
---
contents: ""
ifNotExists: true
mode: "0400"

24
nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml
View File

@ -40,7 +40,7 @@ contents: |
- --requestheader-group-headers=X-Remote-Group
- --requestheader-username-headers=X-Remote-User
- --secure-port=443
- --service-account-key-file=/srv/kubernetes/service-account.key
- --service-account-key-file=/srv/kubernetes/service-account.pub
- --service-cluster-ip-range=100.64.0.0/13
- --storage-backend=etcd3
- --tls-cert-file=/srv/kubernetes/server.crt
@ -188,6 +188,28 @@ mode: "0600"
path: /srv/kubernetes/kubelet-api.key
type: file
---
contents: |
-----BEGIN RSA PUBLIC KEY-----
MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm
XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ==
-----END RSA PUBLIC KEY-----
-----BEGIN RSA PUBLIC KEY-----
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l
Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD
ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp
OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m
74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG
kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF
6QIDAQAB
-----END RSA PUBLIC KEY-----
-----BEGIN RSA PUBLIC KEY-----
MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF
Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ==
-----END RSA PUBLIC KEY-----
mode: "0600"
path: /srv/kubernetes/service-account.pub
type: file
---
contents: ""
ifNotExists: true
mode: "0400"

24
nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml
View File

@ -39,7 +39,7 @@ contents: |
- --requestheader-group-headers=X-Remote-Group
- --requestheader-username-headers=X-Remote-User
- --secure-port=443
- --service-account-key-file=/srv/kubernetes/service-account.key
- --service-account-key-file=/srv/kubernetes/service-account.pub
- --service-cluster-ip-range=100.64.0.0/13
- --storage-backend=etcd3
- --tls-cert-file=/srv/kubernetes/server.crt
@ -187,6 +187,28 @@ mode: "0600"
path: /srv/kubernetes/kubelet-api.key
type: file
---
contents: |
-----BEGIN RSA PUBLIC KEY-----
MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm
XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ==
-----END RSA PUBLIC KEY-----
-----BEGIN RSA PUBLIC KEY-----
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l
Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD
ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp
OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m
74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG
kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF
6QIDAQAB
-----END RSA PUBLIC KEY-----
-----BEGIN RSA PUBLIC KEY-----
MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF
Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ==
-----END RSA PUBLIC KEY-----
mode: "0600"
path: /srv/kubernetes/service-account.pub
type: file
---
contents: ""
ifNotExists: true
mode: "0400"

3
pkg/apis/kops/keyset.go
View File

@ -67,6 +67,9 @@ type KeysetSpec struct {
// Type is the type of the Keyset (PKI keypair, or secret token)
Type KeysetType `json:"type,omitempty"`
// PrimaryId is the id of the key used to make new signatures.
PrimaryId string `json:"primaryId,omitempty"`
// Keys is the set of keys that make up the keyset
Keys []KeysetItem `json:"keys,omitempty"`
}

3
pkg/apis/kops/v1alpha2/keyset.go
View File

@ -68,6 +68,9 @@ type KeysetSpec struct {
// Type is the type of the Keyset (PKI keypair, or secret token)
Type KeysetType `json:"type,omitempty"`
// PrimaryId is the id of the key used to make new signatures.
PrimaryId string `json:"primaryId,omitempty"`
// Keys is the set of keys that make up the keyset
Keys []KeysetItem `json:"keys,omitempty"`
}

2
pkg/apis/kops/v1alpha2/zz_generated.conversion.go generated
View File

@ -4458,6 +4458,7 @@ func Convert_kops_KeysetList_To_v1alpha2_KeysetList(in *kops.KeysetList, out *Ke
func autoConvert_v1alpha2_KeysetSpec_To_kops_KeysetSpec(in *KeysetSpec, out *kops.KeysetSpec, s conversion.Scope) error {
out.Type = kops.KeysetType(in.Type)
out.PrimaryId = in.PrimaryId
if in.Keys != nil {
in, out := &in.Keys, &out.Keys
*out = make([]kops.KeysetItem, len(*in))
@ -4479,6 +4480,7 @@ func Convert_v1alpha2_KeysetSpec_To_kops_KeysetSpec(in *KeysetSpec, out *kops.Ke
func autoConvert_kops_KeysetSpec_To_v1alpha2_KeysetSpec(in *kops.KeysetSpec, out *KeysetSpec, s conversion.Scope) error {
out.Type = KeysetType(in.Type)
out.PrimaryId = in.PrimaryId
if in.Keys != nil {
in, out := &in.Keys, &out.Keys
*out = make([]KeysetItem, len(*in))

26
pkg/configserver/keystore.go
View File

@ -38,22 +38,27 @@ func NewKeyStore(nodeConfig *nodeup.NodeConfig) fi.CAStore {
}
}
// FindKeypair implements fi.Keystore
func (s *configserverKeyStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) {
return nil, nil, false, fmt.Errorf("FindKeypair %q not supported by configserverKeyStore", name)
// FindPrimaryKeypair implements pki.Keystore
func (s *configserverKeyStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) {
return nil, nil, fmt.Errorf("FindPrimaryKeypair %q not supported by configserverKeyStore", name)
}
// FindKeypair implements fi.Keystore
// FindKeyset implements fi.Keystore
func (s *configserverKeyStore) FindKeyset(name string) (*fi.Keyset, error) {
return nil, fmt.Errorf("FindKeyset %q not supported by configserverKeyStore", name)
}
// CreateKeypair implements fi.Keystore
func (s *configserverKeyStore) CreateKeypair(signer string, name string, template *x509.Certificate, privateKey *pki.PrivateKey) (*pki.Certificate, error) {
return nil, fmt.Errorf("CreateKeypair not supported by configserverKeyStore")
}
// FindKeypair implements fi.Keystore
func (s *configserverKeyStore) StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error {
return fmt.Errorf("StoreKeypair not supported by configserverKeyStore")
// StoreKeyset implements fi.Keystore
func (s *configserverKeyStore) StoreKeyset(name string, keyset *fi.Keyset) error {
return fmt.Errorf("StoreKeyset not supported by configserverKeyStore")
}
// FindKeypair implements fi.Keystore
// MirrorTo implements fi.Keystore
func (s *configserverKeyStore) MirrorTo(basedir vfs.Path) error {
return fmt.Errorf("MirrorTo not supported by configserverKeyStore")
}
@ -104,11 +109,6 @@ func (s *configserverKeyStore) ListKeysets() ([]*kops.Keyset, error) {
return nil, fmt.Errorf("ListKeysets not supported by configserverKeyStore")
}
// AddCert implements fi.CAStore
func (s *configserverKeyStore) AddCert(name string, cert *pki.Certificate) error {
return fmt.Errorf("AddCert not supported by configserverKeyStore")
}
// DeleteKeysetItem implements fi.CAStore
func (s *configserverKeyStore) DeleteKeysetItem(item *kops.Keyset, id string) error {
return fmt.Errorf("DeleteKeysetItem not supported by configserverKeyStore")

6
pkg/kubeconfig/create_kubecfg.go
View File

@ -110,12 +110,12 @@ func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.Se
// add the CA Cert to the kubeconfig only if we didn't specify a certificate for the LB
// or if we're using admin credentials and the secondary port
if cluster.Spec.API == nil || cluster.Spec.API.LoadBalancer == nil || cluster.Spec.API.LoadBalancer.SSLCertificate == "" || cluster.Spec.API.LoadBalancer.Class == kops.LoadBalancerClassNetwork || internal {
cert, _, _, err := keyStore.FindKeypair(fi.CertificateIDCA)
keySet, err := keyStore.FindKeyset(fi.CertificateIDCA)
if err != nil {
return nil, fmt.Errorf("error fetching CA keypair: %v", err)
}
if cert != nil {
b.CACert, err = cert.AsBytes()
if keySet != nil && keySet.Primary != nil && keySet.Primary.Certificate != nil {
b.CACert, err = keySet.Primary.Certificate.AsBytes()
if err != nil {
return nil, err
}

46
pkg/kubeconfig/create_kubecfg_test.go
View File

@ -85,21 +85,25 @@ func (f fakeStatusCloud) FindClusterStatus(cluster *kops.Cluster) (*kops.Cluster
// mock a fake key store
type fakeKeyStore struct {
FindKeypairFn func(name string) (*pki.Certificate, *pki.PrivateKey, bool, error)
FindKeysetFn func(name string) (*fi.Keyset, error)
// StoreKeypair writes the keypair to the store
StoreKeypairFn func(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error
// StoreKeysetFn writes the keyset to the store.
StoreKeysetFn func(name string, keyset *fi.Keyset) error
// MirrorTo will copy secrets to a vfs.Path, which is often easier for a machine to read
MirrorToFn func(basedir vfs.Path) error
}
func (f fakeKeyStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) {
return f.FindKeypairFn(name)
func (f fakeKeyStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) {
return fi.FindPrimaryKeypair(f, name)
}
func (f fakeKeyStore) StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error {
return f.StoreKeypairFn(id, cert, privateKey)
func (f fakeKeyStore) FindKeyset(name string) (*fi.Keyset, error) {
return f.FindKeysetFn(name)
}
func (f fakeKeyStore) StoreKeyset(name string, keyset *fi.Keyset) error {
return f.StoreKeysetFn(name, keyset)
}
func (f fakeKeyStore) MirrorTo(basedir vfs.Path) error {
@ -124,16 +128,22 @@ func buildMinimalCluster(clusterName string, masterPublicName string, lbCert boo
return cluster
}
// create a fake certificate
func fakeCertificate() *pki.Certificate {
// create a fake keyset
func fakeKeyset() *fi.Keyset {
cert, _ := pki.ParsePEMCertificate([]byte(certData))
return cert
}
// create a fake private key
func fakePrivateKey() *pki.PrivateKey {
key, _ := pki.ParsePEMPrivateKey([]byte(privatekeyData))
return key
keysetItem := &fi.KeysetItem{
Id: "99",
Certificate: cert,
PrivateKey: key,
}
return &fi.Keyset{
LegacyFormat: true,
Items: map[string]*fi.KeysetItem{
"99": keysetItem,
},
Primary: keysetItem,
}
}
func TestBuildKubecfg(t *testing.T) {
@ -350,10 +360,8 @@ func TestBuildKubecfg(t *testing.T) {
kopsStateStore := "memfs://example-state-store"
keyStore := fakeKeyStore{
FindKeypairFn: func(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) {
return fakeCertificate(),
fakePrivateKey(),
true,
FindKeysetFn: func(name string) (*fi.Keyset, error) {
return fakeKeyset(),
nil
},
}

3
pkg/model/bootstrapscript.go
View File

@ -231,7 +231,8 @@ func (b *BootstrapScriptBuilder) ResourceNodeUp(c *fi.ModelBuilderContext, ig *k
ig: ig,
builder: b,
caTask: caTask,
ca: caTask.Certificate(),
// TODO: use caTask.Keyset() and expose all CA certificates
ca: caTask.Certificate(),
}
task.resource.Task = task
task.auxConfig.Task = task

42
pkg/model/issuerdiscovery.go
View File

@ -22,9 +22,9 @@ import (
"crypto/x509"
"encoding/base64"
"encoding/json"
"encoding/pem"
"fmt"
"io"
"sort"
"gopkg.in/square/go-jose.v2"
"k8s.io/kops/pkg/apis/kops"
@ -128,38 +128,32 @@ func (o *OIDCKeys) GetDependencies(tasks map[string]fi.Task) []fi.Task {
}
}
func (o *OIDCKeys) Open() (io.Reader, error) {
keyset := o.SigningKey.Keyset()
var keys []jose.JSONWebKey
certBytes, err := fi.ResourceAsBytes(o.SigningKey.Certificate())
if err != nil {
return nil, fmt.Errorf("failed to get cert: %w", err)
}
block, _ := pem.Decode(certBytes)
cert, err := x509.ParseCertificate(block.Bytes)
if err != nil {
return nil, fmt.Errorf("failed to parse cert: %w", err)
}
for _, item := range keyset.Items {
publicKey := item.Certificate.PublicKey
publicKeyDERBytes, err := x509.MarshalPKIXPublicKey(publicKey)
if err != nil {
return nil, fmt.Errorf("failed to serialize public key to DER format: %v", err)
}
publicKey := cert.PublicKey
hasher := crypto.SHA256.New()
hasher.Write(publicKeyDERBytes)
publicKeyDERHash := hasher.Sum(nil)
publicKeyDERBytes, err := x509.MarshalPKIXPublicKey(publicKey)
if err != nil {
return nil, fmt.Errorf("failed to serialize public key to DER format: %v", err)
}
keyID := base64.RawURLEncoding.EncodeToString(publicKeyDERHash)
hasher := crypto.SHA256.New()
hasher.Write(publicKeyDERBytes)
publicKeyDERHash := hasher.Sum(nil)
keyID := base64.RawURLEncoding.EncodeToString(publicKeyDERHash)
keys := []jose.JSONWebKey{
{
keys = append(keys, jose.JSONWebKey{
Key: publicKey,
KeyID: keyID,
Algorithm: string(jose.RS256),
Use: "sig",
},
})
}
sort.Slice(keys, func(i, j int) bool {
return keys[i].KeyID < keys[j].KeyID
})
keyResponse := KeyResponse{Keys: keys}
jsonBytes, err := json.MarshalIndent(keyResponse, "", "")

9
pkg/pki/issue.go
View File

@ -56,12 +56,9 @@ type IssueCertRequest struct {
}
type Keystore interface {
// FindKeypair finds a cert & private key, returning nil where either is not found
// FindPrimaryKeypair finds a cert & private key, returning nil where either is not found
// (if the certificate is found but not keypair, that is not an error: only the cert will be returned).
// This func returns a cert, private key and a bool. The bool value is whether the keypair is stored
// in a legacy format. This bool is used by a keypair
// task to convert a Legacy Keypair to the new Keypair API format.
FindKeypair(name string) (*Certificate, *PrivateKey, bool, error)
FindPrimaryKeypair(name string) (*Certificate, *PrivateKey, error)
}
// IssueCert issues a certificate, either a self-signed CA or from a CA in a keystore.
@ -119,7 +116,7 @@ func IssueCert(request *IssueCertRequest, keystore Keystore) (issuedCertificate
var signer *x509.Certificate
if !template.IsCA {
var err error
caCertificate, caPrivateKey, _, err = keystore.FindKeypair(request.Signer)
caCertificate, caPrivateKey, err = keystore.FindPrimaryKeypair(request.Signer)
if err != nil {
return nil, nil, nil, err
}

4
pkg/pki/issue_test.go
View File

@ -38,11 +38,11 @@ type mockKeystore struct {
invoked bool
}
func (m *mockKeystore) FindKeypair(name string) (*Certificate, *PrivateKey, bool, error) {
func (m *mockKeystore) FindPrimaryKeypair(name string) (*Certificate, *PrivateKey, error) {
assert.False(m.t, m.invoked, "invoked already")
m.invoked = true
assert.Equal(m.t, m.signer, name, "name argument")
return m.cert, m.key, false, nil
return m.cert, m.key, nil
}
func TestIssueCert(t *testing.T) {

1
upup/pkg/fi/BUILD.bazel generated
View File

@ -67,7 +67,6 @@ go_library(
go_test(
name = "go_default_test",
size = "small",
srcs = [
"dryruntarget_test.go",
"files_test.go",

103
upup/pkg/fi/ca.go
View File

@ -18,7 +18,12 @@ package fi
import (
"bytes"
"crypto/x509"
"encoding/pem"
"fmt"
"math/big"
"sort"
"strconv"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/pki"
@ -43,17 +48,30 @@ type KeystoreItem struct {
Data []byte
}
// Keyset is a parsed api.Keyset.
type Keyset struct {
// LegacyFormat instructs a keypair task to convert a Legacy Keyset to the new Keyset API format.
LegacyFormat bool
Items map[string]*KeysetItem
Primary *KeysetItem
}
// KeysetItem is a certificate/key pair in a Keyset.
type KeysetItem struct {
Id string
Certificate *pki.Certificate
PrivateKey *pki.PrivateKey
}
// Keystore contains just the functions we need to issue keypairs, not to list / manage them
type Keystore interface {
// FindKeypair finds a cert & private key, returning nil where either is not found
// (if the certificate is found but not keypair, that is not an error: only the cert will be returned).
// This func returns a cert, private key and a bool. The bool value is whether the keypair is stored
// in a legacy format. This bool is used by a keypair
// task to convert a Legacy Keypair to the new Keypair API format.
FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error)
pki.Keystore
// StoreKeypair writes the keypair to the store
StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error
// FindKeyset finds a Keyset.
FindKeyset(name string) (*Keyset, error)
// StoreKeyset writes a Keyset to the store.
StoreKeyset(name string, keyset *Keyset) error
// MirrorTo will copy secrets to a vfs.Path, which is often easier for a machine to read
MirrorTo(basedir vfs.Path) error
@ -86,9 +104,6 @@ type CAStore interface {
// The key material is not guaranteed to be populated - metadata like the name will be.
ListKeysets() ([]*kops.Keyset, error)
// AddCert adds an alternative certificate to the pool (primarily useful for CAs)
AddCert(name string, cert *pki.Certificate) error
// DeleteKeysetItem will delete the specified item from the Keyset
DeleteKeysetItem(item *kops.Keyset, id string) error
}
@ -145,3 +160,69 @@ func (c *CertificatePool) AsString() (string, error) {
}
return data.String(), nil
}
// FindPrimaryKeypair is a common implementation of pki.FindPrimaryKeypair.
func FindPrimaryKeypair(c Keystore, name string) (*pki.Certificate, *pki.PrivateKey, error) {
keyset, err := c.FindKeyset(name)
if err != nil {
return nil, nil, err
}
if keyset == nil || keyset.Primary == nil {
return nil, nil, nil
}
return keyset.Primary.Certificate, keyset.Primary.PrivateKey, nil
}
// AddCert adds an alternative certificate to the keyset (primarily useful for CAs)
func AddCert(keyset *Keyset, cert *pki.Certificate) {
serial := 0
for keyset.Items[strconv.Itoa(serial)] != nil {
serial++
}
keyset.Items[strconv.Itoa(serial)] = &KeysetItem{
Id: strconv.Itoa(serial),
Certificate: cert,
}
}
// KeysetItemIdOlder returns whether the KeysetItem Id a is older than b.
func KeysetItemIdOlder(a, b string) bool {
aVersion, aOk := big.NewInt(0).SetString(a, 10)
bVersion, bOk := big.NewInt(0).SetString(b, 10)
if aOk {
if !bOk {
return false
}
return aVersion.Cmp(bVersion) < 0
} else {
if bOk {
return true
}
return a < b
}
}
func (k *Keyset) ToPublicKeyBytes() ([]byte, error) {
keys := make([]string, 0, len(k.Items))
for k := range k.Items {
keys = append(keys, k)
}
sort.Slice(keys, func(i, j int) bool {
return KeysetItemIdOlder(k.Items[keys[i]].Id, k.Items[keys[j]].Id)
})
buf := new(bytes.Buffer)
for _, key := range keys {
item := k.Items[key]
publicKeyData, err := x509.MarshalPKIXPublicKey(item.Certificate.PublicKey)
if err != nil {
return nil, fmt.Errorf("marshalling public key %s: %v", item.Id, err)
}
if err = pem.Encode(buf, &pem.Block{Type: "RSA PUBLIC KEY", Bytes: publicKeyData}); err != nil {
return nil, fmt.Errorf("encoding public key %s: %v", item.Id, err)
}
}
return buf.Bytes(), nil
}

199
upup/pkg/fi/clientset_castore.go
View File

@ -21,6 +21,7 @@ import (
"context"
"fmt"
"math/big"
"sort"
"golang.org/x/crypto/ssh"
"k8s.io/apimachinery/pkg/api/errors"
@ -65,30 +66,16 @@ func NewClientsetSSHCredentialStore(cluster *kops.Cluster, clientset kopsinterna
return c
}
// keyset is a parsed Keyset
type keyset struct {
legacyFormat bool
items map[string]*keysetItem
primary *keysetItem
}
// keysetItem is a parsed KeysetItem
type keysetItem struct {
id string
certificate *pki.Certificate
privateKey *pki.PrivateKey
}
func parseKeyset(o *kops.Keyset) (*keyset, error) {
func parseKeyset(o *kops.Keyset) (*Keyset, error) {
name := o.Name
keyset := &keyset{
items: make(map[string]*keysetItem),
keyset := &Keyset{
Items: make(map[string]*KeysetItem),
}
for _, key := range o.Spec.Keys {
ki := &keysetItem{
id: key.Id,
ki := &KeysetItem{
Id: key.Id,
}
if len(key.PublicMaterial) != 0 {
cert, err := pki.ParsePEMCertificate(key.PublicMaterial)
@ -96,7 +83,7 @@ func parseKeyset(o *kops.Keyset) (*keyset, error) {
klog.Warningf("key public material was %s", key.PublicMaterial)
return nil, fmt.Errorf("error loading certificate %s/%s: %v", name, key.Id, err)
}
ki.certificate = cert
ki.Certificate = cert
}
if len(key.PrivateMaterial) != 0 {
@ -104,19 +91,19 @@ func parseKeyset(o *kops.Keyset) (*keyset, error) {
if err != nil {
return nil, fmt.Errorf("error loading private key %s/%s: %v", name, key.Id, err)
}
ki.privateKey = privateKey
ki.PrivateKey = privateKey
}
keyset.items[key.Id] = ki
keyset.Items[key.Id] = ki
}
keyset.primary = keyset.findPrimary()
keyset.Primary = keyset.Items[FindPrimary(o).Id]
return keyset, nil
}
// loadKeyset gets the named keyset and the format of the Keyset.
func (c *ClientsetCAStore) loadKeyset(ctx context.Context, name string) (*keyset, error) {
// loadKeyset gets the named Keyset and the format of the Keyset.
func (c *ClientsetCAStore) loadKeyset(ctx context.Context, name string) (*Keyset, error) {
o, err := c.clientset.Keysets(c.namespace).Get(ctx, name, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
@ -132,30 +119,13 @@ func (c *ClientsetCAStore) loadKeyset(ctx context.Context, name string) (*keyset
return keyset, nil
}
// findPrimary returns the primary keysetItem in the keyset
func (k *keyset) findPrimary() *keysetItem {
var primary *keysetItem
var primaryVersion *big.Int
for _, item := range k.items {
version, ok := big.NewInt(0).SetString(item.id, 10)
if !ok {
klog.Warningf("Ignoring key item with non-integer version: %q", item.id)
continue
}
if primaryVersion == nil || version.Cmp(primaryVersion) > 0 {
primary = item
primaryVersion = version
}
}
return primary
}
// FindPrimary returns the primary KeysetItem in the Keyset
func FindPrimary(keyset *kops.Keyset) *kops.KeysetItem {
var primary *kops.KeysetItem
var primaryVersion *big.Int
primaryId := keyset.Spec.PrimaryId
for i := range keyset.Spec.Keys {
item := &keyset.Spec.Keys[i]
version, ok := big.NewInt(0).SetString(item.Id, 10)
@ -164,6 +134,10 @@ func FindPrimary(keyset *kops.Keyset) *kops.KeysetItem {
continue
}
if item.Id == primaryId {
return item
}
if primaryVersion == nil || version.Cmp(primaryVersion) > 0 {
primary = item
primaryVersion = version
@ -172,19 +146,15 @@ func FindPrimary(keyset *kops.Keyset) *kops.KeysetItem {
return primary
}
// FindKeypair implements CAStore::FindKeypair
func (c *ClientsetCAStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) {
// FindPrimaryKeypair implements PKI::FindPrimaryKeypair
func (c *ClientsetCAStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) {
return FindPrimaryKeypair(c, name)
}
// FindKeyset implements CAStore::FindKeyset
func (c *ClientsetCAStore) FindKeyset(name string) (*Keyset, error) {
ctx := context.TODO()
keyset, err := c.loadKeyset(ctx, name)
if err != nil {
return nil, nil, false, err
}
if keyset != nil && keyset.primary != nil {
return keyset.primary.certificate, keyset.primary.privateKey, keyset.legacyFormat, nil
}
return nil, nil, false, nil
return c.loadKeyset(ctx, name)
}
// FindCert implements CAStore::FindCert
@ -195,8 +165,8 @@ func (c *ClientsetCAStore) FindCert(name string) (*pki.Certificate, error) {
return nil, err
}
if keyset != nil && keyset.primary != nil {
return keyset.primary.certificate, nil
if keyset != nil && keyset.Primary != nil {
return keyset.Primary.Certificate, nil
}
return nil, nil
@ -213,15 +183,15 @@ func (c *ClientsetCAStore) FindCertificatePool(name string) (*CertificatePool, e
pool := &CertificatePool{}
if keyset != nil {
if keyset.primary != nil {
pool.Primary = keyset.primary.certificate
if keyset.Primary != nil {
pool.Primary = keyset.Primary.Certificate
}
for id, item := range keyset.items {
if id == keyset.primary.id {
for id, item := range keyset.Items {
if id == keyset.Primary.Id {
continue
}
pool.Secondary = append(pool.Secondary, item.certificate)
pool.Secondary = append(pool.Secondary, item.Certificate)
}
}
return pool, nil
@ -288,26 +258,10 @@ func (c *ClientsetCAStore) ListSSHCredentials() ([]*kops.SSHCredential, error) {
return items, nil
}
// StoreKeypair implements CAStore::StoreKeypair
func (c *ClientsetCAStore) StoreKeypair(name string, cert *pki.Certificate, privateKey *pki.PrivateKey) error {
// StoreKeyset implements CAStore::StoreKeyset
func (c *ClientsetCAStore) StoreKeyset(name string, keyset *Keyset) error {
ctx := context.TODO()
return c.storeKeypair(ctx, name, cert.Certificate.SerialNumber.String(), cert, privateKey)
}
// AddCert implements CAStore::AddCert
func (c *ClientsetCAStore) AddCert(name string, cert *pki.Certificate) error {
ctx := context.TODO()
klog.Infof("Adding TLS certificate: %q", name)
// We add with a timestamp of zero so this will never be the newest cert
serial := pki.BuildPKISerial(0)
err := c.storeKeypair(ctx, name, serial.String(), cert, nil)
if err != nil {
return err
}
return nil
return c.storeKeyset(ctx, name, keyset, kops.SecretTypeKeypair)
}
// FindPrivateKey implements CAStore::FindPrivateKey
@ -318,8 +272,8 @@ func (c *ClientsetCAStore) FindPrivateKey(name string) (*pki.PrivateKey, error)
return nil, err
}
if keyset != nil && keyset.primary != nil {
return keyset.primary.privateKey, nil
if keyset != nil && keyset.Primary != nil {
return keyset.Primary.PrivateKey, nil
}
return nil, nil
}
@ -337,38 +291,69 @@ func (c *ClientsetCAStore) FindPrivateKeyset(name string) (*kops.Keyset, error)
return o, nil
}
// addKey saves the specified key to the registry
func (c *ClientsetCAStore) addKey(ctx context.Context, name string, keysetType kops.KeysetType, item *kops.KeysetItem) error {
// storeKeyset saves the specified keyset to the registry.
func (c *ClientsetCAStore) storeKeyset(ctx context.Context, name string, keyset *Keyset, keysetType kops.KeysetType) error {
create := false
client := c.clientset.Keysets(c.namespace)
keyset, err := client.Get(ctx, name, metav1.GetOptions{})
kopsKeyset, err := client.Get(ctx, name, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
keyset = nil
kopsKeyset = nil
} else {
return fmt.Errorf("error reading keyset %q: %v", name, err)
}
}
if keyset == nil {
keyset = &kops.Keyset{}
keyset.Name = name
keyset.Spec.Type = keysetType
if kopsKeyset == nil {
kopsKeyset = &kops.Keyset{}
kopsKeyset.Name = name
kopsKeyset.Spec.Type = keysetType
create = true
}
keyset.Spec.Keys = append(keyset.Spec.Keys, *item)
kopsKeyset.Spec.Keys = nil
kopsKeyset.Spec.PrimaryId = keyset.Primary.Id
keys := make([]string, 0, len(keyset.Items))
for k := range keyset.Items {
keys = append(keys, k)
}
sort.Slice(keys, func(i, j int) bool {
return KeysetItemIdOlder(keyset.Items[keys[i]].Id, keyset.Items[keys[j]].Id)
})
for _, key := range keys {
item := keyset.Items[key]
var publicMaterial bytes.Buffer
if _, err := item.Certificate.WriteTo(&publicMaterial); err != nil {
return err
}
var privateMaterial bytes.Buffer
if _, err := item.PrivateKey.WriteTo(&privateMaterial); err != nil {
return err
}
kopsKeyset.Spec.Keys = append(kopsKeyset.Spec.Keys, kops.KeysetItem{
Id: item.Id,
PublicMaterial: publicMaterial.Bytes(),
PrivateMaterial: privateMaterial.Bytes(),
})
}
if create {
if _, err := client.Create(ctx, keyset, metav1.CreateOptions{}); err != nil {
if _, err := client.Create(ctx, kopsKeyset, metav1.CreateOptions{}); err != nil {
return fmt.Errorf("error creating keyset %q: %v", name, err)
}
} else {
if _, err := client.Update(ctx, keyset, metav1.UpdateOptions{}); err != nil {
if _, err := client.Update(ctx, kopsKeyset, metav1.UpdateOptions{}); err != nil {
return fmt.Errorf("error updating keyset %q: %v", name, err)
}
}
return nil
}
// deleteKeysetItem deletes the specified key from the registry; deleting the whole keyset if it was the last one
// deleteKeysetItem deletes the specified key from the registry; deleting the whole Keyset if it was the last one.
func deleteKeysetItem(client kopsinternalversion.KeysetInterface, name string, keysetType kops.KeysetType, id string) error {
ctx := context.TODO()
@ -449,26 +434,6 @@ func (c *ClientsetCAStore) deleteSSHCredential(ctx context.Context, name string)
return nil
}
// addKey saves the specified keypair to the registry
func (c *ClientsetCAStore) storeKeypair(ctx context.Context, name string, id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error {
var publicMaterial bytes.Buffer
if _, err := cert.WriteTo(&publicMaterial); err != nil {
return err
}
var privateMaterial bytes.Buffer
if _, err := privateKey.WriteTo(&privateMaterial); err != nil {
return err
}
item := &kops.KeysetItem{
Id: id,
PublicMaterial: publicMaterial.Bytes(),
PrivateMaterial: privateMaterial.Bytes(),
}
return c.addKey(ctx, name, kops.SecretTypeKeypair, item)
}
// AddSSHPublicKey implements CAStore::AddSSHPublicKey
func (c *ClientsetCAStore) AddSSHPublicKey(name string, pubkey []byte) error {
ctx := context.TODO()

66
upup/pkg/fi/fitasks/keypair.go
View File

@ -17,7 +17,6 @@ limitations under the License.
package fitasks
import (
"crypto/sha1"
"crypto/x509/pkix"
"fmt"
"sort"
@ -46,8 +45,8 @@ type Keypair struct {
// LegacyFormat is whether the keypair is stored in a legacy format.
LegacyFormat bool `json:"oldFormat"`
certificate *fi.TaskDependentResource
certificateSHA1Fingerprint *fi.TaskDependentResource
certificate *fi.TaskDependentResource
keyset *fi.Keyset
}
var _ fi.HasCheckExisting = &Keypair{}
@ -70,14 +69,15 @@ func (e *Keypair) Find(c *fi.Context) (*Keypair, error) {
return nil, nil
}
cert, key, legacyFormat, err := c.Keystore.FindKeypair(name)
keyset, err := c.Keystore.FindKeyset(name)
if err != nil {
return nil, err
}
if cert == nil {
if keyset == nil || keyset.Primary == nil || keyset.Primary.Certificate == nil {
return nil, nil
}
if key == nil {
cert := keyset.Primary.Certificate
if keyset.Primary.PrivateKey == nil {
return nil, fmt.Errorf("found cert in store, but did not find private key: %q", name)
}
@ -94,7 +94,7 @@ func (e *Keypair) Find(c *fi.Context) (*Keypair, error) {
AlternateNames: alternateNames,
Subject: pki.PkixNameToString(&cert.Subject),
Type: pki.BuildTypeDescription(cert.Certificate),
LegacyFormat: legacyFormat,
LegacyFormat: keyset.LegacyFormat,
}
actual.Signer = &Keypair{Subject: pki.PkixNameToString(&cert.Certificate.Issuer)}
@ -102,7 +102,7 @@ func (e *Keypair) Find(c *fi.Context) (*Keypair, error) {
// Avoid spurious changes
actual.Lifecycle = e.Lifecycle
if err := e.setResources(cert); err != nil {
if err := e.setResources(keyset); err != nil {
return nil, fmt.Errorf("error setting resources: %v", err)
}
@ -175,14 +175,23 @@ func (_ *Keypair) Render(c *fi.Context, a, e, changes *Keypair) error {
if createCertificate {
klog.V(2).Infof("Creating PKI keypair %q", name)
_, privateKey, _, err := c.Keystore.FindKeypair(name)
keyset, err := c.Keystore.FindKeyset(name)
if err != nil {
return err
}
if keyset == nil {
keyset = &fi.Keyset{
Items: map[string]*fi.KeysetItem{},
}
}
// We always reuse the private key if it exists,
// if we change keys we often have to regenerate e.g. the service accounts
// TODO: Eventually rotate keys / don't always reuse?
var privateKey *pki.PrivateKey
if keyset.Primary != nil {
privateKey = keyset.Primary.PrivateKey
}
if privateKey == nil {
klog.V(2).Infof("Creating privateKey %q", name)
}
@ -217,17 +226,28 @@ func (_ *Keypair) Render(c *fi.Context, a, e, changes *Keypair) error {
if err != nil {
return err
}
err = c.Keystore.StoreKeypair(name, cert, privateKey)
serialString := cert.Certificate.SerialNumber.String()
ki := &fi.KeysetItem{
Id: serialString,
Certificate: cert,
PrivateKey: privateKey,
}
keyset.LegacyFormat = false
keyset.Items[ki.Id] = ki
keyset.Primary = ki
err = c.Keystore.StoreKeyset(name, keyset)
if err != nil {
return err
}
if err := e.setResources(cert); err != nil {
if err := e.setResources(keyset); err != nil {
return fmt.Errorf("error setting resources: %v", err)
}
// Make double-sure it round-trips
_, _, _, err = c.Keystore.FindKeypair(name)
_, err = c.Keystore.FindKeyset(name)
if err != nil {
return err
}
@ -240,11 +260,12 @@ func (_ *Keypair) Render(c *fi.Context, a, e, changes *Keypair) error {
if changeStoredFormat {
// We fetch and reinsert the same keypair, forcing an update to our preferred format
// TODO: We're assuming that we want to save in the preferred format
cert, privateKey, _, err := c.Keystore.FindKeypair(name)
keyset, err := c.Keystore.FindKeyset(name)
if err != nil {
return err
}
err = c.Keystore.StoreKeypair(name, cert, privateKey)
keyset.LegacyFormat = false
err = c.Keystore.StoreKeyset(name, keyset)
if err != nil {
return err
}
@ -285,30 +306,23 @@ func (e *Keypair) ensureResources() {
if e.certificate == nil {
e.certificate = &fi.TaskDependentResource{Task: e}
}
if e.certificateSHA1Fingerprint == nil {
e.certificateSHA1Fingerprint = &fi.TaskDependentResource{Task: e}
}
}
func (e *Keypair) setResources(cert *pki.Certificate) error {
func (e *Keypair) setResources(keyset *fi.Keyset) error {
e.ensureResources()
s, err := cert.AsString()
s, err := keyset.Primary.Certificate.AsString()
if err != nil {
return err
}
e.certificate.Resource = fi.NewStringResource(s)
fingerprint := sha1.Sum(cert.Certificate.Raw)
hex := fmt.Sprintf("%x", fingerprint)
e.certificateSHA1Fingerprint.Resource = fi.NewStringResource(hex)
e.keyset = keyset
return nil
}
func (e *Keypair) CertificateSHA1Fingerprint() fi.Resource {
e.ensureResources()
return e.certificateSHA1Fingerprint
func (e *Keypair) Keyset() *fi.Keyset {
return e.keyset
}
func (e *Keypair) Certificate() fi.Resource {

329
upup/pkg/fi/vfs_castore.go
View File

@ -21,6 +21,7 @@ import (
"fmt"
"math/big"
"os"
"sort"
"strings"
"sync"
@ -40,7 +41,7 @@ type VFSCAStore struct {
cluster *kops.Cluster
mutex sync.Mutex
cachedCA *keyset
cachedCA *Keyset
}
var _ CAStore = &VFSCAStore{}
@ -66,8 +67,8 @@ func NewVFSSSHCredentialStore(cluster *kops.Cluster, basedir vfs.Path) SSHCreden
return c
}
func (s *VFSCAStore) VFSPath() vfs.Path {
return s.basedir
func (c *VFSCAStore) VFSPath() vfs.Path {
return c.basedir
}
func (c *VFSCAStore) buildCertificatePoolPath(name string) vfs.Path {
@ -106,11 +107,12 @@ func (c *VFSCAStore) parseKeysetYaml(data []byte) (*kops.Keyset, bool, error) {
return keyset, gvk.Version != keysetFormatLatest, nil
}
// loadCertificatesBundle loads a keyset from the path
// loadKeyset loads a Keyset from the path.
// Returns (nil, nil) if the file is not found
// Bundles avoid the need for a list-files permission, which can be tricky on e.g. GCE
func (c *VFSCAStore) loadKeysetBundle(p vfs.Path) (*keyset, error) {
data, err := p.ReadFile()
func (c *VFSCAStore) loadKeyset(p vfs.Path) (*Keyset, error) {
bundlePath := p.Join("keyset.yaml")
data, err := bundlePath.ReadFile()
if err != nil {
if os.IsNotExist(err) {
return nil, nil
@ -128,31 +130,40 @@ func (c *VFSCAStore) loadKeysetBundle(p vfs.Path) (*keyset, error) {
return nil, fmt.Errorf("error mapping bundle %q: %v", p, err)
}
keyset.legacyFormat = legacyFormat
keyset.LegacyFormat = legacyFormat
return keyset, nil
}
func (k *keyset) ToAPIObject(name string, includePrivateKeyMaterial bool) (*kops.Keyset, error) {
func (k *Keyset) ToAPIObject(name string, includePrivateKeyMaterial bool) (*kops.Keyset, error) {
o := &kops.Keyset{}
o.Name = name
o.Spec.Type = kops.SecretTypeKeypair
for _, ki := range k.items {
keys := make([]string, 0, len(k.Items))
for k := range k.Items {
keys = append(keys, k)
}
sort.Slice(keys, func(i, j int) bool {
return KeysetItemIdOlder(k.Items[keys[i]].Id, k.Items[keys[j]].Id)
})
for _, key := range keys {
ki := k.Items[key]
oki := kops.KeysetItem{
Id: ki.id,
Id: ki.Id,
}
if ki.certificate != nil {
if ki.Certificate != nil {
var publicMaterial bytes.Buffer
if _, err := ki.certificate.WriteTo(&publicMaterial); err != nil {
if _, err := ki.Certificate.WriteTo(&publicMaterial); err != nil {
return nil, err
}
oki.PublicMaterial = publicMaterial.Bytes()
}
if includePrivateKeyMaterial && ki.privateKey != nil {
if includePrivateKeyMaterial && ki.PrivateKey != nil {
var privateMaterial bytes.Buffer
if _, err := ki.privateKey.WriteTo(&privateMaterial); err != nil {
if _, err := ki.PrivateKey.WriteTo(&privateMaterial); err != nil {
return nil, err
}
@ -161,11 +172,14 @@ func (k *keyset) ToAPIObject(name string, includePrivateKeyMaterial bool) (*kops
o.Spec.Keys = append(o.Spec.Keys, oki)
}
if k.Primary != nil {
o.Spec.PrimaryId = k.Primary.Id
}
return o, nil
}
// writeKeysetBundle writes a keyset bundle to VFS
func (c *VFSCAStore) writeKeysetBundle(p vfs.Path, name string, keyset *keyset, includePrivateKeyMaterial bool) error {
// writeKeysetBundle writes a Keyset bundle to VFS.
func (c *VFSCAStore) writeKeysetBundle(p vfs.Path, name string, keyset *Keyset, includePrivateKeyMaterial bool) error {
p = p.Join("keyset.yaml")
o, err := keyset.ToAPIObject(name, includePrivateKeyMaterial)
@ -185,7 +199,7 @@ func (c *VFSCAStore) writeKeysetBundle(p vfs.Path, name string, keyset *keyset,
return p.WriteFile(bytes.NewReader(objectData), acl)
}
// serializeKeysetBundle converts a keyset bundle to yaml, for writing to VFS
// serializeKeysetBundle converts a Keyset bundle to yaml, for writing to VFS.
func serializeKeysetBundle(o *kops.Keyset) ([]byte, error) {
var objectData bytes.Buffer
codecs := kopscodecs.Codecs
@ -203,88 +217,69 @@ func serializeKeysetBundle(o *kops.Keyset) ([]byte, error) {
// removePrivateKeyMaterial returns a copy of the Keyset with the private key data removed
func removePrivateKeyMaterial(o *kops.Keyset) *kops.Keyset {
copy := o.DeepCopy()
c := o.DeepCopy()
for i := range copy.Spec.Keys {
copy.Spec.Keys[i].PrivateMaterial = nil
for i := range c.Spec.Keys {
c.Spec.Keys[i].PrivateMaterial = nil
}
return copy
return c
}
func SerializeKeyset(o *kops.Keyset) ([]byte, error) {
var objectData bytes.Buffer
{
codecs := kopscodecs.Codecs
yaml, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), "application/yaml")
if !ok {
klog.Fatalf("no YAML serializer registered")
}
encoder := codecs.EncoderForVersion(yaml.Serializer, v1alpha2.SchemeGroupVersion)
if err := encoder.Encode(o, &objectData); err != nil {
return nil, fmt.Errorf("error serializing keyset: %v", err)
}
}
return objectData.Bytes(), nil
func (c *VFSCAStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) {
return FindPrimaryKeypair(c, name)
}
func (c *VFSCAStore) loadCertificates(p vfs.Path) (*keyset, error) {
bundlePath := p.Join("keyset.yaml")
bundle, err := c.loadKeysetBundle(bundlePath)
return bundle, err
}
func (c *VFSCAStore) FindKeyset(id string) (*Keyset, error) {
certs, err := c.loadKeyset(c.buildCertificatePoolPath(id))
func (c *VFSCAStore) loadOneCertificate(p vfs.Path) (*pki.Certificate, error) {
data, err := p.ReadFile()
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
cert, err := pki.ParsePEMCertificate(data)
if err != nil {
return nil, err
}
if cert == nil {
return nil, nil
}
return cert, nil
}
func (c *VFSCAStore) FindKeypair(id string) (*pki.Certificate, *pki.PrivateKey, bool, error) {
cert, legacyFormat, err := c.findCert(id)
if (cert == nil || os.IsNotExist(err)) && id == "service-account" {
if (certs == nil || os.IsNotExist(err)) && id == "service-account" {
// The strange name is because Kops prior to 1.19 used the api-server TLS key for this.
id = "master"
cert, _, err = c.findCert(id)
legacyFormat = true
certs, err = c.loadKeyset(c.buildCertificatePoolPath(id))
if certs != nil {
certs.LegacyFormat = true
}
}
if err != nil {
return nil, nil, false, err
return nil, err
}
key, err := c.FindPrivateKey(id)
keys, err := c.findPrivateKeyset(id)
if err != nil {
return nil, nil, false, err
return nil, err
}
return cert, key, legacyFormat, nil
if certs != nil {
if keys == nil {
return certs, nil
}
if certs.LegacyFormat {
keys.LegacyFormat = true
}
for key, certItem := range certs.Items {
keyItem := keys.Items[key]
if keyItem == nil {
keys.Items[key] = certItem
} else if keyItem.Certificate == nil {
keyItem.Certificate = certItem.Certificate
}
}
}
return keys, nil
}
func (c *VFSCAStore) findCert(name string) (*pki.Certificate, bool, error) {
p := c.buildCertificatePoolPath(name)
certs, err := c.loadCertificates(p)
certs, err := c.loadKeyset(p)
if err != nil {
return nil, false, fmt.Errorf("error in 'FindCert' attempting to load cert %q: %v", name, err)
}
if certs != nil && certs.primary != nil {
return certs.primary.certificate, certs.legacyFormat, nil
if certs != nil && certs.Primary != nil {
return certs.Primary.Certificate, certs.LegacyFormat, nil
}
return nil, false, nil
@ -296,11 +291,11 @@ func (c *VFSCAStore) FindCert(name string) (*pki.Certificate, error) {
}
func (c *VFSCAStore) FindCertificatePool(name string) (*CertificatePool, error) {
var certs *keyset
var certs *Keyset
var err error
p := c.buildCertificatePoolPath(name)
certs, err = c.loadCertificates(p)
certs, err = c.loadKeyset(p)
if err != nil {
return nil, fmt.Errorf("error in 'FindCertificatePool' attempting to load cert %q: %v", name, err)
}
@ -308,18 +303,18 @@ func (c *VFSCAStore) FindCertificatePool(name string) (*CertificatePool, error)
pool := &CertificatePool{}
if certs != nil {
if certs.primary != nil {
pool.Primary = certs.primary.certificate
if certs.Primary != nil {
pool.Primary = certs.Primary.Certificate
}
for k, cert := range certs.items {
if certs.primary != nil && k == certs.primary.id {
for k, cert := range certs.Items {
if certs.Primary != nil && k == certs.Primary.Id {
continue
}
if cert.certificate == nil {
if cert.Certificate == nil {
continue
}
pool.Secondary = append(pool.Secondary, cert.certificate)
pool.Secondary = append(pool.Secondary, cert.Certificate)
}
}
return pool, nil
@ -327,7 +322,7 @@ func (c *VFSCAStore) FindCertificatePool(name string) (*CertificatePool, error)
func (c *VFSCAStore) FindCertificateKeyset(name string) (*kops.Keyset, error) {
p := c.buildCertificatePoolPath(name)
certs, err := c.loadCertificates(p)
certs, err := c.loadKeyset(p)
if err != nil {
return nil, fmt.Errorf("error in 'FindCertificatePool' attempting to load cert %q: %v", name, err)
}
@ -464,7 +459,7 @@ func (c *VFSCAStore) MirrorTo(basedir vfs.Path) error {
return nil
}
// mirrorKeyset writes keyset bundles for the certificates & privatekeys
// mirrorKeyset writes Keyset bundles for the certificates & privatekeys.
func mirrorKeyset(cluster *kops.Cluster, basedir vfs.Path, keyset *kops.Keyset) error {
primary := FindPrimary(keyset)
if primary == nil {
@ -535,64 +530,26 @@ func mirrorSSHCredential(cluster *kops.Cluster, basedir vfs.Path, sshCredential
return nil
}
func (c *VFSCAStore) StoreKeypair(name string, cert *pki.Certificate, privateKey *pki.PrivateKey) error {
serial := cert.Certificate.SerialNumber.String()
ki := &keysetItem{
id: serial,
certificate: cert,
privateKey: privateKey,
}
func (c *VFSCAStore) StoreKeyset(name string, keyset *Keyset) error {
{
err := c.storePrivateKey(name, ki)
if err != nil {
return err
p := c.buildPrivateKeyPoolPath(name)
if err := c.writeKeysetBundle(p, name, keyset, true); err != nil {
return fmt.Errorf("writing private bundle: %v", err)
}
}
{
err := c.storeCertificate(name, ki)
if err != nil {
// TODO: Delete private key?
return err
p := c.buildCertificatePoolPath(name)
if err := c.writeKeysetBundle(p, name, keyset, false); err != nil {
return fmt.Errorf("writing certificate bundle: %v", err)
}
}
return nil
}
func (c *VFSCAStore) AddCert(name string, cert *pki.Certificate) error {
klog.Infof("Adding TLS certificate: %q", name)
// We add with a timestamp of zero so this will never be the newest cert
serial := pki.BuildPKISerial(0).String()
p := c.buildCertificatePath(name, serial)
ki := &keysetItem{
id: serial,
certificate: cert,
}
err := c.storeCertificate(name, ki)
if err != nil {
return err
}
// Make double-sure it round-trips
_, err = c.loadOneCertificate(p)
return err
}
func (c *VFSCAStore) loadPrivateKeys(p vfs.Path) (*keyset, error) {
bundlePath := p.Join("keyset.yaml")
bundle, err := c.loadKeysetBundle(bundlePath)
return bundle, err
}
func (c *VFSCAStore) findPrivateKeyset(id string) (*keyset, error) {
var keys *keyset
func (c *VFSCAStore) findPrivateKeyset(id string) (*Keyset, error) {
var keys *Keyset
var err error
if id == CertificateIDCA {
c.mutex.Lock()
@ -603,7 +560,7 @@ func (c *VFSCAStore) findPrivateKeyset(id string) (*keyset, error) {
return cached, nil
}
keys, err = c.loadPrivateKeys(c.buildPrivateKeyPoolPath(id))
keys, err = c.loadKeyset(c.buildPrivateKeyPoolPath(id))
if err != nil {
return nil, err
}
@ -616,7 +573,7 @@ func (c *VFSCAStore) findPrivateKeyset(id string) (*keyset, error) {
}
} else {
p := c.buildPrivateKeyPoolPath(id)
keys, err = c.loadPrivateKeys(p)
keys, err = c.loadKeyset(p)
if err != nil {
return nil, err
}
@ -632,8 +589,8 @@ func (c *VFSCAStore) FindPrivateKey(id string) (*pki.PrivateKey, error) {
}
var key *pki.PrivateKey
if keys != nil && keys.primary != nil {
key = keys.primary.privateKey
if keys != nil && keys.Primary != nil {
key = keys.Primary.PrivateKey
}
return key, nil
}
@ -652,92 +609,6 @@ func (c *VFSCAStore) FindPrivateKeyset(name string) (*kops.Keyset, error) {
return o, nil
}
func (c *VFSCAStore) storePrivateKey(name string, ki *keysetItem) error {
if ki.privateKey == nil {
return fmt.Errorf("privateKey not provided to storeCertificate")
}
// Write the bundle
{
p := c.buildPrivateKeyPoolPath(name)
ks, err := c.loadPrivateKeys(p)
if err != nil {
return err
}
if ks == nil {
ks = &keyset{}
}
if ks.items == nil {
ks.items = make(map[string]*keysetItem)
}
ks.items[ki.id] = ki
if err := c.writeKeysetBundle(p, name, ks, true); err != nil {
return fmt.Errorf("error writing bundle: %v", err)
}
}
// TODO stop writing and remove legacy format files after rollback to pre-kops 1.18 not needed
// Write the data
{
var data bytes.Buffer
if _, err := ki.privateKey.WriteTo(&data); err != nil {
return err
}
p := c.buildPrivateKeyPath(name, ki.id)
acl, err := acls.GetACL(p, c.cluster)
if err != nil {
return err
}
return p.WriteFile(bytes.NewReader(data.Bytes()), acl)
}
}
func (c *VFSCAStore) storeCertificate(name string, ki *keysetItem) error {
if ki.certificate == nil {
return fmt.Errorf("certificate not provided to storeCertificate")
}
// Write the bundle
{
p := c.buildCertificatePoolPath(name)
ks, err := c.loadCertificates(p)
if err != nil {
return err
}
if ks == nil {
ks = &keyset{}
}
if ks.items == nil {
ks.items = make(map[string]*keysetItem)
}
ks.items[ki.id] = ki
if err := c.writeKeysetBundle(p, name, ks, false); err != nil {
return fmt.Errorf("error writing bundle: %v", err)
}
}
// TODO stop writing and remove legacy format files after rollback to pre-kops 1.18 not needed
// Write the data
{
var data bytes.Buffer
if _, err := ki.certificate.WriteTo(&data); err != nil {
return err
}
p := c.buildCertificatePath(name, ki.id)
acl, err := acls.GetACL(p, c.cluster)
if err != nil {
return err
}
return p.WriteFile(bytes.NewReader(data.Bytes()), acl)
}
}
func (c *VFSCAStore) deletePrivateKey(name string, id string) (bool, error) {
// Delete the file itself
{
@ -751,15 +622,18 @@ func (c *VFSCAStore) deletePrivateKey(name string, id string) (bool, error) {
// Update the bundle
{
p := c.buildPrivateKeyPoolPath(name)
ks, err := c.loadPrivateKeys(p)
ks, err := c.loadKeyset(p)
if err != nil {
return false, err
}
if ks == nil || ks.items[id] == nil {
if ks == nil || ks.Items[id] == nil {
return false, nil
}
delete(ks.items, id)
delete(ks.Items, id)
if ks.Primary != nil && ks.Primary.Id == id {
ks.Primary = nil
}
if err := c.writeKeysetBundle(p, name, ks, true); err != nil {
return false, fmt.Errorf("error writing bundle: %v", err)
@ -781,15 +655,18 @@ func (c *VFSCAStore) deleteCertificate(name string, id string) (bool, error) {
// Update the bundle
{
p := c.buildCertificatePoolPath(name)
ks, err := c.loadCertificates(p)
ks, err := c.loadKeyset(p)
if err != nil {
return false, err
}
if ks == nil || ks.items[id] == nil {
if ks == nil || ks.Items[id] == nil {
return false, nil
}
delete(ks.items, id)
delete(ks.Items, id)
if ks.Primary != nil && ks.Primary.Id == id {
ks.Primary = nil
}
if err := c.writeKeysetBundle(p, name, ks, false); err != nil {
return false, fmt.Errorf("error writing bundle: %v", err)

119
upup/pkg/fi/vfs_castore_test.go
View File

@ -24,7 +24,6 @@ import (
"testing"
"time"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/pki"
"k8s.io/kops/util/pkg/vfs"
)
@ -71,8 +70,19 @@ func TestVFSCAStoreRoundTrip(t *testing.T) {
t.Fatalf("error from ParsePEMCertificate: %v", err)
}
if err := s.StoreKeypair("ca", cert, privateKey); err != nil {
t.Fatalf("error from StoreKeypair: %v", err)
item := &KeysetItem{
Id: "237054359138908419352140518924933177492",
Certificate: cert,
PrivateKey: privateKey,
}
keyset := &Keyset{
Items: map[string]*KeysetItem{
"237054359138908419352140518924933177492": item,
},
Primary: item,
}
if err := s.StoreKeyset("ca", keyset); err != nil {
t.Fatalf("error from StoreKeyset: %v", err)
}
paths, err := basePath.ReadTree()
@ -87,16 +97,14 @@ func TestVFSCAStoreRoundTrip(t *testing.T) {
for _, p := range []string{
"memfs://tests/issued/ca/keyset.yaml",
"memfs://tests/issued/ca/237054359138908419352140518924933177492.crt",
"memfs://tests/private/ca/keyset.yaml",
"memfs://tests/private/ca/237054359138908419352140518924933177492.key",
} {
if _, found := pathMap[p]; !found {
t.Fatalf("file not found: %v", p)
}
}
if len(pathMap) != 4 {
if len(pathMap) != 2 {
t.Fatalf("unexpected pathMap: %v", pathMap)
}
@ -117,6 +125,7 @@ spec:
keys:
- id: "237054359138908419352140518924933177492"
publicMaterial: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMyRENDQWNDZ0F3SUJBZ0lSQUxKWEFrVmo5NjR0cTY3d01TSThvSlF3RFFZSktvWklodmNOQVFFTEJRQXcKRlRFVE1CRUdBMVVFQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4TnpFeU1qY3lNelV5TkRCYUZ3MHlOekV5TWpjeQpNelV5TkRCYU1CVXhFekFSQmdOVkJBTVRDbXQxWW1WeWJtVjBaWE13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBCkE0SUJEd0F3Z2dFS0FvSUJBUURnbkNrU210bm1meEVnUzNxTlBhVUNINVFPQkdESC9pbkhiV0NPRExCQ0s5Z2QKWEVjQmw3RlZ2OFQya0ZyMURZYjBIVkR0TUk3dGl4UlZGRExna3dObFczNHh3V2RaWEI3R2VvRmdVMXhXT1FTWQpPQUNDOEpnWVRRLzEzOUhCRXZncTRzZWo2N3ArL3MvU05jdzM0S2s3SEl1RmhsazFyUms1a01leEtJbEpCS1AxCllZVVlldHNKL1FwVU9rcUo1SFc0R29ldEU3Nll0SG5PUmZZdm55YnZpU01yaDJ3R0dhTjZyL3M0Q2hPYUliWkMKQW44L1lpUEtHSURhWkdwajZHWG5tWEFSUlgvVElkZ1NRa0x3dDBhVERCblBaNFh2dHBJOGFhTDhEWUpJcUF6QQpOUEgyYjQvdU55bGF0NWpEbzBiMEc1NGFnTWk5NysyQVVyQzlVVVhwQWdNQkFBR2pJekFoTUE0R0ExVWREd0VCCi93UUVBd0lCQmpBUEJnTlZIUk1CQWY4RUJUQURBUUgvTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFCVkdSMnIKaHpYelJNVTV3cmlQUUFKU2Nzek5PUnZvQnBYZlpvWjA5Rkl1cHVkRnhCVlUzZDRoVjlTdEtuUWdQU0dBNVhRTwpIRTk3K0J4SkR1QS9yQjVvQlVzTUJqYzd5MWNkZS9UNmhtaTNyTG9FWUJTblN1ZENPWEpFNEc5LzBmOGJ5QUplCnJOOCtObzFyMlZnWnZaaDZwNzRURWtYdi9sM0hCUFdNN0lkVVYwSE85SkRoU2dPVkYxZnlRS0p4UnVMSlI4anQKTzZtUEgyVVgwdk13VmE0anZ3dGtkZHFrMk9BZFlRdkg5cmJEampiemFpVzBLbm1kdWVSbzkyS0hBTjdCc0RaeQpWcFhIcHFvMUt6ZzdEM2ZwYVhDZjVzaTdscXFyZEpWWEg0SkM3Mnp4c1BlaHFnaThlSXVxT0JraURXbVJ4QXhoCjh5R2VSeDlBYmtuSGg0SWEKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
primaryId: "237054359138908419352140518924933177492"
type: Keypair
`
@ -147,18 +156,6 @@ spec:
}
}
// Check issued/ca/237054359138908419352140518924933177492.crt round-tripped
{
roundTrip, err := pathMap["memfs://tests/issued/ca/237054359138908419352140518924933177492.crt"].ReadFile()
if err != nil {
t.Fatalf("error reading file memfs://tests/issued/ca/237054359138908419352140518924933177492.crt: %v", err)
}
if string(roundTrip) != certData {
t.Fatalf("unexpected round-tripped certificate data: %q", string(roundTrip))
}
}
// Check private/ca/keyset.yaml round-tripped
{
privateKeysetYaml, err := pathMap["memfs://tests/private/ca/keyset.yaml"].ReadFile()
@ -177,6 +174,7 @@ spec:
- id: "237054359138908419352140518924933177492"
privateMaterial: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBNEp3cEVwclo1bjhSSUV0NmpUMmxBaCtVRGdSZ3gvNHB4MjFnamd5d1FpdllIVnhICkFaZXhWYi9FOXBCYTlRMkc5QjFRN1RDTzdZc1VWUlF5NEpNRFpWdCtNY0ZuV1Z3ZXhucUJZRk5jVmprRW1EZ0EKZ3ZDWUdFMFA5ZC9Sd1JMNEt1TEhvK3U2ZnY3UDBqWE1OK0NwT3h5TGhZWlpOYTBaT1pESHNTaUpTUVNqOVdHRgpHSHJiQ2YwS1ZEcEtpZVIxdUJxSHJSTyttTFI1emtYMkw1OG03NGtqSzRkc0JobWplcS83T0FvVG1pRzJRZ0ovClAySWp5aGlBMm1ScVkraGw1NWx3RVVWLzB5SFlFa0pDOExkR2t3d1p6MmVGNzdhU1BHbWkvQTJDU0tnTXdEVHgKOW0rUDdqY3BXcmVZdzZORzlCdWVHb0RJdmUvdGdGS3d2VkZGNlFJREFRQUJBb0lCQUEwa3RqYVRmeXJBeHNUSQpCZXpiN1pyNU5CVzU1ZHZ1SUkyOTljZDZNSm8rckkvVFJZaHZVdjQ4a1k4SUZYcC9oeVVqemdlREx1bnhtSWY5Ci9aZ3NvaWM5T2w0NC9nNDVtTWR1aGNHWVB6QUFlQ2RjSjVPQjlyUjlWZkRDWHlqWUxsTjhIOGlVMDczNHRUcU0KMFYxM3RROXpkU3FrR1BaT0ljcS9rUi9weWxiT1phUU1lOTdCVGxzQW5PTVNNS0RnbmZ0WTQxMjJMcTNHWXkrdAp2cHIrYktWYVFad3ZrTG9TVTNyRUNDYUthZ2hnd0N5WDdqZnQ5YUVraGRKditLbHdic0dZNldFcnZ4T2FMV0hkCmN1TVFqR2FwWTFGYS80VUQwMG12ckEyNjBOeUtmenJwNitQNDZSclZNd0VZUkpNSVE4WUJBazZONkhoN2RjMEcKOFo2aTFtMENnWUVBOUhlQ0pSMFRTd2JJUTFiRFhVcnpwZnRIdWlkRzVCblNCdGF4L05EOXFJUGhSL0ZCVzVuagoyMm53TGM0OEtreWlybGZJVUxkMGFlNHFWWEpuN3dmWWN1WC9jSk1MRG1TVnRsTTVEem1pLzkxeFJpRmdJengxCkFzYkJ6YUZqSVNQMkhwU2dMK2U5RnRTWGFhcWVaVnJmbGl0VmhZS1VwSS9BS1YzMXFHSGYwNHNDZ1lFQTZ6VFYKOTlTYjQ5V2RsbnM1SWdzZm5YbDZUb1J0dEIxOGxmRUtjVmZqQU00ZnJua2swNkpwRkFaZVIrOUdHS1VYWkhxcwp6MnFjcGx3NGQvbW9DQzZwM3JZUEJNTFhzckdORVVGWnFCbGd6NzJRQTZCQnEzWDBDZzFCYzJaYks1Vkl6d2tnClNUMlNTdXg2Y2NST2ZnVUxtTjVaaUxPdGRVS05FWnBGRjNpM3F0c0NnWUFEVC9zN2RZRmxhdG9iejNrbU1uWEsKc2ZUdTJNbGxIZFJ5czBZR0h1N1E4YmlEdVFraHJKd2h4UFcwS1M4M2c0SlF5bSswYUVmemgzNmJXY2wrdTZSNwpLaEtqKzlvU2Y5cG5kZ2szNDVnSnozNVJiUEpZaCtFdUFITnZ6ZGdDQXZLNngxakVUV2VLZjZidGo1cEYxVTFpClE0UU5Jdy9RaXdJWGpXWmV1YlRHc1FLQmdRQ2JkdUx1MnJMbmx5eUFhSlpNOERsSFp5SDJnQVhiQlpweHFVOFQKdDltdGtKRFVTL0tSaUVvWUdGVjlDcVMwYVhyYXlWTXNEZlhZNkIvUy9VdVpqTzV1N0x0a2xEenFPZjFhS0czUQpkR1hQS2lia25xcUpZSCtiblVOanVZWU5lckVUVjU3bGlqTUdIdVNZQ2Y4dndMbjNveEJmRVJSWDYxTS9EVThaCndvcnovUUtCZ1FEQ1RKSTIramRYZzI2WHVZVW1NNFhYZm5vY2Z6QVhoWEJVTHQxbkVOY29nTmYxZmNwdEFWdHUKQkFpejQvSGlwUUtxb1dWVVlteGZnYmJMUktLTEswczBsT1dLYllkVmpoRW0vbTJaVTh3dFhUYWdOd2tJR295cQpZL0MxTG94NGYxUk9KbkNqYy9oZmNPamN4WDVNOEE4cGVlY0hXbFZ0VVBLVEpneFE3b01LY3c9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo=
publicMaterial: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMyRENDQWNDZ0F3SUJBZ0lSQUxKWEFrVmo5NjR0cTY3d01TSThvSlF3RFFZSktvWklodmNOQVFFTEJRQXcKRlRFVE1CRUdBMVVFQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4TnpFeU1qY3lNelV5TkRCYUZ3MHlOekV5TWpjeQpNelV5TkRCYU1CVXhFekFSQmdOVkJBTVRDbXQxWW1WeWJtVjBaWE13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBCkE0SUJEd0F3Z2dFS0FvSUJBUURnbkNrU210bm1meEVnUzNxTlBhVUNINVFPQkdESC9pbkhiV0NPRExCQ0s5Z2QKWEVjQmw3RlZ2OFQya0ZyMURZYjBIVkR0TUk3dGl4UlZGRExna3dObFczNHh3V2RaWEI3R2VvRmdVMXhXT1FTWQpPQUNDOEpnWVRRLzEzOUhCRXZncTRzZWo2N3ArL3MvU05jdzM0S2s3SEl1RmhsazFyUms1a01leEtJbEpCS1AxCllZVVlldHNKL1FwVU9rcUo1SFc0R29ldEU3Nll0SG5PUmZZdm55YnZpU01yaDJ3R0dhTjZyL3M0Q2hPYUliWkMKQW44L1lpUEtHSURhWkdwajZHWG5tWEFSUlgvVElkZ1NRa0x3dDBhVERCblBaNFh2dHBJOGFhTDhEWUpJcUF6QQpOUEgyYjQvdU55bGF0NWpEbzBiMEc1NGFnTWk5NysyQVVyQzlVVVhwQWdNQkFBR2pJekFoTUE0R0ExVWREd0VCCi93UUVBd0lCQmpBUEJnTlZIUk1CQWY4RUJUQURBUUgvTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFCVkdSMnIKaHpYelJNVTV3cmlQUUFKU2Nzek5PUnZvQnBYZlpvWjA5Rkl1cHVkRnhCVlUzZDRoVjlTdEtuUWdQU0dBNVhRTwpIRTk3K0J4SkR1QS9yQjVvQlVzTUJqYzd5MWNkZS9UNmhtaTNyTG9FWUJTblN1ZENPWEpFNEc5LzBmOGJ5QUplCnJOOCtObzFyMlZnWnZaaDZwNzRURWtYdi9sM0hCUFdNN0lkVVYwSE85SkRoU2dPVkYxZnlRS0p4UnVMSlI4anQKTzZtUEgyVVgwdk13VmE0anZ3dGtkZHFrMk9BZFlRdkg5cmJEampiemFpVzBLbm1kdWVSbzkyS0hBTjdCc0RaeQpWcFhIcHFvMUt6ZzdEM2ZwYVhDZjVzaTdscXFyZEpWWEg0SkM3Mnp4c1BlaHFnaThlSXVxT0JraURXbVJ4QXhoCjh5R2VSeDlBYmtuSGg0SWEKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
primaryId: "237054359138908419352140518924933177492"
type: Keypair
`
@ -202,34 +200,6 @@ spec:
t.Fatalf("unexpected round-tripped private key data: %q", roundTrip)
}
}
// Check private/ca/237054359138908419352140518924933177492.key round-tripped
{
roundTrip, err := pathMap["memfs://tests/private/ca/237054359138908419352140518924933177492.key"].ReadFile()
if err != nil {
t.Fatalf("error reading file memfs://tests/private/ca/237054359138908419352140518924933177492.key: %v", err)
}
if string(roundTrip) != privateKeyData {
t.Fatalf("unexpected round-tripped private key data: %q", string(roundTrip))
}
}
// Check that keyset gets deleted
{
keyset := &kops.Keyset{}
keyset.Name = "ca"
keyset.Spec.Type = kops.SecretTypeKeypair
s.DeleteKeysetItem(keyset, "237054359138908419352140518924933177492")
_, err := pathMap["memfs://tests/private/ca/237054359138908419352140518924933177492.key"].ReadFile()
if err == nil {
t.Fatalf("File memfs://tests/private/ca/237054359138908419352140518924933177492.key still exists")
}
}
}
func TestVFSCAStoreRoundTripWithVault(t *testing.T) {
@ -263,8 +233,19 @@ func TestVFSCAStoreRoundTripWithVault(t *testing.T) {
t.Fatalf("error from ParsePEMCertificate: %v", err)
}
if err := s.StoreKeypair("ca", cert, privateKey); err != nil {
t.Fatalf("error from StoreKeypair: %v", err)
item := &KeysetItem{
Id: "237054359138908419352140518924933177492",
Certificate: cert,
PrivateKey: privateKey,
}
keyset := &Keyset{
Items: map[string]*KeysetItem{
"237054359138908419352140518924933177492": item,
},
Primary: item,
}
if err := s.StoreKeyset("ca", keyset); err != nil {
t.Fatalf("error from StoreKeyset: %v", err)
}
paths, err := basePath.ReadTree()
@ -281,9 +262,7 @@ func TestVFSCAStoreRoundTripWithVault(t *testing.T) {
for _, p := range []string{
bp + "/issued/ca/keyset.yaml",
bp + "/issued/ca/237054359138908419352140518924933177492.crt",
bp + "/private/ca/keyset.yaml",
bp + "/private/ca/237054359138908419352140518924933177492.key",
} {
if _, found := pathMap[p]; !found {
t.Fatalf("file not found: %v", p)
@ -341,18 +320,6 @@ spec:
}
}
// Check issued/ca/237054359138908419352140518924933177492.crt round-tripped
{
roundTrip, err := pathMap[bp+"/issued/ca/237054359138908419352140518924933177492.crt"].ReadFile()
if err != nil {
t.Fatalf("error reading file issued/ca/237054359138908419352140518924933177492.crt: %v", err)
}
if string(roundTrip) != certData {
t.Fatalf("unexpected round-tripped certificate data: %q", string(roundTrip))
}
}
// Check private/ca/keyset.yaml round-tripped
{
privateKeysetYaml, err := pathMap[bp+"/private/ca/keyset.yaml"].ReadFile()
@ -396,32 +363,4 @@ spec:
t.Fatalf("unexpected round-tripped private key data: %q", roundTrip)
}
}
// Check private/ca/237054359138908419352140518924933177492.key round-tripped
{
roundTrip, err := pathMap[bp+"/private/ca/237054359138908419352140518924933177492.key"].ReadFile()
if err != nil {
t.Fatalf("error reading file private/ca/237054359138908419352140518924933177492.key: %v", err)
}
if string(roundTrip) != privateKeyData {
t.Fatalf("unexpected round-tripped private key data: %q", string(roundTrip))
}
}
// Check that keyset gets deleted
{
keyset := &kops.Keyset{}
keyset.Name = "ca"
keyset.Spec.Type = kops.SecretTypeKeypair
s.DeleteKeysetItem(keyset, "237054359138908419352140518924933177492")
_, err := pathMap[bp+"/private/ca/237054359138908419352140518924933177492.key"].ReadFile()
if err == nil {
t.Fatalf("File private/ca/237054359138908419352140518924933177492.key still exists")
}
}
}

18
upup/pkg/kutil/convert_kubeup_cluster.go
View File

@ -487,17 +487,19 @@ func (x *ConvertKubeupCluster) Upgrade(ctx context.Context) error {
if oldCACertPool == nil {
return fmt.Errorf("cannot find certificate pool %q", fi.CertificateIDCA)
}
keyset, err := newKeyStore.FindKeyset(fi.CertificateIDCA)
if err != nil {
return fmt.Errorf("error reading new CA certs: %v", err)
}
for _, ca := range oldCACertPool.Secondary {
err := newKeyStore.AddCert(fi.CertificateIDCA, ca)
if err != nil {
return fmt.Errorf("error importing old CA certs: %v", err)
}
fi.AddCert(keyset, ca)
}
if oldCACertPool.Primary != nil {
err := newKeyStore.AddCert(fi.CertificateIDCA, oldCACertPool.Primary)
if err != nil {
return fmt.Errorf("error importing old CA certs: %v", err)
}
fi.AddCert(keyset, oldCACertPool.Primary)
}
err = newKeyStore.StoreKeyset(fi.CertificateIDCA, keyset)
if err != nil {
return fmt.Errorf("error importing old CA certs: %v", err)
}
return nil

7
upup/pkg/kutil/import_cluster.go
View File

@ -424,7 +424,12 @@ func (x *ImportCluster) ImportAWSCluster(ctx context.Context) error {
if err != nil {
return err
}
err = keyStore.AddCert(fi.CertificateIDCA, caCert)
keyset, err := keyStore.FindKeyset(fi.CertificateIDCA)
if err != nil {
return err
}
fi.AddCert(keyset, caCert)
err = keyStore.StoreKeyset(fi.CertificateIDCA, keyset)
if err != nil {
return err
}