diff --git a/cmd/kops-controller/pkg/server/keystore.go b/cmd/kops-controller/pkg/server/keystore.go index 52dfb8a52b..baa45cdeb0 100644 --- a/cmd/kops-controller/pkg/server/keystore.go +++ b/cmd/kops-controller/pkg/server/keystore.go @@ -35,12 +35,12 @@ type keystoreEntry struct { var _ pki.Keystore = keystore{} -func (k keystore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) { +func (k keystore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { entry, ok := k.keys[name] if !ok { - return nil, nil, false, fmt.Errorf("unknown CA %q", name) + return nil, nil, fmt.Errorf("unknown CA %q", name) } - return entry.certificate, entry.key, false, nil + return entry.certificate, entry.key, nil } func newKeystore(basePath string, cas []string) (pki.Keystore, error) { diff --git a/cmd/kops-controller/pkg/server/node_config.go b/cmd/kops-controller/pkg/server/node_config.go index 305a06ecf6..0ec76032e6 100644 --- a/cmd/kops-controller/pkg/server/node_config.go +++ b/cmd/kops-controller/pkg/server/node_config.go @@ -73,7 +73,7 @@ func (s *Server) getNodeConfig(ctx context.Context, req *nodeup.BootstrapRequest // We populate some certificates that we know the node will need. for _, name := range []string{"ca"} { - cert, _, _, err := s.keystore.FindKeypair(name) + cert, _, err := s.keystore.FindPrimaryKeypair(name) if err != nil { return nil, fmt.Errorf("error getting certificate %q: %w", name, err) } diff --git a/cmd/kops/create_secret_keypair_ca.go b/cmd/kops/create_secret_keypair_ca.go index 5944a0f910..077067744d 100644 --- a/cmd/kops/create_secret_keypair_ca.go +++ b/cmd/kops/create_secret_keypair_ca.go @@ -133,7 +133,20 @@ func RunCreateSecretCaCert(ctx context.Context, f *util.Factory, out io.Writer, return fmt.Errorf("error loading certificate %q: %v", options.CaCertPath, err) } - err = keyStore.StoreKeypair(fi.CertificateIDCA, cert, privateKey) + serialString := cert.Certificate.SerialNumber.String() + ki := &fi.KeysetItem{ + Id: serialString, + Certificate: cert, + PrivateKey: privateKey, + } + + err = keyStore.StoreKeyset(fi.CertificateIDCA, &fi.Keyset{ + LegacyFormat: false, + Items: map[string]*fi.KeysetItem{ + serialString: ki, + }, + Primary: ki, + }) if err != nil { return fmt.Errorf("error storing user provided keys %q %q: %v", options.CaCertPath, options.CaPrivateKeyPath, err) } diff --git a/k8s/crds/kops.k8s.io_keysets.yaml b/k8s/crds/kops.k8s.io_keysets.yaml index dd02cb401b..9329eac8d2 100644 --- a/k8s/crds/kops.k8s.io_keysets.yaml +++ b/k8s/crds/kops.k8s.io_keysets.yaml @@ -59,6 +59,9 @@ spec: type: string type: object type: array + primaryId: + description: PrimaryId is the id of the key used to make new signatures. + type: string type: description: Type is the type of the Keyset (PKI keypair, or secret token) diff --git a/nodeup/pkg/model/fakes_test.go b/nodeup/pkg/model/fakes_test.go index f126c8cdfe..4c04240c5b 100644 --- a/nodeup/pkg/model/fakes_test.go +++ b/nodeup/pkg/model/fakes_test.go @@ -18,6 +18,7 @@ package model import ( "crypto/x509" + "fmt" "testing" "k8s.io/kops/pkg/apis/kops" @@ -26,40 +27,69 @@ import ( "k8s.io/kops/util/pkg/vfs" ) -// fakeKeyStore mocks out some of fi.KeyStore, for our tests. -type fakeKeyStore struct { - T *testing.T -} - -var _ fi.Keystore = &fakeKeyStore{} - -func (k fakeKeyStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) { - panic("fakeKeyStore does not implement FindKeypair") -} - -func (k fakeKeyStore) CreateKeypair(signer string, name string, template *x509.Certificate, privateKey *pki.PrivateKey) (*pki.Certificate, error) { - panic("fakeKeyStore does not implement CreateKeypair") -} - -func (k fakeKeyStore) StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error { - panic("fakeKeyStore does not implement StoreKeypair") -} - -func (k fakeKeyStore) MirrorTo(basedir vfs.Path) error { - panic("fakeKeyStore does not implement MirrorTo") -} - // fakeCAStore mocks out some of fi.CAStore, for our tests. -// Although CAStore currently embeds KeyStore, we maintain the split here in the hope we can clean this up in future. type fakeCAStore struct { - fakeKeyStore - - privateKeys map[string]*pki.PrivateKey - certs map[string]*pki.Certificate + T *testing.T + privateKeysets map[string]*kops.Keyset + certs map[string]*pki.Certificate } var _ fi.CAStore = &fakeCAStore{} +func (k fakeCAStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { + panic("fakeCAStore does not implement FindPrimaryKeypair") +} + +func (k fakeCAStore) FindKeyset(name string) (*fi.Keyset, error) { + kopsKeyset := k.privateKeysets[name] + if kopsKeyset == nil { + return nil, nil + } + + keyset := &fi.Keyset{ + Items: make(map[string]*fi.KeysetItem), + } + + for _, key := range kopsKeyset.Spec.Keys { + ki := &fi.KeysetItem{ + Id: key.Id, + } + if len(key.PublicMaterial) != 0 { + cert, err := pki.ParsePEMCertificate(key.PublicMaterial) + if err != nil { + return nil, fmt.Errorf("error loading certificate %s/%s: %v", name, key.Id, err) + } + ki.Certificate = cert + } + + if len(key.PrivateMaterial) != 0 { + privateKey, err := pki.ParsePEMPrivateKey(key.PrivateMaterial) + if err != nil { + return nil, fmt.Errorf("error loading private key %s/%s: %v", name, key.Id, err) + } + ki.PrivateKey = privateKey + } + + keyset.Items[key.Id] = ki + } + + keyset.Primary = keyset.Items[fi.FindPrimary(kopsKeyset).Id] + + return keyset, nil +} + +func (k fakeCAStore) CreateKeypair(signer string, name string, template *x509.Certificate, privateKey *pki.PrivateKey) (*pki.Certificate, error) { + panic("fakeCAStore does not implement CreateKeypair") +} + +func (k fakeCAStore) StoreKeyset(name string, keyset *fi.Keyset) error { + panic("fakeCAStore does not implement StoreKeyset") +} + +func (k fakeCAStore) MirrorTo(basedir vfs.Path) error { + panic("fakeCAStore does not implement MirrorTo") +} + func (k fakeCAStore) FindCertificatePool(name string) (*fi.CertificatePool, error) { panic("fakeCAStore does not implement FindCertificatePool") } @@ -69,11 +99,17 @@ func (k fakeCAStore) FindCertificateKeyset(name string) (*kops.Keyset, error) { } func (k fakeCAStore) FindPrivateKey(name string) (*pki.PrivateKey, error) { - return k.privateKeys[name], nil + primaryId := k.privateKeysets[name].Spec.PrimaryId + for _, item := range k.privateKeysets[name].Spec.Keys { + if item.Id == primaryId { + return pki.ParsePEMPrivateKey(item.PrivateMaterial) + } + } + return nil, nil } func (k fakeCAStore) FindPrivateKeyset(name string) (*kops.Keyset, error) { - panic("fakeCAStore does not implement FindPrivateKeyset") + return k.privateKeysets[name], nil } func (k fakeCAStore) FindCert(name string) (*pki.Certificate, error) { @@ -84,10 +120,6 @@ func (k fakeCAStore) ListKeysets() ([]*kops.Keyset, error) { panic("fakeCAStore does not implement ListKeysets") } -func (k fakeCAStore) AddCert(name string, cert *pki.Certificate) error { - panic("fakeCAStore does not implement AddCert") -} - func (k fakeCAStore) DeleteKeysetItem(item *kops.Keyset, id string) error { panic("fakeCAStore does not implement DeleteKeysetItem") } diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go index f7ff9a43bc..1b61d471b6 100644 --- a/nodeup/pkg/model/kube_apiserver.go +++ b/nodeup/pkg/model/kube_apiserver.go @@ -81,6 +81,28 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error { } } } + { + keyset, err := b.KeyStore.FindKeyset("service-account") + if err != nil { + return err + } + + if keyset == nil { + return fmt.Errorf("service-account keyset not found") + } + + buf, err := keyset.ToPublicKeyBytes() + if err != nil { + return err + } + + c.AddTask(&nodetasks.File{ + Path: filepath.Join(b.PathSrvKubernetes(), "service-account.pub"), + Contents: fi.NewBytesResource(buf), + Type: nodetasks.FileType_File, + Mode: s("0600"), + }) + } { pod, err := b.buildPod() if err != nil { @@ -282,8 +304,7 @@ func (b *KubeAPIServerBuilder) writeAuthenticationConfig(c *fi.ModelBuilderConte func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { kubeAPIServer := b.Cluster.Spec.KubeAPIServer - // TODO pass the public key instead. We would first need to segregate the secrets better. - kubeAPIServer.ServiceAccountKeyFile = append(kubeAPIServer.ServiceAccountKeyFile, filepath.Join(b.PathSrvKubernetes(), "service-account.key")) + kubeAPIServer.ServiceAccountKeyFile = append(kubeAPIServer.ServiceAccountKeyFile, filepath.Join(b.PathSrvKubernetes(), "service-account.pub")) // Set the signing key if we're using Service Account Token VolumeProjection if kubeAPIServer.ServiceAccountSigningKeyFile == nil { diff --git a/nodeup/pkg/model/kubelet_test.go b/nodeup/pkg/model/kubelet_test.go index 84c252f542..86e92e2471 100644 --- a/nodeup/pkg/model/kubelet_test.go +++ b/nodeup/pkg/model/kubelet_test.go @@ -269,13 +269,48 @@ func mockedPopulateClusterSpec(c *kops.Cluster, cloud fi.Cloud) (*kops.Cluster, const dummyCertificate = "-----BEGIN CERTIFICATE-----\nMIIC2DCCAcCgAwIBAgIRALJXAkVj964tq67wMSI8oJQwDQYJKoZIhvcNAQELBQAw\nFTETMBEGA1UEAxMKa3ViZXJuZXRlczAeFw0xNzEyMjcyMzUyNDBaFw0yNzEyMjcy\nMzUyNDBaMBUxEzARBgNVBAMTCmt1YmVybmV0ZXMwggEiMA0GCSqGSIb3DQEBAQUA\nA4IBDwAwggEKAoIBAQDgnCkSmtnmfxEgS3qNPaUCH5QOBGDH/inHbWCODLBCK9gd\nXEcBl7FVv8T2kFr1DYb0HVDtMI7tixRVFDLgkwNlW34xwWdZXB7GeoFgU1xWOQSY\nOACC8JgYTQ/139HBEvgq4sej67p+/s/SNcw34Kk7HIuFhlk1rRk5kMexKIlJBKP1\nYYUYetsJ/QpUOkqJ5HW4GoetE76YtHnORfYvnybviSMrh2wGGaN6r/s4ChOaIbZC\nAn8/YiPKGIDaZGpj6GXnmXARRX/TIdgSQkLwt0aTDBnPZ4XvtpI8aaL8DYJIqAzA\nNPH2b4/uNylat5jDo0b0G54agMi97+2AUrC9UUXpAgMBAAGjIzAhMA4GA1UdDwEB\n/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBVGR2r\nhzXzRMU5wriPQAJScszNORvoBpXfZoZ09FIupudFxBVU3d4hV9StKnQgPSGA5XQO\nHE97+BxJDuA/rB5oBUsMBjc7y1cde/T6hmi3rLoEYBSnSudCOXJE4G9/0f8byAJe\nrN8+No1r2VgZvZh6p74TEkXv/l3HBPWM7IdUV0HO9JDhSgOVF1fyQKJxRuLJR8jt\nO6mPH2UX0vMwVa4jvwtkddqk2OAdYQvH9rbDjjbzaiW0KnmdueRo92KHAN7BsDZy\nVpXHpqo1Kzg7D3fpaXCf5si7lqqrdJVXH4JC72zxsPehqgi8eIuqOBkiDWmRxAxh\n8yGeRx9AbknHh4Ia\n-----END CERTIFICATE-----\n" const dummyKey = "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA4JwpEprZ5n8RIEt6jT2lAh+UDgRgx/4px21gjgywQivYHVxH\nAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMDZVt+McFnWVwexnqBYFNcVjkEmDgA\ngvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+CpOxyLhYZZNa0ZOZDHsSiJSQSj9WGF\nGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m74kjK4dsBhmjeq/7OAoTmiG2QgJ/\nP2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdGkwwZz2eF77aSPGmi/A2CSKgMwDTx\n9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF6QIDAQABAoIBAA0ktjaTfyrAxsTI\nBezb7Zr5NBW55dvuII299cd6MJo+rI/TRYhvUv48kY8IFXp/hyUjzgeDLunxmIf9\n/Zgsoic9Ol44/g45mMduhcGYPzAAeCdcJ5OB9rR9VfDCXyjYLlN8H8iU0734tTqM\n0V13tQ9zdSqkGPZOIcq/kR/pylbOZaQMe97BTlsAnOMSMKDgnftY4122Lq3GYy+t\nvpr+bKVaQZwvkLoSU3rECCaKaghgwCyX7jft9aEkhdJv+KlwbsGY6WErvxOaLWHd\ncuMQjGapY1Fa/4UD00mvrA260NyKfzrp6+P46RrVMwEYRJMIQ8YBAk6N6Hh7dc0G\n8Z6i1m0CgYEA9HeCJR0TSwbIQ1bDXUrzpftHuidG5BnSBtax/ND9qIPhR/FBW5nj\n22nwLc48KkyirlfIULd0ae4qVXJn7wfYcuX/cJMLDmSVtlM5Dzmi/91xRiFgIzx1\nAsbBzaFjISP2HpSgL+e9FtSXaaqeZVrflitVhYKUpI/AKV31qGHf04sCgYEA6zTV\n99Sb49Wdlns5IgsfnXl6ToRttB18lfEKcVfjAM4frnkk06JpFAZeR+9GGKUXZHqs\nz2qcplw4d/moCC6p3rYPBMLXsrGNEUFZqBlgz72QA6BBq3X0Cg1Bc2ZbK5VIzwkg\nST2SSux6ccROfgULmN5ZiLOtdUKNEZpFF3i3qtsCgYADT/s7dYFlatobz3kmMnXK\nsfTu2MllHdRys0YGHu7Q8biDuQkhrJwhxPW0KS83g4JQym+0aEfzh36bWcl+u6R7\nKhKj+9oSf9pndgk345gJz35RbPJYh+EuAHNvzdgCAvK6x1jETWeKf6btj5pF1U1i\nQ4QNIw/QiwIXjWZeubTGsQKBgQCbduLu2rLnlyyAaJZM8DlHZyH2gAXbBZpxqU8T\nt9mtkJDUS/KRiEoYGFV9CqS0aXrayVMsDfXY6B/S/UuZjO5u7LtklDzqOf1aKG3Q\ndGXPKibknqqJYH+bnUNjuYYNerETV57lijMGHuSYCf8vwLn3oxBfERRX61M/DU8Z\nworz/QKBgQDCTJI2+jdXg26XuYUmM4XXfnocfzAXhXBULt1nENcogNf1fcptAVtu\nBAiz4/HipQKqoWVUYmxfgbbLRKKLK0s0lOWKbYdVjhEm/m2ZU8wtXTagNwkIGoyq\nY/C1Lox4f1ROJnCjc/hfcOjcxX5M8A8peecHWlVtUPKTJgxQ7oMKcw==\n-----END RSA PRIVATE KEY-----\n" +const previousCertificate = "-----BEGIN CERTIFICATE-----\nMIIBZzCCARGgAwIBAgIBAjANBgkqhkiG9w0BAQsFADAaMRgwFgYDVQQDEw9zZXJ2\naWNlLWFjY291bnQwHhcNMjEwNTAyMjAzMDA2WhcNMzEwNTAyMjAzMDA2WjAaMRgw\nFgYDVQQDEw9zZXJ2aWNlLWFjY291bnQwXDANBgkqhkiG9w0BAQEFAANLADBIAkEA\n2JbeF8dNwqfEKKD65aGlVs58fWkA0qZdVLKw8qATzRBJTi1nqbj2kAR4gyy/C8Mx\nouxva/om9d7Sq8Ka55T7+wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T\nAQH/BAUwAwEB/zAdBgNVHQ4EFgQUI5beFHueAGyT1pQ6UTOdbMfj3gQwDQYJKoZI\nhvcNAQELBQADQQBwPLO+Np8o6k3aNBGKE4JTCOs06X72OXNivkWWWP/9XGz6x4DI\nHPU65kbUn/pWXBUVVlpsKsdmWA2Bu8pd/vD+\n-----END CERTIFICATE-----\n" +const previousKey = "-----BEGIN RSA PRIVATE KEY-----\nMIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4\n9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R\n2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo\nxTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+\nZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr\nKl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh\nAOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY\n-----END RSA PRIVATE KEY-----" +const nextCertificate = "-----BEGIN CERTIFICATE-----\nMIIBZzCCARGgAwIBAgIBBDANBgkqhkiG9w0BAQsFADAaMRgwFgYDVQQDEw9zZXJ2\naWNlLWFjY291bnQwHhcNMjEwNTAyMjAzMjE3WhcNMzEwNTAyMjAzMjE3WjAaMRgw\nFgYDVQQDEw9zZXJ2aWNlLWFjY291bnQwXDANBgkqhkiG9w0BAQEFAANLADBIAkEA\no4Tridlsf4Yz3UAiup/scSTiG/OqxkUW3Fz7zGKvVcLeYj9GEIKuzoB1VFk1nboD\nq4cCuGLfdzaQdCQKPIsDuwIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T\nAQH/BAUwAwEB/zAdBgNVHQ4EFgQUhPbxEmUbwVOCa+fZgxreFhf67UEwDQYJKoZI\nhvcNAQELBQADQQALMsyK2Q7C/bk27eCvXyZKUfrLvor10hEjwGhv14zsKWDeTj/J\nA1LPYp7U9VtFfgFOkVbkLE9Rstc0ltNrPqxA\n-----END CERTIFICATE-----\n" +const nextKey = "-----BEGIN RSA PRIVATE KEY-----\nMIIBOgIBAAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCC\nrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQJAejInjmEzqmzQr0NxcIN4\nPukwK3FBKl+RAOZfqNIKcww14mfOn7Gc6lF2zEC4GnLiB3tthbSXoBGi54nkW4ki\nyQIhANZNne9UhQlwyjsd3WxDWWrl6OOZ3J8ppMOIQni9WRLlAiEAw1XEdxPOSOSO\nB6rucpTT1QivVvyEFIb/ukvPm769Mh8CIQDNQwKnHdlfNX0+KljPPaMD1LrAZbr/\naC+8aWLhqtsKUQIgF7gUcTkwdV17eabh6Xv09Qtm7zMefred2etWvFy+8JUCIECv\nFYOKQVWHX+Q7CHX2K1oTECVnZuW1UItdDYVlFYxQ\n-----END RSA PRIVATE KEY-----" -func mustParsePrivateKey(s string) *pki.PrivateKey { - k, err := pki.ParsePEMPrivateKey([]byte(s)) - if err != nil { - klog.Fatalf("error parsing private key %v", err) +func simplePrivateKeyset(s string) *kops.Keyset { + return &kops.Keyset{ + Spec: kops.KeysetSpec{ + PrimaryId: "3", + Keys: []kops.KeysetItem{ + { + Id: "3", + PrivateMaterial: []byte(s), + }, + }, + }, + } +} + +func rotatingPrivateKeyset() *kops.Keyset { + return &kops.Keyset{ + Spec: kops.KeysetSpec{ + PrimaryId: "3", + Keys: []kops.KeysetItem{ + { + Id: "2", + PrivateMaterial: []byte(previousKey), + PublicMaterial: []byte(previousCertificate), + }, + { + Id: "3", + PrivateMaterial: []byte(dummyKey), + PublicMaterial: []byte(dummyCertificate), + }, + { + Id: "4", + PrivateMaterial: []byte(nextKey), + PublicMaterial: []byte(nextCertificate), + }, + }, + }, } - return k } func mustParseCertificate(s string) *pki.Certificate { @@ -303,13 +338,13 @@ func RunGoldenTest(t *testing.T, basedir string, key string, builder func(*Nodeu keystore := &fakeCAStore{} keystore.T = t - keystore.privateKeys = map[string]*pki.PrivateKey{ - "ca": mustParsePrivateKey(dummyKey), - "apiserver-aggregator-ca": mustParsePrivateKey(dummyKey), - "kube-controller-manager": mustParsePrivateKey(dummyKey), - "kube-proxy": mustParsePrivateKey(dummyKey), - "kube-scheduler": mustParsePrivateKey(dummyKey), - "service-account": mustParsePrivateKey(dummyKey), + keystore.privateKeysets = map[string]*kops.Keyset{ + "ca": simplePrivateKeyset(dummyKey), + "apiserver-aggregator-ca": simplePrivateKeyset(dummyKey), + "kube-controller-manager": simplePrivateKeyset(dummyKey), + "kube-proxy": simplePrivateKeyset(dummyKey), + "kube-scheduler": simplePrivateKeyset(dummyKey), + "service-account": rotatingPrivateKeyset(), } keystore.certs = map[string]*pki.Certificate{ "ca": mustParseCertificate(dummyCertificate), diff --git a/nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml index 65a4c0aa01..fdf2cdc6bf 100644 --- a/nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml +++ b/nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml @@ -62,7 +62,7 @@ contents: | - --requestheader-group-headers=X-Remote-Group - --requestheader-username-headers=X-Remote-User - --secure-port=443 - - --service-account-key-file=/srv/kubernetes/service-account.key + - --service-account-key-file=/srv/kubernetes/service-account.pub - --service-cluster-ip-range=100.64.0.0/13 - --storage-backend=etcd3 - --tls-cert-file=/srv/kubernetes/server.crt @@ -248,6 +248,28 @@ mode: "0600" path: /srv/kubernetes/kubelet-api.key type: file --- +contents: | + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm + XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ== + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l + Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD + ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp + OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m + 74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG + kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF + 6QIDAQAB + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF + Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ== + -----END RSA PUBLIC KEY----- +mode: "0600" +path: /srv/kubernetes/service-account.pub +type: file +--- contents: "" ifNotExists: true mode: "0400" diff --git a/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml index d368426378..daff77983b 100644 --- a/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml +++ b/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml @@ -40,7 +40,7 @@ contents: | - --requestheader-group-headers=X-Remote-Group - --requestheader-username-headers=X-Remote-User - --secure-port=443 - - --service-account-key-file=/srv/kubernetes/service-account.key + - --service-account-key-file=/srv/kubernetes/service-account.pub - --service-cluster-ip-range=100.64.0.0/13 - --storage-backend=etcd3 - --tls-cert-file=/srv/kubernetes/server.crt @@ -188,6 +188,28 @@ mode: "0600" path: /srv/kubernetes/kubelet-api.key type: file --- +contents: | + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm + XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ== + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l + Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD + ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp + OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m + 74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG + kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF + 6QIDAQAB + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF + Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ== + -----END RSA PUBLIC KEY----- +mode: "0600" +path: /srv/kubernetes/service-account.pub +type: file +--- contents: "" ifNotExists: true mode: "0400" diff --git a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml index 917e69ef52..442853faa9 100644 --- a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml +++ b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml @@ -40,7 +40,7 @@ contents: | - --requestheader-group-headers=X-Remote-Group - --requestheader-username-headers=X-Remote-User - --secure-port=443 - - --service-account-key-file=/srv/kubernetes/service-account.key + - --service-account-key-file=/srv/kubernetes/service-account.pub - --service-cluster-ip-range=100.64.0.0/13 - --storage-backend=etcd3 - --tls-cert-file=/srv/kubernetes/server.crt @@ -188,6 +188,28 @@ mode: "0600" path: /srv/kubernetes/kubelet-api.key type: file --- +contents: | + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm + XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ== + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l + Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD + ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp + OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m + 74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG + kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF + 6QIDAQAB + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF + Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ== + -----END RSA PUBLIC KEY----- +mode: "0600" +path: /srv/kubernetes/service-account.pub +type: file +--- contents: "" ifNotExists: true mode: "0400" diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml index 3ffccbc69e..80c1c18e16 100644 --- a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml +++ b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml @@ -40,7 +40,7 @@ contents: | - --requestheader-group-headers=X-Remote-Group - --requestheader-username-headers=X-Remote-User - --secure-port=443 - - --service-account-key-file=/srv/kubernetes/service-account.key + - --service-account-key-file=/srv/kubernetes/service-account.pub - --service-cluster-ip-range=100.64.0.0/13 - --storage-backend=etcd3 - --tls-cert-file=/srv/kubernetes/server.crt @@ -188,6 +188,28 @@ mode: "0600" path: /srv/kubernetes/kubelet-api.key type: file --- +contents: | + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm + XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ== + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l + Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD + ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp + OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m + 74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG + kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF + 6QIDAQAB + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF + Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ== + -----END RSA PUBLIC KEY----- +mode: "0600" +path: /srv/kubernetes/service-account.pub +type: file +--- contents: "" ifNotExists: true mode: "0400" diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml index 44597ad845..e0c1424da8 100644 --- a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml +++ b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml @@ -40,7 +40,7 @@ contents: | - --requestheader-group-headers=X-Remote-Group - --requestheader-username-headers=X-Remote-User - --secure-port=443 - - --service-account-key-file=/srv/kubernetes/service-account.key + - --service-account-key-file=/srv/kubernetes/service-account.pub - --service-cluster-ip-range=100.64.0.0/13 - --storage-backend=etcd3 - --tls-cert-file=/srv/kubernetes/server.crt @@ -188,6 +188,28 @@ mode: "0600" path: /srv/kubernetes/kubelet-api.key type: file --- +contents: | + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm + XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ== + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l + Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD + ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp + OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m + 74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG + kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF + 6QIDAQAB + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF + Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ== + -----END RSA PUBLIC KEY----- +mode: "0600" +path: /srv/kubernetes/service-account.pub +type: file +--- contents: "" ifNotExists: true mode: "0400" diff --git a/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml index 4c43f9a280..6baff4de69 100644 --- a/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml +++ b/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml @@ -39,7 +39,7 @@ contents: | - --requestheader-group-headers=X-Remote-Group - --requestheader-username-headers=X-Remote-User - --secure-port=443 - - --service-account-key-file=/srv/kubernetes/service-account.key + - --service-account-key-file=/srv/kubernetes/service-account.pub - --service-cluster-ip-range=100.64.0.0/13 - --storage-backend=etcd3 - --tls-cert-file=/srv/kubernetes/server.crt @@ -187,6 +187,28 @@ mode: "0600" path: /srv/kubernetes/kubelet-api.key type: file --- +contents: | + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm + XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ== + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l + Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD + ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp + OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m + 74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG + kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF + 6QIDAQAB + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF + Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ== + -----END RSA PUBLIC KEY----- +mode: "0600" +path: /srv/kubernetes/service-account.pub +type: file +--- contents: "" ifNotExists: true mode: "0400" diff --git a/pkg/apis/kops/keyset.go b/pkg/apis/kops/keyset.go index 459f6901bc..db05c68dfd 100644 --- a/pkg/apis/kops/keyset.go +++ b/pkg/apis/kops/keyset.go @@ -67,6 +67,9 @@ type KeysetSpec struct { // Type is the type of the Keyset (PKI keypair, or secret token) Type KeysetType `json:"type,omitempty"` + // PrimaryId is the id of the key used to make new signatures. + PrimaryId string `json:"primaryId,omitempty"` + // Keys is the set of keys that make up the keyset Keys []KeysetItem `json:"keys,omitempty"` } diff --git a/pkg/apis/kops/v1alpha2/keyset.go b/pkg/apis/kops/v1alpha2/keyset.go index 1bbb4aba75..05069e7256 100644 --- a/pkg/apis/kops/v1alpha2/keyset.go +++ b/pkg/apis/kops/v1alpha2/keyset.go @@ -68,6 +68,9 @@ type KeysetSpec struct { // Type is the type of the Keyset (PKI keypair, or secret token) Type KeysetType `json:"type,omitempty"` + // PrimaryId is the id of the key used to make new signatures. + PrimaryId string `json:"primaryId,omitempty"` + // Keys is the set of keys that make up the keyset Keys []KeysetItem `json:"keys,omitempty"` } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index ec1cde12fb..93fed20a50 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -4458,6 +4458,7 @@ func Convert_kops_KeysetList_To_v1alpha2_KeysetList(in *kops.KeysetList, out *Ke func autoConvert_v1alpha2_KeysetSpec_To_kops_KeysetSpec(in *KeysetSpec, out *kops.KeysetSpec, s conversion.Scope) error { out.Type = kops.KeysetType(in.Type) + out.PrimaryId = in.PrimaryId if in.Keys != nil { in, out := &in.Keys, &out.Keys *out = make([]kops.KeysetItem, len(*in)) @@ -4479,6 +4480,7 @@ func Convert_v1alpha2_KeysetSpec_To_kops_KeysetSpec(in *KeysetSpec, out *kops.Ke func autoConvert_kops_KeysetSpec_To_v1alpha2_KeysetSpec(in *kops.KeysetSpec, out *KeysetSpec, s conversion.Scope) error { out.Type = KeysetType(in.Type) + out.PrimaryId = in.PrimaryId if in.Keys != nil { in, out := &in.Keys, &out.Keys *out = make([]KeysetItem, len(*in)) diff --git a/pkg/configserver/keystore.go b/pkg/configserver/keystore.go index ee9f87bc5c..e6e1d7ba37 100644 --- a/pkg/configserver/keystore.go +++ b/pkg/configserver/keystore.go @@ -38,22 +38,27 @@ func NewKeyStore(nodeConfig *nodeup.NodeConfig) fi.CAStore { } } -// FindKeypair implements fi.Keystore -func (s *configserverKeyStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) { - return nil, nil, false, fmt.Errorf("FindKeypair %q not supported by configserverKeyStore", name) +// FindPrimaryKeypair implements pki.Keystore +func (s *configserverKeyStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { + return nil, nil, fmt.Errorf("FindPrimaryKeypair %q not supported by configserverKeyStore", name) } -// FindKeypair implements fi.Keystore +// FindKeyset implements fi.Keystore +func (s *configserverKeyStore) FindKeyset(name string) (*fi.Keyset, error) { + return nil, fmt.Errorf("FindKeyset %q not supported by configserverKeyStore", name) +} + +// CreateKeypair implements fi.Keystore func (s *configserverKeyStore) CreateKeypair(signer string, name string, template *x509.Certificate, privateKey *pki.PrivateKey) (*pki.Certificate, error) { return nil, fmt.Errorf("CreateKeypair not supported by configserverKeyStore") } -// FindKeypair implements fi.Keystore -func (s *configserverKeyStore) StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error { - return fmt.Errorf("StoreKeypair not supported by configserverKeyStore") +// StoreKeyset implements fi.Keystore +func (s *configserverKeyStore) StoreKeyset(name string, keyset *fi.Keyset) error { + return fmt.Errorf("StoreKeyset not supported by configserverKeyStore") } -// FindKeypair implements fi.Keystore +// MirrorTo implements fi.Keystore func (s *configserverKeyStore) MirrorTo(basedir vfs.Path) error { return fmt.Errorf("MirrorTo not supported by configserverKeyStore") } @@ -104,11 +109,6 @@ func (s *configserverKeyStore) ListKeysets() ([]*kops.Keyset, error) { return nil, fmt.Errorf("ListKeysets not supported by configserverKeyStore") } -// AddCert implements fi.CAStore -func (s *configserverKeyStore) AddCert(name string, cert *pki.Certificate) error { - return fmt.Errorf("AddCert not supported by configserverKeyStore") -} - // DeleteKeysetItem implements fi.CAStore func (s *configserverKeyStore) DeleteKeysetItem(item *kops.Keyset, id string) error { return fmt.Errorf("DeleteKeysetItem not supported by configserverKeyStore") diff --git a/pkg/kubeconfig/create_kubecfg.go b/pkg/kubeconfig/create_kubecfg.go index f9c2d21d68..2b27fd6549 100644 --- a/pkg/kubeconfig/create_kubecfg.go +++ b/pkg/kubeconfig/create_kubecfg.go @@ -110,12 +110,12 @@ func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.Se // add the CA Cert to the kubeconfig only if we didn't specify a certificate for the LB // or if we're using admin credentials and the secondary port if cluster.Spec.API == nil || cluster.Spec.API.LoadBalancer == nil || cluster.Spec.API.LoadBalancer.SSLCertificate == "" || cluster.Spec.API.LoadBalancer.Class == kops.LoadBalancerClassNetwork || internal { - cert, _, _, err := keyStore.FindKeypair(fi.CertificateIDCA) + keySet, err := keyStore.FindKeyset(fi.CertificateIDCA) if err != nil { return nil, fmt.Errorf("error fetching CA keypair: %v", err) } - if cert != nil { - b.CACert, err = cert.AsBytes() + if keySet != nil && keySet.Primary != nil && keySet.Primary.Certificate != nil { + b.CACert, err = keySet.Primary.Certificate.AsBytes() if err != nil { return nil, err } diff --git a/pkg/kubeconfig/create_kubecfg_test.go b/pkg/kubeconfig/create_kubecfg_test.go index 3deaf29c6d..c367101217 100644 --- a/pkg/kubeconfig/create_kubecfg_test.go +++ b/pkg/kubeconfig/create_kubecfg_test.go @@ -85,21 +85,25 @@ func (f fakeStatusCloud) FindClusterStatus(cluster *kops.Cluster) (*kops.Cluster // mock a fake key store type fakeKeyStore struct { - FindKeypairFn func(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) + FindKeysetFn func(name string) (*fi.Keyset, error) - // StoreKeypair writes the keypair to the store - StoreKeypairFn func(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error + // StoreKeysetFn writes the keyset to the store. + StoreKeysetFn func(name string, keyset *fi.Keyset) error // MirrorTo will copy secrets to a vfs.Path, which is often easier for a machine to read MirrorToFn func(basedir vfs.Path) error } -func (f fakeKeyStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) { - return f.FindKeypairFn(name) +func (f fakeKeyStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { + return fi.FindPrimaryKeypair(f, name) } -func (f fakeKeyStore) StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error { - return f.StoreKeypairFn(id, cert, privateKey) +func (f fakeKeyStore) FindKeyset(name string) (*fi.Keyset, error) { + return f.FindKeysetFn(name) +} + +func (f fakeKeyStore) StoreKeyset(name string, keyset *fi.Keyset) error { + return f.StoreKeysetFn(name, keyset) } func (f fakeKeyStore) MirrorTo(basedir vfs.Path) error { @@ -124,16 +128,22 @@ func buildMinimalCluster(clusterName string, masterPublicName string, lbCert boo return cluster } -// create a fake certificate -func fakeCertificate() *pki.Certificate { +// create a fake keyset +func fakeKeyset() *fi.Keyset { cert, _ := pki.ParsePEMCertificate([]byte(certData)) - return cert -} - -// create a fake private key -func fakePrivateKey() *pki.PrivateKey { key, _ := pki.ParsePEMPrivateKey([]byte(privatekeyData)) - return key + keysetItem := &fi.KeysetItem{ + Id: "99", + Certificate: cert, + PrivateKey: key, + } + return &fi.Keyset{ + LegacyFormat: true, + Items: map[string]*fi.KeysetItem{ + "99": keysetItem, + }, + Primary: keysetItem, + } } func TestBuildKubecfg(t *testing.T) { @@ -350,10 +360,8 @@ func TestBuildKubecfg(t *testing.T) { kopsStateStore := "memfs://example-state-store" keyStore := fakeKeyStore{ - FindKeypairFn: func(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) { - return fakeCertificate(), - fakePrivateKey(), - true, + FindKeysetFn: func(name string) (*fi.Keyset, error) { + return fakeKeyset(), nil }, } diff --git a/pkg/model/bootstrapscript.go b/pkg/model/bootstrapscript.go index 404f895717..9539887cf1 100644 --- a/pkg/model/bootstrapscript.go +++ b/pkg/model/bootstrapscript.go @@ -231,7 +231,8 @@ func (b *BootstrapScriptBuilder) ResourceNodeUp(c *fi.ModelBuilderContext, ig *k ig: ig, builder: b, caTask: caTask, - ca: caTask.Certificate(), + // TODO: use caTask.Keyset() and expose all CA certificates + ca: caTask.Certificate(), } task.resource.Task = task task.auxConfig.Task = task diff --git a/pkg/model/issuerdiscovery.go b/pkg/model/issuerdiscovery.go index cb84f67a73..dc56c5655c 100644 --- a/pkg/model/issuerdiscovery.go +++ b/pkg/model/issuerdiscovery.go @@ -22,9 +22,9 @@ import ( "crypto/x509" "encoding/base64" "encoding/json" - "encoding/pem" "fmt" "io" + "sort" "gopkg.in/square/go-jose.v2" "k8s.io/kops/pkg/apis/kops" @@ -128,38 +128,32 @@ func (o *OIDCKeys) GetDependencies(tasks map[string]fi.Task) []fi.Task { } } func (o *OIDCKeys) Open() (io.Reader, error) { + keyset := o.SigningKey.Keyset() + var keys []jose.JSONWebKey - certBytes, err := fi.ResourceAsBytes(o.SigningKey.Certificate()) - if err != nil { - return nil, fmt.Errorf("failed to get cert: %w", err) - } - block, _ := pem.Decode(certBytes) - cert, err := x509.ParseCertificate(block.Bytes) - if err != nil { - return nil, fmt.Errorf("failed to parse cert: %w", err) - } + for _, item := range keyset.Items { + publicKey := item.Certificate.PublicKey + publicKeyDERBytes, err := x509.MarshalPKIXPublicKey(publicKey) + if err != nil { + return nil, fmt.Errorf("failed to serialize public key to DER format: %v", err) + } - publicKey := cert.PublicKey + hasher := crypto.SHA256.New() + hasher.Write(publicKeyDERBytes) + publicKeyDERHash := hasher.Sum(nil) - publicKeyDERBytes, err := x509.MarshalPKIXPublicKey(publicKey) - if err != nil { - return nil, fmt.Errorf("failed to serialize public key to DER format: %v", err) - } + keyID := base64.RawURLEncoding.EncodeToString(publicKeyDERHash) - hasher := crypto.SHA256.New() - hasher.Write(publicKeyDERBytes) - publicKeyDERHash := hasher.Sum(nil) - - keyID := base64.RawURLEncoding.EncodeToString(publicKeyDERHash) - - keys := []jose.JSONWebKey{ - { + keys = append(keys, jose.JSONWebKey{ Key: publicKey, KeyID: keyID, Algorithm: string(jose.RS256), Use: "sig", - }, + }) } + sort.Slice(keys, func(i, j int) bool { + return keys[i].KeyID < keys[j].KeyID + }) keyResponse := KeyResponse{Keys: keys} jsonBytes, err := json.MarshalIndent(keyResponse, "", "") diff --git a/pkg/pki/issue.go b/pkg/pki/issue.go index a35e00b317..df83d568e3 100644 --- a/pkg/pki/issue.go +++ b/pkg/pki/issue.go @@ -56,12 +56,9 @@ type IssueCertRequest struct { } type Keystore interface { - // FindKeypair finds a cert & private key, returning nil where either is not found + // FindPrimaryKeypair finds a cert & private key, returning nil where either is not found // (if the certificate is found but not keypair, that is not an error: only the cert will be returned). - // This func returns a cert, private key and a bool. The bool value is whether the keypair is stored - // in a legacy format. This bool is used by a keypair - // task to convert a Legacy Keypair to the new Keypair API format. - FindKeypair(name string) (*Certificate, *PrivateKey, bool, error) + FindPrimaryKeypair(name string) (*Certificate, *PrivateKey, error) } // IssueCert issues a certificate, either a self-signed CA or from a CA in a keystore. @@ -119,7 +116,7 @@ func IssueCert(request *IssueCertRequest, keystore Keystore) (issuedCertificate var signer *x509.Certificate if !template.IsCA { var err error - caCertificate, caPrivateKey, _, err = keystore.FindKeypair(request.Signer) + caCertificate, caPrivateKey, err = keystore.FindPrimaryKeypair(request.Signer) if err != nil { return nil, nil, nil, err } diff --git a/pkg/pki/issue_test.go b/pkg/pki/issue_test.go index 4496dd2878..d2ad1860d7 100644 --- a/pkg/pki/issue_test.go +++ b/pkg/pki/issue_test.go @@ -38,11 +38,11 @@ type mockKeystore struct { invoked bool } -func (m *mockKeystore) FindKeypair(name string) (*Certificate, *PrivateKey, bool, error) { +func (m *mockKeystore) FindPrimaryKeypair(name string) (*Certificate, *PrivateKey, error) { assert.False(m.t, m.invoked, "invoked already") m.invoked = true assert.Equal(m.t, m.signer, name, "name argument") - return m.cert, m.key, false, nil + return m.cert, m.key, nil } func TestIssueCert(t *testing.T) { diff --git a/upup/pkg/fi/BUILD.bazel b/upup/pkg/fi/BUILD.bazel index 2226f923c1..040b2632ee 100644 --- a/upup/pkg/fi/BUILD.bazel +++ b/upup/pkg/fi/BUILD.bazel @@ -67,7 +67,6 @@ go_library( go_test( name = "go_default_test", - size = "small", srcs = [ "dryruntarget_test.go", "files_test.go", diff --git a/upup/pkg/fi/ca.go b/upup/pkg/fi/ca.go index 4b9402feb7..eeadebb939 100644 --- a/upup/pkg/fi/ca.go +++ b/upup/pkg/fi/ca.go @@ -18,7 +18,12 @@ package fi import ( "bytes" + "crypto/x509" + "encoding/pem" "fmt" + "math/big" + "sort" + "strconv" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/pki" @@ -43,17 +48,30 @@ type KeystoreItem struct { Data []byte } +// Keyset is a parsed api.Keyset. +type Keyset struct { + // LegacyFormat instructs a keypair task to convert a Legacy Keyset to the new Keyset API format. + LegacyFormat bool + Items map[string]*KeysetItem + Primary *KeysetItem +} + +// KeysetItem is a certificate/key pair in a Keyset. +type KeysetItem struct { + Id string + Certificate *pki.Certificate + PrivateKey *pki.PrivateKey +} + // Keystore contains just the functions we need to issue keypairs, not to list / manage them type Keystore interface { - // FindKeypair finds a cert & private key, returning nil where either is not found - // (if the certificate is found but not keypair, that is not an error: only the cert will be returned). - // This func returns a cert, private key and a bool. The bool value is whether the keypair is stored - // in a legacy format. This bool is used by a keypair - // task to convert a Legacy Keypair to the new Keypair API format. - FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) + pki.Keystore - // StoreKeypair writes the keypair to the store - StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error + // FindKeyset finds a Keyset. + FindKeyset(name string) (*Keyset, error) + + // StoreKeyset writes a Keyset to the store. + StoreKeyset(name string, keyset *Keyset) error // MirrorTo will copy secrets to a vfs.Path, which is often easier for a machine to read MirrorTo(basedir vfs.Path) error @@ -86,9 +104,6 @@ type CAStore interface { // The key material is not guaranteed to be populated - metadata like the name will be. ListKeysets() ([]*kops.Keyset, error) - // AddCert adds an alternative certificate to the pool (primarily useful for CAs) - AddCert(name string, cert *pki.Certificate) error - // DeleteKeysetItem will delete the specified item from the Keyset DeleteKeysetItem(item *kops.Keyset, id string) error } @@ -145,3 +160,69 @@ func (c *CertificatePool) AsString() (string, error) { } return data.String(), nil } + +// FindPrimaryKeypair is a common implementation of pki.FindPrimaryKeypair. +func FindPrimaryKeypair(c Keystore, name string) (*pki.Certificate, *pki.PrivateKey, error) { + keyset, err := c.FindKeyset(name) + if err != nil { + return nil, nil, err + } + if keyset == nil || keyset.Primary == nil { + return nil, nil, nil + } + return keyset.Primary.Certificate, keyset.Primary.PrivateKey, nil +} + +// AddCert adds an alternative certificate to the keyset (primarily useful for CAs) +func AddCert(keyset *Keyset, cert *pki.Certificate) { + serial := 0 + + for keyset.Items[strconv.Itoa(serial)] != nil { + serial++ + } + + keyset.Items[strconv.Itoa(serial)] = &KeysetItem{ + Id: strconv.Itoa(serial), + Certificate: cert, + } +} + +// KeysetItemIdOlder returns whether the KeysetItem Id a is older than b. +func KeysetItemIdOlder(a, b string) bool { + aVersion, aOk := big.NewInt(0).SetString(a, 10) + bVersion, bOk := big.NewInt(0).SetString(b, 10) + if aOk { + if !bOk { + return false + } + return aVersion.Cmp(bVersion) < 0 + } else { + if bOk { + return true + } + return a < b + } +} + +func (k *Keyset) ToPublicKeyBytes() ([]byte, error) { + keys := make([]string, 0, len(k.Items)) + for k := range k.Items { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + return KeysetItemIdOlder(k.Items[keys[i]].Id, k.Items[keys[j]].Id) + }) + + buf := new(bytes.Buffer) + for _, key := range keys { + item := k.Items[key] + publicKeyData, err := x509.MarshalPKIXPublicKey(item.Certificate.PublicKey) + if err != nil { + return nil, fmt.Errorf("marshalling public key %s: %v", item.Id, err) + } + if err = pem.Encode(buf, &pem.Block{Type: "RSA PUBLIC KEY", Bytes: publicKeyData}); err != nil { + return nil, fmt.Errorf("encoding public key %s: %v", item.Id, err) + } + } + return buf.Bytes(), nil +} diff --git a/upup/pkg/fi/clientset_castore.go b/upup/pkg/fi/clientset_castore.go index 3bac591722..859234aec5 100644 --- a/upup/pkg/fi/clientset_castore.go +++ b/upup/pkg/fi/clientset_castore.go @@ -21,6 +21,7 @@ import ( "context" "fmt" "math/big" + "sort" "golang.org/x/crypto/ssh" "k8s.io/apimachinery/pkg/api/errors" @@ -65,30 +66,16 @@ func NewClientsetSSHCredentialStore(cluster *kops.Cluster, clientset kopsinterna return c } -// keyset is a parsed Keyset -type keyset struct { - legacyFormat bool - items map[string]*keysetItem - primary *keysetItem -} - -// keysetItem is a parsed KeysetItem -type keysetItem struct { - id string - certificate *pki.Certificate - privateKey *pki.PrivateKey -} - -func parseKeyset(o *kops.Keyset) (*keyset, error) { +func parseKeyset(o *kops.Keyset) (*Keyset, error) { name := o.Name - keyset := &keyset{ - items: make(map[string]*keysetItem), + keyset := &Keyset{ + Items: make(map[string]*KeysetItem), } for _, key := range o.Spec.Keys { - ki := &keysetItem{ - id: key.Id, + ki := &KeysetItem{ + Id: key.Id, } if len(key.PublicMaterial) != 0 { cert, err := pki.ParsePEMCertificate(key.PublicMaterial) @@ -96,7 +83,7 @@ func parseKeyset(o *kops.Keyset) (*keyset, error) { klog.Warningf("key public material was %s", key.PublicMaterial) return nil, fmt.Errorf("error loading certificate %s/%s: %v", name, key.Id, err) } - ki.certificate = cert + ki.Certificate = cert } if len(key.PrivateMaterial) != 0 { @@ -104,19 +91,19 @@ func parseKeyset(o *kops.Keyset) (*keyset, error) { if err != nil { return nil, fmt.Errorf("error loading private key %s/%s: %v", name, key.Id, err) } - ki.privateKey = privateKey + ki.PrivateKey = privateKey } - keyset.items[key.Id] = ki + keyset.Items[key.Id] = ki } - keyset.primary = keyset.findPrimary() + keyset.Primary = keyset.Items[FindPrimary(o).Id] return keyset, nil } -// loadKeyset gets the named keyset and the format of the Keyset. -func (c *ClientsetCAStore) loadKeyset(ctx context.Context, name string) (*keyset, error) { +// loadKeyset gets the named Keyset and the format of the Keyset. +func (c *ClientsetCAStore) loadKeyset(ctx context.Context, name string) (*Keyset, error) { o, err := c.clientset.Keysets(c.namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { @@ -132,30 +119,13 @@ func (c *ClientsetCAStore) loadKeyset(ctx context.Context, name string) (*keyset return keyset, nil } -// findPrimary returns the primary keysetItem in the keyset -func (k *keyset) findPrimary() *keysetItem { - var primary *keysetItem - var primaryVersion *big.Int - - for _, item := range k.items { - version, ok := big.NewInt(0).SetString(item.id, 10) - if !ok { - klog.Warningf("Ignoring key item with non-integer version: %q", item.id) - continue - } - - if primaryVersion == nil || version.Cmp(primaryVersion) > 0 { - primary = item - primaryVersion = version - } - } - return primary -} - // FindPrimary returns the primary KeysetItem in the Keyset func FindPrimary(keyset *kops.Keyset) *kops.KeysetItem { var primary *kops.KeysetItem var primaryVersion *big.Int + + primaryId := keyset.Spec.PrimaryId + for i := range keyset.Spec.Keys { item := &keyset.Spec.Keys[i] version, ok := big.NewInt(0).SetString(item.Id, 10) @@ -164,6 +134,10 @@ func FindPrimary(keyset *kops.Keyset) *kops.KeysetItem { continue } + if item.Id == primaryId { + return item + } + if primaryVersion == nil || version.Cmp(primaryVersion) > 0 { primary = item primaryVersion = version @@ -172,19 +146,15 @@ func FindPrimary(keyset *kops.Keyset) *kops.KeysetItem { return primary } -// FindKeypair implements CAStore::FindKeypair -func (c *ClientsetCAStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) { +// FindPrimaryKeypair implements PKI::FindPrimaryKeypair +func (c *ClientsetCAStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { + return FindPrimaryKeypair(c, name) +} + +// FindKeyset implements CAStore::FindKeyset +func (c *ClientsetCAStore) FindKeyset(name string) (*Keyset, error) { ctx := context.TODO() - keyset, err := c.loadKeyset(ctx, name) - if err != nil { - return nil, nil, false, err - } - - if keyset != nil && keyset.primary != nil { - return keyset.primary.certificate, keyset.primary.privateKey, keyset.legacyFormat, nil - } - - return nil, nil, false, nil + return c.loadKeyset(ctx, name) } // FindCert implements CAStore::FindCert @@ -195,8 +165,8 @@ func (c *ClientsetCAStore) FindCert(name string) (*pki.Certificate, error) { return nil, err } - if keyset != nil && keyset.primary != nil { - return keyset.primary.certificate, nil + if keyset != nil && keyset.Primary != nil { + return keyset.Primary.Certificate, nil } return nil, nil @@ -213,15 +183,15 @@ func (c *ClientsetCAStore) FindCertificatePool(name string) (*CertificatePool, e pool := &CertificatePool{} if keyset != nil { - if keyset.primary != nil { - pool.Primary = keyset.primary.certificate + if keyset.Primary != nil { + pool.Primary = keyset.Primary.Certificate } - for id, item := range keyset.items { - if id == keyset.primary.id { + for id, item := range keyset.Items { + if id == keyset.Primary.Id { continue } - pool.Secondary = append(pool.Secondary, item.certificate) + pool.Secondary = append(pool.Secondary, item.Certificate) } } return pool, nil @@ -288,26 +258,10 @@ func (c *ClientsetCAStore) ListSSHCredentials() ([]*kops.SSHCredential, error) { return items, nil } -// StoreKeypair implements CAStore::StoreKeypair -func (c *ClientsetCAStore) StoreKeypair(name string, cert *pki.Certificate, privateKey *pki.PrivateKey) error { +// StoreKeyset implements CAStore::StoreKeyset +func (c *ClientsetCAStore) StoreKeyset(name string, keyset *Keyset) error { ctx := context.TODO() - return c.storeKeypair(ctx, name, cert.Certificate.SerialNumber.String(), cert, privateKey) -} - -// AddCert implements CAStore::AddCert -func (c *ClientsetCAStore) AddCert(name string, cert *pki.Certificate) error { - ctx := context.TODO() - klog.Infof("Adding TLS certificate: %q", name) - - // We add with a timestamp of zero so this will never be the newest cert - serial := pki.BuildPKISerial(0) - - err := c.storeKeypair(ctx, name, serial.String(), cert, nil) - if err != nil { - return err - } - - return nil + return c.storeKeyset(ctx, name, keyset, kops.SecretTypeKeypair) } // FindPrivateKey implements CAStore::FindPrivateKey @@ -318,8 +272,8 @@ func (c *ClientsetCAStore) FindPrivateKey(name string) (*pki.PrivateKey, error) return nil, err } - if keyset != nil && keyset.primary != nil { - return keyset.primary.privateKey, nil + if keyset != nil && keyset.Primary != nil { + return keyset.Primary.PrivateKey, nil } return nil, nil } @@ -337,38 +291,69 @@ func (c *ClientsetCAStore) FindPrivateKeyset(name string) (*kops.Keyset, error) return o, nil } -// addKey saves the specified key to the registry -func (c *ClientsetCAStore) addKey(ctx context.Context, name string, keysetType kops.KeysetType, item *kops.KeysetItem) error { +// storeKeyset saves the specified keyset to the registry. +func (c *ClientsetCAStore) storeKeyset(ctx context.Context, name string, keyset *Keyset, keysetType kops.KeysetType) error { create := false client := c.clientset.Keysets(c.namespace) - keyset, err := client.Get(ctx, name, metav1.GetOptions{}) + kopsKeyset, err := client.Get(ctx, name, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { - keyset = nil + kopsKeyset = nil } else { return fmt.Errorf("error reading keyset %q: %v", name, err) } } - if keyset == nil { - keyset = &kops.Keyset{} - keyset.Name = name - keyset.Spec.Type = keysetType + + if kopsKeyset == nil { + kopsKeyset = &kops.Keyset{} + kopsKeyset.Name = name + kopsKeyset.Spec.Type = keysetType create = true } - keyset.Spec.Keys = append(keyset.Spec.Keys, *item) + + kopsKeyset.Spec.Keys = nil + kopsKeyset.Spec.PrimaryId = keyset.Primary.Id + + keys := make([]string, 0, len(keyset.Items)) + for k := range keyset.Items { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + return KeysetItemIdOlder(keyset.Items[keys[i]].Id, keyset.Items[keys[j]].Id) + }) + + for _, key := range keys { + item := keyset.Items[key] + var publicMaterial bytes.Buffer + if _, err := item.Certificate.WriteTo(&publicMaterial); err != nil { + return err + } + + var privateMaterial bytes.Buffer + if _, err := item.PrivateKey.WriteTo(&privateMaterial); err != nil { + return err + } + + kopsKeyset.Spec.Keys = append(kopsKeyset.Spec.Keys, kops.KeysetItem{ + Id: item.Id, + PublicMaterial: publicMaterial.Bytes(), + PrivateMaterial: privateMaterial.Bytes(), + }) + } + if create { - if _, err := client.Create(ctx, keyset, metav1.CreateOptions{}); err != nil { + if _, err := client.Create(ctx, kopsKeyset, metav1.CreateOptions{}); err != nil { return fmt.Errorf("error creating keyset %q: %v", name, err) } } else { - if _, err := client.Update(ctx, keyset, metav1.UpdateOptions{}); err != nil { + if _, err := client.Update(ctx, kopsKeyset, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("error updating keyset %q: %v", name, err) } } return nil } -// deleteKeysetItem deletes the specified key from the registry; deleting the whole keyset if it was the last one +// deleteKeysetItem deletes the specified key from the registry; deleting the whole Keyset if it was the last one. func deleteKeysetItem(client kopsinternalversion.KeysetInterface, name string, keysetType kops.KeysetType, id string) error { ctx := context.TODO() @@ -449,26 +434,6 @@ func (c *ClientsetCAStore) deleteSSHCredential(ctx context.Context, name string) return nil } -// addKey saves the specified keypair to the registry -func (c *ClientsetCAStore) storeKeypair(ctx context.Context, name string, id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error { - var publicMaterial bytes.Buffer - if _, err := cert.WriteTo(&publicMaterial); err != nil { - return err - } - - var privateMaterial bytes.Buffer - if _, err := privateKey.WriteTo(&privateMaterial); err != nil { - return err - } - - item := &kops.KeysetItem{ - Id: id, - PublicMaterial: publicMaterial.Bytes(), - PrivateMaterial: privateMaterial.Bytes(), - } - return c.addKey(ctx, name, kops.SecretTypeKeypair, item) -} - // AddSSHPublicKey implements CAStore::AddSSHPublicKey func (c *ClientsetCAStore) AddSSHPublicKey(name string, pubkey []byte) error { ctx := context.TODO() diff --git a/upup/pkg/fi/fitasks/keypair.go b/upup/pkg/fi/fitasks/keypair.go index a2fc296e24..4240d67b44 100644 --- a/upup/pkg/fi/fitasks/keypair.go +++ b/upup/pkg/fi/fitasks/keypair.go @@ -17,7 +17,6 @@ limitations under the License. package fitasks import ( - "crypto/sha1" "crypto/x509/pkix" "fmt" "sort" @@ -46,8 +45,8 @@ type Keypair struct { // LegacyFormat is whether the keypair is stored in a legacy format. LegacyFormat bool `json:"oldFormat"` - certificate *fi.TaskDependentResource - certificateSHA1Fingerprint *fi.TaskDependentResource + certificate *fi.TaskDependentResource + keyset *fi.Keyset } var _ fi.HasCheckExisting = &Keypair{} @@ -70,14 +69,15 @@ func (e *Keypair) Find(c *fi.Context) (*Keypair, error) { return nil, nil } - cert, key, legacyFormat, err := c.Keystore.FindKeypair(name) + keyset, err := c.Keystore.FindKeyset(name) if err != nil { return nil, err } - if cert == nil { + if keyset == nil || keyset.Primary == nil || keyset.Primary.Certificate == nil { return nil, nil } - if key == nil { + cert := keyset.Primary.Certificate + if keyset.Primary.PrivateKey == nil { return nil, fmt.Errorf("found cert in store, but did not find private key: %q", name) } @@ -94,7 +94,7 @@ func (e *Keypair) Find(c *fi.Context) (*Keypair, error) { AlternateNames: alternateNames, Subject: pki.PkixNameToString(&cert.Subject), Type: pki.BuildTypeDescription(cert.Certificate), - LegacyFormat: legacyFormat, + LegacyFormat: keyset.LegacyFormat, } actual.Signer = &Keypair{Subject: pki.PkixNameToString(&cert.Certificate.Issuer)} @@ -102,7 +102,7 @@ func (e *Keypair) Find(c *fi.Context) (*Keypair, error) { // Avoid spurious changes actual.Lifecycle = e.Lifecycle - if err := e.setResources(cert); err != nil { + if err := e.setResources(keyset); err != nil { return nil, fmt.Errorf("error setting resources: %v", err) } @@ -175,14 +175,23 @@ func (_ *Keypair) Render(c *fi.Context, a, e, changes *Keypair) error { if createCertificate { klog.V(2).Infof("Creating PKI keypair %q", name) - _, privateKey, _, err := c.Keystore.FindKeypair(name) + keyset, err := c.Keystore.FindKeyset(name) if err != nil { return err } + if keyset == nil { + keyset = &fi.Keyset{ + Items: map[string]*fi.KeysetItem{}, + } + } // We always reuse the private key if it exists, // if we change keys we often have to regenerate e.g. the service accounts // TODO: Eventually rotate keys / don't always reuse? + var privateKey *pki.PrivateKey + if keyset.Primary != nil { + privateKey = keyset.Primary.PrivateKey + } if privateKey == nil { klog.V(2).Infof("Creating privateKey %q", name) } @@ -217,17 +226,28 @@ func (_ *Keypair) Render(c *fi.Context, a, e, changes *Keypair) error { if err != nil { return err } - err = c.Keystore.StoreKeypair(name, cert, privateKey) + + serialString := cert.Certificate.SerialNumber.String() + ki := &fi.KeysetItem{ + Id: serialString, + Certificate: cert, + PrivateKey: privateKey, + } + + keyset.LegacyFormat = false + keyset.Items[ki.Id] = ki + keyset.Primary = ki + err = c.Keystore.StoreKeyset(name, keyset) if err != nil { return err } - if err := e.setResources(cert); err != nil { + if err := e.setResources(keyset); err != nil { return fmt.Errorf("error setting resources: %v", err) } // Make double-sure it round-trips - _, _, _, err = c.Keystore.FindKeypair(name) + _, err = c.Keystore.FindKeyset(name) if err != nil { return err } @@ -240,11 +260,12 @@ func (_ *Keypair) Render(c *fi.Context, a, e, changes *Keypair) error { if changeStoredFormat { // We fetch and reinsert the same keypair, forcing an update to our preferred format // TODO: We're assuming that we want to save in the preferred format - cert, privateKey, _, err := c.Keystore.FindKeypair(name) + keyset, err := c.Keystore.FindKeyset(name) if err != nil { return err } - err = c.Keystore.StoreKeypair(name, cert, privateKey) + keyset.LegacyFormat = false + err = c.Keystore.StoreKeyset(name, keyset) if err != nil { return err } @@ -285,30 +306,23 @@ func (e *Keypair) ensureResources() { if e.certificate == nil { e.certificate = &fi.TaskDependentResource{Task: e} } - if e.certificateSHA1Fingerprint == nil { - e.certificateSHA1Fingerprint = &fi.TaskDependentResource{Task: e} - } } -func (e *Keypair) setResources(cert *pki.Certificate) error { +func (e *Keypair) setResources(keyset *fi.Keyset) error { e.ensureResources() - s, err := cert.AsString() + s, err := keyset.Primary.Certificate.AsString() if err != nil { return err } e.certificate.Resource = fi.NewStringResource(s) - fingerprint := sha1.Sum(cert.Certificate.Raw) - hex := fmt.Sprintf("%x", fingerprint) - e.certificateSHA1Fingerprint.Resource = fi.NewStringResource(hex) - + e.keyset = keyset return nil } -func (e *Keypair) CertificateSHA1Fingerprint() fi.Resource { - e.ensureResources() - return e.certificateSHA1Fingerprint +func (e *Keypair) Keyset() *fi.Keyset { + return e.keyset } func (e *Keypair) Certificate() fi.Resource { diff --git a/upup/pkg/fi/vfs_castore.go b/upup/pkg/fi/vfs_castore.go index a281063e85..063af57805 100644 --- a/upup/pkg/fi/vfs_castore.go +++ b/upup/pkg/fi/vfs_castore.go @@ -21,6 +21,7 @@ import ( "fmt" "math/big" "os" + "sort" "strings" "sync" @@ -40,7 +41,7 @@ type VFSCAStore struct { cluster *kops.Cluster mutex sync.Mutex - cachedCA *keyset + cachedCA *Keyset } var _ CAStore = &VFSCAStore{} @@ -66,8 +67,8 @@ func NewVFSSSHCredentialStore(cluster *kops.Cluster, basedir vfs.Path) SSHCreden return c } -func (s *VFSCAStore) VFSPath() vfs.Path { - return s.basedir +func (c *VFSCAStore) VFSPath() vfs.Path { + return c.basedir } func (c *VFSCAStore) buildCertificatePoolPath(name string) vfs.Path { @@ -106,11 +107,12 @@ func (c *VFSCAStore) parseKeysetYaml(data []byte) (*kops.Keyset, bool, error) { return keyset, gvk.Version != keysetFormatLatest, nil } -// loadCertificatesBundle loads a keyset from the path +// loadKeyset loads a Keyset from the path. // Returns (nil, nil) if the file is not found // Bundles avoid the need for a list-files permission, which can be tricky on e.g. GCE -func (c *VFSCAStore) loadKeysetBundle(p vfs.Path) (*keyset, error) { - data, err := p.ReadFile() +func (c *VFSCAStore) loadKeyset(p vfs.Path) (*Keyset, error) { + bundlePath := p.Join("keyset.yaml") + data, err := bundlePath.ReadFile() if err != nil { if os.IsNotExist(err) { return nil, nil @@ -128,31 +130,40 @@ func (c *VFSCAStore) loadKeysetBundle(p vfs.Path) (*keyset, error) { return nil, fmt.Errorf("error mapping bundle %q: %v", p, err) } - keyset.legacyFormat = legacyFormat + keyset.LegacyFormat = legacyFormat return keyset, nil } -func (k *keyset) ToAPIObject(name string, includePrivateKeyMaterial bool) (*kops.Keyset, error) { +func (k *Keyset) ToAPIObject(name string, includePrivateKeyMaterial bool) (*kops.Keyset, error) { o := &kops.Keyset{} o.Name = name o.Spec.Type = kops.SecretTypeKeypair - for _, ki := range k.items { + keys := make([]string, 0, len(k.Items)) + for k := range k.Items { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + return KeysetItemIdOlder(k.Items[keys[i]].Id, k.Items[keys[j]].Id) + }) + + for _, key := range keys { + ki := k.Items[key] oki := kops.KeysetItem{ - Id: ki.id, + Id: ki.Id, } - if ki.certificate != nil { + if ki.Certificate != nil { var publicMaterial bytes.Buffer - if _, err := ki.certificate.WriteTo(&publicMaterial); err != nil { + if _, err := ki.Certificate.WriteTo(&publicMaterial); err != nil { return nil, err } oki.PublicMaterial = publicMaterial.Bytes() } - if includePrivateKeyMaterial && ki.privateKey != nil { + if includePrivateKeyMaterial && ki.PrivateKey != nil { var privateMaterial bytes.Buffer - if _, err := ki.privateKey.WriteTo(&privateMaterial); err != nil { + if _, err := ki.PrivateKey.WriteTo(&privateMaterial); err != nil { return nil, err } @@ -161,11 +172,14 @@ func (k *keyset) ToAPIObject(name string, includePrivateKeyMaterial bool) (*kops o.Spec.Keys = append(o.Spec.Keys, oki) } + if k.Primary != nil { + o.Spec.PrimaryId = k.Primary.Id + } return o, nil } -// writeKeysetBundle writes a keyset bundle to VFS -func (c *VFSCAStore) writeKeysetBundle(p vfs.Path, name string, keyset *keyset, includePrivateKeyMaterial bool) error { +// writeKeysetBundle writes a Keyset bundle to VFS. +func (c *VFSCAStore) writeKeysetBundle(p vfs.Path, name string, keyset *Keyset, includePrivateKeyMaterial bool) error { p = p.Join("keyset.yaml") o, err := keyset.ToAPIObject(name, includePrivateKeyMaterial) @@ -185,7 +199,7 @@ func (c *VFSCAStore) writeKeysetBundle(p vfs.Path, name string, keyset *keyset, return p.WriteFile(bytes.NewReader(objectData), acl) } -// serializeKeysetBundle converts a keyset bundle to yaml, for writing to VFS +// serializeKeysetBundle converts a Keyset bundle to yaml, for writing to VFS. func serializeKeysetBundle(o *kops.Keyset) ([]byte, error) { var objectData bytes.Buffer codecs := kopscodecs.Codecs @@ -203,88 +217,69 @@ func serializeKeysetBundle(o *kops.Keyset) ([]byte, error) { // removePrivateKeyMaterial returns a copy of the Keyset with the private key data removed func removePrivateKeyMaterial(o *kops.Keyset) *kops.Keyset { - copy := o.DeepCopy() + c := o.DeepCopy() - for i := range copy.Spec.Keys { - copy.Spec.Keys[i].PrivateMaterial = nil + for i := range c.Spec.Keys { + c.Spec.Keys[i].PrivateMaterial = nil } - return copy + return c } -func SerializeKeyset(o *kops.Keyset) ([]byte, error) { - var objectData bytes.Buffer - { - codecs := kopscodecs.Codecs - yaml, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), "application/yaml") - if !ok { - klog.Fatalf("no YAML serializer registered") - } - encoder := codecs.EncoderForVersion(yaml.Serializer, v1alpha2.SchemeGroupVersion) - - if err := encoder.Encode(o, &objectData); err != nil { - return nil, fmt.Errorf("error serializing keyset: %v", err) - } - } - - return objectData.Bytes(), nil +func (c *VFSCAStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { + return FindPrimaryKeypair(c, name) } -func (c *VFSCAStore) loadCertificates(p vfs.Path) (*keyset, error) { - bundlePath := p.Join("keyset.yaml") - bundle, err := c.loadKeysetBundle(bundlePath) - return bundle, err -} +func (c *VFSCAStore) FindKeyset(id string) (*Keyset, error) { + certs, err := c.loadKeyset(c.buildCertificatePoolPath(id)) -func (c *VFSCAStore) loadOneCertificate(p vfs.Path) (*pki.Certificate, error) { - data, err := p.ReadFile() - if err != nil { - if os.IsNotExist(err) { - return nil, nil - } - return nil, err - } - cert, err := pki.ParsePEMCertificate(data) - if err != nil { - return nil, err - } - if cert == nil { - return nil, nil - } - return cert, nil -} - -func (c *VFSCAStore) FindKeypair(id string) (*pki.Certificate, *pki.PrivateKey, bool, error) { - cert, legacyFormat, err := c.findCert(id) - - if (cert == nil || os.IsNotExist(err)) && id == "service-account" { + if (certs == nil || os.IsNotExist(err)) && id == "service-account" { // The strange name is because Kops prior to 1.19 used the api-server TLS key for this. id = "master" - cert, _, err = c.findCert(id) - legacyFormat = true + certs, err = c.loadKeyset(c.buildCertificatePoolPath(id)) + if certs != nil { + certs.LegacyFormat = true + } } if err != nil { - return nil, nil, false, err + return nil, err } - key, err := c.FindPrivateKey(id) + keys, err := c.findPrivateKeyset(id) if err != nil { - return nil, nil, false, err + return nil, err } - return cert, key, legacyFormat, nil + if certs != nil { + if keys == nil { + return certs, nil + } + if certs.LegacyFormat { + keys.LegacyFormat = true + } + for key, certItem := range certs.Items { + keyItem := keys.Items[key] + if keyItem == nil { + keys.Items[key] = certItem + } else if keyItem.Certificate == nil { + keyItem.Certificate = certItem.Certificate + } + } + } + + return keys, nil } func (c *VFSCAStore) findCert(name string) (*pki.Certificate, bool, error) { p := c.buildCertificatePoolPath(name) - certs, err := c.loadCertificates(p) + certs, err := c.loadKeyset(p) if err != nil { return nil, false, fmt.Errorf("error in 'FindCert' attempting to load cert %q: %v", name, err) } - if certs != nil && certs.primary != nil { - return certs.primary.certificate, certs.legacyFormat, nil + if certs != nil && certs.Primary != nil { + return certs.Primary.Certificate, certs.LegacyFormat, nil } return nil, false, nil @@ -296,11 +291,11 @@ func (c *VFSCAStore) FindCert(name string) (*pki.Certificate, error) { } func (c *VFSCAStore) FindCertificatePool(name string) (*CertificatePool, error) { - var certs *keyset + var certs *Keyset var err error p := c.buildCertificatePoolPath(name) - certs, err = c.loadCertificates(p) + certs, err = c.loadKeyset(p) if err != nil { return nil, fmt.Errorf("error in 'FindCertificatePool' attempting to load cert %q: %v", name, err) } @@ -308,18 +303,18 @@ func (c *VFSCAStore) FindCertificatePool(name string) (*CertificatePool, error) pool := &CertificatePool{} if certs != nil { - if certs.primary != nil { - pool.Primary = certs.primary.certificate + if certs.Primary != nil { + pool.Primary = certs.Primary.Certificate } - for k, cert := range certs.items { - if certs.primary != nil && k == certs.primary.id { + for k, cert := range certs.Items { + if certs.Primary != nil && k == certs.Primary.Id { continue } - if cert.certificate == nil { + if cert.Certificate == nil { continue } - pool.Secondary = append(pool.Secondary, cert.certificate) + pool.Secondary = append(pool.Secondary, cert.Certificate) } } return pool, nil @@ -327,7 +322,7 @@ func (c *VFSCAStore) FindCertificatePool(name string) (*CertificatePool, error) func (c *VFSCAStore) FindCertificateKeyset(name string) (*kops.Keyset, error) { p := c.buildCertificatePoolPath(name) - certs, err := c.loadCertificates(p) + certs, err := c.loadKeyset(p) if err != nil { return nil, fmt.Errorf("error in 'FindCertificatePool' attempting to load cert %q: %v", name, err) } @@ -464,7 +459,7 @@ func (c *VFSCAStore) MirrorTo(basedir vfs.Path) error { return nil } -// mirrorKeyset writes keyset bundles for the certificates & privatekeys +// mirrorKeyset writes Keyset bundles for the certificates & privatekeys. func mirrorKeyset(cluster *kops.Cluster, basedir vfs.Path, keyset *kops.Keyset) error { primary := FindPrimary(keyset) if primary == nil { @@ -535,64 +530,26 @@ func mirrorSSHCredential(cluster *kops.Cluster, basedir vfs.Path, sshCredential return nil } -func (c *VFSCAStore) StoreKeypair(name string, cert *pki.Certificate, privateKey *pki.PrivateKey) error { - serial := cert.Certificate.SerialNumber.String() - - ki := &keysetItem{ - id: serial, - certificate: cert, - privateKey: privateKey, - } - +func (c *VFSCAStore) StoreKeyset(name string, keyset *Keyset) error { { - err := c.storePrivateKey(name, ki) - if err != nil { - return err + p := c.buildPrivateKeyPoolPath(name) + if err := c.writeKeysetBundle(p, name, keyset, true); err != nil { + return fmt.Errorf("writing private bundle: %v", err) } } { - err := c.storeCertificate(name, ki) - if err != nil { - // TODO: Delete private key? - return err + p := c.buildCertificatePoolPath(name) + if err := c.writeKeysetBundle(p, name, keyset, false); err != nil { + return fmt.Errorf("writing certificate bundle: %v", err) } } return nil } -func (c *VFSCAStore) AddCert(name string, cert *pki.Certificate) error { - klog.Infof("Adding TLS certificate: %q", name) - - // We add with a timestamp of zero so this will never be the newest cert - serial := pki.BuildPKISerial(0).String() - - p := c.buildCertificatePath(name, serial) - - ki := &keysetItem{ - id: serial, - certificate: cert, - } - err := c.storeCertificate(name, ki) - if err != nil { - return err - } - - // Make double-sure it round-trips - _, err = c.loadOneCertificate(p) - return err -} - -func (c *VFSCAStore) loadPrivateKeys(p vfs.Path) (*keyset, error) { - bundlePath := p.Join("keyset.yaml") - bundle, err := c.loadKeysetBundle(bundlePath) - - return bundle, err -} - -func (c *VFSCAStore) findPrivateKeyset(id string) (*keyset, error) { - var keys *keyset +func (c *VFSCAStore) findPrivateKeyset(id string) (*Keyset, error) { + var keys *Keyset var err error if id == CertificateIDCA { c.mutex.Lock() @@ -603,7 +560,7 @@ func (c *VFSCAStore) findPrivateKeyset(id string) (*keyset, error) { return cached, nil } - keys, err = c.loadPrivateKeys(c.buildPrivateKeyPoolPath(id)) + keys, err = c.loadKeyset(c.buildPrivateKeyPoolPath(id)) if err != nil { return nil, err } @@ -616,7 +573,7 @@ func (c *VFSCAStore) findPrivateKeyset(id string) (*keyset, error) { } } else { p := c.buildPrivateKeyPoolPath(id) - keys, err = c.loadPrivateKeys(p) + keys, err = c.loadKeyset(p) if err != nil { return nil, err } @@ -632,8 +589,8 @@ func (c *VFSCAStore) FindPrivateKey(id string) (*pki.PrivateKey, error) { } var key *pki.PrivateKey - if keys != nil && keys.primary != nil { - key = keys.primary.privateKey + if keys != nil && keys.Primary != nil { + key = keys.Primary.PrivateKey } return key, nil } @@ -652,92 +609,6 @@ func (c *VFSCAStore) FindPrivateKeyset(name string) (*kops.Keyset, error) { return o, nil } -func (c *VFSCAStore) storePrivateKey(name string, ki *keysetItem) error { - if ki.privateKey == nil { - return fmt.Errorf("privateKey not provided to storeCertificate") - } - - // Write the bundle - { - p := c.buildPrivateKeyPoolPath(name) - ks, err := c.loadPrivateKeys(p) - if err != nil { - return err - } - - if ks == nil { - ks = &keyset{} - } - if ks.items == nil { - ks.items = make(map[string]*keysetItem) - } - ks.items[ki.id] = ki - - if err := c.writeKeysetBundle(p, name, ks, true); err != nil { - return fmt.Errorf("error writing bundle: %v", err) - } - } - - // TODO stop writing and remove legacy format files after rollback to pre-kops 1.18 not needed - // Write the data - { - var data bytes.Buffer - if _, err := ki.privateKey.WriteTo(&data); err != nil { - return err - } - - p := c.buildPrivateKeyPath(name, ki.id) - acl, err := acls.GetACL(p, c.cluster) - if err != nil { - return err - } - return p.WriteFile(bytes.NewReader(data.Bytes()), acl) - } -} - -func (c *VFSCAStore) storeCertificate(name string, ki *keysetItem) error { - if ki.certificate == nil { - return fmt.Errorf("certificate not provided to storeCertificate") - } - - // Write the bundle - { - p := c.buildCertificatePoolPath(name) - ks, err := c.loadCertificates(p) - if err != nil { - return err - } - - if ks == nil { - ks = &keyset{} - } - if ks.items == nil { - ks.items = make(map[string]*keysetItem) - } - ks.items[ki.id] = ki - - if err := c.writeKeysetBundle(p, name, ks, false); err != nil { - return fmt.Errorf("error writing bundle: %v", err) - } - } - - // TODO stop writing and remove legacy format files after rollback to pre-kops 1.18 not needed - // Write the data - { - var data bytes.Buffer - if _, err := ki.certificate.WriteTo(&data); err != nil { - return err - } - - p := c.buildCertificatePath(name, ki.id) - acl, err := acls.GetACL(p, c.cluster) - if err != nil { - return err - } - return p.WriteFile(bytes.NewReader(data.Bytes()), acl) - } -} - func (c *VFSCAStore) deletePrivateKey(name string, id string) (bool, error) { // Delete the file itself { @@ -751,15 +622,18 @@ func (c *VFSCAStore) deletePrivateKey(name string, id string) (bool, error) { // Update the bundle { p := c.buildPrivateKeyPoolPath(name) - ks, err := c.loadPrivateKeys(p) + ks, err := c.loadKeyset(p) if err != nil { return false, err } - if ks == nil || ks.items[id] == nil { + if ks == nil || ks.Items[id] == nil { return false, nil } - delete(ks.items, id) + delete(ks.Items, id) + if ks.Primary != nil && ks.Primary.Id == id { + ks.Primary = nil + } if err := c.writeKeysetBundle(p, name, ks, true); err != nil { return false, fmt.Errorf("error writing bundle: %v", err) @@ -781,15 +655,18 @@ func (c *VFSCAStore) deleteCertificate(name string, id string) (bool, error) { // Update the bundle { p := c.buildCertificatePoolPath(name) - ks, err := c.loadCertificates(p) + ks, err := c.loadKeyset(p) if err != nil { return false, err } - if ks == nil || ks.items[id] == nil { + if ks == nil || ks.Items[id] == nil { return false, nil } - delete(ks.items, id) + delete(ks.Items, id) + if ks.Primary != nil && ks.Primary.Id == id { + ks.Primary = nil + } if err := c.writeKeysetBundle(p, name, ks, false); err != nil { return false, fmt.Errorf("error writing bundle: %v", err) diff --git a/upup/pkg/fi/vfs_castore_test.go b/upup/pkg/fi/vfs_castore_test.go index 55b360378f..87da24ff28 100644 --- a/upup/pkg/fi/vfs_castore_test.go +++ b/upup/pkg/fi/vfs_castore_test.go @@ -24,7 +24,6 @@ import ( "testing" "time" - "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/pki" "k8s.io/kops/util/pkg/vfs" ) @@ -71,8 +70,19 @@ func TestVFSCAStoreRoundTrip(t *testing.T) { t.Fatalf("error from ParsePEMCertificate: %v", err) } - if err := s.StoreKeypair("ca", cert, privateKey); err != nil { - t.Fatalf("error from StoreKeypair: %v", err) + item := &KeysetItem{ + Id: "237054359138908419352140518924933177492", + Certificate: cert, + PrivateKey: privateKey, + } + keyset := &Keyset{ + Items: map[string]*KeysetItem{ + "237054359138908419352140518924933177492": item, + }, + Primary: item, + } + if err := s.StoreKeyset("ca", keyset); err != nil { + t.Fatalf("error from StoreKeyset: %v", err) } paths, err := basePath.ReadTree() @@ -87,16 +97,14 @@ func TestVFSCAStoreRoundTrip(t *testing.T) { for _, p := range []string{ "memfs://tests/issued/ca/keyset.yaml", - "memfs://tests/issued/ca/237054359138908419352140518924933177492.crt", "memfs://tests/private/ca/keyset.yaml", - "memfs://tests/private/ca/237054359138908419352140518924933177492.key", } { if _, found := pathMap[p]; !found { t.Fatalf("file not found: %v", p) } } - if len(pathMap) != 4 { + if len(pathMap) != 2 { t.Fatalf("unexpected pathMap: %v", pathMap) } @@ -117,6 +125,7 @@ spec: keys: - id: "237054359138908419352140518924933177492" publicMaterial: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMyRENDQWNDZ0F3SUJBZ0lSQUxKWEFrVmo5NjR0cTY3d01TSThvSlF3RFFZSktvWklodmNOQVFFTEJRQXcKRlRFVE1CRUdBMVVFQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4TnpFeU1qY3lNelV5TkRCYUZ3MHlOekV5TWpjeQpNelV5TkRCYU1CVXhFekFSQmdOVkJBTVRDbXQxWW1WeWJtVjBaWE13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBCkE0SUJEd0F3Z2dFS0FvSUJBUURnbkNrU210bm1meEVnUzNxTlBhVUNINVFPQkdESC9pbkhiV0NPRExCQ0s5Z2QKWEVjQmw3RlZ2OFQya0ZyMURZYjBIVkR0TUk3dGl4UlZGRExna3dObFczNHh3V2RaWEI3R2VvRmdVMXhXT1FTWQpPQUNDOEpnWVRRLzEzOUhCRXZncTRzZWo2N3ArL3MvU05jdzM0S2s3SEl1RmhsazFyUms1a01leEtJbEpCS1AxCllZVVlldHNKL1FwVU9rcUo1SFc0R29ldEU3Nll0SG5PUmZZdm55YnZpU01yaDJ3R0dhTjZyL3M0Q2hPYUliWkMKQW44L1lpUEtHSURhWkdwajZHWG5tWEFSUlgvVElkZ1NRa0x3dDBhVERCblBaNFh2dHBJOGFhTDhEWUpJcUF6QQpOUEgyYjQvdU55bGF0NWpEbzBiMEc1NGFnTWk5NysyQVVyQzlVVVhwQWdNQkFBR2pJekFoTUE0R0ExVWREd0VCCi93UUVBd0lCQmpBUEJnTlZIUk1CQWY4RUJUQURBUUgvTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFCVkdSMnIKaHpYelJNVTV3cmlQUUFKU2Nzek5PUnZvQnBYZlpvWjA5Rkl1cHVkRnhCVlUzZDRoVjlTdEtuUWdQU0dBNVhRTwpIRTk3K0J4SkR1QS9yQjVvQlVzTUJqYzd5MWNkZS9UNmhtaTNyTG9FWUJTblN1ZENPWEpFNEc5LzBmOGJ5QUplCnJOOCtObzFyMlZnWnZaaDZwNzRURWtYdi9sM0hCUFdNN0lkVVYwSE85SkRoU2dPVkYxZnlRS0p4UnVMSlI4anQKTzZtUEgyVVgwdk13VmE0anZ3dGtkZHFrMk9BZFlRdkg5cmJEampiemFpVzBLbm1kdWVSbzkyS0hBTjdCc0RaeQpWcFhIcHFvMUt6ZzdEM2ZwYVhDZjVzaTdscXFyZEpWWEg0SkM3Mnp4c1BlaHFnaThlSXVxT0JraURXbVJ4QXhoCjh5R2VSeDlBYmtuSGg0SWEKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + primaryId: "237054359138908419352140518924933177492" type: Keypair ` @@ -147,18 +156,6 @@ spec: } } - // Check issued/ca/237054359138908419352140518924933177492.crt round-tripped - { - roundTrip, err := pathMap["memfs://tests/issued/ca/237054359138908419352140518924933177492.crt"].ReadFile() - if err != nil { - t.Fatalf("error reading file memfs://tests/issued/ca/237054359138908419352140518924933177492.crt: %v", err) - } - - if string(roundTrip) != certData { - t.Fatalf("unexpected round-tripped certificate data: %q", string(roundTrip)) - } - } - // Check private/ca/keyset.yaml round-tripped { privateKeysetYaml, err := pathMap["memfs://tests/private/ca/keyset.yaml"].ReadFile() @@ -177,6 +174,7 @@ spec: - id: "237054359138908419352140518924933177492" privateMaterial: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBNEp3cEVwclo1bjhSSUV0NmpUMmxBaCtVRGdSZ3gvNHB4MjFnamd5d1FpdllIVnhICkFaZXhWYi9FOXBCYTlRMkc5QjFRN1RDTzdZc1VWUlF5NEpNRFpWdCtNY0ZuV1Z3ZXhucUJZRk5jVmprRW1EZ0EKZ3ZDWUdFMFA5ZC9Sd1JMNEt1TEhvK3U2ZnY3UDBqWE1OK0NwT3h5TGhZWlpOYTBaT1pESHNTaUpTUVNqOVdHRgpHSHJiQ2YwS1ZEcEtpZVIxdUJxSHJSTyttTFI1emtYMkw1OG03NGtqSzRkc0JobWplcS83T0FvVG1pRzJRZ0ovClAySWp5aGlBMm1ScVkraGw1NWx3RVVWLzB5SFlFa0pDOExkR2t3d1p6MmVGNzdhU1BHbWkvQTJDU0tnTXdEVHgKOW0rUDdqY3BXcmVZdzZORzlCdWVHb0RJdmUvdGdGS3d2VkZGNlFJREFRQUJBb0lCQUEwa3RqYVRmeXJBeHNUSQpCZXpiN1pyNU5CVzU1ZHZ1SUkyOTljZDZNSm8rckkvVFJZaHZVdjQ4a1k4SUZYcC9oeVVqemdlREx1bnhtSWY5Ci9aZ3NvaWM5T2w0NC9nNDVtTWR1aGNHWVB6QUFlQ2RjSjVPQjlyUjlWZkRDWHlqWUxsTjhIOGlVMDczNHRUcU0KMFYxM3RROXpkU3FrR1BaT0ljcS9rUi9weWxiT1phUU1lOTdCVGxzQW5PTVNNS0RnbmZ0WTQxMjJMcTNHWXkrdAp2cHIrYktWYVFad3ZrTG9TVTNyRUNDYUthZ2hnd0N5WDdqZnQ5YUVraGRKditLbHdic0dZNldFcnZ4T2FMV0hkCmN1TVFqR2FwWTFGYS80VUQwMG12ckEyNjBOeUtmenJwNitQNDZSclZNd0VZUkpNSVE4WUJBazZONkhoN2RjMEcKOFo2aTFtMENnWUVBOUhlQ0pSMFRTd2JJUTFiRFhVcnpwZnRIdWlkRzVCblNCdGF4L05EOXFJUGhSL0ZCVzVuagoyMm53TGM0OEtreWlybGZJVUxkMGFlNHFWWEpuN3dmWWN1WC9jSk1MRG1TVnRsTTVEem1pLzkxeFJpRmdJengxCkFzYkJ6YUZqSVNQMkhwU2dMK2U5RnRTWGFhcWVaVnJmbGl0VmhZS1VwSS9BS1YzMXFHSGYwNHNDZ1lFQTZ6VFYKOTlTYjQ5V2RsbnM1SWdzZm5YbDZUb1J0dEIxOGxmRUtjVmZqQU00ZnJua2swNkpwRkFaZVIrOUdHS1VYWkhxcwp6MnFjcGx3NGQvbW9DQzZwM3JZUEJNTFhzckdORVVGWnFCbGd6NzJRQTZCQnEzWDBDZzFCYzJaYks1Vkl6d2tnClNUMlNTdXg2Y2NST2ZnVUxtTjVaaUxPdGRVS05FWnBGRjNpM3F0c0NnWUFEVC9zN2RZRmxhdG9iejNrbU1uWEsKc2ZUdTJNbGxIZFJ5czBZR0h1N1E4YmlEdVFraHJKd2h4UFcwS1M4M2c0SlF5bSswYUVmemgzNmJXY2wrdTZSNwpLaEtqKzlvU2Y5cG5kZ2szNDVnSnozNVJiUEpZaCtFdUFITnZ6ZGdDQXZLNngxakVUV2VLZjZidGo1cEYxVTFpClE0UU5Jdy9RaXdJWGpXWmV1YlRHc1FLQmdRQ2JkdUx1MnJMbmx5eUFhSlpNOERsSFp5SDJnQVhiQlpweHFVOFQKdDltdGtKRFVTL0tSaUVvWUdGVjlDcVMwYVhyYXlWTXNEZlhZNkIvUy9VdVpqTzV1N0x0a2xEenFPZjFhS0czUQpkR1hQS2lia25xcUpZSCtiblVOanVZWU5lckVUVjU3bGlqTUdIdVNZQ2Y4dndMbjNveEJmRVJSWDYxTS9EVThaCndvcnovUUtCZ1FEQ1RKSTIramRYZzI2WHVZVW1NNFhYZm5vY2Z6QVhoWEJVTHQxbkVOY29nTmYxZmNwdEFWdHUKQkFpejQvSGlwUUtxb1dWVVlteGZnYmJMUktLTEswczBsT1dLYllkVmpoRW0vbTJaVTh3dFhUYWdOd2tJR295cQpZL0MxTG94NGYxUk9KbkNqYy9oZmNPamN4WDVNOEE4cGVlY0hXbFZ0VVBLVEpneFE3b01LY3c9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= publicMaterial: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMyRENDQWNDZ0F3SUJBZ0lSQUxKWEFrVmo5NjR0cTY3d01TSThvSlF3RFFZSktvWklodmNOQVFFTEJRQXcKRlRFVE1CRUdBMVVFQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4TnpFeU1qY3lNelV5TkRCYUZ3MHlOekV5TWpjeQpNelV5TkRCYU1CVXhFekFSQmdOVkJBTVRDbXQxWW1WeWJtVjBaWE13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBCkE0SUJEd0F3Z2dFS0FvSUJBUURnbkNrU210bm1meEVnUzNxTlBhVUNINVFPQkdESC9pbkhiV0NPRExCQ0s5Z2QKWEVjQmw3RlZ2OFQya0ZyMURZYjBIVkR0TUk3dGl4UlZGRExna3dObFczNHh3V2RaWEI3R2VvRmdVMXhXT1FTWQpPQUNDOEpnWVRRLzEzOUhCRXZncTRzZWo2N3ArL3MvU05jdzM0S2s3SEl1RmhsazFyUms1a01leEtJbEpCS1AxCllZVVlldHNKL1FwVU9rcUo1SFc0R29ldEU3Nll0SG5PUmZZdm55YnZpU01yaDJ3R0dhTjZyL3M0Q2hPYUliWkMKQW44L1lpUEtHSURhWkdwajZHWG5tWEFSUlgvVElkZ1NRa0x3dDBhVERCblBaNFh2dHBJOGFhTDhEWUpJcUF6QQpOUEgyYjQvdU55bGF0NWpEbzBiMEc1NGFnTWk5NysyQVVyQzlVVVhwQWdNQkFBR2pJekFoTUE0R0ExVWREd0VCCi93UUVBd0lCQmpBUEJnTlZIUk1CQWY4RUJUQURBUUgvTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFCVkdSMnIKaHpYelJNVTV3cmlQUUFKU2Nzek5PUnZvQnBYZlpvWjA5Rkl1cHVkRnhCVlUzZDRoVjlTdEtuUWdQU0dBNVhRTwpIRTk3K0J4SkR1QS9yQjVvQlVzTUJqYzd5MWNkZS9UNmhtaTNyTG9FWUJTblN1ZENPWEpFNEc5LzBmOGJ5QUplCnJOOCtObzFyMlZnWnZaaDZwNzRURWtYdi9sM0hCUFdNN0lkVVYwSE85SkRoU2dPVkYxZnlRS0p4UnVMSlI4anQKTzZtUEgyVVgwdk13VmE0anZ3dGtkZHFrMk9BZFlRdkg5cmJEampiemFpVzBLbm1kdWVSbzkyS0hBTjdCc0RaeQpWcFhIcHFvMUt6ZzdEM2ZwYVhDZjVzaTdscXFyZEpWWEg0SkM3Mnp4c1BlaHFnaThlSXVxT0JraURXbVJ4QXhoCjh5R2VSeDlBYmtuSGg0SWEKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + primaryId: "237054359138908419352140518924933177492" type: Keypair ` @@ -202,34 +200,6 @@ spec: t.Fatalf("unexpected round-tripped private key data: %q", roundTrip) } } - - // Check private/ca/237054359138908419352140518924933177492.key round-tripped - { - roundTrip, err := pathMap["memfs://tests/private/ca/237054359138908419352140518924933177492.key"].ReadFile() - if err != nil { - t.Fatalf("error reading file memfs://tests/private/ca/237054359138908419352140518924933177492.key: %v", err) - } - - if string(roundTrip) != privateKeyData { - t.Fatalf("unexpected round-tripped private key data: %q", string(roundTrip)) - } - } - - // Check that keyset gets deleted - { - keyset := &kops.Keyset{} - keyset.Name = "ca" - keyset.Spec.Type = kops.SecretTypeKeypair - - s.DeleteKeysetItem(keyset, "237054359138908419352140518924933177492") - - _, err := pathMap["memfs://tests/private/ca/237054359138908419352140518924933177492.key"].ReadFile() - if err == nil { - t.Fatalf("File memfs://tests/private/ca/237054359138908419352140518924933177492.key still exists") - } - - } - } func TestVFSCAStoreRoundTripWithVault(t *testing.T) { @@ -263,8 +233,19 @@ func TestVFSCAStoreRoundTripWithVault(t *testing.T) { t.Fatalf("error from ParsePEMCertificate: %v", err) } - if err := s.StoreKeypair("ca", cert, privateKey); err != nil { - t.Fatalf("error from StoreKeypair: %v", err) + item := &KeysetItem{ + Id: "237054359138908419352140518924933177492", + Certificate: cert, + PrivateKey: privateKey, + } + keyset := &Keyset{ + Items: map[string]*KeysetItem{ + "237054359138908419352140518924933177492": item, + }, + Primary: item, + } + if err := s.StoreKeyset("ca", keyset); err != nil { + t.Fatalf("error from StoreKeyset: %v", err) } paths, err := basePath.ReadTree() @@ -281,9 +262,7 @@ func TestVFSCAStoreRoundTripWithVault(t *testing.T) { for _, p := range []string{ bp + "/issued/ca/keyset.yaml", - bp + "/issued/ca/237054359138908419352140518924933177492.crt", bp + "/private/ca/keyset.yaml", - bp + "/private/ca/237054359138908419352140518924933177492.key", } { if _, found := pathMap[p]; !found { t.Fatalf("file not found: %v", p) @@ -341,18 +320,6 @@ spec: } } - // Check issued/ca/237054359138908419352140518924933177492.crt round-tripped - { - roundTrip, err := pathMap[bp+"/issued/ca/237054359138908419352140518924933177492.crt"].ReadFile() - if err != nil { - t.Fatalf("error reading file issued/ca/237054359138908419352140518924933177492.crt: %v", err) - } - - if string(roundTrip) != certData { - t.Fatalf("unexpected round-tripped certificate data: %q", string(roundTrip)) - } - } - // Check private/ca/keyset.yaml round-tripped { privateKeysetYaml, err := pathMap[bp+"/private/ca/keyset.yaml"].ReadFile() @@ -396,32 +363,4 @@ spec: t.Fatalf("unexpected round-tripped private key data: %q", roundTrip) } } - - // Check private/ca/237054359138908419352140518924933177492.key round-tripped - { - roundTrip, err := pathMap[bp+"/private/ca/237054359138908419352140518924933177492.key"].ReadFile() - if err != nil { - t.Fatalf("error reading file private/ca/237054359138908419352140518924933177492.key: %v", err) - } - - if string(roundTrip) != privateKeyData { - t.Fatalf("unexpected round-tripped private key data: %q", string(roundTrip)) - } - } - - // Check that keyset gets deleted - { - keyset := &kops.Keyset{} - keyset.Name = "ca" - keyset.Spec.Type = kops.SecretTypeKeypair - - s.DeleteKeysetItem(keyset, "237054359138908419352140518924933177492") - - _, err := pathMap[bp+"/private/ca/237054359138908419352140518924933177492.key"].ReadFile() - if err == nil { - t.Fatalf("File private/ca/237054359138908419352140518924933177492.key still exists") - } - - } - } diff --git a/upup/pkg/kutil/convert_kubeup_cluster.go b/upup/pkg/kutil/convert_kubeup_cluster.go index f335517848..73e0d825fb 100644 --- a/upup/pkg/kutil/convert_kubeup_cluster.go +++ b/upup/pkg/kutil/convert_kubeup_cluster.go @@ -487,17 +487,19 @@ func (x *ConvertKubeupCluster) Upgrade(ctx context.Context) error { if oldCACertPool == nil { return fmt.Errorf("cannot find certificate pool %q", fi.CertificateIDCA) } + keyset, err := newKeyStore.FindKeyset(fi.CertificateIDCA) + if err != nil { + return fmt.Errorf("error reading new CA certs: %v", err) + } for _, ca := range oldCACertPool.Secondary { - err := newKeyStore.AddCert(fi.CertificateIDCA, ca) - if err != nil { - return fmt.Errorf("error importing old CA certs: %v", err) - } + fi.AddCert(keyset, ca) } if oldCACertPool.Primary != nil { - err := newKeyStore.AddCert(fi.CertificateIDCA, oldCACertPool.Primary) - if err != nil { - return fmt.Errorf("error importing old CA certs: %v", err) - } + fi.AddCert(keyset, oldCACertPool.Primary) + } + err = newKeyStore.StoreKeyset(fi.CertificateIDCA, keyset) + if err != nil { + return fmt.Errorf("error importing old CA certs: %v", err) } return nil diff --git a/upup/pkg/kutil/import_cluster.go b/upup/pkg/kutil/import_cluster.go index e0f68d9b4c..08bd9d105c 100644 --- a/upup/pkg/kutil/import_cluster.go +++ b/upup/pkg/kutil/import_cluster.go @@ -424,7 +424,12 @@ func (x *ImportCluster) ImportAWSCluster(ctx context.Context) error { if err != nil { return err } - err = keyStore.AddCert(fi.CertificateIDCA, caCert) + keyset, err := keyStore.FindKeyset(fi.CertificateIDCA) + if err != nil { + return err + } + fi.AddCert(keyset, caCert) + err = keyStore.StoreKeyset(fi.CertificateIDCA, keyset) if err != nil { return err }