Merge pull request #11219 from johngmyers/refactor-keypair

Refactor keypair code in preparation for secret rotation

cmd/kops-controller/pkg/server/keystore.go

@@ -35,12 +35,12 @@ type keystoreEntry struct {
 
 var _ pki.Keystore = keystore{}
 
-func (k keystore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) {
+func (k keystore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) {
 	entry, ok := k.keys[name]
 	if !ok {
-		return nil, nil, false, fmt.Errorf("unknown CA %q", name)
+		return nil, nil, fmt.Errorf("unknown CA %q", name)
 	}
-	return entry.certificate, entry.key, false, nil
+	return entry.certificate, entry.key, nil
 }
 
 func newKeystore(basePath string, cas []string) (pki.Keystore, error) {

cmd/kops-controller/pkg/server/node_config.go

@@ -73,7 +73,7 @@ func (s *Server) getNodeConfig(ctx context.Context, req *nodeup.BootstrapRequest
 
 	// We populate some certificates that we know the node will need.
 	for _, name := range []string{"ca"} {
-		cert, _, _, err := s.keystore.FindKeypair(name)
+		cert, _, err := s.keystore.FindPrimaryKeypair(name)
 		if err != nil {
 			return nil, fmt.Errorf("error getting certificate %q: %w", name, err)
 		}

cmd/kops/create_secret_keypair_ca.go

@@ -133,7 +133,20 @@ func RunCreateSecretCaCert(ctx context.Context, f *util.Factory, out io.Writer,
 		return fmt.Errorf("error loading certificate %q: %v", options.CaCertPath, err)
 	}
 
-	err = keyStore.StoreKeypair(fi.CertificateIDCA, cert, privateKey)
+	serialString := cert.Certificate.SerialNumber.String()
+	ki := &fi.KeysetItem{
+		Id:          serialString,
+		Certificate: cert,
+		PrivateKey:  privateKey,
+	}
+
+	err = keyStore.StoreKeyset(fi.CertificateIDCA, &fi.Keyset{
+		LegacyFormat: false,
+		Items: map[string]*fi.KeysetItem{
+			serialString: ki,
+		},
+		Primary: ki,
+	})
 	if err != nil {
 		return fmt.Errorf("error storing user provided keys %q %q: %v", options.CaCertPath, options.CaPrivateKeyPath, err)
 	}

k8s/crds/kops.k8s.io_keysets.yaml

@@ -59,6 +59,9 @@ spec:
                       type: string
                   type: object
                 type: array
+              primaryId:
+                description: PrimaryId is the id of the key used to make new signatures.
+                type: string
               type:
                 description: Type is the type of the Keyset (PKI keypair, or secret
                   token)

nodeup/pkg/model/fakes_test.go

@@ -18,6 +18,7 @@ package model
 
 import (
 	"crypto/x509"
+	"fmt"
 	"testing"
 
 	"k8s.io/kops/pkg/apis/kops"
@@ -26,40 +27,69 @@ import (
 	"k8s.io/kops/util/pkg/vfs"
 )
 
-// fakeKeyStore mocks out some of fi.KeyStore, for our tests.
-type fakeKeyStore struct {
-	T *testing.T
-}
-
-var _ fi.Keystore = &fakeKeyStore{}
-
-func (k fakeKeyStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) {
-	panic("fakeKeyStore does not implement FindKeypair")
-}
-
-func (k fakeKeyStore) CreateKeypair(signer string, name string, template *x509.Certificate, privateKey *pki.PrivateKey) (*pki.Certificate, error) {
-	panic("fakeKeyStore does not implement CreateKeypair")
-}
-
-func (k fakeKeyStore) StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error {
-	panic("fakeKeyStore does not implement StoreKeypair")
-}
-
-func (k fakeKeyStore) MirrorTo(basedir vfs.Path) error {
-	panic("fakeKeyStore does not implement MirrorTo")
-}
-
 // fakeCAStore mocks out some of fi.CAStore, for our tests.
-// Although CAStore currently embeds KeyStore, we maintain the split here in the hope we can clean this up in future.
 type fakeCAStore struct {
-	fakeKeyStore
+	T              *testing.T
-
+	privateKeysets map[string]*kops.Keyset
-	privateKeys map[string]*pki.PrivateKey
 	certs          map[string]*pki.Certificate
 }
 
 var _ fi.CAStore = &fakeCAStore{}
 
+func (k fakeCAStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) {
+	panic("fakeCAStore does not implement FindPrimaryKeypair")
+}
+
+func (k fakeCAStore) FindKeyset(name string) (*fi.Keyset, error) {
+	kopsKeyset := k.privateKeysets[name]
+	if kopsKeyset == nil {
+		return nil, nil
+	}
+
+	keyset := &fi.Keyset{
+		Items: make(map[string]*fi.KeysetItem),
+	}
+
+	for _, key := range kopsKeyset.Spec.Keys {
+		ki := &fi.KeysetItem{
+			Id: key.Id,
+		}
+		if len(key.PublicMaterial) != 0 {
+			cert, err := pki.ParsePEMCertificate(key.PublicMaterial)
+			if err != nil {
+				return nil, fmt.Errorf("error loading certificate %s/%s: %v", name, key.Id, err)
+			}
+			ki.Certificate = cert
+		}
+
+		if len(key.PrivateMaterial) != 0 {
+			privateKey, err := pki.ParsePEMPrivateKey(key.PrivateMaterial)
+			if err != nil {
+				return nil, fmt.Errorf("error loading private key %s/%s: %v", name, key.Id, err)
+			}
+			ki.PrivateKey = privateKey
+		}
+
+		keyset.Items[key.Id] = ki
+	}
+
+	keyset.Primary = keyset.Items[fi.FindPrimary(kopsKeyset).Id]
+
+	return keyset, nil
+}
+
+func (k fakeCAStore) CreateKeypair(signer string, name string, template *x509.Certificate, privateKey *pki.PrivateKey) (*pki.Certificate, error) {
+	panic("fakeCAStore does not implement CreateKeypair")
+}
+
+func (k fakeCAStore) StoreKeyset(name string, keyset *fi.Keyset) error {
+	panic("fakeCAStore does not implement StoreKeyset")
+}
+
+func (k fakeCAStore) MirrorTo(basedir vfs.Path) error {
+	panic("fakeCAStore does not implement MirrorTo")
+}
+
 func (k fakeCAStore) FindCertificatePool(name string) (*fi.CertificatePool, error) {
 	panic("fakeCAStore does not implement FindCertificatePool")
 }
@@ -69,11 +99,17 @@ func (k fakeCAStore) FindCertificateKeyset(name string) (*kops.Keyset, error) {
 }
 
 func (k fakeCAStore) FindPrivateKey(name string) (*pki.PrivateKey, error) {
-	return k.privateKeys[name], nil
+	primaryId := k.privateKeysets[name].Spec.PrimaryId
+	for _, item := range k.privateKeysets[name].Spec.Keys {
+		if item.Id == primaryId {
+			return pki.ParsePEMPrivateKey(item.PrivateMaterial)
+		}
+	}
+	return nil, nil
 }
 
 func (k fakeCAStore) FindPrivateKeyset(name string) (*kops.Keyset, error) {
-	panic("fakeCAStore does not implement FindPrivateKeyset")
+	return k.privateKeysets[name], nil
 }
 
 func (k fakeCAStore) FindCert(name string) (*pki.Certificate, error) {
@@ -84,10 +120,6 @@ func (k fakeCAStore) ListKeysets() ([]*kops.Keyset, error) {
 	panic("fakeCAStore does not implement ListKeysets")
 }
 
-func (k fakeCAStore) AddCert(name string, cert *pki.Certificate) error {
-	panic("fakeCAStore does not implement AddCert")
-}
-
 func (k fakeCAStore) DeleteKeysetItem(item *kops.Keyset, id string) error {
 	panic("fakeCAStore does not implement DeleteKeysetItem")
 }

nodeup/pkg/model/kube_apiserver.go

@@ -81,6 +81,28 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error {
 			}
 		}
 	}
+	{
+		keyset, err := b.KeyStore.FindKeyset("service-account")
+		if err != nil {
+			return err
+		}
+
+		if keyset == nil {
+			return fmt.Errorf("service-account keyset not found")
+		}
+
+		buf, err := keyset.ToPublicKeyBytes()
+		if err != nil {
+			return err
+		}
+
+		c.AddTask(&nodetasks.File{
+			Path:     filepath.Join(b.PathSrvKubernetes(), "service-account.pub"),
+			Contents: fi.NewBytesResource(buf),
+			Type:     nodetasks.FileType_File,
+			Mode:     s("0600"),
+		})
+	}
 	{
 		pod, err := b.buildPod()
 		if err != nil {
@@ -282,8 +304,7 @@ func (b *KubeAPIServerBuilder) writeAuthenticationConfig(c *fi.ModelBuilderConte
 func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
 	kubeAPIServer := b.Cluster.Spec.KubeAPIServer
 
-	// TODO pass the public key instead. We would first need to segregate the secrets better.
+	kubeAPIServer.ServiceAccountKeyFile = append(kubeAPIServer.ServiceAccountKeyFile, filepath.Join(b.PathSrvKubernetes(), "service-account.pub"))
-	kubeAPIServer.ServiceAccountKeyFile = append(kubeAPIServer.ServiceAccountKeyFile, filepath.Join(b.PathSrvKubernetes(), "service-account.key"))
 
 	// Set the signing key if we're using Service Account Token VolumeProjection
 	if kubeAPIServer.ServiceAccountSigningKeyFile == nil {

nodeup/pkg/model/kubelet_test.go

@@ -269,13 +269,48 @@ func mockedPopulateClusterSpec(c *kops.Cluster, cloud fi.Cloud) (*kops.Cluster,
 
 const dummyCertificate = "-----BEGIN CERTIFICATE-----\nMIIC2DCCAcCgAwIBAgIRALJXAkVj964tq67wMSI8oJQwDQYJKoZIhvcNAQELBQAw\nFTETMBEGA1UEAxMKa3ViZXJuZXRlczAeFw0xNzEyMjcyMzUyNDBaFw0yNzEyMjcy\nMzUyNDBaMBUxEzARBgNVBAMTCmt1YmVybmV0ZXMwggEiMA0GCSqGSIb3DQEBAQUA\nA4IBDwAwggEKAoIBAQDgnCkSmtnmfxEgS3qNPaUCH5QOBGDH/inHbWCODLBCK9gd\nXEcBl7FVv8T2kFr1DYb0HVDtMI7tixRVFDLgkwNlW34xwWdZXB7GeoFgU1xWOQSY\nOACC8JgYTQ/139HBEvgq4sej67p+/s/SNcw34Kk7HIuFhlk1rRk5kMexKIlJBKP1\nYYUYetsJ/QpUOkqJ5HW4GoetE76YtHnORfYvnybviSMrh2wGGaN6r/s4ChOaIbZC\nAn8/YiPKGIDaZGpj6GXnmXARRX/TIdgSQkLwt0aTDBnPZ4XvtpI8aaL8DYJIqAzA\nNPH2b4/uNylat5jDo0b0G54agMi97+2AUrC9UUXpAgMBAAGjIzAhMA4GA1UdDwEB\n/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBVGR2r\nhzXzRMU5wriPQAJScszNORvoBpXfZoZ09FIupudFxBVU3d4hV9StKnQgPSGA5XQO\nHE97+BxJDuA/rB5oBUsMBjc7y1cde/T6hmi3rLoEYBSnSudCOXJE4G9/0f8byAJe\nrN8+No1r2VgZvZh6p74TEkXv/l3HBPWM7IdUV0HO9JDhSgOVF1fyQKJxRuLJR8jt\nO6mPH2UX0vMwVa4jvwtkddqk2OAdYQvH9rbDjjbzaiW0KnmdueRo92KHAN7BsDZy\nVpXHpqo1Kzg7D3fpaXCf5si7lqqrdJVXH4JC72zxsPehqgi8eIuqOBkiDWmRxAxh\n8yGeRx9AbknHh4Ia\n-----END CERTIFICATE-----\n"
 const dummyKey = "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA4JwpEprZ5n8RIEt6jT2lAh+UDgRgx/4px21gjgywQivYHVxH\nAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMDZVt+McFnWVwexnqBYFNcVjkEmDgA\ngvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+CpOxyLhYZZNa0ZOZDHsSiJSQSj9WGF\nGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m74kjK4dsBhmjeq/7OAoTmiG2QgJ/\nP2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdGkwwZz2eF77aSPGmi/A2CSKgMwDTx\n9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF6QIDAQABAoIBAA0ktjaTfyrAxsTI\nBezb7Zr5NBW55dvuII299cd6MJo+rI/TRYhvUv48kY8IFXp/hyUjzgeDLunxmIf9\n/Zgsoic9Ol44/g45mMduhcGYPzAAeCdcJ5OB9rR9VfDCXyjYLlN8H8iU0734tTqM\n0V13tQ9zdSqkGPZOIcq/kR/pylbOZaQMe97BTlsAnOMSMKDgnftY4122Lq3GYy+t\nvpr+bKVaQZwvkLoSU3rECCaKaghgwCyX7jft9aEkhdJv+KlwbsGY6WErvxOaLWHd\ncuMQjGapY1Fa/4UD00mvrA260NyKfzrp6+P46RrVMwEYRJMIQ8YBAk6N6Hh7dc0G\n8Z6i1m0CgYEA9HeCJR0TSwbIQ1bDXUrzpftHuidG5BnSBtax/ND9qIPhR/FBW5nj\n22nwLc48KkyirlfIULd0ae4qVXJn7wfYcuX/cJMLDmSVtlM5Dzmi/91xRiFgIzx1\nAsbBzaFjISP2HpSgL+e9FtSXaaqeZVrflitVhYKUpI/AKV31qGHf04sCgYEA6zTV\n99Sb49Wdlns5IgsfnXl6ToRttB18lfEKcVfjAM4frnkk06JpFAZeR+9GGKUXZHqs\nz2qcplw4d/moCC6p3rYPBMLXsrGNEUFZqBlgz72QA6BBq3X0Cg1Bc2ZbK5VIzwkg\nST2SSux6ccROfgULmN5ZiLOtdUKNEZpFF3i3qtsCgYADT/s7dYFlatobz3kmMnXK\nsfTu2MllHdRys0YGHu7Q8biDuQkhrJwhxPW0KS83g4JQym+0aEfzh36bWcl+u6R7\nKhKj+9oSf9pndgk345gJz35RbPJYh+EuAHNvzdgCAvK6x1jETWeKf6btj5pF1U1i\nQ4QNIw/QiwIXjWZeubTGsQKBgQCbduLu2rLnlyyAaJZM8DlHZyH2gAXbBZpxqU8T\nt9mtkJDUS/KRiEoYGFV9CqS0aXrayVMsDfXY6B/S/UuZjO5u7LtklDzqOf1aKG3Q\ndGXPKibknqqJYH+bnUNjuYYNerETV57lijMGHuSYCf8vwLn3oxBfERRX61M/DU8Z\nworz/QKBgQDCTJI2+jdXg26XuYUmM4XXfnocfzAXhXBULt1nENcogNf1fcptAVtu\nBAiz4/HipQKqoWVUYmxfgbbLRKKLK0s0lOWKbYdVjhEm/m2ZU8wtXTagNwkIGoyq\nY/C1Lox4f1ROJnCjc/hfcOjcxX5M8A8peecHWlVtUPKTJgxQ7oMKcw==\n-----END RSA PRIVATE KEY-----\n"
+const previousCertificate = "-----BEGIN CERTIFICATE-----\nMIIBZzCCARGgAwIBAgIBAjANBgkqhkiG9w0BAQsFADAaMRgwFgYDVQQDEw9zZXJ2\naWNlLWFjY291bnQwHhcNMjEwNTAyMjAzMDA2WhcNMzEwNTAyMjAzMDA2WjAaMRgw\nFgYDVQQDEw9zZXJ2aWNlLWFjY291bnQwXDANBgkqhkiG9w0BAQEFAANLADBIAkEA\n2JbeF8dNwqfEKKD65aGlVs58fWkA0qZdVLKw8qATzRBJTi1nqbj2kAR4gyy/C8Mx\nouxva/om9d7Sq8Ka55T7+wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T\nAQH/BAUwAwEB/zAdBgNVHQ4EFgQUI5beFHueAGyT1pQ6UTOdbMfj3gQwDQYJKoZI\nhvcNAQELBQADQQBwPLO+Np8o6k3aNBGKE4JTCOs06X72OXNivkWWWP/9XGz6x4DI\nHPU65kbUn/pWXBUVVlpsKsdmWA2Bu8pd/vD+\n-----END CERTIFICATE-----\n"
+const previousKey = "-----BEGIN RSA PRIVATE KEY-----\nMIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4\n9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R\n2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo\nxTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+\nZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr\nKl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh\nAOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY\n-----END RSA PRIVATE KEY-----"
+const nextCertificate = "-----BEGIN CERTIFICATE-----\nMIIBZzCCARGgAwIBAgIBBDANBgkqhkiG9w0BAQsFADAaMRgwFgYDVQQDEw9zZXJ2\naWNlLWFjY291bnQwHhcNMjEwNTAyMjAzMjE3WhcNMzEwNTAyMjAzMjE3WjAaMRgw\nFgYDVQQDEw9zZXJ2aWNlLWFjY291bnQwXDANBgkqhkiG9w0BAQEFAANLADBIAkEA\no4Tridlsf4Yz3UAiup/scSTiG/OqxkUW3Fz7zGKvVcLeYj9GEIKuzoB1VFk1nboD\nq4cCuGLfdzaQdCQKPIsDuwIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T\nAQH/BAUwAwEB/zAdBgNVHQ4EFgQUhPbxEmUbwVOCa+fZgxreFhf67UEwDQYJKoZI\nhvcNAQELBQADQQALMsyK2Q7C/bk27eCvXyZKUfrLvor10hEjwGhv14zsKWDeTj/J\nA1LPYp7U9VtFfgFOkVbkLE9Rstc0ltNrPqxA\n-----END CERTIFICATE-----\n"
+const nextKey = "-----BEGIN RSA PRIVATE KEY-----\nMIIBOgIBAAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCC\nrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQJAejInjmEzqmzQr0NxcIN4\nPukwK3FBKl+RAOZfqNIKcww14mfOn7Gc6lF2zEC4GnLiB3tthbSXoBGi54nkW4ki\nyQIhANZNne9UhQlwyjsd3WxDWWrl6OOZ3J8ppMOIQni9WRLlAiEAw1XEdxPOSOSO\nB6rucpTT1QivVvyEFIb/ukvPm769Mh8CIQDNQwKnHdlfNX0+KljPPaMD1LrAZbr/\naC+8aWLhqtsKUQIgF7gUcTkwdV17eabh6Xv09Qtm7zMefred2etWvFy+8JUCIECv\nFYOKQVWHX+Q7CHX2K1oTECVnZuW1UItdDYVlFYxQ\n-----END RSA PRIVATE KEY-----"
 
-func mustParsePrivateKey(s string) *pki.PrivateKey {
+func simplePrivateKeyset(s string) *kops.Keyset {
-	k, err := pki.ParsePEMPrivateKey([]byte(s))
+	return &kops.Keyset{
-	if err != nil {
+		Spec: kops.KeysetSpec{
-		klog.Fatalf("error parsing private key %v", err)
+			PrimaryId: "3",
+			Keys: []kops.KeysetItem{
+				{
+					Id:              "3",
+					PrivateMaterial: []byte(s),
+				},
+			},
+		},
+	}
+}
+
+func rotatingPrivateKeyset() *kops.Keyset {
+	return &kops.Keyset{
+		Spec: kops.KeysetSpec{
+			PrimaryId: "3",
+			Keys: []kops.KeysetItem{
+				{
+					Id:              "2",
+					PrivateMaterial: []byte(previousKey),
+					PublicMaterial:  []byte(previousCertificate),
+				},
+				{
+					Id:              "3",
+					PrivateMaterial: []byte(dummyKey),
+					PublicMaterial:  []byte(dummyCertificate),
+				},
+				{
+					Id:              "4",
+					PrivateMaterial: []byte(nextKey),
+					PublicMaterial:  []byte(nextCertificate),
+				},
+			},
+		},
 	}
-	return k
 }
 
 func mustParseCertificate(s string) *pki.Certificate {
@@ -303,13 +338,13 @@ func RunGoldenTest(t *testing.T, basedir string, key string, builder func(*Nodeu
 
 	keystore := &fakeCAStore{}
 	keystore.T = t
-	keystore.privateKeys = map[string]*pki.PrivateKey{
+	keystore.privateKeysets = map[string]*kops.Keyset{
-		"ca":                      mustParsePrivateKey(dummyKey),
+		"ca":                      simplePrivateKeyset(dummyKey),
-		"apiserver-aggregator-ca": mustParsePrivateKey(dummyKey),
+		"apiserver-aggregator-ca": simplePrivateKeyset(dummyKey),
-		"kube-controller-manager": mustParsePrivateKey(dummyKey),
+		"kube-controller-manager": simplePrivateKeyset(dummyKey),
-		"kube-proxy":              mustParsePrivateKey(dummyKey),
+		"kube-proxy":              simplePrivateKeyset(dummyKey),
-		"kube-scheduler":          mustParsePrivateKey(dummyKey),
+		"kube-scheduler":          simplePrivateKeyset(dummyKey),
-		"service-account":         mustParsePrivateKey(dummyKey),
+		"service-account":         rotatingPrivateKeyset(),
 	}
 	keystore.certs = map[string]*pki.Certificate{
 		"ca":                      mustParseCertificate(dummyCertificate),

nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml

@@ -62,7 +62,7 @@ contents: |
       - --requestheader-group-headers=X-Remote-Group
       - --requestheader-username-headers=X-Remote-User
       - --secure-port=443
-      - --service-account-key-file=/srv/kubernetes/service-account.key
+      - --service-account-key-file=/srv/kubernetes/service-account.pub
       - --service-cluster-ip-range=100.64.0.0/13
       - --storage-backend=etcd3
       - --tls-cert-file=/srv/kubernetes/server.crt
@@ -248,6 +248,28 @@ mode: "0600"
 path: /srv/kubernetes/kubelet-api.key
 type: file
 ---
+contents: |
+  -----BEGIN RSA PUBLIC KEY-----
+  MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm
+  XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ==
+  -----END RSA PUBLIC KEY-----
+  -----BEGIN RSA PUBLIC KEY-----
+  MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l
+  Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD
+  ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp
+  OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m
+  74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG
+  kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF
+  6QIDAQAB
+  -----END RSA PUBLIC KEY-----
+  -----BEGIN RSA PUBLIC KEY-----
+  MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF
+  Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ==
+  -----END RSA PUBLIC KEY-----
+mode: "0600"
+path: /srv/kubernetes/service-account.pub
+type: file
+---
 contents: ""
 ifNotExists: true
 mode: "0400"

nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml

@@ -40,7 +40,7 @@ contents: |
       - --requestheader-group-headers=X-Remote-Group
       - --requestheader-username-headers=X-Remote-User
       - --secure-port=443
-      - --service-account-key-file=/srv/kubernetes/service-account.key
+      - --service-account-key-file=/srv/kubernetes/service-account.pub
       - --service-cluster-ip-range=100.64.0.0/13
       - --storage-backend=etcd3
       - --tls-cert-file=/srv/kubernetes/server.crt
@@ -188,6 +188,28 @@ mode: "0600"
 path: /srv/kubernetes/kubelet-api.key
 type: file
 ---
+contents: |
+  -----BEGIN RSA PUBLIC KEY-----
+  MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm
+  XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ==
+  -----END RSA PUBLIC KEY-----
+  -----BEGIN RSA PUBLIC KEY-----
+  MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l
+  Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD
+  ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp
+  OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m
+  74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG
+  kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF
+  6QIDAQAB
+  -----END RSA PUBLIC KEY-----
+  -----BEGIN RSA PUBLIC KEY-----
+  MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF
+  Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ==
+  -----END RSA PUBLIC KEY-----
+mode: "0600"
+path: /srv/kubernetes/service-account.pub
+type: file
+---
 contents: ""
 ifNotExists: true
 mode: "0400"

nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml

@@ -40,7 +40,7 @@ contents: |
       - --requestheader-group-headers=X-Remote-Group
       - --requestheader-username-headers=X-Remote-User
       - --secure-port=443
-      - --service-account-key-file=/srv/kubernetes/service-account.key
+      - --service-account-key-file=/srv/kubernetes/service-account.pub
       - --service-cluster-ip-range=100.64.0.0/13
       - --storage-backend=etcd3
       - --tls-cert-file=/srv/kubernetes/server.crt
@@ -188,6 +188,28 @@ mode: "0600"
 path: /srv/kubernetes/kubelet-api.key
 type: file
 ---
+contents: |
+  -----BEGIN RSA PUBLIC KEY-----
+  MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm
+  XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ==
+  -----END RSA PUBLIC KEY-----
+  -----BEGIN RSA PUBLIC KEY-----
+  MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l
+  Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD
+  ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp
+  OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m
+  74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG
+  kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF
+  6QIDAQAB
+  -----END RSA PUBLIC KEY-----
+  -----BEGIN RSA PUBLIC KEY-----
+  MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF
+  Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ==
+  -----END RSA PUBLIC KEY-----
+mode: "0600"
+path: /srv/kubernetes/service-account.pub
+type: file
+---
 contents: ""
 ifNotExists: true
 mode: "0400"

nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml

@@ -40,7 +40,7 @@ contents: |
       - --requestheader-group-headers=X-Remote-Group
       - --requestheader-username-headers=X-Remote-User
       - --secure-port=443
-      - --service-account-key-file=/srv/kubernetes/service-account.key
+      - --service-account-key-file=/srv/kubernetes/service-account.pub
       - --service-cluster-ip-range=100.64.0.0/13
       - --storage-backend=etcd3
       - --tls-cert-file=/srv/kubernetes/server.crt
@@ -188,6 +188,28 @@ mode: "0600"
 path: /srv/kubernetes/kubelet-api.key
 type: file
 ---
+contents: |
+  -----BEGIN RSA PUBLIC KEY-----
+  MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm
+  XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ==
+  -----END RSA PUBLIC KEY-----
+  -----BEGIN RSA PUBLIC KEY-----
+  MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l
+  Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD
+  ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp
+  OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m
+  74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG
+  kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF
+  6QIDAQAB
+  -----END RSA PUBLIC KEY-----
+  -----BEGIN RSA PUBLIC KEY-----
+  MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF
+  Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ==
+  -----END RSA PUBLIC KEY-----
+mode: "0600"
+path: /srv/kubernetes/service-account.pub
+type: file
+---
 contents: ""
 ifNotExists: true
 mode: "0400"

nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml

@@ -40,7 +40,7 @@ contents: |
       - --requestheader-group-headers=X-Remote-Group
       - --requestheader-username-headers=X-Remote-User
       - --secure-port=443
-      - --service-account-key-file=/srv/kubernetes/service-account.key
+      - --service-account-key-file=/srv/kubernetes/service-account.pub
       - --service-cluster-ip-range=100.64.0.0/13
       - --storage-backend=etcd3
       - --tls-cert-file=/srv/kubernetes/server.crt
@@ -188,6 +188,28 @@ mode: "0600"
 path: /srv/kubernetes/kubelet-api.key
 type: file
 ---
+contents: |
+  -----BEGIN RSA PUBLIC KEY-----
+  MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm
+  XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ==
+  -----END RSA PUBLIC KEY-----
+  -----BEGIN RSA PUBLIC KEY-----
+  MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l
+  Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD
+  ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp
+  OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m
+  74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG
+  kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF
+  6QIDAQAB
+  -----END RSA PUBLIC KEY-----
+  -----BEGIN RSA PUBLIC KEY-----
+  MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF
+  Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ==
+  -----END RSA PUBLIC KEY-----
+mode: "0600"
+path: /srv/kubernetes/service-account.pub
+type: file
+---
 contents: ""
 ifNotExists: true
 mode: "0400"

nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml

@@ -39,7 +39,7 @@ contents: |
       - --requestheader-group-headers=X-Remote-Group
       - --requestheader-username-headers=X-Remote-User
       - --secure-port=443
-      - --service-account-key-file=/srv/kubernetes/service-account.key
+      - --service-account-key-file=/srv/kubernetes/service-account.pub
       - --service-cluster-ip-range=100.64.0.0/13
       - --storage-backend=etcd3
       - --tls-cert-file=/srv/kubernetes/server.crt
@@ -187,6 +187,28 @@ mode: "0600"
 path: /srv/kubernetes/kubelet-api.key
 type: file
 ---
+contents: |
+  -----BEGIN RSA PUBLIC KEY-----
+  MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm
+  XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ==
+  -----END RSA PUBLIC KEY-----
+  -----BEGIN RSA PUBLIC KEY-----
+  MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l
+  Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD
+  ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp
+  OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m
+  74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG
+  kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF
+  6QIDAQAB
+  -----END RSA PUBLIC KEY-----
+  -----BEGIN RSA PUBLIC KEY-----
+  MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF
+  Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ==
+  -----END RSA PUBLIC KEY-----
+mode: "0600"
+path: /srv/kubernetes/service-account.pub
+type: file
+---
 contents: ""
 ifNotExists: true
 mode: "0400"

pkg/apis/kops/keyset.go

@@ -67,6 +67,9 @@ type KeysetSpec struct {
 	// Type is the type of the Keyset (PKI keypair, or secret token)
 	Type KeysetType `json:"type,omitempty"`
 
+	// PrimaryId is the id of the key used to make new signatures.
+	PrimaryId string `json:"primaryId,omitempty"`
+
 	// Keys is the set of keys that make up the keyset
 	Keys []KeysetItem `json:"keys,omitempty"`
 }

pkg/apis/kops/v1alpha2/keyset.go

@@ -68,6 +68,9 @@ type KeysetSpec struct {
 	// Type is the type of the Keyset (PKI keypair, or secret token)
 	Type KeysetType `json:"type,omitempty"`
 
+	// PrimaryId is the id of the key used to make new signatures.
+	PrimaryId string `json:"primaryId,omitempty"`
+
 	// Keys is the set of keys that make up the keyset
 	Keys []KeysetItem `json:"keys,omitempty"`
 }

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -4458,6 +4458,7 @@ func Convert_kops_KeysetList_To_v1alpha2_KeysetList(in *kops.KeysetList, out *Ke
 
 func autoConvert_v1alpha2_KeysetSpec_To_kops_KeysetSpec(in *KeysetSpec, out *kops.KeysetSpec, s conversion.Scope) error {
 	out.Type = kops.KeysetType(in.Type)
+	out.PrimaryId = in.PrimaryId
 	if in.Keys != nil {
 		in, out := &in.Keys, &out.Keys
 		*out = make([]kops.KeysetItem, len(*in))
@@ -4479,6 +4480,7 @@ func Convert_v1alpha2_KeysetSpec_To_kops_KeysetSpec(in *KeysetSpec, out *kops.Ke
 
 func autoConvert_kops_KeysetSpec_To_v1alpha2_KeysetSpec(in *kops.KeysetSpec, out *KeysetSpec, s conversion.Scope) error {
 	out.Type = KeysetType(in.Type)
+	out.PrimaryId = in.PrimaryId
 	if in.Keys != nil {
 		in, out := &in.Keys, &out.Keys
 		*out = make([]KeysetItem, len(*in))

pkg/configserver/keystore.go

@@ -38,22 +38,27 @@ func NewKeyStore(nodeConfig *nodeup.NodeConfig) fi.CAStore {
 	}
 }
 
-// FindKeypair implements fi.Keystore
+// FindPrimaryKeypair implements pki.Keystore
-func (s *configserverKeyStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) {
+func (s *configserverKeyStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) {
-	return nil, nil, false, fmt.Errorf("FindKeypair %q not supported by configserverKeyStore", name)
+	return nil, nil, fmt.Errorf("FindPrimaryKeypair %q not supported by configserverKeyStore", name)
 }
 
-// FindKeypair implements fi.Keystore
+// FindKeyset implements fi.Keystore
+func (s *configserverKeyStore) FindKeyset(name string) (*fi.Keyset, error) {
+	return nil, fmt.Errorf("FindKeyset %q not supported by configserverKeyStore", name)
+}
+
+// CreateKeypair implements fi.Keystore
 func (s *configserverKeyStore) CreateKeypair(signer string, name string, template *x509.Certificate, privateKey *pki.PrivateKey) (*pki.Certificate, error) {
 	return nil, fmt.Errorf("CreateKeypair not supported by configserverKeyStore")
 }
 
-// FindKeypair implements fi.Keystore
+// StoreKeyset implements fi.Keystore
-func (s *configserverKeyStore) StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error {
+func (s *configserverKeyStore) StoreKeyset(name string, keyset *fi.Keyset) error {
-	return fmt.Errorf("StoreKeypair not supported by configserverKeyStore")
+	return fmt.Errorf("StoreKeyset not supported by configserverKeyStore")
 }
 
-// FindKeypair implements fi.Keystore
+// MirrorTo implements fi.Keystore
 func (s *configserverKeyStore) MirrorTo(basedir vfs.Path) error {
 	return fmt.Errorf("MirrorTo not supported by configserverKeyStore")
 }
@@ -104,11 +109,6 @@ func (s *configserverKeyStore) ListKeysets() ([]*kops.Keyset, error) {
 	return nil, fmt.Errorf("ListKeysets not supported by configserverKeyStore")
 }
 
-// AddCert implements fi.CAStore
-func (s *configserverKeyStore) AddCert(name string, cert *pki.Certificate) error {
-	return fmt.Errorf("AddCert not supported by configserverKeyStore")
-}
-
 // DeleteKeysetItem implements fi.CAStore
 func (s *configserverKeyStore) DeleteKeysetItem(item *kops.Keyset, id string) error {
 	return fmt.Errorf("DeleteKeysetItem not supported by configserverKeyStore")

pkg/kubeconfig/create_kubecfg.go

@@ -110,12 +110,12 @@ func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.Se
 	// add the CA Cert to the kubeconfig only if we didn't specify a certificate for the LB
 	//  or if we're using admin credentials and the secondary port
 	if cluster.Spec.API == nil || cluster.Spec.API.LoadBalancer == nil || cluster.Spec.API.LoadBalancer.SSLCertificate == "" || cluster.Spec.API.LoadBalancer.Class == kops.LoadBalancerClassNetwork || internal {
-		cert, _, _, err := keyStore.FindKeypair(fi.CertificateIDCA)
+		keySet, err := keyStore.FindKeyset(fi.CertificateIDCA)
 		if err != nil {
 			return nil, fmt.Errorf("error fetching CA keypair: %v", err)
 		}
-		if cert != nil {
+		if keySet != nil && keySet.Primary != nil && keySet.Primary.Certificate != nil {
-			b.CACert, err = cert.AsBytes()
+			b.CACert, err = keySet.Primary.Certificate.AsBytes()
 			if err != nil {
 				return nil, err
 			}

pkg/kubeconfig/create_kubecfg_test.go

@@ -85,21 +85,25 @@ func (f fakeStatusCloud) FindClusterStatus(cluster *kops.Cluster) (*kops.Cluster
 
 // mock a fake key store
 type fakeKeyStore struct {
-	FindKeypairFn func(name string) (*pki.Certificate, *pki.PrivateKey, bool, error)
+	FindKeysetFn func(name string) (*fi.Keyset, error)
 
-	// StoreKeypair writes the keypair to the store
+	// StoreKeysetFn writes the keyset to the store.
-	StoreKeypairFn func(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error
+	StoreKeysetFn func(name string, keyset *fi.Keyset) error
 
 	// MirrorTo will copy secrets to a vfs.Path, which is often easier for a machine to read
 	MirrorToFn func(basedir vfs.Path) error
 }
 
-func (f fakeKeyStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) {
+func (f fakeKeyStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) {
-	return f.FindKeypairFn(name)
+	return fi.FindPrimaryKeypair(f, name)
 }
 
-func (f fakeKeyStore) StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error {
+func (f fakeKeyStore) FindKeyset(name string) (*fi.Keyset, error) {
-	return f.StoreKeypairFn(id, cert, privateKey)
+	return f.FindKeysetFn(name)
+}
+
+func (f fakeKeyStore) StoreKeyset(name string, keyset *fi.Keyset) error {
+	return f.StoreKeysetFn(name, keyset)
 }
 
 func (f fakeKeyStore) MirrorTo(basedir vfs.Path) error {
@@ -124,16 +128,22 @@ func buildMinimalCluster(clusterName string, masterPublicName string, lbCert boo
 	return cluster
 }
 
-// create a fake certificate
+// create a fake keyset
-func fakeCertificate() *pki.Certificate {
+func fakeKeyset() *fi.Keyset {
 	cert, _ := pki.ParsePEMCertificate([]byte(certData))
-	return cert
-}
-
-// create a fake private key
-func fakePrivateKey() *pki.PrivateKey {
 	key, _ := pki.ParsePEMPrivateKey([]byte(privatekeyData))
-	return key
+	keysetItem := &fi.KeysetItem{
+		Id:          "99",
+		Certificate: cert,
+		PrivateKey:  key,
+	}
+	return &fi.Keyset{
+		LegacyFormat: true,
+		Items: map[string]*fi.KeysetItem{
+			"99": keysetItem,
+		},
+		Primary: keysetItem,
+	}
 }
 
 func TestBuildKubecfg(t *testing.T) {
@@ -350,10 +360,8 @@ func TestBuildKubecfg(t *testing.T) {
 			kopsStateStore := "memfs://example-state-store"
 
 			keyStore := fakeKeyStore{
-				FindKeypairFn: func(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) {
+				FindKeysetFn: func(name string) (*fi.Keyset, error) {
-					return fakeCertificate(),
+					return fakeKeyset(),
-						fakePrivateKey(),
-						true,
 						nil
 				},
 			}

pkg/model/bootstrapscript.go

@@ -231,6 +231,7 @@ func (b *BootstrapScriptBuilder) ResourceNodeUp(c *fi.ModelBuilderContext, ig *k
 		ig:      ig,
 		builder: b,
 		caTask:  caTask,
+		// TODO: use caTask.Keyset() and expose all CA certificates
 		ca: caTask.Certificate(),
 	}
 	task.resource.Task = task

pkg/model/issuerdiscovery.go

@@ -22,9 +22,9 @@ import (
 	"crypto/x509"
 	"encoding/base64"
 	"encoding/json"
-	"encoding/pem"
 	"fmt"
 	"io"
+	"sort"
 
 	"gopkg.in/square/go-jose.v2"
 	"k8s.io/kops/pkg/apis/kops"
@@ -128,19 +128,11 @@ func (o *OIDCKeys) GetDependencies(tasks map[string]fi.Task) []fi.Task {
 	}
 }
 func (o *OIDCKeys) Open() (io.Reader, error) {
+	keyset := o.SigningKey.Keyset()
+	var keys []jose.JSONWebKey
 
-	certBytes, err := fi.ResourceAsBytes(o.SigningKey.Certificate())
+	for _, item := range keyset.Items {
-	if err != nil {
+		publicKey := item.Certificate.PublicKey
-		return nil, fmt.Errorf("failed to get cert: %w", err)
-	}
-	block, _ := pem.Decode(certBytes)
-	cert, err := x509.ParseCertificate(block.Bytes)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse cert: %w", err)
-	}
-
-	publicKey := cert.PublicKey
-
 		publicKeyDERBytes, err := x509.MarshalPKIXPublicKey(publicKey)
 		if err != nil {
 			return nil, fmt.Errorf("failed to serialize public key to DER format: %v", err)
@@ -152,14 +144,16 @@ func (o *OIDCKeys) Open() (io.Reader, error) {
 
 		keyID := base64.RawURLEncoding.EncodeToString(publicKeyDERHash)
 
-	keys := []jose.JSONWebKey{
+		keys = append(keys, jose.JSONWebKey{
-		{
 			Key:       publicKey,
 			KeyID:     keyID,
 			Algorithm: string(jose.RS256),
 			Use:       "sig",
-		},
+		})
 	}
+	sort.Slice(keys, func(i, j int) bool {
+		return keys[i].KeyID < keys[j].KeyID
+	})
 
 	keyResponse := KeyResponse{Keys: keys}
 	jsonBytes, err := json.MarshalIndent(keyResponse, "", "")

pkg/pki/issue.go

@@ -56,12 +56,9 @@ type IssueCertRequest struct {
 }
 
 type Keystore interface {
-	// FindKeypair finds a cert & private key, returning nil where either is not found
+	// FindPrimaryKeypair finds a cert & private key, returning nil where either is not found
 	// (if the certificate is found but not keypair, that is not an error: only the cert will be returned).
-	// This func returns a cert, private key and a bool.  The bool value is whether the keypair is stored
+	FindPrimaryKeypair(name string) (*Certificate, *PrivateKey, error)
-	// in a legacy format. This bool is used by a keypair
-	// task to convert a Legacy Keypair to the new Keypair API format.
-	FindKeypair(name string) (*Certificate, *PrivateKey, bool, error)
 }
 
 // IssueCert issues a certificate, either a self-signed CA or from a CA in a keystore.
@@ -119,7 +116,7 @@ func IssueCert(request *IssueCertRequest, keystore Keystore) (issuedCertificate
 	var signer *x509.Certificate
 	if !template.IsCA {
 		var err error
-		caCertificate, caPrivateKey, _, err = keystore.FindKeypair(request.Signer)
+		caCertificate, caPrivateKey, err = keystore.FindPrimaryKeypair(request.Signer)
 		if err != nil {
 			return nil, nil, nil, err
 		}

pkg/pki/issue_test.go

@@ -38,11 +38,11 @@ type mockKeystore struct {
 	invoked bool
 }
 
-func (m *mockKeystore) FindKeypair(name string) (*Certificate, *PrivateKey, bool, error) {
+func (m *mockKeystore) FindPrimaryKeypair(name string) (*Certificate, *PrivateKey, error) {
 	assert.False(m.t, m.invoked, "invoked already")
 	m.invoked = true
 	assert.Equal(m.t, m.signer, name, "name argument")
-	return m.cert, m.key, false, nil
+	return m.cert, m.key, nil
 }
 
 func TestIssueCert(t *testing.T) {

upup/pkg/fi/BUILD.bazel

@@ -67,7 +67,6 @@ go_library(
 
 go_test(
     name = "go_default_test",
-    size = "small",
     srcs = [
         "dryruntarget_test.go",
         "files_test.go",

upup/pkg/fi/ca.go

@@ -18,7 +18,12 @@ package fi
 
 import (
 	"bytes"
+	"crypto/x509"
+	"encoding/pem"
 	"fmt"
+	"math/big"
+	"sort"
+	"strconv"
 
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/pki"
@@ -43,17 +48,30 @@ type KeystoreItem struct {
 	Data []byte
 }
 
+// Keyset is a parsed api.Keyset.
+type Keyset struct {
+	// LegacyFormat instructs a keypair task to convert a Legacy Keyset to the new Keyset API format.
+	LegacyFormat bool
+	Items        map[string]*KeysetItem
+	Primary      *KeysetItem
+}
+
+// KeysetItem is a certificate/key pair in a Keyset.
+type KeysetItem struct {
+	Id          string
+	Certificate *pki.Certificate
+	PrivateKey  *pki.PrivateKey
+}
+
 // Keystore contains just the functions we need to issue keypairs, not to list / manage them
 type Keystore interface {
-	// FindKeypair finds a cert & private key, returning nil where either is not found
+	pki.Keystore
-	// (if the certificate is found but not keypair, that is not an error: only the cert will be returned).
-	// This func returns a cert, private key and a bool.  The bool value is whether the keypair is stored
-	// in a legacy format. This bool is used by a keypair
-	// task to convert a Legacy Keypair to the new Keypair API format.
-	FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error)
 
-	// StoreKeypair writes the keypair to the store
+	// FindKeyset finds a Keyset.
-	StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error
+	FindKeyset(name string) (*Keyset, error)
+
+	// StoreKeyset writes a Keyset to the store.
+	StoreKeyset(name string, keyset *Keyset) error
 
 	// MirrorTo will copy secrets to a vfs.Path, which is often easier for a machine to read
 	MirrorTo(basedir vfs.Path) error
@@ -86,9 +104,6 @@ type CAStore interface {
 	// The key material is not guaranteed to be populated - metadata like the name will be.
 	ListKeysets() ([]*kops.Keyset, error)
 
-	// AddCert adds an alternative certificate to the pool (primarily useful for CAs)
-	AddCert(name string, cert *pki.Certificate) error
-
 	// DeleteKeysetItem will delete the specified item from the Keyset
 	DeleteKeysetItem(item *kops.Keyset, id string) error
 }
@@ -145,3 +160,69 @@ func (c *CertificatePool) AsString() (string, error) {
 	}
 	return data.String(), nil
 }
+
+// FindPrimaryKeypair is a common implementation of pki.FindPrimaryKeypair.
+func FindPrimaryKeypair(c Keystore, name string) (*pki.Certificate, *pki.PrivateKey, error) {
+	keyset, err := c.FindKeyset(name)
+	if err != nil {
+		return nil, nil, err
+	}
+	if keyset == nil || keyset.Primary == nil {
+		return nil, nil, nil
+	}
+	return keyset.Primary.Certificate, keyset.Primary.PrivateKey, nil
+}
+
+// AddCert adds an alternative certificate to the keyset (primarily useful for CAs)
+func AddCert(keyset *Keyset, cert *pki.Certificate) {
+	serial := 0
+
+	for keyset.Items[strconv.Itoa(serial)] != nil {
+		serial++
+	}
+
+	keyset.Items[strconv.Itoa(serial)] = &KeysetItem{
+		Id:          strconv.Itoa(serial),
+		Certificate: cert,
+	}
+}
+
+// KeysetItemIdOlder returns whether the KeysetItem Id a is older than b.
+func KeysetItemIdOlder(a, b string) bool {
+	aVersion, aOk := big.NewInt(0).SetString(a, 10)
+	bVersion, bOk := big.NewInt(0).SetString(b, 10)
+	if aOk {
+		if !bOk {
+			return false
+		}
+		return aVersion.Cmp(bVersion) < 0
+	} else {
+		if bOk {
+			return true
+		}
+		return a < b
+	}
+}
+
+func (k *Keyset) ToPublicKeyBytes() ([]byte, error) {
+	keys := make([]string, 0, len(k.Items))
+	for k := range k.Items {
+		keys = append(keys, k)
+	}
+	sort.Slice(keys, func(i, j int) bool {
+		return KeysetItemIdOlder(k.Items[keys[i]].Id, k.Items[keys[j]].Id)
+	})
+
+	buf := new(bytes.Buffer)
+	for _, key := range keys {
+		item := k.Items[key]
+		publicKeyData, err := x509.MarshalPKIXPublicKey(item.Certificate.PublicKey)
+		if err != nil {
+			return nil, fmt.Errorf("marshalling public key %s: %v", item.Id, err)
+		}
+		if err = pem.Encode(buf, &pem.Block{Type: "RSA PUBLIC KEY", Bytes: publicKeyData}); err != nil {
+			return nil, fmt.Errorf("encoding public key %s: %v", item.Id, err)
+		}
+	}
+	return buf.Bytes(), nil
+}

upup/pkg/fi/clientset_castore.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"fmt"
 	"math/big"
+	"sort"
 
 	"golang.org/x/crypto/ssh"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -65,30 +66,16 @@ func NewClientsetSSHCredentialStore(cluster *kops.Cluster, clientset kopsinterna
 	return c
 }
 
-// keyset is a parsed Keyset
+func parseKeyset(o *kops.Keyset) (*Keyset, error) {
-type keyset struct {
-	legacyFormat bool
-	items        map[string]*keysetItem
-	primary      *keysetItem
-}
-
-// keysetItem is a parsed KeysetItem
-type keysetItem struct {
-	id          string
-	certificate *pki.Certificate
-	privateKey  *pki.PrivateKey
-}
-
-func parseKeyset(o *kops.Keyset) (*keyset, error) {
 	name := o.Name
 
-	keyset := &keyset{
+	keyset := &Keyset{
-		items: make(map[string]*keysetItem),
+		Items: make(map[string]*KeysetItem),
 	}
 
 	for _, key := range o.Spec.Keys {
-		ki := &keysetItem{
+		ki := &KeysetItem{
-			id: key.Id,
+			Id: key.Id,
 		}
 		if len(key.PublicMaterial) != 0 {
 			cert, err := pki.ParsePEMCertificate(key.PublicMaterial)
@@ -96,7 +83,7 @@ func parseKeyset(o *kops.Keyset) (*keyset, error) {
 				klog.Warningf("key public material was %s", key.PublicMaterial)
 				return nil, fmt.Errorf("error loading certificate %s/%s: %v", name, key.Id, err)
 			}
-			ki.certificate = cert
+			ki.Certificate = cert
 		}
 
 		if len(key.PrivateMaterial) != 0 {
@@ -104,19 +91,19 @@ func parseKeyset(o *kops.Keyset) (*keyset, error) {
 			if err != nil {
 				return nil, fmt.Errorf("error loading private key %s/%s: %v", name, key.Id, err)
 			}
-			ki.privateKey = privateKey
+			ki.PrivateKey = privateKey
 		}
 
-		keyset.items[key.Id] = ki
+		keyset.Items[key.Id] = ki
 	}
 
-	keyset.primary = keyset.findPrimary()
+	keyset.Primary = keyset.Items[FindPrimary(o).Id]
 
 	return keyset, nil
 }
 
-// loadKeyset gets the named keyset and the format of the Keyset.
+// loadKeyset gets the named Keyset and the format of the Keyset.
-func (c *ClientsetCAStore) loadKeyset(ctx context.Context, name string) (*keyset, error) {
+func (c *ClientsetCAStore) loadKeyset(ctx context.Context, name string) (*Keyset, error) {
 	o, err := c.clientset.Keysets(c.namespace).Get(ctx, name, metav1.GetOptions{})
 	if err != nil {
 		if errors.IsNotFound(err) {
@@ -132,30 +119,13 @@ func (c *ClientsetCAStore) loadKeyset(ctx context.Context, name string) (*keyset
 	return keyset, nil
 }
 
-// findPrimary returns the primary keysetItem in the keyset
-func (k *keyset) findPrimary() *keysetItem {
-	var primary *keysetItem
-	var primaryVersion *big.Int
-
-	for _, item := range k.items {
-		version, ok := big.NewInt(0).SetString(item.id, 10)
-		if !ok {
-			klog.Warningf("Ignoring key item with non-integer version: %q", item.id)
-			continue
-		}
-
-		if primaryVersion == nil || version.Cmp(primaryVersion) > 0 {
-			primary = item
-			primaryVersion = version
-		}
-	}
-	return primary
-}
-
 // FindPrimary returns the primary KeysetItem in the Keyset
 func FindPrimary(keyset *kops.Keyset) *kops.KeysetItem {
 	var primary *kops.KeysetItem
 	var primaryVersion *big.Int
+
+	primaryId := keyset.Spec.PrimaryId
+
 	for i := range keyset.Spec.Keys {
 		item := &keyset.Spec.Keys[i]
 		version, ok := big.NewInt(0).SetString(item.Id, 10)
@@ -164,6 +134,10 @@ func FindPrimary(keyset *kops.Keyset) *kops.KeysetItem {
 			continue
 		}
 
+		if item.Id == primaryId {
+			return item
+		}
+
 		if primaryVersion == nil || version.Cmp(primaryVersion) > 0 {
 			primary = item
 			primaryVersion = version
@@ -172,19 +146,15 @@ func FindPrimary(keyset *kops.Keyset) *kops.KeysetItem {
 	return primary
 }
 
-// FindKeypair implements CAStore::FindKeypair
+// FindPrimaryKeypair implements PKI::FindPrimaryKeypair
-func (c *ClientsetCAStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) {
+func (c *ClientsetCAStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) {
+	return FindPrimaryKeypair(c, name)
+}
+
+// FindKeyset implements CAStore::FindKeyset
+func (c *ClientsetCAStore) FindKeyset(name string) (*Keyset, error) {
 	ctx := context.TODO()
-	keyset, err := c.loadKeyset(ctx, name)
+	return c.loadKeyset(ctx, name)
-	if err != nil {
-		return nil, nil, false, err
-	}
-
-	if keyset != nil && keyset.primary != nil {
-		return keyset.primary.certificate, keyset.primary.privateKey, keyset.legacyFormat, nil
-	}
-
-	return nil, nil, false, nil
 }
 
 // FindCert implements CAStore::FindCert
@@ -195,8 +165,8 @@ func (c *ClientsetCAStore) FindCert(name string) (*pki.Certificate, error) {
 		return nil, err
 	}
 
-	if keyset != nil && keyset.primary != nil {
+	if keyset != nil && keyset.Primary != nil {
-		return keyset.primary.certificate, nil
+		return keyset.Primary.Certificate, nil
 	}
 
 	return nil, nil
@@ -213,15 +183,15 @@ func (c *ClientsetCAStore) FindCertificatePool(name string) (*CertificatePool, e
 	pool := &CertificatePool{}
 
 	if keyset != nil {
-		if keyset.primary != nil {
+		if keyset.Primary != nil {
-			pool.Primary = keyset.primary.certificate
+			pool.Primary = keyset.Primary.Certificate
 		}
 
-		for id, item := range keyset.items {
+		for id, item := range keyset.Items {
-			if id == keyset.primary.id {
+			if id == keyset.Primary.Id {
 				continue
 			}
-			pool.Secondary = append(pool.Secondary, item.certificate)
+			pool.Secondary = append(pool.Secondary, item.Certificate)
 		}
 	}
 	return pool, nil
@@ -288,26 +258,10 @@ func (c *ClientsetCAStore) ListSSHCredentials() ([]*kops.SSHCredential, error) {
 	return items, nil
 }
 
-// StoreKeypair implements CAStore::StoreKeypair
+// StoreKeyset implements CAStore::StoreKeyset
-func (c *ClientsetCAStore) StoreKeypair(name string, cert *pki.Certificate, privateKey *pki.PrivateKey) error {
+func (c *ClientsetCAStore) StoreKeyset(name string, keyset *Keyset) error {
 	ctx := context.TODO()
-	return c.storeKeypair(ctx, name, cert.Certificate.SerialNumber.String(), cert, privateKey)
+	return c.storeKeyset(ctx, name, keyset, kops.SecretTypeKeypair)
-}
-
-// AddCert implements CAStore::AddCert
-func (c *ClientsetCAStore) AddCert(name string, cert *pki.Certificate) error {
-	ctx := context.TODO()
-	klog.Infof("Adding TLS certificate: %q", name)
-
-	// We add with a timestamp of zero so this will never be the newest cert
-	serial := pki.BuildPKISerial(0)
-
-	err := c.storeKeypair(ctx, name, serial.String(), cert, nil)
-	if err != nil {
-		return err
-	}
-
-	return nil
 }
 
 // FindPrivateKey implements CAStore::FindPrivateKey
@@ -318,8 +272,8 @@ func (c *ClientsetCAStore) FindPrivateKey(name string) (*pki.PrivateKey, error)
 		return nil, err
 	}
 
-	if keyset != nil && keyset.primary != nil {
+	if keyset != nil && keyset.Primary != nil {
-		return keyset.primary.privateKey, nil
+		return keyset.Primary.PrivateKey, nil
 	}
 	return nil, nil
 }
@@ -337,38 +291,69 @@ func (c *ClientsetCAStore) FindPrivateKeyset(name string) (*kops.Keyset, error)
 	return o, nil
 }
 
-// addKey saves the specified key to the registry
+// storeKeyset saves the specified keyset to the registry.
-func (c *ClientsetCAStore) addKey(ctx context.Context, name string, keysetType kops.KeysetType, item *kops.KeysetItem) error {
+func (c *ClientsetCAStore) storeKeyset(ctx context.Context, name string, keyset *Keyset, keysetType kops.KeysetType) error {
 	create := false
 	client := c.clientset.Keysets(c.namespace)
-	keyset, err := client.Get(ctx, name, metav1.GetOptions{})
+	kopsKeyset, err := client.Get(ctx, name, metav1.GetOptions{})
 	if err != nil {
 		if errors.IsNotFound(err) {
-			keyset = nil
+			kopsKeyset = nil
 		} else {
 			return fmt.Errorf("error reading keyset %q: %v", name, err)
 		}
 	}
-	if keyset == nil {
+
-		keyset = &kops.Keyset{}
+	if kopsKeyset == nil {
-		keyset.Name = name
+		kopsKeyset = &kops.Keyset{}
-		keyset.Spec.Type = keysetType
+		kopsKeyset.Name = name
+		kopsKeyset.Spec.Type = keysetType
 		create = true
 	}
-	keyset.Spec.Keys = append(keyset.Spec.Keys, *item)
+
+	kopsKeyset.Spec.Keys = nil
+	kopsKeyset.Spec.PrimaryId = keyset.Primary.Id
+
+	keys := make([]string, 0, len(keyset.Items))
+	for k := range keyset.Items {
+		keys = append(keys, k)
+	}
+	sort.Slice(keys, func(i, j int) bool {
+		return KeysetItemIdOlder(keyset.Items[keys[i]].Id, keyset.Items[keys[j]].Id)
+	})
+
+	for _, key := range keys {
+		item := keyset.Items[key]
+		var publicMaterial bytes.Buffer
+		if _, err := item.Certificate.WriteTo(&publicMaterial); err != nil {
+			return err
+		}
+
+		var privateMaterial bytes.Buffer
+		if _, err := item.PrivateKey.WriteTo(&privateMaterial); err != nil {
+			return err
+		}
+
+		kopsKeyset.Spec.Keys = append(kopsKeyset.Spec.Keys, kops.KeysetItem{
+			Id:              item.Id,
+			PublicMaterial:  publicMaterial.Bytes(),
+			PrivateMaterial: privateMaterial.Bytes(),
+		})
+	}
+
 	if create {
-		if _, err := client.Create(ctx, keyset, metav1.CreateOptions{}); err != nil {
+		if _, err := client.Create(ctx, kopsKeyset, metav1.CreateOptions{}); err != nil {
 			return fmt.Errorf("error creating keyset %q: %v", name, err)
 		}
 	} else {
-		if _, err := client.Update(ctx, keyset, metav1.UpdateOptions{}); err != nil {
+		if _, err := client.Update(ctx, kopsKeyset, metav1.UpdateOptions{}); err != nil {
 			return fmt.Errorf("error updating keyset %q: %v", name, err)
 		}
 	}
 	return nil
 }
 
-// deleteKeysetItem deletes the specified key from the registry; deleting the whole keyset if it was the last one
+// deleteKeysetItem deletes the specified key from the registry; deleting the whole Keyset if it was the last one.
 func deleteKeysetItem(client kopsinternalversion.KeysetInterface, name string, keysetType kops.KeysetType, id string) error {
 	ctx := context.TODO()
 
@@ -449,26 +434,6 @@ func (c *ClientsetCAStore) deleteSSHCredential(ctx context.Context, name string)
 	return nil
 }
 
-// addKey saves the specified keypair to the registry
-func (c *ClientsetCAStore) storeKeypair(ctx context.Context, name string, id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error {
-	var publicMaterial bytes.Buffer
-	if _, err := cert.WriteTo(&publicMaterial); err != nil {
-		return err
-	}
-
-	var privateMaterial bytes.Buffer
-	if _, err := privateKey.WriteTo(&privateMaterial); err != nil {
-		return err
-	}
-
-	item := &kops.KeysetItem{
-		Id:              id,
-		PublicMaterial:  publicMaterial.Bytes(),
-		PrivateMaterial: privateMaterial.Bytes(),
-	}
-	return c.addKey(ctx, name, kops.SecretTypeKeypair, item)
-}
-
 // AddSSHPublicKey implements CAStore::AddSSHPublicKey
 func (c *ClientsetCAStore) AddSSHPublicKey(name string, pubkey []byte) error {
 	ctx := context.TODO()

upup/pkg/fi/fitasks/keypair.go

@@ -17,7 +17,6 @@ limitations under the License.
 package fitasks
 
 import (
-	"crypto/sha1"
 	"crypto/x509/pkix"
 	"fmt"
 	"sort"
@@ -47,7 +46,7 @@ type Keypair struct {
 	LegacyFormat bool `json:"oldFormat"`
 
 	certificate *fi.TaskDependentResource
-	certificateSHA1Fingerprint *fi.TaskDependentResource
+	keyset      *fi.Keyset
 }
 
 var _ fi.HasCheckExisting = &Keypair{}
@@ -70,14 +69,15 @@ func (e *Keypair) Find(c *fi.Context) (*Keypair, error) {
 		return nil, nil
 	}
 
-	cert, key, legacyFormat, err := c.Keystore.FindKeypair(name)
+	keyset, err := c.Keystore.FindKeyset(name)
 	if err != nil {
 		return nil, err
 	}
-	if cert == nil {
+	if keyset == nil || keyset.Primary == nil || keyset.Primary.Certificate == nil {
 		return nil, nil
 	}
-	if key == nil {
+	cert := keyset.Primary.Certificate
+	if keyset.Primary.PrivateKey == nil {
 		return nil, fmt.Errorf("found cert in store, but did not find private key: %q", name)
 	}
 
@@ -94,7 +94,7 @@ func (e *Keypair) Find(c *fi.Context) (*Keypair, error) {
 		AlternateNames: alternateNames,
 		Subject:        pki.PkixNameToString(&cert.Subject),
 		Type:           pki.BuildTypeDescription(cert.Certificate),
-		LegacyFormat:   legacyFormat,
+		LegacyFormat:   keyset.LegacyFormat,
 	}
 
 	actual.Signer = &Keypair{Subject: pki.PkixNameToString(&cert.Certificate.Issuer)}
@@ -102,7 +102,7 @@ func (e *Keypair) Find(c *fi.Context) (*Keypair, error) {
 	// Avoid spurious changes
 	actual.Lifecycle = e.Lifecycle
 
-	if err := e.setResources(cert); err != nil {
+	if err := e.setResources(keyset); err != nil {
 		return nil, fmt.Errorf("error setting resources: %v", err)
 	}
 
@@ -175,14 +175,23 @@ func (_ *Keypair) Render(c *fi.Context, a, e, changes *Keypair) error {
 	if createCertificate {
 		klog.V(2).Infof("Creating PKI keypair %q", name)
 
-		_, privateKey, _, err := c.Keystore.FindKeypair(name)
+		keyset, err := c.Keystore.FindKeyset(name)
 		if err != nil {
 			return err
 		}
+		if keyset == nil {
+			keyset = &fi.Keyset{
+				Items: map[string]*fi.KeysetItem{},
+			}
+		}
 
 		// We always reuse the private key if it exists,
 		// if we change keys we often have to regenerate e.g. the service accounts
 		// TODO: Eventually rotate keys / don't always reuse?
+		var privateKey *pki.PrivateKey
+		if keyset.Primary != nil {
+			privateKey = keyset.Primary.PrivateKey
+		}
 		if privateKey == nil {
 			klog.V(2).Infof("Creating privateKey %q", name)
 		}
@@ -217,17 +226,28 @@ func (_ *Keypair) Render(c *fi.Context, a, e, changes *Keypair) error {
 		if err != nil {
 			return err
 		}
-		err = c.Keystore.StoreKeypair(name, cert, privateKey)
+
+		serialString := cert.Certificate.SerialNumber.String()
+		ki := &fi.KeysetItem{
+			Id:          serialString,
+			Certificate: cert,
+			PrivateKey:  privateKey,
+		}
+
+		keyset.LegacyFormat = false
+		keyset.Items[ki.Id] = ki
+		keyset.Primary = ki
+		err = c.Keystore.StoreKeyset(name, keyset)
 		if err != nil {
 			return err
 		}
 
-		if err := e.setResources(cert); err != nil {
+		if err := e.setResources(keyset); err != nil {
 			return fmt.Errorf("error setting resources: %v", err)
 		}
 
 		// Make double-sure it round-trips
-		_, _, _, err = c.Keystore.FindKeypair(name)
+		_, err = c.Keystore.FindKeyset(name)
 		if err != nil {
 			return err
 		}
@@ -240,11 +260,12 @@ func (_ *Keypair) Render(c *fi.Context, a, e, changes *Keypair) error {
 	if changeStoredFormat {
 		// We fetch and reinsert the same keypair, forcing an update to our preferred format
 		// TODO: We're assuming that we want to save in the preferred format
-		cert, privateKey, _, err := c.Keystore.FindKeypair(name)
+		keyset, err := c.Keystore.FindKeyset(name)
 		if err != nil {
 			return err
 		}
-		err = c.Keystore.StoreKeypair(name, cert, privateKey)
+		keyset.LegacyFormat = false
+		err = c.Keystore.StoreKeyset(name, keyset)
 		if err != nil {
 			return err
 		}
@@ -285,30 +306,23 @@ func (e *Keypair) ensureResources() {
 	if e.certificate == nil {
 		e.certificate = &fi.TaskDependentResource{Task: e}
 	}
-	if e.certificateSHA1Fingerprint == nil {
-		e.certificateSHA1Fingerprint = &fi.TaskDependentResource{Task: e}
-	}
 }
 
-func (e *Keypair) setResources(cert *pki.Certificate) error {
+func (e *Keypair) setResources(keyset *fi.Keyset) error {
 	e.ensureResources()
 
-	s, err := cert.AsString()
+	s, err := keyset.Primary.Certificate.AsString()
 	if err != nil {
 		return err
 	}
 	e.certificate.Resource = fi.NewStringResource(s)
 
-	fingerprint := sha1.Sum(cert.Certificate.Raw)
+	e.keyset = keyset
-	hex := fmt.Sprintf("%x", fingerprint)
-	e.certificateSHA1Fingerprint.Resource = fi.NewStringResource(hex)
-
 	return nil
 }
 
-func (e *Keypair) CertificateSHA1Fingerprint() fi.Resource {
+func (e *Keypair) Keyset() *fi.Keyset {
-	e.ensureResources()
+	return e.keyset
-	return e.certificateSHA1Fingerprint
 }
 
 func (e *Keypair) Certificate() fi.Resource {

upup/pkg/fi/vfs_castore.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"math/big"
 	"os"
+	"sort"
 	"strings"
 	"sync"
 
@@ -40,7 +41,7 @@ type VFSCAStore struct {
 	cluster *kops.Cluster
 
 	mutex    sync.Mutex
-	cachedCA *keyset
+	cachedCA *Keyset
 }
 
 var _ CAStore = &VFSCAStore{}
@@ -66,8 +67,8 @@ func NewVFSSSHCredentialStore(cluster *kops.Cluster, basedir vfs.Path) SSHCreden
 	return c
 }
 
-func (s *VFSCAStore) VFSPath() vfs.Path {
+func (c *VFSCAStore) VFSPath() vfs.Path {
-	return s.basedir
+	return c.basedir
 }
 
 func (c *VFSCAStore) buildCertificatePoolPath(name string) vfs.Path {
@@ -106,11 +107,12 @@ func (c *VFSCAStore) parseKeysetYaml(data []byte) (*kops.Keyset, bool, error) {
 	return keyset, gvk.Version != keysetFormatLatest, nil
 }
 
-// loadCertificatesBundle loads a keyset from the path
+// loadKeyset loads a Keyset from the path.
 // Returns (nil, nil) if the file is not found
 // Bundles avoid the need for a list-files permission, which can be tricky on e.g. GCE
-func (c *VFSCAStore) loadKeysetBundle(p vfs.Path) (*keyset, error) {
+func (c *VFSCAStore) loadKeyset(p vfs.Path) (*Keyset, error) {
-	data, err := p.ReadFile()
+	bundlePath := p.Join("keyset.yaml")
+	data, err := bundlePath.ReadFile()
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil, nil
@@ -128,31 +130,40 @@ func (c *VFSCAStore) loadKeysetBundle(p vfs.Path) (*keyset, error) {
 		return nil, fmt.Errorf("error mapping bundle %q: %v", p, err)
 	}
 
-	keyset.legacyFormat = legacyFormat
+	keyset.LegacyFormat = legacyFormat
 	return keyset, nil
 }
 
-func (k *keyset) ToAPIObject(name string, includePrivateKeyMaterial bool) (*kops.Keyset, error) {
+func (k *Keyset) ToAPIObject(name string, includePrivateKeyMaterial bool) (*kops.Keyset, error) {
 	o := &kops.Keyset{}
 	o.Name = name
 	o.Spec.Type = kops.SecretTypeKeypair
 
-	for _, ki := range k.items {
+	keys := make([]string, 0, len(k.Items))
+	for k := range k.Items {
+		keys = append(keys, k)
+	}
+	sort.Slice(keys, func(i, j int) bool {
+		return KeysetItemIdOlder(k.Items[keys[i]].Id, k.Items[keys[j]].Id)
+	})
+
+	for _, key := range keys {
+		ki := k.Items[key]
 		oki := kops.KeysetItem{
-			Id: ki.id,
+			Id: ki.Id,
 		}
 
-		if ki.certificate != nil {
+		if ki.Certificate != nil {
 			var publicMaterial bytes.Buffer
-			if _, err := ki.certificate.WriteTo(&publicMaterial); err != nil {
+			if _, err := ki.Certificate.WriteTo(&publicMaterial); err != nil {
 				return nil, err
 			}
 			oki.PublicMaterial = publicMaterial.Bytes()
 		}
 
-		if includePrivateKeyMaterial && ki.privateKey != nil {
+		if includePrivateKeyMaterial && ki.PrivateKey != nil {
 			var privateMaterial bytes.Buffer
-			if _, err := ki.privateKey.WriteTo(&privateMaterial); err != nil {
+			if _, err := ki.PrivateKey.WriteTo(&privateMaterial); err != nil {
 				return nil, err
 			}
 
@@ -161,11 +172,14 @@ func (k *keyset) ToAPIObject(name string, includePrivateKeyMaterial bool) (*kops
 
 		o.Spec.Keys = append(o.Spec.Keys, oki)
 	}
+	if k.Primary != nil {
+		o.Spec.PrimaryId = k.Primary.Id
+	}
 	return o, nil
 }
 
-// writeKeysetBundle writes a keyset bundle to VFS
+// writeKeysetBundle writes a Keyset bundle to VFS.
-func (c *VFSCAStore) writeKeysetBundle(p vfs.Path, name string, keyset *keyset, includePrivateKeyMaterial bool) error {
+func (c *VFSCAStore) writeKeysetBundle(p vfs.Path, name string, keyset *Keyset, includePrivateKeyMaterial bool) error {
 	p = p.Join("keyset.yaml")
 
 	o, err := keyset.ToAPIObject(name, includePrivateKeyMaterial)
@@ -185,7 +199,7 @@ func (c *VFSCAStore) writeKeysetBundle(p vfs.Path, name string, keyset *keyset,
 	return p.WriteFile(bytes.NewReader(objectData), acl)
 }
 
-// serializeKeysetBundle converts a keyset bundle to yaml, for writing to VFS
+// serializeKeysetBundle converts a Keyset bundle to yaml, for writing to VFS.
 func serializeKeysetBundle(o *kops.Keyset) ([]byte, error) {
 	var objectData bytes.Buffer
 	codecs := kopscodecs.Codecs
@@ -203,88 +217,69 @@ func serializeKeysetBundle(o *kops.Keyset) ([]byte, error) {
 
 // removePrivateKeyMaterial returns a copy of the Keyset with the private key data removed
 func removePrivateKeyMaterial(o *kops.Keyset) *kops.Keyset {
-	copy := o.DeepCopy()
+	c := o.DeepCopy()
 
-	for i := range copy.Spec.Keys {
+	for i := range c.Spec.Keys {
-		copy.Spec.Keys[i].PrivateMaterial = nil
+		c.Spec.Keys[i].PrivateMaterial = nil
 	}
 
-	return copy
+	return c
 }
 
-func SerializeKeyset(o *kops.Keyset) ([]byte, error) {
+func (c *VFSCAStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) {
-	var objectData bytes.Buffer
+	return FindPrimaryKeypair(c, name)
-	{
-		codecs := kopscodecs.Codecs
-		yaml, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), "application/yaml")
-		if !ok {
-			klog.Fatalf("no YAML serializer registered")
-		}
-		encoder := codecs.EncoderForVersion(yaml.Serializer, v1alpha2.SchemeGroupVersion)
-
-		if err := encoder.Encode(o, &objectData); err != nil {
-			return nil, fmt.Errorf("error serializing keyset: %v", err)
-		}
 }
 
-	return objectData.Bytes(), nil
+func (c *VFSCAStore) FindKeyset(id string) (*Keyset, error) {
-}
+	certs, err := c.loadKeyset(c.buildCertificatePoolPath(id))
 
-func (c *VFSCAStore) loadCertificates(p vfs.Path) (*keyset, error) {
+	if (certs == nil || os.IsNotExist(err)) && id == "service-account" {
-	bundlePath := p.Join("keyset.yaml")
-	bundle, err := c.loadKeysetBundle(bundlePath)
-	return bundle, err
-}
-
-func (c *VFSCAStore) loadOneCertificate(p vfs.Path) (*pki.Certificate, error) {
-	data, err := p.ReadFile()
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil, nil
-		}
-		return nil, err
-	}
-	cert, err := pki.ParsePEMCertificate(data)
-	if err != nil {
-		return nil, err
-	}
-	if cert == nil {
-		return nil, nil
-	}
-	return cert, nil
-}
-
-func (c *VFSCAStore) FindKeypair(id string) (*pki.Certificate, *pki.PrivateKey, bool, error) {
-	cert, legacyFormat, err := c.findCert(id)
-
-	if (cert == nil || os.IsNotExist(err)) && id == "service-account" {
 		// The strange name is because Kops prior to 1.19 used the api-server TLS key for this.
 		id = "master"
-		cert, _, err = c.findCert(id)
+		certs, err = c.loadKeyset(c.buildCertificatePoolPath(id))
-		legacyFormat = true
+		if certs != nil {
+			certs.LegacyFormat = true
+		}
 	}
 
 	if err != nil {
-		return nil, nil, false, err
+		return nil, err
 	}
 
-	key, err := c.FindPrivateKey(id)
+	keys, err := c.findPrivateKeyset(id)
 	if err != nil {
-		return nil, nil, false, err
+		return nil, err
 	}
 
-	return cert, key, legacyFormat, nil
+	if certs != nil {
+		if keys == nil {
+			return certs, nil
+		}
+		if certs.LegacyFormat {
+			keys.LegacyFormat = true
+		}
+		for key, certItem := range certs.Items {
+			keyItem := keys.Items[key]
+			if keyItem == nil {
+				keys.Items[key] = certItem
+			} else if keyItem.Certificate == nil {
+				keyItem.Certificate = certItem.Certificate
+			}
+		}
+	}
+
+	return keys, nil
 }
 
 func (c *VFSCAStore) findCert(name string) (*pki.Certificate, bool, error) {
 	p := c.buildCertificatePoolPath(name)
-	certs, err := c.loadCertificates(p)
+	certs, err := c.loadKeyset(p)
 	if err != nil {
 		return nil, false, fmt.Errorf("error in 'FindCert' attempting to load cert %q: %v", name, err)
 	}
 
-	if certs != nil && certs.primary != nil {
+	if certs != nil && certs.Primary != nil {
-		return certs.primary.certificate, certs.legacyFormat, nil
+		return certs.Primary.Certificate, certs.LegacyFormat, nil
 	}
 
 	return nil, false, nil
@@ -296,11 +291,11 @@ func (c *VFSCAStore) FindCert(name string) (*pki.Certificate, error) {
 }
 
 func (c *VFSCAStore) FindCertificatePool(name string) (*CertificatePool, error) {
-	var certs *keyset
+	var certs *Keyset
 
 	var err error
 	p := c.buildCertificatePoolPath(name)
-	certs, err = c.loadCertificates(p)
+	certs, err = c.loadKeyset(p)
 	if err != nil {
 		return nil, fmt.Errorf("error in 'FindCertificatePool' attempting to load cert %q: %v", name, err)
 	}
@@ -308,18 +303,18 @@ func (c *VFSCAStore) FindCertificatePool(name string) (*CertificatePool, error)
 	pool := &CertificatePool{}
 
 	if certs != nil {
-		if certs.primary != nil {
+		if certs.Primary != nil {
-			pool.Primary = certs.primary.certificate
+			pool.Primary = certs.Primary.Certificate
 		}
 
-		for k, cert := range certs.items {
+		for k, cert := range certs.Items {
-			if certs.primary != nil && k == certs.primary.id {
+			if certs.Primary != nil && k == certs.Primary.Id {
 				continue
 			}
-			if cert.certificate == nil {
+			if cert.Certificate == nil {
 				continue
 			}
-			pool.Secondary = append(pool.Secondary, cert.certificate)
+			pool.Secondary = append(pool.Secondary, cert.Certificate)
 		}
 	}
 	return pool, nil
@@ -327,7 +322,7 @@ func (c *VFSCAStore) FindCertificatePool(name string) (*CertificatePool, error)
 
 func (c *VFSCAStore) FindCertificateKeyset(name string) (*kops.Keyset, error) {
 	p := c.buildCertificatePoolPath(name)
-	certs, err := c.loadCertificates(p)
+	certs, err := c.loadKeyset(p)
 	if err != nil {
 		return nil, fmt.Errorf("error in 'FindCertificatePool' attempting to load cert %q: %v", name, err)
 	}
@@ -464,7 +459,7 @@ func (c *VFSCAStore) MirrorTo(basedir vfs.Path) error {
 	return nil
 }
 
-// mirrorKeyset writes keyset bundles for the certificates & privatekeys
+// mirrorKeyset writes Keyset bundles for the certificates & privatekeys.
 func mirrorKeyset(cluster *kops.Cluster, basedir vfs.Path, keyset *kops.Keyset) error {
 	primary := FindPrimary(keyset)
 	if primary == nil {
@@ -535,64 +530,26 @@ func mirrorSSHCredential(cluster *kops.Cluster, basedir vfs.Path, sshCredential
 	return nil
 }
 
-func (c *VFSCAStore) StoreKeypair(name string, cert *pki.Certificate, privateKey *pki.PrivateKey) error {
+func (c *VFSCAStore) StoreKeyset(name string, keyset *Keyset) error {
-	serial := cert.Certificate.SerialNumber.String()
-
-	ki := &keysetItem{
-		id:          serial,
-		certificate: cert,
-		privateKey:  privateKey,
-	}
-
 	{
-		err := c.storePrivateKey(name, ki)
+		p := c.buildPrivateKeyPoolPath(name)
-		if err != nil {
+		if err := c.writeKeysetBundle(p, name, keyset, true); err != nil {
-			return err
+			return fmt.Errorf("writing private bundle: %v", err)
 		}
 	}
 
 	{
-		err := c.storeCertificate(name, ki)
+		p := c.buildCertificatePoolPath(name)
-		if err != nil {
+		if err := c.writeKeysetBundle(p, name, keyset, false); err != nil {
-			// TODO: Delete private key?
+			return fmt.Errorf("writing certificate bundle: %v", err)
-			return err
 		}
 	}
 
 	return nil
 }
 
-func (c *VFSCAStore) AddCert(name string, cert *pki.Certificate) error {
+func (c *VFSCAStore) findPrivateKeyset(id string) (*Keyset, error) {
-	klog.Infof("Adding TLS certificate: %q", name)
+	var keys *Keyset
-
-	// We add with a timestamp of zero so this will never be the newest cert
-	serial := pki.BuildPKISerial(0).String()
-
-	p := c.buildCertificatePath(name, serial)
-
-	ki := &keysetItem{
-		id:          serial,
-		certificate: cert,
-	}
-	err := c.storeCertificate(name, ki)
-	if err != nil {
-		return err
-	}
-
-	// Make double-sure it round-trips
-	_, err = c.loadOneCertificate(p)
-	return err
-}
-
-func (c *VFSCAStore) loadPrivateKeys(p vfs.Path) (*keyset, error) {
-	bundlePath := p.Join("keyset.yaml")
-	bundle, err := c.loadKeysetBundle(bundlePath)
-
-	return bundle, err
-}
-
-func (c *VFSCAStore) findPrivateKeyset(id string) (*keyset, error) {
-	var keys *keyset
 	var err error
 	if id == CertificateIDCA {
 		c.mutex.Lock()
@@ -603,7 +560,7 @@ func (c *VFSCAStore) findPrivateKeyset(id string) (*keyset, error) {
 			return cached, nil
 		}
 
-		keys, err = c.loadPrivateKeys(c.buildPrivateKeyPoolPath(id))
+		keys, err = c.loadKeyset(c.buildPrivateKeyPoolPath(id))
 		if err != nil {
 			return nil, err
 		}
@@ -616,7 +573,7 @@ func (c *VFSCAStore) findPrivateKeyset(id string) (*keyset, error) {
 		}
 	} else {
 		p := c.buildPrivateKeyPoolPath(id)
-		keys, err = c.loadPrivateKeys(p)
+		keys, err = c.loadKeyset(p)
 		if err != nil {
 			return nil, err
 		}
@@ -632,8 +589,8 @@ func (c *VFSCAStore) FindPrivateKey(id string) (*pki.PrivateKey, error) {
 	}
 
 	var key *pki.PrivateKey
-	if keys != nil && keys.primary != nil {
+	if keys != nil && keys.Primary != nil {
-		key = keys.primary.privateKey
+		key = keys.Primary.PrivateKey
 	}
 	return key, nil
 }
@@ -652,92 +609,6 @@ func (c *VFSCAStore) FindPrivateKeyset(name string) (*kops.Keyset, error) {
 	return o, nil
 }
 
-func (c *VFSCAStore) storePrivateKey(name string, ki *keysetItem) error {
-	if ki.privateKey == nil {
-		return fmt.Errorf("privateKey not provided to storeCertificate")
-	}
-
-	// Write the bundle
-	{
-		p := c.buildPrivateKeyPoolPath(name)
-		ks, err := c.loadPrivateKeys(p)
-		if err != nil {
-			return err
-		}
-
-		if ks == nil {
-			ks = &keyset{}
-		}
-		if ks.items == nil {
-			ks.items = make(map[string]*keysetItem)
-		}
-		ks.items[ki.id] = ki
-
-		if err := c.writeKeysetBundle(p, name, ks, true); err != nil {
-			return fmt.Errorf("error writing bundle: %v", err)
-		}
-	}
-
-	// TODO stop writing and remove legacy format files after rollback to pre-kops 1.18 not needed
-	// Write the data
-	{
-		var data bytes.Buffer
-		if _, err := ki.privateKey.WriteTo(&data); err != nil {
-			return err
-		}
-
-		p := c.buildPrivateKeyPath(name, ki.id)
-		acl, err := acls.GetACL(p, c.cluster)
-		if err != nil {
-			return err
-		}
-		return p.WriteFile(bytes.NewReader(data.Bytes()), acl)
-	}
-}
-
-func (c *VFSCAStore) storeCertificate(name string, ki *keysetItem) error {
-	if ki.certificate == nil {
-		return fmt.Errorf("certificate not provided to storeCertificate")
-	}
-
-	// Write the bundle
-	{
-		p := c.buildCertificatePoolPath(name)
-		ks, err := c.loadCertificates(p)
-		if err != nil {
-			return err
-		}
-
-		if ks == nil {
-			ks = &keyset{}
-		}
-		if ks.items == nil {
-			ks.items = make(map[string]*keysetItem)
-		}
-		ks.items[ki.id] = ki
-
-		if err := c.writeKeysetBundle(p, name, ks, false); err != nil {
-			return fmt.Errorf("error writing bundle: %v", err)
-		}
-	}
-
-	// TODO stop writing and remove legacy format files after rollback to pre-kops 1.18 not needed
-	// Write the data
-	{
-		var data bytes.Buffer
-		if _, err := ki.certificate.WriteTo(&data); err != nil {
-			return err
-		}
-
-		p := c.buildCertificatePath(name, ki.id)
-		acl, err := acls.GetACL(p, c.cluster)
-		if err != nil {
-			return err
-		}
-		return p.WriteFile(bytes.NewReader(data.Bytes()), acl)
-	}
-}
-
 func (c *VFSCAStore) deletePrivateKey(name string, id string) (bool, error) {
 	// Delete the file itself
 	{
@@ -751,15 +622,18 @@ func (c *VFSCAStore) deletePrivateKey(name string, id string) (bool, error) {
 	// Update the bundle
 	{
 		p := c.buildPrivateKeyPoolPath(name)
-		ks, err := c.loadPrivateKeys(p)
+		ks, err := c.loadKeyset(p)
 		if err != nil {
 			return false, err
 		}
 
-		if ks == nil || ks.items[id] == nil {
+		if ks == nil || ks.Items[id] == nil {
 			return false, nil
 		}
-		delete(ks.items, id)
+		delete(ks.Items, id)
+		if ks.Primary != nil && ks.Primary.Id == id {
+			ks.Primary = nil
+		}
 
 		if err := c.writeKeysetBundle(p, name, ks, true); err != nil {
 			return false, fmt.Errorf("error writing bundle: %v", err)
@@ -781,15 +655,18 @@ func (c *VFSCAStore) deleteCertificate(name string, id string) (bool, error) {
 	// Update the bundle
 	{
 		p := c.buildCertificatePoolPath(name)
-		ks, err := c.loadCertificates(p)
+		ks, err := c.loadKeyset(p)
 		if err != nil {
 			return false, err
 		}
 
-		if ks == nil || ks.items[id] == nil {
+		if ks == nil || ks.Items[id] == nil {
 			return false, nil
 		}
-		delete(ks.items, id)
+		delete(ks.Items, id)
+		if ks.Primary != nil && ks.Primary.Id == id {
+			ks.Primary = nil
+		}
 
 		if err := c.writeKeysetBundle(p, name, ks, false); err != nil {
 			return false, fmt.Errorf("error writing bundle: %v", err)

upup/pkg/fi/vfs_castore_test.go

@@ -24,7 +24,6 @@ import (
 	"testing"
 	"time"
 
-	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/pki"
 	"k8s.io/kops/util/pkg/vfs"
 )
@@ -71,8 +70,19 @@ func TestVFSCAStoreRoundTrip(t *testing.T) {
 		t.Fatalf("error from ParsePEMCertificate: %v", err)
 	}
 
-	if err := s.StoreKeypair("ca", cert, privateKey); err != nil {
+	item := &KeysetItem{
-		t.Fatalf("error from StoreKeypair: %v", err)
+		Id:          "237054359138908419352140518924933177492",
+		Certificate: cert,
+		PrivateKey:  privateKey,
+	}
+	keyset := &Keyset{
+		Items: map[string]*KeysetItem{
+			"237054359138908419352140518924933177492": item,
+		},
+		Primary: item,
+	}
+	if err := s.StoreKeyset("ca", keyset); err != nil {
+		t.Fatalf("error from StoreKeyset: %v", err)
 	}
 
 	paths, err := basePath.ReadTree()
@@ -87,16 +97,14 @@ func TestVFSCAStoreRoundTrip(t *testing.T) {
 
 	for _, p := range []string{
 		"memfs://tests/issued/ca/keyset.yaml",
-		"memfs://tests/issued/ca/237054359138908419352140518924933177492.crt",
 		"memfs://tests/private/ca/keyset.yaml",
-		"memfs://tests/private/ca/237054359138908419352140518924933177492.key",
 	} {
 		if _, found := pathMap[p]; !found {
 			t.Fatalf("file not found: %v", p)
 		}
 	}
 
-	if len(pathMap) != 4 {
+	if len(pathMap) != 2 {
 		t.Fatalf("unexpected pathMap: %v", pathMap)
 	}
 
@@ -117,6 +125,7 @@ spec:
   keys:
   - id: "237054359138908419352140518924933177492"
     publicMaterial: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMyRENDQWNDZ0F3SUJBZ0lSQUxKWEFrVmo5NjR0cTY3d01TSThvSlF3RFFZSktvWklodmNOQVFFTEJRQXcKRlRFVE1CRUdBMVVFQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4TnpFeU1qY3lNelV5TkRCYUZ3MHlOekV5TWpjeQpNelV5TkRCYU1CVXhFekFSQmdOVkJBTVRDbXQxWW1WeWJtVjBaWE13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBCkE0SUJEd0F3Z2dFS0FvSUJBUURnbkNrU210bm1meEVnUzNxTlBhVUNINVFPQkdESC9pbkhiV0NPRExCQ0s5Z2QKWEVjQmw3RlZ2OFQya0ZyMURZYjBIVkR0TUk3dGl4UlZGRExna3dObFczNHh3V2RaWEI3R2VvRmdVMXhXT1FTWQpPQUNDOEpnWVRRLzEzOUhCRXZncTRzZWo2N3ArL3MvU05jdzM0S2s3SEl1RmhsazFyUms1a01leEtJbEpCS1AxCllZVVlldHNKL1FwVU9rcUo1SFc0R29ldEU3Nll0SG5PUmZZdm55YnZpU01yaDJ3R0dhTjZyL3M0Q2hPYUliWkMKQW44L1lpUEtHSURhWkdwajZHWG5tWEFSUlgvVElkZ1NRa0x3dDBhVERCblBaNFh2dHBJOGFhTDhEWUpJcUF6QQpOUEgyYjQvdU55bGF0NWpEbzBiMEc1NGFnTWk5NysyQVVyQzlVVVhwQWdNQkFBR2pJekFoTUE0R0ExVWREd0VCCi93UUVBd0lCQmpBUEJnTlZIUk1CQWY4RUJUQURBUUgvTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFCVkdSMnIKaHpYelJNVTV3cmlQUUFKU2Nzek5PUnZvQnBYZlpvWjA5Rkl1cHVkRnhCVlUzZDRoVjlTdEtuUWdQU0dBNVhRTwpIRTk3K0J4SkR1QS9yQjVvQlVzTUJqYzd5MWNkZS9UNmhtaTNyTG9FWUJTblN1ZENPWEpFNEc5LzBmOGJ5QUplCnJOOCtObzFyMlZnWnZaaDZwNzRURWtYdi9sM0hCUFdNN0lkVVYwSE85SkRoU2dPVkYxZnlRS0p4UnVMSlI4anQKTzZtUEgyVVgwdk13VmE0anZ3dGtkZHFrMk9BZFlRdkg5cmJEampiemFpVzBLbm1kdWVSbzkyS0hBTjdCc0RaeQpWcFhIcHFvMUt6ZzdEM2ZwYVhDZjVzaTdscXFyZEpWWEg0SkM3Mnp4c1BlaHFnaThlSXVxT0JraURXbVJ4QXhoCjh5R2VSeDlBYmtuSGg0SWEKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
+  primaryId: "237054359138908419352140518924933177492"
   type: Keypair
 `
 
@@ -147,18 +156,6 @@ spec:
 		}
 	}
 
-	// Check issued/ca/237054359138908419352140518924933177492.crt round-tripped
-	{
-		roundTrip, err := pathMap["memfs://tests/issued/ca/237054359138908419352140518924933177492.crt"].ReadFile()
-		if err != nil {
-			t.Fatalf("error reading file memfs://tests/issued/ca/237054359138908419352140518924933177492.crt: %v", err)
-		}
-
-		if string(roundTrip) != certData {
-			t.Fatalf("unexpected round-tripped certificate data: %q", string(roundTrip))
-		}
-	}
-
 	// Check private/ca/keyset.yaml round-tripped
 	{
 		privateKeysetYaml, err := pathMap["memfs://tests/private/ca/keyset.yaml"].ReadFile()
@@ -177,6 +174,7 @@ spec:
   - id: "237054359138908419352140518924933177492"
     privateMaterial: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBNEp3cEVwclo1bjhSSUV0NmpUMmxBaCtVRGdSZ3gvNHB4MjFnamd5d1FpdllIVnhICkFaZXhWYi9FOXBCYTlRMkc5QjFRN1RDTzdZc1VWUlF5NEpNRFpWdCtNY0ZuV1Z3ZXhucUJZRk5jVmprRW1EZ0EKZ3ZDWUdFMFA5ZC9Sd1JMNEt1TEhvK3U2ZnY3UDBqWE1OK0NwT3h5TGhZWlpOYTBaT1pESHNTaUpTUVNqOVdHRgpHSHJiQ2YwS1ZEcEtpZVIxdUJxSHJSTyttTFI1emtYMkw1OG03NGtqSzRkc0JobWplcS83T0FvVG1pRzJRZ0ovClAySWp5aGlBMm1ScVkraGw1NWx3RVVWLzB5SFlFa0pDOExkR2t3d1p6MmVGNzdhU1BHbWkvQTJDU0tnTXdEVHgKOW0rUDdqY3BXcmVZdzZORzlCdWVHb0RJdmUvdGdGS3d2VkZGNlFJREFRQUJBb0lCQUEwa3RqYVRmeXJBeHNUSQpCZXpiN1pyNU5CVzU1ZHZ1SUkyOTljZDZNSm8rckkvVFJZaHZVdjQ4a1k4SUZYcC9oeVVqemdlREx1bnhtSWY5Ci9aZ3NvaWM5T2w0NC9nNDVtTWR1aGNHWVB6QUFlQ2RjSjVPQjlyUjlWZkRDWHlqWUxsTjhIOGlVMDczNHRUcU0KMFYxM3RROXpkU3FrR1BaT0ljcS9rUi9weWxiT1phUU1lOTdCVGxzQW5PTVNNS0RnbmZ0WTQxMjJMcTNHWXkrdAp2cHIrYktWYVFad3ZrTG9TVTNyRUNDYUthZ2hnd0N5WDdqZnQ5YUVraGRKditLbHdic0dZNldFcnZ4T2FMV0hkCmN1TVFqR2FwWTFGYS80VUQwMG12ckEyNjBOeUtmenJwNitQNDZSclZNd0VZUkpNSVE4WUJBazZONkhoN2RjMEcKOFo2aTFtMENnWUVBOUhlQ0pSMFRTd2JJUTFiRFhVcnpwZnRIdWlkRzVCblNCdGF4L05EOXFJUGhSL0ZCVzVuagoyMm53TGM0OEtreWlybGZJVUxkMGFlNHFWWEpuN3dmWWN1WC9jSk1MRG1TVnRsTTVEem1pLzkxeFJpRmdJengxCkFzYkJ6YUZqSVNQMkhwU2dMK2U5RnRTWGFhcWVaVnJmbGl0VmhZS1VwSS9BS1YzMXFHSGYwNHNDZ1lFQTZ6VFYKOTlTYjQ5V2RsbnM1SWdzZm5YbDZUb1J0dEIxOGxmRUtjVmZqQU00ZnJua2swNkpwRkFaZVIrOUdHS1VYWkhxcwp6MnFjcGx3NGQvbW9DQzZwM3JZUEJNTFhzckdORVVGWnFCbGd6NzJRQTZCQnEzWDBDZzFCYzJaYks1Vkl6d2tnClNUMlNTdXg2Y2NST2ZnVUxtTjVaaUxPdGRVS05FWnBGRjNpM3F0c0NnWUFEVC9zN2RZRmxhdG9iejNrbU1uWEsKc2ZUdTJNbGxIZFJ5czBZR0h1N1E4YmlEdVFraHJKd2h4UFcwS1M4M2c0SlF5bSswYUVmemgzNmJXY2wrdTZSNwpLaEtqKzlvU2Y5cG5kZ2szNDVnSnozNVJiUEpZaCtFdUFITnZ6ZGdDQXZLNngxakVUV2VLZjZidGo1cEYxVTFpClE0UU5Jdy9RaXdJWGpXWmV1YlRHc1FLQmdRQ2JkdUx1MnJMbmx5eUFhSlpNOERsSFp5SDJnQVhiQlpweHFVOFQKdDltdGtKRFVTL0tSaUVvWUdGVjlDcVMwYVhyYXlWTXNEZlhZNkIvUy9VdVpqTzV1N0x0a2xEenFPZjFhS0czUQpkR1hQS2lia25xcUpZSCtiblVOanVZWU5lckVUVjU3bGlqTUdIdVNZQ2Y4dndMbjNveEJmRVJSWDYxTS9EVThaCndvcnovUUtCZ1FEQ1RKSTIramRYZzI2WHVZVW1NNFhYZm5vY2Z6QVhoWEJVTHQxbkVOY29nTmYxZmNwdEFWdHUKQkFpejQvSGlwUUtxb1dWVVlteGZnYmJMUktLTEswczBsT1dLYllkVmpoRW0vbTJaVTh3dFhUYWdOd2tJR295cQpZL0MxTG94NGYxUk9KbkNqYy9oZmNPamN4WDVNOEE4cGVlY0hXbFZ0VVBLVEpneFE3b01LY3c9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo=
     publicMaterial: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMyRENDQWNDZ0F3SUJBZ0lSQUxKWEFrVmo5NjR0cTY3d01TSThvSlF3RFFZSktvWklodmNOQVFFTEJRQXcKRlRFVE1CRUdBMVVFQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4TnpFeU1qY3lNelV5TkRCYUZ3MHlOekV5TWpjeQpNelV5TkRCYU1CVXhFekFSQmdOVkJBTVRDbXQxWW1WeWJtVjBaWE13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBCkE0SUJEd0F3Z2dFS0FvSUJBUURnbkNrU210bm1meEVnUzNxTlBhVUNINVFPQkdESC9pbkhiV0NPRExCQ0s5Z2QKWEVjQmw3RlZ2OFQya0ZyMURZYjBIVkR0TUk3dGl4UlZGRExna3dObFczNHh3V2RaWEI3R2VvRmdVMXhXT1FTWQpPQUNDOEpnWVRRLzEzOUhCRXZncTRzZWo2N3ArL3MvU05jdzM0S2s3SEl1RmhsazFyUms1a01leEtJbEpCS1AxCllZVVlldHNKL1FwVU9rcUo1SFc0R29ldEU3Nll0SG5PUmZZdm55YnZpU01yaDJ3R0dhTjZyL3M0Q2hPYUliWkMKQW44L1lpUEtHSURhWkdwajZHWG5tWEFSUlgvVElkZ1NRa0x3dDBhVERCblBaNFh2dHBJOGFhTDhEWUpJcUF6QQpOUEgyYjQvdU55bGF0NWpEbzBiMEc1NGFnTWk5NysyQVVyQzlVVVhwQWdNQkFBR2pJekFoTUE0R0ExVWREd0VCCi93UUVBd0lCQmpBUEJnTlZIUk1CQWY4RUJUQURBUUgvTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFCVkdSMnIKaHpYelJNVTV3cmlQUUFKU2Nzek5PUnZvQnBYZlpvWjA5Rkl1cHVkRnhCVlUzZDRoVjlTdEtuUWdQU0dBNVhRTwpIRTk3K0J4SkR1QS9yQjVvQlVzTUJqYzd5MWNkZS9UNmhtaTNyTG9FWUJTblN1ZENPWEpFNEc5LzBmOGJ5QUplCnJOOCtObzFyMlZnWnZaaDZwNzRURWtYdi9sM0hCUFdNN0lkVVYwSE85SkRoU2dPVkYxZnlRS0p4UnVMSlI4anQKTzZtUEgyVVgwdk13VmE0anZ3dGtkZHFrMk9BZFlRdkg5cmJEampiemFpVzBLbm1kdWVSbzkyS0hBTjdCc0RaeQpWcFhIcHFvMUt6ZzdEM2ZwYVhDZjVzaTdscXFyZEpWWEg0SkM3Mnp4c1BlaHFnaThlSXVxT0JraURXbVJ4QXhoCjh5R2VSeDlBYmtuSGg0SWEKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
+  primaryId: "237054359138908419352140518924933177492"
   type: Keypair
 `
 
@@ -202,34 +200,6 @@ spec:
 			t.Fatalf("unexpected round-tripped private key data: %q", roundTrip)
 		}
 	}
-
-	// Check private/ca/237054359138908419352140518924933177492.key round-tripped
-	{
-		roundTrip, err := pathMap["memfs://tests/private/ca/237054359138908419352140518924933177492.key"].ReadFile()
-		if err != nil {
-			t.Fatalf("error reading file memfs://tests/private/ca/237054359138908419352140518924933177492.key: %v", err)
-		}
-
-		if string(roundTrip) != privateKeyData {
-			t.Fatalf("unexpected round-tripped private key data: %q", string(roundTrip))
-		}
-	}
-
-	// Check that keyset gets deleted
-	{
-		keyset := &kops.Keyset{}
-		keyset.Name = "ca"
-		keyset.Spec.Type = kops.SecretTypeKeypair
-
-		s.DeleteKeysetItem(keyset, "237054359138908419352140518924933177492")
-
-		_, err := pathMap["memfs://tests/private/ca/237054359138908419352140518924933177492.key"].ReadFile()
-		if err == nil {
-			t.Fatalf("File memfs://tests/private/ca/237054359138908419352140518924933177492.key still exists")
-		}
-
-	}
-
 }
 
 func TestVFSCAStoreRoundTripWithVault(t *testing.T) {
@@ -263,8 +233,19 @@ func TestVFSCAStoreRoundTripWithVault(t *testing.T) {
 		t.Fatalf("error from ParsePEMCertificate: %v", err)
 	}
 
-	if err := s.StoreKeypair("ca", cert, privateKey); err != nil {
+	item := &KeysetItem{
-		t.Fatalf("error from StoreKeypair: %v", err)
+		Id:          "237054359138908419352140518924933177492",
+		Certificate: cert,
+		PrivateKey:  privateKey,
+	}
+	keyset := &Keyset{
+		Items: map[string]*KeysetItem{
+			"237054359138908419352140518924933177492": item,
+		},
+		Primary: item,
+	}
+	if err := s.StoreKeyset("ca", keyset); err != nil {
+		t.Fatalf("error from StoreKeyset: %v", err)
 	}
 
 	paths, err := basePath.ReadTree()
@@ -281,9 +262,7 @@ func TestVFSCAStoreRoundTripWithVault(t *testing.T) {
 
 	for _, p := range []string{
 		bp + "/issued/ca/keyset.yaml",
-		bp + "/issued/ca/237054359138908419352140518924933177492.crt",
 		bp + "/private/ca/keyset.yaml",
-		bp + "/private/ca/237054359138908419352140518924933177492.key",
 	} {
 		if _, found := pathMap[p]; !found {
 			t.Fatalf("file not found: %v", p)
@@ -341,18 +320,6 @@ spec:
 		}
 	}
 
-	// Check issued/ca/237054359138908419352140518924933177492.crt round-tripped
-	{
-		roundTrip, err := pathMap[bp+"/issued/ca/237054359138908419352140518924933177492.crt"].ReadFile()
-		if err != nil {
-			t.Fatalf("error reading file issued/ca/237054359138908419352140518924933177492.crt: %v", err)
-		}
-
-		if string(roundTrip) != certData {
-			t.Fatalf("unexpected round-tripped certificate data: %q", string(roundTrip))
-		}
-	}
-
 	// Check private/ca/keyset.yaml round-tripped
 	{
 		privateKeysetYaml, err := pathMap[bp+"/private/ca/keyset.yaml"].ReadFile()
@@ -396,32 +363,4 @@ spec:
 			t.Fatalf("unexpected round-tripped private key data: %q", roundTrip)
 		}
 	}
-
-	// Check private/ca/237054359138908419352140518924933177492.key round-tripped
-	{
-		roundTrip, err := pathMap[bp+"/private/ca/237054359138908419352140518924933177492.key"].ReadFile()
-		if err != nil {
-			t.Fatalf("error reading file private/ca/237054359138908419352140518924933177492.key: %v", err)
-		}
-
-		if string(roundTrip) != privateKeyData {
-			t.Fatalf("unexpected round-tripped private key data: %q", string(roundTrip))
-		}
-	}
-
-	// Check that keyset gets deleted
-	{
-		keyset := &kops.Keyset{}
-		keyset.Name = "ca"
-		keyset.Spec.Type = kops.SecretTypeKeypair
-
-		s.DeleteKeysetItem(keyset, "237054359138908419352140518924933177492")
-
-		_, err := pathMap[bp+"/private/ca/237054359138908419352140518924933177492.key"].ReadFile()
-		if err == nil {
-			t.Fatalf("File private/ca/237054359138908419352140518924933177492.key still exists")
-		}
-
-	}
-
 }

upup/pkg/kutil/convert_kubeup_cluster.go

@@ -487,18 +487,20 @@ func (x *ConvertKubeupCluster) Upgrade(ctx context.Context) error {
 	if oldCACertPool == nil {
 		return fmt.Errorf("cannot find certificate pool %q", fi.CertificateIDCA)
 	}
-	for _, ca := range oldCACertPool.Secondary {
+	keyset, err := newKeyStore.FindKeyset(fi.CertificateIDCA)
-		err := newKeyStore.AddCert(fi.CertificateIDCA, ca)
 	if err != nil {
-			return fmt.Errorf("error importing old CA certs: %v", err)
+		return fmt.Errorf("error reading new CA certs: %v", err)
 	}
+	for _, ca := range oldCACertPool.Secondary {
+		fi.AddCert(keyset, ca)
 	}
 	if oldCACertPool.Primary != nil {
-		err := newKeyStore.AddCert(fi.CertificateIDCA, oldCACertPool.Primary)
+		fi.AddCert(keyset, oldCACertPool.Primary)
+	}
+	err = newKeyStore.StoreKeyset(fi.CertificateIDCA, keyset)
 	if err != nil {
 		return fmt.Errorf("error importing old CA certs: %v", err)
 	}
-	}
 
 	return nil
 }

upup/pkg/kutil/import_cluster.go

@@ -424,7 +424,12 @@ func (x *ImportCluster) ImportAWSCluster(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	err = keyStore.AddCert(fi.CertificateIDCA, caCert)
+	keyset, err := keyStore.FindKeyset(fi.CertificateIDCA)
+	if err != nil {
+		return err
+	}
+	fi.AddCert(keyset, caCert)
+	err = keyStore.StoreKeyset(fi.CertificateIDCA, keyset)
 	if err != nil {
 		return err
 	}