From 91852b931323071e0e97ec239f480a4f8ea38a42 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Sat, 10 Apr 2021 21:59:46 -0700 Subject: [PATCH 01/15] Simplify keyset loading --- upup/pkg/fi/vfs_castore.go | 38 +++++++++++++------------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/upup/pkg/fi/vfs_castore.go b/upup/pkg/fi/vfs_castore.go index a281063e85..1f3e8aa430 100644 --- a/upup/pkg/fi/vfs_castore.go +++ b/upup/pkg/fi/vfs_castore.go @@ -106,11 +106,12 @@ func (c *VFSCAStore) parseKeysetYaml(data []byte) (*kops.Keyset, bool, error) { return keyset, gvk.Version != keysetFormatLatest, nil } -// loadCertificatesBundle loads a keyset from the path +// loadKeyset loads a keyset from the path // Returns (nil, nil) if the file is not found // Bundles avoid the need for a list-files permission, which can be tricky on e.g. GCE -func (c *VFSCAStore) loadKeysetBundle(p vfs.Path) (*keyset, error) { - data, err := p.ReadFile() +func (c *VFSCAStore) loadKeyset(p vfs.Path) (*keyset, error) { + bundlePath := p.Join("keyset.yaml") + data, err := bundlePath.ReadFile() if err != nil { if os.IsNotExist(err) { return nil, nil @@ -230,12 +231,6 @@ func SerializeKeyset(o *kops.Keyset) ([]byte, error) { return objectData.Bytes(), nil } -func (c *VFSCAStore) loadCertificates(p vfs.Path) (*keyset, error) { - bundlePath := p.Join("keyset.yaml") - bundle, err := c.loadKeysetBundle(bundlePath) - return bundle, err -} - func (c *VFSCAStore) loadOneCertificate(p vfs.Path) (*pki.Certificate, error) { data, err := p.ReadFile() if err != nil { @@ -278,7 +273,7 @@ func (c *VFSCAStore) FindKeypair(id string) (*pki.Certificate, *pki.PrivateKey, func (c *VFSCAStore) findCert(name string) (*pki.Certificate, bool, error) { p := c.buildCertificatePoolPath(name) - certs, err := c.loadCertificates(p) + certs, err := c.loadKeyset(p) if err != nil { return nil, false, fmt.Errorf("error in 'FindCert' attempting to load cert %q: %v", name, err) } @@ -300,7 +295,7 @@ func (c *VFSCAStore) FindCertificatePool(name string) (*CertificatePool, error) var err error p := c.buildCertificatePoolPath(name) - certs, err = c.loadCertificates(p) + certs, err = c.loadKeyset(p) if err != nil { return nil, fmt.Errorf("error in 'FindCertificatePool' attempting to load cert %q: %v", name, err) } @@ -327,7 +322,7 @@ func (c *VFSCAStore) FindCertificatePool(name string) (*CertificatePool, error) func (c *VFSCAStore) FindCertificateKeyset(name string) (*kops.Keyset, error) { p := c.buildCertificatePoolPath(name) - certs, err := c.loadCertificates(p) + certs, err := c.loadKeyset(p) if err != nil { return nil, fmt.Errorf("error in 'FindCertificatePool' attempting to load cert %q: %v", name, err) } @@ -584,13 +579,6 @@ func (c *VFSCAStore) AddCert(name string, cert *pki.Certificate) error { return err } -func (c *VFSCAStore) loadPrivateKeys(p vfs.Path) (*keyset, error) { - bundlePath := p.Join("keyset.yaml") - bundle, err := c.loadKeysetBundle(bundlePath) - - return bundle, err -} - func (c *VFSCAStore) findPrivateKeyset(id string) (*keyset, error) { var keys *keyset var err error @@ -603,7 +591,7 @@ func (c *VFSCAStore) findPrivateKeyset(id string) (*keyset, error) { return cached, nil } - keys, err = c.loadPrivateKeys(c.buildPrivateKeyPoolPath(id)) + keys, err = c.loadKeyset(c.buildPrivateKeyPoolPath(id)) if err != nil { return nil, err } @@ -616,7 +604,7 @@ func (c *VFSCAStore) findPrivateKeyset(id string) (*keyset, error) { } } else { p := c.buildPrivateKeyPoolPath(id) - keys, err = c.loadPrivateKeys(p) + keys, err = c.loadKeyset(p) if err != nil { return nil, err } @@ -660,7 +648,7 @@ func (c *VFSCAStore) storePrivateKey(name string, ki *keysetItem) error { // Write the bundle { p := c.buildPrivateKeyPoolPath(name) - ks, err := c.loadPrivateKeys(p) + ks, err := c.loadKeyset(p) if err != nil { return err } @@ -703,7 +691,7 @@ func (c *VFSCAStore) storeCertificate(name string, ki *keysetItem) error { // Write the bundle { p := c.buildCertificatePoolPath(name) - ks, err := c.loadCertificates(p) + ks, err := c.loadKeyset(p) if err != nil { return err } @@ -751,7 +739,7 @@ func (c *VFSCAStore) deletePrivateKey(name string, id string) (bool, error) { // Update the bundle { p := c.buildPrivateKeyPoolPath(name) - ks, err := c.loadPrivateKeys(p) + ks, err := c.loadKeyset(p) if err != nil { return false, err } @@ -781,7 +769,7 @@ func (c *VFSCAStore) deleteCertificate(name string, id string) (bool, error) { // Update the bundle { p := c.buildCertificatePoolPath(name) - ks, err := c.loadCertificates(p) + ks, err := c.loadKeyset(p) if err != nil { return false, err } From b21370d1185726c33930e80dd26496f9381f08a3 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Sat, 10 Apr 2021 22:14:42 -0700 Subject: [PATCH 02/15] Add PrimaryId field to KeysetSpec --- pkg/apis/kops/keyset.go | 3 +++ pkg/apis/kops/v1alpha2/keyset.go | 3 +++ upup/pkg/fi/ca.go | 2 +- upup/pkg/fi/clientset_castore.go | 29 ++++++++--------------------- upup/pkg/fi/vfs_castore.go | 11 +++++++++++ 5 files changed, 26 insertions(+), 22 deletions(-) diff --git a/pkg/apis/kops/keyset.go b/pkg/apis/kops/keyset.go index 459f6901bc..61f6d49379 100644 --- a/pkg/apis/kops/keyset.go +++ b/pkg/apis/kops/keyset.go @@ -67,6 +67,9 @@ type KeysetSpec struct { // Type is the type of the Keyset (PKI keypair, or secret token) Type KeysetType `json:"type,omitempty"` + // PrimaryId is the id of the key used to mint new things. + PrimaryId string `json:"primaryId,omitempty"` + // Keys is the set of keys that make up the keyset Keys []KeysetItem `json:"keys,omitempty"` } diff --git a/pkg/apis/kops/v1alpha2/keyset.go b/pkg/apis/kops/v1alpha2/keyset.go index 1bbb4aba75..942ed5117e 100644 --- a/pkg/apis/kops/v1alpha2/keyset.go +++ b/pkg/apis/kops/v1alpha2/keyset.go @@ -68,6 +68,9 @@ type KeysetSpec struct { // Type is the type of the Keyset (PKI keypair, or secret token) Type KeysetType `json:"type,omitempty"` + // PrimaryId is the id of the key used to mint new things. + PrimaryId string `json:"primaryId,omitempty"` + // Keys is the set of keys that make up the keyset Keys []KeysetItem `json:"keys,omitempty"` } diff --git a/upup/pkg/fi/ca.go b/upup/pkg/fi/ca.go index 4b9402feb7..5490df0d2a 100644 --- a/upup/pkg/fi/ca.go +++ b/upup/pkg/fi/ca.go @@ -52,7 +52,7 @@ type Keystore interface { // task to convert a Legacy Keypair to the new Keypair API format. FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) - // StoreKeypair writes the keypair to the store + // StoreKeypair writes the keypair to the store, making it the primary. StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error // MirrorTo will copy secrets to a vfs.Path, which is often easier for a machine to read diff --git a/upup/pkg/fi/clientset_castore.go b/upup/pkg/fi/clientset_castore.go index 3bac591722..d844d1b2aa 100644 --- a/upup/pkg/fi/clientset_castore.go +++ b/upup/pkg/fi/clientset_castore.go @@ -110,7 +110,7 @@ func parseKeyset(o *kops.Keyset) (*keyset, error) { keyset.items[key.Id] = ki } - keyset.primary = keyset.findPrimary() + keyset.primary = keyset.items[FindPrimary(o).Id] return keyset, nil } @@ -132,30 +132,13 @@ func (c *ClientsetCAStore) loadKeyset(ctx context.Context, name string) (*keyset return keyset, nil } -// findPrimary returns the primary keysetItem in the keyset -func (k *keyset) findPrimary() *keysetItem { - var primary *keysetItem - var primaryVersion *big.Int - - for _, item := range k.items { - version, ok := big.NewInt(0).SetString(item.id, 10) - if !ok { - klog.Warningf("Ignoring key item with non-integer version: %q", item.id) - continue - } - - if primaryVersion == nil || version.Cmp(primaryVersion) > 0 { - primary = item - primaryVersion = version - } - } - return primary -} - // FindPrimary returns the primary KeysetItem in the Keyset func FindPrimary(keyset *kops.Keyset) *kops.KeysetItem { var primary *kops.KeysetItem var primaryVersion *big.Int + + primaryId := keyset.Spec.PrimaryId + for i := range keyset.Spec.Keys { item := &keyset.Spec.Keys[i] version, ok := big.NewInt(0).SetString(item.Id, 10) @@ -164,6 +147,10 @@ func FindPrimary(keyset *kops.Keyset) *kops.KeysetItem { continue } + if item.Id == primaryId { + return item + } + if primaryVersion == nil || version.Cmp(primaryVersion) > 0 { primary = item primaryVersion = version diff --git a/upup/pkg/fi/vfs_castore.go b/upup/pkg/fi/vfs_castore.go index 1f3e8aa430..000c302a42 100644 --- a/upup/pkg/fi/vfs_castore.go +++ b/upup/pkg/fi/vfs_castore.go @@ -162,6 +162,9 @@ func (k *keyset) ToAPIObject(name string, includePrivateKeyMaterial bool) (*kops o.Spec.Keys = append(o.Spec.Keys, oki) } + if k.primary != nil { + o.Spec.PrimaryId = k.primary.id + } return o, nil } @@ -660,6 +663,7 @@ func (c *VFSCAStore) storePrivateKey(name string, ki *keysetItem) error { ks.items = make(map[string]*keysetItem) } ks.items[ki.id] = ki + ks.primary = ki if err := c.writeKeysetBundle(p, name, ks, true); err != nil { return fmt.Errorf("error writing bundle: %v", err) @@ -703,6 +707,7 @@ func (c *VFSCAStore) storeCertificate(name string, ki *keysetItem) error { ks.items = make(map[string]*keysetItem) } ks.items[ki.id] = ki + ks.primary = ki if err := c.writeKeysetBundle(p, name, ks, false); err != nil { return fmt.Errorf("error writing bundle: %v", err) @@ -748,6 +753,9 @@ func (c *VFSCAStore) deletePrivateKey(name string, id string) (bool, error) { return false, nil } delete(ks.items, id) + if ks.primary != nil && ks.primary.id == id { + ks.primary = nil + } if err := c.writeKeysetBundle(p, name, ks, true); err != nil { return false, fmt.Errorf("error writing bundle: %v", err) @@ -778,6 +786,9 @@ func (c *VFSCAStore) deleteCertificate(name string, id string) (bool, error) { return false, nil } delete(ks.items, id) + if ks.primary != nil && ks.primary.id == id { + ks.primary = nil + } if err := c.writeKeysetBundle(p, name, ks, false); err != nil { return false, fmt.Errorf("error writing bundle: %v", err) From 3b54486cdd908442397160fd30eb118f84a54d6a Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Sat, 10 Apr 2021 22:32:24 -0700 Subject: [PATCH 03/15] make apimachinery crds --- k8s/crds/kops.k8s.io_keysets.yaml | 3 +++ pkg/apis/kops/v1alpha2/zz_generated.conversion.go | 2 ++ 2 files changed, 5 insertions(+) diff --git a/k8s/crds/kops.k8s.io_keysets.yaml b/k8s/crds/kops.k8s.io_keysets.yaml index dd02cb401b..d21d2a32b2 100644 --- a/k8s/crds/kops.k8s.io_keysets.yaml +++ b/k8s/crds/kops.k8s.io_keysets.yaml @@ -59,6 +59,9 @@ spec: type: string type: object type: array + primaryId: + description: PrimaryId is the id of the key used to mint new things. + type: string type: description: Type is the type of the Keyset (PKI keypair, or secret token) diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 60c526f31a..e78d6f7302 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -4454,6 +4454,7 @@ func Convert_kops_KeysetList_To_v1alpha2_KeysetList(in *kops.KeysetList, out *Ke func autoConvert_v1alpha2_KeysetSpec_To_kops_KeysetSpec(in *KeysetSpec, out *kops.KeysetSpec, s conversion.Scope) error { out.Type = kops.KeysetType(in.Type) + out.PrimaryId = in.PrimaryId if in.Keys != nil { in, out := &in.Keys, &out.Keys *out = make([]kops.KeysetItem, len(*in)) @@ -4475,6 +4476,7 @@ func Convert_v1alpha2_KeysetSpec_To_kops_KeysetSpec(in *KeysetSpec, out *kops.Ke func autoConvert_kops_KeysetSpec_To_v1alpha2_KeysetSpec(in *kops.KeysetSpec, out *KeysetSpec, s conversion.Scope) error { out.Type = KeysetType(in.Type) + out.PrimaryId = in.PrimaryId if in.Keys != nil { in, out := &in.Keys, &out.Keys *out = make([]KeysetItem, len(*in)) From 6b2250a9afff82f153104bdbb2038fb20aec6145 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Sat, 10 Apr 2021 23:14:36 -0700 Subject: [PATCH 04/15] Have apiserver trust all service-account keys --- nodeup/pkg/model/BUILD.bazel | 1 + nodeup/pkg/model/fakes_test.go | 14 +++-- nodeup/pkg/model/kube_apiserver.go | 41 +++++++++++++- nodeup/pkg/model/kubelet_test.go | 54 ++++++++++++++----- .../golden/awsiam/tasks-kube-apiserver.yaml | 24 ++++++++- .../tasks-kube-apiserver.yaml | 24 ++++++++- .../golden/minimal/tasks-kube-apiserver.yaml | 24 ++++++++- .../tasks-kube-apiserver-amd64.yaml | 24 ++++++++- .../tasks-kube-apiserver-arm64.yaml | 24 ++++++++- .../tasks-kube-apiserver.yaml | 24 ++++++++- upup/pkg/fi/vfs_castore_test.go | 2 + 11 files changed, 232 insertions(+), 24 deletions(-) diff --git a/nodeup/pkg/model/BUILD.bazel b/nodeup/pkg/model/BUILD.bazel index 36d2fd4541..c592121064 100644 --- a/nodeup/pkg/model/BUILD.bazel +++ b/nodeup/pkg/model/BUILD.bazel @@ -53,6 +53,7 @@ go_library( "//pkg/kubemanifest:go_default_library", "//pkg/model/components:go_default_library", "//pkg/nodelabels:go_default_library", + "//pkg/pki:go_default_library", "//pkg/rbac:go_default_library", "//pkg/systemd:go_default_library", "//pkg/tokens:go_default_library", diff --git a/nodeup/pkg/model/fakes_test.go b/nodeup/pkg/model/fakes_test.go index f126c8cdfe..6da59dfde5 100644 --- a/nodeup/pkg/model/fakes_test.go +++ b/nodeup/pkg/model/fakes_test.go @@ -54,8 +54,8 @@ func (k fakeKeyStore) MirrorTo(basedir vfs.Path) error { type fakeCAStore struct { fakeKeyStore - privateKeys map[string]*pki.PrivateKey - certs map[string]*pki.Certificate + privateKeysets map[string]*kops.Keyset + certs map[string]*pki.Certificate } var _ fi.CAStore = &fakeCAStore{} @@ -69,11 +69,17 @@ func (k fakeCAStore) FindCertificateKeyset(name string) (*kops.Keyset, error) { } func (k fakeCAStore) FindPrivateKey(name string) (*pki.PrivateKey, error) { - return k.privateKeys[name], nil + primaryId := k.privateKeysets[name].Spec.PrimaryId + for _, item := range k.privateKeysets[name].Spec.Keys { + if item.Id == primaryId { + return pki.ParsePEMPrivateKey(item.PrivateMaterial) + } + } + return nil, nil } func (k fakeCAStore) FindPrivateKeyset(name string) (*kops.Keyset, error) { - panic("fakeCAStore does not implement FindPrivateKeyset") + return k.privateKeysets[name], nil } func (k fakeCAStore) FindCert(name string) (*pki.Certificate, error) { diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go index f7ff9a43bc..1657970aa6 100644 --- a/nodeup/pkg/model/kube_apiserver.go +++ b/nodeup/pkg/model/kube_apiserver.go @@ -17,6 +17,10 @@ limitations under the License. package model import ( + "bytes" + "crypto/rsa" + "crypto/x509" + "encoding/pem" "fmt" "path/filepath" "strings" @@ -26,6 +30,7 @@ import ( "k8s.io/kops/pkg/k8scodecs" "k8s.io/kops/pkg/kubeconfig" "k8s.io/kops/pkg/kubemanifest" + "k8s.io/kops/pkg/pki" "k8s.io/kops/pkg/wellknownports" "k8s.io/kops/pkg/wellknownusers" "k8s.io/kops/upup/pkg/fi" @@ -81,6 +86,39 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error { } } } + { + keyset, err := b.KeyStore.FindPrivateKeyset("service-account") + if err != nil { + return err + } + + if keyset == nil { + return fmt.Errorf("service-account keyset not found") + } + + buf := new(bytes.Buffer) + for _, keyItem := range keyset.Spec.Keys { + privateKey, err := pki.ParsePEMPrivateKey(keyItem.PrivateMaterial) + if err != nil { + return fmt.Errorf("error loading service-account private key %s: %v", keyItem.Id, err) + } + pkData, err := x509.MarshalPKIXPublicKey(privateKey.Key.(*rsa.PrivateKey).Public()) + if err != nil { + return fmt.Errorf("marshalling public key: %v", err) + } + err = pem.Encode(buf, &pem.Block{Type: "RSA PUBLIC KEY", Bytes: pkData}) + if err != nil { + return fmt.Errorf("encoding public key: %v", err) + } + } + + c.AddTask(&nodetasks.File{ + Path: filepath.Join(b.PathSrvKubernetes(), "service-account.pub"), + Contents: fi.NewBytesResource(buf.Bytes()), + Type: nodetasks.FileType_File, + Mode: s("0600"), + }) + } { pod, err := b.buildPod() if err != nil { @@ -282,8 +320,7 @@ func (b *KubeAPIServerBuilder) writeAuthenticationConfig(c *fi.ModelBuilderConte func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { kubeAPIServer := b.Cluster.Spec.KubeAPIServer - // TODO pass the public key instead. We would first need to segregate the secrets better. - kubeAPIServer.ServiceAccountKeyFile = append(kubeAPIServer.ServiceAccountKeyFile, filepath.Join(b.PathSrvKubernetes(), "service-account.key")) + kubeAPIServer.ServiceAccountKeyFile = append(kubeAPIServer.ServiceAccountKeyFile, filepath.Join(b.PathSrvKubernetes(), "service-account.pub")) // Set the signing key if we're using Service Account Token VolumeProjection if kubeAPIServer.ServiceAccountSigningKeyFile == nil { diff --git a/nodeup/pkg/model/kubelet_test.go b/nodeup/pkg/model/kubelet_test.go index 06f9f72fa2..5f10d52008 100644 --- a/nodeup/pkg/model/kubelet_test.go +++ b/nodeup/pkg/model/kubelet_test.go @@ -268,13 +268,43 @@ func mockedPopulateClusterSpec(c *kops.Cluster, cloud fi.Cloud) (*kops.Cluster, const dummyCertificate = "-----BEGIN CERTIFICATE-----\nMIIC2DCCAcCgAwIBAgIRALJXAkVj964tq67wMSI8oJQwDQYJKoZIhvcNAQELBQAw\nFTETMBEGA1UEAxMKa3ViZXJuZXRlczAeFw0xNzEyMjcyMzUyNDBaFw0yNzEyMjcy\nMzUyNDBaMBUxEzARBgNVBAMTCmt1YmVybmV0ZXMwggEiMA0GCSqGSIb3DQEBAQUA\nA4IBDwAwggEKAoIBAQDgnCkSmtnmfxEgS3qNPaUCH5QOBGDH/inHbWCODLBCK9gd\nXEcBl7FVv8T2kFr1DYb0HVDtMI7tixRVFDLgkwNlW34xwWdZXB7GeoFgU1xWOQSY\nOACC8JgYTQ/139HBEvgq4sej67p+/s/SNcw34Kk7HIuFhlk1rRk5kMexKIlJBKP1\nYYUYetsJ/QpUOkqJ5HW4GoetE76YtHnORfYvnybviSMrh2wGGaN6r/s4ChOaIbZC\nAn8/YiPKGIDaZGpj6GXnmXARRX/TIdgSQkLwt0aTDBnPZ4XvtpI8aaL8DYJIqAzA\nNPH2b4/uNylat5jDo0b0G54agMi97+2AUrC9UUXpAgMBAAGjIzAhMA4GA1UdDwEB\n/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBVGR2r\nhzXzRMU5wriPQAJScszNORvoBpXfZoZ09FIupudFxBVU3d4hV9StKnQgPSGA5XQO\nHE97+BxJDuA/rB5oBUsMBjc7y1cde/T6hmi3rLoEYBSnSudCOXJE4G9/0f8byAJe\nrN8+No1r2VgZvZh6p74TEkXv/l3HBPWM7IdUV0HO9JDhSgOVF1fyQKJxRuLJR8jt\nO6mPH2UX0vMwVa4jvwtkddqk2OAdYQvH9rbDjjbzaiW0KnmdueRo92KHAN7BsDZy\nVpXHpqo1Kzg7D3fpaXCf5si7lqqrdJVXH4JC72zxsPehqgi8eIuqOBkiDWmRxAxh\n8yGeRx9AbknHh4Ia\n-----END CERTIFICATE-----\n" const dummyKey = "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA4JwpEprZ5n8RIEt6jT2lAh+UDgRgx/4px21gjgywQivYHVxH\nAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMDZVt+McFnWVwexnqBYFNcVjkEmDgA\ngvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+CpOxyLhYZZNa0ZOZDHsSiJSQSj9WGF\nGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m74kjK4dsBhmjeq/7OAoTmiG2QgJ/\nP2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdGkwwZz2eF77aSPGmi/A2CSKgMwDTx\n9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF6QIDAQABAoIBAA0ktjaTfyrAxsTI\nBezb7Zr5NBW55dvuII299cd6MJo+rI/TRYhvUv48kY8IFXp/hyUjzgeDLunxmIf9\n/Zgsoic9Ol44/g45mMduhcGYPzAAeCdcJ5OB9rR9VfDCXyjYLlN8H8iU0734tTqM\n0V13tQ9zdSqkGPZOIcq/kR/pylbOZaQMe97BTlsAnOMSMKDgnftY4122Lq3GYy+t\nvpr+bKVaQZwvkLoSU3rECCaKaghgwCyX7jft9aEkhdJv+KlwbsGY6WErvxOaLWHd\ncuMQjGapY1Fa/4UD00mvrA260NyKfzrp6+P46RrVMwEYRJMIQ8YBAk6N6Hh7dc0G\n8Z6i1m0CgYEA9HeCJR0TSwbIQ1bDXUrzpftHuidG5BnSBtax/ND9qIPhR/FBW5nj\n22nwLc48KkyirlfIULd0ae4qVXJn7wfYcuX/cJMLDmSVtlM5Dzmi/91xRiFgIzx1\nAsbBzaFjISP2HpSgL+e9FtSXaaqeZVrflitVhYKUpI/AKV31qGHf04sCgYEA6zTV\n99Sb49Wdlns5IgsfnXl6ToRttB18lfEKcVfjAM4frnkk06JpFAZeR+9GGKUXZHqs\nz2qcplw4d/moCC6p3rYPBMLXsrGNEUFZqBlgz72QA6BBq3X0Cg1Bc2ZbK5VIzwkg\nST2SSux6ccROfgULmN5ZiLOtdUKNEZpFF3i3qtsCgYADT/s7dYFlatobz3kmMnXK\nsfTu2MllHdRys0YGHu7Q8biDuQkhrJwhxPW0KS83g4JQym+0aEfzh36bWcl+u6R7\nKhKj+9oSf9pndgk345gJz35RbPJYh+EuAHNvzdgCAvK6x1jETWeKf6btj5pF1U1i\nQ4QNIw/QiwIXjWZeubTGsQKBgQCbduLu2rLnlyyAaJZM8DlHZyH2gAXbBZpxqU8T\nt9mtkJDUS/KRiEoYGFV9CqS0aXrayVMsDfXY6B/S/UuZjO5u7LtklDzqOf1aKG3Q\ndGXPKibknqqJYH+bnUNjuYYNerETV57lijMGHuSYCf8vwLn3oxBfERRX61M/DU8Z\nworz/QKBgQDCTJI2+jdXg26XuYUmM4XXfnocfzAXhXBULt1nENcogNf1fcptAVtu\nBAiz4/HipQKqoWVUYmxfgbbLRKKLK0s0lOWKbYdVjhEm/m2ZU8wtXTagNwkIGoyq\nY/C1Lox4f1ROJnCjc/hfcOjcxX5M8A8peecHWlVtUPKTJgxQ7oMKcw==\n-----END RSA PRIVATE KEY-----\n" +const previousKey = "-----BEGIN RSA PRIVATE KEY-----\nMIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4\n9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R\n2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo\nxTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+\nZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr\nKl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh\nAOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY\n-----END RSA PRIVATE KEY-----" +const nextKey = "-----BEGIN RSA PRIVATE KEY-----\nMIIBOgIBAAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCC\nrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQJAejInjmEzqmzQr0NxcIN4\nPukwK3FBKl+RAOZfqNIKcww14mfOn7Gc6lF2zEC4GnLiB3tthbSXoBGi54nkW4ki\nyQIhANZNne9UhQlwyjsd3WxDWWrl6OOZ3J8ppMOIQni9WRLlAiEAw1XEdxPOSOSO\nB6rucpTT1QivVvyEFIb/ukvPm769Mh8CIQDNQwKnHdlfNX0+KljPPaMD1LrAZbr/\naC+8aWLhqtsKUQIgF7gUcTkwdV17eabh6Xv09Qtm7zMefred2etWvFy+8JUCIECv\nFYOKQVWHX+Q7CHX2K1oTECVnZuW1UItdDYVlFYxQ\n-----END RSA PRIVATE KEY-----" -func mustParsePrivateKey(s string) *pki.PrivateKey { - k, err := pki.ParsePEMPrivateKey([]byte(s)) - if err != nil { - klog.Fatalf("error parsing private key %v", err) +func simplePrivateKeyset(s string) *kops.Keyset { + return &kops.Keyset{ + Spec: kops.KeysetSpec{ + PrimaryId: "3", + Keys: []kops.KeysetItem{ + { + Id: "3", + PrivateMaterial: []byte(s), + }, + }, + }, + } +} + +func rotatingPrivateKeyset(s string) *kops.Keyset { + return &kops.Keyset{ + Spec: kops.KeysetSpec{ + PrimaryId: "3", + Keys: []kops.KeysetItem{ + { + Id: "2", + PrivateMaterial: []byte(previousKey), + }, + { + Id: "3", + PrivateMaterial: []byte(s), + }, + { + Id: "4", + PrivateMaterial: []byte(nextKey), + }, + }, + }, } - return k } func mustParseCertificate(s string) *pki.Certificate { @@ -302,13 +332,13 @@ func RunGoldenTest(t *testing.T, basedir string, key string, builder func(*Nodeu keystore := &fakeCAStore{} keystore.T = t - keystore.privateKeys = map[string]*pki.PrivateKey{ - "ca": mustParsePrivateKey(dummyKey), - "apiserver-aggregator-ca": mustParsePrivateKey(dummyKey), - "kube-controller-manager": mustParsePrivateKey(dummyKey), - "kube-proxy": mustParsePrivateKey(dummyKey), - "kube-scheduler": mustParsePrivateKey(dummyKey), - "service-account": mustParsePrivateKey(dummyKey), + keystore.privateKeysets = map[string]*kops.Keyset{ + "ca": simplePrivateKeyset(dummyKey), + "apiserver-aggregator-ca": simplePrivateKeyset(dummyKey), + "kube-controller-manager": simplePrivateKeyset(dummyKey), + "kube-proxy": simplePrivateKeyset(dummyKey), + "kube-scheduler": simplePrivateKeyset(dummyKey), + "service-account": rotatingPrivateKeyset(dummyKey), } keystore.certs = map[string]*pki.Certificate{ "ca": mustParseCertificate(dummyCertificate), diff --git a/nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml index 65a4c0aa01..fdf2cdc6bf 100644 --- a/nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml +++ b/nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml @@ -62,7 +62,7 @@ contents: | - --requestheader-group-headers=X-Remote-Group - --requestheader-username-headers=X-Remote-User - --secure-port=443 - - --service-account-key-file=/srv/kubernetes/service-account.key + - --service-account-key-file=/srv/kubernetes/service-account.pub - --service-cluster-ip-range=100.64.0.0/13 - --storage-backend=etcd3 - --tls-cert-file=/srv/kubernetes/server.crt @@ -248,6 +248,28 @@ mode: "0600" path: /srv/kubernetes/kubelet-api.key type: file --- +contents: | + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm + XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ== + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l + Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD + ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp + OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m + 74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG + kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF + 6QIDAQAB + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF + Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ== + -----END RSA PUBLIC KEY----- +mode: "0600" +path: /srv/kubernetes/service-account.pub +type: file +--- contents: "" ifNotExists: true mode: "0400" diff --git a/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml index d368426378..daff77983b 100644 --- a/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml +++ b/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml @@ -40,7 +40,7 @@ contents: | - --requestheader-group-headers=X-Remote-Group - --requestheader-username-headers=X-Remote-User - --secure-port=443 - - --service-account-key-file=/srv/kubernetes/service-account.key + - --service-account-key-file=/srv/kubernetes/service-account.pub - --service-cluster-ip-range=100.64.0.0/13 - --storage-backend=etcd3 - --tls-cert-file=/srv/kubernetes/server.crt @@ -188,6 +188,28 @@ mode: "0600" path: /srv/kubernetes/kubelet-api.key type: file --- +contents: | + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm + XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ== + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l + Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD + ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp + OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m + 74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG + kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF + 6QIDAQAB + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF + Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ== + -----END RSA PUBLIC KEY----- +mode: "0600" +path: /srv/kubernetes/service-account.pub +type: file +--- contents: "" ifNotExists: true mode: "0400" diff --git a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml index 917e69ef52..442853faa9 100644 --- a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml +++ b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml @@ -40,7 +40,7 @@ contents: | - --requestheader-group-headers=X-Remote-Group - --requestheader-username-headers=X-Remote-User - --secure-port=443 - - --service-account-key-file=/srv/kubernetes/service-account.key + - --service-account-key-file=/srv/kubernetes/service-account.pub - --service-cluster-ip-range=100.64.0.0/13 - --storage-backend=etcd3 - --tls-cert-file=/srv/kubernetes/server.crt @@ -188,6 +188,28 @@ mode: "0600" path: /srv/kubernetes/kubelet-api.key type: file --- +contents: | + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm + XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ== + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l + Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD + ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp + OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m + 74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG + kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF + 6QIDAQAB + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF + Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ== + -----END RSA PUBLIC KEY----- +mode: "0600" +path: /srv/kubernetes/service-account.pub +type: file +--- contents: "" ifNotExists: true mode: "0400" diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml index 3ffccbc69e..80c1c18e16 100644 --- a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml +++ b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml @@ -40,7 +40,7 @@ contents: | - --requestheader-group-headers=X-Remote-Group - --requestheader-username-headers=X-Remote-User - --secure-port=443 - - --service-account-key-file=/srv/kubernetes/service-account.key + - --service-account-key-file=/srv/kubernetes/service-account.pub - --service-cluster-ip-range=100.64.0.0/13 - --storage-backend=etcd3 - --tls-cert-file=/srv/kubernetes/server.crt @@ -188,6 +188,28 @@ mode: "0600" path: /srv/kubernetes/kubelet-api.key type: file --- +contents: | + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm + XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ== + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l + Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD + ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp + OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m + 74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG + kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF + 6QIDAQAB + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF + Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ== + -----END RSA PUBLIC KEY----- +mode: "0600" +path: /srv/kubernetes/service-account.pub +type: file +--- contents: "" ifNotExists: true mode: "0400" diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml index 44597ad845..e0c1424da8 100644 --- a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml +++ b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml @@ -40,7 +40,7 @@ contents: | - --requestheader-group-headers=X-Remote-Group - --requestheader-username-headers=X-Remote-User - --secure-port=443 - - --service-account-key-file=/srv/kubernetes/service-account.key + - --service-account-key-file=/srv/kubernetes/service-account.pub - --service-cluster-ip-range=100.64.0.0/13 - --storage-backend=etcd3 - --tls-cert-file=/srv/kubernetes/server.crt @@ -188,6 +188,28 @@ mode: "0600" path: /srv/kubernetes/kubelet-api.key type: file --- +contents: | + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm + XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ== + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l + Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD + ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp + OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m + 74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG + kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF + 6QIDAQAB + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF + Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ== + -----END RSA PUBLIC KEY----- +mode: "0600" +path: /srv/kubernetes/service-account.pub +type: file +--- contents: "" ifNotExists: true mode: "0400" diff --git a/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml index 4c43f9a280..6baff4de69 100644 --- a/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml +++ b/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml @@ -39,7 +39,7 @@ contents: | - --requestheader-group-headers=X-Remote-Group - --requestheader-username-headers=X-Remote-User - --secure-port=443 - - --service-account-key-file=/srv/kubernetes/service-account.key + - --service-account-key-file=/srv/kubernetes/service-account.pub - --service-cluster-ip-range=100.64.0.0/13 - --storage-backend=etcd3 - --tls-cert-file=/srv/kubernetes/server.crt @@ -187,6 +187,28 @@ mode: "0600" path: /srv/kubernetes/kubelet-api.key type: file --- +contents: | + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm + XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ== + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4JwpEprZ5n8RIEt6jT2l + Ah+UDgRgx/4px21gjgywQivYHVxHAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMD + ZVt+McFnWVwexnqBYFNcVjkEmDgAgvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+Cp + OxyLhYZZNa0ZOZDHsSiJSQSj9WGFGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m + 74kjK4dsBhmjeq/7OAoTmiG2QgJ/P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdG + kwwZz2eF77aSPGmi/A2CSKgMwDTx9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF + 6QIDAQAB + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF + Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ== + -----END RSA PUBLIC KEY----- +mode: "0600" +path: /srv/kubernetes/service-account.pub +type: file +--- contents: "" ifNotExists: true mode: "0400" diff --git a/upup/pkg/fi/vfs_castore_test.go b/upup/pkg/fi/vfs_castore_test.go index 55b360378f..93d6ec00a0 100644 --- a/upup/pkg/fi/vfs_castore_test.go +++ b/upup/pkg/fi/vfs_castore_test.go @@ -117,6 +117,7 @@ spec: keys: - id: "237054359138908419352140518924933177492" publicMaterial: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMyRENDQWNDZ0F3SUJBZ0lSQUxKWEFrVmo5NjR0cTY3d01TSThvSlF3RFFZSktvWklodmNOQVFFTEJRQXcKRlRFVE1CRUdBMVVFQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4TnpFeU1qY3lNelV5TkRCYUZ3MHlOekV5TWpjeQpNelV5TkRCYU1CVXhFekFSQmdOVkJBTVRDbXQxWW1WeWJtVjBaWE13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBCkE0SUJEd0F3Z2dFS0FvSUJBUURnbkNrU210bm1meEVnUzNxTlBhVUNINVFPQkdESC9pbkhiV0NPRExCQ0s5Z2QKWEVjQmw3RlZ2OFQya0ZyMURZYjBIVkR0TUk3dGl4UlZGRExna3dObFczNHh3V2RaWEI3R2VvRmdVMXhXT1FTWQpPQUNDOEpnWVRRLzEzOUhCRXZncTRzZWo2N3ArL3MvU05jdzM0S2s3SEl1RmhsazFyUms1a01leEtJbEpCS1AxCllZVVlldHNKL1FwVU9rcUo1SFc0R29ldEU3Nll0SG5PUmZZdm55YnZpU01yaDJ3R0dhTjZyL3M0Q2hPYUliWkMKQW44L1lpUEtHSURhWkdwajZHWG5tWEFSUlgvVElkZ1NRa0x3dDBhVERCblBaNFh2dHBJOGFhTDhEWUpJcUF6QQpOUEgyYjQvdU55bGF0NWpEbzBiMEc1NGFnTWk5NysyQVVyQzlVVVhwQWdNQkFBR2pJekFoTUE0R0ExVWREd0VCCi93UUVBd0lCQmpBUEJnTlZIUk1CQWY4RUJUQURBUUgvTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFCVkdSMnIKaHpYelJNVTV3cmlQUUFKU2Nzek5PUnZvQnBYZlpvWjA5Rkl1cHVkRnhCVlUzZDRoVjlTdEtuUWdQU0dBNVhRTwpIRTk3K0J4SkR1QS9yQjVvQlVzTUJqYzd5MWNkZS9UNmhtaTNyTG9FWUJTblN1ZENPWEpFNEc5LzBmOGJ5QUplCnJOOCtObzFyMlZnWnZaaDZwNzRURWtYdi9sM0hCUFdNN0lkVVYwSE85SkRoU2dPVkYxZnlRS0p4UnVMSlI4anQKTzZtUEgyVVgwdk13VmE0anZ3dGtkZHFrMk9BZFlRdkg5cmJEampiemFpVzBLbm1kdWVSbzkyS0hBTjdCc0RaeQpWcFhIcHFvMUt6ZzdEM2ZwYVhDZjVzaTdscXFyZEpWWEg0SkM3Mnp4c1BlaHFnaThlSXVxT0JraURXbVJ4QXhoCjh5R2VSeDlBYmtuSGg0SWEKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + primaryId: "237054359138908419352140518924933177492" type: Keypair ` @@ -177,6 +178,7 @@ spec: - id: "237054359138908419352140518924933177492" privateMaterial: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBNEp3cEVwclo1bjhSSUV0NmpUMmxBaCtVRGdSZ3gvNHB4MjFnamd5d1FpdllIVnhICkFaZXhWYi9FOXBCYTlRMkc5QjFRN1RDTzdZc1VWUlF5NEpNRFpWdCtNY0ZuV1Z3ZXhucUJZRk5jVmprRW1EZ0EKZ3ZDWUdFMFA5ZC9Sd1JMNEt1TEhvK3U2ZnY3UDBqWE1OK0NwT3h5TGhZWlpOYTBaT1pESHNTaUpTUVNqOVdHRgpHSHJiQ2YwS1ZEcEtpZVIxdUJxSHJSTyttTFI1emtYMkw1OG03NGtqSzRkc0JobWplcS83T0FvVG1pRzJRZ0ovClAySWp5aGlBMm1ScVkraGw1NWx3RVVWLzB5SFlFa0pDOExkR2t3d1p6MmVGNzdhU1BHbWkvQTJDU0tnTXdEVHgKOW0rUDdqY3BXcmVZdzZORzlCdWVHb0RJdmUvdGdGS3d2VkZGNlFJREFRQUJBb0lCQUEwa3RqYVRmeXJBeHNUSQpCZXpiN1pyNU5CVzU1ZHZ1SUkyOTljZDZNSm8rckkvVFJZaHZVdjQ4a1k4SUZYcC9oeVVqemdlREx1bnhtSWY5Ci9aZ3NvaWM5T2w0NC9nNDVtTWR1aGNHWVB6QUFlQ2RjSjVPQjlyUjlWZkRDWHlqWUxsTjhIOGlVMDczNHRUcU0KMFYxM3RROXpkU3FrR1BaT0ljcS9rUi9weWxiT1phUU1lOTdCVGxzQW5PTVNNS0RnbmZ0WTQxMjJMcTNHWXkrdAp2cHIrYktWYVFad3ZrTG9TVTNyRUNDYUthZ2hnd0N5WDdqZnQ5YUVraGRKditLbHdic0dZNldFcnZ4T2FMV0hkCmN1TVFqR2FwWTFGYS80VUQwMG12ckEyNjBOeUtmenJwNitQNDZSclZNd0VZUkpNSVE4WUJBazZONkhoN2RjMEcKOFo2aTFtMENnWUVBOUhlQ0pSMFRTd2JJUTFiRFhVcnpwZnRIdWlkRzVCblNCdGF4L05EOXFJUGhSL0ZCVzVuagoyMm53TGM0OEtreWlybGZJVUxkMGFlNHFWWEpuN3dmWWN1WC9jSk1MRG1TVnRsTTVEem1pLzkxeFJpRmdJengxCkFzYkJ6YUZqSVNQMkhwU2dMK2U5RnRTWGFhcWVaVnJmbGl0VmhZS1VwSS9BS1YzMXFHSGYwNHNDZ1lFQTZ6VFYKOTlTYjQ5V2RsbnM1SWdzZm5YbDZUb1J0dEIxOGxmRUtjVmZqQU00ZnJua2swNkpwRkFaZVIrOUdHS1VYWkhxcwp6MnFjcGx3NGQvbW9DQzZwM3JZUEJNTFhzckdORVVGWnFCbGd6NzJRQTZCQnEzWDBDZzFCYzJaYks1Vkl6d2tnClNUMlNTdXg2Y2NST2ZnVUxtTjVaaUxPdGRVS05FWnBGRjNpM3F0c0NnWUFEVC9zN2RZRmxhdG9iejNrbU1uWEsKc2ZUdTJNbGxIZFJ5czBZR0h1N1E4YmlEdVFraHJKd2h4UFcwS1M4M2c0SlF5bSswYUVmemgzNmJXY2wrdTZSNwpLaEtqKzlvU2Y5cG5kZ2szNDVnSnozNVJiUEpZaCtFdUFITnZ6ZGdDQXZLNngxakVUV2VLZjZidGo1cEYxVTFpClE0UU5Jdy9RaXdJWGpXWmV1YlRHc1FLQmdRQ2JkdUx1MnJMbmx5eUFhSlpNOERsSFp5SDJnQVhiQlpweHFVOFQKdDltdGtKRFVTL0tSaUVvWUdGVjlDcVMwYVhyYXlWTXNEZlhZNkIvUy9VdVpqTzV1N0x0a2xEenFPZjFhS0czUQpkR1hQS2lia25xcUpZSCtiblVOanVZWU5lckVUVjU3bGlqTUdIdVNZQ2Y4dndMbjNveEJmRVJSWDYxTS9EVThaCndvcnovUUtCZ1FEQ1RKSTIramRYZzI2WHVZVW1NNFhYZm5vY2Z6QVhoWEJVTHQxbkVOY29nTmYxZmNwdEFWdHUKQkFpejQvSGlwUUtxb1dWVVlteGZnYmJMUktLTEswczBsT1dLYllkVmpoRW0vbTJaVTh3dFhUYWdOd2tJR295cQpZL0MxTG94NGYxUk9KbkNqYy9oZmNPamN4WDVNOEE4cGVlY0hXbFZ0VVBLVEpneFE3b01LY3c9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= publicMaterial: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMyRENDQWNDZ0F3SUJBZ0lSQUxKWEFrVmo5NjR0cTY3d01TSThvSlF3RFFZSktvWklodmNOQVFFTEJRQXcKRlRFVE1CRUdBMVVFQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4TnpFeU1qY3lNelV5TkRCYUZ3MHlOekV5TWpjeQpNelV5TkRCYU1CVXhFekFSQmdOVkJBTVRDbXQxWW1WeWJtVjBaWE13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBCkE0SUJEd0F3Z2dFS0FvSUJBUURnbkNrU210bm1meEVnUzNxTlBhVUNINVFPQkdESC9pbkhiV0NPRExCQ0s5Z2QKWEVjQmw3RlZ2OFQya0ZyMURZYjBIVkR0TUk3dGl4UlZGRExna3dObFczNHh3V2RaWEI3R2VvRmdVMXhXT1FTWQpPQUNDOEpnWVRRLzEzOUhCRXZncTRzZWo2N3ArL3MvU05jdzM0S2s3SEl1RmhsazFyUms1a01leEtJbEpCS1AxCllZVVlldHNKL1FwVU9rcUo1SFc0R29ldEU3Nll0SG5PUmZZdm55YnZpU01yaDJ3R0dhTjZyL3M0Q2hPYUliWkMKQW44L1lpUEtHSURhWkdwajZHWG5tWEFSUlgvVElkZ1NRa0x3dDBhVERCblBaNFh2dHBJOGFhTDhEWUpJcUF6QQpOUEgyYjQvdU55bGF0NWpEbzBiMEc1NGFnTWk5NysyQVVyQzlVVVhwQWdNQkFBR2pJekFoTUE0R0ExVWREd0VCCi93UUVBd0lCQmpBUEJnTlZIUk1CQWY4RUJUQURBUUgvTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFCVkdSMnIKaHpYelJNVTV3cmlQUUFKU2Nzek5PUnZvQnBYZlpvWjA5Rkl1cHVkRnhCVlUzZDRoVjlTdEtuUWdQU0dBNVhRTwpIRTk3K0J4SkR1QS9yQjVvQlVzTUJqYzd5MWNkZS9UNmhtaTNyTG9FWUJTblN1ZENPWEpFNEc5LzBmOGJ5QUplCnJOOCtObzFyMlZnWnZaaDZwNzRURWtYdi9sM0hCUFdNN0lkVVYwSE85SkRoU2dPVkYxZnlRS0p4UnVMSlI4anQKTzZtUEgyVVgwdk13VmE0anZ3dGtkZHFrMk9BZFlRdkg5cmJEampiemFpVzBLbm1kdWVSbzkyS0hBTjdCc0RaeQpWcFhIcHFvMUt6ZzdEM2ZwYVhDZjVzaTdscXFyZEpWWEg0SkM3Mnp4c1BlaHFnaThlSXVxT0JraURXbVJ4QXhoCjh5R2VSeDlBYmtuSGg0SWEKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + primaryId: "237054359138908419352140518924933177492" type: Keypair ` From 927b321e45179cbe65689fa1bc2577d4bd87341f Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Sun, 11 Apr 2021 10:59:45 -0700 Subject: [PATCH 05/15] Make parsed Keyset type public --- upup/pkg/fi/ca.go | 14 ++++ upup/pkg/fi/clientset_castore.go | 60 ++++++--------- upup/pkg/fi/vfs_castore.go | 126 +++++++++++++++---------------- 3 files changed, 100 insertions(+), 100 deletions(-) diff --git a/upup/pkg/fi/ca.go b/upup/pkg/fi/ca.go index 5490df0d2a..518e9820bc 100644 --- a/upup/pkg/fi/ca.go +++ b/upup/pkg/fi/ca.go @@ -43,6 +43,20 @@ type KeystoreItem struct { Data []byte } +// Keyset is a parsed api.Keyset. +type Keyset struct { + LegacyFormat bool + Items map[string]*KeysetItem + Primary *KeysetItem +} + +// KeysetItem is a certificate/key pair in a Keyset. +type KeysetItem struct { + Id string + Certificate *pki.Certificate + PrivateKey *pki.PrivateKey +} + // Keystore contains just the functions we need to issue keypairs, not to list / manage them type Keystore interface { // FindKeypair finds a cert & private key, returning nil where either is not found diff --git a/upup/pkg/fi/clientset_castore.go b/upup/pkg/fi/clientset_castore.go index d844d1b2aa..18355e9f2e 100644 --- a/upup/pkg/fi/clientset_castore.go +++ b/upup/pkg/fi/clientset_castore.go @@ -65,30 +65,16 @@ func NewClientsetSSHCredentialStore(cluster *kops.Cluster, clientset kopsinterna return c } -// keyset is a parsed Keyset -type keyset struct { - legacyFormat bool - items map[string]*keysetItem - primary *keysetItem -} - -// keysetItem is a parsed KeysetItem -type keysetItem struct { - id string - certificate *pki.Certificate - privateKey *pki.PrivateKey -} - -func parseKeyset(o *kops.Keyset) (*keyset, error) { +func parseKeyset(o *kops.Keyset) (*Keyset, error) { name := o.Name - keyset := &keyset{ - items: make(map[string]*keysetItem), + keyset := &Keyset{ + Items: make(map[string]*KeysetItem), } for _, key := range o.Spec.Keys { - ki := &keysetItem{ - id: key.Id, + ki := &KeysetItem{ + Id: key.Id, } if len(key.PublicMaterial) != 0 { cert, err := pki.ParsePEMCertificate(key.PublicMaterial) @@ -96,7 +82,7 @@ func parseKeyset(o *kops.Keyset) (*keyset, error) { klog.Warningf("key public material was %s", key.PublicMaterial) return nil, fmt.Errorf("error loading certificate %s/%s: %v", name, key.Id, err) } - ki.certificate = cert + ki.Certificate = cert } if len(key.PrivateMaterial) != 0 { @@ -104,19 +90,19 @@ func parseKeyset(o *kops.Keyset) (*keyset, error) { if err != nil { return nil, fmt.Errorf("error loading private key %s/%s: %v", name, key.Id, err) } - ki.privateKey = privateKey + ki.PrivateKey = privateKey } - keyset.items[key.Id] = ki + keyset.Items[key.Id] = ki } - keyset.primary = keyset.items[FindPrimary(o).Id] + keyset.Primary = keyset.Items[FindPrimary(o).Id] return keyset, nil } -// loadKeyset gets the named keyset and the format of the Keyset. -func (c *ClientsetCAStore) loadKeyset(ctx context.Context, name string) (*keyset, error) { +// loadKeyset gets the named Keyset and the format of the Keyset. +func (c *ClientsetCAStore) loadKeyset(ctx context.Context, name string) (*Keyset, error) { o, err := c.clientset.Keysets(c.namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { @@ -167,8 +153,8 @@ func (c *ClientsetCAStore) FindKeypair(name string) (*pki.Certificate, *pki.Priv return nil, nil, false, err } - if keyset != nil && keyset.primary != nil { - return keyset.primary.certificate, keyset.primary.privateKey, keyset.legacyFormat, nil + if keyset != nil && keyset.Primary != nil { + return keyset.Primary.Certificate, keyset.Primary.PrivateKey, keyset.LegacyFormat, nil } return nil, nil, false, nil @@ -182,8 +168,8 @@ func (c *ClientsetCAStore) FindCert(name string) (*pki.Certificate, error) { return nil, err } - if keyset != nil && keyset.primary != nil { - return keyset.primary.certificate, nil + if keyset != nil && keyset.Primary != nil { + return keyset.Primary.Certificate, nil } return nil, nil @@ -200,15 +186,15 @@ func (c *ClientsetCAStore) FindCertificatePool(name string) (*CertificatePool, e pool := &CertificatePool{} if keyset != nil { - if keyset.primary != nil { - pool.Primary = keyset.primary.certificate + if keyset.Primary != nil { + pool.Primary = keyset.Primary.Certificate } - for id, item := range keyset.items { - if id == keyset.primary.id { + for id, item := range keyset.Items { + if id == keyset.Primary.Id { continue } - pool.Secondary = append(pool.Secondary, item.certificate) + pool.Secondary = append(pool.Secondary, item.Certificate) } } return pool, nil @@ -305,8 +291,8 @@ func (c *ClientsetCAStore) FindPrivateKey(name string) (*pki.PrivateKey, error) return nil, err } - if keyset != nil && keyset.primary != nil { - return keyset.primary.privateKey, nil + if keyset != nil && keyset.Primary != nil { + return keyset.Primary.PrivateKey, nil } return nil, nil } @@ -355,7 +341,7 @@ func (c *ClientsetCAStore) addKey(ctx context.Context, name string, keysetType k return nil } -// deleteKeysetItem deletes the specified key from the registry; deleting the whole keyset if it was the last one +// deleteKeysetItem deletes the specified key from the registry; deleting the whole Keyset if it was the last one. func deleteKeysetItem(client kopsinternalversion.KeysetInterface, name string, keysetType kops.KeysetType, id string) error { ctx := context.TODO() diff --git a/upup/pkg/fi/vfs_castore.go b/upup/pkg/fi/vfs_castore.go index 000c302a42..3bde22077f 100644 --- a/upup/pkg/fi/vfs_castore.go +++ b/upup/pkg/fi/vfs_castore.go @@ -40,7 +40,7 @@ type VFSCAStore struct { cluster *kops.Cluster mutex sync.Mutex - cachedCA *keyset + cachedCA *Keyset } var _ CAStore = &VFSCAStore{} @@ -106,10 +106,10 @@ func (c *VFSCAStore) parseKeysetYaml(data []byte) (*kops.Keyset, bool, error) { return keyset, gvk.Version != keysetFormatLatest, nil } -// loadKeyset loads a keyset from the path +// loadKeyset loads a Keyset from the path. // Returns (nil, nil) if the file is not found // Bundles avoid the need for a list-files permission, which can be tricky on e.g. GCE -func (c *VFSCAStore) loadKeyset(p vfs.Path) (*keyset, error) { +func (c *VFSCAStore) loadKeyset(p vfs.Path) (*Keyset, error) { bundlePath := p.Join("keyset.yaml") data, err := bundlePath.ReadFile() if err != nil { @@ -129,31 +129,31 @@ func (c *VFSCAStore) loadKeyset(p vfs.Path) (*keyset, error) { return nil, fmt.Errorf("error mapping bundle %q: %v", p, err) } - keyset.legacyFormat = legacyFormat + keyset.LegacyFormat = legacyFormat return keyset, nil } -func (k *keyset) ToAPIObject(name string, includePrivateKeyMaterial bool) (*kops.Keyset, error) { +func (k *Keyset) ToAPIObject(name string, includePrivateKeyMaterial bool) (*kops.Keyset, error) { o := &kops.Keyset{} o.Name = name o.Spec.Type = kops.SecretTypeKeypair - for _, ki := range k.items { + for _, ki := range k.Items { oki := kops.KeysetItem{ - Id: ki.id, + Id: ki.Id, } - if ki.certificate != nil { + if ki.Certificate != nil { var publicMaterial bytes.Buffer - if _, err := ki.certificate.WriteTo(&publicMaterial); err != nil { + if _, err := ki.Certificate.WriteTo(&publicMaterial); err != nil { return nil, err } oki.PublicMaterial = publicMaterial.Bytes() } - if includePrivateKeyMaterial && ki.privateKey != nil { + if includePrivateKeyMaterial && ki.PrivateKey != nil { var privateMaterial bytes.Buffer - if _, err := ki.privateKey.WriteTo(&privateMaterial); err != nil { + if _, err := ki.PrivateKey.WriteTo(&privateMaterial); err != nil { return nil, err } @@ -162,14 +162,14 @@ func (k *keyset) ToAPIObject(name string, includePrivateKeyMaterial bool) (*kops o.Spec.Keys = append(o.Spec.Keys, oki) } - if k.primary != nil { - o.Spec.PrimaryId = k.primary.id + if k.Primary != nil { + o.Spec.PrimaryId = k.Primary.Id } return o, nil } -// writeKeysetBundle writes a keyset bundle to VFS -func (c *VFSCAStore) writeKeysetBundle(p vfs.Path, name string, keyset *keyset, includePrivateKeyMaterial bool) error { +// writeKeysetBundle writes a Keyset bundle to VFS. +func (c *VFSCAStore) writeKeysetBundle(p vfs.Path, name string, keyset *Keyset, includePrivateKeyMaterial bool) error { p = p.Join("keyset.yaml") o, err := keyset.ToAPIObject(name, includePrivateKeyMaterial) @@ -189,7 +189,7 @@ func (c *VFSCAStore) writeKeysetBundle(p vfs.Path, name string, keyset *keyset, return p.WriteFile(bytes.NewReader(objectData), acl) } -// serializeKeysetBundle converts a keyset bundle to yaml, for writing to VFS +// serializeKeysetBundle converts a Keyset bundle to yaml, for writing to VFS. func serializeKeysetBundle(o *kops.Keyset) ([]byte, error) { var objectData bytes.Buffer codecs := kopscodecs.Codecs @@ -281,8 +281,8 @@ func (c *VFSCAStore) findCert(name string) (*pki.Certificate, bool, error) { return nil, false, fmt.Errorf("error in 'FindCert' attempting to load cert %q: %v", name, err) } - if certs != nil && certs.primary != nil { - return certs.primary.certificate, certs.legacyFormat, nil + if certs != nil && certs.Primary != nil { + return certs.Primary.Certificate, certs.LegacyFormat, nil } return nil, false, nil @@ -294,7 +294,7 @@ func (c *VFSCAStore) FindCert(name string) (*pki.Certificate, error) { } func (c *VFSCAStore) FindCertificatePool(name string) (*CertificatePool, error) { - var certs *keyset + var certs *Keyset var err error p := c.buildCertificatePoolPath(name) @@ -306,18 +306,18 @@ func (c *VFSCAStore) FindCertificatePool(name string) (*CertificatePool, error) pool := &CertificatePool{} if certs != nil { - if certs.primary != nil { - pool.Primary = certs.primary.certificate + if certs.Primary != nil { + pool.Primary = certs.Primary.Certificate } - for k, cert := range certs.items { - if certs.primary != nil && k == certs.primary.id { + for k, cert := range certs.Items { + if certs.Primary != nil && k == certs.Primary.Id { continue } - if cert.certificate == nil { + if cert.Certificate == nil { continue } - pool.Secondary = append(pool.Secondary, cert.certificate) + pool.Secondary = append(pool.Secondary, cert.Certificate) } } return pool, nil @@ -462,7 +462,7 @@ func (c *VFSCAStore) MirrorTo(basedir vfs.Path) error { return nil } -// mirrorKeyset writes keyset bundles for the certificates & privatekeys +// mirrorKeyset writes Keyset bundles for the certificates & privatekeys. func mirrorKeyset(cluster *kops.Cluster, basedir vfs.Path, keyset *kops.Keyset) error { primary := FindPrimary(keyset) if primary == nil { @@ -536,10 +536,10 @@ func mirrorSSHCredential(cluster *kops.Cluster, basedir vfs.Path, sshCredential func (c *VFSCAStore) StoreKeypair(name string, cert *pki.Certificate, privateKey *pki.PrivateKey) error { serial := cert.Certificate.SerialNumber.String() - ki := &keysetItem{ - id: serial, - certificate: cert, - privateKey: privateKey, + ki := &KeysetItem{ + Id: serial, + Certificate: cert, + PrivateKey: privateKey, } { @@ -568,9 +568,9 @@ func (c *VFSCAStore) AddCert(name string, cert *pki.Certificate) error { p := c.buildCertificatePath(name, serial) - ki := &keysetItem{ - id: serial, - certificate: cert, + ki := &KeysetItem{ + Id: serial, + Certificate: cert, } err := c.storeCertificate(name, ki) if err != nil { @@ -582,8 +582,8 @@ func (c *VFSCAStore) AddCert(name string, cert *pki.Certificate) error { return err } -func (c *VFSCAStore) findPrivateKeyset(id string) (*keyset, error) { - var keys *keyset +func (c *VFSCAStore) findPrivateKeyset(id string) (*Keyset, error) { + var keys *Keyset var err error if id == CertificateIDCA { c.mutex.Lock() @@ -623,8 +623,8 @@ func (c *VFSCAStore) FindPrivateKey(id string) (*pki.PrivateKey, error) { } var key *pki.PrivateKey - if keys != nil && keys.primary != nil { - key = keys.primary.privateKey + if keys != nil && keys.Primary != nil { + key = keys.Primary.PrivateKey } return key, nil } @@ -643,8 +643,8 @@ func (c *VFSCAStore) FindPrivateKeyset(name string) (*kops.Keyset, error) { return o, nil } -func (c *VFSCAStore) storePrivateKey(name string, ki *keysetItem) error { - if ki.privateKey == nil { +func (c *VFSCAStore) storePrivateKey(name string, ki *KeysetItem) error { + if ki.PrivateKey == nil { return fmt.Errorf("privateKey not provided to storeCertificate") } @@ -657,13 +657,13 @@ func (c *VFSCAStore) storePrivateKey(name string, ki *keysetItem) error { } if ks == nil { - ks = &keyset{} + ks = &Keyset{} } - if ks.items == nil { - ks.items = make(map[string]*keysetItem) + if ks.Items == nil { + ks.Items = make(map[string]*KeysetItem) } - ks.items[ki.id] = ki - ks.primary = ki + ks.Items[ki.Id] = ki + ks.Primary = ki if err := c.writeKeysetBundle(p, name, ks, true); err != nil { return fmt.Errorf("error writing bundle: %v", err) @@ -674,11 +674,11 @@ func (c *VFSCAStore) storePrivateKey(name string, ki *keysetItem) error { // Write the data { var data bytes.Buffer - if _, err := ki.privateKey.WriteTo(&data); err != nil { + if _, err := ki.PrivateKey.WriteTo(&data); err != nil { return err } - p := c.buildPrivateKeyPath(name, ki.id) + p := c.buildPrivateKeyPath(name, ki.Id) acl, err := acls.GetACL(p, c.cluster) if err != nil { return err @@ -687,8 +687,8 @@ func (c *VFSCAStore) storePrivateKey(name string, ki *keysetItem) error { } } -func (c *VFSCAStore) storeCertificate(name string, ki *keysetItem) error { - if ki.certificate == nil { +func (c *VFSCAStore) storeCertificate(name string, ki *KeysetItem) error { + if ki.Certificate == nil { return fmt.Errorf("certificate not provided to storeCertificate") } @@ -701,13 +701,13 @@ func (c *VFSCAStore) storeCertificate(name string, ki *keysetItem) error { } if ks == nil { - ks = &keyset{} + ks = &Keyset{} } - if ks.items == nil { - ks.items = make(map[string]*keysetItem) + if ks.Items == nil { + ks.Items = make(map[string]*KeysetItem) } - ks.items[ki.id] = ki - ks.primary = ki + ks.Items[ki.Id] = ki + ks.Primary = ki if err := c.writeKeysetBundle(p, name, ks, false); err != nil { return fmt.Errorf("error writing bundle: %v", err) @@ -718,11 +718,11 @@ func (c *VFSCAStore) storeCertificate(name string, ki *keysetItem) error { // Write the data { var data bytes.Buffer - if _, err := ki.certificate.WriteTo(&data); err != nil { + if _, err := ki.Certificate.WriteTo(&data); err != nil { return err } - p := c.buildCertificatePath(name, ki.id) + p := c.buildCertificatePath(name, ki.Id) acl, err := acls.GetACL(p, c.cluster) if err != nil { return err @@ -749,12 +749,12 @@ func (c *VFSCAStore) deletePrivateKey(name string, id string) (bool, error) { return false, err } - if ks == nil || ks.items[id] == nil { + if ks == nil || ks.Items[id] == nil { return false, nil } - delete(ks.items, id) - if ks.primary != nil && ks.primary.id == id { - ks.primary = nil + delete(ks.Items, id) + if ks.Primary != nil && ks.Primary.Id == id { + ks.Primary = nil } if err := c.writeKeysetBundle(p, name, ks, true); err != nil { @@ -782,12 +782,12 @@ func (c *VFSCAStore) deleteCertificate(name string, id string) (bool, error) { return false, err } - if ks == nil || ks.items[id] == nil { + if ks == nil || ks.Items[id] == nil { return false, nil } - delete(ks.items, id) - if ks.primary != nil && ks.primary.id == id { - ks.primary = nil + delete(ks.Items, id) + if ks.Primary != nil && ks.Primary.Id == id { + ks.Primary = nil } if err := c.writeKeysetBundle(p, name, ks, false); err != nil { From 0364a3af25a259eb7178365f17c3e47273885914 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Sun, 11 Apr 2021 13:04:21 -0700 Subject: [PATCH 06/15] Refactor FindKeypair interfaces --- cmd/kops-controller/pkg/server/keystore.go | 6 +-- cmd/kops-controller/pkg/server/node_config.go | 2 +- nodeup/pkg/model/fakes_test.go | 6 ++- pkg/configserver/keystore.go | 17 +++++--- pkg/kubeconfig/create_kubecfg.go | 6 +-- pkg/kubeconfig/create_kubecfg_test.go | 38 +++++++++++------- pkg/pki/issue.go | 7 +--- pkg/pki/issue_test.go | 4 +- upup/pkg/fi/ca.go | 23 ++++++++--- upup/pkg/fi/clientset_castore.go | 20 ++++------ upup/pkg/fi/fitasks/keypair.go | 24 ++++++++---- upup/pkg/fi/vfs_castore.go | 39 +++++++++++++++---- 12 files changed, 122 insertions(+), 70 deletions(-) diff --git a/cmd/kops-controller/pkg/server/keystore.go b/cmd/kops-controller/pkg/server/keystore.go index 52dfb8a52b..44ef8b5b86 100644 --- a/cmd/kops-controller/pkg/server/keystore.go +++ b/cmd/kops-controller/pkg/server/keystore.go @@ -35,12 +35,12 @@ type keystoreEntry struct { var _ pki.Keystore = keystore{} -func (k keystore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) { +func (k keystore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { entry, ok := k.keys[name] if !ok { - return nil, nil, false, fmt.Errorf("unknown CA %q", name) + return nil, nil, fmt.Errorf("unknown CA %q", name) } - return entry.certificate, entry.key, false, nil + return entry.certificate, entry.key, nil } func newKeystore(basePath string, cas []string) (pki.Keystore, error) { diff --git a/cmd/kops-controller/pkg/server/node_config.go b/cmd/kops-controller/pkg/server/node_config.go index 771360acaf..baf885f394 100644 --- a/cmd/kops-controller/pkg/server/node_config.go +++ b/cmd/kops-controller/pkg/server/node_config.go @@ -63,7 +63,7 @@ func (s *Server) getNodeConfig(ctx context.Context, req *nodeup.BootstrapRequest // We populate some certificates that we know the node will need. for _, name := range []string{"ca"} { - cert, _, _, err := s.keystore.FindKeypair(name) + cert, _, err := s.keystore.FindKeypair(name) if err != nil { return nil, fmt.Errorf("error getting certificate %q: %w", name, err) } diff --git a/nodeup/pkg/model/fakes_test.go b/nodeup/pkg/model/fakes_test.go index 6da59dfde5..c87c296992 100644 --- a/nodeup/pkg/model/fakes_test.go +++ b/nodeup/pkg/model/fakes_test.go @@ -33,10 +33,14 @@ type fakeKeyStore struct { var _ fi.Keystore = &fakeKeyStore{} -func (k fakeKeyStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) { +func (k fakeKeyStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { panic("fakeKeyStore does not implement FindKeypair") } +func (k fakeKeyStore) FindKeyset(name string) (*fi.Keyset, error) { + panic("fakeKeyStore does not implement FindKeyset") +} + func (k fakeKeyStore) CreateKeypair(signer string, name string, template *x509.Certificate, privateKey *pki.PrivateKey) (*pki.Certificate, error) { panic("fakeKeyStore does not implement CreateKeypair") } diff --git a/pkg/configserver/keystore.go b/pkg/configserver/keystore.go index ee9f87bc5c..da107f2af6 100644 --- a/pkg/configserver/keystore.go +++ b/pkg/configserver/keystore.go @@ -38,22 +38,27 @@ func NewKeyStore(nodeConfig *nodeup.NodeConfig) fi.CAStore { } } -// FindKeypair implements fi.Keystore -func (s *configserverKeyStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) { - return nil, nil, false, fmt.Errorf("FindKeypair %q not supported by configserverKeyStore", name) +// FindKeypair implements pki.Keystore +func (s *configserverKeyStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { + return nil, nil, fmt.Errorf("FindKeypair %q not supported by configserverKeyStore", name) } -// FindKeypair implements fi.Keystore +// FindKeyset implements fi.Keystore +func (s *configserverKeyStore) FindKeyset(name string) (*fi.Keyset, error) { + return nil, fmt.Errorf("FindKeyset %q not supported by configserverKeyStore", name) +} + +// CreateKeypair implements fi.Keystore func (s *configserverKeyStore) CreateKeypair(signer string, name string, template *x509.Certificate, privateKey *pki.PrivateKey) (*pki.Certificate, error) { return nil, fmt.Errorf("CreateKeypair not supported by configserverKeyStore") } -// FindKeypair implements fi.Keystore +// StoreKeypair implements fi.Keystore func (s *configserverKeyStore) StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error { return fmt.Errorf("StoreKeypair not supported by configserverKeyStore") } -// FindKeypair implements fi.Keystore +// MirrorTo implements fi.Keystore func (s *configserverKeyStore) MirrorTo(basedir vfs.Path) error { return fmt.Errorf("MirrorTo not supported by configserverKeyStore") } diff --git a/pkg/kubeconfig/create_kubecfg.go b/pkg/kubeconfig/create_kubecfg.go index f9c2d21d68..2b27fd6549 100644 --- a/pkg/kubeconfig/create_kubecfg.go +++ b/pkg/kubeconfig/create_kubecfg.go @@ -110,12 +110,12 @@ func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.Se // add the CA Cert to the kubeconfig only if we didn't specify a certificate for the LB // or if we're using admin credentials and the secondary port if cluster.Spec.API == nil || cluster.Spec.API.LoadBalancer == nil || cluster.Spec.API.LoadBalancer.SSLCertificate == "" || cluster.Spec.API.LoadBalancer.Class == kops.LoadBalancerClassNetwork || internal { - cert, _, _, err := keyStore.FindKeypair(fi.CertificateIDCA) + keySet, err := keyStore.FindKeyset(fi.CertificateIDCA) if err != nil { return nil, fmt.Errorf("error fetching CA keypair: %v", err) } - if cert != nil { - b.CACert, err = cert.AsBytes() + if keySet != nil && keySet.Primary != nil && keySet.Primary.Certificate != nil { + b.CACert, err = keySet.Primary.Certificate.AsBytes() if err != nil { return nil, err } diff --git a/pkg/kubeconfig/create_kubecfg_test.go b/pkg/kubeconfig/create_kubecfg_test.go index 3deaf29c6d..4f5a9c0c9a 100644 --- a/pkg/kubeconfig/create_kubecfg_test.go +++ b/pkg/kubeconfig/create_kubecfg_test.go @@ -85,7 +85,7 @@ func (f fakeStatusCloud) FindClusterStatus(cluster *kops.Cluster) (*kops.Cluster // mock a fake key store type fakeKeyStore struct { - FindKeypairFn func(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) + FindKeysetFn func(name string) (*fi.Keyset, error) // StoreKeypair writes the keypair to the store StoreKeypairFn func(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error @@ -94,8 +94,12 @@ type fakeKeyStore struct { MirrorToFn func(basedir vfs.Path) error } -func (f fakeKeyStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) { - return f.FindKeypairFn(name) +func (f fakeKeyStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { + return fi.FindKeypair(f, name) +} + +func (f fakeKeyStore) FindKeyset(name string) (*fi.Keyset, error) { + return f.FindKeysetFn(name) } func (f fakeKeyStore) StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error { @@ -124,16 +128,22 @@ func buildMinimalCluster(clusterName string, masterPublicName string, lbCert boo return cluster } -// create a fake certificate -func fakeCertificate() *pki.Certificate { +// create a fake keyset +func fakeKeyset() *fi.Keyset { cert, _ := pki.ParsePEMCertificate([]byte(certData)) - return cert -} - -// create a fake private key -func fakePrivateKey() *pki.PrivateKey { key, _ := pki.ParsePEMPrivateKey([]byte(privatekeyData)) - return key + keysetItem := &fi.KeysetItem{ + Id: "99", + Certificate: cert, + PrivateKey: key, + } + return &fi.Keyset{ + LegacyFormat: true, + Items: map[string]*fi.KeysetItem{ + "99": keysetItem, + }, + Primary: keysetItem, + } } func TestBuildKubecfg(t *testing.T) { @@ -350,10 +360,8 @@ func TestBuildKubecfg(t *testing.T) { kopsStateStore := "memfs://example-state-store" keyStore := fakeKeyStore{ - FindKeypairFn: func(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) { - return fakeCertificate(), - fakePrivateKey(), - true, + FindKeysetFn: func(name string) (*fi.Keyset, error) { + return fakeKeyset(), nil }, } diff --git a/pkg/pki/issue.go b/pkg/pki/issue.go index a35e00b317..4c7b09b066 100644 --- a/pkg/pki/issue.go +++ b/pkg/pki/issue.go @@ -58,10 +58,7 @@ type IssueCertRequest struct { type Keystore interface { // FindKeypair finds a cert & private key, returning nil where either is not found // (if the certificate is found but not keypair, that is not an error: only the cert will be returned). - // This func returns a cert, private key and a bool. The bool value is whether the keypair is stored - // in a legacy format. This bool is used by a keypair - // task to convert a Legacy Keypair to the new Keypair API format. - FindKeypair(name string) (*Certificate, *PrivateKey, bool, error) + FindKeypair(name string) (*Certificate, *PrivateKey, error) } // IssueCert issues a certificate, either a self-signed CA or from a CA in a keystore. @@ -119,7 +116,7 @@ func IssueCert(request *IssueCertRequest, keystore Keystore) (issuedCertificate var signer *x509.Certificate if !template.IsCA { var err error - caCertificate, caPrivateKey, _, err = keystore.FindKeypair(request.Signer) + caCertificate, caPrivateKey, err = keystore.FindKeypair(request.Signer) if err != nil { return nil, nil, nil, err } diff --git a/pkg/pki/issue_test.go b/pkg/pki/issue_test.go index 4496dd2878..8ebe7abf05 100644 --- a/pkg/pki/issue_test.go +++ b/pkg/pki/issue_test.go @@ -38,11 +38,11 @@ type mockKeystore struct { invoked bool } -func (m *mockKeystore) FindKeypair(name string) (*Certificate, *PrivateKey, bool, error) { +func (m *mockKeystore) FindKeypair(name string) (*Certificate, *PrivateKey, error) { assert.False(m.t, m.invoked, "invoked already") m.invoked = true assert.Equal(m.t, m.signer, name, "name argument") - return m.cert, m.key, false, nil + return m.cert, m.key, nil } func TestIssueCert(t *testing.T) { diff --git a/upup/pkg/fi/ca.go b/upup/pkg/fi/ca.go index 518e9820bc..af3ad99710 100644 --- a/upup/pkg/fi/ca.go +++ b/upup/pkg/fi/ca.go @@ -45,6 +45,7 @@ type KeystoreItem struct { // Keyset is a parsed api.Keyset. type Keyset struct { + // LegacyFormat instructs a keypair task to convert a Legacy Keyset to the new Keyset API format. LegacyFormat bool Items map[string]*KeysetItem Primary *KeysetItem @@ -59,12 +60,10 @@ type KeysetItem struct { // Keystore contains just the functions we need to issue keypairs, not to list / manage them type Keystore interface { - // FindKeypair finds a cert & private key, returning nil where either is not found - // (if the certificate is found but not keypair, that is not an error: only the cert will be returned). - // This func returns a cert, private key and a bool. The bool value is whether the keypair is stored - // in a legacy format. This bool is used by a keypair - // task to convert a Legacy Keypair to the new Keypair API format. - FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) + pki.Keystore + + // FindKeyset finds a Keyset. + FindKeyset(name string) (*Keyset, error) // StoreKeypair writes the keypair to the store, making it the primary. StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error @@ -159,3 +158,15 @@ func (c *CertificatePool) AsString() (string, error) { } return data.String(), nil } + +// FindKeypair is a common implementation of pki.FindKeypair. +func FindKeypair(c Keystore, name string) (*pki.Certificate, *pki.PrivateKey, error) { + keyset, err := c.FindKeyset(name) + if err != nil { + return nil, nil, err + } + if keyset == nil || keyset.Primary == nil { + return nil, nil, nil + } + return keyset.Primary.Certificate, keyset.Primary.PrivateKey, nil +} diff --git a/upup/pkg/fi/clientset_castore.go b/upup/pkg/fi/clientset_castore.go index 18355e9f2e..2ea4ef41f6 100644 --- a/upup/pkg/fi/clientset_castore.go +++ b/upup/pkg/fi/clientset_castore.go @@ -145,19 +145,15 @@ func FindPrimary(keyset *kops.Keyset) *kops.KeysetItem { return primary } -// FindKeypair implements CAStore::FindKeypair -func (c *ClientsetCAStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) { +// FindKeypair implements PKI::FindKeypair +func (c *ClientsetCAStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { + return FindKeypair(c, name) +} + +// FindKeyset implements CAStore::FindKeyset +func (c *ClientsetCAStore) FindKeyset(name string) (*Keyset, error) { ctx := context.TODO() - keyset, err := c.loadKeyset(ctx, name) - if err != nil { - return nil, nil, false, err - } - - if keyset != nil && keyset.Primary != nil { - return keyset.Primary.Certificate, keyset.Primary.PrivateKey, keyset.LegacyFormat, nil - } - - return nil, nil, false, nil + return c.loadKeyset(ctx, name) } // FindCert implements CAStore::FindCert diff --git a/upup/pkg/fi/fitasks/keypair.go b/upup/pkg/fi/fitasks/keypair.go index a2fc296e24..ab2450ecb9 100644 --- a/upup/pkg/fi/fitasks/keypair.go +++ b/upup/pkg/fi/fitasks/keypair.go @@ -70,14 +70,15 @@ func (e *Keypair) Find(c *fi.Context) (*Keypair, error) { return nil, nil } - cert, key, legacyFormat, err := c.Keystore.FindKeypair(name) + keyset, err := c.Keystore.FindKeyset(name) if err != nil { return nil, err } - if cert == nil { + if keyset == nil || keyset.Primary == nil || keyset.Primary.Certificate == nil { return nil, nil } - if key == nil { + cert := keyset.Primary.Certificate + if keyset.Primary.PrivateKey == nil { return nil, fmt.Errorf("found cert in store, but did not find private key: %q", name) } @@ -94,7 +95,7 @@ func (e *Keypair) Find(c *fi.Context) (*Keypair, error) { AlternateNames: alternateNames, Subject: pki.PkixNameToString(&cert.Subject), Type: pki.BuildTypeDescription(cert.Certificate), - LegacyFormat: legacyFormat, + LegacyFormat: keyset.LegacyFormat, } actual.Signer = &Keypair{Subject: pki.PkixNameToString(&cert.Certificate.Issuer)} @@ -175,14 +176,21 @@ func (_ *Keypair) Render(c *fi.Context, a, e, changes *Keypair) error { if createCertificate { klog.V(2).Infof("Creating PKI keypair %q", name) - _, privateKey, _, err := c.Keystore.FindKeypair(name) + keyset, err := c.Keystore.FindKeyset(name) if err != nil { return err } + if keyset == nil { + keyset = &fi.Keyset{} + } // We always reuse the private key if it exists, // if we change keys we often have to regenerate e.g. the service accounts // TODO: Eventually rotate keys / don't always reuse? + var privateKey *pki.PrivateKey + if keyset.Primary != nil { + privateKey = keyset.Primary.PrivateKey + } if privateKey == nil { klog.V(2).Infof("Creating privateKey %q", name) } @@ -227,7 +235,7 @@ func (_ *Keypair) Render(c *fi.Context, a, e, changes *Keypair) error { } // Make double-sure it round-trips - _, _, _, err = c.Keystore.FindKeypair(name) + _, err = c.Keystore.FindKeyset(name) if err != nil { return err } @@ -240,11 +248,11 @@ func (_ *Keypair) Render(c *fi.Context, a, e, changes *Keypair) error { if changeStoredFormat { // We fetch and reinsert the same keypair, forcing an update to our preferred format // TODO: We're assuming that we want to save in the preferred format - cert, privateKey, _, err := c.Keystore.FindKeypair(name) + keyset, err := c.Keystore.FindKeyset(name) if err != nil { return err } - err = c.Keystore.StoreKeypair(name, cert, privateKey) + err = c.Keystore.StoreKeypair(name, keyset.Primary.Certificate, keyset.Primary.PrivateKey) if err != nil { return err } diff --git a/upup/pkg/fi/vfs_castore.go b/upup/pkg/fi/vfs_castore.go index 3bde22077f..1ed767c82b 100644 --- a/upup/pkg/fi/vfs_castore.go +++ b/upup/pkg/fi/vfs_castore.go @@ -252,26 +252,49 @@ func (c *VFSCAStore) loadOneCertificate(p vfs.Path) (*pki.Certificate, error) { return cert, nil } -func (c *VFSCAStore) FindKeypair(id string) (*pki.Certificate, *pki.PrivateKey, bool, error) { - cert, legacyFormat, err := c.findCert(id) +func (c *VFSCAStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { + return FindKeypair(c, name) +} + +func (c *VFSCAStore) FindKeyset(id string) (*Keyset, error) { + certs, err := c.loadKeyset(c.buildCertificatePoolPath(id)) if (cert == nil || os.IsNotExist(err)) && id == "service-account" { // The strange name is because Kops prior to 1.19 used the api-server TLS key for this. id = "master" - cert, _, err = c.findCert(id) - legacyFormat = true + certs, err = c.loadKeyset(c.buildCertificatePoolPath(id)) + if certs != nil { + certs.LegacyFormat = true + } } if err != nil { - return nil, nil, false, err + return nil, err } - key, err := c.FindPrivateKey(id) + keys, err := c.findPrivateKeyset(id) if err != nil { - return nil, nil, false, err + return nil, err } - return cert, key, legacyFormat, nil + if certs != nil { + if keys == nil { + return certs, nil + } + if certs.LegacyFormat { + keys.LegacyFormat = true + } + for key, certItem := range certs.Items { + keyItem := keys.Items[key] + if keyItem == nil { + keys.Items[key] = certItem + } else if keyItem.Certificate == nil { + keyItem.Certificate = certItem.Certificate + } + } + } + + return keys, nil } func (c *VFSCAStore) findCert(name string) (*pki.Certificate, bool, error) { From ed1f6ff79e3bcfbca9b2985cc1a58494c03b39a7 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Sun, 11 Apr 2021 18:59:50 -0700 Subject: [PATCH 07/15] Refactor StoreKeypair and AddCert --- cmd/kops/create_secret_keypair_ca.go | 15 +- nodeup/pkg/model/fakes_test.go | 6 +- pkg/configserver/keystore.go | 7 +- pkg/kubeconfig/create_kubecfg_test.go | 8 +- upup/pkg/fi/BUILD.bazel | 1 - upup/pkg/fi/ca.go | 22 ++- upup/pkg/fi/clientset_castore.go | 83 +++++------ upup/pkg/fi/fitasks/keypair.go | 19 ++- upup/pkg/fi/vfs_castore.go | 181 ++--------------------- upup/pkg/fi/vfs_castore_test.go | 113 ++++---------- upup/pkg/kutil/convert_kubeup_cluster.go | 18 ++- upup/pkg/kutil/import_cluster.go | 7 +- 12 files changed, 142 insertions(+), 338 deletions(-) diff --git a/cmd/kops/create_secret_keypair_ca.go b/cmd/kops/create_secret_keypair_ca.go index 5944a0f910..1646dcea13 100644 --- a/cmd/kops/create_secret_keypair_ca.go +++ b/cmd/kops/create_secret_keypair_ca.go @@ -133,7 +133,20 @@ func RunCreateSecretCaCert(ctx context.Context, f *util.Factory, out io.Writer, return fmt.Errorf("error loading certificate %q: %v", options.CaCertPath, err) } - err = keyStore.StoreKeypair(fi.CertificateIDCA, cert, privateKey) + serialString := cert.Certificate.SerialNumber.String() + ki := &fi.KeysetItem{ + Id: serialString, + Certificate: cert, + PrivateKey: privateKey, + } + + err = keyStore.StoreKeypair(fi.CertificateIDCA, &fi.Keyset{ + LegacyFormat: false, + Items: map[string]*fi.KeysetItem{ + serialString: ki, + }, + Primary: ki, + }) if err != nil { return fmt.Errorf("error storing user provided keys %q %q: %v", options.CaCertPath, options.CaPrivateKeyPath, err) } diff --git a/nodeup/pkg/model/fakes_test.go b/nodeup/pkg/model/fakes_test.go index c87c296992..9dad8510e4 100644 --- a/nodeup/pkg/model/fakes_test.go +++ b/nodeup/pkg/model/fakes_test.go @@ -45,7 +45,7 @@ func (k fakeKeyStore) CreateKeypair(signer string, name string, template *x509.C panic("fakeKeyStore does not implement CreateKeypair") } -func (k fakeKeyStore) StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error { +func (k fakeKeyStore) StoreKeypair(id string, keyset *fi.Keyset) error { panic("fakeKeyStore does not implement StoreKeypair") } @@ -94,10 +94,6 @@ func (k fakeCAStore) ListKeysets() ([]*kops.Keyset, error) { panic("fakeCAStore does not implement ListKeysets") } -func (k fakeCAStore) AddCert(name string, cert *pki.Certificate) error { - panic("fakeCAStore does not implement AddCert") -} - func (k fakeCAStore) DeleteKeysetItem(item *kops.Keyset, id string) error { panic("fakeCAStore does not implement DeleteKeysetItem") } diff --git a/pkg/configserver/keystore.go b/pkg/configserver/keystore.go index da107f2af6..d46c905fd6 100644 --- a/pkg/configserver/keystore.go +++ b/pkg/configserver/keystore.go @@ -54,7 +54,7 @@ func (s *configserverKeyStore) CreateKeypair(signer string, name string, templat } // StoreKeypair implements fi.Keystore -func (s *configserverKeyStore) StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error { +func (s *configserverKeyStore) StoreKeypair(id string, keyset *fi.Keyset) error { return fmt.Errorf("StoreKeypair not supported by configserverKeyStore") } @@ -109,11 +109,6 @@ func (s *configserverKeyStore) ListKeysets() ([]*kops.Keyset, error) { return nil, fmt.Errorf("ListKeysets not supported by configserverKeyStore") } -// AddCert implements fi.CAStore -func (s *configserverKeyStore) AddCert(name string, cert *pki.Certificate) error { - return fmt.Errorf("AddCert not supported by configserverKeyStore") -} - // DeleteKeysetItem implements fi.CAStore func (s *configserverKeyStore) DeleteKeysetItem(item *kops.Keyset, id string) error { return fmt.Errorf("DeleteKeysetItem not supported by configserverKeyStore") diff --git a/pkg/kubeconfig/create_kubecfg_test.go b/pkg/kubeconfig/create_kubecfg_test.go index 4f5a9c0c9a..7c7e6c0539 100644 --- a/pkg/kubeconfig/create_kubecfg_test.go +++ b/pkg/kubeconfig/create_kubecfg_test.go @@ -87,8 +87,8 @@ func (f fakeStatusCloud) FindClusterStatus(cluster *kops.Cluster) (*kops.Cluster type fakeKeyStore struct { FindKeysetFn func(name string) (*fi.Keyset, error) - // StoreKeypair writes the keypair to the store - StoreKeypairFn func(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error + // StoreKeypair writes the keypair to the store. + StoreKeypairFn func(id string, keyset *fi.Keyset) error // MirrorTo will copy secrets to a vfs.Path, which is often easier for a machine to read MirrorToFn func(basedir vfs.Path) error @@ -102,8 +102,8 @@ func (f fakeKeyStore) FindKeyset(name string) (*fi.Keyset, error) { return f.FindKeysetFn(name) } -func (f fakeKeyStore) StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error { - return f.StoreKeypairFn(id, cert, privateKey) +func (f fakeKeyStore) StoreKeypair(id string, keyset *fi.Keyset) error { + return f.StoreKeypairFn(id, keyset) } func (f fakeKeyStore) MirrorTo(basedir vfs.Path) error { diff --git a/upup/pkg/fi/BUILD.bazel b/upup/pkg/fi/BUILD.bazel index 2226f923c1..040b2632ee 100644 --- a/upup/pkg/fi/BUILD.bazel +++ b/upup/pkg/fi/BUILD.bazel @@ -67,7 +67,6 @@ go_library( go_test( name = "go_default_test", - size = "small", srcs = [ "dryruntarget_test.go", "files_test.go", diff --git a/upup/pkg/fi/ca.go b/upup/pkg/fi/ca.go index af3ad99710..fa853e4c4c 100644 --- a/upup/pkg/fi/ca.go +++ b/upup/pkg/fi/ca.go @@ -19,6 +19,7 @@ package fi import ( "bytes" "fmt" + "strconv" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/pki" @@ -65,8 +66,8 @@ type Keystore interface { // FindKeyset finds a Keyset. FindKeyset(name string) (*Keyset, error) - // StoreKeypair writes the keypair to the store, making it the primary. - StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error + // StoreKeypair writes the keyset to the store. + StoreKeypair(id string, keyset *Keyset) error // MirrorTo will copy secrets to a vfs.Path, which is often easier for a machine to read MirrorTo(basedir vfs.Path) error @@ -99,9 +100,6 @@ type CAStore interface { // The key material is not guaranteed to be populated - metadata like the name will be. ListKeysets() ([]*kops.Keyset, error) - // AddCert adds an alternative certificate to the pool (primarily useful for CAs) - AddCert(name string, cert *pki.Certificate) error - // DeleteKeysetItem will delete the specified item from the Keyset DeleteKeysetItem(item *kops.Keyset, id string) error } @@ -170,3 +168,17 @@ func FindKeypair(c Keystore, name string) (*pki.Certificate, *pki.PrivateKey, er } return keyset.Primary.Certificate, keyset.Primary.PrivateKey, nil } + +// AddCert adds an alternative certificate to the keyset (primarily useful for CAs) +func AddCert(keyset *Keyset, cert *pki.Certificate) { + serial := 0 + + for keyset.Items[strconv.Itoa(serial)] != nil { + serial++ + } + + keyset.Items[strconv.Itoa(serial)] = &KeysetItem{ + Id: strconv.Itoa(serial), + Certificate: cert, + } +} diff --git a/upup/pkg/fi/clientset_castore.go b/upup/pkg/fi/clientset_castore.go index 2ea4ef41f6..bcbf792635 100644 --- a/upup/pkg/fi/clientset_castore.go +++ b/upup/pkg/fi/clientset_castore.go @@ -258,25 +258,9 @@ func (c *ClientsetCAStore) ListSSHCredentials() ([]*kops.SSHCredential, error) { } // StoreKeypair implements CAStore::StoreKeypair -func (c *ClientsetCAStore) StoreKeypair(name string, cert *pki.Certificate, privateKey *pki.PrivateKey) error { +func (c *ClientsetCAStore) StoreKeypair(name string, keyset *Keyset) error { ctx := context.TODO() - return c.storeKeypair(ctx, name, cert.Certificate.SerialNumber.String(), cert, privateKey) -} - -// AddCert implements CAStore::AddCert -func (c *ClientsetCAStore) AddCert(name string, cert *pki.Certificate) error { - ctx := context.TODO() - klog.Infof("Adding TLS certificate: %q", name) - - // We add with a timestamp of zero so this will never be the newest cert - serial := pki.BuildPKISerial(0) - - err := c.storeKeypair(ctx, name, serial.String(), cert, nil) - if err != nil { - return err - } - - return nil + return c.storeKeyset(ctx, name, keyset, kops.SecretTypeKeypair) } // FindPrivateKey implements CAStore::FindPrivateKey @@ -306,31 +290,52 @@ func (c *ClientsetCAStore) FindPrivateKeyset(name string) (*kops.Keyset, error) return o, nil } -// addKey saves the specified key to the registry -func (c *ClientsetCAStore) addKey(ctx context.Context, name string, keysetType kops.KeysetType, item *kops.KeysetItem) error { +// storeKeyset saves the specified keyset to the registry. +func (c *ClientsetCAStore) storeKeyset(ctx context.Context, name string, keyset *Keyset, keysetType kops.KeysetType) error { create := false client := c.clientset.Keysets(c.namespace) - keyset, err := client.Get(ctx, name, metav1.GetOptions{}) + kopsKeyset, err := client.Get(ctx, name, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { - keyset = nil + kopsKeyset = nil } else { return fmt.Errorf("error reading keyset %q: %v", name, err) } } - if keyset == nil { - keyset = &kops.Keyset{} - keyset.Name = name - keyset.Spec.Type = keysetType + + if kopsKeyset == nil { + kopsKeyset = &kops.Keyset{} + kopsKeyset.Name = name + kopsKeyset.Spec.Type = keysetType create = true } - keyset.Spec.Keys = append(keyset.Spec.Keys, *item) + + kopsKeyset.Spec.Keys = nil + kopsKeyset.Spec.PrimaryId = keyset.Primary.Id + for _, item := range keyset.Items { + var publicMaterial bytes.Buffer + if _, err := item.Certificate.WriteTo(&publicMaterial); err != nil { + return err + } + + var privateMaterial bytes.Buffer + if _, err := item.PrivateKey.WriteTo(&privateMaterial); err != nil { + return err + } + + kopsKeyset.Spec.Keys = append(kopsKeyset.Spec.Keys, kops.KeysetItem{ + Id: item.Id, + PublicMaterial: publicMaterial.Bytes(), + PrivateMaterial: privateMaterial.Bytes(), + }) + } + if create { - if _, err := client.Create(ctx, keyset, metav1.CreateOptions{}); err != nil { + if _, err := client.Create(ctx, kopsKeyset, metav1.CreateOptions{}); err != nil { return fmt.Errorf("error creating keyset %q: %v", name, err) } } else { - if _, err := client.Update(ctx, keyset, metav1.UpdateOptions{}); err != nil { + if _, err := client.Update(ctx, kopsKeyset, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("error updating keyset %q: %v", name, err) } } @@ -418,26 +423,6 @@ func (c *ClientsetCAStore) deleteSSHCredential(ctx context.Context, name string) return nil } -// addKey saves the specified keypair to the registry -func (c *ClientsetCAStore) storeKeypair(ctx context.Context, name string, id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error { - var publicMaterial bytes.Buffer - if _, err := cert.WriteTo(&publicMaterial); err != nil { - return err - } - - var privateMaterial bytes.Buffer - if _, err := privateKey.WriteTo(&privateMaterial); err != nil { - return err - } - - item := &kops.KeysetItem{ - Id: id, - PublicMaterial: publicMaterial.Bytes(), - PrivateMaterial: privateMaterial.Bytes(), - } - return c.addKey(ctx, name, kops.SecretTypeKeypair, item) -} - // AddSSHPublicKey implements CAStore::AddSSHPublicKey func (c *ClientsetCAStore) AddSSHPublicKey(name string, pubkey []byte) error { ctx := context.TODO() diff --git a/upup/pkg/fi/fitasks/keypair.go b/upup/pkg/fi/fitasks/keypair.go index ab2450ecb9..a7efd720ed 100644 --- a/upup/pkg/fi/fitasks/keypair.go +++ b/upup/pkg/fi/fitasks/keypair.go @@ -225,7 +225,21 @@ func (_ *Keypair) Render(c *fi.Context, a, e, changes *Keypair) error { if err != nil { return err } - err = c.Keystore.StoreKeypair(name, cert, privateKey) + + serialString := cert.Certificate.SerialNumber.String() + ki := &fi.KeysetItem{ + Id: serialString, + Certificate: cert, + PrivateKey: privateKey, + } + + err = c.Keystore.StoreKeypair(name, &fi.Keyset{ + LegacyFormat: false, + Items: map[string]*fi.KeysetItem{ + serialString: ki, + }, + Primary: ki, + }) if err != nil { return err } @@ -252,7 +266,8 @@ func (_ *Keypair) Render(c *fi.Context, a, e, changes *Keypair) error { if err != nil { return err } - err = c.Keystore.StoreKeypair(name, keyset.Primary.Certificate, keyset.Primary.PrivateKey) + keyset.LegacyFormat = false + err = c.Keystore.StoreKeypair(name, keyset) if err != nil { return err } diff --git a/upup/pkg/fi/vfs_castore.go b/upup/pkg/fi/vfs_castore.go index 1ed767c82b..1ad94ee871 100644 --- a/upup/pkg/fi/vfs_castore.go +++ b/upup/pkg/fi/vfs_castore.go @@ -66,8 +66,8 @@ func NewVFSSSHCredentialStore(cluster *kops.Cluster, basedir vfs.Path) SSHCreden return c } -func (s *VFSCAStore) VFSPath() vfs.Path { - return s.basedir +func (c *VFSCAStore) VFSPath() vfs.Path { + return c.basedir } func (c *VFSCAStore) buildCertificatePoolPath(name string) vfs.Path { @@ -207,49 +207,13 @@ func serializeKeysetBundle(o *kops.Keyset) ([]byte, error) { // removePrivateKeyMaterial returns a copy of the Keyset with the private key data removed func removePrivateKeyMaterial(o *kops.Keyset) *kops.Keyset { - copy := o.DeepCopy() + c := o.DeepCopy() - for i := range copy.Spec.Keys { - copy.Spec.Keys[i].PrivateMaterial = nil + for i := range c.Spec.Keys { + c.Spec.Keys[i].PrivateMaterial = nil } - return copy -} - -func SerializeKeyset(o *kops.Keyset) ([]byte, error) { - var objectData bytes.Buffer - { - codecs := kopscodecs.Codecs - yaml, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), "application/yaml") - if !ok { - klog.Fatalf("no YAML serializer registered") - } - encoder := codecs.EncoderForVersion(yaml.Serializer, v1alpha2.SchemeGroupVersion) - - if err := encoder.Encode(o, &objectData); err != nil { - return nil, fmt.Errorf("error serializing keyset: %v", err) - } - } - - return objectData.Bytes(), nil -} - -func (c *VFSCAStore) loadOneCertificate(p vfs.Path) (*pki.Certificate, error) { - data, err := p.ReadFile() - if err != nil { - if os.IsNotExist(err) { - return nil, nil - } - return nil, err - } - cert, err := pki.ParsePEMCertificate(data) - if err != nil { - return nil, err - } - if cert == nil { - return nil, nil - } - return cert, nil + return c } func (c *VFSCAStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { @@ -556,55 +520,24 @@ func mirrorSSHCredential(cluster *kops.Cluster, basedir vfs.Path, sshCredential return nil } -func (c *VFSCAStore) StoreKeypair(name string, cert *pki.Certificate, privateKey *pki.PrivateKey) error { - serial := cert.Certificate.SerialNumber.String() - - ki := &KeysetItem{ - Id: serial, - Certificate: cert, - PrivateKey: privateKey, - } - +func (c *VFSCAStore) StoreKeypair(name string, keyset *Keyset) error { { - err := c.storePrivateKey(name, ki) - if err != nil { - return err + p := c.buildPrivateKeyPoolPath(name) + if err := c.writeKeysetBundle(p, name, keyset, true); err != nil { + return fmt.Errorf("writing private bundle: %v", err) } } { - err := c.storeCertificate(name, ki) - if err != nil { - // TODO: Delete private key? - return err + p := c.buildCertificatePoolPath(name) + if err := c.writeKeysetBundle(p, name, keyset, false); err != nil { + return fmt.Errorf("writing certificate bundle: %v", err) } } return nil } -func (c *VFSCAStore) AddCert(name string, cert *pki.Certificate) error { - klog.Infof("Adding TLS certificate: %q", name) - - // We add with a timestamp of zero so this will never be the newest cert - serial := pki.BuildPKISerial(0).String() - - p := c.buildCertificatePath(name, serial) - - ki := &KeysetItem{ - Id: serial, - Certificate: cert, - } - err := c.storeCertificate(name, ki) - if err != nil { - return err - } - - // Make double-sure it round-trips - _, err = c.loadOneCertificate(p) - return err -} - func (c *VFSCAStore) findPrivateKeyset(id string) (*Keyset, error) { var keys *Keyset var err error @@ -666,94 +599,6 @@ func (c *VFSCAStore) FindPrivateKeyset(name string) (*kops.Keyset, error) { return o, nil } -func (c *VFSCAStore) storePrivateKey(name string, ki *KeysetItem) error { - if ki.PrivateKey == nil { - return fmt.Errorf("privateKey not provided to storeCertificate") - } - - // Write the bundle - { - p := c.buildPrivateKeyPoolPath(name) - ks, err := c.loadKeyset(p) - if err != nil { - return err - } - - if ks == nil { - ks = &Keyset{} - } - if ks.Items == nil { - ks.Items = make(map[string]*KeysetItem) - } - ks.Items[ki.Id] = ki - ks.Primary = ki - - if err := c.writeKeysetBundle(p, name, ks, true); err != nil { - return fmt.Errorf("error writing bundle: %v", err) - } - } - - // TODO stop writing and remove legacy format files after rollback to pre-kops 1.18 not needed - // Write the data - { - var data bytes.Buffer - if _, err := ki.PrivateKey.WriteTo(&data); err != nil { - return err - } - - p := c.buildPrivateKeyPath(name, ki.Id) - acl, err := acls.GetACL(p, c.cluster) - if err != nil { - return err - } - return p.WriteFile(bytes.NewReader(data.Bytes()), acl) - } -} - -func (c *VFSCAStore) storeCertificate(name string, ki *KeysetItem) error { - if ki.Certificate == nil { - return fmt.Errorf("certificate not provided to storeCertificate") - } - - // Write the bundle - { - p := c.buildCertificatePoolPath(name) - ks, err := c.loadKeyset(p) - if err != nil { - return err - } - - if ks == nil { - ks = &Keyset{} - } - if ks.Items == nil { - ks.Items = make(map[string]*KeysetItem) - } - ks.Items[ki.Id] = ki - ks.Primary = ki - - if err := c.writeKeysetBundle(p, name, ks, false); err != nil { - return fmt.Errorf("error writing bundle: %v", err) - } - } - - // TODO stop writing and remove legacy format files after rollback to pre-kops 1.18 not needed - // Write the data - { - var data bytes.Buffer - if _, err := ki.Certificate.WriteTo(&data); err != nil { - return err - } - - p := c.buildCertificatePath(name, ki.Id) - acl, err := acls.GetACL(p, c.cluster) - if err != nil { - return err - } - return p.WriteFile(bytes.NewReader(data.Bytes()), acl) - } -} - func (c *VFSCAStore) deletePrivateKey(name string, id string) (bool, error) { // Delete the file itself { diff --git a/upup/pkg/fi/vfs_castore_test.go b/upup/pkg/fi/vfs_castore_test.go index 93d6ec00a0..61f07b89d4 100644 --- a/upup/pkg/fi/vfs_castore_test.go +++ b/upup/pkg/fi/vfs_castore_test.go @@ -24,7 +24,6 @@ import ( "testing" "time" - "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/pki" "k8s.io/kops/util/pkg/vfs" ) @@ -71,7 +70,18 @@ func TestVFSCAStoreRoundTrip(t *testing.T) { t.Fatalf("error from ParsePEMCertificate: %v", err) } - if err := s.StoreKeypair("ca", cert, privateKey); err != nil { + item := &KeysetItem{ + Id: "237054359138908419352140518924933177492", + Certificate: cert, + PrivateKey: privateKey, + } + keyset := &Keyset{ + Items: map[string]*KeysetItem{ + "237054359138908419352140518924933177492": item, + }, + Primary: item, + } + if err := s.StoreKeypair("ca", keyset); err != nil { t.Fatalf("error from StoreKeypair: %v", err) } @@ -87,16 +97,14 @@ func TestVFSCAStoreRoundTrip(t *testing.T) { for _, p := range []string{ "memfs://tests/issued/ca/keyset.yaml", - "memfs://tests/issued/ca/237054359138908419352140518924933177492.crt", "memfs://tests/private/ca/keyset.yaml", - "memfs://tests/private/ca/237054359138908419352140518924933177492.key", } { if _, found := pathMap[p]; !found { t.Fatalf("file not found: %v", p) } } - if len(pathMap) != 4 { + if len(pathMap) != 2 { t.Fatalf("unexpected pathMap: %v", pathMap) } @@ -148,18 +156,6 @@ spec: } } - // Check issued/ca/237054359138908419352140518924933177492.crt round-tripped - { - roundTrip, err := pathMap["memfs://tests/issued/ca/237054359138908419352140518924933177492.crt"].ReadFile() - if err != nil { - t.Fatalf("error reading file memfs://tests/issued/ca/237054359138908419352140518924933177492.crt: %v", err) - } - - if string(roundTrip) != certData { - t.Fatalf("unexpected round-tripped certificate data: %q", string(roundTrip)) - } - } - // Check private/ca/keyset.yaml round-tripped { privateKeysetYaml, err := pathMap["memfs://tests/private/ca/keyset.yaml"].ReadFile() @@ -204,34 +200,6 @@ spec: t.Fatalf("unexpected round-tripped private key data: %q", roundTrip) } } - - // Check private/ca/237054359138908419352140518924933177492.key round-tripped - { - roundTrip, err := pathMap["memfs://tests/private/ca/237054359138908419352140518924933177492.key"].ReadFile() - if err != nil { - t.Fatalf("error reading file memfs://tests/private/ca/237054359138908419352140518924933177492.key: %v", err) - } - - if string(roundTrip) != privateKeyData { - t.Fatalf("unexpected round-tripped private key data: %q", string(roundTrip)) - } - } - - // Check that keyset gets deleted - { - keyset := &kops.Keyset{} - keyset.Name = "ca" - keyset.Spec.Type = kops.SecretTypeKeypair - - s.DeleteKeysetItem(keyset, "237054359138908419352140518924933177492") - - _, err := pathMap["memfs://tests/private/ca/237054359138908419352140518924933177492.key"].ReadFile() - if err == nil { - t.Fatalf("File memfs://tests/private/ca/237054359138908419352140518924933177492.key still exists") - } - - } - } func TestVFSCAStoreRoundTripWithVault(t *testing.T) { @@ -265,7 +233,18 @@ func TestVFSCAStoreRoundTripWithVault(t *testing.T) { t.Fatalf("error from ParsePEMCertificate: %v", err) } - if err := s.StoreKeypair("ca", cert, privateKey); err != nil { + item := &KeysetItem{ + Id: "237054359138908419352140518924933177492", + Certificate: cert, + PrivateKey: privateKey, + } + keyset := &Keyset{ + Items: map[string]*KeysetItem{ + "237054359138908419352140518924933177492": item, + }, + Primary: item, + } + if err := s.StoreKeypair("ca", keyset); err != nil { t.Fatalf("error from StoreKeypair: %v", err) } @@ -283,9 +262,7 @@ func TestVFSCAStoreRoundTripWithVault(t *testing.T) { for _, p := range []string{ bp + "/issued/ca/keyset.yaml", - bp + "/issued/ca/237054359138908419352140518924933177492.crt", bp + "/private/ca/keyset.yaml", - bp + "/private/ca/237054359138908419352140518924933177492.key", } { if _, found := pathMap[p]; !found { t.Fatalf("file not found: %v", p) @@ -343,18 +320,6 @@ spec: } } - // Check issued/ca/237054359138908419352140518924933177492.crt round-tripped - { - roundTrip, err := pathMap[bp+"/issued/ca/237054359138908419352140518924933177492.crt"].ReadFile() - if err != nil { - t.Fatalf("error reading file issued/ca/237054359138908419352140518924933177492.crt: %v", err) - } - - if string(roundTrip) != certData { - t.Fatalf("unexpected round-tripped certificate data: %q", string(roundTrip)) - } - } - // Check private/ca/keyset.yaml round-tripped { privateKeysetYaml, err := pathMap[bp+"/private/ca/keyset.yaml"].ReadFile() @@ -398,32 +363,4 @@ spec: t.Fatalf("unexpected round-tripped private key data: %q", roundTrip) } } - - // Check private/ca/237054359138908419352140518924933177492.key round-tripped - { - roundTrip, err := pathMap[bp+"/private/ca/237054359138908419352140518924933177492.key"].ReadFile() - if err != nil { - t.Fatalf("error reading file private/ca/237054359138908419352140518924933177492.key: %v", err) - } - - if string(roundTrip) != privateKeyData { - t.Fatalf("unexpected round-tripped private key data: %q", string(roundTrip)) - } - } - - // Check that keyset gets deleted - { - keyset := &kops.Keyset{} - keyset.Name = "ca" - keyset.Spec.Type = kops.SecretTypeKeypair - - s.DeleteKeysetItem(keyset, "237054359138908419352140518924933177492") - - _, err := pathMap[bp+"/private/ca/237054359138908419352140518924933177492.key"].ReadFile() - if err == nil { - t.Fatalf("File private/ca/237054359138908419352140518924933177492.key still exists") - } - - } - } diff --git a/upup/pkg/kutil/convert_kubeup_cluster.go b/upup/pkg/kutil/convert_kubeup_cluster.go index 550a621cc9..2d66731f1a 100644 --- a/upup/pkg/kutil/convert_kubeup_cluster.go +++ b/upup/pkg/kutil/convert_kubeup_cluster.go @@ -487,17 +487,19 @@ func (x *ConvertKubeupCluster) Upgrade(ctx context.Context) error { if oldCACertPool == nil { return fmt.Errorf("cannot find certificate pool %q", fi.CertificateIDCA) } + keyset, err := newKeyStore.FindKeyset(fi.CertificateIDCA) + if err != nil { + return fmt.Errorf("error reading new CA certs: %v", err) + } for _, ca := range oldCACertPool.Secondary { - err := newKeyStore.AddCert(fi.CertificateIDCA, ca) - if err != nil { - return fmt.Errorf("error importing old CA certs: %v", err) - } + fi.AddCert(keyset, ca) } if oldCACertPool.Primary != nil { - err := newKeyStore.AddCert(fi.CertificateIDCA, oldCACertPool.Primary) - if err != nil { - return fmt.Errorf("error importing old CA certs: %v", err) - } + fi.AddCert(keyset, oldCACertPool.Primary) + } + err = newKeyStore.StoreKeypair(fi.CertificateIDCA, keyset) + if err != nil { + return fmt.Errorf("error importing old CA certs: %v", err) } return nil diff --git a/upup/pkg/kutil/import_cluster.go b/upup/pkg/kutil/import_cluster.go index e0f68d9b4c..cd2417e6d3 100644 --- a/upup/pkg/kutil/import_cluster.go +++ b/upup/pkg/kutil/import_cluster.go @@ -424,7 +424,12 @@ func (x *ImportCluster) ImportAWSCluster(ctx context.Context) error { if err != nil { return err } - err = keyStore.AddCert(fi.CertificateIDCA, caCert) + keyset, err := keyStore.FindKeyset(fi.CertificateIDCA) + if err != nil { + return err + } + fi.AddCert(keyset, caCert) + err = keyStore.StoreKeypair(fi.CertificateIDCA, keyset) if err != nil { return err } From 3127dacc0ccfd5f1dcfcf6646609b0d8e0f96c2c Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Sun, 11 Apr 2021 21:27:21 -0700 Subject: [PATCH 08/15] Expose all service-account keys through OIDC --- pkg/model/bootstrapscript.go | 3 ++- pkg/model/issuerdiscovery.go | 42 +++++++++++++++------------------- upup/pkg/fi/fitasks/keypair.go | 31 ++++++++++--------------- 3 files changed, 32 insertions(+), 44 deletions(-) diff --git a/pkg/model/bootstrapscript.go b/pkg/model/bootstrapscript.go index 11da889e81..c5170aaa62 100644 --- a/pkg/model/bootstrapscript.go +++ b/pkg/model/bootstrapscript.go @@ -219,7 +219,8 @@ func (b *BootstrapScriptBuilder) ResourceNodeUp(c *fi.ModelBuilderContext, ig *k ig: ig, builder: b, caTask: caTask, - ca: caTask.Certificate(), + // TODO: use caTask.Keyset() and expose all CA certificates + ca: caTask.Certificate(), } task.resource.Task = task c.AddTask(task) diff --git a/pkg/model/issuerdiscovery.go b/pkg/model/issuerdiscovery.go index cb84f67a73..dc56c5655c 100644 --- a/pkg/model/issuerdiscovery.go +++ b/pkg/model/issuerdiscovery.go @@ -22,9 +22,9 @@ import ( "crypto/x509" "encoding/base64" "encoding/json" - "encoding/pem" "fmt" "io" + "sort" "gopkg.in/square/go-jose.v2" "k8s.io/kops/pkg/apis/kops" @@ -128,38 +128,32 @@ func (o *OIDCKeys) GetDependencies(tasks map[string]fi.Task) []fi.Task { } } func (o *OIDCKeys) Open() (io.Reader, error) { + keyset := o.SigningKey.Keyset() + var keys []jose.JSONWebKey - certBytes, err := fi.ResourceAsBytes(o.SigningKey.Certificate()) - if err != nil { - return nil, fmt.Errorf("failed to get cert: %w", err) - } - block, _ := pem.Decode(certBytes) - cert, err := x509.ParseCertificate(block.Bytes) - if err != nil { - return nil, fmt.Errorf("failed to parse cert: %w", err) - } + for _, item := range keyset.Items { + publicKey := item.Certificate.PublicKey + publicKeyDERBytes, err := x509.MarshalPKIXPublicKey(publicKey) + if err != nil { + return nil, fmt.Errorf("failed to serialize public key to DER format: %v", err) + } - publicKey := cert.PublicKey + hasher := crypto.SHA256.New() + hasher.Write(publicKeyDERBytes) + publicKeyDERHash := hasher.Sum(nil) - publicKeyDERBytes, err := x509.MarshalPKIXPublicKey(publicKey) - if err != nil { - return nil, fmt.Errorf("failed to serialize public key to DER format: %v", err) - } + keyID := base64.RawURLEncoding.EncodeToString(publicKeyDERHash) - hasher := crypto.SHA256.New() - hasher.Write(publicKeyDERBytes) - publicKeyDERHash := hasher.Sum(nil) - - keyID := base64.RawURLEncoding.EncodeToString(publicKeyDERHash) - - keys := []jose.JSONWebKey{ - { + keys = append(keys, jose.JSONWebKey{ Key: publicKey, KeyID: keyID, Algorithm: string(jose.RS256), Use: "sig", - }, + }) } + sort.Slice(keys, func(i, j int) bool { + return keys[i].KeyID < keys[j].KeyID + }) keyResponse := KeyResponse{Keys: keys} jsonBytes, err := json.MarshalIndent(keyResponse, "", "") diff --git a/upup/pkg/fi/fitasks/keypair.go b/upup/pkg/fi/fitasks/keypair.go index a7efd720ed..050e0eeffb 100644 --- a/upup/pkg/fi/fitasks/keypair.go +++ b/upup/pkg/fi/fitasks/keypair.go @@ -17,7 +17,6 @@ limitations under the License. package fitasks import ( - "crypto/sha1" "crypto/x509/pkix" "fmt" "sort" @@ -46,8 +45,8 @@ type Keypair struct { // LegacyFormat is whether the keypair is stored in a legacy format. LegacyFormat bool `json:"oldFormat"` - certificate *fi.TaskDependentResource - certificateSHA1Fingerprint *fi.TaskDependentResource + certificate *fi.TaskDependentResource + keyset *fi.Keyset } var _ fi.HasCheckExisting = &Keypair{} @@ -103,7 +102,7 @@ func (e *Keypair) Find(c *fi.Context) (*Keypair, error) { // Avoid spurious changes actual.Lifecycle = e.Lifecycle - if err := e.setResources(cert); err != nil { + if err := e.setResources(keyset); err != nil { return nil, fmt.Errorf("error setting resources: %v", err) } @@ -233,18 +232,19 @@ func (_ *Keypair) Render(c *fi.Context, a, e, changes *Keypair) error { PrivateKey: privateKey, } - err = c.Keystore.StoreKeypair(name, &fi.Keyset{ + keyset = &fi.Keyset{ LegacyFormat: false, Items: map[string]*fi.KeysetItem{ serialString: ki, }, Primary: ki, - }) + } + err = c.Keystore.StoreKeypair(name, keyset) if err != nil { return err } - if err := e.setResources(cert); err != nil { + if err := e.setResources(keyset); err != nil { return fmt.Errorf("error setting resources: %v", err) } @@ -308,30 +308,23 @@ func (e *Keypair) ensureResources() { if e.certificate == nil { e.certificate = &fi.TaskDependentResource{Task: e} } - if e.certificateSHA1Fingerprint == nil { - e.certificateSHA1Fingerprint = &fi.TaskDependentResource{Task: e} - } } -func (e *Keypair) setResources(cert *pki.Certificate) error { +func (e *Keypair) setResources(keyset *fi.Keyset) error { e.ensureResources() - s, err := cert.AsString() + s, err := keyset.Primary.Certificate.AsString() if err != nil { return err } e.certificate.Resource = fi.NewStringResource(s) - fingerprint := sha1.Sum(cert.Certificate.Raw) - hex := fmt.Sprintf("%x", fingerprint) - e.certificateSHA1Fingerprint.Resource = fi.NewStringResource(hex) - + e.keyset = keyset return nil } -func (e *Keypair) CertificateSHA1Fingerprint() fi.Resource { - e.ensureResources() - return e.certificateSHA1Fingerprint +func (e *Keypair) Keyset() *fi.Keyset { + return e.keyset } func (e *Keypair) Certificate() fi.Resource { From 2a431c03a937d9e3921d4d1202364873d710f3fb Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Mon, 3 May 2021 20:55:01 -0700 Subject: [PATCH 09/15] Improve description of PrimaryId --- k8s/crds/kops.k8s.io_keysets.yaml | 2 +- pkg/apis/kops/keyset.go | 2 +- pkg/apis/kops/v1alpha2/keyset.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/k8s/crds/kops.k8s.io_keysets.yaml b/k8s/crds/kops.k8s.io_keysets.yaml index d21d2a32b2..9329eac8d2 100644 --- a/k8s/crds/kops.k8s.io_keysets.yaml +++ b/k8s/crds/kops.k8s.io_keysets.yaml @@ -60,7 +60,7 @@ spec: type: object type: array primaryId: - description: PrimaryId is the id of the key used to mint new things. + description: PrimaryId is the id of the key used to make new signatures. type: string type: description: Type is the type of the Keyset (PKI keypair, or secret diff --git a/pkg/apis/kops/keyset.go b/pkg/apis/kops/keyset.go index 61f6d49379..db05c68dfd 100644 --- a/pkg/apis/kops/keyset.go +++ b/pkg/apis/kops/keyset.go @@ -67,7 +67,7 @@ type KeysetSpec struct { // Type is the type of the Keyset (PKI keypair, or secret token) Type KeysetType `json:"type,omitempty"` - // PrimaryId is the id of the key used to mint new things. + // PrimaryId is the id of the key used to make new signatures. PrimaryId string `json:"primaryId,omitempty"` // Keys is the set of keys that make up the keyset diff --git a/pkg/apis/kops/v1alpha2/keyset.go b/pkg/apis/kops/v1alpha2/keyset.go index 942ed5117e..05069e7256 100644 --- a/pkg/apis/kops/v1alpha2/keyset.go +++ b/pkg/apis/kops/v1alpha2/keyset.go @@ -68,7 +68,7 @@ type KeysetSpec struct { // Type is the type of the Keyset (PKI keypair, or secret token) Type KeysetType `json:"type,omitempty"` - // PrimaryId is the id of the key used to mint new things. + // PrimaryId is the id of the key used to make new signatures. PrimaryId string `json:"primaryId,omitempty"` // Keys is the set of keys that make up the keyset From 2300d895911171b2f2abc43db2cbbdab6914a85d Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Mon, 3 May 2021 20:59:54 -0700 Subject: [PATCH 10/15] Rename pki.FindKeypair to FindPrimaryKeypair --- cmd/kops-controller/pkg/server/keystore.go | 2 +- cmd/kops-controller/pkg/server/node_config.go | 2 +- nodeup/pkg/model/fakes_test.go | 4 ++-- pkg/configserver/keystore.go | 6 +++--- pkg/kubeconfig/create_kubecfg_test.go | 4 ++-- pkg/pki/issue.go | 6 +++--- pkg/pki/issue_test.go | 2 +- upup/pkg/fi/ca.go | 4 ++-- upup/pkg/fi/clientset_castore.go | 6 +++--- upup/pkg/fi/vfs_castore.go | 4 ++-- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/cmd/kops-controller/pkg/server/keystore.go b/cmd/kops-controller/pkg/server/keystore.go index 44ef8b5b86..baa45cdeb0 100644 --- a/cmd/kops-controller/pkg/server/keystore.go +++ b/cmd/kops-controller/pkg/server/keystore.go @@ -35,7 +35,7 @@ type keystoreEntry struct { var _ pki.Keystore = keystore{} -func (k keystore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { +func (k keystore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { entry, ok := k.keys[name] if !ok { return nil, nil, fmt.Errorf("unknown CA %q", name) diff --git a/cmd/kops-controller/pkg/server/node_config.go b/cmd/kops-controller/pkg/server/node_config.go index baf885f394..94011bc718 100644 --- a/cmd/kops-controller/pkg/server/node_config.go +++ b/cmd/kops-controller/pkg/server/node_config.go @@ -63,7 +63,7 @@ func (s *Server) getNodeConfig(ctx context.Context, req *nodeup.BootstrapRequest // We populate some certificates that we know the node will need. for _, name := range []string{"ca"} { - cert, _, err := s.keystore.FindKeypair(name) + cert, _, err := s.keystore.FindPrimaryKeypair(name) if err != nil { return nil, fmt.Errorf("error getting certificate %q: %w", name, err) } diff --git a/nodeup/pkg/model/fakes_test.go b/nodeup/pkg/model/fakes_test.go index 9dad8510e4..fa2b63e878 100644 --- a/nodeup/pkg/model/fakes_test.go +++ b/nodeup/pkg/model/fakes_test.go @@ -33,8 +33,8 @@ type fakeKeyStore struct { var _ fi.Keystore = &fakeKeyStore{} -func (k fakeKeyStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { - panic("fakeKeyStore does not implement FindKeypair") +func (k fakeKeyStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { + panic("fakeKeyStore does not implement FindPrimaryKeypair") } func (k fakeKeyStore) FindKeyset(name string) (*fi.Keyset, error) { diff --git a/pkg/configserver/keystore.go b/pkg/configserver/keystore.go index d46c905fd6..2f7acc1783 100644 --- a/pkg/configserver/keystore.go +++ b/pkg/configserver/keystore.go @@ -38,9 +38,9 @@ func NewKeyStore(nodeConfig *nodeup.NodeConfig) fi.CAStore { } } -// FindKeypair implements pki.Keystore -func (s *configserverKeyStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { - return nil, nil, fmt.Errorf("FindKeypair %q not supported by configserverKeyStore", name) +// FindPrimaryKeypair implements pki.Keystore +func (s *configserverKeyStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { + return nil, nil, fmt.Errorf("FindPrimaryKeypair %q not supported by configserverKeyStore", name) } // FindKeyset implements fi.Keystore diff --git a/pkg/kubeconfig/create_kubecfg_test.go b/pkg/kubeconfig/create_kubecfg_test.go index 7c7e6c0539..fb961875d3 100644 --- a/pkg/kubeconfig/create_kubecfg_test.go +++ b/pkg/kubeconfig/create_kubecfg_test.go @@ -94,8 +94,8 @@ type fakeKeyStore struct { MirrorToFn func(basedir vfs.Path) error } -func (f fakeKeyStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { - return fi.FindKeypair(f, name) +func (f fakeKeyStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { + return fi.FindPrimaryKeypair(f, name) } func (f fakeKeyStore) FindKeyset(name string) (*fi.Keyset, error) { diff --git a/pkg/pki/issue.go b/pkg/pki/issue.go index 4c7b09b066..df83d568e3 100644 --- a/pkg/pki/issue.go +++ b/pkg/pki/issue.go @@ -56,9 +56,9 @@ type IssueCertRequest struct { } type Keystore interface { - // FindKeypair finds a cert & private key, returning nil where either is not found + // FindPrimaryKeypair finds a cert & private key, returning nil where either is not found // (if the certificate is found but not keypair, that is not an error: only the cert will be returned). - FindKeypair(name string) (*Certificate, *PrivateKey, error) + FindPrimaryKeypair(name string) (*Certificate, *PrivateKey, error) } // IssueCert issues a certificate, either a self-signed CA or from a CA in a keystore. @@ -116,7 +116,7 @@ func IssueCert(request *IssueCertRequest, keystore Keystore) (issuedCertificate var signer *x509.Certificate if !template.IsCA { var err error - caCertificate, caPrivateKey, err = keystore.FindKeypair(request.Signer) + caCertificate, caPrivateKey, err = keystore.FindPrimaryKeypair(request.Signer) if err != nil { return nil, nil, nil, err } diff --git a/pkg/pki/issue_test.go b/pkg/pki/issue_test.go index 8ebe7abf05..d2ad1860d7 100644 --- a/pkg/pki/issue_test.go +++ b/pkg/pki/issue_test.go @@ -38,7 +38,7 @@ type mockKeystore struct { invoked bool } -func (m *mockKeystore) FindKeypair(name string) (*Certificate, *PrivateKey, error) { +func (m *mockKeystore) FindPrimaryKeypair(name string) (*Certificate, *PrivateKey, error) { assert.False(m.t, m.invoked, "invoked already") m.invoked = true assert.Equal(m.t, m.signer, name, "name argument") diff --git a/upup/pkg/fi/ca.go b/upup/pkg/fi/ca.go index fa853e4c4c..95dd910bc8 100644 --- a/upup/pkg/fi/ca.go +++ b/upup/pkg/fi/ca.go @@ -157,8 +157,8 @@ func (c *CertificatePool) AsString() (string, error) { return data.String(), nil } -// FindKeypair is a common implementation of pki.FindKeypair. -func FindKeypair(c Keystore, name string) (*pki.Certificate, *pki.PrivateKey, error) { +// FindPrimaryKeypair is a common implementation of pki.FindPrimaryKeypair. +func FindPrimaryKeypair(c Keystore, name string) (*pki.Certificate, *pki.PrivateKey, error) { keyset, err := c.FindKeyset(name) if err != nil { return nil, nil, err diff --git a/upup/pkg/fi/clientset_castore.go b/upup/pkg/fi/clientset_castore.go index bcbf792635..3379dbd6f6 100644 --- a/upup/pkg/fi/clientset_castore.go +++ b/upup/pkg/fi/clientset_castore.go @@ -145,9 +145,9 @@ func FindPrimary(keyset *kops.Keyset) *kops.KeysetItem { return primary } -// FindKeypair implements PKI::FindKeypair -func (c *ClientsetCAStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { - return FindKeypair(c, name) +// FindPrimaryKeypair implements PKI::FindPrimaryKeypair +func (c *ClientsetCAStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { + return FindPrimaryKeypair(c, name) } // FindKeyset implements CAStore::FindKeyset diff --git a/upup/pkg/fi/vfs_castore.go b/upup/pkg/fi/vfs_castore.go index 1ad94ee871..5a47aef184 100644 --- a/upup/pkg/fi/vfs_castore.go +++ b/upup/pkg/fi/vfs_castore.go @@ -216,8 +216,8 @@ func removePrivateKeyMaterial(o *kops.Keyset) *kops.Keyset { return c } -func (c *VFSCAStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { - return FindKeypair(c, name) +func (c *VFSCAStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { + return FindPrimaryKeypair(c, name) } func (c *VFSCAStore) FindKeyset(id string) (*Keyset, error) { From fa77f8b964c5f8a2953a9880d3c54707b5a75327 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Mon, 3 May 2021 21:12:27 -0700 Subject: [PATCH 11/15] Rename fi.Keystore.StoreKeypair to StoreKeyset --- cmd/kops/create_secret_keypair_ca.go | 2 +- nodeup/pkg/model/fakes_test.go | 4 ++-- pkg/configserver/keystore.go | 6 +++--- pkg/kubeconfig/create_kubecfg_test.go | 8 ++++---- upup/pkg/fi/ca.go | 4 ++-- upup/pkg/fi/clientset_castore.go | 4 ++-- upup/pkg/fi/fitasks/keypair.go | 4 ++-- upup/pkg/fi/vfs_castore.go | 2 +- upup/pkg/fi/vfs_castore_test.go | 8 ++++---- upup/pkg/kutil/convert_kubeup_cluster.go | 2 +- upup/pkg/kutil/import_cluster.go | 2 +- 11 files changed, 23 insertions(+), 23 deletions(-) diff --git a/cmd/kops/create_secret_keypair_ca.go b/cmd/kops/create_secret_keypair_ca.go index 1646dcea13..077067744d 100644 --- a/cmd/kops/create_secret_keypair_ca.go +++ b/cmd/kops/create_secret_keypair_ca.go @@ -140,7 +140,7 @@ func RunCreateSecretCaCert(ctx context.Context, f *util.Factory, out io.Writer, PrivateKey: privateKey, } - err = keyStore.StoreKeypair(fi.CertificateIDCA, &fi.Keyset{ + err = keyStore.StoreKeyset(fi.CertificateIDCA, &fi.Keyset{ LegacyFormat: false, Items: map[string]*fi.KeysetItem{ serialString: ki, diff --git a/nodeup/pkg/model/fakes_test.go b/nodeup/pkg/model/fakes_test.go index fa2b63e878..90f075d4a0 100644 --- a/nodeup/pkg/model/fakes_test.go +++ b/nodeup/pkg/model/fakes_test.go @@ -45,8 +45,8 @@ func (k fakeKeyStore) CreateKeypair(signer string, name string, template *x509.C panic("fakeKeyStore does not implement CreateKeypair") } -func (k fakeKeyStore) StoreKeypair(id string, keyset *fi.Keyset) error { - panic("fakeKeyStore does not implement StoreKeypair") +func (k fakeKeyStore) StoreKeyset(name string, keyset *fi.Keyset) error { + panic("fakeKeyStore does not implement StoreKeyset") } func (k fakeKeyStore) MirrorTo(basedir vfs.Path) error { diff --git a/pkg/configserver/keystore.go b/pkg/configserver/keystore.go index 2f7acc1783..e6e1d7ba37 100644 --- a/pkg/configserver/keystore.go +++ b/pkg/configserver/keystore.go @@ -53,9 +53,9 @@ func (s *configserverKeyStore) CreateKeypair(signer string, name string, templat return nil, fmt.Errorf("CreateKeypair not supported by configserverKeyStore") } -// StoreKeypair implements fi.Keystore -func (s *configserverKeyStore) StoreKeypair(id string, keyset *fi.Keyset) error { - return fmt.Errorf("StoreKeypair not supported by configserverKeyStore") +// StoreKeyset implements fi.Keystore +func (s *configserverKeyStore) StoreKeyset(name string, keyset *fi.Keyset) error { + return fmt.Errorf("StoreKeyset not supported by configserverKeyStore") } // MirrorTo implements fi.Keystore diff --git a/pkg/kubeconfig/create_kubecfg_test.go b/pkg/kubeconfig/create_kubecfg_test.go index fb961875d3..c367101217 100644 --- a/pkg/kubeconfig/create_kubecfg_test.go +++ b/pkg/kubeconfig/create_kubecfg_test.go @@ -87,8 +87,8 @@ func (f fakeStatusCloud) FindClusterStatus(cluster *kops.Cluster) (*kops.Cluster type fakeKeyStore struct { FindKeysetFn func(name string) (*fi.Keyset, error) - // StoreKeypair writes the keypair to the store. - StoreKeypairFn func(id string, keyset *fi.Keyset) error + // StoreKeysetFn writes the keyset to the store. + StoreKeysetFn func(name string, keyset *fi.Keyset) error // MirrorTo will copy secrets to a vfs.Path, which is often easier for a machine to read MirrorToFn func(basedir vfs.Path) error @@ -102,8 +102,8 @@ func (f fakeKeyStore) FindKeyset(name string) (*fi.Keyset, error) { return f.FindKeysetFn(name) } -func (f fakeKeyStore) StoreKeypair(id string, keyset *fi.Keyset) error { - return f.StoreKeypairFn(id, keyset) +func (f fakeKeyStore) StoreKeyset(name string, keyset *fi.Keyset) error { + return f.StoreKeysetFn(name, keyset) } func (f fakeKeyStore) MirrorTo(basedir vfs.Path) error { diff --git a/upup/pkg/fi/ca.go b/upup/pkg/fi/ca.go index 95dd910bc8..22aad02e2c 100644 --- a/upup/pkg/fi/ca.go +++ b/upup/pkg/fi/ca.go @@ -66,8 +66,8 @@ type Keystore interface { // FindKeyset finds a Keyset. FindKeyset(name string) (*Keyset, error) - // StoreKeypair writes the keyset to the store. - StoreKeypair(id string, keyset *Keyset) error + // StoreKeyset writes a Keyset to the store. + StoreKeyset(name string, keyset *Keyset) error // MirrorTo will copy secrets to a vfs.Path, which is often easier for a machine to read MirrorTo(basedir vfs.Path) error diff --git a/upup/pkg/fi/clientset_castore.go b/upup/pkg/fi/clientset_castore.go index 3379dbd6f6..963b1ec2cb 100644 --- a/upup/pkg/fi/clientset_castore.go +++ b/upup/pkg/fi/clientset_castore.go @@ -257,8 +257,8 @@ func (c *ClientsetCAStore) ListSSHCredentials() ([]*kops.SSHCredential, error) { return items, nil } -// StoreKeypair implements CAStore::StoreKeypair -func (c *ClientsetCAStore) StoreKeypair(name string, keyset *Keyset) error { +// StoreKeyset implements CAStore::StoreKeyset +func (c *ClientsetCAStore) StoreKeyset(name string, keyset *Keyset) error { ctx := context.TODO() return c.storeKeyset(ctx, name, keyset, kops.SecretTypeKeypair) } diff --git a/upup/pkg/fi/fitasks/keypair.go b/upup/pkg/fi/fitasks/keypair.go index 050e0eeffb..469b147a80 100644 --- a/upup/pkg/fi/fitasks/keypair.go +++ b/upup/pkg/fi/fitasks/keypair.go @@ -239,7 +239,7 @@ func (_ *Keypair) Render(c *fi.Context, a, e, changes *Keypair) error { }, Primary: ki, } - err = c.Keystore.StoreKeypair(name, keyset) + err = c.Keystore.StoreKeyset(name, keyset) if err != nil { return err } @@ -267,7 +267,7 @@ func (_ *Keypair) Render(c *fi.Context, a, e, changes *Keypair) error { return err } keyset.LegacyFormat = false - err = c.Keystore.StoreKeypair(name, keyset) + err = c.Keystore.StoreKeyset(name, keyset) if err != nil { return err } diff --git a/upup/pkg/fi/vfs_castore.go b/upup/pkg/fi/vfs_castore.go index 5a47aef184..8aa7d927a9 100644 --- a/upup/pkg/fi/vfs_castore.go +++ b/upup/pkg/fi/vfs_castore.go @@ -520,7 +520,7 @@ func mirrorSSHCredential(cluster *kops.Cluster, basedir vfs.Path, sshCredential return nil } -func (c *VFSCAStore) StoreKeypair(name string, keyset *Keyset) error { +func (c *VFSCAStore) StoreKeyset(name string, keyset *Keyset) error { { p := c.buildPrivateKeyPoolPath(name) if err := c.writeKeysetBundle(p, name, keyset, true); err != nil { diff --git a/upup/pkg/fi/vfs_castore_test.go b/upup/pkg/fi/vfs_castore_test.go index 61f07b89d4..87da24ff28 100644 --- a/upup/pkg/fi/vfs_castore_test.go +++ b/upup/pkg/fi/vfs_castore_test.go @@ -81,8 +81,8 @@ func TestVFSCAStoreRoundTrip(t *testing.T) { }, Primary: item, } - if err := s.StoreKeypair("ca", keyset); err != nil { - t.Fatalf("error from StoreKeypair: %v", err) + if err := s.StoreKeyset("ca", keyset); err != nil { + t.Fatalf("error from StoreKeyset: %v", err) } paths, err := basePath.ReadTree() @@ -244,8 +244,8 @@ func TestVFSCAStoreRoundTripWithVault(t *testing.T) { }, Primary: item, } - if err := s.StoreKeypair("ca", keyset); err != nil { - t.Fatalf("error from StoreKeypair: %v", err) + if err := s.StoreKeyset("ca", keyset); err != nil { + t.Fatalf("error from StoreKeyset: %v", err) } paths, err := basePath.ReadTree() diff --git a/upup/pkg/kutil/convert_kubeup_cluster.go b/upup/pkg/kutil/convert_kubeup_cluster.go index 2d66731f1a..4ca57a169f 100644 --- a/upup/pkg/kutil/convert_kubeup_cluster.go +++ b/upup/pkg/kutil/convert_kubeup_cluster.go @@ -497,7 +497,7 @@ func (x *ConvertKubeupCluster) Upgrade(ctx context.Context) error { if oldCACertPool.Primary != nil { fi.AddCert(keyset, oldCACertPool.Primary) } - err = newKeyStore.StoreKeypair(fi.CertificateIDCA, keyset) + err = newKeyStore.StoreKeyset(fi.CertificateIDCA, keyset) if err != nil { return fmt.Errorf("error importing old CA certs: %v", err) } diff --git a/upup/pkg/kutil/import_cluster.go b/upup/pkg/kutil/import_cluster.go index cd2417e6d3..08bd9d105c 100644 --- a/upup/pkg/kutil/import_cluster.go +++ b/upup/pkg/kutil/import_cluster.go @@ -429,7 +429,7 @@ func (x *ImportCluster) ImportAWSCluster(ctx context.Context) error { return err } fi.AddCert(keyset, caCert) - err = keyStore.StoreKeypair(fi.CertificateIDCA, keyset) + err = keyStore.StoreKeyset(fi.CertificateIDCA, keyset) if err != nil { return err } From 15319ae43282a948a2e318a6511e288bd3985e1b Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Mon, 3 May 2021 22:27:46 -0700 Subject: [PATCH 12/15] Make serialization of keyset items stable --- upup/pkg/fi/ca.go | 18 ++++++++++++++++++ upup/pkg/fi/clientset_castore.go | 13 ++++++++++++- upup/pkg/fi/vfs_castore.go | 12 +++++++++++- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/upup/pkg/fi/ca.go b/upup/pkg/fi/ca.go index 22aad02e2c..7eb5703d9e 100644 --- a/upup/pkg/fi/ca.go +++ b/upup/pkg/fi/ca.go @@ -19,6 +19,7 @@ package fi import ( "bytes" "fmt" + "math/big" "strconv" "k8s.io/kops/pkg/apis/kops" @@ -182,3 +183,20 @@ func AddCert(keyset *Keyset, cert *pki.Certificate) { Certificate: cert, } } + +// KeysetItemIdOlder returns whether the KeysetItem Id a is older than b. +func KeysetItemIdOlder(a, b string) bool { + aVersion, aOk := big.NewInt(0).SetString(a, 10) + bVersion, bOk := big.NewInt(0).SetString(b, 10) + if aOk { + if !bOk { + return false + } + return aVersion.Cmp(bVersion) < 0 + } else { + if bOk { + return true + } + return a < b + } +} diff --git a/upup/pkg/fi/clientset_castore.go b/upup/pkg/fi/clientset_castore.go index 963b1ec2cb..859234aec5 100644 --- a/upup/pkg/fi/clientset_castore.go +++ b/upup/pkg/fi/clientset_castore.go @@ -21,6 +21,7 @@ import ( "context" "fmt" "math/big" + "sort" "golang.org/x/crypto/ssh" "k8s.io/apimachinery/pkg/api/errors" @@ -312,7 +313,17 @@ func (c *ClientsetCAStore) storeKeyset(ctx context.Context, name string, keyset kopsKeyset.Spec.Keys = nil kopsKeyset.Spec.PrimaryId = keyset.Primary.Id - for _, item := range keyset.Items { + + keys := make([]string, 0, len(keyset.Items)) + for k := range keyset.Items { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + return KeysetItemIdOlder(keyset.Items[keys[i]].Id, keyset.Items[keys[j]].Id) + }) + + for _, key := range keys { + item := keyset.Items[key] var publicMaterial bytes.Buffer if _, err := item.Certificate.WriteTo(&publicMaterial); err != nil { return err diff --git a/upup/pkg/fi/vfs_castore.go b/upup/pkg/fi/vfs_castore.go index 8aa7d927a9..ba510c1db4 100644 --- a/upup/pkg/fi/vfs_castore.go +++ b/upup/pkg/fi/vfs_castore.go @@ -21,6 +21,7 @@ import ( "fmt" "math/big" "os" + "sort" "strings" "sync" @@ -138,7 +139,16 @@ func (k *Keyset) ToAPIObject(name string, includePrivateKeyMaterial bool) (*kops o.Name = name o.Spec.Type = kops.SecretTypeKeypair - for _, ki := range k.Items { + keys := make([]string, 0, len(k.Items)) + for k := range k.Items { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + return KeysetItemIdOlder(k.Items[keys[i]].Id, k.Items[keys[j]].Id) + }) + + for _, key := range keys { + ki := k.Items[key] oki := kops.KeysetItem{ Id: ki.Id, } From 7240c5bb8d5665939505190d69c80f79210eb491 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Mon, 3 May 2021 22:50:41 -0700 Subject: [PATCH 13/15] Preserve old keys/certs in Keypair.Render() --- upup/pkg/fi/fitasks/keypair.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/upup/pkg/fi/fitasks/keypair.go b/upup/pkg/fi/fitasks/keypair.go index 469b147a80..4240d67b44 100644 --- a/upup/pkg/fi/fitasks/keypair.go +++ b/upup/pkg/fi/fitasks/keypair.go @@ -180,7 +180,9 @@ func (_ *Keypair) Render(c *fi.Context, a, e, changes *Keypair) error { return err } if keyset == nil { - keyset = &fi.Keyset{} + keyset = &fi.Keyset{ + Items: map[string]*fi.KeysetItem{}, + } } // We always reuse the private key if it exists, @@ -232,13 +234,9 @@ func (_ *Keypair) Render(c *fi.Context, a, e, changes *Keypair) error { PrivateKey: privateKey, } - keyset = &fi.Keyset{ - LegacyFormat: false, - Items: map[string]*fi.KeysetItem{ - serialString: ki, - }, - Primary: ki, - } + keyset.LegacyFormat = false + keyset.Items[ki.Id] = ki + keyset.Primary = ki err = c.Keystore.StoreKeyset(name, keyset) if err != nil { return err From bceb901ce4a5aba52f39323c90b24964d106e226 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Mon, 3 May 2021 23:17:06 -0700 Subject: [PATCH 14/15] Fix merge error --- upup/pkg/fi/vfs_castore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upup/pkg/fi/vfs_castore.go b/upup/pkg/fi/vfs_castore.go index ba510c1db4..063af57805 100644 --- a/upup/pkg/fi/vfs_castore.go +++ b/upup/pkg/fi/vfs_castore.go @@ -233,7 +233,7 @@ func (c *VFSCAStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.Pri func (c *VFSCAStore) FindKeyset(id string) (*Keyset, error) { certs, err := c.loadKeyset(c.buildCertificatePoolPath(id)) - if (cert == nil || os.IsNotExist(err)) && id == "service-account" { + if (certs == nil || os.IsNotExist(err)) && id == "service-account" { // The strange name is because Kops prior to 1.19 used the api-server TLS key for this. id = "master" certs, err = c.loadKeyset(c.buildCertificatePoolPath(id)) From 12465ac27c6ad47d021935ee5e4da70e7fb42f64 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Tue, 4 May 2021 08:36:14 -0700 Subject: [PATCH 15/15] Simplify extraction of service-account public keys --- nodeup/pkg/model/BUILD.bazel | 1 - nodeup/pkg/model/fakes_test.go | 86 +++++++++++++++++++----------- nodeup/pkg/model/kube_apiserver.go | 26 ++------- nodeup/pkg/model/kubelet_test.go | 11 ++-- upup/pkg/fi/ca.go | 26 +++++++++ 5 files changed, 95 insertions(+), 55 deletions(-) diff --git a/nodeup/pkg/model/BUILD.bazel b/nodeup/pkg/model/BUILD.bazel index c592121064..36d2fd4541 100644 --- a/nodeup/pkg/model/BUILD.bazel +++ b/nodeup/pkg/model/BUILD.bazel @@ -53,7 +53,6 @@ go_library( "//pkg/kubemanifest:go_default_library", "//pkg/model/components:go_default_library", "//pkg/nodelabels:go_default_library", - "//pkg/pki:go_default_library", "//pkg/rbac:go_default_library", "//pkg/systemd:go_default_library", "//pkg/tokens:go_default_library", diff --git a/nodeup/pkg/model/fakes_test.go b/nodeup/pkg/model/fakes_test.go index 90f075d4a0..4c04240c5b 100644 --- a/nodeup/pkg/model/fakes_test.go +++ b/nodeup/pkg/model/fakes_test.go @@ -18,6 +18,7 @@ package model import ( "crypto/x509" + "fmt" "testing" "k8s.io/kops/pkg/apis/kops" @@ -26,44 +27,69 @@ import ( "k8s.io/kops/util/pkg/vfs" ) -// fakeKeyStore mocks out some of fi.KeyStore, for our tests. -type fakeKeyStore struct { - T *testing.T -} - -var _ fi.Keystore = &fakeKeyStore{} - -func (k fakeKeyStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { - panic("fakeKeyStore does not implement FindPrimaryKeypair") -} - -func (k fakeKeyStore) FindKeyset(name string) (*fi.Keyset, error) { - panic("fakeKeyStore does not implement FindKeyset") -} - -func (k fakeKeyStore) CreateKeypair(signer string, name string, template *x509.Certificate, privateKey *pki.PrivateKey) (*pki.Certificate, error) { - panic("fakeKeyStore does not implement CreateKeypair") -} - -func (k fakeKeyStore) StoreKeyset(name string, keyset *fi.Keyset) error { - panic("fakeKeyStore does not implement StoreKeyset") -} - -func (k fakeKeyStore) MirrorTo(basedir vfs.Path) error { - panic("fakeKeyStore does not implement MirrorTo") -} - // fakeCAStore mocks out some of fi.CAStore, for our tests. -// Although CAStore currently embeds KeyStore, we maintain the split here in the hope we can clean this up in future. type fakeCAStore struct { - fakeKeyStore - + T *testing.T privateKeysets map[string]*kops.Keyset certs map[string]*pki.Certificate } var _ fi.CAStore = &fakeCAStore{} +func (k fakeCAStore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { + panic("fakeCAStore does not implement FindPrimaryKeypair") +} + +func (k fakeCAStore) FindKeyset(name string) (*fi.Keyset, error) { + kopsKeyset := k.privateKeysets[name] + if kopsKeyset == nil { + return nil, nil + } + + keyset := &fi.Keyset{ + Items: make(map[string]*fi.KeysetItem), + } + + for _, key := range kopsKeyset.Spec.Keys { + ki := &fi.KeysetItem{ + Id: key.Id, + } + if len(key.PublicMaterial) != 0 { + cert, err := pki.ParsePEMCertificate(key.PublicMaterial) + if err != nil { + return nil, fmt.Errorf("error loading certificate %s/%s: %v", name, key.Id, err) + } + ki.Certificate = cert + } + + if len(key.PrivateMaterial) != 0 { + privateKey, err := pki.ParsePEMPrivateKey(key.PrivateMaterial) + if err != nil { + return nil, fmt.Errorf("error loading private key %s/%s: %v", name, key.Id, err) + } + ki.PrivateKey = privateKey + } + + keyset.Items[key.Id] = ki + } + + keyset.Primary = keyset.Items[fi.FindPrimary(kopsKeyset).Id] + + return keyset, nil +} + +func (k fakeCAStore) CreateKeypair(signer string, name string, template *x509.Certificate, privateKey *pki.PrivateKey) (*pki.Certificate, error) { + panic("fakeCAStore does not implement CreateKeypair") +} + +func (k fakeCAStore) StoreKeyset(name string, keyset *fi.Keyset) error { + panic("fakeCAStore does not implement StoreKeyset") +} + +func (k fakeCAStore) MirrorTo(basedir vfs.Path) error { + panic("fakeCAStore does not implement MirrorTo") +} + func (k fakeCAStore) FindCertificatePool(name string) (*fi.CertificatePool, error) { panic("fakeCAStore does not implement FindCertificatePool") } diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go index 1657970aa6..1b61d471b6 100644 --- a/nodeup/pkg/model/kube_apiserver.go +++ b/nodeup/pkg/model/kube_apiserver.go @@ -17,10 +17,6 @@ limitations under the License. package model import ( - "bytes" - "crypto/rsa" - "crypto/x509" - "encoding/pem" "fmt" "path/filepath" "strings" @@ -30,7 +26,6 @@ import ( "k8s.io/kops/pkg/k8scodecs" "k8s.io/kops/pkg/kubeconfig" "k8s.io/kops/pkg/kubemanifest" - "k8s.io/kops/pkg/pki" "k8s.io/kops/pkg/wellknownports" "k8s.io/kops/pkg/wellknownusers" "k8s.io/kops/upup/pkg/fi" @@ -87,7 +82,7 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error { } } { - keyset, err := b.KeyStore.FindPrivateKeyset("service-account") + keyset, err := b.KeyStore.FindKeyset("service-account") if err != nil { return err } @@ -96,25 +91,14 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error { return fmt.Errorf("service-account keyset not found") } - buf := new(bytes.Buffer) - for _, keyItem := range keyset.Spec.Keys { - privateKey, err := pki.ParsePEMPrivateKey(keyItem.PrivateMaterial) - if err != nil { - return fmt.Errorf("error loading service-account private key %s: %v", keyItem.Id, err) - } - pkData, err := x509.MarshalPKIXPublicKey(privateKey.Key.(*rsa.PrivateKey).Public()) - if err != nil { - return fmt.Errorf("marshalling public key: %v", err) - } - err = pem.Encode(buf, &pem.Block{Type: "RSA PUBLIC KEY", Bytes: pkData}) - if err != nil { - return fmt.Errorf("encoding public key: %v", err) - } + buf, err := keyset.ToPublicKeyBytes() + if err != nil { + return err } c.AddTask(&nodetasks.File{ Path: filepath.Join(b.PathSrvKubernetes(), "service-account.pub"), - Contents: fi.NewBytesResource(buf.Bytes()), + Contents: fi.NewBytesResource(buf), Type: nodetasks.FileType_File, Mode: s("0600"), }) diff --git a/nodeup/pkg/model/kubelet_test.go b/nodeup/pkg/model/kubelet_test.go index 5f10d52008..3d01594cf8 100644 --- a/nodeup/pkg/model/kubelet_test.go +++ b/nodeup/pkg/model/kubelet_test.go @@ -268,7 +268,9 @@ func mockedPopulateClusterSpec(c *kops.Cluster, cloud fi.Cloud) (*kops.Cluster, const dummyCertificate = "-----BEGIN CERTIFICATE-----\nMIIC2DCCAcCgAwIBAgIRALJXAkVj964tq67wMSI8oJQwDQYJKoZIhvcNAQELBQAw\nFTETMBEGA1UEAxMKa3ViZXJuZXRlczAeFw0xNzEyMjcyMzUyNDBaFw0yNzEyMjcy\nMzUyNDBaMBUxEzARBgNVBAMTCmt1YmVybmV0ZXMwggEiMA0GCSqGSIb3DQEBAQUA\nA4IBDwAwggEKAoIBAQDgnCkSmtnmfxEgS3qNPaUCH5QOBGDH/inHbWCODLBCK9gd\nXEcBl7FVv8T2kFr1DYb0HVDtMI7tixRVFDLgkwNlW34xwWdZXB7GeoFgU1xWOQSY\nOACC8JgYTQ/139HBEvgq4sej67p+/s/SNcw34Kk7HIuFhlk1rRk5kMexKIlJBKP1\nYYUYetsJ/QpUOkqJ5HW4GoetE76YtHnORfYvnybviSMrh2wGGaN6r/s4ChOaIbZC\nAn8/YiPKGIDaZGpj6GXnmXARRX/TIdgSQkLwt0aTDBnPZ4XvtpI8aaL8DYJIqAzA\nNPH2b4/uNylat5jDo0b0G54agMi97+2AUrC9UUXpAgMBAAGjIzAhMA4GA1UdDwEB\n/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBVGR2r\nhzXzRMU5wriPQAJScszNORvoBpXfZoZ09FIupudFxBVU3d4hV9StKnQgPSGA5XQO\nHE97+BxJDuA/rB5oBUsMBjc7y1cde/T6hmi3rLoEYBSnSudCOXJE4G9/0f8byAJe\nrN8+No1r2VgZvZh6p74TEkXv/l3HBPWM7IdUV0HO9JDhSgOVF1fyQKJxRuLJR8jt\nO6mPH2UX0vMwVa4jvwtkddqk2OAdYQvH9rbDjjbzaiW0KnmdueRo92KHAN7BsDZy\nVpXHpqo1Kzg7D3fpaXCf5si7lqqrdJVXH4JC72zxsPehqgi8eIuqOBkiDWmRxAxh\n8yGeRx9AbknHh4Ia\n-----END CERTIFICATE-----\n" const dummyKey = "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA4JwpEprZ5n8RIEt6jT2lAh+UDgRgx/4px21gjgywQivYHVxH\nAZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMDZVt+McFnWVwexnqBYFNcVjkEmDgA\ngvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+CpOxyLhYZZNa0ZOZDHsSiJSQSj9WGF\nGHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m74kjK4dsBhmjeq/7OAoTmiG2QgJ/\nP2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdGkwwZz2eF77aSPGmi/A2CSKgMwDTx\n9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF6QIDAQABAoIBAA0ktjaTfyrAxsTI\nBezb7Zr5NBW55dvuII299cd6MJo+rI/TRYhvUv48kY8IFXp/hyUjzgeDLunxmIf9\n/Zgsoic9Ol44/g45mMduhcGYPzAAeCdcJ5OB9rR9VfDCXyjYLlN8H8iU0734tTqM\n0V13tQ9zdSqkGPZOIcq/kR/pylbOZaQMe97BTlsAnOMSMKDgnftY4122Lq3GYy+t\nvpr+bKVaQZwvkLoSU3rECCaKaghgwCyX7jft9aEkhdJv+KlwbsGY6WErvxOaLWHd\ncuMQjGapY1Fa/4UD00mvrA260NyKfzrp6+P46RrVMwEYRJMIQ8YBAk6N6Hh7dc0G\n8Z6i1m0CgYEA9HeCJR0TSwbIQ1bDXUrzpftHuidG5BnSBtax/ND9qIPhR/FBW5nj\n22nwLc48KkyirlfIULd0ae4qVXJn7wfYcuX/cJMLDmSVtlM5Dzmi/91xRiFgIzx1\nAsbBzaFjISP2HpSgL+e9FtSXaaqeZVrflitVhYKUpI/AKV31qGHf04sCgYEA6zTV\n99Sb49Wdlns5IgsfnXl6ToRttB18lfEKcVfjAM4frnkk06JpFAZeR+9GGKUXZHqs\nz2qcplw4d/moCC6p3rYPBMLXsrGNEUFZqBlgz72QA6BBq3X0Cg1Bc2ZbK5VIzwkg\nST2SSux6ccROfgULmN5ZiLOtdUKNEZpFF3i3qtsCgYADT/s7dYFlatobz3kmMnXK\nsfTu2MllHdRys0YGHu7Q8biDuQkhrJwhxPW0KS83g4JQym+0aEfzh36bWcl+u6R7\nKhKj+9oSf9pndgk345gJz35RbPJYh+EuAHNvzdgCAvK6x1jETWeKf6btj5pF1U1i\nQ4QNIw/QiwIXjWZeubTGsQKBgQCbduLu2rLnlyyAaJZM8DlHZyH2gAXbBZpxqU8T\nt9mtkJDUS/KRiEoYGFV9CqS0aXrayVMsDfXY6B/S/UuZjO5u7LtklDzqOf1aKG3Q\ndGXPKibknqqJYH+bnUNjuYYNerETV57lijMGHuSYCf8vwLn3oxBfERRX61M/DU8Z\nworz/QKBgQDCTJI2+jdXg26XuYUmM4XXfnocfzAXhXBULt1nENcogNf1fcptAVtu\nBAiz4/HipQKqoWVUYmxfgbbLRKKLK0s0lOWKbYdVjhEm/m2ZU8wtXTagNwkIGoyq\nY/C1Lox4f1ROJnCjc/hfcOjcxX5M8A8peecHWlVtUPKTJgxQ7oMKcw==\n-----END RSA PRIVATE KEY-----\n" +const previousCertificate = "-----BEGIN CERTIFICATE-----\nMIIBZzCCARGgAwIBAgIBAjANBgkqhkiG9w0BAQsFADAaMRgwFgYDVQQDEw9zZXJ2\naWNlLWFjY291bnQwHhcNMjEwNTAyMjAzMDA2WhcNMzEwNTAyMjAzMDA2WjAaMRgw\nFgYDVQQDEw9zZXJ2aWNlLWFjY291bnQwXDANBgkqhkiG9w0BAQEFAANLADBIAkEA\n2JbeF8dNwqfEKKD65aGlVs58fWkA0qZdVLKw8qATzRBJTi1nqbj2kAR4gyy/C8Mx\nouxva/om9d7Sq8Ka55T7+wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T\nAQH/BAUwAwEB/zAdBgNVHQ4EFgQUI5beFHueAGyT1pQ6UTOdbMfj3gQwDQYJKoZI\nhvcNAQELBQADQQBwPLO+Np8o6k3aNBGKE4JTCOs06X72OXNivkWWWP/9XGz6x4DI\nHPU65kbUn/pWXBUVVlpsKsdmWA2Bu8pd/vD+\n-----END CERTIFICATE-----\n" const previousKey = "-----BEGIN RSA PRIVATE KEY-----\nMIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4\n9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R\n2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo\nxTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+\nZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr\nKl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh\nAOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY\n-----END RSA PRIVATE KEY-----" +const nextCertificate = "-----BEGIN CERTIFICATE-----\nMIIBZzCCARGgAwIBAgIBBDANBgkqhkiG9w0BAQsFADAaMRgwFgYDVQQDEw9zZXJ2\naWNlLWFjY291bnQwHhcNMjEwNTAyMjAzMjE3WhcNMzEwNTAyMjAzMjE3WjAaMRgw\nFgYDVQQDEw9zZXJ2aWNlLWFjY291bnQwXDANBgkqhkiG9w0BAQEFAANLADBIAkEA\no4Tridlsf4Yz3UAiup/scSTiG/OqxkUW3Fz7zGKvVcLeYj9GEIKuzoB1VFk1nboD\nq4cCuGLfdzaQdCQKPIsDuwIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T\nAQH/BAUwAwEB/zAdBgNVHQ4EFgQUhPbxEmUbwVOCa+fZgxreFhf67UEwDQYJKoZI\nhvcNAQELBQADQQALMsyK2Q7C/bk27eCvXyZKUfrLvor10hEjwGhv14zsKWDeTj/J\nA1LPYp7U9VtFfgFOkVbkLE9Rstc0ltNrPqxA\n-----END CERTIFICATE-----\n" const nextKey = "-----BEGIN RSA PRIVATE KEY-----\nMIIBOgIBAAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCC\nrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQJAejInjmEzqmzQr0NxcIN4\nPukwK3FBKl+RAOZfqNIKcww14mfOn7Gc6lF2zEC4GnLiB3tthbSXoBGi54nkW4ki\nyQIhANZNne9UhQlwyjsd3WxDWWrl6OOZ3J8ppMOIQni9WRLlAiEAw1XEdxPOSOSO\nB6rucpTT1QivVvyEFIb/ukvPm769Mh8CIQDNQwKnHdlfNX0+KljPPaMD1LrAZbr/\naC+8aWLhqtsKUQIgF7gUcTkwdV17eabh6Xv09Qtm7zMefred2etWvFy+8JUCIECv\nFYOKQVWHX+Q7CHX2K1oTECVnZuW1UItdDYVlFYxQ\n-----END RSA PRIVATE KEY-----" func simplePrivateKeyset(s string) *kops.Keyset { @@ -285,7 +287,7 @@ func simplePrivateKeyset(s string) *kops.Keyset { } } -func rotatingPrivateKeyset(s string) *kops.Keyset { +func rotatingPrivateKeyset() *kops.Keyset { return &kops.Keyset{ Spec: kops.KeysetSpec{ PrimaryId: "3", @@ -293,14 +295,17 @@ func rotatingPrivateKeyset(s string) *kops.Keyset { { Id: "2", PrivateMaterial: []byte(previousKey), + PublicMaterial: []byte(previousCertificate), }, { Id: "3", - PrivateMaterial: []byte(s), + PrivateMaterial: []byte(dummyKey), + PublicMaterial: []byte(dummyCertificate), }, { Id: "4", PrivateMaterial: []byte(nextKey), + PublicMaterial: []byte(nextCertificate), }, }, }, @@ -338,7 +343,7 @@ func RunGoldenTest(t *testing.T, basedir string, key string, builder func(*Nodeu "kube-controller-manager": simplePrivateKeyset(dummyKey), "kube-proxy": simplePrivateKeyset(dummyKey), "kube-scheduler": simplePrivateKeyset(dummyKey), - "service-account": rotatingPrivateKeyset(dummyKey), + "service-account": rotatingPrivateKeyset(), } keystore.certs = map[string]*pki.Certificate{ "ca": mustParseCertificate(dummyCertificate), diff --git a/upup/pkg/fi/ca.go b/upup/pkg/fi/ca.go index 7eb5703d9e..eeadebb939 100644 --- a/upup/pkg/fi/ca.go +++ b/upup/pkg/fi/ca.go @@ -18,8 +18,11 @@ package fi import ( "bytes" + "crypto/x509" + "encoding/pem" "fmt" "math/big" + "sort" "strconv" "k8s.io/kops/pkg/apis/kops" @@ -200,3 +203,26 @@ func KeysetItemIdOlder(a, b string) bool { return a < b } } + +func (k *Keyset) ToPublicKeyBytes() ([]byte, error) { + keys := make([]string, 0, len(k.Items)) + for k := range k.Items { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + return KeysetItemIdOlder(k.Items[keys[i]].Id, k.Items[keys[j]].Id) + }) + + buf := new(bytes.Buffer) + for _, key := range keys { + item := k.Items[key] + publicKeyData, err := x509.MarshalPKIXPublicKey(item.Certificate.PublicKey) + if err != nil { + return nil, fmt.Errorf("marshalling public key %s: %v", item.Id, err) + } + if err = pem.Encode(buf, &pem.Block{Type: "RSA PUBLIC KEY", Bytes: publicKeyData}); err != nil { + return nil, fmt.Errorf("encoding public key %s: %v", item.Id, err) + } + } + return buf.Bytes(), nil +}