From 1e0c6cb1aaad0b3e37b3533e242b22aad776a528 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Thu, 1 Jul 2021 14:17:29 -0700 Subject: [PATCH] Refactor apiserver-aggregator-ca --- cmd/kops/create_keypair.go | 1 + cmd/kops/integration_test.go | 6 +++ nodeup/pkg/model/kube_apiserver.go | 45 +++++++++++--------- nodeup/pkg/model/secrets.go | 15 ------- pkg/model/awsmodel/autoscalinggroup_test.go | 6 +++ pkg/model/bootstrapscript.go | 2 +- pkg/model/bootstrapscript_test.go | 6 +++ pkg/model/openstackmodel/servergroup_test.go | 6 +++ upup/pkg/fi/cloudup/apply_cluster.go | 3 ++ 9 files changed, 54 insertions(+), 36 deletions(-) diff --git a/cmd/kops/create_keypair.go b/cmd/kops/create_keypair.go index de176d30f2..4532466cbe 100644 --- a/cmd/kops/create_keypair.go +++ b/cmd/kops/create_keypair.go @@ -79,6 +79,7 @@ type CreateKeypairOptions struct { } var rotatableKeysets = sets.NewString( + "apiserver-aggregator-ca", "ca", "etcd-clients-ca-cilium", "service-account", diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index b1780cc940..fafbb4f87d 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -734,6 +734,12 @@ func (i *integrationTest) setupCluster(t *testing.T, inputYAML string, ctx conte secondaryKey: "-----BEGIN RSA PRIVATE KEY-----\nMIIBOgIBAAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCC\nrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQJAejInjmEzqmzQr0NxcIN4\nPukwK3FBKl+RAOZfqNIKcww14mfOn7Gc6lF2zEC4GnLiB3tthbSXoBGi54nkW4ki\nyQIhANZNne9UhQlwyjsd3WxDWWrl6OOZ3J8ppMOIQni9WRLlAiEAw1XEdxPOSOSO\nB6rucpTT1QivVvyEFIb/ukvPm769Mh8CIQDNQwKnHdlfNX0+KljPPaMD1LrAZbr/\naC+8aWLhqtsKUQIgF7gUcTkwdV17eabh6Xv09Qtm7zMefred2etWvFy+8JUCIECv\nFYOKQVWHX+Q7CHX2K1oTECVnZuW1UItdDYVlFYxQ\n-----END RSA PRIVATE KEY-----\n", secondaryCertificate: "-----BEGIN CERTIFICATE-----\nMIIBaDCCARKgAwIBAgIMFoq6PeyECsgUTfc2MA0GCSqGSIb3DQEBCwUAMBUxEzAR\nBgNVBAMTCmt1YmVybmV0ZXMwHhcNMjEwNjE5MjI0MzEwWhcNMzEwNjE5MjI0MzEw\nWjAVMRMwEQYDVQQDEwprdWJlcm5ldGVzMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJB\nAKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCCrs6AdVRZNZ26\nA6uHArhi33c2kHQkCjyLA7sCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud\nEwEB/wQFMAMBAf8wHQYDVR0OBBYEFIT28RJlG8FTgmvn2YMa3hYX+u1BMA0GCSqG\nSIb3DQEBCwUAA0EAKuaE5wKMP26AyfxkWu83iHoTPFtdjabXF0JcyPy0ijQZxfJq\n9xc2CkttvgaDtT4H+E/ryQ3iq6kSfEYYPi8c0w==\n-----END CERTIFICATE-----", }) + storeKeyset(t, keyStore, "apiserver-aggregator-ca", &testingKeyset{ + primaryKey: "-----BEGIN RSA PRIVATE KEY-----\nMIIBOwIBAAJBAMshO9QDlN4KOVxXoC0On4nSNC4YTMews6U84dsVinB1H2zSO4rY\nCbwv/hpchuVvgxeVe22tCCYkC7Bb3tKC3XsCAwEAAQJAe4xCLGjlQcvsKYsuZFlR\nle0hSawD/y0thuIp6SwH4O92AOsfrWDdiWIVCP6S47oBv351BOcoPbOjxfMTN+f6\naQIhAPIfBCHL/GecX1IVyitI1ueG1z0n5DDOKQAxmxTg82SnAiEA1sYK+vXMIV/e\nCl/CHxKwu7f+ufh1bV0OFyd+eI2+Vw0CICs6eG1kUzNYivhH5ammvp/lxkYn+ijw\nlgdv0+V9aFdfAiEAsTUytiK8zQTGthSQnQbU3+5OtK82ZIgVKjGh/mIlnLkCIQC1\neG3yBXM7/cxw1doWZ7AzMncufx9R8Q2Hblm80UrpaQ==\n-----END RSA PRIVATE KEY-----", + primaryCertificate: "-----BEGIN CERTIFICATE-----\nMIIBgjCCASygAwIBAgIMFo3gINaZLHjisEcbMA0GCSqGSIb3DQEBCwUAMCIxIDAe\nBgNVBAMTF2FwaXNlcnZlci1hZ2dyZWdhdG9yLWNhMB4XDTIxMDYzMDA0NTExMloX\nDTMxMDYzMDA0NTExMlowIjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3It\nY2EwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAyyE71AOU3go5XFegLQ6fidI0LhhM\nx7CzpTzh2xWKcHUfbNI7itgJvC/+GlyG5W+DF5V7ba0IJiQLsFve0oLdewIDAQAB\no0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU\nALfqF5ZmfqvqORuJIFilZYKF3d0wDQYJKoZIhvcNAQELBQADQQAHAomFKsF4jvYX\nWM/UzQXDj9nSAFTf8dBPCXyZZNotsOH7+P6W4mMiuVs8bAuGiXGUdbsQ2lpiT/Rk\nCzMeMdr4\n-----END CERTIFICATE-----", + secondaryKey: "-----BEGIN RSA PRIVATE KEY-----\nMIIBOwIBAAJBAMshO9QDlN4KOVxXoC0On4nSNC4YTMews6U84dsVinB1H2zSO4rY\nCbwv/hpchuVvgxeVe22tCCYkC7Bb3tKC3XsCAwEAAQJAe4xCLGjlQcvsKYsuZFlR\nle0hSawD/y0thuIp6SwH4O92AOsfrWDdiWIVCP6S47oBv351BOcoPbOjxfMTN+f6\naQIhAPIfBCHL/GecX1IVyitI1ueG1z0n5DDOKQAxmxTg82SnAiEA1sYK+vXMIV/e\nCl/CHxKwu7f+ufh1bV0OFyd+eI2+Vw0CICs6eG1kUzNYivhH5ammvp/lxkYn+ijw\nlgdv0+V9aFdfAiEAsTUytiK8zQTGthSQnQbU3+5OtK82ZIgVKjGh/mIlnLkCIQC1\neG3yBXM7/cxw1doWZ7AzMncufx9R8Q2Hblm80UrpaQ==\n-----END RSA PRIVATE KEY-----", + secondaryCertificate: "-----BEGIN CERTIFICATE-----\nMIIBgjCCASygAwIBAgIMFo3gM0nxQpiX/agfMA0GCSqGSIb3DQEBCwUAMCIxIDAe\nBgNVBAMTF2FwaXNlcnZlci1hZ2dyZWdhdG9yLWNhMB4XDTIxMDYzMDA0NTIzMVoX\nDTMxMDYzMDA0NTIzMVowIjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3It\nY2EwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAyyE71AOU3go5XFegLQ6fidI0LhhM\nx7CzpTzh2xWKcHUfbNI7itgJvC/+GlyG5W+DF5V7ba0IJiQLsFve0oLdewIDAQAB\no0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU\nALfqF5ZmfqvqORuJIFilZYKF3d0wDQYJKoZIhvcNAQELBQADQQCXsoezoxXu2CEN\nQdlXZOfmBT6cqxIX/RMHXhpHwRiqPsTO8IO2bVA8CSzxNwMuSv/ZtrMHoh8+PcVW\nHLtkTXH8\n-----END CERTIFICATE-----", + }) storeKeyset(t, keyStore, "etcd-clients-ca", &testingKeyset{ primaryKey: "-----BEGIN RSA PRIVATE KEY-----\nMIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4\n9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R\n2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo\nxTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+\nZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr\nKl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh\nAOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY\n-----END RSA PRIVATE KEY-----", primaryCertificate: "-----BEGIN CERTIFICATE-----\nMIIBcjCCARygAwIBAgIMFo1ogHnr26DL9YkqMA0GCSqGSIb3DQEBCwUAMBoxGDAW\nBgNVBAMTD2V0Y2QtY2xpZW50cy1jYTAeFw0yMTA2MjgxNjE5MDFaFw0zMTA2Mjgx\nNjE5MDFaMBoxGDAWBgNVBAMTD2V0Y2QtY2xpZW50cy1jYTBcMA0GCSqGSIb3DQEB\nAQUAA0sAMEgCQQDYlt4Xx03Cp8QooPrloaVWznx9aQDSpl1UsrDyoBPNEElOLWep\nuPaQBHiDLL8LwzGi7G9r+ib13tKrwprnlPv7AgMBAAGjQjBAMA4GA1UdDwEB/wQE\nAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQjlt4Ue54AbJPWlDpRM51s\nx+PeBDANBgkqhkiG9w0BAQsFAANBAAZAdf8ROEVkr3Rf7I+s+CQOil2toadlKWOY\nqCeJ2XaEROfp9aUTEIU1MGM3g57MPyAPPU7mURskuOQz6B1UFaY=\n-----END CERTIFICATE-----", diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go index bb163529a8..cd44c4c66d 100644 --- a/nodeup/pkg/model/kube_apiserver.go +++ b/nodeup/pkg/model/kube_apiserver.go @@ -124,6 +124,31 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error { kubeAPIServer.EtcdKeyFile = filepath.Join(b.PathSrvKubernetes(), "etcd-client-key.pem") } + { + c.AddTask(&nodetasks.File{ + Path: filepath.Join(pathSrvKAPI, "apiserver-aggregator-ca.crt"), + Contents: fi.NewStringResource(b.NodeupConfig.CAs["apiserver-aggregator-ca"]), + Type: nodetasks.FileType_File, + Mode: fi.String("0644"), + }) + kubeAPIServer.RequestheaderClientCAFile = filepath.Join(pathSrvKAPI, "apiserver-aggregator-ca.crt") + + issueCert := &nodetasks.IssueCert{ + Name: "apiserver-aggregator", + Signer: "apiserver-aggregator-ca", + Type: "client", + // Must match RequestheaderAllowedNames + Subject: nodetasks.PKIXName{CommonName: "aggregator"}, + } + c.AddTask(issueCert) + err := issueCert.AddFileTasks(c, pathSrvKAPI, "apiserver-aggregator", "", nil) + if err != nil { + return err + } + kubeAPIServer.ProxyClientCertFile = fi.String(filepath.Join(pathSrvKAPI, "apiserver-aggregator.crt")) + kubeAPIServer.ProxyClientKeyFile = fi.String(filepath.Join(pathSrvKAPI, "apiserver-aggregator.key")) + } + { pod, err := b.buildPod(&kubeAPIServer) if err != nil { @@ -371,26 +396,6 @@ func (b *KubeAPIServerBuilder) buildPod(kubeAPIServer *kops.KubeAPIServerConfig) kubeAPIServer.KubeletClientCertificate = filepath.Join(b.PathSrvKubernetes(), "kubelet-api.crt") kubeAPIServer.KubeletClientKey = filepath.Join(b.PathSrvKubernetes(), "kubelet-api.key") - { - certPath := filepath.Join(b.PathSrvKubernetes(), "apiserver-aggregator.crt") - kubeAPIServer.ProxyClientCertFile = &certPath - keyPath := filepath.Join(b.PathSrvKubernetes(), "apiserver-aggregator.key") - kubeAPIServer.ProxyClientKeyFile = &keyPath - } - - // APIServer aggregation options - { - cert, err := b.KeyStore.FindCert("apiserver-aggregator-ca") - if err != nil { - return nil, fmt.Errorf("apiserver aggregator CA cert lookup failed: %v", err.Error()) - } - - if cert != nil { - certPath := filepath.Join(b.PathSrvKubernetes(), "apiserver-aggregator-ca.crt") - kubeAPIServer.RequestheaderClientCAFile = certPath - } - } - // @fixup: the admission controller migrated from --admission-control to --enable-admission-plugins, but // most people will still have c.Spec.KubeAPIServer.AdmissionControl references into their configuration we need // to fix up. A PR https://github.com/kubernetes/kops/pull/5221/ introduced the issue and since the command line diff --git a/nodeup/pkg/model/secrets.go b/nodeup/pkg/model/secrets.go index c357cbd339..00915071da 100644 --- a/nodeup/pkg/model/secrets.go +++ b/nodeup/pkg/model/secrets.go @@ -134,21 +134,6 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error { } } - { - issueCert := &nodetasks.IssueCert{ - Name: "apiserver-aggregator", - Signer: "apiserver-aggregator-ca", - Type: "client", - // Must match RequestheaderAllowedNames - Subject: nodetasks.PKIXName{CommonName: "aggregator"}, - } - c.AddTask(issueCert) - err := issueCert.AddFileTasks(c, b.PathSrvKubernetes(), "apiserver-aggregator", "apiserver-aggregator-ca", nil) - if err != nil { - return err - } - } - if err := b.BuildPrivateKeyTask(c, "service-account", "service-account.key", nil); err != nil { return err } diff --git a/pkg/model/awsmodel/autoscalinggroup_test.go b/pkg/model/awsmodel/autoscalinggroup_test.go index 441a35f4f4..6551428bb7 100644 --- a/pkg/model/awsmodel/autoscalinggroup_test.go +++ b/pkg/model/awsmodel/autoscalinggroup_test.go @@ -184,6 +184,12 @@ func TestAPIServerAdditionalSecurityGroupsWithNLB(t *testing.T) { Type: "ca", } c.AddTask(caTask) + aggregatorTask := &fitasks.Keypair{ + Name: fi.String("apiserver-aggregator-ca"), + Subject: "cn=apiserver-aggregator-ca", + Type: "ca", + } + c.AddTask(aggregatorTask) saTask := &fitasks.Keypair{ Name: fi.String("service-account"), Subject: "cn=service-account", diff --git a/pkg/model/bootstrapscript.go b/pkg/model/bootstrapscript.go index b78404e0f5..13349778e9 100644 --- a/pkg/model/bootstrapscript.go +++ b/pkg/model/bootstrapscript.go @@ -220,7 +220,7 @@ func (b *BootstrapScriptBuilder) ResourceNodeUp(c *fi.ModelBuilderContext, ig *k } } if ig.HasAPIServer() { - keypairs = append(keypairs, "service-account") + keypairs = append(keypairs, "apiserver-aggregator-ca", "service-account") if b.UseEtcdManager() { keypairs = append(keypairs, "etcd-clients-ca") } diff --git a/pkg/model/bootstrapscript_test.go b/pkg/model/bootstrapscript_test.go index ce83621842..9cc73c8a0e 100644 --- a/pkg/model/bootstrapscript_test.go +++ b/pkg/model/bootstrapscript_test.go @@ -140,6 +140,12 @@ func TestBootstrapUserData(t *testing.T) { Type: "ca", } c.AddTask(caTask) + aggregatorTask := &fitasks.Keypair{ + Name: fi.String("apiserver-aggregator-ca"), + Subject: "cn=apiserver-aggregator-ca", + Type: "ca", + } + c.AddTask(aggregatorTask) saTask := &fitasks.Keypair{ Name: fi.String("service-account"), Subject: "cn=service-account", diff --git a/pkg/model/openstackmodel/servergroup_test.go b/pkg/model/openstackmodel/servergroup_test.go index b76bcb1118..8ea252e8b8 100644 --- a/pkg/model/openstackmodel/servergroup_test.go +++ b/pkg/model/openstackmodel/servergroup_test.go @@ -1067,6 +1067,12 @@ func RunGoldenTest(t *testing.T, basedir string, testCase serverGroupModelBuilde Type: "ca", } context.AddTask(caTask) + aggregatorTask := &fitasks.Keypair{ + Name: fi.String("apiserver-aggregator-ca"), + Subject: "cn=apiserver-aggregator-ca", + Type: "ca", + } + context.AddTask(aggregatorTask) saTask := &fitasks.Keypair{ Name: fi.String("service-account"), Subject: "cn=service-account", diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index 4d1de1458d..c2106bb7b0 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -1316,6 +1316,9 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAddit } if isMaster || role == kops.InstanceGroupRoleAPIServer { + if err := getTasksCertificate(caTasks, "apiserver-aggregator-ca", config); err != nil { + return nil, nil, err + } if caTasks["etcd-clients-ca"] != nil { if err := getTasksCertificate(caTasks, "etcd-clients-ca", config); err != nil { return nil, nil, err