From c8b1a586b80be40b9ea689cd6e18ff9d95016f70 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Fri, 16 Jul 2021 22:42:23 -0700 Subject: [PATCH 1/5] Refactor kube-apiserver server certificate --- nodeup/pkg/model/kube_apiserver.go | 83 +++++++++++++++++++++++++++--- nodeup/pkg/model/secrets.go | 64 +---------------------- 2 files changed, 77 insertions(+), 70 deletions(-) diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go index 33a57cd84f..b35d038555 100644 --- a/nodeup/pkg/model/kube_apiserver.go +++ b/nodeup/pkg/model/kube_apiserver.go @@ -26,6 +26,7 @@ import ( "k8s.io/kops/pkg/k8scodecs" "k8s.io/kops/pkg/kubeconfig" "k8s.io/kops/pkg/kubemanifest" + "k8s.io/kops/pkg/model/components" "k8s.io/kops/pkg/wellknownports" "k8s.io/kops/pkg/wellknownusers" "k8s.io/kops/upup/pkg/fi" @@ -39,17 +40,17 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) -// PathAuthnConfig is the path to the custom webhook authentication config +// PathAuthnConfig is the path to the custom webhook authentication config. const PathAuthnConfig = "/etc/kubernetes/authn.config" -// KubeAPIServerBuilder install kube-apiserver (just the manifest at the moment) +// KubeAPIServerBuilder installs kube-apiserver. type KubeAPIServerBuilder struct { *NodeupModelContext } var _ fi.ModelBuilder = &KubeAPIServerBuilder{} -// Build is responsible for generating the configuration for the kube-apiserver +// Build is responsible for generating the configuration for the kube-apiserver. func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error { if !b.HasAPIServer { return nil @@ -162,6 +163,10 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error { kubeAPIServer.ProxyClientKeyFile = fi.String(filepath.Join(pathSrvKAPI, "apiserver-aggregator.key")) } + if err := b.writeServerCertificate(c, &kubeAPIServer); err != nil { + return err + } + { pod, err := b.buildPod(&kubeAPIServer) if err != nil { @@ -349,15 +354,79 @@ func (b *KubeAPIServerBuilder) writeAuthenticationConfig(c *fi.ModelBuilderConte return fmt.Errorf("unrecognized authentication config %v", b.Cluster.Spec.Authentication) } -// buildPod is responsible for generating the kube-apiserver pod and thus manifest file -func (b *KubeAPIServerBuilder) buildPod(kubeAPIServer *kops.KubeAPIServerConfig) (*v1.Pod, error) { +func (b *KubeAPIServerBuilder) writeServerCertificate(c *fi.ModelBuilderContext, kubeAPIServer *kops.KubeAPIServerConfig) error { + pathSrvKAPI := filepath.Join(b.PathSrvKubernetes(), "kube-apiserver") + + { + // A few names used from inside the cluster, which all resolve the same based on our default suffixes + alternateNames := []string{ + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc." + b.Cluster.Spec.ClusterDNSDomain, + } + + // Names specified in the cluster spec + alternateNames = append(alternateNames, b.Cluster.Spec.MasterPublicName) + alternateNames = append(alternateNames, b.Cluster.Spec.MasterInternalName) + alternateNames = append(alternateNames, b.Cluster.Spec.AdditionalSANs...) + + // Load balancer IPs passed in through NodeupConfig + alternateNames = append(alternateNames, b.NodeupConfig.ApiserverAdditionalIPs...) + + // Referencing it by internal IP should work also + { + ip, err := components.WellKnownServiceIP(&b.Cluster.Spec, 1) + if err != nil { + return err + } + alternateNames = append(alternateNames, ip.String()) + } + + // We also want to be able to reference it locally via https://127.0.0.1 + alternateNames = append(alternateNames, "127.0.0.1") + + if b.Cluster.Spec.CloudProvider == "openstack" { + if b.Cluster.Spec.Topology != nil && b.Cluster.Spec.Topology.Masters == kops.TopologyPrivate { + instanceAddress, err := getInstanceAddress() + if err != nil { + return err + } + alternateNames = append(alternateNames, instanceAddress) + } + } + + issueCert := &nodetasks.IssueCert{ + Name: "master", + Signer: fi.CertificateIDCA, + KeypairID: b.NodeupConfig.KeypairIDs[fi.CertificateIDCA], + Type: "server", + Subject: nodetasks.PKIXName{CommonName: "kubernetes-master"}, + AlternateNames: alternateNames, + } + + // Including the CA certificate is more correct, and is needed for e.g. AWS WebIdentity federation + issueCert.IncludeRootCertificate = true + + c.AddTask(issueCert) + err := issueCert.AddFileTasks(c, pathSrvKAPI, "server", "", nil) + if err != nil { + return err + } + } + // If clientCAFile is not specified, set it to the default value ${PathSrvKubernetes}/ca.crt if kubeAPIServer.ClientCAFile == "" { kubeAPIServer.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt") } - kubeAPIServer.TLSCertFile = filepath.Join(b.PathSrvKubernetes(), "server.crt") - kubeAPIServer.TLSPrivateKeyFile = filepath.Join(b.PathSrvKubernetes(), "server.key") + kubeAPIServer.TLSCertFile = filepath.Join(pathSrvKAPI, "server.crt") + kubeAPIServer.TLSPrivateKeyFile = filepath.Join(pathSrvKAPI, "server.key") + return nil +} + +// buildPod is responsible for generating the kube-apiserver pod and thus manifest file +func (b *KubeAPIServerBuilder) buildPod(kubeAPIServer *kops.KubeAPIServerConfig) (*v1.Pod, error) { // Support for basic auth was deprecated 1.16 and removed in 1.19 // https://github.com/kubernetes/kubernetes/pull/89069 if b.IsKubernetesLT("1.18") { diff --git a/nodeup/pkg/model/secrets.go b/nodeup/pkg/model/secrets.go index cabc6342bc..8d1d23f4ff 100644 --- a/nodeup/pkg/model/secrets.go +++ b/nodeup/pkg/model/secrets.go @@ -21,14 +21,10 @@ import ( "path/filepath" "strings" - "k8s.io/kops/util/pkg/vfs" - - "k8s.io/kops/pkg/apis/kops" - - "k8s.io/kops/pkg/model/components" "k8s.io/kops/pkg/tokens" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" + "k8s.io/kops/util/pkg/vfs" ) // SecretBuilder writes secrets @@ -77,64 +73,6 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error { return nil } - { - // A few names used from inside the cluster, which all resolve the same based on our default suffixes - alternateNames := []string{ - "kubernetes", - "kubernetes.default", - "kubernetes.default.svc", - "kubernetes.default.svc." + b.Cluster.Spec.ClusterDNSDomain, - } - - // Names specified in the cluster spec - alternateNames = append(alternateNames, b.Cluster.Spec.MasterPublicName) - alternateNames = append(alternateNames, b.Cluster.Spec.MasterInternalName) - alternateNames = append(alternateNames, b.Cluster.Spec.AdditionalSANs...) - - // Load balancer IPs passed in through NodeupConfig - alternateNames = append(alternateNames, b.NodeupConfig.ApiserverAdditionalIPs...) - - // Referencing it by internal IP should work also - { - ip, err := components.WellKnownServiceIP(&b.Cluster.Spec, 1) - if err != nil { - return err - } - alternateNames = append(alternateNames, ip.String()) - } - - // We also want to be able to reference it locally via https://127.0.0.1 - alternateNames = append(alternateNames, "127.0.0.1") - - if b.Cluster.Spec.CloudProvider == "openstack" { - if b.Cluster.Spec.Topology != nil && b.Cluster.Spec.Topology.Masters == kops.TopologyPrivate { - instanceAddress, err := getInstanceAddress() - if err != nil { - return err - } - alternateNames = append(alternateNames, instanceAddress) - } - } - - issueCert := &nodetasks.IssueCert{ - Name: "master", - Signer: fi.CertificateIDCA, - KeypairID: b.NodeupConfig.KeypairIDs[fi.CertificateIDCA], - Type: "server", - Subject: nodetasks.PKIXName{CommonName: "kubernetes-master"}, - AlternateNames: alternateNames, - } - - // Including the CA certificate is more correct, and is needed for e.g. AWS WebIdentity federation - issueCert.IncludeRootCertificate = true - - c.AddTask(issueCert) - err := issueCert.AddFileTasks(c, b.PathSrvKubernetes(), "server", "", nil) - if err != nil { - return err - } - } - // Support for basic auth was deprecated 1.16 and removed in 1.19 // https://github.com/kubernetes/kubernetes/pull/89069 if b.IsKubernetesLT("1.19") && b.SecretStore != nil { From 781b302faca4d7c44393d0595498036a0cb6a40b Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Fri, 16 Jul 2021 22:46:41 -0700 Subject: [PATCH 2/5] hack/update-expected.sh --- .../golden/awsiam/tasks-kube-apiserver.yaml | 65 ++++++++++++++++++- .../tasks-kube-apiserver.yaml | 65 ++++++++++++++++++- .../golden/minimal/tasks-kube-apiserver.yaml | 65 ++++++++++++++++++- .../tests/golden/minimal/tasks-secret.yaml | 65 ------------------- .../tasks-kube-apiserver-amd64.yaml | 65 ++++++++++++++++++- .../tasks-kube-apiserver-arm64.yaml | 65 ++++++++++++++++++- .../tasks-kube-apiserver.yaml | 65 ++++++++++++++++++- 7 files changed, 378 insertions(+), 77 deletions(-) diff --git a/nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml index 31bc01b570..c20ebe3a7a 100644 --- a/nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml +++ b/nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml @@ -65,8 +65,8 @@ contents: | - --service-account-key-file=/srv/kubernetes/kube-apiserver/service-account.pub - --service-cluster-ip-range=100.64.0.0/13 - --storage-backend=etcd3 - - --tls-cert-file=/srv/kubernetes/server.crt - - --tls-private-key-file=/srv/kubernetes/server.key + - --tls-cert-file=/srv/kubernetes/kube-apiserver/server.crt + - --tls-private-key-file=/srv/kubernetes/kube-apiserver/server.key - --v=2 - --logtostderr=false - --alsologtostderr @@ -284,6 +284,50 @@ mode: "0600" path: /srv/kubernetes/kube-apiserver/etcd-client.key type: file --- +contents: + task: + Name: master + alternateNames: + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - kubernetes.default.svc.cluster.local + - api.minimal.example.com + - api.internal.minimal.example.com + - 100.64.0.1 + - 127.0.0.1 + includeRootCertificate: true + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubernetes-master + type: server +mode: "0644" +path: /srv/kubernetes/kube-apiserver/server.crt +type: file +--- +contents: + task: + Name: master + alternateNames: + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - kubernetes.default.svc.cluster.local + - api.minimal.example.com + - api.internal.minimal.example.com + - 100.64.0.1 + - 127.0.0.1 + includeRootCertificate: true + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubernetes-master + type: server +mode: "0600" +path: /srv/kubernetes/kube-apiserver/server.key +type: file +--- contents: | -----BEGIN RSA PUBLIC KEY----- MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm @@ -358,6 +402,23 @@ subject: CommonName: kubelet-api type: client --- +Name: master +alternateNames: +- kubernetes +- kubernetes.default +- kubernetes.default.svc +- kubernetes.default.svc.cluster.local +- api.minimal.example.com +- api.internal.minimal.example.com +- 100.64.0.1 +- 127.0.0.1 +includeRootCertificate: true +keypairID: "3" +signer: kubernetes-ca +subject: + CommonName: kubernetes-master +type: server +--- Name: aws-iam-authenticator home: /srv/kubernetes/aws-iam-authenticator shell: /sbin/nologin diff --git a/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml index eac2a3f686..cfd4fa93e6 100644 --- a/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml +++ b/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml @@ -43,8 +43,8 @@ contents: | - --service-account-key-file=/srv/kubernetes/kube-apiserver/service-account.pub - --service-cluster-ip-range=100.64.0.0/13 - --storage-backend=etcd3 - - --tls-cert-file=/srv/kubernetes/server.crt - - --tls-private-key-file=/srv/kubernetes/server.key + - --tls-cert-file=/srv/kubernetes/kube-apiserver/server.crt + - --tls-private-key-file=/srv/kubernetes/kube-apiserver/server.key - --v=2 - --logtostderr=false - --alsologtostderr @@ -222,6 +222,50 @@ mode: "0600" path: /srv/kubernetes/kube-apiserver/etcd-client.key type: file --- +contents: + task: + Name: master + alternateNames: + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - kubernetes.default.svc.cluster.local + - api.minimal.example.com + - api.internal.minimal.example.com + - 100.64.0.1 + - 127.0.0.1 + includeRootCertificate: true + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubernetes-master + type: server +mode: "0644" +path: /srv/kubernetes/kube-apiserver/server.crt +type: file +--- +contents: + task: + Name: master + alternateNames: + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - kubernetes.default.svc.cluster.local + - api.minimal.example.com + - api.internal.minimal.example.com + - 100.64.0.1 + - 127.0.0.1 + includeRootCertificate: true + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubernetes-master + type: server +mode: "0600" +path: /srv/kubernetes/kube-apiserver/server.key +type: file +--- contents: | -----BEGIN RSA PUBLIC KEY----- MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm @@ -285,3 +329,20 @@ signer: kubernetes-ca subject: CommonName: kubelet-api type: client +--- +Name: master +alternateNames: +- kubernetes +- kubernetes.default +- kubernetes.default.svc +- kubernetes.default.svc.cluster.local +- api.minimal.example.com +- api.internal.minimal.example.com +- 100.64.0.1 +- 127.0.0.1 +includeRootCertificate: true +keypairID: "3" +signer: kubernetes-ca +subject: + CommonName: kubernetes-master +type: server diff --git a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml index 6919669054..26337065be 100644 --- a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml +++ b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml @@ -43,8 +43,8 @@ contents: | - --service-account-key-file=/srv/kubernetes/kube-apiserver/service-account.pub - --service-cluster-ip-range=100.64.0.0/13 - --storage-backend=etcd3 - - --tls-cert-file=/srv/kubernetes/server.crt - - --tls-private-key-file=/srv/kubernetes/server.key + - --tls-cert-file=/srv/kubernetes/kube-apiserver/server.crt + - --tls-private-key-file=/srv/kubernetes/kube-apiserver/server.key - --v=2 - --logtostderr=false - --alsologtostderr @@ -222,6 +222,50 @@ mode: "0600" path: /srv/kubernetes/kube-apiserver/etcd-client.key type: file --- +contents: + task: + Name: master + alternateNames: + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - kubernetes.default.svc.cluster.local + - api.minimal.example.com + - api.internal.minimal.example.com + - 100.64.0.1 + - 127.0.0.1 + includeRootCertificate: true + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubernetes-master + type: server +mode: "0644" +path: /srv/kubernetes/kube-apiserver/server.crt +type: file +--- +contents: + task: + Name: master + alternateNames: + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - kubernetes.default.svc.cluster.local + - api.minimal.example.com + - api.internal.minimal.example.com + - 100.64.0.1 + - 127.0.0.1 + includeRootCertificate: true + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubernetes-master + type: server +mode: "0600" +path: /srv/kubernetes/kube-apiserver/server.key +type: file +--- contents: | -----BEGIN RSA PUBLIC KEY----- MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm @@ -285,3 +329,20 @@ signer: kubernetes-ca subject: CommonName: kubelet-api type: client +--- +Name: master +alternateNames: +- kubernetes +- kubernetes.default +- kubernetes.default.svc +- kubernetes.default.svc.cluster.local +- api.minimal.example.com +- api.internal.minimal.example.com +- 100.64.0.1 +- 127.0.0.1 +includeRootCertificate: true +keypairID: "3" +signer: kubernetes-ca +subject: + CommonName: kubernetes-master +type: server diff --git a/nodeup/pkg/model/tests/golden/minimal/tasks-secret.yaml b/nodeup/pkg/model/tests/golden/minimal/tasks-secret.yaml index 08127ad960..3bcafb654d 100644 --- a/nodeup/pkg/model/tests/golden/minimal/tasks-secret.yaml +++ b/nodeup/pkg/model/tests/golden/minimal/tasks-secret.yaml @@ -1,7 +1,3 @@ -mode: "0755" -path: /srv/kubernetes -type: directory ---- contents: | -----BEGIN CERTIFICATE----- MIIC2DCCAcCgAwIBAgIRALJXAkVj964tq67wMSI8oJQwDQYJKoZIhvcNAQELBQAw @@ -34,64 +30,3 @@ contents: | mode: "0600" path: /srv/kubernetes/ca.crt type: file ---- -contents: - task: - Name: master - alternateNames: - - kubernetes - - kubernetes.default - - kubernetes.default.svc - - kubernetes.default.svc.cluster.local - - api.minimal.example.com - - api.internal.minimal.example.com - - 100.64.0.1 - - 127.0.0.1 - includeRootCertificate: true - keypairID: "3" - signer: kubernetes-ca - subject: - CommonName: kubernetes-master - type: server -mode: "0644" -path: /srv/kubernetes/server.crt -type: file ---- -contents: - task: - Name: master - alternateNames: - - kubernetes - - kubernetes.default - - kubernetes.default.svc - - kubernetes.default.svc.cluster.local - - api.minimal.example.com - - api.internal.minimal.example.com - - 100.64.0.1 - - 127.0.0.1 - includeRootCertificate: true - keypairID: "3" - signer: kubernetes-ca - subject: - CommonName: kubernetes-master - type: server -mode: "0600" -path: /srv/kubernetes/server.key -type: file ---- -Name: master -alternateNames: -- kubernetes -- kubernetes.default -- kubernetes.default.svc -- kubernetes.default.svc.cluster.local -- api.minimal.example.com -- api.internal.minimal.example.com -- 100.64.0.1 -- 127.0.0.1 -includeRootCertificate: true -keypairID: "3" -signer: kubernetes-ca -subject: - CommonName: kubernetes-master -type: server diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml index a82513c373..0bf86d9f8e 100644 --- a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml +++ b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml @@ -43,8 +43,8 @@ contents: | - --service-account-key-file=/srv/kubernetes/kube-apiserver/service-account.pub - --service-cluster-ip-range=100.64.0.0/13 - --storage-backend=etcd3 - - --tls-cert-file=/srv/kubernetes/server.crt - - --tls-private-key-file=/srv/kubernetes/server.key + - --tls-cert-file=/srv/kubernetes/kube-apiserver/server.crt + - --tls-private-key-file=/srv/kubernetes/kube-apiserver/server.key - --v=2 - --logtostderr=false - --alsologtostderr @@ -222,6 +222,50 @@ mode: "0600" path: /srv/kubernetes/kube-apiserver/etcd-client.key type: file --- +contents: + task: + Name: master + alternateNames: + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - kubernetes.default.svc.cluster.local + - api.minimal.example.com + - api.internal.minimal.example.com + - 100.64.0.1 + - 127.0.0.1 + includeRootCertificate: true + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubernetes-master + type: server +mode: "0644" +path: /srv/kubernetes/kube-apiserver/server.crt +type: file +--- +contents: + task: + Name: master + alternateNames: + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - kubernetes.default.svc.cluster.local + - api.minimal.example.com + - api.internal.minimal.example.com + - 100.64.0.1 + - 127.0.0.1 + includeRootCertificate: true + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubernetes-master + type: server +mode: "0600" +path: /srv/kubernetes/kube-apiserver/server.key +type: file +--- contents: | -----BEGIN RSA PUBLIC KEY----- MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm @@ -285,3 +329,20 @@ signer: kubernetes-ca subject: CommonName: kubelet-api type: client +--- +Name: master +alternateNames: +- kubernetes +- kubernetes.default +- kubernetes.default.svc +- kubernetes.default.svc.cluster.local +- api.minimal.example.com +- api.internal.minimal.example.com +- 100.64.0.1 +- 127.0.0.1 +includeRootCertificate: true +keypairID: "3" +signer: kubernetes-ca +subject: + CommonName: kubernetes-master +type: server diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml index 30422c0658..375a0ff7fc 100644 --- a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml +++ b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml @@ -43,8 +43,8 @@ contents: | - --service-account-key-file=/srv/kubernetes/kube-apiserver/service-account.pub - --service-cluster-ip-range=100.64.0.0/13 - --storage-backend=etcd3 - - --tls-cert-file=/srv/kubernetes/server.crt - - --tls-private-key-file=/srv/kubernetes/server.key + - --tls-cert-file=/srv/kubernetes/kube-apiserver/server.crt + - --tls-private-key-file=/srv/kubernetes/kube-apiserver/server.key - --v=2 - --logtostderr=false - --alsologtostderr @@ -222,6 +222,50 @@ mode: "0600" path: /srv/kubernetes/kube-apiserver/etcd-client.key type: file --- +contents: + task: + Name: master + alternateNames: + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - kubernetes.default.svc.cluster.local + - api.minimal.example.com + - api.internal.minimal.example.com + - 100.64.0.1 + - 127.0.0.1 + includeRootCertificate: true + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubernetes-master + type: server +mode: "0644" +path: /srv/kubernetes/kube-apiserver/server.crt +type: file +--- +contents: + task: + Name: master + alternateNames: + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - kubernetes.default.svc.cluster.local + - api.minimal.example.com + - api.internal.minimal.example.com + - 100.64.0.1 + - 127.0.0.1 + includeRootCertificate: true + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubernetes-master + type: server +mode: "0600" +path: /srv/kubernetes/kube-apiserver/server.key +type: file +--- contents: | -----BEGIN RSA PUBLIC KEY----- MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm @@ -285,3 +329,20 @@ signer: kubernetes-ca subject: CommonName: kubelet-api type: client +--- +Name: master +alternateNames: +- kubernetes +- kubernetes.default +- kubernetes.default.svc +- kubernetes.default.svc.cluster.local +- api.minimal.example.com +- api.internal.minimal.example.com +- 100.64.0.1 +- 127.0.0.1 +includeRootCertificate: true +keypairID: "3" +signer: kubernetes-ca +subject: + CommonName: kubernetes-master +type: server diff --git a/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml index 8fe68e0d27..8c49cf161c 100644 --- a/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml +++ b/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml @@ -42,8 +42,8 @@ contents: | - --service-account-key-file=/srv/kubernetes/kube-apiserver/service-account.pub - --service-cluster-ip-range=100.64.0.0/13 - --storage-backend=etcd3 - - --tls-cert-file=/srv/kubernetes/server.crt - - --tls-private-key-file=/srv/kubernetes/server.key + - --tls-cert-file=/srv/kubernetes/kube-apiserver/server.crt + - --tls-private-key-file=/srv/kubernetes/kube-apiserver/server.key - --v=2 - --logtostderr=false - --alsologtostderr @@ -221,6 +221,50 @@ mode: "0600" path: /srv/kubernetes/kube-apiserver/etcd-client.key type: file --- +contents: + task: + Name: master + alternateNames: + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - kubernetes.default.svc.cluster.local + - api.minimal.example.com + - api.internal.minimal.example.com + - 100.64.0.1 + - 127.0.0.1 + includeRootCertificate: true + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubernetes-master + type: server +mode: "0644" +path: /srv/kubernetes/kube-apiserver/server.crt +type: file +--- +contents: + task: + Name: master + alternateNames: + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - kubernetes.default.svc.cluster.local + - api.minimal.example.com + - api.internal.minimal.example.com + - 100.64.0.1 + - 127.0.0.1 + includeRootCertificate: true + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubernetes-master + type: server +mode: "0600" +path: /srv/kubernetes/kube-apiserver/server.key +type: file +--- contents: | -----BEGIN RSA PUBLIC KEY----- MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm @@ -284,3 +328,20 @@ signer: kubernetes-ca subject: CommonName: kubelet-api type: client +--- +Name: master +alternateNames: +- kubernetes +- kubernetes.default +- kubernetes.default.svc +- kubernetes.default.svc.cluster.local +- api.minimal.example.com +- api.internal.minimal.example.com +- 100.64.0.1 +- 127.0.0.1 +includeRootCertificate: true +keypairID: "3" +signer: kubernetes-ca +subject: + CommonName: kubernetes-master +type: server From 68bb8f5ddbca64b95bdf82f743c6a0119c43e02a Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Fri, 16 Jul 2021 22:55:50 -0700 Subject: [PATCH 3/5] Refactor kube-apiserver static credentials --- nodeup/pkg/model/kube_apiserver.go | 84 ++++++++++++++++++++++++++++-- nodeup/pkg/model/secrets.go | 75 -------------------------- 2 files changed, 79 insertions(+), 80 deletions(-) diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go index b35d038555..4226b9bfb9 100644 --- a/nodeup/pkg/model/kube_apiserver.go +++ b/nodeup/pkg/model/kube_apiserver.go @@ -27,6 +27,7 @@ import ( "k8s.io/kops/pkg/kubeconfig" "k8s.io/kops/pkg/kubemanifest" "k8s.io/kops/pkg/model/components" + "k8s.io/kops/pkg/tokens" "k8s.io/kops/pkg/wellknownports" "k8s.io/kops/pkg/wellknownusers" "k8s.io/kops/upup/pkg/fi" @@ -167,6 +168,10 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error { return err } + if err := b.writeStaticCredentials(c, &kubeAPIServer); err != nil { + return err + } + { pod, err := b.buildPod(&kubeAPIServer) if err != nil { @@ -425,21 +430,90 @@ func (b *KubeAPIServerBuilder) writeServerCertificate(c *fi.ModelBuilderContext, return nil } -// buildPod is responsible for generating the kube-apiserver pod and thus manifest file -func (b *KubeAPIServerBuilder) buildPod(kubeAPIServer *kops.KubeAPIServerConfig) (*v1.Pod, error) { +func (b *KubeAPIServerBuilder) writeStaticCredentials(c *fi.ModelBuilderContext, kubeAPIServer *kops.KubeAPIServerConfig) error { + pathSrvKAPI := filepath.Join(b.PathSrvKubernetes(), "kube-apiserver") + + // Support for basic auth was deprecated 1.16 and removed in 1.19 + // https://github.com/kubernetes/kubernetes/pull/89069 + if b.IsKubernetesLT("1.19") && b.SecretStore != nil { + key := "kube" + token, err := b.SecretStore.FindSecret(key) + if err != nil { + return err + } + if token == nil { + return fmt.Errorf("token not found: %q", key) + } + csv := string(token.Data) + "," + adminUser + "," + adminUser + "," + adminGroup + + t := &nodetasks.File{ + Path: filepath.Join(pathSrvKAPI, "basic_auth.csv"), + Contents: fi.NewStringResource(csv), + Type: nodetasks.FileType_File, + Mode: s("0600"), + } + c.AddTask(t) + } + + if b.SecretStore != nil { + allTokens, err := b.allAuthTokens() + if err != nil { + return err + } + + var lines []string + for id, token := range allTokens { + if id == adminUser { + lines = append(lines, token+","+id+","+id+","+adminGroup) + } else { + lines = append(lines, token+","+id+","+id) + } + } + csv := strings.Join(lines, "\n") + + c.AddTask(&nodetasks.File{ + Path: filepath.Join(pathSrvKAPI, "known_tokens.csv"), + Contents: fi.NewStringResource(csv), + Type: nodetasks.FileType_File, + Mode: s("0600"), + }) + } + // Support for basic auth was deprecated 1.16 and removed in 1.19 // https://github.com/kubernetes/kubernetes/pull/89069 if b.IsKubernetesLT("1.18") { - kubeAPIServer.TokenAuthFile = filepath.Join(b.PathSrvKubernetes(), "known_tokens.csv") + kubeAPIServer.TokenAuthFile = filepath.Join(pathSrvKAPI, "known_tokens.csv") if kubeAPIServer.DisableBasicAuth == nil || !*kubeAPIServer.DisableBasicAuth { - kubeAPIServer.BasicAuthFile = filepath.Join(b.PathSrvKubernetes(), "basic_auth.csv") + kubeAPIServer.BasicAuthFile = filepath.Join(pathSrvKAPI, "basic_auth.csv") } } else if b.IsKubernetesLT("1.19") { if kubeAPIServer.DisableBasicAuth != nil && !*kubeAPIServer.DisableBasicAuth { - kubeAPIServer.BasicAuthFile = filepath.Join(b.PathSrvKubernetes(), "basic_auth.csv") + kubeAPIServer.BasicAuthFile = filepath.Join(pathSrvKAPI, "basic_auth.csv") } } + return nil +} + +// allTokens returns a map of all auth tokens that are present +func (b *KubeAPIServerBuilder) allAuthTokens() (map[string]string, error) { + possibleTokens := tokens.GetKubernetesAuthTokens_Deprecated() + + tokens := make(map[string]string) + for _, id := range possibleTokens { + token, err := b.SecretStore.FindSecret(id) + if err != nil { + return nil, err + } + if token != nil { + tokens[id] = string(token.Data) + } + } + return tokens, nil +} + +// buildPod is responsible for generating the kube-apiserver pod and thus manifest file +func (b *KubeAPIServerBuilder) buildPod(kubeAPIServer *kops.KubeAPIServerConfig) (*v1.Pod, error) { // we need to replace 127.0.0.1 for etcd urls with the dns names in case this apiserver is not // running on master nodes if !b.IsMaster { diff --git a/nodeup/pkg/model/secrets.go b/nodeup/pkg/model/secrets.go index 8d1d23f4ff..b90e7da8c0 100644 --- a/nodeup/pkg/model/secrets.go +++ b/nodeup/pkg/model/secrets.go @@ -17,11 +17,8 @@ limitations under the License. package model import ( - "fmt" "path/filepath" - "strings" - "k8s.io/kops/pkg/tokens" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" "k8s.io/kops/util/pkg/vfs" @@ -41,10 +38,6 @@ const ( // Build is responsible for pulling down the secrets func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error { - if b.KeyStore == nil { - return fmt.Errorf("KeyStore not set") - } - // @step: write out the platform ca c.AddTask(&nodetasks.File{ Path: filepath.Join(b.PathSrvKubernetes(), "ca.crt"), @@ -68,77 +61,9 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error { } } - // If we do not run the Kubernetes API server we can stop here. - if !b.HasAPIServer { - return nil - } - - // Support for basic auth was deprecated 1.16 and removed in 1.19 - // https://github.com/kubernetes/kubernetes/pull/89069 - if b.IsKubernetesLT("1.19") && b.SecretStore != nil { - key := "kube" - token, err := b.SecretStore.FindSecret(key) - if err != nil { - return err - } - if token == nil { - return fmt.Errorf("token not found: %q", key) - } - csv := string(token.Data) + "," + adminUser + "," + adminUser + "," + adminGroup - - t := &nodetasks.File{ - Path: filepath.Join(b.PathSrvKubernetes(), "basic_auth.csv"), - Contents: fi.NewStringResource(csv), - Type: nodetasks.FileType_File, - Mode: s("0600"), - } - c.AddTask(t) - } - - if b.SecretStore != nil { - allTokens, err := b.allAuthTokens() - if err != nil { - return err - } - - var lines []string - for id, token := range allTokens { - if id == adminUser { - lines = append(lines, token+","+id+","+id+","+adminGroup) - } else { - lines = append(lines, token+","+id+","+id) - } - } - csv := strings.Join(lines, "\n") - - c.AddTask(&nodetasks.File{ - Path: filepath.Join(b.PathSrvKubernetes(), "known_tokens.csv"), - Contents: fi.NewStringResource(csv), - Type: nodetasks.FileType_File, - Mode: s("0600"), - }) - } - return nil } -// allTokens returns a map of all auth tokens that are present -func (b *SecretBuilder) allAuthTokens() (map[string]string, error) { - possibleTokens := tokens.GetKubernetesAuthTokens_Deprecated() - - tokens := make(map[string]string) - for _, id := range possibleTokens { - token, err := b.SecretStore.FindSecret(id) - if err != nil { - return nil, err - } - if token != nil { - tokens[id] = string(token.Data) - } - } - return tokens, nil -} - func getInstanceAddress() (string, error) { addrBytes, err := vfs.Context.ReadFile("metadata://openstack/local-ipv4") From 7c1ed8de6680f48d7b141095a7103a6334f9d3b2 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Fri, 16 Jul 2021 23:07:14 -0700 Subject: [PATCH 4/5] Refactor kube-apiserver kubelet-api certificate --- nodeup/pkg/model/kube_apiserver.go | 44 ++++++++++++++++++------------ 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go index 4226b9bfb9..ad223d04ed 100644 --- a/nodeup/pkg/model/kube_apiserver.go +++ b/nodeup/pkg/model/kube_apiserver.go @@ -168,6 +168,10 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error { return err } + if err := b.writeKubeletAPICertificate(c, &kubeAPIServer); err != nil { + return err + } + if err := b.writeStaticCredentials(c, &kubeAPIServer); err != nil { return err } @@ -197,19 +201,6 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error { } } - issueCert := &nodetasks.IssueCert{ - Name: "kubelet-api", - Signer: fi.CertificateIDCA, - KeypairID: b.NodeupConfig.KeypairIDs[fi.CertificateIDCA], - Type: "client", - Subject: nodetasks.PKIXName{CommonName: "kubelet-api"}, - } - c.AddTask(issueCert) - err := issueCert.AddFileTasks(c, b.PathSrvKubernetes(), "kubelet-api", "", nil) - if err != nil { - return err - } - c.AddTask(&nodetasks.File{ Path: "/var/log/kube-apiserver.log", Contents: fi.NewStringResource(""), @@ -430,6 +421,29 @@ func (b *KubeAPIServerBuilder) writeServerCertificate(c *fi.ModelBuilderContext, return nil } +func (b *KubeAPIServerBuilder) writeKubeletAPICertificate(c *fi.ModelBuilderContext, kubeAPIServer *kops.KubeAPIServerConfig) error { + pathSrvKAPI := filepath.Join(b.PathSrvKubernetes(), "kube-apiserver") + + issueCert := &nodetasks.IssueCert{ + Name: "kubelet-api", + Signer: fi.CertificateIDCA, + KeypairID: b.NodeupConfig.KeypairIDs[fi.CertificateIDCA], + Type: "client", + Subject: nodetasks.PKIXName{CommonName: "kubelet-api"}, + } + c.AddTask(issueCert) + err := issueCert.AddFileTasks(c, pathSrvKAPI, "kubelet-api", "", nil) + if err != nil { + return err + } + + // @note we are making assumption were using the ones created by the pki model, not custom defined ones + kubeAPIServer.KubeletClientCertificate = filepath.Join(pathSrvKAPI, "kubelet-api.crt") + kubeAPIServer.KubeletClientKey = filepath.Join(pathSrvKAPI, "kubelet-api.key") + + return nil +} + func (b *KubeAPIServerBuilder) writeStaticCredentials(c *fi.ModelBuilderContext, kubeAPIServer *kops.KubeAPIServerConfig) error { pathSrvKAPI := filepath.Join(b.PathSrvKubernetes(), "kube-apiserver") @@ -530,10 +544,6 @@ func (b *KubeAPIServerBuilder) buildPod(kubeAPIServer *kops.KubeAPIServerConfig) } } - // @note we are making assumption were using the ones created by the pki model, not custom defined ones - kubeAPIServer.KubeletClientCertificate = filepath.Join(b.PathSrvKubernetes(), "kubelet-api.crt") - kubeAPIServer.KubeletClientKey = filepath.Join(b.PathSrvKubernetes(), "kubelet-api.key") - // @fixup: the admission controller migrated from --admission-control to --enable-admission-plugins, but // most people will still have c.Spec.KubeAPIServer.AdmissionControl references into their configuration we need // to fix up. A PR https://github.com/kubernetes/kops/pull/5221/ introduced the issue and since the command line From 12c988160c16b37a1abd7570cf1088205b81a6b4 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Fri, 16 Jul 2021 23:12:22 -0700 Subject: [PATCH 5/5] hack/update-expected.sh --- .../golden/awsiam/tasks-kube-apiserver.yaml | 56 +++++++++---------- .../tasks-kube-apiserver.yaml | 56 +++++++++---------- .../golden/minimal/tasks-kube-apiserver.yaml | 56 +++++++++---------- .../tasks-kube-apiserver-amd64.yaml | 56 +++++++++---------- .../tasks-kube-apiserver-arm64.yaml | 56 +++++++++---------- .../tasks-kube-apiserver.yaml | 56 +++++++++---------- 6 files changed, 156 insertions(+), 180 deletions(-) diff --git a/nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml index c20ebe3a7a..6f896fb16a 100644 --- a/nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml +++ b/nodeup/pkg/model/tests/golden/awsiam/tasks-kube-apiserver.yaml @@ -51,8 +51,8 @@ contents: | - --etcd-servers-overrides=/events#https://127.0.0.1:4002 - --etcd-servers=https://127.0.0.1:4001 - --insecure-port=0 - - --kubelet-client-certificate=/srv/kubernetes/kubelet-api.crt - - --kubelet-client-key=/srv/kubernetes/kubelet-api.key + - --kubelet-client-certificate=/srv/kubernetes/kube-apiserver/kubelet-api.crt + - --kubelet-client-key=/srv/kubernetes/kube-apiserver/kubelet-api.key - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP - --proxy-client-cert-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.crt - --proxy-client-key-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.key @@ -184,10 +184,6 @@ contents: | path: /etc/kubernetes/manifests/kube-apiserver.manifest type: file --- -mode: "0755" -path: /srv/kubernetes -type: directory ---- contents: task: Name: aws-iam-authenticator @@ -284,6 +280,30 @@ mode: "0600" path: /srv/kubernetes/kube-apiserver/etcd-client.key type: file --- +contents: + task: + Name: kubelet-api + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubelet-api + type: client +mode: "0644" +path: /srv/kubernetes/kube-apiserver/kubelet-api.crt +type: file +--- +contents: + task: + Name: kubelet-api + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubelet-api + type: client +mode: "0600" +path: /srv/kubernetes/kube-apiserver/kubelet-api.key +type: file +--- contents: task: Name: master @@ -341,30 +361,6 @@ mode: "0600" path: /srv/kubernetes/kube-apiserver/service-account.pub type: file --- -contents: - task: - Name: kubelet-api - keypairID: "3" - signer: kubernetes-ca - subject: - CommonName: kubelet-api - type: client -mode: "0644" -path: /srv/kubernetes/kubelet-api.crt -type: file ---- -contents: - task: - Name: kubelet-api - keypairID: "3" - signer: kubernetes-ca - subject: - CommonName: kubelet-api - type: client -mode: "0600" -path: /srv/kubernetes/kubelet-api.key -type: file ---- contents: "" ifNotExists: true mode: "0400" diff --git a/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml index cfd4fa93e6..40076b8a30 100644 --- a/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml +++ b/nodeup/pkg/model/tests/golden/dedicated-apiserver/tasks-kube-apiserver.yaml @@ -29,8 +29,8 @@ contents: | - --etcd-servers-overrides=/events#https://events.etcd.minimal.example.com:4002 - --etcd-servers=https://main.etcd.minimal.example.com:4001 - --insecure-port=0 - - --kubelet-client-certificate=/srv/kubernetes/kubelet-api.crt - - --kubelet-client-key=/srv/kubernetes/kubelet-api.key + - --kubelet-client-certificate=/srv/kubernetes/kube-apiserver/kubelet-api.crt + - --kubelet-client-key=/srv/kubernetes/kube-apiserver/kubelet-api.key - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP - --proxy-client-cert-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.crt - --proxy-client-key-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.key @@ -157,10 +157,6 @@ path: /etc/kubernetes/manifests/kube-apiserver.manifest type: file --- mode: "0755" -path: /srv/kubernetes -type: directory ---- -mode: "0755" path: /srv/kubernetes/kube-apiserver type: directory --- @@ -222,6 +218,30 @@ mode: "0600" path: /srv/kubernetes/kube-apiserver/etcd-client.key type: file --- +contents: + task: + Name: kubelet-api + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubelet-api + type: client +mode: "0644" +path: /srv/kubernetes/kube-apiserver/kubelet-api.crt +type: file +--- +contents: + task: + Name: kubelet-api + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubelet-api + type: client +mode: "0600" +path: /srv/kubernetes/kube-apiserver/kubelet-api.key +type: file +--- contents: task: Name: master @@ -279,30 +299,6 @@ mode: "0600" path: /srv/kubernetes/kube-apiserver/service-account.pub type: file --- -contents: - task: - Name: kubelet-api - keypairID: "3" - signer: kubernetes-ca - subject: - CommonName: kubelet-api - type: client -mode: "0644" -path: /srv/kubernetes/kubelet-api.crt -type: file ---- -contents: - task: - Name: kubelet-api - keypairID: "3" - signer: kubernetes-ca - subject: - CommonName: kubelet-api - type: client -mode: "0600" -path: /srv/kubernetes/kubelet-api.key -type: file ---- contents: "" ifNotExists: true mode: "0400" diff --git a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml index 26337065be..fefdd9f8d1 100644 --- a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml +++ b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-apiserver.yaml @@ -29,8 +29,8 @@ contents: | - --etcd-servers-overrides=/events#https://127.0.0.1:4002 - --etcd-servers=https://127.0.0.1:4001 - --insecure-port=0 - - --kubelet-client-certificate=/srv/kubernetes/kubelet-api.crt - - --kubelet-client-key=/srv/kubernetes/kubelet-api.key + - --kubelet-client-certificate=/srv/kubernetes/kube-apiserver/kubelet-api.crt + - --kubelet-client-key=/srv/kubernetes/kube-apiserver/kubelet-api.key - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP - --proxy-client-cert-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.crt - --proxy-client-key-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.key @@ -157,10 +157,6 @@ path: /etc/kubernetes/manifests/kube-apiserver.manifest type: file --- mode: "0755" -path: /srv/kubernetes -type: directory ---- -mode: "0755" path: /srv/kubernetes/kube-apiserver type: directory --- @@ -222,6 +218,30 @@ mode: "0600" path: /srv/kubernetes/kube-apiserver/etcd-client.key type: file --- +contents: + task: + Name: kubelet-api + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubelet-api + type: client +mode: "0644" +path: /srv/kubernetes/kube-apiserver/kubelet-api.crt +type: file +--- +contents: + task: + Name: kubelet-api + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubelet-api + type: client +mode: "0600" +path: /srv/kubernetes/kube-apiserver/kubelet-api.key +type: file +--- contents: task: Name: master @@ -279,30 +299,6 @@ mode: "0600" path: /srv/kubernetes/kube-apiserver/service-account.pub type: file --- -contents: - task: - Name: kubelet-api - keypairID: "3" - signer: kubernetes-ca - subject: - CommonName: kubelet-api - type: client -mode: "0644" -path: /srv/kubernetes/kubelet-api.crt -type: file ---- -contents: - task: - Name: kubelet-api - keypairID: "3" - signer: kubernetes-ca - subject: - CommonName: kubelet-api - type: client -mode: "0600" -path: /srv/kubernetes/kubelet-api.key -type: file ---- contents: "" ifNotExists: true mode: "0400" diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml index 0bf86d9f8e..7e400e721d 100644 --- a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml +++ b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml @@ -29,8 +29,8 @@ contents: | - --etcd-servers-overrides=/events#https://127.0.0.1:4002 - --etcd-servers=https://127.0.0.1:4001 - --insecure-port=0 - - --kubelet-client-certificate=/srv/kubernetes/kubelet-api.crt - - --kubelet-client-key=/srv/kubernetes/kubelet-api.key + - --kubelet-client-certificate=/srv/kubernetes/kube-apiserver/kubelet-api.crt + - --kubelet-client-key=/srv/kubernetes/kube-apiserver/kubelet-api.key - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP - --proxy-client-cert-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.crt - --proxy-client-key-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.key @@ -157,10 +157,6 @@ path: /etc/kubernetes/manifests/kube-apiserver.manifest type: file --- mode: "0755" -path: /srv/kubernetes -type: directory ---- -mode: "0755" path: /srv/kubernetes/kube-apiserver type: directory --- @@ -222,6 +218,30 @@ mode: "0600" path: /srv/kubernetes/kube-apiserver/etcd-client.key type: file --- +contents: + task: + Name: kubelet-api + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubelet-api + type: client +mode: "0644" +path: /srv/kubernetes/kube-apiserver/kubelet-api.crt +type: file +--- +contents: + task: + Name: kubelet-api + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubelet-api + type: client +mode: "0600" +path: /srv/kubernetes/kube-apiserver/kubelet-api.key +type: file +--- contents: task: Name: master @@ -279,30 +299,6 @@ mode: "0600" path: /srv/kubernetes/kube-apiserver/service-account.pub type: file --- -contents: - task: - Name: kubelet-api - keypairID: "3" - signer: kubernetes-ca - subject: - CommonName: kubelet-api - type: client -mode: "0644" -path: /srv/kubernetes/kubelet-api.crt -type: file ---- -contents: - task: - Name: kubelet-api - keypairID: "3" - signer: kubernetes-ca - subject: - CommonName: kubelet-api - type: client -mode: "0600" -path: /srv/kubernetes/kubelet-api.key -type: file ---- contents: "" ifNotExists: true mode: "0400" diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml index 375a0ff7fc..dd682eb84b 100644 --- a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml +++ b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml @@ -29,8 +29,8 @@ contents: | - --etcd-servers-overrides=/events#https://127.0.0.1:4002 - --etcd-servers=https://127.0.0.1:4001 - --insecure-port=0 - - --kubelet-client-certificate=/srv/kubernetes/kubelet-api.crt - - --kubelet-client-key=/srv/kubernetes/kubelet-api.key + - --kubelet-client-certificate=/srv/kubernetes/kube-apiserver/kubelet-api.crt + - --kubelet-client-key=/srv/kubernetes/kube-apiserver/kubelet-api.key - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP - --proxy-client-cert-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.crt - --proxy-client-key-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.key @@ -157,10 +157,6 @@ path: /etc/kubernetes/manifests/kube-apiserver.manifest type: file --- mode: "0755" -path: /srv/kubernetes -type: directory ---- -mode: "0755" path: /srv/kubernetes/kube-apiserver type: directory --- @@ -222,6 +218,30 @@ mode: "0600" path: /srv/kubernetes/kube-apiserver/etcd-client.key type: file --- +contents: + task: + Name: kubelet-api + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubelet-api + type: client +mode: "0644" +path: /srv/kubernetes/kube-apiserver/kubelet-api.crt +type: file +--- +contents: + task: + Name: kubelet-api + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubelet-api + type: client +mode: "0600" +path: /srv/kubernetes/kube-apiserver/kubelet-api.key +type: file +--- contents: task: Name: master @@ -279,30 +299,6 @@ mode: "0600" path: /srv/kubernetes/kube-apiserver/service-account.pub type: file --- -contents: - task: - Name: kubelet-api - keypairID: "3" - signer: kubernetes-ca - subject: - CommonName: kubelet-api - type: client -mode: "0644" -path: /srv/kubernetes/kubelet-api.crt -type: file ---- -contents: - task: - Name: kubelet-api - keypairID: "3" - signer: kubernetes-ca - subject: - CommonName: kubelet-api - type: client -mode: "0600" -path: /srv/kubernetes/kubelet-api.key -type: file ---- contents: "" ifNotExists: true mode: "0400" diff --git a/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml b/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml index 8c49cf161c..b8b782f9ca 100644 --- a/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml +++ b/nodeup/pkg/model/tests/golden/without-etcd-events/tasks-kube-apiserver.yaml @@ -28,8 +28,8 @@ contents: | - --etcd-keyfile=/srv/kubernetes/kube-apiserver/etcd-client.key - --etcd-servers=https://127.0.0.1:4001 - --insecure-port=0 - - --kubelet-client-certificate=/srv/kubernetes/kubelet-api.crt - - --kubelet-client-key=/srv/kubernetes/kubelet-api.key + - --kubelet-client-certificate=/srv/kubernetes/kube-apiserver/kubelet-api.crt + - --kubelet-client-key=/srv/kubernetes/kube-apiserver/kubelet-api.key - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP - --proxy-client-cert-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.crt - --proxy-client-key-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.key @@ -156,10 +156,6 @@ path: /etc/kubernetes/manifests/kube-apiserver.manifest type: file --- mode: "0755" -path: /srv/kubernetes -type: directory ---- -mode: "0755" path: /srv/kubernetes/kube-apiserver type: directory --- @@ -221,6 +217,30 @@ mode: "0600" path: /srv/kubernetes/kube-apiserver/etcd-client.key type: file --- +contents: + task: + Name: kubelet-api + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubelet-api + type: client +mode: "0644" +path: /srv/kubernetes/kube-apiserver/kubelet-api.crt +type: file +--- +contents: + task: + Name: kubelet-api + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kubelet-api + type: client +mode: "0600" +path: /srv/kubernetes/kube-apiserver/kubelet-api.key +type: file +--- contents: task: Name: master @@ -278,30 +298,6 @@ mode: "0600" path: /srv/kubernetes/kube-apiserver/service-account.pub type: file --- -contents: - task: - Name: kubelet-api - keypairID: "3" - signer: kubernetes-ca - subject: - CommonName: kubelet-api - type: client -mode: "0644" -path: /srv/kubernetes/kubelet-api.crt -type: file ---- -contents: - task: - Name: kubelet-api - keypairID: "3" - signer: kubernetes-ca - subject: - CommonName: kubelet-api - type: client -mode: "0600" -path: /srv/kubernetes/kubelet-api.key -type: file ---- contents: "" ifNotExists: true mode: "0400"