From 3a53fdb139929ef81a7a7fea1337153b47b47426 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Tue, 20 Jul 2021 17:25:54 -0700 Subject: [PATCH 1/3] Provision TLS server certs for controller-manager and scheduler --- k8s/crds/kops.k8s.io_clusters.yaml | 16 +++++ nodeup/pkg/model/kube_controller_manager.go | 52 +++++++++++++--- nodeup/pkg/model/kube_scheduler.go | 60 +++++++++++++++---- pkg/apis/kops/componentconfig.go | 8 +++ pkg/apis/kops/v1alpha2/componentconfig.go | 8 +++ .../kops/v1alpha2/zz_generated.conversion.go | 8 +++ .../kops/v1alpha2/zz_generated.deepcopy.go | 10 ++++ pkg/apis/kops/zz_generated.deepcopy.go | 10 ++++ 8 files changed, 153 insertions(+), 19 deletions(-) diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 7cacb0dcb4..24e1e461bb 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -1973,6 +1973,10 @@ spec: garbage collector is disabled. format: int32 type: integer + tlsCertFile: + description: TLSCertFile is the file containing the TLS server + certificate. + type: string tlsCipherSuites: description: TLSCipherSuites indicates the allowed TLS cipher suite @@ -1982,6 +1986,10 @@ spec: tlsMinVersion: description: TLSMinVersion indicates the minimum TLS version allowed type: string + tlsPrivateKeyFile: + description: TLSPrivateKeyFile is the file containing the private + key for the TLS server certificate. + type: string useServiceAccountCredentials: description: UseServiceAccountCredentials controls whether we use individual service account credentials for each controller. @@ -2295,6 +2303,14 @@ spec: the burst quota is exhausted pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true + tlsCertFile: + description: TLSCertFile is the file containing the TLS server + certificate. + type: string + tlsPrivateKeyFile: + description: TLSPrivateKeyFile is the file containing the private + key for the TLS server certificate. + type: string usePolicyConfigMap: description: UsePolicyConfigMap enable setting the scheduler policy from a configmap diff --git a/nodeup/pkg/model/kube_controller_manager.go b/nodeup/pkg/model/kube_controller_manager.go index 190cdbaebc..dd36de3214 100644 --- a/nodeup/pkg/model/kube_controller_manager.go +++ b/nodeup/pkg/model/kube_controller_manager.go @@ -21,6 +21,7 @@ import ( "path/filepath" "strings" + "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/flagbuilder" "k8s.io/kops/pkg/k8scodecs" "k8s.io/kops/pkg/kubemanifest" @@ -52,6 +53,9 @@ func (b *KubeControllerManagerBuilder) Build(c *fi.ModelBuilderContext) error { pathSrvKCM := filepath.Join(b.PathSrvKubernetes(), "kube-controller-manager") + kcm := *b.Cluster.Spec.KubeControllerManager + kcm.RootCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt") + // Include the CA Key // @TODO: use a per-machine key? use KMS? if err := b.BuildCertificatePairTask(c, fi.CertificateIDCA, pathSrvKCM, "ca", nil, nil); err != nil { @@ -61,9 +65,14 @@ func (b *KubeControllerManagerBuilder) Build(c *fi.ModelBuilderContext) error { if err := b.BuildPrivateKeyTask(c, "service-account", pathSrvKCM, "service-account", nil, nil); err != nil { return err } + kcm.ServiceAccountPrivateKeyFile = filepath.Join(pathSrvKCM, "service-account.key") + + if err := b.writeServerCertificate(c, &kcm); err != nil { + return err + } { - pod, err := b.buildPod() + pod, err := b.buildPod(&kcm) if err != nil { return fmt.Errorf("error building kube-controller-manager pod: %v", err) } @@ -104,14 +113,39 @@ func (b *KubeControllerManagerBuilder) Build(c *fi.ModelBuilderContext) error { return nil } -// buildPod is responsible for building the kubernetes manifest for the controller-manager -func (b *KubeControllerManagerBuilder) buildPod() (*v1.Pod, error) { - pathSrvKubernetes := b.PathSrvKubernetes() - pathSrvKCM := filepath.Join(pathSrvKubernetes, "kube-controller-manager") +func (b *KubeControllerManagerBuilder) writeServerCertificate(c *fi.ModelBuilderContext, kcm *kops.KubeControllerManagerConfig) error { + pathSrvKCM := filepath.Join(b.PathSrvKubernetes(), "kube-controller-manager") - kcm := b.Cluster.Spec.KubeControllerManager - kcm.RootCAFile = filepath.Join(pathSrvKubernetes, "ca.crt") - kcm.ServiceAccountPrivateKeyFile = filepath.Join(pathSrvKCM, "service-account.key") + if kcm.TLSCertFile == nil { + alternateNames := []string{ + "kube-controller-manager.kube-system.svc." + b.Cluster.Spec.ClusterDNSDomain, + } + + issueCert := &nodetasks.IssueCert{ + Name: "kube-controller-manager-server", + Signer: fi.CertificateIDCA, + KeypairID: b.NodeupConfig.KeypairIDs[fi.CertificateIDCA], + Type: "server", + Subject: nodetasks.PKIXName{CommonName: "kube-controller-manager"}, + AlternateNames: alternateNames, + } + + c.AddTask(issueCert) + err := issueCert.AddFileTasks(c, pathSrvKCM, "server", "", nil) + if err != nil { + return err + } + + kcm.TLSCertFile = fi.String(filepath.Join(pathSrvKCM, "server.crt")) + kcm.TLSPrivateKeyFile = filepath.Join(pathSrvKCM, "server.key") + } + + return nil +} + +// buildPod is responsible for building the kubernetes manifest for the controller-manager +func (b *KubeControllerManagerBuilder) buildPod(kcm *kops.KubeControllerManagerConfig) (*v1.Pod, error) { + pathSrvKCM := filepath.Join(b.PathSrvKubernetes(), "kube-controller-manager") flags, err := flagbuilder.BuildFlagsList(kcm) if err != nil { @@ -220,7 +254,7 @@ func (b *KubeControllerManagerBuilder) buildPod() (*v1.Pod, error) { addHostPathMapping(pod, container, "cloudconfig", CloudConfigFilePath) } - addHostPathMapping(pod, container, "cabundle", filepath.Join(pathSrvKubernetes, "ca.crt")) + addHostPathMapping(pod, container, "cabundle", filepath.Join(b.PathSrvKubernetes(), "ca.crt")) addHostPathMapping(pod, container, "srvkcm", pathSrvKCM) diff --git a/nodeup/pkg/model/kube_scheduler.go b/nodeup/pkg/model/kube_scheduler.go index 30b31a6247..5aadc3603b 100644 --- a/nodeup/pkg/model/kube_scheduler.go +++ b/nodeup/pkg/model/kube_scheduler.go @@ -18,9 +18,11 @@ package model import ( "fmt" + "path/filepath" "strconv" "strings" + "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/configbuilder" "k8s.io/kops/pkg/flagbuilder" "k8s.io/kops/pkg/k8scodecs" @@ -65,8 +67,15 @@ func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error { if !b.IsMaster { return nil } + + kubeScheduler := *b.Cluster.Spec.KubeScheduler + + if err := b.writeServerCertificate(c, &kubeScheduler); err != nil { + return err + } + { - pod, err := b.buildPod() + pod, err := b.buildPod(&kubeScheduler) if err != nil { return fmt.Errorf("error building kube-scheduler pod: %v", err) } @@ -103,7 +112,7 @@ func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error { config = NewSchedulerConfig("kubescheduler.config.k8s.io/v1alpha1") } - manifest, err := configbuilder.BuildConfigYaml(b.Cluster.Spec.KubeScheduler, config) + manifest, err := configbuilder.BuildConfigYaml(&kubeScheduler, config) if err != nil { return err } @@ -139,11 +148,41 @@ func NewSchedulerConfig(apiVersion string) *SchedulerConfig { return schedConfig } -// buildPod is responsible for constructing the pod specification -func (b *KubeSchedulerBuilder) buildPod() (*v1.Pod, error) { - c := b.Cluster.Spec.KubeScheduler +func (b *KubeSchedulerBuilder) writeServerCertificate(c *fi.ModelBuilderContext, kubeScheduler *kops.KubeSchedulerConfig) error { + pathSrvScheduler := filepath.Join(b.PathSrvKubernetes(), "kube-scheduler") - flags, err := flagbuilder.BuildFlagsList(c) + if kubeScheduler.TLSCertFile == nil { + alternateNames := []string{ + "kube-scheduler.kube-system.svc." + b.Cluster.Spec.ClusterDNSDomain, + } + + issueCert := &nodetasks.IssueCert{ + Name: "kube-scheduler-server", + Signer: fi.CertificateIDCA, + KeypairID: b.NodeupConfig.KeypairIDs[fi.CertificateIDCA], + Type: "server", + Subject: nodetasks.PKIXName{CommonName: "kube-scheduler"}, + AlternateNames: alternateNames, + } + + c.AddTask(issueCert) + err := issueCert.AddFileTasks(c, pathSrvScheduler, "server", "", nil) + if err != nil { + return err + } + + kubeScheduler.TLSCertFile = fi.String(filepath.Join(pathSrvScheduler, "server.crt")) + kubeScheduler.TLSPrivateKeyFile = filepath.Join(pathSrvScheduler, "server.key") + } + + return nil +} + +// buildPod is responsible for constructing the pod specification +func (b *KubeSchedulerBuilder) buildPod(kubeScheduler *kops.KubeSchedulerConfig) (*v1.Pod, error) { + pathSrvScheduler := filepath.Join(b.PathSrvKubernetes(), "kube-scheduler") + + flags, err := flagbuilder.BuildFlagsList(kubeScheduler) if err != nil { return nil, fmt.Errorf("error building kube-scheduler flags: %v", err) } @@ -155,7 +194,7 @@ func (b *KubeSchedulerBuilder) buildPod() (*v1.Pod, error) { flags = append(flags, "--"+flag+"kubeconfig="+defaultKubeConfig) } - if c.UsePolicyConfigMap != nil { + if kubeScheduler.UsePolicyConfigMap != nil { flags = append(flags, "--policy-configmap=scheduler-policy", "--policy-configmap-namespace=kube-system") } @@ -176,7 +215,7 @@ func (b *KubeSchedulerBuilder) buildPod() (*v1.Pod, error) { }, } - image := c.Image + image := kubeScheduler.Image if b.Architecture != architectures.ArchitectureAmd64 { image = strings.Replace(image, "-amd64", "-"+string(b.Architecture), 1) } @@ -203,6 +242,7 @@ func (b *KubeSchedulerBuilder) buildPod() (*v1.Pod, error) { }, } addHostPathMapping(pod, container, "varlibkubescheduler", "/var/lib/kube-scheduler") + addHostPathMapping(pod, container, "srvscheduler", pathSrvScheduler) // Log both to docker and to the logfile addHostPathMapping(pod, container, "logfile", "/var/log/kube-scheduler.log").ReadOnly = false @@ -215,10 +255,10 @@ func (b *KubeSchedulerBuilder) buildPod() (*v1.Pod, error) { "--alsologtostderr", "--log-file=/var/log/kube-scheduler.log") - if c.MaxPersistentVolumes != nil { + if kubeScheduler.MaxPersistentVolumes != nil { maxPDV := v1.EnvVar{ Name: "KUBE_MAX_PD_VOLS", // https://kubernetes.io/docs/concepts/storage/storage-limits/ - Value: strconv.Itoa(int(*c.MaxPersistentVolumes)), + Value: strconv.Itoa(int(*kubeScheduler.MaxPersistentVolumes)), } container.Env = append(container.Env, maxPDV) } diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index 6a64625d86..113e5b1f74 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -604,10 +604,14 @@ type KubeControllerManagerConfig struct { ExperimentalClusterSigningDuration *metav1.Duration `json:"experimentalClusterSigningDuration,omitempty" flag:"experimental-cluster-signing-duration"` // FeatureGates is set of key=value pairs that describe feature gates for alpha/experimental features. FeatureGates map[string]string `json:"featureGates,omitempty" flag:"feature-gates"` + // TLSCertFile is the file containing the TLS server certificate. + TLSCertFile *string `json:"tlsCertFile,omitempty" flag:"tls-cert-file"` // TLSCipherSuites indicates the allowed TLS cipher suite TLSCipherSuites []string `json:"tlsCipherSuites,omitempty" flag:"tls-cipher-suites"` // TLSMinVersion indicates the minimum TLS version allowed TLSMinVersion string `json:"tlsMinVersion,omitempty" flag:"tls-min-version"` + // TLSPrivateKeyFile is the file containing the private key for the TLS server certificate. + TLSPrivateKeyFile string `json:"tlsPrivateKeyFile,omitempty" flag:"tls-private-key-file"` // MinResyncPeriod indicates the resync period in reflectors. // The resync period will be random between MinResyncPeriod and 2*MinResyncPeriod. (default 12h0m0s) MinResyncPeriod string `json:"minResyncPeriod,omitempty" flag:"min-resync-period"` @@ -709,6 +713,10 @@ type KubeSchedulerConfig struct { // EnableProfiling enables profiling via web interface host:port/debug/pprof/ EnableProfiling *bool `json:"enableProfiling,omitempty" flag:"profiling"` + // TLSCertFile is the file containing the TLS server certificate. + TLSCertFile *string `json:"tlsCertFile,omitempty" flag:"tls-cert-file"` + // TLSPrivateKeyFile is the file containing the private key for the TLS server certificate. + TLSPrivateKeyFile string `json:"tlsPrivateKeyFile,omitempty" flag:"tls-private-key-file"` } // LeaderElectionConfiguration defines the configuration of leader election diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go index 6353909b63..5f9f8a8a33 100644 --- a/pkg/apis/kops/v1alpha2/componentconfig.go +++ b/pkg/apis/kops/v1alpha2/componentconfig.go @@ -604,10 +604,14 @@ type KubeControllerManagerConfig struct { ExperimentalClusterSigningDuration *metav1.Duration `json:"experimentalClusterSigningDuration,omitempty" flag:"experimental-cluster-signing-duration"` // FeatureGates is set of key=value pairs that describe feature gates for alpha/experimental features. FeatureGates map[string]string `json:"featureGates,omitempty" flag:"feature-gates"` + // TLSCertFile is the file containing the TLS server certificate. + TLSCertFile *string `json:"tlsCertFile,omitempty" flag:"tls-cert-file"` // TLSCipherSuites indicates the allowed TLS cipher suite TLSCipherSuites []string `json:"tlsCipherSuites,omitempty" flag:"tls-cipher-suites"` // TLSMinVersion indicates the minimum TLS version allowed TLSMinVersion string `json:"tlsMinVersion,omitempty" flag:"tls-min-version"` + // TLSPrivateKeyFile is the file containing the private key for the TLS server certificate. + TLSPrivateKeyFile string `json:"tlsPrivateKeyFile,omitempty" flag:"tls-private-key-file"` // MinResyncPeriod indicates the resync period in reflectors. // The resync period will be random between MinResyncPeriod and 2*MinResyncPeriod. (default 12h0m0s) MinResyncPeriod string `json:"minResyncPeriod,omitempty" flag:"min-resync-period"` @@ -708,6 +712,10 @@ type KubeSchedulerConfig struct { // EnableProfiling enables profiling via web interface host:port/debug/pprof/ EnableProfiling *bool `json:"enableProfiling,omitempty" flag:"profiling"` + // TLSCertFile is the file containing the TLS server certificate. + TLSCertFile *string `json:"tlsCertFile,omitempty" flag:"tls-cert-file"` + // TLSPrivateKeyFile is the file containing the private key for the TLS server certificate. + TLSPrivateKeyFile string `json:"tlsPrivateKeyFile,omitempty" flag:"tls-private-key-file"` } // LeaderElectionConfiguration defines the configuration of leader election diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 6ecf83083c..06ffd87afa 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -4851,8 +4851,10 @@ func autoConvert_v1alpha2_KubeControllerManagerConfig_To_kops_KubeControllerMana out.HorizontalPodAutoscalerUseRestClients = in.HorizontalPodAutoscalerUseRestClients out.ExperimentalClusterSigningDuration = in.ExperimentalClusterSigningDuration out.FeatureGates = in.FeatureGates + out.TLSCertFile = in.TLSCertFile out.TLSCipherSuites = in.TLSCipherSuites out.TLSMinVersion = in.TLSMinVersion + out.TLSPrivateKeyFile = in.TLSPrivateKeyFile out.MinResyncPeriod = in.MinResyncPeriod out.KubeAPIQPS = in.KubeAPIQPS out.KubeAPIBurst = in.KubeAPIBurst @@ -4918,8 +4920,10 @@ func autoConvert_kops_KubeControllerManagerConfig_To_v1alpha2_KubeControllerMana out.HorizontalPodAutoscalerUseRestClients = in.HorizontalPodAutoscalerUseRestClients out.ExperimentalClusterSigningDuration = in.ExperimentalClusterSigningDuration out.FeatureGates = in.FeatureGates + out.TLSCertFile = in.TLSCertFile out.TLSCipherSuites = in.TLSCipherSuites out.TLSMinVersion = in.TLSMinVersion + out.TLSPrivateKeyFile = in.TLSPrivateKeyFile out.MinResyncPeriod = in.MinResyncPeriod out.KubeAPIQPS = in.KubeAPIQPS out.KubeAPIBurst = in.KubeAPIBurst @@ -5091,6 +5095,8 @@ func autoConvert_v1alpha2_KubeSchedulerConfig_To_kops_KubeSchedulerConfig(in *Ku out.AuthorizationKubeconfig = in.AuthorizationKubeconfig out.AuthorizationAlwaysAllowPaths = in.AuthorizationAlwaysAllowPaths out.EnableProfiling = in.EnableProfiling + out.TLSCertFile = in.TLSCertFile + out.TLSPrivateKeyFile = in.TLSPrivateKeyFile return nil } @@ -5122,6 +5128,8 @@ func autoConvert_kops_KubeSchedulerConfig_To_v1alpha2_KubeSchedulerConfig(in *ko out.AuthorizationKubeconfig = in.AuthorizationKubeconfig out.AuthorizationAlwaysAllowPaths = in.AuthorizationAlwaysAllowPaths out.EnableProfiling = in.EnableProfiling + out.TLSCertFile = in.TLSCertFile + out.TLSPrivateKeyFile = in.TLSPrivateKeyFile return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index 1410e4de58..5c06cd0916 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -2961,6 +2961,11 @@ func (in *KubeControllerManagerConfig) DeepCopyInto(out *KubeControllerManagerCo (*out)[key] = val } } + if in.TLSCertFile != nil { + in, out := &in.TLSCertFile, &out.TLSCertFile + *out = new(string) + **out = **in + } if in.TLSCipherSuites != nil { in, out := &in.TLSCipherSuites, &out.TLSCipherSuites *out = make([]string, len(*in)) @@ -3198,6 +3203,11 @@ func (in *KubeSchedulerConfig) DeepCopyInto(out *KubeSchedulerConfig) { *out = new(bool) **out = **in } + if in.TLSCertFile != nil { + in, out := &in.TLSCertFile, &out.TLSCertFile + *out = new(string) + **out = **in + } return } diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index bfd34e3b6d..fdf27bb8a7 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -3127,6 +3127,11 @@ func (in *KubeControllerManagerConfig) DeepCopyInto(out *KubeControllerManagerCo (*out)[key] = val } } + if in.TLSCertFile != nil { + in, out := &in.TLSCertFile, &out.TLSCertFile + *out = new(string) + **out = **in + } if in.TLSCipherSuites != nil { in, out := &in.TLSCipherSuites, &out.TLSCipherSuites *out = make([]string, len(*in)) @@ -3364,6 +3369,11 @@ func (in *KubeSchedulerConfig) DeepCopyInto(out *KubeSchedulerConfig) { *out = new(bool) **out = **in } + if in.TLSCertFile != nil { + in, out := &in.TLSCertFile, &out.TLSCertFile + *out = new(string) + **out = **in + } return } From beb974194329c0fc2665d6682ab92ae6b9dbd19f Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Tue, 20 Jul 2021 17:37:13 -0700 Subject: [PATCH 2/3] hack/update-expected.sh --- .../tasks-kube-controller-manager.yaml | 43 ++++++++++++++++ .../golden/minimal/tasks-kube-scheduler.yaml | 49 +++++++++++++++++++ .../tasks-kube-controller-manager-amd64.yaml | 43 ++++++++++++++++ .../tasks-kube-controller-manager-arm64.yaml | 43 ++++++++++++++++ .../tasks-kube-scheduler-amd64.yaml | 49 +++++++++++++++++++ .../tasks-kube-scheduler-arm64.yaml | 49 +++++++++++++++++++ 6 files changed, 276 insertions(+) diff --git a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-controller-manager.yaml b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-controller-manager.yaml index 13624a02b1..bc9c851012 100644 --- a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-controller-manager.yaml +++ b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-controller-manager.yaml @@ -28,6 +28,8 @@ contents: | - --leader-elect=true - --root-ca-file=/srv/kubernetes/ca.crt - --service-account-private-key-file=/srv/kubernetes/kube-controller-manager/service-account.key + - --tls-cert-file=/srv/kubernetes/kube-controller-manager/server.crt + - --tls-private-key-file=/srv/kubernetes/kube-controller-manager/server.key - --use-service-account-credentials=true - --v=2 - --logtostderr=false @@ -147,6 +149,10 @@ contents: | path: /etc/kubernetes/manifests/kube-controller-manager.manifest type: file --- +mode: "0755" +path: /srv/kubernetes/kube-controller-manager +type: directory +--- contents: | -----BEGIN CERTIFICATE----- MIIC2DCCAcCgAwIBAgIRALJXAkVj964tq67wMSI8oJQwDQYJKoZIhvcNAQELBQAw @@ -202,6 +208,34 @@ mode: "0600" path: /srv/kubernetes/kube-controller-manager/ca.key type: file --- +contents: + task: + Name: kube-controller-manager-server + alternateNames: + - kube-controller-manager.kube-system.svc.cluster.local + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kube-controller-manager + type: server +mode: "0644" +path: /srv/kubernetes/kube-controller-manager/server.crt +type: file +--- +contents: + task: + Name: kube-controller-manager-server + alternateNames: + - kube-controller-manager.kube-system.svc.cluster.local + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kube-controller-manager + type: server +mode: "0600" +path: /srv/kubernetes/kube-controller-manager/server.key +type: file +--- contents: | -----BEGIN RSA PRIVATE KEY----- MIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4 @@ -261,6 +295,15 @@ subject: CommonName: system:kube-controller-manager type: client --- +Name: kube-controller-manager-server +alternateNames: +- kube-controller-manager.kube-system.svc.cluster.local +keypairID: "3" +signer: kubernetes-ca +subject: + CommonName: kube-controller-manager +type: server +--- CA: task: Name: kube-controller-manager diff --git a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-scheduler.yaml b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-scheduler.yaml index 8de8f70629..b2718cd0e8 100644 --- a/nodeup/pkg/model/tests/golden/minimal/tasks-kube-scheduler.yaml +++ b/nodeup/pkg/model/tests/golden/minimal/tasks-kube-scheduler.yaml @@ -16,6 +16,8 @@ contents: | - --authorization-kubeconfig=/var/lib/kube-scheduler/kubeconfig - --config=/var/lib/kube-scheduler/config.yaml - --leader-elect=true + - --tls-cert-file=/srv/kubernetes/kube-scheduler/server.crt + - --tls-private-key-file=/srv/kubernetes/kube-scheduler/server.key - --v=2 - --logtostderr=false - --alsologtostderr @@ -38,6 +40,9 @@ contents: | - mountPath: /var/lib/kube-scheduler name: varlibkubescheduler readOnly: true + - mountPath: /srv/kubernetes/kube-scheduler + name: srvscheduler + readOnly: true - mountPath: /var/log/kube-scheduler.log name: logfile hostNetwork: true @@ -49,6 +54,9 @@ contents: | - hostPath: path: /var/lib/kube-scheduler name: varlibkubescheduler + - hostPath: + path: /srv/kubernetes/kube-scheduler + name: srvscheduler - hostPath: path: /var/log/kube-scheduler.log name: logfile @@ -56,6 +64,38 @@ contents: | path: /etc/kubernetes/manifests/kube-scheduler.manifest type: file --- +mode: "0755" +path: /srv/kubernetes/kube-scheduler +type: directory +--- +contents: + task: + Name: kube-scheduler-server + alternateNames: + - kube-scheduler.kube-system.svc.cluster.local + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kube-scheduler + type: server +mode: "0644" +path: /srv/kubernetes/kube-scheduler/server.crt +type: file +--- +contents: + task: + Name: kube-scheduler-server + alternateNames: + - kube-scheduler.kube-system.svc.cluster.local + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kube-scheduler + type: server +mode: "0600" +path: /srv/kubernetes/kube-scheduler/server.key +type: file +--- contents: | apiVersion: kubescheduler.config.k8s.io/v1alpha2 clientConnection: @@ -110,6 +150,15 @@ subject: CommonName: system:kube-scheduler type: client --- +Name: kube-scheduler-server +alternateNames: +- kube-scheduler.kube-system.svc.cluster.local +keypairID: "3" +signer: kubernetes-ca +subject: + CommonName: kube-scheduler +type: server +--- CA: task: Name: kube-scheduler diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-amd64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-amd64.yaml index 2eb47b78f2..6277cec360 100644 --- a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-amd64.yaml +++ b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-amd64.yaml @@ -28,6 +28,8 @@ contents: | - --leader-elect=true - --root-ca-file=/srv/kubernetes/ca.crt - --service-account-private-key-file=/srv/kubernetes/kube-controller-manager/service-account.key + - --tls-cert-file=/srv/kubernetes/kube-controller-manager/server.crt + - --tls-private-key-file=/srv/kubernetes/kube-controller-manager/server.key - --use-service-account-credentials=true - --v=2 - --logtostderr=false @@ -147,6 +149,10 @@ contents: | path: /etc/kubernetes/manifests/kube-controller-manager.manifest type: file --- +mode: "0755" +path: /srv/kubernetes/kube-controller-manager +type: directory +--- contents: | -----BEGIN CERTIFICATE----- MIIC2DCCAcCgAwIBAgIRALJXAkVj964tq67wMSI8oJQwDQYJKoZIhvcNAQELBQAw @@ -202,6 +208,34 @@ mode: "0600" path: /srv/kubernetes/kube-controller-manager/ca.key type: file --- +contents: + task: + Name: kube-controller-manager-server + alternateNames: + - kube-controller-manager.kube-system.svc.cluster.local + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kube-controller-manager + type: server +mode: "0644" +path: /srv/kubernetes/kube-controller-manager/server.crt +type: file +--- +contents: + task: + Name: kube-controller-manager-server + alternateNames: + - kube-controller-manager.kube-system.svc.cluster.local + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kube-controller-manager + type: server +mode: "0600" +path: /srv/kubernetes/kube-controller-manager/server.key +type: file +--- contents: | -----BEGIN RSA PRIVATE KEY----- MIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4 @@ -261,6 +295,15 @@ subject: CommonName: system:kube-controller-manager type: client --- +Name: kube-controller-manager-server +alternateNames: +- kube-controller-manager.kube-system.svc.cluster.local +keypairID: "3" +signer: kubernetes-ca +subject: + CommonName: kube-controller-manager +type: server +--- CA: task: Name: kube-controller-manager diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-arm64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-arm64.yaml index b181477d96..6837be39d8 100644 --- a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-arm64.yaml +++ b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-arm64.yaml @@ -28,6 +28,8 @@ contents: | - --leader-elect=true - --root-ca-file=/srv/kubernetes/ca.crt - --service-account-private-key-file=/srv/kubernetes/kube-controller-manager/service-account.key + - --tls-cert-file=/srv/kubernetes/kube-controller-manager/server.crt + - --tls-private-key-file=/srv/kubernetes/kube-controller-manager/server.key - --use-service-account-credentials=true - --v=2 - --logtostderr=false @@ -147,6 +149,10 @@ contents: | path: /etc/kubernetes/manifests/kube-controller-manager.manifest type: file --- +mode: "0755" +path: /srv/kubernetes/kube-controller-manager +type: directory +--- contents: | -----BEGIN CERTIFICATE----- MIIC2DCCAcCgAwIBAgIRALJXAkVj964tq67wMSI8oJQwDQYJKoZIhvcNAQELBQAw @@ -202,6 +208,34 @@ mode: "0600" path: /srv/kubernetes/kube-controller-manager/ca.key type: file --- +contents: + task: + Name: kube-controller-manager-server + alternateNames: + - kube-controller-manager.kube-system.svc.cluster.local + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kube-controller-manager + type: server +mode: "0644" +path: /srv/kubernetes/kube-controller-manager/server.crt +type: file +--- +contents: + task: + Name: kube-controller-manager-server + alternateNames: + - kube-controller-manager.kube-system.svc.cluster.local + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kube-controller-manager + type: server +mode: "0600" +path: /srv/kubernetes/kube-controller-manager/server.key +type: file +--- contents: | -----BEGIN RSA PRIVATE KEY----- MIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4 @@ -261,6 +295,15 @@ subject: CommonName: system:kube-controller-manager type: client --- +Name: kube-controller-manager-server +alternateNames: +- kube-controller-manager.kube-system.svc.cluster.local +keypairID: "3" +signer: kubernetes-ca +subject: + CommonName: kube-controller-manager +type: server +--- CA: task: Name: kube-controller-manager diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-scheduler-amd64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-scheduler-amd64.yaml index f05f707958..6a839544ed 100644 --- a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-scheduler-amd64.yaml +++ b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-scheduler-amd64.yaml @@ -16,6 +16,8 @@ contents: | - --authorization-kubeconfig=/var/lib/kube-scheduler/kubeconfig - --config=/var/lib/kube-scheduler/config.yaml - --leader-elect=true + - --tls-cert-file=/srv/kubernetes/kube-scheduler/server.crt + - --tls-private-key-file=/srv/kubernetes/kube-scheduler/server.key - --v=2 - --logtostderr=false - --alsologtostderr @@ -38,6 +40,9 @@ contents: | - mountPath: /var/lib/kube-scheduler name: varlibkubescheduler readOnly: true + - mountPath: /srv/kubernetes/kube-scheduler + name: srvscheduler + readOnly: true - mountPath: /var/log/kube-scheduler.log name: logfile hostNetwork: true @@ -49,6 +54,9 @@ contents: | - hostPath: path: /var/lib/kube-scheduler name: varlibkubescheduler + - hostPath: + path: /srv/kubernetes/kube-scheduler + name: srvscheduler - hostPath: path: /var/log/kube-scheduler.log name: logfile @@ -56,6 +64,38 @@ contents: | path: /etc/kubernetes/manifests/kube-scheduler.manifest type: file --- +mode: "0755" +path: /srv/kubernetes/kube-scheduler +type: directory +--- +contents: + task: + Name: kube-scheduler-server + alternateNames: + - kube-scheduler.kube-system.svc.cluster.local + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kube-scheduler + type: server +mode: "0644" +path: /srv/kubernetes/kube-scheduler/server.crt +type: file +--- +contents: + task: + Name: kube-scheduler-server + alternateNames: + - kube-scheduler.kube-system.svc.cluster.local + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kube-scheduler + type: server +mode: "0600" +path: /srv/kubernetes/kube-scheduler/server.key +type: file +--- contents: | apiVersion: kubescheduler.config.k8s.io/v1alpha2 clientConnection: @@ -110,6 +150,15 @@ subject: CommonName: system:kube-scheduler type: client --- +Name: kube-scheduler-server +alternateNames: +- kube-scheduler.kube-system.svc.cluster.local +keypairID: "3" +signer: kubernetes-ca +subject: + CommonName: kube-scheduler +type: server +--- CA: task: Name: kube-scheduler diff --git a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-scheduler-arm64.yaml b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-scheduler-arm64.yaml index 4fffb09f50..5773837629 100644 --- a/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-scheduler-arm64.yaml +++ b/nodeup/pkg/model/tests/golden/side-loading/tasks-kube-scheduler-arm64.yaml @@ -16,6 +16,8 @@ contents: | - --authorization-kubeconfig=/var/lib/kube-scheduler/kubeconfig - --config=/var/lib/kube-scheduler/config.yaml - --leader-elect=true + - --tls-cert-file=/srv/kubernetes/kube-scheduler/server.crt + - --tls-private-key-file=/srv/kubernetes/kube-scheduler/server.key - --v=2 - --logtostderr=false - --alsologtostderr @@ -38,6 +40,9 @@ contents: | - mountPath: /var/lib/kube-scheduler name: varlibkubescheduler readOnly: true + - mountPath: /srv/kubernetes/kube-scheduler + name: srvscheduler + readOnly: true - mountPath: /var/log/kube-scheduler.log name: logfile hostNetwork: true @@ -49,6 +54,9 @@ contents: | - hostPath: path: /var/lib/kube-scheduler name: varlibkubescheduler + - hostPath: + path: /srv/kubernetes/kube-scheduler + name: srvscheduler - hostPath: path: /var/log/kube-scheduler.log name: logfile @@ -56,6 +64,38 @@ contents: | path: /etc/kubernetes/manifests/kube-scheduler.manifest type: file --- +mode: "0755" +path: /srv/kubernetes/kube-scheduler +type: directory +--- +contents: + task: + Name: kube-scheduler-server + alternateNames: + - kube-scheduler.kube-system.svc.cluster.local + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kube-scheduler + type: server +mode: "0644" +path: /srv/kubernetes/kube-scheduler/server.crt +type: file +--- +contents: + task: + Name: kube-scheduler-server + alternateNames: + - kube-scheduler.kube-system.svc.cluster.local + keypairID: "3" + signer: kubernetes-ca + subject: + CommonName: kube-scheduler + type: server +mode: "0600" +path: /srv/kubernetes/kube-scheduler/server.key +type: file +--- contents: | apiVersion: kubescheduler.config.k8s.io/v1alpha2 clientConnection: @@ -110,6 +150,15 @@ subject: CommonName: system:kube-scheduler type: client --- +Name: kube-scheduler-server +alternateNames: +- kube-scheduler.kube-system.svc.cluster.local +keypairID: "3" +signer: kubernetes-ca +subject: + CommonName: kube-scheduler +type: server +--- CA: task: Name: kube-scheduler From 76255842ff6700fe8e5b2855bd58658c56d497d1 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Fri, 23 Jul 2021 22:01:24 -0700 Subject: [PATCH 3/3] Add release note on TLS server cert provisioning --- docs/releases/1.22-NOTES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/releases/1.22-NOTES.md b/docs/releases/1.22-NOTES.md index 9d2634b31a..3433006199 100644 --- a/docs/releases/1.22-NOTES.md +++ b/docs/releases/1.22-NOTES.md @@ -93,6 +93,10 @@ spec: * There is a new command `kops get assets` for listing image and file assets used by a cluster. It also includes a `--copy` flag to copy the assets to local repositories. See the documentation on [Using local asset repositories](../operations/asset-repository.md) for more information. + +* kOps now provisions TLS server certificates signed by the Kubernetes general CA to kube-controller-manager and kube-scheduler. + The previous behavior of using self-signed certs may be restored by setting `kubeControllerManager.tlsCertFile` and/or + `kubeScheduler.tlsCertFile` to `""` in the cluster spec. # Full change list since 1.21.0 release