From a0350a0dfa8c102fca022a21a0d7eff30b8b8494 Mon Sep 17 00:00:00 2001 From: shil Date: Mon, 1 Feb 2021 15:26:09 -0800 Subject: [PATCH 1/6] Use the kubeApiServerConfig clientCAFile field --- k8s/crds/kops.k8s.io_clusters.yaml | 3 ++- nodeup/pkg/model/kube_apiserver.go | 3 +++ nodeup/pkg/model/kube_apiserver_test.go | 6 ++++++ pkg/apis/kops/componentconfig.go | 2 +- pkg/apis/kops/v1alpha2/componentconfig.go | 2 +- 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 5495c6651b..e682d705c4 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -1269,7 +1269,8 @@ spec: kubernetes API type: string clientCAFile: - description: 'TODO: Remove unused ClientCAFile' + description: ClientCAFile is the file used by apisever that contains + the client CA type: string cloudProvider: description: CloudProvider is the name of the cloudProvider we diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go index 201bd64b74..1afabb0094 100644 --- a/nodeup/pkg/model/kube_apiserver.go +++ b/nodeup/pkg/model/kube_apiserver.go @@ -298,6 +298,9 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { } kubeAPIServer.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt") + if b.Cluster.Spec.KubeAPIServer.ClientCAFile != "" { + kubeAPIServer.ClientCAFile = b.Cluster.Spec.KubeAPIServer.ClientCAFile + } kubeAPIServer.TLSCertFile = filepath.Join(b.PathSrvKubernetes(), "server.crt") kubeAPIServer.TLSPrivateKeyFile = filepath.Join(b.PathSrvKubernetes(), "server.key") diff --git a/nodeup/pkg/model/kube_apiserver_test.go b/nodeup/pkg/model/kube_apiserver_test.go index 38b621623f..b116818d8e 100644 --- a/nodeup/pkg/model/kube_apiserver_test.go +++ b/nodeup/pkg/model/kube_apiserver_test.go @@ -131,6 +131,12 @@ func Test_KubeAPIServer_BuildFlags(t *testing.T) { }, "--insecure-port=0 --profiling=false --secure-port=0", }, + { + kops.KubeAPIServerConfig{ + ClientCAFile: "client-ca.crt", + }, + "--client-ca-file=client-ca.crt --insecure-port=0 --secure-port=0", + }, } for _, g := range grid { diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index 5543824a6c..a3f1df17ec 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -311,7 +311,7 @@ type KubeAPIServerConfig struct { EtcdKeyFile string `json:"etcdKeyFile,omitempty" flag:"etcd-keyfile"` // TODO: Remove unused BasicAuthFile BasicAuthFile string `json:"basicAuthFile,omitempty" flag:"basic-auth-file"` - // TODO: Remove unused ClientCAFile + // ClientCAFile is the file used by apisever that contains the client CA ClientCAFile string `json:"clientCAFile,omitempty" flag:"client-ca-file"` // TODO: Remove unused TLSCertFile TLSCertFile string `json:"tlsCertFile,omitempty" flag:"tls-cert-file"` diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go index 18323b5145..bf1f1a98d1 100644 --- a/pkg/apis/kops/v1alpha2/componentconfig.go +++ b/pkg/apis/kops/v1alpha2/componentconfig.go @@ -311,7 +311,7 @@ type KubeAPIServerConfig struct { EtcdKeyFile string `json:"etcdKeyFile,omitempty" flag:"etcd-keyfile"` // TODO: Remove unused BasicAuthFile BasicAuthFile string `json:"basicAuthFile,omitempty" flag:"basic-auth-file"` - // TODO: Remove unused ClientCAFile + // ClientCAFile is the file used by apisever that contains the client CA ClientCAFile string `json:"clientCAFile,omitempty" flag:"client-ca-file"` // TODO: Remove unused TLSCertFile TLSCertFile string `json:"tlsCertFile,omitempty" flag:"tls-cert-file"` From dc03028e5da40eb00bb43b00b9b554bff90b43fb Mon Sep 17 00:00:00 2001 From: shil Date: Tue, 2 Feb 2021 12:10:43 -0800 Subject: [PATCH 2/6] Update the logic to set kubeAPIServer.ClientCAFile --- nodeup/pkg/model/kube_apiserver.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go index 1afabb0094..45a90fe33a 100644 --- a/nodeup/pkg/model/kube_apiserver.go +++ b/nodeup/pkg/model/kube_apiserver.go @@ -296,10 +296,9 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { kubeAPIServer.ServiceAccountSigningKeyFile = &s } } - - kubeAPIServer.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt") - if b.Cluster.Spec.KubeAPIServer.ClientCAFile != "" { - kubeAPIServer.ClientCAFile = b.Cluster.Spec.KubeAPIServer.ClientCAFile + // If clientCAFile is not specified, set it to the default value ${PathSrvKubernetes}/ca.crt + if kubeAPIServer.ClientCAFile == "" { + kubeAPIServer.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt") } kubeAPIServer.TLSCertFile = filepath.Join(b.PathSrvKubernetes(), "server.crt") kubeAPIServer.TLSPrivateKeyFile = filepath.Join(b.PathSrvKubernetes(), "server.key") From be2a146fe6246575527ce88f4aabd6faec7ff35a Mon Sep 17 00:00:00 2001 From: shil Date: Tue, 2 Feb 2021 17:57:58 -0800 Subject: [PATCH 3/6] Add example in cluster_spec.md to customize kube-apiserver clientCAFile --- docs/cluster_spec.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index 60c934ba04..923d121ed2 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -429,6 +429,16 @@ spec: serviceNodePortRange: 30000-33000 ``` +### Customize client-ca file + +This value is passed as `--client-ca-file` for `kube-apiserver`. (default: `/srv/kubernetes/ca.crt`) + +```yaml +spec: + kubeAPIServer: + clientCAFile: /srv/kubernetes/client-ca.crt +``` + ### Disable Basic Auth Support for basic authentication was removed in Kubernetes 1.19. For previous versions From f34690b219e55f0750ac30d7c4fe60cb2bc6bee5 Mon Sep 17 00:00:00 2001 From: shil Date: Wed, 3 Feb 2021 21:43:23 -0800 Subject: [PATCH 4/6] Add more details on how to use the clientCAFile config --- docs/cluster_spec.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index 923d121ed2..27fc4d2c0c 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -438,6 +438,11 @@ spec: kubeAPIServer: clientCAFile: /srv/kubernetes/client-ca.crt ``` +There are certain cases that the user may want to use a customized client CA file other than the default one generated for Kubernetes. In that case, the user can use this flag to specify the client-ca file to use. + +To prepare the customized client-ca file on master nodes, the user can either use the [fileAssets](https://github.com/kubernetes/kops/blob/master/docs/cluster_spec.md#fileassets) feature to push an client-ca file, or embed the customized client-ca file in the master AMI. + +See also [Kubernetes certificates](https://kubernetes.io/docs/concepts/cluster-administration/certificates/) ### Disable Basic Auth From ab3a10f0fae9c60ba0dfc3b37b850b5cfe238f49 Mon Sep 17 00:00:00 2001 From: shil Date: Sun, 7 Feb 2021 20:54:06 -0800 Subject: [PATCH 5/6] Add more details about ca.crt append and refresh in case cert rotation happens. --- docs/cluster_spec.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index 27fc4d2c0c..51fa4f7909 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -442,6 +442,10 @@ There are certain cases that the user may want to use a customized client CA fil To prepare the customized client-ca file on master nodes, the user can either use the [fileAssets](https://github.com/kubernetes/kops/blob/master/docs/cluster_spec.md#fileassets) feature to push an client-ca file, or embed the customized client-ca file in the master AMI. +In the case that the user would use a customized client-ca file, it is common that the kubernetes CA (`/srv/kubernetes/ca/crt`) need to be appended to the end of the client-ca file. One way to append the ca.crt to the end of the customized client-ca file is to write an [kop-hook](https://github.com/kubernetes/kops/blob/master/docs/cluster_spec.md#hooks) to do the append logic. + +Kops also has CA rotation feature, which would refresh the kubernetes cert files, including the ca.crt. If a customized client-ca file is used, when kops cert rotation happens, the user is responsible to update the ca.crt in the customized client-ca file. The refresh ca.crt logic can also be achieved by writing a kops hook. + See also [Kubernetes certificates](https://kubernetes.io/docs/concepts/cluster-administration/certificates/) ### Disable Basic Auth From 11098d072cd630762782138697cf39b00c69dcae Mon Sep 17 00:00:00 2001 From: shil Date: Mon, 8 Feb 2021 11:11:02 -0800 Subject: [PATCH 6/6] Update doc with doc links --- docs/cluster_spec.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index 51fa4f7909..55720b5cfe 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -440,11 +440,11 @@ spec: ``` There are certain cases that the user may want to use a customized client CA file other than the default one generated for Kubernetes. In that case, the user can use this flag to specify the client-ca file to use. -To prepare the customized client-ca file on master nodes, the user can either use the [fileAssets](https://github.com/kubernetes/kops/blob/master/docs/cluster_spec.md#fileassets) feature to push an client-ca file, or embed the customized client-ca file in the master AMI. +To prepare the customized client-ca file on master nodes, the user can either use the [fileAssets](https://kops.sigs.k8s.io/cluster_spec/#fileassets) feature to push an client-ca file, or embed the customized client-ca file in the master AMI. -In the case that the user would use a customized client-ca file, it is common that the kubernetes CA (`/srv/kubernetes/ca/crt`) need to be appended to the end of the client-ca file. One way to append the ca.crt to the end of the customized client-ca file is to write an [kop-hook](https://github.com/kubernetes/kops/blob/master/docs/cluster_spec.md#hooks) to do the append logic. +In the case that the user would use a customized client-ca file, it is common that the kubernetes CA (`/srv/kubernetes/ca/crt`) need to be appended to the end of the client-ca file. One way to append the ca.crt to the end of the customized client-ca file is to write an [kop-hook](https://kops.sigs.k8s.io/cluster_spec/#hooks) to do the append logic. -Kops also has CA rotation feature, which would refresh the kubernetes cert files, including the ca.crt. If a customized client-ca file is used, when kops cert rotation happens, the user is responsible to update the ca.crt in the customized client-ca file. The refresh ca.crt logic can also be achieved by writing a kops hook. +Kops will have [CA rotation](https://kops.sigs.k8s.io/rotate-secrets/) feature soon, which would refresh the kubernetes cert files, including the ca.crt. If a customized client-ca file is used, when kops cert rotation happens, the user is responsible to update the ca.crt in the customized client-ca file. The refresh ca.crt logic can also be achieved by writing a kops hook. See also [Kubernetes certificates](https://kubernetes.io/docs/concepts/cluster-administration/certificates/)