Issue the cilium etcd client cert out of kops-controller

cmd/kops-controller/pkg/server/server.go

@@ -165,6 +165,11 @@ func (s *Server) issueCert(name string, pubKey string, id *fi.VerifyResult, vali
 		return "", fmt.Errorf("key name not enabled")
 	}
 	switch name {
+	case "etcd-client-cilium":
+		issueReq.Signer = "etcd-clients-ca-cilium"
+		issueReq.Subject = pkix.Name{
+			CommonName: "cilium",
+		}
 	case "kubelet":
 		issueReq.Subject = pkix.Name{
 			CommonName:   fmt.Sprintf("system:node:%s", id.NodeName),

nodeup/pkg/model/context.go

@@ -220,9 +220,8 @@ func (c *NodeupModelContext) BuildIssuedKubeconfig(name string, subject nodetask
 	return kubeConfig.GetConfig()
 }
 
-// BuildBootstrapKubeconfig generates a kubeconfig with a client certificate from either kops-controller or the state store.
-func (c *NodeupModelContext) BuildBootstrapKubeconfig(name string, ctx *fi.ModelBuilderContext) (fi.Resource, error) {
-	if c.UseKopsControllerForNodeBootstrap() {
+// GetBootstrapCert requests a certificate keypair from kops-controller.
+func (c *NodeupModelContext) GetBootstrapCert(name string) (cert, key fi.Resource) {
 	b, ok := c.bootstrapCerts[name]
 	if !ok {
 		b = &nodetasks.BootstrapCert{
@@ -231,6 +230,13 @@ func (c *NodeupModelContext) BuildBootstrapKubeconfig(name string, ctx *fi.Model
 		}
 		c.bootstrapCerts[name] = b
 	}
+	return b.Cert, b.Key
+}
+
+// BuildBootstrapKubeconfig generates a kubeconfig with a client certificate from either kops-controller or the state store.
+func (c *NodeupModelContext) BuildBootstrapKubeconfig(name string, ctx *fi.ModelBuilderContext) (fi.Resource, error) {
+	if c.UseKopsControllerForNodeBootstrap() {
+		cert, key := c.GetBootstrapCert(name)
 
 		ca, err := c.GetCert(fi.CertificateIDCA)
 		if err != nil {
@@ -239,8 +245,8 @@ func (c *NodeupModelContext) BuildBootstrapKubeconfig(name string, ctx *fi.Model
 
 		kubeConfig := &nodetasks.KubeConfig{
 			Name: name,
-			Cert: b.Cert,
-			Key:  b.Key,
+			Cert: cert,
+			Key:  key,
 			CA:   fi.NewBytesResource(ca),
 		}
 		if c.IsMaster {

nodeup/pkg/model/kops_controller.go

@@ -19,6 +19,7 @@ package model
 import (
 	"path/filepath"
 
+	"k8s.io/kops/pkg/apis/kops/model"
 	"k8s.io/kops/pkg/wellknownusers"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
@@ -81,7 +82,11 @@ func (b *KopsControllerBuilder) Build(c *fi.ModelBuilderContext) error {
 		Owner:    s(wellknownusers.KopsControllerName),
 	})
 
-	for _, cert := range []string{fi.CertificateIDCA} {
+	caList := []string{fi.CertificateIDCA}
+	if model.UseCiliumEtcd(b.Cluster) {
+		caList = append(caList, "etcd-clients-ca-cilium")
+	}
+	for _, cert := range caList {
 		owner := wellknownusers.KopsControllerName
 		err := b.BuildCertificatePairTask(c, cert, pkiDir, cert, &owner)
 		if err != nil {

nodeup/pkg/model/networking/BUILD.bazel

@@ -14,6 +14,7 @@ go_library(
     deps = [
         "//nodeup/pkg/model:go_default_library",
         "//pkg/apis/kops:go_default_library",
+        "//pkg/apis/kops/model:go_default_library",
         "//pkg/rbac:go_default_library",
         "//upup/pkg/fi:go_default_library",
         "//upup/pkg/fi/nodeup/nodetasks:go_default_library",

nodeup/pkg/model/networking/cilium.go

@@ -18,10 +18,11 @@ package networking
 
 import (
 	"fmt"
+	"path/filepath"
 
 	"golang.org/x/sys/unix"
-
 	"k8s.io/kops/nodeup/pkg/model"
+	apiModel "k8s.io/kops/pkg/apis/kops/model"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
 )
@@ -38,16 +39,7 @@ func (b *CiliumBuilder) Build(c *fi.ModelBuilderContext) error {
 	networking := b.Cluster.Spec.Networking
 
 	// As long as the Cilium Etcd cluster exists, we should do this
-	ciliumEtcd := false
-
-	for _, cluster := range b.Cluster.Spec.EtcdClusters {
-		if cluster.Name == "cilium" {
-			ciliumEtcd = true
-			break
-		}
-	}
-
-	if ciliumEtcd {
+	if apiModel.UseCiliumEtcd(b.Cluster) {
 		if err := b.buildCiliumEtcdSecrets(c); err != nil {
 			return err
 		}
@@ -133,14 +125,36 @@ func (b *CiliumBuilder) buildCiliumEtcdSecrets(c *fi.ModelBuilderContext) error
 	}
 
 	name := "etcd-client-cilium"
+	dir := "/etc/kubernetes/pki/cilium"
+	signer := "etcd-clients-ca-cilium"
+	if b.UseKopsControllerForNodeBootstrap() && !b.IsMaster {
+		cert, key := b.GetBootstrapCert(name)
+
+		c.AddTask(&nodetasks.File{
+			Path:     filepath.Join(dir, name+".crt"),
+			Contents: cert,
+			Type:     nodetasks.FileType_File,
+			Mode:     fi.String("0644"),
+		})
+
+		c.AddTask(&nodetasks.File{
+			Path:     filepath.Join(dir, name+".key"),
+			Contents: key,
+			Type:     nodetasks.FileType_File,
+			Mode:     fi.String("0400"),
+		})
+
+		return b.BuildCertificateTask(c, signer, filepath.Join(dir, "etcd-ca.crt"), nil)
+	} else {
 		issueCert := &nodetasks.IssueCert{
 			Name:   name,
-		Signer: "etcd-clients-ca-cilium",
+			Signer: signer,
 			Type:   "client",
 			Subject: nodetasks.PKIXName{
 				CommonName: "cilium",
 			},
 		}
 		c.AddTask(issueCert)
-	return issueCert.AddFileTasks(c, "/etc/kubernetes/pki/cilium", name, "etcd-ca", nil)
+		return issueCert.AddFileTasks(c, dir, name, "etcd-ca", nil)
+	}
 }

pkg/apis/kops/model/features.go

@@ -24,3 +24,18 @@ import (
 func UseKopsControllerForNodeBootstrap(cluster *kops.Cluster) bool {
 	return kops.CloudProviderID(cluster.Spec.CloudProvider) == kops.CloudProviderAWS && cluster.IsKubernetesGTE("1.19")
 }
+
+// UseCiliumEtcd is true if we are using the Cilium etcd cluster.
+func UseCiliumEtcd(cluster *kops.Cluster) bool {
+	if cluster.Spec.Networking.Cilium == nil {
+		return false
+	}
+
+	for _, cluster := range cluster.Spec.EtcdClusters {
+		if cluster.Name == "cilium" {
+			return true
+		}
+	}
+
+	return false
+}

pkg/model/iam/iam_builder.go

@@ -485,15 +485,7 @@ func ReadableStatePaths(cluster *kops.Cluster, role kops.InstanceGroupRole) ([]s
 
 			// @check if cilium is enabled as the CNI provider and permit access to the cilium etc client TLS certificate by default
 			// As long as the Cilium Etcd cluster exists, we should do this
-			ciliumEtcd := false
-
-			for _, cluster := range cluster.Spec.EtcdClusters {
-				if cluster.Name == "cilium" {
-					ciliumEtcd = true
-					break
-				}
-			}
-			if networkingSpec.Cilium != nil && ciliumEtcd {
+			if networkingSpec.Cilium != nil && model.UseCiliumEtcd(cluster) && !model.UseKopsControllerForNodeBootstrap(cluster) {
 				paths = append(paths, "/pki/private/etcd-clients-ca-cilium/*")
 			}
 		}

upup/pkg/fi/cloudup/template_functions.go

@@ -42,6 +42,7 @@ import (
 	"k8s.io/klog/v2"
 	kopscontrollerconfig "k8s.io/kops/cmd/kops-controller/pkg/config"
 	"k8s.io/kops/pkg/apis/kops"
+	apiModel "k8s.io/kops/pkg/apis/kops/model"
 	"k8s.io/kops/pkg/apis/kops/util"
 	"k8s.io/kops/pkg/dns"
 	"k8s.io/kops/pkg/featureflag"
@@ -384,6 +385,11 @@ func (tf *TemplateFunctions) KopsControllerConfig() (string, error) {
 
 	if tf.UseKopsControllerForNodeBootstrap() {
 		certNames := []string{"kubelet"}
+		signingCAs := []string{fi.CertificateIDCA}
+		if apiModel.UseCiliumEtcd(cluster) {
+			certNames = append(certNames, "etcd-client-cilium")
+			signingCAs = append(signingCAs, "etcd-clients-ca-cilium")
+		}
 		if cluster.Spec.KubeProxy.Enabled == nil || *cluster.Spec.KubeProxy.Enabled {
 			certNames = append(certNames, "kube-proxy")
 		}
@@ -397,7 +403,7 @@ func (tf *TemplateFunctions) KopsControllerConfig() (string, error) {
 			ServerCertificatePath: path.Join(pkiDir, "kops-controller.crt"),
 			ServerKeyPath:         path.Join(pkiDir, "kops-controller.key"),
 			CABasePath:            pkiDir,
-			SigningCAs:            []string{fi.CertificateIDCA},
+			SigningCAs:            signingCAs,
 			CertNames:             certNames,
 		}