Merge pull request #9451 from olemarkus/fix-cilium-etcd-migration

Fix cilium etcd migration

docs/networking/cilium.md

@@ -27,7 +27,9 @@ kops create cluster \
 
 ### Using etcd for agent state sync
 
-By default, Cilium will use CRDs for synchronizing agent state. This can cause performance problems on larger clusters. As of kops 1.18, kops can manage an etcd cluster using etcd-manager dedicated for cilium agent state sync. The [Cilium docs](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-external-etcd/) contains recommendations for this must be enabled.
+This feature is in beta state as of kops 1.18.
+
+By default, Cilium will use CRDs for synchronizing agent state. This can cause performance problems on larger clusters. As of kops 1.18, kops can manage an etcd cluster using etcd-manager dedicated for cilium agent state sync. The [Cilium docs](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-external-etcd/) contains recommendations for when this must be enabled.
 
 Add the following to `spec.etcdClusters`:
 Make sure `instanceGroup` match the other etcd clusters.
@@ -43,6 +45,15 @@ Make sure `instanceGroup` match the other etcd clusters.
     name: cilium
 ```
 
+If this is an existing cluster, it is important that you perform a rolling update on the entire cluster so that all the nodes can connect to the new etcd cluster.
+
+```sh
+kops update cluster
+kops update cluster --yes
+kops rolling-update cluster --force --yes
+
+```
+
 Then enable etcd as kvstore:
 
 ```yaml
@@ -60,6 +71,8 @@ Read more about this in the [Cilium docs](https://docs.cilium.io/en/stable/getti
 
 Be aware that you need to use an AMI with at least Linux 4.19.57 for this feature to work.
 
+Also be aware that while enabling this on an existing cluster is safe, disabling this is disruptive and requires you to run `kops rolling-upgrade cluster --cloudonly`.
+
 ```yaml
   kubeProxy:
     enabled: false
@@ -70,6 +83,8 @@ Be aware that you need to use an AMI with at least Linux 4.19.57 for this featur
 
 ### Enabling Cilium ENI IPAM
 
+This feature is in beta state as of kops 1.18.
+
 As of Kops 1.18, you can have Cilium provision AWS managed adresses and attach them directly to Pods much like Lyft VPC and AWS VPC. See [the Cilium docs for more information](https://docs.cilium.io/en/v1.6/concepts/ipam/eni/)
 
 When using ENI IPAM you need to disable masquerading in Cilium as well.

nodeup/pkg/model/networking/cilium.go

@@ -37,6 +37,22 @@ var _ fi.ModelBuilder = &CiliumBuilder{}
 func (b *CiliumBuilder) Build(c *fi.ModelBuilderContext) error {
 	networking := b.Cluster.Spec.Networking
 
+	// As long as the Cilium Etcd cluster exists, we should do this
+	ciliumEtcd := false
+
+	for _, cluster := range b.Cluster.Spec.EtcdClusters {
+		if cluster.Name == "cilium" {
+			ciliumEtcd = true
+			break
+		}
+	}
+
+	if ciliumEtcd {
+		if err := b.buildCiliumEtcdSecrets(c); err != nil {
+			return err
+		}
+	}
+
 	if networking.Cilium == nil {
 		return nil
 	}
@@ -45,12 +61,6 @@ func (b *CiliumBuilder) Build(c *fi.ModelBuilderContext) error {
 		return err
 	}
 
-	if networking.Cilium.EtcdManaged {
-		if err := b.buildCiliumEtcdSecrets(c); err != nil {
-			return err
-		}
-	}
-
 	return nil
 
 }

pkg/model/iam/iam_builder.go

@@ -478,7 +478,16 @@ func ReadableStatePaths(cluster *kops.Cluster, role kops.InstanceGroupRole) ([]s
 			}
 
 			// @check if cilium is enabled as the CNI provider and permit access to the cilium etc client TLS certificate by default
-			if networkingSpec.Cilium != nil && networkingSpec.Cilium.EtcdManaged {
+			// As long as the Cilium Etcd cluster exists, we should do this
+			ciliumEtcd := false
+
+			for _, cluster := range cluster.Spec.EtcdClusters {
+				if cluster.Name == "cilium" {
+					ciliumEtcd = true
+					break
+				}
+			}
+			if networkingSpec.Cilium != nil && ciliumEtcd {
 				paths = append(paths, "/pki/private/etcd-clients-ca-cilium/*")
 			}
 		}

upup/models/bindata.go

@@ -4088,8 +4088,8 @@ data:
       - https://{{ $.MasterInternalName }}:4003
 
     trusted-ca-file: '/var/lib/etcd-secrets/etcd-ca.crt'
-    key-file: '/var/lib/etcd-secrets/etcd-client.key'
-    cert-file: '/var/lib/etcd-secrets/etcd-client.crt'
+    key-file: '/var/lib/etcd-secrets/etcd-client-cilium.key'
+    cert-file: '/var/lib/etcd-secrets/etcd-client-cilium.crt'
 {{ end }}
 
   # Identity allocation mode selects how identities are shared between cilium

upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template

@@ -28,8 +28,8 @@ data:
       - https://{{ $.MasterInternalName }}:4003
 
     trusted-ca-file: '/var/lib/etcd-secrets/etcd-ca.crt'
-    key-file: '/var/lib/etcd-secrets/etcd-client.key'
-    cert-file: '/var/lib/etcd-secrets/etcd-client.crt'
+    key-file: '/var/lib/etcd-secrets/etcd-client-cilium.key'
+    cert-file: '/var/lib/etcd-secrets/etcd-client-cilium.crt'
 {{ end }}
 
   # Identity allocation mode selects how identities are shared between cilium