Add option to use etcd managed by cilium-etcd-operator as kvstore

k8s/crds/kops.k8s.io_clusters.yaml

@@ -2715,6 +2715,11 @@ spec:
                       description: EnvoyLog is not implemented and may be removed
                         in the future. Setting this has no effect.
                       type: string
+                    etcdManaged:
+                      description: 'EtcdManagd installs an additional etcd cluster
+                        that is used for Cilium state change. The cluster is operated
+                        by cilium-etcd-operator. Default: false'
+                      type: boolean
                     ipam:
                       description: Ipam specifies the IP address allocation mode to
                         use. Possible values are "crd" and "eni". "eni" will use AWS

pkg/apis/kops/networking.go

@@ -402,6 +402,10 @@ type CiliumNetworkingSpec struct {
 	// Requires spec.kubeProxy.enabled be set to false.
 	// Default: false
 	EnableNodePort bool `json:"enableNodePort"`
+	// EtcdManagd installs an additional etcd cluster that is used for Cilium state change.
+	// The cluster is operated by cilium-etcd-operator.
+	// Default: false
+	EtcdManaged bool `json:"etcdManaged,omitempty"`
 
 	// RemoveCbrBridge is not implemented and may be removed in the future.
 	// Setting this has no effect.

pkg/apis/kops/v1alpha1/networking.go

@@ -400,6 +400,10 @@ type CiliumNetworkingSpec struct {
 	// Requires spec.kubeProxy.enabled be set to false.
 	// Default: false
 	EnableNodePort bool `json:"enableNodePort"`
+	// EtcdManagd installs an additional etcd cluster that is used for Cilium state change.
+	// The cluster is operated by cilium-etcd-operator.
+	// Default: false
+	EtcdManaged bool `json:"etcdManaged,omitempty"`
 
 	// RemoveCbrBridge is not implemented and may be removed in the future.
 	// Setting this has no effect.

pkg/apis/kops/v1alpha1/zz_generated.conversion.go

@@ -1334,6 +1334,7 @@ func autoConvert_v1alpha1_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in *
 	out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall
 	out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes
 	out.EnableNodePort = in.EnableNodePort
+	out.EtcdManaged = in.EtcdManaged
 	out.RemoveCbrBridge = in.RemoveCbrBridge
 	out.RestartPods = in.RestartPods
 	out.ReconfigureKubelet = in.ReconfigureKubelet
@@ -1413,6 +1414,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha1_CiliumNetworkingSpec(in *
 	out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall
 	out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes
 	out.EnableNodePort = in.EnableNodePort
+	out.EtcdManaged = in.EtcdManaged
 	out.RemoveCbrBridge = in.RemoveCbrBridge
 	out.RestartPods = in.RestartPods
 	out.ReconfigureKubelet = in.ReconfigureKubelet

pkg/apis/kops/v1alpha2/networking.go

@@ -400,6 +400,10 @@ type CiliumNetworkingSpec struct {
 	// Requires spec.kubeProxy.enabled be set to false.
 	// Default: false
 	EnableNodePort bool `json:"enableNodePort"`
+	// EtcdManagd installs an additional etcd cluster that is used for Cilium state change.
+	// The cluster is operated by cilium-etcd-operator.
+	// Default: false
+	EtcdManaged bool `json:"etcdManaged,omitempty"`
 
 	// RemoveCbrBridge is not implemented and may be removed in the future.
 	// Setting this has no effect.

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -1376,6 +1376,7 @@ func autoConvert_v1alpha2_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in *
 	out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall
 	out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes
 	out.EnableNodePort = in.EnableNodePort
+	out.EtcdManaged = in.EtcdManaged
 	out.RemoveCbrBridge = in.RemoveCbrBridge
 	out.RestartPods = in.RestartPods
 	out.ReconfigureKubelet = in.ReconfigureKubelet
@@ -1455,6 +1456,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha2_CiliumNetworkingSpec(in *
 	out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall
 	out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes
 	out.EnableNodePort = in.EnableNodePort
+	out.EtcdManaged = in.EtcdManaged
 	out.RemoveCbrBridge = in.RemoveCbrBridge
 	out.RestartPods = in.RestartPods
 	out.ReconfigureKubelet = in.ReconfigureKubelet

upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template

@@ -7,6 +7,21 @@ metadata:
     role.kubernetes.io/networking: "1"
 data:
 {{ with .Networking.Cilium }}
+
+{{- if .EtcdManaged }}
+  kvstore: etcd
+  kvstore-opt: '{"etcd.config": "/var/lib/etcd-config/etcd.config", "etcd.operator": "true"}'
+
+  etcd-config: |-
+    ---
+    endpoints:
+      - https://cilium-etcd-client.kube-system.svc:2379
+
+    trusted-ca-file: '/var/lib/etcd-secrets/etcd-client-ca.crt'
+    key-file: '/var/lib/etcd-secrets/etcd-client.key'
+    cert-file: '/var/lib/etcd-secrets/etcd-client.crt'
+{{ end }}
+
   # Identity allocation mode selects how identities are shared between cilium
   # nodes by setting how they are stored. The options are "crd" or "kvstore".
   # - "crd" stores identities in kubernetes as CRDs (custom resource definition).
@@ -460,6 +475,14 @@ spec:
           name: cni-path
         - mountPath: /host/etc/cni/net.d
           name: etc-cni-netd
+{{ if .EtcdManaged }}
+        - mountPath: /var/lib/etcd-config
+          name: etcd-config-path
+          readOnly: true
+        - mountPath: /var/lib/etcd-secrets
+          name: etcd-secrets
+          readOnly: true
+{{ end }}
         - mountPath: /var/lib/cilium/clustermesh
           name: clustermesh-secrets
           readOnly: true
@@ -548,6 +571,22 @@ spec:
           type: FileOrCreate
         name: xtables-lock
         # To read the clustermesh configuration
+{{- if .Networking.Cilium.EtcdManaged }}
+        # To read the etcd config stored in config maps
+      - configMap:
+          defaultMode: 420
+          items:
+          - key: etcd-config
+            path: etcd.config
+          name: cilium-config
+        name: etcd-config-path
+        # To read the k8s etcd secrets in case the user might want to use TLS
+      - name: etcd-secrets
+        secret:
+          defaultMode: 420
+          optional: true
+          secretName: cilium-etcd-secrets
+{{- end }}
       - name: clustermesh-secrets
         secret:
           defaultMode: 420
@@ -699,10 +738,42 @@ spec:
           initialDelaySeconds: 60
           periodSeconds: 10
           timeoutSeconds: 3
+{{- if .EtcdManaged }}
+        volumeMounts:
+        - mountPath: /var/lib/etcd-config
+          name: etcd-config-path
+          readOnly: true
+        - mountPath: /var/lib/etcd-secrets
+          name: etcd-secrets
+          readOnly: true
+{{- end }}
       hostNetwork: true
+{{- if .EtcdManaged }}
+      # In managed etcd mode, Cilium must be able to resolve the DNS name of
+      # the etcd service
+{{- end }}
+
       restartPolicy: Always
       serviceAccount: cilium-operator
       serviceAccountName: cilium-operator
+{{- if .EtcdManaged }}
+      volumes:
+      # To read the etcd config stored in config maps
+      - configMap:
+          defaultMode: 420
+          items:
+          - key: etcd-config
+            path: etcd.config
+          name: cilium-config
+        name: etcd-config-path
+        # To read the k8s etcd secrets in case the user might want to use TLS
+      - name: etcd-secrets
+        secret:
+          defaultMode: 420
+          optional: true
+          secretName: cilium-etcd-secrets
+{{- end }}
+
       {{ if eq .Ipam "eni" }}
       nodeSelector:
         node-role.kubernetes.io/master: ""
@@ -719,3 +790,241 @@ spec:
         tolerationSeconds: 300
       {{ end }}
 {{ end }}
+
+{{ if .Networking.Cilium.EtcdManaged }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    role.kubernetes.io/networking: "1"
+  name: cilium-etcd-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cilium-etcd-operator
+subjects:
+- kind: ServiceAccount
+  name: cilium-etcd-operator
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cilium-etcd-operator
+rules:
+- apiGroups:
+  - etcd.database.coreos.com
+  resources:
+  - etcdclusters
+  verbs:
+  - get
+  - delete
+  - create
+  - update
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - delete
+  - get
+  - create
+- apiGroups:
+  - ""
+  resources:
+  - deployments
+  verbs:
+  - delete
+  - create
+  - get
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - list
+  - get
+  - delete
+- apiGroups:
+  - apps
+  resources:
+  - deployments
+  verbs:
+  - delete
+  - create
+  - get
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - componentstatuses
+  verbs:
+  - get
+- apiGroups:
+  - extensions
+  resources:
+  - deployments
+  verbs:
+  - delete
+  - create
+  - get
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - create
+  - delete
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    io.cilium/app: etcd-operator
+    name: cilium-etcd-operator
+  name: cilium-etcd-operator
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      io.cilium/app: etcd-operator
+      name: cilium-etcd-operator
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        io.cilium/app: etcd-operator
+        name: cilium-etcd-operator
+    spec:
+      containers:
+      - command:
+        - /usr/bin/cilium-etcd-operator
+        env:
+        - name: CILIUM_ETCD_OPERATOR_CLUSTER_DOMAIN
+          value: "{{ $.ClusterDNSDomain }}"
+        - name: CILIUM_ETCD_OPERATOR_ETCD_CLUSTER_SIZE
+          value: "3"
+        - name: CILIUM_ETCD_OPERATOR_NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        - name: CILIUM_ETCD_OPERATOR_POD_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.name
+        - name: CILIUM_ETCD_OPERATOR_POD_UID
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.uid
+        - name: CILIUM_ETCD_META_ETCD_AUTO_COMPACTION_MODE
+          value: "revision"
+        - name: CILIUM_ETCD_META_ETCD_AUTO_COMPACTION_RETENTION
+          value: "25000"
+        image: "cilium/cilium-etcd-operator:v2.0.7"
+        name: cilium-etcd-operator
+      dnsPolicy: ClusterFirst
+      hostNetwork: true
+      restartPolicy: Always
+      serviceAccount: cilium-etcd-operator
+      serviceAccountName: cilium-etcd-operator
+      tolerations:
+      - operator: Exists
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    role.kubernetes.io/networking: "1"
+  name: cilium-etcd-operator
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    role.kubernetes.io/networking: "1"
+  name: etcd-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: etcd-operator
+subjects:
+- kind: ServiceAccount
+  name: cilium-etcd-sa
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    role.kubernetes.io/networking: "1"
+  name: etcd-operator
+rules:
+- apiGroups:
+  - etcd.database.coreos.com
+  resources:
+  - etcdclusters
+  - etcdbackups
+  - etcdrestores
+  verbs:
+  - '*'
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - services
+  - endpoints
+  - persistentvolumeclaims
+  - events
+  - deployments
+  verbs:
+  - '*'
+- apiGroups:
+  - apps
+  resources:
+  - deployments
+  verbs:
+  - '*'
+- apiGroups:
+  - extensions
+  resources:
+  - deployments
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    role.kubernetes.io/networking: "1"
+  name: cilium-etcd-sa
+  namespace: kube-system
+{{ end }}