Add option to use ENI as IPAM mode for Cilium

* Force cilium-operator run on master nodes
* Add option for setting cilium ipam mode
* If cilium ipam mode is eni, add additional permissions to master nodes
* Allow NonMasqueradeCIDR overlap with NetworkCIDR when Cilium ENI is enabled

k8s/crds/kops.k8s.io_clusters.yaml

@@ -2604,6 +2604,8 @@ spec:
                       type: boolean
                     envoyLog:
                       type: string
+                    ipam:
+                      type: string
                     ipv4ClusterCidrMaskSize:
                       type: integer
                     ipv4Node:

nodeup/pkg/model/context.go

@@ -360,11 +360,7 @@ func (c *NodeupModelContext) UseNodeAuthorizer() bool {
 
 // UsesSecondaryIP checks if the CNI in use attaches secondary interfaces to the host.
 func (c *NodeupModelContext) UsesSecondaryIP() bool {
-	if (c.Cluster.Spec.Networking.CNI != nil && c.Cluster.Spec.Networking.CNI.UsesSecondaryIP) || c.Cluster.Spec.Networking.AmazonVPC != nil || c.Cluster.Spec.Networking.LyftVPC != nil {
+	return (c.Cluster.Spec.Networking.CNI != nil && c.Cluster.Spec.Networking.CNI.UsesSecondaryIP) || c.Cluster.Spec.Networking.AmazonVPC != nil || c.Cluster.Spec.Networking.LyftVPC != nil || (c.Cluster.Spec.Networking.Cilium != nil && c.Cluster.Spec.Networking.Cilium.Ipam == kops.CiliumIpamEni)
-		return true
-	}
-
-	return false
 }
 
 // UseBootstrapTokens checks if we are using bootstrap tokens

pkg/apis/kops/networking.go

@@ -192,6 +192,7 @@ type AmazonVPCNetworkingSpec struct {
 }
 
 const CiliumDefaultVersion = "v1.6.6"
+const CiliumIpamEni = "eni"
 
 // CiliumNetworkingSpec declares that we want Cilium networking
 type CiliumNetworkingSpec struct {
@@ -261,6 +262,7 @@ type CiliumNetworkingSpec struct {
 	IPTablesRulesNoinstall bool   `json:"IPTablesRulesNoinstall"`
 	AutoDirectNodeRoutes   bool   `json:"autoDirectNodeRoutes"`
 	EnableNodePort         bool   `json:"enableNodePort"`
+	Ipam                   string `json:"ipam,omitempty"`
 
 	//node init options
 	RemoveCbrBridge       bool   `json:"removeCbrBridge"`

pkg/apis/kops/v1alpha1/networking.go

@@ -258,6 +258,7 @@ type CiliumNetworkingSpec struct {
 	IPTablesRulesNoinstall bool   `json:"IPTablesRulesNoinstall"`
 	AutoDirectNodeRoutes   bool   `json:"autoDirectNodeRoutes"`
 	EnableNodePort         bool   `json:"enableNodePort"`
+	Ipam                   string `json:"ipam,omitempty"`
 
 	//node init options
 	RemoveCbrBridge       bool   `json:"removeCbrBridge"`

pkg/apis/kops/v1alpha1/zz_generated.conversion.go

@@ -1311,6 +1311,7 @@ func autoConvert_v1alpha1_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in *
 	out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall
 	out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes
 	out.EnableNodePort = in.EnableNodePort
+	out.Ipam = in.Ipam
 	out.RemoveCbrBridge = in.RemoveCbrBridge
 	out.RestartPods = in.RestartPods
 	out.ReconfigureKubelet = in.ReconfigureKubelet
@@ -1389,6 +1390,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha1_CiliumNetworkingSpec(in *
 	out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall
 	out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes
 	out.EnableNodePort = in.EnableNodePort
+	out.Ipam = in.Ipam
 	out.RemoveCbrBridge = in.RemoveCbrBridge
 	out.RestartPods = in.RestartPods
 	out.ReconfigureKubelet = in.ReconfigureKubelet

pkg/apis/kops/v1alpha2/networking.go

@@ -259,6 +259,7 @@ type CiliumNetworkingSpec struct {
 	IPTablesRulesNoinstall bool   `json:"IPTablesRulesNoinstall"`
 	AutoDirectNodeRoutes   bool   `json:"autoDirectNodeRoutes"`
 	EnableNodePort         bool   `json:"enableNodePort"`
+	Ipam                   string `json:"ipam,omitempty"`
 
 	//node init options
 	RemoveCbrBridge       bool   `json:"removeCbrBridge"`

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -1353,6 +1353,7 @@ func autoConvert_v1alpha2_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in *
 	out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall
 	out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes
 	out.EnableNodePort = in.EnableNodePort
+	out.Ipam = in.Ipam
 	out.RemoveCbrBridge = in.RemoveCbrBridge
 	out.RestartPods = in.RestartPods
 	out.ReconfigureKubelet = in.ReconfigureKubelet
@@ -1431,6 +1432,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha2_CiliumNetworkingSpec(in *
 	out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall
 	out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes
 	out.EnableNodePort = in.EnableNodePort
+	out.Ipam = in.Ipam
 	out.RemoveCbrBridge = in.RemoveCbrBridge
 	out.RestartPods = in.RestartPods
 	out.ReconfigureKubelet = in.ReconfigureKubelet

pkg/apis/kops/validation/legacy.go

@@ -204,7 +204,7 @@ func ValidateCluster(c *kops.Cluster, strict bool) field.ErrorList {
 				allErrs = append(allErrs, field.Invalid(fieldSpec.Child("nonMasqueradeCIDR"), nonMasqueradeCIDRString, "Cluster had an invalid nonMasqueradeCIDR"))
 			}
 
-			if networkCIDR != nil && subnet.Overlap(nonMasqueradeCIDR, networkCIDR) && c.Spec.Networking != nil && c.Spec.Networking.AmazonVPC == nil && c.Spec.Networking.LyftVPC == nil {
+			if networkCIDR != nil && subnet.Overlap(nonMasqueradeCIDR, networkCIDR) && c.Spec.Networking != nil && c.Spec.Networking.AmazonVPC == nil && c.Spec.Networking.LyftVPC == nil && (c.Spec.Networking.Cilium == nil || c.Spec.Networking.Cilium.Ipam != kops.CiliumIpamEni) {
 				allErrs = append(allErrs, field.Invalid(fieldSpec.Child("nonMasqueradeCIDR"), nonMasqueradeCIDRString, fmt.Sprintf("nonMasqueradeCIDR %q cannot overlap with networkCIDR %q", nonMasqueradeCIDRString, c.Spec.NetworkCIDR)))
 			}
 
@@ -602,8 +602,22 @@ func ValidateCluster(c *kops.Cluster, strict bool) field.ErrorList {
 
 	allErrs = append(allErrs, newValidateCluster(c)...)
 
-	if c.Spec.Networking != nil && c.Spec.Networking.Cilium != nil && c.Spec.Networking.Cilium.EnableNodePort && c.Spec.KubeProxy != nil && *c.Spec.KubeProxy.Enabled {
+	if c.Spec.Networking != nil && c.Spec.Networking.Cilium != nil {
-		allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeProxy").Child("enabled"), "When kilium NodePort is enabled, kubeProxy must be disabled"))
+		ciliumSpec := c.Spec.Networking.Cilium
+
+		if ciliumSpec.EnableNodePort && c.Spec.KubeProxy != nil && *c.Spec.KubeProxy.Enabled {
+			allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeProxy").Child("enabled"), "When Cilium NodePort is enabled, kubeProxy must be disabled"))
+		}
+
+		if ciliumSpec.Ipam == kops.CiliumIpamEni {
+			if c.Spec.CloudProvider != string(kops.CloudProviderAWS) {
+				allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("cilium").Child("ipam"), "Cilum ENI IPAM is supported only in AWS"))
+			}
+			if !ciliumSpec.DisableMasquerade {
+				allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("cilium").Child("disableMasquerade"), "Masquerade must be disabled when ENI IPAM is used"))
+			}
+
+		}
 	}
 
 	return allErrs

pkg/model/iam/iam_builder.go

@@ -192,6 +192,10 @@ func (b *PolicyBuilder) BuildAWSPolicyMaster() (*Policy, error) {
 		addLyftVPCPermissions(p, resource, b.Cluster.Spec.IAM.Legacy, b.Cluster.GetName())
 	}
 
+	if b.Cluster.Spec.Networking != nil && b.Cluster.Spec.Networking.Cilium != nil && b.Cluster.Spec.Networking.Cilium.Ipam == kops.CiliumIpamEni {
+		addCiliumEniPermissions(p, resource, b.Cluster.Spec.IAM.Legacy)
+	}
+
 	return p, nil
 }
 
@@ -861,6 +865,34 @@ func addLyftVPCPermissions(p *Policy, resource stringorslice.StringOrSlice, lega
 	)
 }
 
+func addCiliumEniPermissions(p *Policy, resource stringorslice.StringOrSlice, legacyIAM bool) {
+	if legacyIAM {
+		// Legacy IAM provides ec2:*, so no additional permissions required
+		return
+	}
+
+	p.Statement = append(p.Statement,
+		&Statement{
+			Effect: StatementEffectAllow,
+			Action: stringorslice.Slice([]string{
+				"ec2:DescribeSubnets",
+				"ec2:AttachNetworkInterface",
+				"ec2:AssignPrivateIpAddresses",
+				"ec2:UnassignPrivateIpAddresses",
+				"ec2:CreateNetworkInterface",
+				"ec2:DescribeNetworkInterfaces",
+				"ec2:DescribeVpcPeeringConnections",
+				"ec2:DescribeSecurityGroups",
+				"ec2:DetachNetworkInterface",
+				"ec2:DeleteNetworkInterface",
+				"ec2:ModifyNetworkInterfaceAttribute",
+				"ec2:DescribeVpcs",
+			}),
+			Resource: resource,
+		},
+	)
+}
+
 func addAmazonVPCCNIPermissions(p *Policy, resource stringorslice.StringOrSlice, legacyIAM bool, clusterName string, iamPrefix string) {
 	if legacyIAM {
 		// Legacy IAM provides ec2:*, so no additional permissions required

upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template

@@ -121,6 +121,14 @@ data:
   install-iptables-rules: "{{- if .IPTablesRulesNoinstall -}}false{{- else -}}true{{- end -}}"
   auto-direct-node-routes: "{{- if .AutoDirectNodeRoutes -}}true{{- else -}}false{{- end -}}"
   enable-node-port: "{{- if .EnableNodePort -}}true{{- else -}}false{{- end -}}"
+  {{ with .Ipam }}
+  ipam: {{ . }}
+  {{ if eq . "eni" }}
+  enable-endpoint-routes: "true"
+  auto-create-cilium-node-resource: "true"
+  blacklist-conflicting-routes: "false"
+  {{ end }}
+  {{ end }}
 {{ end }} # With .Networking.Cilium end
 ---
 apiVersion: v1
@@ -662,7 +670,6 @@ spec:
           name: prometheus
           protocol: TCP
         {{ end }}
-{{ end }}
         livenessProbe:
           httpGet:
             path: /healthz
@@ -675,3 +682,19 @@ spec:
       restartPolicy: Always
       serviceAccount: cilium-operator
       serviceAccountName: cilium-operator
+      {{ if eq .Ipam "eni" }}
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+      - effect: NoExecute
+        key: node.kubernetes.io/not-ready
+        operator: Exists
+        tolerationSeconds: 300
+      - effect: NoExecute
+        key: node.kubernetes.io/unreachable
+        operator: Exists
+        tolerationSeconds: 300
+      {{ end }}
+{{ end }}

upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template

@@ -121,6 +121,14 @@ data:
   install-iptables-rules: "{{- if .IPTablesRulesNoinstall -}}false{{- else -}}true{{- end -}}"
   auto-direct-node-routes: "{{- if .AutoDirectNodeRoutes -}}true{{- else -}}false{{- end -}}"
   enable-node-port: "{{- if .EnableNodePort -}}true{{- else -}}false{{- end -}}"
+  {{ with .Ipam }}
+  ipam: {{ . }}
+  {{ if eq . "eni" }}
+  enable-endpoint-routes: "true"
+  auto-create-cilium-node-resource: "true"
+  blacklist-conflicting-routes: "false"
+  {{ end }}
+  {{ end }}
 {{ end }} # With .Networking.Cilium end
 ---
 apiVersion: v1
@@ -654,7 +662,6 @@ spec:
           name: prometheus
           protocol: TCP
         {{ end }}
-{{ end }}
         livenessProbe:
           httpGet:
             path: /healthz
@@ -667,3 +674,19 @@ spec:
       restartPolicy: Always
       serviceAccount: cilium-operator
       serviceAccountName: cilium-operator
+      {{if eq .Ipam "eni" }}
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+      - effect: NoExecute
+        key: node.kubernetes.io/not-ready
+        operator: Exists
+        tolerationSeconds: 300
+      - effect: NoExecute
+        key: node.kubernetes.io/unreachable
+        operator: Exists
+        tolerationSeconds: 300
+       {{ end }}
+{{ end }}