diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index d8f9063846..5d4b31a169 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -2604,6 +2604,8 @@ spec: type: boolean envoyLog: type: string + ipam: + type: string ipv4ClusterCidrMaskSize: type: integer ipv4Node: diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go index 5e54fb2314..0e2dd90f8d 100644 --- a/nodeup/pkg/model/context.go +++ b/nodeup/pkg/model/context.go @@ -360,11 +360,7 @@ func (c *NodeupModelContext) UseNodeAuthorizer() bool { // UsesSecondaryIP checks if the CNI in use attaches secondary interfaces to the host. func (c *NodeupModelContext) UsesSecondaryIP() bool { - if (c.Cluster.Spec.Networking.CNI != nil && c.Cluster.Spec.Networking.CNI.UsesSecondaryIP) || c.Cluster.Spec.Networking.AmazonVPC != nil || c.Cluster.Spec.Networking.LyftVPC != nil { - return true - } - - return false + return (c.Cluster.Spec.Networking.CNI != nil && c.Cluster.Spec.Networking.CNI.UsesSecondaryIP) || c.Cluster.Spec.Networking.AmazonVPC != nil || c.Cluster.Spec.Networking.LyftVPC != nil || (c.Cluster.Spec.Networking.Cilium != nil && c.Cluster.Spec.Networking.Cilium.Ipam == kops.CiliumIpamEni) } // UseBootstrapTokens checks if we are using bootstrap tokens diff --git a/pkg/apis/kops/networking.go b/pkg/apis/kops/networking.go index f28af387bd..507f087333 100644 --- a/pkg/apis/kops/networking.go +++ b/pkg/apis/kops/networking.go @@ -192,6 +192,7 @@ type AmazonVPCNetworkingSpec struct { } const CiliumDefaultVersion = "v1.6.6" +const CiliumIpamEni = "eni" // CiliumNetworkingSpec declares that we want Cilium networking type CiliumNetworkingSpec struct { @@ -261,6 +262,7 @@ type CiliumNetworkingSpec struct { IPTablesRulesNoinstall bool `json:"IPTablesRulesNoinstall"` AutoDirectNodeRoutes bool `json:"autoDirectNodeRoutes"` EnableNodePort bool `json:"enableNodePort"` + Ipam string `json:"ipam,omitempty"` //node init options RemoveCbrBridge bool `json:"removeCbrBridge"` diff --git a/pkg/apis/kops/v1alpha1/networking.go b/pkg/apis/kops/v1alpha1/networking.go index 0581ce4ceb..c286330bab 100644 --- a/pkg/apis/kops/v1alpha1/networking.go +++ b/pkg/apis/kops/v1alpha1/networking.go @@ -258,6 +258,7 @@ type CiliumNetworkingSpec struct { IPTablesRulesNoinstall bool `json:"IPTablesRulesNoinstall"` AutoDirectNodeRoutes bool `json:"autoDirectNodeRoutes"` EnableNodePort bool `json:"enableNodePort"` + Ipam string `json:"ipam,omitempty"` //node init options RemoveCbrBridge bool `json:"removeCbrBridge"` diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index 096bfe0c54..e58eea7f9a 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -1311,6 +1311,7 @@ func autoConvert_v1alpha1_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in * out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes out.EnableNodePort = in.EnableNodePort + out.Ipam = in.Ipam out.RemoveCbrBridge = in.RemoveCbrBridge out.RestartPods = in.RestartPods out.ReconfigureKubelet = in.ReconfigureKubelet @@ -1389,6 +1390,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha1_CiliumNetworkingSpec(in * out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes out.EnableNodePort = in.EnableNodePort + out.Ipam = in.Ipam out.RemoveCbrBridge = in.RemoveCbrBridge out.RestartPods = in.RestartPods out.ReconfigureKubelet = in.ReconfigureKubelet diff --git a/pkg/apis/kops/v1alpha2/networking.go b/pkg/apis/kops/v1alpha2/networking.go index 5a4a8acdb7..744b825253 100644 --- a/pkg/apis/kops/v1alpha2/networking.go +++ b/pkg/apis/kops/v1alpha2/networking.go @@ -259,6 +259,7 @@ type CiliumNetworkingSpec struct { IPTablesRulesNoinstall bool `json:"IPTablesRulesNoinstall"` AutoDirectNodeRoutes bool `json:"autoDirectNodeRoutes"` EnableNodePort bool `json:"enableNodePort"` + Ipam string `json:"ipam,omitempty"` //node init options RemoveCbrBridge bool `json:"removeCbrBridge"` diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index aa7ee793d4..0c66aa7434 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -1353,6 +1353,7 @@ func autoConvert_v1alpha2_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in * out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes out.EnableNodePort = in.EnableNodePort + out.Ipam = in.Ipam out.RemoveCbrBridge = in.RemoveCbrBridge out.RestartPods = in.RestartPods out.ReconfigureKubelet = in.ReconfigureKubelet @@ -1431,6 +1432,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha2_CiliumNetworkingSpec(in * out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes out.EnableNodePort = in.EnableNodePort + out.Ipam = in.Ipam out.RemoveCbrBridge = in.RemoveCbrBridge out.RestartPods = in.RestartPods out.ReconfigureKubelet = in.ReconfigureKubelet diff --git a/pkg/apis/kops/validation/legacy.go b/pkg/apis/kops/validation/legacy.go index 572316aeef..9aa55529ce 100644 --- a/pkg/apis/kops/validation/legacy.go +++ b/pkg/apis/kops/validation/legacy.go @@ -204,7 +204,7 @@ func ValidateCluster(c *kops.Cluster, strict bool) field.ErrorList { allErrs = append(allErrs, field.Invalid(fieldSpec.Child("nonMasqueradeCIDR"), nonMasqueradeCIDRString, "Cluster had an invalid nonMasqueradeCIDR")) } - if networkCIDR != nil && subnet.Overlap(nonMasqueradeCIDR, networkCIDR) && c.Spec.Networking != nil && c.Spec.Networking.AmazonVPC == nil && c.Spec.Networking.LyftVPC == nil { + if networkCIDR != nil && subnet.Overlap(nonMasqueradeCIDR, networkCIDR) && c.Spec.Networking != nil && c.Spec.Networking.AmazonVPC == nil && c.Spec.Networking.LyftVPC == nil && (c.Spec.Networking.Cilium == nil || c.Spec.Networking.Cilium.Ipam != kops.CiliumIpamEni) { allErrs = append(allErrs, field.Invalid(fieldSpec.Child("nonMasqueradeCIDR"), nonMasqueradeCIDRString, fmt.Sprintf("nonMasqueradeCIDR %q cannot overlap with networkCIDR %q", nonMasqueradeCIDRString, c.Spec.NetworkCIDR))) } @@ -602,8 +602,22 @@ func ValidateCluster(c *kops.Cluster, strict bool) field.ErrorList { allErrs = append(allErrs, newValidateCluster(c)...) - if c.Spec.Networking != nil && c.Spec.Networking.Cilium != nil && c.Spec.Networking.Cilium.EnableNodePort && c.Spec.KubeProxy != nil && *c.Spec.KubeProxy.Enabled { - allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeProxy").Child("enabled"), "When kilium NodePort is enabled, kubeProxy must be disabled")) + if c.Spec.Networking != nil && c.Spec.Networking.Cilium != nil { + ciliumSpec := c.Spec.Networking.Cilium + + if ciliumSpec.EnableNodePort && c.Spec.KubeProxy != nil && *c.Spec.KubeProxy.Enabled { + allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeProxy").Child("enabled"), "When Cilium NodePort is enabled, kubeProxy must be disabled")) + } + + if ciliumSpec.Ipam == kops.CiliumIpamEni { + if c.Spec.CloudProvider != string(kops.CloudProviderAWS) { + allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("cilium").Child("ipam"), "Cilum ENI IPAM is supported only in AWS")) + } + if !ciliumSpec.DisableMasquerade { + allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("cilium").Child("disableMasquerade"), "Masquerade must be disabled when ENI IPAM is used")) + } + + } } return allErrs diff --git a/pkg/model/iam/iam_builder.go b/pkg/model/iam/iam_builder.go index 9c7aca0cdf..d82c73023f 100644 --- a/pkg/model/iam/iam_builder.go +++ b/pkg/model/iam/iam_builder.go @@ -192,6 +192,10 @@ func (b *PolicyBuilder) BuildAWSPolicyMaster() (*Policy, error) { addLyftVPCPermissions(p, resource, b.Cluster.Spec.IAM.Legacy, b.Cluster.GetName()) } + if b.Cluster.Spec.Networking != nil && b.Cluster.Spec.Networking.Cilium != nil && b.Cluster.Spec.Networking.Cilium.Ipam == kops.CiliumIpamEni { + addCiliumEniPermissions(p, resource, b.Cluster.Spec.IAM.Legacy) + } + return p, nil } @@ -861,6 +865,34 @@ func addLyftVPCPermissions(p *Policy, resource stringorslice.StringOrSlice, lega ) } +func addCiliumEniPermissions(p *Policy, resource stringorslice.StringOrSlice, legacyIAM bool) { + if legacyIAM { + // Legacy IAM provides ec2:*, so no additional permissions required + return + } + + p.Statement = append(p.Statement, + &Statement{ + Effect: StatementEffectAllow, + Action: stringorslice.Slice([]string{ + "ec2:DescribeSubnets", + "ec2:AttachNetworkInterface", + "ec2:AssignPrivateIpAddresses", + "ec2:UnassignPrivateIpAddresses", + "ec2:CreateNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeVpcPeeringConnections", + "ec2:DescribeSecurityGroups", + "ec2:DetachNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:ModifyNetworkInterfaceAttribute", + "ec2:DescribeVpcs", + }), + Resource: resource, + }, + ) +} + func addAmazonVPCCNIPermissions(p *Policy, resource stringorslice.StringOrSlice, legacyIAM bool, clusterName string, iamPrefix string) { if legacyIAM { // Legacy IAM provides ec2:*, so no additional permissions required diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template index 89179e4084..9e561703a8 100644 --- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template @@ -121,6 +121,14 @@ data: install-iptables-rules: "{{- if .IPTablesRulesNoinstall -}}false{{- else -}}true{{- end -}}" auto-direct-node-routes: "{{- if .AutoDirectNodeRoutes -}}true{{- else -}}false{{- end -}}" enable-node-port: "{{- if .EnableNodePort -}}true{{- else -}}false{{- end -}}" + {{ with .Ipam }} + ipam: {{ . }} + {{ if eq . "eni" }} + enable-endpoint-routes: "true" + auto-create-cilium-node-resource: "true" + blacklist-conflicting-routes: "false" + {{ end }} + {{ end }} {{ end }} # With .Networking.Cilium end --- apiVersion: v1 @@ -662,7 +670,6 @@ spec: name: prometheus protocol: TCP {{ end }} -{{ end }} livenessProbe: httpGet: path: /healthz @@ -675,3 +682,19 @@ spec: restartPolicy: Always serviceAccount: cilium-operator serviceAccountName: cilium-operator + {{ if eq .Ipam "eni" }} + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoExecute + key: node.kubernetes.io/not-ready + operator: Exists + tolerationSeconds: 300 + - effect: NoExecute + key: node.kubernetes.io/unreachable + operator: Exists + tolerationSeconds: 300 + {{ end }} +{{ end }} \ No newline at end of file diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template index 0f4a766a9d..6ad0a79f85 100644 --- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template @@ -121,6 +121,14 @@ data: install-iptables-rules: "{{- if .IPTablesRulesNoinstall -}}false{{- else -}}true{{- end -}}" auto-direct-node-routes: "{{- if .AutoDirectNodeRoutes -}}true{{- else -}}false{{- end -}}" enable-node-port: "{{- if .EnableNodePort -}}true{{- else -}}false{{- end -}}" + {{ with .Ipam }} + ipam: {{ . }} + {{ if eq . "eni" }} + enable-endpoint-routes: "true" + auto-create-cilium-node-resource: "true" + blacklist-conflicting-routes: "false" + {{ end }} + {{ end }} {{ end }} # With .Networking.Cilium end --- apiVersion: v1 @@ -654,7 +662,6 @@ spec: name: prometheus protocol: TCP {{ end }} -{{ end }} livenessProbe: httpGet: path: /healthz @@ -667,3 +674,19 @@ spec: restartPolicy: Always serviceAccount: cilium-operator serviceAccountName: cilium-operator + {{if eq .Ipam "eni" }} + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoExecute + key: node.kubernetes.io/not-ready + operator: Exists + tolerationSeconds: 300 + - effect: NoExecute + key: node.kubernetes.io/unreachable + operator: Exists + tolerationSeconds: 300 + {{ end }} +{{ end }} \ No newline at end of file