kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add option to use ENI as IPAM mode for Cilium 

* Force cilium-operator run on master nodes
* Add option for setting cilium ipam mode
* If cilium ipam mode is eni, add additional permissions to master nodes
* Allow NonMasqueradeCIDR overlap with NetworkCIDR when Cilium ENI is enabled
This commit is contained in:
Ole Markus With 2020-01-11 07:33:06 +01:00
parent 0c2fe666e4
commit ced8f00201
11 changed files with 108 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
k8s/crds/kops.k8s.io_clusters.yaml
View File

@ -2604,6 +2604,8 @@ spec:
type: boolean
envoyLog:
type: string
ipam:
type: string
ipv4ClusterCidrMaskSize:
type: integer
ipv4Node:

6
nodeup/pkg/model/context.go
View File

@ -360,11 +360,7 @@ func (c *NodeupModelContext) UseNodeAuthorizer() bool {
// UsesSecondaryIP checks if the CNI in use attaches secondary interfaces to the host.
func (c *NodeupModelContext) UsesSecondaryIP() bool {
if (c.Cluster.Spec.Networking.CNI != nil && c.Cluster.Spec.Networking.CNI.UsesSecondaryIP) || c.Cluster.Spec.Networking.AmazonVPC != nil || c.Cluster.Spec.Networking.LyftVPC != nil {
return true
}
return false
return (c.Cluster.Spec.Networking.CNI != nil && c.Cluster.Spec.Networking.CNI.UsesSecondaryIP) || c.Cluster.Spec.Networking.AmazonVPC != nil || c.Cluster.Spec.Networking.LyftVPC != nil || (c.Cluster.Spec.Networking.Cilium != nil && c.Cluster.Spec.Networking.Cilium.Ipam == kops.CiliumIpamEni)
}
// UseBootstrapTokens checks if we are using bootstrap tokens

2
pkg/apis/kops/networking.go
View File

@ -192,6 +192,7 @@ type AmazonVPCNetworkingSpec struct {
}
const CiliumDefaultVersion = "v1.6.6"
const CiliumIpamEni = "eni"
// CiliumNetworkingSpec declares that we want Cilium networking
type CiliumNetworkingSpec struct {
@ -261,6 +262,7 @@ type CiliumNetworkingSpec struct {
IPTablesRulesNoinstall bool `json:"IPTablesRulesNoinstall"`
AutoDirectNodeRoutes bool `json:"autoDirectNodeRoutes"`
EnableNodePort bool `json:"enableNodePort"`
Ipam string `json:"ipam,omitempty"`
//node init options
RemoveCbrBridge bool `json:"removeCbrBridge"`

1
pkg/apis/kops/v1alpha1/networking.go
View File

@ -258,6 +258,7 @@ type CiliumNetworkingSpec struct {
IPTablesRulesNoinstall bool `json:"IPTablesRulesNoinstall"`
AutoDirectNodeRoutes bool `json:"autoDirectNodeRoutes"`
EnableNodePort bool `json:"enableNodePort"`
Ipam string `json:"ipam,omitempty"`
//node init options
RemoveCbrBridge bool `json:"removeCbrBridge"`

2
pkg/apis/kops/v1alpha1/zz_generated.conversion.go
View File

@ -1311,6 +1311,7 @@ func autoConvert_v1alpha1_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in *
out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall
out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes
out.EnableNodePort = in.EnableNodePort
out.Ipam = in.Ipam
out.RemoveCbrBridge = in.RemoveCbrBridge
out.RestartPods = in.RestartPods
out.ReconfigureKubelet = in.ReconfigureKubelet
@ -1389,6 +1390,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha1_CiliumNetworkingSpec(in *
out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall
out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes
out.EnableNodePort = in.EnableNodePort
out.Ipam = in.Ipam
out.RemoveCbrBridge = in.RemoveCbrBridge
out.RestartPods = in.RestartPods
out.ReconfigureKubelet = in.ReconfigureKubelet

1
pkg/apis/kops/v1alpha2/networking.go
View File

@ -259,6 +259,7 @@ type CiliumNetworkingSpec struct {
IPTablesRulesNoinstall bool `json:"IPTablesRulesNoinstall"`
AutoDirectNodeRoutes bool `json:"autoDirectNodeRoutes"`
EnableNodePort bool `json:"enableNodePort"`
Ipam string `json:"ipam,omitempty"`
//node init options
RemoveCbrBridge bool `json:"removeCbrBridge"`

2
pkg/apis/kops/v1alpha2/zz_generated.conversion.go
View File

@ -1353,6 +1353,7 @@ func autoConvert_v1alpha2_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in *
out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall
out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes
out.EnableNodePort = in.EnableNodePort
out.Ipam = in.Ipam
out.RemoveCbrBridge = in.RemoveCbrBridge
out.RestartPods = in.RestartPods
out.ReconfigureKubelet = in.ReconfigureKubelet
@ -1431,6 +1432,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha2_CiliumNetworkingSpec(in *
out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall
out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes
out.EnableNodePort = in.EnableNodePort
out.Ipam = in.Ipam
out.RemoveCbrBridge = in.RemoveCbrBridge
out.RestartPods = in.RestartPods
out.ReconfigureKubelet = in.ReconfigureKubelet

20
pkg/apis/kops/validation/legacy.go
View File

@ -204,7 +204,7 @@ func ValidateCluster(c *kops.Cluster, strict bool) field.ErrorList {
allErrs = append(allErrs, field.Invalid(fieldSpec.Child("nonMasqueradeCIDR"), nonMasqueradeCIDRString, "Cluster had an invalid nonMasqueradeCIDR"))
}
if networkCIDR != nil && subnet.Overlap(nonMasqueradeCIDR, networkCIDR) && c.Spec.Networking != nil && c.Spec.Networking.AmazonVPC == nil && c.Spec.Networking.LyftVPC == nil {
if networkCIDR != nil && subnet.Overlap(nonMasqueradeCIDR, networkCIDR) && c.Spec.Networking != nil && c.Spec.Networking.AmazonVPC == nil && c.Spec.Networking.LyftVPC == nil && (c.Spec.Networking.Cilium == nil || c.Spec.Networking.Cilium.Ipam != kops.CiliumIpamEni) {
allErrs = append(allErrs, field.Invalid(fieldSpec.Child("nonMasqueradeCIDR"), nonMasqueradeCIDRString, fmt.Sprintf("nonMasqueradeCIDR %q cannot overlap with networkCIDR %q", nonMasqueradeCIDRString, c.Spec.NetworkCIDR)))
}
@ -602,8 +602,22 @@ func ValidateCluster(c *kops.Cluster, strict bool) field.ErrorList {
allErrs = append(allErrs, newValidateCluster(c)...)
if c.Spec.Networking != nil && c.Spec.Networking.Cilium != nil && c.Spec.Networking.Cilium.EnableNodePort && c.Spec.KubeProxy != nil && *c.Spec.KubeProxy.Enabled {
allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeProxy").Child("enabled"), "When kilium NodePort is enabled, kubeProxy must be disabled"))
if c.Spec.Networking != nil && c.Spec.Networking.Cilium != nil {
ciliumSpec := c.Spec.Networking.Cilium
if ciliumSpec.EnableNodePort && c.Spec.KubeProxy != nil && *c.Spec.KubeProxy.Enabled {
allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeProxy").Child("enabled"), "When Cilium NodePort is enabled, kubeProxy must be disabled"))
}
if ciliumSpec.Ipam == kops.CiliumIpamEni {
if c.Spec.CloudProvider != string(kops.CloudProviderAWS) {
allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("cilium").Child("ipam"), "Cilum ENI IPAM is supported only in AWS"))
}
if !ciliumSpec.DisableMasquerade {
allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("cilium").Child("disableMasquerade"), "Masquerade must be disabled when ENI IPAM is used"))
}
}
}
return allErrs

32
pkg/model/iam/iam_builder.go
View File

@ -192,6 +192,10 @@ func (b *PolicyBuilder) BuildAWSPolicyMaster() (*Policy, error) {
addLyftVPCPermissions(p, resource, b.Cluster.Spec.IAM.Legacy, b.Cluster.GetName())
}
if b.Cluster.Spec.Networking != nil && b.Cluster.Spec.Networking.Cilium != nil && b.Cluster.Spec.Networking.Cilium.Ipam == kops.CiliumIpamEni {
addCiliumEniPermissions(p, resource, b.Cluster.Spec.IAM.Legacy)
}
return p, nil
}
@ -861,6 +865,34 @@ func addLyftVPCPermissions(p *Policy, resource stringorslice.StringOrSlice, lega
)
}
func addCiliumEniPermissions(p *Policy, resource stringorslice.StringOrSlice, legacyIAM bool) {
if legacyIAM {
// Legacy IAM provides ec2:*, so no additional permissions required
return
}
p.Statement = append(p.Statement,
&Statement{
Effect: StatementEffectAllow,
Action: stringorslice.Slice([]string{
"ec2:DescribeSubnets",
"ec2:AttachNetworkInterface",
"ec2:AssignPrivateIpAddresses",
"ec2:UnassignPrivateIpAddresses",
"ec2:CreateNetworkInterface",
"ec2:DescribeNetworkInterfaces",
"ec2:DescribeVpcPeeringConnections",
"ec2:DescribeSecurityGroups",
"ec2:DetachNetworkInterface",
"ec2:DeleteNetworkInterface",
"ec2:ModifyNetworkInterfaceAttribute",
"ec2:DescribeVpcs",
}),
Resource: resource,
},
)
}
func addAmazonVPCCNIPermissions(p *Policy, resource stringorslice.StringOrSlice, legacyIAM bool, clusterName string, iamPrefix string) {
if legacyIAM {
// Legacy IAM provides ec2:*, so no additional permissions required

25
upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template
View File

@ -121,6 +121,14 @@ data:
install-iptables-rules: "{{- if .IPTablesRulesNoinstall -}}false{{- else -}}true{{- end -}}"
auto-direct-node-routes: "{{- if .AutoDirectNodeRoutes -}}true{{- else -}}false{{- end -}}"
enable-node-port: "{{- if .EnableNodePort -}}true{{- else -}}false{{- end -}}"
{{ with .Ipam }}
ipam: {{ . }}
{{ if eq . "eni" }}
enable-endpoint-routes: "true"
auto-create-cilium-node-resource: "true"
blacklist-conflicting-routes: "false"
{{ end }}
{{ end }}
{{ end }} # With .Networking.Cilium end
---
apiVersion: v1
@ -662,7 +670,6 @@ spec:
name: prometheus
protocol: TCP
{{ end }}
{{ end }}
livenessProbe:
httpGet:
path: /healthz
@ -675,3 +682,19 @@ spec:
restartPolicy: Always
serviceAccount: cilium-operator
serviceAccountName: cilium-operator
{{ if eq .Ipam "eni" }}
nodeSelector:
node-role.kubernetes.io/master: ""
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 300
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 300
{{ end }}
{{ end }}

25
upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template
View File

@ -121,6 +121,14 @@ data:
install-iptables-rules: "{{- if .IPTablesRulesNoinstall -}}false{{- else -}}true{{- end -}}"
auto-direct-node-routes: "{{- if .AutoDirectNodeRoutes -}}true{{- else -}}false{{- end -}}"
enable-node-port: "{{- if .EnableNodePort -}}true{{- else -}}false{{- end -}}"
{{ with .Ipam }}
ipam: {{ . }}
{{ if eq . "eni" }}
enable-endpoint-routes: "true"
auto-create-cilium-node-resource: "true"
blacklist-conflicting-routes: "false"
{{ end }}
{{ end }}
{{ end }} # With .Networking.Cilium end
---
apiVersion: v1
@ -654,7 +662,6 @@ spec:
name: prometheus
protocol: TCP
{{ end }}
{{ end }}
livenessProbe:
httpGet:
path: /healthz
@ -667,3 +674,19 @@ spec:
restartPolicy: Always
serviceAccount: cilium-operator
serviceAccountName: cilium-operator
{{if eq .Ipam "eni" }}
nodeSelector:
node-role.kubernetes.io/master: ""
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 300
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 300
{{ end }}
{{ end }}