Merge pull request #14694 from olemarkus/cilium-eni-fix

Fix Cilium ENI ipam

pkg/apis/kops/validation/validation.go

@@ -1041,8 +1041,8 @@ func validateNetworkingCilium(cluster *kops.Cluster, v *kops.CiliumNetworkingSpe
 			if c.GetCloudProvider() != kops.CloudProviderAWS {
 				allErrs = append(allErrs, field.Forbidden(fldPath.Child("ipam"), "Cilum ENI IPAM is supported only in AWS"))
 			}
-			if v.Masquerade != nil && *v.Masquerade {
+			if v.Masquerade != nil && !*v.Masquerade {
-				allErrs = append(allErrs, field.Forbidden(fldPath.Child("masquerade"), "Masquerade must be disabled when ENI IPAM is used"))
+				allErrs = append(allErrs, field.Forbidden(fldPath.Child("masquerade"), "Masquerade must be enabled when ENI IPAM is used"))
 			}
 			if c.IsIPv6Only() {
 				allErrs = append(allErrs, field.Forbidden(fldPath.Child("ipam"), "Cilium ENI IPAM does not support IPv6"))

pkg/apis/kops/validation/validation_test.go

@@ -843,7 +843,7 @@ func Test_Validate_Cilium(t *testing.T) {
 		},
 		{
 			Cilium: kops.CiliumNetworkingSpec{
-				Masquerade: fi.PtrTo(false),
+				Masquerade: fi.PtrTo(true),
 				IPAM:       "eni",
 			},
 			Spec: kops.ClusterSpec{
@@ -860,7 +860,7 @@ func Test_Validate_Cilium(t *testing.T) {
 		},
 		{
 			Cilium: kops.CiliumNetworkingSpec{
-				Masquerade: fi.PtrTo(true),
+				Masquerade: fi.PtrTo(false),
 				IPAM:       "eni",
 			},
 			Spec: kops.ClusterSpec{

pkg/model/components/cilium.go

@@ -112,7 +112,7 @@ func (b *CiliumOptionsBuilder) BuildOptions(o interface{}) error {
 	}
 
 	if c.Masquerade == nil {
-		c.Masquerade = fi.PtrTo(!clusterSpec.IsIPv6Only() && c.IPAM != "eni")
+		c.Masquerade = fi.PtrTo(!clusterSpec.IsIPv6Only())
 	}
 
 	if c.Tunnel == "" {
@@ -128,7 +128,7 @@ func (b *CiliumOptionsBuilder) BuildOptions(o interface{}) error {
 	}
 
 	if c.EnableBPFMasquerade == nil {
-		c.EnableBPFMasquerade = fi.PtrTo(false)
+		c.EnableBPFMasquerade = fi.PtrTo(c.IPAM == "eni")
 	}
 
 	if c.EnableL7Proxy == nil {

tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_minimal-ipv6.example.com-addons-bootstrap_content

@@ -55,7 +55,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.16
     manifest: networking.cilium.io/k8s-1.16-v1.11.yaml
-    manifestHash: 8dca7741f5f2c8cea1f5dd5e2b4fb5c6833816bf6a5968117406e4ab9737985a
+    manifestHash: 8e0768117104113c52ed1ff4bcc311914aa326187a3d10fe18ed63954f16ba0f
     name: networking.cilium.io
     needsRollingUpdate: all
     selector:

tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_minimal-ipv6.example.com-addons-networking.cilium.io-k8s-1.16_content

@@ -44,6 +44,7 @@ data:
   enable-bpf-masquerade: "false"
   enable-endpoint-health-checking: "true"
   enable-ipv4: "false"
+  enable-ipv4-masquerade: "false"
   enable-ipv6: "true"
   enable-ipv6-masquerade: "false"
   enable-l7-proxy: "true"
@@ -55,7 +56,6 @@ data:
   install-iptables-rules: "true"
   ipam: kubernetes
   kube-proxy-replacement: partial
-  masquerade: "false"
   monitor-aggregation: medium
   nodes-gc-interval: 5m0s
   preallocate-bpf-maps: "false"

tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_minimal-warmpool.example.com-addons-bootstrap_content

@@ -48,7 +48,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.16
     manifest: networking.cilium.io/k8s-1.16-v1.11.yaml
-    manifestHash: 925bfdc2b33c36273d5c2b1589b801bcf8d1d2b789ff5bd2bd80a840e278795a
+    manifestHash: c3ae71c91e47dbeda0c0a427f4262d3190ad5cb4efaf787033d793ed05c46f63
     name: networking.cilium.io
     needsRollingUpdate: all
     selector:

tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_minimal-warmpool.example.com-addons-networking.cilium.io-k8s-1.16_content

@@ -44,6 +44,7 @@ data:
   enable-bpf-masquerade: "false"
   enable-endpoint-health-checking: "true"
   enable-ipv4: "true"
+  enable-ipv4-masquerade: "true"
   enable-ipv6: "false"
   enable-ipv6-masquerade: "false"
   enable-l7-proxy: "true"
@@ -55,7 +56,6 @@ data:
   install-iptables-rules: "true"
   ipam: kubernetes
   kube-proxy-replacement: partial
-  masquerade: "true"
   monitor-aggregation: medium
   nodes-gc-interval: 5m0s
   preallocate-bpf-maps: "false"

tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_cluster-completed.spec_content

@@ -177,8 +177,8 @@ spec:
       clusterName: default
       cpuRequest: 25m
       disableCNPStatusUpdates: true
-      disableMasquerade: true
+      disableMasquerade: false
-      enableBPFMasquerade: false
+      enableBPFMasquerade: true
       enableEndpointHealthChecking: true
       enableL7Proxy: true
       enableRemoteNodeIdentity: true

tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content

@@ -48,7 +48,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.16
     manifest: networking.cilium.io/k8s-1.16-v1.11.yaml
-    manifestHash: 8a5107386f0fea73a5d7e14cd94fc20219ac1672e6e35bb9aa529128b0d9bec9
+    manifestHash: e3eb2b6494c1a24704d9663423e8d388acf23a0aabb90651d178a675738f1462
     name: networking.cilium.io
     needsRollingUpdate: all
     selector:

tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content

@@ -43,10 +43,11 @@ data:
   debug: "false"
   disable-cnp-status-updates: "true"
   disable-endpoint-crd: "false"
-  enable-bpf-masquerade: "false"
+  enable-bpf-masquerade: "true"
   enable-endpoint-health-checking: "true"
   enable-endpoint-routes: "true"
   enable-ipv4: "true"
+  enable-ipv4-masquerade: "true"
   enable-ipv6: "false"
   enable-ipv6-masquerade: "false"
   enable-l7-proxy: "true"
@@ -58,7 +59,6 @@ data:
   install-iptables-rules: "true"
   ipam: eni
   kube-proxy-replacement: partial
-  masquerade: "false"
   monitor-aggregation: medium
   nodes-gc-interval: 5m0s
   preallocate-bpf-maps: "false"

tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content

@@ -48,7 +48,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.16
     manifest: networking.cilium.io/k8s-1.16-v1.11.yaml
-    manifestHash: b8e6ace39f88ef81ca852eb08adc0f0fa294449c3387849a5a8b5808ad08207b
+    manifestHash: 26c6d43928b2338a73b52d857d7f7bf2676e6cbd6d5c57725f53b6cb45432929
     name: networking.cilium.io
     needsRollingUpdate: all
     selector:

tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content

@@ -44,6 +44,7 @@ data:
   enable-bpf-masquerade: "false"
   enable-endpoint-health-checking: "true"
   enable-ipv4: "true"
+  enable-ipv4-masquerade: "true"
   enable-ipv6: "false"
   enable-ipv6-masquerade: "false"
   enable-l7-proxy: "true"
@@ -55,7 +56,6 @@ data:
   install-iptables-rules: "true"
   ipam: kubernetes
   kube-proxy-replacement: partial
-  masquerade: "true"
   monitor-aggregation: medium
   nodes-gc-interval: 5m0s
   preallocate-bpf-maps: "false"

tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content

@@ -61,7 +61,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.16
     manifest: networking.cilium.io/k8s-1.16-v1.11.yaml
-    manifestHash: e2e5c9b1c0641e661bedfa24749597167be409d1e13e4511c62b68af88f09dc5
+    manifestHash: 83b60d444aea65103ec26335fe93bed3f428a2fcfabf6f5fabfa83521e85f19d
     name: networking.cilium.io
     needsPKI: true
     needsRollingUpdate: all

tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content

@@ -58,6 +58,7 @@ data:
   enable-endpoint-health-checking: "true"
   enable-hubble: "true"
   enable-ipv4: "true"
+  enable-ipv4-masquerade: "true"
   enable-ipv6: "false"
   enable-ipv6-masquerade: "false"
   enable-l7-proxy: "true"
@@ -75,7 +76,6 @@ data:
   install-iptables-rules: "true"
   ipam: kubernetes
   kube-proxy-replacement: partial
-  masquerade: "true"
   monitor-aggregation: medium
   nodes-gc-interval: 5m0s
   preallocate-bpf-maps: "false"

tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_cluster-completed.spec_content

@@ -185,8 +185,8 @@ spec:
       clusterName: default
       cpuRequest: 25m
       disableCNPStatusUpdates: true
-      disableMasquerade: true
+      disableMasquerade: false
-      enableBPFMasquerade: false
+      enableBPFMasquerade: true
       enableEndpointHealthChecking: true
       enableL7Proxy: true
       enableNodePort: true

tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_privateciliumadvanced.example.com-addons-bootstrap_content

@@ -48,7 +48,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.16
     manifest: networking.cilium.io/k8s-1.16-v1.11.yaml
-    manifestHash: e4a99245537437ec9596da5feea66740b45d873c840ae3b7dc917884cda582eb
+    manifestHash: b6dde3049975e0e183acfe020a65a5ea08202e02589a536184487c17bfb6b598
     name: networking.cilium.io
     needsRollingUpdate: all
     selector:

tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_privateciliumadvanced.example.com-addons-networking.cilium.io-k8s-1.16_content

@@ -43,10 +43,11 @@ data:
   debug: "false"
   disable-cnp-status-updates: "true"
   disable-endpoint-crd: "false"
-  enable-bpf-masquerade: "false"
+  enable-bpf-masquerade: "true"
   enable-endpoint-health-checking: "true"
   enable-endpoint-routes: "true"
   enable-ipv4: "true"
+  enable-ipv4-masquerade: "true"
   enable-ipv6: "false"
   enable-ipv6-masquerade: "false"
   enable-k8s-event-handover: "true"
@@ -69,7 +70,6 @@ data:
   kube-proxy-replacement: strict
   kvstore: etcd
   kvstore-opt: '{"etcd.config": "/var/lib/etcd-config/etcd.config"}'
-  masquerade: "false"
   monitor-aggregation: medium
   nodes-gc-interval: 5m0s
   preallocate-bpf-maps: "false"

tests/integration/update_cluster/privateciliumadvanced/in-v1alpha2.yaml

@@ -34,7 +34,7 @@ spec:
     cilium:
       enableNodePort: true
       etcdManaged: true
-      disableMasquerade: true
+      disableMasquerade: false
       ipam: eni
   nonMasqueradeCIDR: 100.64.0.0/10
   sshAccess:

upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.11.yaml.template

@@ -231,7 +231,7 @@ data:
   # - none
   # - auto (automatically detect the container runtime)
   #
-  masquerade: "{{ .Masquerade }}"
+  enable-ipv4-masquerade: "{{ .Masquerade }}"
   enable-ipv6-masquerade: "false"
   install-iptables-rules: "{{ WithDefaultBool .InstallIptablesRules true }}"
   auto-direct-node-routes: "{{ .AutoDirectNodeRoutes }}"

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml

@@ -48,7 +48,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.16
     manifest: networking.cilium.io/k8s-1.16-v1.11.yaml
-    manifestHash: 678572068ce63e2afc9fafd712bdef61d715f2fec45f04a2c2de875271dd6e6d
+    manifestHash: 225f529de36a87bacd6d60df52f0b11c82b2f1b93b880adfd2d76cf625dea72a
     name: networking.cilium.io
     needsRollingUpdate: all
     selector:

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml

@@ -55,7 +55,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.16
     manifest: networking.cilium.io/k8s-1.16-v1.11.yaml
-    manifestHash: 678572068ce63e2afc9fafd712bdef61d715f2fec45f04a2c2de875271dd6e6d
+    manifestHash: 225f529de36a87bacd6d60df52f0b11c82b2f1b93b880adfd2d76cf625dea72a
     name: networking.cilium.io
     needsRollingUpdate: all
     selector:

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml

@@ -62,7 +62,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.16
     manifest: networking.cilium.io/k8s-1.16-v1.11.yaml
-    manifestHash: 678572068ce63e2afc9fafd712bdef61d715f2fec45f04a2c2de875271dd6e6d
+    manifestHash: 225f529de36a87bacd6d60df52f0b11c82b2f1b93b880adfd2d76cf625dea72a
     name: networking.cilium.io
     needsRollingUpdate: all
     selector: