cilium: Add unreachable route for pod IP on deletion option

When a pod is deleted, the route to its IP is replaced with an unreachable route.
When a pod is created, the route is replaced with a route to the pod veth (so if an unreachable existed, it's replaced).

Ref:
 - https://github.com/cilium/cilium/pull/18505

k8s/crds/kops.k8s.io_clusters.yaml

@@ -4887,6 +4887,10 @@ spec:
                       enableTracing:
                         description: EnableTracing is unused.
                         type: boolean
+                      enableUnreachableRoutes:
+                        description: 'EnableUnreachableRoutes enables unreachable
+                          routes on pod deletion. Default: false'
+                        type: boolean
                       enableipv4:
                         description: EnableIpv4 is unused.
                         type: boolean

pkg/apis/kops/networking.go

@@ -473,6 +473,9 @@ type CiliumNetworkingSpec struct {
 	// EnableRemoteNodeIdentity enables the remote-node-identity.
 	// Default: true
 	EnableRemoteNodeIdentity *bool `json:"enableRemoteNodeIdentity,omitempty"`
+	// EnableUnreachableRoutes enables unreachable routes on pod deletion.
+	// Default: false
+	EnableUnreachableRoutes *bool `json:"enableUnreachableRoutes,omitempty"`
 	// Hubble configures the Hubble service on the Cilium agent.
 	Hubble *HubbleSpec `json:"hubble,omitempty"`
 

pkg/apis/kops/v1alpha2/networking.go

@@ -579,6 +579,9 @@ type CiliumNetworkingSpec struct {
 	// EnableRemoteNodeIdentity enables the remote-node-identity.
 	// Default: true
 	EnableRemoteNodeIdentity *bool `json:"enableRemoteNodeIdentity,omitempty"`
+	// EnableUnreachableRoutes enables unreachable routes on pod deletion.
+	// Default: false
+	EnableUnreachableRoutes *bool `json:"enableUnreachableRoutes,omitempty"`
 	// Hubble configures the Hubble service on the Cilium agent.
 	Hubble *HubbleSpec `json:"hubble,omitempty"`
 

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -1997,6 +1997,7 @@ func autoConvert_v1alpha2_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in *
 	out.EnableNodePort = in.EnableNodePort
 	out.EtcdManaged = in.EtcdManaged
 	out.EnableRemoteNodeIdentity = in.EnableRemoteNodeIdentity
+	out.EnableUnreachableRoutes = in.EnableUnreachableRoutes
 	if in.Hubble != nil {
 		in, out := &in.Hubble, &out.Hubble
 		*out = new(kops.HubbleSpec)
@@ -2059,6 +2060,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha2_CiliumNetworkingSpec(in *
 	out.EnableNodePort = in.EnableNodePort
 	out.EtcdManaged = in.EtcdManaged
 	out.EnableRemoteNodeIdentity = in.EnableRemoteNodeIdentity
+	out.EnableUnreachableRoutes = in.EnableUnreachableRoutes
 	if in.Hubble != nil {
 		in, out := &in.Hubble, &out.Hubble
 		*out = new(HubbleSpec)

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -614,6 +614,11 @@ func (in *CiliumNetworkingSpec) DeepCopyInto(out *CiliumNetworkingSpec) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.EnableUnreachableRoutes != nil {
+		in, out := &in.EnableUnreachableRoutes, &out.EnableUnreachableRoutes
+		*out = new(bool)
+		**out = **in
+	}
 	if in.Hubble != nil {
 		in, out := &in.Hubble, &out.Hubble
 		*out = new(HubbleSpec)

pkg/apis/kops/v1alpha3/networking.go

@@ -436,6 +436,9 @@ type CiliumNetworkingSpec struct {
 	// EnableRemoteNodeIdentity enables the remote-node-identity.
 	// Default: true
 	EnableRemoteNodeIdentity *bool `json:"enableRemoteNodeIdentity,omitempty"`
+	// EnableUnreachableRoutes enables unreachable routes on pod deletion.
+	// Default: false
+	EnableUnreachableRoutes *bool `json:"enableUnreachableRoutes,omitempty"`
 	// Hubble configures the Hubble service on the Cilium agent.
 	Hubble *HubbleSpec `json:"hubble,omitempty"`
 

pkg/apis/kops/v1alpha3/zz_generated.conversion.go

@@ -2120,6 +2120,7 @@ func autoConvert_v1alpha3_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in *
 	out.EnableNodePort = in.EnableNodePort
 	out.EtcdManaged = in.EtcdManaged
 	out.EnableRemoteNodeIdentity = in.EnableRemoteNodeIdentity
+	out.EnableUnreachableRoutes = in.EnableUnreachableRoutes
 	if in.Hubble != nil {
 		in, out := &in.Hubble, &out.Hubble
 		*out = new(kops.HubbleSpec)
@@ -2182,6 +2183,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha3_CiliumNetworkingSpec(in *
 	out.EnableNodePort = in.EnableNodePort
 	out.EtcdManaged = in.EtcdManaged
 	out.EnableRemoteNodeIdentity = in.EnableRemoteNodeIdentity
+	out.EnableUnreachableRoutes = in.EnableUnreachableRoutes
 	if in.Hubble != nil {
 		in, out := &in.Hubble, &out.Hubble
 		*out = new(HubbleSpec)

pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go

@@ -631,6 +631,11 @@ func (in *CiliumNetworkingSpec) DeepCopyInto(out *CiliumNetworkingSpec) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.EnableUnreachableRoutes != nil {
+		in, out := &in.EnableUnreachableRoutes, &out.EnableUnreachableRoutes
+		*out = new(bool)
+		**out = **in
+	}
 	if in.Hubble != nil {
 		in, out := &in.Hubble, &out.Hubble
 		*out = new(HubbleSpec)

pkg/apis/kops/zz_generated.deepcopy.go

@@ -712,6 +712,11 @@ func (in *CiliumNetworkingSpec) DeepCopyInto(out *CiliumNetworkingSpec) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.EnableUnreachableRoutes != nil {
+		in, out := &in.EnableUnreachableRoutes, &out.EnableUnreachableRoutes
+		*out = new(bool)
+		**out = **in
+	}
 	if in.Hubble != nil {
 		in, out := &in.Hubble, &out.Hubble
 		*out = new(HubbleSpec)

pkg/model/components/cilium.go

@@ -127,6 +127,10 @@ func (b *CiliumOptionsBuilder) BuildOptions(o interface{}) error {
 		c.EnableRemoteNodeIdentity = fi.PtrTo(true)
 	}
 
+	if c.EnableUnreachableRoutes == nil {
+		c.EnableUnreachableRoutes = fi.PtrTo(false)
+	}
+
 	if c.EnableBPFMasquerade == nil {
 		c.EnableBPFMasquerade = fi.PtrTo(c.IPAM == "eni")
 	}

tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_cluster-completed.spec_content

@@ -225,6 +225,7 @@ spec:
       enableEndpointHealthChecking: true
       enableL7Proxy: true
       enableRemoteNodeIdentity: true
+      enableUnreachableRoutes: false
       hubble:
         enabled: false
       identityAllocationMode: crd

tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_minimal-ipv6.example.com-addons-bootstrap_content

@@ -55,7 +55,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.16
     manifest: networking.cilium.io/k8s-1.16-v1.12.yaml
-    manifestHash: 61473dd1bb279c61903a8127af34fa01cfc748ea2be8eb8d3728a4606b1237e6
+    manifestHash: f067524e5a9b34b2ed9533fe81e308cc7d25723ffbbd54be681be00f9edf155c
     name: networking.cilium.io
     needsRollingUpdate: all
     selector:

tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_minimal-ipv6.example.com-addons-networking.cilium.io-k8s-1.16_content

@@ -51,6 +51,7 @@ data:
   enable-node-port: "false"
   enable-remote-node-identity: "true"
   enable-service-topology: "false"
+  enable-unreachable-routes: "false"
   identity-allocation-mode: crd
   identity-change-grace-period: 5s
   install-iptables-rules: "true"

tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_cluster-completed.spec_content

@@ -216,6 +216,7 @@ spec:
       enableEndpointHealthChecking: true
       enableL7Proxy: true
       enableRemoteNodeIdentity: true
+      enableUnreachableRoutes: false
       hubble:
         enabled: false
       identityAllocationMode: crd

tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_minimal-warmpool.example.com-addons-bootstrap_content

@@ -48,7 +48,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.16
     manifest: networking.cilium.io/k8s-1.16-v1.12.yaml
-    manifestHash: 030973549f54d32700b7e6dfad4436c6e0030693e8cbc31e5e47048651cc262b
+    manifestHash: e94026a9dabe207b365e65f483c6f584be7b0ac125767a4e8487472741297b18
     name: networking.cilium.io
     needsRollingUpdate: all
     selector:

tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_minimal-warmpool.example.com-addons-networking.cilium.io-k8s-1.16_content

@@ -51,6 +51,7 @@ data:
   enable-node-port: "false"
   enable-remote-node-identity: "true"
   enable-service-topology: "false"
+  enable-unreachable-routes: "false"
   identity-allocation-mode: crd
   identity-change-grace-period: 5s
   install-iptables-rules: "true"

tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_cluster-completed.spec_content

@@ -218,6 +218,7 @@ spec:
       enableEndpointHealthChecking: true
       enableL7Proxy: true
       enableRemoteNodeIdentity: true
+      enableUnreachableRoutes: false
       hubble:
         enabled: false
       identityAllocationMode: crd

tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content

@@ -48,7 +48,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.16
     manifest: networking.cilium.io/k8s-1.16-v1.12.yaml
-    manifestHash: 769fa3eac2e71553adb9a5965a876f3918488fcc41353c49b8c0f3701ce52358
+    manifestHash: a74648938bd05093db333999da4d5acb9277c5d4111f5919a19d1e980f544e4b
     name: networking.cilium.io
     needsRollingUpdate: all
     selector:

tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content

@@ -54,6 +54,7 @@ data:
   enable-node-port: "false"
   enable-remote-node-identity: "true"
   enable-service-topology: "false"
+  enable-unreachable-routes: "false"
   identity-allocation-mode: crd
   identity-change-grace-period: 5s
   install-iptables-rules: "true"

tests/integration/update_cluster/privatecilium/data/aws_s3_object_cluster-completed.spec_content

@@ -222,6 +222,7 @@ spec:
       enableEndpointHealthChecking: true
       enableL7Proxy: true
       enableRemoteNodeIdentity: true
+      enableUnreachableRoutes: false
       hubble:
         enabled: false
       identityAllocationMode: crd

tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content

@@ -48,7 +48,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.16
     manifest: networking.cilium.io/k8s-1.16-v1.12.yaml
-    manifestHash: 197952743ca3f39f89b954c04b40349b3bf1192dccb0bae40c4008be5f9fd787
+    manifestHash: 2a1ee49e7de7f6109240c11f32555b269d41ac87416a0203f2cbac9fda60283a
     name: networking.cilium.io
     needsRollingUpdate: all
     selector:

tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content

@@ -51,6 +51,7 @@ data:
   enable-node-port: "false"
   enable-remote-node-identity: "true"
   enable-service-topology: "false"
+  enable-unreachable-routes: "false"
   identity-allocation-mode: crd
   identity-change-grace-period: 5s
   install-iptables-rules: "true"

tests/integration/update_cluster/privatecilium2/data/aws_s3_object_cluster-completed.spec_content

@@ -221,6 +221,7 @@ spec:
       enableEndpointHealthChecking: true
       enableL7Proxy: true
       enableRemoteNodeIdentity: true
+      enableUnreachableRoutes: false
       hubble:
         enabled: true
       identityAllocationMode: crd

tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content

@@ -111,7 +111,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.16
     manifest: networking.cilium.io/k8s-1.16-v1.12.yaml
-    manifestHash: d0fef712bba535f969cf39fbdc2a81be2d85d08c1ca7a716fa9a110620400b39
+    manifestHash: ba167eb44300511acb7079eaa68f90d368ef4c469b9e16b1dc96b687ff6ea5ed
     name: networking.cilium.io
     needsPKI: true
     needsRollingUpdate: all

tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content

@@ -65,6 +65,7 @@ data:
   enable-node-port: "false"
   enable-remote-node-identity: "true"
   enable-service-topology: "false"
+  enable-unreachable-routes: "false"
   hubble-disable-tls: "false"
   hubble-listen-address: :4244
   hubble-socket-path: /var/run/cilium/hubble.sock

tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_cluster-completed.spec_content

@@ -232,6 +232,7 @@ spec:
       enableL7Proxy: true
       enableNodePort: true
       enableRemoteNodeIdentity: true
+      enableUnreachableRoutes: false
       etcdManaged: true
       hubble:
         enabled: false

tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_privateciliumadvanced.example.com-addons-bootstrap_content

@@ -48,7 +48,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.16
     manifest: networking.cilium.io/k8s-1.16-v1.12.yaml
-    manifestHash: ead84fffa6d9e5335e6b981c9c916af2eb902b1c2af8cfb919b5eaa48979950a
+    manifestHash: c562a1618bc207bcf3727a40e87ae098596e7281afc035425172d23814d2100c
     name: networking.cilium.io
     needsRollingUpdate: all
     selector:

tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_privateciliumadvanced.example.com-addons-networking.cilium.io-k8s-1.16_content

@@ -55,6 +55,7 @@ data:
   enable-node-port: "true"
   enable-remote-node-identity: "true"
   enable-service-topology: "false"
+  enable-unreachable-routes: "false"
   etcd-config: |-
     ---
     endpoints:

upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.12.yaml.template

@@ -259,6 +259,9 @@ data:
   # Enable use of remote node identity (default false)
   enable-remote-node-identity: "{{ .EnableRemoteNodeIdentity }}"
 
+  # Enable unreachable routes on pod deletion (default false)
+  enable-unreachable-routes: "{{ .EnableUnreachableRoutes }}"
+
   # enable-l7-proxy enables L7 proxy for L7 policy enforcement. (default true)
   enable-l7-proxy: "{{ .EnableL7Proxy }}"
 

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml

@@ -48,7 +48,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.16
     manifest: networking.cilium.io/k8s-1.16-v1.12.yaml
-    manifestHash: 586c92a965bab52296ba54e59e91c45731dc0d7449f98858ca33e74b6c010ba6
+    manifestHash: d0d0425155f3251dda3a1f9fdfd7e9bae02f50380d0a503f47968887ae0a3767
     name: networking.cilium.io
     needsRollingUpdate: all
     selector:

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml

@@ -55,7 +55,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.16
     manifest: networking.cilium.io/k8s-1.16-v1.12.yaml
-    manifestHash: 586c92a965bab52296ba54e59e91c45731dc0d7449f98858ca33e74b6c010ba6
+    manifestHash: d0d0425155f3251dda3a1f9fdfd7e9bae02f50380d0a503f47968887ae0a3767
     name: networking.cilium.io
     needsRollingUpdate: all
     selector:

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml

@@ -112,7 +112,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.16
     manifest: networking.cilium.io/k8s-1.16-v1.12.yaml
-    manifestHash: 586c92a965bab52296ba54e59e91c45731dc0d7449f98858ca33e74b6c010ba6
+    manifestHash: d0d0425155f3251dda3a1f9fdfd7e9bae02f50380d0a503f47968887ae0a3767
     name: networking.cilium.io
     needsRollingUpdate: all
     selector: