cilium: Add unreachable route for pod IP on deletion option

When a pod is deleted, the route to its IP is replaced with an unreachable route. When a pod is created, the route is replaced with a route to the pod veth (so if an unreachable existed, it's replaced). Ref: - https://github.com/cilium/cilium/pull/18505
2023-01-24 12:43:58 +01:00 · 2023-01-24 12:43:58 +01:00 · 484bde5b9b
parent 68705f39c9
commit 484bde5b9b
32 changed files with 60 additions and 9 deletions
--- a/k8s/crds/kops.k8s.io_clusters.yaml
+++ b/k8s/crds/kops.k8s.io_clusters.yaml
@ -4887,6 +4887,10 @@ spec:
                      enableTracing:
                        description: EnableTracing is unused.
                        type: boolean
+                      enableUnreachableRoutes:
+                        description: 'EnableUnreachableRoutes enables unreachable
+                          routes on pod deletion. Default: false'
+                        type: boolean
                      enableipv4:
                        description: EnableIpv4 is unused.
                        type: boolean
--- a/pkg/apis/kops/networking.go
+++ b/pkg/apis/kops/networking.go
@ -473,6 +473,9 @@ type CiliumNetworkingSpec struct {
 	// EnableRemoteNodeIdentity enables the remote-node-identity.
 	// Default: true
 	EnableRemoteNodeIdentity *bool `json:"enableRemoteNodeIdentity,omitempty"`
+	// EnableUnreachableRoutes enables unreachable routes on pod deletion.
+	// Default: false
+	EnableUnreachableRoutes *bool `json:"enableUnreachableRoutes,omitempty"`
 	// Hubble configures the Hubble service on the Cilium agent.
 	Hubble *HubbleSpec `json:"hubble,omitempty"`

--- a/pkg/apis/kops/v1alpha2/networking.go
+++ b/pkg/apis/kops/v1alpha2/networking.go
@ -579,6 +579,9 @@ type CiliumNetworkingSpec struct {
 	// EnableRemoteNodeIdentity enables the remote-node-identity.
 	// Default: true
 	EnableRemoteNodeIdentity *bool `json:"enableRemoteNodeIdentity,omitempty"`
+	// EnableUnreachableRoutes enables unreachable routes on pod deletion.
+	// Default: false
+	EnableUnreachableRoutes *bool `json:"enableUnreachableRoutes,omitempty"`
 	// Hubble configures the Hubble service on the Cilium agent.
 	Hubble *HubbleSpec `json:"hubble,omitempty"`

--- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
+++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
@ -1997,6 +1997,7 @@ func autoConvert_v1alpha2_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in *
 	out.EnableNodePort = in.EnableNodePort
 	out.EtcdManaged = in.EtcdManaged
 	out.EnableRemoteNodeIdentity = in.EnableRemoteNodeIdentity
+	out.EnableUnreachableRoutes = in.EnableUnreachableRoutes
 	if in.Hubble != nil {
 		in, out := &in.Hubble, &out.Hubble
 		*out = new(kops.HubbleSpec)
@ -2059,6 +2060,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha2_CiliumNetworkingSpec(in *
 	out.EnableNodePort = in.EnableNodePort
 	out.EtcdManaged = in.EtcdManaged
 	out.EnableRemoteNodeIdentity = in.EnableRemoteNodeIdentity
+	out.EnableUnreachableRoutes = in.EnableUnreachableRoutes
 	if in.Hubble != nil {
 		in, out := &in.Hubble, &out.Hubble
 		*out = new(HubbleSpec)
--- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go
+++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go
@ -614,6 +614,11 @@ func (in *CiliumNetworkingSpec) DeepCopyInto(out *CiliumNetworkingSpec) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.EnableUnreachableRoutes != nil {
+		in, out := &in.EnableUnreachableRoutes, &out.EnableUnreachableRoutes
+		*out = new(bool)
+		**out = **in
+	}
 	if in.Hubble != nil {
 		in, out := &in.Hubble, &out.Hubble
 		*out = new(HubbleSpec)
--- a/pkg/apis/kops/v1alpha3/networking.go
+++ b/pkg/apis/kops/v1alpha3/networking.go
@ -436,6 +436,9 @@ type CiliumNetworkingSpec struct {
 	// EnableRemoteNodeIdentity enables the remote-node-identity.
 	// Default: true
 	EnableRemoteNodeIdentity *bool `json:"enableRemoteNodeIdentity,omitempty"`
+	// EnableUnreachableRoutes enables unreachable routes on pod deletion.
+	// Default: false
+	EnableUnreachableRoutes *bool `json:"enableUnreachableRoutes,omitempty"`
 	// Hubble configures the Hubble service on the Cilium agent.
 	Hubble *HubbleSpec `json:"hubble,omitempty"`

--- a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go
+++ b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go
@ -2120,6 +2120,7 @@ func autoConvert_v1alpha3_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in *
 	out.EnableNodePort = in.EnableNodePort
 	out.EtcdManaged = in.EtcdManaged
 	out.EnableRemoteNodeIdentity = in.EnableRemoteNodeIdentity
+	out.EnableUnreachableRoutes = in.EnableUnreachableRoutes
 	if in.Hubble != nil {
 		in, out := &in.Hubble, &out.Hubble
 		*out = new(kops.HubbleSpec)
@ -2182,6 +2183,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha3_CiliumNetworkingSpec(in *
 	out.EnableNodePort = in.EnableNodePort
 	out.EtcdManaged = in.EtcdManaged
 	out.EnableRemoteNodeIdentity = in.EnableRemoteNodeIdentity
+	out.EnableUnreachableRoutes = in.EnableUnreachableRoutes
 	if in.Hubble != nil {
 		in, out := &in.Hubble, &out.Hubble
 		*out = new(HubbleSpec)
--- a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go
+++ b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go
@ -631,6 +631,11 @@ func (in *CiliumNetworkingSpec) DeepCopyInto(out *CiliumNetworkingSpec) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.EnableUnreachableRoutes != nil {
+		in, out := &in.EnableUnreachableRoutes, &out.EnableUnreachableRoutes
+		*out = new(bool)
+		**out = **in
+	}
 	if in.Hubble != nil {
 		in, out := &in.Hubble, &out.Hubble
 		*out = new(HubbleSpec)
--- a/pkg/apis/kops/zz_generated.deepcopy.go
+++ b/pkg/apis/kops/zz_generated.deepcopy.go
@ -712,6 +712,11 @@ func (in *CiliumNetworkingSpec) DeepCopyInto(out *CiliumNetworkingSpec) {
 		*out = new(bool)
 		**out = **in
 	}
+	if in.EnableUnreachableRoutes != nil {
+		in, out := &in.EnableUnreachableRoutes, &out.EnableUnreachableRoutes
+		*out = new(bool)
+		**out = **in
+	}
 	if in.Hubble != nil {
 		in, out := &in.Hubble, &out.Hubble
 		*out = new(HubbleSpec)
--- a/pkg/model/components/cilium.go
+++ b/pkg/model/components/cilium.go
@ -127,6 +127,10 @@ func (b *CiliumOptionsBuilder) BuildOptions(o interface{}) error {
 		c.EnableRemoteNodeIdentity = fi.PtrTo(true)
 	}

+	if c.EnableUnreachableRoutes == nil {
+		c.EnableUnreachableRoutes = fi.PtrTo(false)
+	}
+
 	if c.EnableBPFMasquerade == nil {
 		c.EnableBPFMasquerade = fi.PtrTo(c.IPAM == "eni")
 	}
--- a/tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_cluster-completed.spec_content
@ -225,6 +225,7 @@ spec:
      enableEndpointHealthChecking: true
      enableL7Proxy: true
      enableRemoteNodeIdentity: true
+      enableUnreachableRoutes: false
      hubble:
        enabled: false
      identityAllocationMode: crd
--- a/tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_minimal-ipv6.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_minimal-ipv6.example.com-addons-bootstrap_content
@ -55,7 +55,7 @@ spec:
    version: 9.99.0
  - id: k8s-1.16
    manifest: networking.cilium.io/k8s-1.16-v1.12.yaml
-    manifestHash: 61473dd1bb279c61903a8127af34fa01cfc748ea2be8eb8d3728a4606b1237e6
+    manifestHash: f067524e5a9b34b2ed9533fe81e308cc7d25723ffbbd54be681be00f9edf155c
    name: networking.cilium.io
    needsRollingUpdate: all
    selector:
--- a/tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_minimal-ipv6.example.com-addons-networking.cilium.io-k8s-1.16_content
+++ b/tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_object_minimal-ipv6.example.com-addons-networking.cilium.io-k8s-1.16_content
@ -51,6 +51,7 @@ data:
  enable-node-port: "false"
  enable-remote-node-identity: "true"
  enable-service-topology: "false"
+  enable-unreachable-routes: "false"
  identity-allocation-mode: crd
  identity-change-grace-period: 5s
  install-iptables-rules: "true"
--- a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_cluster-completed.spec_content
@ -216,6 +216,7 @@ spec:
      enableEndpointHealthChecking: true
      enableL7Proxy: true
      enableRemoteNodeIdentity: true
+      enableUnreachableRoutes: false
      hubble:
        enabled: false
      identityAllocationMode: crd
--- a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_minimal-warmpool.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_minimal-warmpool.example.com-addons-bootstrap_content
@ -48,7 +48,7 @@ spec:
    version: 9.99.0
  - id: k8s-1.16
    manifest: networking.cilium.io/k8s-1.16-v1.12.yaml
-    manifestHash: 030973549f54d32700b7e6dfad4436c6e0030693e8cbc31e5e47048651cc262b
+    manifestHash: e94026a9dabe207b365e65f483c6f584be7b0ac125767a4e8487472741297b18
    name: networking.cilium.io
    needsRollingUpdate: all
    selector:
--- a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_minimal-warmpool.example.com-addons-networking.cilium.io-k8s-1.16_content
+++ b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_object_minimal-warmpool.example.com-addons-networking.cilium.io-k8s-1.16_content
@ -51,6 +51,7 @@ data:
  enable-node-port: "false"
  enable-remote-node-identity: "true"
  enable-service-topology: "false"
+  enable-unreachable-routes: "false"
  identity-allocation-mode: crd
  identity-change-grace-period: 5s
  install-iptables-rules: "true"
--- a/tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_cluster-completed.spec_content
@ -218,6 +218,7 @@ spec:
      enableEndpointHealthChecking: true
      enableL7Proxy: true
      enableRemoteNodeIdentity: true
+      enableUnreachableRoutes: false
      hubble:
        enabled: false
      identityAllocationMode: crd
--- a/tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content
@ -48,7 +48,7 @@ spec:
    version: 9.99.0
  - id: k8s-1.16
    manifest: networking.cilium.io/k8s-1.16-v1.12.yaml
-    manifestHash: 769fa3eac2e71553adb9a5965a876f3918488fcc41353c49b8c0f3701ce52358
+    manifestHash: a74648938bd05093db333999da4d5acb9277c5d4111f5919a19d1e980f544e4b
    name: networking.cilium.io
    needsRollingUpdate: all
    selector:
--- a/tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content
+++ b/tests/integration/update_cluster/privatecilium-eni/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content
@ -54,6 +54,7 @@ data:
  enable-node-port: "false"
  enable-remote-node-identity: "true"
  enable-service-topology: "false"
+  enable-unreachable-routes: "false"
  identity-allocation-mode: crd
  identity-change-grace-period: 5s
  install-iptables-rules: "true"
--- a/tests/integration/update_cluster/privatecilium/data/aws_s3_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/privatecilium/data/aws_s3_object_cluster-completed.spec_content
@ -222,6 +222,7 @@ spec:
      enableEndpointHealthChecking: true
      enableL7Proxy: true
      enableRemoteNodeIdentity: true
+      enableUnreachableRoutes: false
      hubble:
        enabled: false
      identityAllocationMode: crd
--- a/tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content
@ -48,7 +48,7 @@ spec:
    version: 9.99.0
  - id: k8s-1.16
    manifest: networking.cilium.io/k8s-1.16-v1.12.yaml
-    manifestHash: 197952743ca3f39f89b954c04b40349b3bf1192dccb0bae40c4008be5f9fd787
+    manifestHash: 2a1ee49e7de7f6109240c11f32555b269d41ac87416a0203f2cbac9fda60283a
    name: networking.cilium.io
    needsRollingUpdate: all
    selector:
--- a/tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content
+++ b/tests/integration/update_cluster/privatecilium/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content
@ -51,6 +51,7 @@ data:
  enable-node-port: "false"
  enable-remote-node-identity: "true"
  enable-service-topology: "false"
+  enable-unreachable-routes: "false"
  identity-allocation-mode: crd
  identity-change-grace-period: 5s
  install-iptables-rules: "true"
--- a/tests/integration/update_cluster/privatecilium2/data/aws_s3_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/privatecilium2/data/aws_s3_object_cluster-completed.spec_content
@ -221,6 +221,7 @@ spec:
      enableEndpointHealthChecking: true
      enableL7Proxy: true
      enableRemoteNodeIdentity: true
+      enableUnreachableRoutes: false
      hubble:
        enabled: true
      identityAllocationMode: crd
--- a/tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content
@ -111,7 +111,7 @@ spec:
    version: 9.99.0
  - id: k8s-1.16
    manifest: networking.cilium.io/k8s-1.16-v1.12.yaml
-    manifestHash: d0fef712bba535f969cf39fbdc2a81be2d85d08c1ca7a716fa9a110620400b39
+    manifestHash: ba167eb44300511acb7079eaa68f90d368ef4c469b9e16b1dc96b687ff6ea5ed
    name: networking.cilium.io
    needsPKI: true
    needsRollingUpdate: all
--- a/tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content
+++ b/tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content
@ -65,6 +65,7 @@ data:
  enable-node-port: "false"
  enable-remote-node-identity: "true"
  enable-service-topology: "false"
+  enable-unreachable-routes: "false"
  hubble-disable-tls: "false"
  hubble-listen-address: :4244
  hubble-socket-path: /var/run/cilium/hubble.sock
--- a/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_cluster-completed.spec_content
@ -232,6 +232,7 @@ spec:
      enableL7Proxy: true
      enableNodePort: true
      enableRemoteNodeIdentity: true
+      enableUnreachableRoutes: false
      etcdManaged: true
      hubble:
        enabled: false
--- a/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_privateciliumadvanced.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_privateciliumadvanced.example.com-addons-bootstrap_content
@ -48,7 +48,7 @@ spec:
    version: 9.99.0
  - id: k8s-1.16
    manifest: networking.cilium.io/k8s-1.16-v1.12.yaml
-    manifestHash: ead84fffa6d9e5335e6b981c9c916af2eb902b1c2af8cfb919b5eaa48979950a
+    manifestHash: c562a1618bc207bcf3727a40e87ae098596e7281afc035425172d23814d2100c
    name: networking.cilium.io
    needsRollingUpdate: all
    selector:
--- a/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_privateciliumadvanced.example.com-addons-networking.cilium.io-k8s-1.16_content
+++ b/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_object_privateciliumadvanced.example.com-addons-networking.cilium.io-k8s-1.16_content
@ -55,6 +55,7 @@ data:
  enable-node-port: "true"
  enable-remote-node-identity: "true"
  enable-service-topology: "false"
+  enable-unreachable-routes: "false"
  etcd-config: |-
    ---
    endpoints:
--- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.12.yaml.template
+++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.12.yaml.template
@ -259,6 +259,9 @@ data:
  # Enable use of remote node identity (default false)
  enable-remote-node-identity: "{{ .EnableRemoteNodeIdentity }}"

+  # Enable unreachable routes on pod deletion (default false)
+  enable-unreachable-routes: "{{ .EnableUnreachableRoutes }}"
+
  # enable-l7-proxy enables L7 proxy for L7 policy enforcement. (default true)
  enable-l7-proxy: "{{ .EnableL7Proxy }}"

--- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml
+++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml
@ -48,7 +48,7 @@ spec:
    version: 9.99.0
  - id: k8s-1.16
    manifest: networking.cilium.io/k8s-1.16-v1.12.yaml
-    manifestHash: 586c92a965bab52296ba54e59e91c45731dc0d7449f98858ca33e74b6c010ba6
+    manifestHash: d0d0425155f3251dda3a1f9fdfd7e9bae02f50380d0a503f47968887ae0a3767
    name: networking.cilium.io
    needsRollingUpdate: all
    selector:
--- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml
+++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml
@ -55,7 +55,7 @@ spec:
    version: 9.99.0
  - id: k8s-1.16
    manifest: networking.cilium.io/k8s-1.16-v1.12.yaml
-    manifestHash: 586c92a965bab52296ba54e59e91c45731dc0d7449f98858ca33e74b6c010ba6
+    manifestHash: d0d0425155f3251dda3a1f9fdfd7e9bae02f50380d0a503f47968887ae0a3767
    name: networking.cilium.io
    needsRollingUpdate: all
    selector:
--- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml
+++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml
@ -112,7 +112,7 @@ spec:
    version: 9.99.0
  - id: k8s-1.16
    manifest: networking.cilium.io/k8s-1.16-v1.12.yaml
-    manifestHash: 586c92a965bab52296ba54e59e91c45731dc0d7449f98858ca33e74b6c010ba6
+    manifestHash: d0d0425155f3251dda3a1f9fdfd7e9bae02f50380d0a503f47968887ae0a3767
    name: networking.cilium.io
    needsRollingUpdate: all
    selector: