From 88ed9f68bd10b96dac36e4a143221b56d0ca8347 Mon Sep 17 00:00:00 2001 From: bjhaid Date: Tue, 27 Apr 2021 10:55:33 -0500 Subject: [PATCH] Add support for configuring Cilium enable-host-reachable-services. After upgrading Cilium to 1.8 via kops one of our clusters had a total outage due to cilium reporting errors as below: ``` level=error msg="endpoint regeneration failed" containerID= datapathPolicyRevision=0 desiredPolicyRevision=1 endpointID=592 error="Failed to load tc filter: exit status 1" identity=40147 ipv4= ipv6= k8sPodName=/ subsys=endpoint ``` upon searching Cilium slack we found the below thread: https://cilium.slack.com/archives/C1MATJ5U5/p1616400216167600 which recommended setting `enable-host-reachable-services` to true will address the problems. We set the field and it fixed our issues too, however we observed that kops does not have a means to configure this hence this PR. We will like to have this backported after it has been merged. --- k8s/crds/kops.k8s.io_clusters.yaml | 6 ++++++ pkg/apis/kops/networking.go | 5 +++++ pkg/apis/kops/v1alpha2/networking.go | 5 +++++ pkg/apis/kops/v1alpha2/zz_generated.conversion.go | 2 ++ pkg/commands/set_cluster_test.go | 15 +++++++++++++++ .../k8s-1.12-v1.9.yaml.template | 3 +++ 6 files changed, 36 insertions(+) diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 1f29a7b909..52a658f2e4 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -3459,6 +3459,12 @@ spec: description: 'EnableEncryption enables Cilium Encryption. Default: false' type: boolean + enableHostReachableServices: + description: 'EnableHostReachableServices configures Cilium + to enable services to be reached from the host namespace + in addition to pod namespaces. https://docs.cilium.io/en/v1.9/gettingstarted/host-services/ + Default: false' + type: boolean enableNodePort: description: 'EnableNodePort replaces kube-proxy with Cilium''s BPF implementation. Requires spec.kubeProxy.enabled be set diff --git a/pkg/apis/kops/networking.go b/pkg/apis/kops/networking.go index d1f9aac134..db139a8124 100644 --- a/pkg/apis/kops/networking.go +++ b/pkg/apis/kops/networking.go @@ -471,6 +471,11 @@ type CiliumNetworkingSpec struct { // AutoDirectNodeRoutes adds automatic L2 routing between nodes. // Default: false AutoDirectNodeRoutes bool `json:"autoDirectNodeRoutes,omitempty"` + // EnableHostReachableServices configures Cilium to enable services to be + // reached from the host namespace in addition to pod namespaces. + // https://docs.cilium.io/en/v1.9/gettingstarted/host-services/ + // Default: false + EnableHostReachableServices bool `json:"enableHostReachableServices,omitempty"` // EnableNodePort replaces kube-proxy with Cilium's BPF implementation. // Requires spec.kubeProxy.enabled be set to false. // Default: false diff --git a/pkg/apis/kops/v1alpha2/networking.go b/pkg/apis/kops/v1alpha2/networking.go index 0af7746a7b..5acefec4a6 100644 --- a/pkg/apis/kops/v1alpha2/networking.go +++ b/pkg/apis/kops/v1alpha2/networking.go @@ -469,6 +469,11 @@ type CiliumNetworkingSpec struct { // AutoDirectNodeRoutes adds automatic L2 routing between nodes. // Default: false AutoDirectNodeRoutes bool `json:"autoDirectNodeRoutes,omitempty"` + // EnableHostReachableServices configures Cilium to enable services to be + // reached from the host namespace in addition to pod namespaces. + // https://docs.cilium.io/en/v1.9/gettingstarted/host-services/ + // Default: false + EnableHostReachableServices bool `json:"enableHostReachableServices,omitempty"` // EnableNodePort replaces kube-proxy with Cilium's BPF implementation. // Requires spec.kubeProxy.enabled be set to false. // Default: false diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index cc6aaf1746..4c878c4fae 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -1729,6 +1729,7 @@ func autoConvert_v1alpha2_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in * out.Ipam = in.Ipam out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes + out.EnableHostReachableServices = in.EnableHostReachableServices out.EnableNodePort = in.EnableNodePort out.EtcdManaged = in.EtcdManaged out.EnableRemoteNodeIdentity = in.EnableRemoteNodeIdentity @@ -1823,6 +1824,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha2_CiliumNetworkingSpec(in * out.Ipam = in.Ipam out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes + out.EnableHostReachableServices = in.EnableHostReachableServices out.EnableNodePort = in.EnableNodePort out.EtcdManaged = in.EtcdManaged out.EnableRemoteNodeIdentity = in.EnableRemoteNodeIdentity diff --git a/pkg/commands/set_cluster_test.go b/pkg/commands/set_cluster_test.go index 31cf70cdee..8caeedeccf 100644 --- a/pkg/commands/set_cluster_test.go +++ b/pkg/commands/set_cluster_test.go @@ -249,6 +249,21 @@ func TestSetClusterFields(t *testing.T) { }, }, }, + { + Fields: []string{ + "cluster.spec.networking.cilium.enableHostReachableServices=true", + }, + Input: kops.Cluster{}, + Output: kops.Cluster{ + Spec: kops.ClusterSpec{ + Networking: &kops.NetworkingSpec{ + Cilium: &kops.CiliumNetworkingSpec{ + EnableHostReachableServices: true, + }, + }, + }, + }, + }, { Fields: []string{ "cluster.spec.networking.cilium.enableNodePort=true", diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.9.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.9.yaml.template index dc20570d13..959f56451e 100644 --- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.9.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.9.yaml.template @@ -173,6 +173,9 @@ data: masquerade: "{{- if .DisableMasquerade -}}false{{- else -}}true{{- end -}}" install-iptables-rules: "{{- if .IPTablesRulesNoinstall -}}false{{- else -}}true{{- end -}}" auto-direct-node-routes: "{{ .AutoDirectNodeRoutes }}" + {{ if .EnableHostReachableServices }} + enable-host-reachable-services: "{{ .EnableHostReachableServices }}" + {{ end }} enable-node-port: "{{ .EnableNodePort }}" kube-proxy-replacement: "{{- if .EnableNodePort -}}strict{{- else -}}partial{{- end -}}" enable-remote-node-identity: "{{ .EnableRemoteNodeIdentity -}}"