Merge pull request #12394 from ReillyBrogan/reilly/ciliumBidirectionalMount

Add bidirectional BPF mount for Cilium >= 1.9.10 or >= 1.10.4
2021-09-25 09:42:21 -07:00 · 2021-09-25 09:42:21 -07:00 · ef22270b3f
parent 8225c78eb8 9c6bf83c93
commit ef22270b3f
21 changed files with 42 additions and 79 deletions
--- a/pkg/model/components/cilium.go
+++ b/pkg/model/components/cilium.go
@ -39,7 +39,7 @@ func (b *CiliumOptionsBuilder) BuildOptions(o interface{}) error {
 	}

 	if c.Version == "" {
-		c.Version = "v1.10.3"
+		c.Version = "v1.10.4"
 	}

 	if c.EnableEndpointHealthChecking == nil {
--- a/tests/integration/update_cluster/minimal-warmpool/data/aws_launch_template_nodes.minimal-warmpool.example.com_user_data
+++ b/tests/integration/update_cluster/minimal-warmpool/data/aws_launch_template_nodes.minimal-warmpool.example.com_user_data
@ -166,7 +166,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/minimal-warmpool.example.com
 InstanceGroupName: nodes
 InstanceGroupRole: Node
-NodeupConfigHash: wMzxC2J+2fQle4swMnOsp6CHale02Q9EaudaIpKUJWs=
+NodeupConfigHash: iDgKk2qqfmWExfsA3lmRJnHUoxMWVld8fDhunD6gjN8=

 __EOF_KUBE_ENV

--- a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_cluster-completed.spec_content
@ -206,7 +206,7 @@ spec:
      sidecarIstioProxyImage: cilium/istio_proxy
      toFqdnsDnsRejectResponseCode: refused
      tunnel: vxlan
-      version: v1.10.3
+      version: v1.10.4
  nonMasqueradeCIDR: 100.64.0.0/10
  podCIDR: 100.96.0.0/11
  secretStore: memfs://clusters.example.com/minimal-warmpool.example.com/secrets
--- a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-bootstrap_content
@ -47,7 +47,7 @@ spec:
      k8s-addon: storage-aws.addons.k8s.io
  - id: k8s-1.16
    manifest: networking.cilium.io/k8s-1.16-v1.10.yaml
-    manifestHash: 561acbb6fc8947695eb062473e9aa2297a9350fea1508d0939189940f48a5b32
+    manifestHash: 79105657d58e949d8612de05cf98900e30b0e7fb5c6e2c06a7c35692c91f302e
    name: networking.cilium.io
    needsRollingUpdate: all
    selector:
--- a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-networking.cilium.io-k8s-1.16_content
+++ b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-networking.cilium.io-k8s-1.16_content
@ -60,7 +60,6 @@ data:
  tofqdns-dns-reject-response-code: refused
  tofqdns-enable-poller: "false"
  tunnel: vxlan
-  wait-bpf-mount: "false"
 kind: ConfigMap
 metadata:
  creationTimestamp: null
@ -385,7 +384,7 @@ spec:
          value: api.internal.minimal-warmpool.example.com
        - name: KUBERNETES_SERVICE_PORT
          value: "443"
-        image: quay.io/cilium/cilium:v1.10.3
+        image: quay.io/cilium/cilium:v1.10.4
        imagePullPolicy: IfNotPresent
        lifecycle:
          postStart:
@ -430,10 +429,6 @@ spec:
            cpu: 25m
            memory: 128Mi
        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - SYS_MODULE
          privileged: true
        startupProbe:
          failureThreshold: 105
@ -449,6 +444,7 @@ spec:
          successThreshold: null
        volumeMounts:
        - mountPath: /sys/fs/bpf
+          mountPropagation: Bidirectional
          name: bpf-maps
        - mountPath: /var/run/cilium
          name: cilium-run
@ -484,13 +480,7 @@ spec:
              key: clean-cilium-bpf-state
              name: cilium-config
              optional: true
-        - name: CILIUM_WAIT_BPF_MOUNT
-          valueFrom:
-            configMapKeyRef:
-              key: wait-bpf-mount
-              name: cilium-config
-              optional: true
-        image: quay.io/cilium/cilium:v1.10.3
+        image: quay.io/cilium/cilium:v1.10.4
        imagePullPolicy: IfNotPresent
        name: clean-cilium-state
        resources:
@ -500,13 +490,9 @@ spec:
            cpu: 100m
            memory: 100Mi
        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
          privileged: true
        volumeMounts:
        - mountPath: /sys/fs/bpf
-          mountPropagation: HostToContainer
          name: bpf-maps
        - mountPath: /sys/fs/cgroup/unified
          mountPropagation: HostToContainer
@ -618,7 +604,7 @@ spec:
          value: api.internal.minimal-warmpool.example.com
        - name: KUBERNETES_SERVICE_PORT
          value: "443"
-        image: quay.io/cilium/operator:v1.10.3
+        image: quay.io/cilium/operator:v1.10.4
        imagePullPolicy: IfNotPresent
        livenessProbe:
          httpGet:
--- a/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_nodeupconfig-nodes_content
+++ b/tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_nodeupconfig-nodes_content
@ -69,5 +69,5 @@ warmPoolImages:
 - k8s.gcr.io/provider-aws/aws-ebs-csi-driver:v1.2.1
 - k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0
 - k8s.gcr.io/sig-storage/livenessprobe:v2.2.0
- quay.io/cilium/cilium:v1.10.3
- quay.io/cilium/operator:v1.10.3
+- quay.io/cilium/cilium:v1.10.4
+- quay.io/cilium/operator:v1.10.4
--- a/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_cluster-completed.spec_content
@ -199,7 +199,7 @@ spec:
      sidecarIstioProxyImage: cilium/istio_proxy
      toFqdnsDnsRejectResponseCode: refused
      tunnel: vxlan
-      version: v1.10.3
+      version: v1.10.4
  nonMasqueradeCIDR: 100.64.0.0/10
  podCIDR: 100.96.0.0/11
  secretStore: memfs://clusters.example.com/privatecilium.example.com/secrets
--- a/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-bootstrap_content
@ -47,7 +47,7 @@ spec:
      k8s-addon: storage-aws.addons.k8s.io
  - id: k8s-1.16
    manifest: networking.cilium.io/k8s-1.16-v1.10.yaml
-    manifestHash: 215a45b620ee9eb02bf3e22ba5a4a5bbd678b6dd2473ad9adc06baac1c1fbecb
+    manifestHash: 39ec0f473a3c9479e15718e23b965dd5670502132723c7d2d9377f6b800be453
    name: networking.cilium.io
    needsRollingUpdate: all
    selector:
--- a/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content
+++ b/tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-networking.cilium.io-k8s-1.16_content
@ -60,7 +60,6 @@ data:
  tofqdns-dns-reject-response-code: refused
  tofqdns-enable-poller: "false"
  tunnel: vxlan
-  wait-bpf-mount: "false"
 kind: ConfigMap
 metadata:
  creationTimestamp: null
@ -385,7 +384,7 @@ spec:
          value: api.internal.privatecilium.example.com
        - name: KUBERNETES_SERVICE_PORT
          value: "443"
-        image: quay.io/cilium/cilium:v1.10.3
+        image: quay.io/cilium/cilium:v1.10.4
        imagePullPolicy: IfNotPresent
        lifecycle:
          postStart:
@ -430,10 +429,6 @@ spec:
            cpu: 25m
            memory: 128Mi
        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - SYS_MODULE
          privileged: true
        startupProbe:
          failureThreshold: 105
@ -449,6 +444,7 @@ spec:
          successThreshold: null
        volumeMounts:
        - mountPath: /sys/fs/bpf
+          mountPropagation: Bidirectional
          name: bpf-maps
        - mountPath: /var/run/cilium
          name: cilium-run
@ -484,13 +480,7 @@ spec:
              key: clean-cilium-bpf-state
              name: cilium-config
              optional: true
-        - name: CILIUM_WAIT_BPF_MOUNT
-          valueFrom:
-            configMapKeyRef:
-              key: wait-bpf-mount
-              name: cilium-config
-              optional: true
-        image: quay.io/cilium/cilium:v1.10.3
+        image: quay.io/cilium/cilium:v1.10.4
        imagePullPolicy: IfNotPresent
        name: clean-cilium-state
        resources:
@ -500,13 +490,9 @@ spec:
            cpu: 100m
            memory: 100Mi
        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
          privileged: true
        volumeMounts:
        - mountPath: /sys/fs/bpf
-          mountPropagation: HostToContainer
          name: bpf-maps
        - mountPath: /sys/fs/cgroup/unified
          mountPropagation: HostToContainer
@ -618,7 +604,7 @@ spec:
          value: api.internal.privatecilium.example.com
        - name: KUBERNETES_SERVICE_PORT
          value: "443"
-        image: quay.io/cilium/operator:v1.10.3
+        image: quay.io/cilium/operator:v1.10.4
        imagePullPolicy: IfNotPresent
        livenessProbe:
          httpGet:
--- a/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_bucket_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_bucket_object_cluster-completed.spec_content
@ -212,7 +212,7 @@ spec:
      sidecarIstioProxyImage: cilium/istio_proxy
      toFqdnsDnsRejectResponseCode: refused
      tunnel: disabled
-      version: v1.10.3
+      version: v1.10.4
  nonMasqueradeCIDR: 100.64.0.0/10
  podCIDR: 100.96.0.0/11
  secretStore: memfs://clusters.example.com/privateciliumadvanced.example.com/secrets
--- a/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_bucket_object_privateciliumadvanced.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_bucket_object_privateciliumadvanced.example.com-addons-bootstrap_content
@ -47,7 +47,7 @@ spec:
      k8s-addon: storage-aws.addons.k8s.io
  - id: k8s-1.16
    manifest: networking.cilium.io/k8s-1.16-v1.10.yaml
-    manifestHash: a27941351ef6ecb7bee109b5cbd773a7e8bde30626d1cfb1fef8bdcfcb55b38b
+    manifestHash: 6fb92a78a0b2be6a6a21dd312e41be98245f64ad104add14d900de041e41ce41
    name: networking.cilium.io
    needsRollingUpdate: all
    selector:
--- a/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_bucket_object_privateciliumadvanced.example.com-addons-networking.cilium.io-k8s-1.16_content
+++ b/tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_bucket_object_privateciliumadvanced.example.com-addons-networking.cilium.io-k8s-1.16_content
@ -74,7 +74,6 @@ data:
  tofqdns-dns-reject-response-code: refused
  tofqdns-enable-poller: "false"
  tunnel: disabled
-  wait-bpf-mount: "false"
 kind: ConfigMap
 metadata:
  creationTimestamp: null
@ -399,7 +398,7 @@ spec:
          value: api.internal.privateciliumadvanced.example.com
        - name: KUBERNETES_SERVICE_PORT
          value: "443"
-        image: quay.io/cilium/cilium:v1.10.3
+        image: quay.io/cilium/cilium:v1.10.4
        imagePullPolicy: IfNotPresent
        lifecycle:
          postStart:
@ -444,10 +443,6 @@ spec:
            cpu: 25m
            memory: 128Mi
        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - SYS_MODULE
          privileged: true
        startupProbe:
          failureThreshold: 105
@ -463,6 +458,7 @@ spec:
          successThreshold: null
        volumeMounts:
        - mountPath: /sys/fs/bpf
+          mountPropagation: Bidirectional
          name: bpf-maps
        - mountPath: /var/run/cilium
          name: cilium-run
@ -504,13 +500,7 @@ spec:
              key: clean-cilium-bpf-state
              name: cilium-config
              optional: true
-        - name: CILIUM_WAIT_BPF_MOUNT
-          valueFrom:
-            configMapKeyRef:
-              key: wait-bpf-mount
-              name: cilium-config
-              optional: true
-        image: quay.io/cilium/cilium:v1.10.3
+        image: quay.io/cilium/cilium:v1.10.4
        imagePullPolicy: IfNotPresent
        name: clean-cilium-state
        resources:
@ -520,13 +510,9 @@ spec:
            cpu: 100m
            memory: 100Mi
        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
          privileged: true
        volumeMounts:
        - mountPath: /sys/fs/bpf
-          mountPropagation: HostToContainer
          name: bpf-maps
        - mountPath: /sys/fs/cgroup/unified
          mountPropagation: HostToContainer
@ -649,7 +635,7 @@ spec:
          value: api.internal.privateciliumadvanced.example.com
        - name: KUBERNETES_SERVICE_PORT
          value: "443"
-        image: quay.io/cilium/operator:v1.10.3
+        image: quay.io/cilium/operator:v1.10.4
        imagePullPolicy: IfNotPresent
        livenessProbe:
          httpGet:
--- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.8.yaml.template
+++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.8.yaml.template
@ -488,7 +488,7 @@ spec:
          value: {{ . }}
        {{ end }}
 {{ with .Networking.Cilium }}
-        image: "quay.io/cilium/cilium:{{ .Version  }}"
+        image: "quay.io/cilium/cilium:{{ .Version }}"
        imagePullPolicy: IfNotPresent
        lifecycle:
          postStart:
--- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.9.yaml.template
+++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.9.yaml.template
@ -654,7 +654,7 @@ spec:
        - name: CILIUM_ENABLE_POLICY
          value: {{ . }}
        {{ end }}
-        image: "quay.io/cilium/cilium:{{ .Version  }}"
+        image: "quay.io/cilium/cilium:{{ .Version }}"
        imagePullPolicy: IfNotPresent
        lifecycle:
          postStart:
--- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.10.yaml.template
+++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.16-v1.10.yaml.template
@ -1,4 +1,5 @@
 {{ with .Networking.Cilium }}
+{{ $semver := (trimPrefix "v" .Version) }}
 {{- if CiliumSecret }}
 apiVersion: v1
 kind: Secret
@ -192,8 +193,10 @@ data:
  # [0] http://docs.cilium.io/en/stable/policy/language/#dns-based
  # [1] http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action
  tofqdns-enable-poller: "{{- if .ToFqdnsEnablePoller -}}true{{- else -}}false{{- end -}}"
+  {{- if not (semverCompare ">=1.10.4 || ~1.9.10" $semver) }}
  # wait-bpf-mount makes init container wait until bpf filesystem is mounted
  wait-bpf-mount: "false"
+  {{- end }}
  # Enable fetching of container-runtime specific metadata
  #
  # By default, the Kubernetes pod and namespace labels are retrieved and
@ -687,7 +690,7 @@ spec:
        - name: CILIUM_ENABLE_POLICY
          value: {{ . }}
        {{ end }}
-        image: "quay.io/cilium/cilium:{{ .Version  }}"
+        image: "quay.io/cilium/cilium:{{ .Version }}"
        imagePullPolicy: IfNotPresent
        lifecycle:
          postStart:
@ -716,14 +719,13 @@ spec:
        {{ end }}

        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - SYS_MODULE
          privileged: true
        volumeMounts:
        - mountPath: /sys/fs/bpf
          name: bpf-maps
+          {{- if semverCompare ">=1.10.4 || ~1.9.10" $semver }}
+          mountPropagation: Bidirectional
+          {{- end }}
        - mountPath: /var/run/cilium
          name: cilium-run
        - mountPath: /host/opt/cni/bin
@ -776,24 +778,25 @@ spec:
              key: clean-cilium-bpf-state
              name: cilium-config
              optional: true
+        {{- if not (semverCompare ">=1.10.4 || ~1.9.10" $semver) }}
        - name: CILIUM_WAIT_BPF_MOUNT
          valueFrom:
            configMapKeyRef:
              key: wait-bpf-mount
              name: cilium-config
              optional: true
+        {{- end }}
        image: "quay.io/cilium/cilium:{{ .Version }}"
        imagePullPolicy: IfNotPresent
        name: clean-cilium-state
        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
          privileged: true
        volumeMounts:
        - mountPath: /sys/fs/bpf
          name: bpf-maps
+          {{- if not (semverCompare ">=1.10.4 || ~1.9.10" $semver) }}
          mountPropagation: HostToContainer
+          {{- end }}
         # Required to mount cgroup filesystem from the host to cilium agent pod
        - mountPath: /sys/fs/cgroup/unified
          name: cilium-cgroup
--- a/upup/pkg/fi/cloudup/template_functions.go
+++ b/upup/pkg/fi/cloudup/template_functions.go
@ -86,6 +86,8 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS
 	sprigTxtFuncMap := sprig.TxtFuncMap()
 	dest["indent"] = sprigTxtFuncMap["indent"]
 	dest["contains"] = sprigTxtFuncMap["contains"]
+	dest["trimPrefix"] = sprigTxtFuncMap["trimPrefix"]
+	dest["semverCompare"] = sprigTxtFuncMap["semverCompare"]

 	dest["ClusterName"] = tf.ClusterName
 	dest["WithDefaultBool"] = func(v *bool, defaultValue bool) bool {
--- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml
+++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml
@ -53,7 +53,7 @@ spec:
      k8s-addon: storage-aws.addons.k8s.io
  - id: k8s-1.16
    manifest: networking.cilium.io/k8s-1.16-v1.10.yaml
-    manifestHash: 3308ec4fa8e2ca93d839d9fb1e683be766f066f1764eb010f53b12e2df047a92
+    manifestHash: 2aa3ec34168ba0988b888912ecfe9fdc653b3caef70ea5a504aa77f662006a7e
    name: networking.cilium.io
    needsRollingUpdate: all
    selector:
--- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.18/manifest.yaml
+++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.18/manifest.yaml
@ -59,7 +59,7 @@ spec:
      k8s-addon: storage-aws.addons.k8s.io
  - id: k8s-1.16
    manifest: networking.cilium.io/k8s-1.16-v1.10.yaml
-    manifestHash: 3308ec4fa8e2ca93d839d9fb1e683be766f066f1764eb010f53b12e2df047a92
+    manifestHash: 2aa3ec34168ba0988b888912ecfe9fdc653b3caef70ea5a504aa77f662006a7e
    name: networking.cilium.io
    needsRollingUpdate: all
    selector:
--- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml
+++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml
@ -53,7 +53,7 @@ spec:
      k8s-addon: storage-aws.addons.k8s.io
  - id: k8s-1.16
    manifest: networking.cilium.io/k8s-1.16-v1.10.yaml
-    manifestHash: 3308ec4fa8e2ca93d839d9fb1e683be766f066f1764eb010f53b12e2df047a92
+    manifestHash: 2aa3ec34168ba0988b888912ecfe9fdc653b3caef70ea5a504aa77f662006a7e
    name: networking.cilium.io
    needsRollingUpdate: all
    selector:
--- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.18/manifest.yaml
+++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.18/manifest.yaml
@ -65,7 +65,7 @@ spec:
      k8s-addon: storage-aws.addons.k8s.io
  - id: k8s-1.16
    manifest: networking.cilium.io/k8s-1.16-v1.10.yaml
-    manifestHash: 3308ec4fa8e2ca93d839d9fb1e683be766f066f1764eb010f53b12e2df047a92
+    manifestHash: 2aa3ec34168ba0988b888912ecfe9fdc653b3caef70ea5a504aa77f662006a7e
    name: networking.cilium.io
    needsRollingUpdate: all
    selector:
--- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml
+++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml
@ -59,7 +59,7 @@ spec:
      k8s-addon: storage-aws.addons.k8s.io
  - id: k8s-1.16
    manifest: networking.cilium.io/k8s-1.16-v1.10.yaml
-    manifestHash: 3308ec4fa8e2ca93d839d9fb1e683be766f066f1764eb010f53b12e2df047a92
+    manifestHash: 2aa3ec34168ba0988b888912ecfe9fdc653b3caef70ea5a504aa77f662006a7e
    name: networking.cilium.io
    needsRollingUpdate: all
    selector: