Validate CNI configurations during pod startup (#9678)

When users use CNI, we want to ensure that network rewriting inside the pod is setup before allowing linkerd to start. When rewriting isn't happening, we want to exit with a clear error message and enough information in the container log for the administrator to either file a bug report with us or fix their configuration.

This change adds a validator initContainer to all injected workloads, when linkerd is installed with "cniEnabled=false". The validator replaces the noop init container, and will prevent pods from starting up if iptables is not configured.

Part of #8120

Signed-off-by: Steve Jenson <stevej@buoyant.io>

Dockerfile-proxy

@@ -20,6 +20,8 @@ RUN (proxy=$(bin/fetch-proxy $(cat proxy-version) $TARGETARCH) && \
     mv "$proxy" linkerd2-proxy)
 ARG LINKERD_AWAIT_VERSION=v0.2.6
 RUN bin/scurl -o linkerd-await https://github.com/linkerd/linkerd-await/releases/download/release%2F${LINKERD_AWAIT_VERSION}/linkerd-await-${LINKERD_AWAIT_VERSION}-${TARGETARCH} && chmod +x linkerd-await
+ARG LINKERD_VALIDATOR_VERSION=v0.1.0
+RUN bin/scurl -o linkerd-network-validator https://github.com/linkerd/linkerd2-proxy-init/releases/download/validator%2F${LINKERD_VALIDATOR_VERSION}/linkerd-network-validator-${LINKERD_VALIDATOR_VERSION}-${TARGETARCH} && chmod +x linkerd-network-validator
 
 ## compile proxy-identity agent
 FROM go-deps as golang
@@ -38,6 +40,7 @@ COPY --from=fetch /build/target/proxy/LICENSE /usr/lib/linkerd/LICENSE
 COPY --from=fetch /build/proxy-version /usr/lib/linkerd/linkerd2-proxy-version.txt
 COPY --from=fetch /build/linkerd2-proxy /usr/lib/linkerd/linkerd2-proxy
 COPY --from=fetch /build/linkerd-await /usr/lib/linkerd/linkerd-await
+COPY --from=fetch /build/linkerd-network-validator /usr/lib/linkerd/linkerd2-network-validator
 COPY --from=golang /out/proxy-identity /usr/lib/linkerd/linkerd2-proxy-identity
 COPY --from=debian:bullseye-slim /bin/sleep /bin/sleep
 ARG LINKERD_VERSION

charts/linkerd-control-plane/README.md

@@ -166,6 +166,11 @@ Kubernetes: `>=1.21.0-0`
 | imagePullPolicy | string | `"IfNotPresent"` | Docker image pull policy |
 | imagePullSecrets | list | `[]` | For Private docker registries, authentication is needed.  Registry secrets are applied to the respective service accounts |
 | linkerdVersion | string | `"linkerdVersionValue"` | control plane version. See Proxy section for proxy version |
+| networkValidator.connectAddr | string | `"1.1.1.1:20001"` | Address to which the network-validator will attempt to connect. we expect this to be rewritten |
+| networkValidator.listenAddr | string | `"0.0.0.0:4140"` | Address to which network-validator listens to requests from itself |
+| networkValidator.logFormat | string | plain | Log format (`plain` or `json`) for network-validator |
+| networkValidator.logLevel | string | debug | Log level for the network-validator |
+| networkValidator.timeout | string | `"10s"` | Timeout before network-validator fails to validate the pod's network connectivity |
 | nodeSelector | object | `{"kubernetes.io/os":"linux"}` | NodeSelector section, See the [K8S documentation](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) for more information |
 | podAnnotations | object | `{}` | Additional annotations to add to all pods |
 | podLabels | object | `{}` | Additional labels to add to all pods |

charts/linkerd-control-plane/templates/destination.yaml

@@ -304,7 +304,7 @@ spec:
           readOnly: true
       initContainers:
       {{ if .Values.cniEnabled -}}
-      - {{- include "partials.noop" $tree | indent 8 | trimPrefix (repeat 7 " ") }}
+      - {{- include "partials.network-validator" $tree | indent 8 | trimPrefix (repeat 7 " ") }}
       {{ else -}}
       {{- /*
         The destination controller needs to connect to the Kubernetes API before the proxy is able

charts/linkerd-control-plane/templates/identity.yaml

@@ -201,7 +201,7 @@ spec:
       - {{- include "partials.proxy" $tree | indent 8 | trimPrefix (repeat 7 " ") }}
       initContainers:
       {{ if .Values.cniEnabled -}}
-      - {{- include "partials.noop" $tree | indent 8 | trimPrefix (repeat 7 " ") }}
+      - {{- include "partials.network-validator" $tree | indent 8 | trimPrefix (repeat 7 " ") }}
       {{ else -}}
       {{- /*
         The identity controller needs to connect to the Kubernetes API before the proxy is able to

charts/linkerd-control-plane/templates/proxy-injector.yaml

@@ -108,7 +108,7 @@ spec:
           readOnly: true
       initContainers:
       {{ if .Values.cniEnabled -}}
-      - {{- include "partials.noop" $tree | indent 8 | trimPrefix (repeat 7 " ") }}
+      - {{- include "partials.network-validator" $tree | indent 8 | trimPrefix (repeat 7 " ") }}
       {{ else -}}
       - {{- include "partials.proxy-init" $tree | indent 8 | trimPrefix (repeat 7 " ") }}
       {{ end -}}

charts/linkerd-control-plane/values.yaml

@@ -231,6 +231,23 @@ proxyInit:
     mountPath: /run
     name: linkerd-proxy-init-xtables-lock
 
+# network validator configuration
+# This runs on a host that uses iptables to reroute network traffic. The validator
+# ensures that iptables is correctly routing requests before we start linkerd.
+networkValidator:
+  # -- Log level for the network-validator
+  # @default -- debug
+  logLevel: debug
+  # -- Log format (`plain` or `json`) for network-validator
+  # @default -- plain
+  logFormat: plain
+  # -- Address to which the network-validator will attempt to connect. we expect this to be rewritten
+  connectAddr: "1.1.1.1:20001"
+  # -- Address to which network-validator listens to requests from itself
+  listenAddr: "0.0.0.0:4140"
+  # -- Timeout before network-validator fails to validate the pod's network connectivity
+  timeout: "10s"
+
 # -- For Private docker registries, authentication is needed.
 #  Registry secrets are applied to the respective service accounts
 imagePullSecrets: []
@@ -298,7 +315,6 @@ identity:
       # install
       keyPEM: |
 
-
 # -|- CPU, Memory and Ephemeral Storage resources required by the identity controller (see `proxy.resources` for sub-fields)
 #identityResources:
 # -|- CPU, Memory and Ephemeral Storage resources required by proxy injected into identity pod (see `proxy.resources` for sub-fields)

charts/partials/templates/_network-validator.tpl

@@ -0,0 +1,24 @@
+{{- define "partials.network-validator" -}}
+name: linkerd-network-validator
+image: {{.Values.proxy.image.name}}:{{.Values.proxy.image.version | default .Values.linkerdVersion }}
+imagePullPolicy: {{.Values.proxy.image.pullPolicy | default .Values.imagePullPolicy}}
+securityContext:
+  runAsUser: 65534
+  capabilities:
+    drop:
+      - all
+command:
+  - /usr/lib/linkerd/linkerd2-network-validator
+args:
+  - --log-format
+  - {{ .Values.networkValidator.logFormat }}
+  - --log-level
+  - {{ .Values.networkValidator.logLevel }}
+  - --connect-addr
+  - {{ .Values.networkValidator.connectAddr }}
+  - --listen-addr
+  - {{ .Values.networkValidator.listenAddr }}
+  - --timeout
+  - {{ .Values.networkValidator.timeout }}
+
+{{- end -}}

charts/partials/templates/_noop.tpl

@@ -1,6 +0,0 @@
-{{- define "partials.noop" -}}
-args:
-- -v
-image: gcr.io/google_containers/pause:3.2
-name: noop
-{{- end -}}

charts/patch/templates/patch.json

@@ -71,7 +71,7 @@
     "op": "add",
     "path": "{{$prefix}}/spec/initContainers/-",
     "value":
-      {{- include "partials.noop" . | fromYaml | toPrettyJson | nindent 6 }}
+      {{- include "partials.network-validator" . | fromYaml | toPrettyJson | nindent 6 }}
   },
   {{- end }}
   {{- if .Values.debugContainer }}

cli/cmd/install_test.go

@@ -127,6 +127,13 @@ func TestRender(t *testing.T) {
 			RunAsRoot: false,
 			RunAsUser: 65534,
 		},
+		NetworkValidator: &charts.NetworkValidator{
+			LogLevel:    "debug",
+			LogFormat:   "plain",
+			ConnectAddr: "1.1.1.1:20001",
+			ListenAddr:  "0.0.0.0:4140",
+			Timeout:     "10s",
+		},
 		Configs: charts.ConfigJSONs{
 			Global:  "GlobalConfig",
 			Proxy:   "ProxyConfig",

cli/cmd/testdata/inject_emojivoto_deployment_no_init_container.golden.yml

@@ -165,9 +165,26 @@ spec:
           name: http
       initContainers:
       - args:
-        - -v
+        - --log-format
-        image: gcr.io/google_containers/pause:3.2
+        - plain
-        name: noop
+        - --log-level
+        - debug
+        - --connect-addr
+        - 1.1.1.1:20001
+        - --listen-addr
+        - 0.0.0.0:4140
+        - --timeout
+        - 10s
+        command:
+        - /usr/lib/linkerd/linkerd2-network-validator
+        image: cr.l5d.io/linkerd/proxy:test-inject-proxy-version
+        imagePullPolicy: IfNotPresent
+        name: linkerd-network-validator
+        securityContext:
+          capabilities:
+            drop:
+            - all
+          runAsUser: 65534
       volumes:
       - emptyDir:
           medium: Memory

cli/cmd/testdata/install_controlplane_tracing_output.golden

@@ -467,6 +467,12 @@ data:
     imagePullPolicy: IfNotPresent
     imagePullSecrets: []
     linkerdVersion: install-control-plane-version
+    networkValidator:
+      connectAddr: 1.1.1.1:20001
+      listenAddr: 0.0.0.0:4140
+      logFormat: plain
+      logLevel: debug
+      timeout: 10s
     nodeAffinity: null
     nodeSelector:
       kubernetes.io/os: linux

cli/cmd/testdata/install_custom_domain.golden

@@ -467,6 +467,12 @@ data:
     imagePullPolicy: IfNotPresent
     imagePullSecrets: []
     linkerdVersion: install-control-plane-version
+    networkValidator:
+      connectAddr: 1.1.1.1:20001
+      listenAddr: 0.0.0.0:4140
+      logFormat: plain
+      logLevel: debug
+      timeout: 10s
     nodeAffinity: null
     nodeSelector:
       kubernetes.io/os: linux

cli/cmd/testdata/install_custom_registry.golden

@@ -467,6 +467,12 @@ data:
     imagePullPolicy: IfNotPresent
     imagePullSecrets: []
     linkerdVersion: install-control-plane-version
+    networkValidator:
+      connectAddr: 1.1.1.1:20001
+      listenAddr: 0.0.0.0:4140
+      logFormat: plain
+      logLevel: debug
+      timeout: 10s
     nodeAffinity: null
     nodeSelector:
       kubernetes.io/os: linux

cli/cmd/testdata/install_default.golden

@@ -467,6 +467,12 @@ data:
     imagePullPolicy: IfNotPresent
     imagePullSecrets: []
     linkerdVersion: install-control-plane-version
+    networkValidator:
+      connectAddr: 1.1.1.1:20001
+      listenAddr: 0.0.0.0:4140
+      logFormat: plain
+      logLevel: debug
+      timeout: 10s
     nodeAffinity: null
     nodeSelector:
       kubernetes.io/os: linux

cli/cmd/testdata/install_default_override_dst_get_nets.golden

@@ -467,6 +467,12 @@ data:
     imagePullPolicy: IfNotPresent
     imagePullSecrets: []
     linkerdVersion: install-control-plane-version
+    networkValidator:
+      connectAddr: 1.1.1.1:20001
+      listenAddr: 0.0.0.0:4140
+      logFormat: plain
+      logLevel: debug
+      timeout: 10s
     nodeAffinity: null
     nodeSelector:
       kubernetes.io/os: linux

cli/cmd/testdata/install_default_token.golden

@@ -467,6 +467,12 @@ data:
     imagePullPolicy: IfNotPresent
     imagePullSecrets: []
     linkerdVersion: install-control-plane-version
+    networkValidator:
+      connectAddr: 1.1.1.1:20001
+      listenAddr: 0.0.0.0:4140
+      logFormat: plain
+      logLevel: debug
+      timeout: 10s
     nodeAffinity: null
     nodeSelector:
       kubernetes.io/os: linux

cli/cmd/testdata/install_ha_output.golden

@@ -494,6 +494,12 @@ data:
     imagePullPolicy: IfNotPresent
     imagePullSecrets: null
     linkerdVersion: install-control-plane-version
+    networkValidator:
+      connectAddr: 1.1.1.1:20001
+      listenAddr: 0.0.0.0:4140
+      logFormat: plain
+      logLevel: debug
+      timeout: 10s
     nodeAffinity: null
     nodeSelector:
       kubernetes.io/os: linux

cli/cmd/testdata/install_ha_with_overrides_output.golden

@@ -494,6 +494,12 @@ data:
     imagePullPolicy: IfNotPresent
     imagePullSecrets: null
     linkerdVersion: install-control-plane-version
+    networkValidator:
+      connectAddr: 1.1.1.1:20001
+      listenAddr: 0.0.0.0:4140
+      logFormat: plain
+      logLevel: debug
+      timeout: 10s
     nodeAffinity: null
     nodeSelector:
       kubernetes.io/os: linux

cli/cmd/testdata/install_heartbeat_disabled_output.golden

@@ -398,6 +398,12 @@ data:
     imagePullPolicy: IfNotPresent
     imagePullSecrets: []
     linkerdVersion: install-control-plane-version
+    networkValidator:
+      connectAddr: 1.1.1.1:20001
+      listenAddr: 0.0.0.0:4140
+      logFormat: plain
+      logLevel: debug
+      timeout: 10s
     nodeAffinity: null
     nodeSelector:
       kubernetes.io/os: linux

cli/cmd/testdata/install_helm_control_plane_output.golden

@@ -445,6 +445,12 @@ data:
     imagePullPolicy: IfNotPresent
     imagePullSecrets: []
     linkerdVersion: linkerd-version
+    networkValidator:
+      connectAddr: 1.1.1.1:20001
+      listenAddr: 0.0.0.0:4140
+      logFormat: plain
+      logLevel: debug
+      timeout: 10s
     nodeAffinity: null
     nodeSelector:
       kubernetes.io/os: linux

cli/cmd/testdata/install_helm_control_plane_output_ha.golden

@@ -472,6 +472,12 @@ data:
     imagePullPolicy: IfNotPresent
     imagePullSecrets: null
     linkerdVersion: linkerd-version
+    networkValidator:
+      connectAddr: 1.1.1.1:20001
+      listenAddr: 0.0.0.0:4140
+      logFormat: plain
+      logLevel: debug
+      timeout: 10s
     nodeAffinity: null
     nodeSelector:
       kubernetes.io/os: linux

cli/cmd/testdata/install_helm_output_ha_labels.golden

@@ -472,6 +472,12 @@ data:
     imagePullPolicy: IfNotPresent
     imagePullSecrets: null
     linkerdVersion: linkerd-version
+    networkValidator:
+      connectAddr: 1.1.1.1:20001
+      listenAddr: 0.0.0.0:4140
+      logFormat: plain
+      logLevel: debug
+      timeout: 10s
     nodeAffinity: null
     nodeSelector:
       kubernetes.io/os: linux

cli/cmd/testdata/install_helm_output_ha_namespace_selector.golden

@@ -467,6 +467,12 @@ data:
     imagePullPolicy: IfNotPresent
     imagePullSecrets: null
     linkerdVersion: linkerd-version
+    networkValidator:
+      connectAddr: 1.1.1.1:20001
+      listenAddr: 0.0.0.0:4140
+      logFormat: plain
+      logLevel: debug
+      timeout: 10s
     nodeAffinity: null
     nodeSelector:
       kubernetes.io/os: linux

cli/cmd/testdata/install_no_init_container.golden

@@ -293,6 +293,7 @@ metadata:
   labels:
     linkerd.io/control-plane-component: heartbeat
     linkerd.io/control-plane-ns: linkerd
+
 ---
 ###
 ### Proxy Injector RBAC
@@ -467,6 +468,12 @@ data:
     imagePullPolicy: IfNotPresent
     imagePullSecrets: []
     linkerdVersion: install-control-plane-version
+    networkValidator:
+      connectAddr: 1.1.1.1:20001
+      listenAddr: 0.0.0.0:4140
+      logFormat: plain
+      logLevel: debug
+      timeout: 10s
     nodeAffinity: null
     nodeSelector:
       kubernetes.io/os: linux
@@ -901,10 +908,27 @@ spec:
         - mountPath: /var/run/secrets/tokens
           name: linkerd-identity-token
       initContainers:
-      - args:
+      - name: linkerd-network-validator
-          - -v
+        image: cr.l5d.io/linkerd/proxy:install-proxy-version
-        image: gcr.io/google_containers/pause:3.2
+        imagePullPolicy: IfNotPresent
-        name: noop
+        securityContext:
+          runAsUser: 65534
+          capabilities:
+            drop:
+              - all
+        command:
+          - /usr/lib/linkerd/linkerd2-network-validator
+        args:
+          - --log-format
+          - plain
+          - --log-level
+          - debug
+          - --connect-addr
+          - 1.1.1.1:20001
+          - --listen-addr
+          - 0.0.0.0:4140
+          - --timeout
+          - 10s
       serviceAccountName: linkerd-identity
       volumes:
       - name: identity-issuer
@@ -1289,10 +1313,27 @@ spec:
           name: policy-tls
           readOnly: true
       initContainers:
-      - args:
+      - name: linkerd-network-validator
-          - -v
+        image: cr.l5d.io/linkerd/proxy:install-proxy-version
-        image: gcr.io/google_containers/pause:3.2
+        imagePullPolicy: IfNotPresent
-        name: noop
+        securityContext:
+          runAsUser: 65534
+          capabilities:
+            drop:
+              - all
+        command:
+          - /usr/lib/linkerd/linkerd2-network-validator
+        args:
+          - --log-format
+          - plain
+          - --log-level
+          - debug
+          - --connect-addr
+          - 1.1.1.1:20001
+          - --listen-addr
+          - 0.0.0.0:4140
+          - --timeout
+          - 10s
       serviceAccountName: linkerd-destination
       volumes:
       - name: sp-tls
@@ -1563,10 +1604,27 @@ spec:
           name: tls
           readOnly: true
       initContainers:
-      - args:
+      - name: linkerd-network-validator
-          - -v
+        image: cr.l5d.io/linkerd/proxy:install-proxy-version
-        image: gcr.io/google_containers/pause:3.2
+        imagePullPolicy: IfNotPresent
-        name: noop
+        securityContext:
+          runAsUser: 65534
+          capabilities:
+            drop:
+              - all
+        command:
+          - /usr/lib/linkerd/linkerd2-network-validator
+        args:
+          - --log-format
+          - plain
+          - --log-level
+          - debug
+          - --connect-addr
+          - 1.1.1.1:20001
+          - --listen-addr
+          - 0.0.0.0:4140
+          - --timeout
+          - 10s
       serviceAccountName: linkerd-proxy-injector
       volumes:
       - configMap:

cli/cmd/testdata/install_output.golden

@@ -290,6 +290,7 @@ metadata:
   labels:
     linkerd.io/control-plane-component: heartbeat
     linkerd.io/control-plane-ns: linkerd
+
 ---
 ###
 ### Proxy Injector RBAC
@@ -460,6 +461,12 @@ data:
     imagePullPolicy: ImagePullPolicy
     imagePullSecrets: null
     linkerdVersion: LinkerdVersion
+    networkValidator:
+      connectAddr: 1.1.1.1:20001
+      listenAddr: 0.0.0.0:4140
+      logFormat: plain
+      logLevel: debug
+      timeout: 10s
     nodeAffinity: null
     nodeSelector:
       kubernetes.io/os: linux
@@ -879,6 +886,7 @@ spec:
         - mountPath: /var/run/secrets/tokens
           name: linkerd-identity-token
       initContainers:
+      
       - args:
         - --incoming-proxy-port
         - "4143"
@@ -1305,6 +1313,7 @@ spec:
           name: policy-tls
           readOnly: true
       initContainers:
+      
       - args:
         - --incoming-proxy-port
         - "4143"

cli/cmd/testdata/install_proxy_ignores.golden

@@ -467,6 +467,12 @@ data:
     imagePullPolicy: IfNotPresent
     imagePullSecrets: []
     linkerdVersion: install-control-plane-version
+    networkValidator:
+      connectAddr: 1.1.1.1:20001
+      listenAddr: 0.0.0.0:4140
+      logFormat: plain
+      logLevel: debug
+      timeout: 10s
     nodeAffinity: null
     nodeSelector:
       kubernetes.io/os: linux

cli/cmd/testdata/install_values_file.golden

@@ -467,6 +467,12 @@ data:
     imagePullPolicy: IfNotPresent
     imagePullSecrets: []
     linkerdVersion: install-control-plane-version
+    networkValidator:
+      connectAddr: 1.1.1.1:20001
+      listenAddr: 0.0.0.0:4140
+      logFormat: plain
+      logLevel: debug
+      timeout: 10s
     nodeAffinity: null
     nodeSelector:
       kubernetes.io/os: linux

pkg/charts/charts.go

@@ -27,7 +27,7 @@ var (
 		"charts/partials/templates/_helpers.tpl",
 		"charts/partials/templates/_metadata.tpl",
 		"charts/partials/templates/_nodeselector.tpl",
-		"charts/partials/templates/_noop.tpl",
+		"charts/partials/templates/_network-validator.tpl",
 		"charts/partials/templates/_proxy-config-ann.tpl",
 		"charts/partials/templates/_proxy-init.tpl",
 		"charts/partials/templates/_proxy.tpl",

pkg/charts/linkerd2/values.go

@@ -64,6 +64,7 @@ type (
 		PolicyController *PolicyController `json:"policyController"`
 		Proxy            *Proxy            `json:"proxy"`
 		ProxyInit        *ProxyInit        `json:"proxyInit"`
+		NetworkValidator *NetworkValidator `json:"networkValidator"`
 		Identity         *Identity         `json:"identity"`
 		DebugContainer   *DebugContainer   `json:"debugContainer"`
 		ProxyInjector    *Webhook          `json:"proxyInjector"`
@@ -134,6 +135,14 @@ type (
 		IptablesMode         string           `json:"iptablesMode"`
 	}
 
+	NetworkValidator struct {
+		LogLevel    string `json:"logLevel"`
+		LogFormat   string `json:"logFormat"`
+		ConnectAddr string `json:"connectAddr"`
+		ListenAddr  string `json:"listenAddr"`
+		Timeout     string `json:"timeout"`
+	}
+
 	// DebugContainer contains the fields to set the debugging sidecar
 	DebugContainer struct {
 		Image *Image `json:"image"`

pkg/charts/linkerd2/values_test.go

@@ -158,6 +158,13 @@ func TestNewValues(t *testing.T) {
 			RunAsRoot: false,
 			RunAsUser: 65534,
 		},
+		NetworkValidator: &NetworkValidator{
+			LogLevel:    "debug",
+			LogFormat:   "plain",
+			ConnectAddr: "1.1.1.1:20001",
+			ListenAddr:  "0.0.0.0:4140",
+			Timeout:     "10s",
+		},
 		Identity: &Identity{
 			ServiceAccountTokenProjection: true,
 			Issuer: &Issuer{