inject: support `config.linkerd.io/access-log` annotation (#7689)

With #7661, the proxy supports a `LINKERD2_PROXY_ACCESS_LOG` configuration with the values `apache` or `json`. This configuration causes the proxy to emit access logs to stderr. This branch makes it possible for users to enable access logging by adding an annotation, `config.linkerd.io/access-log`, that tells the proxy injector to set this environment variable. I've also added some tests to ensure that the annotation and the environment variable are set correctly. I tried to follow the existing tests as examples of how we do this, but please let me know if I've overlooked anything! Closes #7662 #1913 Signed-off-by: Eliza Weisman <eliza@buoyant.io>
2022-01-24 14:02:19 -08:00 · 2022-01-24 14:02:19 -08:00 · 9e9c9457ae
parent 16337575b5
commit 9e9c9457ae
25 changed files with 264 additions and 0 deletions
--- a/charts/partials/templates/_proxy.tpl
+++ b/charts/partials/templates/_proxy.tpl
@ -129,6 +129,10 @@ be used in other contexts.
 - name: LINKERD2_PROXY_POLICY_SVC_NAME
  value: linkerd-destination.{{.Release.Namespace}}.serviceaccount.identity.{{.Release.Namespace}}.{{$trustDomain}}
 {{ end -}}
+{{ if .Values.proxy.accessLog -}}
+- name: LINKERD2_PROXY_ACCESS_LOG
+  value: {{.Values.proxy.accessLog | quote}}
+{{ end -}}
 image: {{.Values.proxy.image.name}}:{{.Values.proxy.image.version | default .Values.linkerdVersion}}
 imagePullPolicy: {{.Values.proxy.image.pullPolicy | default .Values.imagePullPolicy}}
 livenessProbe:
--- a/cli/cmd/inject.go
+++ b/cli/cmd/inject.go
@ -490,6 +490,10 @@ func getOverrideAnnotations(values *charts.Values, base *charts.Values) map[stri
 		overrideAnnotations[k8s.ProxyDefaultInboundPolicyAnnotation] = proxy.DefaultInboundPolicy
 	}

+	if proxy.AccessLog != baseProxy.AccessLog {
+		overrideAnnotations[k8s.ProxyAccessLogAnnotation] = proxy.AccessLog
+	}
+
 	return overrideAnnotations
 }

--- a/cli/cmd/inject_test.go
+++ b/cli/cmd/inject_test.go
@ -133,6 +133,17 @@ func TestUninjectAndInject(t *testing.T) {
 				return values
 			}(),
 		},
+		{
+			inputFileName:  "inject_emojivoto_deployment.input.yml",
+			goldenFileName: "inject_emojivoto_deployment_access_log.golden.yml",
+			reportFileName: "inject_emojivoto_deployment.report",
+			injectProxy:    true,
+			testInjectConfig: func() *linkerd2.Values {
+				values := defaultConfig()
+				values.Proxy.AccessLog = "apache"
+				return values
+			}(),
+		},
 		{
 			inputFileName:    "inject_emojivoto_list.input.yml",
 			goldenFileName:   "inject_emojivoto_list.golden.yml",
@ -661,6 +672,7 @@ func TestProxyConfigurationAnnotations(t *testing.T) {
 	values.Proxy.Resources.Memory.Limit = "50Mi"
 	values.Proxy.WaitBeforeExitSeconds = 10
 	values.Proxy.Await = false
+	values.Proxy.AccessLog = "apache"

 	expectedOverrides := map[string]string{
 		k8s.ProxyIgnoreInboundPortsAnnotation:  "8500-8505",
@ -681,6 +693,7 @@ func TestProxyConfigurationAnnotations(t *testing.T) {
 		k8s.ProxyMemoryLimitAnnotation:            "50Mi",
 		k8s.ProxyWaitBeforeExitSecondsAnnotation:  "10",
 		k8s.ProxyAwait:                            "disabled",
+		k8s.ProxyAccessLogAnnotation:              "apache",
 	}

 	overrides := getOverrideAnnotations(values, baseValues)
--- a/cli/cmd/testdata/inject_emojivoto_deployment_access_log.golden.yml
+++ b/cli/cmd/testdata/inject_emojivoto_deployment_access_log.golden.yml
@ -0,0 +1,212 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: web
+  namespace: emojivoto
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: web-svc
+  template:
+    metadata:
+      annotations:
+        config.linkerd.io/access-log: apache
+        linkerd.io/created-by: linkerd/cli dev-undefined
+        linkerd.io/identity-mode: default
+        linkerd.io/proxy-version: test-inject-proxy-version
+      labels:
+        app: web-svc
+        linkerd.io/control-plane-ns: linkerd
+        linkerd.io/proxy-deployment: web
+        linkerd.io/workload-ns: emojivoto
+    spec:
+      containers:
+      - env:
+        - name: _pod_name
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: _pod_ns
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: _pod_nodeName
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: LINKERD2_PROXY_LOG
+          value: warn,linkerd=info
+        - name: LINKERD2_PROXY_LOG_FORMAT
+          value: plain
+        - name: LINKERD2_PROXY_DESTINATION_SVC_ADDR
+          value: linkerd-dst-headless.linkerd.svc.cluster.local.:8086
+        - name: LINKERD2_PROXY_DESTINATION_PROFILE_NETWORKS
+          value: 10.0.0.0/8,100.64.0.0/10,172.16.0.0/12,192.168.0.0/16
+        - name: LINKERD2_PROXY_POLICY_SVC_ADDR
+          value: linkerd-policy.linkerd.svc.cluster.local.:8090
+        - name: LINKERD2_PROXY_POLICY_WORKLOAD
+          value: $(_pod_ns):$(_pod_name)
+        - name: LINKERD2_PROXY_INBOUND_DEFAULT_POLICY
+          value: all-unauthenticated
+        - name: LINKERD2_PROXY_POLICY_CLUSTER_NETWORKS
+          value: 10.0.0.0/8,100.64.0.0/10,172.16.0.0/12,192.168.0.0/16
+        - name: LINKERD2_PROXY_INBOUND_CONNECT_TIMEOUT
+          value: 100ms
+        - name: LINKERD2_PROXY_OUTBOUND_CONNECT_TIMEOUT
+          value: 1000ms
+        - name: LINKERD2_PROXY_CONTROL_LISTEN_ADDR
+          value: 0.0.0.0:4190
+        - name: LINKERD2_PROXY_ADMIN_LISTEN_ADDR
+          value: 0.0.0.0:4191
+        - name: LINKERD2_PROXY_OUTBOUND_LISTEN_ADDR
+          value: 127.0.0.1:4140
+        - name: LINKERD2_PROXY_INBOUND_LISTEN_ADDR
+          value: 0.0.0.0:4143
+        - name: LINKERD2_PROXY_INBOUND_IPS
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIPs
+        - name: LINKERD2_PROXY_INBOUND_PORTS
+          value: "80"
+        - name: LINKERD2_PROXY_DESTINATION_PROFILE_SUFFIXES
+          value: svc.cluster.local.
+        - name: LINKERD2_PROXY_INBOUND_ACCEPT_KEEPALIVE
+          value: 10000ms
+        - name: LINKERD2_PROXY_OUTBOUND_CONNECT_KEEPALIVE
+          value: 10000ms
+        - name: LINKERD2_PROXY_INBOUND_PORTS_DISABLE_PROTOCOL_DETECTION
+          value: 25,587,3306,4444,5432,6379,9300,11211
+        - name: LINKERD2_PROXY_DESTINATION_CONTEXT
+          value: |
+            {"ns":"$(_pod_ns)", "nodeName":"$(_pod_nodeName)"}
+        - name: _pod_sa
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.serviceAccountName
+        - name: LINKERD2_PROXY_IDENTITY_DIR
+          value: /var/run/linkerd/identity/end-entity
+        - name: LINKERD2_PROXY_IDENTITY_TRUST_ANCHORS
+          value: |
+            -----BEGIN CERTIFICATE-----
+            MIIBwTCCAWagAwIBAgIQeDZp5lDaIygQ5UfMKZrFATAKBggqhkjOPQQDAjApMScw
+            JQYDVQQDEx5pZGVudGl0eS5saW5rZXJkLmNsdXN0ZXIubG9jYWwwHhcNMjAwODI4
+            MDcxMjQ3WhcNMzAwODI2MDcxMjQ3WjApMScwJQYDVQQDEx5pZGVudGl0eS5saW5r
+            ZXJkLmNsdXN0ZXIubG9jYWwwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARqc70Z
+            l1vgw79rjB5uSITICUA6GyfvSFfcuIis7B/XFSkkwAHU5S/s1AAP+R0TX7HBWUC4
+            uaG4WWsiwJKNn7mgo3AwbjAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB
+            /wIBATAdBgNVHQ4EFgQU5YtjVVPfd7I7NLHsn2C26EByGV0wKQYDVR0RBCIwIIIe
+            aWRlbnRpdHkubGlua2VyZC5jbHVzdGVyLmxvY2FsMAoGCCqGSM49BAMCA0kAMEYC
+            IQCN7lBFLDDvjx6V0+XkjpKERRsJYf5adMvnloFl48ilJgIhANtxhndcr+QJPuC8
+            vgUC0d2/9FMueIVMb+46WTCOjsqr
+            -----END CERTIFICATE-----
+        - name: LINKERD2_PROXY_IDENTITY_TOKEN_FILE
+          value: /var/run/secrets/tokens/linkerd-identity-token
+        - name: LINKERD2_PROXY_IDENTITY_SVC_ADDR
+          value: linkerd-identity-headless.linkerd.svc.cluster.local.:8080
+        - name: LINKERD2_PROXY_IDENTITY_LOCAL_NAME
+          value: $(_pod_sa).$(_pod_ns).serviceaccount.identity.linkerd.cluster.local
+        - name: LINKERD2_PROXY_IDENTITY_SVC_NAME
+          value: linkerd-identity.linkerd.serviceaccount.identity.linkerd.cluster.local
+        - name: LINKERD2_PROXY_DESTINATION_SVC_NAME
+          value: linkerd-destination.linkerd.serviceaccount.identity.linkerd.cluster.local
+        - name: LINKERD2_PROXY_POLICY_SVC_NAME
+          value: linkerd-destination.linkerd.serviceaccount.identity.linkerd.cluster.local
+        - name: LINKERD2_PROXY_ACCESS_LOG
+          value: apache
+        image: cr.l5d.io/linkerd/proxy:test-inject-proxy-version
+        imagePullPolicy: IfNotPresent
+        lifecycle:
+          postStart:
+            exec:
+              command:
+              - /usr/lib/linkerd/linkerd-await
+        livenessProbe:
+          httpGet:
+            path: /live
+            port: 4191
+          initialDelaySeconds: 10
+        name: linkerd-proxy
+        ports:
+        - containerPort: 4143
+          name: linkerd-proxy
+        - containerPort: 4191
+          name: linkerd-admin
+        readinessProbe:
+          httpGet:
+            path: /ready
+            port: 4191
+          initialDelaySeconds: 2
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsUser: 2102
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /var/run/linkerd/identity/end-entity
+          name: linkerd-identity-end-entity
+        - mountPath: /var/run/secrets/tokens
+          name: linkerd-identity-token
+      - env:
+        - name: WEB_PORT
+          value: "80"
+        - name: EMOJISVC_HOST
+          value: emoji-svc.emojivoto:8080
+        - name: VOTINGSVC_HOST
+          value: voting-svc.emojivoto:8080
+        - name: INDEX_BUNDLE
+          value: dist/index_bundle.js
+        image: buoyantio/emojivoto-web:v10
+        name: web-svc
+        ports:
+        - containerPort: 80
+          name: http
+      initContainers:
+      - args:
+        - --incoming-proxy-port
+        - "4143"
+        - --outgoing-proxy-port
+        - "4140"
+        - --proxy-uid
+        - "2102"
+        - --inbound-ports-to-ignore
+        - 4190,4191,4567,4568
+        - --outbound-ports-to-ignore
+        - 4567,4568
+        image: cr.l5d.io/linkerd/proxy-init:v1.5.2
+        imagePullPolicy: IfNotPresent
+        name: linkerd-init
+        resources:
+          limits:
+            cpu: 100m
+            memory: 50Mi
+          requests:
+            cpu: 10m
+            memory: 10Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_ADMIN
+            - NET_RAW
+          privileged: false
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /run
+          name: linkerd-proxy-init-xtables-lock
+      volumes:
+      - emptyDir: {}
+        name: linkerd-proxy-init-xtables-lock
+      - emptyDir:
+          medium: Memory
+        name: linkerd-identity-end-entity
+      - name: linkerd-identity-token
+        projected:
+          sources:
+          - serviceAccountToken:
+              audience: identity.l5d.io
+              expirationSeconds: 86400
+              path: linkerd-identity-token
+---
--- a/cli/cmd/testdata/install_controlplane_tracing_output.golden
+++ b/cli/cmd/testdata/install_controlplane_tracing_output.golden
@ -1206,6 +1206,7 @@ data:
          - disabled
    prometheusUrl: ""
    proxy:
+      accessLog: ""
      await: true
      capabilities: null
      defaultInboundPolicy: ""
--- a/cli/cmd/testdata/install_custom_domain.golden
+++ b/cli/cmd/testdata/install_custom_domain.golden
@ -1206,6 +1206,7 @@ data:
          - disabled
    prometheusUrl: ""
    proxy:
+      accessLog: ""
      await: true
      capabilities: null
      defaultInboundPolicy: ""
--- a/cli/cmd/testdata/install_custom_registry.golden
+++ b/cli/cmd/testdata/install_custom_registry.golden
@ -1206,6 +1206,7 @@ data:
          - disabled
    prometheusUrl: ""
    proxy:
+      accessLog: ""
      await: true
      capabilities: null
      defaultInboundPolicy: ""
--- a/cli/cmd/testdata/install_default.golden
+++ b/cli/cmd/testdata/install_default.golden
@ -1206,6 +1206,7 @@ data:
          - disabled
    prometheusUrl: ""
    proxy:
+      accessLog: ""
      await: true
      capabilities: null
      defaultInboundPolicy: ""
--- a/cli/cmd/testdata/install_default_override_dst_get_nets.golden
+++ b/cli/cmd/testdata/install_default_override_dst_get_nets.golden
@ -1206,6 +1206,7 @@ data:
          - disabled
    prometheusUrl: ""
    proxy:
+      accessLog: ""
      await: true
      capabilities: null
      defaultInboundPolicy: ""
--- a/cli/cmd/testdata/install_default_token.golden
+++ b/cli/cmd/testdata/install_default_token.golden
@ -1206,6 +1206,7 @@ data:
          - disabled
    prometheusUrl: ""
    proxy:
+      accessLog: ""
      await: true
      capabilities: null
      defaultInboundPolicy: ""
--- a/cli/cmd/testdata/install_ha_output.golden
+++ b/cli/cmd/testdata/install_ha_output.golden
@ -1233,6 +1233,7 @@ data:
          - disabled
    prometheusUrl: ""
    proxy:
+      accessLog: ""
      await: true
      capabilities: null
      defaultInboundPolicy: ""
--- a/cli/cmd/testdata/install_ha_with_overrides_output.golden
+++ b/cli/cmd/testdata/install_ha_with_overrides_output.golden
@ -1233,6 +1233,7 @@ data:
          - disabled
    prometheusUrl: ""
    proxy:
+      accessLog: ""
      await: true
      capabilities: null
      defaultInboundPolicy: ""
--- a/cli/cmd/testdata/install_heartbeat_disabled_output.golden
+++ b/cli/cmd/testdata/install_heartbeat_disabled_output.golden
@ -1137,6 +1137,7 @@ data:
          - disabled
    prometheusUrl: ""
    proxy:
+      accessLog: ""
      await: true
      capabilities: null
      defaultInboundPolicy: ""
--- a/cli/cmd/testdata/install_helm_control_plane_output.golden
+++ b/cli/cmd/testdata/install_helm_control_plane_output.golden
@ -472,6 +472,7 @@ data:
          - disabled
    prometheusUrl: ""
    proxy:
+      accessLog: ""
      await: true
      capabilities: null
      defaultInboundPolicy: ""
--- a/cli/cmd/testdata/install_helm_control_plane_output_ha.golden
+++ b/cli/cmd/testdata/install_helm_control_plane_output_ha.golden
@ -499,6 +499,7 @@ data:
          - disabled
    prometheusUrl: ""
    proxy:
+      accessLog: ""
      await: true
      capabilities: null
      defaultInboundPolicy: ""
--- a/cli/cmd/testdata/install_helm_output_ha_labels.golden
+++ b/cli/cmd/testdata/install_helm_output_ha_labels.golden
@ -503,6 +503,7 @@ data:
          - disabled
    prometheusUrl: ""
    proxy:
+      accessLog: ""
      await: true
      capabilities: null
      defaultInboundPolicy: ""
--- a/cli/cmd/testdata/install_helm_output_ha_namespace_selector.golden
+++ b/cli/cmd/testdata/install_helm_output_ha_namespace_selector.golden
@ -499,6 +499,7 @@ data:
          - enabled
    prometheusUrl: ""
    proxy:
+      accessLog: ""
      await: true
      capabilities: null
      defaultInboundPolicy: ""
--- a/cli/cmd/testdata/install_no_init_container.golden
+++ b/cli/cmd/testdata/install_no_init_container.golden
@ -1206,6 +1206,7 @@ data:
          - disabled
    prometheusUrl: ""
    proxy:
+      accessLog: ""
      await: true
      capabilities: null
      defaultInboundPolicy: ""
--- a/cli/cmd/testdata/install_output.golden
+++ b/cli/cmd/testdata/install_output.golden
@ -1203,6 +1203,7 @@ data:
          - disabled
    prometheusUrl: ""
    proxy:
+      accessLog: ""
      await: false
      capabilities: null
      defaultInboundPolicy: ""
--- a/cli/cmd/testdata/install_proxy_ignores.golden
+++ b/cli/cmd/testdata/install_proxy_ignores.golden
@ -1206,6 +1206,7 @@ data:
          - disabled
    prometheusUrl: ""
    proxy:
+      accessLog: ""
      await: true
      capabilities: null
      defaultInboundPolicy: ""
--- a/cli/cmd/testdata/install_values_file.golden
+++ b/cli/cmd/testdata/install_values_file.golden
@ -1206,6 +1206,7 @@ data:
          - disabled
    prometheusUrl: ""
    proxy:
+      accessLog: ""
      await: true
      capabilities: null
      defaultInboundPolicy: ""
--- a/pkg/charts/linkerd2/values.go
+++ b/pkg/charts/linkerd2/values.go
@ -111,6 +111,7 @@ type (
 		OpaquePorts                   string           `json:"opaquePorts"`
 		Await                         bool             `json:"await"`
 		DefaultInboundPolicy          string           `json:"defaultInboundPolicy"`
+		AccessLog                     string           `json:"accessLog"`
 	}

 	// ProxyInit contains the fields to set the proxy-init container
--- a/pkg/inject/inject.go
+++ b/pkg/inject/inject.go
@ -70,6 +70,7 @@ var (
 		k8s.ProxyAwait,
 		k8s.ProxyDefaultInboundPolicyAnnotation,
 		k8s.ProxySkipSubnetsAnnotation,
+		k8s.ProxyAccessLogAnnotation,
 	}
 	// ProxyAlphaConfigAnnotations is the list of all alpha configuration
 	// (config.alpha prefix) that can be applied to a pod or namespace.
@ -1085,6 +1086,10 @@ func (conf *ResourceConfig) applyAnnotationOverrides(values *l5dcharts.Values) {
 	if override, ok := annotations[k8s.ProxySkipSubnetsAnnotation]; ok {
 		values.ProxyInit.SkipSubnets = override
 	}
+
+	if override, ok := annotations[k8s.ProxyAccessLogAnnotation]; ok {
+		values.Proxy.AccessLog = override
+	}
 }

 // GetOverriddenConfiguration returns a map of the overridden proxy annotations
--- a/pkg/inject/inject_test.go
+++ b/pkg/inject/inject_test.go
@ -69,6 +69,7 @@ func TestGetOverriddenValues(t *testing.T) {
 							k8s.ProxyOpaquePortsAnnotation:                   "4320-4325,3306",
 							k8s.ProxyAwait:                                   "enabled",
 							k8s.ProxySkipSubnetsAnnotation:                   "172.17.0.0/16",
+							k8s.ProxyAccessLogAnnotation:                     "apache",
 						},
 					},
 					Spec: corev1.PodSpec{},
@ -116,6 +117,7 @@ func TestGetOverriddenValues(t *testing.T) {
 				values.Proxy.InboundConnectTimeout = "600ms"
 				values.Proxy.OpaquePorts = "4320,4321,4322,4323,4324,4325,3306"
 				values.Proxy.Await = true
+				values.Proxy.AccessLog = "apache"
 				return values
 			},
 		},
@ -159,6 +161,7 @@ func TestGetOverriddenValues(t *testing.T) {
 				k8s.ProxyInboundConnectTimeout:            "600ms",
 				k8s.ProxyOpaquePortsAnnotation:            "4320-4325,3306",
 				k8s.ProxyAwait:                            "enabled",
+				k8s.ProxyAccessLogAnnotation:              "apache",
 			},
 			spec: appsv1.DeploymentSpec{
 				Template: corev1.PodTemplateSpec{
@ -201,6 +204,7 @@ func TestGetOverriddenValues(t *testing.T) {
 				values.Proxy.InboundConnectTimeout = "600ms"
 				values.Proxy.OpaquePorts = "4320,4321,4322,4323,4324,4325,3306"
 				values.Proxy.Await = true
+				values.Proxy.AccessLog = "apache"
 				return values
 			},
 		},
--- a/pkg/k8s/labels.go
+++ b/pkg/k8s/labels.go
@ -257,6 +257,10 @@ const (
 	// inbound policy of the proxy
 	ProxyDefaultInboundPolicyAnnotation = ProxyConfigAnnotationsPrefix + "/default-inbound-policy"

+	// ProxyAccessLogAnnotation configures whether HTTP access logging is
+	// enabled, and what access log format is used.
+	ProxyAccessLogAnnotation = ProxyConfigAnnotationsPrefix + "/access-log"
+
 	// IdentityModeDefault is assigned to IdentityModeAnnotation to
 	// use the control plane's default identity scheme.
 	IdentityModeDefault = "default"