Add native sidecar support (#11465)

* Add native sidecar support

Kubernetes will be providing beta support for native sidecar containers in version 1.29.  This feature improves network proxy sidecar compatibility for jobs and initContainers.

Introduce a new annotation config.alpha.linkerd.io/proxy-enable-native-sidecar and configuration option Proxy.NativeSidecar that causes the proxy container to run as an init-container.

Fixes: #11461

Signed-off-by: TJ Miller <millert@us.ibm.com>
This commit is contained in:
TJ Miller 2023-11-22 09:23:24 -08:00 committed by GitHub
parent e849b1ab1a
commit 1b37e1989f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
40 changed files with 458 additions and 27 deletions

View File

@ -239,6 +239,7 @@ Kubernetes: `>=1.21.0-0`
| proxy.inboundDiscoveryCacheUnusedTimeout | string | `"90s"` | Maximum time allowed before an unused inbound discovery result is evicted from the cache |
| proxy.logFormat | string | `"plain"` | Log format (`plain` or `json`) for the proxy |
| proxy.logLevel | string | `"warn,linkerd=info,trust_dns=error"` | Log level for the proxy |
| proxy.nativeSidecar | bool | `false` | Enable KEP-753 native sidecars This is an experimental feature. It requires Kubernetes >= 1.29. If enabled, .proxy.waitBeforeExitSeconds should not be used. |
| proxy.opaquePorts | string | `"25,587,3306,4444,5432,6379,9300,11211"` | Default set of opaque ports - SMTP (25,587) server-first - MYSQL (3306) server-first - Galera (4444) server-first - PostgreSQL (5432) server-first - Redis (6379) server-first - ElasticSearch (9300) server-first - Memcached (11211) clients do not issue any preamble, which breaks detection |
| proxy.outboundConnectTimeout | string | `"1000ms"` | Maximum time allowed for the proxy to establish an outbound TCP connection |
| proxy.outboundDiscoveryCacheUnusedTimeout | string | `"5s"` | Maximum time allowed before an unused outbound discovery result is evicted from the cache |
@ -254,6 +255,7 @@ Kubernetes: `>=1.21.0-0`
| proxy.resources.memory.limit | string | `""` | Maximum amount of memory that the proxy can use |
| proxy.resources.memory.request | string | `""` | Maximum amount of memory that the proxy requests |
| proxy.shutdownGracePeriod | string | `""` | Grace period for graceful proxy shutdowns. If this timeout elapses before all open connections have completed, the proxy will terminate forcefully, closing any remaining connections. |
| proxy.startupProbe | object | `{"failureThreshold":120,"initialDelaySeconds":0,"periodSeconds":1}` | Native sidecar proxy startup probe parameters. |
| proxy.uid | int | `2102` | User id under which the proxy runs |
| proxy.waitBeforeExitSeconds | int | `0` | If set the injected proxy sidecars in the data plane will stay alive for at least the given period before receiving the SIGTERM signal from Kubernetes but no longer than the pod's `terminationGracePeriodSeconds`. See [Lifecycle hooks](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks) for more info on container lifecycle hooks. |
| proxyInit.closeWaitTimeoutSecs | int | `0` | |

View File

@ -190,7 +190,9 @@ spec:
*/}}
{{- $_ := set $tree.Values.proxy "defaultInboundPolicy" "all-unauthenticated" }}
{{- $_ := set $tree.Values.proxy "capabilities" (dict "drop" (list "ALL")) }}
{{- if not $tree.Values.proxy.nativeSidecar }}
- {{- include "partials.proxy" $tree | indent 8 | trimPrefix (repeat 7 " ") }}
{{- end }}
- args:
- destination
- -addr=:8086
@ -341,6 +343,12 @@ spec:
{{- $_ := set $tree.Values.proxyInit "ignoreOutboundPorts" .Values.proxyInit.kubeAPIServerPorts -}}
- {{- include "partials.proxy-init" $tree | indent 8 | trimPrefix (repeat 7 " ") }}
{{ end -}}
{{- if $tree.Values.proxy.nativeSidecar }}
{{- $_ := set $tree.Values.proxy "startupProbeInitialDelaySeconds" 35 }}
{{- $_ := set $tree.Values.proxy "startupProbePeriodSeconds" 5 }}
{{- $_ := set $tree.Values.proxy "startupProbeFailureThreshold" 20 }}
- {{- include "partials.proxy" $tree | indent 8 | trimPrefix (repeat 7 " ") }}
{{ end -}}
{{- if .Values.priorityClassName -}}
priorityClassName: {{ .Values.priorityClassName }}
{{ end -}}

View File

@ -206,6 +206,7 @@ spec:
{{- $_ := set $tree.Values.proxy "await" false }}
{{- $_ := set $tree.Values.proxy "loadTrustBundleFromConfigMap" true }}
{{- $_ := set $tree.Values.proxy "podInboundPorts" "8080,9990" }}
{{- $_ := set $tree.Values.proxy "nativeSidecar" false }}
{{- /*
The identity controller cannot discover policies, so we configure it with defaults that
enforce TLS on the identity service.

View File

@ -70,7 +70,9 @@ spec:
{{- $_ := set $tree.Values.proxy "capabilities" (dict "drop" (list "ALL")) }}
{{- $_ := set $tree.Values.proxy "outboundDiscoveryCacheUnusedTimeout" "5s" }}
{{- $_ := set $tree.Values.proxy "inboundDiscoveryCacheUnusedTimeout" "90s" }}
{{- if not $tree.Values.proxy.nativeSidecar }}
- {{- include "partials.proxy" $tree | indent 8 | trimPrefix (repeat 7 " ") }}
{{- end }}
- args:
- proxy-injector
- -log-level={{.Values.controllerLogLevel}}
@ -127,6 +129,12 @@ spec:
{{- $_ := set $tree.Values.proxyInit "ignoreOutboundPorts" .Values.proxyInit.kubeAPIServerPorts -}}
- {{- include "partials.proxy-init" $tree | indent 8 | trimPrefix (repeat 7 " ") }}
{{ end -}}
{{- if $tree.Values.proxy.nativeSidecar }}
{{- $_ := set $tree.Values.proxy "startupProbeInitialDelaySeconds" 35 }}
{{- $_ := set $tree.Values.proxy "startupProbePeriodSeconds" 5 }}
{{- $_ := set $tree.Values.proxy "startupProbeFailureThreshold" 20 }}
- {{- include "partials.proxy" $tree | indent 8 | trimPrefix (repeat 7 " ") }}
{{ end -}}
{{- if .Values.priorityClassName -}}
priorityClassName: {{ .Values.priorityClassName }}
{{ end -}}

View File

@ -191,6 +191,15 @@ proxy:
# "all-unauthenticated", "cluster-authenticated", "cluster-unauthenticated", "deny"
# @default -- "all-unauthenticated"
defaultInboundPolicy: "all-unauthenticated"
# -- Enable KEP-753 native sidecars
# This is an experimental feature. It requires Kubernetes >= 1.29.
# If enabled, .proxy.waitBeforeExitSeconds should not be used.
nativeSidecar: false
# -- Native sidecar proxy startup probe parameters.
startupProbe:
initialDelaySeconds: 0
periodSeconds: 1
failureThreshold: 120
# proxy-init configuration
proxyInit:

View File

@ -1,4 +1,7 @@
{{ define "partials.proxy" -}}
{{ if and .Values.proxy.nativeSidecar .Values.proxy.waitBeforeExitSeconds }}
{{ fail "proxy.nativeSidecar and waitBeforeExitSeconds cannot be used simultaneously" }}
{{- end }}
{{- $trustDomain := (.Values.identityTrustDomain | default .Values.clusterDomain) -}}
env:
- name: _pod_name
@ -168,6 +171,15 @@ readinessProbe:
path: /ready
port: {{.Values.proxy.ports.admin}}
initialDelaySeconds: 2
{{- if and .Values.proxy.nativeSidecar .Values.proxy.await }}
startupProbe:
httpGet:
path: /ready
port: {{.Values.proxy.ports.admin}}
initialDelaySeconds: {{.Values.proxy.startupProbe.initialDelaySeconds}}
periodSeconds: {{.Values.proxy.startupProbe.periodSeconds}}
failureThreshold: {{.Values.proxy.startupProbe.failureThreshold}}
{{- end }}
{{- if .Values.proxy.resources }}
{{ include "partials.resources" .Values.proxy.resources }}
{{- end }}
@ -182,7 +194,7 @@ securityContext:
seccompProfile:
type: RuntimeDefault
terminationMessagePolicy: FallbackToLogsOnError
{{- if or (.Values.proxy.await) (.Values.proxy.waitBeforeExitSeconds) }}
{{- if and (not .Values.proxy.nativeSidecar) (or .Values.proxy.await .Values.proxy.waitBeforeExitSeconds) }}
lifecycle:
{{- if .Values.proxy.await }}
postStart:
@ -212,4 +224,7 @@ volumeMounts:
name: {{.Values.proxy.saMountPath.name}}
readOnly: {{.Values.proxy.saMountPath.readOnly}}
{{- end -}}
{{- if .Values.proxy.nativeSidecar }}
restartPolicy: Always
{{- end -}}
{{- end }}

View File

@ -1,4 +1,14 @@
{{ $prefix := .Values.pathPrefix -}}
{{/*
$initIndex represents the patch insertion index of the next initContainer when
proxy.nativeSidecar is true. If enabled, the proxy-init or network-validator
should run first, immediately followed by the proxy. This ordering allows us
to proxy traffic in subsequent initContainers.
Note: dig is not used directly on .Values because it rejects chartutil.Values
structs.
*/}}
{{- $initIndex := ternary "0" "-" (.Values.proxy | default (dict) | dig "nativeSidecar" false) -}}
[
{{- if .Values.addRootMetadata }}
{
@ -62,14 +72,14 @@
},
{
"op": "add",
"path": "{{$prefix}}/spec/initContainers/-",
"path": "{{$prefix}}/spec/initContainers/{{$initIndex}}{{$initIndex = add1 $initIndex}}",
"value":
{{- include "partials.proxy-init" . | fromYaml | toPrettyJson | nindent 6 }}
},
{{- else if and .Values.proxy .Values.cniEnabled }}
{
"op": "add",
"path": "{{$prefix}}/spec/initContainers/-",
"path": "{{$prefix}}/spec/initContainers/{{$initIndex}}{{$initIndex = add1 $initIndex}}",
"value":
{{- include "partials.network-validator" . | fromYaml | toPrettyJson | nindent 6 }}
},
@ -103,7 +113,9 @@
{{- end }}
{
"op": "add",
{{- if .Values.proxy.await }}
{{- if .Values.proxy.nativeSidecar }}
"path": "{{$prefix}}/spec/initContainers/{{$initIndex}}",
{{- else if .Values.proxy.await }}
"path": "{{$prefix}}/spec/containers/0",
{{- else }}
"path": "{{$prefix}}/spec/containers/-",

View File

@ -280,5 +280,9 @@ func generateAnnotationsDocs() []annotationDoc {
Name: k8s.ProxyShutdownGracePeriodAnnotation,
Description: "Grace period for graceful proxy shutdowns. If this timeout elapses before all open connections have completed, the proxy will terminate forcefully, closing any remaining connections.",
},
{
Name: k8s.ProxyEnableNativeSidecarAnnotation,
Description: "Enable KEP-753 native sidecars. This is an experimental feature. It requires Kubernetes >= 1.29. If enabled, .proxy.waitBeforeExitSeconds should not be used.",
},
}
}

View File

@ -491,6 +491,10 @@ func getOverrideAnnotations(values *linkerd2.Values, base *linkerd2.Values) map[
overrideAnnotations[k8s.ProxyShutdownGracePeriodAnnotation] = proxy.ShutdownGracePeriod
}
if proxy.NativeSidecar != baseProxy.NativeSidecar {
overrideAnnotations[k8s.ProxyEnableNativeSidecarAnnotation] = strconv.FormatBool(proxy.NativeSidecar)
}
return overrideAnnotations
}

View File

@ -344,6 +344,17 @@ func TestUninjectAndInject(t *testing.T) {
return values
}(),
},
{
inputFileName: "inject_emojivoto_deployment.input.yml",
goldenFileName: "inject_emojivoto_deployment_native_sidecar.golden.yml",
reportFileName: "inject_emojivoto_deployment.report",
injectProxy: true,
testInjectConfig: func() *linkerd2.Values {
values := defaultConfig()
values.Proxy.NativeSidecar = true
return values
}(),
},
}
for i, tc := range testCases {
@ -678,6 +689,7 @@ func TestProxyConfigurationAnnotations(t *testing.T) {
values.Proxy.Await = false
values.Proxy.AccessLog = "apache"
values.Proxy.ShutdownGracePeriod = "60s"
values.Proxy.NativeSidecar = true
expectedOverrides := map[string]string{
k8s.ProxyIgnoreInboundPortsAnnotation: "8500-8505",
@ -699,6 +711,7 @@ func TestProxyConfigurationAnnotations(t *testing.T) {
k8s.ProxyAwait: "disabled",
k8s.ProxyAccessLogAnnotation: "apache",
k8s.ProxyShutdownGracePeriodAnnotation: "60s",
k8s.ProxyEnableNativeSidecarAnnotation: "true",
}
overrides := getOverrideAnnotations(values, baseValues)

View File

@ -441,6 +441,12 @@ func makeInjectFlags(defaults *l5dcharts.Values) ([]flag.Flag, *pflag.FlagSet) {
injectFlags := pflag.NewFlagSet("inject", pflag.ExitOnError)
flags := []flag.Flag{
flag.NewBoolFlag(injectFlags, "native-sidecar", false, "Enable native sidecar",
func(values *l5dcharts.Values, value bool) error {
values.Proxy.NativeSidecar = value
return nil
}),
flag.NewInt64Flag(injectFlags, "wait-before-exit-seconds", int64(defaults.Proxy.WaitBeforeExitSeconds),
"The period during which the proxy sidecar must stay alive while its pod is terminating. "+
"Must be smaller than terminationGracePeriodSeconds for the pod (default 0)",

View File

@ -0,0 +1,226 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: web
namespace: emojivoto
spec:
replicas: 1
selector:
matchLabels:
app: web-svc
template:
metadata:
annotations:
config.alpha.linkerd.io/proxy-enable-native-sidecar: "true"
linkerd.io/created-by: linkerd/cli dev-undefined
linkerd.io/proxy-version: test-inject-proxy-version
linkerd.io/trust-root-sha256: 8dc603abd4e755c25c94da05abbf29b9b283a784733651020d72f97ca8ab98e4
labels:
app: web-svc
linkerd.io/control-plane-ns: linkerd
linkerd.io/proxy-deployment: web
linkerd.io/workload-ns: emojivoto
spec:
containers:
- env:
- name: WEB_PORT
value: "80"
- name: EMOJISVC_HOST
value: emoji-svc.emojivoto:8080
- name: VOTINGSVC_HOST
value: voting-svc.emojivoto:8080
- name: INDEX_BUNDLE
value: dist/index_bundle.js
image: buoyantio/emojivoto-web:v10
name: web-svc
ports:
- containerPort: 80
name: http
initContainers:
- args:
- --incoming-proxy-port
- "4143"
- --outgoing-proxy-port
- "4140"
- --proxy-uid
- "2102"
- --inbound-ports-to-ignore
- 4190,4191,4567,4568
- --outbound-ports-to-ignore
- 4567,4568
image: cr.l5d.io/linkerd/proxy-init:v2.2.3
imagePullPolicy: IfNotPresent
name: linkerd-init
resources:
limits:
cpu: 100m
memory: 20Mi
requests:
cpu: 100m
memory: 20Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- NET_ADMIN
- NET_RAW
privileged: false
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 65534
seccompProfile:
type: RuntimeDefault
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /run
name: linkerd-proxy-init-xtables-lock
- env:
- name: _pod_name
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: _pod_ns
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: _pod_nodeName
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: LINKERD2_PROXY_LOG
value: warn,linkerd=info,trust_dns=error
- name: LINKERD2_PROXY_LOG_FORMAT
value: plain
- name: LINKERD2_PROXY_DESTINATION_SVC_ADDR
value: linkerd-dst-headless.linkerd.svc.cluster.local.:8086
- name: LINKERD2_PROXY_DESTINATION_PROFILE_NETWORKS
value: 10.0.0.0/8,100.64.0.0/10,172.16.0.0/12,192.168.0.0/16
- name: LINKERD2_PROXY_POLICY_SVC_ADDR
value: linkerd-policy.linkerd.svc.cluster.local.:8090
- name: LINKERD2_PROXY_POLICY_WORKLOAD
value: $(_pod_ns):$(_pod_name)
- name: LINKERD2_PROXY_INBOUND_DEFAULT_POLICY
value: all-unauthenticated
- name: LINKERD2_PROXY_POLICY_CLUSTER_NETWORKS
value: 10.0.0.0/8,100.64.0.0/10,172.16.0.0/12,192.168.0.0/16
- name: LINKERD2_PROXY_INBOUND_CONNECT_TIMEOUT
value: 100ms
- name: LINKERD2_PROXY_OUTBOUND_CONNECT_TIMEOUT
value: 1000ms
- name: LINKERD2_PROXY_OUTBOUND_DISCOVERY_IDLE_TIMEOUT
value: 5s
- name: LINKERD2_PROXY_INBOUND_DISCOVERY_IDLE_TIMEOUT
value: 90s
- name: LINKERD2_PROXY_CONTROL_LISTEN_ADDR
value: 0.0.0.0:4190
- name: LINKERD2_PROXY_ADMIN_LISTEN_ADDR
value: 0.0.0.0:4191
- name: LINKERD2_PROXY_OUTBOUND_LISTEN_ADDR
value: 127.0.0.1:4140
- name: LINKERD2_PROXY_INBOUND_LISTEN_ADDR
value: 0.0.0.0:4143
- name: LINKERD2_PROXY_INBOUND_IPS
valueFrom:
fieldRef:
fieldPath: status.podIPs
- name: LINKERD2_PROXY_INBOUND_PORTS
value: "80"
- name: LINKERD2_PROXY_DESTINATION_PROFILE_SUFFIXES
value: svc.cluster.local.
- name: LINKERD2_PROXY_INBOUND_ACCEPT_KEEPALIVE
value: 10000ms
- name: LINKERD2_PROXY_OUTBOUND_CONNECT_KEEPALIVE
value: 10000ms
- name: LINKERD2_PROXY_INBOUND_PORTS_DISABLE_PROTOCOL_DETECTION
value: 25,587,3306,4444,5432,6379,9300,11211
- name: LINKERD2_PROXY_DESTINATION_CONTEXT
value: |
{"ns":"$(_pod_ns)", "nodeName":"$(_pod_nodeName)", "pod":"$(_pod_name)"}
- name: _pod_sa
valueFrom:
fieldRef:
fieldPath: spec.serviceAccountName
- name: _l5d_ns
value: linkerd
- name: _l5d_trustdomain
value: cluster.local
- name: LINKERD2_PROXY_IDENTITY_DIR
value: /var/run/linkerd/identity/end-entity
- name: LINKERD2_PROXY_IDENTITY_TRUST_ANCHORS
value: |
-----BEGIN CERTIFICATE-----
MIIBwTCCAWagAwIBAgIQeDZp5lDaIygQ5UfMKZrFATAKBggqhkjOPQQDAjApMScw
JQYDVQQDEx5pZGVudGl0eS5saW5rZXJkLmNsdXN0ZXIubG9jYWwwHhcNMjAwODI4
MDcxMjQ3WhcNMzAwODI2MDcxMjQ3WjApMScwJQYDVQQDEx5pZGVudGl0eS5saW5r
ZXJkLmNsdXN0ZXIubG9jYWwwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARqc70Z
l1vgw79rjB5uSITICUA6GyfvSFfcuIis7B/XFSkkwAHU5S/s1AAP+R0TX7HBWUC4
uaG4WWsiwJKNn7mgo3AwbjAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB
/wIBATAdBgNVHQ4EFgQU5YtjVVPfd7I7NLHsn2C26EByGV0wKQYDVR0RBCIwIIIe
aWRlbnRpdHkubGlua2VyZC5jbHVzdGVyLmxvY2FsMAoGCCqGSM49BAMCA0kAMEYC
IQCN7lBFLDDvjx6V0+XkjpKERRsJYf5adMvnloFl48ilJgIhANtxhndcr+QJPuC8
vgUC0d2/9FMueIVMb+46WTCOjsqr
-----END CERTIFICATE-----
- name: LINKERD2_PROXY_IDENTITY_TOKEN_FILE
value: /var/run/secrets/tokens/linkerd-identity-token
- name: LINKERD2_PROXY_IDENTITY_SVC_ADDR
value: linkerd-identity-headless.linkerd.svc.cluster.local.:8080
- name: LINKERD2_PROXY_IDENTITY_LOCAL_NAME
value: $(_pod_sa).$(_pod_ns).serviceaccount.identity.linkerd.cluster.local
- name: LINKERD2_PROXY_IDENTITY_SVC_NAME
value: linkerd-identity.linkerd.serviceaccount.identity.linkerd.cluster.local
- name: LINKERD2_PROXY_DESTINATION_SVC_NAME
value: linkerd-destination.linkerd.serviceaccount.identity.linkerd.cluster.local
- name: LINKERD2_PROXY_POLICY_SVC_NAME
value: linkerd-destination.linkerd.serviceaccount.identity.linkerd.cluster.local
image: cr.l5d.io/linkerd/proxy:test-inject-proxy-version
imagePullPolicy: IfNotPresent
livenessProbe:
httpGet:
path: /live
port: 4191
initialDelaySeconds: 10
name: linkerd-proxy
ports:
- containerPort: 4143
name: linkerd-proxy
- containerPort: 4191
name: linkerd-admin
readinessProbe:
httpGet:
path: /ready
port: 4191
initialDelaySeconds: 2
restartPolicy: Always
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 2102
seccompProfile:
type: RuntimeDefault
startupProbe:
failureThreshold: 120
httpGet:
path: /ready
port: 4191
periodSeconds: 1
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /var/run/linkerd/identity/end-entity
name: linkerd-identity-end-entity
- mountPath: /var/run/secrets/tokens
name: linkerd-identity-token
volumes:
- emptyDir: {}
name: linkerd-proxy-init-xtables-lock
- emptyDir:
medium: Memory
name: linkerd-identity-end-entity
- name: linkerd-identity-token
projected:
sources:
- serviceAccountToken:
audience: identity.l5d.io
expirationSeconds: 86400
path: linkerd-identity-token
---

View File

@ -634,6 +634,7 @@ data:
isIngress: false
logFormat: plain
logLevel: warn,linkerd=info,trust_dns=error
nativeSidecar: false
opaquePorts: 25,587,3306,4444,5432,6379,9300,11211
outboundConnectTimeout: 1000ms
outboundDiscoveryCacheUnusedTimeout: 5s
@ -656,6 +657,10 @@ data:
request: ""
saMountPath: null
shutdownGracePeriod: ""
startupProbe:
failureThreshold: 120
initialDelaySeconds: 0
periodSeconds: 1
uid: 2102
waitBeforeExitSeconds: 0
proxyContainerName: linkerd-proxy

View File

@ -634,6 +634,7 @@ data:
isIngress: false
logFormat: plain
logLevel: warn,linkerd=info,trust_dns=error
nativeSidecar: false
opaquePorts: 25,587,3306,4444,5432,6379,9300,11211
outboundConnectTimeout: 1000ms
outboundDiscoveryCacheUnusedTimeout: 5s
@ -656,6 +657,10 @@ data:
request: ""
saMountPath: null
shutdownGracePeriod: ""
startupProbe:
failureThreshold: 120
initialDelaySeconds: 0
periodSeconds: 1
uid: 2102
waitBeforeExitSeconds: 0
proxyContainerName: linkerd-proxy

View File

@ -634,6 +634,7 @@ data:
isIngress: false
logFormat: plain
logLevel: warn,linkerd=info,trust_dns=error
nativeSidecar: false
opaquePorts: 25,587,3306,4444,5432,6379,9300,11211
outboundConnectTimeout: 1000ms
outboundDiscoveryCacheUnusedTimeout: 5s
@ -656,6 +657,10 @@ data:
request: ""
saMountPath: null
shutdownGracePeriod: ""
startupProbe:
failureThreshold: 120
initialDelaySeconds: 0
periodSeconds: 1
uid: 2102
waitBeforeExitSeconds: 0
proxyContainerName: linkerd-proxy

View File

@ -634,6 +634,7 @@ data:
isIngress: false
logFormat: plain
logLevel: warn,linkerd=info,trust_dns=error
nativeSidecar: false
opaquePorts: 25,587,3306,4444,5432,6379,9300,11211
outboundConnectTimeout: 1000ms
outboundDiscoveryCacheUnusedTimeout: 5s
@ -656,6 +657,10 @@ data:
request: ""
saMountPath: null
shutdownGracePeriod: ""
startupProbe:
failureThreshold: 120
initialDelaySeconds: 0
periodSeconds: 1
uid: 2102
waitBeforeExitSeconds: 0
proxyContainerName: linkerd-proxy

View File

@ -634,6 +634,7 @@ data:
isIngress: false
logFormat: plain
logLevel: warn,linkerd=info,trust_dns=error
nativeSidecar: false
opaquePorts: 25,587,3306,4444,5432,6379,9300,11211
outboundConnectTimeout: 1000ms
outboundDiscoveryCacheUnusedTimeout: 5s
@ -656,6 +657,10 @@ data:
request: ""
saMountPath: null
shutdownGracePeriod: ""
startupProbe:
failureThreshold: 120
initialDelaySeconds: 0
periodSeconds: 1
uid: 2102
waitBeforeExitSeconds: 0
proxyContainerName: linkerd-proxy

View File

@ -634,6 +634,7 @@ data:
isIngress: false
logFormat: plain
logLevel: warn,linkerd=info,trust_dns=error
nativeSidecar: false
opaquePorts: 25,587,3306,4444,5432,6379,9300,11211
outboundConnectTimeout: 1000ms
outboundDiscoveryCacheUnusedTimeout: 5s
@ -656,6 +657,10 @@ data:
request: ""
saMountPath: null
shutdownGracePeriod: ""
startupProbe:
failureThreshold: 120
initialDelaySeconds: 0
periodSeconds: 1
uid: 2102
waitBeforeExitSeconds: 0
proxyContainerName: linkerd-proxy

View File

@ -661,6 +661,7 @@ data:
isIngress: false
logFormat: plain
logLevel: warn,linkerd=info,trust_dns=error
nativeSidecar: false
opaquePorts: 25,587,3306,4444,5432,6379,9300,11211
outboundConnectTimeout: 1000ms
outboundDiscoveryCacheUnusedTimeout: 5s
@ -683,6 +684,10 @@ data:
request: 20Mi
saMountPath: null
shutdownGracePeriod: ""
startupProbe:
failureThreshold: 120
initialDelaySeconds: 0
periodSeconds: 1
uid: 2102
waitBeforeExitSeconds: 0
proxyContainerName: linkerd-proxy

View File

@ -661,6 +661,7 @@ data:
isIngress: false
logFormat: plain
logLevel: warn,linkerd=info,trust_dns=error
nativeSidecar: false
opaquePorts: 25,587,3306,4444,5432,6379,9300,11211
outboundConnectTimeout: 1000ms
outboundDiscoveryCacheUnusedTimeout: 5s
@ -683,6 +684,10 @@ data:
request: 300Mi
saMountPath: null
shutdownGracePeriod: ""
startupProbe:
failureThreshold: 120
initialDelaySeconds: 0
periodSeconds: 1
uid: 2102
waitBeforeExitSeconds: 0
proxyContainerName: linkerd-proxy

View File

@ -565,6 +565,7 @@ data:
isIngress: false
logFormat: plain
logLevel: warn,linkerd=info,trust_dns=error
nativeSidecar: false
opaquePorts: 25,587,3306,4444,5432,6379,9300,11211
outboundConnectTimeout: 1000ms
outboundDiscoveryCacheUnusedTimeout: 5s
@ -587,6 +588,10 @@ data:
request: ""
saMountPath: null
shutdownGracePeriod: ""
startupProbe:
failureThreshold: 120
initialDelaySeconds: 0
periodSeconds: 1
uid: 2102
waitBeforeExitSeconds: 0
proxyContainerName: linkerd-proxy

View File

@ -611,6 +611,7 @@ data:
isIngress: false
logFormat: plain
logLevel: warn,linkerd=info,trust_dns=error
nativeSidecar: false
opaquePorts: 25,587,3306,4444,5432,6379,9300,11211
outboundConnectTimeout: 1000ms
outboundDiscoveryCacheUnusedTimeout: 5s
@ -633,6 +634,10 @@ data:
request: ""
saMountPath: null
shutdownGracePeriod: ""
startupProbe:
failureThreshold: 120
initialDelaySeconds: 0
periodSeconds: 1
uid: 2102
waitBeforeExitSeconds: 0
proxyContainerName: linkerd-proxy

View File

@ -638,6 +638,7 @@ data:
isIngress: false
logFormat: plain
logLevel: warn,linkerd=info,trust_dns=error
nativeSidecar: false
opaquePorts: 25,587,3306,4444,5432,6379,9300,11211
outboundConnectTimeout: 1000ms
outboundDiscoveryCacheUnusedTimeout: 5s
@ -660,6 +661,10 @@ data:
request: 20Mi
saMountPath: null
shutdownGracePeriod: ""
startupProbe:
failureThreshold: 120
initialDelaySeconds: 0
periodSeconds: 1
uid: 2102
waitBeforeExitSeconds: 0
proxyContainerName: linkerd-proxy

View File

@ -642,6 +642,7 @@ data:
isIngress: false
logFormat: plain
logLevel: warn,linkerd=info,trust_dns=error
nativeSidecar: false
opaquePorts: 25,587,3306,4444,5432,6379,9300,11211
outboundConnectTimeout: 1000ms
outboundDiscoveryCacheUnusedTimeout: 5s
@ -664,6 +665,10 @@ data:
request: 20Mi
saMountPath: null
shutdownGracePeriod: ""
startupProbe:
failureThreshold: 120
initialDelaySeconds: 0
periodSeconds: 1
uid: 2102
waitBeforeExitSeconds: 0
proxyContainerName: linkerd-proxy

View File

@ -633,6 +633,7 @@ data:
isIngress: false
logFormat: plain
logLevel: warn,linkerd=info,trust_dns=error
nativeSidecar: false
opaquePorts: 25,587,3306,4444,5432,6379,9300,11211
outboundConnectTimeout: 1000ms
outboundDiscoveryCacheUnusedTimeout: 5s
@ -655,6 +656,10 @@ data:
request: 20Mi
saMountPath: null
shutdownGracePeriod: ""
startupProbe:
failureThreshold: 120
initialDelaySeconds: 0
periodSeconds: 1
uid: 2102
waitBeforeExitSeconds: 0
proxyContainerName: linkerd-proxy

View File

@ -634,6 +634,7 @@ data:
isIngress: false
logFormat: plain
logLevel: warn,linkerd=info,trust_dns=error
nativeSidecar: false
opaquePorts: 25,587,3306,4444,5432,6379,9300,11211
outboundConnectTimeout: 1000ms
outboundDiscoveryCacheUnusedTimeout: 5s
@ -656,6 +657,10 @@ data:
request: ""
saMountPath: null
shutdownGracePeriod: ""
startupProbe:
failureThreshold: 120
initialDelaySeconds: 0
periodSeconds: 1
uid: 2102
waitBeforeExitSeconds: 0
proxyContainerName: linkerd-proxy

File diff suppressed because one or more lines are too long

View File

@ -634,6 +634,7 @@ data:
isIngress: false
logFormat: plain
logLevel: warn,linkerd=info,trust_dns=error
nativeSidecar: false
opaquePorts: 25,587,3306,4444,5432,6379,9300,11211
outboundConnectTimeout: 1000ms
outboundDiscoveryCacheUnusedTimeout: 5s
@ -656,6 +657,10 @@ data:
request: ""
saMountPath: null
shutdownGracePeriod: ""
startupProbe:
failureThreshold: 120
initialDelaySeconds: 0
periodSeconds: 1
uid: 2102
waitBeforeExitSeconds: 0
proxyContainerName: linkerd-proxy

View File

@ -634,6 +634,7 @@ data:
isIngress: false
logFormat: plain
logLevel: warn,linkerd=info,trust_dns=error
nativeSidecar: false
opaquePorts: 25,587,3306,4444,5432,6379,9300,11211
outboundConnectTimeout: 1000ms
outboundDiscoveryCacheUnusedTimeout: 5s
@ -656,6 +657,10 @@ data:
request: ""
saMountPath: null
shutdownGracePeriod: ""
startupProbe:
failureThreshold: 120
initialDelaySeconds: 0
periodSeconds: 1
uid: 2102
waitBeforeExitSeconds: 0
proxyContainerName: linkerd-proxy

View File

@ -1,17 +1,24 @@
package webhook
import (
"fmt"
labels "github.com/linkerd/linkerd2/pkg/k8s"
corev1 "k8s.io/api/core/v1"
)
// GetProxyContainerIndex gets the proxy container index of a pod; the index
// is required in webhooks because of how patches are created.
func GetProxyContainerIndex(containers []corev1.Container) int {
for i, c := range containers {
// GetProxyContainerPath gets the proxy container jsonpath of a pod relative to spec;
// this path is required in webhooks because of how patches are created.
func GetProxyContainerPath(spec corev1.PodSpec) string {
for i, c := range spec.Containers {
if c.Name == labels.ProxyContainerName {
return i
return fmt.Sprintf("containers/%d", i)
}
}
return -1
for i, c := range spec.InitContainers {
if c.Name == labels.ProxyContainerName {
return fmt.Sprintf("initContainers/%d", i)
}
}
return ""
}

View File

@ -8,7 +8,7 @@ const tpl = `[
},
{
"op": "add",
"path": "/spec/containers/{{.ProxyIndex}}/env/-",
"path": "/spec/{{.ProxyPath}}/env/-",
"value": {
"name": "LINKERD2_PROXY_TRACE_ATTRIBUTES_PATH",
"value": "/var/run/linkerd/podinfo/labels"
@ -16,7 +16,7 @@ const tpl = `[
},
{
"op": "add",
"path": "/spec/containers/{{.ProxyIndex}}/env/-",
"path": "/spec/{{.ProxyPath}}/env/-",
"value": {
"name": "LINKERD2_PROXY_TRACE_COLLECTOR_SVC_ADDR",
"value": "{{.CollectorSvcAddr}}"
@ -24,7 +24,7 @@ const tpl = `[
},
{
"op": "add",
"path": "/spec/containers/{{.ProxyIndex}}/env/-",
"path": "/spec/{{.ProxyPath}}/env/-",
"value": {
"name": "LINKERD2_PROXY_TRACE_COLLECTOR_SVC_NAME",
"value": "{{.CollectorSvcAccount}}.serviceaccount.identity.{{.LinkerdNamespace}}.{{.ClusterDomain}}"
@ -32,7 +32,7 @@ const tpl = `[
},
{
"op": "add",
"path": "/spec/containers/{{.ProxyIndex}}/volumeMounts/-",
"path": "/spec/{{.ProxyPath}}/volumeMounts/-",
"value": {
"mountPath": "var/run/linkerd/podinfo",
"name": "podinfo"
@ -44,13 +44,13 @@ const tpl = `[
"value": {
"downwardAPI": {
"items": [
{
{
"fieldRef": {
"fieldPath": "metadata.labels"
},
},
"path": "labels"
}
]
}
]
},
"name": "podinfo"
}

View File

@ -27,7 +27,7 @@ const (
// Params holds the values used in the patch template
type Params struct {
ProxyIndex int
ProxyPath string
CollectorSvcAddr string
CollectorSvcAccount string
ClusterDomain string
@ -59,13 +59,13 @@ func Mutate(collectorSvcAddr, collectorSvcAccount, clusterDomain, linkerdNamespa
return nil, err
}
params := Params{
ProxyIndex: webhook.GetProxyContainerIndex(pod.Spec.Containers),
ProxyPath: webhook.GetProxyContainerPath(pod.Spec),
CollectorSvcAddr: collectorSvcAddr,
CollectorSvcAccount: collectorSvcAccount,
ClusterDomain: clusterDomain,
LinkerdNamespace: linkerdNamespace,
}
if params.ProxyIndex < 0 || labels.IsTracingEnabled(pod) {
if params.ProxyPath == "" || labels.IsTracingEnabled(pod) {
return admissionResponse, nil
}

View File

@ -119,6 +119,8 @@ type (
DefaultInboundPolicy string `json:"defaultInboundPolicy"`
AccessLog string `json:"accessLog"`
ShutdownGracePeriod string `json:"shutdownGracePeriod"`
NativeSidecar bool `json:"nativeSidecar"`
StartupProbe *StartupProbe `json:"startupProbe"`
}
// ProxyInit contains the fields to set the proxy-init container
@ -226,6 +228,13 @@ type (
EphemeralStorage Constraints `json:"ephemeral-storage"`
}
// StartupProbe represents the initContainer startup probe parameters for the proxy
StartupProbe struct {
InitialDelaySeconds uint `json:"initialDelaySeconds"`
PeriodSeconds uint `json:"periodSeconds"`
FailureThreshold uint `json:"failureThreshold"`
}
// Identity contains the fields to set the identity variables in the proxy
// sidecar container
Identity struct {

View File

@ -134,6 +134,11 @@ func TestNewValues(t *testing.T) {
InboundDiscoveryCacheUnusedTimeout: "90s",
DisableOutboundProtocolDetectTimeout: false,
DisableInboundProtocolDetectTimeout: false,
StartupProbe: &StartupProbe{
FailureThreshold: 120,
InitialDelaySeconds: 0,
PeriodSeconds: 1,
},
},
ProxyInit: &ProxyInit{
IptablesMode: "legacy",

View File

@ -82,6 +82,7 @@ var (
// (config.alpha prefix) that can be applied to a pod or namespace.
ProxyAlphaConfigAnnotations = []string{
k8s.ProxyWaitBeforeExitSecondsAnnotation,
k8s.ProxyEnableNativeSidecarAnnotation,
}
)
@ -1000,6 +1001,13 @@ func (conf *ResourceConfig) applyAnnotationOverrides(values *l5dcharts.Values) {
}
}
if override, ok := annotations[k8s.ProxyEnableNativeSidecarAnnotation]; ok {
value, err := strconv.ParseBool(override)
if err == nil {
values.Proxy.NativeSidecar = value
}
}
if override, ok := annotations[k8s.ProxyCPURequestAnnotation]; ok {
_, err := k8sResource.ParseQuantity(override)
if err != nil {

View File

@ -74,6 +74,7 @@ func TestGetOverriddenValues(t *testing.T) {
k8s.ProxyInboundDiscoveryCacheUnusedTimeout: "900s",
k8s.ProxyDisableOutboundProtocolDetectTimeout: "true",
k8s.ProxyDisableInboundProtocolDetectTimeout: "true",
k8s.ProxyEnableNativeSidecarAnnotation: "true",
},
},
Spec: corev1.PodSpec{},
@ -126,6 +127,7 @@ func TestGetOverriddenValues(t *testing.T) {
values.Proxy.InboundDiscoveryCacheUnusedTimeout = "900s"
values.Proxy.DisableOutboundProtocolDetectTimeout = true
values.Proxy.DisableInboundProtocolDetectTimeout = true
values.Proxy.NativeSidecar = true
return values
},
},
@ -174,6 +176,7 @@ func TestGetOverriddenValues(t *testing.T) {
k8s.ProxyInboundDiscoveryCacheUnusedTimeout: "6000ms",
k8s.ProxyDisableOutboundProtocolDetectTimeout: "true",
k8s.ProxyDisableInboundProtocolDetectTimeout: "false",
k8s.ProxyEnableNativeSidecarAnnotation: "true",
},
spec: appsv1.DeploymentSpec{
Template: corev1.PodTemplateSpec{
@ -221,6 +224,7 @@ func TestGetOverriddenValues(t *testing.T) {
values.Proxy.InboundDiscoveryCacheUnusedTimeout = "6s"
values.Proxy.DisableOutboundProtocolDetectTimeout = true
values.Proxy.DisableInboundProtocolDetectTimeout = false
values.Proxy.NativeSidecar = true
return values
},
},

View File

@ -264,6 +264,9 @@ const (
// configured for the Pod
ProxyWaitBeforeExitSecondsAnnotation = ProxyConfigAnnotationsPrefixAlpha + "/proxy-wait-before-exit-seconds"
// ProxyEnableNativeSidecarAnnotation enables the new native initContainer sidecar
ProxyEnableNativeSidecarAnnotation = ProxyConfigAnnotationsPrefixAlpha + "/proxy-enable-native-sidecar"
// ProxyAwait can be used to force the application to wait for the proxy
// to be ready.
ProxyAwait = ProxyConfigAnnotationsPrefix + "/proxy-await"

View File

@ -43,7 +43,12 @@ pub(crate) fn tcp_ports_by_name(spec: &k8s::PodSpec) -> HashMap<String, PortSet>
/// Pod and the paths for which probes are expected.
pub(crate) fn pod_http_probes(pod: &k8s::PodSpec) -> PortMap<BTreeSet<String>> {
let mut probes = PortMap::<BTreeSet<String>>::default();
for (port, path) in pod.containers.iter().flat_map(container_http_probe_paths) {
for (port, path) in pod
.containers
.iter()
.chain(pod.init_containers.iter().flatten())
.flat_map(container_http_probe_paths)
{
probes.entry(port).or_default().insert(path);
}
probes

View File

@ -8,7 +8,7 @@ var tpl = `[
},
{
"op": "add",
"path": "/spec/containers/{{.ProxyIndex}}/env/-",
"path": "/spec/{{.ProxyPath}}/env/-",
"value": {
"name": "LINKERD2_PROXY_TAP_SVC_NAME",
"value": "{{.ProxyTapSvcName}}"

View File

@ -17,7 +17,7 @@ import (
// Params holds the values used in the patch template.
type Params struct {
ProxyIndex int
ProxyPath string
ProxyTapSvcName string
}
@ -41,10 +41,10 @@ func Mutate(tapSvcName string) webhook.Handler {
return nil, err
}
params := Params{
ProxyIndex: webhook.GetProxyContainerIndex(pod.Spec.Containers),
ProxyPath: webhook.GetProxyContainerPath(pod.Spec),
ProxyTapSvcName: tapSvcName,
}
if params.ProxyIndex < 0 || vizLabels.IsTapEnabled(pod) {
if params.ProxyPath == "" || vizLabels.IsTapEnabled(pod) {
return admissionResponse, nil
}
namespace, err := k8sAPI.Get(k8s.NS, request.Namespace)