From 6bea77d89b8759a0fdd1a49626abbe3c6f15c696 Mon Sep 17 00:00:00 2001 From: Matei David Date: Wed, 10 May 2023 16:27:37 +0100 Subject: [PATCH] Add cache configuration annotation support (#10871) The proxy caches discovery results in-memory. Linkerd supports overriding the default eviction timeout for cached discovery results through install (i.e. helm) values. However, it is currently not possible to configure timeouts on a workload-per-workload basis, or to configure the values after Linkerd has been installed (or upgraded). This change adds support for annotation based configuration. Workloads and namespaces now support two new configuration annotations that will override the install values when specified. Additionally, a typo has been fixed on the internal type representation. This is not a breaking change since the type itself is not exposed to users and is parsed correctly in the values.yaml file (or CLI) Signed-off-by: Matei David Co-authored-by: Eliza Weisman --- cli/cmd/doc.go | 8 +++ pkg/charts/linkerd2/values.go | 2 +- pkg/charts/linkerd2/values_test.go | 2 +- pkg/inject/inject.go | 20 ++++++++ pkg/inject/inject_test.go | 78 ++++++++++++++++++------------ pkg/k8s/labels.go | 8 +++ 6 files changed, 84 insertions(+), 34 deletions(-) diff --git a/cli/cmd/doc.go b/cli/cmd/doc.go index 4947a089c..592a9e620 100644 --- a/cli/cmd/doc.go +++ b/cli/cmd/doc.go @@ -244,6 +244,14 @@ func generateAnnotationsDocs() []annotationDoc { Name: k8s.ProxyInboundConnectTimeout, Description: "Inbound TCP connection timeout in the proxy", }, + { + Name: k8s.ProxyOutboundDiscoveryCacheUnusedTimeout, + Description: "Maximum time allowed before an unused outbound discovery result is evicted from the cache. Defaults to `5s`", + }, + { + Name: k8s.ProxyInboundDiscoveryCacheUnusedTimeout, + Description: "Maximum time allowed before an unused inbound discovery result is evicted from the cache. Defaults to `90s`", + }, { Name: k8s.ProxyWaitBeforeExitSecondsAnnotation, Description: "The proxy sidecar will stay alive for at least the given period after receiving SIGTERM signal from Kubernetes but no longer than pod's `terminationGracePeriodSeconds`. Defaults to `0`", diff --git a/pkg/charts/linkerd2/values.go b/pkg/charts/linkerd2/values.go index 292f38595..370ec9232 100644 --- a/pkg/charts/linkerd2/values.go +++ b/pkg/charts/linkerd2/values.go @@ -110,7 +110,7 @@ type ( OutboundConnectTimeout string `json:"outboundConnectTimeout"` InboundConnectTimeout string `json:"inboundConnectTimeout"` OutboundDiscoveryCacheUnusedTimeout string `json:"outboundDiscoveryCacheUnusedTimeout"` - InboundDiscoveryCacheUusedTimeout string `json:"inboundDiscoveryCacheUnusedTimeout"` + InboundDiscoveryCacheUnusedTimeout string `json:"inboundDiscoveryCacheUnusedTimeout"` PodInboundPorts string `json:"podInboundPorts"` OpaquePorts string `json:"opaquePorts"` Await bool `json:"await"` diff --git a/pkg/charts/linkerd2/values_test.go b/pkg/charts/linkerd2/values_test.go index 4714d86c7..d9101d616 100644 --- a/pkg/charts/linkerd2/values_test.go +++ b/pkg/charts/linkerd2/values_test.go @@ -131,7 +131,7 @@ func TestNewValues(t *testing.T) { Await: true, DefaultInboundPolicy: "all-unauthenticated", OutboundDiscoveryCacheUnusedTimeout: "5s", - InboundDiscoveryCacheUusedTimeout: "90s", + InboundDiscoveryCacheUnusedTimeout: "90s", }, ProxyInit: &ProxyInit{ IptablesMode: "legacy", diff --git a/pkg/inject/inject.go b/pkg/inject/inject.go index 2437d4850..edf7f5461 100644 --- a/pkg/inject/inject.go +++ b/pkg/inject/inject.go @@ -73,6 +73,8 @@ var ( k8s.ProxySkipSubnetsAnnotation, k8s.ProxyAccessLogAnnotation, k8s.ProxyShutdownGracePeriodAnnotation, + k8s.ProxyOutboundDiscoveryCacheUnusedTimeout, + k8s.ProxyInboundDiscoveryCacheUnusedTimeout, } // ProxyAlphaConfigAnnotations is the list of all alpha configuration // (config.alpha prefix) that can be applied to a pod or namespace. @@ -934,6 +936,24 @@ func (conf *ResourceConfig) applyAnnotationOverrides(values *l5dcharts.Values) { } } + if override, ok := annotations[k8s.ProxyOutboundDiscoveryCacheUnusedTimeout]; ok { + duration, err := time.ParseDuration(override) + if err != nil { + log.Warnf("unrecognized duration value used on pod annotation %s: %s", k8s.ProxyOutboundDiscoveryCacheUnusedTimeout, err.Error()) + } else { + values.Proxy.OutboundDiscoveryCacheUnusedTimeout = fmt.Sprintf("%ds", int(duration.Seconds())) + } + } + + if override, ok := annotations[k8s.ProxyInboundDiscoveryCacheUnusedTimeout]; ok { + duration, err := time.ParseDuration(override) + if err != nil { + log.Warnf("unrecognized duration value used on pod annotation %s: %s", k8s.ProxyInboundDiscoveryCacheUnusedTimeout, err.Error()) + } else { + values.Proxy.InboundDiscoveryCacheUnusedTimeout = fmt.Sprintf("%ds", int(duration.Seconds())) + } + } + if override, ok := annotations[k8s.ProxyShutdownGracePeriodAnnotation]; ok { duration, err := time.ParseDuration(override) if err != nil { diff --git a/pkg/inject/inject_test.go b/pkg/inject/inject_test.go index 451f9eb65..3f8599c07 100644 --- a/pkg/inject/inject_test.go +++ b/pkg/inject/inject_test.go @@ -70,6 +70,8 @@ func TestGetOverriddenValues(t *testing.T) { k8s.ProxySkipSubnetsAnnotation: "172.17.0.0/16", k8s.ProxyAccessLogAnnotation: "apache", k8s.ProxyShutdownGracePeriodAnnotation: "30s", + k8s.ProxyOutboundDiscoveryCacheUnusedTimeout: "50000ms", + k8s.ProxyInboundDiscoveryCacheUnusedTimeout: "900s", }, }, Spec: corev1.PodSpec{}, @@ -118,6 +120,8 @@ func TestGetOverriddenValues(t *testing.T) { values.Proxy.Await = true values.Proxy.AccessLog = "apache" values.Proxy.ShutdownGracePeriod = "30000ms" + values.Proxy.OutboundDiscoveryCacheUnusedTimeout = "50s" + values.Proxy.InboundDiscoveryCacheUnusedTimeout = "900s" return values }, }, @@ -136,32 +140,34 @@ func TestGetOverriddenValues(t *testing.T) { }, {id: "use namespace overrides", nsAnnotations: map[string]string{ - k8s.ProxyImageAnnotation: "cr.l5d.io/linkerd/proxy", - k8s.ProxyImagePullPolicyAnnotation: pullPolicy, - k8s.ProxyInitImageAnnotation: "cr.l5d.io/linkerd/proxy-init", - k8s.ProxyControlPortAnnotation: "4000", - k8s.ProxyInboundPortAnnotation: "5000", - k8s.ProxyAdminPortAnnotation: "5001", - k8s.ProxyOutboundPortAnnotation: "5002", - k8s.ProxyPodInboundPortsAnnotation: "1234,5678", - k8s.ProxyIgnoreInboundPortsAnnotation: "4222,6222", - k8s.ProxyIgnoreOutboundPortsAnnotation: "8079,8080", - k8s.ProxyCPURequestAnnotation: "0.15", - k8s.ProxyMemoryRequestAnnotation: "120", - k8s.ProxyCPULimitAnnotation: "1.5", - k8s.ProxyMemoryLimitAnnotation: "256", - k8s.ProxyUIDAnnotation: "8500", - k8s.ProxyLogLevelAnnotation: "debug,linkerd=debug", - k8s.ProxyLogFormatAnnotation: "json", - k8s.ProxyEnableExternalProfilesAnnotation: "false", - k8s.ProxyVersionOverrideAnnotation: proxyVersionOverride, - k8s.ProxyWaitBeforeExitSecondsAnnotation: "123", - k8s.ProxyOutboundConnectTimeout: "6000ms", - k8s.ProxyInboundConnectTimeout: "600ms", - k8s.ProxyOpaquePortsAnnotation: "4320-4325,3306", - k8s.ProxyAwait: "enabled", - k8s.ProxyAccessLogAnnotation: "apache", - k8s.ProxyInjectAnnotation: "ingress", + k8s.ProxyImageAnnotation: "cr.l5d.io/linkerd/proxy", + k8s.ProxyImagePullPolicyAnnotation: pullPolicy, + k8s.ProxyInitImageAnnotation: "cr.l5d.io/linkerd/proxy-init", + k8s.ProxyControlPortAnnotation: "4000", + k8s.ProxyInboundPortAnnotation: "5000", + k8s.ProxyAdminPortAnnotation: "5001", + k8s.ProxyOutboundPortAnnotation: "5002", + k8s.ProxyPodInboundPortsAnnotation: "1234,5678", + k8s.ProxyIgnoreInboundPortsAnnotation: "4222,6222", + k8s.ProxyIgnoreOutboundPortsAnnotation: "8079,8080", + k8s.ProxyCPURequestAnnotation: "0.15", + k8s.ProxyMemoryRequestAnnotation: "120", + k8s.ProxyCPULimitAnnotation: "1.5", + k8s.ProxyMemoryLimitAnnotation: "256", + k8s.ProxyUIDAnnotation: "8500", + k8s.ProxyLogLevelAnnotation: "debug,linkerd=debug", + k8s.ProxyLogFormatAnnotation: "json", + k8s.ProxyEnableExternalProfilesAnnotation: "false", + k8s.ProxyVersionOverrideAnnotation: proxyVersionOverride, + k8s.ProxyWaitBeforeExitSecondsAnnotation: "123", + k8s.ProxyOutboundConnectTimeout: "6000ms", + k8s.ProxyInboundConnectTimeout: "600ms", + k8s.ProxyOpaquePortsAnnotation: "4320-4325,3306", + k8s.ProxyAwait: "enabled", + k8s.ProxyAccessLogAnnotation: "apache", + k8s.ProxyInjectAnnotation: "ingress", + k8s.ProxyOutboundDiscoveryCacheUnusedTimeout: "50s", + k8s.ProxyInboundDiscoveryCacheUnusedTimeout: "6000ms", }, spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ @@ -205,13 +211,17 @@ func TestGetOverriddenValues(t *testing.T) { values.Proxy.Await = true values.Proxy.AccessLog = "apache" values.Proxy.IsIngress = true + values.Proxy.OutboundDiscoveryCacheUnusedTimeout = "50s" + values.Proxy.InboundDiscoveryCacheUnusedTimeout = "6s" return values }, }, - {id: "use invalid duration for TCP connect timeouts", + {id: "use invalid duration for proxy timeouts", nsAnnotations: map[string]string{ - k8s.ProxyOutboundConnectTimeout: "6000", - k8s.ProxyInboundConnectTimeout: "600", + k8s.ProxyOutboundConnectTimeout: "6000", + k8s.ProxyInboundConnectTimeout: "600", + k8s.ProxyOutboundDiscoveryCacheUnusedTimeout: "50", + k8s.ProxyInboundDiscoveryCacheUnusedTimeout: "5000", }, spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ @@ -224,11 +234,13 @@ func TestGetOverriddenValues(t *testing.T) { return values }, }, - {id: "use valid duration for TCP connect timeouts", + {id: "use valid duration for proxy timeouts", nsAnnotations: map[string]string{ // Validate we're converting time values into ms for the proxy to parse correctly. - k8s.ProxyOutboundConnectTimeout: "6s5ms", - k8s.ProxyInboundConnectTimeout: "2s5ms", + k8s.ProxyOutboundConnectTimeout: "6s5ms", + k8s.ProxyInboundConnectTimeout: "2s5ms", + k8s.ProxyOutboundDiscoveryCacheUnusedTimeout: "6s5000ms", + k8s.ProxyInboundDiscoveryCacheUnusedTimeout: "6s5000ms", }, spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ @@ -240,6 +252,8 @@ func TestGetOverriddenValues(t *testing.T) { values, _ := l5dcharts.NewValues() values.Proxy.OutboundConnectTimeout = "6005ms" values.Proxy.InboundConnectTimeout = "2005ms" + values.Proxy.OutboundDiscoveryCacheUnusedTimeout = "11s" + values.Proxy.InboundDiscoveryCacheUnusedTimeout = "11s" return values }, }, diff --git a/pkg/k8s/labels.go b/pkg/k8s/labels.go index 707573f24..13205f983 100644 --- a/pkg/k8s/labels.go +++ b/pkg/k8s/labels.go @@ -230,6 +230,14 @@ const ( // timeout in the proxy ProxyInboundConnectTimeout = ProxyConfigAnnotationsPrefix + "/proxy-inbound-connect-timeout" + // ProxyOutboundDiscoveryCacheTimeout can be used to configure the timeout + // that will evict unused outbound discovery results + ProxyOutboundDiscoveryCacheUnusedTimeout = ProxyConfigAnnotationsPrefix + "/proxy-outbound-discovery-cache-unused-timeout" + + // ProxyInboundDiscoveryCacheUnusedTimeout can be used to configure the timeout + // that will evict unused inbound discovery results + ProxyInboundDiscoveryCacheUnusedTimeout = ProxyConfigAnnotationsPrefix + "/proxy-inbound-discovery-cache-unused-timeout" + // ProxyEnableGatewayAnnotation can be used to configure the proxy // to operate as a gateway, routing requests that target the inbound router. ProxyEnableGatewayAnnotation = ProxyConfigAnnotationsPrefix + "/enable-gateway"