Merge pull request #127053 from dashpole/tracing_context_propagation

APIServerTracing: Respect trace context only for privileged users

Kubernetes-commit: b2b6c4d0235085d090ca73b62a1361302db81afd

pkg/endpoints/filters/traces.go

@@ -24,6 +24,7 @@ import (
 	"go.opentelemetry.io/otel/trace"
 	"k8s.io/apiserver/pkg/endpoints/request"
 
+	"k8s.io/apiserver/pkg/authentication/user"
 	tracing "k8s.io/component-base/tracing"
 )
 
@@ -31,7 +32,7 @@ import (
 func WithTracing(handler http.Handler, tp trace.TracerProvider) http.Handler {
 	opts := []otelhttp.Option{
 		otelhttp.WithPropagators(tracing.Propagators()),
-		otelhttp.WithPublicEndpoint(),
+		otelhttp.WithPublicEndpointFn(notSystemPrivilegedGroup),
 		otelhttp.WithTracerProvider(tp),
 		otelhttp.WithSpanNameFormatter(func(operation string, r *http.Request) string {
 			ctx := r.Context()
@@ -43,6 +44,11 @@ func WithTracing(handler http.Handler, tp trace.TracerProvider) http.Handler {
 		}),
 	}
 	wrappedHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Adjust otelhttp tracing start time to match the start time used
+		// for Prometheus metrics.
+		if startTime, ok := request.ReceivedTimestampFrom(r.Context()); ok {
+			r = r.WithContext(otelhttp.ContextWithStartTime(r.Context(), startTime))
+		}
 		// Add the http.target attribute to the otelhttp span
 		// Workaround for https://github.com/open-telemetry/opentelemetry-go-contrib/issues/3743
 		if r.URL != nil {
@@ -73,3 +79,14 @@ func getSpanNameFromRequestInfo(info *request.RequestInfo, r *http.Request) stri
 	}
 	return r.Method + " " + spanName
 }
+
+func notSystemPrivilegedGroup(req *http.Request) bool {
+	if u, ok := request.UserFrom(req.Context()); ok {
+		for _, group := range u.GetGroups() {
+			if group == user.SystemPrivilegedGroup || group == user.MonitoringGroup {
+				return false
+			}
+		}
+	}
+	return true
+}

pkg/server/config.go

@@ -1039,6 +1039,11 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
 	failedHandler := genericapifilters.Unauthorized(c.Serializer)
 	failedHandler = genericapifilters.WithFailedAuthenticationAudit(failedHandler, c.AuditBackend, c.AuditPolicyRuleEvaluator)
 
+	// WithTracing comes after authentication so we can allow authenticated
+	// clients to influence sampling.
+	if c.FeatureGate.Enabled(genericfeatures.APIServerTracing) {
+		handler = genericapifilters.WithTracing(handler, c.TracerProvider)
+	}
 	failedHandler = filterlatency.TrackCompleted(failedHandler)
 	handler = filterlatency.TrackCompleted(handler)
 	handler = genericapifilters.WithAuthentication(handler, c.Authentication.Authenticator, failedHandler, c.Authentication.APIAudiences, c.Authentication.RequestHeaderConfig)
@@ -1069,9 +1074,6 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
 		handler = genericfilters.WithRetryAfter(handler, c.lifecycleSignals.NotAcceptingNewRequest.Signaled())
 	}
 	handler = genericfilters.WithHTTPLogging(handler)
-	if c.FeatureGate.Enabled(genericfeatures.APIServerTracing) {
-		handler = genericapifilters.WithTracing(handler, c.TracerProvider)
-	}
 	handler = genericapifilters.WithLatencyTrackers(handler)
 	// WithRoutine will execute future handlers in a separate goroutine and serving
 	// handler in current goroutine to minimize the stack memory usage. It must be