diff --git a/pkg/endpoints/metrics/metrics.go b/pkg/endpoints/metrics/metrics.go index 31d1c8a07..af09ef50b 100644 --- a/pkg/endpoints/metrics/metrics.go +++ b/pkg/endpoints/metrics/metrics.go @@ -177,7 +177,7 @@ var ( }, []string{"group", "version", "kind"}, ) - // Because of volatality of the base metric this is pre-aggregated one. Instead of reporing current usage all the time + // Because of volatility of the base metric this is pre-aggregated one. Instead of reporting current usage all the time // it reports maximal usage during the last second. currentInflightRequests = compbasemetrics.NewGaugeVec( &compbasemetrics.GaugeOpts{ @@ -187,6 +187,15 @@ var ( }, []string{"requestKind"}, ) + + requestErrorTotal = compbasemetrics.NewCounterVec( + &compbasemetrics.CounterOpts{ + Name: "apiserver_request_error_total", + Help: "Number of requests which have resulted in an apiserver response error", + StabilityLevel: compbasemetrics.ALPHA, + }, + []string{"verb", "group", "version", "resource", "subresource", "scope", "component", "code"}, + ) kubectlExeRegexp = regexp.MustCompile(`^.*((?i:kubectl\.exe))`) metrics = []resettableCollector{ @@ -203,6 +212,7 @@ var ( WatchEvents, WatchEventsSizes, currentInflightRequests, + requestErrorTotal, } ) @@ -236,10 +246,9 @@ func UpdateInflightRequestMetrics(nonmutating, mutating int) { currentInflightRequests.WithLabelValues(MutatingKind).Set(float64(mutating)) } -// Record records a single request to the standard metrics endpoints. For use by handlers that perform their own -// processing. All API paths should use InstrumentRouteFunc implicitly. Use this instead of MonitorRequest if -// you already have a RequestInfo object. -func Record(req *http.Request, requestInfo *request.RequestInfo, component, contentType string, code int, responseSizeInBytes int, elapsed time.Duration) { +// RecordRequestError records the occurrence of a request error during apiserver handling (e.g. timeouts, +// maxinflight throttling, proxyHandler errors). +func RecordRequestError(req *http.Request, requestInfo *request.RequestInfo, component string, code int) { if requestInfo == nil { requestInfo = &request.RequestInfo{Verb: req.Method, Path: req.URL.Path} } @@ -251,9 +260,9 @@ func Record(req *http.Request, requestInfo *request.RequestInfo, component, cont // However, we need to tweak it e.g. to differentiate GET from LIST. verb := canonicalVerb(strings.ToUpper(req.Method), scope) if requestInfo.IsResourceRequest { - MonitorRequest(req, verb, requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope, component, contentType, code, responseSizeInBytes, elapsed) + requestErrorTotal.WithLabelValues(cleanVerb(verb, req), requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope, component, codeToString(code)).Inc() } else { - MonitorRequest(req, verb, "", "", "", requestInfo.Path, scope, component, contentType, code, responseSizeInBytes, elapsed) + requestErrorTotal.WithLabelValues(cleanVerb(verb, req), "", "", "", requestInfo.Path, scope, component, codeToString(code)).Inc() } } diff --git a/pkg/server/filters/maxinflight.go b/pkg/server/filters/maxinflight.go index f7bc691c7..d77676b0a 100644 --- a/pkg/server/filters/maxinflight.go +++ b/pkg/server/filters/maxinflight.go @@ -178,7 +178,7 @@ func WithMaxInFlightLimit( } } } - metrics.Record(r, requestInfo, metrics.APIServerComponent, "", http.StatusTooManyRequests, 0, 0) + metrics.RecordRequestError(r, requestInfo, metrics.APIServerComponent, http.StatusTooManyRequests) tooManyRequests(r, w) } } diff --git a/pkg/server/filters/timeout.go b/pkg/server/filters/timeout.go index 23ac0034d..251597584 100644 --- a/pkg/server/filters/timeout.go +++ b/pkg/server/filters/timeout.go @@ -59,7 +59,7 @@ func WithTimeoutForNonLongRunningRequests(handler http.Handler, longRunning apir postTimeoutFn := func() { cancel() - metrics.Record(req, requestInfo, metrics.APIServerComponent, "", http.StatusGatewayTimeout, 0, 0) + metrics.RecordRequestError(req, requestInfo, metrics.APIServerComponent, http.StatusGatewayTimeout) } return req, time.After(timeout), postTimeoutFn, apierrors.NewTimeoutError(fmt.Sprintf("request did not complete within %s", timeout), 0) }