kubernetes
/
apiserver
mirror of https://github.com/kubernetes/apiserver.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #83427 from logicalhan/metrics-timeout 

Fix double counting issue for request metrics on timeout.

Kubernetes-commit: aa25739da416b68e966bef3c15e4f5281ebd1e85
This commit is contained in:
Kubernetes Publisher 2019-10-21 21:11:22 -07:00
parent 550b75f0da 00d376192a
commit 7b8c488e22
6 changed files with 24 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Godeps/Godeps.json generated
View File

@ -524,7 +524,7 @@
},
{
"ImportPath": "k8s.io/apimachinery",
"Rev": "6e68a40eebf9"
"Rev": "6c8691705fc5"
},
{
"ImportPath": "k8s.io/client-go",

4
go.mod
View File

@ -50,7 +50,7 @@ require (
gopkg.in/yaml.v2 v2.2.4
gotest.tools v2.2.0+incompatible // indirect
k8s.io/api v0.0.0-20191016225839-816a9b7df678
k8s.io/apimachinery v0.0.0-20191017185446-6e68a40eebf9
k8s.io/apimachinery v0.0.0-20191020214737-6c8691705fc5
k8s.io/client-go v0.0.0-20191016230210-14c42cd304d9
k8s.io/component-base v0.0.0-20191016230640-d338b9159fb6
k8s.io/klog v1.0.0
@ -69,7 +69,7 @@ replace (
golang.org/x/text => golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db
golang.org/x/time => golang.org/x/time v0.0.0-20161028155119-f51c12702a4d
k8s.io/api => k8s.io/api v0.0.0-20191016225839-816a9b7df678
k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20191017185446-6e68a40eebf9
k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20191020214737-6c8691705fc5
k8s.io/client-go => k8s.io/client-go v0.0.0-20191016230210-14c42cd304d9
k8s.io/component-base => k8s.io/component-base v0.0.0-20191016230640-d338b9159fb6
)

2
go.sum
View File

@ -297,7 +297,7 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
k8s.io/api v0.0.0-20191016225839-816a9b7df678/go.mod h1:LZQaT8MvVpl7Bg2lYFcQm7+Mpdxq8p1NFl3yh+5DCwY=
k8s.io/apimachinery v0.0.0-20191017185446-6e68a40eebf9/go.mod h1:92mWDd8Ji2sw2157KIgino5wCxffA8KSvhW2oY4ypdw=
k8s.io/apimachinery v0.0.0-20191020214737-6c8691705fc5/go.mod h1:92mWDd8Ji2sw2157KIgino5wCxffA8KSvhW2oY4ypdw=
k8s.io/client-go v0.0.0-20191016230210-14c42cd304d9/go.mod h1:ct8FBj9BiF4WYNmJoE+SiuhAgSrFs9cyTE7icW+iVr4=
k8s.io/component-base v0.0.0-20191016230640-d338b9159fb6/go.mod h1:L2lcIF6P6N33EyqL0ntnoBvJ6t724ev4LzCc0yjn26g=
k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=

25
pkg/endpoints/metrics/metrics.go
View File

@ -177,7 +177,7 @@ var (
},
[]string{"group", "version", "kind"},
)
// Because of volatality of the base metric this is pre-aggregated one. Instead of reporing current usage all the time
// Because of volatility of the base metric this is pre-aggregated one. Instead of reporting current usage all the time
// it reports maximal usage during the last second.
currentInflightRequests = compbasemetrics.NewGaugeVec(
&compbasemetrics.GaugeOpts{
@ -187,6 +187,15 @@ var (
},
[]string{"requestKind"},
)
requestTerminationsTotal = compbasemetrics.NewCounterVec(
&compbasemetrics.CounterOpts{
Name: "apiserver_request_terminations_total",
Help: "Number of requests which apiserver terminated in self-defense.",
StabilityLevel: compbasemetrics.ALPHA,
},
[]string{"verb", "group", "version", "resource", "subresource", "scope", "component", "code"},
)
kubectlExeRegexp = regexp.MustCompile(`^.*((?i:kubectl\.exe))`)
metrics = []resettableCollector{
@ -203,6 +212,7 @@ var (
WatchEvents,
WatchEventsSizes,
currentInflightRequests,
requestTerminationsTotal,
}
)
@ -236,10 +246,11 @@ func UpdateInflightRequestMetrics(nonmutating, mutating int) {
currentInflightRequests.WithLabelValues(MutatingKind).Set(float64(mutating))
}
// Record records a single request to the standard metrics endpoints. For use by handlers that perform their own
// processing. All API paths should use InstrumentRouteFunc implicitly. Use this instead of MonitorRequest if
// you already have a RequestInfo object.
func Record(req *http.Request, requestInfo *request.RequestInfo, component, contentType string, code int, responseSizeInBytes int, elapsed time.Duration) {
// RecordRequestTermination records that the request was terminated early as part of a resource
// preservation or apiserver self-defense mechanism (e.g. timeouts, maxinflight throttling,
// proxyHandler errors). RecordRequestTermination should only be called zero or one times
// per request.
func RecordRequestTermination(req *http.Request, requestInfo *request.RequestInfo, component string, code int) {
if requestInfo == nil {
requestInfo = &request.RequestInfo{Verb: req.Method, Path: req.URL.Path}
}
@ -251,9 +262,9 @@ func Record(req *http.Request, requestInfo *request.RequestInfo, component, cont
// However, we need to tweak it e.g. to differentiate GET from LIST.
verb := canonicalVerb(strings.ToUpper(req.Method), scope)
if requestInfo.IsResourceRequest {
MonitorRequest(req, verb, requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope, component, contentType, code, responseSizeInBytes, elapsed)
requestTerminationsTotal.WithLabelValues(cleanVerb(verb, req), requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope, component, codeToString(code)).Inc()
} else {
MonitorRequest(req, verb, "", "", "", requestInfo.Path, scope, component, contentType, code, responseSizeInBytes, elapsed)
requestTerminationsTotal.WithLabelValues(cleanVerb(verb, req), "", "", "", requestInfo.Path, scope, component, codeToString(code)).Inc()
}
}

2
pkg/server/filters/maxinflight.go
View File

@ -178,7 +178,7 @@ func WithMaxInFlightLimit(
}
}
}
metrics.Record(r, requestInfo, metrics.APIServerComponent, "", http.StatusTooManyRequests, 0, 0)
metrics.RecordRequestTermination(r, requestInfo, metrics.APIServerComponent, http.StatusTooManyRequests)
tooManyRequests(r, w)
}
}

2
pkg/server/filters/timeout.go
View File

@ -59,7 +59,7 @@ func WithTimeoutForNonLongRunningRequests(handler http.Handler, longRunning apir
postTimeoutFn := func() {
cancel()
metrics.Record(req, requestInfo, metrics.APIServerComponent, "", http.StatusGatewayTimeout, 0, 0)
metrics.RecordRequestTermination(req, requestInfo, metrics.APIServerComponent, http.StatusGatewayTimeout)
}
return req, time.After(timeout), postTimeoutFn, apierrors.NewTimeoutError(fmt.Sprintf("request did not complete within %s", timeout), 0)
}