apiserver: add group+version to request metrics
Kubernetes-commit: ff6e028755163ef52ad135da9f5d800c5749632c
This commit is contained in:
Dr. Stefan Schimanski
Kubernetes Publisher
parent
3f030212ae
commit
0f8de8e61f
|
|
@ -176,6 +176,8 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
|
|||
return nil, err
|
||||
}
|
||||
|
||||
group, version := a.group.GroupVersion.Group, a.group.GroupVersion.Version
|
||||
|
||||
fqKindToRegister, err := GetResourceKind(a.group.GroupVersion, storage, a.group.Typer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -571,9 +573,9 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
|
|||
|
||||
if needOverride {
|
||||
// need change the reported verb
|
||||
handler = metrics.InstrumentRouteFunc(verbOverrider.OverrideMetricsVerb(action.Verb), resource, subresource, requestScope, handler)
|
||||
handler = metrics.InstrumentRouteFunc(verbOverrider.OverrideMetricsVerb(action.Verb), group, version, resource, subresource, requestScope, handler)
|
||||
} else {
|
||||
handler = metrics.InstrumentRouteFunc(action.Verb, resource, subresource, requestScope, handler)
|
||||
handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, handler)
|
||||
}
|
||||
|
||||
if a.enableAPIResponseCompression {
|
||||
|
|
@ -607,7 +609,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
|
|||
if isSubresource {
|
||||
doc = "list " + subresource + " of objects of kind " + kind
|
||||
}
|
||||
handler := metrics.InstrumentRouteFunc(action.Verb, resource, subresource, requestScope, restfulListResource(lister, watcher, reqScope, false, a.minRequestTimeout))
|
||||
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, restfulListResource(lister, watcher, reqScope, false, a.minRequestTimeout))
|
||||
if a.enableAPIResponseCompression {
|
||||
handler = genericfilters.RestfulWithCompression(handler)
|
||||
}
|
||||
|
|
@ -642,7 +644,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
|
|||
if isSubresource {
|
||||
doc = "replace " + subresource + " of the specified " + kind
|
||||
}
|
||||
handler := metrics.InstrumentRouteFunc(action.Verb, resource, subresource, requestScope, restfulUpdateResource(updater, reqScope, admit))
|
||||
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, restfulUpdateResource(updater, reqScope, admit))
|
||||
route := ws.PUT(action.Path).To(handler).
|
||||
Doc(doc).
|
||||
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
|
||||
|
|
@ -669,7 +671,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
|
|||
string(types.MergePatchType),
|
||||
string(types.StrategicMergePatchType),
|
||||
}
|
||||
handler := metrics.InstrumentRouteFunc(action.Verb, resource, subresource, requestScope, restfulPatchResource(patcher, reqScope, admit, supportedTypes))
|
||||
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, restfulPatchResource(patcher, reqScope, admit, supportedTypes))
|
||||
route := ws.PATCH(action.Path).To(handler).
|
||||
Doc(doc).
|
||||
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
|
||||
|
|
@ -691,7 +693,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
|
|||
} else {
|
||||
handler = restfulCreateResource(creater, reqScope, admit)
|
||||
}
|
||||
handler = metrics.InstrumentRouteFunc(action.Verb, resource, subresource, requestScope, handler)
|
||||
handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, handler)
|
||||
article := getArticleForNoun(kind, " ")
|
||||
doc := "create" + article + kind
|
||||
if isSubresource {
|
||||
|
|
@ -720,7 +722,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
|
|||
if isSubresource {
|
||||
doc = "delete " + subresource + " of" + article + kind
|
||||
}
|
||||
handler := metrics.InstrumentRouteFunc(action.Verb, resource, subresource, requestScope, restfulDeleteResource(gracefulDeleter, isGracefulDeleter, reqScope, admit))
|
||||
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, restfulDeleteResource(gracefulDeleter, isGracefulDeleter, reqScope, admit))
|
||||
route := ws.DELETE(action.Path).To(handler).
|
||||
Doc(doc).
|
||||
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
|
||||
|
|
@ -743,7 +745,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
|
|||
if isSubresource {
|
||||
doc = "delete collection of " + subresource + " of a " + kind
|
||||
}
|
||||
handler := metrics.InstrumentRouteFunc(action.Verb, resource, subresource, requestScope, restfulDeleteCollection(collectionDeleter, isCollectionDeleter, reqScope, admit))
|
||||
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, restfulDeleteCollection(collectionDeleter, isCollectionDeleter, reqScope, admit))
|
||||
route := ws.DELETE(action.Path).To(handler).
|
||||
Doc(doc).
|
||||
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
|
||||
|
|
@ -763,7 +765,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
|
|||
doc = "watch changes to " + subresource + " of an object of kind " + kind
|
||||
}
|
||||
doc += ". deprecated: use the 'watch' parameter with a list operation instead, filtered to a single item with the 'fieldSelector' parameter."
|
||||
handler := metrics.InstrumentRouteFunc(action.Verb, resource, subresource, requestScope, restfulListResource(lister, watcher, reqScope, true, a.minRequestTimeout))
|
||||
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, restfulListResource(lister, watcher, reqScope, true, a.minRequestTimeout))
|
||||
route := ws.GET(action.Path).To(handler).
|
||||
Doc(doc).
|
||||
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
|
||||
|
|
@ -783,7 +785,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
|
|||
doc = "watch individual changes to a list of " + subresource + " of " + kind
|
||||
}
|
||||
doc += ". deprecated: use the 'watch' parameter with a list operation instead."
|
||||
handler := metrics.InstrumentRouteFunc(action.Verb, resource, subresource, requestScope, restfulListResource(lister, watcher, reqScope, true, a.minRequestTimeout))
|
||||
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, restfulListResource(lister, watcher, reqScope, true, a.minRequestTimeout))
|
||||
route := ws.GET(action.Path).To(handler).
|
||||
Doc(doc).
|
||||
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
|
||||
|
|
@ -806,7 +808,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
|
|||
if isSubresource {
|
||||
doc = "connect " + method + " requests to " + subresource + " of " + kind
|
||||
}
|
||||
handler := metrics.InstrumentRouteFunc(action.Verb, resource, subresource, requestScope, restfulConnectResource(connecter, reqScope, admit, path, isSubresource))
|
||||
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, restfulConnectResource(connecter, reqScope, admit, path, isSubresource))
|
||||
route := ws.Method(method).Path(action.Path).
|
||||
To(handler).
|
||||
Doc(doc).
|
||||
|
|
|
|||
|
|
@ -46,35 +46,35 @@ var (
|
|||
requestCounter = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "apiserver_request_count",
|
||||
Help: "Counter of apiserver requests broken out for each verb, API resource, client, and HTTP response contentType and code.",
|
||||
Help: "Counter of apiserver requests broken out for each verb, group, version, resource, scope, client, and HTTP response contentType and code.",
|
||||
},
|
||||
[]string{"verb", "resource", "subresource", "scope", "client", "contentType", "code"},
|
||||
[]string{"verb", "group", "version", "resource", "subresource", "scope", "client", "contentType", "code"},
|
||||
)
|
||||
longRunningRequestGauge = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "apiserver_longrunning_gauge",
|
||||
Help: "Gauge of all active long-running apiserver requests broken out by verb, API resource, and scope. Not all requests are tracked this way.",
|
||||
Help: "Gauge of all active long-running apiserver requests broken out by verb, group, version, resource, and scope. Not all requests are tracked this way.",
|
||||
},
|
||||
[]string{"verb", "resource", "subresource", "scope"},
|
||||
[]string{"verb", "group", "version", "resource", "subresource", "scope"},
|
||||
)
|
||||
requestLatencies = prometheus.NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
Name: "apiserver_request_latencies",
|
||||
Help: "Response latency distribution in microseconds for each verb, resource and subresource.",
|
||||
Help: "Response latency distribution in microseconds for each verb, group, version, resource, subresource, and scope.",
|
||||
// Use buckets ranging from 125 ms to 8 seconds.
|
||||
Buckets: prometheus.ExponentialBuckets(125000, 2.0, 7),
|
||||
},
|
||||
[]string{"verb", "resource", "subresource", "scope"},
|
||||
[]string{"verb", "group", "version", "resource", "subresource", "scope"},
|
||||
)
|
||||
requestLatenciesSummary = prometheus.NewSummaryVec(
|
||||
prometheus.SummaryOpts{
|
||||
Name: "apiserver_request_latencies_summary",
|
||||
Help: "Response latency summary in microseconds for each verb, resource and subresource.",
|
||||
Help: "Response latency summary in microseconds for each verb, group, version, resource, subresource, and scope.",
|
||||
// Make the sliding window of 5h.
|
||||
// TODO: The value for this should be based on our SLI definition (medium term).
|
||||
MaxAge: 5 * time.Hour,
|
||||
},
|
||||
[]string{"verb", "resource", "subresource", "scope"},
|
||||
[]string{"verb", "group", "version", "resource", "subresource", "scope"},
|
||||
)
|
||||
responseSizes = prometheus.NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
|
|
@ -83,7 +83,7 @@ var (
|
|||
// Use buckets ranging from 1000 bytes (1KB) to 10^9 bytes (1GB).
|
||||
Buckets: prometheus.ExponentialBuckets(1000, 10.0, 7),
|
||||
},
|
||||
[]string{"verb", "resource", "subresource", "scope"},
|
||||
[]string{"verb", "group", "version", "resource", "subresource", "scope"},
|
||||
)
|
||||
// DroppedRequests is a number of requests dropped with 'Try again later' response"
|
||||
DroppedRequests = prometheus.NewCounterVec(
|
||||
|
|
@ -163,9 +163,9 @@ func Record(req *http.Request, requestInfo *request.RequestInfo, contentType str
|
|||
}
|
||||
scope := CleanScope(requestInfo)
|
||||
if requestInfo.IsResourceRequest {
|
||||
MonitorRequest(req, strings.ToUpper(requestInfo.Verb), requestInfo.Resource, requestInfo.Subresource, scope, contentType, code, responseSizeInBytes, elapsed)
|
||||
MonitorRequest(req, strings.ToUpper(requestInfo.Verb), requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope, contentType, code, responseSizeInBytes, elapsed)
|
||||
} else {
|
||||
MonitorRequest(req, strings.ToUpper(requestInfo.Verb), "", requestInfo.Path, scope, contentType, code, responseSizeInBytes, elapsed)
|
||||
MonitorRequest(req, strings.ToUpper(requestInfo.Verb), "", "", "", requestInfo.Path, scope, contentType, code, responseSizeInBytes, elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -179,9 +179,9 @@ func RecordLongRunning(req *http.Request, requestInfo *request.RequestInfo, fn f
|
|||
scope := CleanScope(requestInfo)
|
||||
reportedVerb := cleanVerb(strings.ToUpper(requestInfo.Verb), req)
|
||||
if requestInfo.IsResourceRequest {
|
||||
g = longRunningRequestGauge.WithLabelValues(reportedVerb, requestInfo.Resource, requestInfo.Subresource, scope)
|
||||
g = longRunningRequestGauge.WithLabelValues(reportedVerb, requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope)
|
||||
} else {
|
||||
g = longRunningRequestGauge.WithLabelValues(reportedVerb, "", requestInfo.Path, scope)
|
||||
g = longRunningRequestGauge.WithLabelValues(reportedVerb, "", "", "", requestInfo.Path, scope)
|
||||
}
|
||||
g.Inc()
|
||||
defer g.Dec()
|
||||
|
|
@ -190,22 +190,22 @@ func RecordLongRunning(req *http.Request, requestInfo *request.RequestInfo, fn f
|
|||
|
||||
// MonitorRequest handles standard transformations for client and the reported verb and then invokes Monitor to record
|
||||
// a request. verb must be uppercase to be backwards compatible with existing monitoring tooling.
|
||||
func MonitorRequest(req *http.Request, verb, resource, subresource, scope, contentType string, httpCode, respSize int, elapsed time.Duration) {
|
||||
func MonitorRequest(req *http.Request, verb, group, version, resource, subresource, scope, contentType string, httpCode, respSize int, elapsed time.Duration) {
|
||||
reportedVerb := cleanVerb(verb, req)
|
||||
client := cleanUserAgent(utilnet.GetHTTPClient(req))
|
||||
elapsedMicroseconds := float64(elapsed / time.Microsecond)
|
||||
requestCounter.WithLabelValues(reportedVerb, resource, subresource, scope, client, contentType, codeToString(httpCode)).Inc()
|
||||
requestLatencies.WithLabelValues(reportedVerb, resource, subresource, scope).Observe(elapsedMicroseconds)
|
||||
requestLatenciesSummary.WithLabelValues(reportedVerb, resource, subresource, scope).Observe(elapsedMicroseconds)
|
||||
requestCounter.WithLabelValues(reportedVerb, group, version, resource, subresource, scope, client, contentType, codeToString(httpCode)).Inc()
|
||||
requestLatencies.WithLabelValues(reportedVerb, group, version, resource, subresource, scope).Observe(elapsedMicroseconds)
|
||||
requestLatenciesSummary.WithLabelValues(reportedVerb, group, version, resource, subresource, scope).Observe(elapsedMicroseconds)
|
||||
// We are only interested in response sizes of read requests.
|
||||
if verb == "GET" || verb == "LIST" {
|
||||
responseSizes.WithLabelValues(reportedVerb, resource, subresource, scope).Observe(float64(respSize))
|
||||
responseSizes.WithLabelValues(reportedVerb, group, version, resource, subresource, scope).Observe(float64(respSize))
|
||||
}
|
||||
}
|
||||
|
||||
// InstrumentRouteFunc works like Prometheus' InstrumentHandlerFunc but wraps
|
||||
// the go-restful RouteFunction instead of a HandlerFunc plus some Kubernetes endpoint specific information.
|
||||
func InstrumentRouteFunc(verb, resource, subresource, scope string, routeFunc restful.RouteFunction) restful.RouteFunction {
|
||||
func InstrumentRouteFunc(verb, group, version, resource, subresource, scope string, routeFunc restful.RouteFunction) restful.RouteFunction {
|
||||
return restful.RouteFunction(func(request *restful.Request, response *restful.Response) {
|
||||
now := time.Now()
|
||||
|
||||
|
|
@ -224,12 +224,12 @@ func InstrumentRouteFunc(verb, resource, subresource, scope string, routeFunc re
|
|||
|
||||
routeFunc(request, response)
|
||||
|
||||
MonitorRequest(request.Request, verb, resource, subresource, scope, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(now))
|
||||
MonitorRequest(request.Request, verb, group, version, resource, subresource, scope, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(now))
|
||||
})
|
||||
}
|
||||
|
||||
// InstrumentHandlerFunc works like Prometheus' InstrumentHandlerFunc but adds some Kubernetes endpoint specific information.
|
||||
func InstrumentHandlerFunc(verb, resource, subresource, scope string, handler http.HandlerFunc) http.HandlerFunc {
|
||||
func InstrumentHandlerFunc(verb, group, version, resource, subresource, scope string, handler http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, req *http.Request) {
|
||||
now := time.Now()
|
||||
|
||||
|
|
@ -246,7 +246,7 @@ func InstrumentHandlerFunc(verb, resource, subresource, scope string, handler ht
|
|||
|
||||
handler(w, req)
|
||||
|
||||
MonitorRequest(req, verb, resource, subresource, scope, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(now))
|
||||
MonitorRequest(req, verb, group, version, resource, subresource, scope, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(now))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue