kubernetes
/
apiserver
mirror of https://github.com/kubernetes/apiserver.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

migrate kube-apiserver metrics to stability framework 

Kubernetes-commit: 466980dd747e06e55451301c624eecccfa505123
This commit is contained in:
Han Kang 2019-08-22 15:38:42 -07:00 committed by Kubernetes Publisher
parent 3b38fd7ad4
commit b9084e350a
12 changed files with 242 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
pkg/admission/metrics/metrics.go
View File

@ -22,9 +22,9 @@ import (
"strconv" "strconv"
"time" "time"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/admission"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
) )
const ( const (
@ -154,33 +154,35 @@ func (m *AdmissionMetrics) ObserveWebhook(elapsed time.Duration, rejected bool,
} }
type metricSet struct { type metricSet struct {
latencies *prometheus.HistogramVec latencies *metrics.HistogramVec
latenciesSummary *prometheus.SummaryVec latenciesSummary *metrics.SummaryVec
} }
func newMetricSet(name string, labels []string, helpTemplate string, hasSummary bool) *metricSet { func newMetricSet(name string, labels []string, helpTemplate string, hasSummary bool) *metricSet {
var summary *prometheus.SummaryVec var summary *metrics.SummaryVec
if hasSummary { if hasSummary {
summary = prometheus.NewSummaryVec( summary = metrics.NewSummaryVec(
prometheus.SummaryOpts{ &metrics.SummaryOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: fmt.Sprintf("%s_admission_duration_seconds_summary", name), Name: fmt.Sprintf("%s_admission_duration_seconds_summary", name),
Help: fmt.Sprintf(helpTemplate, "latency summary in seconds"), Help: fmt.Sprintf(helpTemplate, "latency summary in seconds"),
MaxAge: latencySummaryMaxAge, MaxAge: latencySummaryMaxAge,
StabilityLevel: metrics.ALPHA,
}, },
labels, labels,
) )
} }
return &metricSet{ return &metricSet{
latencies: prometheus.NewHistogramVec( latencies: metrics.NewHistogramVec(
prometheus.HistogramOpts{ &metrics.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: fmt.Sprintf("%s_admission_duration_seconds", name), Name: fmt.Sprintf("%s_admission_duration_seconds", name),
Help: fmt.Sprintf(helpTemplate, "latency histogram in seconds"), Help: fmt.Sprintf(helpTemplate, "latency histogram in seconds"),
Buckets: latencyBuckets, Buckets: latencyBuckets,
StabilityLevel: metrics.ALPHA,
}, },
labels, labels,
), ),
@ -191,9 +193,9 @@ func newMetricSet(name string, labels []string, helpTemplate string, hasSummary
// MustRegister registers all the prometheus metrics in the metricSet. // MustRegister registers all the prometheus metrics in the metricSet.
func (m *metricSet) mustRegister() { func (m *metricSet) mustRegister() {
prometheus.MustRegister(m.latencies) legacyregistry.MustRegister(m.latencies)
if m.latenciesSummary != nil { if m.latenciesSummary != nil {
prometheus.MustRegister(m.latenciesSummary) legacyregistry.MustRegister(m.latenciesSummary)
} }
} }

43
pkg/audit/metrics.go
View File

@ -19,8 +19,9 @@ package audit
import ( import (
"fmt" "fmt"
"github.com/prometheus/client_golang/prometheus"
auditinternal "k8s.io/apiserver/pkg/apis/audit" auditinternal "k8s.io/apiserver/pkg/apis/audit"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
"k8s.io/klog" "k8s.io/klog"
) )
@ -29,45 +30,49 @@ const (
) )
var ( var (
eventCounter = prometheus.NewCounter( eventCounter = metrics.NewCounter(
prometheus.CounterOpts{ &metrics.CounterOpts{
Subsystem: subsystem, Subsystem: subsystem,
Name: "event_total", Name: "event_total",
Help: "Counter of audit events generated and sent to the audit backend.", Help: "Counter of audit events generated and sent to the audit backend.",
StabilityLevel: metrics.ALPHA,
}) })
errorCounter = prometheus.NewCounterVec( errorCounter = metrics.NewCounterVec(
prometheus.CounterOpts{ &metrics.CounterOpts{
Subsystem: subsystem, Subsystem: subsystem,
Name: "error_total", Name: "error_total",
Help: "Counter of audit events that failed to be audited properly. " + Help: "Counter of audit events that failed to be audited properly. " +
"Plugin identifies the plugin affected by the error.", "Plugin identifies the plugin affected by the error.",
StabilityLevel: metrics.ALPHA,
}, },
[]string{"plugin"}, []string{"plugin"},
) )
levelCounter = prometheus.NewCounterVec( levelCounter = metrics.NewCounterVec(
prometheus.CounterOpts{ &metrics.CounterOpts{
Subsystem: subsystem, Subsystem: subsystem,
Name: "level_total", Name: "level_total",
Help: "Counter of policy levels for audit events (1 per request).", Help: "Counter of policy levels for audit events (1 per request).",
StabilityLevel: metrics.ALPHA,
}, },
[]string{"level"}, []string{"level"},
) )
ApiserverAuditDroppedCounter = prometheus.NewCounter( ApiserverAuditDroppedCounter = metrics.NewCounter(
prometheus.CounterOpts{ &metrics.CounterOpts{
Subsystem: subsystem, Subsystem: subsystem,
Name: "requests_rejected_total", Name: "requests_rejected_total",
Help: "Counter of apiserver requests rejected due to an error " + Help: "Counter of apiserver requests rejected due to an error " +
"in audit logging backend.", "in audit logging backend.",
StabilityLevel: metrics.ALPHA,
}, },
) )
) )
func init() { func init() {
prometheus.MustRegister(eventCounter) legacyregistry.MustRegister(eventCounter)
prometheus.MustRegister(errorCounter) legacyregistry.MustRegister(errorCounter)
prometheus.MustRegister(levelCounter) legacyregistry.MustRegister(levelCounter)
prometheus.MustRegister(ApiserverAuditDroppedCounter) legacyregistry.MustRegister(ApiserverAuditDroppedCounter)
} }
// ObserveEvent updates the relevant prometheus metrics for the generated audit event. // ObserveEvent updates the relevant prometheus metrics for the generated audit event.

11
pkg/authentication/request/x509/x509.go
View File

@ -23,16 +23,16 @@ import (
"net/http" "net/http"
"time" "time"
"github.com/prometheus/client_golang/prometheus"
utilerrors "k8s.io/apimachinery/pkg/util/errors" utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authentication/user"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
) )
var clientCertificateExpirationHistogram = prometheus.NewHistogram( var clientCertificateExpirationHistogram = metrics.NewHistogram(
prometheus.HistogramOpts{ &metrics.HistogramOpts{
Namespace: "apiserver", Namespace: "apiserver",
Subsystem: "client", Subsystem: "client",
Name: "certificate_expiration_seconds", Name: "certificate_expiration_seconds",
@ -53,11 +53,12 @@ var clientCertificateExpirationHistogram = prometheus.NewHistogram(
(6 * 30 * 24 * time.Hour).Seconds(), (6 * 30 * 24 * time.Hour).Seconds(),
(12 * 30 * 24 * time.Hour).Seconds(), (12 * 30 * 24 * time.Hour).Seconds(),
}, },
StabilityLevel: metrics.ALPHA,
}, },
) )
func init() { func init() {
prometheus.MustRegister(clientCertificateExpirationHistogram) legacyregistry.MustRegister(clientCertificateExpirationHistogram)
} }
// UserConversion defines an interface for extracting user info from a client certificate chain // UserConversion defines an interface for extracting user info from a client certificate chain

17
pkg/endpoints/filters/authentication.go
View File

@ -21,29 +21,30 @@ import (
"net/http" "net/http"
"strings" "strings"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/klog"
apierrors "k8s.io/apimachinery/pkg/api/errors" apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
genericapirequest "k8s.io/apiserver/pkg/endpoints/request" genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
"k8s.io/klog"
) )
var ( var (
authenticatedUserCounter = prometheus.NewCounterVec( authenticatedUserCounter = metrics.NewCounterVec(
prometheus.CounterOpts{ &metrics.CounterOpts{
Name: "authenticated_user_requests", Name: "authenticated_user_requests",
Help: "Counter of authenticated requests broken out by username.", Help: "Counter of authenticated requests broken out by username.",
StabilityLevel: metrics.ALPHA,
}, },
[]string{"username"}, []string{"username"},
) )
) )
func init() { func init() {
prometheus.MustRegister(authenticatedUserCounter) legacyregistry.MustRegister(authenticatedUserCounter)
} }
// WithAuthentication creates an http handler that tries to authenticate the given request as a user, and then // WithAuthentication creates an http handler that tries to authenticate the given request as a user, and then

123
pkg/endpoints/metrics/metrics.go
View File

@ -27,6 +27,9 @@ import (
"sync" "sync"
"time" "time"
restful "github.com/emicklei/go-restful"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/apimachinery/pkg/apis/meta/v1/validation" "k8s.io/apimachinery/pkg/apis/meta/v1/validation"
"k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/types"
utilnet "k8s.io/apimachinery/pkg/util/net" utilnet "k8s.io/apimachinery/pkg/util/net"
@ -34,15 +37,14 @@ import (
"k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/features"
utilfeature "k8s.io/apiserver/pkg/util/feature" utilfeature "k8s.io/apiserver/pkg/util/feature"
compbasemetrics "k8s.io/component-base/metrics"
restful "github.com/emicklei/go-restful" "k8s.io/component-base/metrics/legacyregistry"
"github.com/prometheus/client_golang/prometheus"
) )
// resettableCollector is the interface implemented by prometheus.MetricVec // resettableCollector is the interface implemented by prometheus.MetricVec
// that can be used by Prometheus to collect metrics and reset their values. // that can be used by Prometheus to collect metrics and reset their values.
type resettableCollector interface { type resettableCollector interface {
prometheus.Collector compbasemetrics.Registerable
Reset() Reset()
} }
@ -53,10 +55,11 @@ const (
var ( var (
// TODO(a-robinson): Add unit tests for the handling of these metrics once // TODO(a-robinson): Add unit tests for the handling of these metrics once
// the upstream library supports it. // the upstream library supports it.
requestCounter = prometheus.NewCounterVec( requestCounter = compbasemetrics.NewCounterVec(
prometheus.CounterOpts{ &compbasemetrics.CounterOpts{
Name: "apiserver_request_total", Name: "apiserver_request_total",
Help: "Counter of apiserver requests broken out for each verb, dry run value, group, version, resource, scope, component, client, and HTTP response contentType and code.", Help: "Counter of apiserver requests broken out for each verb, dry run value, group, version, resource, scope, component, client, and HTTP response contentType and code.",
StabilityLevel: compbasemetrics.ALPHA,
}, },
// The label_name contentType doesn't follow the label_name convention defined here: // The label_name contentType doesn't follow the label_name convention defined here:
// https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/instrumentation.md // https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/instrumentation.md
@ -64,22 +67,24 @@ var (
// should be all lowercase and separated by underscores. // should be all lowercase and separated by underscores.
[]string{"verb", "dry_run", "group", "version", "resource", "subresource", "scope", "component", "client", "contentType", "code"}, []string{"verb", "dry_run", "group", "version", "resource", "subresource", "scope", "component", "client", "contentType", "code"},
) )
deprecatedRequestCounter = prometheus.NewCounterVec( deprecatedRequestCounter = compbasemetrics.NewCounterVec(
prometheus.CounterOpts{ &compbasemetrics.CounterOpts{
Name: "apiserver_request_count", Name: "apiserver_request_count",
Help: "(Deprecated) Counter of apiserver requests broken out for each verb, group, version, resource, scope, component, client, and HTTP response contentType and code.", Help: "(Deprecated) Counter of apiserver requests broken out for each verb, group, version, resource, scope, component, client, and HTTP response contentType and code.",
StabilityLevel: compbasemetrics.ALPHA,
}, },
[]string{"verb", "group", "version", "resource", "subresource", "scope", "component", "client", "contentType", "code"}, []string{"verb", "group", "version", "resource", "subresource", "scope", "component", "client", "contentType", "code"},
) )
longRunningRequestGauge = prometheus.NewGaugeVec( longRunningRequestGauge = compbasemetrics.NewGaugeVec(
prometheus.GaugeOpts{ &compbasemetrics.GaugeOpts{
Name: "apiserver_longrunning_gauge", Name: "apiserver_longrunning_gauge",
Help: "Gauge of all active long-running apiserver requests broken out by verb, group, version, resource, scope and component. Not all requests are tracked this way.", Help: "Gauge of all active long-running apiserver requests broken out by verb, group, version, resource, scope and component. Not all requests are tracked this way.",
StabilityLevel: compbasemetrics.ALPHA,
}, },
[]string{"verb", "group", "version", "resource", "subresource", "scope", "component"}, []string{"verb", "group", "version", "resource", "subresource", "scope", "component"},
) )
requestLatencies = prometheus.NewHistogramVec( requestLatencies = compbasemetrics.NewHistogramVec(
prometheus.HistogramOpts{ &compbasemetrics.HistogramOpts{
Name: "apiserver_request_duration_seconds", Name: "apiserver_request_duration_seconds",
Help: "Response latency distribution in seconds for each verb, dry run value, group, version, resource, subresource, scope and component.", Help: "Response latency distribution in seconds for each verb, dry run value, group, version, resource, subresource, scope and component.",
// This metric is used for verifying api call latencies SLO, // This metric is used for verifying api call latencies SLO,
@ -87,81 +92,91 @@ var (
// Thus we customize buckets significantly, to empower both usecases. // Thus we customize buckets significantly, to empower both usecases.
Buckets: []float64{0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, Buckets: []float64{0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0,
1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60}, 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60},
StabilityLevel: compbasemetrics.ALPHA,
}, },
[]string{"verb", "dry_run", "group", "version", "resource", "subresource", "scope", "component"}, []string{"verb", "dry_run", "group", "version", "resource", "subresource", "scope", "component"},
) )
deprecatedRequestLatencies = prometheus.NewHistogramVec( deprecatedRequestLatencies = compbasemetrics.NewHistogramVec(
prometheus.HistogramOpts{ &compbasemetrics.HistogramOpts{
Name: "apiserver_request_latencies", Name: "apiserver_request_latencies",
Help: "(Deprecated) Response latency distribution in microseconds for each verb, group, version, resource, subresource, scope and component.", Help: "(Deprecated) Response latency distribution in microseconds for each verb, group, version, resource, subresource, scope and component.",
// Use buckets ranging from 125 ms to 8 seconds. // Use buckets ranging from 125 ms to 8 seconds.
Buckets: prometheus.ExponentialBuckets(125000, 2.0, 7), Buckets: prometheus.ExponentialBuckets(125000, 2.0, 7),
StabilityLevel: compbasemetrics.ALPHA,
}, },
[]string{"verb", "group", "version", "resource", "subresource", "scope", "component"}, []string{"verb", "group", "version", "resource", "subresource", "scope", "component"},
) )
deprecatedRequestLatenciesSummary = prometheus.NewSummaryVec( deprecatedRequestLatenciesSummary = compbasemetrics.NewSummaryVec(
prometheus.SummaryOpts{ &compbasemetrics.SummaryOpts{
Name: "apiserver_request_latencies_summary", Name: "apiserver_request_latencies_summary",
Help: "(Deprecated) Response latency summary in microseconds for each verb, group, version, resource, subresource, scope and component.", Help: "(Deprecated) Response latency summary in microseconds for each verb, group, version, resource, subresource, scope and component.",
// Make the sliding window of 5h. // Make the sliding window of 5h.
// TODO: The value for this should be based on our SLI definition (medium term). // TODO: The value for this should be based on our SLI definition (medium term).
MaxAge: 5 * time.Hour, MaxAge: 5 * time.Hour,
StabilityLevel: compbasemetrics.ALPHA,
}, },
[]string{"verb", "group", "version", "resource", "subresource", "scope", "component"}, []string{"verb", "group", "version", "resource", "subresource", "scope", "component"},
) )
responseSizes = prometheus.NewHistogramVec( responseSizes = compbasemetrics.NewHistogramVec(
prometheus.HistogramOpts{ &compbasemetrics.HistogramOpts{
Name: "apiserver_response_sizes", Name: "apiserver_response_sizes",
Help: "Response size distribution in bytes for each group, version, verb, resource, subresource, scope and component.", Help: "Response size distribution in bytes for each group, version, verb, resource, subresource, scope and component.",
// Use buckets ranging from 1000 bytes (1KB) to 10^9 bytes (1GB). // Use buckets ranging from 1000 bytes (1KB) to 10^9 bytes (1GB).
Buckets: prometheus.ExponentialBuckets(1000, 10.0, 7), Buckets: prometheus.ExponentialBuckets(1000, 10.0, 7),
StabilityLevel: compbasemetrics.ALPHA,
}, },
[]string{"verb", "group", "version", "resource", "subresource", "scope", "component"}, []string{"verb", "group", "version", "resource", "subresource", "scope", "component"},
) )
// DroppedRequests is a number of requests dropped with 'Try again later' response" // DroppedRequests is a number of requests dropped with 'Try again later' response"
DroppedRequests = prometheus.NewCounterVec( DroppedRequests = compbasemetrics.NewCounterVec(
prometheus.CounterOpts{ &compbasemetrics.CounterOpts{
Name: "apiserver_dropped_requests_total", Name: "apiserver_dropped_requests_total",
Help: "Number of requests dropped with 'Try again later' response", Help: "Number of requests dropped with 'Try again later' response",
StabilityLevel: compbasemetrics.ALPHA,
}, },
[]string{"requestKind"}, []string{"requestKind"},
) )
DeprecatedDroppedRequests = prometheus.NewCounterVec( DeprecatedDroppedRequests = compbasemetrics.NewCounterVec(
prometheus.CounterOpts{ &compbasemetrics.CounterOpts{
Name: "apiserver_dropped_requests", Name: "apiserver_dropped_requests",
Help: "(Deprecated) Number of requests dropped with 'Try again later' response", Help: "(Deprecated) Number of requests dropped with 'Try again later' response",
StabilityLevel: compbasemetrics.ALPHA,
}, },
[]string{"requestKind"}, []string{"requestKind"},
) )
// RegisteredWatchers is a number of currently registered watchers splitted by resource. // RegisteredWatchers is a number of currently registered watchers splitted by resource.
RegisteredWatchers = prometheus.NewGaugeVec( RegisteredWatchers = compbasemetrics.NewGaugeVec(
prometheus.GaugeOpts{ &compbasemetrics.GaugeOpts{
Name: "apiserver_registered_watchers", Name: "apiserver_registered_watchers",
Help: "Number of currently registered watchers for a given resources", Help: "Number of currently registered watchers for a given resources",
StabilityLevel: compbasemetrics.ALPHA,
}, },
[]string{"group", "version", "kind"}, []string{"group", "version", "kind"},
) )
WatchEvents = prometheus.NewCounterVec( WatchEvents = compbasemetrics.NewCounterVec(
prometheus.CounterOpts{ &compbasemetrics.CounterOpts{
Name: "apiserver_watch_events_total", Name: "apiserver_watch_events_total",
Help: "Number of events sent in watch clients", Help: "Number of events sent in watch clients",
StabilityLevel: compbasemetrics.ALPHA,
}, },
[]string{"group", "version", "kind"}, []string{"group", "version", "kind"},
) )
WatchEventsSizes = prometheus.NewHistogramVec( WatchEventsSizes = compbasemetrics.NewHistogramVec(
prometheus.HistogramOpts{ &compbasemetrics.HistogramOpts{
Name: "apiserver_watch_events_sizes", Name: "apiserver_watch_events_sizes",
Help: "Watch event size distribution in bytes", Help: "Watch event size distribution in bytes",
Buckets: prometheus.ExponentialBuckets(1024, 2.0, 8), // 1K, 2K, 4K, 8K, ..., 128K. Buckets: prometheus.ExponentialBuckets(1024, 2.0, 8), // 1K, 2K, 4K, 8K, ..., 128K.
StabilityLevel: compbasemetrics.ALPHA,
}, },
[]string{"group", "version", "kind"}, []string{"group", "version", "kind"},
) )
// Because of volatality of the base metric this is pre-aggregated one. Instead of reporing current usage all the time // Because of volatality of the base metric this is pre-aggregated one. Instead of reporing current usage all the time
// it reports maximal usage during the last second. // it reports maximal usage during the last second.
currentInflightRequests = prometheus.NewGaugeVec( currentInflightRequests = compbasemetrics.NewGaugeVec(
prometheus.GaugeOpts{ &compbasemetrics.GaugeOpts{
Name: "apiserver_current_inflight_requests", Name: "apiserver_current_inflight_requests",
Help: "Maximal number of currently used inflight request limit of this apiserver per request kind in last second.", Help: "Maximal number of currently used inflight request limit of this apiserver per request kind in last second.",
StabilityLevel: compbasemetrics.ALPHA,
}, },
[]string{"requestKind"}, []string{"requestKind"},
) )
@ -197,7 +212,7 @@ var registerMetrics sync.Once
func Register() { func Register() {
registerMetrics.Do(func() { registerMetrics.Do(func() {
for _, metric := range metrics { for _, metric := range metrics {
prometheus.MustRegister(metric) legacyregistry.MustRegister(metric)
} }
}) })
} }
@ -241,7 +256,7 @@ func RecordLongRunning(req *http.Request, requestInfo *request.RequestInfo, comp
if requestInfo == nil { if requestInfo == nil {
requestInfo = &request.RequestInfo{Verb: req.Method, Path: req.URL.Path} requestInfo = &request.RequestInfo{Verb: req.Method, Path: req.URL.Path}
} }
var g prometheus.Gauge var g compbasemetrics.GaugeMetric
scope := CleanScope(requestInfo) scope := CleanScope(requestInfo)
// We don't use verb from <requestInfo>, as for the healthy path // We don't use verb from <requestInfo>, as for the healthy path
// MonitorRequest is called from InstrumentRouteFunc which is registered // MonitorRequest is called from InstrumentRouteFunc which is registered

7
pkg/server/routes/metrics.go
View File

@ -23,8 +23,7 @@ import (
apimetrics "k8s.io/apiserver/pkg/endpoints/metrics" apimetrics "k8s.io/apiserver/pkg/endpoints/metrics"
"k8s.io/apiserver/pkg/server/mux" "k8s.io/apiserver/pkg/server/mux"
etcd3metrics "k8s.io/apiserver/pkg/storage/etcd3/metrics" etcd3metrics "k8s.io/apiserver/pkg/storage/etcd3/metrics"
"k8s.io/component-base/metrics/legacyregistry"
"github.com/prometheus/client_golang/prometheus"
) )
// DefaultMetrics installs the default prometheus metrics handler // DefaultMetrics installs the default prometheus metrics handler
@ -33,7 +32,7 @@ type DefaultMetrics struct{}
// Install adds the DefaultMetrics handler // Install adds the DefaultMetrics handler
func (m DefaultMetrics) Install(c *mux.PathRecorderMux) { func (m DefaultMetrics) Install(c *mux.PathRecorderMux) {
register() register()
c.Handle("/metrics", prometheus.Handler()) c.Handle("/metrics", legacyregistry.Handler())
} }
// MetricsWithReset install the prometheus metrics handler extended with support for the DELETE method // MetricsWithReset install the prometheus metrics handler extended with support for the DELETE method
@ -43,7 +42,7 @@ type MetricsWithReset struct{}
// Install adds the MetricsWithReset handler // Install adds the MetricsWithReset handler
func (m MetricsWithReset) Install(c *mux.PathRecorderMux) { func (m MetricsWithReset) Install(c *mux.PathRecorderMux) {
register() register()
defaultMetricsHandler := prometheus.Handler().ServeHTTP defaultMetricsHandler := legacyregistry.Handler().ServeHTTP
c.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) { c.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) {
if req.Method == "DELETE" { if req.Method == "DELETE" {
apimetrics.Reset() apimetrics.Reset()

18
pkg/storage/cacher/cacher.go
View File

@ -24,8 +24,6 @@ import (
"sync" "sync"
"time" "time"
"k8s.io/klog"
"k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -41,16 +39,18 @@ import (
"k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage"
utilfeature "k8s.io/apiserver/pkg/util/feature" utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/cache"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
"k8s.io/klog"
utiltrace "k8s.io/utils/trace" utiltrace "k8s.io/utils/trace"
"github.com/prometheus/client_golang/prometheus"
) )
var ( var (
initCounter = prometheus.NewCounterVec( initCounter = metrics.NewCounterVec(
prometheus.CounterOpts{ &metrics.CounterOpts{
Name: "apiserver_init_events_total", Name: "apiserver_init_events_total",
Help: "Counter of init events processed in watchcache broken by resource type", Help: "Counter of init events processed in watchcache broken by resource type",
StabilityLevel: metrics.ALPHA,
}, },
[]string{"resource"}, []string{"resource"},
) )
@ -64,7 +64,7 @@ const (
) )
func init() { func init() {
prometheus.MustRegister(initCounter) legacyregistry.MustRegister(initCounter)
} }
// Config contains the configuration for a given Cache. // Config contains the configuration for a given Cache.

36
pkg/storage/etcd3/metrics/metrics.go
View File

@ -20,29 +20,33 @@ import (
"sync" "sync"
"time" "time"
"github.com/prometheus/client_golang/prometheus" compbasemetrics "k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
) )
var ( var (
etcdRequestLatency = prometheus.NewHistogramVec( etcdRequestLatency = compbasemetrics.NewHistogramVec(
prometheus.HistogramOpts{ &compbasemetrics.HistogramOpts{
Name: "etcd_request_duration_seconds", Name: "etcd_request_duration_seconds",
Help: "Etcd request latency in seconds for each operation and object type.", Help: "Etcd request latency in seconds for each operation and object type.",
StabilityLevel: compbasemetrics.ALPHA,
}, },
[]string{"operation", "type"}, []string{"operation", "type"},
) )
objectCounts = prometheus.NewGaugeVec( objectCounts = compbasemetrics.NewGaugeVec(
prometheus.GaugeOpts{ &compbasemetrics.GaugeOpts{
Name: "etcd_object_counts", Name: "etcd_object_counts",
Help: "Number of stored objects at the time of last check split by kind.", Help: "Number of stored objects at the time of last check split by kind.",
StabilityLevel: compbasemetrics.ALPHA,
}, },
[]string{"resource"}, []string{"resource"},
) )
deprecatedEtcdRequestLatenciesSummary = prometheus.NewSummaryVec( deprecatedEtcdRequestLatenciesSummary = compbasemetrics.NewSummaryVec(
prometheus.SummaryOpts{ &compbasemetrics.SummaryOpts{
Name: "etcd_request_latencies_summary", Name: "etcd_request_latencies_summary",
Help: "(Deprecated) Etcd request latency summary in microseconds for each operation and object type.", Help: "(Deprecated) Etcd request latency summary in microseconds for each operation and object type.",
StabilityLevel: compbasemetrics.ALPHA,
}, },
[]string{"operation", "type"}, []string{"operation", "type"},
) )
@ -54,11 +58,11 @@ var registerMetrics sync.Once
func Register() { func Register() {
// Register the metrics. // Register the metrics.
registerMetrics.Do(func() { registerMetrics.Do(func() {
prometheus.MustRegister(etcdRequestLatency) legacyregistry.MustRegister(etcdRequestLatency)
prometheus.MustRegister(objectCounts) legacyregistry.MustRegister(objectCounts)
// TODO(danielqsj): Remove the following metrics, they are deprecated // TODO(danielqsj): Remove the following metrics, they are deprecated
prometheus.MustRegister(deprecatedEtcdRequestLatenciesSummary) legacyregistry.MustRegister(deprecatedEtcdRequestLatenciesSummary)
}) })
} }

5
pkg/storage/storagebackend/factory/etcd3.go
View File

@ -34,6 +34,7 @@ import (
"k8s.io/apiserver/pkg/storage/etcd3" "k8s.io/apiserver/pkg/storage/etcd3"
"k8s.io/apiserver/pkg/storage/storagebackend" "k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/apiserver/pkg/storage/value" "k8s.io/apiserver/pkg/storage/value"
"k8s.io/component-base/metrics/legacyregistry"
) )
// The short keepalive timeout and interval have been chosen to aggressively // The short keepalive timeout and interval have been chosen to aggressively
@ -46,6 +47,10 @@ const keepaliveTimeout = 10 * time.Second
// on heavily loaded arm64 CPUs (issue #64649) // on heavily loaded arm64 CPUs (issue #64649)
const dialTimeout = 20 * time.Second const dialTimeout = 20 * time.Second
func init() {
legacyregistry.RawMustRegister(grpcprom.DefaultClientMetrics)
}
func newETCD3HealthCheck(c storagebackend.Config) (func() error, error) { func newETCD3HealthCheck(c storagebackend.Config) (func() error, error) {
// constructing the etcd v3 client blocks and times out if etcd is not available. // constructing the etcd v3 client blocks and times out if etcd is not available.
// retry in a loop in the background until we successfully create the client, storing the client or error encountered // retry in a loop in the background until we successfully create the client, storing the client or error encountered

116
pkg/storage/value/metrics.go
View File

@ -20,9 +20,11 @@ import (
"sync" "sync"
"time" "time"
"github.com/prometheus/client_golang/prometheus"
"google.golang.org/grpc/status" "google.golang.org/grpc/status"
"github.com/prometheus/client_golang/prometheus" "k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
) )
const ( const (
@ -31,84 +33,92 @@ const (
) )
var ( var (
transformerLatencies = prometheus.NewHistogramVec( transformerLatencies = metrics.NewHistogramVec(
prometheus.HistogramOpts{ &metrics.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "transformation_duration_seconds", Name: "transformation_duration_seconds",
Help: "Latencies in seconds of value transformation operations.", Help: "Latencies in seconds of value transformation operations.",
// In-process transformations (ex. AES CBC) complete on the order of 20 microseconds. However, when // In-process transformations (ex. AES CBC) complete on the order of 20 microseconds. However, when
// external KMS is involved latencies may climb into milliseconds. // external KMS is involved latencies may climb into milliseconds.
Buckets: prometheus.ExponentialBuckets(5e-6, 2, 14), Buckets: prometheus.ExponentialBuckets(5e-6, 2, 14),
StabilityLevel: metrics.ALPHA,
}, },
[]string{"transformation_type"}, []string{"transformation_type"},
) )
deprecatedTransformerLatencies = prometheus.NewHistogramVec( deprecatedTransformerLatencies = metrics.NewHistogramVec(
prometheus.HistogramOpts{ &metrics.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "transformation_latencies_microseconds", Name: "transformation_latencies_microseconds",
Help: "(Deprecated) Latencies in microseconds of value transformation operations.", Help: "(Deprecated) Latencies in microseconds of value transformation operations.",
// In-process transformations (ex. AES CBC) complete on the order of 20 microseconds. However, when // In-process transformations (ex. AES CBC) complete on the order of 20 microseconds. However, when
// external KMS is involved latencies may climb into milliseconds. // external KMS is involved latencies may climb into milliseconds.
Buckets: prometheus.ExponentialBuckets(5, 2, 14), Buckets: prometheus.ExponentialBuckets(5, 2, 14),
StabilityLevel: metrics.ALPHA,
}, },
[]string{"transformation_type"}, []string{"transformation_type"},
) )
transformerOperationsTotal = prometheus.NewCounterVec( transformerOperationsTotal = metrics.NewCounterVec(
prometheus.CounterOpts{ &metrics.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "transformation_operations_total", Name: "transformation_operations_total",
Help: "Total number of transformations.", Help: "Total number of transformations.",
StabilityLevel: metrics.ALPHA,
}, },
[]string{"transformation_type", "transformer_prefix", "status"}, []string{"transformation_type", "transformer_prefix", "status"},
) )
deprecatedTransformerFailuresTotal = prometheus.NewCounterVec( deprecatedTransformerFailuresTotal = metrics.NewCounterVec(
prometheus.CounterOpts{ &metrics.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "transformation_failures_total", Name: "transformation_failures_total",
Help: "(Deprecated) Total number of failed transformation operations.", Help: "(Deprecated) Total number of failed transformation operations.",
StabilityLevel: metrics.ALPHA,
}, },
[]string{"transformation_type"}, []string{"transformation_type"},
) )
envelopeTransformationCacheMissTotal = prometheus.NewCounter( envelopeTransformationCacheMissTotal = metrics.NewCounter(
prometheus.CounterOpts{ &metrics.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "envelope_transformation_cache_misses_total", Name: "envelope_transformation_cache_misses_total",
Help: "Total number of cache misses while accessing key decryption key(KEK).", Help: "Total number of cache misses while accessing key decryption key(KEK).",
StabilityLevel: metrics.ALPHA,
}, },
) )
dataKeyGenerationLatencies = prometheus.NewHistogram( dataKeyGenerationLatencies = metrics.NewHistogram(
prometheus.HistogramOpts{ &metrics.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "data_key_generation_duration_seconds", Name: "data_key_generation_duration_seconds",
Help: "Latencies in seconds of data encryption key(DEK) generation operations.", Help: "Latencies in seconds of data encryption key(DEK) generation operations.",
Buckets: prometheus.ExponentialBuckets(5e-6, 2, 14), Buckets: prometheus.ExponentialBuckets(5e-6, 2, 14),
StabilityLevel: metrics.ALPHA,
}, },
) )
deprecatedDataKeyGenerationLatencies = prometheus.NewHistogram( deprecatedDataKeyGenerationLatencies = metrics.NewHistogram(
prometheus.HistogramOpts{ &metrics.HistogramOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "data_key_generation_latencies_microseconds", Name: "data_key_generation_latencies_microseconds",
Help: "(Deprecated) Latencies in microseconds of data encryption key(DEK) generation operations.", Help: "(Deprecated) Latencies in microseconds of data encryption key(DEK) generation operations.",
Buckets: prometheus.ExponentialBuckets(5, 2, 14), Buckets: prometheus.ExponentialBuckets(5, 2, 14),
StabilityLevel: metrics.ALPHA,
}, },
) )
dataKeyGenerationFailuresTotal = prometheus.NewCounter( dataKeyGenerationFailuresTotal = metrics.NewCounter(
prometheus.CounterOpts{ &metrics.CounterOpts{
Namespace: namespace, Namespace: namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "data_key_generation_failures_total", Name: "data_key_generation_failures_total",
Help: "Total number of failed data encryption key(DEK) generation operations.", Help: "Total number of failed data encryption key(DEK) generation operations.",
StabilityLevel: metrics.ALPHA,
}, },
) )
) )
@ -117,14 +127,14 @@ var registerMetrics sync.Once
func RegisterMetrics() { func RegisterMetrics() {
registerMetrics.Do(func() { registerMetrics.Do(func() {
prometheus.MustRegister(transformerLatencies) legacyregistry.MustRegister(transformerLatencies)
prometheus.MustRegister(deprecatedTransformerLatencies) legacyregistry.MustRegister(deprecatedTransformerLatencies)
prometheus.MustRegister(transformerOperationsTotal) legacyregistry.MustRegister(transformerOperationsTotal)
prometheus.MustRegister(deprecatedTransformerFailuresTotal) legacyregistry.MustRegister(deprecatedTransformerFailuresTotal)
prometheus.MustRegister(envelopeTransformationCacheMissTotal) legacyregistry.MustRegister(envelopeTransformationCacheMissTotal)
prometheus.MustRegister(dataKeyGenerationLatencies) legacyregistry.MustRegister(dataKeyGenerationLatencies)
prometheus.MustRegister(deprecatedDataKeyGenerationLatencies) legacyregistry.MustRegister(deprecatedDataKeyGenerationLatencies)
prometheus.MustRegister(dataKeyGenerationFailuresTotal) legacyregistry.MustRegister(dataKeyGenerationFailuresTotal)
}) })
} }

10
pkg/storage/value/metrics_test.go
View File

@ -44,10 +44,10 @@ func TestTotals(t *testing.T) {
}, },
error: errors.New("foo"), error: errors.New("foo"),
want: ` want: `
# HELP apiserver_storage_transformation_failures_total (Deprecated) Total number of failed transformation operations. # HELP apiserver_storage_transformation_failures_total [ALPHA] (Deprecated) Total number of failed transformation operations.
# TYPE apiserver_storage_transformation_failures_total counter # TYPE apiserver_storage_transformation_failures_total counter
apiserver_storage_transformation_failures_total{transformation_type="encrypt"} 1 apiserver_storage_transformation_failures_total{transformation_type="encrypt"} 1
# HELP apiserver_storage_transformation_operations_total Total number of transformations. # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations.
# TYPE apiserver_storage_transformation_operations_total counter # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="Unknown",transformation_type="encrypt",transformer_prefix="k8s:enc:kms:v1:"} 1 apiserver_storage_transformation_operations_total{status="Unknown",transformation_type="encrypt",transformer_prefix="k8s:enc:kms:v1:"} 1
`, `,
@ -59,7 +59,7 @@ func TestTotals(t *testing.T) {
"apiserver_storage_transformation_failures_total", "apiserver_storage_transformation_failures_total",
}, },
want: ` want: `
# HELP apiserver_storage_transformation_operations_total Total number of transformations. # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations.
# TYPE apiserver_storage_transformation_operations_total counter # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="OK",transformation_type="encrypt",transformer_prefix="k8s:enc:kms:v1:"} 1 apiserver_storage_transformation_operations_total{status="OK",transformation_type="encrypt",transformer_prefix="k8s:enc:kms:v1:"} 1
`, `,
@ -72,10 +72,10 @@ func TestTotals(t *testing.T) {
}, },
error: status.Error(codes.FailedPrecondition, "foo"), error: status.Error(codes.FailedPrecondition, "foo"),
want: ` want: `
# HELP apiserver_storage_transformation_failures_total (Deprecated) Total number of failed transformation operations. # HELP apiserver_storage_transformation_failures_total [ALPHA] (Deprecated) Total number of failed transformation operations.
# TYPE apiserver_storage_transformation_failures_total counter # TYPE apiserver_storage_transformation_failures_total counter
apiserver_storage_transformation_failures_total{transformation_type="encrypt"} 1 apiserver_storage_transformation_failures_total{transformation_type="encrypt"} 1
# HELP apiserver_storage_transformation_operations_total Total number of transformations. # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations.
# TYPE apiserver_storage_transformation_operations_total counter # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="FailedPrecondition",transformation_type="encrypt",transformer_prefix="k8s:enc:kms:v1:"} 1 apiserver_storage_transformation_operations_total{status="FailedPrecondition",transformation_type="encrypt",transformer_prefix="k8s:enc:kms:v1:"} 1
`, `,

12
pkg/storage/value/transformer_test.go
View File

@ -128,7 +128,7 @@ func TestPrefixFromMetrics(t *testing.T) {
"apiserver_storage_transformation_operations_total", "apiserver_storage_transformation_operations_total",
}, },
want: ` want: `
# HELP apiserver_storage_transformation_operations_total Total number of transformations. # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations.
# TYPE apiserver_storage_transformation_operations_total counter # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="OK",transformation_type="from_storage",transformer_prefix="identity"} 1 apiserver_storage_transformation_operations_total{status="OK",transformation_type="from_storage",transformer_prefix="identity"} 1
`, `,
@ -142,7 +142,7 @@ func TestPrefixFromMetrics(t *testing.T) {
"apiserver_storage_transformation_operations_total", "apiserver_storage_transformation_operations_total",
}, },
want: ` want: `
# HELP apiserver_storage_transformation_operations_total Total number of transformations. # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations.
# TYPE apiserver_storage_transformation_operations_total counter # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="OK",transformation_type="from_storage",transformer_prefix="other:"} 1 apiserver_storage_transformation_operations_total{status="OK",transformation_type="from_storage",transformer_prefix="other:"} 1
`, `,
@ -156,7 +156,7 @@ func TestPrefixFromMetrics(t *testing.T) {
"apiserver_storage_transformation_operations_total", "apiserver_storage_transformation_operations_total",
}, },
want: ` want: `
# HELP apiserver_storage_transformation_operations_total Total number of transformations. # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations.
# TYPE apiserver_storage_transformation_operations_total counter # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="Unknown",transformation_type="from_storage",transformer_prefix="other:"} 1 apiserver_storage_transformation_operations_total{status="Unknown",transformation_type="from_storage",transformer_prefix="other:"} 1
`, `,
@ -170,7 +170,7 @@ func TestPrefixFromMetrics(t *testing.T) {
"apiserver_storage_transformation_operations_total", "apiserver_storage_transformation_operations_total",
}, },
want: ` want: `
# HELP apiserver_storage_transformation_operations_total Total number of transformations. # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations.
# TYPE apiserver_storage_transformation_operations_total counter # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="Unknown",transformation_type="from_storage",transformer_prefix="unknown"} 1 apiserver_storage_transformation_operations_total{status="Unknown",transformation_type="from_storage",transformer_prefix="unknown"} 1
`, `,
@ -214,7 +214,7 @@ func TestPrefixToMetrics(t *testing.T) {
"apiserver_storage_transformation_operations_total", "apiserver_storage_transformation_operations_total",
}, },
want: ` want: `
# HELP apiserver_storage_transformation_operations_total Total number of transformations. # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations.
# TYPE apiserver_storage_transformation_operations_total counter # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="OK",transformation_type="to_storage",transformer_prefix="other:"} 1 apiserver_storage_transformation_operations_total{status="OK",transformation_type="to_storage",transformer_prefix="other:"} 1
`, `,
@ -228,7 +228,7 @@ func TestPrefixToMetrics(t *testing.T) {
"apiserver_storage_transformation_operations_total", "apiserver_storage_transformation_operations_total",
}, },
want: ` want: `
# HELP apiserver_storage_transformation_operations_total Total number of transformations. # HELP apiserver_storage_transformation_operations_total [ALPHA] Total number of transformations.
# TYPE apiserver_storage_transformation_operations_total counter # TYPE apiserver_storage_transformation_operations_total counter
apiserver_storage_transformation_operations_total{status="Unknown",transformation_type="to_storage",transformer_prefix="other:"} 1 apiserver_storage_transformation_operations_total{status="Unknown",transformation_type="to_storage",transformer_prefix="other:"} 1
`, `,