Merge pull request #106737 from wojtek-t/cacher_events_processed_metrics

Add watchcache metrics to tracking its progress

Kubernetes-commit: d7bcd8e7c0ce0e4476e1aa7041003ba7fe74fa10

go.mod

@@ -46,7 +46,7 @@ require (
 	k8s.io/api v0.0.0-20211203085948-25b7aa9e86de
 	k8s.io/apimachinery v0.0.0-20211208045959-67ffd82e6cff
 	k8s.io/client-go v0.0.0-20211208050435-c713f38723cb
-	k8s.io/component-base v0.0.0-20211203092501-60bec488f080
+	k8s.io/component-base v0.0.0-20211208050949-17742f0cef83
 	k8s.io/klog/v2 v2.30.0
 	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65
 	k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b
@@ -60,5 +60,5 @@ replace (
 	k8s.io/api => k8s.io/api v0.0.0-20211203085948-25b7aa9e86de
 	k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20211208045959-67ffd82e6cff
 	k8s.io/client-go => k8s.io/client-go v0.0.0-20211208050435-c713f38723cb
-	k8s.io/component-base => k8s.io/component-base v0.0.0-20211203092501-60bec488f080
+	k8s.io/component-base => k8s.io/component-base v0.0.0-20211208050949-17742f0cef83
 )

go.sum

@@ -959,8 +959,8 @@ k8s.io/apimachinery v0.0.0-20211208045959-67ffd82e6cff h1:YOhVGWI7F5otLrzreq4Y6t
 k8s.io/apimachinery v0.0.0-20211208045959-67ffd82e6cff/go.mod h1:fFCTTBKvKcwTPFzjlcxp91uPFZr+JA0FubU4fLzzFYc=
 k8s.io/client-go v0.0.0-20211208050435-c713f38723cb h1:tbntDZ7c+LkEPnURV1oMy9HtSAU7vag2pDilT68PBLg=
 k8s.io/client-go v0.0.0-20211208050435-c713f38723cb/go.mod h1:4l8IJYdxbF/Uo0zdB+OB48wOhs2/7FtuppsKYTXj1Fw=
-k8s.io/component-base v0.0.0-20211203092501-60bec488f080 h1:G2FyK080Yrlu0bZrru3nchG0ABRGFNaRIet230edgtw=
+k8s.io/component-base v0.0.0-20211208050949-17742f0cef83 h1:/YKdVgYQ4uRg+fks7/YhFFI/BuzCgYPQuNpbssNr31Y=
-k8s.io/component-base v0.0.0-20211203092501-60bec488f080/go.mod h1:ps6SHYeOsJvoz1zvLfjkYts/xUlWlYsBjgki6W4Faxg=
+k8s.io/component-base v0.0.0-20211208050949-17742f0cef83/go.mod h1:Ezju43f/aFnlJPxnOeCGzPSqp/Xvhv/9nyyPLoeHhtk=
 k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=

pkg/storage/cacher/cacher.go

@@ -408,6 +408,7 @@ func (c *Cacher) startCaching(stopChannel <-chan struct{}) {
 		successfulList = true
 		c.ready.set(true)
 		klog.V(1).Infof("cacher (%v): initialized", c.objectType.String())
+		watchCacheInitializations.WithLabelValues(c.objectType.String()).Inc()
 	})
 	defer func() {
 		if successfulList {
@@ -822,6 +823,7 @@ func (c *Cacher) dispatchEvents() {
 				c.dispatchEvent(&event)
 			}
 			lastProcessedResourceVersion = event.ResourceVersion
+			eventsCounter.WithLabelValues(c.objectType.String()).Inc()
 		case <-bookmarkTimer.C():
 			bookmarkTimer.Reset(wait.Jitter(time.Second, 0.25))
 			// Never send a bookmark event if we did not see an event here, this is fine

pkg/storage/cacher/metrics.go

@@ -21,6 +21,11 @@ import (
 	"k8s.io/component-base/metrics/legacyregistry"
 )
 
+const (
+	namespace = "apiserver"
+	subsystem = "watch_cache"
+)
+
 /*
  * By default, all the following metrics are defined as falling under
  * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/1209-metrics-stability/kubernetes-control-plane-metrics-stability.md#stability-classes)
@@ -39,6 +44,17 @@ var (
 		[]string{"resource"},
 	)
 
+	eventsCounter = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "events_dispatched_total",
+			Help:           "Counter of events dispatched in watch cache broken by resource type.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"resource"},
+	)
+
 	terminatedWatchersCounter = metrics.NewCounterVec(
 		&metrics.CounterOpts{
 			Name:           "apiserver_terminated_watchers_total",
@@ -74,14 +90,27 @@ var (
 		},
 		[]string{"resource"},
 	)
+
+	watchCacheInitializations = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "watch_cache_initializations_total",
+			Help:           "Counter of watch cache initializations broken by resource type.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"resource"},
+	)
 )
 
 func init() {
 	legacyregistry.MustRegister(initCounter)
+	legacyregistry.MustRegister(eventsCounter)
 	legacyregistry.MustRegister(terminatedWatchersCounter)
 	legacyregistry.MustRegister(watchCacheCapacityIncreaseTotal)
 	legacyregistry.MustRegister(watchCacheCapacityDecreaseTotal)
 	legacyregistry.MustRegister(watchCacheCapacity)
+	legacyregistry.MustRegister(watchCacheInitializations)
 }
 
 // recordsWatchCacheCapacityChange record watchCache capacity resize(increase or decrease) operations.