register metrics from comp-base

Kubernetes-commit: 11656478be93d4a9e54129ec35cd2b9558e901ac

go.mod

@@ -29,12 +29,12 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
 	github.com/pkg/errors v0.8.1
 	github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021 // indirect
-	github.com/prometheus/client_golang v1.0.0
 	github.com/prometheus/client_model v0.2.0
 	github.com/sirupsen/logrus v1.4.2 // indirect
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.4.0
 	go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738
+	go.uber.org/atomic v1.3.2
 	go.uber.org/zap v1.10.0
 	golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975
 	golang.org/x/net v0.0.0-20191004110552-13f9640d40b9
@@ -44,10 +44,10 @@ require (
 	gopkg.in/square/go-jose.v2 v2.2.2
 	gopkg.in/yaml.v2 v2.2.8
 	gotest.tools v2.2.0+incompatible // indirect
-	k8s.io/api v0.0.0-20200302082247-8f54d34188b0
+	k8s.io/api v0.0.0
-	k8s.io/apimachinery v0.0.0-20200302045842-b9f0d37e94c6
+	k8s.io/apimachinery v0.0.0
-	k8s.io/client-go v0.0.0-20200302082525-ca7edf3d8a93
+	k8s.io/client-go v0.0.0
-	k8s.io/component-base v0.0.0-20200227163006-6f93f319a354
+	k8s.io/component-base v0.0.0
 	k8s.io/klog v1.0.0
 	k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c
 	k8s.io/utils v0.0.0-20200117235808-5f6fbceb4c31
@@ -59,8 +59,9 @@ require (
 replace (
 	golang.org/x/sys => golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a // pinned to release-branch.go1.13
 	golang.org/x/tools => golang.org/x/tools v0.0.0-20190821162956-65e3620a7ae7 // pinned to release-branch.go1.13
-	k8s.io/api => k8s.io/api v0.0.0-20200302082247-8f54d34188b0
+	k8s.io/api => ../api
-	k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20200302045842-b9f0d37e94c6
+	k8s.io/apimachinery => ../apimachinery
-	k8s.io/client-go => k8s.io/client-go v0.0.0-20200302082525-ca7edf3d8a93
+	k8s.io/apiserver => ../apiserver
-	k8s.io/component-base => k8s.io/component-base v0.0.0-20200227163006-6f93f319a354
+	k8s.io/client-go => ../client-go
+	k8s.io/component-base => ../component-base
 )

go.sum

@@ -337,10 +337,6 @@ gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-k8s.io/api v0.0.0-20200302082247-8f54d34188b0/go.mod h1:+2jnw1NMpdXlVDlcy5KjXl7Gh4M0HZ0AhKakQo+KiV8=
-k8s.io/apimachinery v0.0.0-20200302045842-b9f0d37e94c6/go.mod h1:5X8oEhnd931nEg6/Nkumo00nT6ZsCLp2h7Xwd7Ym6P4=
-k8s.io/client-go v0.0.0-20200302082525-ca7edf3d8a93/go.mod h1:2DzSdhxUXsi3Ln8q5B+GHLG6b2cQN64WNsTnBYZ8Y4Y=
-k8s.io/component-base v0.0.0-20200227163006-6f93f319a354/go.mod h1:HDvqB92UBNY8EfzZuIRA2S7PmDF5CfiRwS88LNkTRnE=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=

pkg/server/filters/priority-and-fairness.go

@@ -21,11 +21,9 @@ import (
 	"fmt"
 	"net/http"
 
-	// TODO: decide whether to also generate the old metrics, which
+	_ "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
-	// categorize according to mutating vs readonly.
-
-	// "k8s.io/apiserver/pkg/endpoints/metrics"
 
+	"go.uber.org/atomic"
 	fcv1a1 "k8s.io/api/flowcontrol/v1alpha1"
 	apitypes "k8s.io/apimachinery/pkg/types"
 	apirequest "k8s.io/apiserver/pkg/endpoints/request"
@@ -38,8 +36,8 @@ type priorityAndFairnessKeyType int
 const priorityAndFairnessKey priorityAndFairnessKeyType = iota
 
 const (
-	responseHeaderMatchedPriorityLevelConfigurationUID = "X-Kubernetes-PF-PriorityLevelUID"
+	responseHeaderMatchedPriorityLevelConfigurationUID = "X-Kubernetes-PF-PriorityLevel-UID"
-	responseHeaderMatchedFlowSchemaUID                 = "X-Kubernetes-PF-FlowSchemaUID"
+	responseHeaderMatchedFlowSchemaUID                 = "X-Kubernetes-PF-FlowSchema-UID"
 )
 
 // PriorityAndFairnessClassification identifies the results of
@@ -57,6 +55,8 @@ func GetClassification(ctx context.Context) *PriorityAndFairnessClassification {
 	return ctx.Value(priorityAndFairnessKey).(*PriorityAndFairnessClassification)
 }
 
+var atomicMutatingLen, atomicNonMutatingLen = atomic.NewInt32(0), atomic.NewInt32(0)
+
 // WithPriorityAndFairness limits the number of in-flight
 // requests in a fine-grained way.
 func WithPriorityAndFairness(
@@ -68,7 +68,7 @@ func WithPriorityAndFairness(
 		klog.Warningf("priority and fairness support not found, skipping")
 		return handler
 	}
-
+	startOnce.Do(startRecordingUsage)
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := r.Context()
 		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
@@ -97,8 +97,25 @@ func WithPriorityAndFairness(
 				PriorityLevelName: pl.Name,
 				PriorityLevelUID:  pl.UID}
 		}
+
 		var served bool
+		isMutatingRequest := !nonMutatingRequestVerbs.Has(requestInfo.Verb)
 		execute := func() {
+			var mutatingLen, readOnlyLen int
+			if isMutatingRequest {
+				mutatingLen = int(atomicMutatingLen.Inc())
+			} else {
+				readOnlyLen = int(atomicNonMutatingLen.Inc())
+			}
+			defer func() {
+				if isMutatingRequest {
+					atomicMutatingLen.Dec()
+					watermark.recordMutating(mutatingLen)
+				} else {
+					atomicNonMutatingLen.Dec()
+					watermark.recordReadOnly(readOnlyLen)
+				}
+			}()
 			served = true
 			innerCtx := context.WithValue(ctx, priorityAndFairnessKey, classification)
 			innerReq := r.Clone(innerCtx)

pkg/server/routes/metrics.go

@@ -23,6 +23,7 @@ import (
 	apimetrics "k8s.io/apiserver/pkg/endpoints/metrics"
 	"k8s.io/apiserver/pkg/server/mux"
 	etcd3metrics "k8s.io/apiserver/pkg/storage/etcd3/metrics"
+	flowcontrolmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
 	"k8s.io/component-base/metrics/legacyregistry"
 )
 
@@ -58,4 +59,5 @@ func (m MetricsWithReset) Install(c *mux.PathRecorderMux) {
 func register() {
 	apimetrics.Register()
 	etcd3metrics.Register()
+	flowcontrolmetrics.Register()
 }

pkg/util/flowcontrol/metrics/metrics.go

@@ -17,9 +17,11 @@ limitations under the License.
 package metrics
 
 import (
+	"sync"
 	"time"
 
-	"github.com/prometheus/client_golang/prometheus"
+	compbasemetrics "k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
 )
 
 const (
@@ -37,28 +39,29 @@ var (
 	requestDurationSecondsBuckets = []float64{0, 0.005, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 30}
 )
 
-func init() {
+var registerMetrics sync.Once
-	prometheus.MustRegister(apiserverRejectedRequests)
+
-	prometheus.MustRegister(apiserverCurrentInqueueRequests)
+// Register all metrics.
-	prometheus.MustRegister(apiserverRequestQueueLength)
+func Register() {
-	prometheus.MustRegister(apiserverRequestConcurrencyLimit)
+	registerMetrics.Do(func() {
-	prometheus.MustRegister(apiserverCurrentExecutingRequests)
+		for _, metric := range metrics {
-	prometheus.MustRegister(apiserverRequestWaitingSeconds)
+			legacyregistry.MustRegister(metric)
-	prometheus.MustRegister(apiserverRequestExecutionSeconds)
+		}
+	})
 }
 
 var (
-	apiserverRejectedRequests = prometheus.NewCounterVec(
+	apiserverRejectedRequestsTotal = compbasemetrics.NewCounterVec(
-		prometheus.CounterOpts{
+		&compbasemetrics.CounterOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
-			Name:      "rejected_requests",
+			Name:      "rejected_requests_total",
 			Help:      "Number of rejected requests by api priority and fairness system",
 		},
 		[]string{priorityLevel, "reason"},
 	)
-	apiserverCurrentInqueueRequests = prometheus.NewGaugeVec(
+	apiserverCurrentInqueueRequests = compbasemetrics.NewGaugeVec(
-		prometheus.GaugeOpts{
+		&compbasemetrics.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
 			Name:      "current_inqueue_requests",
@@ -66,8 +69,8 @@ var (
 		},
 		[]string{priorityLevel},
 	)
-	apiserverRequestQueueLength = prometheus.NewHistogramVec(
+	apiserverRequestQueueLength = compbasemetrics.NewHistogramVec(
-		prometheus.HistogramOpts{
+		&compbasemetrics.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
 			Name:      "request_queue_length",
@@ -76,8 +79,8 @@ var (
 		},
 		[]string{priorityLevel},
 	)
-	apiserverRequestConcurrencyLimit = prometheus.NewGaugeVec(
+	apiserverRequestConcurrencyLimit = compbasemetrics.NewGaugeVec(
-		prometheus.GaugeOpts{
+		&compbasemetrics.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
 			Name:      "request_concurrency_limit",
@@ -85,8 +88,8 @@ var (
 		},
 		[]string{priorityLevel},
 	)
-	apiserverCurrentExecutingRequests = prometheus.NewGaugeVec(
+	apiserverCurrentExecutingRequests = compbasemetrics.NewGaugeVec(
-		prometheus.GaugeOpts{
+		&compbasemetrics.GaugeOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
 			Name:      "current_executing_requests",
@@ -94,8 +97,8 @@ var (
 		},
 		[]string{priorityLevel},
 	)
-	apiserverRequestWaitingSeconds = prometheus.NewHistogramVec(
+	apiserverRequestWaitingSeconds = compbasemetrics.NewHistogramVec(
-		prometheus.HistogramOpts{
+		&compbasemetrics.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
 			Name:      "request_wait_duration_seconds",
@@ -104,8 +107,8 @@ var (
 		},
 		[]string{priorityLevel, flowSchema, "execute"},
 	)
-	apiserverRequestExecutionSeconds = prometheus.NewHistogramVec(
+	apiserverRequestExecutionSeconds = compbasemetrics.NewHistogramVec(
-		prometheus.HistogramOpts{
+		&compbasemetrics.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
 			Name:      "request_execution_seconds",
@@ -114,6 +117,15 @@ var (
 		},
 		[]string{priorityLevel, flowSchema},
 	)
+	metrics = []compbasemetrics.Registerable{
+		apiserverRejectedRequestsTotal,
+		apiserverCurrentInqueueRequests,
+		apiserverRequestQueueLength,
+		apiserverRequestConcurrencyLimit,
+		apiserverCurrentExecutingRequests,
+		apiserverRequestWaitingSeconds,
+		apiserverRequestExecutionSeconds,
+	}
 )
 
 // UpdateFlowControlRequestsInQueue updates the value for the # of requests in the specified queues in flow control
@@ -133,7 +145,7 @@ func UpdateSharedConcurrencyLimit(priorityLevel string, limit int) {
 
 // AddReject increments the # of rejected requests for flow control
 func AddReject(priorityLevel string, reason string) {
-	apiserverRejectedRequests.WithLabelValues(priorityLevel, reason).Add(1)
+	apiserverRejectedRequestsTotal.WithLabelValues(priorityLevel, reason).Add(1)
 }
 
 // ObserveQueueLength observes the queue length for flow control