apf: add a metric to count seat samples

Kubernetes-commit: bb15bdf15c1cc4d5a4380f3f6ed46d4adc9662a1

pkg/server/filters/priority-and-fairness.go

@@ -111,7 +111,6 @@ func WithPriorityAndFairness(
 			httplog.AddKeyValue(ctx, "apf_pl", truncateLogField(pl.Name))
 			httplog.AddKeyValue(ctx, "apf_fs", truncateLogField(fs.Name))
 			httplog.AddKeyValue(ctx, "apf_fd", truncateLogField(flowDistinguisher))
-<<<<<<< HEAD
 		}
 		// estimateWork is called, if at all, after noteFn
 		estimateWork := func() flowcontrolrequest.WorkEstimate {
@@ -125,6 +124,8 @@ func WithPriorityAndFairness(
 			}
 
 			workEstimate := workEstimator(r, classification.FlowSchemaName, classification.PriorityLevelName)
+
+			fcmetrics.ObserveWorkEstimatedSeats(classification.PriorityLevelName, classification.FlowSchemaName, workEstimate.MaxSeats())
 			if klog.V(4).Enabled() {
 				httplog.AddKeyValue(ctx, "apf_iseats", workEstimate.InitialSeats)
 				httplog.AddKeyValue(ctx, "apf_fseats", workEstimate.FinalSeats)

pkg/util/flowcontrol/metrics/metrics.go

@@ -311,6 +311,19 @@ var (
 		},
 		[]string{priorityLevel, "success"},
 	)
+	apiserverWorkEstimatedSeats = compbasemetrics.NewHistogramVec(
+		&compbasemetrics.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "work_estimated_seats",
+			Help:      "Number of estimated seats (maximum of initial and final seats) associated with requests in API Priority and Fairness",
+			// the upper bound comes from the maximum number of seats a request
+			// can occupy which is currently set at 10.
+			Buckets:        []float64{1, 2, 4, 10},
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{priorityLevel, flowSchema},
+	)
 
 	metrics = Registerables{
 		apiserverRejectedRequestsTotal,
@@ -329,6 +342,7 @@ var (
 		apiserverRequestExecutionSeconds,
 		watchCountSamples,
 		apiserverEpochAdvances,
+		apiserverWorkEstimatedSeats,
 	}.
 		Append(PriorityLevelExecutionSeatsObserverGenerator.metrics()...).
 		Append(PriorityLevelConcurrencyObserverPairGenerator.metrics()...).
@@ -409,3 +423,8 @@ func ObserveWatchCount(ctx context.Context, priorityLevel, flowSchema string, co
 func AddEpochAdvance(ctx context.Context, priorityLevel string, success bool) {
 	apiserverEpochAdvances.WithContext(ctx).WithLabelValues(priorityLevel, strconv.FormatBool(success)).Inc()
 }
+
+// ObserveWorkEstimatedSeats notes a sampling of estimated seats associated with a request
+func ObserveWorkEstimatedSeats(priorityLevel, flowSchema string, seats int) {
+	apiserverWorkEstimatedSeats.WithLabelValues(priorityLevel, flowSchema).Observe(float64(seats))
+}

pkg/util/flowcontrol/request/width.go

@@ -30,6 +30,10 @@ const (
 	minimumSeats = 1
 
 	// the maximum number of seats a request can occupy
+	//
+	// NOTE: work_estimate_seats_samples metric uses the value of maximumSeats
+	// as the upper bound, so when we change maximumSeats we should also
+	// update the buckets of the metric.
 	maximumSeats = 10
 )