Merge pull request #123611 from ritazh/authz-mcmetrics

Add authz webhook matchcondition metrics

Kubernetes-commit: 3e1da218014b5a4e5c95ee79404093302104438b

go.mod

@@ -44,7 +44,7 @@ require (
 	gopkg.in/square/go-jose.v2 v2.6.0
 	k8s.io/api v0.0.0-20240301204737-cd36300dc819
 	k8s.io/apimachinery v0.0.0-20240302004725-df38a01ea799
-	k8s.io/client-go v0.0.0-20240301224102-d48adf87e6e2
+	k8s.io/client-go v0.0.0-20240302051254-1002c2f9bdc6
 	k8s.io/component-base v0.0.0-20240301210028-15d726cdca18
 	k8s.io/klog/v2 v2.120.1
 	k8s.io/kms v0.0.0-20240301210546-4a4bf5f9988c
@@ -127,7 +127,7 @@ require (
 replace (
 	k8s.io/api => k8s.io/api v0.0.0-20240301204737-cd36300dc819
 	k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20240302004725-df38a01ea799
-	k8s.io/client-go => k8s.io/client-go v0.0.0-20240301224102-d48adf87e6e2
+	k8s.io/client-go => k8s.io/client-go v0.0.0-20240302051254-1002c2f9bdc6
 	k8s.io/component-base => k8s.io/component-base v0.0.0-20240301210028-15d726cdca18
 	k8s.io/kms => k8s.io/kms v0.0.0-20240301210546-4a4bf5f9988c
 )

go.sum

@@ -389,8 +389,8 @@ k8s.io/api v0.0.0-20240301204737-cd36300dc819 h1:F87SZX4P+r/LNtLhFtsZjylRqRLRSX7
 k8s.io/api v0.0.0-20240301204737-cd36300dc819/go.mod h1:TYylmz5ON3nmsvimIN46iaRIjQwS/RcA5nYFRkdJmT4=
 k8s.io/apimachinery v0.0.0-20240302004725-df38a01ea799 h1:QqDm+JeV6HCqng5kBgyWDazPe4nK0P20XhjX5Bx9elE=
 k8s.io/apimachinery v0.0.0-20240302004725-df38a01ea799/go.mod h1:qPsrq6INURDMMgqxK78MEuC8GzI1f2oHvfHzg5ZOa6s=
-k8s.io/client-go v0.0.0-20240301224102-d48adf87e6e2 h1:aAiHBa00sgAneHg4BXYbeFj20+c2GdlGERMK3tINtpQ=
+k8s.io/client-go v0.0.0-20240302051254-1002c2f9bdc6 h1:lNZRgV27p39rVqmofLPOTGmsvXW/IO++hB6VrgYL8a8=
-k8s.io/client-go v0.0.0-20240301224102-d48adf87e6e2/go.mod h1:ilQTB01iT0Gh0yvPf71CUaqqEbcNdr9xaG/WvftEjBk=
+k8s.io/client-go v0.0.0-20240302051254-1002c2f9bdc6/go.mod h1:8W5nYx/9kzuguai+FbcWMfIj8s2vArWs0bJH4jr/LmU=
 k8s.io/component-base v0.0.0-20240301210028-15d726cdca18 h1:kiqSkGxkfImyPIdgBwK6mLoS8yLDijT7wKm35LduYLs=
 k8s.io/component-base v0.0.0-20240301210028-15d726cdca18/go.mod h1:ovtVM/EGyY/M89mMKkFZ3tdQREOE2u9pODk7+C6VENQ=
 k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw=

pkg/authorization/authorizerfactory/delegating.go

@@ -26,7 +26,7 @@ import (
 	authorizationclient "k8s.io/client-go/kubernetes/typed/authorization/v1"
 )
 
-// DelegatingAuthorizerConfig is the minimal configuration needed to create an authenticator
+// DelegatingAuthorizerConfig is the minimal configuration needed to create an authorizer
 // built to delegate authorization to a kube API server
 type DelegatingAuthorizerConfig struct {
 	SubjectAccessReviewClient authorizationclient.AuthorizationV1Interface
@@ -55,9 +55,6 @@ func (c DelegatingAuthorizerConfig) New() (authorizer.Authorizer, error) {
 		c.DenyCacheTTL,
 		*c.WebhookRetryBackoff,
 		authorizer.DecisionNoOpinion,
-		webhook.AuthorizerMetrics{
+		NewDelegatingAuthorizerMetrics(),
-			RecordRequestTotal:   RecordRequestTotal,
-			RecordRequestLatency: RecordRequestLatency,
-		},
 	)
 }

pkg/authorization/authorizerfactory/metrics.go

@@ -18,18 +18,22 @@ package authorizerfactory
 
 import (
 	"context"
+	"sync"
 
+	celmetrics "k8s.io/apiserver/pkg/authorization/cel"
+	webhookmetrics "k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics"
 	compbasemetrics "k8s.io/component-base/metrics"
 	"k8s.io/component-base/metrics/legacyregistry"
 )
 
-type registerables []compbasemetrics.Registerable
+var registerMetrics sync.Once
 
-// init registers all metrics
+// RegisterMetrics registers authorizer metrics.
-func init() {
+func RegisterMetrics() {
-	for _, metric := range metrics {
+	registerMetrics.Do(func() {
-		legacyregistry.MustRegister(metric)
+		legacyregistry.MustRegister(requestTotal)
-	}
+		legacyregistry.MustRegister(requestLatency)
+	})
 }
 
 var (
@@ -51,19 +55,26 @@ var (
 		},
 		[]string{"code"},
 	)
-
-	metrics = registerables{
-		requestTotal,
-		requestLatency,
-	}
 )
 
+var _ = webhookmetrics.AuthorizerMetrics(delegatingAuthorizerMetrics{})
+
+type delegatingAuthorizerMetrics struct {
+	// no-op for matchCondition metrics for now, delegating authorization doesn't configure match conditions
+	celmetrics.NoopMatcherMetrics
+}
+
+func NewDelegatingAuthorizerMetrics() delegatingAuthorizerMetrics {
+	RegisterMetrics()
+	return delegatingAuthorizerMetrics{}
+}
+
 // RecordRequestTotal increments the total number of requests for the delegated authorization.
-func RecordRequestTotal(ctx context.Context, code string) {
+func (delegatingAuthorizerMetrics) RecordRequestTotal(ctx context.Context, code string) {
 	requestTotal.WithContext(ctx).WithLabelValues(code).Add(1)
 }
 
 // RecordRequestLatency measures request latency in seconds for the delegated authorization. Broken down by status code.
-func RecordRequestLatency(ctx context.Context, code string, latency float64) {
+func (delegatingAuthorizerMetrics) RecordRequestLatency(ctx context.Context, code string, latency float64) {
 	requestLatency.WithContext(ctx).WithLabelValues(code).Observe(latency)
 }

pkg/authorization/cel/matcher.go

@@ -19,6 +19,7 @@ package cel
 import (
 	"context"
 	"fmt"
+	"time"
 
 	celgo "github.com/google/cel-go/cel"
 
@@ -28,11 +29,29 @@ import (
 
 type CELMatcher struct {
 	CompilationResults []CompilationResult
+
+	// These are optional fields which can be populated if metrics reporting is desired
+	Metrics        MatcherMetrics
+	AuthorizerType string
+	AuthorizerName string
 }
 
 // eval evaluates the given SubjectAccessReview against all cel matchCondition expression
 func (c *CELMatcher) Eval(ctx context.Context, r *authorizationv1.SubjectAccessReview) (bool, error) {
 	var evalErrors []error
+
+	metrics := c.Metrics
+	if metrics == nil {
+		metrics = NoopMatcherMetrics{}
+	}
+	start := time.Now()
+	defer func() {
+		metrics.RecordAuthorizationMatchConditionEvaluation(ctx, c.AuthorizerType, c.AuthorizerName, time.Since(start))
+		if len(evalErrors) > 0 {
+			metrics.RecordAuthorizationMatchConditionEvaluationFailure(ctx, c.AuthorizerType, c.AuthorizerName)
+		}
+	}()
+
 	va := map[string]interface{}{
 		"request": convertObjectToUnstructured(&r.Spec),
 	}
@@ -54,6 +73,7 @@ func (c *CELMatcher) Eval(ctx context.Context, r *authorizationv1.SubjectAccessR
 		// If at least one matchCondition successfully evaluates to FALSE,
 		// return early
 		if !match {
+			metrics.RecordAuthorizationMatchConditionExclusion(ctx, c.AuthorizerType, c.AuthorizerName)
 			return false, nil
 		}
 	}

pkg/authorization/cel/metrics.go

@@ -0,0 +1,120 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cel
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+// MatcherMetrics defines methods for reporting matchCondition metrics
+type MatcherMetrics interface {
+	// RecordAuthorizationMatchConditionEvaluation records the total time taken to evaluate matchConditions for an Authorize() call to the given authorizer
+	RecordAuthorizationMatchConditionEvaluation(ctx context.Context, authorizerType, authorizerName string, elapsed time.Duration)
+	// RecordAuthorizationMatchConditionEvaluationFailure increments if any evaluation error was encountered evaluating matchConditions for an Authorize() call to the given authorizer
+	RecordAuthorizationMatchConditionEvaluationFailure(ctx context.Context, authorizerType, authorizerName string)
+	// RecordAuthorizationMatchConditionExclusion records increments when at least one matchCondition evaluates to false and excludes an Authorize() call to the given authorizer
+	RecordAuthorizationMatchConditionExclusion(ctx context.Context, authorizerType, authorizerName string)
+}
+
+type NoopMatcherMetrics struct{}
+
+func (NoopMatcherMetrics) RecordAuthorizationMatchConditionEvaluation(ctx context.Context, authorizerType, authorizerName string, elapsed time.Duration) {
+}
+func (NoopMatcherMetrics) RecordAuthorizationMatchConditionEvaluationFailure(ctx context.Context, authorizerType, authorizerName string) {
+}
+func (NoopMatcherMetrics) RecordAuthorizationMatchConditionExclusion(ctx context.Context, authorizerType, authorizerName string) {
+}
+
+type matcherMetrics struct{}
+
+func NewMatcherMetrics() MatcherMetrics {
+	RegisterMetrics()
+	return matcherMetrics{}
+}
+
+const (
+	namespace = "apiserver"
+	subsystem = "authorization"
+)
+
+var (
+	authorizationMatchConditionEvaluationErrorsTotal = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "match_condition_evaluation_errors_total",
+			Help:           "Total number of errors when an authorization webhook encounters a match condition error split by authorizer type and name.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"type", "name"},
+	)
+	authorizationMatchConditionExclusionsTotal = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "match_condition_exclusions_total",
+			Help:           "Total number of exclusions when an authorization webhook is skipped because match conditions exclude it.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"type", "name"},
+	)
+	authorizationMatchConditionEvaluationSeconds = metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "match_condition_evaluation_seconds",
+			Help:           "Authorization match condition evaluation time in seconds, split by authorizer type and name.",
+			Buckets:        []float64{0.001, 0.005, 0.01, 0.025, 0.1, 0.2, 0.25},
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"type", "name"},
+	)
+)
+
+var registerMetrics sync.Once
+
+func RegisterMetrics() {
+	registerMetrics.Do(func() {
+		legacyregistry.MustRegister(authorizationMatchConditionEvaluationErrorsTotal)
+		legacyregistry.MustRegister(authorizationMatchConditionExclusionsTotal)
+		legacyregistry.MustRegister(authorizationMatchConditionEvaluationSeconds)
+	})
+}
+
+func ResetMetricsForTest() {
+	authorizationMatchConditionEvaluationErrorsTotal.Reset()
+	authorizationMatchConditionExclusionsTotal.Reset()
+	authorizationMatchConditionEvaluationSeconds.Reset()
+}
+
+func (matcherMetrics) RecordAuthorizationMatchConditionEvaluationFailure(ctx context.Context, authorizerType, authorizerName string) {
+	authorizationMatchConditionEvaluationErrorsTotal.WithContext(ctx).WithLabelValues(authorizerType, authorizerName).Inc()
+}
+
+func (matcherMetrics) RecordAuthorizationMatchConditionExclusion(ctx context.Context, authorizerType, authorizerName string) {
+	authorizationMatchConditionExclusionsTotal.WithContext(ctx).WithLabelValues(authorizerType, authorizerName).Inc()
+}
+
+func (matcherMetrics) RecordAuthorizationMatchConditionEvaluation(ctx context.Context, authorizerType, authorizerName string, elapsed time.Duration) {
+	elapsedSeconds := elapsed.Seconds()
+	authorizationMatchConditionEvaluationSeconds.WithContext(ctx).WithLabelValues(authorizerType, authorizerName).Observe(elapsedSeconds)
+}

pkg/authorization/cel/metrics_test.go

@@ -0,0 +1,81 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cel
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/component-base/metrics/testutil"
+)
+
+func TestRecordAuthorizationMatchConditionEvaluationFailure(t *testing.T) {
+	testCases := []struct {
+		desc      string
+		metrics   []string
+		name      string
+		authztype string
+		want      string
+	}{
+		{
+			desc: "evaluation failure total",
+			metrics: []string{
+				"apiserver_authorization_match_condition_evaluation_errors_total",
+				"apiserver_authorization_match_condition_exclusions_total",
+				"apiserver_authorization_match_condition_evaluation_seconds",
+			},
+			name:      "wh1.example.com",
+			authztype: "Webhook",
+			want: `
+			# HELP apiserver_authorization_match_condition_evaluation_errors_total [ALPHA] Total number of errors when an authorization webhook encounters a match condition error split by authorizer type and name.
+			# TYPE apiserver_authorization_match_condition_evaluation_errors_total counter
+			apiserver_authorization_match_condition_evaluation_errors_total{name="wh1.example.com",type="Webhook"} 1
+			# HELP apiserver_authorization_match_condition_evaluation_seconds [ALPHA] Authorization match condition evaluation time in seconds, split by authorizer type and name.
+        	# TYPE apiserver_authorization_match_condition_evaluation_seconds histogram
+        	apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.001"} 0
+        	apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.005"} 0
+        	apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.01"} 0
+        	apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.025"} 0
+        	apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.1"} 0
+        	apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.2"} 0
+        	apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="0.25"} 0
+        	apiserver_authorization_match_condition_evaluation_seconds_bucket{name="wh1.example.com",type="Webhook",le="+Inf"} 1
+        	apiserver_authorization_match_condition_evaluation_seconds_sum{name="wh1.example.com",type="Webhook"} 1
+        	apiserver_authorization_match_condition_evaluation_seconds_count{name="wh1.example.com",type="Webhook"} 1
+			# HELP apiserver_authorization_match_condition_exclusions_total [ALPHA] Total number of exclusions when an authorization webhook is skipped because match conditions exclude it.
+			# TYPE apiserver_authorization_match_condition_exclusions_total counter
+			apiserver_authorization_match_condition_exclusions_total{name="wh1.example.com",type="Webhook"} 1
+			`,
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.desc, func(t *testing.T) {
+			ResetMetricsForTest()
+			m := NewMatcherMetrics()
+			m.RecordAuthorizationMatchConditionEvaluationFailure(context.Background(), tt.authztype, tt.name)
+			m.RecordAuthorizationMatchConditionExclusion(context.Background(), tt.authztype, tt.name)
+			m.RecordAuthorizationMatchConditionEvaluation(context.Background(), tt.authztype, tt.name, time.Duration(1*time.Second))
+			if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}

plugin/pkg/authorizer/webhook/metrics/metrics.go

@@ -14,22 +14,36 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package webhook
+package metrics
 
 import (
 	"context"
+
+	"k8s.io/apiserver/pkg/authorization/cel"
 )
 
 // AuthorizerMetrics specifies a set of methods that are used to register various metrics for the webhook authorizer
-type AuthorizerMetrics struct {
+type AuthorizerMetrics interface {
-	// RecordRequestTotal increments the total number of requests for the webhook authorizer
+	// Request total and latency metrics
-	RecordRequestTotal func(ctx context.Context, code string)
+	RequestMetrics
-
+	// match condition metrics
-	// RecordRequestLatency measures request latency in seconds for webhooks. Broken down by status code.
+	cel.MatcherMetrics
-	RecordRequestLatency func(ctx context.Context, code string, latency float64)
 }
 
-type noopMetrics struct{}
+type NoopAuthorizerMetrics struct {
+	NoopRequestMetrics
+	cel.NoopMatcherMetrics
+}
 
-func (noopMetrics) RecordRequestTotal(context.Context, string)            {}
+type RequestMetrics interface {
-func (noopMetrics) RecordRequestLatency(context.Context, string, float64) {}
+	// RecordRequestTotal increments the total number of requests for the webhook authorizer
+	RecordRequestTotal(ctx context.Context, code string)
+
+	// RecordRequestLatency measures request latency in seconds for webhooks. Broken down by status code.
+	RecordRequestLatency(ctx context.Context, code string, latency float64)
+}
+
+type NoopRequestMetrics struct{}
+
+func (NoopRequestMetrics) RecordRequestTotal(context.Context, string)            {}
+func (NoopRequestMetrics) RecordRequestLatency(context.Context, string, float64) {}

plugin/pkg/authorizer/webhook/metrics_test.go

@@ -23,6 +23,7 @@ import (
 	"k8s.io/apiserver/pkg/apis/apiserver"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
+	"k8s.io/apiserver/pkg/authorization/cel"
 )
 
 func TestAuthorizerMetrics(t *testing.T) {
@@ -76,11 +77,7 @@ func TestAuthorizerMetrics(t *testing.T) {
 			defer server.Close()
 
 			fakeAuthzMetrics := &fakeAuthorizerMetrics{}
-			authzMetrics := AuthorizerMetrics{
+			wh, err := newV1Authorizer(server.URL, scenario.clientCert, scenario.clientKey, scenario.clientCA, 0, fakeAuthzMetrics, []apiserver.WebhookMatchCondition{}, "")
-				RecordRequestTotal:   fakeAuthzMetrics.RequestTotal,
-				RecordRequestLatency: fakeAuthzMetrics.RequestLatency,
-			}
-			wh, err := newV1Authorizer(server.URL, scenario.clientCert, scenario.clientKey, scenario.clientCA, 0, authzMetrics, []apiserver.WebhookMatchCondition{})
 			if err != nil {
 				t.Error("failed to create client")
 				return
@@ -110,13 +107,15 @@ type fakeAuthorizerMetrics struct {
 
 	latency     float64
 	latencyCode string
+
+	cel.NoopMatcherMetrics
 }
 
-func (f *fakeAuthorizerMetrics) RequestTotal(_ context.Context, code string) {
+func (f *fakeAuthorizerMetrics) RecordRequestTotal(_ context.Context, code string) {
 	f.totalCode = code
 }
 
-func (f *fakeAuthorizerMetrics) RequestLatency(_ context.Context, code string, latency float64) {
+func (f *fakeAuthorizerMetrics) RecordRequestLatency(_ context.Context, code string, latency float64) {
 	f.latency = latency
 	f.latencyCode = code
 }

plugin/pkg/authorizer/webhook/webhook.go

@@ -39,6 +39,7 @@ import (
 	"k8s.io/apiserver/pkg/features"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/apiserver/pkg/util/webhook"
+	"k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics"
 	"k8s.io/client-go/kubernetes/scheme"
 	authorizationv1client "k8s.io/client-go/kubernetes/typed/authorization/v1"
 	"k8s.io/client-go/rest"
@@ -70,13 +71,14 @@ type WebhookAuthorizer struct {
 	unauthorizedTTL     time.Duration
 	retryBackoff        wait.Backoff
 	decisionOnError     authorizer.Decision
-	metrics             AuthorizerMetrics
+	metrics             metrics.AuthorizerMetrics
 	celMatcher          *authorizationcel.CELMatcher
+	name                string
 }
 
 // NewFromInterface creates a WebhookAuthorizer using the given subjectAccessReview client
-func NewFromInterface(subjectAccessReview authorizationv1client.AuthorizationV1Interface, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, metrics AuthorizerMetrics) (*WebhookAuthorizer, error) {
+func NewFromInterface(subjectAccessReview authorizationv1client.AuthorizationV1Interface, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, metrics metrics.AuthorizerMetrics) (*WebhookAuthorizer, error) {
-	return newWithBackoff(&subjectAccessReviewV1Client{subjectAccessReview.RESTClient()}, authorizedTTL, unauthorizedTTL, retryBackoff, decisionOnError, nil, metrics)
+	return newWithBackoff(&subjectAccessReviewV1Client{subjectAccessReview.RESTClient()}, authorizedTTL, unauthorizedTTL, retryBackoff, decisionOnError, nil, metrics, "")
 }
 
 // New creates a new WebhookAuthorizer from the provided kubeconfig file.
@@ -98,24 +100,26 @@ func NewFromInterface(subjectAccessReview authorizationv1client.AuthorizationV1I
 //
 // For additional HTTP configuration, refer to the kubeconfig documentation
 // https://kubernetes.io/docs/user-guide/kubeconfig-file/.
-func New(config *rest.Config, version string, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, matchConditions []apiserver.WebhookMatchCondition) (*WebhookAuthorizer, error) {
+func New(config *rest.Config, version string, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, matchConditions []apiserver.WebhookMatchCondition, name string, metrics metrics.AuthorizerMetrics) (*WebhookAuthorizer, error) {
 	subjectAccessReview, err := subjectAccessReviewInterfaceFromConfig(config, version, retryBackoff)
 	if err != nil {
 		return nil, err
 	}
-	return newWithBackoff(subjectAccessReview, authorizedTTL, unauthorizedTTL, retryBackoff, decisionOnError, matchConditions, AuthorizerMetrics{
+	return newWithBackoff(subjectAccessReview, authorizedTTL, unauthorizedTTL, retryBackoff, decisionOnError, matchConditions, metrics, name)
-		RecordRequestTotal:   noopMetrics{}.RecordRequestTotal,
-		RecordRequestLatency: noopMetrics{}.RecordRequestLatency,
-	})
 }
 
 // newWithBackoff allows tests to skip the sleep.
-func newWithBackoff(subjectAccessReview subjectAccessReviewer, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, matchConditions []apiserver.WebhookMatchCondition, metrics AuthorizerMetrics) (*WebhookAuthorizer, error) {
+func newWithBackoff(subjectAccessReview subjectAccessReviewer, authorizedTTL, unauthorizedTTL time.Duration, retryBackoff wait.Backoff, decisionOnError authorizer.Decision, matchConditions []apiserver.WebhookMatchCondition, am metrics.AuthorizerMetrics, name string) (*WebhookAuthorizer, error) {
 	// compile all expressions once in validation and save the results to be used for eval later
 	cm, fieldErr := apiservervalidation.ValidateAndCompileMatchConditions(matchConditions)
 	if err := fieldErr.ToAggregate(); err != nil {
 		return nil, err
 	}
+	if cm != nil {
+		cm.AuthorizerType = "Webhook"
+		cm.AuthorizerName = name
+		cm.Metrics = am
+	}
 	return &WebhookAuthorizer{
 		subjectAccessReview: subjectAccessReview,
 		responseCache:       cache.NewLRUExpireCache(8192),
@@ -123,8 +127,9 @@ func newWithBackoff(subjectAccessReview subjectAccessReviewer, authorizedTTL, un
 		unauthorizedTTL:     unauthorizedTTL,
 		retryBackoff:        retryBackoff,
 		decisionOnError:     decisionOnError,
-		metrics:             metrics,
+		metrics:             am,
 		celMatcher:          cm,
+		name:                name,
 	}, nil
 }
 

plugin/pkg/authorizer/webhook/webhook_v1_test.go

@@ -44,11 +44,15 @@ import (
 	"k8s.io/apiserver/pkg/apis/apiserver"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
+	celmetrics "k8s.io/apiserver/pkg/authorization/cel"
 	"k8s.io/apiserver/pkg/features"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	webhookutil "k8s.io/apiserver/pkg/util/webhook"
+	"k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics"
 	v1 "k8s.io/client-go/tools/clientcmd/api/v1"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
+	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/component-base/metrics/testutil"
 )
 
 var testRetryBackoff = wait.Backoff{
@@ -210,7 +214,7 @@ current-context: default
 			if err != nil {
 				return fmt.Errorf("error building sar client: %v", err)
 			}
-			_, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff, authorizer.DecisionNoOpinion, []apiserver.WebhookMatchCondition{}, noopAuthorizerMetrics())
+			_, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff, authorizer.DecisionNoOpinion, []apiserver.WebhookMatchCondition{}, noopAuthorizerMetrics(), "")
 			return err
 		}()
 		if err != nil && !tt.wantErr {
@@ -323,7 +327,7 @@ func (m *mockV1Service) HTTPStatusCode() int { return m.statusCode }
 
 // newV1Authorizer creates a temporary kubeconfig file from the provided arguments and attempts to load
 // a new WebhookAuthorizer from it.
-func newV1Authorizer(callbackURL string, clientCert, clientKey, ca []byte, cacheTime time.Duration, metrics AuthorizerMetrics, expressions []apiserver.WebhookMatchCondition) (*WebhookAuthorizer, error) {
+func newV1Authorizer(callbackURL string, clientCert, clientKey, ca []byte, cacheTime time.Duration, metrics metrics.AuthorizerMetrics, expressions []apiserver.WebhookMatchCondition, authzName string) (*WebhookAuthorizer, error) {
 	tempfile, err := ioutil.TempFile("", "")
 	if err != nil {
 		return nil, err
@@ -353,7 +357,7 @@ func newV1Authorizer(callbackURL string, clientCert, clientKey, ca []byte, cache
 	if err != nil {
 		return nil, fmt.Errorf("error building sar client: %v", err)
 	}
-	return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff, authorizer.DecisionNoOpinion, expressions, metrics)
+	return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff, authorizer.DecisionNoOpinion, expressions, metrics, authzName)
 }
 
 func TestV1TLSConfig(t *testing.T) {
@@ -412,7 +416,7 @@ func TestV1TLSConfig(t *testing.T) {
 			}
 			defer server.Close()
 
-			wh, err := newV1Authorizer(server.URL, tt.clientCert, tt.clientKey, tt.clientCA, 0, noopAuthorizerMetrics(), []apiserver.WebhookMatchCondition{})
+			wh, err := newV1Authorizer(server.URL, tt.clientCert, tt.clientKey, tt.clientCA, 0, noopAuthorizerMetrics(), []apiserver.WebhookMatchCondition{}, "")
 			if err != nil {
 				t.Errorf("%s: failed to create client: %v", tt.test, err)
 				return
@@ -477,7 +481,7 @@ func TestV1Webhook(t *testing.T) {
 	}
 	defer s.Close()
 
-	wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), []apiserver.WebhookMatchCondition{})
+	wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), []apiserver.WebhookMatchCondition{}, "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -584,7 +588,7 @@ func TestV1WebhookCache(t *testing.T) {
 		},
 	}
 	// Create an authorizer that caches successful responses "forever" (100 days).
-	wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 2400*time.Hour, noopAuthorizerMetrics(), expressions)
+	wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 2400*time.Hour, noopAuthorizerMetrics(), expressions, "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -760,7 +764,7 @@ func TestStructuredAuthzConfigFeatureEnablement(t *testing.T) {
 	for i, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StructuredAuthorizationConfiguration, test.featureEnabled)()
-			wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), test.expressions)
+			wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), test.expressions, "")
 			if test.expectedCompileErr && err == nil {
 				t.Fatalf("%d: Expected compile error", i)
 			} else if !test.expectedCompileErr && err != nil {
@@ -782,6 +786,112 @@ func TestStructuredAuthzConfigFeatureEnablement(t *testing.T) {
 	}
 }
 
+func TestWebhookMetrics(t *testing.T) {
+	service := new(mockV1Service)
+	service.statusCode = 200
+	service.Allow()
+	s, err := NewV1TestServer(service, serverCert, serverKey, caCert)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StructuredAuthorizationConfiguration, true)()
+
+	aliceAttr := authorizer.AttributesRecord{
+		User: &user.DefaultInfo{
+			Name: "alice",
+			UID:  "1",
+		},
+	}
+
+	testCases := []struct {
+		name         string
+		attr         authorizer.AttributesRecord
+		expressions1 []apiserver.WebhookMatchCondition
+		expressions2 []apiserver.WebhookMatchCondition
+		metrics      []string
+		want         string
+	}{
+		{
+			name: "should have one evaluation error from multiple failed match conditions",
+			attr: aliceAttr,
+			expressions1: []apiserver.WebhookMatchCondition{
+				{
+					Expression: "request.user == 'alice'",
+				},
+				{
+					Expression: "request.resourceAttributes.verb == 'get'",
+				},
+				{
+					Expression: "request.resourceAttributes.namespace == 'kittensandponies'",
+				},
+			},
+			expressions2: []apiserver.WebhookMatchCondition{
+				{
+					Expression: "request.user == 'alice'",
+				},
+			},
+			metrics: []string{
+				"apiserver_authorization_match_condition_evaluation_errors_total",
+			},
+			want: fmt.Sprintf(`
+					# HELP apiserver_authorization_match_condition_evaluation_errors_total [ALPHA] Total number of errors when an authorization webhook encounters a match condition error split by authorizer type and name.
+					# TYPE apiserver_authorization_match_condition_evaluation_errors_total counter
+					apiserver_authorization_match_condition_evaluation_errors_total{name="%s",type="%s"} 1
+					`, "wh1.example.com", "Webhook"),
+		},
+		{
+			name: "should have two webhook exclusions due to match condition",
+			attr: aliceAttr,
+			expressions1: []apiserver.WebhookMatchCondition{
+				{
+					Expression: "request.user == 'alice2'",
+				},
+				{
+					Expression: "request.uid == '1'",
+				},
+			},
+			expressions2: []apiserver.WebhookMatchCondition{
+				{
+					Expression: "request.user == 'alice1'",
+				},
+			},
+			metrics: []string{
+				"apiserver_authorization_match_condition_exclusions_total",
+			},
+			want: fmt.Sprintf(`
+					# HELP apiserver_authorization_match_condition_exclusions_total [ALPHA] Total number of exclusions when an authorization webhook is skipped because match conditions exclude it.
+					# TYPE apiserver_authorization_match_condition_exclusions_total counter
+					apiserver_authorization_match_condition_exclusions_total{name="%s",type="%s"} 1
+					apiserver_authorization_match_condition_exclusions_total{name="%s",type="%s"} 1
+					`, "wh1.example.com", "Webhook", "wh2.example.com", "Webhook"),
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			celmetrics.ResetMetricsForTest()
+			defer celmetrics.ResetMetricsForTest()
+			wh1, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, celAuthorizerMetrics(), tt.expressions1, "wh1.example.com")
+			if err != nil {
+				t.Fatal(err)
+			}
+			wh2, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, celAuthorizerMetrics(), tt.expressions2, "wh2.example.com")
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err == nil {
+				_, _, _ = wh1.Authorize(context.Background(), tt.attr)
+				_, _, _ = wh2.Authorize(context.Background(), tt.attr)
+			}
+
+			if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), tt.metrics...); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
 func BenchmarkNoCELExpressionFeatureOff(b *testing.B) {
 	expressions := []apiserver.WebhookMatchCondition{}
 	b.Run("compile", func(b *testing.B) {
@@ -942,7 +1052,7 @@ func benchmarkNewWebhookAuthorizer(b *testing.B, expressions []apiserver.Webhook
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		// Create an authorizer with or without expressions to compile
-		_, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), expressions)
+		_, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), expressions, "")
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -972,7 +1082,7 @@ func benchmarkWebhookAuthorize(b *testing.B, expressions []apiserver.WebhookMatc
 	defer s.Close()
 	defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.StructuredAuthorizationConfiguration, featureEnabled)()
 	// Create an authorizer with or without expressions to compile
-	wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), expressions)
+	wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), expressions, "")
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -1259,7 +1369,7 @@ func TestV1WebhookMatchConditions(t *testing.T) {
 
 	for i, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), test.expressions)
+			wh, err := newV1Authorizer(s.URL, clientCert, clientKey, caCert, 0, noopAuthorizerMetrics(), test.expressions, "")
 			if len(test.expectedCompileErr) > 0 && err == nil {
 				t.Fatalf("%d: Expected compile error", i)
 			} else if len(test.expectedCompileErr) == 0 && err != nil {
@@ -1292,9 +1402,17 @@ func TestV1WebhookMatchConditions(t *testing.T) {
 	}
 }
 
-func noopAuthorizerMetrics() AuthorizerMetrics {
+func noopAuthorizerMetrics() metrics.AuthorizerMetrics {
-	return AuthorizerMetrics{
+	return metrics.NoopAuthorizerMetrics{}
-		RecordRequestTotal:   noopMetrics{}.RecordRequestTotal,
+}
-		RecordRequestLatency: noopMetrics{}.RecordRequestLatency,
+
+func celAuthorizerMetrics() metrics.AuthorizerMetrics {
+	return celAuthorizerMetricsType{
+		MatcherMetrics: celmetrics.NewMatcherMetrics(),
 	}
 }
+
+type celAuthorizerMetricsType struct {
+	metrics.NoopRequestMetrics
+	celmetrics.MatcherMetrics
+}

plugin/pkg/authorizer/webhook/webhook_v1beta1_test.go

@@ -197,7 +197,7 @@ current-context: default
 			if err != nil {
 				return fmt.Errorf("error building sar client: %v", err)
 			}
-			_, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff, authorizer.DecisionNoOpinion, []authzconfig.WebhookMatchCondition{}, noopAuthorizerMetrics())
+			_, err = newWithBackoff(sarClient, 0, 0, testRetryBackoff, authorizer.DecisionNoOpinion, []authzconfig.WebhookMatchCondition{}, noopAuthorizerMetrics(), "")
 			return err
 		}()
 		if err != nil && !tt.wantErr {
@@ -340,7 +340,7 @@ func newV1beta1Authorizer(callbackURL string, clientCert, clientKey, ca []byte,
 	if err != nil {
 		return nil, fmt.Errorf("error building sar client: %v", err)
 	}
-	return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff, authorizer.DecisionNoOpinion, []authzconfig.WebhookMatchCondition{}, noopAuthorizerMetrics())
+	return newWithBackoff(sarClient, cacheTime, cacheTime, testRetryBackoff, authorizer.DecisionNoOpinion, []authzconfig.WebhookMatchCondition{}, noopAuthorizerMetrics(), "")
 }
 
 func TestV1beta1TLSConfig(t *testing.T) {