diff --git a/go.mod b/go.mod index 9afed5bc1..41ffad96d 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( gopkg.in/square/go-jose.v2 v2.2.2 k8s.io/api v0.0.0-20221028075226-689257039cfb k8s.io/apimachinery v0.0.0-20221028155017-b03a432a2a6d - k8s.io/client-go v0.0.0-20221028155553-a232cf00b972 + k8s.io/client-go v0.0.0-20221028155554-0d5739633518 k8s.io/component-base v0.0.0-20221028160413-57523092d0c2 k8s.io/klog/v2 v2.80.1 k8s.io/kms v0.0.0-20221028080743-a9ba1c11c0c6 @@ -124,7 +124,7 @@ require ( replace ( k8s.io/api => k8s.io/api v0.0.0-20221028075226-689257039cfb k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20221028155017-b03a432a2a6d - k8s.io/client-go => k8s.io/client-go v0.0.0-20221028155553-a232cf00b972 + k8s.io/client-go => k8s.io/client-go v0.0.0-20221028155554-0d5739633518 k8s.io/component-base => k8s.io/component-base v0.0.0-20221028160413-57523092d0c2 k8s.io/kms => k8s.io/kms v0.0.0-20221028080743-a9ba1c11c0c6 ) diff --git a/go.sum b/go.sum index 5d72ad802..cefb162f0 100644 --- a/go.sum +++ b/go.sum @@ -989,8 +989,8 @@ k8s.io/api v0.0.0-20221028075226-689257039cfb h1:QZsHf1k3xHSbYozSHt8DZOrH36F8Mbd k8s.io/api v0.0.0-20221028075226-689257039cfb/go.mod h1:96woDPRgmgH0tVTXgsLdv0cwHXPUvZ97ghr8sMUtxx0= k8s.io/apimachinery v0.0.0-20221028155017-b03a432a2a6d h1:fg/DbLqFKxFESf3AnU5iwCexZWOUnFFLb0JraG20wZo= k8s.io/apimachinery v0.0.0-20221028155017-b03a432a2a6d/go.mod h1:zSkBXgO5G/dSQOe256tx5Yo2OJytojpY3bsXu/4/ZJE= -k8s.io/client-go v0.0.0-20221028155553-a232cf00b972 h1:Xhhs4dBca/3rT0is6GDMtFjKyviLZkdlw1xRwvYFsww= -k8s.io/client-go v0.0.0-20221028155553-a232cf00b972/go.mod h1:9OZTm80DH1AI7P4cpx8yehVlTU1xZQCsMtAtlJYLWDw= +k8s.io/client-go v0.0.0-20221028155554-0d5739633518 h1:KlSjZkXeVyocbVpEU157nadMdQIfWXchaItVtjmMVUE= +k8s.io/client-go v0.0.0-20221028155554-0d5739633518/go.mod h1:9OZTm80DH1AI7P4cpx8yehVlTU1xZQCsMtAtlJYLWDw= k8s.io/component-base v0.0.0-20221028160413-57523092d0c2 h1:lHXBae/IFHPh8Lu9rDRTzt6KyTiBkKvRJWeLTYpXFDA= k8s.io/component-base v0.0.0-20221028160413-57523092d0c2/go.mod h1:ng9M2gjWKyA3UKzYYImYBFvpk731uuzazqB7Ti2cKmA= k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= diff --git a/pkg/admission/cel/metrics.go b/pkg/admission/cel/metrics.go new file mode 100644 index 000000000..fe14d0682 --- /dev/null +++ b/pkg/admission/cel/metrics.go @@ -0,0 +1,105 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cel + +import ( + "context" + "time" + + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" +) + +const ( + metricsNamespace = "apiserver" + metricsSubsystem = "validating_admission_policy" +) + +var ( + // Metrics provides access to validation admission metrics. + Metrics = newValidationAdmissionMetrics() +) + +// ValidatingAdmissionPolicyMetrics aggregates Prometheus metrics related to validation admission control. +type ValidatingAdmissionPolicyMetrics struct { + policyCheck *metrics.CounterVec + policyDefinition *metrics.CounterVec + policyLatency *metrics.HistogramVec +} + +func newValidationAdmissionMetrics() *ValidatingAdmissionPolicyMetrics { + check := metrics.NewCounterVec( + &metrics.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "check_total", + Help: "Validation admission policy check total, labeled by policy and param resource, and further identified by binding, validation expression, enforcement action taken, and state.", + StabilityLevel: metrics.ALPHA, + }, + []string{"policy", "policy_binding", "validation_expression", "enforcement_action", "params", "state"}, + ) + definition := metrics.NewCounterVec(&metrics.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "definition_total", + Help: "Validation admission policy count total, labeled by state and enforcement action.", + StabilityLevel: metrics.ALPHA, + }, + []string{"state", "enforcement_action"}, + ) + latency := metrics.NewHistogramVec(&metrics.HistogramOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "check_duration_seconds", + Help: "Validation admission latency for individual validation expressions in seconds, labeled by policy and param resource, further including binding, state and enforcement action taken.", + // the bucket distribution here is based oo the benchmark suite at + // github.com/DangerOnTheRanger/cel-benchmark performed on 16-core Intel Xeon + // the lowest bucket was based around the 180ns/op figure for BenchmarkAccess, + // plus some additional leeway to account for the apiserver doing other things + // the largest bucket was chosen based on the fact that benchmarks indicate the + // same Xeon running a CEL expression close to the estimated cost limit takes + // around 760ms, so that bucket should only ever have the slowest CEL expressions + // in it + Buckets: []float64{0.0000005, 0.001, 0.01, 0.1, 1.0}, + StabilityLevel: metrics.ALPHA, + }, + []string{"policy", "policy_binding", "validation_expression", "enforcement_action", "params", "state"}, + ) + + legacyregistry.MustRegister(check) + legacyregistry.MustRegister(definition) + legacyregistry.MustRegister(latency) + return &ValidatingAdmissionPolicyMetrics{policyCheck: check, policyDefinition: definition, policyLatency: latency} +} + +// Reset resets all validation admission-related Prometheus metrics. +func (m *ValidatingAdmissionPolicyMetrics) Reset() { + m.policyCheck.Reset() + m.policyDefinition.Reset() + m.policyLatency.Reset() +} + +// ObserveDefinition observes a policy definition. +func (m *ValidatingAdmissionPolicyMetrics) ObserveDefinition(ctx context.Context, state, enforcementAction string) { + m.policyDefinition.WithContext(ctx).WithLabelValues(state, enforcementAction).Inc() +} + +// ObserveCheck observes a policy validation check. +func (m *ValidatingAdmissionPolicyMetrics) ObserveCheck(ctx context.Context, elapsed time.Duration, policy, binding, expression, enforcementAction, params, state string) { + m.policyCheck.WithContext(ctx).WithLabelValues(policy, binding, expression, enforcementAction, params, state).Inc() + m.policyLatency.WithContext(ctx).WithLabelValues(policy, binding, expression, enforcementAction, params, state).Observe(elapsed.Seconds()) +} diff --git a/pkg/admission/cel/metrics_test.go b/pkg/admission/cel/metrics_test.go new file mode 100644 index 000000000..2820b3a1d --- /dev/null +++ b/pkg/admission/cel/metrics_test.go @@ -0,0 +1,88 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cel + +import ( + "context" + "strings" + "testing" + "time" + + "k8s.io/component-base/metrics/legacyregistry" + "k8s.io/component-base/metrics/testutil" +) + +type metricsObserver func() + +func TestNoUtils(t *testing.T) { + + metrics := []string{ + "apiserver_validating_admission_policy_definition_total", + "apiserver_validating_admission_policy_check_total", + "apiserver_validating_admission_policy_check_duration_seconds", + } + + testCases := []struct { + desc string + want string + observer metricsObserver + }{ + { + desc: "observe policy check", + want: ` + # HELP apiserver_validating_admission_policy_check_duration_seconds [ALPHA] Validation admission latency for individual validation expressions in seconds, labeled by policy and param resource, further including binding, state and enforcement action taken. + # TYPE apiserver_validating_admission_policy_check_duration_seconds histogram + apiserver_validating_admission_policy_check_duration_seconds_bucket{enforcement_action="allow",params="params.example.com",policy="policy.example.com",policy_binding="binding.example.com",state="active",validation_expression="true",le="0.0000005"} 0 + apiserver_validating_admission_policy_check_duration_seconds_bucket{enforcement_action="allow",params="params.example.com",policy="policy.example.com",policy_binding="binding.example.com",state="active",validation_expression="true",le="0.001"} 0 + apiserver_validating_admission_policy_check_duration_seconds_bucket{enforcement_action="allow",params="params.example.com",policy="policy.example.com",policy_binding="binding.example.com",state="active",validation_expression="true",le="0.01"} 0 + apiserver_validating_admission_policy_check_duration_seconds_bucket{enforcement_action="allow",params="params.example.com",policy="policy.example.com",policy_binding="binding.example.com",state="active",validation_expression="true",le="0.1"} 0 + apiserver_validating_admission_policy_check_duration_seconds_bucket{enforcement_action="allow",params="params.example.com",policy="policy.example.com",policy_binding="binding.example.com",state="active",validation_expression="true",le="1"} 0 + apiserver_validating_admission_policy_check_duration_seconds_bucket{enforcement_action="allow",params="params.example.com",policy="policy.example.com",policy_binding="binding.example.com",state="active",validation_expression="true",le="+Inf"} 1 + apiserver_validating_admission_policy_check_duration_seconds_sum{enforcement_action="allow",params="params.example.com",policy="policy.example.com",policy_binding="binding.example.com",state="active",validation_expression="true"} 10 + apiserver_validating_admission_policy_check_duration_seconds_count{enforcement_action="allow",params="params.example.com",policy="policy.example.com",policy_binding="binding.example.com",state="active",validation_expression="true"} 1 + # HELP apiserver_validating_admission_policy_check_total [ALPHA] Validation admission policy check total, labeled by policy and param resource, and further identified by binding, validation expression, enforcement action taken, and state. + # TYPE apiserver_validating_admission_policy_check_total counter + apiserver_validating_admission_policy_check_total{enforcement_action="allow",params="params.example.com",policy="policy.example.com",policy_binding="binding.example.com",state="active",validation_expression="true"} 1 + `, + observer: func() { + Metrics.ObserveCheck(context.TODO(), time.Duration(10)*time.Second, "policy.example.com", "binding.example.com", "true", "allow", "params.example.com", "active") + }, + }, + { + desc: "observe policy definition", + want: `# HELP apiserver_validating_admission_policy_definition_total [ALPHA] Validation admission policy count total, labeled by state and enforcement action. + # TYPE apiserver_validating_admission_policy_definition_total counter + apiserver_validating_admission_policy_definition_total{enforcement_action="deny",state="active"} 1 + `, + observer: func() { + Metrics.ObserveDefinition(context.TODO(), "active", "deny") + }, + }, + } + + Metrics.Reset() + + for _, tt := range testCases { + t.Run(tt.desc, func(t *testing.T) { + defer Metrics.Reset() + tt.observer() + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tt.want), metrics...); err != nil { + t.Fatal(err) + } + }) + } +}