kubernetes
/
kube-state-metrics
mirror of https://github.com/kubernetes/kube-state-metrics.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

*: add webhooks client config service metrics 

In order to better identify, prioritize, and debug webhook latency
issues it is important to have a metric that would point to the resource
it is responsible for.  However, it is not possible to have that
dimension in the metrics exposed by Kubernetes because of the unbound
cardinality that such a label would have.

The name of the webhook could be an alternative since it usually
contains some information about the resource that the webhook targets,
however this is not very practical to use in multi-tenants
environments.

A solution for these kind of platform is to tie a specific webhook to a
namespace in order to be able to know which tenant manages it and take
actions depending on that. This is achieveable by leveraging the client
config information of webhooks configured via WebhookConfiguration
resources since Services are namespaced objects.

With these new metrics, users will be able to split the alerting
severity of webhook latency / rejection rate per namespace on top of
being able to do it based on the webhook name. This is key in
environment where administrators don't have control over the webhooks
installed by the various tenants.

Signed-off-by: Damien Grisonnet <dgrisonn@redhat.com>
This commit is contained in:
Damien Grisonnet 2023-07-04 17:56:28 +02:00
parent 31d6e8fc26
commit 229befd41d
6 changed files with 118 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/mutatingwebhookconfiguration-metrics.md
View File

@ -5,3 +5,4 @@
| kube_mutatingwebhookconfiguration_info | Gauge | `mutatingwebhookconfiguration`=&lt;mutatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;mutatingwebhookconfiguration-namespace&gt; | EXPERIMENTAL |
| kube_mutatingwebhookconfiguration_created | Gauge | `mutatingwebhookconfiguration`=&lt;mutatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;mutatingwebhookconfiguration-namespace&gt; | EXPERIMENTAL |
| kube_mutatingwebhookconfiguration_metadata_resource_version | Gauge | `mutatingwebhookconfiguration`=&lt;mutatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;mutatingwebhookconfiguration-namespace&gt; | EXPERIMENTAL |
| kube_mutatingwebhookconfiguration_webhook_clientconfig_service | Gauge | `mutatingwebhookconfiguration`=&lt;mutatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;mutatingwebhookconfiguration-namespace&gt; <br> `webhook_name`=&lt;webhook-name&gt; <br> `service_name`=&lt;webhook-service-name&gt; <br> `service_namespace`=&lt;webhook-service-namespace&gt;| EXPERIMENTAL |

1
docs/validatingwebhookconfiguration-metrics.md
View File

@ -5,3 +5,4 @@
| kube_validatingwebhookconfiguration_info | Gauge | `validatingwebhookconfiguration`=&lt;validatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;validatingwebhookconfiguration-namespace&gt; | EXPERIMENTAL |
| kube_validatingwebhookconfiguration_created | Gauge | `validatingwebhookconfiguration`=&lt;validatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;validatingwebhookconfiguration-namespace&gt; | EXPERIMENTAL |
| kube_validatingwebhookconfiguration_metadata_resource_version | Gauge | `validatingwebhookconfiguration`=&lt;validatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;validatingwebhookconfiguration-namespace&gt; | EXPERIMENTAL |
| kube_validatingwebhookconfiguration_webhook_clientconfig_service | Gauge | `validatingwebhookconfiguration`=&lt;validatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;validatingwebhookconfiguration-namespace&gt; <br> `webhook_name`=&lt;webhook-name&gt; <br> `service_name`=&lt;webhook-service-name&gt; <br> `service_namespace`=&lt;webhook-service-namespace&gt;| EXPERIMENTAL |

26
internal/store/mutatingwebhookconfiguration.go
View File

@ -82,6 +82,32 @@ var (
}
}),
),
*generator.NewFamilyGeneratorWithStability(
"kube_mutatingwebhookconfiguration_webhook_clientconfig_service",
"Service used by the apiserver to connect to a mutating webhook.",
metric.Gauge,
basemetrics.ALPHA,
"",
wrapMutatingWebhookConfigurationFunc(func(mwc *admissionregistrationv1.MutatingWebhookConfiguration) *metric.Family {
ms := []*metric.Metric{}
for _, webhook := range mwc.Webhooks {
var serviceName, serviceNamespace string
if webhook.ClientConfig.Service != nil {
serviceName = webhook.ClientConfig.Service.Name
serviceNamespace = webhook.ClientConfig.Service.Namespace
}
ms = append(ms, &metric.Metric{
LabelKeys: []string{"webhook_name", "service_name", "service_namespace"},
LabelValues: []string{webhook.Name, serviceName, serviceNamespace},
Value: 1,
})
}
return &metric.Family{
Metrics: ms,
}
}),
),
}
)

32
internal/store/mutatingwebhookconfiguration_test.go
View File

@ -28,6 +28,7 @@ import (
func TestMutatingWebhookConfigurationStore(t *testing.T) {
startTime := 1501569018
metav1StartTime := metav1.Unix(int64(startTime), 0)
externalURL := "example.com"
cases := []generateMetricsTestCase{
{
@ -69,6 +70,37 @@ func TestMutatingWebhookConfigurationStore(t *testing.T) {
`,
MetricNames: []string{"kube_mutatingwebhookconfiguration_created", "kube_mutatingwebhookconfiguration_info", "kube_mutatingwebhookconfiguration_metadata_resource_version"},
},
{
Obj: &admissionregistrationv1.MutatingWebhookConfiguration{
ObjectMeta: metav1.ObjectMeta{
Name: "mutatingwebhookconfiguration3",
Namespace: "ns3",
CreationTimestamp: metav1StartTime,
ResourceVersion: "abcdef",
},
Webhooks: []admissionregistrationv1.MutatingWebhook{
{
Name: "webhook_with_service",
ClientConfig: admissionregistrationv1.WebhookClientConfig{
Service: &admissionregistrationv1.ServiceReference{Name: "svc", Namespace: "ns"},
},
},
{
Name: "webhook_with_external_url",
ClientConfig: admissionregistrationv1.WebhookClientConfig{
URL: &externalURL,
},
},
},
},
Want: `
# HELP kube_mutatingwebhookconfiguration_webhook_clientconfig_service Service used by the apiserver to connect to a mutating webhook.
# TYPE kube_mutatingwebhookconfiguration_webhook_clientconfig_service gauge
kube_mutatingwebhookconfiguration_webhook_clientconfig_service{webhook_name="webhook_with_external_url",namespace="ns3",service_name="",service_namespace="",mutatingwebhookconfiguration="mutatingwebhookconfiguration3"} 1
kube_mutatingwebhookconfiguration_webhook_clientconfig_service{webhook_name="webhook_with_service",namespace="ns3",service_name="svc",service_namespace="ns",mutatingwebhookconfiguration="mutatingwebhookconfiguration3"} 1
`,
MetricNames: []string{"kube_mutatingwebhookconfiguration_webhook_clientconfig_service"},
},
}
for i, c := range cases {
c.Func = generator.ComposeMetricGenFuncs(mutatingWebhookConfigurationMetricFamilies)

26
internal/store/validatingwebhookconfiguration.go
View File

@ -82,6 +82,32 @@ var (
}
}),
),
*generator.NewFamilyGeneratorWithStability(
"kube_validatingwebhookconfiguration_webhook_clientconfig_service",
"Service used by the apiserver to connect to a validating webhook.",
metric.Gauge,
basemetrics.ALPHA,
"",
wrapValidatingWebhookConfigurationFunc(func(vwc *admissionregistrationv1.ValidatingWebhookConfiguration) *metric.Family {
ms := []*metric.Metric{}
for _, webhook := range vwc.Webhooks {
var serviceName, serviceNamespace string
if webhook.ClientConfig.Service != nil {
serviceName = webhook.ClientConfig.Service.Name
serviceNamespace = webhook.ClientConfig.Service.Namespace
}
ms = append(ms, &metric.Metric{
LabelKeys: []string{"webhook_name", "service_name", "service_namespace"},
LabelValues: []string{webhook.Name, serviceName, serviceNamespace},
Value: 1,
})
}
return &metric.Family{
Metrics: ms,
}
}),
),
}
)

32
internal/store/validatingwebhookconfiguration_test.go
View File

@ -28,6 +28,7 @@ import (
func TestValidatingWebhookConfigurationStore(t *testing.T) {
startTime := 1501569018
metav1StartTime := metav1.Unix(int64(startTime), 0)
externalURL := "example.com"
cases := []generateMetricsTestCase{
{
@ -69,6 +70,37 @@ func TestValidatingWebhookConfigurationStore(t *testing.T) {
`,
MetricNames: []string{"kube_validatingwebhookconfiguration_created", "kube_validatingwebhookconfiguration_info", "kube_validatingwebhookconfiguration_metadata_resource_version"},
},
{
Obj: &admissionregistrationv1.ValidatingWebhookConfiguration{
ObjectMeta: metav1.ObjectMeta{
Name: "validatingwebhookconfiguration3",
Namespace: "ns3",
CreationTimestamp: metav1StartTime,
ResourceVersion: "abcdef",
},
Webhooks: []admissionregistrationv1.ValidatingWebhook{
{
Name: "webhook_with_service",
ClientConfig: admissionregistrationv1.WebhookClientConfig{
Service: &admissionregistrationv1.ServiceReference{Name: "svc", Namespace: "ns"},
},
},
{
Name: "webhook_with_external_url",
ClientConfig: admissionregistrationv1.WebhookClientConfig{
URL: &externalURL,
},
},
},
},
Want: `
# HELP kube_validatingwebhookconfiguration_webhook_clientconfig_service Service used by the apiserver to connect to a validating webhook.
# TYPE kube_validatingwebhookconfiguration_webhook_clientconfig_service gauge
kube_validatingwebhookconfiguration_webhook_clientconfig_service{webhook_name="webhook_with_external_url",namespace="ns3",service_name="",service_namespace="",validatingwebhookconfiguration="validatingwebhookconfiguration3"} 1
kube_validatingwebhookconfiguration_webhook_clientconfig_service{webhook_name="webhook_with_service",namespace="ns3",service_name="svc",service_namespace="ns",validatingwebhookconfiguration="validatingwebhookconfiguration3"} 1
`,
MetricNames: []string{"kube_validatingwebhookconfiguration_webhook_clientconfig_service"},
},
}
for i, c := range cases {
c.Func = generator.ComposeMetricGenFuncs(validatingWebhookConfigurationMetricFamilies)