Merge pull request #119008 from nilekhc/hotreload-update-metrics
[KMSv2] feat: implements metrics for encryption config hot reload Kubernetes-commit: e3bc35bc1bc7b399b564f2c4efc75eb9959e70f7
This commit is contained in:
Kubernetes Publisher
commit
b928ae8c42
|
|
@ -27,6 +27,7 @@ import (
|
|||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/apiserver/pkg/server/healthz"
|
||||
"k8s.io/apiserver/pkg/server/options/encryptionconfig"
|
||||
"k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics"
|
||||
"k8s.io/client-go/util/workqueue"
|
||||
"k8s.io/klog/v2"
|
||||
)
|
||||
|
|
@ -163,16 +164,19 @@ func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem(serverCtx contex
|
|||
ctx, closeTransformers := context.WithCancel(serverCtx)
|
||||
|
||||
defer func() {
|
||||
// TODO: increment success metric when updatedEffectiveConfig=true
|
||||
|
||||
// TODO can work queue metrics help here?
|
||||
|
||||
if !updatedEffectiveConfig {
|
||||
// avoid leaking if we're not using the newly constructed transformers (due to an error or them not being changed)
|
||||
closeTransformers()
|
||||
}
|
||||
|
||||
if updatedEffectiveConfig && err == nil {
|
||||
metrics.RecordEncryptionConfigAutomaticReloadSuccess()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
// TODO: increment failure metric
|
||||
metrics.RecordEncryptionConfigAutomaticReloadFailure()
|
||||
utilruntime.HandleError(fmt.Errorf("error processing encryption config file %s: %v", d.filePath, err))
|
||||
// add dummy item back to the queue to trigger file content processing.
|
||||
d.queue.AddRateLimited(key)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,86 @@
|
|||
/*
|
||||
Copyright 2023 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"k8s.io/component-base/metrics"
|
||||
"k8s.io/component-base/metrics/legacyregistry"
|
||||
)
|
||||
|
||||
const (
|
||||
namespace = "apiserver"
|
||||
subsystem = "encryption_config_controller"
|
||||
)
|
||||
|
||||
var (
|
||||
encryptionConfigAutomaticReloadFailureTotal = metrics.NewCounter(
|
||||
&metrics.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "automatic_reload_failures_total",
|
||||
Help: "Total number of failed automatic reloads of encryption configuration.",
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
)
|
||||
|
||||
encryptionConfigAutomaticReloadSuccessTotal = metrics.NewCounter(
|
||||
&metrics.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "automatic_reload_success_total",
|
||||
Help: "Total number of successful automatic reloads of encryption configuration.",
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
)
|
||||
|
||||
encryptionConfigAutomaticReloadLastTimestampSeconds = metrics.NewGaugeVec(
|
||||
&metrics.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "automatic_reload_last_timestamp_seconds",
|
||||
Help: "Timestamp of the last successful or failed automatic reload of encryption configuration.",
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
[]string{"status"},
|
||||
)
|
||||
)
|
||||
|
||||
var registerMetrics sync.Once
|
||||
|
||||
func RegisterMetrics() {
|
||||
registerMetrics.Do(func() {
|
||||
legacyregistry.MustRegister(encryptionConfigAutomaticReloadFailureTotal)
|
||||
legacyregistry.MustRegister(encryptionConfigAutomaticReloadSuccessTotal)
|
||||
legacyregistry.MustRegister(encryptionConfigAutomaticReloadLastTimestampSeconds)
|
||||
})
|
||||
}
|
||||
|
||||
func RecordEncryptionConfigAutomaticReloadFailure() {
|
||||
encryptionConfigAutomaticReloadFailureTotal.Inc()
|
||||
recordEncryptionConfigAutomaticReloadTimestamp("failure")
|
||||
}
|
||||
|
||||
func RecordEncryptionConfigAutomaticReloadSuccess() {
|
||||
encryptionConfigAutomaticReloadSuccessTotal.Inc()
|
||||
recordEncryptionConfigAutomaticReloadTimestamp("success")
|
||||
}
|
||||
|
||||
func recordEncryptionConfigAutomaticReloadTimestamp(result string) {
|
||||
encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result).SetToCurrentTime()
|
||||
}
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
/*
|
||||
Copyright 2023 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"k8s.io/component-base/metrics/legacyregistry"
|
||||
"k8s.io/component-base/metrics/testutil"
|
||||
)
|
||||
|
||||
func TestRecordEncryptionConfigAutomaticReloadFailure(t *testing.T) {
|
||||
expectedValue := `
|
||||
# HELP apiserver_encryption_config_controller_automatic_reload_failures_total [ALPHA] Total number of failed automatic reloads of encryption configuration.
|
||||
# TYPE apiserver_encryption_config_controller_automatic_reload_failures_total counter
|
||||
apiserver_encryption_config_controller_automatic_reload_failures_total 1
|
||||
`
|
||||
metrics := []string{
|
||||
namespace + "_" + subsystem + "_automatic_reload_failures_total",
|
||||
}
|
||||
|
||||
encryptionConfigAutomaticReloadFailureTotal.Reset()
|
||||
RegisterMetrics()
|
||||
|
||||
RecordEncryptionConfigAutomaticReloadFailure()
|
||||
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordEncryptionConfigAutomaticReloadSuccess(t *testing.T) {
|
||||
expectedValue := `
|
||||
# HELP apiserver_encryption_config_controller_automatic_reload_success_total [ALPHA] Total number of successful automatic reloads of encryption configuration.
|
||||
# TYPE apiserver_encryption_config_controller_automatic_reload_success_total counter
|
||||
apiserver_encryption_config_controller_automatic_reload_success_total 1
|
||||
`
|
||||
metrics := []string{
|
||||
namespace + "_" + subsystem + "_automatic_reload_success_total",
|
||||
}
|
||||
|
||||
encryptionConfigAutomaticReloadSuccessTotal.Reset()
|
||||
RegisterMetrics()
|
||||
|
||||
RecordEncryptionConfigAutomaticReloadSuccess()
|
||||
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptionConfigAutomaticReloadLastTimestampSeconds(t *testing.T) {
|
||||
testCases := []struct {
|
||||
expectedValue string
|
||||
resultLabel string
|
||||
timestamp int64
|
||||
}{
|
||||
{
|
||||
expectedValue: `
|
||||
# HELP apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last successful or failed automatic reload of encryption configuration.
|
||||
# TYPE apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds gauge
|
||||
apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{status="failure"} 1.689101941e+09
|
||||
`,
|
||||
resultLabel: "failure",
|
||||
timestamp: 1689101941,
|
||||
},
|
||||
{
|
||||
expectedValue: `
|
||||
# HELP apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds [ALPHA] Timestamp of the last successful or failed automatic reload of encryption configuration.
|
||||
# TYPE apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds gauge
|
||||
apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{status="success"} 1.689101941e+09
|
||||
`,
|
||||
resultLabel: "success",
|
||||
timestamp: 1689101941,
|
||||
},
|
||||
}
|
||||
|
||||
metrics := []string{
|
||||
namespace + "_" + subsystem + "_automatic_reload_last_timestamp_seconds",
|
||||
}
|
||||
RegisterMetrics()
|
||||
|
||||
for _, tc := range testCases {
|
||||
encryptionConfigAutomaticReloadLastTimestampSeconds.Reset()
|
||||
encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(tc.resultLabel).Set(float64(tc.timestamp))
|
||||
|
||||
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tc.expectedValue), metrics...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue