Add automatic_reload_last_config_info metric for auth configs

Signed-off-by: Anish Ramasekar <anish.ramasekar@gmail.com>

Kubernetes-commit: 21e2fcea9ec25c79d4ceea0c8a931c2655bf5d93
This commit is contained in:
Anish Ramasekar 2025-06-11 13:13:04 -07:00 committed by Kubernetes Publisher
parent f6d9871507
commit 31fd0fd4cd
11 changed files with 237 additions and 29 deletions

View File

@ -21,6 +21,7 @@ import (
"fmt"
"sync"
"k8s.io/apiserver/pkg/util/configmetrics"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)
@ -52,14 +53,25 @@ var (
},
[]string{"status", "apiserver_id_hash"},
)
authenticationConfigLastConfigInfo = metrics.NewDesc(
metrics.BuildFQName(namespace, subsystem, "last_config_info"),
"Information about the last applied authentication configuration with hash as label, split by apiserver identity.",
[]string{"apiserver_id_hash", "hash"},
nil,
metrics.ALPHA,
"",
)
)
var registerMetrics sync.Once
var configHashProvider = configmetrics.NewAtomicHashProvider()
func RegisterMetrics() {
registerMetrics.Do(func() {
legacyregistry.MustRegister(authenticationConfigAutomaticReloadsTotal)
legacyregistry.MustRegister(authenticationConfigAutomaticReloadLastTimestampSeconds)
legacyregistry.CustomMustRegister(configmetrics.NewConfigInfoCustomCollector(authenticationConfigLastConfigInfo, configHashProvider))
})
}
@ -74,10 +86,16 @@ func RecordAuthenticationConfigAutomaticReloadFailure(apiServerID string) {
authenticationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("failure", apiServerIDHash).SetToCurrentTime()
}
func RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID string) {
func RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID, authConfigData string) {
apiServerIDHash := getHash(apiServerID)
authenticationConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc()
authenticationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("success", apiServerIDHash).SetToCurrentTime()
RecordAuthenticationConfigLastConfigInfo(apiServerID, authConfigData)
}
func RecordAuthenticationConfigLastConfigInfo(apiServerID, authConfigData string) {
configHashProvider.SetHashes(getHash(apiServerID), getHash(authConfigData))
}
func getHash(data string) string {

View File

@ -27,6 +27,18 @@ import (
const (
testAPIServerID = "testAPIServerID"
testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"
testConfigData = `
apiVersion: apiserver.config.k8s.io/v1
kind: AuthenticationConfiguration
jwt:
- issuer:
url: https://test-issuer
audiences: [ "aud" ]
claimMappings:
username:
claim: sub
prefix: ""
`
)
func TestRecordAuthenticationConfigAutomaticReloadFailure(t *testing.T) {
@ -53,15 +65,19 @@ func TestRecordAuthenticationConfigAutomaticReloadSuccess(t *testing.T) {
# HELP apiserver_authentication_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authentication configuration split by status and apiserver identity.
# TYPE apiserver_authentication_config_controller_automatic_reloads_total counter
apiserver_authentication_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1
# HELP apiserver_authentication_config_controller_last_config_info [ALPHA] Information about the last applied authentication configuration with hash as label, split by apiserver identity.
# TYPE apiserver_authentication_config_controller_last_config_info gauge
apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:ccbcaf98557c273dfc779222d54a5bd3e785ea5330048f3bf4278cf3997b669c"} 1
`
metrics := []string{
namespace + "_" + subsystem + "_automatic_reloads_total",
namespace + "_" + subsystem + "_last_config_info",
}
authenticationConfigAutomaticReloadsTotal.Reset()
ResetMetricsForTest()
RegisterMetrics()
RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID)
RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID, testConfigData)
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
t.Fatal(err)
}
@ -107,3 +123,34 @@ func TestAuthenticationConfigAutomaticReloadLastTimestampSeconds(t *testing.T) {
}
}
}
func TestRecordAuthenticationConfigAutomaticReloadSuccess_StaleMetricCleanup(t *testing.T) {
ResetMetricsForTest()
RegisterMetrics()
// Record initial success with first config
firstConfig := "config1"
RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID, firstConfig)
// Record success with different config - should clean up old metric
secondConfig := "config2"
RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID, secondConfig)
// Verify only the latest hash is present
expectedValue := `
# HELP apiserver_authentication_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authentication configuration split by status and apiserver identity.
# TYPE apiserver_authentication_config_controller_automatic_reloads_total counter
apiserver_authentication_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 2
# HELP apiserver_authentication_config_controller_last_config_info [ALPHA] Information about the last applied authentication configuration with hash as label, split by apiserver identity.
# TYPE apiserver_authentication_config_controller_last_config_info gauge
apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:f309dd9c31fe24b3e594d2f9420419c48dfe954523245d5f35dc37739970d881"} 1
`
metrics := []string{
namespace + "_" + subsystem + "_automatic_reloads_total",
namespace + "_" + subsystem + "_last_config_info",
}
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
t.Fatal(err)
}
}

View File

@ -22,6 +22,7 @@ import (
"hash"
"sync"
"k8s.io/apiserver/pkg/util/configmetrics"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)
@ -53,10 +54,20 @@ var (
},
[]string{"status", "apiserver_id_hash"},
)
authorizationConfigLastConfigInfo = metrics.NewDesc(
metrics.BuildFQName(namespace, subsystem, "last_config_info"),
"Information about the last applied authorization configuration with hash as label, split by apiserver identity.",
[]string{"apiserver_id_hash", "hash"},
nil,
metrics.ALPHA,
"",
)
)
var registerMetrics sync.Once
var hashPool *sync.Pool
var configHashProvider = configmetrics.NewAtomicHashProvider()
func RegisterMetrics() {
registerMetrics.Do(func() {
@ -67,6 +78,7 @@ func RegisterMetrics() {
}
legacyregistry.MustRegister(authorizationConfigAutomaticReloadsTotal)
legacyregistry.MustRegister(authorizationConfigAutomaticReloadLastTimestampSeconds)
legacyregistry.CustomMustRegister(configmetrics.NewConfigInfoCustomCollector(authorizationConfigLastConfigInfo, configHashProvider))
})
}
@ -81,10 +93,16 @@ func RecordAuthorizationConfigAutomaticReloadFailure(apiServerID string) {
authorizationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("failure", apiServerIDHash).SetToCurrentTime()
}
func RecordAuthorizationConfigAutomaticReloadSuccess(apiServerID string) {
func RecordAuthorizationConfigAutomaticReloadSuccess(apiServerID, authzConfigData string) {
apiServerIDHash := getHash(apiServerID)
authorizationConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc()
authorizationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("success", apiServerIDHash).SetToCurrentTime()
RecordAuthorizationConfigLastConfigInfo(apiServerID, authzConfigData)
}
func RecordAuthorizationConfigLastConfigInfo(apiServerID, authzConfigData string) {
configHashProvider.SetHashes(getHash(apiServerID), getHash(authzConfigData))
}
func getHash(data string) string {

View File

@ -27,6 +27,27 @@ import (
const (
testAPIServerID = "testAPIServerID"
testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"
testConfigData = `
apiVersion: apiserver.config.k8s.io/v1
kind: AuthorizationConfiguration
authorizers:
- type: Webhook
name: error.example.com
webhook:
timeout: 5s
failurePolicy: Deny
subjectAccessReviewVersion: v1
matchConditionSubjectAccessReviewVersion: v1
authorizedTTL: 1ms
unauthorizedTTL: 1ms
connectionInfo:
type: KubeConfigFile
kubeConfigFile: /path/to/kubeconfig
matchConditions:
- expression: has(request.resourceAttributes)
- expression: 'request.resourceAttributes.namespace == "fail"'
- expression: 'request.resourceAttributes.name == "error"'
`
)
func TestRecordAuthorizationConfigAutomaticReloadFailure(t *testing.T) {
@ -53,15 +74,19 @@ func TestRecordAuthorizationConfigAutomaticReloadSuccess(t *testing.T) {
# HELP apiserver_authorization_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authorization configuration split by status and apiserver identity.
# TYPE apiserver_authorization_config_controller_automatic_reloads_total counter
apiserver_authorization_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1
# HELP apiserver_authorization_config_controller_last_config_info [ALPHA] Information about the last applied authorization configuration with hash as label, split by apiserver identity.
# TYPE apiserver_authorization_config_controller_last_config_info gauge
apiserver_authorization_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:2a66c95e5593fc74e6c3dbbb708741361f0690ed45d17e4d4ac0b9c282b6538f"} 1
`
metrics := []string{
namespace + "_" + subsystem + "_automatic_reloads_total",
namespace + "_" + subsystem + "_last_config_info",
}
authorizationConfigAutomaticReloadsTotal.Reset()
ResetMetricsForTest()
RegisterMetrics()
RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID)
RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID, testConfigData)
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
t.Fatal(err)
}
@ -107,3 +132,34 @@ func TestAuthorizationConfigAutomaticReloadLastTimestampSeconds(t *testing.T) {
}
}
}
func TestRecordAuthorizationConfigAutomaticReloadSuccess_StaleMetricCleanup(t *testing.T) {
ResetMetricsForTest()
RegisterMetrics()
// Record initial success with first config
firstConfig := "config1"
RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID, firstConfig)
// Record success with different config - should clean up old metric
secondConfig := "config2"
RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID, secondConfig)
// Verify only the latest hash is present
expectedValue := `
# HELP apiserver_authorization_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authorization configuration split by status and apiserver identity.
# TYPE apiserver_authorization_config_controller_automatic_reloads_total counter
apiserver_authorization_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 2
# HELP apiserver_authorization_config_controller_last_config_info [ALPHA] Information about the last applied authorization configuration with hash as label, split by apiserver identity.
# TYPE apiserver_authorization_config_controller_last_config_info gauge
apiserver_authorization_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:f309dd9c31fe24b3e594d2f9420419c48dfe954523245d5f35dc37739970d881"} 1
`
metrics := []string{
namespace + "_" + subsystem + "_automatic_reloads_total",
namespace + "_" + subsystem + "_last_config_info",
}
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
t.Fatal(err)
}
}

View File

@ -907,9 +907,8 @@ func (u unionTransformers) TransformToStorage(ctx context.Context, data []byte,
// computeEncryptionConfigHash returns the expected hash for an encryption config file that has been loaded as bytes.
// We use a hash instead of the raw file contents when tracking changes to avoid holding any encryption keys in memory outside of their associated transformers.
// This hash must be used in-memory and not externalized to the process because it has no cross-release stability guarantees.
func computeEncryptionConfigHash(data []byte) string {
return fmt.Sprintf("k8s:enc:unstable:1:%x", sha256.Sum256(data))
return fmt.Sprintf("sha256:%x", sha256.Sum256(data))
}
var _ storagevalue.ResourceTransformers = &DynamicTransformers{}

View File

@ -1843,7 +1843,7 @@ func errString(err error) string {
func TestComputeEncryptionConfigHash(t *testing.T) {
// hash the empty string to be sure that sha256 is being used
expect := "k8s:enc:unstable:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
expect := "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
sum := computeEncryptionConfigHash([]byte(""))
if expect != sum {
t.Errorf("expected hash %q but got %q", expect, sum)
@ -2220,7 +2220,7 @@ func TestGetEncryptionConfigHash(t *testing.T) {
{
name: "valid file",
filepath: "testdata/valid-configs/secret-box-first.yaml",
wantHash: "k8s:enc:unstable:1:c638c0327dbc3276dd1fcf3e67895d19ebca16b91ae0d19af24ef0759b8e0f66",
wantHash: "sha256:c638c0327dbc3276dd1fcf3e67895d19ebca16b91ae0d19af24ef0759b8e0f66",
wantErr: ``,
},
}

View File

@ -174,7 +174,7 @@ func (d *DynamicEncryptionConfigContent) processWorkItem(serverCtx context.Conte
}
if updatedEffectiveConfig && err == nil {
metrics.RecordEncryptionConfigAutomaticReloadSuccess(d.apiServerID)
metrics.RecordEncryptionConfigAutomaticReloadSuccess(d.apiServerID, encryptionConfiguration.EncryptionFileContentHash)
}
if err != nil {

View File

@ -47,6 +47,9 @@ func TestController(t *testing.T) {
# HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity.
# TYPE apiserver_encryption_config_controller_automatic_reloads_total counter
apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:cd8a60cec6134082e9f37e7a4146b4bc14a0bf8a863237c36ec8fdb658c3e027",status="success"} 1
# HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity.
# TYPE apiserver_encryption_config_controller_last_config_info gauge
apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:cd8a60cec6134082e9f37e7a4146b4bc14a0bf8a863237c36ec8fdb658c3e027",hash="sha256:6f6af143a5aec5c4056d759f2bdf8b6ffe218a2bf3846c4fd42b6baef037c5ef"} 1
`
const expectedFailureMetricValue = `
# HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity.
@ -67,7 +70,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
}{
{
name: "when invalid config is provided previous config shouldn't be changed",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 1,
wantHashCalls: 1,
wantTransformerClosed: true,
@ -82,7 +85,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
},
{
name: "when new valid config is provided it should be updated",
wantECFileHash: "some new config hash",
wantECFileHash: "sha256:6f6af143a5aec5c4056d759f2bdf8b6ffe218a2bf3846c4fd42b6baef037c5ef", // some new config hash
wantLoadCalls: 1,
wantHashCalls: 1,
wantMetrics: expectedSuccessMetricValue,
@ -98,13 +101,13 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
err: nil,
},
},
EncryptionFileContentHash: "some new config hash",
EncryptionFileContentHash: "sha256:6f6af143a5aec5c4056d759f2bdf8b6ffe218a2bf3846c4fd42b6baef037c5ef", // some new config hash
}, nil
},
},
{
name: "when same valid config is provided previous config shouldn't be changed",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 1,
wantHashCalls: 1,
wantTransformerClosed: true,
@ -122,13 +125,13 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
},
},
// hash of initial "testdata/ec_config.yaml" config file before reloading
EncryptionFileContentHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
EncryptionFileContentHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
}, nil
},
},
{
name: "when transformer's health check fails previous config shouldn't be changed",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 1,
wantHashCalls: 1,
wantTransformerClosed: true,
@ -152,7 +155,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
},
{
name: "when multiple health checks are present previous config shouldn't be changed",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 1,
wantHashCalls: 1,
wantTransformerClosed: true,
@ -179,7 +182,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
},
{
name: "when invalid health check URL is provided previous config shouldn't be changed",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 1,
wantHashCalls: 1,
wantTransformerClosed: true,
@ -202,7 +205,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
},
{
name: "when config is not updated transformers are closed correctly",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 1,
wantHashCalls: 1,
wantTransformerClosed: true,
@ -220,13 +223,13 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
},
},
// hash of initial "testdata/ec_config.yaml" config file before reloading
EncryptionFileContentHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
EncryptionFileContentHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
}, nil
},
},
{
name: "when config hash is not updated transformers are closed correctly",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 0,
wantHashCalls: 1,
wantTransformerClosed: true,
@ -234,7 +237,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
wantAddRateLimitedCount: 0,
mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
// hash of initial "testdata/ec_config.yaml" config file before reloading
return "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", nil
return "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", nil
},
mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
return nil, fmt.Errorf("should not be called")
@ -242,7 +245,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
},
{
name: "when config hash errors transformers are closed correctly",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 0,
wantHashCalls: 1,
wantTransformerClosed: true,
@ -332,7 +335,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
}
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(test.wantMetrics),
"apiserver_encryption_config_controller_automatic_reloads_total",
"apiserver_encryption_config_controller_automatic_reloads_total", "apiserver_encryption_config_controller_automatic_reload_last_config_info",
); err != nil {
t.Errorf("failed to validate metrics: %v", err)
}

View File

@ -22,6 +22,7 @@ import (
"hash"
"sync"
"k8s.io/apiserver/pkg/util/configmetrics"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)
@ -53,10 +54,20 @@ var (
},
[]string{"status", "apiserver_id_hash"},
)
encryptionConfigLastConfigInfo = metrics.NewDesc(
metrics.BuildFQName(namespace, subsystem, "last_config_info"),
"Information about the last applied encryption configuration with hash as label, split by apiserver identity.",
[]string{"apiserver_id_hash", "hash"},
nil,
metrics.ALPHA,
"",
)
)
var registerMetrics sync.Once
var hashPool *sync.Pool
var configHashProvider = configmetrics.NewAtomicHashProvider()
func RegisterMetrics() {
registerMetrics.Do(func() {
@ -67,25 +78,37 @@ func RegisterMetrics() {
}
legacyregistry.MustRegister(encryptionConfigAutomaticReloadsTotal)
legacyregistry.MustRegister(encryptionConfigAutomaticReloadLastTimestampSeconds)
legacyregistry.CustomMustRegister(configmetrics.NewConfigInfoCustomCollector(encryptionConfigLastConfigInfo, configHashProvider))
})
}
func ResetMetricsForTest() {
encryptionConfigAutomaticReloadsTotal.Reset()
encryptionConfigAutomaticReloadLastTimestampSeconds.Reset()
}
func RecordEncryptionConfigAutomaticReloadFailure(apiServerID string) {
apiServerIDHash := getHash(apiServerID)
encryptionConfigAutomaticReloadsTotal.WithLabelValues("failure", apiServerIDHash).Inc()
recordEncryptionConfigAutomaticReloadTimestamp("failure", apiServerIDHash)
}
func RecordEncryptionConfigAutomaticReloadSuccess(apiServerID string) {
func RecordEncryptionConfigAutomaticReloadSuccess(apiServerID, encryptionConfigDataHash string) {
apiServerIDHash := getHash(apiServerID)
encryptionConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc()
recordEncryptionConfigAutomaticReloadTimestamp("success", apiServerIDHash)
RecordEncryptionConfigLastConfigInfo(apiServerID, encryptionConfigDataHash)
}
func recordEncryptionConfigAutomaticReloadTimestamp(result, apiServerIDHash string) {
encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result, apiServerIDHash).SetToCurrentTime()
}
func RecordEncryptionConfigLastConfigInfo(apiServerID, encryptionConfigDataHash string) {
configHashProvider.SetHashes(getHash(apiServerID), encryptionConfigDataHash)
}
func getHash(data string) string {
if len(data) == 0 {
return ""

View File

@ -27,6 +27,7 @@ import (
const (
testAPIServerID = "testAPIServerID"
testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"
testConfigDataHash = "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3"
)
func TestRecordEncryptionConfigAutomaticReloadFailure(t *testing.T) {
@ -53,16 +54,19 @@ func TestRecordEncryptionConfigAutomaticReloadSuccess(t *testing.T) {
# HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity.
# TYPE apiserver_encryption_config_controller_automatic_reloads_total counter
apiserver_encryption_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1
# HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity.
# TYPE apiserver_encryption_config_controller_last_config_info gauge
apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3"} 1
`
metricNames := []string{
namespace + "_" + subsystem + "_automatic_reload_success_total",
namespace + "_" + subsystem + "_automatic_reloads_total",
namespace + "_" + subsystem + "_last_config_info",
}
encryptionConfigAutomaticReloadsTotal.Reset()
ResetMetricsForTest()
RegisterMetrics()
RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID)
RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID, testConfigDataHash)
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metricNames...); err != nil {
t.Fatal(err)
}
@ -108,3 +112,41 @@ func TestEncryptionConfigAutomaticReloadLastTimestampSeconds(t *testing.T) {
}
}
}
func TestRecordEncryptionConfigAutomaticReloadSuccess_StaleMetricCleanup(t *testing.T) {
ResetMetricsForTest()
RegisterMetrics()
// Record first config
firstConfigHash := "sha256:firsthash"
RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID, firstConfigHash)
// Verify first config metric exists
firstExpected := `
# HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity.
# TYPE apiserver_encryption_config_controller_last_config_info gauge
apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:firsthash"} 1
`
metricNames := []string{
namespace + "_" + subsystem + "_last_config_info",
}
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(firstExpected), metricNames...); err != nil {
t.Fatal(err)
}
// Record second config - should clean up first config's metric
secondConfigHash := "sha256:secondhash"
RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID, secondConfigHash)
// Verify only second config metric exists
secondExpected := `
# HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity.
# TYPE apiserver_encryption_config_controller_last_config_info gauge
apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:secondhash"} 1
`
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(secondExpected), metricNames...); err != nil {
t.Fatal(err)
}
}

View File

@ -37,6 +37,7 @@ import (
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/apiserver/pkg/server/options/encryptionconfig"
encryptionconfigcontroller "k8s.io/apiserver/pkg/server/options/encryptionconfig/controller"
encryptionconfigmetrics "k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics"
serverstorage "k8s.io/apiserver/pkg/server/storage"
"k8s.io/apiserver/pkg/storage/etcd3/metrics"
"k8s.io/apiserver/pkg/storage/storagebackend"
@ -298,6 +299,7 @@ func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err erro
if err != nil {
return err
}
encryptionconfigmetrics.RecordEncryptionConfigLastConfigInfo(c.APIServerID, encryptionConfiguration.EncryptionFileContentHash)
if s.EncryptionProviderConfigAutomaticReload {
// with reload=true we will always have 1 health check