diff --git a/pkg/server/options/authenticationconfig/metrics/metrics.go b/pkg/server/options/authenticationconfig/metrics/metrics.go index 74925972b..ed878d94e 100644 --- a/pkg/server/options/authenticationconfig/metrics/metrics.go +++ b/pkg/server/options/authenticationconfig/metrics/metrics.go @@ -21,6 +21,7 @@ import ( "fmt" "sync" + "k8s.io/apiserver/pkg/util/configmetrics" "k8s.io/component-base/metrics" "k8s.io/component-base/metrics/legacyregistry" ) @@ -52,14 +53,25 @@ var ( }, []string{"status", "apiserver_id_hash"}, ) + + authenticationConfigLastConfigInfo = metrics.NewDesc( + metrics.BuildFQName(namespace, subsystem, "last_config_info"), + "Information about the last applied authentication configuration with hash as label, split by apiserver identity.", + []string{"apiserver_id_hash", "hash"}, + nil, + metrics.ALPHA, + "", + ) ) var registerMetrics sync.Once +var configHashProvider = configmetrics.NewAtomicHashProvider() func RegisterMetrics() { registerMetrics.Do(func() { legacyregistry.MustRegister(authenticationConfigAutomaticReloadsTotal) legacyregistry.MustRegister(authenticationConfigAutomaticReloadLastTimestampSeconds) + legacyregistry.CustomMustRegister(configmetrics.NewConfigInfoCustomCollector(authenticationConfigLastConfigInfo, configHashProvider)) }) } @@ -74,10 +86,16 @@ func RecordAuthenticationConfigAutomaticReloadFailure(apiServerID string) { authenticationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("failure", apiServerIDHash).SetToCurrentTime() } -func RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID string) { +func RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID, authConfigData string) { apiServerIDHash := getHash(apiServerID) authenticationConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc() authenticationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("success", apiServerIDHash).SetToCurrentTime() + + RecordAuthenticationConfigLastConfigInfo(apiServerID, authConfigData) +} + +func RecordAuthenticationConfigLastConfigInfo(apiServerID, authConfigData string) { + configHashProvider.SetHashes(getHash(apiServerID), getHash(authConfigData)) } func getHash(data string) string { diff --git a/pkg/server/options/authenticationconfig/metrics/metrics_test.go b/pkg/server/options/authenticationconfig/metrics/metrics_test.go index 172238885..4af9f6446 100644 --- a/pkg/server/options/authenticationconfig/metrics/metrics_test.go +++ b/pkg/server/options/authenticationconfig/metrics/metrics_test.go @@ -27,6 +27,18 @@ import ( const ( testAPIServerID = "testAPIServerID" testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37" + testConfigData = ` +apiVersion: apiserver.config.k8s.io/v1 +kind: AuthenticationConfiguration +jwt: +- issuer: + url: https://test-issuer + audiences: [ "aud" ] + claimMappings: + username: + claim: sub + prefix: "" +` ) func TestRecordAuthenticationConfigAutomaticReloadFailure(t *testing.T) { @@ -53,15 +65,19 @@ func TestRecordAuthenticationConfigAutomaticReloadSuccess(t *testing.T) { # HELP apiserver_authentication_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authentication configuration split by status and apiserver identity. # TYPE apiserver_authentication_config_controller_automatic_reloads_total counter apiserver_authentication_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1 + # HELP apiserver_authentication_config_controller_last_config_info [ALPHA] Information about the last applied authentication configuration with hash as label, split by apiserver identity. + # TYPE apiserver_authentication_config_controller_last_config_info gauge + apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:ccbcaf98557c273dfc779222d54a5bd3e785ea5330048f3bf4278cf3997b669c"} 1 ` metrics := []string{ namespace + "_" + subsystem + "_automatic_reloads_total", + namespace + "_" + subsystem + "_last_config_info", } - authenticationConfigAutomaticReloadsTotal.Reset() + ResetMetricsForTest() RegisterMetrics() - RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID) + RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID, testConfigData) if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil { t.Fatal(err) } @@ -107,3 +123,34 @@ func TestAuthenticationConfigAutomaticReloadLastTimestampSeconds(t *testing.T) { } } } + +func TestRecordAuthenticationConfigAutomaticReloadSuccess_StaleMetricCleanup(t *testing.T) { + ResetMetricsForTest() + RegisterMetrics() + + // Record initial success with first config + firstConfig := "config1" + RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID, firstConfig) + + // Record success with different config - should clean up old metric + secondConfig := "config2" + RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID, secondConfig) + + // Verify only the latest hash is present + expectedValue := ` + # HELP apiserver_authentication_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authentication configuration split by status and apiserver identity. + # TYPE apiserver_authentication_config_controller_automatic_reloads_total counter + apiserver_authentication_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 2 + # HELP apiserver_authentication_config_controller_last_config_info [ALPHA] Information about the last applied authentication configuration with hash as label, split by apiserver identity. + # TYPE apiserver_authentication_config_controller_last_config_info gauge + apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:f309dd9c31fe24b3e594d2f9420419c48dfe954523245d5f35dc37739970d881"} 1 + ` + metrics := []string{ + namespace + "_" + subsystem + "_automatic_reloads_total", + namespace + "_" + subsystem + "_last_config_info", + } + + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil { + t.Fatal(err) + } +} diff --git a/pkg/server/options/authorizationconfig/metrics/metrics.go b/pkg/server/options/authorizationconfig/metrics/metrics.go index e03894cce..1a9d1a09e 100644 --- a/pkg/server/options/authorizationconfig/metrics/metrics.go +++ b/pkg/server/options/authorizationconfig/metrics/metrics.go @@ -22,6 +22,7 @@ import ( "hash" "sync" + "k8s.io/apiserver/pkg/util/configmetrics" "k8s.io/component-base/metrics" "k8s.io/component-base/metrics/legacyregistry" ) @@ -53,10 +54,20 @@ var ( }, []string{"status", "apiserver_id_hash"}, ) + + authorizationConfigLastConfigInfo = metrics.NewDesc( + metrics.BuildFQName(namespace, subsystem, "last_config_info"), + "Information about the last applied authorization configuration with hash as label, split by apiserver identity.", + []string{"apiserver_id_hash", "hash"}, + nil, + metrics.ALPHA, + "", + ) ) var registerMetrics sync.Once var hashPool *sync.Pool +var configHashProvider = configmetrics.NewAtomicHashProvider() func RegisterMetrics() { registerMetrics.Do(func() { @@ -67,6 +78,7 @@ func RegisterMetrics() { } legacyregistry.MustRegister(authorizationConfigAutomaticReloadsTotal) legacyregistry.MustRegister(authorizationConfigAutomaticReloadLastTimestampSeconds) + legacyregistry.CustomMustRegister(configmetrics.NewConfigInfoCustomCollector(authorizationConfigLastConfigInfo, configHashProvider)) }) } @@ -81,10 +93,16 @@ func RecordAuthorizationConfigAutomaticReloadFailure(apiServerID string) { authorizationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("failure", apiServerIDHash).SetToCurrentTime() } -func RecordAuthorizationConfigAutomaticReloadSuccess(apiServerID string) { +func RecordAuthorizationConfigAutomaticReloadSuccess(apiServerID, authzConfigData string) { apiServerIDHash := getHash(apiServerID) authorizationConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc() authorizationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("success", apiServerIDHash).SetToCurrentTime() + + RecordAuthorizationConfigLastConfigInfo(apiServerID, authzConfigData) +} + +func RecordAuthorizationConfigLastConfigInfo(apiServerID, authzConfigData string) { + configHashProvider.SetHashes(getHash(apiServerID), getHash(authzConfigData)) } func getHash(data string) string { diff --git a/pkg/server/options/authorizationconfig/metrics/metrics_test.go b/pkg/server/options/authorizationconfig/metrics/metrics_test.go index 546561b28..21c52dc00 100644 --- a/pkg/server/options/authorizationconfig/metrics/metrics_test.go +++ b/pkg/server/options/authorizationconfig/metrics/metrics_test.go @@ -27,6 +27,27 @@ import ( const ( testAPIServerID = "testAPIServerID" testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37" + testConfigData = ` +apiVersion: apiserver.config.k8s.io/v1 +kind: AuthorizationConfiguration +authorizers: +- type: Webhook + name: error.example.com + webhook: + timeout: 5s + failurePolicy: Deny + subjectAccessReviewVersion: v1 + matchConditionSubjectAccessReviewVersion: v1 + authorizedTTL: 1ms + unauthorizedTTL: 1ms + connectionInfo: + type: KubeConfigFile + kubeConfigFile: /path/to/kubeconfig + matchConditions: + - expression: has(request.resourceAttributes) + - expression: 'request.resourceAttributes.namespace == "fail"' + - expression: 'request.resourceAttributes.name == "error"' +` ) func TestRecordAuthorizationConfigAutomaticReloadFailure(t *testing.T) { @@ -53,15 +74,19 @@ func TestRecordAuthorizationConfigAutomaticReloadSuccess(t *testing.T) { # HELP apiserver_authorization_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authorization configuration split by status and apiserver identity. # TYPE apiserver_authorization_config_controller_automatic_reloads_total counter apiserver_authorization_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1 + # HELP apiserver_authorization_config_controller_last_config_info [ALPHA] Information about the last applied authorization configuration with hash as label, split by apiserver identity. + # TYPE apiserver_authorization_config_controller_last_config_info gauge + apiserver_authorization_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:2a66c95e5593fc74e6c3dbbb708741361f0690ed45d17e4d4ac0b9c282b6538f"} 1 ` metrics := []string{ namespace + "_" + subsystem + "_automatic_reloads_total", + namespace + "_" + subsystem + "_last_config_info", } - authorizationConfigAutomaticReloadsTotal.Reset() + ResetMetricsForTest() RegisterMetrics() - RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID) + RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID, testConfigData) if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil { t.Fatal(err) } @@ -107,3 +132,34 @@ func TestAuthorizationConfigAutomaticReloadLastTimestampSeconds(t *testing.T) { } } } + +func TestRecordAuthorizationConfigAutomaticReloadSuccess_StaleMetricCleanup(t *testing.T) { + ResetMetricsForTest() + RegisterMetrics() + + // Record initial success with first config + firstConfig := "config1" + RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID, firstConfig) + + // Record success with different config - should clean up old metric + secondConfig := "config2" + RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID, secondConfig) + + // Verify only the latest hash is present + expectedValue := ` + # HELP apiserver_authorization_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authorization configuration split by status and apiserver identity. + # TYPE apiserver_authorization_config_controller_automatic_reloads_total counter + apiserver_authorization_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 2 + # HELP apiserver_authorization_config_controller_last_config_info [ALPHA] Information about the last applied authorization configuration with hash as label, split by apiserver identity. + # TYPE apiserver_authorization_config_controller_last_config_info gauge + apiserver_authorization_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:f309dd9c31fe24b3e594d2f9420419c48dfe954523245d5f35dc37739970d881"} 1 + ` + metrics := []string{ + namespace + "_" + subsystem + "_automatic_reloads_total", + namespace + "_" + subsystem + "_last_config_info", + } + + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil { + t.Fatal(err) + } +} diff --git a/pkg/server/options/encryptionconfig/config.go b/pkg/server/options/encryptionconfig/config.go index 48efe7a27..6d0e28428 100644 --- a/pkg/server/options/encryptionconfig/config.go +++ b/pkg/server/options/encryptionconfig/config.go @@ -907,9 +907,8 @@ func (u unionTransformers) TransformToStorage(ctx context.Context, data []byte, // computeEncryptionConfigHash returns the expected hash for an encryption config file that has been loaded as bytes. // We use a hash instead of the raw file contents when tracking changes to avoid holding any encryption keys in memory outside of their associated transformers. -// This hash must be used in-memory and not externalized to the process because it has no cross-release stability guarantees. func computeEncryptionConfigHash(data []byte) string { - return fmt.Sprintf("k8s:enc:unstable:1:%x", sha256.Sum256(data)) + return fmt.Sprintf("sha256:%x", sha256.Sum256(data)) } var _ storagevalue.ResourceTransformers = &DynamicTransformers{} diff --git a/pkg/server/options/encryptionconfig/config_test.go b/pkg/server/options/encryptionconfig/config_test.go index 5e551e096..125d43ef9 100644 --- a/pkg/server/options/encryptionconfig/config_test.go +++ b/pkg/server/options/encryptionconfig/config_test.go @@ -1843,7 +1843,7 @@ func errString(err error) string { func TestComputeEncryptionConfigHash(t *testing.T) { // hash the empty string to be sure that sha256 is being used - expect := "k8s:enc:unstable:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + expect := "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" sum := computeEncryptionConfigHash([]byte("")) if expect != sum { t.Errorf("expected hash %q but got %q", expect, sum) @@ -2220,7 +2220,7 @@ func TestGetEncryptionConfigHash(t *testing.T) { { name: "valid file", filepath: "testdata/valid-configs/secret-box-first.yaml", - wantHash: "k8s:enc:unstable:1:c638c0327dbc3276dd1fcf3e67895d19ebca16b91ae0d19af24ef0759b8e0f66", + wantHash: "sha256:c638c0327dbc3276dd1fcf3e67895d19ebca16b91ae0d19af24ef0759b8e0f66", wantErr: ``, }, } diff --git a/pkg/server/options/encryptionconfig/controller/controller.go b/pkg/server/options/encryptionconfig/controller/controller.go index fd783f41a..5c249bcc0 100644 --- a/pkg/server/options/encryptionconfig/controller/controller.go +++ b/pkg/server/options/encryptionconfig/controller/controller.go @@ -174,7 +174,7 @@ func (d *DynamicEncryptionConfigContent) processWorkItem(serverCtx context.Conte } if updatedEffectiveConfig && err == nil { - metrics.RecordEncryptionConfigAutomaticReloadSuccess(d.apiServerID) + metrics.RecordEncryptionConfigAutomaticReloadSuccess(d.apiServerID, encryptionConfiguration.EncryptionFileContentHash) } if err != nil { diff --git a/pkg/server/options/encryptionconfig/controller/controller_test.go b/pkg/server/options/encryptionconfig/controller/controller_test.go index 900746324..50eca7b93 100644 --- a/pkg/server/options/encryptionconfig/controller/controller_test.go +++ b/pkg/server/options/encryptionconfig/controller/controller_test.go @@ -47,6 +47,9 @@ func TestController(t *testing.T) { # HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity. # TYPE apiserver_encryption_config_controller_automatic_reloads_total counter apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:cd8a60cec6134082e9f37e7a4146b4bc14a0bf8a863237c36ec8fdb658c3e027",status="success"} 1 +# HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity. +# TYPE apiserver_encryption_config_controller_last_config_info gauge +apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:cd8a60cec6134082e9f37e7a4146b4bc14a0bf8a863237c36ec8fdb658c3e027",hash="sha256:6f6af143a5aec5c4056d759f2bdf8b6ffe218a2bf3846c4fd42b6baef037c5ef"} 1 ` const expectedFailureMetricValue = ` # HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity. @@ -67,7 +70,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash }{ { name: "when invalid config is provided previous config shouldn't be changed", - wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", + wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", wantLoadCalls: 1, wantHashCalls: 1, wantTransformerClosed: true, @@ -82,7 +85,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash }, { name: "when new valid config is provided it should be updated", - wantECFileHash: "some new config hash", + wantECFileHash: "sha256:6f6af143a5aec5c4056d759f2bdf8b6ffe218a2bf3846c4fd42b6baef037c5ef", // some new config hash wantLoadCalls: 1, wantHashCalls: 1, wantMetrics: expectedSuccessMetricValue, @@ -98,13 +101,13 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash err: nil, }, }, - EncryptionFileContentHash: "some new config hash", + EncryptionFileContentHash: "sha256:6f6af143a5aec5c4056d759f2bdf8b6ffe218a2bf3846c4fd42b6baef037c5ef", // some new config hash }, nil }, }, { name: "when same valid config is provided previous config shouldn't be changed", - wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", + wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", wantLoadCalls: 1, wantHashCalls: 1, wantTransformerClosed: true, @@ -122,13 +125,13 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash }, }, // hash of initial "testdata/ec_config.yaml" config file before reloading - EncryptionFileContentHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", + EncryptionFileContentHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", }, nil }, }, { name: "when transformer's health check fails previous config shouldn't be changed", - wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", + wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", wantLoadCalls: 1, wantHashCalls: 1, wantTransformerClosed: true, @@ -152,7 +155,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash }, { name: "when multiple health checks are present previous config shouldn't be changed", - wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", + wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", wantLoadCalls: 1, wantHashCalls: 1, wantTransformerClosed: true, @@ -179,7 +182,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash }, { name: "when invalid health check URL is provided previous config shouldn't be changed", - wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", + wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", wantLoadCalls: 1, wantHashCalls: 1, wantTransformerClosed: true, @@ -202,7 +205,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash }, { name: "when config is not updated transformers are closed correctly", - wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", + wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", wantLoadCalls: 1, wantHashCalls: 1, wantTransformerClosed: true, @@ -220,13 +223,13 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash }, }, // hash of initial "testdata/ec_config.yaml" config file before reloading - EncryptionFileContentHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", + EncryptionFileContentHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", }, nil }, }, { name: "when config hash is not updated transformers are closed correctly", - wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", + wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", wantLoadCalls: 0, wantHashCalls: 1, wantTransformerClosed: true, @@ -234,7 +237,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash wantAddRateLimitedCount: 0, mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) { // hash of initial "testdata/ec_config.yaml" config file before reloading - return "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", nil + return "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", nil }, mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) { return nil, fmt.Errorf("should not be called") @@ -242,7 +245,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash }, { name: "when config hash errors transformers are closed correctly", - wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", + wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", wantLoadCalls: 0, wantHashCalls: 1, wantTransformerClosed: true, @@ -332,7 +335,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash } if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(test.wantMetrics), - "apiserver_encryption_config_controller_automatic_reloads_total", + "apiserver_encryption_config_controller_automatic_reloads_total", "apiserver_encryption_config_controller_automatic_reload_last_config_info", ); err != nil { t.Errorf("failed to validate metrics: %v", err) } diff --git a/pkg/server/options/encryptionconfig/metrics/metrics.go b/pkg/server/options/encryptionconfig/metrics/metrics.go index 43b6ebbf3..ed2d5c9a8 100644 --- a/pkg/server/options/encryptionconfig/metrics/metrics.go +++ b/pkg/server/options/encryptionconfig/metrics/metrics.go @@ -22,6 +22,7 @@ import ( "hash" "sync" + "k8s.io/apiserver/pkg/util/configmetrics" "k8s.io/component-base/metrics" "k8s.io/component-base/metrics/legacyregistry" ) @@ -53,10 +54,20 @@ var ( }, []string{"status", "apiserver_id_hash"}, ) + + encryptionConfigLastConfigInfo = metrics.NewDesc( + metrics.BuildFQName(namespace, subsystem, "last_config_info"), + "Information about the last applied encryption configuration with hash as label, split by apiserver identity.", + []string{"apiserver_id_hash", "hash"}, + nil, + metrics.ALPHA, + "", + ) ) var registerMetrics sync.Once var hashPool *sync.Pool +var configHashProvider = configmetrics.NewAtomicHashProvider() func RegisterMetrics() { registerMetrics.Do(func() { @@ -67,25 +78,37 @@ func RegisterMetrics() { } legacyregistry.MustRegister(encryptionConfigAutomaticReloadsTotal) legacyregistry.MustRegister(encryptionConfigAutomaticReloadLastTimestampSeconds) + legacyregistry.CustomMustRegister(configmetrics.NewConfigInfoCustomCollector(encryptionConfigLastConfigInfo, configHashProvider)) }) } +func ResetMetricsForTest() { + encryptionConfigAutomaticReloadsTotal.Reset() + encryptionConfigAutomaticReloadLastTimestampSeconds.Reset() +} + func RecordEncryptionConfigAutomaticReloadFailure(apiServerID string) { apiServerIDHash := getHash(apiServerID) encryptionConfigAutomaticReloadsTotal.WithLabelValues("failure", apiServerIDHash).Inc() recordEncryptionConfigAutomaticReloadTimestamp("failure", apiServerIDHash) } -func RecordEncryptionConfigAutomaticReloadSuccess(apiServerID string) { +func RecordEncryptionConfigAutomaticReloadSuccess(apiServerID, encryptionConfigDataHash string) { apiServerIDHash := getHash(apiServerID) encryptionConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc() recordEncryptionConfigAutomaticReloadTimestamp("success", apiServerIDHash) + + RecordEncryptionConfigLastConfigInfo(apiServerID, encryptionConfigDataHash) } func recordEncryptionConfigAutomaticReloadTimestamp(result, apiServerIDHash string) { encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result, apiServerIDHash).SetToCurrentTime() } +func RecordEncryptionConfigLastConfigInfo(apiServerID, encryptionConfigDataHash string) { + configHashProvider.SetHashes(getHash(apiServerID), encryptionConfigDataHash) +} + func getHash(data string) string { if len(data) == 0 { return "" diff --git a/pkg/server/options/encryptionconfig/metrics/metrics_test.go b/pkg/server/options/encryptionconfig/metrics/metrics_test.go index 281ed9608..6ef751962 100644 --- a/pkg/server/options/encryptionconfig/metrics/metrics_test.go +++ b/pkg/server/options/encryptionconfig/metrics/metrics_test.go @@ -27,6 +27,7 @@ import ( const ( testAPIServerID = "testAPIServerID" testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37" + testConfigDataHash = "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3" ) func TestRecordEncryptionConfigAutomaticReloadFailure(t *testing.T) { @@ -53,16 +54,19 @@ func TestRecordEncryptionConfigAutomaticReloadSuccess(t *testing.T) { # HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity. # TYPE apiserver_encryption_config_controller_automatic_reloads_total counter apiserver_encryption_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1 + # HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity. + # TYPE apiserver_encryption_config_controller_last_config_info gauge + apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3"} 1 ` metricNames := []string{ - namespace + "_" + subsystem + "_automatic_reload_success_total", namespace + "_" + subsystem + "_automatic_reloads_total", + namespace + "_" + subsystem + "_last_config_info", } - encryptionConfigAutomaticReloadsTotal.Reset() + ResetMetricsForTest() RegisterMetrics() - RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID) + RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID, testConfigDataHash) if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metricNames...); err != nil { t.Fatal(err) } @@ -108,3 +112,41 @@ func TestEncryptionConfigAutomaticReloadLastTimestampSeconds(t *testing.T) { } } } + +func TestRecordEncryptionConfigAutomaticReloadSuccess_StaleMetricCleanup(t *testing.T) { + ResetMetricsForTest() + RegisterMetrics() + + // Record first config + firstConfigHash := "sha256:firsthash" + RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID, firstConfigHash) + + // Verify first config metric exists + firstExpected := ` + # HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity. + # TYPE apiserver_encryption_config_controller_last_config_info gauge + apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:firsthash"} 1 + ` + metricNames := []string{ + namespace + "_" + subsystem + "_last_config_info", + } + + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(firstExpected), metricNames...); err != nil { + t.Fatal(err) + } + + // Record second config - should clean up first config's metric + secondConfigHash := "sha256:secondhash" + RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID, secondConfigHash) + + // Verify only second config metric exists + secondExpected := ` + # HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity. + # TYPE apiserver_encryption_config_controller_last_config_info gauge + apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:secondhash"} 1 + ` + + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(secondExpected), metricNames...); err != nil { + t.Fatal(err) + } +} diff --git a/pkg/server/options/etcd.go b/pkg/server/options/etcd.go index 8f73749c5..f52225693 100644 --- a/pkg/server/options/etcd.go +++ b/pkg/server/options/etcd.go @@ -37,6 +37,7 @@ import ( "k8s.io/apiserver/pkg/server/healthz" "k8s.io/apiserver/pkg/server/options/encryptionconfig" encryptionconfigcontroller "k8s.io/apiserver/pkg/server/options/encryptionconfig/controller" + encryptionconfigmetrics "k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics" serverstorage "k8s.io/apiserver/pkg/server/storage" "k8s.io/apiserver/pkg/storage/etcd3/metrics" "k8s.io/apiserver/pkg/storage/storagebackend" @@ -298,6 +299,7 @@ func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err erro if err != nil { return err } + encryptionconfigmetrics.RecordEncryptionConfigLastConfigInfo(c.APIServerID, encryptionConfiguration.EncryptionFileContentHash) if s.EncryptionProviderConfigAutomaticReload { // with reload=true we will always have 1 health check