kubernetes
/
apiserver
mirror of https://github.com/kubernetes/apiserver.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add automatic_reload_last_config_info metric for auth configs 

Signed-off-by: Anish Ramasekar <anish.ramasekar@gmail.com>

Kubernetes-commit: 21e2fcea9ec25c79d4ceea0c8a931c2655bf5d93
This commit is contained in:
Anish Ramasekar 2025-06-11 13:13:04 -07:00 committed by Kubernetes Publisher
parent f6d9871507
commit 31fd0fd4cd
11 changed files with 237 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
pkg/server/options/authenticationconfig/metrics/metrics.go
View File

@ -21,6 +21,7 @@ import (
"fmt"
"sync"
"k8s.io/apiserver/pkg/util/configmetrics"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)
@ -52,14 +53,25 @@ var (
},
[]string{"status", "apiserver_id_hash"},
)
authenticationConfigLastConfigInfo = metrics.NewDesc(
metrics.BuildFQName(namespace, subsystem, "last_config_info"),
"Information about the last applied authentication configuration with hash as label, split by apiserver identity.",
[]string{"apiserver_id_hash", "hash"},
nil,
metrics.ALPHA,
"",
)
)
var registerMetrics sync.Once
var configHashProvider = configmetrics.NewAtomicHashProvider()
func RegisterMetrics() {
registerMetrics.Do(func() {
legacyregistry.MustRegister(authenticationConfigAutomaticReloadsTotal)
legacyregistry.MustRegister(authenticationConfigAutomaticReloadLastTimestampSeconds)
legacyregistry.CustomMustRegister(configmetrics.NewConfigInfoCustomCollector(authenticationConfigLastConfigInfo, configHashProvider))
})
}
@ -74,10 +86,16 @@ func RecordAuthenticationConfigAutomaticReloadFailure(apiServerID string) {
authenticationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("failure", apiServerIDHash).SetToCurrentTime()
}
func RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID string) {
func RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID, authConfigData string) {
apiServerIDHash := getHash(apiServerID)
authenticationConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc()
authenticationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("success", apiServerIDHash).SetToCurrentTime()
RecordAuthenticationConfigLastConfigInfo(apiServerID, authConfigData)
}
func RecordAuthenticationConfigLastConfigInfo(apiServerID, authConfigData string) {
configHashProvider.SetHashes(getHash(apiServerID), getHash(authConfigData))
}
func getHash(data string) string {

51
pkg/server/options/authenticationconfig/metrics/metrics_test.go
View File

@ -27,6 +27,18 @@ import (
const (
testAPIServerID = "testAPIServerID"
testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"
testConfigData = `
apiVersion: apiserver.config.k8s.io/v1
kind: AuthenticationConfiguration
jwt:
- issuer:
url: https://test-issuer
audiences: [ "aud" ]
claimMappings:
username:
claim: sub
prefix: ""
`
)
func TestRecordAuthenticationConfigAutomaticReloadFailure(t *testing.T) {
@ -53,15 +65,19 @@ func TestRecordAuthenticationConfigAutomaticReloadSuccess(t *testing.T) {
# HELP apiserver_authentication_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authentication configuration split by status and apiserver identity.
# TYPE apiserver_authentication_config_controller_automatic_reloads_total counter
apiserver_authentication_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1
# HELP apiserver_authentication_config_controller_last_config_info [ALPHA] Information about the last applied authentication configuration with hash as label, split by apiserver identity.
# TYPE apiserver_authentication_config_controller_last_config_info gauge
apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:ccbcaf98557c273dfc779222d54a5bd3e785ea5330048f3bf4278cf3997b669c"} 1
`
metrics := []string{
namespace + "_" + subsystem + "_automatic_reloads_total",
namespace + "_" + subsystem + "_last_config_info",
}
authenticationConfigAutomaticReloadsTotal.Reset()
ResetMetricsForTest()
RegisterMetrics()
RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID)
RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID, testConfigData)
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
t.Fatal(err)
}
@ -107,3 +123,34 @@ func TestAuthenticationConfigAutomaticReloadLastTimestampSeconds(t *testing.T) {
}
}
}
func TestRecordAuthenticationConfigAutomaticReloadSuccess_StaleMetricCleanup(t *testing.T) {
ResetMetricsForTest()
RegisterMetrics()
// Record initial success with first config
firstConfig := "config1"
RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID, firstConfig)
// Record success with different config - should clean up old metric
secondConfig := "config2"
RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID, secondConfig)
// Verify only the latest hash is present
expectedValue := `
# HELP apiserver_authentication_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authentication configuration split by status and apiserver identity.
# TYPE apiserver_authentication_config_controller_automatic_reloads_total counter
apiserver_authentication_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 2
# HELP apiserver_authentication_config_controller_last_config_info [ALPHA] Information about the last applied authentication configuration with hash as label, split by apiserver identity.
# TYPE apiserver_authentication_config_controller_last_config_info gauge
apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:f309dd9c31fe24b3e594d2f9420419c48dfe954523245d5f35dc37739970d881"} 1
`
metrics := []string{
namespace + "_" + subsystem + "_automatic_reloads_total",
namespace + "_" + subsystem + "_last_config_info",
}
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
t.Fatal(err)
}
}

20
pkg/server/options/authorizationconfig/metrics/metrics.go
View File

@ -22,6 +22,7 @@ import (
"hash"
"sync"
"k8s.io/apiserver/pkg/util/configmetrics"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)
@ -53,10 +54,20 @@ var (
},
[]string{"status", "apiserver_id_hash"},
)
authorizationConfigLastConfigInfo = metrics.NewDesc(
metrics.BuildFQName(namespace, subsystem, "last_config_info"),
"Information about the last applied authorization configuration with hash as label, split by apiserver identity.",
[]string{"apiserver_id_hash", "hash"},
nil,
metrics.ALPHA,
"",
)
)
var registerMetrics sync.Once
var hashPool *sync.Pool
var configHashProvider = configmetrics.NewAtomicHashProvider()
func RegisterMetrics() {
registerMetrics.Do(func() {
@ -67,6 +78,7 @@ func RegisterMetrics() {
}
legacyregistry.MustRegister(authorizationConfigAutomaticReloadsTotal)
legacyregistry.MustRegister(authorizationConfigAutomaticReloadLastTimestampSeconds)
legacyregistry.CustomMustRegister(configmetrics.NewConfigInfoCustomCollector(authorizationConfigLastConfigInfo, configHashProvider))
})
}
@ -81,10 +93,16 @@ func RecordAuthorizationConfigAutomaticReloadFailure(apiServerID string) {
authorizationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("failure", apiServerIDHash).SetToCurrentTime()
}
func RecordAuthorizationConfigAutomaticReloadSuccess(apiServerID string) {
func RecordAuthorizationConfigAutomaticReloadSuccess(apiServerID, authzConfigData string) {
apiServerIDHash := getHash(apiServerID)
authorizationConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc()
authorizationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("success", apiServerIDHash).SetToCurrentTime()
RecordAuthorizationConfigLastConfigInfo(apiServerID, authzConfigData)
}
func RecordAuthorizationConfigLastConfigInfo(apiServerID, authzConfigData string) {
configHashProvider.SetHashes(getHash(apiServerID), getHash(authzConfigData))
}
func getHash(data string) string {

60
pkg/server/options/authorizationconfig/metrics/metrics_test.go
View File

@ -27,6 +27,27 @@ import (
const (
testAPIServerID = "testAPIServerID"
testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"
testConfigData = `
apiVersion: apiserver.config.k8s.io/v1
kind: AuthorizationConfiguration
authorizers:
- type: Webhook
name: error.example.com
webhook:
timeout: 5s
failurePolicy: Deny
subjectAccessReviewVersion: v1
matchConditionSubjectAccessReviewVersion: v1
authorizedTTL: 1ms
unauthorizedTTL: 1ms
connectionInfo:
type: KubeConfigFile
kubeConfigFile: /path/to/kubeconfig
matchConditions:
- expression: has(request.resourceAttributes)
- expression: 'request.resourceAttributes.namespace == "fail"'
- expression: 'request.resourceAttributes.name == "error"'
`
)
func TestRecordAuthorizationConfigAutomaticReloadFailure(t *testing.T) {
@ -53,15 +74,19 @@ func TestRecordAuthorizationConfigAutomaticReloadSuccess(t *testing.T) {
# HELP apiserver_authorization_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authorization configuration split by status and apiserver identity.
# TYPE apiserver_authorization_config_controller_automatic_reloads_total counter
apiserver_authorization_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1
# HELP apiserver_authorization_config_controller_last_config_info [ALPHA] Information about the last applied authorization configuration with hash as label, split by apiserver identity.
# TYPE apiserver_authorization_config_controller_last_config_info gauge
apiserver_authorization_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:2a66c95e5593fc74e6c3dbbb708741361f0690ed45d17e4d4ac0b9c282b6538f"} 1
`
metrics := []string{
namespace + "_" + subsystem + "_automatic_reloads_total",
namespace + "_" + subsystem + "_last_config_info",
}
authorizationConfigAutomaticReloadsTotal.Reset()
ResetMetricsForTest()
RegisterMetrics()
RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID)
RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID, testConfigData)
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
t.Fatal(err)
}
@ -107,3 +132,34 @@ func TestAuthorizationConfigAutomaticReloadLastTimestampSeconds(t *testing.T) {
}
}
}
func TestRecordAuthorizationConfigAutomaticReloadSuccess_StaleMetricCleanup(t *testing.T) {
ResetMetricsForTest()
RegisterMetrics()
// Record initial success with first config
firstConfig := "config1"
RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID, firstConfig)
// Record success with different config - should clean up old metric
secondConfig := "config2"
RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID, secondConfig)
// Verify only the latest hash is present
expectedValue := `
# HELP apiserver_authorization_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authorization configuration split by status and apiserver identity.
# TYPE apiserver_authorization_config_controller_automatic_reloads_total counter
apiserver_authorization_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 2
# HELP apiserver_authorization_config_controller_last_config_info [ALPHA] Information about the last applied authorization configuration with hash as label, split by apiserver identity.
# TYPE apiserver_authorization_config_controller_last_config_info gauge
apiserver_authorization_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:f309dd9c31fe24b3e594d2f9420419c48dfe954523245d5f35dc37739970d881"} 1
`
metrics := []string{
namespace + "_" + subsystem + "_automatic_reloads_total",
namespace + "_" + subsystem + "_last_config_info",
}
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
t.Fatal(err)
}
}

3
pkg/server/options/encryptionconfig/config.go
View File

@ -907,9 +907,8 @@ func (u unionTransformers) TransformToStorage(ctx context.Context, data []byte,
// computeEncryptionConfigHash returns the expected hash for an encryption config file that has been loaded as bytes.
// We use a hash instead of the raw file contents when tracking changes to avoid holding any encryption keys in memory outside of their associated transformers.
// This hash must be used in-memory and not externalized to the process because it has no cross-release stability guarantees.
func computeEncryptionConfigHash(data []byte) string {
return fmt.Sprintf("k8s:enc:unstable:1:%x", sha256.Sum256(data))
return fmt.Sprintf("sha256:%x", sha256.Sum256(data))
}
var _ storagevalue.ResourceTransformers = &DynamicTransformers{}

4
pkg/server/options/encryptionconfig/config_test.go
View File

@ -1843,7 +1843,7 @@ func errString(err error) string {
func TestComputeEncryptionConfigHash(t *testing.T) {
// hash the empty string to be sure that sha256 is being used
expect := "k8s:enc:unstable:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
expect := "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
sum := computeEncryptionConfigHash([]byte(""))
if expect != sum {
t.Errorf("expected hash %q but got %q", expect, sum)
@ -2220,7 +2220,7 @@ func TestGetEncryptionConfigHash(t *testing.T) {
{
name: "valid file",
filepath: "testdata/valid-configs/secret-box-first.yaml",
wantHash: "k8s:enc:unstable:1:c638c0327dbc3276dd1fcf3e67895d19ebca16b91ae0d19af24ef0759b8e0f66",
wantHash: "sha256:c638c0327dbc3276dd1fcf3e67895d19ebca16b91ae0d19af24ef0759b8e0f66",
wantErr: ``,
},
}

2
pkg/server/options/encryptionconfig/controller/controller.go
View File

@ -174,7 +174,7 @@ func (d *DynamicEncryptionConfigContent) processWorkItem(serverCtx context.Conte
}
if updatedEffectiveConfig && err == nil {
metrics.RecordEncryptionConfigAutomaticReloadSuccess(d.apiServerID)
metrics.RecordEncryptionConfigAutomaticReloadSuccess(d.apiServerID, encryptionConfiguration.EncryptionFileContentHash)
}
if err != nil {

31
pkg/server/options/encryptionconfig/controller/controller_test.go
View File

@ -47,6 +47,9 @@ func TestController(t *testing.T) {
# HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity.
# TYPE apiserver_encryption_config_controller_automatic_reloads_total counter
apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:cd8a60cec6134082e9f37e7a4146b4bc14a0bf8a863237c36ec8fdb658c3e027",status="success"} 1
# HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity.
# TYPE apiserver_encryption_config_controller_last_config_info gauge
apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:cd8a60cec6134082e9f37e7a4146b4bc14a0bf8a863237c36ec8fdb658c3e027",hash="sha256:6f6af143a5aec5c4056d759f2bdf8b6ffe218a2bf3846c4fd42b6baef037c5ef"} 1
`
const expectedFailureMetricValue = `
# HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity.
@ -67,7 +70,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
}{
{
name: "when invalid config is provided previous config shouldn't be changed",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 1,
wantHashCalls: 1,
wantTransformerClosed: true,
@ -82,7 +85,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
},
{
name: "when new valid config is provided it should be updated",
wantECFileHash: "some new config hash",
wantECFileHash: "sha256:6f6af143a5aec5c4056d759f2bdf8b6ffe218a2bf3846c4fd42b6baef037c5ef", // some new config hash
wantLoadCalls: 1,
wantHashCalls: 1,
wantMetrics: expectedSuccessMetricValue,
@ -98,13 +101,13 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
err: nil,
},
},
EncryptionFileContentHash: "some new config hash",
EncryptionFileContentHash: "sha256:6f6af143a5aec5c4056d759f2bdf8b6ffe218a2bf3846c4fd42b6baef037c5ef", // some new config hash
}, nil
},
},
{
name: "when same valid config is provided previous config shouldn't be changed",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 1,
wantHashCalls: 1,
wantTransformerClosed: true,
@ -122,13 +125,13 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
},
},
// hash of initial "testdata/ec_config.yaml" config file before reloading
EncryptionFileContentHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
EncryptionFileContentHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
}, nil
},
},
{
name: "when transformer's health check fails previous config shouldn't be changed",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 1,
wantHashCalls: 1,
wantTransformerClosed: true,
@ -152,7 +155,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
},
{
name: "when multiple health checks are present previous config shouldn't be changed",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 1,
wantHashCalls: 1,
wantTransformerClosed: true,
@ -179,7 +182,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
},
{
name: "when invalid health check URL is provided previous config shouldn't be changed",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 1,
wantHashCalls: 1,
wantTransformerClosed: true,
@ -202,7 +205,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
},
{
name: "when config is not updated transformers are closed correctly",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 1,
wantHashCalls: 1,
wantTransformerClosed: true,
@ -220,13 +223,13 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
},
},
// hash of initial "testdata/ec_config.yaml" config file before reloading
EncryptionFileContentHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
EncryptionFileContentHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
}, nil
},
},
{
name: "when config hash is not updated transformers are closed correctly",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 0,
wantHashCalls: 1,
wantTransformerClosed: true,
@ -234,7 +237,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
wantAddRateLimitedCount: 0,
mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
// hash of initial "testdata/ec_config.yaml" config file before reloading
return "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", nil
return "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", nil
},
mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
return nil, fmt.Errorf("should not be called")
@ -242,7 +245,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
},
{
name: "when config hash errors transformers are closed correctly",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 0,
wantHashCalls: 1,
wantTransformerClosed: true,
@ -332,7 +335,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
}
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(test.wantMetrics),
"apiserver_encryption_config_controller_automatic_reloads_total",
"apiserver_encryption_config_controller_automatic_reloads_total", "apiserver_encryption_config_controller_automatic_reload_last_config_info",
); err != nil {
t.Errorf("failed to validate metrics: %v", err)
}

25
pkg/server/options/encryptionconfig/metrics/metrics.go
View File

@ -22,6 +22,7 @@ import (
"hash"
"sync"
"k8s.io/apiserver/pkg/util/configmetrics"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)
@ -53,10 +54,20 @@ var (
},
[]string{"status", "apiserver_id_hash"},
)
encryptionConfigLastConfigInfo = metrics.NewDesc(
metrics.BuildFQName(namespace, subsystem, "last_config_info"),
"Information about the last applied encryption configuration with hash as label, split by apiserver identity.",
[]string{"apiserver_id_hash", "hash"},
nil,
metrics.ALPHA,
"",
)
)
var registerMetrics sync.Once
var hashPool *sync.Pool
var configHashProvider = configmetrics.NewAtomicHashProvider()
func RegisterMetrics() {
registerMetrics.Do(func() {
@ -67,25 +78,37 @@ func RegisterMetrics() {
}
legacyregistry.MustRegister(encryptionConfigAutomaticReloadsTotal)
legacyregistry.MustRegister(encryptionConfigAutomaticReloadLastTimestampSeconds)
legacyregistry.CustomMustRegister(configmetrics.NewConfigInfoCustomCollector(encryptionConfigLastConfigInfo, configHashProvider))
})
}
func ResetMetricsForTest() {
encryptionConfigAutomaticReloadsTotal.Reset()
encryptionConfigAutomaticReloadLastTimestampSeconds.Reset()
}
func RecordEncryptionConfigAutomaticReloadFailure(apiServerID string) {
apiServerIDHash := getHash(apiServerID)
encryptionConfigAutomaticReloadsTotal.WithLabelValues("failure", apiServerIDHash).Inc()
recordEncryptionConfigAutomaticReloadTimestamp("failure", apiServerIDHash)
}
func RecordEncryptionConfigAutomaticReloadSuccess(apiServerID string) {
func RecordEncryptionConfigAutomaticReloadSuccess(apiServerID, encryptionConfigDataHash string) {
apiServerIDHash := getHash(apiServerID)
encryptionConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc()
recordEncryptionConfigAutomaticReloadTimestamp("success", apiServerIDHash)
RecordEncryptionConfigLastConfigInfo(apiServerID, encryptionConfigDataHash)
}
func recordEncryptionConfigAutomaticReloadTimestamp(result, apiServerIDHash string) {
encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result, apiServerIDHash).SetToCurrentTime()
}
func RecordEncryptionConfigLastConfigInfo(apiServerID, encryptionConfigDataHash string) {
configHashProvider.SetHashes(getHash(apiServerID), encryptionConfigDataHash)
}
func getHash(data string) string {
if len(data) == 0 {
return ""

48
pkg/server/options/encryptionconfig/metrics/metrics_test.go
View File

@ -27,6 +27,7 @@ import (
const (
testAPIServerID = "testAPIServerID"
testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"
testConfigDataHash = "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3"
)
func TestRecordEncryptionConfigAutomaticReloadFailure(t *testing.T) {
@ -53,16 +54,19 @@ func TestRecordEncryptionConfigAutomaticReloadSuccess(t *testing.T) {
# HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity.
# TYPE apiserver_encryption_config_controller_automatic_reloads_total counter
apiserver_encryption_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1
# HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity.
# TYPE apiserver_encryption_config_controller_last_config_info gauge
apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3"} 1
`
metricNames := []string{
namespace + "_" + subsystem + "_automatic_reload_success_total",
namespace + "_" + subsystem + "_automatic_reloads_total",
namespace + "_" + subsystem + "_last_config_info",
}
encryptionConfigAutomaticReloadsTotal.Reset()
ResetMetricsForTest()
RegisterMetrics()
RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID)
RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID, testConfigDataHash)
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metricNames...); err != nil {
t.Fatal(err)
}
@ -108,3 +112,41 @@ func TestEncryptionConfigAutomaticReloadLastTimestampSeconds(t *testing.T) {
}
}
}
func TestRecordEncryptionConfigAutomaticReloadSuccess_StaleMetricCleanup(t *testing.T) {
ResetMetricsForTest()
RegisterMetrics()
// Record first config
firstConfigHash := "sha256:firsthash"
RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID, firstConfigHash)
// Verify first config metric exists
firstExpected := `
# HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity.
# TYPE apiserver_encryption_config_controller_last_config_info gauge
apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:firsthash"} 1
`
metricNames := []string{
namespace + "_" + subsystem + "_last_config_info",
}
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(firstExpected), metricNames...); err != nil {
t.Fatal(err)
}
// Record second config - should clean up first config's metric
secondConfigHash := "sha256:secondhash"
RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID, secondConfigHash)
// Verify only second config metric exists
secondExpected := `
# HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity.
# TYPE apiserver_encryption_config_controller_last_config_info gauge
apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:secondhash"} 1
`
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(secondExpected), metricNames...); err != nil {
t.Fatal(err)
}
}

2
pkg/server/options/etcd.go
View File

@ -37,6 +37,7 @@ import (
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/apiserver/pkg/server/options/encryptionconfig"
encryptionconfigcontroller "k8s.io/apiserver/pkg/server/options/encryptionconfig/controller"
encryptionconfigmetrics "k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics"
serverstorage "k8s.io/apiserver/pkg/server/storage"
"k8s.io/apiserver/pkg/storage/etcd3/metrics"
"k8s.io/apiserver/pkg/storage/storagebackend"
@ -298,6 +299,7 @@ func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err erro
if err != nil {
return err
}
encryptionconfigmetrics.RecordEncryptionConfigLastConfigInfo(c.APIServerID, encryptionConfiguration.EncryptionFileContentHash)
if s.EncryptionProviderConfigAutomaticReload {
// with reload=true we will always have 1 health check