diff --git a/pkg/server/options/encryptionconfig/config.go b/pkg/server/options/encryptionconfig/config.go index af7506742..c91d2f94d 100644 --- a/pkg/server/options/encryptionconfig/config.go +++ b/pkg/server/options/encryptionconfig/config.go @@ -82,57 +82,91 @@ type kmsv2PluginProbe struct { l *sync.Mutex } +type kmsHealthChecker []healthz.HealthChecker + +func (k kmsHealthChecker) Name() string { + return "kms-providers" +} + +func (k kmsHealthChecker) Check(req *http.Request) error { + var errs []error + + for i := range k { + checker := k[i] + if err := checker.Check(req); err != nil { + errs = append(errs, fmt.Errorf("%s: %w", checker.Name(), err)) + } + } + + return utilerrors.Reduce(utilerrors.NewAggregate(errs)) +} + func (h *kmsPluginProbe) toHealthzCheck(idx int) healthz.HealthChecker { return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error { return h.check() }) } -func (p *kmsv2PluginProbe) toHealthzCheck(idx int) healthz.HealthChecker { +func (h *kmsv2PluginProbe) toHealthzCheck(idx int) healthz.HealthChecker { return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error { - return p.check(r.Context()) + return h.check(r.Context()) }) } -func LoadEncryptionConfig(filepath string, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, error) { +// LoadEncryptionConfig parses and validates the encryption config specified by filepath. +// It may launch multiple go routines whose lifecycle is controlled by stopCh. +// If reload is true, or KMS v2 plugins are used with no KMS v1 plugins, the returned slice of health checkers will always be of length 1. +func LoadEncryptionConfig(filepath string, reload bool, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, error) { config, err := loadConfig(filepath) if err != nil { - return nil, nil, fmt.Errorf("error while parsing file: %v", err) + return nil, nil, fmt.Errorf("error while parsing file: %w", err) } - return getTransformerOverridesAndKMSPluginHealthzCheckers(config, stopCh) + transformers, kmsHealthChecks, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(config, stopCh) + if err != nil { + return nil, nil, fmt.Errorf("error while building transformers: %w", err) + } + + if reload || (kmsUsed.v2Used && !kmsUsed.v1Used) { + kmsHealthChecks = []healthz.HealthChecker{kmsHealthChecker(kmsHealthChecks)} + } + + return transformers, kmsHealthChecks, nil } -func getTransformerOverridesAndKMSPluginHealthzCheckers(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, error) { +func getTransformerOverridesAndKMSPluginHealthzCheckers(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, *kmsState, error) { var kmsHealthChecks []healthz.HealthChecker - transformers, probes, err := getTransformerOverridesAndKMSPluginProbes(config, stopCh) + transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(config, stopCh) if err != nil { - return nil, nil, err + return nil, nil, nil, err } for i := range probes { probe := probes[i] kmsHealthChecks = append(kmsHealthChecks, probe.toHealthzCheck(i)) } - return transformers, kmsHealthChecks, nil + return transformers, kmsHealthChecks, kmsUsed, nil } type healthChecker interface { toHealthzCheck(idx int) healthz.HealthChecker } -func getTransformerOverridesAndKMSPluginProbes(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthChecker, error) { +func getTransformerOverridesAndKMSPluginProbes(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthChecker, *kmsState, error) { resourceToPrefixTransformer := map[schema.GroupResource][]value.PrefixTransformer{} var probes []healthChecker + var kmsUsed kmsState // For each entry in the configuration for _, resourceConfig := range config.Resources { resourceConfig := resourceConfig - transformers, p, err := prefixTransformersAndProbes(resourceConfig, stopCh) + transformers, p, used, err := prefixTransformersAndProbes(resourceConfig, stopCh) if err != nil { - return nil, nil, err + return nil, nil, nil, err } + kmsUsed.v1Used = kmsUsed.v1Used || used.v1Used + kmsUsed.v2Used = kmsUsed.v2Used || used.v2Used // For each resource, create a list of providers to use for _, resource := range resourceConfig.Resources { @@ -152,7 +186,7 @@ func getTransformerOverridesAndKMSPluginProbes(config *apiserverconfig.Encryptio transformers[gr] = value.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...) } - return transformers, probes, nil + return transformers, probes, &kmsUsed, nil } // check encrypts and decrypts test data against KMS-Plugin's gRPC endpoint. @@ -168,13 +202,13 @@ func (h *kmsPluginProbe) check() error { if err != nil { h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()} h.ttl = kmsPluginHealthzNegativeTTL - return fmt.Errorf("failed to perform encrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err) + return fmt.Errorf("failed to perform encrypt section of the healthz check for KMS Provider %s, error: %w", h.name, err) } if _, err := h.service.Decrypt(p); err != nil { h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()} h.ttl = kmsPluginHealthzNegativeTTL - return fmt.Errorf("failed to perform decrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err) + return fmt.Errorf("failed to perform decrypt section of the healthz check for KMS Provider %s, error: %w", h.name, err) } h.lastResponse = &kmsPluginHealthzResponse{err: nil, received: time.Now()} @@ -195,7 +229,7 @@ func (h *kmsv2PluginProbe) check(ctx context.Context) error { if err != nil { h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()} h.ttl = kmsPluginHealthzNegativeTTL - return fmt.Errorf("failed to perform status section of the healthz check for KMS Provider %s, error: %v", h.name, err) + return fmt.Errorf("failed to perform status section of the healthz check for KMS Provider %s, error: %w", h.name, err) } if err := isKMSv2ProviderHealthy(h.name, p); err != nil { @@ -223,7 +257,7 @@ func isKMSv2ProviderHealthy(name string, response *envelopekmsv2.StatusResponse) } if err := utilerrors.Reduce(utilerrors.NewAggregate(errs)); err != nil { - return fmt.Errorf("kmsv2 Provider %s is not healthy, error: %v", name, err) + return fmt.Errorf("kmsv2 Provider %s is not healthy, error: %w", name, err) } return nil } @@ -231,13 +265,13 @@ func isKMSv2ProviderHealthy(name string, response *envelopekmsv2.StatusResponse) func loadConfig(filepath string) (*apiserverconfig.EncryptionConfiguration, error) { f, err := os.Open(filepath) if err != nil { - return nil, fmt.Errorf("error opening encryption provider configuration file %q: %v", filepath, err) + return nil, fmt.Errorf("error opening encryption provider configuration file %q: %w", filepath, err) } defer f.Close() data, err := io.ReadAll(f) if err != nil { - return nil, fmt.Errorf("could not read contents: %v", err) + return nil, fmt.Errorf("could not read contents: %w", err) } if len(data) == 0 { return nil, fmt.Errorf("encryption provider configuration file %q is empty", filepath) @@ -260,9 +294,10 @@ func loadConfig(filepath string) (*apiserverconfig.EncryptionConfiguration, erro return config, validation.ValidateEncryptionConfiguration(config).ToAggregate() } -func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, stopCh <-chan struct{}) ([]value.PrefixTransformer, []healthChecker, error) { +func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, stopCh <-chan struct{}) ([]value.PrefixTransformer, []healthChecker, *kmsState, error) { var transformers []value.PrefixTransformer var probes []healthChecker + var kmsUsed kmsState for _, provider := range config.Providers { provider := provider @@ -270,6 +305,7 @@ func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, s transformer value.PrefixTransformer transformerErr error probe healthChecker + used *kmsState ) switch { @@ -283,9 +319,11 @@ func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, s transformer, transformerErr = secretboxPrefixTransformer(provider.Secretbox) case provider.KMS != nil: - transformer, probe, transformerErr = kmsPrefixTransformer(provider.KMS, stopCh) + transformer, probe, used, transformerErr = kmsPrefixTransformer(provider.KMS, stopCh) if transformerErr == nil { probes = append(probes, probe) + kmsUsed.v1Used = kmsUsed.v1Used || used.v1Used + kmsUsed.v2Used = kmsUsed.v2Used || used.v2Used } case provider.Identity != nil: @@ -295,17 +333,17 @@ func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, s } default: - return nil, nil, errors.New("provider does not contain any of the expected providers: KMS, AESGCM, AESCBC, Secretbox, Identity") + return nil, nil, nil, errors.New("provider does not contain any of the expected providers: KMS, AESGCM, AESCBC, Secretbox, Identity") } if transformerErr != nil { - return nil, nil, transformerErr + return nil, nil, nil, transformerErr } transformers = append(transformers, transformer) } - return transformers, probes, nil + return transformers, probes, &kmsUsed, nil } type blockTransformerFunc func(cipher.Block) value.Transformer @@ -419,7 +457,11 @@ var ( EnvelopeKMSv2ServiceFactory = envelopekmsv2.NewGRPCService ) -func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-chan struct{}) (value.PrefixTransformer, healthChecker, error) { +type kmsState struct { + v1Used, v2Used bool +} + +func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-chan struct{}) (value.PrefixTransformer, healthChecker, *kmsState, error) { // we ignore the cancel func because this context should only be canceled when stopCh is closed ctx, _ := wait.ContextForChannel(stopCh) @@ -428,7 +470,7 @@ func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-cha case kmsAPIVersionV1: envelopeService, err := envelopeServiceFactory(ctx, config.Endpoint, config.Timeout.Duration) if err != nil { - return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %v", kmsName, err) + return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %w", kmsName, err) } probe := &kmsPluginProbe{ @@ -441,16 +483,16 @@ func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-cha transformer := envelopePrefixTransformer(config, envelopeService, kmsTransformerPrefixV1) - return transformer, probe, nil + return transformer, probe, &kmsState{v1Used: true}, nil case kmsAPIVersionV2: if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) { - return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", kmsName) + return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", kmsName) } envelopeService, err := EnvelopeKMSv2ServiceFactory(ctx, config.Endpoint, config.Timeout.Duration) if err != nil { - return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %v", kmsName, err) + return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %w", kmsName, err) } probe := &kmsv2PluginProbe{ @@ -467,10 +509,10 @@ func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-cha Prefix: []byte(kmsTransformerPrefixV2 + kmsName + ":"), } - return transformer, probe, nil + return transformer, probe, &kmsState{v2Used: true}, nil default: - return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", kmsName, config.APIVersion) + return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", kmsName, config.APIVersion) } } diff --git a/pkg/server/options/encryptionconfig/config_test.go b/pkg/server/options/encryptionconfig/config_test.go index 15c0e93cb..4d1dfdb83 100644 --- a/pkg/server/options/encryptionconfig/config_test.go +++ b/pkg/server/options/encryptionconfig/config_test.go @@ -177,37 +177,37 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) { // Transforms data using one of them, and tries to untransform using the others. // Repeats this for all possible combinations. correctConfigWithIdentityFirst := "testdata/valid-configs/identity-first.yaml" - identityFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithIdentityFirst, ctx.Done()) + identityFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithIdentityFirst, false, ctx.Done()) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithIdentityFirst) } correctConfigWithAesGcmFirst := "testdata/valid-configs/aes-gcm-first.yaml" - aesGcmFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithAesGcmFirst, ctx.Done()) + aesGcmFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithAesGcmFirst, false, ctx.Done()) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesGcmFirst) } correctConfigWithAesCbcFirst := "testdata/valid-configs/aes-cbc-first.yaml" - aesCbcFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithAesCbcFirst, ctx.Done()) + aesCbcFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithAesCbcFirst, false, ctx.Done()) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesCbcFirst) } correctConfigWithSecretboxFirst := "testdata/valid-configs/secret-box-first.yaml" - secretboxFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithSecretboxFirst, ctx.Done()) + secretboxFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithSecretboxFirst, false, ctx.Done()) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithSecretboxFirst) } correctConfigWithKMSFirst := "testdata/valid-configs/kms-first.yaml" - kmsFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithKMSFirst, ctx.Done()) + kmsFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithKMSFirst, false, ctx.Done()) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSFirst) } correctConfigWithKMSv2First := "testdata/valid-configs/kmsv2-first.yaml" - kmsv2FirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithKMSv2First, ctx.Done()) + kmsv2FirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithKMSv2First, false, ctx.Done()) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSv2First) } @@ -264,6 +264,8 @@ func TestKMSPluginHealthz(t *testing.T) { config string want []healthChecker wantErr string + kmsv2 bool + kmsv1 bool }{ { desc: "Install Healthz", @@ -274,6 +276,7 @@ func TestKMSPluginHealthz(t *testing.T) { ttl: 3 * time.Second, }, }, + kmsv1: true, }, { desc: "Install multiple healthz", @@ -288,6 +291,7 @@ func TestKMSPluginHealthz(t *testing.T) { ttl: 3 * time.Second, }, }, + kmsv1: true, }, { desc: "No KMS Providers", @@ -306,6 +310,8 @@ func TestKMSPluginHealthz(t *testing.T) { ttl: 3 * time.Second, }, }, + kmsv2: true, + kmsv1: true, }, { desc: "Invalid API version", @@ -325,7 +331,7 @@ func TestKMSPluginHealthz(t *testing.T) { return } - _, got, err := getTransformerOverridesAndKMSPluginProbes(config, testContext(t).Done()) + _, got, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(config, testContext(t).Done()) if err != nil { t.Fatal(err) } @@ -347,6 +353,13 @@ func TestKMSPluginHealthz(t *testing.T) { } } + if tt.kmsv2 != kmsUsed.v2Used { + t.Errorf("incorrect kms v2 detection: want=%v got=%v", tt.kmsv2, kmsUsed.v2Used) + } + if tt.kmsv1 != kmsUsed.v1Used { + t.Errorf("incorrect kms v1 detection: want=%v got=%v", tt.kmsv1, kmsUsed.v1Used) + } + if d := cmp.Diff(tt.want, got, cmp.Comparer(func(a, b *kmsPluginProbe) bool { return *a == *b @@ -528,7 +541,7 @@ func getTransformerFromEncryptionConfig(t *testing.T, encryptionConfigPath strin ctx := testContext(t) t.Helper() - transformers, _, err := LoadEncryptionConfig(encryptionConfigPath, ctx.Done()) + transformers, _, err := LoadEncryptionConfig(encryptionConfigPath, false, ctx.Done()) if err != nil { t.Fatal(err) } diff --git a/pkg/server/options/etcd.go b/pkg/server/options/etcd.go index 83210381b..683cc003c 100644 --- a/pkg/server/options/etcd.go +++ b/pkg/server/options/etcd.go @@ -43,8 +43,9 @@ import ( type EtcdOptions struct { // The value of Paging on StorageConfig will be overridden by the // calculated feature gate value. - StorageConfig storagebackend.Config - EncryptionProviderConfigFilepath string + StorageConfig storagebackend.Config + EncryptionProviderConfigFilepath string + EncryptionProviderConfigAutomaticReload bool EtcdServersOverrides []string @@ -117,6 +118,10 @@ func (s *EtcdOptions) Validate() []error { } + if len(s.EncryptionProviderConfigFilepath) == 0 && s.EncryptionProviderConfigAutomaticReload { + allErrors = append(allErrors, fmt.Errorf("--encryption-provider-config-automatic-reload must be set with --encryption-provider-config")) + } + return allErrors } @@ -182,6 +187,10 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.EncryptionProviderConfigFilepath, "encryption-provider-config", s.EncryptionProviderConfigFilepath, "The file containing configuration for encryption providers to be used for storing secrets in etcd") + fs.BoolVar(&s.EncryptionProviderConfigAutomaticReload, "encryption-provider-config-automatic-reload", s.EncryptionProviderConfigAutomaticReload, + "Determines if the file set by --encryption-provider-config should be automatically reloaded if the disk contents change. "+ + "Setting this to true disables the ability to uniquely identify distinct KMS plugins via the API server healthz endpoints.") + fs.DurationVar(&s.StorageConfig.CompactionInterval, "etcd-compaction-interval", s.StorageConfig.CompactionInterval, "The interval of compaction requests. If 0, the compaction request from apiserver is disabled.") @@ -214,7 +223,7 @@ func (s *EtcdOptions) Complete(storageObjectCountTracker flowcontrolrequest.Stor } if len(s.EncryptionProviderConfigFilepath) != 0 { - transformerOverrides, kmsPluginHealthzChecks, err := encryptionconfig.LoadEncryptionConfig(s.EncryptionProviderConfigFilepath, stopCh) + transformerOverrides, kmsPluginHealthzChecks, err := encryptionconfig.LoadEncryptionConfig(s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload, stopCh) if err != nil { return err } diff --git a/pkg/server/options/etcd_test.go b/pkg/server/options/etcd_test.go index 6fac3d53b..0527d3ba7 100644 --- a/pkg/server/options/etcd_test.go +++ b/pkg/server/options/etcd_test.go @@ -25,9 +25,13 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/healthz" "k8s.io/apiserver/pkg/storage/storagebackend" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" ) func TestEtcdOptionsValidate(t *testing.T) { @@ -108,6 +112,31 @@ func TestEtcdOptionsValidate(t *testing.T) { }, expectErr: "--etcd-servers-overrides invalid, must be of format: group/resource#servers, where servers are URLs, semicolon separated", }, + { + name: "test when encryption-provider-config-automatic-reload is invalid", + testOptions: &EtcdOptions{ + StorageConfig: storagebackend.Config{ + Type: "etcd3", + Prefix: "/registry", + Transport: storagebackend.TransportConfig{ + ServerList: []string{"http://127.0.0.1"}, + KeyFile: "/var/run/kubernetes/etcd.key", + TrustedCAFile: "/var/run/kubernetes/etcdca.crt", + CertFile: "/var/run/kubernetes/etcdce.crt", + }, + CompactionInterval: storagebackend.DefaultCompactInterval, + CountMetricPollPeriod: time.Minute, + }, + EncryptionProviderConfigAutomaticReload: true, + DefaultStorageMediaType: "application/vnd.kubernetes.protobuf", + DeleteCollectionWorkers: 1, + EnableGarbageCollection: true, + EnableWatchCache: true, + DefaultWatchCacheSize: 100, + EtcdServersOverrides: []string{"/events#http://127.0.0.1:4002"}, + }, + expectErr: "--encryption-provider-config-automatic-reload must be set with --encryption-provider-config", + }, { name: "test when EtcdOptions is valid", testOptions: &EtcdOptions{ @@ -200,12 +229,26 @@ func TestParseWatchCacheSizes(t *testing.T) { } func TestKMSHealthzEndpoint(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)() + testCases := []struct { name string encryptionConfigPath string wantChecks []string skipHealth bool + reload bool }{ + { + name: "no kms-provider, expect no kms healthz check", + encryptionConfigPath: "testdata/encryption-configs/no-kms-provider.yaml", + wantChecks: []string{"etcd"}, + }, + { + name: "no kms-provider+reload, expect single kms healthz check", + encryptionConfigPath: "testdata/encryption-configs/no-kms-provider.yaml", + reload: true, + wantChecks: []string{"etcd", "kms-providers"}, + }, { name: "single kms-provider, expect single kms healthz check", encryptionConfigPath: "testdata/encryption-configs/single-kms-provider.yaml", @@ -216,6 +259,34 @@ func TestKMSHealthzEndpoint(t *testing.T) { encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers.yaml", wantChecks: []string{"etcd", "kms-provider-0", "kms-provider-1"}, }, + { + name: "two kms-providers+reload, expect single kms healthz check", + encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers.yaml", + reload: true, + wantChecks: []string{"etcd", "kms-providers"}, + }, + { + name: "kms v1+v2, expect three kms healthz checks", + encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers-with-v2.yaml", + wantChecks: []string{"etcd", "kms-provider-0", "kms-provider-1", "kms-provider-2"}, + }, + { + name: "kms v1+v2+reload, expect single kms healthz check", + encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers-with-v2.yaml", + reload: true, + wantChecks: []string{"etcd", "kms-providers"}, + }, + { + name: "multiple kms v2, expect single kms healthz check", + encryptionConfigPath: "testdata/encryption-configs/multiple-kms-v2-providers.yaml", + wantChecks: []string{"etcd", "kms-providers"}, + }, + { + name: "multiple kms v2+reload, expect single kms healthz check", + encryptionConfigPath: "testdata/encryption-configs/multiple-kms-v2-providers.yaml", + reload: true, + wantChecks: []string{"etcd", "kms-providers"}, + }, { name: "two kms-providers with skip, expect zero kms healthz checks", encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers.yaml", @@ -231,8 +302,9 @@ func TestKMSHealthzEndpoint(t *testing.T) { t.Run(tc.name, func(t *testing.T) { serverConfig := server.NewConfig(codecs) etcdOptions := &EtcdOptions{ - EncryptionProviderConfigFilepath: tc.encryptionConfigPath, - SkipHealthEndpoints: tc.skipHealth, + EncryptionProviderConfigFilepath: tc.encryptionConfigPath, + EncryptionProviderConfigAutomaticReload: tc.reload, + SkipHealthEndpoints: tc.skipHealth, } if err := etcdOptions.Complete(serverConfig.StorageObjectCountTracker, serverConfig.DrainedNotify()); err != nil { t.Fatal(err) @@ -241,11 +313,7 @@ func TestKMSHealthzEndpoint(t *testing.T) { t.Fatalf("Failed to add healthz error: %v", err) } - for _, n := range tc.wantChecks { - if !hasCheck(n, serverConfig.HealthzChecks) { - t.Errorf("Missing HealthzChecker %s", n) - } - } + healthChecksAreEqual(t, tc.wantChecks, serverConfig.HealthzChecks) }) } } @@ -284,25 +352,25 @@ func TestReadinessCheck(t *testing.T) { t.Fatalf("Failed to add healthz error: %v", err) } - for _, n := range tc.wantReadyzChecks { - if !hasCheck(n, serverConfig.ReadyzChecks) { - t.Errorf("Missing ReadyzChecker %s", n) - } - } - for _, n := range tc.wantHealthzChecks { - if !hasCheck(n, serverConfig.HealthzChecks) { - t.Errorf("Missing HealthzChecker %s", n) - } - } + healthChecksAreEqual(t, tc.wantReadyzChecks, serverConfig.ReadyzChecks) + healthChecksAreEqual(t, tc.wantHealthzChecks, serverConfig.HealthzChecks) }) } } -func hasCheck(want string, healthchecks []healthz.HealthChecker) bool { - for _, h := range healthchecks { - if want == h.Name() { - return true - } +func healthChecksAreEqual(t *testing.T, want []string, healthChecks []healthz.HealthChecker) { + t.Helper() + + wantSet := sets.NewString(want...) + gotSet := sets.NewString() + + for _, h := range healthChecks { + gotSet.Insert(h.Name()) + } + + gotSet.Delete("log", "ping") // not relevant for our tests + + if !wantSet.Equal(gotSet) { + t.Errorf("healthz checks are not equal, missing=%q, extra=%q", wantSet.Difference(gotSet).List(), gotSet.Difference(wantSet).List()) } - return false } diff --git a/pkg/server/options/testdata/encryption-configs/multiple-kms-providers-with-v2.yaml b/pkg/server/options/testdata/encryption-configs/multiple-kms-providers-with-v2.yaml new file mode 100644 index 000000000..6003b60e1 --- /dev/null +++ b/pkg/server/options/testdata/encryption-configs/multiple-kms-providers-with-v2.yaml @@ -0,0 +1,18 @@ +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 +resources: + - resources: + - secrets + providers: + - kms: + name: kms-provider-1 + cachesize: 1000 + endpoint: unix:///@provider1.sock + - kms: + name: kms-provider-2 + cachesize: 1000 + endpoint: unix:///@provider2.sock + - kms: + apiVersion: v2 + name: kms-provider-3 + endpoint: unix:///@provider2.sock diff --git a/pkg/server/options/testdata/encryption-configs/multiple-kms-v2-providers.yaml b/pkg/server/options/testdata/encryption-configs/multiple-kms-v2-providers.yaml new file mode 100644 index 000000000..4ea8b0129 --- /dev/null +++ b/pkg/server/options/testdata/encryption-configs/multiple-kms-v2-providers.yaml @@ -0,0 +1,20 @@ +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 +resources: + - resources: + - secrets + providers: + - kms: + apiVersion: v2 + name: kms-provider-1 + cachesize: 1000 + endpoint: unix:///@provider1.sock + - kms: + apiVersion: v2 + name: kms-provider-2 + cachesize: 1000 + endpoint: unix:///@provider2.sock + - kms: + apiVersion: v2 + name: kms-provider-3 + endpoint: unix:///@provider2.sock diff --git a/pkg/server/options/testdata/encryption-configs/no-kms-provider.yaml b/pkg/server/options/testdata/encryption-configs/no-kms-provider.yaml new file mode 100644 index 000000000..37e871094 --- /dev/null +++ b/pkg/server/options/testdata/encryption-configs/no-kms-provider.yaml @@ -0,0 +1,10 @@ +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 +resources: + - resources: + - secrets + providers: + - aesgcm: + keys: + - name: key1 + secret: c2VjcmV0IGlzIHNlY3VyZQ==