From e137a3da77d46c87005252c7e76e5b4e2b650750 Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Wed, 2 Nov 2022 01:08:42 -0400 Subject: [PATCH] kms: add wiring to support automatic encryption config reload This change adds a flag --encryption-provider-config-automatic-reload which will be used to drive automatic reloading of the encryption config at runtime. While this flag is set to true, or when KMS v2 plugins are used without KMS v1 plugins, the /healthz endpoints associated with said plugins are collapsed into a single endpoint at /healthz/kms-providers - in this state, it is not possible to configure exclusions for specific KMS providers while including the remaining ones - ex: using /readyz?exclude=kms-provider-1 to exclude a particular KMS is not possible. This single healthz check handles checking all configured KMS providers. When reloading is enabled but no KMS providers are configured, it is a no-op. k8s.io/apiserver does not support dynamic addition and removal of healthz checks at runtime. Reloading will instead have a single static healthz check and swap the underlying implementation at runtime when a config change occurs. Signed-off-by: Monis Khan Kubernetes-commit: 22e540bc48d9bf698c4f381ccb56ed57dea0dae2 --- pkg/server/options/encryptionconfig/config.go | 104 +++++++++++----- .../options/encryptionconfig/config_test.go | 29 +++-- pkg/server/options/etcd.go | 15 ++- pkg/server/options/etcd_test.go | 114 ++++++++++++++---- .../multiple-kms-providers-with-v2.yaml | 18 +++ .../multiple-kms-v2-providers.yaml | 20 +++ .../encryption-configs/no-kms-provider.yaml | 10 ++ 7 files changed, 245 insertions(+), 65 deletions(-) create mode 100644 pkg/server/options/testdata/encryption-configs/multiple-kms-providers-with-v2.yaml create mode 100644 pkg/server/options/testdata/encryption-configs/multiple-kms-v2-providers.yaml create mode 100644 pkg/server/options/testdata/encryption-configs/no-kms-provider.yaml diff --git a/pkg/server/options/encryptionconfig/config.go b/pkg/server/options/encryptionconfig/config.go index af7506742..c91d2f94d 100644 --- a/pkg/server/options/encryptionconfig/config.go +++ b/pkg/server/options/encryptionconfig/config.go @@ -82,57 +82,91 @@ type kmsv2PluginProbe struct { l *sync.Mutex } +type kmsHealthChecker []healthz.HealthChecker + +func (k kmsHealthChecker) Name() string { + return "kms-providers" +} + +func (k kmsHealthChecker) Check(req *http.Request) error { + var errs []error + + for i := range k { + checker := k[i] + if err := checker.Check(req); err != nil { + errs = append(errs, fmt.Errorf("%s: %w", checker.Name(), err)) + } + } + + return utilerrors.Reduce(utilerrors.NewAggregate(errs)) +} + func (h *kmsPluginProbe) toHealthzCheck(idx int) healthz.HealthChecker { return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error { return h.check() }) } -func (p *kmsv2PluginProbe) toHealthzCheck(idx int) healthz.HealthChecker { +func (h *kmsv2PluginProbe) toHealthzCheck(idx int) healthz.HealthChecker { return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error { - return p.check(r.Context()) + return h.check(r.Context()) }) } -func LoadEncryptionConfig(filepath string, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, error) { +// LoadEncryptionConfig parses and validates the encryption config specified by filepath. +// It may launch multiple go routines whose lifecycle is controlled by stopCh. +// If reload is true, or KMS v2 plugins are used with no KMS v1 plugins, the returned slice of health checkers will always be of length 1. +func LoadEncryptionConfig(filepath string, reload bool, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, error) { config, err := loadConfig(filepath) if err != nil { - return nil, nil, fmt.Errorf("error while parsing file: %v", err) + return nil, nil, fmt.Errorf("error while parsing file: %w", err) } - return getTransformerOverridesAndKMSPluginHealthzCheckers(config, stopCh) + transformers, kmsHealthChecks, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(config, stopCh) + if err != nil { + return nil, nil, fmt.Errorf("error while building transformers: %w", err) + } + + if reload || (kmsUsed.v2Used && !kmsUsed.v1Used) { + kmsHealthChecks = []healthz.HealthChecker{kmsHealthChecker(kmsHealthChecks)} + } + + return transformers, kmsHealthChecks, nil } -func getTransformerOverridesAndKMSPluginHealthzCheckers(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, error) { +func getTransformerOverridesAndKMSPluginHealthzCheckers(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, *kmsState, error) { var kmsHealthChecks []healthz.HealthChecker - transformers, probes, err := getTransformerOverridesAndKMSPluginProbes(config, stopCh) + transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(config, stopCh) if err != nil { - return nil, nil, err + return nil, nil, nil, err } for i := range probes { probe := probes[i] kmsHealthChecks = append(kmsHealthChecks, probe.toHealthzCheck(i)) } - return transformers, kmsHealthChecks, nil + return transformers, kmsHealthChecks, kmsUsed, nil } type healthChecker interface { toHealthzCheck(idx int) healthz.HealthChecker } -func getTransformerOverridesAndKMSPluginProbes(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthChecker, error) { +func getTransformerOverridesAndKMSPluginProbes(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthChecker, *kmsState, error) { resourceToPrefixTransformer := map[schema.GroupResource][]value.PrefixTransformer{} var probes []healthChecker + var kmsUsed kmsState // For each entry in the configuration for _, resourceConfig := range config.Resources { resourceConfig := resourceConfig - transformers, p, err := prefixTransformersAndProbes(resourceConfig, stopCh) + transformers, p, used, err := prefixTransformersAndProbes(resourceConfig, stopCh) if err != nil { - return nil, nil, err + return nil, nil, nil, err } + kmsUsed.v1Used = kmsUsed.v1Used || used.v1Used + kmsUsed.v2Used = kmsUsed.v2Used || used.v2Used // For each resource, create a list of providers to use for _, resource := range resourceConfig.Resources { @@ -152,7 +186,7 @@ func getTransformerOverridesAndKMSPluginProbes(config *apiserverconfig.Encryptio transformers[gr] = value.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...) } - return transformers, probes, nil + return transformers, probes, &kmsUsed, nil } // check encrypts and decrypts test data against KMS-Plugin's gRPC endpoint. @@ -168,13 +202,13 @@ func (h *kmsPluginProbe) check() error { if err != nil { h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()} h.ttl = kmsPluginHealthzNegativeTTL - return fmt.Errorf("failed to perform encrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err) + return fmt.Errorf("failed to perform encrypt section of the healthz check for KMS Provider %s, error: %w", h.name, err) } if _, err := h.service.Decrypt(p); err != nil { h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()} h.ttl = kmsPluginHealthzNegativeTTL - return fmt.Errorf("failed to perform decrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err) + return fmt.Errorf("failed to perform decrypt section of the healthz check for KMS Provider %s, error: %w", h.name, err) } h.lastResponse = &kmsPluginHealthzResponse{err: nil, received: time.Now()} @@ -195,7 +229,7 @@ func (h *kmsv2PluginProbe) check(ctx context.Context) error { if err != nil { h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()} h.ttl = kmsPluginHealthzNegativeTTL - return fmt.Errorf("failed to perform status section of the healthz check for KMS Provider %s, error: %v", h.name, err) + return fmt.Errorf("failed to perform status section of the healthz check for KMS Provider %s, error: %w", h.name, err) } if err := isKMSv2ProviderHealthy(h.name, p); err != nil { @@ -223,7 +257,7 @@ func isKMSv2ProviderHealthy(name string, response *envelopekmsv2.StatusResponse) } if err := utilerrors.Reduce(utilerrors.NewAggregate(errs)); err != nil { - return fmt.Errorf("kmsv2 Provider %s is not healthy, error: %v", name, err) + return fmt.Errorf("kmsv2 Provider %s is not healthy, error: %w", name, err) } return nil } @@ -231,13 +265,13 @@ func isKMSv2ProviderHealthy(name string, response *envelopekmsv2.StatusResponse) func loadConfig(filepath string) (*apiserverconfig.EncryptionConfiguration, error) { f, err := os.Open(filepath) if err != nil { - return nil, fmt.Errorf("error opening encryption provider configuration file %q: %v", filepath, err) + return nil, fmt.Errorf("error opening encryption provider configuration file %q: %w", filepath, err) } defer f.Close() data, err := io.ReadAll(f) if err != nil { - return nil, fmt.Errorf("could not read contents: %v", err) + return nil, fmt.Errorf("could not read contents: %w", err) } if len(data) == 0 { return nil, fmt.Errorf("encryption provider configuration file %q is empty", filepath) @@ -260,9 +294,10 @@ func loadConfig(filepath string) (*apiserverconfig.EncryptionConfiguration, erro return config, validation.ValidateEncryptionConfiguration(config).ToAggregate() } -func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, stopCh <-chan struct{}) ([]value.PrefixTransformer, []healthChecker, error) { +func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, stopCh <-chan struct{}) ([]value.PrefixTransformer, []healthChecker, *kmsState, error) { var transformers []value.PrefixTransformer var probes []healthChecker + var kmsUsed kmsState for _, provider := range config.Providers { provider := provider @@ -270,6 +305,7 @@ func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, s transformer value.PrefixTransformer transformerErr error probe healthChecker + used *kmsState ) switch { @@ -283,9 +319,11 @@ func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, s transformer, transformerErr = secretboxPrefixTransformer(provider.Secretbox) case provider.KMS != nil: - transformer, probe, transformerErr = kmsPrefixTransformer(provider.KMS, stopCh) + transformer, probe, used, transformerErr = kmsPrefixTransformer(provider.KMS, stopCh) if transformerErr == nil { probes = append(probes, probe) + kmsUsed.v1Used = kmsUsed.v1Used || used.v1Used + kmsUsed.v2Used = kmsUsed.v2Used || used.v2Used } case provider.Identity != nil: @@ -295,17 +333,17 @@ func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, s } default: - return nil, nil, errors.New("provider does not contain any of the expected providers: KMS, AESGCM, AESCBC, Secretbox, Identity") + return nil, nil, nil, errors.New("provider does not contain any of the expected providers: KMS, AESGCM, AESCBC, Secretbox, Identity") } if transformerErr != nil { - return nil, nil, transformerErr + return nil, nil, nil, transformerErr } transformers = append(transformers, transformer) } - return transformers, probes, nil + return transformers, probes, &kmsUsed, nil } type blockTransformerFunc func(cipher.Block) value.Transformer @@ -419,7 +457,11 @@ var ( EnvelopeKMSv2ServiceFactory = envelopekmsv2.NewGRPCService ) -func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-chan struct{}) (value.PrefixTransformer, healthChecker, error) { +type kmsState struct { + v1Used, v2Used bool +} + +func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-chan struct{}) (value.PrefixTransformer, healthChecker, *kmsState, error) { // we ignore the cancel func because this context should only be canceled when stopCh is closed ctx, _ := wait.ContextForChannel(stopCh) @@ -428,7 +470,7 @@ func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-cha case kmsAPIVersionV1: envelopeService, err := envelopeServiceFactory(ctx, config.Endpoint, config.Timeout.Duration) if err != nil { - return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %v", kmsName, err) + return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %w", kmsName, err) } probe := &kmsPluginProbe{ @@ -441,16 +483,16 @@ func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-cha transformer := envelopePrefixTransformer(config, envelopeService, kmsTransformerPrefixV1) - return transformer, probe, nil + return transformer, probe, &kmsState{v1Used: true}, nil case kmsAPIVersionV2: if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) { - return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", kmsName) + return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", kmsName) } envelopeService, err := EnvelopeKMSv2ServiceFactory(ctx, config.Endpoint, config.Timeout.Duration) if err != nil { - return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %v", kmsName, err) + return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %w", kmsName, err) } probe := &kmsv2PluginProbe{ @@ -467,10 +509,10 @@ func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-cha Prefix: []byte(kmsTransformerPrefixV2 + kmsName + ":"), } - return transformer, probe, nil + return transformer, probe, &kmsState{v2Used: true}, nil default: - return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", kmsName, config.APIVersion) + return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", kmsName, config.APIVersion) } } diff --git a/pkg/server/options/encryptionconfig/config_test.go b/pkg/server/options/encryptionconfig/config_test.go index 15c0e93cb..4d1dfdb83 100644 --- a/pkg/server/options/encryptionconfig/config_test.go +++ b/pkg/server/options/encryptionconfig/config_test.go @@ -177,37 +177,37 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) { // Transforms data using one of them, and tries to untransform using the others. // Repeats this for all possible combinations. correctConfigWithIdentityFirst := "testdata/valid-configs/identity-first.yaml" - identityFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithIdentityFirst, ctx.Done()) + identityFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithIdentityFirst, false, ctx.Done()) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithIdentityFirst) } correctConfigWithAesGcmFirst := "testdata/valid-configs/aes-gcm-first.yaml" - aesGcmFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithAesGcmFirst, ctx.Done()) + aesGcmFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithAesGcmFirst, false, ctx.Done()) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesGcmFirst) } correctConfigWithAesCbcFirst := "testdata/valid-configs/aes-cbc-first.yaml" - aesCbcFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithAesCbcFirst, ctx.Done()) + aesCbcFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithAesCbcFirst, false, ctx.Done()) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesCbcFirst) } correctConfigWithSecretboxFirst := "testdata/valid-configs/secret-box-first.yaml" - secretboxFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithSecretboxFirst, ctx.Done()) + secretboxFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithSecretboxFirst, false, ctx.Done()) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithSecretboxFirst) } correctConfigWithKMSFirst := "testdata/valid-configs/kms-first.yaml" - kmsFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithKMSFirst, ctx.Done()) + kmsFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithKMSFirst, false, ctx.Done()) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSFirst) } correctConfigWithKMSv2First := "testdata/valid-configs/kmsv2-first.yaml" - kmsv2FirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithKMSv2First, ctx.Done()) + kmsv2FirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithKMSv2First, false, ctx.Done()) if err != nil { t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSv2First) } @@ -264,6 +264,8 @@ func TestKMSPluginHealthz(t *testing.T) { config string want []healthChecker wantErr string + kmsv2 bool + kmsv1 bool }{ { desc: "Install Healthz", @@ -274,6 +276,7 @@ func TestKMSPluginHealthz(t *testing.T) { ttl: 3 * time.Second, }, }, + kmsv1: true, }, { desc: "Install multiple healthz", @@ -288,6 +291,7 @@ func TestKMSPluginHealthz(t *testing.T) { ttl: 3 * time.Second, }, }, + kmsv1: true, }, { desc: "No KMS Providers", @@ -306,6 +310,8 @@ func TestKMSPluginHealthz(t *testing.T) { ttl: 3 * time.Second, }, }, + kmsv2: true, + kmsv1: true, }, { desc: "Invalid API version", @@ -325,7 +331,7 @@ func TestKMSPluginHealthz(t *testing.T) { return } - _, got, err := getTransformerOverridesAndKMSPluginProbes(config, testContext(t).Done()) + _, got, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(config, testContext(t).Done()) if err != nil { t.Fatal(err) } @@ -347,6 +353,13 @@ func TestKMSPluginHealthz(t *testing.T) { } } + if tt.kmsv2 != kmsUsed.v2Used { + t.Errorf("incorrect kms v2 detection: want=%v got=%v", tt.kmsv2, kmsUsed.v2Used) + } + if tt.kmsv1 != kmsUsed.v1Used { + t.Errorf("incorrect kms v1 detection: want=%v got=%v", tt.kmsv1, kmsUsed.v1Used) + } + if d := cmp.Diff(tt.want, got, cmp.Comparer(func(a, b *kmsPluginProbe) bool { return *a == *b @@ -528,7 +541,7 @@ func getTransformerFromEncryptionConfig(t *testing.T, encryptionConfigPath strin ctx := testContext(t) t.Helper() - transformers, _, err := LoadEncryptionConfig(encryptionConfigPath, ctx.Done()) + transformers, _, err := LoadEncryptionConfig(encryptionConfigPath, false, ctx.Done()) if err != nil { t.Fatal(err) } diff --git a/pkg/server/options/etcd.go b/pkg/server/options/etcd.go index 83210381b..683cc003c 100644 --- a/pkg/server/options/etcd.go +++ b/pkg/server/options/etcd.go @@ -43,8 +43,9 @@ import ( type EtcdOptions struct { // The value of Paging on StorageConfig will be overridden by the // calculated feature gate value. - StorageConfig storagebackend.Config - EncryptionProviderConfigFilepath string + StorageConfig storagebackend.Config + EncryptionProviderConfigFilepath string + EncryptionProviderConfigAutomaticReload bool EtcdServersOverrides []string @@ -117,6 +118,10 @@ func (s *EtcdOptions) Validate() []error { } + if len(s.EncryptionProviderConfigFilepath) == 0 && s.EncryptionProviderConfigAutomaticReload { + allErrors = append(allErrors, fmt.Errorf("--encryption-provider-config-automatic-reload must be set with --encryption-provider-config")) + } + return allErrors } @@ -182,6 +187,10 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.EncryptionProviderConfigFilepath, "encryption-provider-config", s.EncryptionProviderConfigFilepath, "The file containing configuration for encryption providers to be used for storing secrets in etcd") + fs.BoolVar(&s.EncryptionProviderConfigAutomaticReload, "encryption-provider-config-automatic-reload", s.EncryptionProviderConfigAutomaticReload, + "Determines if the file set by --encryption-provider-config should be automatically reloaded if the disk contents change. "+ + "Setting this to true disables the ability to uniquely identify distinct KMS plugins via the API server healthz endpoints.") + fs.DurationVar(&s.StorageConfig.CompactionInterval, "etcd-compaction-interval", s.StorageConfig.CompactionInterval, "The interval of compaction requests. If 0, the compaction request from apiserver is disabled.") @@ -214,7 +223,7 @@ func (s *EtcdOptions) Complete(storageObjectCountTracker flowcontrolrequest.Stor } if len(s.EncryptionProviderConfigFilepath) != 0 { - transformerOverrides, kmsPluginHealthzChecks, err := encryptionconfig.LoadEncryptionConfig(s.EncryptionProviderConfigFilepath, stopCh) + transformerOverrides, kmsPluginHealthzChecks, err := encryptionconfig.LoadEncryptionConfig(s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload, stopCh) if err != nil { return err } diff --git a/pkg/server/options/etcd_test.go b/pkg/server/options/etcd_test.go index 6fac3d53b..0527d3ba7 100644 --- a/pkg/server/options/etcd_test.go +++ b/pkg/server/options/etcd_test.go @@ -25,9 +25,13 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/healthz" "k8s.io/apiserver/pkg/storage/storagebackend" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" ) func TestEtcdOptionsValidate(t *testing.T) { @@ -108,6 +112,31 @@ func TestEtcdOptionsValidate(t *testing.T) { }, expectErr: "--etcd-servers-overrides invalid, must be of format: group/resource#servers, where servers are URLs, semicolon separated", }, + { + name: "test when encryption-provider-config-automatic-reload is invalid", + testOptions: &EtcdOptions{ + StorageConfig: storagebackend.Config{ + Type: "etcd3", + Prefix: "/registry", + Transport: storagebackend.TransportConfig{ + ServerList: []string{"http://127.0.0.1"}, + KeyFile: "/var/run/kubernetes/etcd.key", + TrustedCAFile: "/var/run/kubernetes/etcdca.crt", + CertFile: "/var/run/kubernetes/etcdce.crt", + }, + CompactionInterval: storagebackend.DefaultCompactInterval, + CountMetricPollPeriod: time.Minute, + }, + EncryptionProviderConfigAutomaticReload: true, + DefaultStorageMediaType: "application/vnd.kubernetes.protobuf", + DeleteCollectionWorkers: 1, + EnableGarbageCollection: true, + EnableWatchCache: true, + DefaultWatchCacheSize: 100, + EtcdServersOverrides: []string{"/events#http://127.0.0.1:4002"}, + }, + expectErr: "--encryption-provider-config-automatic-reload must be set with --encryption-provider-config", + }, { name: "test when EtcdOptions is valid", testOptions: &EtcdOptions{ @@ -200,12 +229,26 @@ func TestParseWatchCacheSizes(t *testing.T) { } func TestKMSHealthzEndpoint(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)() + testCases := []struct { name string encryptionConfigPath string wantChecks []string skipHealth bool + reload bool }{ + { + name: "no kms-provider, expect no kms healthz check", + encryptionConfigPath: "testdata/encryption-configs/no-kms-provider.yaml", + wantChecks: []string{"etcd"}, + }, + { + name: "no kms-provider+reload, expect single kms healthz check", + encryptionConfigPath: "testdata/encryption-configs/no-kms-provider.yaml", + reload: true, + wantChecks: []string{"etcd", "kms-providers"}, + }, { name: "single kms-provider, expect single kms healthz check", encryptionConfigPath: "testdata/encryption-configs/single-kms-provider.yaml", @@ -216,6 +259,34 @@ func TestKMSHealthzEndpoint(t *testing.T) { encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers.yaml", wantChecks: []string{"etcd", "kms-provider-0", "kms-provider-1"}, }, + { + name: "two kms-providers+reload, expect single kms healthz check", + encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers.yaml", + reload: true, + wantChecks: []string{"etcd", "kms-providers"}, + }, + { + name: "kms v1+v2, expect three kms healthz checks", + encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers-with-v2.yaml", + wantChecks: []string{"etcd", "kms-provider-0", "kms-provider-1", "kms-provider-2"}, + }, + { + name: "kms v1+v2+reload, expect single kms healthz check", + encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers-with-v2.yaml", + reload: true, + wantChecks: []string{"etcd", "kms-providers"}, + }, + { + name: "multiple kms v2, expect single kms healthz check", + encryptionConfigPath: "testdata/encryption-configs/multiple-kms-v2-providers.yaml", + wantChecks: []string{"etcd", "kms-providers"}, + }, + { + name: "multiple kms v2+reload, expect single kms healthz check", + encryptionConfigPath: "testdata/encryption-configs/multiple-kms-v2-providers.yaml", + reload: true, + wantChecks: []string{"etcd", "kms-providers"}, + }, { name: "two kms-providers with skip, expect zero kms healthz checks", encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers.yaml", @@ -231,8 +302,9 @@ func TestKMSHealthzEndpoint(t *testing.T) { t.Run(tc.name, func(t *testing.T) { serverConfig := server.NewConfig(codecs) etcdOptions := &EtcdOptions{ - EncryptionProviderConfigFilepath: tc.encryptionConfigPath, - SkipHealthEndpoints: tc.skipHealth, + EncryptionProviderConfigFilepath: tc.encryptionConfigPath, + EncryptionProviderConfigAutomaticReload: tc.reload, + SkipHealthEndpoints: tc.skipHealth, } if err := etcdOptions.Complete(serverConfig.StorageObjectCountTracker, serverConfig.DrainedNotify()); err != nil { t.Fatal(err) @@ -241,11 +313,7 @@ func TestKMSHealthzEndpoint(t *testing.T) { t.Fatalf("Failed to add healthz error: %v", err) } - for _, n := range tc.wantChecks { - if !hasCheck(n, serverConfig.HealthzChecks) { - t.Errorf("Missing HealthzChecker %s", n) - } - } + healthChecksAreEqual(t, tc.wantChecks, serverConfig.HealthzChecks) }) } } @@ -284,25 +352,25 @@ func TestReadinessCheck(t *testing.T) { t.Fatalf("Failed to add healthz error: %v", err) } - for _, n := range tc.wantReadyzChecks { - if !hasCheck(n, serverConfig.ReadyzChecks) { - t.Errorf("Missing ReadyzChecker %s", n) - } - } - for _, n := range tc.wantHealthzChecks { - if !hasCheck(n, serverConfig.HealthzChecks) { - t.Errorf("Missing HealthzChecker %s", n) - } - } + healthChecksAreEqual(t, tc.wantReadyzChecks, serverConfig.ReadyzChecks) + healthChecksAreEqual(t, tc.wantHealthzChecks, serverConfig.HealthzChecks) }) } } -func hasCheck(want string, healthchecks []healthz.HealthChecker) bool { - for _, h := range healthchecks { - if want == h.Name() { - return true - } +func healthChecksAreEqual(t *testing.T, want []string, healthChecks []healthz.HealthChecker) { + t.Helper() + + wantSet := sets.NewString(want...) + gotSet := sets.NewString() + + for _, h := range healthChecks { + gotSet.Insert(h.Name()) + } + + gotSet.Delete("log", "ping") // not relevant for our tests + + if !wantSet.Equal(gotSet) { + t.Errorf("healthz checks are not equal, missing=%q, extra=%q", wantSet.Difference(gotSet).List(), gotSet.Difference(wantSet).List()) } - return false } diff --git a/pkg/server/options/testdata/encryption-configs/multiple-kms-providers-with-v2.yaml b/pkg/server/options/testdata/encryption-configs/multiple-kms-providers-with-v2.yaml new file mode 100644 index 000000000..6003b60e1 --- /dev/null +++ b/pkg/server/options/testdata/encryption-configs/multiple-kms-providers-with-v2.yaml @@ -0,0 +1,18 @@ +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 +resources: + - resources: + - secrets + providers: + - kms: + name: kms-provider-1 + cachesize: 1000 + endpoint: unix:///@provider1.sock + - kms: + name: kms-provider-2 + cachesize: 1000 + endpoint: unix:///@provider2.sock + - kms: + apiVersion: v2 + name: kms-provider-3 + endpoint: unix:///@provider2.sock diff --git a/pkg/server/options/testdata/encryption-configs/multiple-kms-v2-providers.yaml b/pkg/server/options/testdata/encryption-configs/multiple-kms-v2-providers.yaml new file mode 100644 index 000000000..4ea8b0129 --- /dev/null +++ b/pkg/server/options/testdata/encryption-configs/multiple-kms-v2-providers.yaml @@ -0,0 +1,20 @@ +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 +resources: + - resources: + - secrets + providers: + - kms: + apiVersion: v2 + name: kms-provider-1 + cachesize: 1000 + endpoint: unix:///@provider1.sock + - kms: + apiVersion: v2 + name: kms-provider-2 + cachesize: 1000 + endpoint: unix:///@provider2.sock + - kms: + apiVersion: v2 + name: kms-provider-3 + endpoint: unix:///@provider2.sock diff --git a/pkg/server/options/testdata/encryption-configs/no-kms-provider.yaml b/pkg/server/options/testdata/encryption-configs/no-kms-provider.yaml new file mode 100644 index 000000000..37e871094 --- /dev/null +++ b/pkg/server/options/testdata/encryption-configs/no-kms-provider.yaml @@ -0,0 +1,10 @@ +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 +resources: + - resources: + - secrets + providers: + - aesgcm: + keys: + - name: key1 + secret: c2VjcmV0IGlzIHNlY3VyZQ==