kms: add wiring to support automatic encryption config reload

This change adds a flag --encryption-provider-config-automatic-reload
which will be used to drive automatic reloading of the encryption
config at runtime.  While this flag is set to true, or when KMS v2
plugins are used without KMS v1 plugins, the /healthz endpoints
associated with said plugins are collapsed into a single endpoint at
/healthz/kms-providers - in this state, it is not possible to
configure exclusions for specific KMS providers while including the
remaining ones - ex: using /readyz?exclude=kms-provider-1 to exclude
a particular KMS is not possible.  This single healthz check handles
checking  all configured KMS providers.  When reloading is enabled
but no KMS providers are configured, it is a no-op.

k8s.io/apiserver does not support dynamic addition and removal of
healthz checks at runtime.  Reloading will instead have a single
static healthz check and swap the underlying implementation at
runtime when a config change occurs.

Signed-off-by: Monis Khan <mok@microsoft.com>

Kubernetes-commit: 22e540bc48d9bf698c4f381ccb56ed57dea0dae2

pkg/server/options/encryptionconfig/config.go

@@ -82,57 +82,91 @@ type kmsv2PluginProbe struct {
 	l            *sync.Mutex
 }
 
+type kmsHealthChecker []healthz.HealthChecker
+
+func (k kmsHealthChecker) Name() string {
+	return "kms-providers"
+}
+
+func (k kmsHealthChecker) Check(req *http.Request) error {
+	var errs []error
+
+	for i := range k {
+		checker := k[i]
+		if err := checker.Check(req); err != nil {
+			errs = append(errs, fmt.Errorf("%s: %w", checker.Name(), err))
+		}
+	}
+
+	return utilerrors.Reduce(utilerrors.NewAggregate(errs))
+}
+
 func (h *kmsPluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
 	return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error {
 		return h.check()
 	})
 }
 
-func (p *kmsv2PluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
+func (h *kmsv2PluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
 	return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error {
-		return p.check(r.Context())
+		return h.check(r.Context())
 	})
 }
 
-func LoadEncryptionConfig(filepath string, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, error) {
+// LoadEncryptionConfig parses and validates the encryption config specified by filepath.
+// It may launch multiple go routines whose lifecycle is controlled by stopCh.
+// If reload is true, or KMS v2 plugins are used with no KMS v1 plugins, the returned slice of health checkers will always be of length 1.
+func LoadEncryptionConfig(filepath string, reload bool, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, error) {
 	config, err := loadConfig(filepath)
 	if err != nil {
-		return nil, nil, fmt.Errorf("error while parsing file: %v", err)
+		return nil, nil, fmt.Errorf("error while parsing file: %w", err)
 	}
 
-	return getTransformerOverridesAndKMSPluginHealthzCheckers(config, stopCh)
+	transformers, kmsHealthChecks, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(config, stopCh)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error while building transformers: %w", err)
+	}
+
+	if reload || (kmsUsed.v2Used && !kmsUsed.v1Used) {
+		kmsHealthChecks = []healthz.HealthChecker{kmsHealthChecker(kmsHealthChecks)}
+	}
+
+	return transformers, kmsHealthChecks, nil
 }
 
-func getTransformerOverridesAndKMSPluginHealthzCheckers(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, error) {
+func getTransformerOverridesAndKMSPluginHealthzCheckers(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, *kmsState, error) {
 	var kmsHealthChecks []healthz.HealthChecker
-	transformers, probes, err := getTransformerOverridesAndKMSPluginProbes(config, stopCh)
+	transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(config, stopCh)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 	for i := range probes {
 		probe := probes[i]
 		kmsHealthChecks = append(kmsHealthChecks, probe.toHealthzCheck(i))
 	}
 
-	return transformers, kmsHealthChecks, nil
+	return transformers, kmsHealthChecks, kmsUsed, nil
 }
 
 type healthChecker interface {
 	toHealthzCheck(idx int) healthz.HealthChecker
 }
 
-func getTransformerOverridesAndKMSPluginProbes(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthChecker, error) {
+func getTransformerOverridesAndKMSPluginProbes(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthChecker, *kmsState, error) {
 	resourceToPrefixTransformer := map[schema.GroupResource][]value.PrefixTransformer{}
 	var probes []healthChecker
+	var kmsUsed kmsState
 
 	// For each entry in the configuration
 	for _, resourceConfig := range config.Resources {
 		resourceConfig := resourceConfig
 
-		transformers, p, err := prefixTransformersAndProbes(resourceConfig, stopCh)
+		transformers, p, used, err := prefixTransformersAndProbes(resourceConfig, stopCh)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
+		kmsUsed.v1Used = kmsUsed.v1Used || used.v1Used
+		kmsUsed.v2Used = kmsUsed.v2Used || used.v2Used
 
 		// For each resource, create a list of providers to use
 		for _, resource := range resourceConfig.Resources {
@@ -152,7 +186,7 @@ func getTransformerOverridesAndKMSPluginProbes(config *apiserverconfig.Encryptio
 		transformers[gr] = value.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...)
 	}
 
-	return transformers, probes, nil
+	return transformers, probes, &kmsUsed, nil
 }
 
 // check encrypts and decrypts test data against KMS-Plugin's gRPC endpoint.
@@ -168,13 +202,13 @@ func (h *kmsPluginProbe) check() error {
 	if err != nil {
 		h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
 		h.ttl = kmsPluginHealthzNegativeTTL
-		return fmt.Errorf("failed to perform encrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err)
+		return fmt.Errorf("failed to perform encrypt section of the healthz check for KMS Provider %s, error: %w", h.name, err)
 	}
 
 	if _, err := h.service.Decrypt(p); err != nil {
 		h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
 		h.ttl = kmsPluginHealthzNegativeTTL
-		return fmt.Errorf("failed to perform decrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err)
+		return fmt.Errorf("failed to perform decrypt section of the healthz check for KMS Provider %s, error: %w", h.name, err)
 	}
 
 	h.lastResponse = &kmsPluginHealthzResponse{err: nil, received: time.Now()}
@@ -195,7 +229,7 @@ func (h *kmsv2PluginProbe) check(ctx context.Context) error {
 	if err != nil {
 		h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
 		h.ttl = kmsPluginHealthzNegativeTTL
-		return fmt.Errorf("failed to perform status section of the healthz check for KMS Provider %s, error: %v", h.name, err)
+		return fmt.Errorf("failed to perform status section of the healthz check for KMS Provider %s, error: %w", h.name, err)
 	}
 
 	if err := isKMSv2ProviderHealthy(h.name, p); err != nil {
@@ -223,7 +257,7 @@ func isKMSv2ProviderHealthy(name string, response *envelopekmsv2.StatusResponse)
 	}
 
 	if err := utilerrors.Reduce(utilerrors.NewAggregate(errs)); err != nil {
-		return fmt.Errorf("kmsv2 Provider %s is not healthy, error: %v", name, err)
+		return fmt.Errorf("kmsv2 Provider %s is not healthy, error: %w", name, err)
 	}
 	return nil
 }
@@ -231,13 +265,13 @@ func isKMSv2ProviderHealthy(name string, response *envelopekmsv2.StatusResponse)
 func loadConfig(filepath string) (*apiserverconfig.EncryptionConfiguration, error) {
 	f, err := os.Open(filepath)
 	if err != nil {
-		return nil, fmt.Errorf("error opening encryption provider configuration file %q: %v", filepath, err)
+		return nil, fmt.Errorf("error opening encryption provider configuration file %q: %w", filepath, err)
 	}
 	defer f.Close()
 
 	data, err := io.ReadAll(f)
 	if err != nil {
-		return nil, fmt.Errorf("could not read contents: %v", err)
+		return nil, fmt.Errorf("could not read contents: %w", err)
 	}
 	if len(data) == 0 {
 		return nil, fmt.Errorf("encryption provider configuration file %q is empty", filepath)
@@ -260,9 +294,10 @@ func loadConfig(filepath string) (*apiserverconfig.EncryptionConfiguration, erro
 	return config, validation.ValidateEncryptionConfiguration(config).ToAggregate()
 }
 
-func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, stopCh <-chan struct{}) ([]value.PrefixTransformer, []healthChecker, error) {
+func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, stopCh <-chan struct{}) ([]value.PrefixTransformer, []healthChecker, *kmsState, error) {
 	var transformers []value.PrefixTransformer
 	var probes []healthChecker
+	var kmsUsed kmsState
 
 	for _, provider := range config.Providers {
 		provider := provider
@@ -270,6 +305,7 @@ func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, s
 			transformer    value.PrefixTransformer
 			transformerErr error
 			probe          healthChecker
+			used           *kmsState
 		)
 
 		switch {
@@ -283,9 +319,11 @@ func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, s
 			transformer, transformerErr = secretboxPrefixTransformer(provider.Secretbox)
 
 		case provider.KMS != nil:
-			transformer, probe, transformerErr = kmsPrefixTransformer(provider.KMS, stopCh)
+			transformer, probe, used, transformerErr = kmsPrefixTransformer(provider.KMS, stopCh)
 			if transformerErr == nil {
 				probes = append(probes, probe)
+				kmsUsed.v1Used = kmsUsed.v1Used || used.v1Used
+				kmsUsed.v2Used = kmsUsed.v2Used || used.v2Used
 			}
 
 		case provider.Identity != nil:
@@ -295,17 +333,17 @@ func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, s
 			}
 
 		default:
-			return nil, nil, errors.New("provider does not contain any of the expected providers: KMS, AESGCM, AESCBC, Secretbox, Identity")
+			return nil, nil, nil, errors.New("provider does not contain any of the expected providers: KMS, AESGCM, AESCBC, Secretbox, Identity")
 		}
 
 		if transformerErr != nil {
-			return nil, nil, transformerErr
+			return nil, nil, nil, transformerErr
 		}
 
 		transformers = append(transformers, transformer)
 	}
 
-	return transformers, probes, nil
+	return transformers, probes, &kmsUsed, nil
 }
 
 type blockTransformerFunc func(cipher.Block) value.Transformer
@@ -419,7 +457,11 @@ var (
 	EnvelopeKMSv2ServiceFactory = envelopekmsv2.NewGRPCService
 )
 
-func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-chan struct{}) (value.PrefixTransformer, healthChecker, error) {
+type kmsState struct {
+	v1Used, v2Used bool
+}
+
+func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-chan struct{}) (value.PrefixTransformer, healthChecker, *kmsState, error) {
 	// we ignore the cancel func because this context should only be canceled when stopCh is closed
 	ctx, _ := wait.ContextForChannel(stopCh)
 
@@ -428,7 +470,7 @@ func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-cha
 	case kmsAPIVersionV1:
 		envelopeService, err := envelopeServiceFactory(ctx, config.Endpoint, config.Timeout.Duration)
 		if err != nil {
-			return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %v", kmsName, err)
+			return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %w", kmsName, err)
 		}
 
 		probe := &kmsPluginProbe{
@@ -441,16 +483,16 @@ func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-cha
 
 		transformer := envelopePrefixTransformer(config, envelopeService, kmsTransformerPrefixV1)
 
-		return transformer, probe, nil
+		return transformer, probe, &kmsState{v1Used: true}, nil
 
 	case kmsAPIVersionV2:
 		if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) {
-			return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", kmsName)
+			return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", kmsName)
 		}
 
 		envelopeService, err := EnvelopeKMSv2ServiceFactory(ctx, config.Endpoint, config.Timeout.Duration)
 		if err != nil {
-			return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %v", kmsName, err)
+			return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %w", kmsName, err)
 		}
 
 		probe := &kmsv2PluginProbe{
@@ -467,10 +509,10 @@ func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-cha
 			Prefix:      []byte(kmsTransformerPrefixV2 + kmsName + ":"),
 		}
 
-		return transformer, probe, nil
+		return transformer, probe, &kmsState{v2Used: true}, nil
 
 	default:
-		return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", kmsName, config.APIVersion)
+		return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", kmsName, config.APIVersion)
 	}
 }
 

pkg/server/options/encryptionconfig/config_test.go

@@ -177,37 +177,37 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) {
 	// Transforms data using one of them, and tries to untransform using the others.
 	// Repeats this for all possible combinations.
 	correctConfigWithIdentityFirst := "testdata/valid-configs/identity-first.yaml"
-	identityFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithIdentityFirst, ctx.Done())
+	identityFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithIdentityFirst, false, ctx.Done())
 	if err != nil {
 		t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithIdentityFirst)
 	}
 
 	correctConfigWithAesGcmFirst := "testdata/valid-configs/aes-gcm-first.yaml"
-	aesGcmFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithAesGcmFirst, ctx.Done())
+	aesGcmFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithAesGcmFirst, false, ctx.Done())
 	if err != nil {
 		t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesGcmFirst)
 	}
 
 	correctConfigWithAesCbcFirst := "testdata/valid-configs/aes-cbc-first.yaml"
-	aesCbcFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithAesCbcFirst, ctx.Done())
+	aesCbcFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithAesCbcFirst, false, ctx.Done())
 	if err != nil {
 		t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesCbcFirst)
 	}
 
 	correctConfigWithSecretboxFirst := "testdata/valid-configs/secret-box-first.yaml"
-	secretboxFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithSecretboxFirst, ctx.Done())
+	secretboxFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithSecretboxFirst, false, ctx.Done())
 	if err != nil {
 		t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithSecretboxFirst)
 	}
 
 	correctConfigWithKMSFirst := "testdata/valid-configs/kms-first.yaml"
-	kmsFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithKMSFirst, ctx.Done())
+	kmsFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithKMSFirst, false, ctx.Done())
 	if err != nil {
 		t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSFirst)
 	}
 
 	correctConfigWithKMSv2First := "testdata/valid-configs/kmsv2-first.yaml"
-	kmsv2FirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithKMSv2First, ctx.Done())
+	kmsv2FirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithKMSv2First, false, ctx.Done())
 	if err != nil {
 		t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSv2First)
 	}
@@ -264,6 +264,8 @@ func TestKMSPluginHealthz(t *testing.T) {
 		config  string
 		want    []healthChecker
 		wantErr string
+		kmsv2   bool
+		kmsv1   bool
 	}{
 		{
 			desc:   "Install Healthz",
@@ -274,6 +276,7 @@ func TestKMSPluginHealthz(t *testing.T) {
 					ttl:  3 * time.Second,
 				},
 			},
+			kmsv1: true,
 		},
 		{
 			desc:   "Install multiple healthz",
@@ -288,6 +291,7 @@ func TestKMSPluginHealthz(t *testing.T) {
 					ttl:  3 * time.Second,
 				},
 			},
+			kmsv1: true,
 		},
 		{
 			desc:   "No KMS Providers",
@@ -306,6 +310,8 @@ func TestKMSPluginHealthz(t *testing.T) {
 					ttl:  3 * time.Second,
 				},
 			},
+			kmsv2: true,
+			kmsv1: true,
 		},
 		{
 			desc:    "Invalid API version",
@@ -325,7 +331,7 @@ func TestKMSPluginHealthz(t *testing.T) {
 				return
 			}
 
-			_, got, err := getTransformerOverridesAndKMSPluginProbes(config, testContext(t).Done())
+			_, got, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(config, testContext(t).Done())
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -347,6 +353,13 @@ func TestKMSPluginHealthz(t *testing.T) {
 				}
 			}
 
+			if tt.kmsv2 != kmsUsed.v2Used {
+				t.Errorf("incorrect kms v2 detection: want=%v got=%v", tt.kmsv2, kmsUsed.v2Used)
+			}
+			if tt.kmsv1 != kmsUsed.v1Used {
+				t.Errorf("incorrect kms v1 detection: want=%v got=%v", tt.kmsv1, kmsUsed.v1Used)
+			}
+
 			if d := cmp.Diff(tt.want, got,
 				cmp.Comparer(func(a, b *kmsPluginProbe) bool {
 					return *a == *b
@@ -528,7 +541,7 @@ func getTransformerFromEncryptionConfig(t *testing.T, encryptionConfigPath strin
 	ctx := testContext(t)
 
 	t.Helper()
-	transformers, _, err := LoadEncryptionConfig(encryptionConfigPath, ctx.Done())
+	transformers, _, err := LoadEncryptionConfig(encryptionConfigPath, false, ctx.Done())
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/server/options/etcd.go

@@ -43,8 +43,9 @@ import (
 type EtcdOptions struct {
 	// The value of Paging on StorageConfig will be overridden by the
 	// calculated feature gate value.
-	StorageConfig                    storagebackend.Config
+	StorageConfig                           storagebackend.Config
-	EncryptionProviderConfigFilepath string
+	EncryptionProviderConfigFilepath        string
+	EncryptionProviderConfigAutomaticReload bool
 
 	EtcdServersOverrides []string
 
@@ -117,6 +118,10 @@ func (s *EtcdOptions) Validate() []error {
 
 	}
 
+	if len(s.EncryptionProviderConfigFilepath) == 0 && s.EncryptionProviderConfigAutomaticReload {
+		allErrors = append(allErrors, fmt.Errorf("--encryption-provider-config-automatic-reload must be set with --encryption-provider-config"))
+	}
+
 	return allErrors
 }
 
@@ -182,6 +187,10 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.EncryptionProviderConfigFilepath, "encryption-provider-config", s.EncryptionProviderConfigFilepath,
 		"The file containing configuration for encryption providers to be used for storing secrets in etcd")
 
+	fs.BoolVar(&s.EncryptionProviderConfigAutomaticReload, "encryption-provider-config-automatic-reload", s.EncryptionProviderConfigAutomaticReload,
+		"Determines if the file set by --encryption-provider-config should be automatically reloaded if the disk contents change. "+
+			"Setting this to true disables the ability to uniquely identify distinct KMS plugins via the API server healthz endpoints.")
+
 	fs.DurationVar(&s.StorageConfig.CompactionInterval, "etcd-compaction-interval", s.StorageConfig.CompactionInterval,
 		"The interval of compaction requests. If 0, the compaction request from apiserver is disabled.")
 
@@ -214,7 +223,7 @@ func (s *EtcdOptions) Complete(storageObjectCountTracker flowcontrolrequest.Stor
 	}
 
 	if len(s.EncryptionProviderConfigFilepath) != 0 {
-		transformerOverrides, kmsPluginHealthzChecks, err := encryptionconfig.LoadEncryptionConfig(s.EncryptionProviderConfigFilepath, stopCh)
+		transformerOverrides, kmsPluginHealthzChecks, err := encryptionconfig.LoadEncryptionConfig(s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload, stopCh)
 		if err != nil {
 			return err
 		}

pkg/server/options/etcd_test.go

@@ -25,9 +25,13 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
 )
 
 func TestEtcdOptionsValidate(t *testing.T) {
@@ -108,6 +112,31 @@ func TestEtcdOptionsValidate(t *testing.T) {
 			},
 			expectErr: "--etcd-servers-overrides invalid, must be of format: group/resource#servers, where servers are URLs, semicolon separated",
 		},
+		{
+			name: "test when encryption-provider-config-automatic-reload is invalid",
+			testOptions: &EtcdOptions{
+				StorageConfig: storagebackend.Config{
+					Type:   "etcd3",
+					Prefix: "/registry",
+					Transport: storagebackend.TransportConfig{
+						ServerList:    []string{"http://127.0.0.1"},
+						KeyFile:       "/var/run/kubernetes/etcd.key",
+						TrustedCAFile: "/var/run/kubernetes/etcdca.crt",
+						CertFile:      "/var/run/kubernetes/etcdce.crt",
+					},
+					CompactionInterval:    storagebackend.DefaultCompactInterval,
+					CountMetricPollPeriod: time.Minute,
+				},
+				EncryptionProviderConfigAutomaticReload: true,
+				DefaultStorageMediaType:                 "application/vnd.kubernetes.protobuf",
+				DeleteCollectionWorkers:                 1,
+				EnableGarbageCollection:                 true,
+				EnableWatchCache:                        true,
+				DefaultWatchCacheSize:                   100,
+				EtcdServersOverrides:                    []string{"/events#http://127.0.0.1:4002"},
+			},
+			expectErr: "--encryption-provider-config-automatic-reload must be set with --encryption-provider-config",
+		},
 		{
 			name: "test when EtcdOptions is valid",
 			testOptions: &EtcdOptions{
@@ -200,12 +229,26 @@ func TestParseWatchCacheSizes(t *testing.T) {
 }
 
 func TestKMSHealthzEndpoint(t *testing.T) {
+	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
+
 	testCases := []struct {
 		name                 string
 		encryptionConfigPath string
 		wantChecks           []string
 		skipHealth           bool
+		reload               bool
 	}{
+		{
+			name:                 "no kms-provider, expect no kms healthz check",
+			encryptionConfigPath: "testdata/encryption-configs/no-kms-provider.yaml",
+			wantChecks:           []string{"etcd"},
+		},
+		{
+			name:                 "no kms-provider+reload, expect single kms healthz check",
+			encryptionConfigPath: "testdata/encryption-configs/no-kms-provider.yaml",
+			reload:               true,
+			wantChecks:           []string{"etcd", "kms-providers"},
+		},
 		{
 			name:                 "single kms-provider, expect single kms healthz check",
 			encryptionConfigPath: "testdata/encryption-configs/single-kms-provider.yaml",
@@ -216,6 +259,34 @@ func TestKMSHealthzEndpoint(t *testing.T) {
 			encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers.yaml",
 			wantChecks:           []string{"etcd", "kms-provider-0", "kms-provider-1"},
 		},
+		{
+			name:                 "two kms-providers+reload, expect single kms healthz check",
+			encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers.yaml",
+			reload:               true,
+			wantChecks:           []string{"etcd", "kms-providers"},
+		},
+		{
+			name:                 "kms v1+v2, expect three kms healthz checks",
+			encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers-with-v2.yaml",
+			wantChecks:           []string{"etcd", "kms-provider-0", "kms-provider-1", "kms-provider-2"},
+		},
+		{
+			name:                 "kms v1+v2+reload, expect single kms healthz check",
+			encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers-with-v2.yaml",
+			reload:               true,
+			wantChecks:           []string{"etcd", "kms-providers"},
+		},
+		{
+			name:                 "multiple kms v2, expect single kms healthz check",
+			encryptionConfigPath: "testdata/encryption-configs/multiple-kms-v2-providers.yaml",
+			wantChecks:           []string{"etcd", "kms-providers"},
+		},
+		{
+			name:                 "multiple kms v2+reload, expect single kms healthz check",
+			encryptionConfigPath: "testdata/encryption-configs/multiple-kms-v2-providers.yaml",
+			reload:               true,
+			wantChecks:           []string{"etcd", "kms-providers"},
+		},
 		{
 			name:                 "two kms-providers with skip, expect zero kms healthz checks",
 			encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers.yaml",
@@ -231,8 +302,9 @@ func TestKMSHealthzEndpoint(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			serverConfig := server.NewConfig(codecs)
 			etcdOptions := &EtcdOptions{
-				EncryptionProviderConfigFilepath: tc.encryptionConfigPath,
+				EncryptionProviderConfigFilepath:        tc.encryptionConfigPath,
-				SkipHealthEndpoints:              tc.skipHealth,
+				EncryptionProviderConfigAutomaticReload: tc.reload,
+				SkipHealthEndpoints:                     tc.skipHealth,
 			}
 			if err := etcdOptions.Complete(serverConfig.StorageObjectCountTracker, serverConfig.DrainedNotify()); err != nil {
 				t.Fatal(err)
@@ -241,11 +313,7 @@ func TestKMSHealthzEndpoint(t *testing.T) {
 				t.Fatalf("Failed to add healthz error: %v", err)
 			}
 
-			for _, n := range tc.wantChecks {
+			healthChecksAreEqual(t, tc.wantChecks, serverConfig.HealthzChecks)
-				if !hasCheck(n, serverConfig.HealthzChecks) {
-					t.Errorf("Missing HealthzChecker %s", n)
-				}
-			}
 		})
 	}
 }
@@ -284,25 +352,25 @@ func TestReadinessCheck(t *testing.T) {
 				t.Fatalf("Failed to add healthz error: %v", err)
 			}
 
-			for _, n := range tc.wantReadyzChecks {
+			healthChecksAreEqual(t, tc.wantReadyzChecks, serverConfig.ReadyzChecks)
-				if !hasCheck(n, serverConfig.ReadyzChecks) {
+			healthChecksAreEqual(t, tc.wantHealthzChecks, serverConfig.HealthzChecks)
-					t.Errorf("Missing ReadyzChecker %s", n)
-				}
-			}
-			for _, n := range tc.wantHealthzChecks {
-				if !hasCheck(n, serverConfig.HealthzChecks) {
-					t.Errorf("Missing HealthzChecker %s", n)
-				}
-			}
 		})
 	}
 }
 
-func hasCheck(want string, healthchecks []healthz.HealthChecker) bool {
+func healthChecksAreEqual(t *testing.T, want []string, healthChecks []healthz.HealthChecker) {
-	for _, h := range healthchecks {
+	t.Helper()
-		if want == h.Name() {
+
-			return true
+	wantSet := sets.NewString(want...)
-		}
+	gotSet := sets.NewString()
+
+	for _, h := range healthChecks {
+		gotSet.Insert(h.Name())
+	}
+
+	gotSet.Delete("log", "ping") // not relevant for our tests
+
+	if !wantSet.Equal(gotSet) {
+		t.Errorf("healthz checks are not equal, missing=%q, extra=%q", wantSet.Difference(gotSet).List(), gotSet.Difference(wantSet).List())
 	}
-	return false
 }

pkg/server/options/testdata/encryption-configs/multiple-kms-providers-with-v2.yaml

@@ -0,0 +1,18 @@
+kind: EncryptionConfiguration
+apiVersion: apiserver.config.k8s.io/v1
+resources:
+  - resources:
+      - secrets
+    providers:
+      - kms:
+          name: kms-provider-1
+          cachesize: 1000
+          endpoint: unix:///@provider1.sock
+      - kms:
+          name: kms-provider-2
+          cachesize: 1000
+          endpoint: unix:///@provider2.sock
+      - kms:
+          apiVersion: v2
+          name: kms-provider-3
+          endpoint: unix:///@provider2.sock

pkg/server/options/testdata/encryption-configs/multiple-kms-v2-providers.yaml

@@ -0,0 +1,20 @@
+kind: EncryptionConfiguration
+apiVersion: apiserver.config.k8s.io/v1
+resources:
+  - resources:
+      - secrets
+    providers:
+      - kms:
+          apiVersion: v2
+          name: kms-provider-1
+          cachesize: 1000
+          endpoint: unix:///@provider1.sock
+      - kms:
+          apiVersion: v2
+          name: kms-provider-2
+          cachesize: 1000
+          endpoint: unix:///@provider2.sock
+      - kms:
+          apiVersion: v2
+          name: kms-provider-3
+          endpoint: unix:///@provider2.sock

pkg/server/options/testdata/encryption-configs/no-kms-provider.yaml

@@ -0,0 +1,10 @@
+kind: EncryptionConfiguration
+apiVersion: apiserver.config.k8s.io/v1
+resources:
+  - resources:
+      - secrets
+    providers:
+    - aesgcm:
+        keys:
+        - name: key1
+          secret: c2VjcmV0IGlzIHNlY3VyZQ==