kubernetes
/
apiserver
mirror of https://github.com/kubernetes/apiserver.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

kms: add wiring to support automatic encryption config reload 

This change adds a flag --encryption-provider-config-automatic-reload
which will be used to drive automatic reloading of the encryption
config at runtime.  While this flag is set to true, or when KMS v2
plugins are used without KMS v1 plugins, the /healthz endpoints
associated with said plugins are collapsed into a single endpoint at
/healthz/kms-providers - in this state, it is not possible to
configure exclusions for specific KMS providers while including the
remaining ones - ex: using /readyz?exclude=kms-provider-1 to exclude
a particular KMS is not possible.  This single healthz check handles
checking  all configured KMS providers.  When reloading is enabled
but no KMS providers are configured, it is a no-op.

k8s.io/apiserver does not support dynamic addition and removal of
healthz checks at runtime.  Reloading will instead have a single
static healthz check and swap the underlying implementation at
runtime when a config change occurs.

Signed-off-by: Monis Khan <mok@microsoft.com>

Kubernetes-commit: 22e540bc48d9bf698c4f381ccb56ed57dea0dae2
This commit is contained in:
Monis Khan 2022-11-02 01:08:42 -04:00 committed by Kubernetes Publisher
parent a3a25ce15d
commit e137a3da77
7 changed files with 245 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
pkg/server/options/encryptionconfig/config.go
View File

@ -82,57 +82,91 @@ type kmsv2PluginProbe struct {
l *sync.Mutex
}
type kmsHealthChecker []healthz.HealthChecker
func (k kmsHealthChecker) Name() string {
return "kms-providers"
}
func (k kmsHealthChecker) Check(req *http.Request) error {
var errs []error
for i := range k {
checker := k[i]
if err := checker.Check(req); err != nil {
errs = append(errs, fmt.Errorf("%s: %w", checker.Name(), err))
}
}
return utilerrors.Reduce(utilerrors.NewAggregate(errs))
}
func (h *kmsPluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error {
return h.check()
})
}
func (p *kmsv2PluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
func (h *kmsv2PluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error {
return p.check(r.Context())
return h.check(r.Context())
})
}
func LoadEncryptionConfig(filepath string, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, error) {
// LoadEncryptionConfig parses and validates the encryption config specified by filepath.
// It may launch multiple go routines whose lifecycle is controlled by stopCh.
// If reload is true, or KMS v2 plugins are used with no KMS v1 plugins, the returned slice of health checkers will always be of length 1.
func LoadEncryptionConfig(filepath string, reload bool, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, error) {
config, err := loadConfig(filepath)
if err != nil {
return nil, nil, fmt.Errorf("error while parsing file: %v", err)
return nil, nil, fmt.Errorf("error while parsing file: %w", err)
}
return getTransformerOverridesAndKMSPluginHealthzCheckers(config, stopCh)
}
func getTransformerOverridesAndKMSPluginHealthzCheckers(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, error) {
var kmsHealthChecks []healthz.HealthChecker
transformers, probes, err := getTransformerOverridesAndKMSPluginProbes(config, stopCh)
transformers, kmsHealthChecks, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(config, stopCh)
if err != nil {
return nil, nil, err
return nil, nil, fmt.Errorf("error while building transformers: %w", err)
}
if reload || (kmsUsed.v2Used && !kmsUsed.v1Used) {
kmsHealthChecks = []healthz.HealthChecker{kmsHealthChecker(kmsHealthChecks)}
}
return transformers, kmsHealthChecks, nil
}
func getTransformerOverridesAndKMSPluginHealthzCheckers(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, *kmsState, error) {
var kmsHealthChecks []healthz.HealthChecker
transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(config, stopCh)
if err != nil {
return nil, nil, nil, err
}
for i := range probes {
probe := probes[i]
kmsHealthChecks = append(kmsHealthChecks, probe.toHealthzCheck(i))
}
return transformers, kmsHealthChecks, nil
return transformers, kmsHealthChecks, kmsUsed, nil
}
type healthChecker interface {
toHealthzCheck(idx int) healthz.HealthChecker
}
func getTransformerOverridesAndKMSPluginProbes(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthChecker, error) {
func getTransformerOverridesAndKMSPluginProbes(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthChecker, *kmsState, error) {
resourceToPrefixTransformer := map[schema.GroupResource][]value.PrefixTransformer{}
var probes []healthChecker
var kmsUsed kmsState
// For each entry in the configuration
for _, resourceConfig := range config.Resources {
resourceConfig := resourceConfig
transformers, p, err := prefixTransformersAndProbes(resourceConfig, stopCh)
transformers, p, used, err := prefixTransformersAndProbes(resourceConfig, stopCh)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
kmsUsed.v1Used = kmsUsed.v1Used || used.v1Used
kmsUsed.v2Used = kmsUsed.v2Used || used.v2Used
// For each resource, create a list of providers to use
for _, resource := range resourceConfig.Resources {
@ -152,7 +186,7 @@ func getTransformerOverridesAndKMSPluginProbes(config *apiserverconfig.Encryptio
transformers[gr] = value.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...)
}
return transformers, probes, nil
return transformers, probes, &kmsUsed, nil
}
// check encrypts and decrypts test data against KMS-Plugin's gRPC endpoint.
@ -168,13 +202,13 @@ func (h *kmsPluginProbe) check() error {
if err != nil {
h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
h.ttl = kmsPluginHealthzNegativeTTL
return fmt.Errorf("failed to perform encrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err)
return fmt.Errorf("failed to perform encrypt section of the healthz check for KMS Provider %s, error: %w", h.name, err)
}
if _, err := h.service.Decrypt(p); err != nil {
h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
h.ttl = kmsPluginHealthzNegativeTTL
return fmt.Errorf("failed to perform decrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err)
return fmt.Errorf("failed to perform decrypt section of the healthz check for KMS Provider %s, error: %w", h.name, err)
}
h.lastResponse = &kmsPluginHealthzResponse{err: nil, received: time.Now()}
@ -195,7 +229,7 @@ func (h *kmsv2PluginProbe) check(ctx context.Context) error {
if err != nil {
h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
h.ttl = kmsPluginHealthzNegativeTTL
return fmt.Errorf("failed to perform status section of the healthz check for KMS Provider %s, error: %v", h.name, err)
return fmt.Errorf("failed to perform status section of the healthz check for KMS Provider %s, error: %w", h.name, err)
}
if err := isKMSv2ProviderHealthy(h.name, p); err != nil {
@ -223,7 +257,7 @@ func isKMSv2ProviderHealthy(name string, response *envelopekmsv2.StatusResponse)
}
if err := utilerrors.Reduce(utilerrors.NewAggregate(errs)); err != nil {
return fmt.Errorf("kmsv2 Provider %s is not healthy, error: %v", name, err)
return fmt.Errorf("kmsv2 Provider %s is not healthy, error: %w", name, err)
}
return nil
}
@ -231,13 +265,13 @@ func isKMSv2ProviderHealthy(name string, response *envelopekmsv2.StatusResponse)
func loadConfig(filepath string) (*apiserverconfig.EncryptionConfiguration, error) {
f, err := os.Open(filepath)
if err != nil {
return nil, fmt.Errorf("error opening encryption provider configuration file %q: %v", filepath, err)
return nil, fmt.Errorf("error opening encryption provider configuration file %q: %w", filepath, err)
}
defer f.Close()
data, err := io.ReadAll(f)
if err != nil {
return nil, fmt.Errorf("could not read contents: %v", err)
return nil, fmt.Errorf("could not read contents: %w", err)
}
if len(data) == 0 {
return nil, fmt.Errorf("encryption provider configuration file %q is empty", filepath)
@ -260,9 +294,10 @@ func loadConfig(filepath string) (*apiserverconfig.EncryptionConfiguration, erro
return config, validation.ValidateEncryptionConfiguration(config).ToAggregate()
}
func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, stopCh <-chan struct{}) ([]value.PrefixTransformer, []healthChecker, error) {
func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, stopCh <-chan struct{}) ([]value.PrefixTransformer, []healthChecker, *kmsState, error) {
var transformers []value.PrefixTransformer
var probes []healthChecker
var kmsUsed kmsState
for _, provider := range config.Providers {
provider := provider
@ -270,6 +305,7 @@ func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, s
transformer value.PrefixTransformer
transformerErr error
probe healthChecker
used *kmsState
)
switch {
@ -283,9 +319,11 @@ func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, s
transformer, transformerErr = secretboxPrefixTransformer(provider.Secretbox)
case provider.KMS != nil:
transformer, probe, transformerErr = kmsPrefixTransformer(provider.KMS, stopCh)
transformer, probe, used, transformerErr = kmsPrefixTransformer(provider.KMS, stopCh)
if transformerErr == nil {
probes = append(probes, probe)
kmsUsed.v1Used = kmsUsed.v1Used || used.v1Used
kmsUsed.v2Used = kmsUsed.v2Used || used.v2Used
}
case provider.Identity != nil:
@ -295,17 +333,17 @@ func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, s
}
default:
return nil, nil, errors.New("provider does not contain any of the expected providers: KMS, AESGCM, AESCBC, Secretbox, Identity")
return nil, nil, nil, errors.New("provider does not contain any of the expected providers: KMS, AESGCM, AESCBC, Secretbox, Identity")
}
if transformerErr != nil {
return nil, nil, transformerErr
return nil, nil, nil, transformerErr
}
transformers = append(transformers, transformer)
}
return transformers, probes, nil
return transformers, probes, &kmsUsed, nil
}
type blockTransformerFunc func(cipher.Block) value.Transformer
@ -419,7 +457,11 @@ var (
EnvelopeKMSv2ServiceFactory = envelopekmsv2.NewGRPCService
)
func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-chan struct{}) (value.PrefixTransformer, healthChecker, error) {
type kmsState struct {
v1Used, v2Used bool
}
func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-chan struct{}) (value.PrefixTransformer, healthChecker, *kmsState, error) {
// we ignore the cancel func because this context should only be canceled when stopCh is closed
ctx, _ := wait.ContextForChannel(stopCh)
@ -428,7 +470,7 @@ func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-cha
case kmsAPIVersionV1:
envelopeService, err := envelopeServiceFactory(ctx, config.Endpoint, config.Timeout.Duration)
if err != nil {
return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %v", kmsName, err)
return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %w", kmsName, err)
}
probe := &kmsPluginProbe{
@ -441,16 +483,16 @@ func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-cha
transformer := envelopePrefixTransformer(config, envelopeService, kmsTransformerPrefixV1)
return transformer, probe, nil
return transformer, probe, &kmsState{v1Used: true}, nil
case kmsAPIVersionV2:
if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) {
return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", kmsName)
return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", kmsName)
}
envelopeService, err := EnvelopeKMSv2ServiceFactory(ctx, config.Endpoint, config.Timeout.Duration)
if err != nil {
return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %v", kmsName, err)
return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %w", kmsName, err)
}
probe := &kmsv2PluginProbe{
@ -467,10 +509,10 @@ func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-cha
Prefix: []byte(kmsTransformerPrefixV2 + kmsName + ":"),
}
return transformer, probe, nil
return transformer, probe, &kmsState{v2Used: true}, nil
default:
return value.PrefixTransformer{}, nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", kmsName, config.APIVersion)
return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", kmsName, config.APIVersion)
}
}

29
pkg/server/options/encryptionconfig/config_test.go
View File

@ -177,37 +177,37 @@ func TestEncryptionProviderConfigCorrect(t *testing.T) {
// Transforms data using one of them, and tries to untransform using the others.
// Repeats this for all possible combinations.
correctConfigWithIdentityFirst := "testdata/valid-configs/identity-first.yaml"
identityFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithIdentityFirst, ctx.Done())
identityFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithIdentityFirst, false, ctx.Done())
if err != nil {
t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithIdentityFirst)
}
correctConfigWithAesGcmFirst := "testdata/valid-configs/aes-gcm-first.yaml"
aesGcmFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithAesGcmFirst, ctx.Done())
aesGcmFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithAesGcmFirst, false, ctx.Done())
if err != nil {
t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesGcmFirst)
}
correctConfigWithAesCbcFirst := "testdata/valid-configs/aes-cbc-first.yaml"
aesCbcFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithAesCbcFirst, ctx.Done())
aesCbcFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithAesCbcFirst, false, ctx.Done())
if err != nil {
t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithAesCbcFirst)
}
correctConfigWithSecretboxFirst := "testdata/valid-configs/secret-box-first.yaml"
secretboxFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithSecretboxFirst, ctx.Done())
secretboxFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithSecretboxFirst, false, ctx.Done())
if err != nil {
t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithSecretboxFirst)
}
correctConfigWithKMSFirst := "testdata/valid-configs/kms-first.yaml"
kmsFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithKMSFirst, ctx.Done())
kmsFirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithKMSFirst, false, ctx.Done())
if err != nil {
t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSFirst)
}
correctConfigWithKMSv2First := "testdata/valid-configs/kmsv2-first.yaml"
kmsv2FirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithKMSv2First, ctx.Done())
kmsv2FirstTransformerOverrides, _, err := LoadEncryptionConfig(correctConfigWithKMSv2First, false, ctx.Done())
if err != nil {
t.Fatalf("error while parsing configuration file: %s.\nThe file was:\n%s", err, correctConfigWithKMSv2First)
}
@ -264,6 +264,8 @@ func TestKMSPluginHealthz(t *testing.T) {
config string
want []healthChecker
wantErr string
kmsv2 bool
kmsv1 bool
}{
{
desc: "Install Healthz",
@ -274,6 +276,7 @@ func TestKMSPluginHealthz(t *testing.T) {
ttl: 3 * time.Second,
},
},
kmsv1: true,
},
{
desc: "Install multiple healthz",
@ -288,6 +291,7 @@ func TestKMSPluginHealthz(t *testing.T) {
ttl: 3 * time.Second,
},
},
kmsv1: true,
},
{
desc: "No KMS Providers",
@ -306,6 +310,8 @@ func TestKMSPluginHealthz(t *testing.T) {
ttl: 3 * time.Second,
},
},
kmsv2: true,
kmsv1: true,
},
{
desc: "Invalid API version",
@ -325,7 +331,7 @@ func TestKMSPluginHealthz(t *testing.T) {
return
}
_, got, err := getTransformerOverridesAndKMSPluginProbes(config, testContext(t).Done())
_, got, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(config, testContext(t).Done())
if err != nil {
t.Fatal(err)
}
@ -347,6 +353,13 @@ func TestKMSPluginHealthz(t *testing.T) {
}
}
if tt.kmsv2 != kmsUsed.v2Used {
t.Errorf("incorrect kms v2 detection: want=%v got=%v", tt.kmsv2, kmsUsed.v2Used)
}
if tt.kmsv1 != kmsUsed.v1Used {
t.Errorf("incorrect kms v1 detection: want=%v got=%v", tt.kmsv1, kmsUsed.v1Used)
}
if d := cmp.Diff(tt.want, got,
cmp.Comparer(func(a, b *kmsPluginProbe) bool {
return *a == *b
@ -528,7 +541,7 @@ func getTransformerFromEncryptionConfig(t *testing.T, encryptionConfigPath strin
ctx := testContext(t)
t.Helper()
transformers, _, err := LoadEncryptionConfig(encryptionConfigPath, ctx.Done())
transformers, _, err := LoadEncryptionConfig(encryptionConfigPath, false, ctx.Done())
if err != nil {
t.Fatal(err)
}

11
pkg/server/options/etcd.go
View File

@ -45,6 +45,7 @@ type EtcdOptions struct {
// calculated feature gate value.
StorageConfig storagebackend.Config
EncryptionProviderConfigFilepath string
EncryptionProviderConfigAutomaticReload bool
EtcdServersOverrides []string
@ -117,6 +118,10 @@ func (s *EtcdOptions) Validate() []error {
}
if len(s.EncryptionProviderConfigFilepath) == 0 && s.EncryptionProviderConfigAutomaticReload {
allErrors = append(allErrors, fmt.Errorf("--encryption-provider-config-automatic-reload must be set with --encryption-provider-config"))
}
return allErrors
}
@ -182,6 +187,10 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&s.EncryptionProviderConfigFilepath, "encryption-provider-config", s.EncryptionProviderConfigFilepath,
"The file containing configuration for encryption providers to be used for storing secrets in etcd")
fs.BoolVar(&s.EncryptionProviderConfigAutomaticReload, "encryption-provider-config-automatic-reload", s.EncryptionProviderConfigAutomaticReload,
"Determines if the file set by --encryption-provider-config should be automatically reloaded if the disk contents change. "+
"Setting this to true disables the ability to uniquely identify distinct KMS plugins via the API server healthz endpoints.")
fs.DurationVar(&s.StorageConfig.CompactionInterval, "etcd-compaction-interval", s.StorageConfig.CompactionInterval,
"The interval of compaction requests. If 0, the compaction request from apiserver is disabled.")
@ -214,7 +223,7 @@ func (s *EtcdOptions) Complete(storageObjectCountTracker flowcontrolrequest.Stor
}
if len(s.EncryptionProviderConfigFilepath) != 0 {
transformerOverrides, kmsPluginHealthzChecks, err := encryptionconfig.LoadEncryptionConfig(s.EncryptionProviderConfigFilepath, stopCh)
transformerOverrides, kmsPluginHealthzChecks, err := encryptionconfig.LoadEncryptionConfig(s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload, stopCh)
if err != nil {
return err
}

110
pkg/server/options/etcd_test.go
View File

@ -25,9 +25,13 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/apiserver/pkg/storage/storagebackend"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
)
func TestEtcdOptionsValidate(t *testing.T) {
@ -108,6 +112,31 @@ func TestEtcdOptionsValidate(t *testing.T) {
},
expectErr: "--etcd-servers-overrides invalid, must be of format: group/resource#servers, where servers are URLs, semicolon separated",
},
{
name: "test when encryption-provider-config-automatic-reload is invalid",
testOptions: &EtcdOptions{
StorageConfig: storagebackend.Config{
Type: "etcd3",
Prefix: "/registry",
Transport: storagebackend.TransportConfig{
ServerList: []string{"http://127.0.0.1"},
KeyFile: "/var/run/kubernetes/etcd.key",
TrustedCAFile: "/var/run/kubernetes/etcdca.crt",
CertFile: "/var/run/kubernetes/etcdce.crt",
},
CompactionInterval: storagebackend.DefaultCompactInterval,
CountMetricPollPeriod: time.Minute,
},
EncryptionProviderConfigAutomaticReload: true,
DefaultStorageMediaType: "application/vnd.kubernetes.protobuf",
DeleteCollectionWorkers: 1,
EnableGarbageCollection: true,
EnableWatchCache: true,
DefaultWatchCacheSize: 100,
EtcdServersOverrides: []string{"/events#http://127.0.0.1:4002"},
},
expectErr: "--encryption-provider-config-automatic-reload must be set with --encryption-provider-config",
},
{
name: "test when EtcdOptions is valid",
testOptions: &EtcdOptions{
@ -200,12 +229,26 @@ func TestParseWatchCacheSizes(t *testing.T) {
}
func TestKMSHealthzEndpoint(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv2, true)()
testCases := []struct {
name string
encryptionConfigPath string
wantChecks []string
skipHealth bool
reload bool
}{
{
name: "no kms-provider, expect no kms healthz check",
encryptionConfigPath: "testdata/encryption-configs/no-kms-provider.yaml",
wantChecks: []string{"etcd"},
},
{
name: "no kms-provider+reload, expect single kms healthz check",
encryptionConfigPath: "testdata/encryption-configs/no-kms-provider.yaml",
reload: true,
wantChecks: []string{"etcd", "kms-providers"},
},
{
name: "single kms-provider, expect single kms healthz check",
encryptionConfigPath: "testdata/encryption-configs/single-kms-provider.yaml",
@ -216,6 +259,34 @@ func TestKMSHealthzEndpoint(t *testing.T) {
encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers.yaml",
wantChecks: []string{"etcd", "kms-provider-0", "kms-provider-1"},
},
{
name: "two kms-providers+reload, expect single kms healthz check",
encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers.yaml",
reload: true,
wantChecks: []string{"etcd", "kms-providers"},
},
{
name: "kms v1+v2, expect three kms healthz checks",
encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers-with-v2.yaml",
wantChecks: []string{"etcd", "kms-provider-0", "kms-provider-1", "kms-provider-2"},
},
{
name: "kms v1+v2+reload, expect single kms healthz check",
encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers-with-v2.yaml",
reload: true,
wantChecks: []string{"etcd", "kms-providers"},
},
{
name: "multiple kms v2, expect single kms healthz check",
encryptionConfigPath: "testdata/encryption-configs/multiple-kms-v2-providers.yaml",
wantChecks: []string{"etcd", "kms-providers"},
},
{
name: "multiple kms v2+reload, expect single kms healthz check",
encryptionConfigPath: "testdata/encryption-configs/multiple-kms-v2-providers.yaml",
reload: true,
wantChecks: []string{"etcd", "kms-providers"},
},
{
name: "two kms-providers with skip, expect zero kms healthz checks",
encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers.yaml",
@ -232,6 +303,7 @@ func TestKMSHealthzEndpoint(t *testing.T) {
serverConfig := server.NewConfig(codecs)
etcdOptions := &EtcdOptions{
EncryptionProviderConfigFilepath: tc.encryptionConfigPath,
EncryptionProviderConfigAutomaticReload: tc.reload,
SkipHealthEndpoints: tc.skipHealth,
}
if err := etcdOptions.Complete(serverConfig.StorageObjectCountTracker, serverConfig.DrainedNotify()); err != nil {
@ -241,11 +313,7 @@ func TestKMSHealthzEndpoint(t *testing.T) {
t.Fatalf("Failed to add healthz error: %v", err)
}
for _, n := range tc.wantChecks {
if !hasCheck(n, serverConfig.HealthzChecks) {
t.Errorf("Missing HealthzChecker %s", n)
}
}
healthChecksAreEqual(t, tc.wantChecks, serverConfig.HealthzChecks)
})
}
}
@ -284,25 +352,25 @@ func TestReadinessCheck(t *testing.T) {
t.Fatalf("Failed to add healthz error: %v", err)
}
for _, n := range tc.wantReadyzChecks {
if !hasCheck(n, serverConfig.ReadyzChecks) {
t.Errorf("Missing ReadyzChecker %s", n)
}
}
for _, n := range tc.wantHealthzChecks {
if !hasCheck(n, serverConfig.HealthzChecks) {
t.Errorf("Missing HealthzChecker %s", n)
}
}
healthChecksAreEqual(t, tc.wantReadyzChecks, serverConfig.ReadyzChecks)
healthChecksAreEqual(t, tc.wantHealthzChecks, serverConfig.HealthzChecks)
})
}
}
func hasCheck(want string, healthchecks []healthz.HealthChecker) bool {
for _, h := range healthchecks {
if want == h.Name() {
return true
func healthChecksAreEqual(t *testing.T, want []string, healthChecks []healthz.HealthChecker) {
t.Helper()
wantSet := sets.NewString(want...)
gotSet := sets.NewString()
for _, h := range healthChecks {
gotSet.Insert(h.Name())
}
gotSet.Delete("log", "ping") // not relevant for our tests
if !wantSet.Equal(gotSet) {
t.Errorf("healthz checks are not equal, missing=%q, extra=%q", wantSet.Difference(gotSet).List(), gotSet.Difference(wantSet).List())
}
}
return false
}

18
pkg/server/options/testdata/encryption-configs/multiple-kms-providers-with-v2.yaml vendored Normal file
View File

@ -0,0 +1,18 @@
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
- resources:
- secrets
providers:
- kms:
name: kms-provider-1
cachesize: 1000
endpoint: unix:///@provider1.sock
- kms:
name: kms-provider-2
cachesize: 1000
endpoint: unix:///@provider2.sock
- kms:
apiVersion: v2
name: kms-provider-3
endpoint: unix:///@provider2.sock

20
pkg/server/options/testdata/encryption-configs/multiple-kms-v2-providers.yaml vendored Normal file
View File

@ -0,0 +1,20 @@
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
- resources:
- secrets
providers:
- kms:
apiVersion: v2
name: kms-provider-1
cachesize: 1000
endpoint: unix:///@provider1.sock
- kms:
apiVersion: v2
name: kms-provider-2
cachesize: 1000
endpoint: unix:///@provider2.sock
- kms:
apiVersion: v2
name: kms-provider-3
endpoint: unix:///@provider2.sock

10
pkg/server/options/testdata/encryption-configs/no-kms-provider.yaml vendored Normal file
View File

@ -0,0 +1,10 @@
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
- resources:
- secrets
providers:
- aesgcm:
keys:
- name: key1
secret: c2VjcmV0IGlzIHNlY3VyZQ==