397 lines
15 KiB
Go
397 lines
15 KiB
Go
/*
|
|
Copyright 2022 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package controller
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"k8s.io/apiserver/pkg/features"
|
|
"k8s.io/apiserver/pkg/server/healthz"
|
|
"k8s.io/apiserver/pkg/server/options/encryptionconfig"
|
|
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
|
"k8s.io/client-go/util/workqueue"
|
|
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
|
"k8s.io/component-base/metrics/legacyregistry"
|
|
"k8s.io/component-base/metrics/testutil"
|
|
)
|
|
|
|
func TestController(t *testing.T) {
|
|
origMinKMSPluginCloseGracePeriod := minKMSPluginCloseGracePeriod
|
|
t.Cleanup(func() { minKMSPluginCloseGracePeriod = origMinKMSPluginCloseGracePeriod })
|
|
minKMSPluginCloseGracePeriod = 300 * time.Millisecond
|
|
|
|
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KMSv1, true)
|
|
|
|
const expectedSuccessMetricValue = `
|
|
# HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity.
|
|
# TYPE apiserver_encryption_config_controller_automatic_reloads_total counter
|
|
apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:cd8a60cec6134082e9f37e7a4146b4bc14a0bf8a863237c36ec8fdb658c3e027",status="success"} 1
|
|
`
|
|
const expectedFailureMetricValue = `
|
|
# HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity.
|
|
# TYPE apiserver_encryption_config_controller_automatic_reloads_total counter
|
|
apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:cd8a60cec6134082e9f37e7a4146b4bc14a0bf8a863237c36ec8fdb658c3e027",status="failure"} 1
|
|
`
|
|
|
|
tests := []struct {
|
|
name string
|
|
wantECFileHash string
|
|
wantTransformerClosed bool
|
|
wantLoadCalls int
|
|
wantHashCalls int
|
|
wantAddRateLimitedCount uint64
|
|
wantMetrics string
|
|
mockLoadEncryptionConfig func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error)
|
|
mockGetEncryptionConfigHash func(ctx context.Context, filepath string) (string, error)
|
|
}{
|
|
{
|
|
name: "when invalid config is provided previous config shouldn't be changed",
|
|
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
|
wantLoadCalls: 1,
|
|
wantHashCalls: 1,
|
|
wantTransformerClosed: true,
|
|
wantMetrics: expectedFailureMetricValue,
|
|
wantAddRateLimitedCount: 1,
|
|
mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
|
|
return "always changes and never errors", nil
|
|
},
|
|
mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
|
|
return nil, fmt.Errorf("empty config file")
|
|
},
|
|
},
|
|
{
|
|
name: "when new valid config is provided it should be updated",
|
|
wantECFileHash: "some new config hash",
|
|
wantLoadCalls: 1,
|
|
wantHashCalls: 1,
|
|
wantMetrics: expectedSuccessMetricValue,
|
|
wantAddRateLimitedCount: 0,
|
|
mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
|
|
return "always changes and never errors", nil
|
|
},
|
|
mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
|
|
return &encryptionconfig.EncryptionConfiguration{
|
|
HealthChecks: []healthz.HealthChecker{
|
|
&mockHealthChecker{
|
|
pluginName: "valid-plugin",
|
|
err: nil,
|
|
},
|
|
},
|
|
EncryptionFileContentHash: "some new config hash",
|
|
}, nil
|
|
},
|
|
},
|
|
{
|
|
name: "when same valid config is provided previous config shouldn't be changed",
|
|
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
|
wantLoadCalls: 1,
|
|
wantHashCalls: 1,
|
|
wantTransformerClosed: true,
|
|
wantMetrics: "",
|
|
wantAddRateLimitedCount: 0,
|
|
mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
|
|
return "always changes and never errors", nil
|
|
},
|
|
mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
|
|
return &encryptionconfig.EncryptionConfiguration{
|
|
HealthChecks: []healthz.HealthChecker{
|
|
&mockHealthChecker{
|
|
pluginName: "valid-plugin",
|
|
err: nil,
|
|
},
|
|
},
|
|
// hash of initial "testdata/ec_config.yaml" config file before reloading
|
|
EncryptionFileContentHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
|
}, nil
|
|
},
|
|
},
|
|
{
|
|
name: "when transformer's health check fails previous config shouldn't be changed",
|
|
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
|
wantLoadCalls: 1,
|
|
wantHashCalls: 1,
|
|
wantTransformerClosed: true,
|
|
wantMetrics: expectedFailureMetricValue,
|
|
wantAddRateLimitedCount: 1,
|
|
mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
|
|
return "always changes and never errors", nil
|
|
},
|
|
mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
|
|
return &encryptionconfig.EncryptionConfiguration{
|
|
HealthChecks: []healthz.HealthChecker{
|
|
&mockHealthChecker{
|
|
pluginName: "invalid-plugin",
|
|
err: fmt.Errorf("mockingly failing"),
|
|
},
|
|
},
|
|
KMSCloseGracePeriod: 0, // use minKMSPluginCloseGracePeriod
|
|
EncryptionFileContentHash: "anything different",
|
|
}, nil
|
|
},
|
|
},
|
|
{
|
|
name: "when multiple health checks are present previous config shouldn't be changed",
|
|
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
|
wantLoadCalls: 1,
|
|
wantHashCalls: 1,
|
|
wantTransformerClosed: true,
|
|
wantMetrics: expectedFailureMetricValue,
|
|
wantAddRateLimitedCount: 1,
|
|
mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
|
|
return "always changes and never errors", nil
|
|
},
|
|
mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
|
|
return &encryptionconfig.EncryptionConfiguration{
|
|
HealthChecks: []healthz.HealthChecker{
|
|
&mockHealthChecker{
|
|
pluginName: "valid-plugin",
|
|
err: nil,
|
|
},
|
|
&mockHealthChecker{
|
|
pluginName: "another-valid-plugin",
|
|
err: nil,
|
|
},
|
|
},
|
|
EncryptionFileContentHash: "anything different",
|
|
}, nil
|
|
},
|
|
},
|
|
{
|
|
name: "when invalid health check URL is provided previous config shouldn't be changed",
|
|
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
|
wantLoadCalls: 1,
|
|
wantHashCalls: 1,
|
|
wantTransformerClosed: true,
|
|
wantMetrics: expectedFailureMetricValue,
|
|
wantAddRateLimitedCount: 1,
|
|
mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
|
|
return "always changes and never errors", nil
|
|
},
|
|
mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
|
|
return &encryptionconfig.EncryptionConfiguration{
|
|
HealthChecks: []healthz.HealthChecker{
|
|
&mockHealthChecker{
|
|
pluginName: "invalid\nname",
|
|
err: nil,
|
|
},
|
|
},
|
|
EncryptionFileContentHash: "anything different",
|
|
}, nil
|
|
},
|
|
},
|
|
{
|
|
name: "when config is not updated transformers are closed correctly",
|
|
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
|
wantLoadCalls: 1,
|
|
wantHashCalls: 1,
|
|
wantTransformerClosed: true,
|
|
wantMetrics: "",
|
|
wantAddRateLimitedCount: 0,
|
|
mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
|
|
return "always changes and never errors", nil
|
|
},
|
|
mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
|
|
return &encryptionconfig.EncryptionConfiguration{
|
|
HealthChecks: []healthz.HealthChecker{
|
|
&mockHealthChecker{
|
|
pluginName: "valid-plugin",
|
|
err: nil,
|
|
},
|
|
},
|
|
// hash of initial "testdata/ec_config.yaml" config file before reloading
|
|
EncryptionFileContentHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
|
}, nil
|
|
},
|
|
},
|
|
{
|
|
name: "when config hash is not updated transformers are closed correctly",
|
|
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
|
wantLoadCalls: 0,
|
|
wantHashCalls: 1,
|
|
wantTransformerClosed: true,
|
|
wantMetrics: "",
|
|
wantAddRateLimitedCount: 0,
|
|
mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
|
|
// hash of initial "testdata/ec_config.yaml" config file before reloading
|
|
return "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", nil
|
|
},
|
|
mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
|
|
return nil, fmt.Errorf("should not be called")
|
|
},
|
|
},
|
|
{
|
|
name: "when config hash errors transformers are closed correctly",
|
|
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
|
|
wantLoadCalls: 0,
|
|
wantHashCalls: 1,
|
|
wantTransformerClosed: true,
|
|
wantMetrics: expectedFailureMetricValue,
|
|
wantAddRateLimitedCount: 1,
|
|
mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
|
|
return "", fmt.Errorf("some io error")
|
|
},
|
|
mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
|
|
return nil, fmt.Errorf("should not be called")
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
ctxServer, closeServer := context.WithCancel(context.Background())
|
|
ctxTransformers, closeTransformers := context.WithCancel(ctxServer)
|
|
t.Cleanup(closeServer)
|
|
t.Cleanup(closeTransformers)
|
|
|
|
legacyregistry.Reset()
|
|
|
|
// load initial encryption config
|
|
encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(
|
|
ctxTransformers,
|
|
"testdata/ec_config.yaml",
|
|
true,
|
|
"test-apiserver",
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("failed to load encryption config: %v", err)
|
|
}
|
|
|
|
d := NewDynamicEncryptionConfiguration(
|
|
"test-controller",
|
|
"does not matter",
|
|
encryptionconfig.NewDynamicTransformers(
|
|
encryptionConfiguration.Transformers,
|
|
encryptionConfiguration.HealthChecks[0],
|
|
closeTransformers,
|
|
0, // set grace period to 0 so that the time.Sleep in DynamicTransformers.Set finishes quickly
|
|
),
|
|
encryptionConfiguration.EncryptionFileContentHash,
|
|
"test-apiserver",
|
|
)
|
|
d.queue.ShutDown() // we do not use the real queue during tests
|
|
|
|
queue := &mockWorkQueue{
|
|
addCalled: make(chan struct{}),
|
|
cancel: closeServer,
|
|
}
|
|
d.queue = queue
|
|
|
|
var hashCalls, loadCalls int
|
|
d.loadEncryptionConfig = func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
|
|
loadCalls++
|
|
queue.ctx = ctx
|
|
return test.mockLoadEncryptionConfig(ctx, filepath, reload, apiServerID)
|
|
}
|
|
d.getEncryptionConfigHash = func(ctx context.Context, filepath string) (string, error) {
|
|
hashCalls++
|
|
queue.ctx = ctx
|
|
return test.mockGetEncryptionConfigHash(ctx, filepath)
|
|
}
|
|
|
|
d.Run(ctxServer) // this should block and run exactly one iteration of the worker loop
|
|
|
|
if test.wantECFileHash != d.lastLoadedEncryptionConfigHash {
|
|
t.Errorf("expected encryption config hash %q but got %q", test.wantECFileHash, d.lastLoadedEncryptionConfigHash)
|
|
}
|
|
|
|
if test.wantLoadCalls != loadCalls {
|
|
t.Errorf("load calls does not match: want=%v, got=%v", test.wantLoadCalls, loadCalls)
|
|
}
|
|
|
|
if test.wantHashCalls != hashCalls {
|
|
t.Errorf("hash calls does not match: want=%v, got=%v", test.wantHashCalls, hashCalls)
|
|
}
|
|
|
|
if test.wantTransformerClosed != queue.wasCanceled {
|
|
t.Errorf("transformer closed does not match: want=%v, got=%v", test.wantTransformerClosed, queue.wasCanceled)
|
|
}
|
|
|
|
if test.wantAddRateLimitedCount != queue.addRateLimitedCount.Load() {
|
|
t.Errorf("queue addRateLimitedCount does not match: want=%v, got=%v", test.wantAddRateLimitedCount, queue.addRateLimitedCount.Load())
|
|
}
|
|
|
|
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(test.wantMetrics),
|
|
"apiserver_encryption_config_controller_automatic_reloads_total",
|
|
); err != nil {
|
|
t.Errorf("failed to validate metrics: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
type mockWorkQueue struct {
|
|
workqueue.TypedRateLimitingInterface[string] // will panic if any unexpected method is called
|
|
|
|
closeOnce sync.Once
|
|
addCalled chan struct{}
|
|
|
|
count atomic.Uint64
|
|
ctx context.Context
|
|
wasCanceled bool
|
|
cancel func()
|
|
|
|
addRateLimitedCount atomic.Uint64
|
|
}
|
|
|
|
func (m *mockWorkQueue) Done(item string) {
|
|
m.count.Add(1)
|
|
m.wasCanceled = m.ctx.Err() != nil
|
|
m.cancel()
|
|
}
|
|
|
|
func (m *mockWorkQueue) Get() (item string, shutdown bool) {
|
|
<-m.addCalled
|
|
|
|
switch m.count.Load() {
|
|
case 0:
|
|
return "", false
|
|
case 1:
|
|
return "", true
|
|
default:
|
|
panic("too many calls to Get")
|
|
}
|
|
}
|
|
|
|
func (m *mockWorkQueue) Add(item string) {
|
|
m.closeOnce.Do(func() {
|
|
close(m.addCalled)
|
|
})
|
|
}
|
|
|
|
func (m *mockWorkQueue) ShutDown() {}
|
|
func (m *mockWorkQueue) AddRateLimited(item string) { m.addRateLimitedCount.Add(1) }
|
|
|
|
type mockHealthChecker struct {
|
|
pluginName string
|
|
err error
|
|
}
|
|
|
|
func (m *mockHealthChecker) Check(req *http.Request) error {
|
|
return m.err
|
|
}
|
|
|
|
func (m *mockHealthChecker) Name() string {
|
|
return m.pluginName
|
|
}
|