kubernetes
/
apiserver
mirror of https://github.com/kubernetes/apiserver.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #116345 from aramase/aramase/f/kms_cache_key 

[KMSv2] use encDEK, keyID and annotations to generate cache key

Kubernetes-commit: 2467eb8a7b0e988f897d6eee478636d6ff6d5d3f
This commit is contained in:
Kubernetes Publisher 2023-03-14 17:44:25 -07:00
parent bab2edbb73 b21cb57710
commit 121f10f1bd
7 changed files with 257 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
go.mod
View File

@ -42,9 +42,9 @@ require (
google.golang.org/protobuf v1.28.1
gopkg.in/natefinch/lumberjack.v2 v2.0.0
gopkg.in/square/go-jose.v2 v2.6.0
k8s.io/api v0.0.0-20230314091508-112a65bae227
k8s.io/apimachinery v0.0.0-20230314010357-128166500c57
k8s.io/client-go v0.0.0-20230315061817-2a7ba9488095
k8s.io/api v0.0.0-20230315055830-f836919cf7fd
k8s.io/apimachinery v0.0.0-20230315054726-2da2e3c5ec8c
k8s.io/client-go v0.0.0-20230315061838-ef07195878d5
k8s.io/component-base v0.0.0-20230313212246-ce16dede9c0e
k8s.io/klog/v2 v2.90.1
k8s.io/kms v0.0.0-20230313212457-12714b59d299
@ -124,9 +124,9 @@ require (
)
replace (
k8s.io/api => k8s.io/api v0.0.0-20230314091508-112a65bae227
k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20230314010357-128166500c57
k8s.io/client-go => k8s.io/client-go v0.0.0-20230315061817-2a7ba9488095
k8s.io/api => k8s.io/api v0.0.0-20230315055830-f836919cf7fd
k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20230315054726-2da2e3c5ec8c
k8s.io/client-go => k8s.io/client-go v0.0.0-20230315061838-ef07195878d5
k8s.io/component-base => k8s.io/component-base v0.0.0-20230313212246-ce16dede9c0e
k8s.io/kms => k8s.io/kms v0.0.0-20230313212457-12714b59d299
)

12
go.sum
View File

@ -874,12 +874,12 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
k8s.io/api v0.0.0-20230314091508-112a65bae227 h1:Ak4YrHI4101ZMfx2hkXnp//d5r0nKXA8RkNaHCBJ7MA=
k8s.io/api v0.0.0-20230314091508-112a65bae227/go.mod h1:YsFNxBfPYQZAIBg0XvL94rxDDVZvQQQUlo4KbwEEqNU=
k8s.io/apimachinery v0.0.0-20230314010357-128166500c57 h1:Vr1geeI+at1NNCWyTN70NtPSNcveZ+fAcbZivzwHknM=
k8s.io/apimachinery v0.0.0-20230314010357-128166500c57/go.mod h1:1AlvkfXatlv5Kq9dCZg3Ksdu/DyrZ31Q0CncRqQ8Q9I=
k8s.io/client-go v0.0.0-20230315061817-2a7ba9488095 h1:9yBBDqoFcdUPmjREBn9pblz+iOshotP0PgYO3oFnCTg=
k8s.io/client-go v0.0.0-20230315061817-2a7ba9488095/go.mod h1:oWqDJxiXYcSV7S8woQ4H5epopeK85SJyOu5lJsMndcU=
k8s.io/api v0.0.0-20230315055830-f836919cf7fd h1:+dH602TARUCflBOu27VPjQUAWQRipSXs71MGuXEaXoE=
k8s.io/api v0.0.0-20230315055830-f836919cf7fd/go.mod h1:0+pk96f3KZ0I1oIK8l1lCI9gWzpKaAQdU7//+21ulhU=
k8s.io/apimachinery v0.0.0-20230315054726-2da2e3c5ec8c h1:rFeoHo0jyaOib25IhqMPs5LSWtwU3LVmwhES7t4RzS0=
k8s.io/apimachinery v0.0.0-20230315054726-2da2e3c5ec8c/go.mod h1:1AlvkfXatlv5Kq9dCZg3Ksdu/DyrZ31Q0CncRqQ8Q9I=
k8s.io/client-go v0.0.0-20230315061838-ef07195878d5 h1:r1VQdDr96Jc/CJmwSSgwjdViB2rnwz7cnaFr0mqIEn4=
k8s.io/client-go v0.0.0-20230315061838-ef07195878d5/go.mod h1:hTovlUcXHrjXfBPynmQWc9IYuZmAU7m1sWRkEs2N1gQ=
k8s.io/component-base v0.0.0-20230313212246-ce16dede9c0e h1:/mftAl/78q8dPZU8YkshNzt1XpbEJTGsxYZP/WGI4og=
k8s.io/component-base v0.0.0-20230313212246-ce16dede9c0e/go.mod h1:MQKfc/tXtS50042g1VxMb2W2E8PCt97xO8RsLTw3AeI=
k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw=

7
pkg/server/options/encryptionconfig/config.go
View File

@ -358,7 +358,7 @@ func (h *kmsv2PluginProbe) rotateDEKOnKeyIDChange(ctx context.Context, statusKey
return nil
}
transformer, resp, errGen := envelopekmsv2.GenerateTransformer(ctx, uid, h.service)
transformer, resp, cacheKey, errGen := envelopekmsv2.GenerateTransformer(ctx, uid, h.service)
if resp == nil {
resp = &kmsservice.EncryptResponse{} // avoid nil panics
@ -374,6 +374,7 @@ func (h *kmsv2PluginProbe) rotateDEKOnKeyIDChange(ctx context.Context, statusKey
Annotations: resp.Annotations,
UID: uid,
ExpirationTimestamp: expirationTimestamp,
CacheKey: cacheKey,
})
klog.V(6).InfoS("successfully rotated DEK",
"uid", uid,
@ -410,6 +411,10 @@ func (h *kmsv2PluginProbe) getCurrentState() (envelopekmsv2.State, error) {
return envelopekmsv2.State{}, fmt.Errorf("got unexpected zero expirationTimestamp")
}
if len(state.CacheKey) == 0 {
return envelopekmsv2.State{}, fmt.Errorf("got unexpected empty cacheKey")
}
return state, nil
}

3
pkg/server/options/encryptionconfig/config_test.go
View File

@ -1781,7 +1781,7 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
t.Errorf("log mismatch (-want +got):\n%s", diff)
}
ignoredFields := sets.NewString("Transformer", "EncryptedDEK", "UID")
ignoredFields := sets.NewString("Transformer", "EncryptedDEK", "UID", "CacheKey")
if diff := cmp.Diff(tt.wantState, *h.state.Load(),
cmp.FilterPath(func(path cmp.Path) bool { return ignoredFields.Has(path.String()) }, cmp.Ignore()),
@ -1806,6 +1806,7 @@ func validState(keyID string, exp time.Time) envelopekmsv2.State {
EncryptedDEK: []byte{1},
KeyID: keyID,
ExpirationTimestamp: exp,
CacheKey: []byte{1},
}
}

8
pkg/storage/value/encrypt/envelope/kmsv2/cache.go
View File

@ -98,5 +98,11 @@ func (c *simpleCache) keyFunc(s []byte) string {
// toString performs unholy acts to avoid allocations
func toString(b []byte) string {
return *(*string)(unsafe.Pointer(&b))
// unsafe.SliceData relies on cap whereas we want to rely on len
if len(b) == 0 {
return ""
}
// Copied from go 1.20.1 strings.Builder.String
// https://github.com/golang/go/blob/202a1a57064127c3f19d96df57b9f9586145e21c/src/strings/builder.go#L48
return unsafe.String(unsafe.SliceData(b), len(b))
}

90
pkg/storage/value/encrypt/envelope/kmsv2/envelope.go
View File

@ -21,9 +21,12 @@ import (
"context"
"crypto/aes"
"fmt"
"sort"
"time"
"unsafe"
"github.com/gogo/protobuf/proto"
"golang.org/x/crypto/cryptobyte"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/uuid"
@ -87,6 +90,9 @@ type State struct {
UID string
ExpirationTimestamp time.Time
// CacheKey is the key used to cache the DEK in transformer.cache.
CacheKey []byte
}
func (s *State) ValidateEncryptCapability() error {
@ -137,8 +143,13 @@ func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []b
return nil, false, err
}
encryptedObjectCacheKey, err := generateCacheKey(encryptedObject.EncryptedDEK, encryptedObject.KeyID, encryptedObject.Annotations)
if err != nil {
return nil, false, err
}
// Look up the decrypted DEK from cache first
transformer := t.cache.get(encryptedObject.EncryptedDEK)
transformer := t.cache.get(encryptedObjectCacheKey)
// fallback to the envelope service if we do not have the transformer locally
if transformer == nil {
@ -159,7 +170,7 @@ func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []b
return nil, false, fmt.Errorf("failed to decrypt DEK, error: %w", err)
}
transformer, err = t.addTransformerForDecryption(encryptedObject.EncryptedDEK, key)
transformer, err = t.addTransformerForDecryption(encryptedObjectCacheKey, key)
if err != nil {
return nil, false, err
}
@ -190,7 +201,7 @@ func (t *envelopeTransformer) TransformToStorage(ctx context.Context, data []byt
// this has the side benefit of causing the cache to perform a GC
// TODO see if we can do this inside the stateFunc control loop
// TODO(aramase): Add metrics for cache fill percentage with custom cache implementation.
t.cache.set(state.EncryptedDEK, state.Transformer)
t.cache.set(state.CacheKey, state.Transformer)
requestInfo := getRequestInfoFromContext(ctx)
klog.V(6).InfoS("encrypting content using DEK", "uid", state.UID, "key", string(dataCtx.AuthenticatedData()),
@ -216,7 +227,7 @@ func (t *envelopeTransformer) TransformToStorage(ctx context.Context, data []byt
}
// addTransformerForDecryption inserts a new transformer to the Envelope cache of DEKs for future reads.
func (t *envelopeTransformer) addTransformerForDecryption(encKey []byte, key []byte) (decryptTransformer, error) {
func (t *envelopeTransformer) addTransformerForDecryption(cacheKey []byte, key []byte) (decryptTransformer, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
@ -228,7 +239,7 @@ func (t *envelopeTransformer) addTransformerForDecryption(encKey []byte, key []b
return nil, err
}
// TODO(aramase): Add metrics for cache fill percentage with custom cache implementation.
t.cache.set(encKey, transformer)
t.cache.set(cacheKey, transformer)
return transformer, nil
}
@ -254,20 +265,25 @@ func (t *envelopeTransformer) doDecode(originalData []byte) (*kmstypes.Encrypted
return o, nil
}
func GenerateTransformer(ctx context.Context, uid string, envelopeService kmsservice.Service) (value.Transformer, *kmsservice.EncryptResponse, error) {
func GenerateTransformer(ctx context.Context, uid string, envelopeService kmsservice.Service) (value.Transformer, *kmsservice.EncryptResponse, []byte, error) {
transformer, newKey, err := aestransformer.NewGCMTransformerWithUniqueKeyUnsafe()
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
klog.V(6).InfoS("encrypting content using envelope service", "uid", uid)
resp, err := envelopeService.Encrypt(ctx, uid, newKey)
if err != nil {
return nil, nil, fmt.Errorf("failed to encrypt DEK, error: %w", err)
return nil, nil, nil, fmt.Errorf("failed to encrypt DEK, error: %w", err)
}
return transformer, resp, nil
cacheKey, err := generateCacheKey(resp.Ciphertext, resp.KeyID, resp.Annotations)
if err != nil {
return nil, nil, nil, err
}
return transformer, resp, cacheKey, nil
}
func validateEncryptedObject(o *kmstypes.EncryptedObject) error {
@ -339,3 +355,59 @@ func getRequestInfoFromContext(ctx context.Context) *genericapirequest.RequestIn
}
return &genericapirequest.RequestInfo{}
}
// generateCacheKey returns a key for the cache.
// The key is a concatenation of:
// 1. encryptedDEK
// 2. keyID
// 3. length of annotations
// 4. annotations (sorted by key) - each annotation is a concatenation of:
// a. annotation key
// b. annotation value
func generateCacheKey(encryptedDEK []byte, keyID string, annotations map[string][]byte) ([]byte, error) {
// TODO(aramase): use sync pool buffer to avoid allocations
b := cryptobyte.NewBuilder(nil)
b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
b.AddBytes(encryptedDEK)
})
b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
b.AddBytes(toBytes(keyID))
})
if len(annotations) == 0 {
return b.Bytes()
}
// add the length of annotations to the cache key
b.AddUint32(uint32(len(annotations)))
// Sort the annotations by key.
keys := make([]string, 0, len(annotations))
for k := range annotations {
k := k
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
// The maximum size of annotations is annotationsMaxSize (32 kB) so we can safely
// assume that the length of the key and value will fit in a uint16.
b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
b.AddBytes(toBytes(k))
})
b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
b.AddBytes(annotations[k])
})
}
return b.Bytes()
}
// toBytes performs unholy acts to avoid allocations
func toBytes(s string) []byte {
// unsafe.StringData is unspecified for the empty string, so we provide a strict interpretation
if len(s) == 0 {
return nil
}
// Copied from go 1.20.1 os.File.WriteString
// https://github.com/golang/go/blob/202a1a57064127c3f19d96df57b9f9586145e21c/src/os/file.go#L246
return unsafe.Slice(unsafe.StringData(s), len(s))
}

169
pkg/storage/value/encrypt/envelope/kmsv2/envelope_test.go
View File

@ -55,12 +55,15 @@ const (
// testEnvelopeService is a mock Envelope service which can be used to simulate remote Envelope services
// for testing of Envelope based encryption providers.
type testEnvelopeService struct {
annotations map[string][]byte
disabled bool
keyVersion string
annotations map[string][]byte
disabled bool
keyVersion string
ciphertext []byte
decryptCalls int
}
func (t *testEnvelopeService) Decrypt(ctx context.Context, uid string, req *kmsservice.DecryptRequest) ([]byte, error) {
t.decryptCalls++
if t.disabled {
return nil, fmt.Errorf("Envelope service was disabled")
}
@ -88,7 +91,13 @@ func (t *testEnvelopeService) Encrypt(ctx context.Context, uid string, data []by
} else {
annotations["local-kek.kms.kubernetes.io"] = []byte("encrypted-local-kek")
}
return &kmsservice.EncryptResponse{Ciphertext: []byte(base64.StdEncoding.EncodeToString(data)), KeyID: t.keyVersion, Annotations: annotations}, nil
ciphertext := t.ciphertext
if ciphertext == nil {
ciphertext = []byte(base64.StdEncoding.EncodeToString(data))
}
return &kmsservice.EncryptResponse{Ciphertext: ciphertext, KeyID: t.keyVersion, Annotations: annotations}, nil
}
func (t *testEnvelopeService) Status(ctx context.Context) (*kmsservice.StatusResponse, error) {
@ -106,6 +115,10 @@ func (t *testEnvelopeService) SetAnnotations(annotations map[string][]byte) {
t.annotations = annotations
}
func (t *testEnvelopeService) SetCiphertext(ciphertext []byte) {
t.ciphertext = ciphertext
}
func (t *testEnvelopeService) Rotate() {
i, _ := strconv.Atoi(t.keyVersion)
t.keyVersion = strconv.FormatInt(int64(i+1), 10)
@ -124,17 +137,26 @@ func TestEnvelopeCaching(t *testing.T) {
cacheTTL time.Duration
simulateKMSPluginFailure bool
expectedError string
expectedDecryptCalls int
}{
{
desc: "entry in cache should withstand plugin failure",
cacheTTL: 5 * time.Minute,
simulateKMSPluginFailure: true,
expectedDecryptCalls: 0, // should not hit KMS plugin
},
{
desc: "cache entry expired should not withstand plugin failure",
cacheTTL: 1 * time.Millisecond,
simulateKMSPluginFailure: true,
expectedError: "failed to decrypt DEK, error: Envelope service was disabled",
expectedDecryptCalls: 10, // should hit KMS plugin for each read after cache entry expired and fail
},
{
desc: "cache entry expired should work after cache refresh",
cacheTTL: 1 * time.Millisecond,
simulateKMSPluginFailure: false,
expectedDecryptCalls: 1, // should hit KMS plugin just for the 1st read after cache entry expired
},
}
@ -176,30 +198,35 @@ func TestEnvelopeCaching(t *testing.T) {
}
envelopeService.SetDisabledStatus(tt.simulateKMSPluginFailure)
// Subsequent read for the same data should work fine due to caching.
untransformedData, _, err = transformer.TransformFromStorage(ctx, transformedData, dataCtx)
if tt.expectedError != "" {
if err == nil {
t.Fatalf("expected error: %v, got nil", tt.expectedError)
}
if err.Error() != tt.expectedError {
t.Fatalf("expected error: %v, got: %v", tt.expectedError, err)
}
} else {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !bytes.Equal(untransformedData, originalText) {
t.Fatalf("envelopeTransformer transformed data incorrectly. Expected: %v, got %v", originalText, untransformedData)
for i := 0; i < 10; i++ {
// Subsequent reads for the same data should work fine due to caching.
untransformedData, _, err = transformer.TransformFromStorage(ctx, transformedData, dataCtx)
if tt.expectedError != "" {
if err == nil {
t.Fatalf("expected error: %v, got nil", tt.expectedError)
}
if err.Error() != tt.expectedError {
t.Fatalf("expected error: %v, got: %v", tt.expectedError, err)
}
} else {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !bytes.Equal(untransformedData, originalText) {
t.Fatalf("envelopeTransformer transformed data incorrectly. Expected: %v, got %v", originalText, untransformedData)
}
}
}
if envelopeService.decryptCalls != tt.expectedDecryptCalls {
t.Fatalf("expected %d decrypt calls, got %d", tt.expectedDecryptCalls, envelopeService.decryptCalls)
}
})
}
}
func testStateFunc(ctx context.Context, envelopeService kmsservice.Service, clock clock.Clock) func() (State, error) {
return func() (State, error) {
transformer, resp, errGen := GenerateTransformer(ctx, string(uuid.NewUUID()), envelopeService)
transformer, resp, cacheKey, errGen := GenerateTransformer(ctx, string(uuid.NewUUID()), envelopeService)
if errGen != nil {
return State{}, errGen
}
@ -210,6 +237,7 @@ func testStateFunc(ctx context.Context, envelopeService kmsservice.Service, cloc
Annotations: resp.Annotations,
UID: "panda",
ExpirationTimestamp: clock.Now().Add(time.Hour),
CacheKey: cacheKey,
}, nil
}
}
@ -861,6 +889,107 @@ func TestEnvelopeLogging(t *testing.T) {
}
}
func TestCacheNotCorrupted(t *testing.T) {
ctx := testContext(t)
envelopeService := newTestEnvelopeService()
envelopeService.SetAnnotations(map[string][]byte{
"encrypted-dek.kms.kubernetes.io": []byte("encrypted-dek-0"),
})
fakeClock := testingclock.NewFakeClock(time.Now())
state, err := testStateFunc(ctx, envelopeService, fakeClock)()
if err != nil {
t.Fatal(err)
}
transformer := newEnvelopeTransformerWithClock(envelopeService, testProviderName,
func() (State, error) { return state, nil },
1*time.Second, fakeClock)
dataCtx := value.DefaultContext(testContextText)
originalText := []byte(testText)
transformedData1, err := transformer.TransformToStorage(ctx, originalText, dataCtx)
if err != nil {
t.Fatalf("envelopeTransformer: error while transforming data to storage: %s", err)
}
// this is to mimic a plugin that sets a static response for ciphertext
// but uses the annotation field to send the actual encrypted DEK.
envelopeService.SetCiphertext(state.EncryptedDEK)
// for this plugin, it indicates a change in the remote key ID as the returned
// encrypted DEK is different.
envelopeService.SetAnnotations(map[string][]byte{
"encrypted-dek.kms.kubernetes.io": []byte("encrypted-dek-1"),
})
state, err = testStateFunc(ctx, envelopeService, fakeClock)()
if err != nil {
t.Fatal(err)
}
transformer = newEnvelopeTransformerWithClock(envelopeService, testProviderName,
func() (State, error) { return state, nil },
1*time.Second, fakeClock)
transformedData2, err := transformer.TransformToStorage(ctx, originalText, dataCtx)
if err != nil {
t.Fatalf("envelopeTransformer: error while transforming data to storage: %s", err)
}
if _, _, err := transformer.TransformFromStorage(ctx, transformedData1, dataCtx); err != nil {
t.Fatal(err)
}
if _, _, err := transformer.TransformFromStorage(ctx, transformedData2, dataCtx); err != nil {
t.Fatal(err)
}
}
func TestGenerateCacheKey(t *testing.T) {
encryptedDEK1 := []byte{1, 2, 3}
keyID1 := "id1"
annotations1 := map[string][]byte{"a": {4, 5}, "b": {6, 7}}
encryptedDEK2 := []byte{4, 5, 6}
keyID2 := "id2"
annotations2 := map[string][]byte{"x": {9, 10}, "y": {11, 12}}
// generate all possible combinations of the above
testCases := []struct {
encryptedDEK []byte
keyID string
annotations map[string][]byte
}{
{encryptedDEK1, keyID1, annotations1},
{encryptedDEK1, keyID1, annotations2},
{encryptedDEK1, keyID2, annotations1},
{encryptedDEK1, keyID2, annotations2},
{encryptedDEK2, keyID1, annotations1},
{encryptedDEK2, keyID1, annotations2},
{encryptedDEK2, keyID2, annotations1},
{encryptedDEK2, keyID2, annotations2},
}
for _, tc := range testCases {
tc := tc
for _, tc2 := range testCases {
tc2 := tc2
t.Run(fmt.Sprintf("%+v-%+v", tc, tc2), func(t *testing.T) {
key1, err1 := generateCacheKey(tc.encryptedDEK, tc.keyID, tc.annotations)
key2, err2 := generateCacheKey(tc2.encryptedDEK, tc2.keyID, tc2.annotations)
if err1 != nil || err2 != nil {
t.Errorf("generateCacheKey() want err=nil, got err1=%q, err2=%q", errString(err1), errString(err2))
}
if bytes.Equal(key1, key2) != reflect.DeepEqual(tc, tc2) {
t.Errorf("expected %v, got %v", reflect.DeepEqual(tc, tc2), bytes.Equal(key1, key2))
}
})
}
}
}
func errString(err error) string {
if err == nil {
return ""