Merge pull request #116345 from aramase/aramase/f/kms_cache_key

[KMSv2] use encDEK, keyID and annotations to generate cache key

Kubernetes-commit: 2467eb8a7b0e988f897d6eee478636d6ff6d5d3f

go.mod

@@ -42,9 +42,9 @@ require (
 	google.golang.org/protobuf v1.28.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/square/go-jose.v2 v2.6.0
-	k8s.io/api v0.0.0-20230314091508-112a65bae227
+	k8s.io/api v0.0.0-20230315055830-f836919cf7fd
-	k8s.io/apimachinery v0.0.0-20230314010357-128166500c57
+	k8s.io/apimachinery v0.0.0-20230315054726-2da2e3c5ec8c
-	k8s.io/client-go v0.0.0-20230315061817-2a7ba9488095
+	k8s.io/client-go v0.0.0-20230315061838-ef07195878d5
 	k8s.io/component-base v0.0.0-20230313212246-ce16dede9c0e
 	k8s.io/klog/v2 v2.90.1
 	k8s.io/kms v0.0.0-20230313212457-12714b59d299
@@ -124,9 +124,9 @@ require (
 )
 
 replace (
-	k8s.io/api => k8s.io/api v0.0.0-20230314091508-112a65bae227
+	k8s.io/api => k8s.io/api v0.0.0-20230315055830-f836919cf7fd
-	k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20230314010357-128166500c57
+	k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20230315054726-2da2e3c5ec8c
-	k8s.io/client-go => k8s.io/client-go v0.0.0-20230315061817-2a7ba9488095
+	k8s.io/client-go => k8s.io/client-go v0.0.0-20230315061838-ef07195878d5
 	k8s.io/component-base => k8s.io/component-base v0.0.0-20230313212246-ce16dede9c0e
 	k8s.io/kms => k8s.io/kms v0.0.0-20230313212457-12714b59d299
 )

go.sum

@@ -874,12 +874,12 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.0.0-20230314091508-112a65bae227 h1:Ak4YrHI4101ZMfx2hkXnp//d5r0nKXA8RkNaHCBJ7MA=
+k8s.io/api v0.0.0-20230315055830-f836919cf7fd h1:+dH602TARUCflBOu27VPjQUAWQRipSXs71MGuXEaXoE=
-k8s.io/api v0.0.0-20230314091508-112a65bae227/go.mod h1:YsFNxBfPYQZAIBg0XvL94rxDDVZvQQQUlo4KbwEEqNU=
+k8s.io/api v0.0.0-20230315055830-f836919cf7fd/go.mod h1:0+pk96f3KZ0I1oIK8l1lCI9gWzpKaAQdU7//+21ulhU=
-k8s.io/apimachinery v0.0.0-20230314010357-128166500c57 h1:Vr1geeI+at1NNCWyTN70NtPSNcveZ+fAcbZivzwHknM=
+k8s.io/apimachinery v0.0.0-20230315054726-2da2e3c5ec8c h1:rFeoHo0jyaOib25IhqMPs5LSWtwU3LVmwhES7t4RzS0=
-k8s.io/apimachinery v0.0.0-20230314010357-128166500c57/go.mod h1:1AlvkfXatlv5Kq9dCZg3Ksdu/DyrZ31Q0CncRqQ8Q9I=
+k8s.io/apimachinery v0.0.0-20230315054726-2da2e3c5ec8c/go.mod h1:1AlvkfXatlv5Kq9dCZg3Ksdu/DyrZ31Q0CncRqQ8Q9I=
-k8s.io/client-go v0.0.0-20230315061817-2a7ba9488095 h1:9yBBDqoFcdUPmjREBn9pblz+iOshotP0PgYO3oFnCTg=
+k8s.io/client-go v0.0.0-20230315061838-ef07195878d5 h1:r1VQdDr96Jc/CJmwSSgwjdViB2rnwz7cnaFr0mqIEn4=
-k8s.io/client-go v0.0.0-20230315061817-2a7ba9488095/go.mod h1:oWqDJxiXYcSV7S8woQ4H5epopeK85SJyOu5lJsMndcU=
+k8s.io/client-go v0.0.0-20230315061838-ef07195878d5/go.mod h1:hTovlUcXHrjXfBPynmQWc9IYuZmAU7m1sWRkEs2N1gQ=
 k8s.io/component-base v0.0.0-20230313212246-ce16dede9c0e h1:/mftAl/78q8dPZU8YkshNzt1XpbEJTGsxYZP/WGI4og=
 k8s.io/component-base v0.0.0-20230313212246-ce16dede9c0e/go.mod h1:MQKfc/tXtS50042g1VxMb2W2E8PCt97xO8RsLTw3AeI=
 k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw=

pkg/server/options/encryptionconfig/config.go

@@ -358,7 +358,7 @@ func (h *kmsv2PluginProbe) rotateDEKOnKeyIDChange(ctx context.Context, statusKey
 		return nil
 	}
 
-	transformer, resp, errGen := envelopekmsv2.GenerateTransformer(ctx, uid, h.service)
+	transformer, resp, cacheKey, errGen := envelopekmsv2.GenerateTransformer(ctx, uid, h.service)
 
 	if resp == nil {
 		resp = &kmsservice.EncryptResponse{} // avoid nil panics
@@ -374,6 +374,7 @@ func (h *kmsv2PluginProbe) rotateDEKOnKeyIDChange(ctx context.Context, statusKey
 			Annotations:         resp.Annotations,
 			UID:                 uid,
 			ExpirationTimestamp: expirationTimestamp,
+			CacheKey:            cacheKey,
 		})
 		klog.V(6).InfoS("successfully rotated DEK",
 			"uid", uid,
@@ -410,6 +411,10 @@ func (h *kmsv2PluginProbe) getCurrentState() (envelopekmsv2.State, error) {
 		return envelopekmsv2.State{}, fmt.Errorf("got unexpected zero expirationTimestamp")
 	}
 
+	if len(state.CacheKey) == 0 {
+		return envelopekmsv2.State{}, fmt.Errorf("got unexpected empty cacheKey")
+	}
+
 	return state, nil
 }
 

pkg/server/options/encryptionconfig/config_test.go

@@ -1781,7 +1781,7 @@ func Test_kmsv2PluginProbe_rotateDEKOnKeyIDChange(t *testing.T) {
 				t.Errorf("log mismatch (-want +got):\n%s", diff)
 			}
 
-			ignoredFields := sets.NewString("Transformer", "EncryptedDEK", "UID")
+			ignoredFields := sets.NewString("Transformer", "EncryptedDEK", "UID", "CacheKey")
 
 			if diff := cmp.Diff(tt.wantState, *h.state.Load(),
 				cmp.FilterPath(func(path cmp.Path) bool { return ignoredFields.Has(path.String()) }, cmp.Ignore()),
@@ -1806,6 +1806,7 @@ func validState(keyID string, exp time.Time) envelopekmsv2.State {
 		EncryptedDEK:        []byte{1},
 		KeyID:               keyID,
 		ExpirationTimestamp: exp,
+		CacheKey:            []byte{1},
 	}
 }
 

pkg/storage/value/encrypt/envelope/kmsv2/cache.go

@@ -98,5 +98,11 @@ func (c *simpleCache) keyFunc(s []byte) string {
 
 // toString performs unholy acts to avoid allocations
 func toString(b []byte) string {
-	return *(*string)(unsafe.Pointer(&b))
+	// unsafe.SliceData relies on cap whereas we want to rely on len
+	if len(b) == 0 {
+		return ""
+	}
+	// Copied from go 1.20.1 strings.Builder.String
+	// https://github.com/golang/go/blob/202a1a57064127c3f19d96df57b9f9586145e21c/src/strings/builder.go#L48
+	return unsafe.String(unsafe.SliceData(b), len(b))
 }

pkg/storage/value/encrypt/envelope/kmsv2/envelope.go

@@ -21,9 +21,12 @@ import (
 	"context"
 	"crypto/aes"
 	"fmt"
+	"sort"
 	"time"
+	"unsafe"
 
 	"github.com/gogo/protobuf/proto"
+	"golang.org/x/crypto/cryptobyte"
 
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/uuid"
@@ -87,6 +90,9 @@ type State struct {
 	UID string
 
 	ExpirationTimestamp time.Time
+
+	// CacheKey is the key used to cache the DEK in transformer.cache.
+	CacheKey []byte
 }
 
 func (s *State) ValidateEncryptCapability() error {
@@ -137,8 +143,13 @@ func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []b
 		return nil, false, err
 	}
 
+	encryptedObjectCacheKey, err := generateCacheKey(encryptedObject.EncryptedDEK, encryptedObject.KeyID, encryptedObject.Annotations)
+	if err != nil {
+		return nil, false, err
+	}
+
 	// Look up the decrypted DEK from cache first
-	transformer := t.cache.get(encryptedObject.EncryptedDEK)
+	transformer := t.cache.get(encryptedObjectCacheKey)
 
 	// fallback to the envelope service if we do not have the transformer locally
 	if transformer == nil {
@@ -159,7 +170,7 @@ func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []b
 			return nil, false, fmt.Errorf("failed to decrypt DEK, error: %w", err)
 		}
 
-		transformer, err = t.addTransformerForDecryption(encryptedObject.EncryptedDEK, key)
+		transformer, err = t.addTransformerForDecryption(encryptedObjectCacheKey, key)
 		if err != nil {
 			return nil, false, err
 		}
@@ -190,7 +201,7 @@ func (t *envelopeTransformer) TransformToStorage(ctx context.Context, data []byt
 	// this has the side benefit of causing the cache to perform a GC
 	// TODO see if we can do this inside the stateFunc control loop
 	// TODO(aramase): Add metrics for cache fill percentage with custom cache implementation.
-	t.cache.set(state.EncryptedDEK, state.Transformer)
+	t.cache.set(state.CacheKey, state.Transformer)
 
 	requestInfo := getRequestInfoFromContext(ctx)
 	klog.V(6).InfoS("encrypting content using DEK", "uid", state.UID, "key", string(dataCtx.AuthenticatedData()),
@@ -216,7 +227,7 @@ func (t *envelopeTransformer) TransformToStorage(ctx context.Context, data []byt
 }
 
 // addTransformerForDecryption inserts a new transformer to the Envelope cache of DEKs for future reads.
-func (t *envelopeTransformer) addTransformerForDecryption(encKey []byte, key []byte) (decryptTransformer, error) {
+func (t *envelopeTransformer) addTransformerForDecryption(cacheKey []byte, key []byte) (decryptTransformer, error) {
 	block, err := aes.NewCipher(key)
 	if err != nil {
 		return nil, err
@@ -228,7 +239,7 @@ func (t *envelopeTransformer) addTransformerForDecryption(encKey []byte, key []b
 		return nil, err
 	}
 	// TODO(aramase): Add metrics for cache fill percentage with custom cache implementation.
-	t.cache.set(encKey, transformer)
+	t.cache.set(cacheKey, transformer)
 	return transformer, nil
 }
 
@@ -254,20 +265,25 @@ func (t *envelopeTransformer) doDecode(originalData []byte) (*kmstypes.Encrypted
 	return o, nil
 }
 
-func GenerateTransformer(ctx context.Context, uid string, envelopeService kmsservice.Service) (value.Transformer, *kmsservice.EncryptResponse, error) {
+func GenerateTransformer(ctx context.Context, uid string, envelopeService kmsservice.Service) (value.Transformer, *kmsservice.EncryptResponse, []byte, error) {
 	transformer, newKey, err := aestransformer.NewGCMTransformerWithUniqueKeyUnsafe()
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 
 	klog.V(6).InfoS("encrypting content using envelope service", "uid", uid)
 
 	resp, err := envelopeService.Encrypt(ctx, uid, newKey)
 	if err != nil {
-		return nil, nil, fmt.Errorf("failed to encrypt DEK, error: %w", err)
+		return nil, nil, nil, fmt.Errorf("failed to encrypt DEK, error: %w", err)
 	}
 
-	return transformer, resp, nil
+	cacheKey, err := generateCacheKey(resp.Ciphertext, resp.KeyID, resp.Annotations)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	return transformer, resp, cacheKey, nil
 }
 
 func validateEncryptedObject(o *kmstypes.EncryptedObject) error {
@@ -339,3 +355,59 @@ func getRequestInfoFromContext(ctx context.Context) *genericapirequest.RequestIn
 	}
 	return &genericapirequest.RequestInfo{}
 }
+
+// generateCacheKey returns a key for the cache.
+// The key is a concatenation of:
+//  1. encryptedDEK
+//  2. keyID
+//  3. length of annotations
+//  4. annotations (sorted by key) - each annotation is a concatenation of:
+//     a. annotation key
+//     b. annotation value
+func generateCacheKey(encryptedDEK []byte, keyID string, annotations map[string][]byte) ([]byte, error) {
+	// TODO(aramase): use sync pool buffer to avoid allocations
+	b := cryptobyte.NewBuilder(nil)
+	b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
+		b.AddBytes(encryptedDEK)
+	})
+	b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
+		b.AddBytes(toBytes(keyID))
+	})
+	if len(annotations) == 0 {
+		return b.Bytes()
+	}
+
+	// add the length of annotations to the cache key
+	b.AddUint32(uint32(len(annotations)))
+
+	// Sort the annotations by key.
+	keys := make([]string, 0, len(annotations))
+	for k := range annotations {
+		k := k
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		// The maximum size of annotations is annotationsMaxSize (32 kB) so we can safely
+		// assume that the length of the key and value will fit in a uint16.
+		b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
+			b.AddBytes(toBytes(k))
+		})
+		b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
+			b.AddBytes(annotations[k])
+		})
+	}
+
+	return b.Bytes()
+}
+
+// toBytes performs unholy acts to avoid allocations
+func toBytes(s string) []byte {
+	// unsafe.StringData is unspecified for the empty string, so we provide a strict interpretation
+	if len(s) == 0 {
+		return nil
+	}
+	// Copied from go 1.20.1 os.File.WriteString
+	// https://github.com/golang/go/blob/202a1a57064127c3f19d96df57b9f9586145e21c/src/os/file.go#L246
+	return unsafe.Slice(unsafe.StringData(s), len(s))
+}

pkg/storage/value/encrypt/envelope/kmsv2/envelope_test.go

@@ -55,12 +55,15 @@ const (
 // testEnvelopeService is a mock Envelope service which can be used to simulate remote Envelope services
 // for testing of Envelope based encryption providers.
 type testEnvelopeService struct {
-	annotations map[string][]byte
+	annotations  map[string][]byte
-	disabled    bool
+	disabled     bool
-	keyVersion  string
+	keyVersion   string
+	ciphertext   []byte
+	decryptCalls int
 }
 
 func (t *testEnvelopeService) Decrypt(ctx context.Context, uid string, req *kmsservice.DecryptRequest) ([]byte, error) {
+	t.decryptCalls++
 	if t.disabled {
 		return nil, fmt.Errorf("Envelope service was disabled")
 	}
@@ -88,7 +91,13 @@ func (t *testEnvelopeService) Encrypt(ctx context.Context, uid string, data []by
 	} else {
 		annotations["local-kek.kms.kubernetes.io"] = []byte("encrypted-local-kek")
 	}
-	return &kmsservice.EncryptResponse{Ciphertext: []byte(base64.StdEncoding.EncodeToString(data)), KeyID: t.keyVersion, Annotations: annotations}, nil
+
+	ciphertext := t.ciphertext
+	if ciphertext == nil {
+		ciphertext = []byte(base64.StdEncoding.EncodeToString(data))
+	}
+
+	return &kmsservice.EncryptResponse{Ciphertext: ciphertext, KeyID: t.keyVersion, Annotations: annotations}, nil
 }
 
 func (t *testEnvelopeService) Status(ctx context.Context) (*kmsservice.StatusResponse, error) {
@@ -106,6 +115,10 @@ func (t *testEnvelopeService) SetAnnotations(annotations map[string][]byte) {
 	t.annotations = annotations
 }
 
+func (t *testEnvelopeService) SetCiphertext(ciphertext []byte) {
+	t.ciphertext = ciphertext
+}
+
 func (t *testEnvelopeService) Rotate() {
 	i, _ := strconv.Atoi(t.keyVersion)
 	t.keyVersion = strconv.FormatInt(int64(i+1), 10)
@@ -124,17 +137,26 @@ func TestEnvelopeCaching(t *testing.T) {
 		cacheTTL                 time.Duration
 		simulateKMSPluginFailure bool
 		expectedError            string
+		expectedDecryptCalls     int
 	}{
 		{
 			desc:                     "entry in cache should withstand plugin failure",
 			cacheTTL:                 5 * time.Minute,
 			simulateKMSPluginFailure: true,
+			expectedDecryptCalls:     0, // should not hit KMS plugin
 		},
 		{
 			desc:                     "cache entry expired should not withstand plugin failure",
 			cacheTTL:                 1 * time.Millisecond,
 			simulateKMSPluginFailure: true,
 			expectedError:            "failed to decrypt DEK, error: Envelope service was disabled",
+			expectedDecryptCalls:     10, // should hit KMS plugin for each read after cache entry expired and fail
+		},
+		{
+			desc:                     "cache entry expired should work after cache refresh",
+			cacheTTL:                 1 * time.Millisecond,
+			simulateKMSPluginFailure: false,
+			expectedDecryptCalls:     1, // should hit KMS plugin just for the 1st read after cache entry expired
 		},
 	}
 
@@ -176,30 +198,35 @@ func TestEnvelopeCaching(t *testing.T) {
 			}
 			envelopeService.SetDisabledStatus(tt.simulateKMSPluginFailure)
 
-			// Subsequent read for the same data should work fine due to caching.
+			for i := 0; i < 10; i++ {
-			untransformedData, _, err = transformer.TransformFromStorage(ctx, transformedData, dataCtx)
+				// Subsequent reads for the same data should work fine due to caching.
-			if tt.expectedError != "" {
+				untransformedData, _, err = transformer.TransformFromStorage(ctx, transformedData, dataCtx)
-				if err == nil {
+				if tt.expectedError != "" {
-					t.Fatalf("expected error: %v, got nil", tt.expectedError)
+					if err == nil {
-				}
+						t.Fatalf("expected error: %v, got nil", tt.expectedError)
-				if err.Error() != tt.expectedError {
+					}
-					t.Fatalf("expected error: %v, got: %v", tt.expectedError, err)
+					if err.Error() != tt.expectedError {
-				}
+						t.Fatalf("expected error: %v, got: %v", tt.expectedError, err)
-			} else {
+					}
-				if err != nil {
+				} else {
-					t.Fatalf("unexpected error: %v", err)
+					if err != nil {
-				}
+						t.Fatalf("unexpected error: %v", err)
-				if !bytes.Equal(untransformedData, originalText) {
+					}
-					t.Fatalf("envelopeTransformer transformed data incorrectly. Expected: %v, got %v", originalText, untransformedData)
+					if !bytes.Equal(untransformedData, originalText) {
+						t.Fatalf("envelopeTransformer transformed data incorrectly. Expected: %v, got %v", originalText, untransformedData)
+					}
 				}
 			}
+			if envelopeService.decryptCalls != tt.expectedDecryptCalls {
+				t.Fatalf("expected %d decrypt calls, got %d", tt.expectedDecryptCalls, envelopeService.decryptCalls)
+			}
 		})
 	}
 }
 
 func testStateFunc(ctx context.Context, envelopeService kmsservice.Service, clock clock.Clock) func() (State, error) {
 	return func() (State, error) {
-		transformer, resp, errGen := GenerateTransformer(ctx, string(uuid.NewUUID()), envelopeService)
+		transformer, resp, cacheKey, errGen := GenerateTransformer(ctx, string(uuid.NewUUID()), envelopeService)
 		if errGen != nil {
 			return State{}, errGen
 		}
@@ -210,6 +237,7 @@ func testStateFunc(ctx context.Context, envelopeService kmsservice.Service, cloc
 			Annotations:         resp.Annotations,
 			UID:                 "panda",
 			ExpirationTimestamp: clock.Now().Add(time.Hour),
+			CacheKey:            cacheKey,
 		}, nil
 	}
 }
@@ -861,6 +889,107 @@ func TestEnvelopeLogging(t *testing.T) {
 	}
 }
 
+func TestCacheNotCorrupted(t *testing.T) {
+	ctx := testContext(t)
+
+	envelopeService := newTestEnvelopeService()
+	envelopeService.SetAnnotations(map[string][]byte{
+		"encrypted-dek.kms.kubernetes.io": []byte("encrypted-dek-0"),
+	})
+
+	fakeClock := testingclock.NewFakeClock(time.Now())
+
+	state, err := testStateFunc(ctx, envelopeService, fakeClock)()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	transformer := newEnvelopeTransformerWithClock(envelopeService, testProviderName,
+		func() (State, error) { return state, nil },
+		1*time.Second, fakeClock)
+
+	dataCtx := value.DefaultContext(testContextText)
+	originalText := []byte(testText)
+
+	transformedData1, err := transformer.TransformToStorage(ctx, originalText, dataCtx)
+	if err != nil {
+		t.Fatalf("envelopeTransformer: error while transforming data to storage: %s", err)
+	}
+
+	// this is to mimic a plugin that sets a static response for ciphertext
+	// but uses the annotation field to send the actual encrypted DEK.
+	envelopeService.SetCiphertext(state.EncryptedDEK)
+	// for this plugin, it indicates a change in the remote key ID as the returned
+	// encrypted DEK is different.
+	envelopeService.SetAnnotations(map[string][]byte{
+		"encrypted-dek.kms.kubernetes.io": []byte("encrypted-dek-1"),
+	})
+
+	state, err = testStateFunc(ctx, envelopeService, fakeClock)()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	transformer = newEnvelopeTransformerWithClock(envelopeService, testProviderName,
+		func() (State, error) { return state, nil },
+		1*time.Second, fakeClock)
+
+	transformedData2, err := transformer.TransformToStorage(ctx, originalText, dataCtx)
+	if err != nil {
+		t.Fatalf("envelopeTransformer: error while transforming data to storage: %s", err)
+	}
+
+	if _, _, err := transformer.TransformFromStorage(ctx, transformedData1, dataCtx); err != nil {
+		t.Fatal(err)
+	}
+	if _, _, err := transformer.TransformFromStorage(ctx, transformedData2, dataCtx); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestGenerateCacheKey(t *testing.T) {
+	encryptedDEK1 := []byte{1, 2, 3}
+	keyID1 := "id1"
+	annotations1 := map[string][]byte{"a": {4, 5}, "b": {6, 7}}
+
+	encryptedDEK2 := []byte{4, 5, 6}
+	keyID2 := "id2"
+	annotations2 := map[string][]byte{"x": {9, 10}, "y": {11, 12}}
+
+	// generate all possible combinations of the above
+	testCases := []struct {
+		encryptedDEK []byte
+		keyID        string
+		annotations  map[string][]byte
+	}{
+		{encryptedDEK1, keyID1, annotations1},
+		{encryptedDEK1, keyID1, annotations2},
+		{encryptedDEK1, keyID2, annotations1},
+		{encryptedDEK1, keyID2, annotations2},
+		{encryptedDEK2, keyID1, annotations1},
+		{encryptedDEK2, keyID1, annotations2},
+		{encryptedDEK2, keyID2, annotations1},
+		{encryptedDEK2, keyID2, annotations2},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		for _, tc2 := range testCases {
+			tc2 := tc2
+			t.Run(fmt.Sprintf("%+v-%+v", tc, tc2), func(t *testing.T) {
+				key1, err1 := generateCacheKey(tc.encryptedDEK, tc.keyID, tc.annotations)
+				key2, err2 := generateCacheKey(tc2.encryptedDEK, tc2.keyID, tc2.annotations)
+				if err1 != nil || err2 != nil {
+					t.Errorf("generateCacheKey() want err=nil, got err1=%q, err2=%q", errString(err1), errString(err2))
+				}
+				if bytes.Equal(key1, key2) != reflect.DeepEqual(tc, tc2) {
+					t.Errorf("expected %v, got %v", reflect.DeepEqual(tc, tc2), bytes.Equal(key1, key2))
+				}
+			})
+		}
+	}
+}
+
 func errString(err error) string {
 	if err == nil {
 		return ""