Instrument DEK cache fill and request inter-arrival times.

Kubernetes-commit: 684d6fb0ade6ac088af391cedd70bc847941a54f

pkg/storage/value/encrypt/envelope/envelope.go

@@ -33,6 +33,7 @@ import (
 
 func init() {
 	value.RegisterMetrics()
+	registerMetrics()
 }
 
 // Service allows encrypting and decrypting data using an external Key Management Service.
@@ -52,6 +53,7 @@ type envelopeTransformer struct {
 	// baseTransformerFunc creates a new transformer for encrypting the data with the DEK.
 	baseTransformerFunc func(cipher.Block) value.Transformer
 
+	cacheSize    int
 	cacheEnabled bool
 }
 
@@ -76,11 +78,14 @@ func NewEnvelopeTransformer(envelopeService Service, cacheSize int, baseTransfor
 		transformers:        cache,
 		baseTransformerFunc: baseTransformerFunc,
 		cacheEnabled:        cacheSize > 0,
+		cacheSize:           cacheSize,
 	}, nil
 }
 
 // TransformFromStorage decrypts data encrypted by this transformer using envelope encryption.
 func (t *envelopeTransformer) TransformFromStorage(data []byte, context value.Context) ([]byte, bool, error) {
+	recordArrival(fromStorageLabel, time.Now())
+
 	// Read the 16 bit length-of-DEK encoded at the start of the encrypted DEK. 16 bits can
 	// represent a maximum key length of 65536 bytes. We are using a 256 bit key, whose
 	// length cannot fit in 8 bits (1 byte). Thus, we use 16 bits (2 bytes) to store the length.
@@ -117,6 +122,7 @@ func (t *envelopeTransformer) TransformFromStorage(data []byte, context value.Co
 
 // TransformToStorage encrypts data to be written to disk using envelope encryption.
 func (t *envelopeTransformer) TransformToStorage(data []byte, context value.Context) ([]byte, error) {
+	recordArrival(toStorageLabel, time.Now())
 	newKey, err := generateKey(32)
 	if err != nil {
 		return nil, err
@@ -162,6 +168,7 @@ func (t *envelopeTransformer) addTransformer(encKey []byte, key []byte) (value.T
 	// cannot hash []uint8.
 	if t.cacheEnabled {
 		t.transformers.Add(base64.StdEncoding.EncodeToString(encKey), transformer)
+		dekCacheFillPercent.Set(float64(t.transformers.Len()) / float64(t.cacheSize))
 	}
 	return transformer, nil
 }

pkg/storage/value/encrypt/envelope/metrics.go

@@ -0,0 +1,102 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package envelope
+
+import (
+	"sync"
+	"time"
+
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+const (
+	namespace        = "apiserver"
+	subsystem        = "envelope_encryption"
+	fromStorageLabel = "from_storage"
+	toStorageLabel   = "to_storage"
+)
+
+/*
+ * By default, all the following metrics are defined as falling under
+ * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md#stability-classes)
+ *
+ * Promoting the stability level of the metric is a responsibility of the component owner, since it
+ * involves explicitly acknowledging support for the metric across multiple releases, in accordance with
+ * the metric stability policy.
+ */
+var (
+	lockLastFromStorage sync.Mutex
+	lockLastToStorage   sync.Mutex
+
+	lastFromStorage time.Time
+	lastToStorage   time.Time
+
+	dekCacheFillPercent = metrics.NewGauge(
+		&metrics.GaugeOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "dek_cache_fill_percent",
+			Help:           "Percent of the cache slots currently occupied by cached DEKs.",
+			StabilityLevel: metrics.ALPHA,
+		},
+	)
+
+	dekCacheInterArrivals = metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "dek_cache_inter_arrival_time_seconds",
+			Help:           "Time (in seconds) of inter arrival of transformation requests.",
+			StabilityLevel: metrics.ALPHA,
+			Buckets:        metrics.ExponentialBuckets(60, 2, 10),
+		},
+		[]string{"transformation_type"},
+	)
+)
+
+var registerMetricsFunc sync.Once
+
+func registerMetrics() {
+	registerMetricsFunc.Do(func() {
+		legacyregistry.MustRegister(dekCacheFillPercent)
+		legacyregistry.MustRegister(dekCacheInterArrivals)
+	})
+}
+
+func recordArrival(transformationType string, start time.Time) {
+	switch transformationType {
+	case fromStorageLabel:
+		lockLastFromStorage.Lock()
+		defer lockLastFromStorage.Unlock()
+
+		if lastFromStorage.IsZero() {
+			lastFromStorage = start
+		}
+		dekCacheInterArrivals.WithLabelValues(transformationType).Observe(start.Sub(lastFromStorage).Seconds())
+		lastFromStorage = start
+	case toStorageLabel:
+		lockLastToStorage.Lock()
+		defer lockLastToStorage.Unlock()
+
+		if lastToStorage.IsZero() {
+			lastToStorage = start
+		}
+		dekCacheInterArrivals.WithLabelValues(transformationType).Observe(start.Sub(lastToStorage).Seconds())
+		lastToStorage = start
+	}
+}