Merge pull request #88294 from immutableT/envelope-metrics

Instrument DEK cache fill and request inter-arrival times.

Kubernetes-commit: 15515e12401090bff398efa73598f99ebf8c34d4

Godeps/Godeps.json

@@ -588,7 +588,7 @@
 		},
 		{
 			"ImportPath": "k8s.io/api",
-			"Rev": "026463abc787"
+			"Rev": "2db522d7d8a2"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery",
@@ -596,7 +596,7 @@
 		},
 		{
 			"ImportPath": "k8s.io/client-go",
-			"Rev": "5e1786105b6f"
+			"Rev": "f772958f8a29"
 		},
 		{
 			"ImportPath": "k8s.io/component-base",

go.mod

@@ -44,9 +44,9 @@ require (
 	gopkg.in/square/go-jose.v2 v2.2.2
 	gopkg.in/yaml.v2 v2.2.8
 	gotest.tools v2.2.0+incompatible // indirect
-	k8s.io/api v0.0.0-20200214081624-026463abc787
+	k8s.io/api v0.0.0-20200221201225-2db522d7d8a2
 	k8s.io/apimachinery v0.0.0-20200214081019-2373d029717c
-	k8s.io/client-go v0.0.0-20200221163115-5e1786105b6f
+	k8s.io/client-go v0.0.0-20200222082411-f772958f8a29
 	k8s.io/component-base v0.0.0-20200221165142-d6212a66f6f6
 	k8s.io/klog v1.0.0
 	k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c
@@ -59,8 +59,8 @@ require (
 replace (
 	golang.org/x/sys => golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a // pinned to release-branch.go1.13
 	golang.org/x/tools => golang.org/x/tools v0.0.0-20190821162956-65e3620a7ae7 // pinned to release-branch.go1.13
-	k8s.io/api => k8s.io/api v0.0.0-20200214081624-026463abc787
+	k8s.io/api => k8s.io/api v0.0.0-20200221201225-2db522d7d8a2
 	k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20200214081019-2373d029717c
-	k8s.io/client-go => k8s.io/client-go v0.0.0-20200221163115-5e1786105b6f
+	k8s.io/client-go => k8s.io/client-go v0.0.0-20200222082411-f772958f8a29
 	k8s.io/component-base => k8s.io/component-base v0.0.0-20200221165142-d6212a66f6f6
 )

go.sum

@@ -337,9 +337,9 @@ gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-k8s.io/api v0.0.0-20200214081624-026463abc787/go.mod h1:brPp6rLV9ZWi2IgXmvCsY7TKw2l27eF4rfCHlyW88ys=
+k8s.io/api v0.0.0-20200221201225-2db522d7d8a2/go.mod h1:brPp6rLV9ZWi2IgXmvCsY7TKw2l27eF4rfCHlyW88ys=
 k8s.io/apimachinery v0.0.0-20200214081019-2373d029717c/go.mod h1:5X8oEhnd931nEg6/Nkumo00nT6ZsCLp2h7Xwd7Ym6P4=
-k8s.io/client-go v0.0.0-20200221163115-5e1786105b6f/go.mod h1:UmUmKsm/QPFM6PX++FkaGSN2sbquEn4rxsBMAvnX+58=
+k8s.io/client-go v0.0.0-20200222082411-f772958f8a29/go.mod h1:oCEmeN2HgOxtxgMbWWGHNOsipC+qARZIFENn44PJ5k4=
 k8s.io/component-base v0.0.0-20200221165142-d6212a66f6f6/go.mod h1:B8q2AK86xxE1I0R1+CrykxCmsXybANtr7pvlzNE9vjo=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=

pkg/storage/value/encrypt/envelope/envelope.go

@@ -33,6 +33,7 @@ import (
 
 func init() {
 	value.RegisterMetrics()
+	registerMetrics()
 }
 
 // Service allows encrypting and decrypting data using an external Key Management Service.
@@ -52,6 +53,7 @@ type envelopeTransformer struct {
 	// baseTransformerFunc creates a new transformer for encrypting the data with the DEK.
 	baseTransformerFunc func(cipher.Block) value.Transformer
 
+	cacheSize    int
 	cacheEnabled bool
 }
 
@@ -76,11 +78,14 @@ func NewEnvelopeTransformer(envelopeService Service, cacheSize int, baseTransfor
 		transformers:        cache,
 		baseTransformerFunc: baseTransformerFunc,
 		cacheEnabled:        cacheSize > 0,
+		cacheSize:           cacheSize,
 	}, nil
 }
 
 // TransformFromStorage decrypts data encrypted by this transformer using envelope encryption.
 func (t *envelopeTransformer) TransformFromStorage(data []byte, context value.Context) ([]byte, bool, error) {
+	recordArrival(fromStorageLabel, time.Now())
+
 	// Read the 16 bit length-of-DEK encoded at the start of the encrypted DEK. 16 bits can
 	// represent a maximum key length of 65536 bytes. We are using a 256 bit key, whose
 	// length cannot fit in 8 bits (1 byte). Thus, we use 16 bits (2 bytes) to store the length.
@@ -117,6 +122,7 @@ func (t *envelopeTransformer) TransformFromStorage(data []byte, context value.Co
 
 // TransformToStorage encrypts data to be written to disk using envelope encryption.
 func (t *envelopeTransformer) TransformToStorage(data []byte, context value.Context) ([]byte, error) {
+	recordArrival(toStorageLabel, time.Now())
 	newKey, err := generateKey(32)
 	if err != nil {
 		return nil, err
@@ -162,6 +168,7 @@ func (t *envelopeTransformer) addTransformer(encKey []byte, key []byte) (value.T
 	// cannot hash []uint8.
 	if t.cacheEnabled {
 		t.transformers.Add(base64.StdEncoding.EncodeToString(encKey), transformer)
+		dekCacheFillPercent.Set(float64(t.transformers.Len()) / float64(t.cacheSize))
 	}
 	return transformer, nil
 }

pkg/storage/value/encrypt/envelope/metrics.go

@@ -0,0 +1,102 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package envelope
+
+import (
+	"sync"
+	"time"
+
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+const (
+	namespace        = "apiserver"
+	subsystem        = "envelope_encryption"
+	fromStorageLabel = "from_storage"
+	toStorageLabel   = "to_storage"
+)
+
+/*
+ * By default, all the following metrics are defined as falling under
+ * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md#stability-classes)
+ *
+ * Promoting the stability level of the metric is a responsibility of the component owner, since it
+ * involves explicitly acknowledging support for the metric across multiple releases, in accordance with
+ * the metric stability policy.
+ */
+var (
+	lockLastFromStorage sync.Mutex
+	lockLastToStorage   sync.Mutex
+
+	lastFromStorage time.Time
+	lastToStorage   time.Time
+
+	dekCacheFillPercent = metrics.NewGauge(
+		&metrics.GaugeOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "dek_cache_fill_percent",
+			Help:           "Percent of the cache slots currently occupied by cached DEKs.",
+			StabilityLevel: metrics.ALPHA,
+		},
+	)
+
+	dekCacheInterArrivals = metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "dek_cache_inter_arrival_time_seconds",
+			Help:           "Time (in seconds) of inter arrival of transformation requests.",
+			StabilityLevel: metrics.ALPHA,
+			Buckets:        metrics.ExponentialBuckets(60, 2, 10),
+		},
+		[]string{"transformation_type"},
+	)
+)
+
+var registerMetricsFunc sync.Once
+
+func registerMetrics() {
+	registerMetricsFunc.Do(func() {
+		legacyregistry.MustRegister(dekCacheFillPercent)
+		legacyregistry.MustRegister(dekCacheInterArrivals)
+	})
+}
+
+func recordArrival(transformationType string, start time.Time) {
+	switch transformationType {
+	case fromStorageLabel:
+		lockLastFromStorage.Lock()
+		defer lockLastFromStorage.Unlock()
+
+		if lastFromStorage.IsZero() {
+			lastFromStorage = start
+		}
+		dekCacheInterArrivals.WithLabelValues(transformationType).Observe(start.Sub(lastFromStorage).Seconds())
+		lastFromStorage = start
+	case toStorageLabel:
+		lockLastToStorage.Lock()
+		defer lockLastToStorage.Unlock()
+
+		if lastToStorage.IsZero() {
+			lastToStorage = start
+		}
+		dekCacheInterArrivals.WithLabelValues(transformationType).Observe(start.Sub(lastToStorage).Seconds())
+		lastToStorage = start
+	}
+}