From 959ee35394452dd88a07b3e2c389f8e3aaf0aa8c Mon Sep 17 00:00:00 2001 From: immutablet Date: Tue, 27 Feb 2018 17:24:27 -0800 Subject: [PATCH 1/3] Instrument transformer.go with latency metrics. Kubernetes-commit: e54864f53de75cd56c0fe94777e1d3de0c559c7f --- pkg/storage/value/metrics.go | 55 ++++++++++++++++++++++++++++++++ pkg/storage/value/transformer.go | 7 ++++ 2 files changed, 62 insertions(+) create mode 100644 pkg/storage/value/metrics.go diff --git a/pkg/storage/value/metrics.go b/pkg/storage/value/metrics.go new file mode 100644 index 000000000..1cdb006a4 --- /dev/null +++ b/pkg/storage/value/metrics.go @@ -0,0 +1,55 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package value + +import ( + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +const ( + valueSubsystem = "value" +) + +var ( + TransformerOperationalLatencies = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Subsystem: valueSubsystem, + Name: "storage_transformation_latency_microseconds", + Help: "Latency in microseconds of value transformation operations.", + }, + []string{"transformation_type"}, + ) +) + +var registerMetrics sync.Once + +func RegisterMetrics() { + registerMetrics.Do(func() { + prometheus.MustRegister(TransformerOperationalLatencies) + }) +} + +func RecordTransformation(transformationType string, start time.Time) { + TransformerOperationalLatencies.WithLabelValues(transformationType).Observe(sinceInMicroseconds(start)) +} + +func sinceInMicroseconds(start time.Time) float64 { + return float64(time.Since(start).Nanoseconds() / time.Microsecond.Nanoseconds()) +} diff --git a/pkg/storage/value/transformer.go b/pkg/storage/value/transformer.go index 04af5d6db..899814716 100644 --- a/pkg/storage/value/transformer.go +++ b/pkg/storage/value/transformer.go @@ -21,8 +21,13 @@ import ( "bytes" "fmt" "sync" + "time" ) +func init() { + RegisterMetrics() +} + // Context is additional information that a storage transformation may need to verify the data at rest. type Context interface { // AuthenticatedData should return an array of bytes that describes the current value. If the value changes, @@ -80,12 +85,14 @@ func (t *MutableTransformer) Set(transformer Transformer) { } func (t *MutableTransformer) TransformFromStorage(data []byte, context Context) (out []byte, stale bool, err error) { + defer RecordTransformation("from_storage", time.Now()) t.lock.RLock() transformer := t.transformer t.lock.RUnlock() return transformer.TransformFromStorage(data, context) } func (t *MutableTransformer) TransformToStorage(data []byte, context Context) (out []byte, err error) { + defer RecordTransformation("to_storage", time.Now()) t.lock.RLock() transformer := t.transformer t.lock.RUnlock() From f44ea185dae55229044d566697f9f9d778a0c2d1 Mon Sep 17 00:00:00 2001 From: immutablet Date: Thu, 15 Mar 2018 14:13:24 -0700 Subject: [PATCH 2/3] Instrument transformer.go with latency metrics. Kubernetes-commit: 04a6613fb565a54b6a74e5bfad8844928e98a59b --- pkg/storage/value/metrics.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/storage/value/metrics.go b/pkg/storage/value/metrics.go index 1cdb006a4..2a903011b 100644 --- a/pkg/storage/value/metrics.go +++ b/pkg/storage/value/metrics.go @@ -31,7 +31,7 @@ var ( TransformerOperationalLatencies = prometheus.NewSummaryVec( prometheus.SummaryOpts{ Subsystem: valueSubsystem, - Name: "storage_transformation_latency_microseconds", + Name: "apiserver_storage_transformation_latency_microseconds", Help: "Latency in microseconds of value transformation operations.", }, []string{"transformation_type"}, @@ -47,9 +47,9 @@ func RegisterMetrics() { } func RecordTransformation(transformationType string, start time.Time) { - TransformerOperationalLatencies.WithLabelValues(transformationType).Observe(sinceInMicroseconds(start)) + TransformerOperationalLatencies.WithLabelValues(transformationType).Observe(float64(sinceInMicroseconds(start))) } -func sinceInMicroseconds(start time.Time) float64 { - return float64(time.Since(start).Nanoseconds() / time.Microsecond.Nanoseconds()) +func sinceInMicroseconds(start time.Time) time.Duration { + return time.Since(start) / time.Microsecond } From dfdceff3c6e57fa310faf8008a5e2f69e932cb7b Mon Sep 17 00:00:00 2001 From: immutablet Date: Fri, 16 Mar 2018 14:25:26 -0700 Subject: [PATCH 3/3] Instrument transformer.go with latency metrics. Kubernetes-commit: bfcb3cd91f93669b94ea80eadebdff769c88952e --- pkg/storage/value/metrics.go | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/pkg/storage/value/metrics.go b/pkg/storage/value/metrics.go index 2a903011b..0425cd230 100644 --- a/pkg/storage/value/metrics.go +++ b/pkg/storage/value/metrics.go @@ -23,16 +23,16 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const ( - valueSubsystem = "value" -) - var ( - TransformerOperationalLatencies = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Subsystem: valueSubsystem, - Name: "apiserver_storage_transformation_latency_microseconds", - Help: "Latency in microseconds of value transformation operations.", + transformerLatencies = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "apiserver", + Subsystem: "storage", + Name: "transformation_latencies_microseconds", + Help: "Latencies in microseconds of value transformation operations.", + // In-process transformations (ex. AES CBC) complete on the order of 20 microseconds. However, when + // external KMS is involved latencies may climb into milliseconds. + Buckets: prometheus.ExponentialBuckets(5, 2, 14), }, []string{"transformation_type"}, ) @@ -42,14 +42,16 @@ var registerMetrics sync.Once func RegisterMetrics() { registerMetrics.Do(func() { - prometheus.MustRegister(TransformerOperationalLatencies) + prometheus.MustRegister(transformerLatencies) }) } func RecordTransformation(transformationType string, start time.Time) { - TransformerOperationalLatencies.WithLabelValues(transformationType).Observe(float64(sinceInMicroseconds(start))) + since := sinceInMicroseconds(start) + transformerLatencies.WithLabelValues(transformationType).Observe(float64(since)) } -func sinceInMicroseconds(start time.Time) time.Duration { - return time.Since(start) / time.Microsecond +func sinceInMicroseconds(start time.Time) int64 { + elapsedNanoseconds := time.Since(start).Nanoseconds() + return elapsedNanoseconds / int64(time.Microsecond) }