kubernetes
/
apiserver
mirror of https://github.com/kubernetes/apiserver.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #114376 from baomingwang/serialization-error-metric 

Added serialization from etcd error metric

Kubernetes-commit: dfb976e25a6687a2c4ff7d374e3f01006d65df66
This commit is contained in:
Kubernetes Publisher 2023-02-07 13:46:59 -08:00
parent 3cee67d89e 3f56cdd970
commit e3ca625155
5 changed files with 108 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
go.mod
View File

@ -44,8 +44,8 @@ require (
gopkg.in/square/go-jose.v2 v2.2.2
k8s.io/api v0.0.0-20230207050353-0478a3e95231
k8s.io/apimachinery v0.0.0-20230207050124-7687996c715e
k8s.io/client-go v0.0.0-20230207050729-9c5046523c31
k8s.io/component-base v0.0.0-20230207051715-0e730ff5a97c
k8s.io/client-go v0.0.0-20230207170730-00b9d76f44fa
k8s.io/component-base v0.0.0-20230207171128-98a50fa9a53a
k8s.io/klog/v2 v2.80.1
k8s.io/kms v0.0.0-20230207051926-033d6007a685
k8s.io/kube-openapi v0.0.0-20230123231816-1cb3ae25d79a
@ -124,7 +124,7 @@ require (
replace (
k8s.io/api => k8s.io/api v0.0.0-20230207050353-0478a3e95231
k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20230207050124-7687996c715e
k8s.io/client-go => k8s.io/client-go v0.0.0-20230207050729-9c5046523c31
k8s.io/component-base => k8s.io/component-base v0.0.0-20230207051715-0e730ff5a97c
k8s.io/client-go => k8s.io/client-go v0.0.0-20230207170730-00b9d76f44fa
k8s.io/component-base => k8s.io/component-base v0.0.0-20230207171128-98a50fa9a53a
k8s.io/kms => k8s.io/kms v0.0.0-20230207051926-033d6007a685
)

8
go.sum
View File

@ -995,10 +995,10 @@ k8s.io/api v0.0.0-20230207050353-0478a3e95231 h1:Hw1yJYPZz5zc2WFi/BXpNBKGes53T+s
k8s.io/api v0.0.0-20230207050353-0478a3e95231/go.mod h1:AOV8t62tTmWTshfSud6FYWhf3Rm1Q5UxvAqLIqlxkM4=
k8s.io/apimachinery v0.0.0-20230207050124-7687996c715e h1:5ePdsIQsiQekT414IXmCFsnLSOIRJOK5PWGeL572CmU=
k8s.io/apimachinery v0.0.0-20230207050124-7687996c715e/go.mod h1:+fpAUqCL1i4ngcHvZN7X/IpmZkLOuwrbLoazw/uczMg=
k8s.io/client-go v0.0.0-20230207050729-9c5046523c31 h1:DWjKT8WVJJt+EryxnjEYrxMuNX7rYw8FLpAVehncQuU=
k8s.io/client-go v0.0.0-20230207050729-9c5046523c31/go.mod h1:ioQlcLr7B8Ah2uK38HJZAjbXzTANbLLyr1i1BII9pJo=
k8s.io/component-base v0.0.0-20230207051715-0e730ff5a97c h1:BtaYpuZAOEf0xFA63UvwB/cSChQ2yH6L+xSa/ig1iVs=
k8s.io/component-base v0.0.0-20230207051715-0e730ff5a97c/go.mod h1:fbYnBlLfmf+XpdnrYWgDHjNqYLW61/CSfCRIlSY5h8s=
k8s.io/client-go v0.0.0-20230207170730-00b9d76f44fa h1:3Wx0R6PYblpdjecD28Xce4jTRrXEBpyQ1ngFohA7SQk=
k8s.io/client-go v0.0.0-20230207170730-00b9d76f44fa/go.mod h1:ioQlcLr7B8Ah2uK38HJZAjbXzTANbLLyr1i1BII9pJo=
k8s.io/component-base v0.0.0-20230207171128-98a50fa9a53a h1:duAULUYrLV7aCLzA6MLvmn28LjyFW52PFmM1HcTOqbQ=
k8s.io/component-base v0.0.0-20230207171128-98a50fa9a53a/go.mod h1:fbYnBlLfmf+XpdnrYWgDHjNqYLW61/CSfCRIlSY5h8s=
k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4=
k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
k8s.io/kms v0.0.0-20230207051926-033d6007a685 h1:MJkvtoWzMwNe3LYaE6RiDRnds1KXAdtWJKk6nW1oYwo=

15
pkg/storage/etcd3/metrics/metrics.go
View File

@ -113,6 +113,15 @@ var (
},
[]string{"resource"},
)
decodeErrorCounts = compbasemetrics.NewCounterVec(
&compbasemetrics.CounterOpts{
Namespace: "apiserver",
Name: "storage_decode_errors_total",
Help: "Number of stored object decode errors split by object type",
StabilityLevel: compbasemetrics.ALPHA,
},
[]string{"resource"},
)
)
var registerMetrics sync.Once
@ -130,6 +139,7 @@ func Register() {
legacyregistry.MustRegister(listStorageNumFetched)
legacyregistry.MustRegister(listStorageNumSelectorEvals)
legacyregistry.MustRegister(listStorageNumReturned)
legacyregistry.MustRegister(decodeErrorCounts)
})
}
@ -148,6 +158,11 @@ func RecordEtcdBookmark(resource string) {
etcdBookmarkCounts.WithLabelValues(resource).Inc()
}
// RecordDecodeError sets the storage_decode_errors metrics.
func RecordDecodeError(resource string) {
decodeErrorCounts.WithLabelValues(resource).Inc()
}
// Reset resets the etcd_request_duration_seconds metric.
func Reset() {
etcdRequestLatency.Reset()

57
pkg/storage/etcd3/metrics/metrics_test.go Normal file
View File

@ -0,0 +1,57 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package metrics
import (
"strings"
"testing"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/testutil"
)
func TestRecordDecodeError(t *testing.T) {
registry := metrics.NewKubeRegistry()
defer registry.Reset()
registry.Register(decodeErrorCounts)
resourceName := "pods"
testedMetrics := "apiserver_storage_decode_errors_total"
testCases := []struct {
desc string
resource string
want string
}{
{
desc: "test success",
resource: resourceName,
want: `
# HELP apiserver_storage_decode_errors_total [ALPHA] Number of stored object decode errors split by object type
# TYPE apiserver_storage_decode_errors_total counter
apiserver_storage_decode_errors_total{resource="pods"} 1
`,
},
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
RecordDecodeError(test.resource)
if err := testutil.GatherAndCompare(registry, strings.NewReader(test.want), testedMetrics); err != nil {
t.Fatal(err)
}
})
}
}

31
pkg/storage/etcd3/store.go
View File

@ -156,7 +156,12 @@ func (s *store) Get(ctx context.Context, key string, opts storage.GetOptions, ou
return storage.NewInternalError(err.Error())
}
return decode(s.codec, s.versioner, data, out, kv.ModRevision)
err = decode(s.codec, s.versioner, data, out, kv.ModRevision)
if err != nil {
recordDecodeError(s.groupResourceString, preparedKey)
return err
}
return nil
}
// Create implements storage.Interface.Create.
@ -220,6 +225,7 @@ func (s *store) Create(ctx context.Context, key string, obj, out runtime.Object,
err = decode(s.codec, s.versioner, data, out, putResp.Header.Revision)
if err != nil {
span.AddEvent("decode failed", attribute.Int("len", len(data)), attribute.String("err", err.Error()))
recordDecodeError(s.groupResourceString, preparedKey)
return err
}
span.AddEvent("decode succeeded", attribute.Int("len", len(data)))
@ -352,7 +358,12 @@ func (s *store) conditionalDelete(
if deleteResp.Header == nil {
return errors.New("invalid DeleteRange response - nil header")
}
return decode(s.codec, s.versioner, origState.data, out, deleteResp.Header.Revision)
err = decode(s.codec, s.versioner, origState.data, out, deleteResp.Header.Revision)
if err != nil {
recordDecodeError(s.groupResourceString, key)
return err
}
return nil
}
}
@ -470,7 +481,12 @@ func (s *store) GuaranteedUpdate(
}
// recheck that the data from etcd is not stale before short-circuiting a write
if !origState.stale {
return decode(s.codec, s.versioner, origState.data, destination, origState.rev)
err = decode(s.codec, s.versioner, origState.data, destination, origState.rev)
if err != nil {
recordDecodeError(s.groupResourceString, preparedKey)
return err
}
return nil
}
}
@ -518,6 +534,7 @@ func (s *store) GuaranteedUpdate(
err = decode(s.codec, s.versioner, data, destination, putResp.Header.Revision)
if err != nil {
span.AddEvent("decode failed", attribute.Int("len", len(data)), attribute.String("err", err.Error()))
recordDecodeError(s.groupResourceString, preparedKey)
return err
}
span.AddEvent("decode succeeded", attribute.Int("len", len(data)))
@ -745,6 +762,7 @@ func (s *store) GetList(ctx context.Context, key string, opts storage.ListOption
}
if err := appendListItem(v, data, uint64(kv.ModRevision), pred, s.codec, s.versioner, newItemFunc); err != nil {
recordDecodeError(s.groupResourceString, string(kv.Key))
return err
}
numEvald++
@ -884,6 +902,7 @@ func (s *store) getState(ctx context.Context, getResp *clientv3.GetResponse, key
state.data = data
state.stale = stale
if err := decode(s.codec, s.versioner, state.data, state.obj, state.rev); err != nil {
recordDecodeError(s.groupResourceString, key)
return nil, err
}
}
@ -1022,6 +1041,12 @@ func appendListItem(v reflect.Value, data []byte, rev uint64, pred storage.Selec
return nil
}
// recordDecodeError record decode error split by object type.
func recordDecodeError(resource string, key string) {
metrics.RecordDecodeError(resource)
klog.V(4).Infof("Decoding %s \"%s\" failed", resource, key)
}
func notFound(key string) clientv3.Cmp {
return clientv3.Compare(clientv3.ModRevision(key), "=", 0)
}