kubernetes
/
apiserver
mirror of https://github.com/kubernetes/apiserver.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

register metrics from comp-base 

Kubernetes-commit: 11656478be93d4a9e54129ec35cd2b9558e901ac
This commit is contained in:
yue9944882 2020-02-27 17:04:17 +08:00 committed by Kubernetes Publisher
parent dffa3c3d5d
commit f452a698b0
5 changed files with 73 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
go.mod
View File

@ -29,12 +29,12 @@ require (
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
github.com/pkg/errors v0.8.1
github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021 // indirect
github.com/prometheus/client_golang v1.0.0
github.com/prometheus/client_model v0.2.0
github.com/sirupsen/logrus v1.4.2 // indirect
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.4.0
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738
go.uber.org/atomic v1.3.2
go.uber.org/zap v1.10.0
golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9
@ -44,10 +44,10 @@ require (
gopkg.in/square/go-jose.v2 v2.2.2
gopkg.in/yaml.v2 v2.2.8
gotest.tools v2.2.0+incompatible // indirect
k8s.io/api v0.0.0-20200302082247-8f54d34188b0
k8s.io/apimachinery v0.0.0-20200302045842-b9f0d37e94c6
k8s.io/client-go v0.0.0-20200302082525-ca7edf3d8a93
k8s.io/component-base v0.0.0-20200227163006-6f93f319a354
k8s.io/api v0.0.0
k8s.io/apimachinery v0.0.0
k8s.io/client-go v0.0.0
k8s.io/component-base v0.0.0
k8s.io/klog v1.0.0
k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c
k8s.io/utils v0.0.0-20200117235808-5f6fbceb4c31
@ -59,8 +59,9 @@ require (
replace (
golang.org/x/sys => golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a // pinned to release-branch.go1.13
golang.org/x/tools => golang.org/x/tools v0.0.0-20190821162956-65e3620a7ae7 // pinned to release-branch.go1.13
k8s.io/api => k8s.io/api v0.0.0-20200302082247-8f54d34188b0
k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20200302045842-b9f0d37e94c6
k8s.io/client-go => k8s.io/client-go v0.0.0-20200302082525-ca7edf3d8a93
k8s.io/component-base => k8s.io/component-base v0.0.0-20200227163006-6f93f319a354
k8s.io/api => ../api
k8s.io/apimachinery => ../apimachinery
k8s.io/apiserver => ../apiserver
k8s.io/client-go => ../client-go
k8s.io/component-base => ../component-base
)

4
go.sum
View File

@ -337,10 +337,6 @@ gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
k8s.io/api v0.0.0-20200302082247-8f54d34188b0/go.mod h1:+2jnw1NMpdXlVDlcy5KjXl7Gh4M0HZ0AhKakQo+KiV8=
k8s.io/apimachinery v0.0.0-20200302045842-b9f0d37e94c6/go.mod h1:5X8oEhnd931nEg6/Nkumo00nT6ZsCLp2h7Xwd7Ym6P4=
k8s.io/client-go v0.0.0-20200302082525-ca7edf3d8a93/go.mod h1:2DzSdhxUXsi3Ln8q5B+GHLG6b2cQN64WNsTnBYZ8Y4Y=
k8s.io/component-base v0.0.0-20200227163006-6f93f319a354/go.mod h1:HDvqB92UBNY8EfzZuIRA2S7PmDF5CfiRwS88LNkTRnE=
k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=

31
pkg/server/filters/priority-and-fairness.go
View File

@ -21,11 +21,9 @@ import (
"fmt"
"net/http"
// TODO: decide whether to also generate the old metrics, which
// categorize according to mutating vs readonly.
// "k8s.io/apiserver/pkg/endpoints/metrics"
_ "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
"go.uber.org/atomic"
fcv1a1 "k8s.io/api/flowcontrol/v1alpha1"
apitypes "k8s.io/apimachinery/pkg/types"
apirequest "k8s.io/apiserver/pkg/endpoints/request"
@ -38,8 +36,8 @@ type priorityAndFairnessKeyType int
const priorityAndFairnessKey priorityAndFairnessKeyType = iota
const (
responseHeaderMatchedPriorityLevelConfigurationUID = "X-Kubernetes-PF-PriorityLevelUID"
responseHeaderMatchedFlowSchemaUID = "X-Kubernetes-PF-FlowSchemaUID"
responseHeaderMatchedPriorityLevelConfigurationUID = "X-Kubernetes-PF-PriorityLevel-UID"
responseHeaderMatchedFlowSchemaUID = "X-Kubernetes-PF-FlowSchema-UID"
)
// PriorityAndFairnessClassification identifies the results of
@ -57,6 +55,8 @@ func GetClassification(ctx context.Context) *PriorityAndFairnessClassification {
return ctx.Value(priorityAndFairnessKey).(*PriorityAndFairnessClassification)
}
var atomicMutatingLen, atomicNonMutatingLen = atomic.NewInt32(0), atomic.NewInt32(0)
// WithPriorityAndFairness limits the number of in-flight
// requests in a fine-grained way.
func WithPriorityAndFairness(
@ -68,7 +68,7 @@ func WithPriorityAndFairness(
klog.Warningf("priority and fairness support not found, skipping")
return handler
}
startOnce.Do(startRecordingUsage)
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
requestInfo, ok := apirequest.RequestInfoFrom(ctx)
@ -97,8 +97,25 @@ func WithPriorityAndFairness(
PriorityLevelName: pl.Name,
PriorityLevelUID: pl.UID}
}
var served bool
isMutatingRequest := !nonMutatingRequestVerbs.Has(requestInfo.Verb)
execute := func() {
var mutatingLen, readOnlyLen int
if isMutatingRequest {
mutatingLen = int(atomicMutatingLen.Inc())
} else {
readOnlyLen = int(atomicNonMutatingLen.Inc())
}
defer func() {
if isMutatingRequest {
atomicMutatingLen.Dec()
watermark.recordMutating(mutatingLen)
} else {
atomicNonMutatingLen.Dec()
watermark.recordReadOnly(readOnlyLen)
}
}()
served = true
innerCtx := context.WithValue(ctx, priorityAndFairnessKey, classification)
innerReq := r.Clone(innerCtx)

2
pkg/server/routes/metrics.go
View File

@ -23,6 +23,7 @@ import (
apimetrics "k8s.io/apiserver/pkg/endpoints/metrics"
"k8s.io/apiserver/pkg/server/mux"
etcd3metrics "k8s.io/apiserver/pkg/storage/etcd3/metrics"
flowcontrolmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)
@ -58,4 +59,5 @@ func (m MetricsWithReset) Install(c *mux.PathRecorderMux) {
func register() {
apimetrics.Register()
etcd3metrics.Register()
flowcontrolmetrics.Register()
}

62
pkg/util/flowcontrol/metrics/metrics.go
View File

@ -17,9 +17,11 @@ limitations under the License.
package metrics
import (
"sync"
"time"
"github.com/prometheus/client_golang/prometheus"
compbasemetrics "k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)
const (
@ -37,28 +39,29 @@ var (
requestDurationSecondsBuckets = []float64{0, 0.005, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 30}
)
func init() {
prometheus.MustRegister(apiserverRejectedRequests)
prometheus.MustRegister(apiserverCurrentInqueueRequests)
prometheus.MustRegister(apiserverRequestQueueLength)
prometheus.MustRegister(apiserverRequestConcurrencyLimit)
prometheus.MustRegister(apiserverCurrentExecutingRequests)
prometheus.MustRegister(apiserverRequestWaitingSeconds)
prometheus.MustRegister(apiserverRequestExecutionSeconds)
var registerMetrics sync.Once
// Register all metrics.
func Register() {
registerMetrics.Do(func() {
for _, metric := range metrics {
legacyregistry.MustRegister(metric)
}
})
}
var (
apiserverRejectedRequests = prometheus.NewCounterVec(
prometheus.CounterOpts{
apiserverRejectedRequestsTotal = compbasemetrics.NewCounterVec(
&compbasemetrics.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "rejected_requests",
Name: "rejected_requests_total",
Help: "Number of rejected requests by api priority and fairness system",
},
[]string{priorityLevel, "reason"},
)
apiserverCurrentInqueueRequests = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
apiserverCurrentInqueueRequests = compbasemetrics.NewGaugeVec(
&compbasemetrics.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "current_inqueue_requests",
@ -66,8 +69,8 @@ var (
},
[]string{priorityLevel},
)
apiserverRequestQueueLength = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
apiserverRequestQueueLength = compbasemetrics.NewHistogramVec(
&compbasemetrics.HistogramOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "request_queue_length",
@ -76,8 +79,8 @@ var (
},
[]string{priorityLevel},
)
apiserverRequestConcurrencyLimit = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
apiserverRequestConcurrencyLimit = compbasemetrics.NewGaugeVec(
&compbasemetrics.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "request_concurrency_limit",
@ -85,8 +88,8 @@ var (
},
[]string{priorityLevel},
)
apiserverCurrentExecutingRequests = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
apiserverCurrentExecutingRequests = compbasemetrics.NewGaugeVec(
&compbasemetrics.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "current_executing_requests",
@ -94,8 +97,8 @@ var (
},
[]string{priorityLevel},
)
apiserverRequestWaitingSeconds = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
apiserverRequestWaitingSeconds = compbasemetrics.NewHistogramVec(
&compbasemetrics.HistogramOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "request_wait_duration_seconds",
@ -104,8 +107,8 @@ var (
},
[]string{priorityLevel, flowSchema, "execute"},
)
apiserverRequestExecutionSeconds = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
apiserverRequestExecutionSeconds = compbasemetrics.NewHistogramVec(
&compbasemetrics.HistogramOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "request_execution_seconds",
@ -114,6 +117,15 @@ var (
},
[]string{priorityLevel, flowSchema},
)
metrics = []compbasemetrics.Registerable{
apiserverRejectedRequestsTotal,
apiserverCurrentInqueueRequests,
apiserverRequestQueueLength,
apiserverRequestConcurrencyLimit,
apiserverCurrentExecutingRequests,
apiserverRequestWaitingSeconds,
apiserverRequestExecutionSeconds,
}
)
// UpdateFlowControlRequestsInQueue updates the value for the # of requests in the specified queues in flow control
@ -133,7 +145,7 @@ func UpdateSharedConcurrencyLimit(priorityLevel string, limit int) {
// AddReject increments the # of rejected requests for flow control
func AddReject(priorityLevel string, reason string) {
apiserverRejectedRequests.WithLabelValues(priorityLevel, reason).Add(1)
apiserverRejectedRequestsTotal.WithLabelValues(priorityLevel, reason).Add(1)
}
// ObserveQueueLength observes the queue length for flow control