From c522d44f7b7cc9968c19029675c9e9af49847b13 Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Fri, 6 Jul 2018 15:12:58 +0200 Subject: [PATCH] Allow white- and black-listing metrics to be exposed --- CHANGELOG.md | 1 + main.go | 18 ++++- pkg/metrics/metrics.go | 81 +++++++++++++++++++++ pkg/metrics/metrics_test.go | 137 ++++++++++++++++++++++++++++++++++++ pkg/options/options.go | 8 ++- pkg/options/types.go | 37 ++++++++++ 6 files changed, 279 insertions(+), 3 deletions(-) create mode 100644 pkg/metrics/metrics.go create mode 100644 pkg/metrics/metrics_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 1905428c..cf8abb20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ * [CHANGE] `kube_job_status_start_time` and `kube_job_status_completion_time` metric types changed from counter to gauge. * [CHANGE] `job` label to `job_name` as this collides with the Prometheus `job` label. +* [FEATURE] Allow white- and black-listing metrics to be exposed. ## v1.3.1 / 2018-04-12 diff --git a/main.go b/main.go index 88fd9b78..4cfb4edf 100644 --- a/main.go +++ b/main.go @@ -31,10 +31,11 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" clientset "k8s.io/client-go/kubernetes" + _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/client-go/tools/clientcmd" - _ "k8s.io/client-go/plugin/pkg/client/auth" kcollectors "k8s.io/kube-state-metrics/pkg/collectors" + "k8s.io/kube-state-metrics/pkg/metrics" "k8s.io/kube-state-metrics/pkg/options" "k8s.io/kube-state-metrics/pkg/version" ) @@ -91,6 +92,19 @@ func main() { glog.Infof("Using %s namespaces", namespaces) } + if opts.MetricWhitelist.IsEmpty() && opts.MetricBlacklist.IsEmpty() { + glog.Info("No metric whitelist or blacklist set. No filtering of metrics will be done.") + } + if !opts.MetricWhitelist.IsEmpty() && !opts.MetricBlacklist.IsEmpty() { + glog.Fatal("Whitelist and blacklist are both set. They are mutually exclusive, only one of them can be set.") + } + if !opts.MetricWhitelist.IsEmpty() { + glog.Infof("A metric whitelist has been configured. Only the following metrics will be exposed: %s.", opts.MetricWhitelist.String()) + } + if !opts.MetricBlacklist.IsEmpty() { + glog.Infof("A metric blacklist has been configured. The following metrics will not be exposed: %s.", opts.MetricBlacklist.String()) + } + proc.StartReaper() kubeClient, err := createKubeClient(opts.Apiserver, opts.Kubeconfig) @@ -107,7 +121,7 @@ func main() { registry := prometheus.NewRegistry() registerCollectors(registry, kubeClient, collectors, namespaces, opts) - metricsServer(registry, opts.Host, opts.Port) + metricsServer(metrics.FilteredGatherer(registry, opts.MetricWhitelist, opts.MetricBlacklist), opts.Host, opts.Port) } func createKubeClient(apiserver string, kubeconfig string) (clientset.Interface, error) { diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 00000000..926dd369 --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,81 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + + "k8s.io/kube-state-metrics/pkg/options" +) + +type gathererFunc func() ([]*dto.MetricFamily, error) + +func (f gathererFunc) Gather() ([]*dto.MetricFamily, error) { + return f() +} + +// FilteredGatherer wraps a prometheus.Gatherer to filter metrics based on a +// white or blacklist. Whitelist and blacklist are mutually exclusive. +func FilteredGatherer(r prometheus.Gatherer, whitelist options.MetricSet, blacklist options.MetricSet) prometheus.Gatherer { + whitelistEnabled := !whitelist.IsEmpty() + blacklistEnabled := !blacklist.IsEmpty() + + if whitelistEnabled { + return gathererFunc(func() ([]*dto.MetricFamily, error) { + metricFamilies, err := r.Gather() + if err != nil { + return nil, err + } + + newMetricFamilies := []*dto.MetricFamily{} + for _, metricFamily := range metricFamilies { + // deferencing this string may be a performance bottleneck + name := *metricFamily.Name + _, onWhitelist := whitelist[name] + if onWhitelist { + newMetricFamilies = append(newMetricFamilies, metricFamily) + } + } + + return newMetricFamilies, nil + }) + } + + if blacklistEnabled { + return gathererFunc(func() ([]*dto.MetricFamily, error) { + metricFamilies, err := r.Gather() + if err != nil { + return nil, err + } + + newMetricFamilies := []*dto.MetricFamily{} + for _, metricFamily := range metricFamilies { + name := *metricFamily.Name + _, onBlacklist := blacklist[name] + if onBlacklist { + continue + } + newMetricFamilies = append(newMetricFamilies, metricFamily) + } + + return newMetricFamilies, nil + }) + } + + return r +} diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go new file mode 100644 index 00000000..9b4c9acb --- /dev/null +++ b/pkg/metrics/metrics_test.go @@ -0,0 +1,137 @@ +package metrics + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + "k8s.io/kube-state-metrics/pkg/options" +) + +func TestFiltererdGatherer(t *testing.T) { + r := prometheus.NewRegistry() + c1 := prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "test1", + Help: "test1 help", + }, + ) + c2 := prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "test2", + Help: "test2 help", + }, + ) + c1.Inc() + c1.Inc() + c2.Inc() + r.MustRegister(c1) + r.MustRegister(c2) + + res, err := FilteredGatherer(r, nil, nil).Gather() + if err != nil { + t.Fatal(err) + } + + found1 := false + found2 := false + for _, mf := range res { + if *mf.Name == "test1" { + found1 = true + } + if *mf.Name == "test2" { + found2 = true + } + } + + if !found1 || !found2 { + t.Fatal("No results expected to be filtered, but results were filtered.") + } +} + +func TestFiltererdGathererWhitelist(t *testing.T) { + r := prometheus.NewRegistry() + c1 := prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "test1", + Help: "test1 help", + }, + ) + c2 := prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "test2", + Help: "test2 help", + }, + ) + c1.Inc() + c1.Inc() + c2.Inc() + r.MustRegister(c1) + r.MustRegister(c2) + + whitelist := options.MetricSet{} + whitelist.Set("test1") + + res, err := FilteredGatherer(r, whitelist, nil).Gather() + if err != nil { + t.Fatal(err) + } + + found1 := false + found2 := false + for _, mf := range res { + if *mf.Name == "test1" { + found1 = true + } + if *mf.Name == "test2" { + found2 = true + } + } + + if !found1 || found2 { + t.Fatalf("Expected `test2` to be filtered and `test1` not. `test1`: %t ; `test2`: %t.", found1, found2) + } +} + +func TestFiltererdGathererBlacklist(t *testing.T) { + r := prometheus.NewRegistry() + c1 := prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "test1", + Help: "test1 help", + }, + ) + c2 := prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "test2", + Help: "test2 help", + }, + ) + c1.Inc() + c1.Inc() + c2.Inc() + r.MustRegister(c1) + r.MustRegister(c2) + + blacklist := options.MetricSet{} + blacklist.Set("test1") + + res, err := FilteredGatherer(r, nil, blacklist).Gather() + if err != nil { + t.Fatal(err) + } + + found1 := false + found2 := false + for _, mf := range res { + if *mf.Name == "test1" { + found1 = true + } + if *mf.Name == "test2" { + found2 = true + } + } + + if found1 || !found2 { + t.Fatalf("Expected `test1` to be filtered and `test2` not. `test1`: %t ; `test2`: %t.", found1, found2) + } +} diff --git a/pkg/options/options.go b/pkg/options/options.go index f093c744..87e1f0c3 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -34,6 +34,8 @@ type Options struct { TelemetryHost string Collectors CollectorSet Namespaces NamespaceList + MetricBlacklist MetricSet + MetricWhitelist MetricSet Version bool DisablePodNonGenericResourceMetrics bool DisableNodeNonGenericResourceMetrics bool @@ -43,7 +45,9 @@ type Options struct { func NewOptions() *Options { return &Options{ - Collectors: CollectorSet{}, + Collectors: CollectorSet{}, + MetricWhitelist: MetricSet{}, + MetricBlacklist: MetricSet{}, } } @@ -69,6 +73,8 @@ func (o *Options) AddFlags() { o.flags.StringVar(&o.TelemetryHost, "telemetry-host", "0.0.0.0", `Host to expose kube-state-metrics self metrics on.`) o.flags.Var(&o.Collectors, "collectors", fmt.Sprintf("Comma-separated list of collectors to be enabled. Defaults to %q", &DefaultCollectors)) o.flags.Var(&o.Namespaces, "namespace", fmt.Sprintf("Comma-separated list of namespaces to be enabled. Defaults to %q", &DefaultNamespaces)) + o.flags.Var(&o.MetricWhitelist, "metric-whitelist", "Comma-separated list of metrics to be exposed. The whitelist and blacklist are mutually exclusive.") + o.flags.Var(&o.MetricBlacklist, "metric-blacklist", "Comma-separated list of metrics not to be enabled. The whitelist and blacklist are mutually exclusive.") o.flags.BoolVarP(&o.Version, "version", "", false, "kube-state-metrics build version information") o.flags.BoolVarP(&o.DisablePodNonGenericResourceMetrics, "disable-pod-non-generic-resource-metrics", "", false, "Disable pod non generic resource request and limit metrics") o.flags.BoolVarP(&o.DisableNodeNonGenericResourceMetrics, "disable-node-non-generic-resource-metrics", "", false, "Disable node non generic resource request and limit metrics") diff --git a/pkg/options/types.go b/pkg/options/types.go index 848ef5da..970d985f 100644 --- a/pkg/options/types.go +++ b/pkg/options/types.go @@ -25,6 +25,43 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +type MetricSet map[string]struct{} + +func (ms *MetricSet) String() string { + s := *ms + ss := s.asSlice() + sort.Strings(ss) + return strings.Join(ss, ",") +} + +func (ms *MetricSet) Set(value string) error { + s := *ms + metrics := strings.Split(value, ",") + for _, metric := range metrics { + metric = strings.TrimSpace(metric) + if len(metric) != 0 { + s[metric] = struct{}{} + } + } + return nil +} + +func (ms MetricSet) asSlice() []string { + metrics := []string{} + for metric := range ms { + metrics = append(metrics, metric) + } + return metrics +} + +func (ms MetricSet) IsEmpty() bool { + return len(ms.asSlice()) == 0 +} + +func (ms *MetricSet) Type() string { + return "string" +} + type CollectorSet map[string]struct{} func (c *CollectorSet) String() string {