kubernetes
/
kube-state-metrics
mirror of https://github.com/kubernetes/kube-state-metrics.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add pod and node label metrics

This commit is contained in:
Frederic Branczyk 2017-04-28 15:31:04 +02:00
parent b12f83db99
commit 81ba59afd9
No known key found for this signature in database
GPG Key ID: CA14788B1E48B256
5 changed files with 133 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
main.go
View File

@ -28,6 +28,7 @@ import (
"github.com/golang/glog"
"github.com/openshift/origin/pkg/util/proc"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/spf13/pflag"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
@ -47,7 +48,7 @@ var (
"pods": struct{}{},
"nodes": struct{}{},
"resourcequotas": struct{}{},
"limitrange": struct{}{},
"limitrange": struct{}{},
"replicasets": struct{}{},
"replicationcontrollers": struct{}{},
}
@ -57,7 +58,7 @@ var (
"pods": RegisterPodCollector,
"nodes": RegisterNodeCollector,
"resourcequotas": RegisterResourceQuotaCollector,
"limitrange": RegisterLimitRangeCollector,
"limitrange": RegisterLimitRangeCollector,
"replicasets": RegisterReplicaSetCollector,
"replicationcontrollers": RegisterReplicationControllerCollector,
}
@ -160,8 +161,9 @@ func main() {
glog.Fatalf("Failed to create client: %v", err)
}
registerCollectors(kubeClient, collectors)
metricsServer(options.port)
registry := prometheus.NewRegistry()
registerCollectors(registry, kubeClient, collectors)
metricsServer(registry, options.port)
}
func isNotExists(file string) bool {
@ -224,13 +226,13 @@ func createKubeClient(inCluster bool, apiserver string, kubeconfig string) (kube
return kubeClient, nil
}
func metricsServer(port int) {
func metricsServer(registry prometheus.Gatherer, port int) {
// Address to listen on for web interface and telemetry
listenAddress := fmt.Sprintf(":%d", port)
glog.Infof("Starting metrics server: %s", listenAddress)
// Add metricsPath
http.Handle(metricsPath, prometheus.UninstrumentedHandler())
http.Handle(metricsPath, promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
// Add healthzPath
http.HandleFunc(healthzPath, func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(200)
@ -254,12 +256,12 @@ func metricsServer(port int) {
// registerCollectors creates and starts informers and initializes and
// registers metrics for collection.
func registerCollectors(kubeClient clientset.Interface, enabledCollectors collectorSet) {
func registerCollectors(registry prometheus.Registerer, kubeClient clientset.Interface, enabledCollectors collectorSet) {
activeCollectors := []string{}
for c, _ := range enabledCollectors {
f, ok := availableCollectors[c]
if ok {
f(prometheus.DefaultRegisterer, kubeClient)
f(registry, kubeClient)
activeCollectors = append(activeCollectors, c)
}
}

22
node.go
View File

@ -27,6 +27,10 @@ import (
)
var (
descNodeLabelsName = "kube_node_labels"
descNodeLabelsHelp = "Kubernetes labels converted to Prometheus labels."
descNodeLabelsDefaultLabels = []string{"node"}
descNodeInfo = prometheus.NewDesc(
"kube_node_info",
"Information about a cluster node.",
@ -40,6 +44,12 @@ var (
}, nil,
)
descNodeLabels = prometheus.NewDesc(
descNodeLabelsName,
descNodeLabelsHelp,
descNodeLabelsDefaultLabels, nil,
)
descNodeSpecUnschedulable = prometheus.NewDesc(
"kube_node_spec_unschedulable",
"Whether a node can schedule new pods.",
@ -144,6 +154,7 @@ type nodeCollector struct {
// Describe implements the prometheus.Collector interface.
func (nc *nodeCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- descNodeInfo
ch <- descNodeLabels
ch <- descNodeSpecUnschedulable
ch <- descNodeStatusReady
ch <- descNodeStatusMemoryPressure
@ -171,6 +182,15 @@ func (nc *nodeCollector) Collect(ch chan<- prometheus.Metric) {
}
}
func nodeLabelsDesc(labelKeys []string) *prometheus.Desc {
return prometheus.NewDesc(
descNodeLabelsName,
descNodeLabelsHelp,
append(descNodeLabelsDefaultLabels, labelKeys...),
nil,
)
}
func (nc *nodeCollector) collectNode(ch chan<- prometheus.Metric, n v1.Node) {
addGauge := func(desc *prometheus.Desc, v float64, lv ...string) {
lv = append([]string{n.Name}, lv...)
@ -185,6 +205,8 @@ func (nc *nodeCollector) collectNode(ch chan<- prometheus.Metric, n v1.Node) {
n.Status.NodeInfo.KubeletVersion,
n.Status.NodeInfo.KubeProxyVersion,
)
labelKeys, labelValues := kubeLabelsToPrometheusLabels(n.Labels)
addGauge(nodeLabelsDesc(labelKeys), 1, labelValues...)
addGauge(descNodeSpecUnschedulable, boolFloat64(n.Spec.Unschedulable))

19
node_test.go
View File

@ -37,6 +37,8 @@ func TestNodeCollector(t *testing.T) {
const metadata = `
# HELP kube_node_info Information about a cluster node.
# TYPE kube_node_info gauge
# HELP kube_node_labels Kubernetes labels converted to Prometheus labels.
# TYPE kube_node_labels gauge
# HELP kube_node_spec_unschedulable Whether a node can schedule new pods.
# TYPE kube_node_spec_unschedulable gauge
# HELP kube_node_status_ready The ready status of a cluster node.
@ -55,12 +57,12 @@ func TestNodeCollector(t *testing.T) {
# HELP kube_node_status_allocatable_cpu_cores The CPU resources of a node that are available for scheduling.
# TYPE kube_node_status_allocatable_memory_bytes gauge
# HELP kube_node_status_allocatable_memory_bytes The memory resources of a node that are available for scheduling.
# HELP kube_node_status_memory_pressure Whether the kubelet is under pressure due to insufficient available memory.
# TYPE kube_node_status_memory_pressure gauge
# HELP kube_node_status_disk_pressure Whether the kubelet is under pressure due to insufficient available disk.
# TYPE kube_node_status_disk_pressure gauge
# HELP kube_node_status_network_unavailable Whether the network is correctly configured for the node.
# TYPE kube_node_status_network_unavailable gauge
# HELP kube_node_status_memory_pressure Whether the kubelet is under pressure due to insufficient available memory.
# TYPE kube_node_status_memory_pressure gauge
# HELP kube_node_status_disk_pressure Whether the kubelet is under pressure due to insufficient available disk.
# TYPE kube_node_status_disk_pressure gauge
# HELP kube_node_status_network_unavailable Whether the network is correctly configured for the node.
# TYPE kube_node_status_network_unavailable gauge
`
cases := []struct {
nodes []v1.Node
@ -87,6 +89,7 @@ func TestNodeCollector(t *testing.T) {
},
want: metadata + `
kube_node_info{container_runtime_version="rkt",kernel_version="kernel",kubelet_version="kubelet",kubeproxy_version="kubeproxy",node="127.0.0.1",os_image="osimage"} 1
kube_node_labels{node="127.0.0.1"} 1
kube_node_spec_unschedulable{node="127.0.0.1"} 0
`,
},
@ -96,6 +99,9 @@ func TestNodeCollector(t *testing.T) {
{
ObjectMeta: v1.ObjectMeta{
Name: "127.0.0.1",
Labels: map[string]string{
"type": "master",
},
},
Spec: v1.NodeSpec{
Unschedulable: true,
@ -123,6 +129,7 @@ func TestNodeCollector(t *testing.T) {
},
want: metadata + `
kube_node_info{container_runtime_version="rkt",kernel_version="kernel",kubelet_version="kubelet",kubeproxy_version="kubeproxy",node="127.0.0.1",os_image="osimage"} 1
kube_node_labels{label_type="master",node="127.0.0.1"} 1
kube_node_spec_unschedulable{node="127.0.0.1"} 1
kube_node_status_capacity_cpu_cores{node="127.0.0.1"} 4.3
kube_node_status_capacity_memory_bytes{node="127.0.0.1"} 2e9

53
pod.go
View File

@ -18,8 +18,9 @@ package main
import (
"encoding/json"
"fmt"
"fmt"
"regexp"
"github.com/golang/glog"
"github.com/prometheus/client_golang/prometheus"
"golang.org/x/net/context"
@ -30,51 +31,71 @@ import (
)
var (
invalidLabelCharRE = regexp.MustCompile(`[^a-zA-Z0-9_]`)
descPodLabelsName = "kube_pod_labels"
descPodLabelsHelp = "Kubernetes labels converted to Prometheus labels."
descPodLabelsDefaultLabels = []string{"namespace", "pod"}
descPodInfo = prometheus.NewDesc(
"kube_pod_info",
"Information about pod.",
[]string{"namespace", "pod", "host_ip", "pod_ip", "node", "created_by"}, nil,
)
descPodLabels = prometheus.NewDesc(
descPodLabelsName,
descPodLabelsHelp,
descPodLabelsDefaultLabels, nil,
)
descPodStatusPhase = prometheus.NewDesc(
"kube_pod_status_phase",
"The pods current phase.",
[]string{"namespace", "pod", "phase"}, nil,
)
descPodStatusReady = prometheus.NewDesc(
"kube_pod_status_ready",
"Describes whether the pod is ready to serve requests.",
[]string{"namespace", "pod", "condition"}, nil,
)
descPodStatusScheduled = prometheus.NewDesc(
"kube_pod_status_scheduled",
"Describes the status of the scheduling process for the pod.",
[]string{"namespace", "pod", "condition"}, nil,
)
descPodContainerInfo = prometheus.NewDesc(
"kube_pod_container_info",
"Information about a container in a pod.",
[]string{"namespace", "pod", "container", "image", "image_id", "container_id"}, nil,
)
descPodContainerStatusWaiting = prometheus.NewDesc(
"kube_pod_container_status_waiting",
"Describes whether the container is currently in waiting state.",
[]string{"namespace", "pod", "container"}, nil,
)
descPodContainerStatusRunning = prometheus.NewDesc(
"kube_pod_container_status_running",
"Describes whether the container is currently in running state.",
[]string{"namespace", "pod", "container"}, nil,
)
descPodContainerStatusTerminated = prometheus.NewDesc(
"kube_pod_container_status_terminated",
"Describes whether the container is currently in terminated state.",
[]string{"namespace", "pod", "container"}, nil,
)
descPodContainerStatusReady = prometheus.NewDesc(
"kube_pod_container_status_ready",
"Describes whether the containers readiness check succeeded.",
[]string{"namespace", "pod", "container"}, nil,
)
descPodContainerStatusRestarts = prometheus.NewDesc(
"kube_pod_container_status_restarts",
"The number of container restarts per container.",
@ -140,6 +161,7 @@ type podCollector struct {
// Describe implements the prometheus.Collector interface.
func (pc *podCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- descPodInfo
ch <- descPodLabels
ch <- descPodStatusPhase
ch <- descPodStatusReady
ch <- descPodStatusScheduled
@ -179,6 +201,31 @@ func (pc *podCollector) Collect(ch chan<- prometheus.Metric) {
}
}
func kubeLabelsToPrometheusLabels(labels map[string]string) ([]string, []string) {
labelKeys := make([]string, len(labels))
labelValues := make([]string, len(labels))
i := 0
for k, v := range labels {
labelKeys[i] = "label_" + sanitizeLabelName(k)
labelValues[i] = v
i++
}
return labelKeys, labelValues
}
func sanitizeLabelName(s string) string {
return invalidLabelCharRE.ReplaceAllString(s, "_")
}
func podLabelsDesc(labelKeys []string) *prometheus.Desc {
return prometheus.NewDesc(
descPodLabelsName,
descPodLabelsHelp,
append(descPodLabelsDefaultLabels, labelKeys...),
nil,
)
}
func (pc *podCollector) collectPod(ch chan<- prometheus.Metric, p v1.Pod) {
nodeName := p.Spec.NodeName
addConstMetric := func(desc *prometheus.Desc, t prometheus.ValueType, v float64, lv ...string) {
@ -193,6 +240,8 @@ func (pc *podCollector) collectPod(ch chan<- prometheus.Metric, p v1.Pod) {
}
addGauge(descPodInfo, 1, p.Status.HostIP, p.Status.PodIP, nodeName, extractCreatedBy(p.Annotations))
labelKeys, labelValues := kubeLabelsToPrometheusLabels(p.Labels)
addGauge(podLabelsDesc(labelKeys), 1, labelValues...)
addGauge(descPodStatusPhase, 1, string(p.Status.Phase))
for _, c := range p.Status.Conditions {

53
pod_test.go
View File

@ -37,6 +37,8 @@ func TestPodCollector(t *testing.T) {
const metadata = `
# HELP kube_pod_container_info Information about a container in a pod.
# TYPE kube_pod_container_info gauge
# HELP kube_pod_labels Kubernetes labels converted to Prometheus labels.
# TYPE kube_pod_labels gauge
# HELP kube_pod_container_status_ready Describes whether the containers readiness check succeeded.
# TYPE kube_pod_container_status_ready gauge
# HELP kube_pod_container_status_restarts The number of container restarts per container.
@ -468,22 +470,22 @@ func TestPodCollector(t *testing.T) {
},
},
want: metadata + `
kube_pod_container_resource_requests_cpu_cores{container="pod1_con1",namespace="ns1",node="node1",pod="pod1"} 0.2
kube_pod_container_resource_requests_cpu_cores{container="pod1_con2",namespace="ns1",node="node1",pod="pod1"} 0.3
kube_pod_container_resource_requests_cpu_cores{container="pod2_con1",namespace="ns2",node="node2",pod="pod2"} 0.4
kube_pod_container_resource_requests_cpu_cores{container="pod2_con2",namespace="ns2",node="node2",pod="pod2"} 0.5
kube_pod_container_resource_requests_memory_bytes{container="pod1_con1",namespace="ns1",node="node1",pod="pod1"} 1e+08
kube_pod_container_resource_requests_memory_bytes{container="pod1_con2",namespace="ns1",node="node1",pod="pod1"} 2e+08
kube_pod_container_resource_requests_memory_bytes{container="pod2_con1",namespace="ns2",node="node2",pod="pod2"} 3e+08
kube_pod_container_resource_requests_memory_bytes{container="pod2_con2",namespace="ns2",node="node2",pod="pod2"} 4e+08
kube_pod_container_resource_limits_cpu_cores{container="pod1_con1",namespace="ns1",node="node1",pod="pod1"} 0.2
kube_pod_container_resource_limits_cpu_cores{container="pod1_con2",namespace="ns1",node="node1",pod="pod1"} 0.3
kube_pod_container_resource_limits_cpu_cores{container="pod2_con1",namespace="ns2",node="node2",pod="pod2"} 0.4
kube_pod_container_resource_limits_cpu_cores{container="pod2_con2",namespace="ns2",node="node2",pod="pod2"} 0.5
kube_pod_container_resource_limits_memory_bytes{container="pod1_con1",namespace="ns1",node="node1",pod="pod1"} 1e+08
kube_pod_container_resource_limits_memory_bytes{container="pod1_con2",namespace="ns1",node="node1",pod="pod1"} 2e+08
kube_pod_container_resource_limits_memory_bytes{container="pod2_con1",namespace="ns2",node="node2",pod="pod2"} 3e+08
kube_pod_container_resource_limits_memory_bytes{container="pod2_con2",namespace="ns2",node="node2",pod="pod2"} 4e+08
kube_pod_container_resource_requests_cpu_cores{container="pod1_con1",namespace="ns1",node="node1",pod="pod1"} 0.2
kube_pod_container_resource_requests_cpu_cores{container="pod1_con2",namespace="ns1",node="node1",pod="pod1"} 0.3
kube_pod_container_resource_requests_cpu_cores{container="pod2_con1",namespace="ns2",node="node2",pod="pod2"} 0.4
kube_pod_container_resource_requests_cpu_cores{container="pod2_con2",namespace="ns2",node="node2",pod="pod2"} 0.5
kube_pod_container_resource_requests_memory_bytes{container="pod1_con1",namespace="ns1",node="node1",pod="pod1"} 1e+08
kube_pod_container_resource_requests_memory_bytes{container="pod1_con2",namespace="ns1",node="node1",pod="pod1"} 2e+08
kube_pod_container_resource_requests_memory_bytes{container="pod2_con1",namespace="ns2",node="node2",pod="pod2"} 3e+08
kube_pod_container_resource_requests_memory_bytes{container="pod2_con2",namespace="ns2",node="node2",pod="pod2"} 4e+08
kube_pod_container_resource_limits_cpu_cores{container="pod1_con1",namespace="ns1",node="node1",pod="pod1"} 0.2
kube_pod_container_resource_limits_cpu_cores{container="pod1_con2",namespace="ns1",node="node1",pod="pod1"} 0.3
kube_pod_container_resource_limits_cpu_cores{container="pod2_con1",namespace="ns2",node="node2",pod="pod2"} 0.4
kube_pod_container_resource_limits_cpu_cores{container="pod2_con2",namespace="ns2",node="node2",pod="pod2"} 0.5
kube_pod_container_resource_limits_memory_bytes{container="pod1_con1",namespace="ns1",node="node1",pod="pod1"} 1e+08
kube_pod_container_resource_limits_memory_bytes{container="pod1_con2",namespace="ns1",node="node1",pod="pod1"} 2e+08
kube_pod_container_resource_limits_memory_bytes{container="pod2_con1",namespace="ns2",node="node2",pod="pod2"} 3e+08
kube_pod_container_resource_limits_memory_bytes{container="pod2_con2",namespace="ns2",node="node2",pod="pod2"} 4e+08
`,
metrics: []string{
"kube_pod_container_resource_requests_cpu_cores",
@ -491,6 +493,25 @@ func TestPodCollector(t *testing.T) {
"kube_pod_container_resource_limits_cpu_cores",
"kube_pod_container_resource_limits_memory_bytes",
},
}, {
pods: []v1.Pod{
{
ObjectMeta: v1.ObjectMeta{
Name: "pod1",
Namespace: "ns1",
Labels: map[string]string{
"app": "example",
},
},
Spec: v1.PodSpec{},
},
},
want: metadata + `
kube_pod_labels{label_app="example",namespace="ns1",pod="pod1"} 1
`,
metrics: []string{
"kube_pod_labels",
},
},
}
for _, c := range cases {