Merge pull request #1916 from mrueg/config-change-metrics

Add metrics for config file changes

README.md

@@ -165,6 +165,14 @@ please check the blog post [here](https://www.robustperception.io/exposing-the-s
 Sharding metrics expose `--shard` and `--total-shards` flags and can be used to validate
 run-time configuration, see [`/examples/prometheus-alerting-rules`](./examples/prometheus-alerting-rules).
 
+kube-state-metrics also exposes metrics about it config file: 
+
+```
+kube_state_metrics_config_hash{type="config", filename="config.yml"} 4.0061079457904e+13
+kube_state_metrics_config_last_reload_success_timestamp_seconds{type="config", filename="config.yml"} 1.6697483049487052e+09
+kube_state_metrics_config_last_reload_successful{type="config", filename="config.yml"} 1
+```
+
 ### Scaling kube-state-metrics
 
 #### Resource recommendation

internal/wrapper.go

@@ -54,7 +54,7 @@ func RunKubeStateMetricsWrapper(opts *options.Options) {
 		}
 	}
 	ctx, cancel := context.WithCancel(context.Background())
-	if file := options.GetOptsConfigFile(*opts); file != "" {
+	if file := options.GetConfigFile(*opts); file != "" {
 		viper.SetConfigType("yaml")
 		viper.SetConfigFile(file)
 		if err := viper.ReadInConfig(); err != nil {

pkg/app/server.go

@@ -18,6 +18,8 @@ package app
 
 import (
 	"context"
+	"crypto/md5" //nolint:gosec
+	"encoding/binary"
 	"fmt"
 	"net"
 	"net/http"
@@ -99,16 +101,32 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options, factories .
 			ConstLabels: prometheus.Labels{"handler": "metrics"},
 		}, []string{"method"},
 	)
+	configHash := promauto.With(ksmMetricsRegistry).NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "kube_state_metrics_config_hash",
+			Help: "Hash of the currently loaded configuration.",
+		}, []string{"type", "filename"})
+	configSuccess := promauto.With(ksmMetricsRegistry).NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "kube_state_metrics_last_config_reload_successful",
+			Help: "Whether the last configuration reload attempt was successful.",
+		}, []string{"type", "filename"})
+	configSuccessTime := promauto.With(ksmMetricsRegistry).NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "kube_state_metrics_last_config_reload_success_timestamp_seconds",
+			Help: "Timestamp of the last successful configuration reload.",
+		}, []string{"type", "filename"})
+
 	storeBuilder.WithMetrics(ksmMetricsRegistry)
 
-	got := options.GetOptsConfigFile(*opts)
+	got := options.GetConfigFile(*opts)
 	if got != "" {
-		optsConfigFile, err := os.ReadFile(filepath.Clean(got))
+		configFile, err := os.ReadFile(filepath.Clean(got))
 		if err != nil {
 			return fmt.Errorf("failed to read opts config file: %v", err)
 		}
 		// NOTE: Config value will override default values of intersecting options.
-		err = yaml.Unmarshal(optsConfigFile, opts)
+		err = yaml.Unmarshal(configFile, opts)
 		if err != nil {
 			// DO NOT end the process.
 			// We want to allow the user to still be able to fix the misconfigured config (redeploy or edit the configmaps) and reload KSM automatically once that's done.
@@ -116,7 +134,13 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options, factories .
 			// Wait for the next reload.
 			klog.Infof("misconfigured config detected, KSM will automatically reload on next write to the config")
 			klog.Infof("waiting for config to be fixed")
+			configSuccess.WithLabelValues("config", filepath.Clean(got)).Set(0)
 			<-ctx.Done()
+		} else {
+			configSuccess.WithLabelValues("config", filepath.Clean(got)).Set(1)
+			configSuccessTime.WithLabelValues("config", filepath.Clean(got)).SetToCurrentTime()
+			hash := md5HashAsMetricValue(configFile)
+			configHash.WithLabelValues("config", filepath.Clean(got)).Set(hash)
 		}
 	}
 	var resources []string
@@ -371,3 +395,14 @@ func buildMetricsServer(m *metricshandler.MetricsHandler, durationObserver prome
 	})
 	return mux
 }
+
+// md5HashAsMetricValue creates an md5 hash and returns the most significant bytes that fit into a float64
+// Taken from https://github.com/prometheus/alertmanager/blob/6ef6e6868dbeb7984d2d577dd4bf75c65bf1904f/config/coordinator.go#L149
+func md5HashAsMetricValue(data []byte) float64 {
+	sum := md5.Sum(data) //nolint:gosec
+	// We only want 48 bits as a float64 only has a 53 bit mantissa.
+	smallSum := sum[0:6]
+	bytes := make([]byte, 8)
+	copy(bytes, smallSum)
+	return float64(binary.LittleEndian.Uint64(bytes))
+}

pkg/options/options.go

@@ -62,8 +62,8 @@ type Options struct {
 	cmd *cobra.Command
 }
 
-// GetOptsConfigFile is the getter for --options-config-file value.
-func GetOptsConfigFile(opt Options) string {
+// GetConfigFile is the getter for --config value.
+func GetConfigFile(opt Options) string {
 	return opt.Config
 }