Record the ready status as Prometheus metric

Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>

controllers/kustomization_controller.go

@@ -43,6 +43,7 @@ import (
 	"sigs.k8s.io/kustomize/api/krusty"
 	kustypes "sigs.k8s.io/kustomize/api/types"
 
+	"github.com/fluxcd/kustomize-controller/internal/metrics"
 	"github.com/fluxcd/pkg/apis/meta"
 	"github.com/fluxcd/pkg/recorder"
 	"github.com/fluxcd/pkg/runtime/predicates"
@@ -60,6 +61,7 @@ type KustomizationReconciler struct {
 	Scheme                *runtime.Scheme
 	EventRecorder         kuberecorder.EventRecorder
 	ExternalEventRecorder *recorder.EventRecorder
+	MetricsRecorder       *metrics.Recorder
 	StatusPoller          *polling.StatusPoller
 }
 
@@ -101,6 +103,9 @@ func (r *KustomizationReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
 					return ctrl.Result{}, err
 				}
 			}
+			// Record deleted status
+			r.recordReadiness(kustomization, true)
+
 			// Remove our finalizer from the list and update it
 			kustomization.ObjectMeta.Finalizers = removeString(kustomization.ObjectMeta.Finalizers, kustomizev1.KustomizationFinalizer)
 			if err := r.Update(ctx, &kustomization); err != nil {
@@ -118,6 +123,7 @@ func (r *KustomizationReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
 			log.Error(err, "unable to update status")
 			return ctrl.Result{Requeue: true}, err
 		}
+		r.recordReadiness(kustomization, false)
 		log.Info(msg)
 		return ctrl.Result{}, nil
 	}
@@ -142,6 +148,7 @@ func (r *KustomizationReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
 			log.Error(err, "unable to update status")
 			return ctrl.Result{Requeue: true}, err
 		}
+		r.recordReadiness(kustomization, false)
 		log.Info(msg)
 		return ctrl.Result{}, nil
 	}
@@ -160,6 +167,7 @@ func (r *KustomizationReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
 			msg := fmt.Sprintf("Dependencies do not meet ready condition, retrying in %s", r.requeueDependency.String())
 			log.Error(err, msg)
 			r.event(kustomization, source.GetArtifact().Revision, recorder.EventSeverityInfo, msg, nil)
+			r.recordReadiness(kustomization, false)
 			return ctrl.Result{RequeueAfter: r.requeueDependency}, nil
 		}
 		log.Info("All dependencies area ready, proceeding with reconciliation")
@@ -188,11 +196,14 @@ func (r *KustomizationReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
 	// requeue
 	if reconcileErr != nil {
 		// record the reconciliation error
+		r.recordReadiness(reconciledKustomization, false)
 		return ctrl.Result{RequeueAfter: kustomization.Spec.Interval.Duration}, reconcileErr
 	}
 
+	// record the reconciliation result
 	r.event(reconciledKustomization, source.GetArtifact().Revision, recorder.EventSeverityInfo,
 		"Update completed", map[string]string{"commit_status": "update"})
+	r.recordReadiness(reconciledKustomization, false)
 	return ctrl.Result{RequeueAfter: kustomization.Spec.Interval.Duration}, nil
 }
 
@@ -768,6 +779,24 @@ func (r *KustomizationReconciler) event(kustomization kustomizev1.Kustomization,
 	}
 }
 
+func (r *KustomizationReconciler) recordReadiness(kustomization kustomizev1.Kustomization, deleted bool) {
+	if r.MetricsRecorder == nil {
+		return
+	}
+
+	objRef, err := reference.GetReference(r.Scheme, &kustomization)
+	if err != nil {
+		r.Log.WithValues(
+			strings.ToLower(kustomization.Kind),
+			fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()),
+		).Error(err, "unable to record readiness metric")
+		return
+	}
+	if rc := meta.GetCondition(kustomization.Status.Conditions, meta.ReadyCondition); rc != nil {
+		r.MetricsRecorder.RecordReadyStatus(*objRef, *rc, deleted)
+	}
+}
+
 func containsString(slice []string, s string) bool {
 	for _, item := range slice {
 		if item == s {

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c
 	github.com/onsi/ginkgo v1.12.1
 	github.com/onsi/gomega v1.10.1
+	github.com/prometheus/client_golang v1.0.0
 	go.mozilla.org/gopgagent v0.0.0-20170926210634-4d7ea76ff71a
 	go.mozilla.org/sops/v3 v3.6.1
 	golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975

internal/metrics/recorder.go

@@ -0,0 +1,62 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	corev1 "k8s.io/api/core/v1"
+
+	"github.com/fluxcd/pkg/apis/meta"
+)
+
+const (
+	ConditionDeleted = "Deleted"
+)
+
+type Recorder struct {
+	readyGauge *prometheus.GaugeVec
+}
+
+func NewRecorder() *Recorder {
+	return &Recorder{
+		readyGauge: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: "",
+				Subsystem: "",
+				Name:      "gitops_toolkit_ready_condition",
+				Help:      "The current ready condition status of a GitOps Toolkit resource.",
+			},
+			[]string{"kind", "name", "namespace", "status"},
+		),
+	}
+}
+
+func (r *Recorder) Collectors() []prometheus.Collector {
+	return []prometheus.Collector{r.readyGauge}
+}
+
+func (r *Recorder) RecordReadyStatus(ref corev1.ObjectReference, condition meta.Condition, deleted bool) {
+	for _, status := range []string{string(corev1.ConditionTrue), string(corev1.ConditionFalse), string(corev1.ConditionUnknown), ConditionDeleted} {
+		var value float64
+		if deleted {
+			if status == ConditionDeleted {
+				value = 1
+			}
+		} else {
+			switch condition.Status {
+			case corev1.ConditionTrue:
+				if status == string(condition.Status) {
+					value = 1
+				}
+			case corev1.ConditionFalse:
+				if status == string(condition.Status) {
+					value = 1
+				}
+			case corev1.ConditionUnknown:
+				if status == string(condition.Status) {
+					value = 1
+				}
+			}
+		}
+
+		r.readyGauge.WithLabelValues(ref.Kind, ref.Name, ref.Namespace, status).Set(value)
+	}
+}

main.go

@@ -26,9 +26,11 @@ import (
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	"sigs.k8s.io/cli-utils/pkg/kstatus/polling"
 	ctrl "sigs.k8s.io/controller-runtime"
+	crtlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1"
 	"github.com/fluxcd/kustomize-controller/controllers"
+	"github.com/fluxcd/kustomize-controller/internal/metrics"
 	"github.com/fluxcd/pkg/recorder"
 	"github.com/fluxcd/pkg/runtime/logger"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
@@ -85,6 +87,9 @@ func main() {
 		}
 	}
 
+	metricsRecorder := metrics.NewRecorder()
+	crtlmetrics.Registry.MustRegister(metricsRecorder.Collectors()...)
+
 	watchNamespace := ""
 	if !watchAllNamespaces {
 		watchNamespace = os.Getenv("RUNTIME_NAMESPACE")
@@ -126,6 +131,7 @@ func main() {
 		Scheme:                mgr.GetScheme(),
 		EventRecorder:         mgr.GetEventRecorderFor("kustomize-controller"),
 		ExternalEventRecorder: eventRecorder,
+		MetricsRecorder:       metricsRecorder,
 		StatusPoller:          polling.NewStatusPoller(mgr.GetClient(), mgr.GetRESTMapper()),
 	}).SetupWithManager(mgr, controllers.KustomizationReconcilerOptions{
 		MaxConcurrentReconciles:   concurrent,