diff --git a/controllers/alert_controller.go b/controllers/alert_controller.go index 10b3d52..081ed64 100644 --- a/controllers/alert_controller.go +++ b/controllers/alert_controller.go @@ -18,16 +18,20 @@ package controllers import ( "context" + "fmt" "strings" + "time" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/reference" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/fluxcd/pkg/apis/meta" + "github.com/fluxcd/pkg/runtime/metrics" "github.com/fluxcd/notification-controller/api/v1beta1" ) @@ -35,8 +39,9 @@ import ( // AlertReconciler reconciles a Alert object type AlertReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme + Log logr.Logger + Scheme *runtime.Scheme + MetricsRecorder *metrics.Recorder } // +kubebuilder:rbac:groups=notification.toolkit.fluxcd.io,resources=alerts,verbs=get;list;watch;create;update;patch;delete @@ -44,6 +49,7 @@ type AlertReconciler struct { func (r *AlertReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { ctx := context.Background() + reconcileStart := time.Now() var alert v1beta1.Alert if err := r.Get(ctx, req.NamespacedName, &alert); err != nil { @@ -52,6 +58,15 @@ func (r *AlertReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("controller", strings.ToLower(alert.Kind), "request", req.NamespacedName) + // record reconciliation duration + if r.MetricsRecorder != nil { + objRef, err := reference.GetReference(r.Scheme, &alert) + if err != nil { + return ctrl.Result{}, err + } + defer r.MetricsRecorder.RecordDuration(*objRef, reconcileStart) + } + init := true if c := meta.GetCondition(alert.Status.Conditions, meta.ReadyCondition); c != nil { if c.Status == corev1.ConditionTrue { @@ -75,6 +90,8 @@ func (r *AlertReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { log.Info("Alert initialised") } + r.recordReadiness(alert, false) + return ctrl.Result{}, nil } @@ -83,3 +100,26 @@ func (r *AlertReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&v1beta1.Alert{}). Complete(r) } + +func (r *AlertReconciler) recordReadiness(alert v1beta1.Alert, deleted bool) { + if r.MetricsRecorder == nil { + return + } + + objRef, err := reference.GetReference(r.Scheme, &alert) + if err != nil { + r.Log.WithValues( + strings.ToLower(alert.Kind), + fmt.Sprintf("%s/%s", alert.GetNamespace(), alert.GetName()), + ).Error(err, "unable to record readiness metric") + return + } + if rc := meta.GetCondition(alert.Status.Conditions, meta.ReadyCondition); rc != nil { + r.MetricsRecorder.RecordCondition(*objRef, *rc, deleted) + } else { + r.MetricsRecorder.RecordCondition(*objRef, meta.Condition{ + Type: meta.ReadyCondition, + Status: corev1.ConditionUnknown, + }, deleted) + } +} diff --git a/controllers/provider_controller.go b/controllers/provider_controller.go index 901d92a..0ec398e 100644 --- a/controllers/provider_controller.go +++ b/controllers/provider_controller.go @@ -18,16 +18,20 @@ package controllers import ( "context" + "fmt" "strings" + "time" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/reference" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/fluxcd/pkg/apis/meta" + "github.com/fluxcd/pkg/runtime/metrics" "github.com/fluxcd/notification-controller/api/v1beta1" ) @@ -35,8 +39,9 @@ import ( // ProviderReconciler reconciles a Provider object type ProviderReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme + Log logr.Logger + Scheme *runtime.Scheme + MetricsRecorder *metrics.Recorder } // +kubebuilder:rbac:groups=notification.toolkit.fluxcd.io,resources=providers,verbs=get;list;watch;create;update;patch;delete @@ -44,6 +49,7 @@ type ProviderReconciler struct { func (r *ProviderReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { ctx := context.Background() + reconcileStart := time.Now() var provider v1beta1.Provider if err := r.Get(ctx, req.NamespacedName, &provider); err != nil { @@ -52,6 +58,15 @@ func (r *ProviderReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("controller", strings.ToLower(provider.Kind), "request", req.NamespacedName) + // record reconciliation duration + if r.MetricsRecorder != nil { + objRef, err := reference.GetReference(r.Scheme, &provider) + if err != nil { + return ctrl.Result{}, err + } + defer r.MetricsRecorder.RecordDuration(*objRef, reconcileStart) + } + init := true if c := meta.GetCondition(provider.Status.Conditions, meta.ReadyCondition); c != nil { if c.Status == corev1.ConditionTrue { @@ -83,3 +98,26 @@ func (r *ProviderReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&v1beta1.Provider{}). Complete(r) } + +func (r *ProviderReconciler) recordReadiness(provider v1beta1.Provider, deleted bool) { + if r.MetricsRecorder == nil { + return + } + + objRef, err := reference.GetReference(r.Scheme, &provider) + if err != nil { + r.Log.WithValues( + strings.ToLower(provider.Kind), + fmt.Sprintf("%s/%s", provider.GetNamespace(), provider.GetName()), + ).Error(err, "unable to record readiness metric") + return + } + if rc := meta.GetCondition(provider.Status.Conditions, meta.ReadyCondition); rc != nil { + r.MetricsRecorder.RecordCondition(*objRef, *rc, deleted) + } else { + r.MetricsRecorder.RecordCondition(*objRef, meta.Condition{ + Type: meta.ReadyCondition, + Status: corev1.ConditionUnknown, + }, deleted) + } +} diff --git a/controllers/receiver_controller.go b/controllers/receiver_controller.go index 00e6c61..73f4b39 100644 --- a/controllers/receiver_controller.go +++ b/controllers/receiver_controller.go @@ -30,6 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/fluxcd/pkg/apis/meta" + "github.com/fluxcd/pkg/runtime/metrics" "github.com/fluxcd/notification-controller/api/v1beta1" ) @@ -37,8 +38,9 @@ import ( // ReceiverReconciler reconciles a Receiver object type ReceiverReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme + Log logr.Logger + Scheme *runtime.Scheme + MetricsRecorder *metrics.Recorder } // +kubebuilder:rbac:groups=notification.toolkit.fluxcd.io,resources=receivers,verbs=get;list;watch;create;update;patch;delete diff --git a/go.mod b/go.mod index 27742ba..c255def 100644 --- a/go.mod +++ b/go.mod @@ -8,11 +8,11 @@ require ( github.com/fluxcd/notification-controller/api v0.1.0 github.com/fluxcd/pkg/apis/meta v0.0.2 github.com/fluxcd/pkg/recorder v0.0.6 - github.com/fluxcd/pkg/runtime v0.0.6 + github.com/fluxcd/pkg/runtime v0.1.0 github.com/fluxcd/source-controller/api v0.1.0 github.com/go-logr/logr v0.1.0 github.com/google/go-github/v32 v32.0.0 - github.com/hashicorp/go-retryablehttp v0.6.6 + github.com/hashicorp/go-retryablehttp v0.6.7 github.com/onsi/ginkgo v1.12.1 github.com/onsi/gomega v1.10.1 github.com/stretchr/testify v1.6.1 diff --git a/go.sum b/go.sum index 7de65cf..491e12d 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ github.com/fluxcd/pkg/apis/meta v0.0.2 h1:kyA4Y0IzNjf1joBOnFqpWG7aNDHvtLExZcaHQM github.com/fluxcd/pkg/apis/meta v0.0.2/go.mod h1:nCNps5JJOcEQr3MNDmZqI4o0chjePSUYL6Q2ktDtotU= github.com/fluxcd/pkg/recorder v0.0.6 h1:me/n8syeeGXz50OXoPX3jgIj9AtinvhHdKT9Dy+MbHs= github.com/fluxcd/pkg/recorder v0.0.6/go.mod h1:IfQxfVRSNsWs3B0Yp5B6ObEWwKHILlAx8N7XkoDdhFg= -github.com/fluxcd/pkg/runtime v0.0.6 h1:m7qwr2wRePs1vzVlM0Y88vitXSsv1lb3QCJflRpa3qQ= -github.com/fluxcd/pkg/runtime v0.0.6/go.mod h1:iLjncjktQVpqpb1NsY2fW+UYDFOtVyt+yJrxqrrK8A0= +github.com/fluxcd/pkg/runtime v0.1.0 h1:mCLj5GlQZqWtK3tvtZTmfgFOLsTUY1iqg3CmEyS1nRs= +github.com/fluxcd/pkg/runtime v0.1.0/go.mod h1:OXkrYtDLw3GhclbzvnzfSktUyxRmC3FFhXj0tVVaIX8= github.com/fluxcd/source-controller/api v0.1.0 h1:ky3gMs3mnkDl6ClX+7uT2BNxU+sLzW/6a8B/M1KfySw= github.com/fluxcd/source-controller/api v0.1.0/go.mod h1:1ac/vj49YVPKF+xBHTo/9pfFj64TcLc1RLaxi4MwVEM= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -190,6 +190,8 @@ github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrj github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-retryablehttp v0.6.6 h1:HJunrbHTDDbBb/ay4kxa1n+dLmttUlnP3V9oNE4hmsM= github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-retryablehttp v0.6.7 h1:8/CAEZt/+F7kR7GevNHulKkUjLht3CPmn7egmhieNKo= +github.com/hashicorp/go-retryablehttp v0.6.7/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= diff --git a/main.go b/main.go index 473a764..d7b269f 100644 --- a/main.go +++ b/main.go @@ -24,12 +24,15 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" ctrl "sigs.k8s.io/controller-runtime" + crtlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" + + "github.com/fluxcd/pkg/runtime/logger" + "github.com/fluxcd/pkg/runtime/metrics" + sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" "github.com/fluxcd/notification-controller/api/v1beta1" "github.com/fluxcd/notification-controller/controllers" "github.com/fluxcd/notification-controller/internal/server" - "github.com/fluxcd/pkg/runtime/logger" - sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" // +kubebuilder:scaffold:imports ) @@ -74,6 +77,9 @@ func main() { zapLogger := logger.NewLogger(logLevel, logJSON) ctrl.SetLogger(zapLogger) + metricsRecorder := metrics.NewRecorder() + crtlmetrics.Registry.MustRegister(metricsRecorder.Collectors()...) + watchNamespace := "" if !watchAllNamespaces { watchNamespace = os.Getenv("RUNTIME_NAMESPACE") @@ -94,25 +100,28 @@ func main() { } if err = (&controllers.ProviderReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("Provider"), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("Provider"), + Scheme: mgr.GetScheme(), + MetricsRecorder: metricsRecorder, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Provider") os.Exit(1) } if err = (&controllers.AlertReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("Alert"), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("Alert"), + Scheme: mgr.GetScheme(), + MetricsRecorder: metricsRecorder, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Alert") os.Exit(1) } if err = (&controllers.ReceiverReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("Receiver"), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("Receiver"), + Scheme: mgr.GetScheme(), + MetricsRecorder: metricsRecorder, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Receiver") os.Exit(1)