From 385277bef36d891f6f2f6564459fcc1585eb73d1 Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Mon, 23 Nov 2020 11:08:05 +0000 Subject: [PATCH 1/5] Add --watch-all-namespaces flag to controller This is a convention among the GitOps toolkit controllers; if false, the controller will watch only resources in the namespace in which it's running. Signed-off-by: Michael Bridgen --- main.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 33e71fd..ac1e8f3 100644 --- a/main.go +++ b/main.go @@ -49,22 +49,34 @@ func init() { } func main() { - var metricsAddr string - var enableLeaderElection bool + var ( + metricsAddr string + enableLeaderElection bool + watchAllNamespaces bool + ) + flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") + flag.BoolVar(&watchAllNamespaces, "watch-all-namespaces", true, + "Watch for custom resources in all namespaces, if set to false it will only watch the runtime namespace.") flag.Parse() ctrl.SetLogger(zap.New(zap.UseDevMode(true))) + watchNamespace := "" + if !watchAllNamespaces { + watchNamespace = os.Getenv("RUNTIME_NAMESPACE") + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, MetricsBindAddress: metricsAddr, Port: 9443, LeaderElection: enableLeaderElection, LeaderElectionID: "79628f79.fluxcd.io", + Namespace: watchNamespace, }) if err != nil { setupLog.Error(err, "unable to start manager") From 59975f6ad0552fd6b66321da760944107f09680c Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Mon, 23 Nov 2020 11:23:30 +0000 Subject: [PATCH 2/5] Make room for event recorder This gives the controller an event recorder, without using it yet, and makes sure it is registered when setting everything up in main.go. Signed-off-by: Michael Bridgen --- controllers/imageupdateautomation_controller.go | 6 ++++-- main.go | 7 ++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/controllers/imageupdateautomation_controller.go b/controllers/imageupdateautomation_controller.go index d9ec8e7..fb0072e 100644 --- a/controllers/imageupdateautomation_controller.go +++ b/controllers/imageupdateautomation_controller.go @@ -36,6 +36,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + kuberecorder "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -64,8 +65,9 @@ const imagePolicyKey = ".spec.update.imagePolicy" // ImageUpdateAutomationReconciler reconciles a ImageUpdateAutomation object type ImageUpdateAutomationReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme + Log logr.Logger + Scheme *runtime.Scheme + EventRecorder kuberecorder.EventRecorder } // +kubebuilder:rbac:groups=image.toolkit.fluxcd.io,resources=imageupdateautomations,verbs=get;list;watch;create;update;patch;delete diff --git a/main.go b/main.go index ac1e8f3..a9c43a5 100644 --- a/main.go +++ b/main.go @@ -84,9 +84,10 @@ func main() { } if err = (&controllers.ImageUpdateAutomationReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("ImageUpdateAutomation"), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("ImageUpdateAutomation"), + Scheme: mgr.GetScheme(), + EventRecorder: mgr.GetEventRecorderFor("image-automation-controller"), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "ImageUpdateAutomation") os.Exit(1) From c762ce8d6f5f48f8394d18f009468d974d52e667 Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Mon, 23 Nov 2020 12:45:08 +0000 Subject: [PATCH 3/5] Send event on success/failure to commit change The useful events to know about for the update automation are when it either errors out while trying to update the git repo, or succeeds. Signed-off-by: Michael Bridgen --- controllers/imageupdateautomation_controller.go | 13 +++++++++++++ go.mod | 1 + go.sum | 4 ++++ 3 files changed, 18 insertions(+) diff --git a/controllers/imageupdateautomation_controller.go b/controllers/imageupdateautomation_controller.go index fb0072e..944824c 100644 --- a/controllers/imageupdateautomation_controller.go +++ b/controllers/imageupdateautomation_controller.go @@ -44,6 +44,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" imagev1_reflect "github.com/fluxcd/image-reflector-controller/api/v1alpha1" + "github.com/fluxcd/pkg/runtime/events" sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" "github.com/fluxcd/source-controller/pkg/git" @@ -134,6 +135,7 @@ func (r *ImageUpdateAutomationReconciler) Reconcile(req ctrl.Request) (ctrl.Resu } if err := updateAccordingToSetters(ctx, tmp, policies.Items); err != nil { + r.event(auto, events.EventSeverityError, err.Error()) return ctrl.Result{}, err } default: @@ -145,14 +147,17 @@ func (r *ImageUpdateAutomationReconciler) Reconcile(req ctrl.Request) (ctrl.Resu log.V(debug).Info("ran updates to working dir", "working", tmp) var commitMade bool + if rev, err := commitAllAndPush(ctx, repo, access, &auto.Spec.Commit); err != nil { if err == errNoChanges { log.Info("no changes made in working directory; no commit") } else { + r.event(auto, events.EventSeverityError, err.Error()) return ctrl.Result{}, err } } else { commitMade = true + r.event(auto, events.EventSeverityInfo, "committed and pushed change "+rev) log.V(debug).Info("pushed commit to origin", "revision", rev) } @@ -331,6 +336,14 @@ func commitAllAndPush(ctx context.Context, repo *gogit.Repository, access repoAc }) } +// --- events + +func (r *ImageUpdateAutomationReconciler) event(auto imagev1.ImageUpdateAutomation, severity, msg string) { + if r.EventRecorder != nil { + r.EventRecorder.Event(&auto, "Normal", severity, msg) + } +} + // --- updates // updateAccordingToSetters updates files under the root by treating diff --git a/go.mod b/go.mod index aa4a9b9..67ed309 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/fluxcd/image-automation-controller/api v0.0.0-00010101000000-000000000000 github.com/fluxcd/image-reflector-controller v0.0.0-20200810165546-c2265d9b49b9 github.com/fluxcd/pkg/gittestserver v0.0.2 + github.com/fluxcd/pkg/runtime v0.2.0 github.com/fluxcd/source-controller v0.2.2 // If you bump this, change TOOLKIT_VERSION in the Makefile to match github.com/fluxcd/source-controller/api v0.2.2 diff --git a/go.sum b/go.sum index 9f43deb..80b6ba3 100644 --- a/go.sum +++ b/go.sum @@ -295,6 +295,7 @@ github.com/fluxcd/pkg/gittestserver v0.0.2 h1:11aKRVuuHiyeaicdN4wPNSMy/dUarQkrPr github.com/fluxcd/pkg/gittestserver v0.0.2/go.mod h1:GW8N9d1o8/+mXWnSzs02qCB5WlArWQHdMpDPf7b/GZg= github.com/fluxcd/pkg/helmtestserver v0.0.1/go.mod h1:GR8LriiU7PqZSTH4Xe6Cimpig2VVPB29PeUXJjNJYfA= github.com/fluxcd/pkg/lockedfile v0.0.5/go.mod h1:uAtPUBId6a2RqO84MTH5HKGX0SbM1kNW3Wr/FhYyDVA= +github.com/fluxcd/pkg/runtime v0.2.0 h1:aZmSLuyA9pF/KANf4wi7pZIICE19BKTYFSPRbl6WHtY= github.com/fluxcd/pkg/runtime v0.2.0/go.mod h1:P1/S8TOSuJgVPU0SRahWzbNxLWYoUwvBcPCNGc+dWWg= github.com/fluxcd/pkg/ssh v0.0.5 h1:rnbFZ7voy2JBlUfMbfyqArX2FYaLNpDhccGFC3qW83A= github.com/fluxcd/pkg/ssh v0.0.5/go.mod h1:7jXPdXZpc0ttMNz2kD9QuMi3RNn/e0DOFbj0Tij/+Hs= @@ -573,7 +574,9 @@ github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyN github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= @@ -581,6 +584,7 @@ github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1: github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-retryablehttp v0.6.7 h1:8/CAEZt/+F7kR7GevNHulKkUjLht3CPmn7egmhieNKo= github.com/hashicorp/go-retryablehttp v0.6.7/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= From 5bd0bc8cba4bfc147126c4ea0f90b3f0ba44b01d Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Mon, 23 Nov 2020 12:54:15 +0000 Subject: [PATCH 4/5] Set logger up with log level and JSON flags This is the GitOps Toolkit convention. This commit also puts the flags in the base deployment config, in the same order as for other GOTK controllers. Signed-off-by: Michael Bridgen --- config/manager/manager.yaml | 3 +++ main.go | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 1889734..40065a7 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -34,6 +34,9 @@ spec: fieldRef: fieldPath: metadata.namespace args: + - --watch-all-namespaces + - --log-level=info + - --log-json - --enable-leader-election resources: limits: diff --git a/main.go b/main.go index a9c43a5..3e6fca1 100644 --- a/main.go +++ b/main.go @@ -24,9 +24,9 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/log/zap" imagev1alpha1_reflect "github.com/fluxcd/image-reflector-controller/api/v1alpha1" + "github.com/fluxcd/pkg/runtime/logger" sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" imagev1alpha1_auto "github.com/fluxcd/image-automation-controller/api/v1alpha1" @@ -52,6 +52,8 @@ func main() { var ( metricsAddr string enableLeaderElection bool + logLevel string + logJSON bool watchAllNamespaces bool ) @@ -59,11 +61,13 @@ func main() { flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") + flag.StringVar(&logLevel, "log-level", "info", "Set logging level. Can be debug, info or error.") + flag.BoolVar(&logJSON, "log-json", false, "Set logging to JSON format.") flag.BoolVar(&watchAllNamespaces, "watch-all-namespaces", true, "Watch for custom resources in all namespaces, if set to false it will only watch the runtime namespace.") flag.Parse() - ctrl.SetLogger(zap.New(zap.UseDevMode(true))) + ctrl.SetLogger(logger.NewLogger(logLevel, logJSON)) watchNamespace := "" if !watchAllNamespaces { From 311ca212c5f495e13d0dcdc556e94f543ffbce49 Mon Sep 17 00:00:00 2001 From: Michael Bridgen Date: Mon, 23 Nov 2020 13:42:33 +0000 Subject: [PATCH 5/5] Give controller liveness and readiness endpoints - make the healthz endpoint a flag and give the value to the controller runtime - set up probe endpoints; copy the func from elsewhere for now - add the probe defs to the deployment Signed-off-by: Michael Bridgen --- config/manager/manager.yaml | 11 +++++++++++ main.go | 30 ++++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 40065a7..7e5579e 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -28,6 +28,9 @@ spec: ports: - containerPort: 8080 name: http-prom + - containerPort: 9440 + name: healthz + protocol: TCP env: - name: RUNTIME_NAMESPACE valueFrom: @@ -38,6 +41,14 @@ spec: - --log-level=info - --log-json - --enable-leader-election + readinessProbe: + httpGet: + path: /readyz + port: healthz + livenessProbe: + httpGet: + path: /healthz + port: healthz resources: limits: cpu: 1000m diff --git a/main.go b/main.go index 3e6fca1..03773d1 100644 --- a/main.go +++ b/main.go @@ -24,6 +24,7 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" imagev1alpha1_reflect "github.com/fluxcd/image-reflector-controller/api/v1alpha1" "github.com/fluxcd/pkg/runtime/logger" @@ -51,6 +52,7 @@ func init() { func main() { var ( metricsAddr string + healthAddr string enableLeaderElection bool logLevel string logJSON bool @@ -58,6 +60,7 @@ func main() { ) flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&healthAddr, "health-addr", ":9440", "The address the health endpoint binds to.") flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") @@ -75,18 +78,21 @@ func main() { } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: metricsAddr, - Port: 9443, - LeaderElection: enableLeaderElection, - LeaderElectionID: "79628f79.fluxcd.io", - Namespace: watchNamespace, + Scheme: scheme, + MetricsBindAddress: metricsAddr, + HealthProbeBindAddress: healthAddr, + Port: 9443, + LeaderElection: enableLeaderElection, + LeaderElectionID: "79628f79.fluxcd.io", + Namespace: watchNamespace, }) if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) } + setupChecks(mgr) + if err = (&controllers.ImageUpdateAutomationReconciler{ Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("ImageUpdateAutomation"), @@ -104,3 +110,15 @@ func main() { os.Exit(1) } } + +func setupChecks(mgr ctrl.Manager) { + if err := mgr.AddReadyzCheck("ping", healthz.Ping); err != nil { + setupLog.Error(err, "unable to create ready check") + os.Exit(1) + } + + if err := mgr.AddHealthzCheck("ping", healthz.Ping); err != nil { + setupLog.Error(err, "unable to create health check") + os.Exit(1) + } +}