diff --git a/internal/controller/bucket_controller.go b/internal/controller/bucket_controller.go index ccac13ef..8180ebf6 100644 --- a/internal/controller/bucket_controller.go +++ b/internal/controller/bucket_controller.go @@ -42,6 +42,7 @@ import ( "github.com/fluxcd/pkg/apis/meta" "github.com/fluxcd/pkg/runtime/conditions" helper "github.com/fluxcd/pkg/runtime/controller" + "github.com/fluxcd/pkg/runtime/jitter" "github.com/fluxcd/pkg/runtime/patch" "github.com/fluxcd/pkg/runtime/predicates" rreconcile "github.com/fluxcd/pkg/runtime/reconcile" @@ -205,7 +206,9 @@ func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res summarize.RecordContextualError, summarize.RecordReconcileReq, ), - summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}), + summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{ + RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()), + }), summarize.WithPatchFieldOwner(r.ControllerName), } result, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...) diff --git a/internal/controller/bucket_controller_test.go b/internal/controller/bucket_controller_test.go index 93a551d6..ff7b33f6 100644 --- a/internal/controller/bucket_controller_test.go +++ b/internal/controller/bucket_controller_test.go @@ -40,6 +40,7 @@ import ( "github.com/fluxcd/pkg/apis/meta" "github.com/fluxcd/pkg/runtime/conditions" conditionscheck "github.com/fluxcd/pkg/runtime/conditions/check" + "github.com/fluxcd/pkg/runtime/jitter" "github.com/fluxcd/pkg/runtime/patch" sourcev1 "github.com/fluxcd/source-controller/api/v1" @@ -1368,7 +1369,9 @@ func TestBucketReconciler_statusConditions(t *testing.T) { summarize.WithReconcileResult(recResult), summarize.WithReconcileError(retErr), summarize.WithIgnoreNotFound(), - summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}), + summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{ + RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()), + }), summarize.WithPatchFieldOwner("source-controller"), } _, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...) diff --git a/internal/controller/gitrepository_controller.go b/internal/controller/gitrepository_controller.go index dc7d7a0f..e74ed34a 100644 --- a/internal/controller/gitrepository_controller.go +++ b/internal/controller/gitrepository_controller.go @@ -49,6 +49,7 @@ import ( "github.com/fluxcd/pkg/git/repository" "github.com/fluxcd/pkg/runtime/conditions" helper "github.com/fluxcd/pkg/runtime/controller" + "github.com/fluxcd/pkg/runtime/jitter" "github.com/fluxcd/pkg/runtime/patch" "github.com/fluxcd/pkg/runtime/predicates" rreconcile "github.com/fluxcd/pkg/runtime/reconcile" @@ -199,7 +200,9 @@ func (r *GitRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reques summarize.ErrorActionHandler, summarize.RecordReconcileReq, ), - summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}), + summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{ + RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()), + }), summarize.WithPatchFieldOwner(r.ControllerName), } result, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...) diff --git a/internal/controller/gitrepository_controller_test.go b/internal/controller/gitrepository_controller_test.go index cd3c085e..a7740fe4 100644 --- a/internal/controller/gitrepository_controller_test.go +++ b/internal/controller/gitrepository_controller_test.go @@ -51,6 +51,7 @@ import ( "github.com/fluxcd/pkg/gittestserver" "github.com/fluxcd/pkg/runtime/conditions" conditionscheck "github.com/fluxcd/pkg/runtime/conditions/check" + "github.com/fluxcd/pkg/runtime/jitter" "github.com/fluxcd/pkg/runtime/patch" "github.com/fluxcd/pkg/ssh" "github.com/fluxcd/pkg/testserver" @@ -2103,7 +2104,9 @@ func TestGitRepositoryReconciler_statusConditions(t *testing.T) { summarize.WithReconcileResult(recResult), summarize.WithReconcileError(retErr), summarize.WithIgnoreNotFound(), - summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}), + summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{ + RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()), + }), summarize.WithPatchFieldOwner("source-controller"), } _, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...) diff --git a/internal/controller/helmchart_controller.go b/internal/controller/helmchart_controller.go index f2b9eaa1..74651ef8 100644 --- a/internal/controller/helmchart_controller.go +++ b/internal/controller/helmchart_controller.go @@ -54,6 +54,7 @@ import ( "github.com/fluxcd/pkg/git" "github.com/fluxcd/pkg/runtime/conditions" helper "github.com/fluxcd/pkg/runtime/controller" + "github.com/fluxcd/pkg/runtime/jitter" "github.com/fluxcd/pkg/runtime/patch" "github.com/fluxcd/pkg/runtime/predicates" rreconcile "github.com/fluxcd/pkg/runtime/reconcile" @@ -220,7 +221,9 @@ func (r *HelmChartReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( summarize.RecordContextualError, summarize.RecordReconcileReq, ), - summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}), + summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{ + RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()), + }), summarize.WithPatchFieldOwner(r.ControllerName), } result, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...) diff --git a/internal/controller/helmchart_controller_test.go b/internal/controller/helmchart_controller_test.go index ec067fd5..c0ad9438 100644 --- a/internal/controller/helmchart_controller_test.go +++ b/internal/controller/helmchart_controller_test.go @@ -53,6 +53,7 @@ import ( "github.com/fluxcd/pkg/helmtestserver" "github.com/fluxcd/pkg/runtime/conditions" conditionscheck "github.com/fluxcd/pkg/runtime/conditions/check" + "github.com/fluxcd/pkg/runtime/jitter" "github.com/fluxcd/pkg/runtime/patch" "github.com/fluxcd/pkg/testserver" @@ -2107,7 +2108,9 @@ func TestHelmChartReconciler_statusConditions(t *testing.T) { summarize.WithReconcileResult(recResult), summarize.WithReconcileError(retErr), summarize.WithIgnoreNotFound(), - summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}), + summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{ + RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()), + }), summarize.WithPatchFieldOwner("source-controller"), } _, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...) diff --git a/internal/controller/helmrepository_controller.go b/internal/controller/helmrepository_controller.go index 1f9d2616..91275e73 100644 --- a/internal/controller/helmrepository_controller.go +++ b/internal/controller/helmrepository_controller.go @@ -41,6 +41,7 @@ import ( "github.com/fluxcd/pkg/apis/meta" "github.com/fluxcd/pkg/runtime/conditions" helper "github.com/fluxcd/pkg/runtime/controller" + "github.com/fluxcd/pkg/runtime/jitter" "github.com/fluxcd/pkg/runtime/patch" "github.com/fluxcd/pkg/runtime/predicates" rreconcile "github.com/fluxcd/pkg/runtime/reconcile" @@ -182,7 +183,9 @@ func (r *HelmRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reque summarize.RecordContextualError, summarize.RecordReconcileReq, ), - summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}), + summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{ + RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()), + }), summarize.WithPatchFieldOwner(r.ControllerName), } result, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...) diff --git a/internal/controller/helmrepository_controller_oci.go b/internal/controller/helmrepository_controller_oci.go index 87f504be..07a2c837 100644 --- a/internal/controller/helmrepository_controller_oci.go +++ b/internal/controller/helmrepository_controller_oci.go @@ -42,6 +42,7 @@ import ( "github.com/fluxcd/pkg/apis/meta" "github.com/fluxcd/pkg/runtime/conditions" helper "github.com/fluxcd/pkg/runtime/controller" + "github.com/fluxcd/pkg/runtime/jitter" "github.com/fluxcd/pkg/runtime/patch" "github.com/fluxcd/pkg/runtime/predicates" rreconcile "github.com/fluxcd/pkg/runtime/reconcile" @@ -261,6 +262,11 @@ func (r *HelmRepositoryOCIReconciler) reconcile(ctx context.Context, sp *patch.S if conditions.IsReady(oldObj) && !conditions.IsReady(obj) { r.eventLogf(ctx, obj, corev1.EventTypeWarning, ready.Reason, ready.Message) } + + // Apply jitter. + if result.RequeueAfter == obj.GetRequeueAfter() { + result.RequeueAfter = jitter.JitteredIntervalDuration(result.RequeueAfter) + } }() // Set reconciling condition. diff --git a/internal/controller/ocirepository_controller.go b/internal/controller/ocirepository_controller.go index 9986dc9b..20663c68 100644 --- a/internal/controller/ocirepository_controller.go +++ b/internal/controller/ocirepository_controller.go @@ -57,6 +57,7 @@ import ( "github.com/fluxcd/pkg/oci" "github.com/fluxcd/pkg/runtime/conditions" helper "github.com/fluxcd/pkg/runtime/controller" + "github.com/fluxcd/pkg/runtime/jitter" "github.com/fluxcd/pkg/runtime/patch" "github.com/fluxcd/pkg/runtime/predicates" rreconcile "github.com/fluxcd/pkg/runtime/reconcile" @@ -200,7 +201,9 @@ func (r *OCIRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reques summarize.ErrorActionHandler, summarize.RecordReconcileReq, ), - summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}), + summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{ + RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()), + }), summarize.WithPatchFieldOwner(r.ControllerName), } result, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...) diff --git a/main.go b/main.go index 5071f811..762fef71 100644 --- a/main.go +++ b/main.go @@ -42,6 +42,7 @@ import ( helper "github.com/fluxcd/pkg/runtime/controller" "github.com/fluxcd/pkg/runtime/events" feathelper "github.com/fluxcd/pkg/runtime/features" + "github.com/fluxcd/pkg/runtime/jitter" "github.com/fluxcd/pkg/runtime/leaderelection" "github.com/fluxcd/pkg/runtime/logger" "github.com/fluxcd/pkg/runtime/pprof" @@ -104,6 +105,7 @@ func main() { rateLimiterOptions helper.RateLimiterOptions featureGates feathelper.FeatureGates watchOptions helper.WatchOptions + intervalJitterOptions jitter.IntervalOptions helmCacheMaxSize int helmCacheTTL string helmCachePurgeInterval string @@ -155,6 +157,7 @@ func main() { rateLimiterOptions.BindFlags(flag.CommandLine) featureGates.BindFlags(flag.CommandLine) watchOptions.BindFlags(flag.CommandLine) + intervalJitterOptions.BindFlags(flag.CommandLine) flag.Parse() @@ -165,6 +168,11 @@ func main() { os.Exit(1) } + if err := intervalJitterOptions.SetGlobalJitter(nil); err != nil { + setupLog.Error(err, "unable to set global jitter") + os.Exit(1) + } + mgr := mustSetupManager(metricsAddr, healthAddr, concurrent, watchOptions, clientOptions, leaderElectionOptions) probes.SetupChecks(mgr, setupLog)