controller: jitter requeue interval
This adds a `--interval-jitter-percentage` flag to the controller to add a +/- percentage jitter to the interval defined in resources (defaults to 10%). Effectively, this results in a reconcilation every 4.5 - 5.5 minutes for a resource with an interval of 5 minutes. Main reason to add this change is to mitigate spikes in memory and CPU usage caused by many resources being configured with the same interval. Signed-off-by: Hidde Beydals <hidde@hhh.computer>
This commit is contained in:
parent
63f40601a3
commit
6f3eb22613
|
@ -42,6 +42,7 @@ import (
|
|||
"github.com/fluxcd/pkg/apis/meta"
|
||||
"github.com/fluxcd/pkg/runtime/conditions"
|
||||
helper "github.com/fluxcd/pkg/runtime/controller"
|
||||
"github.com/fluxcd/pkg/runtime/jitter"
|
||||
"github.com/fluxcd/pkg/runtime/patch"
|
||||
"github.com/fluxcd/pkg/runtime/predicates"
|
||||
rreconcile "github.com/fluxcd/pkg/runtime/reconcile"
|
||||
|
@ -205,7 +206,9 @@ func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
|
|||
summarize.RecordContextualError,
|
||||
summarize.RecordReconcileReq,
|
||||
),
|
||||
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}),
|
||||
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{
|
||||
RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()),
|
||||
}),
|
||||
summarize.WithPatchFieldOwner(r.ControllerName),
|
||||
}
|
||||
result, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...)
|
||||
|
|
|
@ -40,6 +40,7 @@ import (
|
|||
"github.com/fluxcd/pkg/apis/meta"
|
||||
"github.com/fluxcd/pkg/runtime/conditions"
|
||||
conditionscheck "github.com/fluxcd/pkg/runtime/conditions/check"
|
||||
"github.com/fluxcd/pkg/runtime/jitter"
|
||||
"github.com/fluxcd/pkg/runtime/patch"
|
||||
|
||||
sourcev1 "github.com/fluxcd/source-controller/api/v1"
|
||||
|
@ -1368,7 +1369,9 @@ func TestBucketReconciler_statusConditions(t *testing.T) {
|
|||
summarize.WithReconcileResult(recResult),
|
||||
summarize.WithReconcileError(retErr),
|
||||
summarize.WithIgnoreNotFound(),
|
||||
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}),
|
||||
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{
|
||||
RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()),
|
||||
}),
|
||||
summarize.WithPatchFieldOwner("source-controller"),
|
||||
}
|
||||
_, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...)
|
||||
|
|
|
@ -49,6 +49,7 @@ import (
|
|||
"github.com/fluxcd/pkg/git/repository"
|
||||
"github.com/fluxcd/pkg/runtime/conditions"
|
||||
helper "github.com/fluxcd/pkg/runtime/controller"
|
||||
"github.com/fluxcd/pkg/runtime/jitter"
|
||||
"github.com/fluxcd/pkg/runtime/patch"
|
||||
"github.com/fluxcd/pkg/runtime/predicates"
|
||||
rreconcile "github.com/fluxcd/pkg/runtime/reconcile"
|
||||
|
@ -199,7 +200,9 @@ func (r *GitRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reques
|
|||
summarize.ErrorActionHandler,
|
||||
summarize.RecordReconcileReq,
|
||||
),
|
||||
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}),
|
||||
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{
|
||||
RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()),
|
||||
}),
|
||||
summarize.WithPatchFieldOwner(r.ControllerName),
|
||||
}
|
||||
result, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...)
|
||||
|
|
|
@ -51,6 +51,7 @@ import (
|
|||
"github.com/fluxcd/pkg/gittestserver"
|
||||
"github.com/fluxcd/pkg/runtime/conditions"
|
||||
conditionscheck "github.com/fluxcd/pkg/runtime/conditions/check"
|
||||
"github.com/fluxcd/pkg/runtime/jitter"
|
||||
"github.com/fluxcd/pkg/runtime/patch"
|
||||
"github.com/fluxcd/pkg/ssh"
|
||||
"github.com/fluxcd/pkg/testserver"
|
||||
|
@ -2103,7 +2104,9 @@ func TestGitRepositoryReconciler_statusConditions(t *testing.T) {
|
|||
summarize.WithReconcileResult(recResult),
|
||||
summarize.WithReconcileError(retErr),
|
||||
summarize.WithIgnoreNotFound(),
|
||||
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}),
|
||||
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{
|
||||
RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()),
|
||||
}),
|
||||
summarize.WithPatchFieldOwner("source-controller"),
|
||||
}
|
||||
_, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...)
|
||||
|
|
|
@ -54,6 +54,7 @@ import (
|
|||
"github.com/fluxcd/pkg/git"
|
||||
"github.com/fluxcd/pkg/runtime/conditions"
|
||||
helper "github.com/fluxcd/pkg/runtime/controller"
|
||||
"github.com/fluxcd/pkg/runtime/jitter"
|
||||
"github.com/fluxcd/pkg/runtime/patch"
|
||||
"github.com/fluxcd/pkg/runtime/predicates"
|
||||
rreconcile "github.com/fluxcd/pkg/runtime/reconcile"
|
||||
|
@ -220,7 +221,9 @@ func (r *HelmChartReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
|
|||
summarize.RecordContextualError,
|
||||
summarize.RecordReconcileReq,
|
||||
),
|
||||
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}),
|
||||
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{
|
||||
RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()),
|
||||
}),
|
||||
summarize.WithPatchFieldOwner(r.ControllerName),
|
||||
}
|
||||
result, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...)
|
||||
|
|
|
@ -53,6 +53,7 @@ import (
|
|||
"github.com/fluxcd/pkg/helmtestserver"
|
||||
"github.com/fluxcd/pkg/runtime/conditions"
|
||||
conditionscheck "github.com/fluxcd/pkg/runtime/conditions/check"
|
||||
"github.com/fluxcd/pkg/runtime/jitter"
|
||||
"github.com/fluxcd/pkg/runtime/patch"
|
||||
"github.com/fluxcd/pkg/testserver"
|
||||
|
||||
|
@ -2107,7 +2108,9 @@ func TestHelmChartReconciler_statusConditions(t *testing.T) {
|
|||
summarize.WithReconcileResult(recResult),
|
||||
summarize.WithReconcileError(retErr),
|
||||
summarize.WithIgnoreNotFound(),
|
||||
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}),
|
||||
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{
|
||||
RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()),
|
||||
}),
|
||||
summarize.WithPatchFieldOwner("source-controller"),
|
||||
}
|
||||
_, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...)
|
||||
|
|
|
@ -41,6 +41,7 @@ import (
|
|||
"github.com/fluxcd/pkg/apis/meta"
|
||||
"github.com/fluxcd/pkg/runtime/conditions"
|
||||
helper "github.com/fluxcd/pkg/runtime/controller"
|
||||
"github.com/fluxcd/pkg/runtime/jitter"
|
||||
"github.com/fluxcd/pkg/runtime/patch"
|
||||
"github.com/fluxcd/pkg/runtime/predicates"
|
||||
rreconcile "github.com/fluxcd/pkg/runtime/reconcile"
|
||||
|
@ -182,7 +183,9 @@ func (r *HelmRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reque
|
|||
summarize.RecordContextualError,
|
||||
summarize.RecordReconcileReq,
|
||||
),
|
||||
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}),
|
||||
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{
|
||||
RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()),
|
||||
}),
|
||||
summarize.WithPatchFieldOwner(r.ControllerName),
|
||||
}
|
||||
result, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...)
|
||||
|
|
|
@ -42,6 +42,7 @@ import (
|
|||
"github.com/fluxcd/pkg/apis/meta"
|
||||
"github.com/fluxcd/pkg/runtime/conditions"
|
||||
helper "github.com/fluxcd/pkg/runtime/controller"
|
||||
"github.com/fluxcd/pkg/runtime/jitter"
|
||||
"github.com/fluxcd/pkg/runtime/patch"
|
||||
"github.com/fluxcd/pkg/runtime/predicates"
|
||||
rreconcile "github.com/fluxcd/pkg/runtime/reconcile"
|
||||
|
@ -261,6 +262,11 @@ func (r *HelmRepositoryOCIReconciler) reconcile(ctx context.Context, sp *patch.S
|
|||
if conditions.IsReady(oldObj) && !conditions.IsReady(obj) {
|
||||
r.eventLogf(ctx, obj, corev1.EventTypeWarning, ready.Reason, ready.Message)
|
||||
}
|
||||
|
||||
// Apply jitter.
|
||||
if result.RequeueAfter == obj.GetRequeueAfter() {
|
||||
result.RequeueAfter = jitter.JitteredIntervalDuration(result.RequeueAfter)
|
||||
}
|
||||
}()
|
||||
|
||||
// Set reconciling condition.
|
||||
|
|
|
@ -57,6 +57,7 @@ import (
|
|||
"github.com/fluxcd/pkg/oci"
|
||||
"github.com/fluxcd/pkg/runtime/conditions"
|
||||
helper "github.com/fluxcd/pkg/runtime/controller"
|
||||
"github.com/fluxcd/pkg/runtime/jitter"
|
||||
"github.com/fluxcd/pkg/runtime/patch"
|
||||
"github.com/fluxcd/pkg/runtime/predicates"
|
||||
rreconcile "github.com/fluxcd/pkg/runtime/reconcile"
|
||||
|
@ -200,7 +201,9 @@ func (r *OCIRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reques
|
|||
summarize.ErrorActionHandler,
|
||||
summarize.RecordReconcileReq,
|
||||
),
|
||||
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}),
|
||||
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{
|
||||
RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()),
|
||||
}),
|
||||
summarize.WithPatchFieldOwner(r.ControllerName),
|
||||
}
|
||||
result, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...)
|
||||
|
|
8
main.go
8
main.go
|
@ -42,6 +42,7 @@ import (
|
|||
helper "github.com/fluxcd/pkg/runtime/controller"
|
||||
"github.com/fluxcd/pkg/runtime/events"
|
||||
feathelper "github.com/fluxcd/pkg/runtime/features"
|
||||
"github.com/fluxcd/pkg/runtime/jitter"
|
||||
"github.com/fluxcd/pkg/runtime/leaderelection"
|
||||
"github.com/fluxcd/pkg/runtime/logger"
|
||||
"github.com/fluxcd/pkg/runtime/pprof"
|
||||
|
@ -104,6 +105,7 @@ func main() {
|
|||
rateLimiterOptions helper.RateLimiterOptions
|
||||
featureGates feathelper.FeatureGates
|
||||
watchOptions helper.WatchOptions
|
||||
intervalJitterOptions jitter.IntervalOptions
|
||||
helmCacheMaxSize int
|
||||
helmCacheTTL string
|
||||
helmCachePurgeInterval string
|
||||
|
@ -155,6 +157,7 @@ func main() {
|
|||
rateLimiterOptions.BindFlags(flag.CommandLine)
|
||||
featureGates.BindFlags(flag.CommandLine)
|
||||
watchOptions.BindFlags(flag.CommandLine)
|
||||
intervalJitterOptions.BindFlags(flag.CommandLine)
|
||||
|
||||
flag.Parse()
|
||||
|
||||
|
@ -165,6 +168,11 @@ func main() {
|
|||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := intervalJitterOptions.SetGlobalJitter(nil); err != nil {
|
||||
setupLog.Error(err, "unable to set global jitter")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
mgr := mustSetupManager(metricsAddr, healthAddr, concurrent, watchOptions, clientOptions, leaderElectionOptions)
|
||||
|
||||
probes.SetupChecks(mgr, setupLog)
|
||||
|
|
Loading…
Reference in New Issue