Merge pull request #1184 from fluxcd/jitter

controller: jitter requeue interval
This commit is contained in:
Hidde Beydals 2023-08-07 16:58:04 +02:00 committed by GitHub
commit 727d62b51a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 135 additions and 33 deletions

View File

@ -55,7 +55,9 @@ type GitRepositorySpec struct {
// +optional
SecretRef *meta.LocalObjectReference `json:"secretRef,omitempty"`
// Interval at which to check the GitRepository for updates.
// Interval at which the GitRepository URL is checked for updates.
// This interval is approximate and may be subject to jitter to ensure
// efficient use of resources.
// +kubebuilder:validation:Type=string
// +kubebuilder:validation:Pattern="^([0-9]+(\\.[0-9]+)?(ms|s|m|h))+$"
// +required

View File

@ -78,7 +78,9 @@ type BucketSpec struct {
// +optional
SecretRef *meta.LocalObjectReference `json:"secretRef,omitempty"`
// Interval at which to check the Endpoint for updates.
// Interval at which the Bucket Endpoint is checked for updates.
// This interval is approximate and may be subject to jitter to ensure
// efficient use of resources.
// +kubebuilder:validation:Type=string
// +kubebuilder:validation:Pattern="^([0-9]+(\\.[0-9]+)?(ms|s|m|h))+$"
// +required

View File

@ -46,7 +46,9 @@ type HelmChartSpec struct {
// +required
SourceRef LocalHelmChartSourceReference `json:"sourceRef"`
// Interval is the interval at which to check the Source for updates.
// Interval at which the HelmChart SourceRef is checked for updates.
// This interval is approximate and may be subject to jitter to ensure
// efficient use of resources.
// +kubebuilder:validation:Type=string
// +kubebuilder:validation:Pattern="^([0-9]+(\\.[0-9]+)?(ms|s|m|h))+$"
// +required

View File

@ -72,7 +72,9 @@ type HelmRepositorySpec struct {
// +optional
PassCredentials bool `json:"passCredentials,omitempty"`
// Interval at which to check the URL for updates.
// Interval at which the HelmRepository URL is checked for updates.
// This interval is approximate and may be subject to jitter to ensure
// efficient use of resources.
// +kubebuilder:validation:Type=string
// +kubebuilder:validation:Pattern="^([0-9]+(\\.[0-9]+)?(ms|s|m|h))+$"
// +required

View File

@ -111,7 +111,9 @@ type OCIRepositorySpec struct {
// +optional
CertSecretRef *meta.LocalObjectReference `json:"certSecretRef,omitempty"`
// The interval at which to check for image updates.
// Interval at which the OCIRepository URL is checked for updates.
// This interval is approximate and may be subject to jitter to ensure
// efficient use of resources.
// +kubebuilder:validation:Type=string
// +kubebuilder:validation:Pattern="^([0-9]+(\\.[0-9]+)?(ms|s|m|h))+$"
// +required

View File

@ -326,7 +326,9 @@ spec:
description: Insecure allows connecting to a non-TLS HTTP Endpoint.
type: boolean
interval:
description: Interval at which to check the Endpoint for updates.
description: Interval at which the Bucket Endpoint is checked for
updates. This interval is approximate and may be subject to jitter
to ensure efficient use of resources.
pattern: ^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$
type: string
provider:

View File

@ -87,7 +87,9 @@ spec:
type: object
type: array
interval:
description: Interval at which to check the GitRepository for updates.
description: Interval at which the GitRepository URL is checked for
updates. This interval is approximate and may be subject to jitter
to ensure efficient use of resources.
pattern: ^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$
type: string
proxySecretRef:

View File

@ -346,8 +346,9 @@ spec:
at in the SourceRef.
type: string
interval:
description: Interval is the interval at which to check the Source
for updates.
description: Interval at which the HelmChart SourceRef is checked
for updates. This interval is approximate and may be subject to
jitter to ensure efficient use of resources.
pattern: ^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$
type: string
reconcileStrategy:

View File

@ -309,7 +309,9 @@ spec:
- name
type: object
interval:
description: Interval at which to check the URL for updates.
description: Interval at which the HelmRepository URL is checked for
updates. This interval is approximate and may be subject to jitter
to ensure efficient use of resources.
pattern: ^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$
type: string
passCredentials:

View File

@ -75,7 +75,9 @@ spec:
registry.
type: boolean
interval:
description: The interval at which to check for image updates.
description: Interval at which the OCIRepository URL is checked for
updates. This interval is approximate and may be subject to jitter
to ensure efficient use of resources.
pattern: ^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$
type: string
layerSelector:

View File

@ -108,7 +108,9 @@ Kubernetes meta/v1.Duration
</em>
</td>
<td>
<p>Interval at which to check the GitRepository for updates.</p>
<p>Interval at which the GitRepository URL is checked for updates.
This interval is approximate and may be subject to jitter to ensure
efficient use of resources.</p>
</td>
</tr>
<tr>
@ -559,7 +561,9 @@ Kubernetes meta/v1.Duration
</em>
</td>
<td>
<p>Interval at which to check the GitRepository for updates.</p>
<p>Interval at which the GitRepository URL is checked for updates.
This interval is approximate and may be subject to jitter to ensure
efficient use of resources.</p>
</td>
</tr>
<tr>

View File

@ -161,7 +161,9 @@ Kubernetes meta/v1.Duration
</em>
</td>
<td>
<p>Interval at which to check the Endpoint for updates.</p>
<p>Interval at which the Bucket Endpoint is checked for updates.
This interval is approximate and may be subject to jitter to ensure
efficient use of resources.</p>
</td>
</tr>
<tr>
@ -593,7 +595,9 @@ Kubernetes meta/v1.Duration
</em>
</td>
<td>
<p>Interval is the interval at which to check the Source for updates.</p>
<p>Interval at which the HelmChart SourceRef is checked for updates.
This interval is approximate and may be subject to jitter to ensure
efficient use of resources.</p>
</td>
</tr>
<tr>
@ -840,7 +844,9 @@ Kubernetes meta/v1.Duration
</em>
</td>
<td>
<p>Interval at which to check the URL for updates.</p>
<p>Interval at which the HelmRepository URL is checked for updates.
This interval is approximate and may be subject to jitter to ensure
efficient use of resources.</p>
</td>
</tr>
<tr>
@ -1126,7 +1132,9 @@ Kubernetes meta/v1.Duration
</em>
</td>
<td>
<p>The interval at which to check for image updates.</p>
<p>Interval at which the OCIRepository URL is checked for updates.
This interval is approximate and may be subject to jitter to ensure
efficient use of resources.</p>
</td>
</tr>
<tr>
@ -1424,7 +1432,9 @@ Kubernetes meta/v1.Duration
</em>
</td>
<td>
<p>Interval at which to check the Endpoint for updates.</p>
<p>Interval at which the Bucket Endpoint is checked for updates.
This interval is approximate and may be subject to jitter to ensure
efficient use of resources.</p>
</td>
</tr>
<tr>
@ -2214,7 +2224,9 @@ Kubernetes meta/v1.Duration
</em>
</td>
<td>
<p>Interval is the interval at which to check the Source for updates.</p>
<p>Interval at which the HelmChart SourceRef is checked for updates.
This interval is approximate and may be subject to jitter to ensure
efficient use of resources.</p>
</td>
</tr>
<tr>
@ -2524,7 +2536,9 @@ Kubernetes meta/v1.Duration
</em>
</td>
<td>
<p>Interval at which to check the URL for updates.</p>
<p>Interval at which the HelmRepository URL is checked for updates.
This interval is approximate and may be subject to jitter to ensure
efficient use of resources.</p>
</td>
</tr>
<tr>
@ -3013,7 +3027,9 @@ Kubernetes meta/v1.Duration
</em>
</td>
<td>
<p>The interval at which to check for image updates.</p>
<p>Interval at which the OCIRepository URL is checked for updates.
This interval is approximate and may be subject to jitter to ensure
efficient use of resources.</p>
</td>
</tr>
<tr>

View File

@ -224,6 +224,11 @@ e.g. `10m0s` to reconcile the object every 10 minutes.
If the `.metadata.generation` of a resource changes (due to e.g. a change to
the spec), this is handled instantly outside the interval window.
**Note:** The controller can be configured to apply a jitter to the interval in
order to distribute the load more evenly when multiple GitRepository objects are
set up with the same interval. For more information, please refer to the
[source-controller configuration options](https://fluxcd.io/flux/components/source/options/).
### Timeout
`.spec.timeout` is an optional field to specify a timeout for Git operations

View File

@ -724,7 +724,7 @@ Where the (base64 decoded) value of `.data.serviceaccount` looks like this:
### Interval
`.spec.interval` is a required field that specifices the interval which the
`.spec.interval` is a required field that specifies the interval which the
object storage bucket must be consulted at.
After successfully reconciling a Bucket object, the source-controller requeues
@ -733,7 +733,12 @@ the object for inspection after the specified interval. The value must be in a
e.g. `10m0s` to look at the object storage bucket every 10 minutes.
If the `.metadata.generation` of a resource changes (due to e.g. the apply of a
change to the spec), this is handled instantly outside of the interval window.
change to the spec), this is handled instantly outside the interval window.
**Note:** The controller can be configured to apply a jitter to the interval in
order to distribute the load more evenly when multiple Bucket objects are set up
with the same interval. For more information, please refer to the
[source-controller configuration options](https://fluxcd.io/flux/components/source/options/).
### Endpoint

View File

@ -45,7 +45,7 @@ In the above example:
You can run this example by saving the manifest into `helmchart.yaml`.
**NOTE:** HelmChart is usually used by the helm-controller. Based on the
**Note:** HelmChart is usually used by the helm-controller. Based on the
HelmRelease configuration, an associated HelmChart is created by the
helm-controller.
@ -211,7 +211,7 @@ changes in a `HelmRepository`. `Revision` is used for creating a new artifact
when the source revision changes in a `GitRepository` or a `Bucket` Source. It
defaults to `ChartVersion`.
**NOTE:** If the reconcile strategy is `ChartVersion` and the source reference
**Note:** If the reconcile strategy is `ChartVersion` and the source reference
is a `GitRepository` or a `Bucket`, no new chart artifact is produced on updates
to the source unless the `version` in `Chart.yaml` is incremented. To produce
new chart artifact on change in source revision, set the reconcile strategy to
@ -233,6 +233,11 @@ e.g. `10m0s` to look at the source for updates every 10 minutes.
If the `.metadata.generation` of a resource changes (due to e.g. applying a
change to the spec), this is handled instantly outside the interval window.
**Note:** The controller can be configured to apply a jitter to the interval in
order to distribute the load more evenly when multiple HelmChart objects are set
up with the same interval. For more information, please refer to the
[source-controller configuration options](https://fluxcd.io/flux/components/source/options/).
### Suspend
`.spec.suspend` is an optional field to suspend the reconciliation of a

View File

@ -361,6 +361,11 @@ e.g. `10m0s` to fetch the HelmRepository index YAML every 10 minutes.
If the `.metadata.generation` of a resource changes (due to e.g. applying a
change to the spec), this is handled instantly outside the interval window.
**Note:** The controller can be configured to apply a jitter to the interval in
order to distribute the load more evenly when multiple HelmRepository objects
are set up with the same interval. For more information, please refer to the
[source-controller configuration options](https://fluxcd.io/flux/components/source/options/).
### URL
`.spec.url` is a required field that depending on the [type of the HelmRepository object](#type)

View File

@ -355,6 +355,11 @@ e.g. `10m0s` to reconcile the object every 10 minutes.
If the `.metadata.generation` of a resource changes (due to e.g. a change to
the spec), this is handled instantly outside the interval window.
**Note:** The controller can be configured to apply a jitter to the interval in
order to distribute the load more evenly when multiple OCIRepository objects are
set up with the same interval. For more information, please refer to the
[source-controller configuration options](https://fluxcd.io/flux/components/source/options/).
### Timeout
`.spec.timeout` is an optional field to specify a timeout for OCI operations

View File

@ -42,6 +42,7 @@ import (
"github.com/fluxcd/pkg/apis/meta"
"github.com/fluxcd/pkg/runtime/conditions"
helper "github.com/fluxcd/pkg/runtime/controller"
"github.com/fluxcd/pkg/runtime/jitter"
"github.com/fluxcd/pkg/runtime/patch"
"github.com/fluxcd/pkg/runtime/predicates"
rreconcile "github.com/fluxcd/pkg/runtime/reconcile"
@ -205,7 +206,9 @@ func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
summarize.RecordContextualError,
summarize.RecordReconcileReq,
),
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}),
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{
RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()),
}),
summarize.WithPatchFieldOwner(r.ControllerName),
}
result, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...)

View File

@ -40,6 +40,7 @@ import (
"github.com/fluxcd/pkg/apis/meta"
"github.com/fluxcd/pkg/runtime/conditions"
conditionscheck "github.com/fluxcd/pkg/runtime/conditions/check"
"github.com/fluxcd/pkg/runtime/jitter"
"github.com/fluxcd/pkg/runtime/patch"
sourcev1 "github.com/fluxcd/source-controller/api/v1"
@ -1368,7 +1369,9 @@ func TestBucketReconciler_statusConditions(t *testing.T) {
summarize.WithReconcileResult(recResult),
summarize.WithReconcileError(retErr),
summarize.WithIgnoreNotFound(),
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}),
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{
RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()),
}),
summarize.WithPatchFieldOwner("source-controller"),
}
_, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...)

View File

@ -49,6 +49,7 @@ import (
"github.com/fluxcd/pkg/git/repository"
"github.com/fluxcd/pkg/runtime/conditions"
helper "github.com/fluxcd/pkg/runtime/controller"
"github.com/fluxcd/pkg/runtime/jitter"
"github.com/fluxcd/pkg/runtime/patch"
"github.com/fluxcd/pkg/runtime/predicates"
rreconcile "github.com/fluxcd/pkg/runtime/reconcile"
@ -199,7 +200,9 @@ func (r *GitRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reques
summarize.ErrorActionHandler,
summarize.RecordReconcileReq,
),
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}),
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{
RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()),
}),
summarize.WithPatchFieldOwner(r.ControllerName),
}
result, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...)

View File

@ -51,6 +51,7 @@ import (
"github.com/fluxcd/pkg/gittestserver"
"github.com/fluxcd/pkg/runtime/conditions"
conditionscheck "github.com/fluxcd/pkg/runtime/conditions/check"
"github.com/fluxcd/pkg/runtime/jitter"
"github.com/fluxcd/pkg/runtime/patch"
"github.com/fluxcd/pkg/ssh"
"github.com/fluxcd/pkg/testserver"
@ -2103,7 +2104,9 @@ func TestGitRepositoryReconciler_statusConditions(t *testing.T) {
summarize.WithReconcileResult(recResult),
summarize.WithReconcileError(retErr),
summarize.WithIgnoreNotFound(),
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}),
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{
RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()),
}),
summarize.WithPatchFieldOwner("source-controller"),
}
_, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...)

View File

@ -54,6 +54,7 @@ import (
"github.com/fluxcd/pkg/git"
"github.com/fluxcd/pkg/runtime/conditions"
helper "github.com/fluxcd/pkg/runtime/controller"
"github.com/fluxcd/pkg/runtime/jitter"
"github.com/fluxcd/pkg/runtime/patch"
"github.com/fluxcd/pkg/runtime/predicates"
rreconcile "github.com/fluxcd/pkg/runtime/reconcile"
@ -220,7 +221,9 @@ func (r *HelmChartReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
summarize.RecordContextualError,
summarize.RecordReconcileReq,
),
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}),
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{
RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()),
}),
summarize.WithPatchFieldOwner(r.ControllerName),
}
result, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...)

View File

@ -53,6 +53,7 @@ import (
"github.com/fluxcd/pkg/helmtestserver"
"github.com/fluxcd/pkg/runtime/conditions"
conditionscheck "github.com/fluxcd/pkg/runtime/conditions/check"
"github.com/fluxcd/pkg/runtime/jitter"
"github.com/fluxcd/pkg/runtime/patch"
"github.com/fluxcd/pkg/testserver"
@ -2107,7 +2108,9 @@ func TestHelmChartReconciler_statusConditions(t *testing.T) {
summarize.WithReconcileResult(recResult),
summarize.WithReconcileError(retErr),
summarize.WithIgnoreNotFound(),
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}),
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{
RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()),
}),
summarize.WithPatchFieldOwner("source-controller"),
}
_, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...)

View File

@ -41,6 +41,7 @@ import (
"github.com/fluxcd/pkg/apis/meta"
"github.com/fluxcd/pkg/runtime/conditions"
helper "github.com/fluxcd/pkg/runtime/controller"
"github.com/fluxcd/pkg/runtime/jitter"
"github.com/fluxcd/pkg/runtime/patch"
"github.com/fluxcd/pkg/runtime/predicates"
rreconcile "github.com/fluxcd/pkg/runtime/reconcile"
@ -182,7 +183,9 @@ func (r *HelmRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reque
summarize.RecordContextualError,
summarize.RecordReconcileReq,
),
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}),
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{
RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()),
}),
summarize.WithPatchFieldOwner(r.ControllerName),
}
result, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...)

View File

@ -42,6 +42,7 @@ import (
"github.com/fluxcd/pkg/apis/meta"
"github.com/fluxcd/pkg/runtime/conditions"
helper "github.com/fluxcd/pkg/runtime/controller"
"github.com/fluxcd/pkg/runtime/jitter"
"github.com/fluxcd/pkg/runtime/patch"
"github.com/fluxcd/pkg/runtime/predicates"
rreconcile "github.com/fluxcd/pkg/runtime/reconcile"
@ -261,6 +262,11 @@ func (r *HelmRepositoryOCIReconciler) reconcile(ctx context.Context, sp *patch.S
if conditions.IsReady(oldObj) && !conditions.IsReady(obj) {
r.eventLogf(ctx, obj, corev1.EventTypeWarning, ready.Reason, ready.Message)
}
// Apply jitter.
if result.RequeueAfter == obj.GetRequeueAfter() {
result.RequeueAfter = jitter.JitteredIntervalDuration(result.RequeueAfter)
}
}()
// Set reconciling condition.

View File

@ -57,6 +57,7 @@ import (
"github.com/fluxcd/pkg/oci"
"github.com/fluxcd/pkg/runtime/conditions"
helper "github.com/fluxcd/pkg/runtime/controller"
"github.com/fluxcd/pkg/runtime/jitter"
"github.com/fluxcd/pkg/runtime/patch"
"github.com/fluxcd/pkg/runtime/predicates"
rreconcile "github.com/fluxcd/pkg/runtime/reconcile"
@ -200,7 +201,9 @@ func (r *OCIRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reques
summarize.ErrorActionHandler,
summarize.RecordReconcileReq,
),
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{RequeueAfter: obj.GetRequeueAfter()}),
summarize.WithResultBuilder(sreconcile.AlwaysRequeueResultBuilder{
RequeueAfter: jitter.JitteredIntervalDuration(obj.GetRequeueAfter()),
}),
summarize.WithPatchFieldOwner(r.ControllerName),
}
result, retErr = summarizeHelper.SummarizeAndPatch(ctx, obj, summarizeOpts...)

View File

@ -42,6 +42,7 @@ import (
helper "github.com/fluxcd/pkg/runtime/controller"
"github.com/fluxcd/pkg/runtime/events"
feathelper "github.com/fluxcd/pkg/runtime/features"
"github.com/fluxcd/pkg/runtime/jitter"
"github.com/fluxcd/pkg/runtime/leaderelection"
"github.com/fluxcd/pkg/runtime/logger"
"github.com/fluxcd/pkg/runtime/pprof"
@ -104,6 +105,7 @@ func main() {
rateLimiterOptions helper.RateLimiterOptions
featureGates feathelper.FeatureGates
watchOptions helper.WatchOptions
intervalJitterOptions jitter.IntervalOptions
helmCacheMaxSize int
helmCacheTTL string
helmCachePurgeInterval string
@ -155,6 +157,7 @@ func main() {
rateLimiterOptions.BindFlags(flag.CommandLine)
featureGates.BindFlags(flag.CommandLine)
watchOptions.BindFlags(flag.CommandLine)
intervalJitterOptions.BindFlags(flag.CommandLine)
flag.Parse()
@ -165,6 +168,11 @@ func main() {
os.Exit(1)
}
if err := intervalJitterOptions.SetGlobalJitter(nil); err != nil {
setupLog.Error(err, "unable to set global jitter")
os.Exit(1)
}
mgr := mustSetupManager(metricsAddr, healthAddr, concurrent, watchOptions, clientOptions, leaderElectionOptions)
probes.SetupChecks(mgr, setupLog)