Enable fail-fast behavior for health checks

Fail the health check as soon as a resource becomes stalled
without waiting for the timeout to expire.
This behavior can be disabled using the `DisableFailFastBehavior` feature flag.

Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>

internal/controller/kustomization_controller.go

@@ -89,6 +89,7 @@ type KustomizationReconciler struct {
 	statusManager         string
 	NoCrossNamespaceRefs  bool
 	NoRemoteBases         bool
+	FailFast              bool
 	DefaultServiceAccount string
 	KubeConfigOpts        runtimeClient.KubeConfigOptions
 }
@@ -863,6 +864,7 @@ func (r *KustomizationReconciler) checkHealth(ctx context.Context,
 	if err := manager.WaitForSet(toCheck, ssa.WaitOptions{
 		Interval: 5 * time.Second,
 		Timeout:  obj.GetTimeout(),
+		FailFast: r.FailFast,
 	}); err != nil {
 		conditions.MarkFalse(obj, meta.ReadyCondition, kustomizev1.HealthCheckFailedReason, err.Error())
 		conditions.MarkFalse(obj, kustomizev1.HealthyCondition, kustomizev1.HealthCheckFailedReason, err.Error())

internal/features/features.go

@@ -35,6 +35,10 @@ const (
 	// large number of resources, as it will potentially reduce the amount of
 	// memory used by the controller.
 	DisableStatusPollerCache = "DisableStatusPollerCache"
+
+	// DisableFailFastBehavior controls whether the fail-fast behavior when
+	// waiting for resources to become ready should be disabled.
+	DisableFailFastBehavior = "DisableFailFastBehavior"
 )
 
 var features = map[string]bool{
@@ -44,6 +48,9 @@ var features = map[string]bool{
 	// DisableStatusPollerCache
 	// opt-in from v0.35
 	DisableStatusPollerCache: false,
+	// DisableFailFastBehavior
+	// opt-in from v1.1
+	DisableFailFastBehavior: false,
 }
 
 // FeatureGates contains a list of all supported feature gates and

main.go

@@ -200,6 +200,11 @@ func main() {
 		pollingOpts.ClusterReaderFactory = engine.ClusterReaderFactoryFunc(clusterreader.NewDirectClusterReader)
 	}
 
+	failFast := true
+	if ok, _ := features.Enabled(features.DisableFailFastBehavior); ok {
+		failFast = false
+	}
+
 	if err = (&controller.KustomizationReconciler{
 		ControllerName:        controllerName,
 		DefaultServiceAccount: defaultServiceAccount,
@@ -208,6 +213,7 @@ func main() {
 		EventRecorder:         eventRecorder,
 		NoCrossNamespaceRefs:  aclOptions.NoCrossNamespaceRefs,
 		NoRemoteBases:         noRemoteBases,
+		FailFast:              failFast,
 		KubeConfigOpts:        kubeConfigOpts,
 		PollingOpts:           pollingOpts,
 		StatusPoller:          polling.NewStatusPoller(mgr.GetClient(), mgr.GetRESTMapper(), pollingOpts),