diff --git a/pkg/controllers/binding/binding_controller.go b/pkg/controllers/binding/binding_controller.go index 57756e514..22be7c95c 100644 --- a/pkg/controllers/binding/binding_controller.go +++ b/pkg/controllers/binding/binding_controller.go @@ -13,6 +13,7 @@ import ( "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/dynamic" "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/retry" "k8s.io/klog/v2" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -275,8 +276,11 @@ func (c *ResourceBindingController) updateFullyAppliedCondition(binding *workv1a Message: FullyAppliedSuccessMessage, } - meta.SetStatusCondition(&binding.Status.Conditions, newBindingFullyAppliedCondition) - err := c.Client.Status().Update(context.TODO(), binding) - - return err + return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + if err = c.Get(context.TODO(), client.ObjectKey{Namespace: binding.Namespace, Name: binding.Name}, binding); err != nil { + return err + } + meta.SetStatusCondition(&binding.Status.Conditions, newBindingFullyAppliedCondition) + return c.Status().Update(context.TODO(), binding) + }) } diff --git a/pkg/controllers/binding/cluster_resource_binding_controller.go b/pkg/controllers/binding/cluster_resource_binding_controller.go index fb63a642c..d5ad2e1cb 100644 --- a/pkg/controllers/binding/cluster_resource_binding_controller.go +++ b/pkg/controllers/binding/cluster_resource_binding_controller.go @@ -13,6 +13,7 @@ import ( "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/dynamic" "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/retry" "k8s.io/klog/v2" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -266,8 +267,11 @@ func (c *ClusterResourceBindingController) updateFullyAppliedCondition(binding * Message: FullyAppliedSuccessMessage, } - meta.SetStatusCondition(&binding.Status.Conditions, newBindingFullyAppliedCondition) - err := c.Client.Status().Update(context.TODO(), binding) - - return err + return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + if err = c.Get(context.TODO(), client.ObjectKey{Namespace: binding.Namespace, Name: binding.Name}, binding); err != nil { + return err + } + meta.SetStatusCondition(&binding.Status.Conditions, newBindingFullyAppliedCondition) + return c.Status().Update(context.TODO(), binding) + }) } diff --git a/pkg/controllers/execution/execution_controller.go b/pkg/controllers/execution/execution_controller.go index 89d054487..89ccf7eb5 100644 --- a/pkg/controllers/execution/execution_controller.go +++ b/pkg/controllers/execution/execution_controller.go @@ -11,6 +11,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/retry" "k8s.io/klog/v2" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -241,7 +242,11 @@ func (c *Controller) updateAppliedCondition(work *workv1alpha1.Work, status meta LastTransitionTime: metav1.Now(), } - meta.SetStatusCondition(&work.Status.Conditions, newWorkAppliedCondition) - err := c.Client.Status().Update(context.TODO(), work) - return err + return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + if err = c.Get(context.TODO(), client.ObjectKey{Namespace: work.Namespace, Name: work.Name}, work); err != nil { + return err + } + meta.SetStatusCondition(&work.Status.Conditions, newWorkAppliedCondition) + return c.Status().Update(context.TODO(), work) + }) } diff --git a/pkg/controllers/status/workstatus_controller.go b/pkg/controllers/status/workstatus_controller.go index 65dac5fb1..b3cef8f6c 100644 --- a/pkg/controllers/status/workstatus_controller.go +++ b/pkg/controllers/status/workstatus_controller.go @@ -13,6 +13,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/retry" "k8s.io/klog/v2" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -288,14 +289,13 @@ func (c *WorkStatusController) reflectStatus(work *workv1alpha1.Work, clusterObj manifestStatus.Status = rawExtension } - work.Status.ManifestStatuses = c.mergeStatus(work.Status.ManifestStatuses, manifestStatus) - - if err := c.Client.Status().Update(context.TODO(), work); err != nil { - klog.Errorf("Failed to reflect status to work(%s/%s): %v", work.Namespace, work.Name, err) - return err - } - - return nil + return retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) { + if err = c.Get(context.TODO(), client.ObjectKey{Namespace: work.Namespace, Name: work.Name}, work); err != nil { + return err + } + work.Status.ManifestStatuses = c.mergeStatus(work.Status.ManifestStatuses, manifestStatus) + return c.Status().Update(context.TODO(), work) + }) } func (c *WorkStatusController) buildStatusIdentifier(work *workv1alpha1.Work, clusterObj *unstructured.Unstructured) (*workv1alpha1.ResourceIdentifier, error) {