From 8241b3cab2d39725a590a1ad184fdf5fefe5fb96 Mon Sep 17 00:00:00 2001 From: zhzhuang-zju Date: Wed, 21 Feb 2024 11:07:29 +0800 Subject: [PATCH] update work status condition when recreate failed Signed-off-by: zhzhuang-zju --- .../status/work_status_controller.go | 61 ++++++++++++++++++- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/pkg/controllers/status/work_status_controller.go b/pkg/controllers/status/work_status_controller.go index 17365f724..7a5f79c04 100644 --- a/pkg/controllers/status/work_status_controller.go +++ b/pkg/controllers/status/work_status_controller.go @@ -257,7 +257,7 @@ func (c *WorkStatusController) syncWorkStatus(key util.QueueKey) error { func (c *WorkStatusController) handleDeleteEvent(key keys.FederatedKey) error { executionSpace := names.GenerateExecutionSpaceName(key.Cluster) - // Given the workload might has been deleted from informer cache, so that we can't get work object by it's label, + // Given the workload might have been deleted from informer cache, so that we can't get work object by its label, // we have to get work by naming rule as the work's name is generated by the workload's kind, name and namespace. workName := names.GenerateWorkName(key.Kind, key.Name, key.Namespace) work := &workv1alpha1.Work{} @@ -280,7 +280,13 @@ func (c *WorkStatusController) handleDeleteEvent(key keys.FederatedKey) error { return nil } - return c.recreateResourceIfNeeded(work, key) + reCreateErr := c.recreateResourceIfNeeded(work, key) + if reCreateErr != nil { + c.updateAppliedCondition(work, metav1.ConditionFalse, "ReCreateFailed", reCreateErr.Error()) + return reCreateErr + } + c.updateAppliedCondition(work, metav1.ConditionTrue, "ReCreateSuccessful", "Manifest has been successfully applied") + return nil } func (c *WorkStatusController) recreateResourceIfNeeded(work *workv1alpha1.Work, workloadKey keys.FederatedKey) error { @@ -295,12 +301,52 @@ func (c *WorkStatusController) recreateResourceIfNeeded(work *workv1alpha1.Work, manifest.GetNamespace() == workloadKey.Namespace && manifest.GetName() == workloadKey.Name { klog.Infof("recreating %s", workloadKey.String()) - return c.ObjectWatcher.Create(workloadKey.Cluster, manifest) + err := c.ObjectWatcher.Create(workloadKey.Cluster, manifest) + if err != nil { + c.eventf(manifest, corev1.EventTypeWarning, events.EventReasonSyncWorkloadFailed, "Failed to create or update resource(%s/%s) in member cluster(%s): %v", manifest.GetNamespace(), manifest.GetName(), workloadKey.Cluster, err) + return err + } + c.eventf(manifest, corev1.EventTypeNormal, events.EventReasonSyncWorkloadSucceed, "Successfully applied resource(%s/%s) to cluster %s", manifest.GetNamespace(), manifest.GetName(), workloadKey.Cluster) + return nil } } return nil } +// updateAppliedCondition update the condition for the given Work +func (c *WorkStatusController) updateAppliedCondition(work *workv1alpha1.Work, status metav1.ConditionStatus, reason, message string) { + newWorkAppliedCondition := metav1.Condition{ + Type: workv1alpha1.WorkApplied, + Status: status, + Reason: reason, + Message: message, + LastTransitionTime: metav1.Now(), + } + + err := retry.RetryOnConflict(retry.DefaultRetry, func() (err error) { + workStatus := work.Status.DeepCopy() + meta.SetStatusCondition(&work.Status.Conditions, newWorkAppliedCondition) + if reflect.DeepEqual(*workStatus, work.Status) { + return nil + } + updateErr := c.Status().Update(context.TODO(), work) + if updateErr == nil { + return nil + } + updated := &workv1alpha1.Work{} + if err = c.Get(context.TODO(), client.ObjectKey{Namespace: work.Namespace, Name: work.Name}, updated); err == nil { + work = updated + } else { + klog.Errorf("Failed to get updated work %s/%s: %s", work.Namespace, work.Name, err.Error()) + } + return updateErr + }) + + if err != nil { + klog.Errorf("Failed to update condition of work %s/%s: %s", work.Namespace, work.Name, err.Error()) + } +} + // reflectStatus grabs cluster object's running status then updates to its owner object(Work). func (c *WorkStatusController) reflectStatus(work *workv1alpha1.Work, clusterObj *unstructured.Unstructured) error { statusRaw, err := c.ResourceInterpreter.ReflectStatus(clusterObj) @@ -499,3 +545,12 @@ func (c *WorkStatusController) SetupWithManager(mgr controllerruntime.Manager) e RateLimiter: ratelimiterflag.DefaultControllerRateLimiter(c.RateLimiterOptions), }).Complete(c) } + +func (c *WorkStatusController) eventf(object *unstructured.Unstructured, eventType, reason, messageFmt string, args ...interface{}) { + ref, err := helper.GenEventRef(object) + if err != nil { + klog.Errorf("ignore event(%s) as failed to build event reference for: kind=%s, %s due to %v", reason, object.GetKind(), klog.KObj(object), err) + return + } + c.EventRecorder.Eventf(ref, eventType, reason, messageFmt, args...) +}