[flake] application failover testing

Signed-off-by: Poor12 <shentiecheng@huawei.com>

pkg/controllers/binding/binding_controller.go

@@ -193,8 +193,10 @@ func (c *ResourceBindingController) newOverridePolicyFunc() handler.MapFunc {
 
 			workload, err := helper.FetchResourceTemplate(c.DynamicClient, c.InformerManager, c.RESTMapper, binding.Spec.Resource)
 			if err != nil {
+				// If we cannot fetch resource template from binding, this may be due to the fact that the resource template has been deleted.
+				// Just skip it so that it will not affect other bindings.
 				klog.Errorf("Failed to fetch workload for resourceBinding(%s/%s). Error: %v.", binding.Namespace, binding.Name, err)
-				return nil
+				continue
 			}
 
 			for _, rs := range overrideRS {

pkg/controllers/binding/cluster_resource_binding_controller.go

@@ -180,8 +180,10 @@ func (c *ClusterResourceBindingController) newOverridePolicyFunc() handler.MapFu
 
 			workload, err := helper.FetchResourceTemplate(c.DynamicClient, c.InformerManager, c.RESTMapper, binding.Spec.Resource)
 			if err != nil {
+				// If we cannot fetch resource template from binding, this may be due to the fact that the resource template has been deleted.
+				// Just skip it so that it will not affect other bindings.
 				klog.Errorf("Failed to fetch workload for clusterResourceBinding(%s). Error: %v.", binding.Name, err)
-				return nil
+				continue
 			}
 
 			for _, rs := range overrideRS {

test/e2e/failover_test.go

@@ -335,6 +335,18 @@ var _ = framework.SerialDescribe("failover testing", func() {
 				framework.CreateOverridePolicy(karmadaClient, overridePolicy)
 			})
 
+			ginkgo.By("check if deployment present on member clusters has correct image value", func() {
+				framework.WaitDeploymentPresentOnClustersFitWith(disabledClusters, deployment.Namespace, deployment.Name,
+					func(deployment *appsv1.Deployment) bool {
+						for _, container := range deployment.Spec.Template.Spec.Containers {
+							if container.Image != "fake/nginx:1.19.0" {
+								return false
+							}
+						}
+						return true
+					})
+			})
+
 			ginkgo.By("check whether the failed deployment disappears in the disabledClusters", func() {
 				framework.WaitDeploymentDisappearOnClusters(disabledClusters, deploymentNamespace, deploymentName)
 			})
@@ -451,6 +463,18 @@ var _ = framework.SerialDescribe("failover testing", func() {
 				framework.CreateOverridePolicy(karmadaClient, overridePolicy)
 			})
 
+			ginkgo.By("check if deployment present on member clusters has correct image value", func() {
+				framework.WaitDeploymentPresentOnClustersFitWith(disabledClusters, deployment.Namespace, deployment.Name,
+					func(deployment *appsv1.Deployment) bool {
+						for _, container := range deployment.Spec.Template.Spec.Containers {
+							if container.Image != "fake/nginx:1.19.0" {
+								return false
+							}
+						}
+						return true
+					})
+			})
+
 			ginkgo.By("check whether the failed deployment is rescheduled to other available cluster", func() {
 				gomega.Eventually(func() int {
 					targetClusterNames := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)

test/e2e/framework/deployment.go

@@ -95,7 +95,7 @@ func WaitDeploymentDisappearOnCluster(cluster, namespace, name string) {
 	clusterClient := GetClusterClient(cluster)
 	gomega.Expect(clusterClient).ShouldNot(gomega.BeNil())
 
-	klog.Infof("Waiting for deployment(%s/%s) disappear on cluster(%s)", namespace, name, cluster)
+	klog.Infof("Waiting for deployment(%s/%s) disappears on cluster(%s)", namespace, name, cluster)
 	gomega.Eventually(func() bool {
 		_, err := clusterClient.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{})
 		if err == nil {
@@ -112,7 +112,7 @@ func WaitDeploymentDisappearOnCluster(cluster, namespace, name string) {
 
 // WaitDeploymentDisappearOnClusters wait deployment disappear on member clusters until timeout.
 func WaitDeploymentDisappearOnClusters(clusters []string, namespace, name string) {
-	ginkgo.By(fmt.Sprintf("Check if deployment(%s/%s) diappeare on member clusters", namespace, name), func() {
+	ginkgo.By(fmt.Sprintf("Check if deployment(%s/%s) disappears on member clusters", namespace, name), func() {
 		for _, clusterName := range clusters {
 			WaitDeploymentDisappearOnCluster(clusterName, namespace, name)
 		}