diff --git a/pkg/controllers/status/cluster_status_controller.go b/pkg/controllers/status/cluster_status_controller.go index 73e522020..0a12a7915 100644 --- a/pkg/controllers/status/cluster_status_controller.go +++ b/pkg/controllers/status/cluster_status_controller.go @@ -177,8 +177,6 @@ func (c *ClusterStatusController) syncClusterStatus(cluster *clusterv1alpha1.Clu if !online && readyCondition.Status != metav1.ConditionTrue { klog.V(2).Infof("Cluster(%s) still offline after %s, ensuring offline is set.", cluster.Name, c.ClusterFailureThreshold.Duration) - c.GenericInformerManager.Stop(cluster.Name) - c.TypedInformerManager.Stop(cluster.Name) setTransitionTime(cluster.Status.Conditions, readyCondition) meta.SetStatusCondition(¤tClusterStatus.Conditions, *readyCondition) return c.updateStatusIfNeeded(cluster, currentClusterStatus) diff --git a/test/e2e/resource_test.go b/test/e2e/resource_test.go index 259433def..0b163af04 100644 --- a/test/e2e/resource_test.go +++ b/test/e2e/resource_test.go @@ -15,13 +15,18 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" + clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1" policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" + controllercluster "github.com/karmada-io/karmada/pkg/controllers/cluster" + "github.com/karmada-io/karmada/pkg/util" + "github.com/karmada-io/karmada/pkg/util/helper" "github.com/karmada-io/karmada/pkg/util/names" "github.com/karmada-io/karmada/test/e2e/framework" - "github.com/karmada-io/karmada/test/helper" + testhelper "github.com/karmada-io/karmada/test/helper" ) var _ = ginkgo.Describe("[resource-status collection] resource status collection testing", func() { @@ -45,8 +50,8 @@ var _ = ginkgo.Describe("[resource-status collection] resource status collection deploymentNamespace = testNamespace deploymentName = policyName - deployment = helper.NewDeployment(deploymentNamespace, deploymentName) - policy = helper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{ + deployment = testhelper.NewDeployment(deploymentNamespace, deploymentName) + policy = testhelper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{ { APIVersion: deployment.APIVersion, Kind: deployment.Kind, @@ -120,8 +125,8 @@ var _ = ginkgo.Describe("[resource-status collection] resource status collection serviceNamespace = testNamespace serviceName = policyName - service = helper.NewService(serviceNamespace, serviceName, corev1.ServiceTypeLoadBalancer) - policy = helper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{ + service = testhelper.NewService(serviceNamespace, serviceName, corev1.ServiceTypeLoadBalancer) + policy = testhelper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{ { APIVersion: service.APIVersion, Kind: service.Kind, @@ -192,8 +197,8 @@ var _ = ginkgo.Describe("[resource-status collection] resource status collection serviceNamespace = testNamespace serviceName = policyName - service = helper.NewService(serviceNamespace, serviceName, corev1.ServiceTypeNodePort) - policy = helper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{ + service = testhelper.NewService(serviceNamespace, serviceName, corev1.ServiceTypeNodePort) + policy = testhelper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{ { APIVersion: service.APIVersion, Kind: service.Kind, @@ -261,8 +266,8 @@ var _ = ginkgo.Describe("[resource-status collection] resource status collection ingNamespace = testNamespace ingName = policyName - ingress = helper.NewIngress(ingNamespace, ingName) - policy = helper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{ + ingress = testhelper.NewIngress(ingNamespace, ingName) + policy = testhelper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{ { APIVersion: ingress.APIVersion, Kind: ingress.Kind, @@ -333,8 +338,8 @@ var _ = ginkgo.Describe("[resource-status collection] resource status collection jobNamespace = testNamespace jobName = policyName - job = helper.NewJob(jobNamespace, jobName) - policy = helper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{ + job = testhelper.NewJob(jobNamespace, jobName) + policy = testhelper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{ { APIVersion: job.APIVersion, Kind: job.Kind, @@ -386,8 +391,8 @@ var _ = ginkgo.Describe("[resource-status collection] resource status collection daemonSetNamespace = testNamespace daemonSetName = policyName - daemonSet = helper.NewDaemonSet(daemonSetNamespace, daemonSetName) - policy = helper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{ + daemonSet = testhelper.NewDaemonSet(daemonSetNamespace, daemonSetName) + policy = testhelper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{ { APIVersion: daemonSet.APIVersion, Kind: daemonSet.Kind, @@ -471,8 +476,8 @@ var _ = ginkgo.Describe("[resource-status collection] resource status collection statefulSetNamespace = testNamespace statefulSetName = policyName - statefulSet = helper.NewStatefulSet(statefulSetNamespace, statefulSetName) - policy = helper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{ + statefulSet = testhelper.NewStatefulSet(statefulSetNamespace, statefulSetName) + policy = testhelper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{ { APIVersion: statefulSet.APIVersion, Kind: statefulSet.Kind, @@ -535,3 +540,115 @@ var _ = ginkgo.Describe("[resource-status collection] resource status collection }) }) }) + +var _ = framework.SerialDescribe("workload status synchronization testing", func() { + ginkgo.Context("Deployment status synchronization when cluster failed and recovered soon", func() { + var policyNamespace, policyName string + var deploymentNamespace, deploymentName string + var deployment *appsv1.Deployment + var policy *policyv1alpha1.PropagationPolicy + var originalReplicas, numOfFailedClusters int + + ginkgo.BeforeEach(func() { + policyNamespace = testNamespace + policyName = deploymentNamePrefix + rand.String(RandomStrLength) + deploymentNamespace = testNamespace + deploymentName = policyName + deployment = testhelper.NewDeployment(deploymentNamespace, deploymentName) + numOfFailedClusters = 1 + originalReplicas = 3 + + policy = testhelper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{ + { + APIVersion: deployment.APIVersion, + Kind: deployment.Kind, + Name: deployment.Name, + }, + }, policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + LabelSelector: &metav1.LabelSelector{ + // only test push mode clusters + // because pull mode clusters cannot be disabled by changing APIEndpoint + MatchLabels: pushModeClusterLabels, + }, + }, + ReplicaScheduling: &policyv1alpha1.ReplicaSchedulingStrategy{ + ReplicaSchedulingType: policyv1alpha1.ReplicaSchedulingTypeDuplicated, + }, + }) + }) + + ginkgo.BeforeEach(func() { + framework.CreatePropagationPolicy(karmadaClient, policy) + framework.CreateDeployment(kubeClient, deployment) + ginkgo.DeferCleanup(func() { + framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name) + framework.RemovePropagationPolicy(karmadaClient, policy.Namespace, policy.Name) + }) + }) + + ginkgo.It("deployment status synchronization testing", func() { + var disabledClusters []string + targetClusterNames := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment) + + ginkgo.By("set one cluster condition status to false", func() { + temp := numOfFailedClusters + for _, targetClusterName := range targetClusterNames { + if temp > 0 { + klog.Infof("Set cluster %s to disable.", targetClusterName) + err := disableCluster(controlPlaneClient, targetClusterName) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + // wait for the current cluster status changing to false + framework.WaitClusterFitWith(controlPlaneClient, targetClusterName, func(cluster *clusterv1alpha1.Cluster) bool { + return helper.TaintExists(cluster.Spec.Taints, controllercluster.NotReadyTaintTemplate) + }) + disabledClusters = append(disabledClusters, targetClusterName) + temp-- + } + } + }) + + ginkgo.By("recover not ready cluster", func() { + for _, disabledCluster := range disabledClusters { + fmt.Printf("cluster %s is waiting for recovering\n", disabledCluster) + originalAPIEndpoint := getClusterAPIEndpoint(disabledCluster) + + err := recoverCluster(controlPlaneClient, disabledCluster, originalAPIEndpoint) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + // wait for the disabled cluster recovered + err = wait.PollImmediate(pollInterval, pollTimeout, func() (done bool, err error) { + currentCluster, err := util.GetCluster(controlPlaneClient, disabledCluster) + if err != nil { + return false, err + } + if !helper.TaintExists(currentCluster.Spec.Taints, controllercluster.NotReadyTaintTemplate) { + fmt.Printf("cluster %s recovered\n", disabledCluster) + return true, nil + } + return false, nil + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + } + }) + + ginkgo.By("edit deployment in disabled cluster", func() { + for _, disabledCluster := range disabledClusters { + clusterClient := framework.GetClusterClient(disabledCluster) + framework.UpdateDeploymentReplicas(clusterClient, deployment, updateDeploymentReplicas) + // wait for the status synchronization + err := wait.PollImmediate(pollInterval, pollTimeout, func() (done bool, err error) { + currentDeployment, err := clusterClient.AppsV1().Deployments(testNamespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + if *currentDeployment.Spec.Replicas == int32(originalReplicas) { + return true, nil + } + return false, nil + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + } + }) + }) + }) +})