fix work status not sync to control plane
Signed-off-by: Poor12 <shentiecheng@huawei.com>
This commit is contained in:
Poor12
parent
5f673cbde8
commit
7c4cd0b5b6
|
|
@ -177,8 +177,6 @@ func (c *ClusterStatusController) syncClusterStatus(cluster *clusterv1alpha1.Clu
|
|||
if !online && readyCondition.Status != metav1.ConditionTrue {
|
||||
klog.V(2).Infof("Cluster(%s) still offline after %s, ensuring offline is set.",
|
||||
cluster.Name, c.ClusterFailureThreshold.Duration)
|
||||
c.GenericInformerManager.Stop(cluster.Name)
|
||||
c.TypedInformerManager.Stop(cluster.Name)
|
||||
setTransitionTime(cluster.Status.Conditions, readyCondition)
|
||||
meta.SetStatusCondition(¤tClusterStatus.Conditions, *readyCondition)
|
||||
return c.updateStatusIfNeeded(cluster, currentClusterStatus)
|
||||
|
|
|
|||
|
|
@ -15,13 +15,18 @@ import (
|
|||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/rand"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/klog/v2"
|
||||
|
||||
clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
|
||||
policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
|
||||
workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
|
||||
controllercluster "github.com/karmada-io/karmada/pkg/controllers/cluster"
|
||||
"github.com/karmada-io/karmada/pkg/util"
|
||||
"github.com/karmada-io/karmada/pkg/util/helper"
|
||||
"github.com/karmada-io/karmada/pkg/util/names"
|
||||
"github.com/karmada-io/karmada/test/e2e/framework"
|
||||
"github.com/karmada-io/karmada/test/helper"
|
||||
testhelper "github.com/karmada-io/karmada/test/helper"
|
||||
)
|
||||
|
||||
var _ = ginkgo.Describe("[resource-status collection] resource status collection testing", func() {
|
||||
|
|
@ -45,8 +50,8 @@ var _ = ginkgo.Describe("[resource-status collection] resource status collection
|
|||
deploymentNamespace = testNamespace
|
||||
deploymentName = policyName
|
||||
|
||||
deployment = helper.NewDeployment(deploymentNamespace, deploymentName)
|
||||
policy = helper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
|
||||
deployment = testhelper.NewDeployment(deploymentNamespace, deploymentName)
|
||||
policy = testhelper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
|
||||
{
|
||||
APIVersion: deployment.APIVersion,
|
||||
Kind: deployment.Kind,
|
||||
|
|
@ -120,8 +125,8 @@ var _ = ginkgo.Describe("[resource-status collection] resource status collection
|
|||
serviceNamespace = testNamespace
|
||||
serviceName = policyName
|
||||
|
||||
service = helper.NewService(serviceNamespace, serviceName, corev1.ServiceTypeLoadBalancer)
|
||||
policy = helper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
|
||||
service = testhelper.NewService(serviceNamespace, serviceName, corev1.ServiceTypeLoadBalancer)
|
||||
policy = testhelper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
|
||||
{
|
||||
APIVersion: service.APIVersion,
|
||||
Kind: service.Kind,
|
||||
|
|
@ -192,8 +197,8 @@ var _ = ginkgo.Describe("[resource-status collection] resource status collection
|
|||
serviceNamespace = testNamespace
|
||||
serviceName = policyName
|
||||
|
||||
service = helper.NewService(serviceNamespace, serviceName, corev1.ServiceTypeNodePort)
|
||||
policy = helper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
|
||||
service = testhelper.NewService(serviceNamespace, serviceName, corev1.ServiceTypeNodePort)
|
||||
policy = testhelper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
|
||||
{
|
||||
APIVersion: service.APIVersion,
|
||||
Kind: service.Kind,
|
||||
|
|
@ -261,8 +266,8 @@ var _ = ginkgo.Describe("[resource-status collection] resource status collection
|
|||
ingNamespace = testNamespace
|
||||
ingName = policyName
|
||||
|
||||
ingress = helper.NewIngress(ingNamespace, ingName)
|
||||
policy = helper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
|
||||
ingress = testhelper.NewIngress(ingNamespace, ingName)
|
||||
policy = testhelper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
|
||||
{
|
||||
APIVersion: ingress.APIVersion,
|
||||
Kind: ingress.Kind,
|
||||
|
|
@ -333,8 +338,8 @@ var _ = ginkgo.Describe("[resource-status collection] resource status collection
|
|||
jobNamespace = testNamespace
|
||||
jobName = policyName
|
||||
|
||||
job = helper.NewJob(jobNamespace, jobName)
|
||||
policy = helper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
|
||||
job = testhelper.NewJob(jobNamespace, jobName)
|
||||
policy = testhelper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
|
||||
{
|
||||
APIVersion: job.APIVersion,
|
||||
Kind: job.Kind,
|
||||
|
|
@ -386,8 +391,8 @@ var _ = ginkgo.Describe("[resource-status collection] resource status collection
|
|||
daemonSetNamespace = testNamespace
|
||||
daemonSetName = policyName
|
||||
|
||||
daemonSet = helper.NewDaemonSet(daemonSetNamespace, daemonSetName)
|
||||
policy = helper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
|
||||
daemonSet = testhelper.NewDaemonSet(daemonSetNamespace, daemonSetName)
|
||||
policy = testhelper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
|
||||
{
|
||||
APIVersion: daemonSet.APIVersion,
|
||||
Kind: daemonSet.Kind,
|
||||
|
|
@ -471,8 +476,8 @@ var _ = ginkgo.Describe("[resource-status collection] resource status collection
|
|||
statefulSetNamespace = testNamespace
|
||||
statefulSetName = policyName
|
||||
|
||||
statefulSet = helper.NewStatefulSet(statefulSetNamespace, statefulSetName)
|
||||
policy = helper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
|
||||
statefulSet = testhelper.NewStatefulSet(statefulSetNamespace, statefulSetName)
|
||||
policy = testhelper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
|
||||
{
|
||||
APIVersion: statefulSet.APIVersion,
|
||||
Kind: statefulSet.Kind,
|
||||
|
|
@ -535,3 +540,115 @@ var _ = ginkgo.Describe("[resource-status collection] resource status collection
|
|||
})
|
||||
})
|
||||
})
|
||||
|
||||
var _ = framework.SerialDescribe("workload status synchronization testing", func() {
|
||||
ginkgo.Context("Deployment status synchronization when cluster failed and recovered soon", func() {
|
||||
var policyNamespace, policyName string
|
||||
var deploymentNamespace, deploymentName string
|
||||
var deployment *appsv1.Deployment
|
||||
var policy *policyv1alpha1.PropagationPolicy
|
||||
var originalReplicas, numOfFailedClusters int
|
||||
|
||||
ginkgo.BeforeEach(func() {
|
||||
policyNamespace = testNamespace
|
||||
policyName = deploymentNamePrefix + rand.String(RandomStrLength)
|
||||
deploymentNamespace = testNamespace
|
||||
deploymentName = policyName
|
||||
deployment = testhelper.NewDeployment(deploymentNamespace, deploymentName)
|
||||
numOfFailedClusters = 1
|
||||
originalReplicas = 3
|
||||
|
||||
policy = testhelper.NewPropagationPolicy(policyNamespace, policyName, []policyv1alpha1.ResourceSelector{
|
||||
{
|
||||
APIVersion: deployment.APIVersion,
|
||||
Kind: deployment.Kind,
|
||||
Name: deployment.Name,
|
||||
},
|
||||
}, policyv1alpha1.Placement{
|
||||
ClusterAffinity: &policyv1alpha1.ClusterAffinity{
|
||||
LabelSelector: &metav1.LabelSelector{
|
||||
// only test push mode clusters
|
||||
// because pull mode clusters cannot be disabled by changing APIEndpoint
|
||||
MatchLabels: pushModeClusterLabels,
|
||||
},
|
||||
},
|
||||
ReplicaScheduling: &policyv1alpha1.ReplicaSchedulingStrategy{
|
||||
ReplicaSchedulingType: policyv1alpha1.ReplicaSchedulingTypeDuplicated,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
ginkgo.BeforeEach(func() {
|
||||
framework.CreatePropagationPolicy(karmadaClient, policy)
|
||||
framework.CreateDeployment(kubeClient, deployment)
|
||||
ginkgo.DeferCleanup(func() {
|
||||
framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name)
|
||||
framework.RemovePropagationPolicy(karmadaClient, policy.Namespace, policy.Name)
|
||||
})
|
||||
})
|
||||
|
||||
ginkgo.It("deployment status synchronization testing", func() {
|
||||
var disabledClusters []string
|
||||
targetClusterNames := framework.ExtractTargetClustersFrom(controlPlaneClient, deployment)
|
||||
|
||||
ginkgo.By("set one cluster condition status to false", func() {
|
||||
temp := numOfFailedClusters
|
||||
for _, targetClusterName := range targetClusterNames {
|
||||
if temp > 0 {
|
||||
klog.Infof("Set cluster %s to disable.", targetClusterName)
|
||||
err := disableCluster(controlPlaneClient, targetClusterName)
|
||||
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
|
||||
|
||||
// wait for the current cluster status changing to false
|
||||
framework.WaitClusterFitWith(controlPlaneClient, targetClusterName, func(cluster *clusterv1alpha1.Cluster) bool {
|
||||
return helper.TaintExists(cluster.Spec.Taints, controllercluster.NotReadyTaintTemplate)
|
||||
})
|
||||
disabledClusters = append(disabledClusters, targetClusterName)
|
||||
temp--
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
ginkgo.By("recover not ready cluster", func() {
|
||||
for _, disabledCluster := range disabledClusters {
|
||||
fmt.Printf("cluster %s is waiting for recovering\n", disabledCluster)
|
||||
originalAPIEndpoint := getClusterAPIEndpoint(disabledCluster)
|
||||
|
||||
err := recoverCluster(controlPlaneClient, disabledCluster, originalAPIEndpoint)
|
||||
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
|
||||
// wait for the disabled cluster recovered
|
||||
err = wait.PollImmediate(pollInterval, pollTimeout, func() (done bool, err error) {
|
||||
currentCluster, err := util.GetCluster(controlPlaneClient, disabledCluster)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !helper.TaintExists(currentCluster.Spec.Taints, controllercluster.NotReadyTaintTemplate) {
|
||||
fmt.Printf("cluster %s recovered\n", disabledCluster)
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
|
||||
}
|
||||
})
|
||||
|
||||
ginkgo.By("edit deployment in disabled cluster", func() {
|
||||
for _, disabledCluster := range disabledClusters {
|
||||
clusterClient := framework.GetClusterClient(disabledCluster)
|
||||
framework.UpdateDeploymentReplicas(clusterClient, deployment, updateDeploymentReplicas)
|
||||
// wait for the status synchronization
|
||||
err := wait.PollImmediate(pollInterval, pollTimeout, func() (done bool, err error) {
|
||||
currentDeployment, err := clusterClient.AppsV1().Deployments(testNamespace).Get(context.TODO(), deploymentName, metav1.GetOptions{})
|
||||
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
|
||||
|
||||
if *currentDeployment.Spec.Replicas == int32(originalReplicas) {
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Reference in New Issue