Merge pull request #4363 from jwcesign/jw-1
e2e: Fix the occasional failure of mcs e2e
This commit is contained in:
karmada-bot
GitHub
commit
d9f47afd4a
|
|
@ -74,17 +74,6 @@ var _ = ginkgo.Describe("[karmada-search] karmada search testing", ginkgo.Ordere
|
||||||
var pathNSDeploymentsFmt = pathPrefix + "apis/apps/v1/namespaces/%s/deployments"
|
var pathNSDeploymentsFmt = pathPrefix + "apis/apps/v1/namespaces/%s/deployments"
|
||||||
// var pathWithLabel = pathPrefix + "apis/apps/v1/namespaces/kube-system/deployments?labelSelector=k8s-app=kube-dns"
|
// var pathWithLabel = pathPrefix + "apis/apps/v1/namespaces/kube-system/deployments?labelSelector=k8s-app=kube-dns"
|
||||||
|
|
||||||
// var pollTimeout = 30 * time.Second
|
|
||||||
var searchObject = func(path, target string, exists bool) {
|
|
||||||
gomega.Eventually(func(g gomega.Gomega) (bool, error) {
|
|
||||||
res := karmadaClient.SearchV1alpha1().RESTClient().Get().AbsPath(path).Do(context.TODO())
|
|
||||||
g.Expect(res.Error()).ShouldNot(gomega.HaveOccurred())
|
|
||||||
raw, err := res.Raw()
|
|
||||||
g.Expect(err).ShouldNot(gomega.HaveOccurred())
|
|
||||||
return strings.Contains(string(raw), target), nil
|
|
||||||
}, pollTimeout, pollInterval).Should(gomega.Equal(exists))
|
|
||||||
}
|
|
||||||
|
|
||||||
ginkgo.BeforeAll(func() {
|
ginkgo.BeforeAll(func() {
|
||||||
// get clusters' name
|
// get clusters' name
|
||||||
pushModeClusters := framework.ClusterNamesWithSyncMode(clusterv1alpha1.Push)
|
pushModeClusters := framework.ClusterNamesWithSyncMode(clusterv1alpha1.Push)
|
||||||
|
|
@ -733,193 +722,214 @@ var _ = ginkgo.Describe("[karmada-search] karmada search testing", ginkgo.Ordere
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
})
|
||||||
|
|
||||||
// test when cluster joined, updated, karmada will reconcile search cache
|
// test when cluster joined, updated, karmada will reconcile search cache
|
||||||
ginkgo.Describe("reconcile ResourceRegistry when clusters joined, updated", ginkgo.Ordered, func() {
|
var _ = framework.SerialDescribe("reconcile ResourceRegistry when clusters joined, updated", func() {
|
||||||
ginkgo.Context("when cluster joined", func() {
|
var member1 = "member1"
|
||||||
var (
|
var member2 = "member2"
|
||||||
clusterName string
|
|
||||||
homeDir string
|
|
||||||
kubeConfigPath string
|
|
||||||
controlPlane string
|
|
||||||
clusterContext string
|
|
||||||
f cmdutil.Factory
|
|
||||||
|
|
||||||
rrName string
|
var member1PodName = "etcd-member1-control-plane"
|
||||||
rr *searchv1alpha1.ResourceRegistry
|
var member2PodName = "etcd-member2-control-plane"
|
||||||
resourceSelector searchv1alpha1.ResourceSelector
|
var existsDeploymentName = "coredns"
|
||||||
)
|
|
||||||
|
|
||||||
ginkgo.BeforeAll(func() {
|
var pathPrefix = "/apis/search.karmada.io/v1alpha1/search/cache/"
|
||||||
clusterName = "member-e2e-" + rand.String(RandomStrLength)
|
var pathAllPods = pathPrefix + "api/v1/pods"
|
||||||
homeDir = os.Getenv("HOME")
|
var pathAllDeployments = pathPrefix + "apis/apps/v1/deployments"
|
||||||
kubeConfigPath = fmt.Sprintf("%s/.kube/%s.config", homeDir, clusterName)
|
|
||||||
controlPlane = fmt.Sprintf("%s-control-plane", clusterName)
|
|
||||||
clusterContext = fmt.Sprintf("kind-%s", clusterName)
|
|
||||||
defaultConfigFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag().WithDiscoveryBurst(300).WithDiscoveryQPS(50.0)
|
|
||||||
defaultConfigFlags.Context = &karmadaContext
|
|
||||||
f = cmdutil.NewFactory(defaultConfigFlags)
|
|
||||||
|
|
||||||
// create a cluster
|
ginkgo.Context("when cluster joined", func() {
|
||||||
ginkgo.By(fmt.Sprintf("Creating cluster: %s", clusterName), func() {
|
var (
|
||||||
err := createCluster(clusterName, kubeConfigPath, controlPlane, clusterContext)
|
clusterName string
|
||||||
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
|
homeDir string
|
||||||
})
|
kubeConfigPath string
|
||||||
|
controlPlane string
|
||||||
|
clusterContext string
|
||||||
|
f cmdutil.Factory
|
||||||
|
|
||||||
// create a ResourceRegistry with a non joined cluster name
|
rrName string
|
||||||
rrName = resourceRegistryPrefix + rand.String(RandomStrLength)
|
rr *searchv1alpha1.ResourceRegistry
|
||||||
resourceSelector = searchv1alpha1.ResourceSelector{
|
resourceSelector searchv1alpha1.ResourceSelector
|
||||||
APIVersion: "apps/v1",
|
)
|
||||||
Kind: "Deployment",
|
|
||||||
}
|
ginkgo.BeforeEach(func() {
|
||||||
rr = &searchv1alpha1.ResourceRegistry{
|
clusterName = "member-e2e-" + rand.String(RandomStrLength)
|
||||||
ObjectMeta: metav1.ObjectMeta{
|
homeDir = os.Getenv("HOME")
|
||||||
Name: rrName,
|
kubeConfigPath = fmt.Sprintf("%s/.kube/%s.config", homeDir, clusterName)
|
||||||
},
|
controlPlane = fmt.Sprintf("%s-control-plane", clusterName)
|
||||||
Spec: searchv1alpha1.ResourceRegistrySpec{
|
clusterContext = fmt.Sprintf("kind-%s", clusterName)
|
||||||
TargetCluster: policyv1alpha1.ClusterAffinity{
|
defaultConfigFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag().WithDiscoveryBurst(300).WithDiscoveryQPS(50.0)
|
||||||
ClusterNames: []string{clusterName},
|
defaultConfigFlags.Context = &karmadaContext
|
||||||
},
|
f = cmdutil.NewFactory(defaultConfigFlags)
|
||||||
ResourceSelectors: []searchv1alpha1.ResourceSelector{
|
|
||||||
resourceSelector,
|
// create a cluster
|
||||||
},
|
ginkgo.By(fmt.Sprintf("Creating cluster: %s", clusterName), func() {
|
||||||
},
|
err := createCluster(clusterName, kubeConfigPath, controlPlane, clusterContext)
|
||||||
}
|
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
|
||||||
// create a ResourceRegistry with a non joined cluster name
|
|
||||||
ginkgo.By(fmt.Sprintf("create ResourceRegistry %s with a non joined cluster name %v and resource selector %v", rrName, clusterName, resourceSelector), func() {
|
|
||||||
framework.CreateResourceRegistry(karmadaClient, rr)
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
|
|
||||||
ginkgo.AfterAll(func() {
|
// create a ResourceRegistry with a non joined cluster name
|
||||||
framework.RemoveResourceRegistry(karmadaClient, rrName)
|
rrName = resourceRegistryPrefix + rand.String(RandomStrLength)
|
||||||
|
resourceSelector = searchv1alpha1.ResourceSelector{
|
||||||
ginkgo.By(fmt.Sprintf("Deleting clusters: %s", clusterName), func() {
|
APIVersion: "apps/v1",
|
||||||
err := deleteCluster(clusterName, kubeConfigPath)
|
Kind: "Deployment",
|
||||||
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
|
}
|
||||||
_ = os.Remove(kubeConfigPath)
|
rr = &searchv1alpha1.ResourceRegistry{
|
||||||
})
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
})
|
Name: rrName,
|
||||||
|
},
|
||||||
ginkgo.It("[member clusters joined] could reconcile ResourceRegistry", func() {
|
Spec: searchv1alpha1.ResourceRegistrySpec{
|
||||||
// search cache should not have the deployment
|
TargetCluster: policyv1alpha1.ClusterAffinity{
|
||||||
searchObject(pathAllDeployments, existsDeploymentName, false)
|
ClusterNames: []string{clusterName},
|
||||||
// join the cluster
|
},
|
||||||
ginkgo.By(fmt.Sprintf("Joining cluster: %s", clusterName), func() {
|
ResourceSelectors: []searchv1alpha1.ResourceSelector{
|
||||||
opts := join.CommandJoinOption{
|
resourceSelector,
|
||||||
DryRun: false,
|
},
|
||||||
ClusterNamespace: "karmada-cluster",
|
},
|
||||||
ClusterName: clusterName,
|
}
|
||||||
ClusterContext: clusterContext,
|
// create a ResourceRegistry with a non joined cluster name
|
||||||
ClusterKubeConfig: kubeConfigPath,
|
ginkgo.By(fmt.Sprintf("create ResourceRegistry %s with a non joined cluster name %v and resource selector %v", rrName, clusterName, resourceSelector), func() {
|
||||||
}
|
framework.CreateResourceRegistry(karmadaClient, rr)
|
||||||
err := opts.Run(f)
|
|
||||||
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
|
|
||||||
})
|
|
||||||
// search cache should have the deployment
|
|
||||||
searchObject(pathAllDeployments, existsDeploymentName, true)
|
|
||||||
|
|
||||||
ginkgo.By(fmt.Sprintf("Unjoinning cluster: %s", clusterName), func() {
|
|
||||||
opts := unjoin.CommandUnjoinOption{
|
|
||||||
DryRun: false,
|
|
||||||
ClusterNamespace: "karmada-cluster",
|
|
||||||
ClusterName: clusterName,
|
|
||||||
ClusterContext: clusterContext,
|
|
||||||
ClusterKubeConfig: kubeConfigPath,
|
|
||||||
Wait: 5 * options.DefaultKarmadactlCommandDuration,
|
|
||||||
}
|
|
||||||
err := opts.Run(f)
|
|
||||||
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
ginkgo.Context("when cluster updated", func() {
|
ginkgo.AfterEach(func() {
|
||||||
var (
|
framework.RemoveResourceRegistry(karmadaClient, rrName)
|
||||||
rrName string
|
|
||||||
rr *searchv1alpha1.ResourceRegistry
|
|
||||||
labelKey string
|
|
||||||
labelValue string
|
|
||||||
labelSelector *metav1.LabelSelector
|
|
||||||
resourceSelector searchv1alpha1.ResourceSelector
|
|
||||||
)
|
|
||||||
|
|
||||||
ginkgo.BeforeAll(func() {
|
ginkgo.By(fmt.Sprintf("Deleting clusters: %s", clusterName), func() {
|
||||||
// create a ResourceRegistry with label selector
|
err := deleteCluster(clusterName, kubeConfigPath)
|
||||||
rrName = resourceRegistryPrefix + rand.String(RandomStrLength)
|
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
|
||||||
labelKey = "karmada-cluster.k8s.some.com/managed"
|
_ = os.Remove(kubeConfigPath)
|
||||||
labelValue = "true"
|
|
||||||
labelSelector = &metav1.LabelSelector{
|
|
||||||
MatchExpressions: []metav1.LabelSelectorRequirement{
|
|
||||||
{
|
|
||||||
Key: labelKey,
|
|
||||||
Operator: metav1.LabelSelectorOpExists,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
resourceSelector = searchv1alpha1.ResourceSelector{
|
|
||||||
APIVersion: "v1",
|
|
||||||
Kind: "Pod",
|
|
||||||
}
|
|
||||||
rr = &searchv1alpha1.ResourceRegistry{
|
|
||||||
ObjectMeta: metav1.ObjectMeta{
|
|
||||||
Name: rrName,
|
|
||||||
},
|
|
||||||
Spec: searchv1alpha1.ResourceRegistrySpec{
|
|
||||||
TargetCluster: policyv1alpha1.ClusterAffinity{
|
|
||||||
LabelSelector: labelSelector,
|
|
||||||
},
|
|
||||||
ResourceSelectors: []searchv1alpha1.ResourceSelector{
|
|
||||||
resourceSelector,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
ginkgo.By(fmt.Sprintf("create ResourceRegistry %s with label selector %v and resource selector %v", rrName, labelSelector, resourceSelector), func() {
|
|
||||||
framework.CreateResourceRegistry(karmadaClient, rr)
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
|
})
|
||||||
|
|
||||||
ginkgo.AfterAll(func() {
|
ginkgo.It("[member clusters joined] could reconcile ResourceRegistry", func() {
|
||||||
framework.RemoveResourceRegistry(karmadaClient, rrName)
|
// search cache should not have the deployment
|
||||||
|
searchObject(pathAllDeployments, existsDeploymentName, false)
|
||||||
|
// join the cluster
|
||||||
|
ginkgo.By(fmt.Sprintf("Joining cluster: %s", clusterName), func() {
|
||||||
|
opts := join.CommandJoinOption{
|
||||||
|
DryRun: false,
|
||||||
|
ClusterNamespace: "karmada-cluster",
|
||||||
|
ClusterName: clusterName,
|
||||||
|
ClusterContext: clusterContext,
|
||||||
|
ClusterKubeConfig: kubeConfigPath,
|
||||||
|
}
|
||||||
|
err := opts.Run(f)
|
||||||
|
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
|
||||||
})
|
})
|
||||||
|
// search cache should have the deployment
|
||||||
|
searchObject(pathAllDeployments, existsDeploymentName, true)
|
||||||
|
|
||||||
ginkgo.It("[member clusters updated, deleted label] could reconcile ResourceRegistry", func() {
|
ginkgo.By(fmt.Sprintf("Unjoinning cluster: %s", clusterName), func() {
|
||||||
// search the pod in the ResourceRegistry will fail
|
opts := unjoin.CommandUnjoinOption{
|
||||||
searchObject(pathAllPods, member1PodName, false)
|
DryRun: false,
|
||||||
|
ClusterNamespace: "karmada-cluster",
|
||||||
// add the label to the member1 cluster
|
ClusterName: clusterName,
|
||||||
ginkgo.By(fmt.Sprintf("add label %s=%s to cluster %s", labelKey, labelValue, member1), func() {
|
ClusterContext: clusterContext,
|
||||||
framework.UpdateClusterLabels(karmadaClient, member1, map[string]string{labelKey: labelValue})
|
ClusterKubeConfig: kubeConfigPath,
|
||||||
})
|
Wait: 5 * options.DefaultKarmadactlCommandDuration,
|
||||||
|
}
|
||||||
// search the pod in the ResourceRegistry will success
|
err := opts.Run(f)
|
||||||
searchObject(pathAllPods, member1PodName, true)
|
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
|
||||||
|
|
||||||
// add the label to the member2 cluster
|
|
||||||
ginkgo.By(fmt.Sprintf("add label %s=%s to cluster %s", labelKey, labelValue, member2), func() {
|
|
||||||
framework.UpdateClusterLabels(karmadaClient, member2, map[string]string{labelKey: labelValue})
|
|
||||||
})
|
|
||||||
|
|
||||||
// search the pod in the ResourceRegistry will success
|
|
||||||
searchObject(pathAllPods, member2PodName, true)
|
|
||||||
|
|
||||||
// delete the label of the member2 cluster
|
|
||||||
ginkgo.By(fmt.Sprintf("delete label %s=%s to cluster %s", labelKey, labelValue, member2), func() {
|
|
||||||
framework.DeleteClusterLabels(karmadaClient, member2, map[string]string{labelKey: labelValue})
|
|
||||||
})
|
|
||||||
|
|
||||||
// search the pod in the ResourceRegistry will fail
|
|
||||||
searchObject(pathAllPods, member2PodName, false)
|
|
||||||
|
|
||||||
// delete the label of the member1 cluster
|
|
||||||
ginkgo.By(fmt.Sprintf("delete label %s=%s to cluster %s", labelKey, labelValue, member1), func() {
|
|
||||||
framework.DeleteClusterLabels(karmadaClient, member1, map[string]string{labelKey: labelValue})
|
|
||||||
|
|
||||||
})
|
|
||||||
|
|
||||||
// search the pod in the ResourceRegistry will fail
|
|
||||||
searchObject(pathAllPods, member1PodName, false)
|
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
ginkgo.Context("when cluster updated", func() {
|
||||||
|
var (
|
||||||
|
rrName string
|
||||||
|
rr *searchv1alpha1.ResourceRegistry
|
||||||
|
labelKey string
|
||||||
|
labelValue string
|
||||||
|
labelSelector *metav1.LabelSelector
|
||||||
|
resourceSelector searchv1alpha1.ResourceSelector
|
||||||
|
)
|
||||||
|
|
||||||
|
ginkgo.BeforeEach(func() {
|
||||||
|
// create a ResourceRegistry with label selector
|
||||||
|
rrName = resourceRegistryPrefix + rand.String(RandomStrLength)
|
||||||
|
labelKey = "karmada-cluster.k8s.some.com/managed"
|
||||||
|
labelValue = "true"
|
||||||
|
labelSelector = &metav1.LabelSelector{
|
||||||
|
MatchExpressions: []metav1.LabelSelectorRequirement{
|
||||||
|
{
|
||||||
|
Key: labelKey,
|
||||||
|
Operator: metav1.LabelSelectorOpExists,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
resourceSelector = searchv1alpha1.ResourceSelector{
|
||||||
|
APIVersion: "v1",
|
||||||
|
Kind: "Pod",
|
||||||
|
}
|
||||||
|
rr = &searchv1alpha1.ResourceRegistry{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: rrName,
|
||||||
|
},
|
||||||
|
Spec: searchv1alpha1.ResourceRegistrySpec{
|
||||||
|
TargetCluster: policyv1alpha1.ClusterAffinity{
|
||||||
|
LabelSelector: labelSelector,
|
||||||
|
},
|
||||||
|
ResourceSelectors: []searchv1alpha1.ResourceSelector{
|
||||||
|
resourceSelector,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
ginkgo.By(fmt.Sprintf("create ResourceRegistry %s with label selector %v and resource selector %v", rrName, labelSelector, resourceSelector), func() {
|
||||||
|
framework.CreateResourceRegistry(karmadaClient, rr)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
ginkgo.AfterEach(func() {
|
||||||
|
framework.RemoveResourceRegistry(karmadaClient, rrName)
|
||||||
|
})
|
||||||
|
|
||||||
|
ginkgo.It("[member clusters updated, deleted label] could reconcile ResourceRegistry", func() {
|
||||||
|
// search the pod in the ResourceRegistry will fail
|
||||||
|
searchObject(pathAllPods, member1PodName, false)
|
||||||
|
|
||||||
|
// add the label to the member1 cluster
|
||||||
|
ginkgo.By(fmt.Sprintf("add label %s=%s to cluster %s", labelKey, labelValue, member1), func() {
|
||||||
|
framework.UpdateClusterLabels(karmadaClient, member1, map[string]string{labelKey: labelValue})
|
||||||
|
})
|
||||||
|
|
||||||
|
// search the pod in the ResourceRegistry will success
|
||||||
|
searchObject(pathAllPods, member1PodName, true)
|
||||||
|
|
||||||
|
// add the label to the member2 cluster
|
||||||
|
ginkgo.By(fmt.Sprintf("add label %s=%s to cluster %s", labelKey, labelValue, member2), func() {
|
||||||
|
framework.UpdateClusterLabels(karmadaClient, member2, map[string]string{labelKey: labelValue})
|
||||||
|
})
|
||||||
|
|
||||||
|
// search the pod in the ResourceRegistry will success
|
||||||
|
searchObject(pathAllPods, member2PodName, true)
|
||||||
|
|
||||||
|
// delete the label of the member2 cluster
|
||||||
|
ginkgo.By(fmt.Sprintf("delete label %s=%s to cluster %s", labelKey, labelValue, member2), func() {
|
||||||
|
framework.DeleteClusterLabels(karmadaClient, member2, map[string]string{labelKey: labelValue})
|
||||||
|
})
|
||||||
|
|
||||||
|
// search the pod in the ResourceRegistry will fail
|
||||||
|
searchObject(pathAllPods, member2PodName, false)
|
||||||
|
|
||||||
|
// delete the label of the member1 cluster
|
||||||
|
ginkgo.By(fmt.Sprintf("delete label %s=%s to cluster %s", labelKey, labelValue, member1), func() {
|
||||||
|
framework.DeleteClusterLabels(karmadaClient, member1, map[string]string{labelKey: labelValue})
|
||||||
|
|
||||||
|
})
|
||||||
|
|
||||||
|
// search the pod in the ResourceRegistry will fail
|
||||||
|
searchObject(pathAllPods, member1PodName, false)
|
||||||
|
})
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
func searchObject(path, target string, exists bool) {
|
||||||
|
gomega.Eventually(func(g gomega.Gomega) (bool, error) {
|
||||||
|
res := karmadaClient.SearchV1alpha1().RESTClient().Get().AbsPath(path).Do(context.TODO())
|
||||||
|
g.Expect(res.Error()).ShouldNot(gomega.HaveOccurred())
|
||||||
|
raw, err := res.Raw()
|
||||||
|
g.Expect(err).ShouldNot(gomega.HaveOccurred())
|
||||||
|
return strings.Contains(string(raw), target), nil
|
||||||
|
}, pollTimeout, pollInterval).Should(gomega.Equal(exists))
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue