diff --git a/test/e2e/framework/cluster.go b/test/e2e/framework/cluster.go index fd327d4b8..79a9c920a 100644 --- a/test/e2e/framework/cluster.go +++ b/test/e2e/framework/cluster.go @@ -45,6 +45,17 @@ func ClusterNames() []string { return clusterNames } +// ClusterNamesWithSyncMode will return member clusters' names which matches the sync mode. +func ClusterNamesWithSyncMode(mode clusterv1alpha1.ClusterSyncMode) []string { + res := make([]string, 0, len(clusterNames)) + for _, cluster := range clusters { + if cluster.Spec.SyncMode == mode { + res = append(res, cluster.Name) + } + } + return res +} + // InitClusterInformation init the E2E test's cluster information. func InitClusterInformation(karmadaClient karmada.Interface, controlPlaneClient client.Client) { var err error diff --git a/test/e2e/framework/dynamic.go b/test/e2e/framework/dynamic.go new file mode 100644 index 000000000..ad2d78965 --- /dev/null +++ b/test/e2e/framework/dynamic.go @@ -0,0 +1,36 @@ +package framework + +import ( + "context" + + "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/dynamic" +) + +// GetResourceNames list resources and return their names. +func GetResourceNames(client dynamic.ResourceInterface) sets.String { + list, err := client.List(context.TODO(), metav1.ListOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + names := sets.NewString() + for _, item := range list.Items { + names.Insert(item.GetName()) + } + return names +} + +// GetAnyResourceOrFail list resources and return anyone. Failed if listing empty. +func GetAnyResourceOrFail(client dynamic.ResourceInterface) *unstructured.Unstructured { + list, err := client.List(context.TODO(), metav1.ListOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(list.Items).ShouldNot(gomega.BeEmpty()) + + if len(list.Items) == 1 { + return &list.Items[0] + } + return &list.Items[rand.Intn(len(list.Items))] +} diff --git a/test/e2e/search_test.go b/test/e2e/search_test.go index 92b202673..7480d4e75 100644 --- a/test/e2e/search_test.go +++ b/test/e2e/search_test.go @@ -3,22 +3,30 @@ package e2e import ( "context" "fmt" + "sort" "strings" + "time" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" + clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1" policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" searchv1alpha1 "github.com/karmada-io/karmada/pkg/apis/search/v1alpha1" "github.com/karmada-io/karmada/test/e2e/framework" testhelper "github.com/karmada-io/karmada/test/helper" ) -var _ = ginkgo.Describe("[karmada-search] karmada search testing", func() { +var _ = ginkgo.Describe("[karmada-search] karmada search testing", ginkgo.Ordered, func() { var member1 = "member1" var member2 = "member2" @@ -54,6 +62,20 @@ var _ = ginkgo.Describe("[karmada-search] karmada search testing", func() { }, pollTimeout, pollInterval).Should(gomega.Equal(exists)) } + ginkgo.BeforeAll(func() { + // get clusters' name + pushModeClusters := framework.ClusterNamesWithSyncMode(clusterv1alpha1.Push) + ginkgo.By(fmt.Sprintf("get %v clusters in push mode", len(pushModeClusters))) + gomega.Expect(len(pushModeClusters) >= 2).Should(gomega.BeTrue()) + pushModeClusters = pushModeClusters[:2] + sort.Strings(pushModeClusters) + member1, member2 = pushModeClusters[0], pushModeClusters[1] + ginkgo.By(fmt.Sprintf("test on %v and %v", member1, member2)) + + // clean ResourceRegistries before test + gomega.Expect(karmadaClient.SearchV1alpha1().ResourceRegistries().DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{})).Should(gomega.Succeed()) + }) + // use service as search object ginkgo.Describe("no ResourceRegistry testings", func() { ginkgo.It("[Service] should be not searchable", func() { @@ -320,4 +342,286 @@ var _ = ginkgo.Describe("[karmada-search] karmada search testing", func() { ginkgo.Describe("backend store testing", ginkgo.Ordered, func() { }) + + ginkgo.Describe("karmada proxy testing", ginkgo.Ordered, func() { + var ( + m1Client, m2Client, proxyClient kubernetes.Interface + m1Dynamic, m2Dynamic, proxyDynamic dynamic.Interface + nodeGVR = corev1.SchemeGroupVersion.WithResource("nodes") + ) + + ginkgo.BeforeAll(func() { + m1Dynamic = framework.GetClusterDynamicClient(member1) + gomega.Expect(m1Dynamic).ShouldNot(gomega.BeNil()) + m2Dynamic = framework.GetClusterDynamicClient(member2) + gomega.Expect(m2Dynamic).ShouldNot(gomega.BeNil()) + + m1Client = framework.GetClusterClient(member1) + gomega.Expect(m1Client).ShouldNot(gomega.BeNil()) + m2Client = framework.GetClusterClient(member2) + gomega.Expect(m2Client).ShouldNot(gomega.BeNil()) + + proxyConfig := *restConfig + proxyConfig.Host += "/apis/search.karmada.io/v1alpha1/proxying/karmada/proxy" + proxyClient = kubernetes.NewForConfigOrDie(&proxyConfig) + proxyDynamic = dynamic.NewForConfigOrDie(&proxyConfig) + }) + + ginkgo.Describe("resourceRegistry testings", func() { + var ( + rr1, rr2 *searchv1alpha1.ResourceRegistry + m1Deploy, m2Deploy *appsv1.Deployment + ) + + ginkgo.BeforeAll(func() { + rr1 = &searchv1alpha1.ResourceRegistry{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rr-" + rand.String(RandomStrLength), + }, + Spec: searchv1alpha1.ResourceRegistrySpec{ + TargetCluster: policyv1alpha1.ClusterAffinity{ + ClusterNames: []string{member1, member2}, + }, + ResourceSelectors: []searchv1alpha1.ResourceSelector{ + { + APIVersion: "v1", + Kind: "Node", + }, + }, + }, + } + rr2 = &searchv1alpha1.ResourceRegistry{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rr-" + rand.String(RandomStrLength), + }, + Spec: searchv1alpha1.ResourceRegistrySpec{ + TargetCluster: policyv1alpha1.ClusterAffinity{ + ClusterNames: []string{member1, member2}, + }, + ResourceSelectors: []searchv1alpha1.ResourceSelector{ + { + APIVersion: "v1", + Kind: "Node", + }, + { + APIVersion: "v1", + Kind: "Pod", + }, + }, + }, + } + + m1Deploy = testhelper.NewDeployment(testNamespace, "proxy-m1-"+rand.String(RandomStrLength)) + framework.CreateDeployment(m1Client, m1Deploy) + + m2Deploy = testhelper.NewDeployment(testNamespace, "proxy-m2-"+rand.String(RandomStrLength)) + framework.CreateDeployment(m2Client, m2Deploy) + }) + + ginkgo.AfterAll(func() { + err := karmadaClient.SearchV1alpha1().ResourceRegistries().Delete(context.TODO(), rr1.Name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + } + err = karmadaClient.SearchV1alpha1().ResourceRegistries().Delete(context.TODO(), rr2.Name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + } + + framework.RemoveDeployment(m1Client, testNamespace, m1Deploy.Name) + framework.RemoveDeployment(m2Client, testNamespace, m2Deploy.Name) + }) + + ginkgo.It("create, update, delete resourceRegistry", func() { + ginkgo.By("no resourceRegistries testings", func() { + ginkgo.By("should not list pods", func() { + gomega.Eventually(func(g gomega.Gomega) { + list, err := proxyClient.CoreV1().Pods(testNamespace).List(context.TODO(), metav1.ListOptions{}) + g.Expect(err).ShouldNot(gomega.HaveOccurred()) + g.Expect(list.Items).Should(gomega.BeEmpty()) + }, time.Second*10).Should(gomega.Succeed()) + }) + + ginkgo.By("should not list nodes", func() { + gomega.Eventually(func(g gomega.Gomega) { + list, err := proxyClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + g.Expect(err).ShouldNot(gomega.HaveOccurred()) + g.Expect(list.Items).Should(gomega.BeEmpty()) + }, time.Second*10).Should(gomega.Succeed()) + }) + }) + + ginkgo.By("create resourceRegistry rr1 for nodes", func() { + framework.CreateResourceRegistry(karmadaClient, rr1) + + ginkgo.By("should not list pods", func() { + gomega.Eventually(func(g gomega.Gomega) { + list, err := proxyClient.CoreV1().Pods(testNamespace).List(context.TODO(), metav1.ListOptions{}) + g.Expect(err).ShouldNot(gomega.HaveOccurred()) + g.Expect(list.Items).Should(gomega.BeEmpty()) + }, time.Second*10).Should(gomega.Succeed()) + }) + + ginkgo.By("should list nodes", func() { + gomega.Eventually(func(g gomega.Gomega) { + list, err := proxyClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + g.Expect(err).ShouldNot(gomega.HaveOccurred()) + g.Expect(list.Items).ShouldNot(gomega.BeEmpty()) + }, time.Second*10).Should(gomega.Succeed()) + }) + }) + + ginkgo.By("create resourceRegistry rr2 for pods, nodes", func() { + framework.CreateResourceRegistry(karmadaClient, rr2) + + ginkgo.By("should list pods", func() { + gomega.Eventually(func(g gomega.Gomega) { + list, err := proxyClient.CoreV1().Pods(testNamespace).List(context.TODO(), metav1.ListOptions{}) + g.Expect(err).ShouldNot(gomega.HaveOccurred()) + g.Expect(list.Items).ShouldNot(gomega.BeEmpty()) + }, time.Second*10).Should(gomega.Succeed()) + }) + + ginkgo.By("should list nodes", func() { + gomega.Eventually(func(g gomega.Gomega) { + list, err := proxyClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + g.Expect(err).ShouldNot(gomega.HaveOccurred()) + g.Expect(list.Items).ShouldNot(gomega.BeEmpty()) + }, time.Second*10).Should(gomega.Succeed()) + }) + }) + + ginkgo.By("delete resourceRegistries rr1 and rr2", func() { + framework.RemoveResourceRegistry(karmadaClient, rr1.Name) + framework.RemoveResourceRegistry(karmadaClient, rr2.Name) + + ginkgo.By("should not list pods", func() { + gomega.Eventually(func(g gomega.Gomega) { + list, err := proxyClient.CoreV1().Pods(testNamespace).List(context.TODO(), metav1.ListOptions{}) + g.Expect(err).ShouldNot(gomega.HaveOccurred()) + g.Expect(list.Items).Should(gomega.BeEmpty()) + }, time.Second*10).Should(gomega.Succeed()) + }) + + ginkgo.By("should not list nodes", func() { + gomega.Eventually(func(g gomega.Gomega) { + list, err := proxyClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + g.Expect(err).ShouldNot(gomega.HaveOccurred()) + g.Expect(list.Items).Should(gomega.BeEmpty()) + }, time.Second*10).Should(gomega.Succeed()) + }) + }) + }) + }) + ginkgo.Describe("access resources testings", func() { + ginkgo.Context("caching nodes", func() { + var ( + rr *searchv1alpha1.ResourceRegistry + ) + + ginkgo.BeforeAll(func() { + rr = &searchv1alpha1.ResourceRegistry{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rr-" + rand.String(RandomStrLength), + }, + Spec: searchv1alpha1.ResourceRegistrySpec{ + TargetCluster: policyv1alpha1.ClusterAffinity{ + ClusterNames: []string{member1, member2}, + }, + ResourceSelectors: []searchv1alpha1.ResourceSelector{ + { + APIVersion: "v1", + Kind: "Node", + }, + }, + }, + } + framework.CreateResourceRegistry(karmadaClient, rr) + }) + + ginkgo.AfterAll(func() { + framework.RemoveResourceRegistry(karmadaClient, rr.Name) + }) + + ginkgo.It("could list nodes", func() { + fromM1 := framework.GetResourceNames(m1Dynamic.Resource(nodeGVR)) + ginkgo.By("list nodes from member1: " + strings.Join(fromM1.List(), ",")) + fromM2 := framework.GetResourceNames(m2Dynamic.Resource(nodeGVR)) + ginkgo.By("list nodes from member2: " + strings.Join(fromM2.List(), ",")) + fromMembers := sets.NewString().Union(fromM1).Union(fromM2) + + gomega.Eventually(func(g gomega.Gomega) { + fromProxy := framework.GetResourceNames(proxyDynamic.Resource(nodeGVR)) + g.Expect(fromProxy).Should(gomega.Equal(fromMembers)) + }, time.Second*10).Should(gomega.Succeed()) + }) + + ginkgo.It("could chunk list nodes", func() { + fromM1, err := m1Client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + ginkgo.By(fmt.Sprintf("list %v nodes from member1", len(fromM1.Items))) + + fromM2, err := m2Client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + total := len(fromM1.Items) + len(fromM2.Items) + ginkgo.By(fmt.Sprintf("list %v nodes from member2", len(fromM2.Items))) + + list1, err := proxyClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{Limit: 1}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(list1.Items).Should(gomega.HaveLen(1)) + + if list1.Continue == "" { + ginkgo.By("No more nodes remains") + gomega.Expect(list1.Items).Should(gomega.HaveLen(total)) + ginkgo.Skip("No more nodes remains") + } + + list2, err := proxyClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ + Limit: 999999999999, + Continue: list1.Continue, + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(list2.Items).Should(gomega.HaveLen(total - len(list1.Items))) + }) + + ginkgo.It("could list & watch nodes", func() { + listObj, err := proxyClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + watcher, err := proxyClient.CoreV1().Nodes().Watch(context.TODO(), metav1.ListOptions{ResourceVersion: listObj.ResourceVersion}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + defer watcher.Stop() + + testNode := framework.GetAnyResourceOrFail(m1Dynamic.Resource(nodeGVR)) + + anno := "proxy-ann-" + rand.String(RandomStrLength) + data := []byte(`{"metadata": {"annotations": {"` + anno + `": "true"}}}`) + _, err = m1Client.CoreV1().Nodes().Patch(context.TODO(), testNode.GetName(), types.StrategicMergePatchType, data, metav1.PatchOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + gomega.Eventually(func() bool { + event := <-watcher.ResultChan() + o, ok := event.Object.(*corev1.Node) + if !ok { + return false + } + return o.UID == testNode.GetUID() && metav1.HasAnnotation(o.ObjectMeta, anno) + }, time.Second*10, 0).Should(gomega.BeTrue()) + }) + + ginkgo.It("could path nodes", func() { + testObject := framework.GetAnyResourceOrFail(m1Dynamic.Resource(nodeGVR)) + + anno := "proxy-ann-" + rand.String(RandomStrLength) + data := []byte(`{"metadata": {"annotations": {"` + anno + `": "true"}}}`) + _, err := proxyClient.CoreV1().Nodes().Patch(context.TODO(), testObject.GetName(), types.StrategicMergePatchType, data, metav1.PatchOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + testPod, err := m1Client.CoreV1().Nodes().Get(context.TODO(), testObject.GetName(), metav1.GetOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(testPod.Annotations).Should(gomega.HaveKey(anno)) + }) + }) + }) + }) })