From 0f1ce4e1d9143f9b297315f163d7348cec41f428 Mon Sep 17 00:00:00 2001 From: changzhen Date: Wed, 21 Jul 2021 16:46:10 +0800 Subject: [PATCH] add e2e test for MCS feature Signed-off-by: changzhen --- artifacts/deploy/controller-manager.yaml | 2 +- .../mcs/service_export_controller.go | 4 +- pkg/util/constants.go | 3 + test/e2e/mcs_test.go | 600 ++++++++++++++++++ test/e2e/suite_test.go | 16 + 5 files changed, 622 insertions(+), 3 deletions(-) create mode 100644 test/e2e/mcs_test.go diff --git a/artifacts/deploy/controller-manager.yaml b/artifacts/deploy/controller-manager.yaml index 761f314e4..34b166e6c 100644 --- a/artifacts/deploy/controller-manager.yaml +++ b/artifacts/deploy/controller-manager.yaml @@ -29,7 +29,7 @@ spec: - --bind-address=0.0.0.0 - --cluster-status-update-frequency=10s - --secure-port=10357 - - --v=2 + - --v=4 volumeMounts: - name: kubeconfig subPath: kubeconfig diff --git a/pkg/controllers/mcs/service_export_controller.go b/pkg/controllers/mcs/service_export_controller.go index af69e7c91..58b4d2d77 100644 --- a/pkg/controllers/mcs/service_export_controller.go +++ b/pkg/controllers/mcs/service_export_controller.go @@ -111,14 +111,14 @@ func (c *ServiceExportController) RunWorkQueue() { } func (c *ServiceExportController) syncServiceExportOrEndpointSlice(key util.QueueKey) error { - klog.V(4).Infof("Begin to sync ServiceExport or EndpointSlice %s.", key) - fedKey, ok := key.(keys.FederatedKey) if !ok { klog.Errorf("Failed to sync serviceExport as invalid key: %v", key) return fmt.Errorf("invalid key") } + klog.V(4).Infof("Begin to sync %s %s.", fedKey.Kind, fedKey.NamespaceKey()) + switch fedKey.Kind { case util.ServiceExportKind: if err := c.handleServiceExportEvent(fedKey); err != nil { diff --git a/pkg/util/constants.go b/pkg/util/constants.go index 8e15959c7..9b9ecb52d 100644 --- a/pkg/util/constants.go +++ b/pkg/util/constants.go @@ -99,6 +99,9 @@ const ( ServiceExportKind = "ServiceExport" // ServiceImportKind indicates the target resource is a serviceimport crd ServiceImportKind = "ServiceImport" + + // CRDKind indicated the target resource is a CustomResourceDefinition + CRDKind = "CustomResourceDefinition" ) // Define resource filed diff --git a/test/e2e/mcs_test.go b/test/e2e/mcs_test.go new file mode 100644 index 000000000..199962691 --- /dev/null +++ b/test/e2e/mcs_test.go @@ -0,0 +1,600 @@ +package e2e + +import ( + "context" + "fmt" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + discoveryv1beta1 "k8s.io/api/discovery/v1beta1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog/v2" + mcsv1alpha1 "sigs.k8s.io/mcs-api/pkg/apis/v1alpha1" + + policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" + "github.com/karmada-io/karmada/pkg/util" + "github.com/karmada-io/karmada/pkg/util/helper" + "github.com/karmada-io/karmada/pkg/util/names" + testhelper "github.com/karmada-io/karmada/test/helper" +) + +var ( + serviceExportClusterName = "member1" + serviceImportClusterName = "member2" + + serviceExportResource = "serviceexports" + serviceImportResource = "serviceimports" + + replicaCount = int32(1) + helloDeployment = appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Replicas: &replicaCount, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "hello", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "hello"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "hello-tcp", + Image: "busybox", + Args: []string{"nc", "-lk", "-p", "42", "-v", "-e", "echo", "hello"}, + }, + { + Name: "hello-udp", + Image: "busybox", + Args: []string{"nc", "-lk", "-p", "42", "-u", "-v", "-e", "echo", "hello"}, + }, + }, + }, + }, + }, + } + helloService = corev1.Service{ + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "app": "hello", + }, + Ports: []corev1.ServicePort{ + { + Name: "tcp", + Port: 42, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "udp", + Port: 42, + Protocol: corev1.ProtocolUDP, + }, + }, + }, + } + helloServiceExport = mcsv1alpha1.ServiceExport{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "multicluster.x-k8s.io/v1alpha1", + Kind: "ServiceExport", + }, + } + helloServiceImport = mcsv1alpha1.ServiceImport{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "multicluster.x-k8s.io/v1alpha1", + Kind: "ServiceImport", + }, + Spec: mcsv1alpha1.ServiceImportSpec{ + Type: mcsv1alpha1.ClusterSetIP, + Ports: []mcsv1alpha1.ServicePort{ + { + Name: "tcp", + Port: 42, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "udp", + Port: 42, + Protocol: corev1.ProtocolUDP, + }, + }, + }, + } + requestPod = corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "request", + Image: "busybox", + Args: []string{"nc"}, + }, + }, + }, + } +) + +func getPrepareInfo() (serviceExport mcsv1alpha1.ServiceExport, serviceImport mcsv1alpha1.ServiceImport, exportPolicy, + importPolicy *policyv1alpha1.PropagationPolicy, deployment appsv1.Deployment, service corev1.Service) { + helloNamespace := testNamespace + helloName := fmt.Sprintf("hello-%s", rand.String(RandomStrLength)) + + serviceExport = helloServiceExport + serviceExport.Namespace = helloNamespace + serviceExport.Name = helloName + + serviceImport = helloServiceImport + serviceImport.Namespace = helloNamespace + serviceImport.Name = helloName + + exportPolicyNamespace := serviceExport.Namespace + exportPolicyName := fmt.Sprintf("export-%s-policy", serviceExport.Name) + exportPolicy = testhelper.NewPropagationPolicy(exportPolicyNamespace, exportPolicyName, []policyv1alpha1.ResourceSelector{ + { + APIVersion: serviceExport.APIVersion, + Kind: serviceExport.Kind, + Name: serviceExport.Name, + Namespace: serviceExport.Namespace, + }, + }, policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: []string{serviceExportClusterName}, + }, + }) + + importPolicyNamespace := serviceImport.Namespace + importPolicyName := fmt.Sprintf("import-%s-policy", serviceImport.Name) + importPolicy = testhelper.NewPropagationPolicy(importPolicyNamespace, importPolicyName, []policyv1alpha1.ResourceSelector{ + { + APIVersion: serviceImport.APIVersion, + Kind: serviceImport.Kind, + Name: serviceImport.Name, + Namespace: serviceImport.Namespace, + }, + }, policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: []string{serviceImportClusterName}, + }, + }) + + deployment = helloDeployment + deployment.Namespace = helloNamespace + deployment.Name = helloName + + service = helloService + service.Namespace = helloNamespace + service.Name = helloName + + return +} + +var _ = ginkgo.Describe("[MCS] Multi-Cluster Service testing", func() { + serviceExportPolicyName := fmt.Sprintf("%s-%s-policy", serviceExportResource, rand.String(RandomStrLength)) + serviceImportPolicyName := fmt.Sprintf("%s-%s-policy", serviceImportResource, rand.String(RandomStrLength)) + + serviceExportPolicy := testhelper.NewClusterPropagationPolicy(serviceExportPolicyName, []policyv1alpha1.ResourceSelector{ + { + APIVersion: apiextensionsv1.SchemeGroupVersion.String(), + Kind: util.CRDKind, + Name: fmt.Sprintf("%s.%s", serviceExportResource, mcsv1alpha1.GroupName), + }, + }, policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: clusterNames, + }, + }) + serviceImportPolicy := testhelper.NewClusterPropagationPolicy(serviceImportPolicyName, []policyv1alpha1.ResourceSelector{ + { + APIVersion: apiextensionsv1.SchemeGroupVersion.String(), + Kind: util.CRDKind, + Name: fmt.Sprintf("%s.%s", serviceImportResource, mcsv1alpha1.GroupName), + }, + }, policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: clusterNames, + }, + }) + + ginkgo.BeforeEach(func() { + ginkgo.By(fmt.Sprintf("Create ClusterPropagationPolicy(%s) to Propagation ServiceExport CRD", serviceExportPolicy.Name), func() { + _, err := karmadaClient.PolicyV1alpha1().ClusterPropagationPolicies().Create(context.TODO(), serviceExportPolicy, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By(fmt.Sprintf("Create ClusterPropagationPolicy(%s) to Propagation ServiceImport CRD", serviceImportPolicy.Name), func() { + _, err := karmadaClient.PolicyV1alpha1().ClusterPropagationPolicies().Create(context.TODO(), serviceImportPolicy, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By("Wait ServiceExport CRD present on member clusters", func() { + err := wait.PollImmediate(pollInterval, pollTimeout, func() (done bool, err error) { + clusters, err := fetchClusters(karmadaClient) + if err != nil { + return false, err + } + for _, cluster := range clusters { + if !helper.IsAPIEnabled(cluster.Status.APIEnablements, mcsv1alpha1.GroupVersion.String(), util.ServiceExportKind) { + klog.Infof("Waiting for CRD(%s) present on member cluster(%s)", util.ServiceExportKind, cluster.Name) + return false, nil + } + } + return true, nil + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By("Wait ServiceImport CRD present on member clusters", func() { + err := wait.PollImmediate(pollInterval, pollTimeout, func() (done bool, err error) { + clusters, err := fetchClusters(karmadaClient) + if err != nil { + return false, err + } + for _, cluster := range clusters { + if !helper.IsAPIEnabled(cluster.Status.APIEnablements, mcsv1alpha1.GroupVersion.String(), util.ServiceImportKind) { + klog.Infof("Waiting for CRD(%s) present on member cluster(%s)", util.ServiceImportKind, cluster.Name) + return false, nil + } + } + return true, nil + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + }) + + ginkgo.AfterEach(func() { + ginkgo.By(fmt.Sprintf("Delete ClusterPropagationPolicy %s", serviceExportPolicy.Name), func() { + err := karmadaClient.PolicyV1alpha1().ClusterPropagationPolicies().Delete(context.TODO(), serviceExportPolicy.Name, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By(fmt.Sprintf("Delete ClusterPropagationPolicy %s", serviceImportPolicy.Name), func() { + err := karmadaClient.PolicyV1alpha1().ClusterPropagationPolicies().Delete(context.TODO(), serviceImportPolicy.Name, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + // In order to avoid that current delete operation will delete the ServiceExport and ServiceImport + // CRD which created in next block, here waiting for the deletion action to be end. + + ginkgo.By("Wait ServiceExport CRD disappear on member clusters", func() { + err := wait.PollImmediate(pollInterval, pollTimeout, func() (done bool, err error) { + clusters, err := fetchClusters(karmadaClient) + if err != nil { + return false, err + } + for _, cluster := range clusters { + if helper.IsAPIEnabled(cluster.Status.APIEnablements, mcsv1alpha1.GroupVersion.String(), util.ServiceExportKind) { + return false, nil + } + } + return true, nil + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By("Wait ServiceImport CRD disappear on member clusters", func() { + err := wait.PollImmediate(pollInterval, pollTimeout, func() (done bool, err error) { + clusters, err := fetchClusters(karmadaClient) + if err != nil { + return false, err + } + for _, cluster := range clusters { + if helper.IsAPIEnabled(cluster.Status.APIEnablements, mcsv1alpha1.GroupVersion.String(), util.ServiceImportKind) { + return false, nil + } + } + return true, nil + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + }) + + ginkgo.Context("Connectivity testing", func() { + serviceExport, serviceImport, exportPolicy, importPolicy, demoDeployment, demoService := getPrepareInfo() + + ginkgo.BeforeEach(func() { + exportClusterClient := getClusterClient(serviceExportClusterName) + gomega.Expect(exportClusterClient).ShouldNot(gomega.BeNil()) + + ginkgo.By(fmt.Sprintf("Create Deployment(%s/%s) in %s cluster", demoDeployment.Namespace, demoDeployment.Name, serviceExportClusterName), func() { + _, err := exportClusterClient.AppsV1().Deployments(demoDeployment.Namespace).Create(context.TODO(), &demoDeployment, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By(fmt.Sprintf("Create Service(%s/%s) in %s cluster", demoService.Namespace, demoService.Name, serviceExportClusterName), func() { + _, err := exportClusterClient.CoreV1().Services(demoService.Namespace).Create(context.TODO(), &demoService, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By(fmt.Sprintf("Wait Service(%s/%s)'s EndpointSlice exist in %s cluster", demoService.Namespace, demoService.Name, serviceExportClusterName), func() { + gomega.Eventually(func() int { + endpointSlices, err := exportClusterClient.DiscoveryV1beta1().EndpointSlices(demoService.Namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: labels.Set{discoveryv1beta1.LabelServiceName: demoService.Name}.AsSelector().String(), + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + epNums := 0 + for _, slice := range endpointSlices.Items { + epNums += len(slice.Endpoints) + } + return epNums + }, pollTimeout, pollInterval).Should(gomega.Equal(1)) + }) + }) + + ginkgo.AfterEach(func() { + exportClusterClient := getClusterClient(serviceExportClusterName) + gomega.Expect(exportClusterClient).ShouldNot(gomega.BeNil()) + + ginkgo.By(fmt.Sprintf("Delete Deployment(%s/%s) in %s cluster", demoDeployment.Namespace, demoDeployment.Name, serviceExportClusterName), func() { + err := exportClusterClient.AppsV1().Deployments(demoDeployment.Namespace).Delete(context.TODO(), demoDeployment.Name, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By(fmt.Sprintf("Delete Service(%s/%s) in %s cluster", demoService.Namespace, demoService.Name, serviceExportClusterName), func() { + err := exportClusterClient.CoreV1().Services(demoService.Namespace).Delete(context.TODO(), demoService.Name, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + }) + + ginkgo.It("Export Service from source-clusters, import Service to destination-clusters", func() { + importClusterClient := getClusterClient(serviceImportClusterName) + gomega.Expect(importClusterClient).ShouldNot(gomega.BeNil()) + + ginkgo.By(fmt.Sprintf("Create ServiceExport(%s/%s)", serviceExport.Namespace, serviceExport.Name), func() { + err := controlPlaneClient.Create(context.TODO(), &serviceExport) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By(fmt.Sprintf("Create PropagationPolicy(%s/%s) to propagate ServiceExport", exportPolicy.Namespace, exportPolicy.Name), func() { + _, err := karmadaClient.PolicyV1alpha1().PropagationPolicies(exportPolicy.Namespace).Create(context.TODO(), exportPolicy, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By(fmt.Sprintf("Wait EndpointSlices collected to namespace(%s) in controller-plane", demoService.Namespace), func() { + gomega.Eventually(func() int { + endpointSlices, err := kubeClient.DiscoveryV1beta1().EndpointSlices(demoService.Namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: labels.Set{discoveryv1beta1.LabelServiceName: names.GenerateDerivedServiceName(demoService.Name)}.AsSelector().String(), + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + epNums := 0 + for _, slice := range endpointSlices.Items { + epNums += len(slice.Endpoints) + } + return epNums + }, pollTimeout, pollInterval).Should(gomega.Equal(1)) + }) + + ginkgo.By(fmt.Sprintf("Create ServiceImport(%s/%s)", serviceImport.Namespace, serviceImport.Name), func() { + err := controlPlaneClient.Create(context.TODO(), &serviceImport) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By(fmt.Sprintf("Create PropagationPolicy(%s/%s) to propagate ServiveImport", importPolicy.Namespace, importPolicy.Name), func() { + _, err := karmadaClient.PolicyV1alpha1().PropagationPolicies(importPolicy.Namespace).Create(context.TODO(), importPolicy, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By(fmt.Sprintf("Wait derived-service(%s/%s) exist in %s cluster", demoService.Namespace, names.GenerateDerivedServiceName(demoService.Name), serviceImportClusterName), func() { + err := wait.PollImmediate(pollInterval, pollTimeout, func() (done bool, err error) { + _, err = importClusterClient.CoreV1().Services(demoService.Namespace).Get(context.TODO(), names.GenerateDerivedServiceName(demoService.Name), metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return false, nil + } + return false, err + } + return true, nil + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By(fmt.Sprintf("Wait EndpointSlices have been imported to %s cluster", serviceImportClusterName), func() { + gomega.Eventually(func() int { + endpointSlices, err := importClusterClient.DiscoveryV1beta1().EndpointSlices(demoService.Namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: labels.Set{discoveryv1beta1.LabelServiceName: names.GenerateDerivedServiceName(demoService.Name)}.AsSelector().String(), + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + epNums := 0 + for _, slice := range endpointSlices.Items { + epNums += len(slice.Endpoints) + } + return epNums + }, pollTimeout, pollInterval).Should(gomega.Equal(1)) + }) + + ginkgo.By("TCP connects across clusters using the ClusterIP", func() { + derivedSvc, err := importClusterClient.CoreV1().Services(demoService.Namespace).Get(context.TODO(), names.GenerateDerivedServiceName(demoService.Name), metav1.GetOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + pod := requestPod + pod.Name = fmt.Sprintf("request-%s", rand.String(RandomStrLength)) + pod.Spec.Containers[0].Args = []string{"nc", derivedSvc.Spec.ClusterIP, "42"} + _, err = importClusterClient.CoreV1().Pods(demoService.Namespace).Create(context.TODO(), &pod, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Eventually(func() (string, error) { + return podLogs(context.TODO(), importClusterClient, demoService.Namespace, pod.Name) + }, pollTimeout, pollInterval).Should(gomega.Equal("hello\n")) + }) + + ginkgo.By("UDP connects across clusters using the ClusterIP", func() { + derivedSvc, err := importClusterClient.CoreV1().Services(demoService.Namespace).Get(context.TODO(), names.GenerateDerivedServiceName(demoService.Name), metav1.GetOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + pod := requestPod + pod.Name = fmt.Sprintf("request-%s", rand.String(RandomStrLength)) + pod.Spec.Containers[0].Args = []string{"sh", "-c", fmt.Sprintf("echo hi | nc -u %s 42", derivedSvc.Spec.ClusterIP)} + _, err = importClusterClient.CoreV1().Pods(demoService.Namespace).Create(context.TODO(), &pod, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Eventually(func() (string, error) { + return podLogs(context.TODO(), importClusterClient, demoService.Namespace, pod.Name) + }, pollTimeout, pollInterval).Should(gomega.Equal("hello\n")) + }) + + ginkgo.By("Cleanup", func() { + err := controlPlaneClient.Delete(context.TODO(), &serviceExport) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + err = karmadaClient.PolicyV1alpha1().PropagationPolicies(exportPolicy.Namespace).Delete(context.TODO(), exportPolicy.Name, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + err = controlPlaneClient.Delete(context.TODO(), &serviceImport) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + err = karmadaClient.PolicyV1alpha1().PropagationPolicies(importPolicy.Namespace).Delete(context.TODO(), importPolicy.Name, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + }) + }) + + ginkgo.Context("EndpointSlices change testing", func() { + serviceExport, serviceImport, exportPolicy, importPolicy, demoDeployment, demoService := getPrepareInfo() + + ginkgo.BeforeEach(func() { + exportClusterClient := getClusterClient(serviceExportClusterName) + gomega.Expect(exportClusterClient).ShouldNot(gomega.BeNil()) + + ginkgo.By(fmt.Sprintf("Create Deployment(%s/%s) in %s cluster", demoDeployment.Namespace, demoDeployment.Name, serviceExportClusterName), func() { + _, err := exportClusterClient.AppsV1().Deployments(demoDeployment.Namespace).Create(context.TODO(), &demoDeployment, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By(fmt.Sprintf("Create Service(%s/%s) in %s cluster", demoService.Namespace, demoService.Name, serviceExportClusterName), func() { + _, err := exportClusterClient.CoreV1().Services(demoService.Namespace).Create(context.TODO(), &demoService, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By(fmt.Sprintf("Wait Service(%s/%s)'s EndpointSlice exist in %s cluster", demoService.Namespace, demoService.Name, serviceExportClusterName), func() { + gomega.Eventually(func() int { + endpointSlices, err := exportClusterClient.DiscoveryV1beta1().EndpointSlices(demoService.Namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: labels.Set{discoveryv1beta1.LabelServiceName: demoService.Name}.AsSelector().String(), + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + epNums := 0 + for _, slice := range endpointSlices.Items { + epNums += len(slice.Endpoints) + } + return epNums + }, pollTimeout, pollInterval).Should(gomega.Equal(1)) + }) + }) + + ginkgo.AfterEach(func() { + exportClusterClient := getClusterClient(serviceExportClusterName) + gomega.Expect(exportClusterClient).ShouldNot(gomega.BeNil()) + + ginkgo.By(fmt.Sprintf("Delete Deployment(%s/%s) in %s cluster", demoDeployment.Namespace, demoDeployment.Name, serviceExportClusterName), func() { + err := exportClusterClient.AppsV1().Deployments(demoDeployment.Namespace).Delete(context.TODO(), demoDeployment.Name, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By(fmt.Sprintf("Delete Service(%s/%s) in %s cluster", demoService.Namespace, demoService.Name, serviceExportClusterName), func() { + err := exportClusterClient.CoreV1().Services(demoService.Namespace).Delete(context.TODO(), demoService.Name, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + }) + + ginkgo.It("Update Deployment's replicas", func() { + exportClusterClient := getClusterClient(serviceExportClusterName) + gomega.Expect(exportClusterClient).ShouldNot(gomega.BeNil()) + importClusterClient := getClusterClient(serviceImportClusterName) + gomega.Expect(importClusterClient).ShouldNot(gomega.BeNil()) + + ginkgo.By(fmt.Sprintf("Create ServiceExport(%s/%s)", serviceExport.Namespace, serviceExport.Name), func() { + err := controlPlaneClient.Create(context.TODO(), &serviceExport) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By(fmt.Sprintf("Create PropagationPolicy(%s/%s)", exportPolicy.Namespace, exportPolicy.Name), func() { + _, err := karmadaClient.PolicyV1alpha1().PropagationPolicies(exportPolicy.Namespace).Create(context.TODO(), exportPolicy, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By(fmt.Sprintf("Wait EndpointSlices collected to namespace(%s) in controller-plane", demoService.Namespace), func() { + gomega.Eventually(func() int { + endpointSlices, err := kubeClient.DiscoveryV1beta1().EndpointSlices(demoService.Namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: labels.Set{discoveryv1beta1.LabelServiceName: names.GenerateDerivedServiceName(demoService.Name)}.AsSelector().String(), + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + epNums := 0 + for _, slice := range endpointSlices.Items { + epNums += len(slice.Endpoints) + } + return epNums + }, pollTimeout, pollInterval).Should(gomega.Equal(1)) + }) + + ginkgo.By(fmt.Sprintf("Create ServiceImport(%s/%s)", serviceImport.Namespace, serviceImport.Name), func() { + err := controlPlaneClient.Create(context.TODO(), &serviceImport) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By(fmt.Sprintf("Create PropagationPolicy(%s/%s) to propagate ServiveImport", importPolicy.Namespace, importPolicy.Name), func() { + _, err := karmadaClient.PolicyV1alpha1().PropagationPolicies(importPolicy.Namespace).Create(context.TODO(), importPolicy, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By(fmt.Sprintf("Wait EndpointSlice exist in %s cluster", serviceImportClusterName), func() { + gomega.Eventually(func() int { + endpointSlices, err := importClusterClient.DiscoveryV1beta1().EndpointSlices(demoService.Namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: labels.Set{discoveryv1beta1.LabelServiceName: names.GenerateDerivedServiceName(demoService.Name)}.AsSelector().String(), + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + epNums := 0 + for _, slice := range endpointSlices.Items { + epNums += len(slice.Endpoints) + } + return epNums + }, pollTimeout, pollInterval).Should(gomega.Equal(1)) + }) + + ginkgo.By(fmt.Sprintf("Update Deployment's replicas in %s cluster", serviceExportClusterName), func() { + updateReplicaCount := int32(2) + demoDeployment.Spec.Replicas = &updateReplicaCount + + _, err := exportClusterClient.AppsV1().Deployments(demoDeployment.Namespace).Update(context.TODO(), &demoDeployment, metav1.UpdateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By(fmt.Sprintf("Wait EndpointSlice update in %s cluster", serviceImportClusterName), func() { + gomega.Eventually(func() int { + endpointSlices, err := importClusterClient.DiscoveryV1beta1().EndpointSlices(demoService.Namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: labels.Set{discoveryv1beta1.LabelServiceName: names.GenerateDerivedServiceName(demoService.Name)}.AsSelector().String(), + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + epNums := 0 + for _, slice := range endpointSlices.Items { + epNums += len(slice.Endpoints) + } + return epNums + }, pollTimeout, pollInterval).Should(gomega.Equal(2)) + }) + + ginkgo.By("Cleanup", func() { + err := controlPlaneClient.Delete(context.TODO(), &serviceExport) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + err = karmadaClient.PolicyV1alpha1().PropagationPolicies(exportPolicy.Namespace).Delete(context.TODO(), exportPolicy.Name, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + err = controlPlaneClient.Delete(context.TODO(), &serviceImport) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + err = karmadaClient.PolicyV1alpha1().PropagationPolicies(importPolicy.Namespace).Delete(context.TODO(), importPolicy.Name, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + }) + }) +}) diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index 6131c9f92..8fd1ce055 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -3,6 +3,7 @@ package e2e import ( "context" "fmt" + "io/ioutil" "os" "strings" "testing" @@ -10,6 +11,7 @@ import ( "github.com/onsi/ginkgo" "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/rand" @@ -371,3 +373,17 @@ func DeleteClusterLabel(c client.Client, clusterName string) error { }) return err } + +func podLogs(ctx context.Context, k8s kubernetes.Interface, namespace, name string) (string, error) { + logRequest := k8s.CoreV1().Pods(namespace).GetLogs(name, &corev1.PodLogOptions{}) + logs, err := logRequest.Stream(ctx) + if err != nil { + return "", err + } + defer logs.Close() + data, err := ioutil.ReadAll(logs) + if err != nil { + return "", err + } + return string(data), nil +}