From fa1df5844a880b78eb92e4ebf2fb5e00bea742b8 Mon Sep 17 00:00:00 2001 From: changzhen Date: Sat, 20 Mar 2021 17:41:48 +0800 Subject: [PATCH] add basic crd propagation test Signed-off-by: changzhen --- test/e2e/clusterpropagationpolicy_test.go | 109 ++++++++++++++++++++++ test/e2e/suite_test.go | 43 ++++++--- test/helper/clusterpropagationpolicy.go | 36 +++++++ test/helper/resource.go | 82 ++++++++++++++++ 4 files changed, 258 insertions(+), 12 deletions(-) create mode 100644 test/e2e/clusterpropagationpolicy_test.go create mode 100644 test/helper/clusterpropagationpolicy.go diff --git a/test/e2e/clusterpropagationpolicy_test.go b/test/e2e/clusterpropagationpolicy_test.go new file mode 100644 index 000000000..488d14ac8 --- /dev/null +++ b/test/e2e/clusterpropagationpolicy_test.go @@ -0,0 +1,109 @@ +package e2e + +import ( + "context" + "fmt" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog/v2" + + "github.com/karmada-io/karmada/test/helper" +) + +var _ = ginkgo.Describe("[BasicClusterPropagation] basic cluster propagation testing", func() { + ginkgo.Context("CustomResourceDefinition propagation testing", func() { + crdName := "foos.example.karmada.io" + crdPolicyName := crdName + + crd := helper.NewCustomResourceDefinition(apiextensionsv1.NamespaceScoped) + crdPolicy := helper.NewPolicyWithSingleCRD(crdPolicyName, crd, clusterNames) + + crdGVR := schema.GroupVersionResource{ + Group: "apiextensions.k8s.io", + Version: "v1", + Resource: "customresourcedefinitions", + } + + ginkgo.BeforeEach(func() { + ginkgo.By(fmt.Sprintf("creating crdPolicy(%s)", crdPolicyName), func() { + _, err := karmadaClient.PolicyV1alpha1().ClusterPropagationPolicies().Create(context.TODO(), crdPolicy, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + }) + + ginkgo.AfterEach(func() { + ginkgo.By(fmt.Sprintf("removing crdPolicy(%s)", crdPolicyName), func() { + err := karmadaClient.PolicyV1alpha1().ClusterPropagationPolicies().Delete(context.TODO(), crdPolicyName, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + }) + + ginkgo.It("crd propagation testing", func() { + ginkgo.By(fmt.Sprintf("creating crd(%s)", crdName), func() { + unstructObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(crd) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + _, err = dynamicClient.Resource(crdGVR).Namespace(crd.Namespace).Create(context.TODO(), &unstructured.Unstructured{Object: unstructObj}, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By(fmt.Sprintf("get crd(%s)", crdName), func() { + _, err := dynamicClient.Resource(crdGVR).Namespace(crd.Namespace).Get(context.TODO(), crd.Name, metav1.GetOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By("check if crd present on member clusters", func() { + for _, cluster := range clusters { + clusterDynamicClient := getClusterDynamicClient(cluster.Name) + gomega.Expect(clusterDynamicClient).ShouldNot(gomega.BeNil()) + + klog.Infof("Waiting for crd(%s) present on cluster(%s)", crdName, cluster.Name) + err := wait.Poll(pollInterval, pollTimeout, func() (done bool, err error) { + _, err = clusterDynamicClient.Resource(crdGVR).Namespace(crd.Namespace).Get(context.TODO(), crd.Name, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return false, nil + } + return false, err + } + return true, nil + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + } + }) + + ginkgo.By(fmt.Sprintf("removing crd(%s)", crdPolicyName), func() { + err := dynamicClient.Resource(crdGVR).Namespace(crd.Namespace).Delete(context.TODO(), crd.Name, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + + ginkgo.By("check if crd not present on member clusters", func() { + for _, cluster := range clusters { + clusterDynamicClient := getClusterDynamicClient(cluster.Name) + gomega.Expect(clusterDynamicClient).ShouldNot(gomega.BeNil()) + + klog.Infof("Waiting for crd(%s) present on cluster(%s)", crdName, cluster.Name) + err := wait.Poll(pollInterval, pollTimeout, func() (done bool, err error) { + _, err = clusterDynamicClient.Resource(crdGVR).Namespace(crd.Namespace).Get(context.TODO(), crd.Name, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return true, nil + } + return false, err + } + return false, nil + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + } + }) + }) + }) +}) diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index 06c1b3bce..c7560d9f6 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -9,9 +9,9 @@ import ( "github.com/onsi/ginkgo" "github.com/onsi/gomega" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -44,15 +44,17 @@ const ( ) var ( - kubeconfig string - restConfig *rest.Config - kubeClient kubernetes.Interface - karmadaClient karmada.Interface - clusters []*clusterapi.Cluster - clusterNames []string - clusterClients []*util.ClusterClient - testNamespace = fmt.Sprintf("karmadatest-%s", rand.String(RandomStrLength)) - clusterProvider *cluster.Provider + kubeconfig string + restConfig *rest.Config + kubeClient kubernetes.Interface + karmadaClient karmada.Interface + dynamicClient dynamic.Interface + clusters []*clusterapi.Cluster + clusterNames []string + clusterClients []*util.ClusterClient + clusterDynamicClients []*util.DynamicClusterClient + testNamespace = fmt.Sprintf("karmadatest-%s", rand.String(RandomStrLength)) + clusterProvider *cluster.Provider ) func TestE2E(t *testing.T) { @@ -75,6 +77,9 @@ var _ = ginkgo.BeforeSuite(func() { karmadaClient, err = karmada.NewForConfig(restConfig) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + dynamicClient, err = dynamic.NewForConfig(restConfig) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + clusters, err = fetchClusters(karmadaClient) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) @@ -89,6 +94,10 @@ var _ = ginkgo.BeforeSuite(func() { clusterClient, err := util.NewClusterClientSet(cluster, kubeClient) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) clusterClients = append(clusterClients, clusterClient) + + clusterDynamicClient, err := util.NewClusterDynamicClientSet(cluster, kubeClient) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + clusterDynamicClients = append(clusterDynamicClients, clusterDynamicClient) } gomega.Expect(clusterNames).Should(gomega.HaveLen(len(clusters))) @@ -105,7 +114,7 @@ var _ = ginkgo.AfterSuite(func() { // fetchClusters will fetch all member clusters we have. func fetchClusters(client karmada.Interface) ([]*clusterapi.Cluster, error) { - clusterList, err := client.ClusterV1alpha1().Clusters().List(context.TODO(), v1.ListOptions{}) + clusterList, err := client.ClusterV1alpha1().Clusters().List(context.TODO(), metav1.ListOptions{}) if err != nil { return nil, err } @@ -183,6 +192,16 @@ func getClusterClient(clusterName string) kubernetes.Interface { return nil } +func getClusterDynamicClient(clusterName string) dynamic.Interface { + for _, client := range clusterDynamicClients { + if client.ClusterName == clusterName { + return client.DynamicClientSet + } + } + + return nil +} + func createCluster(clusterName, kubeConfigPath, controlPlane, clusterContext string) error { err := clusterProvider.Create(clusterName, cluster.CreateWithKubeconfigPath(kubeConfigPath)) if err != nil { diff --git a/test/helper/clusterpropagationpolicy.go b/test/helper/clusterpropagationpolicy.go new file mode 100644 index 000000000..b3ff83563 --- /dev/null +++ b/test/helper/clusterpropagationpolicy.go @@ -0,0 +1,36 @@ +package helper + +import ( + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + propagationapi "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" +) + +// NewPolicyWithSingleCRD will build a ClusterPropagationPolicy object. +func NewPolicyWithSingleCRD(name string, crd *apiextensionsv1.CustomResourceDefinition, clusters []string) *propagationapi.ClusterPropagationPolicy { + return newClusterPolicy(name, crd.APIVersion, crd.Kind, crd.Name, clusters) +} + +// newClusterPolicy will build a ClusterPropagationPolicy object. +func newClusterPolicy(policyName, apiVersion, kind, resourceName string, clusters []string) *propagationapi.ClusterPropagationPolicy { + return &propagationapi.ClusterPropagationPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: policyName, + }, + Spec: propagationapi.PropagationSpec{ + ResourceSelectors: []propagationapi.ResourceSelector{ + { + APIVersion: apiVersion, + Kind: kind, + Name: resourceName, + }, + }, + Placement: propagationapi.Placement{ + ClusterAffinity: &propagationapi.ClusterAffinity{ + ClusterNames: clusters, + }, + }, + }, + } +} diff --git a/test/helper/resource.go b/test/helper/resource.go index 9baf71964..fb7902db2 100644 --- a/test/helper/resource.go +++ b/test/helper/resource.go @@ -3,6 +3,7 @@ package helper import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/pointer" @@ -94,3 +95,84 @@ func NewPod(namespace string, name string) *corev1.Pod { }, } } + +// NewCustomResourceDefinition will build a CRD object. +func NewCustomResourceDefinition(scope apiextensionsv1.ResourceScope) *apiextensionsv1.CustomResourceDefinition { + return &apiextensionsv1.CustomResourceDefinition{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apiextensions.k8s.io/v1", + Kind: "CustomResourceDefinition", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "foos.example.karmada.io", + }, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "example.karmada.io", + Names: apiextensionsv1.CustomResourceDefinitionNames{ + Kind: "Foo", + ListKind: "FooList", + Plural: "foos", + Singular: "foo", + }, + Scope: scope, + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1alpha1", + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "apiVersion": {Type: "string"}, + "kind": {Type: "string"}, + "metadata": {Type: "object"}, + "spec": { + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "clusters": { + Items: &apiextensionsv1.JSONSchemaPropsOrArray{ + Schema: &apiextensionsv1.JSONSchemaProps{ + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "name": {Type: "string"}, + }, + Required: []string{"name"}, + Type: "object", + }, + }, + Type: "array", + }, + "resource": { + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "apiVersion": {Type: "string"}, + "kind": {Type: "string"}, + "name": {Type: "string"}, + "namespace": {Type: "string"}, + "resourceVersion": {Type: "string"}, + }, + Required: []string{"apiVersion", "kind", "name"}, + Type: "object", + }, + }, + Required: []string{"resource"}, + Type: "object", + }, + }, + Required: []string{"spec"}, + Type: "object", + }, + }, + Served: true, + Storage: true, + Subresources: &apiextensionsv1.CustomResourceSubresources{ + Status: &apiextensionsv1.CustomResourceSubresourceStatus{}, + }, + }, + }, + }, + Status: apiextensionsv1.CustomResourceDefinitionStatus{ + AcceptedNames: apiextensionsv1.CustomResourceDefinitionNames{ + Kind: "", + Plural: "", + }, + Conditions: []apiextensionsv1.CustomResourceDefinitionCondition{}, + StoredVersions: []string{}, + }, + } +}