From b534e994c2b2b17e3e6ad8910231edcd30792ab1 Mon Sep 17 00:00:00 2001 From: wwwnay Date: Mon, 14 Mar 2022 23:05:23 +0800 Subject: [PATCH] e2etest for aggregated api endpoint Signed-off-by: wwwnay --- test/e2e/aggregatedapi_test.go | 128 ++++++++++++++++ test/e2e/framework/clusterrole.go | 243 ++++++++++++++++++++++++++++++ test/helper/resource.go | 39 +++++ 3 files changed, 410 insertions(+) create mode 100644 test/e2e/aggregatedapi_test.go create mode 100644 test/e2e/framework/clusterrole.go diff --git a/test/e2e/aggregatedapi_test.go b/test/e2e/aggregatedapi_test.go new file mode 100644 index 000000000..78db01f1a --- /dev/null +++ b/test/e2e/aggregatedapi_test.go @@ -0,0 +1,128 @@ +package e2e + +import ( + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + rbacv1 "k8s.io/api/rbac/v1" + + "github.com/karmada-io/karmada/test/e2e/framework" + "github.com/karmada-io/karmada/test/helper" +) + +var _ = ginkgo.Describe("aggregatedapi testing", func() { + + var member1, member2 string + saName := "tom" + saNamespace := testNamespace + saTom := helper.NewServiceaccount(saNamespace, saName) + clusterroleName := "cluster-proxy-clusterrole" + crbSubjectGroupName := "system:serviceaccounts:" + saNamespace + crbSubject := []rbacv1.Subject{{ + Kind: "ServiceAccount", + Name: saName, + Namespace: saTom.Namespace, + }, + { + Kind: "Group", + Name: "system:serviceaccounts", + }, + { + Kind: "Group", + Name: crbSubjectGroupName, + }, + } + saBinding := helper.NewClusterrolebindings(saName, clusterroleName, crbSubject) + saClustrole := helper.NewClusterroles(saName, "*", "*", "") + crbSubject2 := []rbacv1.Subject{{ + Kind: "ServiceAccount", + Name: saName, + Namespace: saTom.Namespace, + }, + } + saBinding2 := helper.NewClusterrolebindings(saName, saName, crbSubject2) + + ginkgo.BeforeEach(func() { + member1 = framework.ClusterNames()[0] + member2 = framework.ClusterNames()[1] + karmdaApiGroup := "cluster.karmada.io" + karmdaResource := "clusters/proxy" + proxyClustrole := helper.NewClusterroles(clusterroleName, karmdaApiGroup, karmdaResource, member1) + framework.CreateMermberclusterSa(member1, saTom) + framework.CreateKarmadaSa(kubeClient, saTom) + framework.CreateClusterrole(kubeClient, proxyClustrole) + framework.CreateClusterroleBinding(kubeClient, saBinding) + }) + + ginkgo.AfterEach(func() { + framework.RemoveMermberclusterSa(member1, saTom.Name, saTom.Namespace) + framework.RemoveKarmadalusterSa(kubeClient, saTom.Name, saTom.Namespace) + framework.RemoveClusterrole(kubeClient, clusterroleName) + framework.RemoveclusterroleBinding(kubeClient, saBinding.Name) + }) + + ginkgo.When("Serviceaccount access test", func() { + ginkgo.Context("Serviceaccount access test", func() { + ginkgo.It("Serviceaccount access in the member1 cluster", func() { + framework.WaitServiceaccountPresentOnCluster(kubeClient, saName, saTom.Namespace) + framework.WaitServiceaccountPresentOnMemberCluster(member1, saName, saTom.Namespace) + framework.WaitClusterrolebindingPresentOnCluster(kubeClient, saBinding.Name) + apiPath := "/apis/cluster.karmada.io/v1alpha1/clusters/" + member1 + "/proxy/apis" + code := framework.GetClusterNode(apiPath, kubeClient, karmadaClient, saName, saTom.Namespace) + gomega.Expect(code).Should(gomega.Equal(200)) + }) + + ginkgo.It("Serviceaccount have no access in the member2 cluster", func() { + framework.WaitServiceaccountPresentOnCluster(kubeClient, saName, saTom.Namespace) + framework.WaitServiceaccountPresentOnMemberCluster(member1, saName, saTom.Namespace) + framework.WaitClusterrolebindingPresentOnCluster(kubeClient, saBinding.Name) + apiPath := "/apis/cluster.karmada.io/v1alpha1/clusters/" + member2 + "/proxy/apis" + code := framework.GetClusterNode(apiPath, kubeClient, karmadaClient, saName, saTom.Namespace) + gomega.Expect(code).ShouldNot(gomega.Equal(200)) + gomega.Expect(code).Should(gomega.Equal(403)) + }) + + ginkgo.It("Serviceaccount does not have any permissions in the member1 cluster", func() { + framework.WaitServiceaccountPresentOnCluster(kubeClient, saName, saTom.Namespace) + framework.WaitServiceaccountPresentOnMemberCluster(member1, saName, saTom.Namespace) + framework.WaitClusterrolebindingPresentOnCluster(kubeClient, saBinding.Name) + apiPath := "/apis/cluster.karmada.io/v1alpha1/clusters/" + member1 + "/proxy/api/v1/nodes" + code := framework.GetClusterNode(apiPath, kubeClient, karmadaClient, saName, saTom.Namespace) + gomega.Expect(code).ShouldNot(gomega.Equal(200)) + gomega.Expect(code).Should(gomega.Equal(403)) + }) + }) + }) + + ginkgo.When("Grant permission to Serviceaccount in member ", func() { + ginkgo.JustBeforeEach(func() { + framework.CreateMemberclusterRole(member1, saClustrole) + framework.CreateMemberclusterRoleBinding(member1, saBinding2) + }) + + ginkgo.JustAfterEach(func() { + framework.RemoveMemberclusterRole(member1, saClustrole.Name) + framework.RemoveMemberclusterRoleBinding(member1, saBinding2.Name) + }) + + ginkgo.It("Grant permission to Serviceaccount in member1 cluster.", func() { + framework.WaitServiceaccountPresentOnCluster(kubeClient, saName, saTom.Namespace) + framework.WaitServiceaccountPresentOnMemberCluster(member1, saName, saTom.Namespace) + framework.WaitClusterrolebindingPresentOnCluster(kubeClient, saBinding.Name) + framework.WaitClusterrolebindingPresentOnMemberCluster(member1, saBinding2.Name) + apiPath := "/apis/cluster.karmada.io/v1alpha1/clusters/" + member1 + "/proxy/api/v1/nodes" + code := framework.GetClusterNode(apiPath, kubeClient, karmadaClient, saName, saTom.Namespace) + gomega.Expect(code).Should(gomega.Equal(200)) + }) + + ginkgo.It("Grant no permission to Serviceaccount in member2 cluster.", func() { + framework.WaitServiceaccountPresentOnCluster(kubeClient, saName, saTom.Namespace) + framework.WaitServiceaccountPresentOnMemberCluster(member1, saName, saTom.Namespace) + framework.WaitClusterrolebindingPresentOnCluster(kubeClient, saBinding.Name) + framework.WaitClusterrolebindingPresentOnMemberCluster(member1, saBinding2.Name) + apiPath := "/apis/cluster.karmada.io/v1alpha1/clusters/" + member2 + "/proxy/api/v1/nodes" + code := framework.GetClusterNode(apiPath, kubeClient, karmadaClient, saName, saTom.Namespace) + gomega.Expect(code).ShouldNot(gomega.Equal(200)) + gomega.Expect(code).Should(gomega.Equal(403)) + }) + }) +}) diff --git a/test/e2e/framework/clusterrole.go b/test/e2e/framework/clusterrole.go new file mode 100644 index 000000000..f94a6ba4c --- /dev/null +++ b/test/e2e/framework/clusterrole.go @@ -0,0 +1,243 @@ +package framework + +import ( + "context" + "crypto/tls" + "encoding/base64" + "fmt" + "net/http" + "os" + "strings" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + karmada "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" +) + +// CreateClusterrole create clusterrole. +func CreateClusterrole(client kubernetes.Interface, clusterrole *rbacv1.ClusterRole) { + ginkgo.By(fmt.Sprintf("Creating ClusterRole(%s)", clusterrole.Name), func() { + _, err := client.RbacV1().ClusterRoles().Create(context.TODO(), clusterrole, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) +} + +// CreateClusterroleBinding create clusterrolebinding. +func CreateClusterroleBinding(client kubernetes.Interface, clusterrolebinding *rbacv1.ClusterRoleBinding) { + ginkgo.By(fmt.Sprintf("Creating ClusterRoleBinding(%s)", clusterrolebinding.Name), func() { + _, err := client.RbacV1().ClusterRoleBindings().Create(context.TODO(), clusterrolebinding, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) +} + +// WaitClusterrolebindingPresentOnCluster wait for Clusterrolebinding ready in cluster until timeout. +func WaitClusterrolebindingPresentOnCluster(client kubernetes.Interface, name string) { + ginkgo.By(fmt.Sprintf("wait for clusterRoleBinding(%s) present ", name), func() { + gomega.Eventually(func(g gomega.Gomega) (bool, error) { + _, err := client.RbacV1().ClusterRoleBindings().Get(context.TODO(), name, metav1.GetOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return true, nil + }, pollTimeout, pollInterval).Should(gomega.Equal(true)) + }) +} + +// RemoveClusterrole delete clusterrole. +func RemoveClusterrole(client kubernetes.Interface, name string) { + ginkgo.By(fmt.Sprintf("Remove ClusterRole(%s)", name), func() { + err := client.RbacV1().ClusterRoles().Delete(context.TODO(), name, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) +} + +// RemoveclusterroleBinding delete clusterrolebinding. +func RemoveclusterroleBinding(client kubernetes.Interface, name string) { + ginkgo.By(fmt.Sprintf("Remove ClusterRole(%s)", name), func() { + err := client.RbacV1().ClusterRoleBindings().Delete(context.TODO(), name, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) +} + +// CreateMermberclusterSa create serviceaccount for member cluster. +func CreateMermberclusterSa(cluster string, serviceaccount *corev1.ServiceAccount) { + ginkgo.By(fmt.Sprintf("Creating serviceaccount(%s) in membercluster (%s) in namespace %s", serviceaccount.Name, cluster, serviceaccount.Namespace), func() { + clusterClient := GetClusterClient(cluster) + gomega.Expect(clusterClient).ShouldNot(gomega.BeNil()) + _, err := clusterClient.CoreV1().ServiceAccounts(serviceaccount.Namespace).Create(context.TODO(), serviceaccount, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) + +} + +// CreateKarmadaSa create serviceaccount for karmada. +func CreateKarmadaSa(client kubernetes.Interface, serviceaccount *corev1.ServiceAccount) { + ginkgo.By(fmt.Sprintf("Creating serviceaccount(%s) in karmadaCluster in namespace %s", serviceaccount.Name, serviceaccount.Namespace), func() { + _, err := client.CoreV1().ServiceAccounts(serviceaccount.Namespace).Create(context.TODO(), serviceaccount, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) +} + +// RemoveMermberclusterSa delete serviceaccount for member cluster. +func RemoveMermberclusterSa(cluster, name, namespace string) { + ginkgo.By(fmt.Sprintf("Remove serviceaccount(%s) in membercluster (%s)", name, cluster), func() { + clusterClient := GetClusterClient(cluster) + gomega.Expect(clusterClient).ShouldNot(gomega.BeNil()) + err := clusterClient.CoreV1().ServiceAccounts(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) +} + +// RemoveKarmadalusterSa delete serviceaccount for karmada cluster. +func RemoveKarmadalusterSa(client kubernetes.Interface, name, namespace string) { + ginkgo.By(fmt.Sprintf("Remove serviceaccount(%s) in karmadaCluster", name), func() { + err := client.CoreV1().ServiceAccounts(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) +} + +// CreateMemberclusterRole create clusterrole with config. +func CreateMemberclusterRole(cluster string, clusterrole *rbacv1.ClusterRole) { + ginkgo.By(fmt.Sprintf("Creating member clusterrole(%s) in clustermember(%s)", clusterrole.Name, cluster), func() { + clusterClient := GetClusterClient(cluster) + gomega.Expect(clusterClient).ShouldNot(gomega.BeNil()) + _, err := clusterClient.RbacV1().ClusterRoles().Create(context.TODO(), clusterrole, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) +} + +// RemoveMemberclusterRole delete member cluster clusterRole. +func RemoveMemberclusterRole(cluster string, clusterroleName string) { + ginkgo.By(fmt.Sprintf("Remove clusterRole(%s) in member(%s) ", clusterroleName, cluster), func() { + clusterClient := GetClusterClient(cluster) + gomega.Expect(clusterClient).ShouldNot(gomega.BeNil()) + err := clusterClient.RbacV1().ClusterRoles().Delete(context.TODO(), clusterroleName, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) +} + +// CreateMemberclusterRoleBinding create membercluster clusterrolebinding. +func CreateMemberclusterRoleBinding(cluster string, clusterrolebinding *rbacv1.ClusterRoleBinding) { + ginkgo.By(fmt.Sprintf("Creating member clusterrolebinding(%s)", clusterrolebinding.Name), func() { + clusterClient := GetClusterClient(cluster) + gomega.Expect(clusterClient).ShouldNot(gomega.BeNil()) + _, err := clusterClient.RbacV1().ClusterRoleBindings().Create(context.TODO(), clusterrolebinding, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) +} + +// WaitClusterrolebindingPresentOnMemberCluster wait for Clusterrolebinding ready in member cluster until timeout. +func WaitClusterrolebindingPresentOnMemberCluster(cluster string, clusterroleBindName string) { + ginkgo.By(fmt.Sprintf("wait for clusterRoleBinding(%s) present in member(%s) cluster ", clusterroleBindName, cluster), func() { + clusterClient := GetClusterClient(cluster) + gomega.Expect(clusterClient).ShouldNot(gomega.BeNil()) + gomega.Eventually(func(g gomega.Gomega) (bool, error) { + _, err := clusterClient.RbacV1().ClusterRoleBindings().Get(context.TODO(), clusterroleBindName, metav1.GetOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return true, nil + }, pollTimeout, pollInterval).Should(gomega.Equal(true)) + }) +} + +// RemoveMemberclusterRoleBinding delete member cluster clusterRoleBinding. +func RemoveMemberclusterRoleBinding(cluster string, clusterroleBindName string) { + ginkgo.By(fmt.Sprintf("Remove clusterRoleBinding(%s) in member(%s) ", clusterroleBindName, cluster), func() { + clusterClient := GetClusterClient(cluster) + gomega.Expect(clusterClient).ShouldNot(gomega.BeNil()) + err := clusterClient.RbacV1().ClusterRoleBindings().Delete(context.TODO(), clusterroleBindName, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) +} + +// WaitServiceaccountPresentOnCluster wait for sa ready on cluster untill timeout. +func WaitServiceaccountPresentOnCluster(client kubernetes.Interface, saname, namespace string) { + ginkgo.By(fmt.Sprintf("wait for serviceaccount (%s) present in karmadacluster in namespace %s", saname, namespace), func() { + gomega.Eventually(func(g gomega.Gomega) (bool, error) { + _, err := client.CoreV1().ServiceAccounts(namespace).Get(context.TODO(), saname, metav1.GetOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return true, nil + }, pollTimeout, pollInterval).Should(gomega.Equal(true)) + }) +} + +// WaitServiceaccountPresentOnMemberCluster wait for sa ready on targetmember cluster untill timeout. +func WaitServiceaccountPresentOnMemberCluster(cluster, saname, namespace string) { + ginkgo.By(fmt.Sprintf("wait for serviceaccount (%s) present in member(%s) cluster in namespace %s ", saname, cluster, namespace), func() { + clusterClient := GetClusterClient(cluster) + gomega.Expect(clusterClient).ShouldNot(gomega.BeNil()) + gomega.Eventually(func(g gomega.Gomega) (bool, error) { + _, err := clusterClient.CoreV1().ServiceAccounts(namespace).Get(context.TODO(), saname, metav1.GetOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return true, nil + }, pollTimeout, pollInterval).Should(gomega.Equal(true)) + }) +} + +// WaitSecretPresentOnCluster wait for Clusterrolebinding ready until timeout. +func WaitSecretPresentOnCluster(client kubernetes.Interface, saname, namespace string) (secretName string) { + ginkgo.By(fmt.Sprintf("wait for secret (%s) present in cluster in namespace %s", saname, namespace), func() { + gomega.Eventually(func(g gomega.Gomega) (bool, error) { + listSecret, err := client.CoreV1().Secrets(namespace).List(context.TODO(), metav1.ListOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + sclist := []string{} + for _, scName := range listSecret.Items { + sclist = append(sclist, scName.Name) + } + for _, k := range sclist { + secretName = k + } + return strings.HasPrefix(secretName, saname), nil + }, pollTimeout, pollInterval).Should(gomega.Equal(true)) + }) + return +} + +// GetSecretInfo get token/url of the karmada cluster. +func GetSecretInfo(client kubernetes.Interface, karmadaClient karmada.Interface, saname, namespace string) (token string, url string) { + secretName := WaitSecretPresentOnCluster(client, saname, namespace) + ta, err := client.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) + if err != nil { + fmt.Println(err.Error()) + } + token = base64.StdEncoding.EncodeToString(ta.Data["token"]) + kubeconfig := os.Getenv("KUBECONFIG") + config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + url = config.Host + if err != nil { + panic(err.Error()) + } + return token, url +} + +// GetClusterNode use bearer token call karmada api. +func GetClusterNode(path string, client kubernetes.Interface, karmadaClient karmada.Interface, saname, namespace string) int { + token, apiurl := GetSecretInfo(client, karmadaClient, saname, namespace) + // changge toekn to decodestring + to1, err := base64.StdEncoding.DecodeString(token) + if err != nil { + fmt.Println(err.Error()) + } + t1 := string(to1) + t := "Bearer " + t1 + url := apiurl + path + // insecure the httprequest + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + httpClient := &http.Client{Transport: tr} + res, err := http.NewRequest("GET", url, nil) + if err != nil { + panic(err) + } + res.Header.Add("Authorization", t) + resp, err := httpClient.Do(res) + if err != nil { + panic(err) + } + defer resp.Body.Close() + return resp.StatusCode +} diff --git a/test/helper/resource.go b/test/helper/resource.go index 8bd261839..a0828cadd 100644 --- a/test/helper/resource.go +++ b/test/helper/resource.go @@ -6,6 +6,7 @@ import ( appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -551,3 +552,41 @@ func NewConfigMap(namespace string, name string, data map[string]string) *corev1 Data: data, } } + +// NewServiceaccount will build a new serviceaccount. +func NewServiceaccount(namespace, name string) *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Secrets: []corev1.ObjectReference{}, + } +} + +// NewClusterroles will build a new clusterrole resource. +func NewClusterroles(name, apigroup, resource, resourcename string) *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: name}, + Rules: []rbacv1.PolicyRule{{ + APIGroups: []string{apigroup}, + Verbs: []string{"*"}, + Resources: []string{resource}, + // ResourceNames: []string{"member1","member2","member3"}, + ResourceNames: []string{resourcename}, + }}, + } +} + +// NewClusterrolebindings will bild a new clusterrolebinding. +func NewClusterrolebindings(name, clsterroleName string, subject []rbacv1.Subject) *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: name}, + Subjects: subject, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: clsterroleName, + }, + } +}