From 92fff3163dce84e950cce2a80f452ef821acc13f Mon Sep 17 00:00:00 2001 From: chaunceyjiang Date: Fri, 16 Sep 2022 15:43:14 +0800 Subject: [PATCH] propagate dependencies support propagate sa Signed-off-by: chaunceyjiang --- .../dependencies_distributor.go | 1 + test/e2e/dependenciesdistributor_test.go | 48 +++++++++++++++++ test/e2e/framework/deployment.go | 11 ++++ test/e2e/framework/rbac.go | 54 +++++++++++++++++++ test/e2e/suite_test.go | 1 + test/helper/resource.go | 35 ++++++++++++ 6 files changed, 150 insertions(+) diff --git a/pkg/dependenciesdistributor/dependencies_distributor.go b/pkg/dependenciesdistributor/dependencies_distributor.go index 00b6a3cff..778d81e8f 100644 --- a/pkg/dependenciesdistributor/dependencies_distributor.go +++ b/pkg/dependenciesdistributor/dependencies_distributor.go @@ -49,6 +49,7 @@ const ( var supportedTypes = []schema.GroupVersionResource{ corev1.SchemeGroupVersion.WithResource("configmaps"), corev1.SchemeGroupVersion.WithResource("secrets"), + corev1.SchemeGroupVersion.WithResource("serviceaccounts"), corev1.SchemeGroupVersion.WithResource("persistentvolumeclaims"), } diff --git a/test/e2e/dependenciesdistributor_test.go b/test/e2e/dependenciesdistributor_test.go index 64e01aa59..652909f66 100644 --- a/test/e2e/dependenciesdistributor_test.go +++ b/test/e2e/dependenciesdistributor_test.go @@ -277,5 +277,53 @@ var _ = ginkgo.Describe("[DependenciesDistributor] automatically propagate relev }) }) }) + + ginkgo.When("serviceAccount propagate automatically", func() { + var saName string + var sa *corev1.ServiceAccount + + ginkgo.BeforeEach(func() { + saName = saNamePrefix + rand.String(RandomStrLength) + sa = testhelper.NewServiceaccount(testNamespace, saName) + + deployment = testhelper.NewDeploymentWithServiceAccount(testNamespace, deploymentName, saName) + + policy = testhelper.NewPropagationPolicy(testNamespace, policyName, []policyv1alpha1.ResourceSelector{ + { + APIVersion: deployment.APIVersion, + Kind: deployment.Kind, + Name: deployment.Name, + }, + }, policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: initClusterNames, + }, + }) + policy.Spec.PropagateDeps = true + }) + + ginkgo.It("serviceAccount automatically propagation testing", func() { + framework.CreateServiceAccount(kubeClient, sa) + ginkgo.DeferCleanup(func() { + framework.RemoveServiceAccount(kubeClient, sa.GetNamespace(), sa.GetName()) + }) + ginkgo.By("check if the serviceAccount is propagated automatically", func() { + framework.WaitDeploymentPresentOnClustersFitWith(initClusterNames, deployment.Namespace, deployment.Name, + func(deployment *appsv1.Deployment) bool { + return true + }) + + framework.WaitServiceAccountPresentOnClustersFitWith(initClusterNames, sa.GetNamespace(), sa.GetName(), + func(sa *corev1.ServiceAccount) bool { + return true + }) + }) + + ginkgo.By("make the sa is not referenced by the deployment ", func() { + framework.UpdateDeploymentServiceAccountName(kubeClient, deployment, "default") + framework.WaitServiceAccountDisappearOnClusters(initClusterNames, sa.GetNamespace(), sa.GetName()) + }) + }) + }) }) }) diff --git a/test/e2e/framework/deployment.go b/test/e2e/framework/deployment.go index 1ee3c1228..7e9f7c234 100644 --- a/test/e2e/framework/deployment.go +++ b/test/e2e/framework/deployment.go @@ -122,6 +122,17 @@ func UpdateDeploymentVolumes(client kubernetes.Interface, deployment *appsv1.Dep }) } +// UpdateDeploymentServiceAccountName update Deployment's serviceAccountName. +func UpdateDeploymentServiceAccountName(client kubernetes.Interface, deployment *appsv1.Deployment, serviceAccountName string) { + ginkgo.By(fmt.Sprintf("Updating Deployment(%s/%s)'s serviceAccountName", deployment.Namespace, deployment.Name), func() { + deployment.Spec.Template.Spec.ServiceAccountName = serviceAccountName + gomega.Eventually(func() error { + _, err := client.AppsV1().Deployments(deployment.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) + return err + }, pollTimeout, pollInterval).ShouldNot(gomega.HaveOccurred()) + }) +} + // ExtractTargetClustersFrom extract the target cluster names from deployment's related resourceBinding Information. func ExtractTargetClustersFrom(c client.Client, deployment *appsv1.Deployment) []string { bindingName := names.GenerateBindingName(deployment.Kind, deployment.Name) diff --git a/test/e2e/framework/rbac.go b/test/e2e/framework/rbac.go index ddb053f91..099472c61 100644 --- a/test/e2e/framework/rbac.go +++ b/test/e2e/framework/rbac.go @@ -11,6 +11,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" ) // CreateClusterRole create clusterRole. @@ -69,3 +70,56 @@ func RemoveServiceAccount(client kubernetes.Interface, namespace, name string) { gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) }) } + +// WaitServiceAccountPresentOnClusterFitWith wait sa present on member clusters sync with fit func. +func WaitServiceAccountPresentOnClusterFitWith(cluster, namespace, name string, fit func(sa *corev1.ServiceAccount) bool) { + clusterClient := GetClusterClient(cluster) + gomega.Expect(clusterClient).ShouldNot(gomega.BeNil()) + + klog.Infof("Waiting for serviceAccount(%s/%s) synced on cluster(%s)", namespace, name, cluster) + gomega.Eventually(func() bool { + sa, err := clusterClient.CoreV1().ServiceAccounts(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return false + } + return fit(sa) + }, pollTimeout, pollInterval).Should(gomega.Equal(true)) +} + +// WaitServiceAccountPresentOnClustersFitWith wait sa present on cluster sync with fit func. +func WaitServiceAccountPresentOnClustersFitWith(clusters []string, namespace, name string, fit func(sa *corev1.ServiceAccount) bool) { + ginkgo.By(fmt.Sprintf("Waiting for pod(%s/%s) synced on member clusters", namespace, name), func() { + for _, clusterName := range clusters { + WaitServiceAccountPresentOnClusterFitWith(clusterName, namespace, name, fit) + } + }) +} + +// WaitServiceAccountDisappearOnCluster wait sa disappear on cluster until timeout. +func WaitServiceAccountDisappearOnCluster(cluster, namespace, name string) { + clusterClient := GetClusterClient(cluster) + gomega.Expect(clusterClient).ShouldNot(gomega.BeNil()) + + klog.Infof("Waiting for sa(%s/%s) disappear on cluster(%s)", namespace, name, cluster) + gomega.Eventually(func() bool { + _, err := clusterClient.CoreV1().ServiceAccounts(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err == nil { + return false + } + if apierrors.IsNotFound(err) { + return true + } + + klog.Errorf("Failed to get sa(%s/%s) on cluster(%s), err: %v", namespace, name, cluster, err) + return false + }, pollTimeout, pollInterval).Should(gomega.Equal(true)) +} + +// WaitServiceAccountDisappearOnClusters wait sa disappear on member clusters until timeout. +func WaitServiceAccountDisappearOnClusters(clusters []string, namespace, name string) { + ginkgo.By(fmt.Sprintf("Check if sa(%s/%s) diappeare on member clusters", namespace, name), func() { + for _, clusterName := range clusters { + WaitServiceAccountDisappearOnCluster(clusterName, namespace, name) + } + }) +} diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index 874bc3030..8f289752e 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -47,6 +47,7 @@ const ( configMapNamePrefix = "configmap-" secretNamePrefix = "secret-" pvcNamePrefix = "pvc-" + saNamePrefix = "sa-" ingressNamePrefix = "ingress-" daemonSetNamePrefix = "daemonset-" statefulSetNamePrefix = "statefulset-" diff --git a/test/helper/resource.go b/test/helper/resource.go index 184355669..415cd12a2 100644 --- a/test/helper/resource.go +++ b/test/helper/resource.go @@ -559,6 +559,41 @@ func NewDeploymentWithVolumes(namespace, deploymentName string, volumes []corev1 } } +// NewDeploymentWithServiceAccount will build a deployment object that with serviceAccount. +func NewDeploymentWithServiceAccount(namespace, deploymentName string, serviceAccountName string) *appsv1.Deployment { + podLabels := map[string]string{"app": "nginx"} + + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "Deployment", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: deploymentName, + }, + Spec: appsv1.DeploymentSpec{ + + Replicas: pointer.Int32Ptr(3), + Selector: &metav1.LabelSelector{ + MatchLabels: podLabels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: podLabels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "nginx", + Image: "nginx:1.19.0", + }}, + ServiceAccountName: serviceAccountName, + }, + }, + }, + } +} + // NewSecret will build a secret object. func NewSecret(namespace string, name string, data map[string][]byte) *corev1.Secret { return &corev1.Secret{