From fb8f99718c70f4e51c893ee82a37aef95604d460 Mon Sep 17 00:00:00 2001 From: changzhen Date: Sat, 25 Dec 2021 16:56:27 +0800 Subject: [PATCH] upgrade scenario for push mode cluster Signed-off-by: changzhen --- .../app/controllermanager.go | 6 +- .../ensure_impersonation_secret.go | 103 ++++++++++++++++++ .../unifiedauth/unified_auth_controller.go | 24 +++- 3 files changed, 126 insertions(+), 7 deletions(-) create mode 100644 pkg/controllers/unifiedauth/ensure_impersonation_secret.go diff --git a/cmd/controller-manager/app/controllermanager.go b/cmd/controller-manager/app/controllermanager.go index 8319ea828..c68aad0a1 100644 --- a/cmd/controller-manager/app/controllermanager.go +++ b/cmd/controller-manager/app/controllermanager.go @@ -319,8 +319,10 @@ func startServiceImportController(ctx controllerscontext.Context) (enabled bool, func startUnifiedAuthController(ctx controllerscontext.Context) (enabled bool, err error) { unifiedAuthController := &unifiedauth.Controller{ - Client: ctx.Mgr.GetClient(), - EventRecorder: ctx.Mgr.GetEventRecorderFor(unifiedauth.ControllerName), + Client: ctx.Mgr.GetClient(), + ControllerPlaneConfig: ctx.Mgr.GetConfig(), + EventRecorder: ctx.Mgr.GetEventRecorderFor(unifiedauth.ControllerName), + ClusterClientSetFunc: util.NewClusterClientSet, } if err := unifiedAuthController.SetupWithManager(ctx.Mgr); err != nil { return false, err diff --git a/pkg/controllers/unifiedauth/ensure_impersonation_secret.go b/pkg/controllers/unifiedauth/ensure_impersonation_secret.go new file mode 100644 index 000000000..1fb909763 --- /dev/null +++ b/pkg/controllers/unifiedauth/ensure_impersonation_secret.go @@ -0,0 +1,103 @@ +package unifiedauth + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeclient "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + + clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1" + "github.com/karmada-io/karmada/pkg/util" + "github.com/karmada-io/karmada/pkg/util/names" +) + +// ensureImpersonationSecret make sure create impersonation secret for all Cluster. +// This logic is used only in the upgrade scenario of the current version +// and can be deleted in the next version. +func (c *Controller) ensureImpersonationSecret() { + controlPlaneKubeClient := kubeclient.NewForConfigOrDie(c.ControllerPlaneConfig) + + clusterList := &clusterv1alpha1.ClusterList{} + if err := c.Client.List(context.TODO(), clusterList); err != nil { + klog.Errorf("Failed to list clusterList, error: %v", err) + return + } + + for index, cluster := range clusterList.Items { + err := c.ensureImpersonationSecretForCluster(controlPlaneKubeClient, &clusterList.Items[index]) + if err != nil { + klog.Errorf("Failed to ensure impersonation secret exist for cluster %s", cluster.Name) + return + } + } +} + +func (c *Controller) ensureImpersonationSecretForCluster(karmadaKubeClient kubeclient.Interface, cluster *clusterv1alpha1.Cluster) error { + klog.V(4).Infof("Create impersonation secret for cluster %s", cluster.Name) + // create a ClusterClient for the given member cluster + clusterClient, err := c.ClusterClientSetFunc(cluster.Name, c.Client, nil) + if err != nil { + klog.Errorf("Failed to create a ClusterClient for the given member cluster: %v, err is : %v", cluster.Name, err) + return err + } + + // clusterNamespace store namespace where serviceaccount and secret exist. + clusterNamespace := cluster.Spec.SecretRef.Namespace + + // create a ServiceAccount for impersonation in cluster. + impersonationSA := &corev1.ServiceAccount{} + impersonationSA.Namespace = clusterNamespace + impersonationSA.Name = names.GenerateServiceAccountName("impersonator") + if impersonationSA, err = c.ensureServiceAccountExist(clusterClient.KubeClient, impersonationSA); err != nil { + return err + } + + clusterImpersonatorSecret, err := util.WaitForServiceAccountSecretCreation(clusterClient.KubeClient, impersonationSA) + if err != nil { + return fmt.Errorf("failed to get serviceAccount secret for impersonation from cluster(%s), error: %v", cluster.Name, err) + } + + // create secret to store impersonation info in control plane + impersonationSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: clusterNamespace, + Name: names.GenerateImpersonationSecretName(cluster.Name), + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(cluster, clusterv1alpha1.SchemeGroupVersion.WithKind("Cluster")), + }, + }, + Data: map[string][]byte{ + clusterv1alpha1.SecretCADataKey: clusterImpersonatorSecret.Data["ca.crt"], + clusterv1alpha1.SecretTokenKey: clusterImpersonatorSecret.Data[clusterv1alpha1.SecretTokenKey], + }, + } + + _, err = util.CreateSecret(karmadaKubeClient, impersonationSecret) + if err != nil { + return fmt.Errorf("failed to create impersonator secret in control plane. error: %v", err) + } + + return nil +} + +// ensureServiceAccountExist makes sure that the specific service account exist in cluster. +// If service account not exit, just create it. +func (c *Controller) ensureServiceAccountExist(client kubeclient.Interface, saObj *corev1.ServiceAccount) (*corev1.ServiceAccount, error) { + exist, err := util.IsServiceAccountExist(client, saObj.Namespace, saObj.Name) + if err != nil { + return nil, fmt.Errorf("failed to check if impersonation service account exist. error: %v", err) + } + if exist { + return saObj, nil + } + + createdObj, err := util.CreateServiceAccount(client, saObj) + if err != nil { + return nil, fmt.Errorf("ensure impersonation service account failed due to create failed, error: %v", err) + } + + return createdObj, nil +} diff --git a/pkg/controllers/unifiedauth/unified_auth_controller.go b/pkg/controllers/unifiedauth/unified_auth_controller.go index d22c481d8..f4072d433 100644 --- a/pkg/controllers/unifiedauth/unified_auth_controller.go +++ b/pkg/controllers/unifiedauth/unified_auth_controller.go @@ -9,7 +9,9 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" "k8s.io/klog/v2" controllerruntime "sigs.k8s.io/controller-runtime" @@ -38,8 +40,10 @@ const ( // Controller is to sync impersonation config to member clusters for unified authentication. type Controller struct { - client.Client // used to operate Cluster resources. - EventRecorder record.EventRecorder + client.Client // used to operate Cluster resources. + ControllerPlaneConfig *rest.Config + EventRecorder record.EventRecorder + ClusterClientSetFunc func(string, client.Client, *util.ClientOption) (*util.ClusterClient, error) } // Reconcile performs a full reconciliation for the object referred to by the Request. @@ -71,6 +75,12 @@ func (c *Controller) Reconcile(ctx context.Context, req controllerruntime.Reques return controllerruntime.Result{}, nil } +// Start starts a goroutine to ensure impersonation secret for upgrade scenario. +func (c *Controller) Start(ctx context.Context) error { + go c.ensureImpersonationSecret() + return nil +} + func (c *Controller) syncImpersonationConfig(cluster *clusterv1alpha1.Cluster) error { // step1: list all clusterroles clusterRoleList := &rbacv1.ClusterRoleList{} @@ -234,9 +244,13 @@ func (c *Controller) SetupWithManager(mgr controllerruntime.Manager) error { }, } - return controllerruntime.NewControllerManagedBy(mgr).For(&clusterv1alpha1.Cluster{}).WithEventFilter(clusterPredicateFunc). - Watches(&source.Kind{Type: &rbacv1.ClusterRole{}}, handler.EnqueueRequestsFromMapFunc(c.newClusterRoleMapFunc())). - Watches(&source.Kind{Type: &rbacv1.ClusterRoleBinding{}}, handler.EnqueueRequestsFromMapFunc(c.newClusterRoleBindingMapFunc())).Complete(c) + return utilerrors.NewAggregate([]error{ + controllerruntime.NewControllerManagedBy(mgr).For(&clusterv1alpha1.Cluster{}).WithEventFilter(clusterPredicateFunc). + Watches(&source.Kind{Type: &rbacv1.ClusterRole{}}, handler.EnqueueRequestsFromMapFunc(c.newClusterRoleMapFunc())). + Watches(&source.Kind{Type: &rbacv1.ClusterRoleBinding{}}, handler.EnqueueRequestsFromMapFunc(c.newClusterRoleBindingMapFunc())).Complete(c), + mgr.Add(c), + }) + } func (c *Controller) newClusterRoleMapFunc() handler.MapFunc {