upgrade scenario for push mode cluster

Signed-off-by: changzhen <changzhen5@huawei.com>

cmd/controller-manager/app/controllermanager.go

@@ -319,8 +319,10 @@ func startServiceImportController(ctx controllerscontext.Context) (enabled bool,
 
 func startUnifiedAuthController(ctx controllerscontext.Context) (enabled bool, err error) {
 	unifiedAuthController := &unifiedauth.Controller{
-		Client:        ctx.Mgr.GetClient(),
+		Client:                ctx.Mgr.GetClient(),
-		EventRecorder: ctx.Mgr.GetEventRecorderFor(unifiedauth.ControllerName),
+		ControllerPlaneConfig: ctx.Mgr.GetConfig(),
+		EventRecorder:         ctx.Mgr.GetEventRecorderFor(unifiedauth.ControllerName),
+		ClusterClientSetFunc:  util.NewClusterClientSet,
 	}
 	if err := unifiedAuthController.SetupWithManager(ctx.Mgr); err != nil {
 		return false, err

pkg/controllers/unifiedauth/ensure_impersonation_secret.go

@@ -0,0 +1,103 @@
+package unifiedauth
+
+import (
+	"context"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kubeclient "k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
+
+	clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
+	"github.com/karmada-io/karmada/pkg/util"
+	"github.com/karmada-io/karmada/pkg/util/names"
+)
+
+// ensureImpersonationSecret make sure create impersonation secret for all Cluster.
+// This logic is used only in the upgrade scenario of the current version
+// and can be deleted in the next version.
+func (c *Controller) ensureImpersonationSecret() {
+	controlPlaneKubeClient := kubeclient.NewForConfigOrDie(c.ControllerPlaneConfig)
+
+	clusterList := &clusterv1alpha1.ClusterList{}
+	if err := c.Client.List(context.TODO(), clusterList); err != nil {
+		klog.Errorf("Failed to list clusterList, error: %v", err)
+		return
+	}
+
+	for index, cluster := range clusterList.Items {
+		err := c.ensureImpersonationSecretForCluster(controlPlaneKubeClient, &clusterList.Items[index])
+		if err != nil {
+			klog.Errorf("Failed to ensure impersonation secret exist for cluster %s", cluster.Name)
+			return
+		}
+	}
+}
+
+func (c *Controller) ensureImpersonationSecretForCluster(karmadaKubeClient kubeclient.Interface, cluster *clusterv1alpha1.Cluster) error {
+	klog.V(4).Infof("Create impersonation secret for cluster %s", cluster.Name)
+	// create a ClusterClient for the given member cluster
+	clusterClient, err := c.ClusterClientSetFunc(cluster.Name, c.Client, nil)
+	if err != nil {
+		klog.Errorf("Failed to create a ClusterClient for the given member cluster: %v, err is : %v", cluster.Name, err)
+		return err
+	}
+
+	// clusterNamespace store namespace where serviceaccount and secret exist.
+	clusterNamespace := cluster.Spec.SecretRef.Namespace
+
+	// create a ServiceAccount for impersonation in cluster.
+	impersonationSA := &corev1.ServiceAccount{}
+	impersonationSA.Namespace = clusterNamespace
+	impersonationSA.Name = names.GenerateServiceAccountName("impersonator")
+	if impersonationSA, err = c.ensureServiceAccountExist(clusterClient.KubeClient, impersonationSA); err != nil {
+		return err
+	}
+
+	clusterImpersonatorSecret, err := util.WaitForServiceAccountSecretCreation(clusterClient.KubeClient, impersonationSA)
+	if err != nil {
+		return fmt.Errorf("failed to get serviceAccount secret for impersonation from cluster(%s), error: %v", cluster.Name, err)
+	}
+
+	// create secret to store impersonation info in control plane
+	impersonationSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: clusterNamespace,
+			Name:      names.GenerateImpersonationSecretName(cluster.Name),
+			OwnerReferences: []metav1.OwnerReference{
+				*metav1.NewControllerRef(cluster, clusterv1alpha1.SchemeGroupVersion.WithKind("Cluster")),
+			},
+		},
+		Data: map[string][]byte{
+			clusterv1alpha1.SecretCADataKey: clusterImpersonatorSecret.Data["ca.crt"],
+			clusterv1alpha1.SecretTokenKey:  clusterImpersonatorSecret.Data[clusterv1alpha1.SecretTokenKey],
+		},
+	}
+
+	_, err = util.CreateSecret(karmadaKubeClient, impersonationSecret)
+	if err != nil {
+		return fmt.Errorf("failed to create impersonator secret in control plane. error: %v", err)
+	}
+
+	return nil
+}
+
+// ensureServiceAccountExist makes sure that the specific service account exist in cluster.
+// If service account not exit, just create it.
+func (c *Controller) ensureServiceAccountExist(client kubeclient.Interface, saObj *corev1.ServiceAccount) (*corev1.ServiceAccount, error) {
+	exist, err := util.IsServiceAccountExist(client, saObj.Namespace, saObj.Name)
+	if err != nil {
+		return nil, fmt.Errorf("failed to check if impersonation service account exist. error: %v", err)
+	}
+	if exist {
+		return saObj, nil
+	}
+
+	createdObj, err := util.CreateServiceAccount(client, saObj)
+	if err != nil {
+		return nil, fmt.Errorf("ensure impersonation service account failed due to create failed, error: %v", err)
+	}
+
+	return createdObj, nil
+}

pkg/controllers/unifiedauth/unified_auth_controller.go

@@ -9,7 +9,9 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
 	controllerruntime "sigs.k8s.io/controller-runtime"
@@ -38,8 +40,10 @@ const (
 
 // Controller is to sync impersonation config to member clusters for unified authentication.
 type Controller struct {
-	client.Client // used to operate Cluster resources.
+	client.Client         // used to operate Cluster resources.
-	EventRecorder record.EventRecorder
+	ControllerPlaneConfig *rest.Config
+	EventRecorder         record.EventRecorder
+	ClusterClientSetFunc  func(string, client.Client, *util.ClientOption) (*util.ClusterClient, error)
 }
 
 // Reconcile performs a full reconciliation for the object referred to by the Request.
@@ -71,6 +75,12 @@ func (c *Controller) Reconcile(ctx context.Context, req controllerruntime.Reques
 	return controllerruntime.Result{}, nil
 }
 
+// Start starts a goroutine to ensure impersonation secret for upgrade scenario.
+func (c *Controller) Start(ctx context.Context) error {
+	go c.ensureImpersonationSecret()
+	return nil
+}
+
 func (c *Controller) syncImpersonationConfig(cluster *clusterv1alpha1.Cluster) error {
 	// step1: list all clusterroles
 	clusterRoleList := &rbacv1.ClusterRoleList{}
@@ -234,9 +244,13 @@ func (c *Controller) SetupWithManager(mgr controllerruntime.Manager) error {
 		},
 	}
 
-	return controllerruntime.NewControllerManagedBy(mgr).For(&clusterv1alpha1.Cluster{}).WithEventFilter(clusterPredicateFunc).
+	return utilerrors.NewAggregate([]error{
-		Watches(&source.Kind{Type: &rbacv1.ClusterRole{}}, handler.EnqueueRequestsFromMapFunc(c.newClusterRoleMapFunc())).
+		controllerruntime.NewControllerManagedBy(mgr).For(&clusterv1alpha1.Cluster{}).WithEventFilter(clusterPredicateFunc).
-		Watches(&source.Kind{Type: &rbacv1.ClusterRoleBinding{}}, handler.EnqueueRequestsFromMapFunc(c.newClusterRoleBindingMapFunc())).Complete(c)
+			Watches(&source.Kind{Type: &rbacv1.ClusterRole{}}, handler.EnqueueRequestsFromMapFunc(c.newClusterRoleMapFunc())).
+			Watches(&source.Kind{Type: &rbacv1.ClusterRoleBinding{}}, handler.EnqueueRequestsFromMapFunc(c.newClusterRoleBindingMapFunc())).Complete(c),
+		mgr.Add(c),
+	})
+
 }
 
 func (c *Controller) newClusterRoleMapFunc() handler.MapFunc {