karmada operator: support install karmada on remote cluster

Signed-off-by: calvin <wen.chen@daocloud.io>

operator/pkg/controller/karmada/planner.go

@@ -4,7 +4,10 @@ import (
 	"context"
 	"fmt"
 
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -33,6 +36,7 @@ type Planner struct {
 	client.Client
 	karmada *operatorv1alpha1.Karmada
 	job     *workflow.Job
+	config  *rest.Config
 }
 
 // NewPlannerFor creates planner, it will recognize the karmada resource action
@@ -68,6 +72,7 @@ func NewPlannerFor(karmada *operatorv1alpha1.Karmada, c client.Client, config *r
 		Client:  c,
 		job:     job,
 		action:  action,
+		config:  config,
 	}, nil
 }
 
@@ -124,18 +129,44 @@ func (p *Planner) runJobErr(err error) error {
 
 func (p *Planner) afterRunJob() error {
 	if p.action == InitAction {
-		// Update the condition to Ready and set kubeconfig of karmada-apiserver to status.
+		// Update the karmada condition to Ready and set kubeconfig of karmada apiserver to karmada status.
 		operatorv1alpha1.KarmadaCompleted(p.karmada, operatorv1alpha1.Ready, "karmada init job is completed")
+
+		if !util.IsInCluster(p.karmada.Spec.HostCluster) {
+			localClusterClient, err := clientset.NewForConfig(p.config)
+			if err != nil {
+				return fmt.Errorf("error when creating local cluster client, err: %w", err)
+			}
+
+			remoteClient, err := util.BuildClientFromSecretRef(localClusterClient, p.karmada.Spec.HostCluster.SecretRef)
+			if err != nil {
+				return fmt.Errorf("error when creating cluster client to install karmada, err: %w", err)
+			}
+
+			secret, err := remoteClient.CoreV1().Secrets(p.karmada.GetNamespace()).Get(context.TODO(), util.AdminKubeconfigSecretName(p.karmada.GetName()), metav1.GetOptions{})
+			if err != nil {
+				return err
+			}
+
+			_, err = localClusterClient.CoreV1().Secrets(p.karmada.GetNamespace()).Create(context.TODO(), &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: p.karmada.GetNamespace(),
+					Name:      util.AdminKubeconfigSecretName(p.karmada.GetName()),
+				},
+				Data: secret.Data,
+			}, metav1.CreateOptions{})
+			if err != nil {
+				return err
+			}
+		}
+
 		p.karmada.Status.SecretRef = &operatorv1alpha1.LocalSecretReference{
 			Namespace: p.karmada.GetNamespace(),
 			Name:      util.AdminKubeconfigSecretName(p.karmada.GetName()),
 		}
-
 		return p.Client.Status().Update(context.TODO(), p.karmada)
 	}
-
 	// if it is deInit workflow, the cr will be deleted with karmada is be deleted, so we need not to
 	// update the karmada status.
-
 	return nil
 }

operator/pkg/deinit.go

@@ -10,6 +10,7 @@ import (
 	operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
 	"github.com/karmada-io/karmada/operator/pkg/constants"
 	tasks "github.com/karmada-io/karmada/operator/pkg/tasks/deinit"
+	"github.com/karmada-io/karmada/operator/pkg/util"
 	"github.com/karmada-io/karmada/operator/pkg/workflow"
 )
 
@@ -44,16 +45,21 @@ func NewDeInitDataJob(opt *DeInitOptions) *workflow.Job {
 	deInitJob.AppendTask(tasks.NewCleanupKubeconfigTask())
 
 	deInitJob.SetDataInitializer(func() (workflow.RunData, error) {
+		localClusterClient, err := clientset.NewForConfig(opt.Kubeconfig)
+		if err != nil {
+			return nil, fmt.Errorf("error when creating local cluster client, err: %w", err)
+		}
+
 		// if there is no endpoint info, we are consider that the local cluster
 		// is remote cluster to install karmada.
 		var remoteClient clientset.Interface
-		if opt.HostCluster.SecretRef == nil && len(opt.HostCluster.APIEndpoint) == 0 {
+		if util.IsInCluster(opt.HostCluster) {
-			client, err := clientset.NewForConfig(opt.Kubeconfig)
+			remoteClient = localClusterClient
+		} else {
+			remoteClient, err = util.BuildClientFromSecretRef(localClusterClient, opt.HostCluster.SecretRef)
 			if err != nil {
-				return nil, fmt.Errorf("error when create cluster client to install karmada, err: %w", err)
+				return nil, fmt.Errorf("error when creating cluster client to install karmada, err: %w", err)
 			}
-
-			remoteClient = client
 		}
 
 		if len(opt.Name) == 0 || len(opt.Namespace) == 0 {

operator/pkg/init.go

@@ -7,6 +7,7 @@ import (
 	"sync"
 
 	corev1 "k8s.io/api/core/v1"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	utilversion "k8s.io/apimachinery/pkg/util/version"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -31,11 +32,42 @@ type InitOptions struct {
 	Namespace      string
 	Kubeconfig     *rest.Config
 	KarmadaVersion string
-	CrdRemoteURL   string
+	CRDRemoteURL   string
 	KarmadaDataDir string
 	Karmada        *operatorv1alpha1.Karmada
 }
 
+// Validate is used to validate the initOptions before creating initJob.
+func (opt *InitOptions) Validate() error {
+	var errs []error
+
+	if len(opt.Name) == 0 || len(opt.Namespace) == 0 {
+		return errors.New("unexpected empty name or namespace")
+	}
+	if len(opt.CRDRemoteURL) > 0 {
+		if _, err := url.Parse(opt.CRDRemoteURL); err != nil {
+			return fmt.Errorf("unexpected invalid crds remote url %s", opt.CRDRemoteURL)
+		}
+	}
+	if !util.IsInCluster(opt.Karmada.Spec.HostCluster) && opt.Karmada.Spec.Components.KarmadaAPIServer.ServiceType == corev1.ServiceTypeClusterIP {
+		return fmt.Errorf("if karmada is installed in a remote cluster, the service type of karmada-apiserver must be either NodePort or LoadBalancer")
+	}
+	_, err := utilversion.ParseGeneric(opt.KarmadaVersion)
+	if err != nil {
+		return fmt.Errorf("unexpected karmada invalid version %s", opt.KarmadaVersion)
+	}
+
+	if opt.Karmada.Spec.Components.Etcd.Local != nil && opt.Karmada.Spec.Components.Etcd.Local.CommonSettings.Replicas != nil {
+		replicas := *opt.Karmada.Spec.Components.Etcd.Local.CommonSettings.Replicas
+
+		if (replicas % 2) == 0 {
+			klog.Warningf("invalid etcd replicas %d, expected an odd number", replicas)
+		}
+	}
+
+	return utilerrors.NewAggregate(errs)
+}
+
 // InitOpt defines a type of function to set InitOptions values.
 type InitOpt func(o *InitOptions)
 
@@ -54,7 +86,7 @@ type initData struct {
 	remoteClient        clientset.Interface
 	karmadaClient       clientset.Interface
 	dnsDomain           string
-	crdRemoteURL        string
+	CRDRemoteURL        string
 	karmadaDataDir      string
 	privateRegistry     string
 	featureGates        map[string]bool
@@ -89,16 +121,25 @@ func NewInitJob(opt *InitOptions) *workflow.Job {
 }
 
 func newRunData(opt *InitOptions) (*initData, error) {
-	// if there is no endpoint info, we are consider that the local cluster
+	if err := opt.Validate(); err != nil {
+		return nil, err
+	}
+
+	localClusterClient, err := clientset.NewForConfig(opt.Kubeconfig)
+	if err != nil {
+		return nil, fmt.Errorf("error when creating local cluster client, err: %w", err)
+	}
+
+	// if there is no endpoint message, we are consider that the local cluster
 	// is remote cluster to install karmada.
 	var remoteClient clientset.Interface
-	if opt.Karmada.Spec.HostCluster.SecretRef == nil && len(opt.Karmada.Spec.HostCluster.APIEndpoint) == 0 {
+	if util.IsInCluster(opt.Karmada.Spec.HostCluster) {
-		client, err := clientset.NewForConfig(opt.Kubeconfig)
+		remoteClient = localClusterClient
+	} else {
+		remoteClient, err = util.BuildClientFromSecretRef(localClusterClient, opt.Karmada.Spec.HostCluster.SecretRef)
 		if err != nil {
-			return nil, fmt.Errorf("error when create cluster client to install karmada, err: %w", err)
+			return nil, fmt.Errorf("error when creating cluster client to install karmada, err: %w", err)
 		}
-
-		remoteClient = client
 	}
 
 	var privateRegistry string
@@ -106,29 +147,11 @@ func newRunData(opt *InitOptions) (*initData, error) {
 		privateRegistry = opt.Karmada.Spec.PrivateRegistry.Registry
 	}
 
-	if len(opt.Name) == 0 || len(opt.Namespace) == 0 {
-		return nil, errors.New("unexpected empty name or namespace")
-	}
-
 	version, err := utilversion.ParseGeneric(opt.KarmadaVersion)
 	if err != nil {
 		return nil, fmt.Errorf("unexpected karmada invalid version %s", opt.KarmadaVersion)
 	}
 
-	if len(opt.CrdRemoteURL) > 0 {
-		if _, err := url.Parse(opt.CrdRemoteURL); err != nil {
-			return nil, fmt.Errorf("unexpected invalid crds remote url %s", opt.CrdRemoteURL)
-		}
-	}
-
-	if opt.Karmada.Spec.Components.Etcd.Local != nil && opt.Karmada.Spec.Components.Etcd.Local.CommonSettings.Replicas != nil {
-		replicas := *opt.Karmada.Spec.Components.Etcd.Local.CommonSettings.Replicas
-
-		if (replicas % 2) == 0 {
-			klog.Warningf("invalid etcd replicas %d, expected an odd number", replicas)
-		}
-	}
-
 	// TODO: Verify whether important values of initData is valid
 	var address string
 	if opt.Karmada.Spec.Components.KarmadaAPIServer.ServiceType == corev1.ServiceTypeNodePort {
@@ -144,7 +167,7 @@ func newRunData(opt *InitOptions) (*initData, error) {
 		karmadaVersion:      version,
 		controlplaneAddress: address,
 		remoteClient:        remoteClient,
-		crdRemoteURL:        opt.CrdRemoteURL,
+		CRDRemoteURL:        opt.CRDRemoteURL,
 		karmadaDataDir:      opt.KarmadaDataDir,
 		privateRegistry:     privateRegistry,
 		components:          opt.Karmada.Spec.Components,
@@ -197,7 +220,7 @@ func (data *initData) DataDir() string {
 }
 
 func (data *initData) CrdsRemoteURL() string {
-	return data.crdRemoteURL
+	return data.CRDRemoteURL
 }
 
 func (data *initData) KarmadaVersion() string {
@@ -230,7 +253,7 @@ func defaultJobInitOptions() *InitOptions {
 	operatorscheme.Scheme.Default(karmada)
 
 	return &InitOptions{
-		CrdRemoteURL:   fmt.Sprintf(defaultCrdURL, operatorv1alpha1.DefaultKarmadaImageVersion),
+		CRDRemoteURL:   fmt.Sprintf(defaultCrdURL, operatorv1alpha1.DefaultKarmadaImageVersion),
 		KarmadaVersion: operatorv1alpha1.DefaultKarmadaImageVersion,
 		KarmadaDataDir: constants.KarmadaDataDir,
 		Karmada:        karmada,

operator/pkg/util/kubeconfig.go

@@ -1,9 +1,15 @@
 package util
 
 import (
+	"context"
 	"fmt"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+
+	operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
 )
 
 // CreateWithCerts creates a KubeConfig object with access to the API server with client certificates
@@ -38,3 +44,41 @@ func CreateBasic(serverURL, clusterName, userName string, caCert []byte) *client
 		CurrentContext: contextName,
 	}
 }
+
+// IsInCluster returns a bool represents whether the remote cluster is the local or not.
+func IsInCluster(hostCluster *operatorv1alpha1.HostCluster) bool {
+	return hostCluster == nil || hostCluster.SecretRef == nil || len(hostCluster.SecretRef.Name) == 0
+}
+
+func BuildClientFromSecretRef(client *clientset.Clientset, ref *operatorv1alpha1.LocalSecretReference) (*clientset.Clientset, error) {
+	secret, err := client.CoreV1().Secrets(ref.Namespace).Get(context.TODO(), ref.Name, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	kubeconfigBytes, ok := secret.Data["kubeconfig"]
+	if !ok {
+		return nil, fmt.Errorf("the kubeconfig or data key 'kubeconfig' is not found, please check the secret %s/%s", secret.Namespace, secret.Name)
+	}
+
+	return newClientSetForConfig(kubeconfigBytes)
+}
+
+func newClientSetForConfig(kubeconfig []byte) (*clientset.Clientset, error) {
+	clientConfig, err := clientcmd.NewClientConfigFromBytes(kubeconfig)
+	if err != nil {
+		return nil, err
+	}
+
+	config, err := clientConfig.ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := clientset.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+
+	return client, nil
+}