From fa0884c3337fac963faaae735acac6b0b808fa4a Mon Sep 17 00:00:00 2001 From: calvin Date: Wed, 9 Aug 2023 19:07:57 +0800 Subject: [PATCH] karmada operator: support install karmada on remote cluster Signed-off-by: calvin --- operator/pkg/controller/karmada/planner.go | 39 +++++++++-- operator/pkg/deinit.go | 16 +++-- operator/pkg/init.go | 81 ++++++++++++++-------- operator/pkg/util/kubeconfig.go | 44 ++++++++++++ 4 files changed, 142 insertions(+), 38 deletions(-) diff --git a/operator/pkg/controller/karmada/planner.go b/operator/pkg/controller/karmada/planner.go index 52c094459..5deec82cd 100644 --- a/operator/pkg/controller/karmada/planner.go +++ b/operator/pkg/controller/karmada/planner.go @@ -4,7 +4,10 @@ import ( "context" "fmt" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" + clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" @@ -33,6 +36,7 @@ type Planner struct { client.Client karmada *operatorv1alpha1.Karmada job *workflow.Job + config *rest.Config } // NewPlannerFor creates planner, it will recognize the karmada resource action @@ -68,6 +72,7 @@ func NewPlannerFor(karmada *operatorv1alpha1.Karmada, c client.Client, config *r Client: c, job: job, action: action, + config: config, }, nil } @@ -124,18 +129,44 @@ func (p *Planner) runJobErr(err error) error { func (p *Planner) afterRunJob() error { if p.action == InitAction { - // Update the condition to Ready and set kubeconfig of karmada-apiserver to status. + // Update the karmada condition to Ready and set kubeconfig of karmada apiserver to karmada status. operatorv1alpha1.KarmadaCompleted(p.karmada, operatorv1alpha1.Ready, "karmada init job is completed") + + if !util.IsInCluster(p.karmada.Spec.HostCluster) { + localClusterClient, err := clientset.NewForConfig(p.config) + if err != nil { + return fmt.Errorf("error when creating local cluster client, err: %w", err) + } + + remoteClient, err := util.BuildClientFromSecretRef(localClusterClient, p.karmada.Spec.HostCluster.SecretRef) + if err != nil { + return fmt.Errorf("error when creating cluster client to install karmada, err: %w", err) + } + + secret, err := remoteClient.CoreV1().Secrets(p.karmada.GetNamespace()).Get(context.TODO(), util.AdminKubeconfigSecretName(p.karmada.GetName()), metav1.GetOptions{}) + if err != nil { + return err + } + + _, err = localClusterClient.CoreV1().Secrets(p.karmada.GetNamespace()).Create(context.TODO(), &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: p.karmada.GetNamespace(), + Name: util.AdminKubeconfigSecretName(p.karmada.GetName()), + }, + Data: secret.Data, + }, metav1.CreateOptions{}) + if err != nil { + return err + } + } + p.karmada.Status.SecretRef = &operatorv1alpha1.LocalSecretReference{ Namespace: p.karmada.GetNamespace(), Name: util.AdminKubeconfigSecretName(p.karmada.GetName()), } - return p.Client.Status().Update(context.TODO(), p.karmada) } - // if it is deInit workflow, the cr will be deleted with karmada is be deleted, so we need not to // update the karmada status. - return nil } diff --git a/operator/pkg/deinit.go b/operator/pkg/deinit.go index c2e9cb0a6..5d224d482 100644 --- a/operator/pkg/deinit.go +++ b/operator/pkg/deinit.go @@ -10,6 +10,7 @@ import ( operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1" "github.com/karmada-io/karmada/operator/pkg/constants" tasks "github.com/karmada-io/karmada/operator/pkg/tasks/deinit" + "github.com/karmada-io/karmada/operator/pkg/util" "github.com/karmada-io/karmada/operator/pkg/workflow" ) @@ -44,16 +45,21 @@ func NewDeInitDataJob(opt *DeInitOptions) *workflow.Job { deInitJob.AppendTask(tasks.NewCleanupKubeconfigTask()) deInitJob.SetDataInitializer(func() (workflow.RunData, error) { + localClusterClient, err := clientset.NewForConfig(opt.Kubeconfig) + if err != nil { + return nil, fmt.Errorf("error when creating local cluster client, err: %w", err) + } + // if there is no endpoint info, we are consider that the local cluster // is remote cluster to install karmada. var remoteClient clientset.Interface - if opt.HostCluster.SecretRef == nil && len(opt.HostCluster.APIEndpoint) == 0 { - client, err := clientset.NewForConfig(opt.Kubeconfig) + if util.IsInCluster(opt.HostCluster) { + remoteClient = localClusterClient + } else { + remoteClient, err = util.BuildClientFromSecretRef(localClusterClient, opt.HostCluster.SecretRef) if err != nil { - return nil, fmt.Errorf("error when create cluster client to install karmada, err: %w", err) + return nil, fmt.Errorf("error when creating cluster client to install karmada, err: %w", err) } - - remoteClient = client } if len(opt.Name) == 0 || len(opt.Namespace) == 0 { diff --git a/operator/pkg/init.go b/operator/pkg/init.go index b1d05858b..adcfde441 100644 --- a/operator/pkg/init.go +++ b/operator/pkg/init.go @@ -7,6 +7,7 @@ import ( "sync" corev1 "k8s.io/api/core/v1" + utilerrors "k8s.io/apimachinery/pkg/util/errors" utilversion "k8s.io/apimachinery/pkg/util/version" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -31,11 +32,42 @@ type InitOptions struct { Namespace string Kubeconfig *rest.Config KarmadaVersion string - CrdRemoteURL string + CRDRemoteURL string KarmadaDataDir string Karmada *operatorv1alpha1.Karmada } +// Validate is used to validate the initOptions before creating initJob. +func (opt *InitOptions) Validate() error { + var errs []error + + if len(opt.Name) == 0 || len(opt.Namespace) == 0 { + return errors.New("unexpected empty name or namespace") + } + if len(opt.CRDRemoteURL) > 0 { + if _, err := url.Parse(opt.CRDRemoteURL); err != nil { + return fmt.Errorf("unexpected invalid crds remote url %s", opt.CRDRemoteURL) + } + } + if !util.IsInCluster(opt.Karmada.Spec.HostCluster) && opt.Karmada.Spec.Components.KarmadaAPIServer.ServiceType == corev1.ServiceTypeClusterIP { + return fmt.Errorf("if karmada is installed in a remote cluster, the service type of karmada-apiserver must be either NodePort or LoadBalancer") + } + _, err := utilversion.ParseGeneric(opt.KarmadaVersion) + if err != nil { + return fmt.Errorf("unexpected karmada invalid version %s", opt.KarmadaVersion) + } + + if opt.Karmada.Spec.Components.Etcd.Local != nil && opt.Karmada.Spec.Components.Etcd.Local.CommonSettings.Replicas != nil { + replicas := *opt.Karmada.Spec.Components.Etcd.Local.CommonSettings.Replicas + + if (replicas % 2) == 0 { + klog.Warningf("invalid etcd replicas %d, expected an odd number", replicas) + } + } + + return utilerrors.NewAggregate(errs) +} + // InitOpt defines a type of function to set InitOptions values. type InitOpt func(o *InitOptions) @@ -54,7 +86,7 @@ type initData struct { remoteClient clientset.Interface karmadaClient clientset.Interface dnsDomain string - crdRemoteURL string + CRDRemoteURL string karmadaDataDir string privateRegistry string featureGates map[string]bool @@ -89,16 +121,25 @@ func NewInitJob(opt *InitOptions) *workflow.Job { } func newRunData(opt *InitOptions) (*initData, error) { - // if there is no endpoint info, we are consider that the local cluster + if err := opt.Validate(); err != nil { + return nil, err + } + + localClusterClient, err := clientset.NewForConfig(opt.Kubeconfig) + if err != nil { + return nil, fmt.Errorf("error when creating local cluster client, err: %w", err) + } + + // if there is no endpoint message, we are consider that the local cluster // is remote cluster to install karmada. var remoteClient clientset.Interface - if opt.Karmada.Spec.HostCluster.SecretRef == nil && len(opt.Karmada.Spec.HostCluster.APIEndpoint) == 0 { - client, err := clientset.NewForConfig(opt.Kubeconfig) + if util.IsInCluster(opt.Karmada.Spec.HostCluster) { + remoteClient = localClusterClient + } else { + remoteClient, err = util.BuildClientFromSecretRef(localClusterClient, opt.Karmada.Spec.HostCluster.SecretRef) if err != nil { - return nil, fmt.Errorf("error when create cluster client to install karmada, err: %w", err) + return nil, fmt.Errorf("error when creating cluster client to install karmada, err: %w", err) } - - remoteClient = client } var privateRegistry string @@ -106,29 +147,11 @@ func newRunData(opt *InitOptions) (*initData, error) { privateRegistry = opt.Karmada.Spec.PrivateRegistry.Registry } - if len(opt.Name) == 0 || len(opt.Namespace) == 0 { - return nil, errors.New("unexpected empty name or namespace") - } - version, err := utilversion.ParseGeneric(opt.KarmadaVersion) if err != nil { return nil, fmt.Errorf("unexpected karmada invalid version %s", opt.KarmadaVersion) } - if len(opt.CrdRemoteURL) > 0 { - if _, err := url.Parse(opt.CrdRemoteURL); err != nil { - return nil, fmt.Errorf("unexpected invalid crds remote url %s", opt.CrdRemoteURL) - } - } - - if opt.Karmada.Spec.Components.Etcd.Local != nil && opt.Karmada.Spec.Components.Etcd.Local.CommonSettings.Replicas != nil { - replicas := *opt.Karmada.Spec.Components.Etcd.Local.CommonSettings.Replicas - - if (replicas % 2) == 0 { - klog.Warningf("invalid etcd replicas %d, expected an odd number", replicas) - } - } - // TODO: Verify whether important values of initData is valid var address string if opt.Karmada.Spec.Components.KarmadaAPIServer.ServiceType == corev1.ServiceTypeNodePort { @@ -144,7 +167,7 @@ func newRunData(opt *InitOptions) (*initData, error) { karmadaVersion: version, controlplaneAddress: address, remoteClient: remoteClient, - crdRemoteURL: opt.CrdRemoteURL, + CRDRemoteURL: opt.CRDRemoteURL, karmadaDataDir: opt.KarmadaDataDir, privateRegistry: privateRegistry, components: opt.Karmada.Spec.Components, @@ -197,7 +220,7 @@ func (data *initData) DataDir() string { } func (data *initData) CrdsRemoteURL() string { - return data.crdRemoteURL + return data.CRDRemoteURL } func (data *initData) KarmadaVersion() string { @@ -230,7 +253,7 @@ func defaultJobInitOptions() *InitOptions { operatorscheme.Scheme.Default(karmada) return &InitOptions{ - CrdRemoteURL: fmt.Sprintf(defaultCrdURL, operatorv1alpha1.DefaultKarmadaImageVersion), + CRDRemoteURL: fmt.Sprintf(defaultCrdURL, operatorv1alpha1.DefaultKarmadaImageVersion), KarmadaVersion: operatorv1alpha1.DefaultKarmadaImageVersion, KarmadaDataDir: constants.KarmadaDataDir, Karmada: karmada, diff --git a/operator/pkg/util/kubeconfig.go b/operator/pkg/util/kubeconfig.go index a0be3e17c..53e265683 100644 --- a/operator/pkg/util/kubeconfig.go +++ b/operator/pkg/util/kubeconfig.go @@ -1,9 +1,15 @@ package util import ( + "context" "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + + operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1" ) // CreateWithCerts creates a KubeConfig object with access to the API server with client certificates @@ -38,3 +44,41 @@ func CreateBasic(serverURL, clusterName, userName string, caCert []byte) *client CurrentContext: contextName, } } + +// IsInCluster returns a bool represents whether the remote cluster is the local or not. +func IsInCluster(hostCluster *operatorv1alpha1.HostCluster) bool { + return hostCluster == nil || hostCluster.SecretRef == nil || len(hostCluster.SecretRef.Name) == 0 +} + +func BuildClientFromSecretRef(client *clientset.Clientset, ref *operatorv1alpha1.LocalSecretReference) (*clientset.Clientset, error) { + secret, err := client.CoreV1().Secrets(ref.Namespace).Get(context.TODO(), ref.Name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + kubeconfigBytes, ok := secret.Data["kubeconfig"] + if !ok { + return nil, fmt.Errorf("the kubeconfig or data key 'kubeconfig' is not found, please check the secret %s/%s", secret.Namespace, secret.Name) + } + + return newClientSetForConfig(kubeconfigBytes) +} + +func newClientSetForConfig(kubeconfig []byte) (*clientset.Clientset, error) { + clientConfig, err := clientcmd.NewClientConfigFromBytes(kubeconfig) + if err != nil { + return nil, err + } + + config, err := clientConfig.ClientConfig() + if err != nil { + return nil, err + } + + client, err := clientset.NewForConfig(config) + if err != nil { + return nil, err + } + + return client, nil +}