karmada-io
/
karmada
mirror of https://github.com/karmada-io/karmada.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

karmada operator: support install karmada on remote cluster 

Signed-off-by: calvin <wen.chen@daocloud.io>
This commit is contained in:
calvin 2023-08-09 19:07:57 +08:00
parent 0fe6f1411e
commit fa0884c333
4 changed files with 142 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
operator/pkg/controller/karmada/planner.go
View File

@ -4,7 +4,10 @@ import (
"context"
"fmt"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/client"
@ -33,6 +36,7 @@ type Planner struct {
client.Client
karmada *operatorv1alpha1.Karmada
job *workflow.Job
config *rest.Config
}
// NewPlannerFor creates planner, it will recognize the karmada resource action
@ -68,6 +72,7 @@ func NewPlannerFor(karmada *operatorv1alpha1.Karmada, c client.Client, config *r
Client: c,
job: job,
action: action,
config: config,
}, nil
}
@ -124,18 +129,44 @@ func (p *Planner) runJobErr(err error) error {
func (p *Planner) afterRunJob() error {
if p.action == InitAction {
// Update the condition to Ready and set kubeconfig of karmada-apiserver to status.
// Update the karmada condition to Ready and set kubeconfig of karmada apiserver to karmada status.
operatorv1alpha1.KarmadaCompleted(p.karmada, operatorv1alpha1.Ready, "karmada init job is completed")
if !util.IsInCluster(p.karmada.Spec.HostCluster) {
localClusterClient, err := clientset.NewForConfig(p.config)
if err != nil {
return fmt.Errorf("error when creating local cluster client, err: %w", err)
}
remoteClient, err := util.BuildClientFromSecretRef(localClusterClient, p.karmada.Spec.HostCluster.SecretRef)
if err != nil {
return fmt.Errorf("error when creating cluster client to install karmada, err: %w", err)
}
secret, err := remoteClient.CoreV1().Secrets(p.karmada.GetNamespace()).Get(context.TODO(), util.AdminKubeconfigSecretName(p.karmada.GetName()), metav1.GetOptions{})
if err != nil {
return err
}
_, err = localClusterClient.CoreV1().Secrets(p.karmada.GetNamespace()).Create(context.TODO(), &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Namespace: p.karmada.GetNamespace(),
Name: util.AdminKubeconfigSecretName(p.karmada.GetName()),
},
Data: secret.Data,
}, metav1.CreateOptions{})
if err != nil {
return err
}
}
p.karmada.Status.SecretRef = &operatorv1alpha1.LocalSecretReference{
Namespace: p.karmada.GetNamespace(),
Name: util.AdminKubeconfigSecretName(p.karmada.GetName()),
}
return p.Client.Status().Update(context.TODO(), p.karmada)
}
// if it is deInit workflow, the cr will be deleted with karmada is be deleted, so we need not to
// update the karmada status.
return nil
}

16
operator/pkg/deinit.go
View File

@ -10,6 +10,7 @@ import (
operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
"github.com/karmada-io/karmada/operator/pkg/constants"
tasks "github.com/karmada-io/karmada/operator/pkg/tasks/deinit"
"github.com/karmada-io/karmada/operator/pkg/util"
"github.com/karmada-io/karmada/operator/pkg/workflow"
)
@ -44,16 +45,21 @@ func NewDeInitDataJob(opt *DeInitOptions) *workflow.Job {
deInitJob.AppendTask(tasks.NewCleanupKubeconfigTask())
deInitJob.SetDataInitializer(func() (workflow.RunData, error) {
localClusterClient, err := clientset.NewForConfig(opt.Kubeconfig)
if err != nil {
return nil, fmt.Errorf("error when creating local cluster client, err: %w", err)
}
// if there is no endpoint info, we are consider that the local cluster
// is remote cluster to install karmada.
var remoteClient clientset.Interface
if opt.HostCluster.SecretRef == nil && len(opt.HostCluster.APIEndpoint) == 0 {
client, err := clientset.NewForConfig(opt.Kubeconfig)
if util.IsInCluster(opt.HostCluster) {
remoteClient = localClusterClient
} else {
remoteClient, err = util.BuildClientFromSecretRef(localClusterClient, opt.HostCluster.SecretRef)
if err != nil {
return nil, fmt.Errorf("error when create cluster client to install karmada, err: %w", err)
return nil, fmt.Errorf("error when creating cluster client to install karmada, err: %w", err)
}
remoteClient = client
}
if len(opt.Name) == 0 || len(opt.Namespace) == 0 {

85
operator/pkg/init.go
View File

@ -7,6 +7,7 @@ import (
"sync"
corev1 "k8s.io/api/core/v1"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilversion "k8s.io/apimachinery/pkg/util/version"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
@ -31,11 +32,42 @@ type InitOptions struct {
Namespace string
Kubeconfig *rest.Config
KarmadaVersion string
CrdRemoteURL string
CRDRemoteURL string
KarmadaDataDir string
Karmada *operatorv1alpha1.Karmada
}
// Validate is used to validate the initOptions before creating initJob.
func (opt *InitOptions) Validate() error {
var errs []error
if len(opt.Name) == 0 || len(opt.Namespace) == 0 {
return errors.New("unexpected empty name or namespace")
}
if len(opt.CRDRemoteURL) > 0 {
if _, err := url.Parse(opt.CRDRemoteURL); err != nil {
return fmt.Errorf("unexpected invalid crds remote url %s", opt.CRDRemoteURL)
}
}
if !util.IsInCluster(opt.Karmada.Spec.HostCluster) && opt.Karmada.Spec.Components.KarmadaAPIServer.ServiceType == corev1.ServiceTypeClusterIP {
return fmt.Errorf("if karmada is installed in a remote cluster, the service type of karmada-apiserver must be either NodePort or LoadBalancer")
}
_, err := utilversion.ParseGeneric(opt.KarmadaVersion)
if err != nil {
return fmt.Errorf("unexpected karmada invalid version %s", opt.KarmadaVersion)
}
if opt.Karmada.Spec.Components.Etcd.Local != nil && opt.Karmada.Spec.Components.Etcd.Local.CommonSettings.Replicas != nil {
replicas := *opt.Karmada.Spec.Components.Etcd.Local.CommonSettings.Replicas
if (replicas % 2) == 0 {
klog.Warningf("invalid etcd replicas %d, expected an odd number", replicas)
}
}
return utilerrors.NewAggregate(errs)
}
// InitOpt defines a type of function to set InitOptions values.
type InitOpt func(o *InitOptions)
@ -54,7 +86,7 @@ type initData struct {
remoteClient clientset.Interface
karmadaClient clientset.Interface
dnsDomain string
crdRemoteURL string
CRDRemoteURL string
karmadaDataDir string
privateRegistry string
featureGates map[string]bool
@ -89,16 +121,25 @@ func NewInitJob(opt *InitOptions) *workflow.Job {
}
func newRunData(opt *InitOptions) (*initData, error) {
// if there is no endpoint info, we are consider that the local cluster
// is remote cluster to install karmada.
var remoteClient clientset.Interface
if opt.Karmada.Spec.HostCluster.SecretRef == nil && len(opt.Karmada.Spec.HostCluster.APIEndpoint) == 0 {
client, err := clientset.NewForConfig(opt.Kubeconfig)
if err != nil {
return nil, fmt.Errorf("error when create cluster client to install karmada, err: %w", err)
if err := opt.Validate(); err != nil {
return nil, err
}
remoteClient = client
localClusterClient, err := clientset.NewForConfig(opt.Kubeconfig)
if err != nil {
return nil, fmt.Errorf("error when creating local cluster client, err: %w", err)
}
// if there is no endpoint message, we are consider that the local cluster
// is remote cluster to install karmada.
var remoteClient clientset.Interface
if util.IsInCluster(opt.Karmada.Spec.HostCluster) {
remoteClient = localClusterClient
} else {
remoteClient, err = util.BuildClientFromSecretRef(localClusterClient, opt.Karmada.Spec.HostCluster.SecretRef)
if err != nil {
return nil, fmt.Errorf("error when creating cluster client to install karmada, err: %w", err)
}
}
var privateRegistry string
@ -106,29 +147,11 @@ func newRunData(opt *InitOptions) (*initData, error) {
privateRegistry = opt.Karmada.Spec.PrivateRegistry.Registry
}
if len(opt.Name) == 0 || len(opt.Namespace) == 0 {
return nil, errors.New("unexpected empty name or namespace")
}
version, err := utilversion.ParseGeneric(opt.KarmadaVersion)
if err != nil {
return nil, fmt.Errorf("unexpected karmada invalid version %s", opt.KarmadaVersion)
}
if len(opt.CrdRemoteURL) > 0 {
if _, err := url.Parse(opt.CrdRemoteURL); err != nil {
return nil, fmt.Errorf("unexpected invalid crds remote url %s", opt.CrdRemoteURL)
}
}
if opt.Karmada.Spec.Components.Etcd.Local != nil && opt.Karmada.Spec.Components.Etcd.Local.CommonSettings.Replicas != nil {
replicas := *opt.Karmada.Spec.Components.Etcd.Local.CommonSettings.Replicas
if (replicas % 2) == 0 {
klog.Warningf("invalid etcd replicas %d, expected an odd number", replicas)
}
}
// TODO: Verify whether important values of initData is valid
var address string
if opt.Karmada.Spec.Components.KarmadaAPIServer.ServiceType == corev1.ServiceTypeNodePort {
@ -144,7 +167,7 @@ func newRunData(opt *InitOptions) (*initData, error) {
karmadaVersion: version,
controlplaneAddress: address,
remoteClient: remoteClient,
crdRemoteURL: opt.CrdRemoteURL,
CRDRemoteURL: opt.CRDRemoteURL,
karmadaDataDir: opt.KarmadaDataDir,
privateRegistry: privateRegistry,
components: opt.Karmada.Spec.Components,
@ -197,7 +220,7 @@ func (data *initData) DataDir() string {
}
func (data *initData) CrdsRemoteURL() string {
return data.crdRemoteURL
return data.CRDRemoteURL
}
func (data *initData) KarmadaVersion() string {
@ -230,7 +253,7 @@ func defaultJobInitOptions() *InitOptions {
operatorscheme.Scheme.Default(karmada)
return &InitOptions{
CrdRemoteURL: fmt.Sprintf(defaultCrdURL, operatorv1alpha1.DefaultKarmadaImageVersion),
CRDRemoteURL: fmt.Sprintf(defaultCrdURL, operatorv1alpha1.DefaultKarmadaImageVersion),
KarmadaVersion: operatorv1alpha1.DefaultKarmadaImageVersion,
KarmadaDataDir: constants.KarmadaDataDir,
Karmada: karmada,

44
operator/pkg/util/kubeconfig.go
View File

@ -1,9 +1,15 @@
package util
import (
"context"
"fmt"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
)
// CreateWithCerts creates a KubeConfig object with access to the API server with client certificates
@ -38,3 +44,41 @@ func CreateBasic(serverURL, clusterName, userName string, caCert []byte) *client
CurrentContext: contextName,
}
}
// IsInCluster returns a bool represents whether the remote cluster is the local or not.
func IsInCluster(hostCluster *operatorv1alpha1.HostCluster) bool {
return hostCluster == nil || hostCluster.SecretRef == nil || len(hostCluster.SecretRef.Name) == 0
}
func BuildClientFromSecretRef(client *clientset.Clientset, ref *operatorv1alpha1.LocalSecretReference) (*clientset.Clientset, error) {
secret, err := client.CoreV1().Secrets(ref.Namespace).Get(context.TODO(), ref.Name, metav1.GetOptions{})
if err != nil {
return nil, err
}
kubeconfigBytes, ok := secret.Data["kubeconfig"]
if !ok {
return nil, fmt.Errorf("the kubeconfig or data key 'kubeconfig' is not found, please check the secret %s/%s", secret.Namespace, secret.Name)
}
return newClientSetForConfig(kubeconfigBytes)
}
func newClientSetForConfig(kubeconfig []byte) (*clientset.Clientset, error) {
clientConfig, err := clientcmd.NewClientConfigFromBytes(kubeconfig)
if err != nil {
return nil, err
}
config, err := clientConfig.ClientConfig()
if err != nil {
return nil, err
}
client, err := clientset.NewForConfig(config)
if err != nil {
return nil, err
}
return client, nil
}