add nodeport service type for apiserver

Signed-off-by: calvin0327 <wen.chen@daocloud.io>

operator/pkg/apis/operator/v1alpha1/defaults.go

@@ -12,7 +12,7 @@ import (
 
 var (
 	etcdImageRepository                       = fmt.Sprintf("%s/%s", constants.KubeDefaultRepository, constants.Etcd)
-	karmadaAPIServiceImageRepository          = fmt.Sprintf("%s/%s", constants.KubeDefaultRepository, constants.KarmadaAPIServer)
+	karmadaAPIServiceImageRepository          = fmt.Sprintf("%s/%s", constants.KubeDefaultRepository, constants.KubeAPIServer)
 	karmadaAggregatedAPIServerImageRepository = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaAggregatedAPIServer)
 	kubeControllerManagerImageRepository      = fmt.Sprintf("%s/%s", constants.KubeDefaultRepository, constants.KubeControllerManager)
 	karmadaControllerManagerImageRepository   = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaControllerManager)

operator/pkg/certs/certs.go

@@ -40,6 +40,7 @@ const (
 type AltNamesMutatorConfig struct {
 	Name                string
 	Namespace           string
+	ControlplaneAddress string
 	Components          *operatorv1alpha1.KarmadaComponents
 }
 
@@ -122,6 +123,20 @@ func KarmadaCertApiserver() *CertConfig {
 	}
 }
 
+// KarmadaCertClient returns karmada client cert config.
+func KarmadaCertClient() *CertConfig {
+	return &CertConfig{
+		Name:   "karmada-client",
+		CAName: constants.CaCertAndKeyName,
+		Config: certutil.Config{
+			CommonName:   "system:admin",
+			Organization: []string{"system:masters"},
+			Usages:       []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+		},
+		AltNamesMutatorFunc: makeAltNamesMutator(apiServerAltNamesMutator),
+	}
+}
+
 // KarmadaCertFrontProxyCA returns karmada front proxy cert config.
 func KarmadaCertFrontProxyCA() *CertConfig {
 	return &CertConfig{
@@ -444,8 +459,12 @@ func apiServerAltNamesMutator(cfg *AltNamesMutatorConfig) (*certutil.AltNames, e
 			"kubernetes.default.svc",
 			fmt.Sprintf("*.%s.svc.cluster.local", cfg.Namespace),
 			fmt.Sprintf("*.%s.svc", cfg.Namespace),
+			cfg.ControlplaneAddress,
+		},
+		IPs: []net.IP{
+			net.IPv4(127, 0, 0, 1),
+			net.ParseIP(cfg.ControlplaneAddress),
 		},
-		IPs: []net.IP{net.IPv4(127, 0, 0, 1)},
 	}
 
 	if len(cfg.Components.KarmadaAPIServer.CertSANs) > 0 {

operator/pkg/constants/constants.go

@@ -27,7 +27,9 @@ const (
 	// Etcd defines the name of the built-in etcd cluster component
 	Etcd = "etcd"
 	// KarmadaAPIServer defines the name of the karmada-apiserver component
-	KarmadaAPIServer = "kube-apiserver"
+	KarmadaAPIServer = "karmada-apiserver"
+	// KubeAPIServer defines the repository name of the kube apiserver
+	KubeAPIServer = "kube-apiserver"
 	// KarmadaAggregatedAPIServer defines the name of the karmada-aggregated-apiserver component
 	KarmadaAggregatedAPIServer = "karmada-aggregated-apiserver"
 	// KubeControllerManager defines the name of the kube-controller-manager component

operator/pkg/controlplane/apiserver/apiserver.go

@@ -18,7 +18,7 @@ import (
 // EnsureKarmadaAPIServer creates karmada apiserver deployment and service resource
 func EnsureKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, name, namespace string) error {
 	if err := installKarmadaAPIServer(client, cfg.KarmadaAPIServer, name, namespace); err != nil {
-		return err
+		return fmt.Errorf("failed to install karmada apiserver, err: %w", err)
 	}
 
 	return createKarmadaAPIServerService(client, cfg.KarmadaAPIServer, name, namespace)

operator/pkg/controlplane/apiserver/mainfests.go

@@ -7,7 +7,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   labels:
-    karmada-app: kube-apiserver
+    karmada-app: karmada-apiserver
     app.kubernetes.io/managed-by: karmada-operator
   name: {{ .DeploymentName }}
   namespace: {{ .Namespace }}
@@ -15,11 +15,11 @@ spec:
   replicas: {{ .Replicas }}
   selector:
     matchLabels:
-      karmada-app: kube-apiserver
+      karmada-app: karmada-apiserver
   template:
     metadata:
       labels:
-        karmada-app: kube-apiserver
+        karmada-app: karmada-apiserver
     spec:
       automountServiceAccountToken: false
       containers:
@@ -127,7 +127,7 @@ spec:
     protocol: TCP
     targetPort: 5443
   selector:
-    karmada-app: kube-apiserver
+    karmada-app: karmada-apiserver
   type: {{ .ServiceType }}
 `
 

operator/pkg/init.go

@@ -6,6 +6,7 @@ import (
 	"net/url"
 	"sync"
 
+	corev1 "k8s.io/api/core/v1"
 	utilversion "k8s.io/apimachinery/pkg/util/version"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -16,6 +17,7 @@ import (
 	"github.com/karmada-io/karmada/operator/pkg/constants"
 	operatorscheme "github.com/karmada-io/karmada/operator/pkg/scheme"
 	tasks "github.com/karmada-io/karmada/operator/pkg/tasks/init"
+	"github.com/karmada-io/karmada/operator/pkg/util"
 	workflow "github.com/karmada-io/karmada/operator/pkg/workflow"
 )
 
@@ -48,6 +50,7 @@ type initData struct {
 	namespace           string
 	karmadaVersion      *utilversion.Version
 	controlplaneConifig *rest.Config
+	controlplaneAddress string
 	remoteClient        clientset.Interface
 	karmadaClient       clientset.Interface
 	dnsDomain           string
@@ -67,10 +70,11 @@ func NewInitJob(opt *InitOptions) *workflow.Job {
 	initJob.AppendTask(tasks.NewPrepareCrdsTask())
 	initJob.AppendTask(tasks.NewCertTask())
 	initJob.AppendTask(tasks.NewNamespaceTask())
-	initJob.AppendTask(tasks.NewUploadKubeconfigTask())
 	initJob.AppendTask(tasks.NewUploadCertsTask())
 	initJob.AppendTask(tasks.NewEtcdTask())
 	initJob.AppendTask(tasks.NewKarmadaApiserverTask())
+	initJob.AppendTask(tasks.NewUploadKubeconfigTask())
+	initJob.AppendTask(tasks.NewKarmadaAggregatedApiserverTask())
 	initJob.AppendTask(tasks.NewCheckApiserverHealthTask())
 	initJob.AppendTask(tasks.NewKarmadaResourcesTask())
 	initJob.AppendTask(tasks.NewComponentTask())
@@ -125,11 +129,19 @@ func newRunData(opt *InitOptions) (*initData, error) {
 	}
 
 	// TODO: Verify whether important values of initData is valid
+	var address string
+	if opt.Karmada.Spec.Components.KarmadaAPIServer.ServiceType == corev1.ServiceTypeNodePort {
+		address, err = util.GetAPIServiceIP(remoteClient)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get a valid node IP for APIServer, err: %w", err)
+		}
+	}
 
 	return &initData{
 		name:                opt.Name,
 		namespace:           opt.Namespace,
 		karmadaVersion:      version,
+		controlplaneAddress: address,
 		remoteClient:        remoteClient,
 		crdRemoteURL:        opt.CrdRemoteURL,
 		karmadaDataDir:      opt.KarmadaDataDir,
@@ -191,6 +203,10 @@ func (data *initData) KarmadaVersion() string {
 	return data.karmadaVersion.String()
 }
 
+func (data *initData) ControlplaneAddress() string {
+	return data.controlplaneAddress
+}
+
 // NewJobInitOptions calls all of InitOpt func to initialize a InitOptions.
 // if there is not InitOpt functions, it will return a default InitOptions.
 func NewJobInitOptions(opts ...InitOpt) *InitOptions {

operator/pkg/tasks/init/apiserver.go

@@ -13,8 +13,7 @@ import (
 	"github.com/karmada-io/karmada/operator/pkg/workflow"
 )
 
-// NewKarmadaApiserverTask init apiserver task to install karmada apiserver and
+// NewKarmadaApiserverTask inits a task to install karmada-apiserver component
-// karmada aggregated apiserver component
 func NewKarmadaApiserverTask() workflow.Task {
 	return workflow.Task{
 		Name:        "apiserver",
@@ -29,6 +28,17 @@ func NewKarmadaApiserverTask() workflow.Task {
 				Name: fmt.Sprintf("%s-%s", "wait", constants.KarmadaAPIserverComponent),
 				Run:  runWaitKarmadaAPIServer,
 			},
+		},
+	}
+}
+
+// NewKarmadaAggregatedApiserverTask inits a task to install karmada-aggregated-apiserver component
+func NewKarmadaAggregatedApiserverTask() workflow.Task {
+	return workflow.Task{
+		Name:        "aggregated-apiserver",
+		Run:         runAggregatedApiserver,
+		RunSubTasks: true,
+		Tasks: []workflow.Task{
 			{
 				Name: constants.KarmadaAggregatedAPIServerComponent,
 				Run:  runKarmadaAggregatedAPIServer,
@@ -41,6 +51,16 @@ func NewKarmadaApiserverTask() workflow.Task {
 	}
 }
 
+func runAggregatedApiserver(r workflow.RunData) error {
+	data, ok := r.(InitData)
+	if !ok {
+		return errors.New("aggregated-apiserver task invoked with an invalid data struct")
+	}
+
+	klog.V(4).InfoS("[aggregated-apiserver] Running aggregated apiserver task", "karmada", klog.KObj(data))
+	return nil
+}
+
 func runApiserver(r workflow.RunData) error {
 	data, ok := r.(InitData)
 	if !ok {
@@ -58,6 +78,7 @@ func runKarmadaAPIServer(r workflow.RunData) error {
 	}
 
 	cfg := data.Components()
+
 	if cfg.KarmadaAPIServer == nil {
 		klog.V(2).InfoS("[KarmadaApiserver] Skip install karmada-apiserver component")
 		return nil

operator/pkg/tasks/init/cert.go

@@ -137,6 +137,7 @@ func mutateCertConfig(data InitData, cc *certs.CertConfig) error {
 			Name:                data.GetName(),
 			Namespace:           data.GetNamespace(),
 			Components:          data.Components(),
+			ControlplaneAddress: data.ControlplaneAddress(),
 		}, cc)
 
 		if err != nil {

operator/pkg/tasks/init/data.go

@@ -15,6 +15,7 @@ type InitData interface {
 	GetNamespace() string
 	SetControlplaneConifg(config *rest.Config)
 	ControlplaneConifg() *rest.Config
+	ControlplaneAddress() string
 	RemoteClient() clientset.Interface
 	KarmadaClient() clientset.Interface
 	DataDir() string

operator/pkg/tasks/init/upload.go

@@ -1,16 +1,13 @@
 package tasks
 
 import (
-	"crypto/x509"
 	"errors"
 	"fmt"
-	"time"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
-	certutil "k8s.io/client-go/util/cert"
 	"k8s.io/klog/v2"
 
 	"github.com/karmada-io/karmada/operator/pkg/certs"
@@ -49,14 +46,25 @@ func runUploadKubeconfig(r workflow.RunData) error {
 func runUploadAdminKubeconfig(r workflow.RunData) error {
 	data, ok := r.(InitData)
 	if !ok {
-		return errors.New("upload-config task invoked with an invalid data struct")
+		return errors.New("UploadAdminKubeconfig task invoked with an invalid data struct")
 	}
 
+	var endpoint string
+	switch data.Components().KarmadaAPIServer.ServiceType {
+	case corev1.ServiceTypeClusterIP:
 		apiserverName := util.KarmadaAPIServerName(data.GetName())
+		endpoint = fmt.Sprintf("https://%s.%s.svc.cluster.local:%d", apiserverName, data.GetNamespace(), constants.KarmadaAPIserverListenClientPort)
 
-	// TODO: How to get controlPlaneEndpoint?
+	case corev1.ServiceTypeNodePort:
-	localEndpoint := fmt.Sprintf("https://%s.%s.svc.cluster.local:%d", apiserverName, data.GetNamespace(), constants.KarmadaAPIserverListenClientPort)
+		service, err := apiclient.GetService(data.RemoteClient(), util.KarmadaAPIServerName(data.GetName()), data.GetNamespace())
-	kubeconfig, err := buildKubeConfigFromSpec(data.GetCert(constants.CaCertAndKeyName), localEndpoint)
+		if err != nil {
+			return err
+		}
+		nodePort := getNodePortFromAPIServerService(service)
+		endpoint = fmt.Sprintf("https://%s:%d", data.ControlplaneAddress(), nodePort)
+	}
+
+	kubeconfig, err := buildKubeConfigFromSpec(data, endpoint)
 	if err != nil {
 		return err
 	}
@@ -85,16 +93,35 @@ func runUploadAdminKubeconfig(r workflow.RunData) error {
 	}
 	data.SetControlplaneConifg(config)
 
-	klog.V(2).InfoS("[upload-config] Successfully created secret of karmada apiserver kubeconfig", "karmada", klog.KObj(data))
+	klog.V(2).InfoS("[UploadAdminKubeconfig] Successfully created secret of karmada apiserver kubeconfig", "karmada", klog.KObj(data))
 	return nil
 }
 
-func buildKubeConfigFromSpec(ca *certs.KarmadaCert, serverURL string) (*clientcmdapi.Config, error) {
+func getNodePortFromAPIServerService(service *corev1.Service) int32 {
+	var nodePort int32
+	if service.Spec.Type == corev1.ServiceTypeNodePort {
+		for _, port := range service.Spec.Ports {
+			if port.Name != "client" {
+				continue
+			}
+			nodePort = port.NodePort
+		}
+	}
+
+	return nodePort
+}
+
+func buildKubeConfigFromSpec(data InitData, serverURL string) (*clientcmdapi.Config, error) {
+	ca := data.GetCert(constants.CaCertAndKeyName)
 	if ca == nil {
 		return nil, errors.New("unable build karmada admin kubeconfig, CA cert is empty")
 	}
 
-	cc := newClientCertConfigFromKubeConfigSpec(nil)
+	cc := certs.KarmadaCertClient()
+
+	if err := mutateCertConfig(data, cc); err != nil {
+		return nil, fmt.Errorf("error when mutate cert altNames for %s, err: %w", cc.Name, err)
+	}
 	client, err := certs.CreateCertAndKeyFilesWithCA(cc, ca.CertData(), ca.KeyData())
 	if err != nil {
 		return nil, fmt.Errorf("failed to generate karmada apiserver client certificate for kubeconfig, err: %w", err)
@@ -235,16 +262,3 @@ func runUploadWebHookCert(r workflow.RunData) error {
 	klog.V(2).InfoS("[upload-webhookCert] Successfully uploaded webhook certs to secret", "karmada", klog.KObj(data))
 	return nil
 }
-
-func newClientCertConfigFromKubeConfigSpec(notAfter *time.Time) *certs.CertConfig {
-	return &certs.CertConfig{
-		Name:   "karmada-client",
-		CAName: constants.CaCertAndKeyName,
-		Config: certutil.Config{
-			CommonName:   "system:admin",
-			Organization: []string{"system:masters"},
-			Usages:       []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
-		},
-		NotAfter: notAfter,
-	}
-}

operator/pkg/util/apiclient/idempotency.go

@@ -230,8 +230,8 @@ func CreateOrUpdateStatefulSet(client clientset.Interface, statefuleSet *appsv1.
 	return nil
 }
 
-// DeleteDeploymentIfHasLabels delete a Deployment that exists the given labels.
+// DeleteDeploymentIfHasLabels deletes a Deployment that exists the given labels.
-func DeleteDeploymentIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Labels) error {
+func DeleteDeploymentIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Set) error {
 	deployment, err := client.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{})
 	if err != nil {
 		if apierrors.IsNotFound(err) {
@@ -241,15 +241,15 @@ func DeleteDeploymentIfHasLabels(client clientset.Interface, name, namespace str
 		return err
 	}
 
-	if match := containsLabels(deployment.ObjectMeta, constants.KarmadaOperatorLabel); !match {
+	if match := containsLabels(deployment.ObjectMeta, ls); !match {
 		klog.V(4).InfoS("Can not delete Deployment, it doesn't have given label", "Deployment", name, "label", constants.KarmadaOperatorLabelKeyName)
 		return nil
 	}
 	return client.AppsV1().Deployments(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
 }
 
-// DeleteStatefulSetIfHasLabels delete a StatefuleSet that exists the given labels.
+// DeleteStatefulSetIfHasLabels deletes a StatefuleSet that exists the given labels.
-func DeleteStatefulSetIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Labels) error {
+func DeleteStatefulSetIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Set) error {
 	sts, err := client.AppsV1().StatefulSets(namespace).Get(context.TODO(), name, metav1.GetOptions{})
 	if err != nil {
 		if apierrors.IsNotFound(err) {
@@ -259,15 +259,15 @@ func DeleteStatefulSetIfHasLabels(client clientset.Interface, name, namespace st
 		return err
 	}
 
-	if match := containsLabels(sts.ObjectMeta, constants.KarmadaOperatorLabel); !match {
+	if match := containsLabels(sts.ObjectMeta, ls); !match {
 		klog.V(4).InfoS("Can not delete StatefulSet, it doesn't have given label", "StatefulSet", name, "label", constants.KarmadaOperatorLabelKeyName)
 		return nil
 	}
 	return client.AppsV1().StatefulSets(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
 }
 
-// DeleteSecretIfHasLabels delete a secret that exists the given labels.
+// DeleteSecretIfHasLabels deletes a secret that exists the given labels.
-func DeleteSecretIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Labels) error {
+func DeleteSecretIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Set) error {
 	sts, err := client.CoreV1().Secrets(namespace).Get(context.TODO(), name, metav1.GetOptions{})
 	if err != nil {
 		if apierrors.IsNotFound(err) {
@@ -277,15 +277,15 @@ func DeleteSecretIfHasLabels(client clientset.Interface, name, namespace string,
 		return err
 	}
 
-	if match := containsLabels(sts.ObjectMeta, constants.KarmadaOperatorLabel); !match {
+	if match := containsLabels(sts.ObjectMeta, ls); !match {
 		klog.V(4).InfoS("Can not delete Secret, it doesn't have given label", "Secret", name, "label", constants.KarmadaOperatorLabelKeyName)
 		return nil
 	}
 	return client.CoreV1().Secrets(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
 }
 
-// DeleteServiceIfHasLabels delete a service that exists the given labels.
+// DeleteServiceIfHasLabels deletes a service that exists the given labels.
-func DeleteServiceIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Labels) error {
+func DeleteServiceIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Set) error {
 	service, err := client.CoreV1().Services(namespace).Get(context.TODO(), name, metav1.GetOptions{})
 	if err != nil {
 		if apierrors.IsNotFound(err) {
@@ -295,13 +295,18 @@ func DeleteServiceIfHasLabels(client clientset.Interface, name, namespace string
 		return err
 	}
 
-	if match := containsLabels(service.ObjectMeta, constants.KarmadaOperatorLabel); !match {
+	if match := containsLabels(service.ObjectMeta, ls); !match {
 		klog.V(4).InfoS("Can not delete Service, it doesn't have given label", "Service", name, "label", constants.KarmadaOperatorLabelKeyName)
 		return nil
 	}
 	return client.CoreV1().Services(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
 }
 
+// GetService returns service resource with specified name and namespace.
+func GetService(client clientset.Interface, name, namespace string) (*corev1.Service, error) {
+	return client.CoreV1().Services(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+}
+
 func containsLabels(object metav1.ObjectMeta, ls labels.Set) bool {
 	return ls.AsSelector().Matches(labels.Set(object.GetLabels()))
 }

operator/pkg/util/endpoint.go

@@ -0,0 +1,57 @@
+package util
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/url"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	clientset "k8s.io/client-go/kubernetes"
+	netutils "k8s.io/utils/net"
+)
+
+// GetControlplaneEndpoint parses an Endpoint and returns it as a string,
+// or returns an error in case it cannot be parsed.
+func GetControlplaneEndpoint(address, port string) (string, error) {
+	var ip = netutils.ParseIPSloppy(address)
+	if ip == nil {
+		return "", fmt.Errorf("invalid value `%s` given for address", address)
+	}
+	url := formatURL(ip.String(), port)
+	return url.String(), nil
+}
+
+// formatURL takes a host and a port string and creates a net.URL using https scheme
+func formatURL(host, port string) *url.URL {
+	return &url.URL{
+		Scheme: "https",
+		Host:   net.JoinHostPort(host, port),
+	}
+}
+
+// GetAPIServiceIP returns a valid node IP address.
+func GetAPIServiceIP(clientset clientset.Interface) (string, error) {
+	nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+	if err != nil || len(nodes.Items) == 0 {
+		return "", fmt.Errorf("there are no nodes in cluster, err: %w", err)
+	}
+
+	var (
+		masterLabel       = labels.Set{"node-role.kubernetes.io/master": ""}
+		controlplaneLabel = labels.Set{"node-role.kubernetes.io/control-plane": ""}
+	)
+	// first, select the master node as the IP of APIServer. if there is
+	// no master nodes, randomly select a worker node.
+	for _, node := range nodes.Items {
+		ls := labels.Set(node.GetLabels())
+
+		if masterLabel.AsSelector().Matches(ls) || controlplaneLabel.AsSelector().Matches(ls) {
+			if ip := netutils.ParseIPSloppy(node.Status.Addresses[0].Address); ip != nil {
+				return ip.String(), nil
+			}
+		}
+	}
+	return nodes.Items[0].Status.Addresses[0].Address, nil
+}