diff --git a/operator/pkg/apis/operator/v1alpha1/defaults.go b/operator/pkg/apis/operator/v1alpha1/defaults.go index 6009768f9..af9754f87 100644 --- a/operator/pkg/apis/operator/v1alpha1/defaults.go +++ b/operator/pkg/apis/operator/v1alpha1/defaults.go @@ -12,7 +12,7 @@ import ( var ( etcdImageRepository = fmt.Sprintf("%s/%s", constants.KubeDefaultRepository, constants.Etcd) - karmadaAPIServiceImageRepository = fmt.Sprintf("%s/%s", constants.KubeDefaultRepository, constants.KarmadaAPIServer) + karmadaAPIServiceImageRepository = fmt.Sprintf("%s/%s", constants.KubeDefaultRepository, constants.KubeAPIServer) karmadaAggregatedAPIServerImageRepository = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaAggregatedAPIServer) kubeControllerManagerImageRepository = fmt.Sprintf("%s/%s", constants.KubeDefaultRepository, constants.KubeControllerManager) karmadaControllerManagerImageRepository = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaControllerManager) diff --git a/operator/pkg/certs/certs.go b/operator/pkg/certs/certs.go index 19e99658d..1fbc24c06 100644 --- a/operator/pkg/certs/certs.go +++ b/operator/pkg/certs/certs.go @@ -38,9 +38,10 @@ const ( // AltNamesMutatorConfig is a config to AltNamesMutator. It includes necessary // configs to AltNamesMutator. type AltNamesMutatorConfig struct { - Name string - Namespace string - Components *operatorv1alpha1.KarmadaComponents + Name string + Namespace string + ControlplaneAddress string + Components *operatorv1alpha1.KarmadaComponents } type altNamesMutatorFunc func(*AltNamesMutatorConfig, *CertConfig) error @@ -122,6 +123,20 @@ func KarmadaCertApiserver() *CertConfig { } } +// KarmadaCertClient returns karmada client cert config. +func KarmadaCertClient() *CertConfig { + return &CertConfig{ + Name: "karmada-client", + CAName: constants.CaCertAndKeyName, + Config: certutil.Config{ + CommonName: "system:admin", + Organization: []string{"system:masters"}, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }, + AltNamesMutatorFunc: makeAltNamesMutator(apiServerAltNamesMutator), + } +} + // KarmadaCertFrontProxyCA returns karmada front proxy cert config. func KarmadaCertFrontProxyCA() *CertConfig { return &CertConfig{ @@ -444,8 +459,12 @@ func apiServerAltNamesMutator(cfg *AltNamesMutatorConfig) (*certutil.AltNames, e "kubernetes.default.svc", fmt.Sprintf("*.%s.svc.cluster.local", cfg.Namespace), fmt.Sprintf("*.%s.svc", cfg.Namespace), + cfg.ControlplaneAddress, + }, + IPs: []net.IP{ + net.IPv4(127, 0, 0, 1), + net.ParseIP(cfg.ControlplaneAddress), }, - IPs: []net.IP{net.IPv4(127, 0, 0, 1)}, } if len(cfg.Components.KarmadaAPIServer.CertSANs) > 0 { diff --git a/operator/pkg/constants/constants.go b/operator/pkg/constants/constants.go index 9d9ece828..33e84066a 100644 --- a/operator/pkg/constants/constants.go +++ b/operator/pkg/constants/constants.go @@ -27,7 +27,9 @@ const ( // Etcd defines the name of the built-in etcd cluster component Etcd = "etcd" // KarmadaAPIServer defines the name of the karmada-apiserver component - KarmadaAPIServer = "kube-apiserver" + KarmadaAPIServer = "karmada-apiserver" + // KubeAPIServer defines the repository name of the kube apiserver + KubeAPIServer = "kube-apiserver" // KarmadaAggregatedAPIServer defines the name of the karmada-aggregated-apiserver component KarmadaAggregatedAPIServer = "karmada-aggregated-apiserver" // KubeControllerManager defines the name of the kube-controller-manager component diff --git a/operator/pkg/controlplane/apiserver/apiserver.go b/operator/pkg/controlplane/apiserver/apiserver.go index de018c66e..21174530a 100644 --- a/operator/pkg/controlplane/apiserver/apiserver.go +++ b/operator/pkg/controlplane/apiserver/apiserver.go @@ -18,7 +18,7 @@ import ( // EnsureKarmadaAPIServer creates karmada apiserver deployment and service resource func EnsureKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, name, namespace string) error { if err := installKarmadaAPIServer(client, cfg.KarmadaAPIServer, name, namespace); err != nil { - return err + return fmt.Errorf("failed to install karmada apiserver, err: %w", err) } return createKarmadaAPIServerService(client, cfg.KarmadaAPIServer, name, namespace) diff --git a/operator/pkg/controlplane/apiserver/mainfests.go b/operator/pkg/controlplane/apiserver/mainfests.go index 2cc4910ef..aa4ab65d1 100644 --- a/operator/pkg/controlplane/apiserver/mainfests.go +++ b/operator/pkg/controlplane/apiserver/mainfests.go @@ -7,7 +7,7 @@ apiVersion: apps/v1 kind: Deployment metadata: labels: - karmada-app: kube-apiserver + karmada-app: karmada-apiserver app.kubernetes.io/managed-by: karmada-operator name: {{ .DeploymentName }} namespace: {{ .Namespace }} @@ -15,11 +15,11 @@ spec: replicas: {{ .Replicas }} selector: matchLabels: - karmada-app: kube-apiserver + karmada-app: karmada-apiserver template: metadata: labels: - karmada-app: kube-apiserver + karmada-app: karmada-apiserver spec: automountServiceAccountToken: false containers: @@ -127,7 +127,7 @@ spec: protocol: TCP targetPort: 5443 selector: - karmada-app: kube-apiserver + karmada-app: karmada-apiserver type: {{ .ServiceType }} ` diff --git a/operator/pkg/init.go b/operator/pkg/init.go index 2cfa066c7..d3e83fde0 100644 --- a/operator/pkg/init.go +++ b/operator/pkg/init.go @@ -6,6 +6,7 @@ import ( "net/url" "sync" + corev1 "k8s.io/api/core/v1" utilversion "k8s.io/apimachinery/pkg/util/version" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -16,6 +17,7 @@ import ( "github.com/karmada-io/karmada/operator/pkg/constants" operatorscheme "github.com/karmada-io/karmada/operator/pkg/scheme" tasks "github.com/karmada-io/karmada/operator/pkg/tasks/init" + "github.com/karmada-io/karmada/operator/pkg/util" workflow "github.com/karmada-io/karmada/operator/pkg/workflow" ) @@ -48,6 +50,7 @@ type initData struct { namespace string karmadaVersion *utilversion.Version controlplaneConifig *rest.Config + controlplaneAddress string remoteClient clientset.Interface karmadaClient clientset.Interface dnsDomain string @@ -67,10 +70,11 @@ func NewInitJob(opt *InitOptions) *workflow.Job { initJob.AppendTask(tasks.NewPrepareCrdsTask()) initJob.AppendTask(tasks.NewCertTask()) initJob.AppendTask(tasks.NewNamespaceTask()) - initJob.AppendTask(tasks.NewUploadKubeconfigTask()) initJob.AppendTask(tasks.NewUploadCertsTask()) initJob.AppendTask(tasks.NewEtcdTask()) initJob.AppendTask(tasks.NewKarmadaApiserverTask()) + initJob.AppendTask(tasks.NewUploadKubeconfigTask()) + initJob.AppendTask(tasks.NewKarmadaAggregatedApiserverTask()) initJob.AppendTask(tasks.NewCheckApiserverHealthTask()) initJob.AppendTask(tasks.NewKarmadaResourcesTask()) initJob.AppendTask(tasks.NewComponentTask()) @@ -125,19 +129,27 @@ func newRunData(opt *InitOptions) (*initData, error) { } // TODO: Verify whether important values of initData is valid + var address string + if opt.Karmada.Spec.Components.KarmadaAPIServer.ServiceType == corev1.ServiceTypeNodePort { + address, err = util.GetAPIServiceIP(remoteClient) + if err != nil { + return nil, fmt.Errorf("failed to get a valid node IP for APIServer, err: %w", err) + } + } return &initData{ - name: opt.Name, - namespace: opt.Namespace, - karmadaVersion: version, - remoteClient: remoteClient, - crdRemoteURL: opt.CrdRemoteURL, - karmadaDataDir: opt.KarmadaDataDir, - privateRegistry: privateRegistry, - components: opt.Karmada.Spec.Components, - featureGates: opt.Karmada.Spec.FeatureGates, - dnsDomain: *opt.Karmada.Spec.HostCluster.Networking.DNSDomain, - CertStore: certs.NewCertStore(), + name: opt.Name, + namespace: opt.Namespace, + karmadaVersion: version, + controlplaneAddress: address, + remoteClient: remoteClient, + crdRemoteURL: opt.CrdRemoteURL, + karmadaDataDir: opt.KarmadaDataDir, + privateRegistry: privateRegistry, + components: opt.Karmada.Spec.Components, + featureGates: opt.Karmada.Spec.FeatureGates, + dnsDomain: *opt.Karmada.Spec.HostCluster.Networking.DNSDomain, + CertStore: certs.NewCertStore(), }, nil } @@ -191,6 +203,10 @@ func (data *initData) KarmadaVersion() string { return data.karmadaVersion.String() } +func (data *initData) ControlplaneAddress() string { + return data.controlplaneAddress +} + // NewJobInitOptions calls all of InitOpt func to initialize a InitOptions. // if there is not InitOpt functions, it will return a default InitOptions. func NewJobInitOptions(opts ...InitOpt) *InitOptions { diff --git a/operator/pkg/tasks/init/apiserver.go b/operator/pkg/tasks/init/apiserver.go index 5c8e088aa..59fe34318 100644 --- a/operator/pkg/tasks/init/apiserver.go +++ b/operator/pkg/tasks/init/apiserver.go @@ -13,8 +13,7 @@ import ( "github.com/karmada-io/karmada/operator/pkg/workflow" ) -// NewKarmadaApiserverTask init apiserver task to install karmada apiserver and -// karmada aggregated apiserver component +// NewKarmadaApiserverTask inits a task to install karmada-apiserver component func NewKarmadaApiserverTask() workflow.Task { return workflow.Task{ Name: "apiserver", @@ -29,6 +28,17 @@ func NewKarmadaApiserverTask() workflow.Task { Name: fmt.Sprintf("%s-%s", "wait", constants.KarmadaAPIserverComponent), Run: runWaitKarmadaAPIServer, }, + }, + } +} + +// NewKarmadaAggregatedApiserverTask inits a task to install karmada-aggregated-apiserver component +func NewKarmadaAggregatedApiserverTask() workflow.Task { + return workflow.Task{ + Name: "aggregated-apiserver", + Run: runAggregatedApiserver, + RunSubTasks: true, + Tasks: []workflow.Task{ { Name: constants.KarmadaAggregatedAPIServerComponent, Run: runKarmadaAggregatedAPIServer, @@ -41,6 +51,16 @@ func NewKarmadaApiserverTask() workflow.Task { } } +func runAggregatedApiserver(r workflow.RunData) error { + data, ok := r.(InitData) + if !ok { + return errors.New("aggregated-apiserver task invoked with an invalid data struct") + } + + klog.V(4).InfoS("[aggregated-apiserver] Running aggregated apiserver task", "karmada", klog.KObj(data)) + return nil +} + func runApiserver(r workflow.RunData) error { data, ok := r.(InitData) if !ok { @@ -58,6 +78,7 @@ func runKarmadaAPIServer(r workflow.RunData) error { } cfg := data.Components() + if cfg.KarmadaAPIServer == nil { klog.V(2).InfoS("[KarmadaApiserver] Skip install karmada-apiserver component") return nil diff --git a/operator/pkg/tasks/init/cert.go b/operator/pkg/tasks/init/cert.go index 049fafc1f..531fab9e0 100644 --- a/operator/pkg/tasks/init/cert.go +++ b/operator/pkg/tasks/init/cert.go @@ -134,9 +134,10 @@ func runCertTask(cc, caCert *certs.CertConfig) func(d workflow.RunData) error { func mutateCertConfig(data InitData, cc *certs.CertConfig) error { if cc.AltNamesMutatorFunc != nil { err := cc.AltNamesMutatorFunc(&certs.AltNamesMutatorConfig{ - Name: data.GetName(), - Namespace: data.GetNamespace(), - Components: data.Components(), + Name: data.GetName(), + Namespace: data.GetNamespace(), + Components: data.Components(), + ControlplaneAddress: data.ControlplaneAddress(), }, cc) if err != nil { diff --git a/operator/pkg/tasks/init/data.go b/operator/pkg/tasks/init/data.go index c41685dc7..99e77cc73 100644 --- a/operator/pkg/tasks/init/data.go +++ b/operator/pkg/tasks/init/data.go @@ -15,6 +15,7 @@ type InitData interface { GetNamespace() string SetControlplaneConifg(config *rest.Config) ControlplaneConifg() *rest.Config + ControlplaneAddress() string RemoteClient() clientset.Interface KarmadaClient() clientset.Interface DataDir() string diff --git a/operator/pkg/tasks/init/upload.go b/operator/pkg/tasks/init/upload.go index 135fba58f..98b6aa90a 100644 --- a/operator/pkg/tasks/init/upload.go +++ b/operator/pkg/tasks/init/upload.go @@ -1,16 +1,13 @@ package tasks import ( - "crypto/x509" "errors" "fmt" - "time" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - certutil "k8s.io/client-go/util/cert" "k8s.io/klog/v2" "github.com/karmada-io/karmada/operator/pkg/certs" @@ -49,14 +46,25 @@ func runUploadKubeconfig(r workflow.RunData) error { func runUploadAdminKubeconfig(r workflow.RunData) error { data, ok := r.(InitData) if !ok { - return errors.New("upload-config task invoked with an invalid data struct") + return errors.New("UploadAdminKubeconfig task invoked with an invalid data struct") } - apiserverName := util.KarmadaAPIServerName(data.GetName()) + var endpoint string + switch data.Components().KarmadaAPIServer.ServiceType { + case corev1.ServiceTypeClusterIP: + apiserverName := util.KarmadaAPIServerName(data.GetName()) + endpoint = fmt.Sprintf("https://%s.%s.svc.cluster.local:%d", apiserverName, data.GetNamespace(), constants.KarmadaAPIserverListenClientPort) - // TODO: How to get controlPlaneEndpoint? - localEndpoint := fmt.Sprintf("https://%s.%s.svc.cluster.local:%d", apiserverName, data.GetNamespace(), constants.KarmadaAPIserverListenClientPort) - kubeconfig, err := buildKubeConfigFromSpec(data.GetCert(constants.CaCertAndKeyName), localEndpoint) + case corev1.ServiceTypeNodePort: + service, err := apiclient.GetService(data.RemoteClient(), util.KarmadaAPIServerName(data.GetName()), data.GetNamespace()) + if err != nil { + return err + } + nodePort := getNodePortFromAPIServerService(service) + endpoint = fmt.Sprintf("https://%s:%d", data.ControlplaneAddress(), nodePort) + } + + kubeconfig, err := buildKubeConfigFromSpec(data, endpoint) if err != nil { return err } @@ -85,16 +93,35 @@ func runUploadAdminKubeconfig(r workflow.RunData) error { } data.SetControlplaneConifg(config) - klog.V(2).InfoS("[upload-config] Successfully created secret of karmada apiserver kubeconfig", "karmada", klog.KObj(data)) + klog.V(2).InfoS("[UploadAdminKubeconfig] Successfully created secret of karmada apiserver kubeconfig", "karmada", klog.KObj(data)) return nil } -func buildKubeConfigFromSpec(ca *certs.KarmadaCert, serverURL string) (*clientcmdapi.Config, error) { +func getNodePortFromAPIServerService(service *corev1.Service) int32 { + var nodePort int32 + if service.Spec.Type == corev1.ServiceTypeNodePort { + for _, port := range service.Spec.Ports { + if port.Name != "client" { + continue + } + nodePort = port.NodePort + } + } + + return nodePort +} + +func buildKubeConfigFromSpec(data InitData, serverURL string) (*clientcmdapi.Config, error) { + ca := data.GetCert(constants.CaCertAndKeyName) if ca == nil { return nil, errors.New("unable build karmada admin kubeconfig, CA cert is empty") } - cc := newClientCertConfigFromKubeConfigSpec(nil) + cc := certs.KarmadaCertClient() + + if err := mutateCertConfig(data, cc); err != nil { + return nil, fmt.Errorf("error when mutate cert altNames for %s, err: %w", cc.Name, err) + } client, err := certs.CreateCertAndKeyFilesWithCA(cc, ca.CertData(), ca.KeyData()) if err != nil { return nil, fmt.Errorf("failed to generate karmada apiserver client certificate for kubeconfig, err: %w", err) @@ -235,16 +262,3 @@ func runUploadWebHookCert(r workflow.RunData) error { klog.V(2).InfoS("[upload-webhookCert] Successfully uploaded webhook certs to secret", "karmada", klog.KObj(data)) return nil } - -func newClientCertConfigFromKubeConfigSpec(notAfter *time.Time) *certs.CertConfig { - return &certs.CertConfig{ - Name: "karmada-client", - CAName: constants.CaCertAndKeyName, - Config: certutil.Config{ - CommonName: "system:admin", - Organization: []string{"system:masters"}, - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - }, - NotAfter: notAfter, - } -} diff --git a/operator/pkg/util/apiclient/idempotency.go b/operator/pkg/util/apiclient/idempotency.go index 20743a1b0..749513e0c 100644 --- a/operator/pkg/util/apiclient/idempotency.go +++ b/operator/pkg/util/apiclient/idempotency.go @@ -230,8 +230,8 @@ func CreateOrUpdateStatefulSet(client clientset.Interface, statefuleSet *appsv1. return nil } -// DeleteDeploymentIfHasLabels delete a Deployment that exists the given labels. -func DeleteDeploymentIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Labels) error { +// DeleteDeploymentIfHasLabels deletes a Deployment that exists the given labels. +func DeleteDeploymentIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Set) error { deployment, err := client.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { @@ -241,15 +241,15 @@ func DeleteDeploymentIfHasLabels(client clientset.Interface, name, namespace str return err } - if match := containsLabels(deployment.ObjectMeta, constants.KarmadaOperatorLabel); !match { + if match := containsLabels(deployment.ObjectMeta, ls); !match { klog.V(4).InfoS("Can not delete Deployment, it doesn't have given label", "Deployment", name, "label", constants.KarmadaOperatorLabelKeyName) return nil } return client.AppsV1().Deployments(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) } -// DeleteStatefulSetIfHasLabels delete a StatefuleSet that exists the given labels. -func DeleteStatefulSetIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Labels) error { +// DeleteStatefulSetIfHasLabels deletes a StatefuleSet that exists the given labels. +func DeleteStatefulSetIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Set) error { sts, err := client.AppsV1().StatefulSets(namespace).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { @@ -259,15 +259,15 @@ func DeleteStatefulSetIfHasLabels(client clientset.Interface, name, namespace st return err } - if match := containsLabels(sts.ObjectMeta, constants.KarmadaOperatorLabel); !match { + if match := containsLabels(sts.ObjectMeta, ls); !match { klog.V(4).InfoS("Can not delete StatefulSet, it doesn't have given label", "StatefulSet", name, "label", constants.KarmadaOperatorLabelKeyName) return nil } return client.AppsV1().StatefulSets(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) } -// DeleteSecretIfHasLabels delete a secret that exists the given labels. -func DeleteSecretIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Labels) error { +// DeleteSecretIfHasLabels deletes a secret that exists the given labels. +func DeleteSecretIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Set) error { sts, err := client.CoreV1().Secrets(namespace).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { @@ -277,15 +277,15 @@ func DeleteSecretIfHasLabels(client clientset.Interface, name, namespace string, return err } - if match := containsLabels(sts.ObjectMeta, constants.KarmadaOperatorLabel); !match { + if match := containsLabels(sts.ObjectMeta, ls); !match { klog.V(4).InfoS("Can not delete Secret, it doesn't have given label", "Secret", name, "label", constants.KarmadaOperatorLabelKeyName) return nil } return client.CoreV1().Secrets(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) } -// DeleteServiceIfHasLabels delete a service that exists the given labels. -func DeleteServiceIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Labels) error { +// DeleteServiceIfHasLabels deletes a service that exists the given labels. +func DeleteServiceIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Set) error { service, err := client.CoreV1().Services(namespace).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { @@ -295,13 +295,18 @@ func DeleteServiceIfHasLabels(client clientset.Interface, name, namespace string return err } - if match := containsLabels(service.ObjectMeta, constants.KarmadaOperatorLabel); !match { + if match := containsLabels(service.ObjectMeta, ls); !match { klog.V(4).InfoS("Can not delete Service, it doesn't have given label", "Service", name, "label", constants.KarmadaOperatorLabelKeyName) return nil } return client.CoreV1().Services(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) } +// GetService returns service resource with specified name and namespace. +func GetService(client clientset.Interface, name, namespace string) (*corev1.Service, error) { + return client.CoreV1().Services(namespace).Get(context.TODO(), name, metav1.GetOptions{}) +} + func containsLabels(object metav1.ObjectMeta, ls labels.Set) bool { return ls.AsSelector().Matches(labels.Set(object.GetLabels())) } diff --git a/operator/pkg/util/endpoint.go b/operator/pkg/util/endpoint.go new file mode 100644 index 000000000..f24e2cce2 --- /dev/null +++ b/operator/pkg/util/endpoint.go @@ -0,0 +1,57 @@ +package util + +import ( + "context" + "fmt" + "net" + "net/url" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + clientset "k8s.io/client-go/kubernetes" + netutils "k8s.io/utils/net" +) + +// GetControlplaneEndpoint parses an Endpoint and returns it as a string, +// or returns an error in case it cannot be parsed. +func GetControlplaneEndpoint(address, port string) (string, error) { + var ip = netutils.ParseIPSloppy(address) + if ip == nil { + return "", fmt.Errorf("invalid value `%s` given for address", address) + } + url := formatURL(ip.String(), port) + return url.String(), nil +} + +// formatURL takes a host and a port string and creates a net.URL using https scheme +func formatURL(host, port string) *url.URL { + return &url.URL{ + Scheme: "https", + Host: net.JoinHostPort(host, port), + } +} + +// GetAPIServiceIP returns a valid node IP address. +func GetAPIServiceIP(clientset clientset.Interface) (string, error) { + nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil || len(nodes.Items) == 0 { + return "", fmt.Errorf("there are no nodes in cluster, err: %w", err) + } + + var ( + masterLabel = labels.Set{"node-role.kubernetes.io/master": ""} + controlplaneLabel = labels.Set{"node-role.kubernetes.io/control-plane": ""} + ) + // first, select the master node as the IP of APIServer. if there is + // no master nodes, randomly select a worker node. + for _, node := range nodes.Items { + ls := labels.Set(node.GetLabels()) + + if masterLabel.AsSelector().Matches(ls) || controlplaneLabel.AsSelector().Matches(ls) { + if ip := netutils.ParseIPSloppy(node.Status.Addresses[0].Address); ip != nil { + return ip.String(), nil + } + } + } + return nodes.Items[0].Status.Addresses[0].Address, nil +}