karmada-io
/
karmada
mirror of https://github.com/karmada-io/karmada.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add nodeport service type for apiserver 

Signed-off-by: calvin0327 <wen.chen@daocloud.io>
This commit is contained in:
calvin0327 2023-04-09 22:56:52 +08:00 committed by calvin
parent 48e93dccb6
commit 428dd0a769
12 changed files with 200 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
operator/pkg/apis/operator/v1alpha1/defaults.go
View File

@ -12,7 +12,7 @@ import (
var (
etcdImageRepository = fmt.Sprintf("%s/%s", constants.KubeDefaultRepository, constants.Etcd)
karmadaAPIServiceImageRepository = fmt.Sprintf("%s/%s", constants.KubeDefaultRepository, constants.KarmadaAPIServer)
karmadaAPIServiceImageRepository = fmt.Sprintf("%s/%s", constants.KubeDefaultRepository, constants.KubeAPIServer)
karmadaAggregatedAPIServerImageRepository = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaAggregatedAPIServer)
kubeControllerManagerImageRepository = fmt.Sprintf("%s/%s", constants.KubeDefaultRepository, constants.KubeControllerManager)
karmadaControllerManagerImageRepository = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaControllerManager)

27
operator/pkg/certs/certs.go
View File

@ -38,9 +38,10 @@ const (
// AltNamesMutatorConfig is a config to AltNamesMutator. It includes necessary
// configs to AltNamesMutator.
type AltNamesMutatorConfig struct {
Name string
Namespace string
Components *operatorv1alpha1.KarmadaComponents
Name string
Namespace string
ControlplaneAddress string
Components *operatorv1alpha1.KarmadaComponents
}
type altNamesMutatorFunc func(*AltNamesMutatorConfig, *CertConfig) error
@ -122,6 +123,20 @@ func KarmadaCertApiserver() *CertConfig {
}
}
// KarmadaCertClient returns karmada client cert config.
func KarmadaCertClient() *CertConfig {
return &CertConfig{
Name: "karmada-client",
CAName: constants.CaCertAndKeyName,
Config: certutil.Config{
CommonName: "system:admin",
Organization: []string{"system:masters"},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
},
AltNamesMutatorFunc: makeAltNamesMutator(apiServerAltNamesMutator),
}
}
// KarmadaCertFrontProxyCA returns karmada front proxy cert config.
func KarmadaCertFrontProxyCA() *CertConfig {
return &CertConfig{
@ -444,8 +459,12 @@ func apiServerAltNamesMutator(cfg *AltNamesMutatorConfig) (*certutil.AltNames, e
"kubernetes.default.svc",
fmt.Sprintf("*.%s.svc.cluster.local", cfg.Namespace),
fmt.Sprintf("*.%s.svc", cfg.Namespace),
cfg.ControlplaneAddress,
},
IPs: []net.IP{
net.IPv4(127, 0, 0, 1),
net.ParseIP(cfg.ControlplaneAddress),
},
IPs: []net.IP{net.IPv4(127, 0, 0, 1)},
}
if len(cfg.Components.KarmadaAPIServer.CertSANs) > 0 {

4
operator/pkg/constants/constants.go
View File

@ -27,7 +27,9 @@ const (
// Etcd defines the name of the built-in etcd cluster component
Etcd = "etcd"
// KarmadaAPIServer defines the name of the karmada-apiserver component
KarmadaAPIServer = "kube-apiserver"
KarmadaAPIServer = "karmada-apiserver"
// KubeAPIServer defines the repository name of the kube apiserver
KubeAPIServer = "kube-apiserver"
// KarmadaAggregatedAPIServer defines the name of the karmada-aggregated-apiserver component
KarmadaAggregatedAPIServer = "karmada-aggregated-apiserver"
// KubeControllerManager defines the name of the kube-controller-manager component

2
operator/pkg/controlplane/apiserver/apiserver.go
View File

@ -18,7 +18,7 @@ import (
// EnsureKarmadaAPIServer creates karmada apiserver deployment and service resource
func EnsureKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, name, namespace string) error {
if err := installKarmadaAPIServer(client, cfg.KarmadaAPIServer, name, namespace); err != nil {
return err
return fmt.Errorf("failed to install karmada apiserver, err: %w", err)
}
return createKarmadaAPIServerService(client, cfg.KarmadaAPIServer, name, namespace)

8
operator/pkg/controlplane/apiserver/mainfests.go
View File

@ -7,7 +7,7 @@ apiVersion: apps/v1
kind: Deployment
metadata:
labels:
karmada-app: kube-apiserver
karmada-app: karmada-apiserver
app.kubernetes.io/managed-by: karmada-operator
name: {{ .DeploymentName }}
namespace: {{ .Namespace }}
@ -15,11 +15,11 @@ spec:
replicas: {{ .Replicas }}
selector:
matchLabels:
karmada-app: kube-apiserver
karmada-app: karmada-apiserver
template:
metadata:
labels:
karmada-app: kube-apiserver
karmada-app: karmada-apiserver
spec:
automountServiceAccountToken: false
containers:
@ -127,7 +127,7 @@ spec:
protocol: TCP
targetPort: 5443
selector:
karmada-app: kube-apiserver
karmada-app: karmada-apiserver
type: {{ .ServiceType }}
`

40
operator/pkg/init.go
View File

@ -6,6 +6,7 @@ import (
"net/url"
"sync"
corev1 "k8s.io/api/core/v1"
utilversion "k8s.io/apimachinery/pkg/util/version"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
@ -16,6 +17,7 @@ import (
"github.com/karmada-io/karmada/operator/pkg/constants"
operatorscheme "github.com/karmada-io/karmada/operator/pkg/scheme"
tasks "github.com/karmada-io/karmada/operator/pkg/tasks/init"
"github.com/karmada-io/karmada/operator/pkg/util"
workflow "github.com/karmada-io/karmada/operator/pkg/workflow"
)
@ -48,6 +50,7 @@ type initData struct {
namespace string
karmadaVersion *utilversion.Version
controlplaneConifig *rest.Config
controlplaneAddress string
remoteClient clientset.Interface
karmadaClient clientset.Interface
dnsDomain string
@ -67,10 +70,11 @@ func NewInitJob(opt *InitOptions) *workflow.Job {
initJob.AppendTask(tasks.NewPrepareCrdsTask())
initJob.AppendTask(tasks.NewCertTask())
initJob.AppendTask(tasks.NewNamespaceTask())
initJob.AppendTask(tasks.NewUploadKubeconfigTask())
initJob.AppendTask(tasks.NewUploadCertsTask())
initJob.AppendTask(tasks.NewEtcdTask())
initJob.AppendTask(tasks.NewKarmadaApiserverTask())
initJob.AppendTask(tasks.NewUploadKubeconfigTask())
initJob.AppendTask(tasks.NewKarmadaAggregatedApiserverTask())
initJob.AppendTask(tasks.NewCheckApiserverHealthTask())
initJob.AppendTask(tasks.NewKarmadaResourcesTask())
initJob.AppendTask(tasks.NewComponentTask())
@ -125,19 +129,27 @@ func newRunData(opt *InitOptions) (*initData, error) {
}
// TODO: Verify whether important values of initData is valid
var address string
if opt.Karmada.Spec.Components.KarmadaAPIServer.ServiceType == corev1.ServiceTypeNodePort {
address, err = util.GetAPIServiceIP(remoteClient)
if err != nil {
return nil, fmt.Errorf("failed to get a valid node IP for APIServer, err: %w", err)
}
}
return &initData{
name: opt.Name,
namespace: opt.Namespace,
karmadaVersion: version,
remoteClient: remoteClient,
crdRemoteURL: opt.CrdRemoteURL,
karmadaDataDir: opt.KarmadaDataDir,
privateRegistry: privateRegistry,
components: opt.Karmada.Spec.Components,
featureGates: opt.Karmada.Spec.FeatureGates,
dnsDomain: *opt.Karmada.Spec.HostCluster.Networking.DNSDomain,
CertStore: certs.NewCertStore(),
name: opt.Name,
namespace: opt.Namespace,
karmadaVersion: version,
controlplaneAddress: address,
remoteClient: remoteClient,
crdRemoteURL: opt.CrdRemoteURL,
karmadaDataDir: opt.KarmadaDataDir,
privateRegistry: privateRegistry,
components: opt.Karmada.Spec.Components,
featureGates: opt.Karmada.Spec.FeatureGates,
dnsDomain: *opt.Karmada.Spec.HostCluster.Networking.DNSDomain,
CertStore: certs.NewCertStore(),
}, nil
}
@ -191,6 +203,10 @@ func (data *initData) KarmadaVersion() string {
return data.karmadaVersion.String()
}
func (data *initData) ControlplaneAddress() string {
return data.controlplaneAddress
}
// NewJobInitOptions calls all of InitOpt func to initialize a InitOptions.
// if there is not InitOpt functions, it will return a default InitOptions.
func NewJobInitOptions(opts ...InitOpt) *InitOptions {

25
operator/pkg/tasks/init/apiserver.go
View File

@ -13,8 +13,7 @@ import (
"github.com/karmada-io/karmada/operator/pkg/workflow"
)
// NewKarmadaApiserverTask init apiserver task to install karmada apiserver and
// karmada aggregated apiserver component
// NewKarmadaApiserverTask inits a task to install karmada-apiserver component
func NewKarmadaApiserverTask() workflow.Task {
return workflow.Task{
Name: "apiserver",
@ -29,6 +28,17 @@ func NewKarmadaApiserverTask() workflow.Task {
Name: fmt.Sprintf("%s-%s", "wait", constants.KarmadaAPIserverComponent),
Run: runWaitKarmadaAPIServer,
},
},
}
}
// NewKarmadaAggregatedApiserverTask inits a task to install karmada-aggregated-apiserver component
func NewKarmadaAggregatedApiserverTask() workflow.Task {
return workflow.Task{
Name: "aggregated-apiserver",
Run: runAggregatedApiserver,
RunSubTasks: true,
Tasks: []workflow.Task{
{
Name: constants.KarmadaAggregatedAPIServerComponent,
Run: runKarmadaAggregatedAPIServer,
@ -41,6 +51,16 @@ func NewKarmadaApiserverTask() workflow.Task {
}
}
func runAggregatedApiserver(r workflow.RunData) error {
data, ok := r.(InitData)
if !ok {
return errors.New("aggregated-apiserver task invoked with an invalid data struct")
}
klog.V(4).InfoS("[aggregated-apiserver] Running aggregated apiserver task", "karmada", klog.KObj(data))
return nil
}
func runApiserver(r workflow.RunData) error {
data, ok := r.(InitData)
if !ok {
@ -58,6 +78,7 @@ func runKarmadaAPIServer(r workflow.RunData) error {
}
cfg := data.Components()
if cfg.KarmadaAPIServer == nil {
klog.V(2).InfoS("[KarmadaApiserver] Skip install karmada-apiserver component")
return nil

7
operator/pkg/tasks/init/cert.go
View File

@ -134,9 +134,10 @@ func runCertTask(cc, caCert *certs.CertConfig) func(d workflow.RunData) error {
func mutateCertConfig(data InitData, cc *certs.CertConfig) error {
if cc.AltNamesMutatorFunc != nil {
err := cc.AltNamesMutatorFunc(&certs.AltNamesMutatorConfig{
Name: data.GetName(),
Namespace: data.GetNamespace(),
Components: data.Components(),
Name: data.GetName(),
Namespace: data.GetNamespace(),
Components: data.Components(),
ControlplaneAddress: data.ControlplaneAddress(),
}, cc)
if err != nil {

1
operator/pkg/tasks/init/data.go
View File

@ -15,6 +15,7 @@ type InitData interface {
GetNamespace() string
SetControlplaneConifg(config *rest.Config)
ControlplaneConifg() *rest.Config
ControlplaneAddress() string
RemoteClient() clientset.Interface
KarmadaClient() clientset.Interface
DataDir() string

62
operator/pkg/tasks/init/upload.go
View File

@ -1,16 +1,13 @@
package tasks
import (
"crypto/x509"
"errors"
"fmt"
"time"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
certutil "k8s.io/client-go/util/cert"
"k8s.io/klog/v2"
"github.com/karmada-io/karmada/operator/pkg/certs"
@ -49,14 +46,25 @@ func runUploadKubeconfig(r workflow.RunData) error {
func runUploadAdminKubeconfig(r workflow.RunData) error {
data, ok := r.(InitData)
if !ok {
return errors.New("upload-config task invoked with an invalid data struct")
return errors.New("UploadAdminKubeconfig task invoked with an invalid data struct")
}
apiserverName := util.KarmadaAPIServerName(data.GetName())
var endpoint string
switch data.Components().KarmadaAPIServer.ServiceType {
case corev1.ServiceTypeClusterIP:
apiserverName := util.KarmadaAPIServerName(data.GetName())
endpoint = fmt.Sprintf("https://%s.%s.svc.cluster.local:%d", apiserverName, data.GetNamespace(), constants.KarmadaAPIserverListenClientPort)
// TODO: How to get controlPlaneEndpoint?
localEndpoint := fmt.Sprintf("https://%s.%s.svc.cluster.local:%d", apiserverName, data.GetNamespace(), constants.KarmadaAPIserverListenClientPort)
kubeconfig, err := buildKubeConfigFromSpec(data.GetCert(constants.CaCertAndKeyName), localEndpoint)
case corev1.ServiceTypeNodePort:
service, err := apiclient.GetService(data.RemoteClient(), util.KarmadaAPIServerName(data.GetName()), data.GetNamespace())
if err != nil {
return err
}
nodePort := getNodePortFromAPIServerService(service)
endpoint = fmt.Sprintf("https://%s:%d", data.ControlplaneAddress(), nodePort)
}
kubeconfig, err := buildKubeConfigFromSpec(data, endpoint)
if err != nil {
return err
}
@ -85,16 +93,35 @@ func runUploadAdminKubeconfig(r workflow.RunData) error {
}
data.SetControlplaneConifg(config)
klog.V(2).InfoS("[upload-config] Successfully created secret of karmada apiserver kubeconfig", "karmada", klog.KObj(data))
klog.V(2).InfoS("[UploadAdminKubeconfig] Successfully created secret of karmada apiserver kubeconfig", "karmada", klog.KObj(data))
return nil
}
func buildKubeConfigFromSpec(ca *certs.KarmadaCert, serverURL string) (*clientcmdapi.Config, error) {
func getNodePortFromAPIServerService(service *corev1.Service) int32 {
var nodePort int32
if service.Spec.Type == corev1.ServiceTypeNodePort {
for _, port := range service.Spec.Ports {
if port.Name != "client" {
continue
}
nodePort = port.NodePort
}
}
return nodePort
}
func buildKubeConfigFromSpec(data InitData, serverURL string) (*clientcmdapi.Config, error) {
ca := data.GetCert(constants.CaCertAndKeyName)
if ca == nil {
return nil, errors.New("unable build karmada admin kubeconfig, CA cert is empty")
}
cc := newClientCertConfigFromKubeConfigSpec(nil)
cc := certs.KarmadaCertClient()
if err := mutateCertConfig(data, cc); err != nil {
return nil, fmt.Errorf("error when mutate cert altNames for %s, err: %w", cc.Name, err)
}
client, err := certs.CreateCertAndKeyFilesWithCA(cc, ca.CertData(), ca.KeyData())
if err != nil {
return nil, fmt.Errorf("failed to generate karmada apiserver client certificate for kubeconfig, err: %w", err)
@ -235,16 +262,3 @@ func runUploadWebHookCert(r workflow.RunData) error {
klog.V(2).InfoS("[upload-webhookCert] Successfully uploaded webhook certs to secret", "karmada", klog.KObj(data))
return nil
}
func newClientCertConfigFromKubeConfigSpec(notAfter *time.Time) *certs.CertConfig {
return &certs.CertConfig{
Name: "karmada-client",
CAName: constants.CaCertAndKeyName,
Config: certutil.Config{
CommonName: "system:admin",
Organization: []string{"system:masters"},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
},
NotAfter: notAfter,
}
}

29
operator/pkg/util/apiclient/idempotency.go
View File

@ -230,8 +230,8 @@ func CreateOrUpdateStatefulSet(client clientset.Interface, statefuleSet *appsv1.
return nil
}
// DeleteDeploymentIfHasLabels delete a Deployment that exists the given labels.
func DeleteDeploymentIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Labels) error {
// DeleteDeploymentIfHasLabels deletes a Deployment that exists the given labels.
func DeleteDeploymentIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Set) error {
deployment, err := client.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
@ -241,15 +241,15 @@ func DeleteDeploymentIfHasLabels(client clientset.Interface, name, namespace str
return err
}
if match := containsLabels(deployment.ObjectMeta, constants.KarmadaOperatorLabel); !match {
if match := containsLabels(deployment.ObjectMeta, ls); !match {
klog.V(4).InfoS("Can not delete Deployment, it doesn't have given label", "Deployment", name, "label", constants.KarmadaOperatorLabelKeyName)
return nil
}
return client.AppsV1().Deployments(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
}
// DeleteStatefulSetIfHasLabels delete a StatefuleSet that exists the given labels.
func DeleteStatefulSetIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Labels) error {
// DeleteStatefulSetIfHasLabels deletes a StatefuleSet that exists the given labels.
func DeleteStatefulSetIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Set) error {
sts, err := client.AppsV1().StatefulSets(namespace).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
@ -259,15 +259,15 @@ func DeleteStatefulSetIfHasLabels(client clientset.Interface, name, namespace st
return err
}
if match := containsLabels(sts.ObjectMeta, constants.KarmadaOperatorLabel); !match {
if match := containsLabels(sts.ObjectMeta, ls); !match {
klog.V(4).InfoS("Can not delete StatefulSet, it doesn't have given label", "StatefulSet", name, "label", constants.KarmadaOperatorLabelKeyName)
return nil
}
return client.AppsV1().StatefulSets(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
}
// DeleteSecretIfHasLabels delete a secret that exists the given labels.
func DeleteSecretIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Labels) error {
// DeleteSecretIfHasLabels deletes a secret that exists the given labels.
func DeleteSecretIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Set) error {
sts, err := client.CoreV1().Secrets(namespace).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
@ -277,15 +277,15 @@ func DeleteSecretIfHasLabels(client clientset.Interface, name, namespace string,
return err
}
if match := containsLabels(sts.ObjectMeta, constants.KarmadaOperatorLabel); !match {
if match := containsLabels(sts.ObjectMeta, ls); !match {
klog.V(4).InfoS("Can not delete Secret, it doesn't have given label", "Secret", name, "label", constants.KarmadaOperatorLabelKeyName)
return nil
}
return client.CoreV1().Secrets(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
}
// DeleteServiceIfHasLabels delete a service that exists the given labels.
func DeleteServiceIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Labels) error {
// DeleteServiceIfHasLabels deletes a service that exists the given labels.
func DeleteServiceIfHasLabels(client clientset.Interface, name, namespace string, ls labels.Set) error {
service, err := client.CoreV1().Services(namespace).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
@ -295,13 +295,18 @@ func DeleteServiceIfHasLabels(client clientset.Interface, name, namespace string
return err
}
if match := containsLabels(service.ObjectMeta, constants.KarmadaOperatorLabel); !match {
if match := containsLabels(service.ObjectMeta, ls); !match {
klog.V(4).InfoS("Can not delete Service, it doesn't have given label", "Service", name, "label", constants.KarmadaOperatorLabelKeyName)
return nil
}
return client.CoreV1().Services(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
}
// GetService returns service resource with specified name and namespace.
func GetService(client clientset.Interface, name, namespace string) (*corev1.Service, error) {
return client.CoreV1().Services(namespace).Get(context.TODO(), name, metav1.GetOptions{})
}
func containsLabels(object metav1.ObjectMeta, ls labels.Set) bool {
return ls.AsSelector().Matches(labels.Set(object.GetLabels()))
}

57
operator/pkg/util/endpoint.go Normal file
View File

@ -0,0 +1,57 @@
package util
import (
"context"
"fmt"
"net"
"net/url"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
clientset "k8s.io/client-go/kubernetes"
netutils "k8s.io/utils/net"
)
// GetControlplaneEndpoint parses an Endpoint and returns it as a string,
// or returns an error in case it cannot be parsed.
func GetControlplaneEndpoint(address, port string) (string, error) {
var ip = netutils.ParseIPSloppy(address)
if ip == nil {
return "", fmt.Errorf("invalid value `%s` given for address", address)
}
url := formatURL(ip.String(), port)
return url.String(), nil
}
// formatURL takes a host and a port string and creates a net.URL using https scheme
func formatURL(host, port string) *url.URL {
return &url.URL{
Scheme: "https",
Host: net.JoinHostPort(host, port),
}
}
// GetAPIServiceIP returns a valid node IP address.
func GetAPIServiceIP(clientset clientset.Interface) (string, error) {
nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
if err != nil || len(nodes.Items) == 0 {
return "", fmt.Errorf("there are no nodes in cluster, err: %w", err)
}
var (
masterLabel = labels.Set{"node-role.kubernetes.io/master": ""}
controlplaneLabel = labels.Set{"node-role.kubernetes.io/control-plane": ""}
)
// first, select the master node as the IP of APIServer. if there is
// no master nodes, randomly select a worker node.
for _, node := range nodes.Items {
ls := labels.Set(node.GetLabels())
if masterLabel.AsSelector().Matches(ls) || controlplaneLabel.AsSelector().Matches(ls) {
if ip := netutils.ParseIPSloppy(node.Status.Addresses[0].Address); ip != nil {
return ip.String(), nil
}
}
}
return nodes.Items[0].Status.Addresses[0].Address, nil
}