karmada-io
/
karmada
mirror of https://github.com/karmada-io/karmada.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3898 from tedli/external-etcd 

karmadactl init with external etcd
This commit is contained in:
karmada-bot 2023-09-04 14:28:46 +08:00 committed by GitHub
parent 44d57c94bc 1ad8fd311a
commit 7c96e0db54
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 208 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/karmadactl/addons/search/manifests.go
View File

@ -48,7 +48,8 @@ spec:
- --audit-log-path=-
- --feature-gates=APIPriorityAndFairness=false
- --audit-log-maxage=0
- --audit-log-maxbackup=0
- --audit-log-maxbackup=0{{- if .KeyPrefix }}
- --etcd-prefix={{ .KeyPrefix }}{{- end }}
livenessProbe:
httpGet:
path: /livez
@ -126,6 +127,7 @@ type DeploymentReplace struct {
Replicas *int32
Image string
ETCDSevers string
KeyPrefix string
}
// ServiceReplace is a struct to help to concrete

60
pkg/karmadactl/addons/search/search.go
View File

@ -11,6 +11,7 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kuberuntime "k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
clientsetscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/klog/v2"
apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
@ -28,6 +29,8 @@ const (
// etcdStatefulSetAndServiceName define etcd statefulSet and serviceName installed by init command
etcdStatefulSetAndServiceName = "etcd"
// karmadaAPIServerDeploymentAndServiceName defines the name of karmada-apiserver deployment and service installed by init command
karmadaAPIServerDeploymentAndServiceName = "karmada-apiserver"
// etcdContainerClientPort define etcd pod installed by init command
etcdContainerClientPort = 2379
@ -134,7 +137,7 @@ func installComponentsOnHostCluster(opts *addoninit.CommandAddonsEnableOption) e
return fmt.Errorf("create karmada search service error: %v", err)
}
etcdServers, err := etcdServers(opts)
etcdServers, keyPrefix, err := etcdServers(opts)
if err != nil {
return err
}
@ -146,6 +149,7 @@ func installComponentsOnHostCluster(opts *addoninit.CommandAddonsEnableOption) e
Namespace: opts.Namespace,
Replicas: &opts.KarmadaSearchReplicas,
ETCDSevers: etcdServers,
KeyPrefix: keyPrefix,
Image: addoninit.KarmadaSearchImage(opts),
})
if err != nil {
@ -212,10 +216,56 @@ func installComponentsOnKarmadaControlPlane(opts *addoninit.CommandAddonsEnableO
return nil
}
func etcdServers(opts *addoninit.CommandAddonsEnableOption) (string, error) {
sts, err := opts.KubeClientSet.AppsV1().StatefulSets(opts.Namespace).Get(context.TODO(), etcdStatefulSetAndServiceName, metav1.GetOptions{})
const (
etcdServerArgPrefix = "--etcd-servers="
etcdServerArgPrefixLength = len(etcdServerArgPrefix)
etcdKeyPrefixArgPrefix = "--etcd-prefix="
etcdKeyPrefixArgPrefixLength = len(etcdKeyPrefixArgPrefix)
)
func getExternalEtcdServerConfig(ctx context.Context, host kubernetes.Interface, namespace string) (servers, prefix string, err error) {
var apiserver *appsv1.Deployment
if apiserver, err = host.AppsV1().Deployments(namespace).Get(
ctx, karmadaAPIServerDeploymentAndServiceName, metav1.GetOptions{}); err != nil {
return
}
// should be only one container, but it may be injected others by mutating webhook of host cluster,
// anyway, a for can handle all cases.
var apiServerContainer *corev1.Container
for i, container := range apiserver.Spec.Template.Spec.Containers {
if container.Name == karmadaAPIServerDeploymentAndServiceName {
apiServerContainer = &apiserver.Spec.Template.Spec.Containers[i]
break
}
}
if apiServerContainer == nil {
return
}
for _, cmd := range apiServerContainer.Command {
if strings.HasPrefix(cmd, etcdServerArgPrefix) {
servers = cmd[etcdServerArgPrefixLength:]
} else if strings.HasPrefix(cmd, etcdKeyPrefixArgPrefix) {
prefix = cmd[etcdKeyPrefixArgPrefixLength:]
}
if servers != "" && prefix != "" {
break
}
}
return
}
func etcdServers(opts *addoninit.CommandAddonsEnableOption) (string, string, error) {
ctx := context.TODO()
sts, err := opts.KubeClientSet.AppsV1().StatefulSets(opts.Namespace).Get(ctx, etcdStatefulSetAndServiceName, metav1.GetOptions{})
if err != nil {
return "", err
if apierrors.IsNotFound(err) {
if servers, prefix, cfgErr := getExternalEtcdServerConfig(ctx, opts.KubeClientSet, opts.Namespace); cfgErr != nil {
return "", "", cfgErr
} else if servers != "" {
return servers, prefix, nil
}
}
return "", "", err
}
etcdReplicas := *sts.Spec.Replicas
@ -225,5 +275,5 @@ func etcdServers(opts *addoninit.CommandAddonsEnableOption) (string, error) {
etcdServers += fmt.Sprintf("https://%s-%v.%s.%s.svc.%s:%v", etcdStatefulSetAndServiceName, v, etcdStatefulSetAndServiceName, opts.Namespace, opts.HostClusterDomain, etcdContainerClientPort) + ","
}
return strings.TrimRight(etcdServers, ","), nil
return strings.TrimRight(etcdServers, ","), "", nil
}

4
pkg/karmadactl/cmdinit/cert/cert.go
View File

@ -285,6 +285,10 @@ func GenCerts(pkiPath string, etcdServerCertCfg, etcdClientCertCfg, karmadaCertC
return err
}
if etcdServerCertCfg == nil && etcdClientCertCfg == nil {
// use external etcd
return nil
}
return genEtcdCerts(pkiPath, etcdServerCertCfg, etcdClientCertCfg)
}

5
pkg/karmadactl/cmdinit/cmdinit.go
View File

@ -122,6 +122,11 @@ func NewCmdInit(parentCommand string) *cobra.Command {
flags.StringVarP(&opts.EtcdHostDataPath, "etcd-data", "", "/var/lib/karmada-etcd", "etcd data path,valid in hostPath mode.")
flags.StringVarP(&opts.EtcdNodeSelectorLabels, "etcd-node-selector-labels", "", "", "etcd pod select the labels of the node. valid in hostPath mode ( e.g. --etcd-node-selector-labels karmada.io/etcd=true)")
flags.StringVarP(&opts.EtcdPersistentVolumeSize, "etcd-pvc-size", "", "5Gi", "etcd data path,valid in pvc mode.")
flags.StringVar(&opts.ExternalEtcdCACertPath, "external-etcd-ca-cert-path", "", "The path of CA certificate of the external etcd cluster in pem format.")
flags.StringVar(&opts.ExternalEtcdClientCertPath, "external-etcd-client-cert-path", "", "The path of client side certificate to the external etcd cluster in pem format.")
flags.StringVar(&opts.ExternalEtcdClientKeyPath, "external-etcd-client-key-path", "", "The path of client side private key to the external etcd cluster in pem format.")
flags.StringVar(&opts.ExternalEtcdServers, "external-etcd-servers", "", "The server urls of external etcd cluster, to be used by kube-apiserver through --etcd-servers.")
flags.StringVar(&opts.ExternalEtcdKeyPrefix, "external-etcd-key-prefix", "", "The key prefix to be configured to kube-apiserver through --etcd-prefix.")
// karmada
flags.StringVar(&opts.CRDs, "crds", kubernetes.DefaultCrdURL, "Karmada crds resource.(local file e.g. --crds /root/crds.tar.gz)")
flags.StringVarP(&opts.KarmadaAPIServerAdvertiseAddress, "karmada-apiserver-advertise-address", "", "", "The IP address the Karmada API Server will advertise it's listening on. If not set, the address on the master node will be used.")

100
pkg/karmadactl/cmdinit/kubernetes/deploy.go
View File

@ -45,6 +45,25 @@ var (
options.FrontProxyClientCertAndKeyName,
}
emptyByteSlice = make([]byte, 0)
externalEtcdCertSpecialization = map[string]func(*CommandInitOption) ([]byte, []byte, error){
options.EtcdCaCertAndKeyName: func(option *CommandInitOption) (cert, key []byte, err error) {
cert, err = os.ReadFile(option.ExternalEtcdCACertPath)
key = emptyByteSlice
return
},
options.EtcdServerCertAndKeyName: func(_ *CommandInitOption) ([]byte, []byte, error) {
return emptyByteSlice, emptyByteSlice, nil
},
options.EtcdClientCertAndKeyName: func(option *CommandInitOption) (cert, key []byte, err error) {
if cert, err = os.ReadFile(option.ExternalEtcdClientCertPath); err != nil {
return
}
key, err = os.ReadFile(option.ExternalEtcdClientKeyPath)
return
},
}
karmadaRelease string
defaultEtcdImage = "etcd:3.5.9-0"
@ -98,6 +117,11 @@ type CommandInitOption struct {
EtcdHostDataPath string
EtcdNodeSelectorLabels string
EtcdPersistentVolumeSize string
ExternalEtcdCACertPath string
ExternalEtcdClientCertPath string
ExternalEtcdClientKeyPath string
ExternalEtcdServers string
ExternalEtcdKeyPrefix string
KarmadaAPIServerImage string
KarmadaAPIServerReplicas int32
KarmadaAPIServerAdvertiseAddress string
@ -131,16 +155,7 @@ type CommandInitOption struct {
WaitComponentReadyTimeout int
}
// Validate Check that there are enough flags to run the command.
//
//nolint:gocyclo
func (i *CommandInitOption) Validate(parentCommand string) error {
if i.KarmadaAPIServerAdvertiseAddress != "" {
if netutils.ParseIPSloppy(i.KarmadaAPIServerAdvertiseAddress) == nil {
return fmt.Errorf("karmada apiserver advertise address is not valid")
}
}
func (i *CommandInitOption) validateLocalEtcd(parentCommand string) error {
if i.EtcdStorageMode == etcdStorageModeHostPath && i.EtcdHostDataPath == "" {
return fmt.Errorf("when etcd storage mode is hostPath, dataPath is not empty. See '%s init --help'", parentCommand)
}
@ -173,6 +188,35 @@ func (i *CommandInitOption) Validate(parentCommand string) error {
return nil
}
func (i *CommandInitOption) validateExternalEtcd(_ string) error {
if (i.ExternalEtcdClientCertPath == "" && i.ExternalEtcdClientKeyPath != "") ||
(i.ExternalEtcdClientCertPath != "" && i.ExternalEtcdClientKeyPath == "") {
return fmt.Errorf("etcd client cert and key should be specified both or none")
}
return nil
}
func (i *CommandInitOption) isExternalEtcdProvided() bool {
return i.ExternalEtcdServers != ""
}
// Validate Check that there are enough flags to run the command.
//
//nolint:gocyclo
func (i *CommandInitOption) Validate(parentCommand string) error {
if i.KarmadaAPIServerAdvertiseAddress != "" {
if netutils.ParseIPSloppy(i.KarmadaAPIServerAdvertiseAddress) == nil {
return fmt.Errorf("karmada apiserver advertise address is not valid")
}
}
if i.isExternalEtcdProvided() {
return i.validateExternalEtcd(parentCommand)
} else {
return i.validateLocalEtcd(parentCommand)
}
}
// Complete Initialize k8s client
func (i *CommandInitOption) Complete() error {
restConfig, err := apiclient.RestConfig(i.Context, i.KubeConfig)
@ -192,7 +236,7 @@ func (i *CommandInitOption) Complete() error {
return fmt.Errorf("nodePort of karmada apiserver %v already exist", i.KarmadaAPIServerNodePort)
}
if i.EtcdStorageMode == "hostPath" && i.EtcdNodeSelectorLabels == "" {
if !i.isExternalEtcdProvided() && i.EtcdStorageMode == "hostPath" && i.EtcdNodeSelectorLabels == "" {
if err := i.AddNodeSelectorLabels(); err != nil {
return err
}
@ -203,7 +247,7 @@ func (i *CommandInitOption) Complete() error {
}
klog.Infof("karmada apiserver ip: %s", i.KarmadaAPIServerIP)
if i.EtcdStorageMode == "hostPath" && i.EtcdNodeSelectorLabels != "" {
if !i.isExternalEtcdProvided() && i.EtcdStorageMode == "hostPath" && i.EtcdNodeSelectorLabels != "" {
if !i.isNodeExist(i.EtcdNodeSelectorLabels) {
return fmt.Errorf("no node found by label %s", i.EtcdNodeSelectorLabels)
}
@ -230,6 +274,8 @@ func initializeDirectory(path string) error {
func (i *CommandInitOption) genCerts() error {
notAfter := time.Now().Add(i.CertValidity).UTC()
var etcdServerCertConfig, etcdClientCertCfg *cert.CertsConfig
if !i.isExternalEtcdProvided() {
etcdServerCertDNS := []string{
"localhost",
}
@ -241,8 +287,9 @@ func (i *CommandInitOption) genCerts() error {
DNSNames: etcdServerCertDNS,
IPs: []net.IP{utils.StringToNetIP("127.0.0.1")},
}
etcdServerCertConfig := cert.NewCertConfig("karmada-etcd-server", []string{}, etcdServerAltNames, &notAfter)
etcdClientCertCfg := cert.NewCertConfig("karmada-etcd-client", []string{}, certutil.AltNames{}, &notAfter)
etcdServerCertConfig = cert.NewCertConfig("karmada-etcd-server", []string{}, etcdServerAltNames, &notAfter)
etcdClientCertCfg = cert.NewCertConfig("karmada-etcd-client", []string{}, certutil.AltNames{}, &notAfter)
}
karmadaDNS := []string{
"localhost",
@ -357,6 +404,7 @@ func (i *CommandInitOption) createCertsSecrets() error {
}
func (i *CommandInitOption) initKarmadaAPIServer() error {
if !i.isExternalEtcdProvided() {
if err := util.CreateOrUpdateService(i.KubeClientSet, i.makeEtcdService(etcdStatefulSetAndServiceName)); err != nil {
return err
}
@ -368,6 +416,7 @@ func (i *CommandInitOption) initKarmadaAPIServer() error {
if err := util.WaitForStatefulSetRollout(i.KubeClientSet, etcdStatefulSet, i.WaitComponentReadyTimeout); err != nil {
klog.Warning(err)
}
}
klog.Info("Create karmada ApiServer Deployment")
if err := util.CreateOrUpdateService(i.KubeClientSet, i.makeKarmadaAPIServerService()); err != nil {
return err
@ -388,7 +437,7 @@ func (i *CommandInitOption) initKarmadaAPIServer() error {
klog.Exitln(err)
}
karmadaAggregatedAPIServerDeployment := i.makeKarmadaAggregatedAPIServerDeployment()
if _, err := i.KubeClientSet.AppsV1().Deployments(i.Namespace).Create(context.TODO(), i.makeKarmadaAggregatedAPIServerDeployment(), metav1.CreateOptions{}); err != nil {
if _, err := i.KubeClientSet.AppsV1().Deployments(i.Namespace).Create(context.TODO(), karmadaAggregatedAPIServerDeployment, metav1.CreateOptions{}); err != nil {
klog.Warning(err)
}
if err := util.WaitForDeploymentRollout(i.KubeClientSet, karmadaAggregatedAPIServerDeployment, i.WaitComponentReadyTimeout); err != nil {
@ -451,6 +500,22 @@ func (i *CommandInitOption) initKarmadaComponent() error {
return nil
}
func (i *CommandInitOption) readExternalEtcdCert(name string) (isExternalEtcdCert bool, err error) {
if !i.isExternalEtcdProvided() {
return
}
var getCertAndKey func(*CommandInitOption) ([]byte, []byte, error)
if getCertAndKey, isExternalEtcdCert = externalEtcdCertSpecialization[name]; isExternalEtcdCert {
var certs, key []byte
if certs, key, err = getCertAndKey(i); err != nil {
return
}
i.CertAndKeyFileData[fmt.Sprintf("%s.crt", name)] = certs
i.CertAndKeyFileData[fmt.Sprintf("%s.key", name)] = key
}
return
}
// RunInit Deploy karmada in kubernetes
func (i *CommandInitOption) RunInit(parentCommand string) error {
// generate certificate
@ -461,6 +526,11 @@ func (i *CommandInitOption) RunInit(parentCommand string) error {
i.CertAndKeyFileData = map[string][]byte{}
for _, v := range certList {
if isExternalEtcdCert, err := i.readExternalEtcdCert(v); err != nil {
return fmt.Errorf("read external etcd certificate failed, %s. %v", v, err)
} else if isExternalEtcdCert {
continue
}
certs, err := utils.FileToBytes(i.KarmadaPkiPath, fmt.Sprintf("%s.crt", v))
if err != nil {
return fmt.Errorf("'%s.crt' conversion failed. %v", v, err)

52
pkg/karmadactl/cmdinit/kubernetes/deployments.go
View File

@ -59,7 +59,11 @@ func (i *CommandInitOption) etcdServers() string {
}
func (i *CommandInitOption) karmadaAPIServerContainerCommand() []string {
return []string{
var etcdServers string
if etcdServers = i.ExternalEtcdServers; etcdServers == "" {
etcdServers = strings.TrimRight(i.etcdServers(), ",")
}
command := []string{
"kube-apiserver",
"--allow-privileged=true",
"--authorization-mode=Node,RBAC",
@ -68,7 +72,7 @@ func (i *CommandInitOption) karmadaAPIServerContainerCommand() []string {
fmt.Sprintf("--etcd-cafile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdCaCertAndKeyName),
fmt.Sprintf("--etcd-certfile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName),
fmt.Sprintf("--etcd-keyfile=%s/%s.key", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName),
fmt.Sprintf("--etcd-servers=%s", strings.TrimRight(i.etcdServers(), ",")),
fmt.Sprintf("--etcd-servers=%s", etcdServers),
"--bind-address=0.0.0.0",
fmt.Sprintf("--kubelet-client-certificate=%s/%s.crt", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName),
fmt.Sprintf("--kubelet-client-key=%s/%s.key", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName),
@ -91,6 +95,10 @@ func (i *CommandInitOption) karmadaAPIServerContainerCommand() []string {
fmt.Sprintf("--tls-cert-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.ApiserverCertAndKeyName),
fmt.Sprintf("--tls-private-key-file=%s/%s.key", karmadaCertsVolumeMountPath, options.ApiserverCertAndKeyName),
}
if i.ExternalEtcdKeyPrefix != "" {
command = append(command, fmt.Sprintf("--etcd-prefix=%s", i.ExternalEtcdKeyPrefix))
}
return command
}
func (i *CommandInitOption) makeKarmadaAPIServerDeployment() *appsv1.Deployment {
@ -775,6 +783,29 @@ func (i *CommandInitOption) makeKarmadaAggregatedAPIServerDeployment() *appsv1.D
TimeoutSeconds: 15,
}
var etcdServers string
if etcdServers = i.ExternalEtcdServers; etcdServers == "" {
etcdServers = strings.TrimRight(i.etcdServers(), ",")
}
command := []string{
"/bin/karmada-aggregated-apiserver",
"--kubeconfig=/etc/kubeconfig",
"--authentication-kubeconfig=/etc/kubeconfig",
"--authorization-kubeconfig=/etc/kubeconfig",
fmt.Sprintf("--etcd-servers=%s", etcdServers),
fmt.Sprintf("--etcd-cafile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdCaCertAndKeyName),
fmt.Sprintf("--etcd-certfile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName),
fmt.Sprintf("--etcd-keyfile=%s/%s.key", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName),
fmt.Sprintf("--tls-cert-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName),
fmt.Sprintf("--tls-private-key-file=%s/%s.key", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName),
"--audit-log-path=-",
"--feature-gates=APIPriorityAndFairness=false",
"--audit-log-maxage=0",
"--audit-log-maxbackup=0",
}
if i.ExternalEtcdKeyPrefix != "" {
command = append(command, fmt.Sprintf("--etcd-prefix=%s", i.ExternalEtcdKeyPrefix))
}
podSpec := corev1.PodSpec{
ImagePullSecrets: i.getImagePullSecrets(),
Affinity: &corev1.Affinity{
@ -800,22 +831,7 @@ func (i *CommandInitOption) makeKarmadaAggregatedAPIServerDeployment() *appsv1.D
{
Name: karmadaAggregatedAPIServerDeploymentAndServiceName,
Image: i.karmadaAggregatedAPIServerImage(),
Command: []string{
"/bin/karmada-aggregated-apiserver",
"--kubeconfig=/etc/kubeconfig",
"--authentication-kubeconfig=/etc/kubeconfig",
"--authorization-kubeconfig=/etc/kubeconfig",
fmt.Sprintf("--etcd-servers=%s", strings.TrimRight(i.etcdServers(), ",")),
fmt.Sprintf("--etcd-cafile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdCaCertAndKeyName),
fmt.Sprintf("--etcd-certfile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName),
fmt.Sprintf("--etcd-keyfile=%s/%s.key", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName),
fmt.Sprintf("--tls-cert-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName),
fmt.Sprintf("--tls-private-key-file=%s/%s.key", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName),
"--audit-log-path=-",
"--feature-gates=APIPriorityAndFairness=false",
"--audit-log-maxage=0",
"--audit-log-maxbackup=0",
},
Command: command,
VolumeMounts: []corev1.VolumeMount{
{
Name: KubeConfigSecretAndMountName,