Merge pull request #3898 from tedli/external-etcd

karmadactl init with external etcd

pkg/karmadactl/addons/search/manifests.go

@@ -48,7 +48,8 @@ spec:
             - --audit-log-path=-
             - --feature-gates=APIPriorityAndFairness=false
             - --audit-log-maxage=0
-            - --audit-log-maxbackup=0
+            - --audit-log-maxbackup=0{{- if .KeyPrefix }}
+            - --etcd-prefix={{ .KeyPrefix }}{{- end }}
           livenessProbe:
             httpGet:
               path: /livez
@@ -126,6 +127,7 @@ type DeploymentReplace struct {
 	Replicas   *int32
 	Image      string
 	ETCDSevers string
+	KeyPrefix  string
 }
 
 // ServiceReplace is a struct to help to concrete

pkg/karmadactl/addons/search/search.go

@@ -11,6 +11,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kuberuntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
 	clientsetscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/klog/v2"
 	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
@@ -28,6 +29,8 @@ const (
 
 	// etcdStatefulSetAndServiceName define etcd statefulSet and serviceName installed by init command
 	etcdStatefulSetAndServiceName = "etcd"
+	// karmadaAPIServerDeploymentAndServiceName defines the name of karmada-apiserver deployment and service installed by init command
+	karmadaAPIServerDeploymentAndServiceName = "karmada-apiserver"
 
 	// etcdContainerClientPort define etcd pod installed by init command
 	etcdContainerClientPort = 2379
@@ -134,7 +137,7 @@ func installComponentsOnHostCluster(opts *addoninit.CommandAddonsEnableOption) e
 		return fmt.Errorf("create karmada search service error: %v", err)
 	}
 
-	etcdServers, err := etcdServers(opts)
+	etcdServers, keyPrefix, err := etcdServers(opts)
 	if err != nil {
 		return err
 	}
@@ -146,6 +149,7 @@ func installComponentsOnHostCluster(opts *addoninit.CommandAddonsEnableOption) e
 		Namespace:  opts.Namespace,
 		Replicas:   &opts.KarmadaSearchReplicas,
 		ETCDSevers: etcdServers,
+		KeyPrefix:  keyPrefix,
 		Image:      addoninit.KarmadaSearchImage(opts),
 	})
 	if err != nil {
@@ -212,10 +216,56 @@ func installComponentsOnKarmadaControlPlane(opts *addoninit.CommandAddonsEnableO
 	return nil
 }
 
-func etcdServers(opts *addoninit.CommandAddonsEnableOption) (string, error) {
+const (
-	sts, err := opts.KubeClientSet.AppsV1().StatefulSets(opts.Namespace).Get(context.TODO(), etcdStatefulSetAndServiceName, metav1.GetOptions{})
+	etcdServerArgPrefix          = "--etcd-servers="
+	etcdServerArgPrefixLength    = len(etcdServerArgPrefix)
+	etcdKeyPrefixArgPrefix       = "--etcd-prefix="
+	etcdKeyPrefixArgPrefixLength = len(etcdKeyPrefixArgPrefix)
+)
+
+func getExternalEtcdServerConfig(ctx context.Context, host kubernetes.Interface, namespace string) (servers, prefix string, err error) {
+	var apiserver *appsv1.Deployment
+	if apiserver, err = host.AppsV1().Deployments(namespace).Get(
+		ctx, karmadaAPIServerDeploymentAndServiceName, metav1.GetOptions{}); err != nil {
+		return
+	}
+	// should be only one container, but it may be injected others by mutating webhook of host cluster,
+	// anyway, a for can handle all cases.
+	var apiServerContainer *corev1.Container
+	for i, container := range apiserver.Spec.Template.Spec.Containers {
+		if container.Name == karmadaAPIServerDeploymentAndServiceName {
+			apiServerContainer = &apiserver.Spec.Template.Spec.Containers[i]
+			break
+		}
+	}
+	if apiServerContainer == nil {
+		return
+	}
+	for _, cmd := range apiServerContainer.Command {
+		if strings.HasPrefix(cmd, etcdServerArgPrefix) {
+			servers = cmd[etcdServerArgPrefixLength:]
+		} else if strings.HasPrefix(cmd, etcdKeyPrefixArgPrefix) {
+			prefix = cmd[etcdKeyPrefixArgPrefixLength:]
+		}
+		if servers != "" && prefix != "" {
+			break
+		}
+	}
+	return
+}
+
+func etcdServers(opts *addoninit.CommandAddonsEnableOption) (string, string, error) {
+	ctx := context.TODO()
+	sts, err := opts.KubeClientSet.AppsV1().StatefulSets(opts.Namespace).Get(ctx, etcdStatefulSetAndServiceName, metav1.GetOptions{})
 	if err != nil {
-		return "", err
+		if apierrors.IsNotFound(err) {
+			if servers, prefix, cfgErr := getExternalEtcdServerConfig(ctx, opts.KubeClientSet, opts.Namespace); cfgErr != nil {
+				return "", "", cfgErr
+			} else if servers != "" {
+				return servers, prefix, nil
+			}
+		}
+		return "", "", err
 	}
 
 	etcdReplicas := *sts.Spec.Replicas
@@ -225,5 +275,5 @@ func etcdServers(opts *addoninit.CommandAddonsEnableOption) (string, error) {
 		etcdServers += fmt.Sprintf("https://%s-%v.%s.%s.svc.%s:%v", etcdStatefulSetAndServiceName, v, etcdStatefulSetAndServiceName, opts.Namespace, opts.HostClusterDomain, etcdContainerClientPort) + ","
 	}
 
-	return strings.TrimRight(etcdServers, ","), nil
+	return strings.TrimRight(etcdServers, ","), "", nil
 }

pkg/karmadactl/cmdinit/cert/cert.go

@@ -285,6 +285,10 @@ func GenCerts(pkiPath string, etcdServerCertCfg, etcdClientCertCfg, karmadaCertC
 		return err
 	}
 
+	if etcdServerCertCfg == nil && etcdClientCertCfg == nil {
+		// use external etcd
+		return nil
+	}
 	return genEtcdCerts(pkiPath, etcdServerCertCfg, etcdClientCertCfg)
 }
 

pkg/karmadactl/cmdinit/cmdinit.go

@@ -122,6 +122,11 @@ func NewCmdInit(parentCommand string) *cobra.Command {
 	flags.StringVarP(&opts.EtcdHostDataPath, "etcd-data", "", "/var/lib/karmada-etcd", "etcd data path,valid in hostPath mode.")
 	flags.StringVarP(&opts.EtcdNodeSelectorLabels, "etcd-node-selector-labels", "", "", "etcd pod select the labels of the node. valid in hostPath mode ( e.g. --etcd-node-selector-labels karmada.io/etcd=true)")
 	flags.StringVarP(&opts.EtcdPersistentVolumeSize, "etcd-pvc-size", "", "5Gi", "etcd data path,valid in pvc mode.")
+	flags.StringVar(&opts.ExternalEtcdCACertPath, "external-etcd-ca-cert-path", "", "The path of CA certificate of the external etcd cluster in pem format.")
+	flags.StringVar(&opts.ExternalEtcdClientCertPath, "external-etcd-client-cert-path", "", "The path of client side certificate to the external etcd cluster in pem format.")
+	flags.StringVar(&opts.ExternalEtcdClientKeyPath, "external-etcd-client-key-path", "", "The path of client side private key to the external etcd cluster in pem format.")
+	flags.StringVar(&opts.ExternalEtcdServers, "external-etcd-servers", "", "The server urls of external etcd cluster, to be used by kube-apiserver through --etcd-servers.")
+	flags.StringVar(&opts.ExternalEtcdKeyPrefix, "external-etcd-key-prefix", "", "The key prefix to be configured to kube-apiserver through --etcd-prefix.")
 	// karmada
 	flags.StringVar(&opts.CRDs, "crds", kubernetes.DefaultCrdURL, "Karmada crds resource.(local file e.g. --crds /root/crds.tar.gz)")
 	flags.StringVarP(&opts.KarmadaAPIServerAdvertiseAddress, "karmada-apiserver-advertise-address", "", "", "The IP address the Karmada API Server will advertise it's listening on. If not set, the address on the master node will be used.")

pkg/karmadactl/cmdinit/kubernetes/deploy.go

@@ -45,6 +45,25 @@ var (
 		options.FrontProxyClientCertAndKeyName,
 	}
 
+	emptyByteSlice                 = make([]byte, 0)
+	externalEtcdCertSpecialization = map[string]func(*CommandInitOption) ([]byte, []byte, error){
+		options.EtcdCaCertAndKeyName: func(option *CommandInitOption) (cert, key []byte, err error) {
+			cert, err = os.ReadFile(option.ExternalEtcdCACertPath)
+			key = emptyByteSlice
+			return
+		},
+		options.EtcdServerCertAndKeyName: func(_ *CommandInitOption) ([]byte, []byte, error) {
+			return emptyByteSlice, emptyByteSlice, nil
+		},
+		options.EtcdClientCertAndKeyName: func(option *CommandInitOption) (cert, key []byte, err error) {
+			if cert, err = os.ReadFile(option.ExternalEtcdClientCertPath); err != nil {
+				return
+			}
+			key, err = os.ReadFile(option.ExternalEtcdClientKeyPath)
+			return
+		},
+	}
+
 	karmadaRelease string
 
 	defaultEtcdImage = "etcd:3.5.9-0"
@@ -98,6 +117,11 @@ type CommandInitOption struct {
 	EtcdHostDataPath                   string
 	EtcdNodeSelectorLabels             string
 	EtcdPersistentVolumeSize           string
+	ExternalEtcdCACertPath             string
+	ExternalEtcdClientCertPath         string
+	ExternalEtcdClientKeyPath          string
+	ExternalEtcdServers                string
+	ExternalEtcdKeyPrefix              string
 	KarmadaAPIServerImage              string
 	KarmadaAPIServerReplicas           int32
 	KarmadaAPIServerAdvertiseAddress   string
@@ -131,16 +155,7 @@ type CommandInitOption struct {
 	WaitComponentReadyTimeout          int
 }
 
-// Validate Check that there are enough flags to run the command.
+func (i *CommandInitOption) validateLocalEtcd(parentCommand string) error {
-//
-//nolint:gocyclo
-func (i *CommandInitOption) Validate(parentCommand string) error {
-	if i.KarmadaAPIServerAdvertiseAddress != "" {
-		if netutils.ParseIPSloppy(i.KarmadaAPIServerAdvertiseAddress) == nil {
-			return fmt.Errorf("karmada apiserver advertise address is not valid")
-		}
-	}
-
 	if i.EtcdStorageMode == etcdStorageModeHostPath && i.EtcdHostDataPath == "" {
 		return fmt.Errorf("when etcd storage mode is hostPath, dataPath is not empty. See '%s init --help'", parentCommand)
 	}
@@ -173,6 +188,35 @@ func (i *CommandInitOption) Validate(parentCommand string) error {
 	return nil
 }
 
+func (i *CommandInitOption) validateExternalEtcd(_ string) error {
+	if (i.ExternalEtcdClientCertPath == "" && i.ExternalEtcdClientKeyPath != "") ||
+		(i.ExternalEtcdClientCertPath != "" && i.ExternalEtcdClientKeyPath == "") {
+		return fmt.Errorf("etcd client cert and key should be specified both or none")
+	}
+	return nil
+}
+
+func (i *CommandInitOption) isExternalEtcdProvided() bool {
+	return i.ExternalEtcdServers != ""
+}
+
+// Validate Check that there are enough flags to run the command.
+//
+//nolint:gocyclo
+func (i *CommandInitOption) Validate(parentCommand string) error {
+	if i.KarmadaAPIServerAdvertiseAddress != "" {
+		if netutils.ParseIPSloppy(i.KarmadaAPIServerAdvertiseAddress) == nil {
+			return fmt.Errorf("karmada apiserver advertise address is not valid")
+		}
+	}
+
+	if i.isExternalEtcdProvided() {
+		return i.validateExternalEtcd(parentCommand)
+	} else {
+		return i.validateLocalEtcd(parentCommand)
+	}
+}
+
 // Complete Initialize k8s client
 func (i *CommandInitOption) Complete() error {
 	restConfig, err := apiclient.RestConfig(i.Context, i.KubeConfig)
@@ -192,7 +236,7 @@ func (i *CommandInitOption) Complete() error {
 		return fmt.Errorf("nodePort of karmada apiserver %v already exist", i.KarmadaAPIServerNodePort)
 	}
 
-	if i.EtcdStorageMode == "hostPath" && i.EtcdNodeSelectorLabels == "" {
+	if !i.isExternalEtcdProvided() && i.EtcdStorageMode == "hostPath" && i.EtcdNodeSelectorLabels == "" {
 		if err := i.AddNodeSelectorLabels(); err != nil {
 			return err
 		}
@@ -203,7 +247,7 @@ func (i *CommandInitOption) Complete() error {
 	}
 	klog.Infof("karmada apiserver ip: %s", i.KarmadaAPIServerIP)
 
-	if i.EtcdStorageMode == "hostPath" && i.EtcdNodeSelectorLabels != "" {
+	if !i.isExternalEtcdProvided() && i.EtcdStorageMode == "hostPath" && i.EtcdNodeSelectorLabels != "" {
 		if !i.isNodeExist(i.EtcdNodeSelectorLabels) {
 			return fmt.Errorf("no node found by label %s", i.EtcdNodeSelectorLabels)
 		}
@@ -230,6 +274,8 @@ func initializeDirectory(path string) error {
 func (i *CommandInitOption) genCerts() error {
 	notAfter := time.Now().Add(i.CertValidity).UTC()
 
+	var etcdServerCertConfig, etcdClientCertCfg *cert.CertsConfig
+	if !i.isExternalEtcdProvided() {
 		etcdServerCertDNS := []string{
 			"localhost",
 		}
@@ -241,8 +287,9 @@ func (i *CommandInitOption) genCerts() error {
 			DNSNames: etcdServerCertDNS,
 			IPs:      []net.IP{utils.StringToNetIP("127.0.0.1")},
 		}
-	etcdServerCertConfig := cert.NewCertConfig("karmada-etcd-server", []string{}, etcdServerAltNames, &notAfter)
+		etcdServerCertConfig = cert.NewCertConfig("karmada-etcd-server", []string{}, etcdServerAltNames, &notAfter)
-	etcdClientCertCfg := cert.NewCertConfig("karmada-etcd-client", []string{}, certutil.AltNames{}, &notAfter)
+		etcdClientCertCfg = cert.NewCertConfig("karmada-etcd-client", []string{}, certutil.AltNames{}, &notAfter)
+	}
 
 	karmadaDNS := []string{
 		"localhost",
@@ -357,6 +404,7 @@ func (i *CommandInitOption) createCertsSecrets() error {
 }
 
 func (i *CommandInitOption) initKarmadaAPIServer() error {
+	if !i.isExternalEtcdProvided() {
 		if err := util.CreateOrUpdateService(i.KubeClientSet, i.makeEtcdService(etcdStatefulSetAndServiceName)); err != nil {
 			return err
 		}
@@ -368,6 +416,7 @@ func (i *CommandInitOption) initKarmadaAPIServer() error {
 		if err := util.WaitForStatefulSetRollout(i.KubeClientSet, etcdStatefulSet, i.WaitComponentReadyTimeout); err != nil {
 			klog.Warning(err)
 		}
+	}
 	klog.Info("Create karmada ApiServer Deployment")
 	if err := util.CreateOrUpdateService(i.KubeClientSet, i.makeKarmadaAPIServerService()); err != nil {
 		return err
@@ -388,7 +437,7 @@ func (i *CommandInitOption) initKarmadaAPIServer() error {
 		klog.Exitln(err)
 	}
 	karmadaAggregatedAPIServerDeployment := i.makeKarmadaAggregatedAPIServerDeployment()
-	if _, err := i.KubeClientSet.AppsV1().Deployments(i.Namespace).Create(context.TODO(), i.makeKarmadaAggregatedAPIServerDeployment(), metav1.CreateOptions{}); err != nil {
+	if _, err := i.KubeClientSet.AppsV1().Deployments(i.Namespace).Create(context.TODO(), karmadaAggregatedAPIServerDeployment, metav1.CreateOptions{}); err != nil {
 		klog.Warning(err)
 	}
 	if err := util.WaitForDeploymentRollout(i.KubeClientSet, karmadaAggregatedAPIServerDeployment, i.WaitComponentReadyTimeout); err != nil {
@@ -451,6 +500,22 @@ func (i *CommandInitOption) initKarmadaComponent() error {
 	return nil
 }
 
+func (i *CommandInitOption) readExternalEtcdCert(name string) (isExternalEtcdCert bool, err error) {
+	if !i.isExternalEtcdProvided() {
+		return
+	}
+	var getCertAndKey func(*CommandInitOption) ([]byte, []byte, error)
+	if getCertAndKey, isExternalEtcdCert = externalEtcdCertSpecialization[name]; isExternalEtcdCert {
+		var certs, key []byte
+		if certs, key, err = getCertAndKey(i); err != nil {
+			return
+		}
+		i.CertAndKeyFileData[fmt.Sprintf("%s.crt", name)] = certs
+		i.CertAndKeyFileData[fmt.Sprintf("%s.key", name)] = key
+	}
+	return
+}
+
 // RunInit Deploy karmada in kubernetes
 func (i *CommandInitOption) RunInit(parentCommand string) error {
 	// generate certificate
@@ -461,6 +526,11 @@ func (i *CommandInitOption) RunInit(parentCommand string) error {
 	i.CertAndKeyFileData = map[string][]byte{}
 
 	for _, v := range certList {
+		if isExternalEtcdCert, err := i.readExternalEtcdCert(v); err != nil {
+			return fmt.Errorf("read external etcd certificate failed, %s. %v", v, err)
+		} else if isExternalEtcdCert {
+			continue
+		}
 		certs, err := utils.FileToBytes(i.KarmadaPkiPath, fmt.Sprintf("%s.crt", v))
 		if err != nil {
 			return fmt.Errorf("'%s.crt' conversion failed. %v", v, err)

pkg/karmadactl/cmdinit/kubernetes/deployments.go

@@ -59,7 +59,11 @@ func (i *CommandInitOption) etcdServers() string {
 }
 
 func (i *CommandInitOption) karmadaAPIServerContainerCommand() []string {
-	return []string{
+	var etcdServers string
+	if etcdServers = i.ExternalEtcdServers; etcdServers == "" {
+		etcdServers = strings.TrimRight(i.etcdServers(), ",")
+	}
+	command := []string{
 		"kube-apiserver",
 		"--allow-privileged=true",
 		"--authorization-mode=Node,RBAC",
@@ -68,7 +72,7 @@ func (i *CommandInitOption) karmadaAPIServerContainerCommand() []string {
 		fmt.Sprintf("--etcd-cafile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdCaCertAndKeyName),
 		fmt.Sprintf("--etcd-certfile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName),
 		fmt.Sprintf("--etcd-keyfile=%s/%s.key", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName),
-		fmt.Sprintf("--etcd-servers=%s", strings.TrimRight(i.etcdServers(), ",")),
+		fmt.Sprintf("--etcd-servers=%s", etcdServers),
 		"--bind-address=0.0.0.0",
 		fmt.Sprintf("--kubelet-client-certificate=%s/%s.crt", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName),
 		fmt.Sprintf("--kubelet-client-key=%s/%s.key", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName),
@@ -91,6 +95,10 @@ func (i *CommandInitOption) karmadaAPIServerContainerCommand() []string {
 		fmt.Sprintf("--tls-cert-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.ApiserverCertAndKeyName),
 		fmt.Sprintf("--tls-private-key-file=%s/%s.key", karmadaCertsVolumeMountPath, options.ApiserverCertAndKeyName),
 	}
+	if i.ExternalEtcdKeyPrefix != "" {
+		command = append(command, fmt.Sprintf("--etcd-prefix=%s", i.ExternalEtcdKeyPrefix))
+	}
+	return command
 }
 
 func (i *CommandInitOption) makeKarmadaAPIServerDeployment() *appsv1.Deployment {
@@ -775,6 +783,29 @@ func (i *CommandInitOption) makeKarmadaAggregatedAPIServerDeployment() *appsv1.D
 		TimeoutSeconds:      15,
 	}
 
+	var etcdServers string
+	if etcdServers = i.ExternalEtcdServers; etcdServers == "" {
+		etcdServers = strings.TrimRight(i.etcdServers(), ",")
+	}
+	command := []string{
+		"/bin/karmada-aggregated-apiserver",
+		"--kubeconfig=/etc/kubeconfig",
+		"--authentication-kubeconfig=/etc/kubeconfig",
+		"--authorization-kubeconfig=/etc/kubeconfig",
+		fmt.Sprintf("--etcd-servers=%s", etcdServers),
+		fmt.Sprintf("--etcd-cafile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdCaCertAndKeyName),
+		fmt.Sprintf("--etcd-certfile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName),
+		fmt.Sprintf("--etcd-keyfile=%s/%s.key", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName),
+		fmt.Sprintf("--tls-cert-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName),
+		fmt.Sprintf("--tls-private-key-file=%s/%s.key", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName),
+		"--audit-log-path=-",
+		"--feature-gates=APIPriorityAndFairness=false",
+		"--audit-log-maxage=0",
+		"--audit-log-maxbackup=0",
+	}
+	if i.ExternalEtcdKeyPrefix != "" {
+		command = append(command, fmt.Sprintf("--etcd-prefix=%s", i.ExternalEtcdKeyPrefix))
+	}
 	podSpec := corev1.PodSpec{
 		ImagePullSecrets: i.getImagePullSecrets(),
 		Affinity: &corev1.Affinity{
@@ -800,22 +831,7 @@ func (i *CommandInitOption) makeKarmadaAggregatedAPIServerDeployment() *appsv1.D
 			{
 				Name:    karmadaAggregatedAPIServerDeploymentAndServiceName,
 				Image:   i.karmadaAggregatedAPIServerImage(),
-				Command: []string{
+				Command: command,
-					"/bin/karmada-aggregated-apiserver",
-					"--kubeconfig=/etc/kubeconfig",
-					"--authentication-kubeconfig=/etc/kubeconfig",
-					"--authorization-kubeconfig=/etc/kubeconfig",
-					fmt.Sprintf("--etcd-servers=%s", strings.TrimRight(i.etcdServers(), ",")),
-					fmt.Sprintf("--etcd-cafile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdCaCertAndKeyName),
-					fmt.Sprintf("--etcd-certfile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName),
-					fmt.Sprintf("--etcd-keyfile=%s/%s.key", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName),
-					fmt.Sprintf("--tls-cert-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName),
-					fmt.Sprintf("--tls-private-key-file=%s/%s.key", karmadaCertsVolumeMountPath, options.KarmadaCertAndKeyName),
-					"--audit-log-path=-",
-					"--feature-gates=APIPriorityAndFairness=false",
-					"--audit-log-maxage=0",
-					"--audit-log-maxbackup=0",
-				},
 				VolumeMounts: []corev1.VolumeMount{
 					{
 						Name:      KubeConfigSecretAndMountName,