karmada-io
/
karmada
mirror of https://github.com/karmada-io/karmada.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5720 from jabellard/external_etcd_part2 

External `etcd` Support for Karmada Operator - Part 2
This commit is contained in:
karmada-bot 2024-10-24 09:21:33 +08:00 committed by GitHub
parent 331145f789 54427360c8
commit 60e8e34009
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 307 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
operator/pkg/constants/constants.go
View File

@ -77,6 +77,16 @@ const (
KarmadaAPIserverListenClientPort = 5443
// EtcdDataVolumeName defines the name to etcd data volume
EtcdDataVolumeName = "etcd-data"
// EtcdClientCredentialsVolumeName defines the name of the volume for the etcd client credentials
EtcdClientCredentialsVolumeName = "etcd-client-cert" // #nosec G101
// EtcdClientCredentialsMountPath defines the mount path for the etcd client credentials data
EtcdClientCredentialsMountPath = "/etc/karmada/pki/etcd-client" // #nosec G101
// CaCertDataKey defines the data key for a CA cert
CaCertDataKey = "ca.crt"
// TLSCertDataKey defines the data key for a TLS cert
TLSCertDataKey = "tls.crt"
// TLSPrivateKeyDataKey defines the data key for a TLS cert private key
TLSPrivateKeyDataKey = "tls.key"
// CertificateValidity Certificate validity period
CertificateValidity = time.Hour * 24 * 365
@ -125,9 +135,6 @@ const (
// APIServiceName defines the karmada aggregated apiserver APIService resource name.
APIServiceName = "v1alpha1.cluster.karmada.io"
// KarmadaApiserverEtcdClientCertNameSuffix defines the suffix for the Karmada API server etcd client cert name
KarmadaApiserverEtcdClientCertNameSuffix = "karmada-apiserver-etcd-client-cert"
)
var (

4
operator/pkg/controller/karmada/controller.go
View File

@ -40,8 +40,8 @@ import (
"sigs.k8s.io/controller-runtime/pkg/predicate"
operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
"github.com/karmada-io/karmada/operator/pkg/constants"
operatorscheme "github.com/karmada-io/karmada/operator/pkg/scheme"
"github.com/karmada-io/karmada/operator/pkg/util"
)
const (
@ -112,7 +112,7 @@ func (ctrl *Controller) Reconcile(ctx context.Context, req controllerruntime.Req
// validateKarmada ensures the Karmada resource adheres to validation rules
func (ctrl *Controller) validateKarmada(karmada *operatorv1alpha1.Karmada) error {
if karmada.Spec.Components.Etcd != nil && karmada.Spec.Components.Etcd.External != nil {
expectedSecretName := fmt.Sprintf("%s-%s", karmada.Name, constants.KarmadaApiserverEtcdClientCertNameSuffix)
expectedSecretName := util.EtcdCertSecretName(karmada.Name)
if karmada.Spec.Components.Etcd.External.SecretRef.Name != expectedSecretName {
errorMessage := fmt.Sprintf("Secret name for external etcd client must be %s, but got %s", expectedSecretName, karmada.Spec.Components.Etcd.External.SecretRef.Name)
ctrl.EventRecorder.Event(karmada, corev1.EventTypeWarning, ValidationErrorReason, errorMessage)

59
operator/pkg/controller/karmada/planner_test.go
View File

@ -51,6 +51,13 @@ func TestNewPlannerFor(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Spec: operatorv1alpha1.KarmadaSpec{
Components: &operatorv1alpha1.KarmadaComponents{
Etcd: &operatorv1alpha1.Etcd{
Local: &operatorv1alpha1.LocalEtcd{},
},
},
},
},
client: fake.NewFakeClient(),
config: &rest.Config{},
@ -65,8 +72,16 @@ func TestNewPlannerFor(t *testing.T) {
DeletionTimestamp: &metav1.Time{
Time: time.Now().Add(-5 * time.Minute),
},
Finalizers: []string{ControllerFinalizerName},
},
Spec: operatorv1alpha1.KarmadaSpec{
Components: &operatorv1alpha1.KarmadaComponents{
Etcd: &operatorv1alpha1.Etcd{
Local: &operatorv1alpha1.LocalEtcd{},
},
},
},
},
client: fake.NewFakeClient(),
config: &rest.Config{},
@ -107,6 +122,13 @@ func TestPreRunJob(t *testing.T) {
Name: name,
Namespace: namespace,
},
Spec: operatorv1alpha1.KarmadaSpec{
Components: &operatorv1alpha1.KarmadaComponents{
Etcd: &operatorv1alpha1.Etcd{
Local: &operatorv1alpha1.LocalEtcd{},
},
},
},
},
config: &rest.Config{},
action: InitAction,
@ -124,6 +146,13 @@ func TestPreRunJob(t *testing.T) {
},
Finalizers: []string{ControllerFinalizerName},
},
Spec: operatorv1alpha1.KarmadaSpec{
Components: &operatorv1alpha1.KarmadaComponents{
Etcd: &operatorv1alpha1.Etcd{
Local: &operatorv1alpha1.LocalEtcd{},
},
},
},
},
config: &rest.Config{},
action: DeInitAction,
@ -137,6 +166,13 @@ func TestPreRunJob(t *testing.T) {
Name: name,
Namespace: namespace,
},
Spec: operatorv1alpha1.KarmadaSpec{
Components: &operatorv1alpha1.KarmadaComponents{
Etcd: &operatorv1alpha1.Etcd{
Local: &operatorv1alpha1.LocalEtcd{},
},
},
},
},
config: &rest.Config{},
action: "UnknownAction",
@ -197,7 +233,13 @@ func TestAfterRunJob(t *testing.T) {
Name: name,
Namespace: namespace,
},
Spec: operatorv1alpha1.KarmadaSpec{},
Spec: operatorv1alpha1.KarmadaSpec{
Components: &operatorv1alpha1.KarmadaComponents{
Etcd: &operatorv1alpha1.Etcd{
Local: &operatorv1alpha1.LocalEtcd{},
},
},
},
},
config: &rest.Config{},
action: InitAction,
@ -233,6 +275,13 @@ func TestAfterRunJob(t *testing.T) {
},
Finalizers: []string{ControllerFinalizerName},
},
Spec: operatorv1alpha1.KarmadaSpec{
Components: &operatorv1alpha1.KarmadaComponents{
Etcd: &operatorv1alpha1.Etcd{
Local: &operatorv1alpha1.LocalEtcd{},
},
},
},
},
config: &rest.Config{},
action: DeInitAction,
@ -288,7 +337,13 @@ func TestRunJobErr(t *testing.T) {
Name: name,
Namespace: namespace,
},
Spec: operatorv1alpha1.KarmadaSpec{},
Spec: operatorv1alpha1.KarmadaSpec{
Components: &operatorv1alpha1.KarmadaComponents{
Etcd: &operatorv1alpha1.Etcd{
Local: &operatorv1alpha1.LocalEtcd{},
},
},
},
},
config: &rest.Config{},
jobErr: errors.New("test error"),

69
operator/pkg/controlplane/apiserver/apiserver.go
View File

@ -27,7 +27,7 @@ import (
clientsetscheme "k8s.io/client-go/kubernetes/scheme"
operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
"github.com/karmada-io/karmada/operator/pkg/constants"
"github.com/karmada-io/karmada/operator/pkg/controlplane/etcd"
"github.com/karmada-io/karmada/operator/pkg/util"
"github.com/karmada-io/karmada/operator/pkg/util/apiclient"
"github.com/karmada-io/karmada/operator/pkg/util/patcher"
@ -35,7 +35,7 @@ import (
// EnsureKarmadaAPIServer creates karmada apiserver deployment and service resource
func EnsureKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, name, namespace string, featureGates map[string]bool) error {
if err := installKarmadaAPIServer(client, cfg.KarmadaAPIServer, name, namespace, featureGates); err != nil {
if err := installKarmadaAPIServer(client, cfg.KarmadaAPIServer, cfg.Etcd, name, namespace, featureGates); err != nil {
return fmt.Errorf("failed to install karmada apiserver, err: %w", err)
}
@ -44,29 +44,25 @@ func EnsureKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.Ka
// EnsureKarmadaAggregatedAPIServer creates karmada aggregated apiserver deployment and service resource
func EnsureKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, name, namespace string, featureGates map[string]bool) error {
if err := installKarmadaAggregatedAPIServer(client, cfg.KarmadaAggregatedAPIServer, name, namespace, featureGates); err != nil {
if err := installKarmadaAggregatedAPIServer(client, cfg.KarmadaAggregatedAPIServer, cfg.Etcd, name, namespace, featureGates); err != nil {
return err
}
return createKarmadaAggregatedAPIServerService(client, name, namespace)
}
func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAPIServer, name, namespace string, _ map[string]bool) error {
func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAPIServer, etcdCfg *operatorv1alpha1.Etcd, name, namespace string, _ map[string]bool) error {
apiserverDeploymentBytes, err := util.ParseTemplate(KarmadaApiserverDeployment, struct {
DeploymentName, Namespace, Image, ImagePullPolicy, EtcdClientService string
ServiceSubnet, KarmadaCertsSecret, EtcdCertsSecret string
Replicas *int32
EtcdListenClientPort int32
DeploymentName, Namespace, Image, ImagePullPolicy string
ServiceSubnet, KarmadaCertsSecret string
Replicas *int32
}{
DeploymentName: util.KarmadaAPIServerName(name),
Namespace: namespace,
Image: cfg.Image.Name(),
ImagePullPolicy: string(cfg.ImagePullPolicy),
EtcdClientService: util.KarmadaEtcdClientName(name),
ServiceSubnet: *cfg.ServiceSubnet,
KarmadaCertsSecret: util.KarmadaCertSecretName(name),
EtcdCertsSecret: util.EtcdCertSecretName(name),
Replicas: cfg.Replicas,
EtcdListenClientPort: constants.EtcdListenClientPort,
DeploymentName: util.KarmadaAPIServerName(name),
Namespace: namespace,
Image: cfg.Image.Name(),
ImagePullPolicy: string(cfg.ImagePullPolicy),
ServiceSubnet: *cfg.ServiceSubnet,
KarmadaCertsSecret: util.KarmadaCertSecretName(name),
Replicas: cfg.Replicas,
})
if err != nil {
return fmt.Errorf("error when parsing karmadaApiserver deployment template: %w", err)
@ -76,6 +72,12 @@ func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.K
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), apiserverDeploymentBytes, apiserverDeployment); err != nil {
return fmt.Errorf("error when decoding karmadaApiserver deployment: %w", err)
}
err = etcd.ConfigureClientCredentials(apiserverDeployment, etcdCfg, name, namespace)
if err != nil {
return err
}
patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
WithExtraArgs(cfg.ExtraArgs).WithExtraVolumeMounts(cfg.ExtraVolumeMounts).
WithExtraVolumes(cfg.ExtraVolumes).WithResources(cfg.Resources).ForDeployment(apiserverDeployment)
@ -112,23 +114,19 @@ func createKarmadaAPIServerService(client clientset.Interface, cfg *operatorv1al
return nil
}
func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAggregatedAPIServer, name, namespace string, featureGates map[string]bool) error {
func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAggregatedAPIServer, etcdCfg *operatorv1alpha1.Etcd, name, namespace string, featureGates map[string]bool) error {
aggregatedAPIServerDeploymentBytes, err := util.ParseTemplate(KarmadaAggregatedAPIServerDeployment, struct {
DeploymentName, Namespace, Image, ImagePullPolicy, EtcdClientService string
KubeconfigSecret, KarmadaCertsSecret, EtcdCertsSecret string
Replicas *int32
EtcdListenClientPort int32
DeploymentName, Namespace, Image, ImagePullPolicy string
KubeconfigSecret, KarmadaCertsSecret string
Replicas *int32
}{
DeploymentName: util.KarmadaAggregatedAPIServerName(name),
Namespace: namespace,
Image: cfg.Image.Name(),
ImagePullPolicy: string(cfg.ImagePullPolicy),
EtcdClientService: util.KarmadaEtcdClientName(name),
KubeconfigSecret: util.AdminKubeconfigSecretName(name),
KarmadaCertsSecret: util.KarmadaCertSecretName(name),
EtcdCertsSecret: util.EtcdCertSecretName(name),
Replicas: cfg.Replicas,
EtcdListenClientPort: constants.EtcdListenClientPort,
DeploymentName: util.KarmadaAggregatedAPIServerName(name),
Namespace: namespace,
Image: cfg.Image.Name(),
ImagePullPolicy: string(cfg.ImagePullPolicy),
KubeconfigSecret: util.AdminKubeconfigSecretName(name),
KarmadaCertsSecret: util.KarmadaCertSecretName(name),
Replicas: cfg.Replicas,
})
if err != nil {
return fmt.Errorf("error when parsing karmadaAggregatedAPIServer deployment template: %w", err)
@ -139,6 +137,11 @@ func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operator
return fmt.Errorf("err when decoding karmadaApiserver deployment: %w", err)
}
err = etcd.ConfigureClientCredentials(aggregatedAPIServerDeployment, etcdCfg, name, namespace)
if err != nil {
return err
}
patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).WithResources(cfg.Resources).ForDeployment(aggregatedAPIServerDeployment)

18
operator/pkg/controlplane/apiserver/apiserver_test.go
View File

@ -54,6 +54,9 @@ func TestEnsureKarmadaAPIServer(t *testing.T) {
ServiceSubnet: ptr.To(serviceSubnet),
ExtraArgs: map[string]string{"cmd1": "arg1", "cmd2": "arg2"},
},
Etcd: &operatorv1alpha1.Etcd{
Local: &operatorv1alpha1.LocalEtcd{},
},
}
fakeClient := fakeclientset.NewSimpleClientset()
@ -90,6 +93,9 @@ func TestEnsureKarmadaAggregatedAPIServer(t *testing.T) {
},
ExtraArgs: map[string]string{"cmd1": "arg1", "cmd2": "arg2"},
},
Etcd: &operatorv1alpha1.Etcd{
Local: &operatorv1alpha1.LocalEtcd{},
},
}
featureGates := map[string]bool{"FeatureA": true}
@ -133,11 +139,13 @@ func TestInstallKarmadaAPIServer(t *testing.T) {
ServiceSubnet: ptr.To(serviceSubnet),
ExtraArgs: map[string]string{"cmd1": "arg1", "cmd2": "arg2"},
}
etcdCfg := &operatorv1alpha1.Etcd{
Local: &operatorv1alpha1.LocalEtcd{},
}
featureGates := map[string]bool{"FeatureA": true}
// Call the function under test.
err := installKarmadaAPIServer(fakeClient, cfg, name, namespace, featureGates)
err := installKarmadaAPIServer(fakeClient, cfg, etcdCfg, name, namespace, featureGates)
if err != nil {
t.Fatalf("expected no error, but got: %v", err)
}
@ -228,8 +236,10 @@ func TestInstallKarmadaAggregatedAPIServer(t *testing.T) {
}
featureGates := map[string]bool{"FeatureA": true}
err := installKarmadaAggregatedAPIServer(fakeClient, cfg, name, namespace, featureGates)
etcdCfg := &operatorv1alpha1.Etcd{
Local: &operatorv1alpha1.LocalEtcd{},
}
err := installKarmadaAggregatedAPIServer(fakeClient, cfg, etcdCfg, name, namespace, featureGates)
if err != nil {
t.Fatalf("Failed to install Karmada Aggregated API Server: %v", err)
}

20
operator/pkg/controlplane/apiserver/mainfests.go
View File

@ -50,10 +50,6 @@ spec:
- --disable-admission-plugins=StorageObjectInUseProtection,ServiceAccount
- --enable-admission-plugins=NodeRestriction
- --enable-bootstrap-token-auth=true
- --etcd-cafile=/etc/etcd/pki/etcd-ca.crt
- --etcd-certfile=/etc/etcd/pki/etcd-client.crt
- --etcd-keyfile=/etc/etcd/pki/etcd-client.key
- --etcd-servers=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenClientPort }}
- --bind-address=0.0.0.0
- --secure-port=5443
- --service-account-issuer=https://kubernetes.default.svc.cluster.local
@ -112,17 +108,11 @@ spec:
- mountPath: /etc/karmada/pki
name: apiserver-cert
readOnly: true
- mountPath: /etc/etcd/pki
name: etcd-cert
readOnly: true
priorityClassName: system-node-critical
volumes:
- name: apiserver-cert
secret:
secretName: {{ .KarmadaCertsSecret }}
- name: etcd-cert
secret:
secretName: {{ .EtcdCertsSecret }}
`
// KarmadaApiserverService is karmada apiserver service manifest
@ -176,10 +166,6 @@ spec:
- --kubeconfig=/etc/karmada/kubeconfig
- --authentication-kubeconfig=/etc/karmada/kubeconfig
- --authorization-kubeconfig=/etc/karmada/kubeconfig
- --etcd-cafile=/etc/etcd/pki/etcd-ca.crt
- --etcd-certfile=/etc/etcd/pki/etcd-client.crt
- --etcd-keyfile=/etc/etcd/pki/etcd-client.key
- --etcd-servers=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenClientPort }}
- --tls-cert-file=/etc/karmada/pki/karmada.crt
- --tls-private-key-file=/etc/karmada/pki/karmada.key
- --tls-min-version=VersionTLS13
@ -190,9 +176,6 @@ spec:
- mountPath: /etc/karmada/kubeconfig
name: kubeconfig
subPath: kubeconfig
- mountPath: /etc/etcd/pki
name: etcd-cert
readOnly: true
- mountPath: /etc/karmada/pki
name: apiserver-cert
readOnly: true
@ -203,9 +186,6 @@ spec:
- name: apiserver-cert
secret:
secretName: {{ .KarmadaCertsSecret }}
- name: etcd-cert
secret:
secretName: {{ .EtcdCertsSecret }}
`
// KarmadaAggregatedAPIServerService is karmada aggregated APIServer Service manifest
KarmadaAggregatedAPIServerService = `

89
operator/pkg/controlplane/etcd/util.go Normal file
View File

@ -0,0 +1,89 @@
/*
Copyright 2024 The Karmada Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package etcd
import (
"fmt"
"strconv"
"strings"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
"github.com/karmada-io/karmada/operator/pkg/constants"
"github.com/karmada-io/karmada/operator/pkg/util"
)
// ConfigureClientCredentials configures etcd client credentials for Karmada core and aggregated API servers
func ConfigureClientCredentials(apiServerDeployment *appsv1.Deployment, etcdCfg *operatorv1alpha1.Etcd, name, namespace string) error {
etcdClientServiceName := util.KarmadaEtcdClientName(name)
etcdCertSecretName := util.EtcdCertSecretName(name)
if etcdCfg.External == nil {
etcdClientCredentialsArgs := []string{
fmt.Sprintf("--etcd-cafile=%s/%s.crt", constants.EtcdClientCredentialsMountPath, constants.EtcdCaCertAndKeyName),
fmt.Sprintf("--etcd-certfile=%s/%s.crt", constants.EtcdClientCredentialsMountPath, constants.EtcdClientCertAndKeyName),
fmt.Sprintf("--etcd-keyfile=%s/%s.key", constants.EtcdClientCredentialsMountPath, constants.EtcdClientCertAndKeyName),
fmt.Sprintf("--etcd-servers=https://%s.%s.svc.cluster.local:%s", etcdClientServiceName, namespace, strconv.Itoa(constants.EtcdListenClientPort)),
}
apiServerDeployment.Spec.Template.Spec.Containers[0].Command = append(apiServerDeployment.Spec.Template.Spec.Containers[0].Command, etcdClientCredentialsArgs...)
etcdClientCredentialsVolumeMount := corev1.VolumeMount{
Name: constants.EtcdClientCredentialsVolumeName,
MountPath: constants.EtcdClientCredentialsMountPath,
ReadOnly: true,
}
apiServerDeployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(apiServerDeployment.Spec.Template.Spec.Containers[0].VolumeMounts, etcdClientCredentialsVolumeMount)
etcdClientCredentialsVolume := corev1.Volume{
Name: constants.EtcdClientCredentialsVolumeName,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: etcdCertSecretName,
},
},
}
apiServerDeployment.Spec.Template.Spec.Volumes = append(apiServerDeployment.Spec.Template.Spec.Volumes, etcdClientCredentialsVolume)
} else {
etcdServers := strings.Join(etcdCfg.External.Endpoints, ",")
etcdClientCredentialsArgs := []string{
fmt.Sprintf("--etcd-cafile=%s/%s", constants.EtcdClientCredentialsMountPath, constants.CaCertDataKey),
fmt.Sprintf("--etcd-certfile=%s/%s", constants.EtcdClientCredentialsMountPath, constants.TLSCertDataKey),
fmt.Sprintf("--etcd-keyfile=%s/%s", constants.EtcdClientCredentialsMountPath, constants.TLSPrivateKeyDataKey),
fmt.Sprintf("--etcd-servers=%s", etcdServers),
}
apiServerDeployment.Spec.Template.Spec.Containers[0].Command = append(apiServerDeployment.Spec.Template.Spec.Containers[0].Command, etcdClientCredentialsArgs...)
etcdClientCredentialsVolumeMount := corev1.VolumeMount{
Name: constants.EtcdClientCredentialsVolumeName,
MountPath: constants.EtcdClientCredentialsMountPath,
ReadOnly: true,
}
apiServerDeployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(apiServerDeployment.Spec.Template.Spec.Containers[0].VolumeMounts, etcdClientCredentialsVolumeMount)
etcdClientCredentialsVolume := corev1.Volume{
Name: constants.EtcdClientCredentialsVolumeName,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: etcdCfg.External.SecretRef.Name,
},
},
}
apiServerDeployment.Spec.Template.Spec.Volumes = append(apiServerDeployment.Spec.Template.Spec.Volumes, etcdClientCredentialsVolume)
}
return nil
}

4
operator/pkg/controlplane/search/mainfests.go
View File

@ -57,10 +57,6 @@ spec:
- --kubeconfig=/etc/kubeconfig
- --authentication-kubeconfig=/etc/kubeconfig
- --authorization-kubeconfig=/etc/kubeconfig
- --etcd-servers=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenClientPort }}
- --etcd-cafile=/etc/karmada/pki/etcd-ca.crt
- --etcd-certfile=/etc/karmada/pki/etcd-client.crt
- --etcd-keyfile=/etc/karmada/pki/etcd-client.key
- --tls-cert-file=/etc/karmada/pki/karmada.crt
- --tls-private-key-file=/etc/karmada/pki/karmada.key
- --tls-min-version=VersionTLS13

32
operator/pkg/controlplane/search/search.go
View File

@ -26,37 +26,34 @@ import (
clientsetscheme "k8s.io/client-go/kubernetes/scheme"
operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
"github.com/karmada-io/karmada/operator/pkg/constants"
"github.com/karmada-io/karmada/operator/pkg/controlplane/etcd"
"github.com/karmada-io/karmada/operator/pkg/util"
"github.com/karmada-io/karmada/operator/pkg/util/apiclient"
"github.com/karmada-io/karmada/operator/pkg/util/patcher"
)
// EnsureKarmadaSearch creates karmada search deployment and service resource.
func EnsureKarmadaSearch(client clientset.Interface, cfg *operatorv1alpha1.KarmadaSearch, name, namespace string, featureGates map[string]bool) error {
if err := installKarmadaSearch(client, cfg, name, namespace, featureGates); err != nil {
func EnsureKarmadaSearch(client clientset.Interface, cfg *operatorv1alpha1.KarmadaSearch, etcdCfg *operatorv1alpha1.Etcd, name, namespace string, featureGates map[string]bool) error {
if err := installKarmadaSearch(client, cfg, etcdCfg, name, namespace, featureGates); err != nil {
return err
}
return createKarmadaSearchService(client, name, namespace)
}
func installKarmadaSearch(client clientset.Interface, cfg *operatorv1alpha1.KarmadaSearch, name, namespace string, _ map[string]bool) error {
func installKarmadaSearch(client clientset.Interface, cfg *operatorv1alpha1.KarmadaSearch, etcdCfg *operatorv1alpha1.Etcd, name, namespace string, _ map[string]bool) error {
searchDeploymentSetBytes, err := util.ParseTemplate(KarmadaSearchDeployment, struct {
DeploymentName, Namespace, Image, ImagePullPolicy, KarmadaCertsSecret string
KubeconfigSecret, EtcdClientService string
KubeconfigSecret string
Replicas *int32
EtcdListenClientPort int32
}{
DeploymentName: util.KarmadaSearchName(name),
Namespace: namespace,
Image: cfg.Image.Name(),
ImagePullPolicy: string(cfg.ImagePullPolicy),
KarmadaCertsSecret: util.KarmadaCertSecretName(name),
Replicas: cfg.Replicas,
KubeconfigSecret: util.AdminKubeconfigSecretName(name),
EtcdClientService: util.KarmadaEtcdClientName(name),
EtcdListenClientPort: constants.EtcdListenClientPort,
DeploymentName: util.KarmadaSearchName(name),
Namespace: namespace,
Image: cfg.Image.Name(),
ImagePullPolicy: string(cfg.ImagePullPolicy),
KarmadaCertsSecret: util.KarmadaCertSecretName(name),
Replicas: cfg.Replicas,
KubeconfigSecret: util.AdminKubeconfigSecretName(name),
})
if err != nil {
return fmt.Errorf("error when parsing KarmadaSearch Deployment template: %w", err)
@ -67,6 +64,11 @@ func installKarmadaSearch(client clientset.Interface, cfg *operatorv1alpha1.Karm
return fmt.Errorf("err when decoding KarmadaSearch Deployment: %w", err)
}
err = etcd.ConfigureClientCredentials(searchDeployment, etcdCfg, name, namespace)
if err != nil {
return err
}
patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
WithExtraArgs(cfg.ExtraArgs).WithResources(cfg.Resources).ForDeployment(searchDeployment)

12
operator/pkg/controlplane/search/search_test.go
View File

@ -55,8 +55,10 @@ func TestEnsureKarmadaSearch(t *testing.T) {
// Create fake clientset.
fakeClient := fakeclientset.NewSimpleClientset()
err := EnsureKarmadaSearch(fakeClient, cfg, name, namespace, map[string]bool{})
etcdCfg := &operatorv1alpha1.Etcd{
Local: &operatorv1alpha1.LocalEtcd{},
}
err := EnsureKarmadaSearch(fakeClient, cfg, etcdCfg, name, namespace, map[string]bool{})
if err != nil {
t.Fatalf("failed to ensure karmada search, but got: %v", err)
}
@ -94,8 +96,10 @@ func TestInstallKarmadaSearch(t *testing.T) {
// Create fake clientset.
fakeClient := fakeclientset.NewSimpleClientset()
err := installKarmadaSearch(fakeClient, cfg, name, namespace, map[string]bool{})
etcdCfg := &operatorv1alpha1.Etcd{
Local: &operatorv1alpha1.LocalEtcd{},
}
err := installKarmadaSearch(fakeClient, cfg, etcdCfg, name, namespace, map[string]bool{})
if err != nil {
t.Fatalf("failed to install karmada search: %v", err)
}

6
operator/pkg/deinit.go
View File

@ -36,6 +36,7 @@ type DeInitOptions struct {
Namespace string
Kubeconfig *rest.Config
HostCluster *operatorv1alpha1.HostCluster
Karmada *operatorv1alpha1.Karmada
}
// DeInitOpt defines a type of function to set DeInitOptions values.
@ -56,8 +57,8 @@ type deInitData struct {
func NewDeInitDataJob(opt *DeInitOptions) *workflow.Job {
deInitJob := workflow.NewJob()
deInitJob.AppendTask(tasks.NewRemoveComponentTask())
deInitJob.AppendTask(tasks.NewCleanupCertTask())
deInitJob.AppendTask(tasks.NewRemoveComponentTask(opt.Karmada))
deInitJob.AppendTask(tasks.NewCleanupCertTask(opt.Karmada))
deInitJob.AppendTask(tasks.NewCleanupKubeconfigTask())
deInitJob.SetDataInitializer(func() (workflow.RunData, error) {
@ -126,6 +127,7 @@ func defaultJobDeInitOptions() *DeInitOptions {
// NewDeInitOptWithKarmada returns a DeInitOpt function to initialize DeInitOptions with karmada resource
func NewDeInitOptWithKarmada(karmada *operatorv1alpha1.Karmada) DeInitOpt {
return func(o *DeInitOptions) {
o.Karmada = karmada
o.Name = karmada.GetName()
o.Namespace = karmada.GetNamespace()

8
operator/pkg/init.go
View File

@ -119,7 +119,13 @@ func NewInitJob(opt *InitOptions) *workflow.Job {
initJob.AppendTask(tasks.NewCertTask())
initJob.AppendTask(tasks.NewNamespaceTask())
initJob.AppendTask(tasks.NewUploadCertsTask())
initJob.AppendTask(tasks.NewEtcdTask())
etcdConfig := opt.Karmada.Spec.Components.Etcd
// Only required if local etcd is configured
if etcdConfig.Local != nil {
initJob.AppendTask(tasks.NewEtcdTask())
}
initJob.AppendTask(tasks.NewKarmadaApiserverTask())
initJob.AppendTask(tasks.NewUploadKubeconfigTask())
initJob.AppendTask(tasks.NewKarmadaAggregatedApiserverTask())

18
operator/pkg/tasks/deinit/cert.go
View File

@ -22,6 +22,7 @@ import (
"k8s.io/klog/v2"
"github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
"github.com/karmada-io/karmada/operator/pkg/constants"
"github.com/karmada-io/karmada/operator/pkg/util"
"github.com/karmada-io/karmada/operator/pkg/util/apiclient"
@ -29,16 +30,21 @@ import (
)
// NewCleanupCertTask init a task to cleanup certs
func NewCleanupCertTask() workflow.Task {
func NewCleanupCertTask(karmada *v1alpha1.Karmada) workflow.Task {
workflowTasks := []workflow.Task{
newCleanupCertSubTask("karmada", util.KarmadaCertSecretName),
newCleanupCertSubTask("webhook", util.WebhookCertSecretName),
}
// Required only if local etcd is configured
if karmada.Spec.Components.Etcd.Local != nil {
cleanupEtcdCertTask := newCleanupCertSubTask("etcd", util.EtcdCertSecretName)
workflowTasks = append(workflowTasks, cleanupEtcdCertTask)
}
return workflow.Task{
Name: "cleanup-cert",
Run: runCleanupCert,
RunSubTasks: true,
Tasks: []workflow.Task{
newCleanupCertSubTask("karmada", util.KarmadaCertSecretName),
newCleanupCertSubTask("etcd", util.EtcdCertSecretName),
newCleanupCertSubTask("webhook", util.WebhookCertSecretName),
},
Tasks: workflowTasks,
}
}

14
operator/pkg/tasks/deinit/cert_test.go
View File

@ -27,6 +27,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
fakeclientset "k8s.io/client-go/kubernetes/fake"
"github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
"github.com/karmada-io/karmada/operator/pkg/constants"
"github.com/karmada-io/karmada/operator/pkg/util"
"github.com/karmada-io/karmada/operator/pkg/workflow"
@ -45,15 +46,24 @@ func TestNewCleanupCertTask(t *testing.T) {
RunSubTasks: true,
Tasks: []workflow.Task{
newCleanupCertSubTask("karmada", util.KarmadaCertSecretName),
newCleanupCertSubTask("etcd", util.EtcdCertSecretName),
newCleanupCertSubTask("webhook", util.WebhookCertSecretName),
newCleanupCertSubTask("etcd", util.EtcdCertSecretName),
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cleanupCertTask := NewCleanupCertTask()
karmada := &v1alpha1.Karmada{
Spec: v1alpha1.KarmadaSpec{
Components: &v1alpha1.KarmadaComponents{
Etcd: &v1alpha1.Etcd{
Local: &v1alpha1.LocalEtcd{},
},
},
},
}
cleanupCertTask := NewCleanupCertTask(karmada)
if err := util.DeepEqualTasks(cleanupCertTask, test.wantTask); err != nil {
t.Errorf("unexpected error: %v", err)
}

38
operator/pkg/tasks/deinit/component.go
View File

@ -22,6 +22,7 @@ import (
"k8s.io/klog/v2"
"github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
"github.com/karmada-io/karmada/operator/pkg/constants"
"github.com/karmada-io/karmada/operator/pkg/util"
"github.com/karmada-io/karmada/operator/pkg/util/apiclient"
@ -29,26 +30,31 @@ import (
)
// NewRemoveComponentTask init a remove karmada components task
func NewRemoveComponentTask() workflow.Task {
func NewRemoveComponentTask(karmada *v1alpha1.Karmada) workflow.Task {
workflowTasks := []workflow.Task{
newRemoveComponentWithServiceSubTask(constants.KarmadaMetricsAdapterComponent, util.KarmadaMetricsAdapterName),
newRemoveComponentSubTask(constants.KarmadaDeschedulerComponent, util.KarmadaDeschedulerName),
newRemoveComponentSubTask(constants.KarmadaSchedulerComponent, util.KarmadaSchedulerName),
newRemoveComponentSubTask(constants.KarmadaControllerManagerComponent, util.KarmadaControllerManagerName),
newRemoveComponentSubTask(constants.KubeControllerManagerComponent, util.KubeControllerManagerName),
newRemoveComponentWithServiceSubTask(constants.KarmadaWebhookComponent, util.KarmadaWebhookName),
newRemoveComponentWithServiceSubTask(constants.KarmadaSearchComponent, util.KarmadaSearchName),
newRemoveComponentWithServiceSubTask(constants.KarmadaAggregatedAPIServerComponent, util.KarmadaAggregatedAPIServerName),
newRemoveComponentWithServiceSubTask(constants.KarmadaAPIserverComponent, util.KarmadaAPIServerName),
}
// Required only if local etcd is configured
if karmada.Spec.Components.Etcd.Local != nil {
removeEtcdTask := workflow.Task{
Name: "remove-etcd",
Run: runRemoveEtcd,
}
workflowTasks = append(workflowTasks, removeEtcdTask)
}
return workflow.Task{
Name: "remove-component",
Run: runRemoveComponent,
RunSubTasks: true,
Tasks: []workflow.Task{
newRemoveComponentWithServiceSubTask(constants.KarmadaMetricsAdapterComponent, util.KarmadaMetricsAdapterName),
newRemoveComponentSubTask(constants.KarmadaDeschedulerComponent, util.KarmadaDeschedulerName),
newRemoveComponentSubTask(constants.KarmadaSchedulerComponent, util.KarmadaSchedulerName),
newRemoveComponentSubTask(constants.KarmadaControllerManagerComponent, util.KarmadaControllerManagerName),
newRemoveComponentSubTask(constants.KubeControllerManagerComponent, util.KubeControllerManagerName),
newRemoveComponentWithServiceSubTask(constants.KarmadaWebhookComponent, util.KarmadaWebhookName),
newRemoveComponentWithServiceSubTask(constants.KarmadaSearchComponent, util.KarmadaSearchName),
newRemoveComponentWithServiceSubTask(constants.KarmadaAggregatedAPIServerComponent, util.KarmadaAggregatedAPIServerName),
newRemoveComponentWithServiceSubTask(constants.KarmadaAPIserverComponent, util.KarmadaAPIServerName),
{
Name: "remove-etcd",
Run: runRemoveEtcd,
},
},
Tasks: workflowTasks,
}
}

18
operator/pkg/tasks/init/apiserver_test.go
View File

@ -194,7 +194,11 @@ func TestRunKarmadaAPIServer(t *testing.T) {
{
name: "RunKarmadaAPIServer_NilKarmadaAPIServer_RunIsCompletedWithoutErrors",
runData: &TestInitData{
ComponentsUnits: &operatorv1alpha1.KarmadaComponents{},
ComponentsUnits: &operatorv1alpha1.KarmadaComponents{
Etcd: &operatorv1alpha1.Etcd{
Local: &operatorv1alpha1.LocalEtcd{},
},
},
},
wantErr: false,
},
@ -213,6 +217,9 @@ func TestRunKarmadaAPIServer(t *testing.T) {
},
ServiceSubnet: ptr.To("10.96.0.0/12"),
},
Etcd: &operatorv1alpha1.Etcd{
Local: &operatorv1alpha1.LocalEtcd{},
},
},
RemoteClientConnector: fakeclientset.NewSimpleClientset(),
FeatureGatesOptions: map[string]bool{
@ -329,7 +336,11 @@ func TestRunKarmadaAggregatedAPIServer(t *testing.T) {
{
name: "RunKarmadaAggregatedAPIServer_NilKarmadaAggregatedAPIServer_RunIsCompletedWithoutErrors",
runData: &TestInitData{
ComponentsUnits: &operatorv1alpha1.KarmadaComponents{},
ComponentsUnits: &operatorv1alpha1.KarmadaComponents{
Etcd: &operatorv1alpha1.Etcd{
Local: &operatorv1alpha1.LocalEtcd{},
},
},
},
prep: func() error { return nil },
wantErr: false,
@ -348,6 +359,9 @@ func TestRunKarmadaAggregatedAPIServer(t *testing.T) {
ImagePullPolicy: corev1.PullIfNotPresent,
},
},
Etcd: &operatorv1alpha1.Etcd{
Local: &operatorv1alpha1.LocalEtcd{},
},
},
RemoteClientConnector: fakeclientset.NewSimpleClientset(),
FeatureGatesOptions: map[string]bool{

1
operator/pkg/tasks/init/component.go
View File

@ -264,6 +264,7 @@ func runKarmadaSearch(r workflow.RunData) error {
err := search.EnsureKarmadaSearch(
data.RemoteClient(),
cfg.KarmadaSearch,
cfg.Etcd,
data.GetName(),
data.GetNamespace(),
data.FeatureGates(),