Merge pull request #3510 from calvin0327/support-karmada-descheduler

karmada operator: implement installing karmada descheduler

operator/pkg/apis/operator/v1alpha1/defaults.go

@@ -18,6 +18,7 @@ var (
 	karmadaControllerManagerImageRepository   = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaControllerManager)
 	karmadaSchedulerImageRepository           = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaScheduler)
 	karmadaWebhookImageRepository             = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaWebhook)
+	karmadaDeschedulerImageRepository         = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaDescheduler)
 )
 
 func addDefaultingFuncs(scheme *runtime.Scheme) error {
@@ -54,6 +55,9 @@ func setDefaultsKarmadaComponents(obj *Karmada) {
 	setDefaultsKarmadaControllerManager(obj.Spec.Components)
 	setDefaultsKarmadaScheduler(obj.Spec.Components)
 	setDefaultsKarmadaWebhook(obj.Spec.Components)
+
+	// set addon defaults
+	setDefaultsKarmadaDescheduler(obj.Spec.Components)
 }
 
 func setDefaultsHostCluster(obj *Karmada) {
@@ -206,3 +210,20 @@ func setDefaultsKarmadaWebhook(obj *KarmadaComponents) {
 		webhook.Replicas = pointer.Int32(1)
 	}
 }
+
+func setDefaultsKarmadaDescheduler(obj *KarmadaComponents) {
+	if obj.KarmadaDescheduler == nil {
+		return
+	}
+
+	descheduler := obj.KarmadaDescheduler
+	if len(descheduler.Image.ImageRepository) == 0 {
+		descheduler.Image.ImageRepository = karmadaDeschedulerImageRepository
+	}
+	if len(descheduler.Image.ImageTag) == 0 {
+		descheduler.Image.ImageTag = constants.KarmadaDefaultVersion
+	}
+	if descheduler.Replicas == nil {
+		descheduler.Replicas = pointer.Int32(1)
+	}
+}

operator/pkg/constants/constants.go

@@ -40,6 +40,8 @@ const (
 	KarmadaScheduler = "karmada-scheduler"
 	// KarmadaWebhook defines the name of the karmada-webhook component
 	KarmadaWebhook = "karmada-webhook"
+	// KarmadaDescheduler defines the name of the karmada-descheduler component
+	KarmadaDescheduler = "karmada-descheduler"
 
 	// KarmadaSystemNamespace defines the leader selection namespace for karmada components
 	KarmadaSystemNamespace = "karmada-system"
@@ -78,18 +80,20 @@ const (
 	// UserName karmada cluster user name
 	UserName = "karmada-admin"
 
-	// KarmadaAPIserverComponent defines the name of karmada apiserver component
+	// KarmadaAPIserverComponent defines the name of karmada-apiserver component
 	KarmadaAPIserverComponent = "KarmadaAPIServer"
-	// KarmadaAggregatedAPIServerComponent defines the name of karmada aggregated apiserver component
+	// KarmadaAggregatedAPIServerComponent defines the name of karmada-aggregated-apiserver component
 	KarmadaAggregatedAPIServerComponent = "KarmadaAggregatedAPIServer"
-	// KubeControllerManagerComponent defines the name of kube controller manager component
+	// KubeControllerManagerComponent defines the name of kube-controller-manager-component
 	KubeControllerManagerComponent = "KubeControllerManager"
-	// KarmadaControllerManagerComponent defines the name of karmada controller manager component
+	// KarmadaControllerManagerComponent defines the name of karmada-controller-manager component
 	KarmadaControllerManagerComponent = "KarmadaControllerManager"
-	// KarmadaSchedulerComponent defines the name of karmada scheduler component
+	// KarmadaSchedulerComponent defines the name of karmada-scheduler component
 	KarmadaSchedulerComponent = "KarmadaScheduler"
 	// KarmadaWebhookComponent defines the name of the karmada-webhook component
 	KarmadaWebhookComponent = "KarmadaWebhook"
+	// KarmadaDeschedulerComponent defines the name of the karmada-descheduler component
+	KarmadaDeschedulerComponent = "KarmadaDescheduler"
 
 	// KarmadaOperatorLabelKeyName defines a label key used by all of resources created by karmada operator
 	KarmadaOperatorLabelKeyName = "app.kubernetes.io/managed-by"

operator/pkg/controlplane/controlplane.go

@@ -7,6 +7,7 @@ import (
 	kuberuntime "k8s.io/apimachinery/pkg/runtime"
 	clientset "k8s.io/client-go/kubernetes"
 	clientsetscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/klog/v2"
 
 	operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
 	"github.com/karmada-io/karmada/operator/pkg/constants"
@@ -24,8 +25,10 @@ func EnsureControlPlaneComponent(component, name, namespace string, client clien
 
 	deployment, ok := deployments[component]
 	if !ok {
-		return fmt.Errorf("no exist manifest for %s", component)
+		klog.Infof("Skip installing component %s(%s/%s)", component, namespace, name)
+		return nil
 	}
+
 	if err := apiclient.CreateOrUpdateDeployment(client, deployment); err != nil {
 		return fmt.Errorf("failed to create deployment resource for component %s, err: %w", component, err)
 	}
@@ -41,16 +44,26 @@ func getComponentManifests(name, namespace string, cfg *operatorv1alpha1.Karmada
 	if err != nil {
 		return nil, err
 	}
-	scheduler, err := getKarmadaSchedulerManifest(name, namespace, cfg.KarmadaScheduler)
+	karmadaScheduler, err := getKarmadaSchedulerManifest(name, namespace, cfg.KarmadaScheduler)
 	if err != nil {
 		return nil, err
 	}
 
-	return map[string]*appsv1.Deployment{
-		constants.KubeControllerManagerComponent:    kubeControllerManager,
+	manifest := map[string]*appsv1.Deployment{
 		constants.KarmadaControllerManagerComponent: karmadaControllerManager,
-		constants.KarmadaSchedulerComponent:         scheduler,
-	}, nil
+		constants.KubeControllerManagerComponent:    kubeControllerManager,
+		constants.KarmadaSchedulerComponent:         karmadaScheduler,
+	}
+
+	if cfg.KarmadaDescheduler != nil {
+		descheduler, err := getKarmadaDeschedulerManifest(name, namespace, cfg.KarmadaDescheduler)
+		if err != nil {
+			return nil, err
+		}
+		manifest[constants.KarmadaDeschedulerComponent] = descheduler
+	}
+
+	return manifest, nil
 }
 
 func getKubeControllerManagerManifest(name, namespace string, cfg *operatorv1alpha1.KubeControllerManager) (*appsv1.Deployment, error) {
@@ -67,12 +80,12 @@ func getKubeControllerManagerManifest(name, namespace string, cfg *operatorv1alp
 		Replicas:           cfg.Replicas,
 	})
 	if err != nil {
-		return nil, fmt.Errorf("error when parsing KubeControllerManager Deployment template: %w", err)
+		return nil, fmt.Errorf("error when parsing kube-controller-manager deployment template: %w", err)
 	}
 
 	kcm := &appsv1.Deployment{}
 	if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), kubeControllerManageretBytes, kcm); err != nil {
-		return nil, fmt.Errorf("err when decoding KubeControllerManager Deployment: %w", err)
+		return nil, fmt.Errorf("err when decoding kube-controller-manager deployment: %w", err)
 	}
 
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(kcm)
@@ -92,12 +105,12 @@ func getKarmadaControllerManagerManifest(name, namespace string, cfg *operatorv1
 		Replicas:         cfg.Replicas,
 	})
 	if err != nil {
-		return nil, fmt.Errorf("error when parsing KarmadaControllerManager Deployment template: %w", err)
+		return nil, fmt.Errorf("error when parsing karmada-controller-manager deployment template: %w", err)
 	}
 
 	kcm := &appsv1.Deployment{}
 	if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), karmadaControllerManageretBytes, kcm); err != nil {
-		return nil, fmt.Errorf("err when decoding KarmadaControllerManager Deployment: %w", err)
+		return nil, fmt.Errorf("err when decoding karmada-controller-manager deployment: %w", err)
 	}
 
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(kcm)
@@ -117,14 +130,39 @@ func getKarmadaSchedulerManifest(name, namespace string, cfg *operatorv1alpha1.K
 		Replicas:         cfg.Replicas,
 	})
 	if err != nil {
-		return nil, fmt.Errorf("error when parsing KarmadaScheduler Deployment template: %w", err)
+		return nil, fmt.Errorf("error when parsing karmada-scheduler deployment template: %w", err)
 	}
 
 	scheduler := &appsv1.Deployment{}
 	if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), karmadaSchedulerBytes, scheduler); err != nil {
-		return nil, fmt.Errorf("err when decoding KarmadaScheduler Deployment: %w", err)
+		return nil, fmt.Errorf("err when decoding karmada-scheduler deployment: %w", err)
 	}
 
 	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(scheduler)
 	return scheduler, nil
 }
+
+func getKarmadaDeschedulerManifest(name, namespace string, cfg *operatorv1alpha1.KarmadaDescheduler) (*appsv1.Deployment, error) {
+	karmadaDeschedulerBytes, err := util.ParseTemplate(KarmadaDeschedulerDeployment, struct {
+		Replicas                  *int32
+		DeploymentName, Namespace string
+		Image, KubeconfigSecret   string
+	}{
+		DeploymentName:   util.KarmadaDeschedulerName(name),
+		Namespace:        namespace,
+		Image:            cfg.Image.Name(),
+		KubeconfigSecret: util.AdminKubeconfigSercretName(name),
+		Replicas:         cfg.Replicas,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error when parsing karmada-descheduler deployment template: %w", err)
+	}
+
+	descheduler := &appsv1.Deployment{}
+	if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), karmadaDeschedulerBytes, descheduler); err != nil {
+		return nil, fmt.Errorf("err when decoding karmada-descheduler deployment: %w", err)
+	}
+
+	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(descheduler)
+	return descheduler, nil
+}

operator/pkg/controlplane/manifests.go

@@ -194,4 +194,56 @@ spec:
           secret:
             secretName: {{ .KubeconfigSecret }}
 `
+
+	// KarmadaDeschedulerDeployment is KarmadaDescheduler Deployment manifest
+	KarmadaDeschedulerDeployment = `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .DeploymentName }}
+  namespace: {{ .Namespace }}
+  labels:
+    karmada-app: karmada-descheduler
+    app.kubernetes.io/managed-by: karmada-operator
+spec:
+  replicas: {{ .Replicas }}
+  selector:
+    matchLabels:
+      karmada-app: karmada-descheduler
+  template:
+    metadata:
+      labels:
+        karmada-app: karmada-descheduler
+    spec:
+      automountServiceAccountToken: false
+      tolerations:
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+      containers:
+      - name: karmada-descheduler
+        image: {{ .Image }}
+        imagePullPolicy: IfNotPresent
+        command:
+        - /bin/karmada-descheduler
+        - --kubeconfig=/etc/karmada/config
+        - --bind-address=0.0.0.0
+        - --v=4
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 10358
+            scheme: HTTP
+          failureThreshold: 3
+          initialDelaySeconds: 15
+          periodSeconds: 15
+          timeoutSeconds: 5
+        volumeMounts:
+        - name: kubeconfig
+          subPath: config
+          mountPath: /etc/karmada/config
+      volumes:
+        - name: kubeconfig
+          secret:
+            secretName: {{ .KubeconfigSecret }}
+`
 )

operator/pkg/tasks/deinit/component.go

@@ -19,6 +19,7 @@ func NewRemoveComponentTask() workflow.Task {
 		Run:         runRemoveComponent,
 		RunSubTasks: true,
 		Tasks: []workflow.Task{
+			newRemoveComponentSubTask(constants.KarmadaDeschedulerComponent, util.KarmadaDeschedulerName),
 			newRemoveComponentSubTask(constants.KarmadaSchedulerComponent, util.KarmadaSchedulerName),
 			newRemoveComponentSubTask(constants.KarmadaControllerManagerComponent, util.KarmadaControllerManagerName),
 			newRemoveComponentSubTask(constants.KubeControllerManagerComponent, util.KubeControllerManagerName),

operator/pkg/tasks/init/component.go

@@ -26,6 +26,7 @@ func NewComponentTask() workflow.Task {
 				Name: "KarmadaWebhook",
 				Run:  runKarmadaWebhook,
 			},
+			newComponentSubTask(constants.KarmadaDeschedulerComponent),
 		},
 	}
 }

operator/pkg/util/naming.go

@@ -8,66 +8,71 @@ import (
 // Namefunc defines a function to generate resource name according to karmada resource name.
 type Namefunc func(karmada string) string
 
-// AdminKubeconfigSercretName return a secret name of karmada admin kubeconfig
+// AdminKubeconfigSercretName returns secret name of karmada-admin kubeconfig
 func AdminKubeconfigSercretName(karmada string) string {
 	return generateResourceName(karmada, "admin-config")
 }
 
-// KarmadaCertSecretName return a secret name of karmada certs
+// KarmadaCertSecretName returns secret name of karmada certs
 func KarmadaCertSecretName(karmada string) string {
 	return generateResourceName(karmada, "cert")
 }
 
-// EtcdCertSecretName return a secret name of etcd cert
+// EtcdCertSecretName returns secret name of etcd cert
 func EtcdCertSecretName(karmada string) string {
 	return generateResourceName(karmada, "etcd-cert")
 }
 
-// WebhookCertSecretName return secret name of karmada webhook cert
+// WebhookCertSecretName returns secret name of karmada-webhook cert
 func WebhookCertSecretName(karmada string) string {
 	return generateResourceName(karmada, "webhook-cert")
 }
 
-// KarmadaAPIServerName return secret name of karmada apiserver
+// KarmadaAPIServerName returns secret name of karmada-apiserver
 func KarmadaAPIServerName(karmada string) string {
 	return generateResourceName(karmada, "apiserver")
 }
 
-// KarmadaAggregatedAPIServerName return secret name of karmada aggregated apiserver
+// KarmadaAggregatedAPIServerName returns secret name of karmada-aggregated-apiserver
 func KarmadaAggregatedAPIServerName(karmada string) string {
 	return generateResourceName(karmada, "aggregated-apiserver")
 }
 
-// KarmadaEtcdName return karmada etcd name
+// KarmadaEtcdName returns name of karmada-etcd
 func KarmadaEtcdName(karmada string) string {
 	return generateResourceName(karmada, "etcd")
 }
 
-// KarmadaEtcdClientName return karmada etcd client name
+// KarmadaEtcdClientName returns name of karmada-etcd client
 func KarmadaEtcdClientName(karmada string) string {
 	return generateResourceName(karmada, "etcd-client")
 }
 
-// KubeControllerManagerName return name of kube controller manager name of karmada
+// KubeControllerManagerName returns name of kube-controller-manager
 func KubeControllerManagerName(karmada string) string {
 	return generateResourceName(karmada, "kube-controller-manager")
 }
 
-// KarmadaControllerManagerName return karmada controller manager name
+// KarmadaControllerManagerName returns name of karmada-controller-manager
 func KarmadaControllerManagerName(karmada string) string {
 	return generateResourceName(karmada, "controller-manager")
 }
 
-// KarmadaSchedulerName return karmada scheduler name
+// KarmadaSchedulerName returns name of karmada-scheduler
 func KarmadaSchedulerName(karmada string) string {
 	return generateResourceName(karmada, "scheduler")
 }
 
-// KarmadaWebhookName return karmada webhook name
+// KarmadaWebhookName returns name of karmada-webhook
 func KarmadaWebhookName(karmada string) string {
 	return generateResourceName(karmada, "webhook")
 }
 
+// KarmadaDeschedulerName returns name of karmada-descheduler
+func KarmadaDeschedulerName(karmada string) string {
+	return generateResourceName(karmada, "descheduler")
+}
+
 func generateResourceName(karmada, suffix string) string {
 	if strings.Contains(karmada, "karmada") {
 		return fmt.Sprintf("%s-%s", karmada, suffix)