Merge pull request #4316 from zhzhuang-zju/operator

feat: support install karmada search with operator

operator/pkg/apis/operator/v1alpha1/defaults.go

@@ -40,6 +40,7 @@ var (
 	karmadaWebhookImageRepository             = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaWebhook)
 	karmadaDeschedulerImageRepository         = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaDescheduler)
 	KarmadaMetricsAdapterImageRepository      = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaMetricsAdapter)
+	karmadaSearchImageRepository              = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaSearch)
 )
 
 func init() {
@@ -87,7 +88,7 @@ func setDefaultsKarmadaComponents(obj *Karmada) {
 	setDefaultsKarmadaScheduler(obj.Spec.Components)
 	setDefaultsKarmadaWebhook(obj.Spec.Components)
 	setDefaultsKarmadaMetricsAdapter(obj.Spec.Components)
-
+	setDefaultsKarmadaSearch(obj.Spec.Components)
 	// set addon defaults
 	setDefaultsKarmadaDescheduler(obj.Spec.Components)
 }
@@ -243,6 +244,23 @@ func setDefaultsKarmadaWebhook(obj *KarmadaComponents) {
 	}
 }
 
+func setDefaultsKarmadaSearch(obj *KarmadaComponents) {
+	if obj.KarmadaSearch == nil {
+		return
+	}
+
+	search := obj.KarmadaSearch
+	if len(search.Image.ImageRepository) == 0 {
+		search.Image.ImageRepository = karmadaSearchImageRepository
+	}
+	if len(search.Image.ImageTag) == 0 {
+		search.Image.ImageTag = DefaultKarmadaImageVersion
+	}
+	if search.Replicas == nil {
+		search.Replicas = pointer.Int32(1)
+	}
+}
+
 func setDefaultsKarmadaDescheduler(obj *KarmadaComponents) {
 	if obj.KarmadaDescheduler == nil {
 		return

operator/pkg/constants/constants.go

@@ -55,6 +55,8 @@ const (
 	KarmadaScheduler = "karmada-scheduler"
 	// KarmadaWebhook defines the name of the karmada-webhook component
 	KarmadaWebhook = "karmada-webhook"
+	// KarmadaSearch defines the name of the karmada-search component
+	KarmadaSearch = "karmada-search"
 	// KarmadaDescheduler defines the name of the karmada-descheduler component
 	KarmadaDescheduler = "karmada-descheduler"
 	// KarmadaMetricsAdapter defines the name of the karmada-metrics-adapter component
@@ -111,6 +113,8 @@ const (
 	KarmadaSchedulerComponent = "KarmadaScheduler"
 	// KarmadaWebhookComponent defines the name of the karmada-webhook component
 	KarmadaWebhookComponent = "KarmadaWebhook"
+	// KarmadaSearchComponent defines the name of the karmada-search component
+	KarmadaSearchComponent = "KarmadaSearch"
 	// KarmadaDeschedulerComponent defines the name of the karmada-descheduler component
 	KarmadaDeschedulerComponent = "KarmadaDescheduler"
 	// KarmadaMetricsAdapterComponent defines the name of the karmada-metrics-adapter component
@@ -133,4 +137,9 @@ var (
 		{Group: "custom.metrics.k8s.io", Version: "v1beta1"},
 		{Group: "custom.metrics.k8s.io", Version: "v1beta2"},
 	}
+
+	// KarmadaSearchAPIServices defines the GroupVersions of all karmada-search APIServices
+	KarmadaSearchAPIServices = []schema.GroupVersion{
+		{Group: "search.karmada.io", Version: "v1alpha1"},
+	}
 )

operator/pkg/controlplane/search/mainfests.go

@@ -0,0 +1,109 @@
+/*
+Copyright 2023 The Karmada Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package search
+
+const (
+	// KarmadaSearchDeployment is karmada search deployment manifest
+	KarmadaSearchDeployment = `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .DeploymentName }}
+  namespace: {{ .Namespace }}
+  labels:
+    karmada-app: karmada-search
+    apiserver: "true"
+spec:
+  selector:
+    matchLabels:
+      karmada-app: karmada-search
+      apiserver: "true"
+  replicas: {{ .Replicas }}
+  template:
+    metadata:
+      labels:
+        karmada-app: karmada-search
+        apiserver: "true"
+    spec:
+      automountServiceAccountToken: false
+      containers:
+        - name: karmada-search
+          image: {{ .Image }}
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: k8s-certs
+              mountPath: /etc/karmada/pki
+              readOnly: true
+            - name: kubeconfig
+              subPath: kubeconfig
+              mountPath: /etc/kubeconfig
+          command:
+            - /bin/karmada-search
+            - --kubeconfig=/etc/kubeconfig
+            - --authentication-kubeconfig=/etc/kubeconfig
+            - --authorization-kubeconfig=/etc/kubeconfig
+            - --etcd-servers=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenClientPort }}
+            - --etcd-cafile=/etc/karmada/pki/etcd-ca.crt
+            - --etcd-certfile=/etc/karmada/pki/etcd-client.crt
+            - --etcd-keyfile=/etc/karmada/pki/etcd-client.key
+            - --tls-cert-file=/etc/karmada/pki/karmada.crt
+            - --tls-private-key-file=/etc/karmada/pki/karmada.key
+            - --tls-min-version=VersionTLS13
+            - --audit-log-path=-
+            - --feature-gates=APIPriorityAndFairness=false
+            - --audit-log-maxage=0
+            - --audit-log-maxbackup=0
+          livenessProbe:
+            httpGet:
+              path: /livez
+              port: 443
+              scheme: HTTPS
+            failureThreshold: 3
+            initialDelaySeconds: 15
+            periodSeconds: 15
+            timeoutSeconds: 5
+          resources:
+            requests:
+              cpu: 100m
+      volumes:
+        - name: k8s-certs
+          secret:
+            secretName: {{ .KarmadaCertsSecret }}
+        - name: kubeconfig
+          secret:
+            secretName: {{ .KubeconfigSecret }}
+`
+
+	// KarmadaSearchService is karmada-search service manifest
+	KarmadaSearchService = `
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .ServiceName }}
+  namespace: {{ .Namespace }}
+  labels:
+    karmada-app: karmada-search
+    apiserver: "true"
+spec:
+  ports:
+    - port: 443
+      protocol: TCP
+      targetPort: 443
+  selector:
+    karmada-app: karmada-search
+`
+)

operator/pkg/controlplane/search/search.go

@@ -0,0 +1,98 @@
+/*
+Copyright 2023 The Karmada Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package search
+
+import (
+	"fmt"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	kuberuntime "k8s.io/apimachinery/pkg/runtime"
+	clientset "k8s.io/client-go/kubernetes"
+	clientsetscheme "k8s.io/client-go/kubernetes/scheme"
+
+	operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
+	"github.com/karmada-io/karmada/operator/pkg/constants"
+	"github.com/karmada-io/karmada/operator/pkg/util"
+	"github.com/karmada-io/karmada/operator/pkg/util/apiclient"
+	"github.com/karmada-io/karmada/operator/pkg/util/patcher"
+)
+
+// EnsureKarmadaSearch creates karmada search deployment and service resource.
+func EnsureKarmadaSearch(client clientset.Interface, cfg *operatorv1alpha1.KarmadaSearch, name, namespace string, featureGates map[string]bool) error {
+	if err := installKarmadaSearch(client, cfg, name, namespace, featureGates); err != nil {
+		return err
+	}
+
+	return createKarmadaSearchService(client, name, namespace)
+}
+
+func installKarmadaSearch(client clientset.Interface, cfg *operatorv1alpha1.KarmadaSearch, name, namespace string, featureGates map[string]bool) error {
+	searchDeploymentSetBytes, err := util.ParseTemplate(KarmadaSearchDeployment, struct {
+		DeploymentName, Namespace, Image, KarmadaCertsSecret string
+		KubeconfigSecret, EtcdClientService                  string
+		Replicas                                             *int32
+		EtcdListenClientPort                                 int32
+	}{
+		DeploymentName:       util.KarmadaSearchName(name),
+		Namespace:            namespace,
+		Image:                cfg.Image.Name(),
+		KarmadaCertsSecret:   util.KarmadaCertSecretName(name),
+		Replicas:             cfg.Replicas,
+		KubeconfigSecret:     util.AdminKubeconfigSecretName(name),
+		EtcdClientService:    util.KarmadaEtcdClientName(name),
+		EtcdListenClientPort: constants.EtcdListenClientPort,
+	})
+	if err != nil {
+		return fmt.Errorf("error when parsing KarmadaSearch Deployment template: %w", err)
+	}
+
+	searchDeployment := &appsv1.Deployment{}
+	if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), searchDeploymentSetBytes, searchDeployment); err != nil {
+		return fmt.Errorf("err when decoding KarmadaSearch Deployment: %w", err)
+	}
+
+	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
+		WithExtraArgs(cfg.ExtraArgs).WithResources(cfg.Resources).ForDeployment(searchDeployment)
+
+	if err := apiclient.CreateOrUpdateDeployment(client, searchDeployment); err != nil {
+		return fmt.Errorf("error when creating deployment for %s, err: %w", searchDeployment.Name, err)
+	}
+	return nil
+}
+
+func createKarmadaSearchService(client clientset.Interface, name, namespace string) error {
+	searchServiceSetBytes, err := util.ParseTemplate(KarmadaSearchService, struct {
+		ServiceName, Namespace string
+	}{
+		ServiceName: util.KarmadaSearchName(name),
+		Namespace:   namespace,
+	})
+	if err != nil {
+		return fmt.Errorf("error when parsing KarmadaSearch Service template: %w", err)
+	}
+
+	searchService := &corev1.Service{}
+	if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), searchServiceSetBytes, searchService); err != nil {
+		return fmt.Errorf("err when decoding KarmadaSearch Service: %w", err)
+	}
+
+	if err := apiclient.CreateOrUpdateService(client, searchService); err != nil {
+		return fmt.Errorf("err when creating service for %s, err: %w", searchService.Name, err)
+	}
+	return nil
+}

operator/pkg/karmadaresource/apiservice/apiservice.go

@@ -169,3 +169,64 @@ func karmadaMetricsAdapterService(client clientset.Interface, karmadaControlPlan
 	}
 	return nil
 }
+
+// EnsureSearchAPIService creates APIService and a service for karmada-metrics-adapter
+func EnsureSearchAPIService(aggregatorClient *aggregator.Clientset, client clientset.Interface, karmadaControlPlaneServiceName, karmadaControlPlaneNamespace, hostClusterServiceName, hostClusterNamespace, caBundle string) error {
+	if err := karmadaSearchService(client, karmadaControlPlaneServiceName, karmadaControlPlaneNamespace, hostClusterServiceName, hostClusterNamespace); err != nil {
+		return err
+	}
+
+	return karmadaSearchAPIService(aggregatorClient, karmadaControlPlaneServiceName, karmadaControlPlaneNamespace, caBundle)
+}
+
+func karmadaSearchAPIService(client *aggregator.Clientset, karmadaControlPlaneServiceName, karmadaControlPlaneNamespace, caBundle string) error {
+	apiServiceBytes, err := util.ParseTemplate(KarmadaSearchAPIService, struct {
+		ServiceName, Namespace string
+		CABundle               string
+	}{
+		Namespace:   karmadaControlPlaneNamespace,
+		ServiceName: util.KarmadaSearchAPIServerName(karmadaControlPlaneServiceName),
+		CABundle:    caBundle,
+	})
+	if err != nil {
+		return fmt.Errorf("error when parsing KarmadaSearch APIService template: %s", err.Error())
+	}
+
+	apiService := &apiregistrationv1.APIService{}
+	if err = runtime.DecodeInto(codecs.UniversalDecoder(), apiServiceBytes, apiService); err != nil {
+		return fmt.Errorf("err when decoding KarmadaSearch APIService: %s", err.Error())
+	}
+
+	if err = apiclient.CreateOrUpdateAPIService(client, apiService); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func karmadaSearchService(client clientset.Interface, karmadaControlPlaneServiceName, karmadaControlPlaneNamespace, hostClusterServiceName, hostClusterNamespace string) error {
+	searchApiserverServiceBytes, err := util.ParseTemplate(KarmadaSearchService, struct {
+		Namespace              string
+		ServiceName            string
+		HostClusterServiceName string
+		HostClusterNamespace   string
+	}{
+		Namespace:              karmadaControlPlaneNamespace,
+		ServiceName:            util.KarmadaSearchName(karmadaControlPlaneServiceName),
+		HostClusterServiceName: util.KarmadaSearchName(hostClusterServiceName),
+		HostClusterNamespace:   hostClusterNamespace,
+	})
+	if err != nil {
+		return fmt.Errorf("error when parsing KarmadaSearch Service template: %w", err)
+	}
+
+	searchService := &corev1.Service{}
+	if err := runtime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), searchApiserverServiceBytes, searchService); err != nil {
+		return fmt.Errorf("err when decoding KarmadaSearch Service: %w", err)
+	}
+
+	if err := apiclient.CreateOrUpdateService(client, searchService); err != nil {
+		return err
+	}
+	return nil
+}

operator/pkg/karmadaresource/apiservice/manifest.go

@@ -70,6 +70,38 @@ spec:
 	KarmadaMetricsAdapterService = `
 apiVersion: v1
 kind: Service
+metadata:
+  name: {{ .ServiceName }}
+  namespace: {{ .Namespace }}
+spec:
+  type: ExternalName
+  externalName: {{ .HostClusterServiceName }}.{{ .HostClusterNamespace }}.svc
+`
+
+	// KarmadaSearchAPIService is karmada-search APIService manifest
+	KarmadaSearchAPIService = `
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  name: v1alpha1.search.karmada.io
+  labels:
+    app: karmada-search
+    apiserver: "true"
+spec:
+  caBundle: {{ .CABundle }}
+  group: search.karmada.io
+  groupPriorityMinimum: 2000
+  service:
+    name: {{ .ServiceName }}
+    namespace: {{ .Namespace }}
+  version: v1alpha1
+  versionPriority: 10
+`
+
+	// KarmadaSearchService is karmada-search service manifest
+	KarmadaSearchService = `
+apiVersion: v1
+kind: Service
 metadata:
   name: {{ .ServiceName }}
   namespace: {{ .Namespace }}

operator/pkg/tasks/deinit/component.go

@@ -41,6 +41,7 @@ func NewRemoveComponentTask() workflow.Task {
 			newRemoveComponentSubTask(constants.KarmadaControllerManagerComponent, util.KarmadaControllerManagerName),
 			newRemoveComponentSubTask(constants.KubeControllerManagerComponent, util.KubeControllerManagerName),
 			newRemoveComponentWithServiceSubTask(constants.KarmadaWebhookComponent, util.KarmadaWebhookName),
+			newRemoveComponentWithServiceSubTask(constants.KarmadaSearchComponent, util.KarmadaSearchName),
 			newRemoveComponentWithServiceSubTask(constants.KarmadaAggregatedAPIServerComponent, util.KarmadaAggregatedAPIServerName),
 			newRemoveComponentWithServiceSubTask(constants.KarmadaAPIserverComponent, util.KarmadaAPIServerName),
 			{

operator/pkg/tasks/init/component.go

@@ -27,6 +27,7 @@ import (
 	"github.com/karmada-io/karmada/operator/pkg/constants"
 	"github.com/karmada-io/karmada/operator/pkg/controlplane"
 	"github.com/karmada-io/karmada/operator/pkg/controlplane/metricsadapter"
+	"github.com/karmada-io/karmada/operator/pkg/controlplane/search"
 	"github.com/karmada-io/karmada/operator/pkg/controlplane/webhook"
 	"github.com/karmada-io/karmada/operator/pkg/karmadaresource/apiservice"
 	"github.com/karmada-io/karmada/operator/pkg/util/apiclient"
@@ -44,11 +45,12 @@ func NewComponentTask() workflow.Task {
 			newComponentSubTask(constants.KarmadaControllerManagerComponent),
 			newComponentSubTask(constants.KarmadaSchedulerComponent),
 			{
-				Name: "KarmadaWebhook",
+				Name: constants.KarmadaWebhookComponent,
 				Run:  runKarmadaWebhook,
 			},
 			newComponentSubTask(constants.KarmadaDeschedulerComponent),
 			newKarmadaMetricsAdapterSubTask(),
+			newKarmadaSearchSubTask(),
 		},
 	}
 }
@@ -228,3 +230,93 @@ func runDeployMetricAdapterAPIService(r workflow.RunData) error {
 
 	return nil
 }
+
+func newKarmadaSearchSubTask() workflow.Task {
+	return workflow.Task{
+		Name:        constants.KarmadaSearchComponent,
+		Run:         runKarmadaResources,
+		RunSubTasks: true,
+		Tasks: []workflow.Task{
+			{
+				Name: "DeployKarmadaSearch",
+				Run:  runKarmadaSearch,
+			},
+			{
+				Name: "DeployKarmadaSearchAPIService",
+				Run:  runDeployKarmadaSearchAPIService,
+			},
+		},
+	}
+}
+
+func runKarmadaSearch(r workflow.RunData) error {
+	data, ok := r.(InitData)
+	if !ok {
+		return errors.New("KarmadaSearch task invoked with an invalid data struct")
+	}
+
+	cfg := data.Components()
+	if cfg.KarmadaSearch == nil {
+		klog.Infof("Skip installing component (%s/%s)", data.GetNamespace(), constants.KarmadaSearchComponent)
+		return nil
+	}
+
+	err := search.EnsureKarmadaSearch(
+		data.RemoteClient(),
+		cfg.KarmadaSearch,
+		data.GetName(),
+		data.GetNamespace(),
+		data.FeatureGates(),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to apply karmada search, err: %w", err)
+	}
+
+	klog.V(2).InfoS("[KarmadaSearch] Successfully applied karmada search component", "karmada", klog.KObj(data))
+	return nil
+}
+
+func runDeployKarmadaSearchAPIService(r workflow.RunData) error {
+	data, ok := r.(InitData)
+	if !ok {
+		return errors.New("DeploySearchAPIService task invoked with an invalid data struct")
+	}
+
+	cfg := data.Components()
+	if cfg.KarmadaSearch == nil {
+		klog.V(2).InfoS("[karmadaSearch] Skip install karmada-search APIService")
+		return nil
+	}
+
+	config := data.ControlplaneConfig()
+	client, err := apiclient.NewAPIRegistrationClient(config)
+	if err != nil {
+		return err
+	}
+
+	cert := data.GetCert(constants.CaCertAndKeyName)
+	if len(cert.CertData()) == 0 {
+		return errors.New("unexpected empty ca cert data for search")
+	}
+	caBase64 := base64.StdEncoding.EncodeToString(cert.CertData())
+
+	err = apiservice.EnsureSearchAPIService(client, data.KarmadaClient(), data.GetName(), constants.KarmadaSystemNamespace, data.GetName(), data.GetNamespace(), caBase64)
+	if err != nil {
+		return fmt.Errorf("failed to apply karmada-metrics-adapter APIService resource to karmada controlplane, err: %w", err)
+	}
+
+	if *cfg.KarmadaSearch.Replicas != 0 {
+		waiter := apiclient.NewKarmadaWaiter(config, nil, time.Second*20)
+		for _, gv := range constants.KarmadaSearchAPIServices {
+			apiServiceName := fmt.Sprintf("%s.%s", gv.Version, gv.Group)
+
+			if err := waiter.WaitForAPIService(apiServiceName); err != nil {
+				return fmt.Errorf("the APIService %s is unhealthy, err: %w", apiServiceName, err)
+			}
+		}
+
+		klog.V(2).InfoS("[DeploySearchAPIService] all karmada-search APIServices status is ready ", "karmada", klog.KObj(data))
+	}
+
+	return nil
+}

operator/pkg/util/naming.go

@@ -54,6 +54,11 @@ func KarmadaAggregatedAPIServerName(karmada string) string {
 	return generateResourceName(karmada, "aggregated-apiserver")
 }
 
+// KarmadaSearchAPIServerName returns secret name of karmada-search
+func KarmadaSearchAPIServerName(karmada string) string {
+	return generateResourceName(karmada, "search")
+}
+
 // KarmadaEtcdName returns name of karmada-etcd
 func KarmadaEtcdName(karmada string) string {
 	return generateResourceName(karmada, "etcd")
@@ -94,6 +99,11 @@ func KarmadaMetricsAdapterName(karmada string) string {
 	return generateResourceName(karmada, "metrics-adapter")
 }
 
+// KarmadaSearchName returns name of karmada-search
+func KarmadaSearchName(karmada string) string {
+	return generateResourceName(karmada, "search")
+}
+
 func generateResourceName(karmada, suffix string) string {
 	if strings.Contains(karmada, "karmada") {
 		return fmt.Sprintf("%s-%s", karmada, suffix)