diff --git a/operator/pkg/apis/operator/v1alpha1/defaults.go b/operator/pkg/apis/operator/v1alpha1/defaults.go index a25af0091..ac5005a97 100644 --- a/operator/pkg/apis/operator/v1alpha1/defaults.go +++ b/operator/pkg/apis/operator/v1alpha1/defaults.go @@ -40,6 +40,7 @@ var ( karmadaWebhookImageRepository = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaWebhook) karmadaDeschedulerImageRepository = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaDescheduler) KarmadaMetricsAdapterImageRepository = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaMetricsAdapter) + karmadaSearchImageRepository = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaSearch) ) func init() { @@ -87,7 +88,7 @@ func setDefaultsKarmadaComponents(obj *Karmada) { setDefaultsKarmadaScheduler(obj.Spec.Components) setDefaultsKarmadaWebhook(obj.Spec.Components) setDefaultsKarmadaMetricsAdapter(obj.Spec.Components) - + setDefaultsKarmadaSearch(obj.Spec.Components) // set addon defaults setDefaultsKarmadaDescheduler(obj.Spec.Components) } @@ -243,6 +244,23 @@ func setDefaultsKarmadaWebhook(obj *KarmadaComponents) { } } +func setDefaultsKarmadaSearch(obj *KarmadaComponents) { + if obj.KarmadaSearch == nil { + return + } + + search := obj.KarmadaSearch + if len(search.Image.ImageRepository) == 0 { + search.Image.ImageRepository = karmadaSearchImageRepository + } + if len(search.Image.ImageTag) == 0 { + search.Image.ImageTag = DefaultKarmadaImageVersion + } + if search.Replicas == nil { + search.Replicas = pointer.Int32(1) + } +} + func setDefaultsKarmadaDescheduler(obj *KarmadaComponents) { if obj.KarmadaDescheduler == nil { return diff --git a/operator/pkg/constants/constants.go b/operator/pkg/constants/constants.go index 008838d20..f2e80c586 100644 --- a/operator/pkg/constants/constants.go +++ b/operator/pkg/constants/constants.go @@ -55,6 +55,8 @@ const ( KarmadaScheduler = "karmada-scheduler" // KarmadaWebhook defines the name of the karmada-webhook component KarmadaWebhook = "karmada-webhook" + // KarmadaSearch defines the name of the karmada-search component + KarmadaSearch = "karmada-search" // KarmadaDescheduler defines the name of the karmada-descheduler component KarmadaDescheduler = "karmada-descheduler" // KarmadaMetricsAdapter defines the name of the karmada-metrics-adapter component @@ -111,6 +113,8 @@ const ( KarmadaSchedulerComponent = "KarmadaScheduler" // KarmadaWebhookComponent defines the name of the karmada-webhook component KarmadaWebhookComponent = "KarmadaWebhook" + // KarmadaSearchComponent defines the name of the karmada-search component + KarmadaSearchComponent = "KarmadaSearch" // KarmadaDeschedulerComponent defines the name of the karmada-descheduler component KarmadaDeschedulerComponent = "KarmadaDescheduler" // KarmadaMetricsAdapterComponent defines the name of the karmada-metrics-adapter component @@ -133,4 +137,9 @@ var ( {Group: "custom.metrics.k8s.io", Version: "v1beta1"}, {Group: "custom.metrics.k8s.io", Version: "v1beta2"}, } + + // KarmadaSearchAPIServices defines the GroupVersions of all karmada-search APIServices + KarmadaSearchAPIServices = []schema.GroupVersion{ + {Group: "search.karmada.io", Version: "v1alpha1"}, + } ) diff --git a/operator/pkg/controlplane/search/mainfests.go b/operator/pkg/controlplane/search/mainfests.go new file mode 100644 index 000000000..b3584fb5d --- /dev/null +++ b/operator/pkg/controlplane/search/mainfests.go @@ -0,0 +1,109 @@ +/* +Copyright 2023 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package search + +const ( + // KarmadaSearchDeployment is karmada search deployment manifest + KarmadaSearchDeployment = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .DeploymentName }} + namespace: {{ .Namespace }} + labels: + karmada-app: karmada-search + apiserver: "true" +spec: + selector: + matchLabels: + karmada-app: karmada-search + apiserver: "true" + replicas: {{ .Replicas }} + template: + metadata: + labels: + karmada-app: karmada-search + apiserver: "true" + spec: + automountServiceAccountToken: false + containers: + - name: karmada-search + image: {{ .Image }} + imagePullPolicy: IfNotPresent + volumeMounts: + - name: k8s-certs + mountPath: /etc/karmada/pki + readOnly: true + - name: kubeconfig + subPath: kubeconfig + mountPath: /etc/kubeconfig + command: + - /bin/karmada-search + - --kubeconfig=/etc/kubeconfig + - --authentication-kubeconfig=/etc/kubeconfig + - --authorization-kubeconfig=/etc/kubeconfig + - --etcd-servers=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenClientPort }} + - --etcd-cafile=/etc/karmada/pki/etcd-ca.crt + - --etcd-certfile=/etc/karmada/pki/etcd-client.crt + - --etcd-keyfile=/etc/karmada/pki/etcd-client.key + - --tls-cert-file=/etc/karmada/pki/karmada.crt + - --tls-private-key-file=/etc/karmada/pki/karmada.key + - --tls-min-version=VersionTLS13 + - --audit-log-path=- + - --feature-gates=APIPriorityAndFairness=false + - --audit-log-maxage=0 + - --audit-log-maxbackup=0 + livenessProbe: + httpGet: + path: /livez + port: 443 + scheme: HTTPS + failureThreshold: 3 + initialDelaySeconds: 15 + periodSeconds: 15 + timeoutSeconds: 5 + resources: + requests: + cpu: 100m + volumes: + - name: k8s-certs + secret: + secretName: {{ .KarmadaCertsSecret }} + - name: kubeconfig + secret: + secretName: {{ .KubeconfigSecret }} +` + + // KarmadaSearchService is karmada-search service manifest + KarmadaSearchService = ` +apiVersion: v1 +kind: Service +metadata: + name: {{ .ServiceName }} + namespace: {{ .Namespace }} + labels: + karmada-app: karmada-search + apiserver: "true" +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 443 + selector: + karmada-app: karmada-search +` +) diff --git a/operator/pkg/controlplane/search/search.go b/operator/pkg/controlplane/search/search.go new file mode 100644 index 000000000..4c3115a87 --- /dev/null +++ b/operator/pkg/controlplane/search/search.go @@ -0,0 +1,98 @@ +/* +Copyright 2023 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package search + +import ( + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + kuberuntime "k8s.io/apimachinery/pkg/runtime" + clientset "k8s.io/client-go/kubernetes" + clientsetscheme "k8s.io/client-go/kubernetes/scheme" + + operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1" + "github.com/karmada-io/karmada/operator/pkg/constants" + "github.com/karmada-io/karmada/operator/pkg/util" + "github.com/karmada-io/karmada/operator/pkg/util/apiclient" + "github.com/karmada-io/karmada/operator/pkg/util/patcher" +) + +// EnsureKarmadaSearch creates karmada search deployment and service resource. +func EnsureKarmadaSearch(client clientset.Interface, cfg *operatorv1alpha1.KarmadaSearch, name, namespace string, featureGates map[string]bool) error { + if err := installKarmadaSearch(client, cfg, name, namespace, featureGates); err != nil { + return err + } + + return createKarmadaSearchService(client, name, namespace) +} + +func installKarmadaSearch(client clientset.Interface, cfg *operatorv1alpha1.KarmadaSearch, name, namespace string, featureGates map[string]bool) error { + searchDeploymentSetBytes, err := util.ParseTemplate(KarmadaSearchDeployment, struct { + DeploymentName, Namespace, Image, KarmadaCertsSecret string + KubeconfigSecret, EtcdClientService string + Replicas *int32 + EtcdListenClientPort int32 + }{ + DeploymentName: util.KarmadaSearchName(name), + Namespace: namespace, + Image: cfg.Image.Name(), + KarmadaCertsSecret: util.KarmadaCertSecretName(name), + Replicas: cfg.Replicas, + KubeconfigSecret: util.AdminKubeconfigSecretName(name), + EtcdClientService: util.KarmadaEtcdClientName(name), + EtcdListenClientPort: constants.EtcdListenClientPort, + }) + if err != nil { + return fmt.Errorf("error when parsing KarmadaSearch Deployment template: %w", err) + } + + searchDeployment := &appsv1.Deployment{} + if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), searchDeploymentSetBytes, searchDeployment); err != nil { + return fmt.Errorf("err when decoding KarmadaSearch Deployment: %w", err) + } + + patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels). + WithExtraArgs(cfg.ExtraArgs).WithResources(cfg.Resources).ForDeployment(searchDeployment) + + if err := apiclient.CreateOrUpdateDeployment(client, searchDeployment); err != nil { + return fmt.Errorf("error when creating deployment for %s, err: %w", searchDeployment.Name, err) + } + return nil +} + +func createKarmadaSearchService(client clientset.Interface, name, namespace string) error { + searchServiceSetBytes, err := util.ParseTemplate(KarmadaSearchService, struct { + ServiceName, Namespace string + }{ + ServiceName: util.KarmadaSearchName(name), + Namespace: namespace, + }) + if err != nil { + return fmt.Errorf("error when parsing KarmadaSearch Service template: %w", err) + } + + searchService := &corev1.Service{} + if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), searchServiceSetBytes, searchService); err != nil { + return fmt.Errorf("err when decoding KarmadaSearch Service: %w", err) + } + + if err := apiclient.CreateOrUpdateService(client, searchService); err != nil { + return fmt.Errorf("err when creating service for %s, err: %w", searchService.Name, err) + } + return nil +} diff --git a/operator/pkg/karmadaresource/apiservice/apiservice.go b/operator/pkg/karmadaresource/apiservice/apiservice.go index 7c161ce8d..4eadfa731 100644 --- a/operator/pkg/karmadaresource/apiservice/apiservice.go +++ b/operator/pkg/karmadaresource/apiservice/apiservice.go @@ -169,3 +169,64 @@ func karmadaMetricsAdapterService(client clientset.Interface, karmadaControlPlan } return nil } + +// EnsureSearchAPIService creates APIService and a service for karmada-metrics-adapter +func EnsureSearchAPIService(aggregatorClient *aggregator.Clientset, client clientset.Interface, karmadaControlPlaneServiceName, karmadaControlPlaneNamespace, hostClusterServiceName, hostClusterNamespace, caBundle string) error { + if err := karmadaSearchService(client, karmadaControlPlaneServiceName, karmadaControlPlaneNamespace, hostClusterServiceName, hostClusterNamespace); err != nil { + return err + } + + return karmadaSearchAPIService(aggregatorClient, karmadaControlPlaneServiceName, karmadaControlPlaneNamespace, caBundle) +} + +func karmadaSearchAPIService(client *aggregator.Clientset, karmadaControlPlaneServiceName, karmadaControlPlaneNamespace, caBundle string) error { + apiServiceBytes, err := util.ParseTemplate(KarmadaSearchAPIService, struct { + ServiceName, Namespace string + CABundle string + }{ + Namespace: karmadaControlPlaneNamespace, + ServiceName: util.KarmadaSearchAPIServerName(karmadaControlPlaneServiceName), + CABundle: caBundle, + }) + if err != nil { + return fmt.Errorf("error when parsing KarmadaSearch APIService template: %s", err.Error()) + } + + apiService := &apiregistrationv1.APIService{} + if err = runtime.DecodeInto(codecs.UniversalDecoder(), apiServiceBytes, apiService); err != nil { + return fmt.Errorf("err when decoding KarmadaSearch APIService: %s", err.Error()) + } + + if err = apiclient.CreateOrUpdateAPIService(client, apiService); err != nil { + return err + } + + return nil +} + +func karmadaSearchService(client clientset.Interface, karmadaControlPlaneServiceName, karmadaControlPlaneNamespace, hostClusterServiceName, hostClusterNamespace string) error { + searchApiserverServiceBytes, err := util.ParseTemplate(KarmadaSearchService, struct { + Namespace string + ServiceName string + HostClusterServiceName string + HostClusterNamespace string + }{ + Namespace: karmadaControlPlaneNamespace, + ServiceName: util.KarmadaSearchName(karmadaControlPlaneServiceName), + HostClusterServiceName: util.KarmadaSearchName(hostClusterServiceName), + HostClusterNamespace: hostClusterNamespace, + }) + if err != nil { + return fmt.Errorf("error when parsing KarmadaSearch Service template: %w", err) + } + + searchService := &corev1.Service{} + if err := runtime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), searchApiserverServiceBytes, searchService); err != nil { + return fmt.Errorf("err when decoding KarmadaSearch Service: %w", err) + } + + if err := apiclient.CreateOrUpdateService(client, searchService); err != nil { + return err + } + return nil +} diff --git a/operator/pkg/karmadaresource/apiservice/manifest.go b/operator/pkg/karmadaresource/apiservice/manifest.go index 2d79d98f2..9261fb632 100644 --- a/operator/pkg/karmadaresource/apiservice/manifest.go +++ b/operator/pkg/karmadaresource/apiservice/manifest.go @@ -70,6 +70,38 @@ spec: KarmadaMetricsAdapterService = ` apiVersion: v1 kind: Service +metadata: + name: {{ .ServiceName }} + namespace: {{ .Namespace }} +spec: + type: ExternalName + externalName: {{ .HostClusterServiceName }}.{{ .HostClusterNamespace }}.svc +` + + // KarmadaSearchAPIService is karmada-search APIService manifest + KarmadaSearchAPIService = ` +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1alpha1.search.karmada.io + labels: + app: karmada-search + apiserver: "true" +spec: + caBundle: {{ .CABundle }} + group: search.karmada.io + groupPriorityMinimum: 2000 + service: + name: {{ .ServiceName }} + namespace: {{ .Namespace }} + version: v1alpha1 + versionPriority: 10 +` + + // KarmadaSearchService is karmada-search service manifest + KarmadaSearchService = ` +apiVersion: v1 +kind: Service metadata: name: {{ .ServiceName }} namespace: {{ .Namespace }} diff --git a/operator/pkg/tasks/deinit/component.go b/operator/pkg/tasks/deinit/component.go index 8d4ef762f..a082a8dba 100644 --- a/operator/pkg/tasks/deinit/component.go +++ b/operator/pkg/tasks/deinit/component.go @@ -41,6 +41,7 @@ func NewRemoveComponentTask() workflow.Task { newRemoveComponentSubTask(constants.KarmadaControllerManagerComponent, util.KarmadaControllerManagerName), newRemoveComponentSubTask(constants.KubeControllerManagerComponent, util.KubeControllerManagerName), newRemoveComponentWithServiceSubTask(constants.KarmadaWebhookComponent, util.KarmadaWebhookName), + newRemoveComponentWithServiceSubTask(constants.KarmadaSearchComponent, util.KarmadaSearchName), newRemoveComponentWithServiceSubTask(constants.KarmadaAggregatedAPIServerComponent, util.KarmadaAggregatedAPIServerName), newRemoveComponentWithServiceSubTask(constants.KarmadaAPIserverComponent, util.KarmadaAPIServerName), { diff --git a/operator/pkg/tasks/init/component.go b/operator/pkg/tasks/init/component.go index ffc5d1233..931090b3f 100644 --- a/operator/pkg/tasks/init/component.go +++ b/operator/pkg/tasks/init/component.go @@ -27,6 +27,7 @@ import ( "github.com/karmada-io/karmada/operator/pkg/constants" "github.com/karmada-io/karmada/operator/pkg/controlplane" "github.com/karmada-io/karmada/operator/pkg/controlplane/metricsadapter" + "github.com/karmada-io/karmada/operator/pkg/controlplane/search" "github.com/karmada-io/karmada/operator/pkg/controlplane/webhook" "github.com/karmada-io/karmada/operator/pkg/karmadaresource/apiservice" "github.com/karmada-io/karmada/operator/pkg/util/apiclient" @@ -44,11 +45,12 @@ func NewComponentTask() workflow.Task { newComponentSubTask(constants.KarmadaControllerManagerComponent), newComponentSubTask(constants.KarmadaSchedulerComponent), { - Name: "KarmadaWebhook", + Name: constants.KarmadaWebhookComponent, Run: runKarmadaWebhook, }, newComponentSubTask(constants.KarmadaDeschedulerComponent), newKarmadaMetricsAdapterSubTask(), + newKarmadaSearchSubTask(), }, } } @@ -228,3 +230,93 @@ func runDeployMetricAdapterAPIService(r workflow.RunData) error { return nil } + +func newKarmadaSearchSubTask() workflow.Task { + return workflow.Task{ + Name: constants.KarmadaSearchComponent, + Run: runKarmadaResources, + RunSubTasks: true, + Tasks: []workflow.Task{ + { + Name: "DeployKarmadaSearch", + Run: runKarmadaSearch, + }, + { + Name: "DeployKarmadaSearchAPIService", + Run: runDeployKarmadaSearchAPIService, + }, + }, + } +} + +func runKarmadaSearch(r workflow.RunData) error { + data, ok := r.(InitData) + if !ok { + return errors.New("KarmadaSearch task invoked with an invalid data struct") + } + + cfg := data.Components() + if cfg.KarmadaSearch == nil { + klog.Infof("Skip installing component (%s/%s)", data.GetNamespace(), constants.KarmadaSearchComponent) + return nil + } + + err := search.EnsureKarmadaSearch( + data.RemoteClient(), + cfg.KarmadaSearch, + data.GetName(), + data.GetNamespace(), + data.FeatureGates(), + ) + if err != nil { + return fmt.Errorf("failed to apply karmada search, err: %w", err) + } + + klog.V(2).InfoS("[KarmadaSearch] Successfully applied karmada search component", "karmada", klog.KObj(data)) + return nil +} + +func runDeployKarmadaSearchAPIService(r workflow.RunData) error { + data, ok := r.(InitData) + if !ok { + return errors.New("DeploySearchAPIService task invoked with an invalid data struct") + } + + cfg := data.Components() + if cfg.KarmadaSearch == nil { + klog.V(2).InfoS("[karmadaSearch] Skip install karmada-search APIService") + return nil + } + + config := data.ControlplaneConfig() + client, err := apiclient.NewAPIRegistrationClient(config) + if err != nil { + return err + } + + cert := data.GetCert(constants.CaCertAndKeyName) + if len(cert.CertData()) == 0 { + return errors.New("unexpected empty ca cert data for search") + } + caBase64 := base64.StdEncoding.EncodeToString(cert.CertData()) + + err = apiservice.EnsureSearchAPIService(client, data.KarmadaClient(), data.GetName(), constants.KarmadaSystemNamespace, data.GetName(), data.GetNamespace(), caBase64) + if err != nil { + return fmt.Errorf("failed to apply karmada-metrics-adapter APIService resource to karmada controlplane, err: %w", err) + } + + if *cfg.KarmadaSearch.Replicas != 0 { + waiter := apiclient.NewKarmadaWaiter(config, nil, time.Second*20) + for _, gv := range constants.KarmadaSearchAPIServices { + apiServiceName := fmt.Sprintf("%s.%s", gv.Version, gv.Group) + + if err := waiter.WaitForAPIService(apiServiceName); err != nil { + return fmt.Errorf("the APIService %s is unhealthy, err: %w", apiServiceName, err) + } + } + + klog.V(2).InfoS("[DeploySearchAPIService] all karmada-search APIServices status is ready ", "karmada", klog.KObj(data)) + } + + return nil +} diff --git a/operator/pkg/util/naming.go b/operator/pkg/util/naming.go index d00b84f1f..dbbecab7a 100644 --- a/operator/pkg/util/naming.go +++ b/operator/pkg/util/naming.go @@ -54,6 +54,11 @@ func KarmadaAggregatedAPIServerName(karmada string) string { return generateResourceName(karmada, "aggregated-apiserver") } +// KarmadaSearchAPIServerName returns secret name of karmada-search +func KarmadaSearchAPIServerName(karmada string) string { + return generateResourceName(karmada, "search") +} + // KarmadaEtcdName returns name of karmada-etcd func KarmadaEtcdName(karmada string) string { return generateResourceName(karmada, "etcd") @@ -94,6 +99,11 @@ func KarmadaMetricsAdapterName(karmada string) string { return generateResourceName(karmada, "metrics-adapter") } +// KarmadaSearchName returns name of karmada-search +func KarmadaSearchName(karmada string) string { + return generateResourceName(karmada, "search") +} + func generateResourceName(karmada, suffix string) string { if strings.Contains(karmada, "karmada") { return fmt.Sprintf("%s-%s", karmada, suffix)