karmada-io
/
karmada
mirror of https://github.com/karmada-io/karmada.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4316 from zhzhuang-zju/operator 

feat: support install karmada search with operator
This commit is contained in:
karmada-bot 2023-12-13 11:28:10 +08:00 committed by GitHub
parent 652c25436d 4916e3b9cd
commit 85a905f955
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 432 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
operator/pkg/apis/operator/v1alpha1/defaults.go
View File

@ -40,6 +40,7 @@ var (
karmadaWebhookImageRepository = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaWebhook)
karmadaDeschedulerImageRepository = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaDescheduler)
KarmadaMetricsAdapterImageRepository = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaMetricsAdapter)
karmadaSearchImageRepository = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaSearch)
)
func init() {
@ -87,7 +88,7 @@ func setDefaultsKarmadaComponents(obj *Karmada) {
setDefaultsKarmadaScheduler(obj.Spec.Components)
setDefaultsKarmadaWebhook(obj.Spec.Components)
setDefaultsKarmadaMetricsAdapter(obj.Spec.Components)
setDefaultsKarmadaSearch(obj.Spec.Components)
// set addon defaults
setDefaultsKarmadaDescheduler(obj.Spec.Components)
}
@ -243,6 +244,23 @@ func setDefaultsKarmadaWebhook(obj *KarmadaComponents) {
}
}
func setDefaultsKarmadaSearch(obj *KarmadaComponents) {
if obj.KarmadaSearch == nil {
return
}
search := obj.KarmadaSearch
if len(search.Image.ImageRepository) == 0 {
search.Image.ImageRepository = karmadaSearchImageRepository
}
if len(search.Image.ImageTag) == 0 {
search.Image.ImageTag = DefaultKarmadaImageVersion
}
if search.Replicas == nil {
search.Replicas = pointer.Int32(1)
}
}
func setDefaultsKarmadaDescheduler(obj *KarmadaComponents) {
if obj.KarmadaDescheduler == nil {
return

9
operator/pkg/constants/constants.go
View File

@ -55,6 +55,8 @@ const (
KarmadaScheduler = "karmada-scheduler"
// KarmadaWebhook defines the name of the karmada-webhook component
KarmadaWebhook = "karmada-webhook"
// KarmadaSearch defines the name of the karmada-search component
KarmadaSearch = "karmada-search"
// KarmadaDescheduler defines the name of the karmada-descheduler component
KarmadaDescheduler = "karmada-descheduler"
// KarmadaMetricsAdapter defines the name of the karmada-metrics-adapter component
@ -111,6 +113,8 @@ const (
KarmadaSchedulerComponent = "KarmadaScheduler"
// KarmadaWebhookComponent defines the name of the karmada-webhook component
KarmadaWebhookComponent = "KarmadaWebhook"
// KarmadaSearchComponent defines the name of the karmada-search component
KarmadaSearchComponent = "KarmadaSearch"
// KarmadaDeschedulerComponent defines the name of the karmada-descheduler component
KarmadaDeschedulerComponent = "KarmadaDescheduler"
// KarmadaMetricsAdapterComponent defines the name of the karmada-metrics-adapter component
@ -133,4 +137,9 @@ var (
{Group: "custom.metrics.k8s.io", Version: "v1beta1"},
{Group: "custom.metrics.k8s.io", Version: "v1beta2"},
}
// KarmadaSearchAPIServices defines the GroupVersions of all karmada-search APIServices
KarmadaSearchAPIServices = []schema.GroupVersion{
{Group: "search.karmada.io", Version: "v1alpha1"},
}
)

109
operator/pkg/controlplane/search/mainfests.go Normal file
View File

@ -0,0 +1,109 @@
/*
Copyright 2023 The Karmada Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package search
const (
// KarmadaSearchDeployment is karmada search deployment manifest
KarmadaSearchDeployment = `
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .DeploymentName }}
namespace: {{ .Namespace }}
labels:
karmada-app: karmada-search
apiserver: "true"
spec:
selector:
matchLabels:
karmada-app: karmada-search
apiserver: "true"
replicas: {{ .Replicas }}
template:
metadata:
labels:
karmada-app: karmada-search
apiserver: "true"
spec:
automountServiceAccountToken: false
containers:
- name: karmada-search
image: {{ .Image }}
imagePullPolicy: IfNotPresent
volumeMounts:
- name: k8s-certs
mountPath: /etc/karmada/pki
readOnly: true
- name: kubeconfig
subPath: kubeconfig
mountPath: /etc/kubeconfig
command:
- /bin/karmada-search
- --kubeconfig=/etc/kubeconfig
- --authentication-kubeconfig=/etc/kubeconfig
- --authorization-kubeconfig=/etc/kubeconfig
- --etcd-servers=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenClientPort }}
- --etcd-cafile=/etc/karmada/pki/etcd-ca.crt
- --etcd-certfile=/etc/karmada/pki/etcd-client.crt
- --etcd-keyfile=/etc/karmada/pki/etcd-client.key
- --tls-cert-file=/etc/karmada/pki/karmada.crt
- --tls-private-key-file=/etc/karmada/pki/karmada.key
- --tls-min-version=VersionTLS13
- --audit-log-path=-
- --feature-gates=APIPriorityAndFairness=false
- --audit-log-maxage=0
- --audit-log-maxbackup=0
livenessProbe:
httpGet:
path: /livez
port: 443
scheme: HTTPS
failureThreshold: 3
initialDelaySeconds: 15
periodSeconds: 15
timeoutSeconds: 5
resources:
requests:
cpu: 100m
volumes:
- name: k8s-certs
secret:
secretName: {{ .KarmadaCertsSecret }}
- name: kubeconfig
secret:
secretName: {{ .KubeconfigSecret }}
`
// KarmadaSearchService is karmada-search service manifest
KarmadaSearchService = `
apiVersion: v1
kind: Service
metadata:
name: {{ .ServiceName }}
namespace: {{ .Namespace }}
labels:
karmada-app: karmada-search
apiserver: "true"
spec:
ports:
- port: 443
protocol: TCP
targetPort: 443
selector:
karmada-app: karmada-search
`
)

98
operator/pkg/controlplane/search/search.go Normal file
View File

@ -0,0 +1,98 @@
/*
Copyright 2023 The Karmada Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package search
import (
"fmt"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
kuberuntime "k8s.io/apimachinery/pkg/runtime"
clientset "k8s.io/client-go/kubernetes"
clientsetscheme "k8s.io/client-go/kubernetes/scheme"
operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
"github.com/karmada-io/karmada/operator/pkg/constants"
"github.com/karmada-io/karmada/operator/pkg/util"
"github.com/karmada-io/karmada/operator/pkg/util/apiclient"
"github.com/karmada-io/karmada/operator/pkg/util/patcher"
)
// EnsureKarmadaSearch creates karmada search deployment and service resource.
func EnsureKarmadaSearch(client clientset.Interface, cfg *operatorv1alpha1.KarmadaSearch, name, namespace string, featureGates map[string]bool) error {
if err := installKarmadaSearch(client, cfg, name, namespace, featureGates); err != nil {
return err
}
return createKarmadaSearchService(client, name, namespace)
}
func installKarmadaSearch(client clientset.Interface, cfg *operatorv1alpha1.KarmadaSearch, name, namespace string, featureGates map[string]bool) error {
searchDeploymentSetBytes, err := util.ParseTemplate(KarmadaSearchDeployment, struct {
DeploymentName, Namespace, Image, KarmadaCertsSecret string
KubeconfigSecret, EtcdClientService string
Replicas *int32
EtcdListenClientPort int32
}{
DeploymentName: util.KarmadaSearchName(name),
Namespace: namespace,
Image: cfg.Image.Name(),
KarmadaCertsSecret: util.KarmadaCertSecretName(name),
Replicas: cfg.Replicas,
KubeconfigSecret: util.AdminKubeconfigSecretName(name),
EtcdClientService: util.KarmadaEtcdClientName(name),
EtcdListenClientPort: constants.EtcdListenClientPort,
})
if err != nil {
return fmt.Errorf("error when parsing KarmadaSearch Deployment template: %w", err)
}
searchDeployment := &appsv1.Deployment{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), searchDeploymentSetBytes, searchDeployment); err != nil {
return fmt.Errorf("err when decoding KarmadaSearch Deployment: %w", err)
}
patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
WithExtraArgs(cfg.ExtraArgs).WithResources(cfg.Resources).ForDeployment(searchDeployment)
if err := apiclient.CreateOrUpdateDeployment(client, searchDeployment); err != nil {
return fmt.Errorf("error when creating deployment for %s, err: %w", searchDeployment.Name, err)
}
return nil
}
func createKarmadaSearchService(client clientset.Interface, name, namespace string) error {
searchServiceSetBytes, err := util.ParseTemplate(KarmadaSearchService, struct {
ServiceName, Namespace string
}{
ServiceName: util.KarmadaSearchName(name),
Namespace: namespace,
})
if err != nil {
return fmt.Errorf("error when parsing KarmadaSearch Service template: %w", err)
}
searchService := &corev1.Service{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), searchServiceSetBytes, searchService); err != nil {
return fmt.Errorf("err when decoding KarmadaSearch Service: %w", err)
}
if err := apiclient.CreateOrUpdateService(client, searchService); err != nil {
return fmt.Errorf("err when creating service for %s, err: %w", searchService.Name, err)
}
return nil
}

61
operator/pkg/karmadaresource/apiservice/apiservice.go
View File

@ -169,3 +169,64 @@ func karmadaMetricsAdapterService(client clientset.Interface, karmadaControlPlan
}
return nil
}
// EnsureSearchAPIService creates APIService and a service for karmada-metrics-adapter
func EnsureSearchAPIService(aggregatorClient *aggregator.Clientset, client clientset.Interface, karmadaControlPlaneServiceName, karmadaControlPlaneNamespace, hostClusterServiceName, hostClusterNamespace, caBundle string) error {
if err := karmadaSearchService(client, karmadaControlPlaneServiceName, karmadaControlPlaneNamespace, hostClusterServiceName, hostClusterNamespace); err != nil {
return err
}
return karmadaSearchAPIService(aggregatorClient, karmadaControlPlaneServiceName, karmadaControlPlaneNamespace, caBundle)
}
func karmadaSearchAPIService(client *aggregator.Clientset, karmadaControlPlaneServiceName, karmadaControlPlaneNamespace, caBundle string) error {
apiServiceBytes, err := util.ParseTemplate(KarmadaSearchAPIService, struct {
ServiceName, Namespace string
CABundle string
}{
Namespace: karmadaControlPlaneNamespace,
ServiceName: util.KarmadaSearchAPIServerName(karmadaControlPlaneServiceName),
CABundle: caBundle,
})
if err != nil {
return fmt.Errorf("error when parsing KarmadaSearch APIService template: %s", err.Error())
}
apiService := &apiregistrationv1.APIService{}
if err = runtime.DecodeInto(codecs.UniversalDecoder(), apiServiceBytes, apiService); err != nil {
return fmt.Errorf("err when decoding KarmadaSearch APIService: %s", err.Error())
}
if err = apiclient.CreateOrUpdateAPIService(client, apiService); err != nil {
return err
}
return nil
}
func karmadaSearchService(client clientset.Interface, karmadaControlPlaneServiceName, karmadaControlPlaneNamespace, hostClusterServiceName, hostClusterNamespace string) error {
searchApiserverServiceBytes, err := util.ParseTemplate(KarmadaSearchService, struct {
Namespace string
ServiceName string
HostClusterServiceName string
HostClusterNamespace string
}{
Namespace: karmadaControlPlaneNamespace,
ServiceName: util.KarmadaSearchName(karmadaControlPlaneServiceName),
HostClusterServiceName: util.KarmadaSearchName(hostClusterServiceName),
HostClusterNamespace: hostClusterNamespace,
})
if err != nil {
return fmt.Errorf("error when parsing KarmadaSearch Service template: %w", err)
}
searchService := &corev1.Service{}
if err := runtime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), searchApiserverServiceBytes, searchService); err != nil {
return fmt.Errorf("err when decoding KarmadaSearch Service: %w", err)
}
if err := apiclient.CreateOrUpdateService(client, searchService); err != nil {
return err
}
return nil
}

32
operator/pkg/karmadaresource/apiservice/manifest.go
View File

@ -70,6 +70,38 @@ spec:
KarmadaMetricsAdapterService = `
apiVersion: v1
kind: Service
metadata:
name: {{ .ServiceName }}
namespace: {{ .Namespace }}
spec:
type: ExternalName
externalName: {{ .HostClusterServiceName }}.{{ .HostClusterNamespace }}.svc
`
// KarmadaSearchAPIService is karmada-search APIService manifest
KarmadaSearchAPIService = `
apiVersion: apiregistration.k8s.io/v1
kind: APIService
metadata:
name: v1alpha1.search.karmada.io
labels:
app: karmada-search
apiserver: "true"
spec:
caBundle: {{ .CABundle }}
group: search.karmada.io
groupPriorityMinimum: 2000
service:
name: {{ .ServiceName }}
namespace: {{ .Namespace }}
version: v1alpha1
versionPriority: 10
`
// KarmadaSearchService is karmada-search service manifest
KarmadaSearchService = `
apiVersion: v1
kind: Service
metadata:
name: {{ .ServiceName }}
namespace: {{ .Namespace }}

1
operator/pkg/tasks/deinit/component.go
View File

@ -41,6 +41,7 @@ func NewRemoveComponentTask() workflow.Task {
newRemoveComponentSubTask(constants.KarmadaControllerManagerComponent, util.KarmadaControllerManagerName),
newRemoveComponentSubTask(constants.KubeControllerManagerComponent, util.KubeControllerManagerName),
newRemoveComponentWithServiceSubTask(constants.KarmadaWebhookComponent, util.KarmadaWebhookName),
newRemoveComponentWithServiceSubTask(constants.KarmadaSearchComponent, util.KarmadaSearchName),
newRemoveComponentWithServiceSubTask(constants.KarmadaAggregatedAPIServerComponent, util.KarmadaAggregatedAPIServerName),
newRemoveComponentWithServiceSubTask(constants.KarmadaAPIserverComponent, util.KarmadaAPIServerName),
{

94
operator/pkg/tasks/init/component.go
View File

@ -27,6 +27,7 @@ import (
"github.com/karmada-io/karmada/operator/pkg/constants"
"github.com/karmada-io/karmada/operator/pkg/controlplane"
"github.com/karmada-io/karmada/operator/pkg/controlplane/metricsadapter"
"github.com/karmada-io/karmada/operator/pkg/controlplane/search"
"github.com/karmada-io/karmada/operator/pkg/controlplane/webhook"
"github.com/karmada-io/karmada/operator/pkg/karmadaresource/apiservice"
"github.com/karmada-io/karmada/operator/pkg/util/apiclient"
@ -44,11 +45,12 @@ func NewComponentTask() workflow.Task {
newComponentSubTask(constants.KarmadaControllerManagerComponent),
newComponentSubTask(constants.KarmadaSchedulerComponent),
{
Name: "KarmadaWebhook",
Name: constants.KarmadaWebhookComponent,
Run: runKarmadaWebhook,
},
newComponentSubTask(constants.KarmadaDeschedulerComponent),
newKarmadaMetricsAdapterSubTask(),
newKarmadaSearchSubTask(),
},
}
}
@ -228,3 +230,93 @@ func runDeployMetricAdapterAPIService(r workflow.RunData) error {
return nil
}
func newKarmadaSearchSubTask() workflow.Task {
return workflow.Task{
Name: constants.KarmadaSearchComponent,
Run: runKarmadaResources,
RunSubTasks: true,
Tasks: []workflow.Task{
{
Name: "DeployKarmadaSearch",
Run: runKarmadaSearch,
},
{
Name: "DeployKarmadaSearchAPIService",
Run: runDeployKarmadaSearchAPIService,
},
},
}
}
func runKarmadaSearch(r workflow.RunData) error {
data, ok := r.(InitData)
if !ok {
return errors.New("KarmadaSearch task invoked with an invalid data struct")
}
cfg := data.Components()
if cfg.KarmadaSearch == nil {
klog.Infof("Skip installing component (%s/%s)", data.GetNamespace(), constants.KarmadaSearchComponent)
return nil
}
err := search.EnsureKarmadaSearch(
data.RemoteClient(),
cfg.KarmadaSearch,
data.GetName(),
data.GetNamespace(),
data.FeatureGates(),
)
if err != nil {
return fmt.Errorf("failed to apply karmada search, err: %w", err)
}
klog.V(2).InfoS("[KarmadaSearch] Successfully applied karmada search component", "karmada", klog.KObj(data))
return nil
}
func runDeployKarmadaSearchAPIService(r workflow.RunData) error {
data, ok := r.(InitData)
if !ok {
return errors.New("DeploySearchAPIService task invoked with an invalid data struct")
}
cfg := data.Components()
if cfg.KarmadaSearch == nil {
klog.V(2).InfoS("[karmadaSearch] Skip install karmada-search APIService")
return nil
}
config := data.ControlplaneConfig()
client, err := apiclient.NewAPIRegistrationClient(config)
if err != nil {
return err
}
cert := data.GetCert(constants.CaCertAndKeyName)
if len(cert.CertData()) == 0 {
return errors.New("unexpected empty ca cert data for search")
}
caBase64 := base64.StdEncoding.EncodeToString(cert.CertData())
err = apiservice.EnsureSearchAPIService(client, data.KarmadaClient(), data.GetName(), constants.KarmadaSystemNamespace, data.GetName(), data.GetNamespace(), caBase64)
if err != nil {
return fmt.Errorf("failed to apply karmada-metrics-adapter APIService resource to karmada controlplane, err: %w", err)
}
if *cfg.KarmadaSearch.Replicas != 0 {
waiter := apiclient.NewKarmadaWaiter(config, nil, time.Second*20)
for _, gv := range constants.KarmadaSearchAPIServices {
apiServiceName := fmt.Sprintf("%s.%s", gv.Version, gv.Group)
if err := waiter.WaitForAPIService(apiServiceName); err != nil {
return fmt.Errorf("the APIService %s is unhealthy, err: %w", apiServiceName, err)
}
}
klog.V(2).InfoS("[DeploySearchAPIService] all karmada-search APIServices status is ready ", "karmada", klog.KObj(data))
}
return nil
}

10
operator/pkg/util/naming.go
View File

@ -54,6 +54,11 @@ func KarmadaAggregatedAPIServerName(karmada string) string {
return generateResourceName(karmada, "aggregated-apiserver")
}
// KarmadaSearchAPIServerName returns secret name of karmada-search
func KarmadaSearchAPIServerName(karmada string) string {
return generateResourceName(karmada, "search")
}
// KarmadaEtcdName returns name of karmada-etcd
func KarmadaEtcdName(karmada string) string {
return generateResourceName(karmada, "etcd")
@ -94,6 +99,11 @@ func KarmadaMetricsAdapterName(karmada string) string {
return generateResourceName(karmada, "metrics-adapter")
}
// KarmadaSearchName returns name of karmada-search
func KarmadaSearchName(karmada string) string {
return generateResourceName(karmada, "search")
}
func generateResourceName(karmada, suffix string) string {
if strings.Contains(karmada, "karmada") {
return fmt.Sprintf("%s-%s", karmada, suffix)