From 1abdba3ec861a585cdbe7ae12b7f625fa1283934 Mon Sep 17 00:00:00 2001 From: calvin Date: Thu, 29 Jun 2023 00:05:42 +0800 Subject: [PATCH] karmada operator: install karmada metrics adapter addon Signed-off-by: calvin --- .../crds/operator.karmada.io_karmadas.yaml | 137 ++++++++++++++++-- .../crds/operator.karmada.io_karmadas.yaml | 137 ++++++++++++++++-- .../pkg/apis/operator/v1alpha1/defaults.go | 20 +++ operator/pkg/apis/operator/v1alpha1/type.go | 42 +++++- .../v1alpha1/zz_generated.deepcopy.go | 29 ++++ operator/pkg/constants/constants.go | 15 ++ .../controlplane/metricsadapter/mainfests.go | 95 ++++++++++++ .../metricsadapter/metricsadapter.go | 77 ++++++++++ .../karmadaresource/apiservice/apiservice.go | 65 +++++++++ .../karmadaresource/apiservice/manifest.go | 30 ++++ operator/pkg/tasks/init/component.go | 110 +++++++++++++- operator/pkg/tasks/init/karmadaresource.go | 2 +- operator/pkg/tasks/init/wait.go | 1 + operator/pkg/util/apiclient/wait.go | 9 +- operator/pkg/util/naming.go | 5 + 15 files changed, 726 insertions(+), 48 deletions(-) create mode 100644 operator/pkg/controlplane/metricsadapter/mainfests.go create mode 100644 operator/pkg/controlplane/metricsadapter/metricsadapter.go diff --git a/charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml b/charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml index a0b7df6f0..2cfbf7fb7 100644 --- a/charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml +++ b/charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml @@ -667,7 +667,7 @@ spec: used. Incorrect settings on this field maybe lead to the corresponding component in an unhealthy state. Before you do it, please confirm that you understand the risks of this - configuration. \n For supported flags, please see https://github.com/karmada-io/karmada/blob/master/cmd/aggregated-apiserver/app/options/options.go + configuration. \n For supported flags, please see https://karmada.io/docs/reference/components/karmada-aggregated-apiserver for details." type: object featureGates: @@ -793,7 +793,7 @@ spec: used. Incorrect settings on this field maybe lead to the corresponding component in an unhealthy state. Before you do it, please confirm that you understand the risks of this - configuration. \n For supported flags, please see https://github.com/karmada-io/karmada/blob/master/cmd/controller-manager/app/options/options.go + configuration. \n For supported flags, please see https://karmada.io/docs/reference/components/karmada-controller-manager for details." type: object featureGates: @@ -908,7 +908,112 @@ spec: on this field maybe lead to the corresponding component in an unhealthy state. Before you do it, please confirm that you understand the risks of this configuration. \n - For supported flags, please see https://github.com/karmada-io/karmada/blob/master/cmd/descheduler/app/options/options.go + For supported flags, please see https://karmada.io/docs/reference/components/karmada-descheduler + for details." + type: object + imageRepository: + description: ImageRepository sets the container registry to + pull images from. if not set, the ImageRepository defined + in KarmadaSpec will be used instead. + type: string + imageTag: + description: ImageTag allows to specify a tag for the image. + In case this value is set, operator does not change automatically + the version of the above components during upgrades. + type: string + labels: + additionalProperties: + type: string + description: 'Map of string keys and values that can be used + to organize and categorize (scope and select) objects. May + match selectors of replication controllers and services. + More info: http://kubernetes.io/docs/user-guide/labels' + type: object + replicas: + description: Number of desired pods. This is a pointer to + distinguish between explicit zero and not specified. Defaults + to 1. + format: int32 + type: integer + resources: + description: 'Compute Resources required by this component. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. \n This field + is immutable. It can only be set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry + in pod.spec.resourceClaims of the Pod where this + field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of + compute resources required. If Requests is omitted for + a container, it defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined value. + Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + type: object + karmadaMetricsAdapter: + description: KarmadaMetricsAdapter holds settings to karmada metrics + adapter component of the karmada. + properties: + annotations: + additionalProperties: + type: string + description: 'Annotations is an unstructured key value map + stored with a resource that may be set by external tools + to store and retrieve arbitrary metadata. They are not queryable + and should be preserved when modifying objects. More info: + http://kubernetes.io/docs/user-guide/annotations' + type: object + extraArgs: + additionalProperties: + type: string + description: "ExtraArgs is an extra set of flags to pass to + the karmada-metrics-adapter component or override. A key + in this map is the flag name as it appears on the command + line except without leading dash(es). \n Note: This is a + temporary solution to allow for the configuration of the + karmada-metrics-adapter component. In the future, we will + provide a more structured way to configure the component. + Once that is done, this field will be discouraged to be + used. Incorrect settings on this field maybe lead to the + corresponding component in an unhealthy state. Before you + do it, please confirm that you understand the risks of this + configuration. \n For supported flags, please see https://karmada.io/docs/reference/components/karmada-metrics-adapter for details." type: object imageRepository: @@ -1013,7 +1118,7 @@ spec: on this field maybe lead to the corresponding component in an unhealthy state. Before you do it, please confirm that you understand the risks of this configuration. \n - For supported flags, please see https://github.com/karmada-io/karmada/blob/master/cmd/scheduler/app/options/options.go + For supported flags, please see https://karmada.io/docs/reference/components/karmada-scheduler for details." type: object featureGates: @@ -1115,17 +1220,17 @@ spec: additionalProperties: type: string description: "ExtraArgs is an extra set of flags to pass to - the karmada-descheduler component or override. A key in - this map is the flag name as it appears on the command line - except without leading dash(es). \n Note: This is a temporary - solution to allow for the configuration of the karmada-descheduler - component. In the future, we will provide a more structured - way to configure the component. Once that is done, this - field will be discouraged to be used. Incorrect settings - on this field maybe lead to the corresponding component - in an unhealthy state. Before you do it, please confirm - that you understand the risks of this configuration. \n - For supported flags, please see https://github.com/karmada-io/karmada/blob/master/cmd/descheduler/app/options/options.go + the karmada-search component or override. A key in this + map is the flag name as it appears on the command line except + without leading dash(es). \n Note: This is a temporary solution + to allow for the configuration of the karmada-search component. + In the future, we will provide a more structured way to + configure the component. Once that is done, this field will + be discouraged to be used. Incorrect settings on this field + maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand + the risks of this configuration. \n For supported flags, + please see https://karmada.io/docs/reference/components/karmada-search for details." type: object imageRepository: @@ -1230,7 +1335,7 @@ spec: maybe lead to the corresponding component in an unhealthy state. Before you do it, please confirm that you understand the risks of this configuration. \n For supported flags, - please see https://github.com/karmada-io/karmada/blob/master/cmd/webhook/app/options/options.go + please see https://karmada.io/docs/reference/components/karmada-webhook for details." type: object imageRepository: diff --git a/operator/config/crds/operator.karmada.io_karmadas.yaml b/operator/config/crds/operator.karmada.io_karmadas.yaml index 024dc1b13..ee6fe97c9 100644 --- a/operator/config/crds/operator.karmada.io_karmadas.yaml +++ b/operator/config/crds/operator.karmada.io_karmadas.yaml @@ -663,7 +663,7 @@ spec: used. Incorrect settings on this field maybe lead to the corresponding component in an unhealthy state. Before you do it, please confirm that you understand the risks of this - configuration. \n For supported flags, please see https://github.com/karmada-io/karmada/blob/master/cmd/aggregated-apiserver/app/options/options.go + configuration. \n For supported flags, please see https://karmada.io/docs/reference/components/karmada-aggregated-apiserver for details." type: object featureGates: @@ -789,7 +789,7 @@ spec: used. Incorrect settings on this field maybe lead to the corresponding component in an unhealthy state. Before you do it, please confirm that you understand the risks of this - configuration. \n For supported flags, please see https://github.com/karmada-io/karmada/blob/master/cmd/controller-manager/app/options/options.go + configuration. \n For supported flags, please see https://karmada.io/docs/reference/components/karmada-controller-manager for details." type: object featureGates: @@ -904,7 +904,112 @@ spec: on this field maybe lead to the corresponding component in an unhealthy state. Before you do it, please confirm that you understand the risks of this configuration. \n - For supported flags, please see https://github.com/karmada-io/karmada/blob/master/cmd/descheduler/app/options/options.go + For supported flags, please see https://karmada.io/docs/reference/components/karmada-descheduler + for details." + type: object + imageRepository: + description: ImageRepository sets the container registry to + pull images from. if not set, the ImageRepository defined + in KarmadaSpec will be used instead. + type: string + imageTag: + description: ImageTag allows to specify a tag for the image. + In case this value is set, operator does not change automatically + the version of the above components during upgrades. + type: string + labels: + additionalProperties: + type: string + description: 'Map of string keys and values that can be used + to organize and categorize (scope and select) objects. May + match selectors of replication controllers and services. + More info: http://kubernetes.io/docs/user-guide/labels' + type: object + replicas: + description: Number of desired pods. This is a pointer to + distinguish between explicit zero and not specified. Defaults + to 1. + format: int32 + type: integer + resources: + description: 'Compute Resources required by this component. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. \n This field + is immutable." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry + in pod.spec.resourceClaims of the Pod where this + field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of + compute resources required. If Requests is omitted for + a container, it defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined value. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + type: object + karmadaMetricsAdapter: + description: KarmadaMetricsAdapter holds settings to karmada metrics + adapter component of the karmada. + properties: + annotations: + additionalProperties: + type: string + description: 'Annotations is an unstructured key value map + stored with a resource that may be set by external tools + to store and retrieve arbitrary metadata. They are not queryable + and should be preserved when modifying objects. More info: + http://kubernetes.io/docs/user-guide/annotations' + type: object + extraArgs: + additionalProperties: + type: string + description: "ExtraArgs is an extra set of flags to pass to + the karmada-metrics-adapter component or override. A key + in this map is the flag name as it appears on the command + line except without leading dash(es). \n Note: This is a + temporary solution to allow for the configuration of the + karmada-metrics-adapter component. In the future, we will + provide a more structured way to configure the component. + Once that is done, this field will be discouraged to be + used. Incorrect settings on this field maybe lead to the + corresponding component in an unhealthy state. Before you + do it, please confirm that you understand the risks of this + configuration. \n For supported flags, please see https://karmada.io/docs/reference/components/karmada-metrics-adapter for details." type: object imageRepository: @@ -1009,7 +1114,7 @@ spec: on this field maybe lead to the corresponding component in an unhealthy state. Before you do it, please confirm that you understand the risks of this configuration. \n - For supported flags, please see https://github.com/karmada-io/karmada/blob/master/cmd/scheduler/app/options/options.go + For supported flags, please see https://karmada.io/docs/reference/components/karmada-scheduler for details." type: object featureGates: @@ -1111,17 +1216,17 @@ spec: additionalProperties: type: string description: "ExtraArgs is an extra set of flags to pass to - the karmada-descheduler component or override. A key in - this map is the flag name as it appears on the command line - except without leading dash(es). \n Note: This is a temporary - solution to allow for the configuration of the karmada-descheduler - component. In the future, we will provide a more structured - way to configure the component. Once that is done, this - field will be discouraged to be used. Incorrect settings - on this field maybe lead to the corresponding component - in an unhealthy state. Before you do it, please confirm - that you understand the risks of this configuration. \n - For supported flags, please see https://github.com/karmada-io/karmada/blob/master/cmd/descheduler/app/options/options.go + the karmada-search component or override. A key in this + map is the flag name as it appears on the command line except + without leading dash(es). \n Note: This is a temporary solution + to allow for the configuration of the karmada-search component. + In the future, we will provide a more structured way to + configure the component. Once that is done, this field will + be discouraged to be used. Incorrect settings on this field + maybe lead to the corresponding component in an unhealthy + state. Before you do it, please confirm that you understand + the risks of this configuration. \n For supported flags, + please see https://karmada.io/docs/reference/components/karmada-search for details." type: object imageRepository: @@ -1226,7 +1331,7 @@ spec: maybe lead to the corresponding component in an unhealthy state. Before you do it, please confirm that you understand the risks of this configuration. \n For supported flags, - please see https://github.com/karmada-io/karmada/blob/master/cmd/webhook/app/options/options.go + please see https://karmada.io/docs/reference/components/karmada-webhook for details." type: object imageRepository: diff --git a/operator/pkg/apis/operator/v1alpha1/defaults.go b/operator/pkg/apis/operator/v1alpha1/defaults.go index af4d2904a..de8ff8543 100644 --- a/operator/pkg/apis/operator/v1alpha1/defaults.go +++ b/operator/pkg/apis/operator/v1alpha1/defaults.go @@ -19,6 +19,7 @@ var ( karmadaSchedulerImageRepository = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaScheduler) karmadaWebhookImageRepository = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaWebhook) karmadaDeschedulerImageRepository = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaDescheduler) + KarmadaMetricsAdapterImageRepository = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaMetricsAdapter) ) func addDefaultingFuncs(scheme *runtime.Scheme) error { @@ -55,6 +56,7 @@ func setDefaultsKarmadaComponents(obj *Karmada) { setDefaultsKarmadaControllerManager(obj.Spec.Components) setDefaultsKarmadaScheduler(obj.Spec.Components) setDefaultsKarmadaWebhook(obj.Spec.Components) + setDefaultsKarmadaMetricsAdapter(obj.Spec.Components) // set addon defaults setDefaultsKarmadaDescheduler(obj.Spec.Components) @@ -227,3 +229,21 @@ func setDefaultsKarmadaDescheduler(obj *KarmadaComponents) { descheduler.Replicas = pointer.Int32(1) } } + +func setDefaultsKarmadaMetricsAdapter(obj *KarmadaComponents) { + if obj.KarmadaMetricsAdapter == nil { + obj.KarmadaMetricsAdapter = &KarmadaMetricsAdapter{} + } + + metricsAdapter := obj.KarmadaMetricsAdapter + if len(metricsAdapter.Image.ImageRepository) == 0 { + metricsAdapter.Image.ImageRepository = KarmadaMetricsAdapterImageRepository + } + if len(metricsAdapter.Image.ImageTag) == 0 { + metricsAdapter.Image.ImageTag = constants.KarmadaDefaultVersion + } + + if metricsAdapter.Replicas == nil { + metricsAdapter.Replicas = pointer.Int32(2) + } +} diff --git a/operator/pkg/apis/operator/v1alpha1/type.go b/operator/pkg/apis/operator/v1alpha1/type.go index f493f9881..bd9bf87db 100644 --- a/operator/pkg/apis/operator/v1alpha1/type.go +++ b/operator/pkg/apis/operator/v1alpha1/type.go @@ -123,6 +123,10 @@ type KarmadaComponents struct { // KarmadaSearch holds settings to karmada search component of the karmada. // +optional KarmadaSearch *KarmadaSearch `json:"karmadaSearch,omitempty"` + + // KarmadaMetricsAdapter holds settings to karmada metrics adapter component of the karmada. + // +optional + KarmadaMetricsAdapter *KarmadaMetricsAdapter `json:"karmadaMetricsAdapter,omitempty"` } // Networking contains elements describing cluster's networking configuration @@ -266,7 +270,7 @@ type KarmadaAggregatedAPIServer struct { // state. Before you do it, please confirm that you understand the risks of this configuration. // // For supported flags, please see - // https://github.com/karmada-io/karmada/blob/master/cmd/aggregated-apiserver/app/options/options.go + // https://karmada.io/docs/reference/components/karmada-aggregated-apiserver // for details. // +optional ExtraArgs map[string]string `json:"extraArgs,omitempty"` @@ -379,7 +383,7 @@ type KarmadaControllerManager struct { // state. Before you do it, please confirm that you understand the risks of this configuration. // // For supported flags, please see - // https://github.com/karmada-io/karmada/blob/master/cmd/controller-manager/app/options/options.go + // https://karmada.io/docs/reference/components/karmada-controller-manager // for details. // +optional ExtraArgs map[string]string `json:"extraArgs,omitempty"` @@ -410,7 +414,7 @@ type KarmadaScheduler struct { // state. Before you do it, please confirm that you understand the risks of this configuration. // // For supported flags, please see - // https://github.com/karmada-io/karmada/blob/master/cmd/scheduler/app/options/options.go + // https://karmada.io/docs/reference/components/karmada-scheduler // for details. // +optional ExtraArgs map[string]string `json:"extraArgs,omitempty"` @@ -438,7 +442,7 @@ type KarmadaDescheduler struct { // state. Before you do it, please confirm that you understand the risks of this configuration. // // For supported flags, please see - // https://github.com/karmada-io/karmada/blob/master/cmd/descheduler/app/options/options.go + // https://karmada.io/docs/reference/components/karmada-descheduler // for details. // +optional ExtraArgs map[string]string `json:"extraArgs,omitempty"` @@ -449,18 +453,40 @@ type KarmadaSearch struct { // CommonSettings holds common settings to karmada search. CommonSettings `json:",inline"` - // ExtraArgs is an extra set of flags to pass to the karmada-descheduler component or override. + // ExtraArgs is an extra set of flags to pass to the karmada-search component or override. // A key in this map is the flag name as it appears on the command line except without // leading dash(es). // - // Note: This is a temporary solution to allow for the configuration of the karmada-descheduler + // Note: This is a temporary solution to allow for the configuration of the karmada-search // component. In the future, we will provide a more structured way to configure the component. // Once that is done, this field will be discouraged to be used. // Incorrect settings on this field maybe lead to the corresponding component in an unhealthy // state. Before you do it, please confirm that you understand the risks of this configuration. // // For supported flags, please see - // https://github.com/karmada-io/karmada/blob/master/cmd/descheduler/app/options/options.go + // https://karmada.io/docs/reference/components/karmada-search + // for details. + // +optional + ExtraArgs map[string]string `json:"extraArgs,omitempty"` +} + +// KarmadaMetricsAdapter holds settings to karmada-metrics-adapter component of the karmada. +type KarmadaMetricsAdapter struct { + // CommonSettings holds common settings to karmada metrics adapter. + CommonSettings `json:",inline"` + + // ExtraArgs is an extra set of flags to pass to the karmada-metrics-adapter component or override. + // A key in this map is the flag name as it appears on the command line except without + // leading dash(es). + // + // Note: This is a temporary solution to allow for the configuration of the karmada-metrics-adapter + // component. In the future, we will provide a more structured way to configure the component. + // Once that is done, this field will be discouraged to be used. + // Incorrect settings on this field maybe lead to the corresponding component in an unhealthy + // state. Before you do it, please confirm that you understand the risks of this configuration. + // + // For supported flags, please see + // https://karmada.io/docs/reference/components/karmada-metrics-adapter // for details. // +optional ExtraArgs map[string]string `json:"extraArgs,omitempty"` @@ -482,7 +508,7 @@ type KarmadaWebhook struct { // state. Before you do it, please confirm that you understand the risks of this configuration. // // For supported flags, please see - // https://github.com/karmada-io/karmada/blob/master/cmd/webhook/app/options/options.go + // https://karmada.io/docs/reference/components/karmada-webhook // for details. // +optional ExtraArgs map[string]string `json:"extraArgs,omitempty"` diff --git a/operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go b/operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go index 8f7025e93..43922d2b2 100644 --- a/operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go @@ -321,6 +321,11 @@ func (in *KarmadaComponents) DeepCopyInto(out *KarmadaComponents) { *out = new(KarmadaSearch) (*in).DeepCopyInto(*out) } + if in.KarmadaMetricsAdapter != nil { + in, out := &in.KarmadaMetricsAdapter, &out.KarmadaMetricsAdapter + *out = new(KarmadaMetricsAdapter) + (*in).DeepCopyInto(*out) + } return } @@ -427,6 +432,30 @@ func (in *KarmadaList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KarmadaMetricsAdapter) DeepCopyInto(out *KarmadaMetricsAdapter) { + *out = *in + in.CommonSettings.DeepCopyInto(&out.CommonSettings) + if in.ExtraArgs != nil { + in, out := &in.ExtraArgs, &out.ExtraArgs + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KarmadaMetricsAdapter. +func (in *KarmadaMetricsAdapter) DeepCopy() *KarmadaMetricsAdapter { + if in == nil { + return nil + } + out := new(KarmadaMetricsAdapter) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KarmadaScheduler) DeepCopyInto(out *KarmadaScheduler) { *out = *in diff --git a/operator/pkg/constants/constants.go b/operator/pkg/constants/constants.go index 6517f15cd..5313b1bc2 100644 --- a/operator/pkg/constants/constants.go +++ b/operator/pkg/constants/constants.go @@ -4,6 +4,7 @@ import ( "time" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" ) const ( @@ -42,6 +43,8 @@ const ( KarmadaWebhook = "karmada-webhook" // KarmadaDescheduler defines the name of the karmada-descheduler component KarmadaDescheduler = "karmada-descheduler" + // KarmadaMetricsAdapter defines the name of the karmada-metrics-adapter component + KarmadaMetricsAdapter = "karmada-metrics-adapter" // KarmadaSystemNamespace defines the leader selection namespace for karmada components KarmadaSystemNamespace = "karmada-system" @@ -96,12 +99,24 @@ const ( KarmadaWebhookComponent = "KarmadaWebhook" // KarmadaDeschedulerComponent defines the name of the karmada-descheduler component KarmadaDeschedulerComponent = "KarmadaDescheduler" + // KarmadaMetricsAdapterComponent defines the name of the karmada-metrics-adapter component + KarmadaMetricsAdapterComponent = "KarmadaMetricsAdapter" // KarmadaOperatorLabelKeyName defines a label key used by all resources created by karmada operator KarmadaOperatorLabelKeyName = "app.kubernetes.io/managed-by" + + // APIServiceName defines the karmada aggregated apiserver APISerivce resource name. + APIServiceName = "v1alpha1.cluster.karmada.io" ) var ( // KarmadaOperatorLabel defines the default labels in the resource create by karmada operator KarmadaOperatorLabel = labels.Set{KarmadaOperatorLabelKeyName: KarmadaOperator} + + // KarmadaMetricsAdapterAPIServices defines the GroupVersions of all karmada-metrics-adapter APIServices + KarmadaMetricsAdapterAPIServices = []schema.GroupVersion{ + {Group: "metrics.k8s.io", Version: "v1beta1"}, + {Group: "custom.metrics.k8s.io", Version: "v1beta1"}, + {Group: "custom.metrics.k8s.io", Version: "v1beta2"}, + } ) diff --git a/operator/pkg/controlplane/metricsadapter/mainfests.go b/operator/pkg/controlplane/metricsadapter/mainfests.go new file mode 100644 index 000000000..1de9ac3fa --- /dev/null +++ b/operator/pkg/controlplane/metricsadapter/mainfests.go @@ -0,0 +1,95 @@ +package metricsadapter + +const ( + // KarmadaMetricsAdapterDeployment is karmada-metrics-adapter deployment manifest + KarmadaMetricsAdapterDeployment = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .DeploymentName }} + namespace: {{ .Namespace }} + labels: + karmada-app: karmada-metrics-adapter + app.kubernetes.io/managed-by: karmada-operator +spec: + replicas: {{ .Replicas }} + selector: + matchLabels: + karmada-app: karmada-metrics-adapter + template: + metadata: + labels: + karmada-app: karmada-metrics-adapter + spec: + automountServiceAccountToken: false + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + containers: + - name: karmada-metrics-adapter + image: {{ .Image }} + imagePullPolicy: IfNotPresent + command: + - /bin/karmada-metrics-adapter + - --kubeconfig=/etc/karmada/config + - --authentication-kubeconfig=/etc/karmada/config + - --authorization-kubeconfig=/etc/karmada/config + - --client-ca-file=/etc/karmada/pki/ca.crt + - --audit-log-path=- + - --audit-log-maxage=0 + - --audit-log-maxbackup=0 + volumeMounts: + - name: kubeconfig + subPath: config + mountPath: /etc/karmada/config + - name: karmada-cert + mountPath: /etc/karmada/pki + readOnly: true + readinessProbe: + httpGet: + path: /readyz + port: 443 + scheme: HTTPS + initialDelaySeconds: 1 + failureThreshold: 3 + periodSeconds: 3 + timeoutSeconds: 15 + livenessProbe: + httpGet: + path: /healthz + port: 443 + scheme: HTTPS + initialDelaySeconds: 10 + failureThreshold: 3 + periodSeconds: 10 + timeoutSeconds: 15 + resources: + requests: + cpu: 100m + volumes: + - name: kubeconfig + secret: + secretName: {{ .KubeconfigSecret }} + - name: karmada-cert + secret: + secretName: {{ .KarmadaCertsSecret }} +` + + // KarmadaMetricsAdapterService is karmada-metrics-adapter service manifest + KarmadaMetricsAdapterService = ` +apiVersion: v1 +kind: Service +metadata: + name: {{ .ServiceName }} + namespace: {{ .Namespace }} + labels: + app.kubernetes.io/managed-by: karmada-operator +spec: + selector: + karmada-app: karmada-metrics-adapter + ports: + - port: 443 + protocol: TCP + targetPort: 443 +` +) diff --git a/operator/pkg/controlplane/metricsadapter/metricsadapter.go b/operator/pkg/controlplane/metricsadapter/metricsadapter.go new file mode 100644 index 000000000..7ef816b6c --- /dev/null +++ b/operator/pkg/controlplane/metricsadapter/metricsadapter.go @@ -0,0 +1,77 @@ +package metricsadapter + +import ( + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + kuberuntime "k8s.io/apimachinery/pkg/runtime" + clientset "k8s.io/client-go/kubernetes" + clientsetscheme "k8s.io/client-go/kubernetes/scheme" + + operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1" + "github.com/karmada-io/karmada/operator/pkg/util" + "github.com/karmada-io/karmada/operator/pkg/util/apiclient" + "github.com/karmada-io/karmada/operator/pkg/util/patcher" +) + +// EnsureKarmadaMetricAdapter creates karmada-metric-adapter deployment and service resource. +func EnsureKarmadaMetricAdapter(client clientset.Interface, cfg *operatorv1alpha1.KarmadaMetricsAdapter, name, namespace string) error { + if err := installKarmadaMetricAdapter(client, cfg, name, namespace); err != nil { + return err + } + + return createKarmadaMetricAdapterService(client, name, namespace) +} + +func installKarmadaMetricAdapter(client clientset.Interface, cfg *operatorv1alpha1.KarmadaMetricsAdapter, name, namespace string) error { + metricAdapterBytes, err := util.ParseTemplate(KarmadaMetricsAdapterDeployment, struct { + DeploymentName, Namespace, Image string + KubeconfigSecret, KarmadaCertsSecret string + Replicas *int32 + }{ + DeploymentName: util.KarmadaMetricsAdapterName(name), + Namespace: namespace, + Image: cfg.Image.Name(), + Replicas: cfg.Replicas, + KubeconfigSecret: util.AdminKubeconfigSecretName(name), + KarmadaCertsSecret: util.KarmadaCertSecretName(name), + }) + if err != nil { + return fmt.Errorf("error when parsing KarmadaMetricAdapter Deployment template: %w", err) + } + + metricAdapter := &appsv1.Deployment{} + if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), metricAdapterBytes, metricAdapter); err != nil { + return fmt.Errorf("err when decoding KarmadaMetricAdapter Deployment: %w", err) + } + + patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(metricAdapter) + + if err := apiclient.CreateOrUpdateDeployment(client, metricAdapter); err != nil { + return fmt.Errorf("error when creating deployment for %s, err: %w", metricAdapter.Name, err) + } + return nil +} + +func createKarmadaMetricAdapterService(client clientset.Interface, name, namespace string) error { + metricAdapterServiceBytes, err := util.ParseTemplate(KarmadaMetricsAdapterService, struct { + ServiceName, Namespace string + }{ + ServiceName: util.KarmadaMetricsAdapterName(name), + Namespace: namespace, + }) + if err != nil { + return fmt.Errorf("error when parsing KarmadaMetricAdapter Service template: %w", err) + } + + metricAdapterService := &corev1.Service{} + if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), metricAdapterServiceBytes, metricAdapterService); err != nil { + return fmt.Errorf("err when decoding KarmadaMetricAdapter Service: %w", err) + } + + if err := apiclient.CreateOrUpdateService(client, metricAdapterService); err != nil { + return fmt.Errorf("err when creating service for %s, err: %w", metricAdapterService.Name, err) + } + return nil +} diff --git a/operator/pkg/karmadaresource/apiservice/apiservice.go b/operator/pkg/karmadaresource/apiservice/apiservice.go index f628edd26..a04f5c597 100644 --- a/operator/pkg/karmadaresource/apiservice/apiservice.go +++ b/operator/pkg/karmadaresource/apiservice/apiservice.go @@ -14,6 +14,7 @@ import ( apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" aggregator "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + "github.com/karmada-io/karmada/operator/pkg/constants" "github.com/karmada-io/karmada/operator/pkg/util" "github.com/karmada-io/karmada/operator/pkg/util/apiclient" ) @@ -82,3 +83,67 @@ func aggregatedApiserverService(client clientset.Interface, name, namespace stri } return nil } + +// EnsureMetricsAdapterAPIService creates APIService and a service for karmada-metrics-adapter +func EnsureMetricsAdapterAPIService(aggregatorClient *aggregator.Clientset, client clientset.Interface, name, namespace string) error { + if err := karmadaMetricsAdapterService(client, name, namespace); err != nil { + return err + } + + return karmadaMetricsAdapterAPIService(aggregatorClient, name, namespace) +} + +func karmadaMetricsAdapterAPIService(client *aggregator.Clientset, name, namespace string) error { + for _, gv := range constants.KarmadaMetricsAdapterAPIServices { + // The APIService name to metrics adapter is "$version.$group" + apiServiceName := fmt.Sprintf("%s.%s", gv.Version, gv.Group) + + apiServiceBytes, err := util.ParseTemplate(KarmadaMetricsAdapterAPIService, struct { + Name, Namespace string + ServiceName, Group, Version string + }{ + Name: apiServiceName, + Namespace: namespace, + Group: gv.Group, + Version: gv.Version, + ServiceName: util.KarmadaMetricsAdapterName(name), + }) + if err != nil { + return fmt.Errorf("error when parsing KarmadaMetricsAdapter APIService %s template: %w", apiServiceName, err) + } + + apiService := &apiregistrationv1.APIService{} + if err := runtime.DecodeInto(codecs.UniversalDecoder(), apiServiceBytes, apiService); err != nil { + return fmt.Errorf("err when decoding KarmadaMetricsAdapter APIService %s: %w", apiServiceName, err) + } + + if err := apiclient.CreateOrUpdateAPIService(client, apiService); err != nil { + return err + } + } + + return nil +} + +func karmadaMetricsAdapterService(client clientset.Interface, name, namespace string) error { + aggregatedApiserverServiceBytes, err := util.ParseTemplate(KarmadaMetricsAdapterService, struct { + Namespace string + ServiceName string + }{ + Namespace: namespace, + ServiceName: util.KarmadaMetricsAdapterName(name), + }) + if err != nil { + return fmt.Errorf("error when parsing KarmadaMetricsAdapter Service template: %w", err) + } + + aggregatedService := &corev1.Service{} + if err := runtime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), aggregatedApiserverServiceBytes, aggregatedService); err != nil { + return fmt.Errorf("err when decoding KarmadaMetricsAdapter Service: %w", err) + } + + if err := apiclient.CreateOrUpdateService(client, aggregatedService); err != nil { + return err + } + return nil +} diff --git a/operator/pkg/karmadaresource/apiservice/manifest.go b/operator/pkg/karmadaresource/apiservice/manifest.go index aed6d5b11..0b4827d86 100644 --- a/operator/pkg/karmadaresource/apiservice/manifest.go +++ b/operator/pkg/karmadaresource/apiservice/manifest.go @@ -20,10 +20,40 @@ spec: version: v1alpha1 versionPriority: 10 ` + // KarmadaAggregatedApiserverService is karmada aggregated apiserver service manifest KarmadaAggregatedApiserverService = ` apiVersion: v1 kind: Service +metadata: + name: {{ .ServiceName }} + namespace: {{ .Namespace }} +spec: + type: ExternalName + externalName: {{ .ServiceName }}.{{ .Namespace }}.svc +` + + // KarmadaMetricsAdapterAPIService is karmada-metrics-adapter APIService manifest + KarmadaMetricsAdapterAPIService = ` +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: {{ .Name }} +spec: + service: + name: {{ .ServiceName }} + namespace: {{ .Namespace }} + group: {{ .Group }} + version: {{ .Version }} + insecureSkipTLSVerify: true + groupPriorityMinimum: 100 + versionPriority: 200 +` + + // KarmadaMetricsAdapterService is karmada-metrics-adapter service manifest + KarmadaMetricsAdapterService = ` +apiVersion: v1 +kind: Service metadata: name: {{ .ServiceName }} namespace: {{ .Namespace }} diff --git a/operator/pkg/tasks/init/component.go b/operator/pkg/tasks/init/component.go index 7d01080f7..a2fe67c81 100644 --- a/operator/pkg/tasks/init/component.go +++ b/operator/pkg/tasks/init/component.go @@ -3,12 +3,16 @@ package tasks import ( "errors" "fmt" + "time" "k8s.io/klog/v2" "github.com/karmada-io/karmada/operator/pkg/constants" "github.com/karmada-io/karmada/operator/pkg/controlplane" + "github.com/karmada-io/karmada/operator/pkg/controlplane/metricsadapter" "github.com/karmada-io/karmada/operator/pkg/controlplane/webhook" + "github.com/karmada-io/karmada/operator/pkg/karmadaresource/apiservice" + "github.com/karmada-io/karmada/operator/pkg/util/apiclient" "github.com/karmada-io/karmada/operator/pkg/workflow" ) @@ -27,6 +31,7 @@ func NewComponentTask() workflow.Task { Run: runKarmadaWebhook, }, newComponentSubTask(constants.KarmadaDeschedulerComponent), + newKarmadaMetricsAdapterSubTask(), }, } } @@ -75,7 +80,7 @@ func runComponentSubTask(component string) func(r workflow.RunData) error { func runKarmadaWebhook(r workflow.RunData) error { data, ok := r.(InitData) if !ok { - return errors.New("certs task invoked with an invalid data struct") + return errors.New("KarmadaWebhook task invoked with an invalid data struct") } cfg := data.Components() @@ -97,3 +102,106 @@ func runKarmadaWebhook(r workflow.RunData) error { klog.V(2).InfoS("[KarmadaWebhook] Successfully applied karmada webhook component", "karmada", klog.KObj(data)) return nil } + +func newKarmadaMetricsAdapterSubTask() workflow.Task { + return workflow.Task{ + Name: constants.KarmadaMetricsAdapterComponent, + Run: runKarmadaMetricsAdapter, + RunSubTasks: true, + Tasks: []workflow.Task{ + { + Name: "DeployMetricAdapter", + Run: runDeployMetricAdapter, + }, + { + Name: "DeployMetricAdapterAPIService", + Run: runDeployMetricAdapterAPIService, + }, + }, + } +} + +func runKarmadaMetricsAdapter(r workflow.RunData) error { + data, ok := r.(InitData) + if !ok { + return errors.New("karmadaMetricsAdapter task invoked with an invalid data struct") + } + + klog.V(4).InfoS("[karmadaMetricsAdapter] Running karmadaMetricsAdapter task", "karmada", klog.KObj(data)) + return nil +} + +func runDeployMetricAdapter(r workflow.RunData) error { + data, ok := r.(InitData) + if !ok { + return errors.New("DeployMetricAdapter task invoked with an invalid data struct") + } + + cfg := data.Components() + if cfg.KarmadaMetricsAdapter == nil { + klog.V(2).InfoS("[karmadaMetricsAdapter] Skip install karmada-metrics-adapter component") + return nil + } + + err := metricsadapter.EnsureKarmadaMetricAdapter( + data.RemoteClient(), + cfg.KarmadaMetricsAdapter, + data.GetName(), + data.GetNamespace(), + ) + if err != nil { + return fmt.Errorf("failed to apply karmada-metrics-adapter, err: %w", err) + } + + klog.V(2).InfoS("[DeployMetricAdapter] Successfully applied karmada-metrics-adapter component", "karmada", klog.KObj(data)) + + if *cfg.KarmadaMetricsAdapter.Replicas != 0 { + waiter := apiclient.NewKarmadaWaiter(data.ControlplaneConfig(), data.RemoteClient(), time.Second*30) + if err = waiter.WaitForSomePods(karmadaMetricAdapterLabels.String(), data.GetNamespace(), 1); err != nil { + return fmt.Errorf("waiting for karmada-metrics-adapter to ready timeout, err: %w", err) + } + + klog.V(2).InfoS("[DeployMetricAdapter] the karmada-metrics-adapter is ready", "karmada", klog.KObj(data)) + } + + return nil +} + +func runDeployMetricAdapterAPIService(r workflow.RunData) error { + data, ok := r.(InitData) + if !ok { + return errors.New("DeployMetricAdapterAPIService task invoked with an invalid data struct") + } + + cfg := data.Components() + if cfg.KarmadaMetricsAdapter == nil { + klog.V(2).InfoS("[karmadaMetricsAdapter] Skip install karmada-metrics-adapter APIService") + return nil + } + + config := data.ControlplaneConfig() + client, err := apiclient.NewAPIRegistrationClient(config) + if err != nil { + return err + } + + err = apiservice.EnsureMetricsAdapterAPIService(client, data.KarmadaClient(), data.GetName(), data.GetNamespace()) + if err != nil { + return fmt.Errorf("failed to apply karmada-metrics-adapter APIService resource to karmada controlplane, err: %w", err) + } + + if *cfg.KarmadaMetricsAdapter.Replicas != 0 { + waiter := apiclient.NewKarmadaWaiter(config, nil, time.Second*20) + for _, gv := range constants.KarmadaMetricsAdapterAPIServices { + apiServiceName := fmt.Sprintf("%s.%s", gv.Version, gv.Group) + + if err := waiter.WaitForAPIService(apiServiceName); err != nil { + return fmt.Errorf("the APIService %s is unhealthy, err: %w", apiServiceName, err) + } + } + + klog.V(2).InfoS("[DeployMetricAdapterAPIService] all karmada-metrics-adapter APIServices status is ready ", "karmada", klog.KObj(data)) + } + + return nil +} diff --git a/operator/pkg/tasks/init/karmadaresource.go b/operator/pkg/tasks/init/karmadaresource.go index b11b6d858..901df91a1 100644 --- a/operator/pkg/tasks/init/karmadaresource.go +++ b/operator/pkg/tasks/init/karmadaresource.go @@ -200,7 +200,7 @@ func runAPIService(r workflow.RunData) error { } waiter := apiclient.NewKarmadaWaiter(config, nil, componentBeReadyTimeout) - if err := apiclient.TryRunCommand(waiter.WaitForAPIService, 3); err != nil { + if err := waiter.WaitForAPIService(constants.APIServiceName); err != nil { return fmt.Errorf("the APIService is unhealthy, err: %w", err) } diff --git a/operator/pkg/tasks/init/wait.go b/operator/pkg/tasks/init/wait.go index 89f412aa0..c52b64200 100644 --- a/operator/pkg/tasks/init/wait.go +++ b/operator/pkg/tasks/init/wait.go @@ -25,6 +25,7 @@ var ( karmadaControllerManagerLabels = labels.Set{"karmada-app": constants.KarmadaControllerManager} karmadaSchedulerLabels = labels.Set{"karmada-app": constants.KarmadaScheduler} karmadaWebhookLabels = labels.Set{"karmada-app": constants.KarmadaWebhook} + karmadaMetricAdapterLabels = labels.Set{"karmada-app": constants.KarmadaMetricsAdapter} ) // NewCheckApiserverHealthTask init wait-apiserver task diff --git a/operator/pkg/util/apiclient/wait.go b/operator/pkg/util/apiclient/wait.go index f66c4f976..42ac8f82e 100644 --- a/operator/pkg/util/apiclient/wait.go +++ b/operator/pkg/util/apiclient/wait.go @@ -18,9 +18,6 @@ import ( const ( // APICallRetryInterval defines how long kubeadm should wait before retrying a failed API operation APICallRetryInterval = 500 * time.Millisecond - - // APIServiceName defines the karmada aggregated apiserver APISerivce resource name. - APIServiceName = "v1alpha1.cluster.karmada.io" ) // Waiter is an interface for waiting for criteria in Karmada to happen @@ -28,7 +25,7 @@ type Waiter interface { // WaitForAPI waits for the API Server's /healthz endpoint to become "ok" WaitForAPI() error // WaitForAPIService waits for the APIService condition to become "true" - WaitForAPIService() error + WaitForAPIService(name string) error // WaitForPods waits for Pods in the namespace to become Ready WaitForPods(label, namespace string) error // WaitForSomePods waits for the specified number of Pods in the namespace to become Ready @@ -67,14 +64,14 @@ func (w *KarmadaWaiter) WaitForAPI() error { } // WaitForAPIService waits for the APIService condition to become "true" -func (w *KarmadaWaiter) WaitForAPIService() error { +func (w *KarmadaWaiter) WaitForAPIService(name string) error { aggregateClient, err := aggregator.NewForConfig(w.karmadaConfig) if err != nil { return err } err = wait.PollImmediate(APICallRetryInterval, w.timeout, func() (done bool, err error) { - apiService, err := aggregateClient.ApiregistrationV1().APIServices().Get(context.TODO(), APIServiceName, metav1.GetOptions{}) + apiService, err := aggregateClient.ApiregistrationV1().APIServices().Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { return false, nil } diff --git a/operator/pkg/util/naming.go b/operator/pkg/util/naming.go index c5a6cc62c..248f85f65 100644 --- a/operator/pkg/util/naming.go +++ b/operator/pkg/util/naming.go @@ -73,6 +73,11 @@ func KarmadaDeschedulerName(karmada string) string { return generateResourceName(karmada, "descheduler") } +// KarmadaMetricsAdapterName returns name of karmada-metric-adapter +func KarmadaMetricsAdapterName(karmada string) string { + return generateResourceName(karmada, "metrics-adapter") +} + func generateResourceName(karmada, suffix string) string { if strings.Contains(karmada, "karmada") { return fmt.Sprintf("%s-%s", karmada, suffix)