karmada-io
/
karmada
mirror of https://github.com/karmada-io/karmada.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

karmada operator: install karmada metrics adapter addon 

Signed-off-by: calvin <wen.chen@daocloud.io>
This commit is contained in:
calvin 2023-06-29 00:05:42 +08:00
parent 19fbab565a
commit 1abdba3ec8
15 changed files with 726 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml
View File

@ -667,7 +667,7 @@ spec:
used. Incorrect settings on this field maybe lead to the
corresponding component in an unhealthy state. Before you
do it, please confirm that you understand the risks of this
configuration. \n For supported flags, please see https://github.com/karmada-io/karmada/blob/master/cmd/aggregated-apiserver/app/options/options.go
configuration. \n For supported flags, please see https://karmada.io/docs/reference/components/karmada-aggregated-apiserver
for details."
type: object
featureGates:
@ -793,7 +793,7 @@ spec:
used. Incorrect settings on this field maybe lead to the
corresponding component in an unhealthy state. Before you
do it, please confirm that you understand the risks of this
configuration. \n For supported flags, please see https://github.com/karmada-io/karmada/blob/master/cmd/controller-manager/app/options/options.go
configuration. \n For supported flags, please see https://karmada.io/docs/reference/components/karmada-controller-manager
for details."
type: object
featureGates:
@ -908,7 +908,112 @@ spec:
on this field maybe lead to the corresponding component
in an unhealthy state. Before you do it, please confirm
that you understand the risks of this configuration. \n
For supported flags, please see https://github.com/karmada-io/karmada/blob/master/cmd/descheduler/app/options/options.go
For supported flags, please see https://karmada.io/docs/reference/components/karmada-descheduler
for details."
type: object
imageRepository:
description: ImageRepository sets the container registry to
pull images from. if not set, the ImageRepository defined
in KarmadaSpec will be used instead.
type: string
imageTag:
description: ImageTag allows to specify a tag for the image.
In case this value is set, operator does not change automatically
the version of the above components during upgrades.
type: string
labels:
additionalProperties:
type: string
description: 'Map of string keys and values that can be used
to organize and categorize (scope and select) objects. May
match selectors of replication controllers and services.
More info: http://kubernetes.io/docs/user-guide/labels'
type: object
replicas:
description: Number of desired pods. This is a pointer to
distinguish between explicit zero and not specified. Defaults
to 1.
format: int32
type: integer
resources:
description: 'Compute Resources required by this component.
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
properties:
claims:
description: "Claims lists the names of resources, defined
in spec.resourceClaims, that are used by this container.
\n This is an alpha field and requires enabling the
DynamicResourceAllocation feature gate. \n This field
is immutable. It can only be set for containers."
items:
description: ResourceClaim references one entry in PodSpec.ResourceClaims.
properties:
name:
description: Name must match the name of one entry
in pod.spec.resourceClaims of the Pod where this
field is used. It makes that resource available
inside a container.
type: string
required:
- name
type: object
type: array
x-kubernetes-list-map-keys:
- name
x-kubernetes-list-type: map
limits:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: 'Limits describes the maximum amount of compute
resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
type: object
requests:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: 'Requests describes the minimum amount of
compute resources required. If Requests is omitted for
a container, it defaults to Limits if that is explicitly
specified, otherwise to an implementation-defined value.
Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
type: object
type: object
type: object
karmadaMetricsAdapter:
description: KarmadaMetricsAdapter holds settings to karmada metrics
adapter component of the karmada.
properties:
annotations:
additionalProperties:
type: string
description: 'Annotations is an unstructured key value map
stored with a resource that may be set by external tools
to store and retrieve arbitrary metadata. They are not queryable
and should be preserved when modifying objects. More info:
http://kubernetes.io/docs/user-guide/annotations'
type: object
extraArgs:
additionalProperties:
type: string
description: "ExtraArgs is an extra set of flags to pass to
the karmada-metrics-adapter component or override. A key
in this map is the flag name as it appears on the command
line except without leading dash(es). \n Note: This is a
temporary solution to allow for the configuration of the
karmada-metrics-adapter component. In the future, we will
provide a more structured way to configure the component.
Once that is done, this field will be discouraged to be
used. Incorrect settings on this field maybe lead to the
corresponding component in an unhealthy state. Before you
do it, please confirm that you understand the risks of this
configuration. \n For supported flags, please see https://karmada.io/docs/reference/components/karmada-metrics-adapter
for details."
type: object
imageRepository:
@ -1013,7 +1118,7 @@ spec:
on this field maybe lead to the corresponding component
in an unhealthy state. Before you do it, please confirm
that you understand the risks of this configuration. \n
For supported flags, please see https://github.com/karmada-io/karmada/blob/master/cmd/scheduler/app/options/options.go
For supported flags, please see https://karmada.io/docs/reference/components/karmada-scheduler
for details."
type: object
featureGates:
@ -1115,17 +1220,17 @@ spec:
additionalProperties:
type: string
description: "ExtraArgs is an extra set of flags to pass to
the karmada-descheduler component or override. A key in
this map is the flag name as it appears on the command line
except without leading dash(es). \n Note: This is a temporary
solution to allow for the configuration of the karmada-descheduler
component. In the future, we will provide a more structured
way to configure the component. Once that is done, this
field will be discouraged to be used. Incorrect settings
on this field maybe lead to the corresponding component
in an unhealthy state. Before you do it, please confirm
that you understand the risks of this configuration. \n
For supported flags, please see https://github.com/karmada-io/karmada/blob/master/cmd/descheduler/app/options/options.go
the karmada-search component or override. A key in this
map is the flag name as it appears on the command line except
without leading dash(es). \n Note: This is a temporary solution
to allow for the configuration of the karmada-search component.
In the future, we will provide a more structured way to
configure the component. Once that is done, this field will
be discouraged to be used. Incorrect settings on this field
maybe lead to the corresponding component in an unhealthy
state. Before you do it, please confirm that you understand
the risks of this configuration. \n For supported flags,
please see https://karmada.io/docs/reference/components/karmada-search
for details."
type: object
imageRepository:
@ -1230,7 +1335,7 @@ spec:
maybe lead to the corresponding component in an unhealthy
state. Before you do it, please confirm that you understand
the risks of this configuration. \n For supported flags,
please see https://github.com/karmada-io/karmada/blob/master/cmd/webhook/app/options/options.go
please see https://karmada.io/docs/reference/components/karmada-webhook
for details."
type: object
imageRepository:

137
operator/config/crds/operator.karmada.io_karmadas.yaml
View File

@ -663,7 +663,7 @@ spec:
used. Incorrect settings on this field maybe lead to the
corresponding component in an unhealthy state. Before you
do it, please confirm that you understand the risks of this
configuration. \n For supported flags, please see https://github.com/karmada-io/karmada/blob/master/cmd/aggregated-apiserver/app/options/options.go
configuration. \n For supported flags, please see https://karmada.io/docs/reference/components/karmada-aggregated-apiserver
for details."
type: object
featureGates:
@ -789,7 +789,7 @@ spec:
used. Incorrect settings on this field maybe lead to the
corresponding component in an unhealthy state. Before you
do it, please confirm that you understand the risks of this
configuration. \n For supported flags, please see https://github.com/karmada-io/karmada/blob/master/cmd/controller-manager/app/options/options.go
configuration. \n For supported flags, please see https://karmada.io/docs/reference/components/karmada-controller-manager
for details."
type: object
featureGates:
@ -904,7 +904,112 @@ spec:
on this field maybe lead to the corresponding component
in an unhealthy state. Before you do it, please confirm
that you understand the risks of this configuration. \n
For supported flags, please see https://github.com/karmada-io/karmada/blob/master/cmd/descheduler/app/options/options.go
For supported flags, please see https://karmada.io/docs/reference/components/karmada-descheduler
for details."
type: object
imageRepository:
description: ImageRepository sets the container registry to
pull images from. if not set, the ImageRepository defined
in KarmadaSpec will be used instead.
type: string
imageTag:
description: ImageTag allows to specify a tag for the image.
In case this value is set, operator does not change automatically
the version of the above components during upgrades.
type: string
labels:
additionalProperties:
type: string
description: 'Map of string keys and values that can be used
to organize and categorize (scope and select) objects. May
match selectors of replication controllers and services.
More info: http://kubernetes.io/docs/user-guide/labels'
type: object
replicas:
description: Number of desired pods. This is a pointer to
distinguish between explicit zero and not specified. Defaults
to 1.
format: int32
type: integer
resources:
description: 'Compute Resources required by this component.
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
properties:
claims:
description: "Claims lists the names of resources, defined
in spec.resourceClaims, that are used by this container.
\n This is an alpha field and requires enabling the
DynamicResourceAllocation feature gate. \n This field
is immutable."
items:
description: ResourceClaim references one entry in PodSpec.ResourceClaims.
properties:
name:
description: Name must match the name of one entry
in pod.spec.resourceClaims of the Pod where this
field is used. It makes that resource available
inside a container.
type: string
required:
- name
type: object
type: array
x-kubernetes-list-map-keys:
- name
x-kubernetes-list-type: map
limits:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: 'Limits describes the maximum amount of compute
resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
type: object
requests:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: 'Requests describes the minimum amount of
compute resources required. If Requests is omitted for
a container, it defaults to Limits if that is explicitly
specified, otherwise to an implementation-defined value.
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
type: object
type: object
type: object
karmadaMetricsAdapter:
description: KarmadaMetricsAdapter holds settings to karmada metrics
adapter component of the karmada.
properties:
annotations:
additionalProperties:
type: string
description: 'Annotations is an unstructured key value map
stored with a resource that may be set by external tools
to store and retrieve arbitrary metadata. They are not queryable
and should be preserved when modifying objects. More info:
http://kubernetes.io/docs/user-guide/annotations'
type: object
extraArgs:
additionalProperties:
type: string
description: "ExtraArgs is an extra set of flags to pass to
the karmada-metrics-adapter component or override. A key
in this map is the flag name as it appears on the command
line except without leading dash(es). \n Note: This is a
temporary solution to allow for the configuration of the
karmada-metrics-adapter component. In the future, we will
provide a more structured way to configure the component.
Once that is done, this field will be discouraged to be
used. Incorrect settings on this field maybe lead to the
corresponding component in an unhealthy state. Before you
do it, please confirm that you understand the risks of this
configuration. \n For supported flags, please see https://karmada.io/docs/reference/components/karmada-metrics-adapter
for details."
type: object
imageRepository:
@ -1009,7 +1114,7 @@ spec:
on this field maybe lead to the corresponding component
in an unhealthy state. Before you do it, please confirm
that you understand the risks of this configuration. \n
For supported flags, please see https://github.com/karmada-io/karmada/blob/master/cmd/scheduler/app/options/options.go
For supported flags, please see https://karmada.io/docs/reference/components/karmada-scheduler
for details."
type: object
featureGates:
@ -1111,17 +1216,17 @@ spec:
additionalProperties:
type: string
description: "ExtraArgs is an extra set of flags to pass to
the karmada-descheduler component or override. A key in
this map is the flag name as it appears on the command line
except without leading dash(es). \n Note: This is a temporary
solution to allow for the configuration of the karmada-descheduler
component. In the future, we will provide a more structured
way to configure the component. Once that is done, this
field will be discouraged to be used. Incorrect settings
on this field maybe lead to the corresponding component
in an unhealthy state. Before you do it, please confirm
that you understand the risks of this configuration. \n
For supported flags, please see https://github.com/karmada-io/karmada/blob/master/cmd/descheduler/app/options/options.go
the karmada-search component or override. A key in this
map is the flag name as it appears on the command line except
without leading dash(es). \n Note: This is a temporary solution
to allow for the configuration of the karmada-search component.
In the future, we will provide a more structured way to
configure the component. Once that is done, this field will
be discouraged to be used. Incorrect settings on this field
maybe lead to the corresponding component in an unhealthy
state. Before you do it, please confirm that you understand
the risks of this configuration. \n For supported flags,
please see https://karmada.io/docs/reference/components/karmada-search
for details."
type: object
imageRepository:
@ -1226,7 +1331,7 @@ spec:
maybe lead to the corresponding component in an unhealthy
state. Before you do it, please confirm that you understand
the risks of this configuration. \n For supported flags,
please see https://github.com/karmada-io/karmada/blob/master/cmd/webhook/app/options/options.go
please see https://karmada.io/docs/reference/components/karmada-webhook
for details."
type: object
imageRepository:

20
operator/pkg/apis/operator/v1alpha1/defaults.go
View File

@ -19,6 +19,7 @@ var (
karmadaSchedulerImageRepository = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaScheduler)
karmadaWebhookImageRepository = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaWebhook)
karmadaDeschedulerImageRepository = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaDescheduler)
KarmadaMetricsAdapterImageRepository = fmt.Sprintf("%s/%s", constants.KarmadaDefaultRepository, constants.KarmadaMetricsAdapter)
)
func addDefaultingFuncs(scheme *runtime.Scheme) error {
@ -55,6 +56,7 @@ func setDefaultsKarmadaComponents(obj *Karmada) {
setDefaultsKarmadaControllerManager(obj.Spec.Components)
setDefaultsKarmadaScheduler(obj.Spec.Components)
setDefaultsKarmadaWebhook(obj.Spec.Components)
setDefaultsKarmadaMetricsAdapter(obj.Spec.Components)
// set addon defaults
setDefaultsKarmadaDescheduler(obj.Spec.Components)
@ -227,3 +229,21 @@ func setDefaultsKarmadaDescheduler(obj *KarmadaComponents) {
descheduler.Replicas = pointer.Int32(1)
}
}
func setDefaultsKarmadaMetricsAdapter(obj *KarmadaComponents) {
if obj.KarmadaMetricsAdapter == nil {
obj.KarmadaMetricsAdapter = &KarmadaMetricsAdapter{}
}
metricsAdapter := obj.KarmadaMetricsAdapter
if len(metricsAdapter.Image.ImageRepository) == 0 {
metricsAdapter.Image.ImageRepository = KarmadaMetricsAdapterImageRepository
}
if len(metricsAdapter.Image.ImageTag) == 0 {
metricsAdapter.Image.ImageTag = constants.KarmadaDefaultVersion
}
if metricsAdapter.Replicas == nil {
metricsAdapter.Replicas = pointer.Int32(2)
}
}

42
operator/pkg/apis/operator/v1alpha1/type.go
View File

@ -123,6 +123,10 @@ type KarmadaComponents struct {
// KarmadaSearch holds settings to karmada search component of the karmada.
// +optional
KarmadaSearch *KarmadaSearch `json:"karmadaSearch,omitempty"`
// KarmadaMetricsAdapter holds settings to karmada metrics adapter component of the karmada.
// +optional
KarmadaMetricsAdapter *KarmadaMetricsAdapter `json:"karmadaMetricsAdapter,omitempty"`
}
// Networking contains elements describing cluster's networking configuration
@ -266,7 +270,7 @@ type KarmadaAggregatedAPIServer struct {
// state. Before you do it, please confirm that you understand the risks of this configuration.
//
// For supported flags, please see
// https://github.com/karmada-io/karmada/blob/master/cmd/aggregated-apiserver/app/options/options.go
// https://karmada.io/docs/reference/components/karmada-aggregated-apiserver
// for details.
// +optional
ExtraArgs map[string]string `json:"extraArgs,omitempty"`
@ -379,7 +383,7 @@ type KarmadaControllerManager struct {
// state. Before you do it, please confirm that you understand the risks of this configuration.
//
// For supported flags, please see
// https://github.com/karmada-io/karmada/blob/master/cmd/controller-manager/app/options/options.go
// https://karmada.io/docs/reference/components/karmada-controller-manager
// for details.
// +optional
ExtraArgs map[string]string `json:"extraArgs,omitempty"`
@ -410,7 +414,7 @@ type KarmadaScheduler struct {
// state. Before you do it, please confirm that you understand the risks of this configuration.
//
// For supported flags, please see
// https://github.com/karmada-io/karmada/blob/master/cmd/scheduler/app/options/options.go
// https://karmada.io/docs/reference/components/karmada-scheduler
// for details.
// +optional
ExtraArgs map[string]string `json:"extraArgs,omitempty"`
@ -438,7 +442,7 @@ type KarmadaDescheduler struct {
// state. Before you do it, please confirm that you understand the risks of this configuration.
//
// For supported flags, please see
// https://github.com/karmada-io/karmada/blob/master/cmd/descheduler/app/options/options.go
// https://karmada.io/docs/reference/components/karmada-descheduler
// for details.
// +optional
ExtraArgs map[string]string `json:"extraArgs,omitempty"`
@ -449,18 +453,40 @@ type KarmadaSearch struct {
// CommonSettings holds common settings to karmada search.
CommonSettings `json:",inline"`
// ExtraArgs is an extra set of flags to pass to the karmada-descheduler component or override.
// ExtraArgs is an extra set of flags to pass to the karmada-search component or override.
// A key in this map is the flag name as it appears on the command line except without
// leading dash(es).
//
// Note: This is a temporary solution to allow for the configuration of the karmada-descheduler
// Note: This is a temporary solution to allow for the configuration of the karmada-search
// component. In the future, we will provide a more structured way to configure the component.
// Once that is done, this field will be discouraged to be used.
// Incorrect settings on this field maybe lead to the corresponding component in an unhealthy
// state. Before you do it, please confirm that you understand the risks of this configuration.
//
// For supported flags, please see
// https://github.com/karmada-io/karmada/blob/master/cmd/descheduler/app/options/options.go
// https://karmada.io/docs/reference/components/karmada-search
// for details.
// +optional
ExtraArgs map[string]string `json:"extraArgs,omitempty"`
}
// KarmadaMetricsAdapter holds settings to karmada-metrics-adapter component of the karmada.
type KarmadaMetricsAdapter struct {
// CommonSettings holds common settings to karmada metrics adapter.
CommonSettings `json:",inline"`
// ExtraArgs is an extra set of flags to pass to the karmada-metrics-adapter component or override.
// A key in this map is the flag name as it appears on the command line except without
// leading dash(es).
//
// Note: This is a temporary solution to allow for the configuration of the karmada-metrics-adapter
// component. In the future, we will provide a more structured way to configure the component.
// Once that is done, this field will be discouraged to be used.
// Incorrect settings on this field maybe lead to the corresponding component in an unhealthy
// state. Before you do it, please confirm that you understand the risks of this configuration.
//
// For supported flags, please see
// https://karmada.io/docs/reference/components/karmada-metrics-adapter
// for details.
// +optional
ExtraArgs map[string]string `json:"extraArgs,omitempty"`
@ -482,7 +508,7 @@ type KarmadaWebhook struct {
// state. Before you do it, please confirm that you understand the risks of this configuration.
//
// For supported flags, please see
// https://github.com/karmada-io/karmada/blob/master/cmd/webhook/app/options/options.go
// https://karmada.io/docs/reference/components/karmada-webhook
// for details.
// +optional
ExtraArgs map[string]string `json:"extraArgs,omitempty"`

29
operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go
View File

@ -321,6 +321,11 @@ func (in *KarmadaComponents) DeepCopyInto(out *KarmadaComponents) {
*out = new(KarmadaSearch)
(*in).DeepCopyInto(*out)
}
if in.KarmadaMetricsAdapter != nil {
in, out := &in.KarmadaMetricsAdapter, &out.KarmadaMetricsAdapter
*out = new(KarmadaMetricsAdapter)
(*in).DeepCopyInto(*out)
}
return
}
@ -427,6 +432,30 @@ func (in *KarmadaList) DeepCopyObject() runtime.Object {
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *KarmadaMetricsAdapter) DeepCopyInto(out *KarmadaMetricsAdapter) {
*out = *in
in.CommonSettings.DeepCopyInto(&out.CommonSettings)
if in.ExtraArgs != nil {
in, out := &in.ExtraArgs, &out.ExtraArgs
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KarmadaMetricsAdapter.
func (in *KarmadaMetricsAdapter) DeepCopy() *KarmadaMetricsAdapter {
if in == nil {
return nil
}
out := new(KarmadaMetricsAdapter)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *KarmadaScheduler) DeepCopyInto(out *KarmadaScheduler) {
*out = *in

15
operator/pkg/constants/constants.go
View File

@ -4,6 +4,7 @@ import (
"time"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime/schema"
)
const (
@ -42,6 +43,8 @@ const (
KarmadaWebhook = "karmada-webhook"
// KarmadaDescheduler defines the name of the karmada-descheduler component
KarmadaDescheduler = "karmada-descheduler"
// KarmadaMetricsAdapter defines the name of the karmada-metrics-adapter component
KarmadaMetricsAdapter = "karmada-metrics-adapter"
// KarmadaSystemNamespace defines the leader selection namespace for karmada components
KarmadaSystemNamespace = "karmada-system"
@ -96,12 +99,24 @@ const (
KarmadaWebhookComponent = "KarmadaWebhook"
// KarmadaDeschedulerComponent defines the name of the karmada-descheduler component
KarmadaDeschedulerComponent = "KarmadaDescheduler"
// KarmadaMetricsAdapterComponent defines the name of the karmada-metrics-adapter component
KarmadaMetricsAdapterComponent = "KarmadaMetricsAdapter"
// KarmadaOperatorLabelKeyName defines a label key used by all resources created by karmada operator
KarmadaOperatorLabelKeyName = "app.kubernetes.io/managed-by"
// APIServiceName defines the karmada aggregated apiserver APISerivce resource name.
APIServiceName = "v1alpha1.cluster.karmada.io"
)
var (
// KarmadaOperatorLabel defines the default labels in the resource create by karmada operator
KarmadaOperatorLabel = labels.Set{KarmadaOperatorLabelKeyName: KarmadaOperator}
// KarmadaMetricsAdapterAPIServices defines the GroupVersions of all karmada-metrics-adapter APIServices
KarmadaMetricsAdapterAPIServices = []schema.GroupVersion{
{Group: "metrics.k8s.io", Version: "v1beta1"},
{Group: "custom.metrics.k8s.io", Version: "v1beta1"},
{Group: "custom.metrics.k8s.io", Version: "v1beta2"},
}
)

95
operator/pkg/controlplane/metricsadapter/mainfests.go Normal file
View File

@ -0,0 +1,95 @@
package metricsadapter
const (
// KarmadaMetricsAdapterDeployment is karmada-metrics-adapter deployment manifest
KarmadaMetricsAdapterDeployment = `
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .DeploymentName }}
namespace: {{ .Namespace }}
labels:
karmada-app: karmada-metrics-adapter
app.kubernetes.io/managed-by: karmada-operator
spec:
replicas: {{ .Replicas }}
selector:
matchLabels:
karmada-app: karmada-metrics-adapter
template:
metadata:
labels:
karmada-app: karmada-metrics-adapter
spec:
automountServiceAccountToken: false
tolerations:
- key: node-role.kubernetes.io/master
operator: Exists
containers:
- name: karmada-metrics-adapter
image: {{ .Image }}
imagePullPolicy: IfNotPresent
command:
- /bin/karmada-metrics-adapter
- --kubeconfig=/etc/karmada/config
- --authentication-kubeconfig=/etc/karmada/config
- --authorization-kubeconfig=/etc/karmada/config
- --client-ca-file=/etc/karmada/pki/ca.crt
- --audit-log-path=-
- --audit-log-maxage=0
- --audit-log-maxbackup=0
volumeMounts:
- name: kubeconfig
subPath: config
mountPath: /etc/karmada/config
- name: karmada-cert
mountPath: /etc/karmada/pki
readOnly: true
readinessProbe:
httpGet:
path: /readyz
port: 443
scheme: HTTPS
initialDelaySeconds: 1
failureThreshold: 3
periodSeconds: 3
timeoutSeconds: 15
livenessProbe:
httpGet:
path: /healthz
port: 443
scheme: HTTPS
initialDelaySeconds: 10
failureThreshold: 3
periodSeconds: 10
timeoutSeconds: 15
resources:
requests:
cpu: 100m
volumes:
- name: kubeconfig
secret:
secretName: {{ .KubeconfigSecret }}
- name: karmada-cert
secret:
secretName: {{ .KarmadaCertsSecret }}
`
// KarmadaMetricsAdapterService is karmada-metrics-adapter service manifest
KarmadaMetricsAdapterService = `
apiVersion: v1
kind: Service
metadata:
name: {{ .ServiceName }}
namespace: {{ .Namespace }}
labels:
app.kubernetes.io/managed-by: karmada-operator
spec:
selector:
karmada-app: karmada-metrics-adapter
ports:
- port: 443
protocol: TCP
targetPort: 443
`
)

77
operator/pkg/controlplane/metricsadapter/metricsadapter.go Normal file
View File

@ -0,0 +1,77 @@
package metricsadapter
import (
"fmt"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
kuberuntime "k8s.io/apimachinery/pkg/runtime"
clientset "k8s.io/client-go/kubernetes"
clientsetscheme "k8s.io/client-go/kubernetes/scheme"
operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
"github.com/karmada-io/karmada/operator/pkg/util"
"github.com/karmada-io/karmada/operator/pkg/util/apiclient"
"github.com/karmada-io/karmada/operator/pkg/util/patcher"
)
// EnsureKarmadaMetricAdapter creates karmada-metric-adapter deployment and service resource.
func EnsureKarmadaMetricAdapter(client clientset.Interface, cfg *operatorv1alpha1.KarmadaMetricsAdapter, name, namespace string) error {
if err := installKarmadaMetricAdapter(client, cfg, name, namespace); err != nil {
return err
}
return createKarmadaMetricAdapterService(client, name, namespace)
}
func installKarmadaMetricAdapter(client clientset.Interface, cfg *operatorv1alpha1.KarmadaMetricsAdapter, name, namespace string) error {
metricAdapterBytes, err := util.ParseTemplate(KarmadaMetricsAdapterDeployment, struct {
DeploymentName, Namespace, Image string
KubeconfigSecret, KarmadaCertsSecret string
Replicas *int32
}{
DeploymentName: util.KarmadaMetricsAdapterName(name),
Namespace: namespace,
Image: cfg.Image.Name(),
Replicas: cfg.Replicas,
KubeconfigSecret: util.AdminKubeconfigSecretName(name),
KarmadaCertsSecret: util.KarmadaCertSecretName(name),
})
if err != nil {
return fmt.Errorf("error when parsing KarmadaMetricAdapter Deployment template: %w", err)
}
metricAdapter := &appsv1.Deployment{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), metricAdapterBytes, metricAdapter); err != nil {
return fmt.Errorf("err when decoding KarmadaMetricAdapter Deployment: %w", err)
}
patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(metricAdapter)
if err := apiclient.CreateOrUpdateDeployment(client, metricAdapter); err != nil {
return fmt.Errorf("error when creating deployment for %s, err: %w", metricAdapter.Name, err)
}
return nil
}
func createKarmadaMetricAdapterService(client clientset.Interface, name, namespace string) error {
metricAdapterServiceBytes, err := util.ParseTemplate(KarmadaMetricsAdapterService, struct {
ServiceName, Namespace string
}{
ServiceName: util.KarmadaMetricsAdapterName(name),
Namespace: namespace,
})
if err != nil {
return fmt.Errorf("error when parsing KarmadaMetricAdapter Service template: %w", err)
}
metricAdapterService := &corev1.Service{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), metricAdapterServiceBytes, metricAdapterService); err != nil {
return fmt.Errorf("err when decoding KarmadaMetricAdapter Service: %w", err)
}
if err := apiclient.CreateOrUpdateService(client, metricAdapterService); err != nil {
return fmt.Errorf("err when creating service for %s, err: %w", metricAdapterService.Name, err)
}
return nil
}

65
operator/pkg/karmadaresource/apiservice/apiservice.go
View File

@ -14,6 +14,7 @@ import (
apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
aggregator "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
"github.com/karmada-io/karmada/operator/pkg/constants"
"github.com/karmada-io/karmada/operator/pkg/util"
"github.com/karmada-io/karmada/operator/pkg/util/apiclient"
)
@ -82,3 +83,67 @@ func aggregatedApiserverService(client clientset.Interface, name, namespace stri
}
return nil
}
// EnsureMetricsAdapterAPIService creates APIService and a service for karmada-metrics-adapter
func EnsureMetricsAdapterAPIService(aggregatorClient *aggregator.Clientset, client clientset.Interface, name, namespace string) error {
if err := karmadaMetricsAdapterService(client, name, namespace); err != nil {
return err
}
return karmadaMetricsAdapterAPIService(aggregatorClient, name, namespace)
}
func karmadaMetricsAdapterAPIService(client *aggregator.Clientset, name, namespace string) error {
for _, gv := range constants.KarmadaMetricsAdapterAPIServices {
// The APIService name to metrics adapter is "$version.$group"
apiServiceName := fmt.Sprintf("%s.%s", gv.Version, gv.Group)
apiServiceBytes, err := util.ParseTemplate(KarmadaMetricsAdapterAPIService, struct {
Name, Namespace string
ServiceName, Group, Version string
}{
Name: apiServiceName,
Namespace: namespace,
Group: gv.Group,
Version: gv.Version,
ServiceName: util.KarmadaMetricsAdapterName(name),
})
if err != nil {
return fmt.Errorf("error when parsing KarmadaMetricsAdapter APIService %s template: %w", apiServiceName, err)
}
apiService := &apiregistrationv1.APIService{}
if err := runtime.DecodeInto(codecs.UniversalDecoder(), apiServiceBytes, apiService); err != nil {
return fmt.Errorf("err when decoding KarmadaMetricsAdapter APIService %s: %w", apiServiceName, err)
}
if err := apiclient.CreateOrUpdateAPIService(client, apiService); err != nil {
return err
}
}
return nil
}
func karmadaMetricsAdapterService(client clientset.Interface, name, namespace string) error {
aggregatedApiserverServiceBytes, err := util.ParseTemplate(KarmadaMetricsAdapterService, struct {
Namespace string
ServiceName string
}{
Namespace: namespace,
ServiceName: util.KarmadaMetricsAdapterName(name),
})
if err != nil {
return fmt.Errorf("error when parsing KarmadaMetricsAdapter Service template: %w", err)
}
aggregatedService := &corev1.Service{}
if err := runtime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), aggregatedApiserverServiceBytes, aggregatedService); err != nil {
return fmt.Errorf("err when decoding KarmadaMetricsAdapter Service: %w", err)
}
if err := apiclient.CreateOrUpdateService(client, aggregatedService); err != nil {
return err
}
return nil
}

30
operator/pkg/karmadaresource/apiservice/manifest.go
View File

@ -20,10 +20,40 @@ spec:
version: v1alpha1
versionPriority: 10
`
// KarmadaAggregatedApiserverService is karmada aggregated apiserver service manifest
KarmadaAggregatedApiserverService = `
apiVersion: v1
kind: Service
metadata:
name: {{ .ServiceName }}
namespace: {{ .Namespace }}
spec:
type: ExternalName
externalName: {{ .ServiceName }}.{{ .Namespace }}.svc
`
// KarmadaMetricsAdapterAPIService is karmada-metrics-adapter APIService manifest
KarmadaMetricsAdapterAPIService = `
apiVersion: apiregistration.k8s.io/v1
kind: APIService
metadata:
name: {{ .Name }}
spec:
service:
name: {{ .ServiceName }}
namespace: {{ .Namespace }}
group: {{ .Group }}
version: {{ .Version }}
insecureSkipTLSVerify: true
groupPriorityMinimum: 100
versionPriority: 200
`
// KarmadaMetricsAdapterService is karmada-metrics-adapter service manifest
KarmadaMetricsAdapterService = `
apiVersion: v1
kind: Service
metadata:
name: {{ .ServiceName }}
namespace: {{ .Namespace }}

110
operator/pkg/tasks/init/component.go
View File

@ -3,12 +3,16 @@ package tasks
import (
"errors"
"fmt"
"time"
"k8s.io/klog/v2"
"github.com/karmada-io/karmada/operator/pkg/constants"
"github.com/karmada-io/karmada/operator/pkg/controlplane"
"github.com/karmada-io/karmada/operator/pkg/controlplane/metricsadapter"
"github.com/karmada-io/karmada/operator/pkg/controlplane/webhook"
"github.com/karmada-io/karmada/operator/pkg/karmadaresource/apiservice"
"github.com/karmada-io/karmada/operator/pkg/util/apiclient"
"github.com/karmada-io/karmada/operator/pkg/workflow"
)
@ -27,6 +31,7 @@ func NewComponentTask() workflow.Task {
Run: runKarmadaWebhook,
},
newComponentSubTask(constants.KarmadaDeschedulerComponent),
newKarmadaMetricsAdapterSubTask(),
},
}
}
@ -75,7 +80,7 @@ func runComponentSubTask(component string) func(r workflow.RunData) error {
func runKarmadaWebhook(r workflow.RunData) error {
data, ok := r.(InitData)
if !ok {
return errors.New("certs task invoked with an invalid data struct")
return errors.New("KarmadaWebhook task invoked with an invalid data struct")
}
cfg := data.Components()
@ -97,3 +102,106 @@ func runKarmadaWebhook(r workflow.RunData) error {
klog.V(2).InfoS("[KarmadaWebhook] Successfully applied karmada webhook component", "karmada", klog.KObj(data))
return nil
}
func newKarmadaMetricsAdapterSubTask() workflow.Task {
return workflow.Task{
Name: constants.KarmadaMetricsAdapterComponent,
Run: runKarmadaMetricsAdapter,
RunSubTasks: true,
Tasks: []workflow.Task{
{
Name: "DeployMetricAdapter",
Run: runDeployMetricAdapter,
},
{
Name: "DeployMetricAdapterAPIService",
Run: runDeployMetricAdapterAPIService,
},
},
}
}
func runKarmadaMetricsAdapter(r workflow.RunData) error {
data, ok := r.(InitData)
if !ok {
return errors.New("karmadaMetricsAdapter task invoked with an invalid data struct")
}
klog.V(4).InfoS("[karmadaMetricsAdapter] Running karmadaMetricsAdapter task", "karmada", klog.KObj(data))
return nil
}
func runDeployMetricAdapter(r workflow.RunData) error {
data, ok := r.(InitData)
if !ok {
return errors.New("DeployMetricAdapter task invoked with an invalid data struct")
}
cfg := data.Components()
if cfg.KarmadaMetricsAdapter == nil {
klog.V(2).InfoS("[karmadaMetricsAdapter] Skip install karmada-metrics-adapter component")
return nil
}
err := metricsadapter.EnsureKarmadaMetricAdapter(
data.RemoteClient(),
cfg.KarmadaMetricsAdapter,
data.GetName(),
data.GetNamespace(),
)
if err != nil {
return fmt.Errorf("failed to apply karmada-metrics-adapter, err: %w", err)
}
klog.V(2).InfoS("[DeployMetricAdapter] Successfully applied karmada-metrics-adapter component", "karmada", klog.KObj(data))
if *cfg.KarmadaMetricsAdapter.Replicas != 0 {
waiter := apiclient.NewKarmadaWaiter(data.ControlplaneConfig(), data.RemoteClient(), time.Second*30)
if err = waiter.WaitForSomePods(karmadaMetricAdapterLabels.String(), data.GetNamespace(), 1); err != nil {
return fmt.Errorf("waiting for karmada-metrics-adapter to ready timeout, err: %w", err)
}
klog.V(2).InfoS("[DeployMetricAdapter] the karmada-metrics-adapter is ready", "karmada", klog.KObj(data))
}
return nil
}
func runDeployMetricAdapterAPIService(r workflow.RunData) error {
data, ok := r.(InitData)
if !ok {
return errors.New("DeployMetricAdapterAPIService task invoked with an invalid data struct")
}
cfg := data.Components()
if cfg.KarmadaMetricsAdapter == nil {
klog.V(2).InfoS("[karmadaMetricsAdapter] Skip install karmada-metrics-adapter APIService")
return nil
}
config := data.ControlplaneConfig()
client, err := apiclient.NewAPIRegistrationClient(config)
if err != nil {
return err
}
err = apiservice.EnsureMetricsAdapterAPIService(client, data.KarmadaClient(), data.GetName(), data.GetNamespace())
if err != nil {
return fmt.Errorf("failed to apply karmada-metrics-adapter APIService resource to karmada controlplane, err: %w", err)
}
if *cfg.KarmadaMetricsAdapter.Replicas != 0 {
waiter := apiclient.NewKarmadaWaiter(config, nil, time.Second*20)
for _, gv := range constants.KarmadaMetricsAdapterAPIServices {
apiServiceName := fmt.Sprintf("%s.%s", gv.Version, gv.Group)
if err := waiter.WaitForAPIService(apiServiceName); err != nil {
return fmt.Errorf("the APIService %s is unhealthy, err: %w", apiServiceName, err)
}
}
klog.V(2).InfoS("[DeployMetricAdapterAPIService] all karmada-metrics-adapter APIServices status is ready ", "karmada", klog.KObj(data))
}
return nil
}

2
operator/pkg/tasks/init/karmadaresource.go
View File

@ -200,7 +200,7 @@ func runAPIService(r workflow.RunData) error {
}
waiter := apiclient.NewKarmadaWaiter(config, nil, componentBeReadyTimeout)
if err := apiclient.TryRunCommand(waiter.WaitForAPIService, 3); err != nil {
if err := waiter.WaitForAPIService(constants.APIServiceName); err != nil {
return fmt.Errorf("the APIService is unhealthy, err: %w", err)
}

1
operator/pkg/tasks/init/wait.go
View File

@ -25,6 +25,7 @@ var (
karmadaControllerManagerLabels = labels.Set{"karmada-app": constants.KarmadaControllerManager}
karmadaSchedulerLabels = labels.Set{"karmada-app": constants.KarmadaScheduler}
karmadaWebhookLabels = labels.Set{"karmada-app": constants.KarmadaWebhook}
karmadaMetricAdapterLabels = labels.Set{"karmada-app": constants.KarmadaMetricsAdapter}
)
// NewCheckApiserverHealthTask init wait-apiserver task

9
operator/pkg/util/apiclient/wait.go
View File

@ -18,9 +18,6 @@ import (
const (
// APICallRetryInterval defines how long kubeadm should wait before retrying a failed API operation
APICallRetryInterval = 500 * time.Millisecond
// APIServiceName defines the karmada aggregated apiserver APISerivce resource name.
APIServiceName = "v1alpha1.cluster.karmada.io"
)
// Waiter is an interface for waiting for criteria in Karmada to happen
@ -28,7 +25,7 @@ type Waiter interface {
// WaitForAPI waits for the API Server's /healthz endpoint to become "ok"
WaitForAPI() error
// WaitForAPIService waits for the APIService condition to become "true"
WaitForAPIService() error
WaitForAPIService(name string) error
// WaitForPods waits for Pods in the namespace to become Ready
WaitForPods(label, namespace string) error
// WaitForSomePods waits for the specified number of Pods in the namespace to become Ready
@ -67,14 +64,14 @@ func (w *KarmadaWaiter) WaitForAPI() error {
}
// WaitForAPIService waits for the APIService condition to become "true"
func (w *KarmadaWaiter) WaitForAPIService() error {
func (w *KarmadaWaiter) WaitForAPIService(name string) error {
aggregateClient, err := aggregator.NewForConfig(w.karmadaConfig)
if err != nil {
return err
}
err = wait.PollImmediate(APICallRetryInterval, w.timeout, func() (done bool, err error) {
apiService, err := aggregateClient.ApiregistrationV1().APIServices().Get(context.TODO(), APIServiceName, metav1.GetOptions{})
apiService, err := aggregateClient.ApiregistrationV1().APIServices().Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
return false, nil
}

5
operator/pkg/util/naming.go
View File

@ -73,6 +73,11 @@ func KarmadaDeschedulerName(karmada string) string {
return generateResourceName(karmada, "descheduler")
}
// KarmadaMetricsAdapterName returns name of karmada-metric-adapter
func KarmadaMetricsAdapterName(karmada string) string {
return generateResourceName(karmada, "metrics-adapter")
}
func generateResourceName(karmada, suffix string) string {
if strings.Contains(karmada, "karmada") {
return fmt.Sprintf("%s-%s", karmada, suffix)