From c3429f940fa005e1e4a4df09e727d56d45fda559 Mon Sep 17 00:00:00 2001 From: calvin Date: Wed, 29 Mar 2023 14:43:32 +0800 Subject: [PATCH] karmada operator: support etcd high availability Signed-off-by: calvin --- .../crds/operator.karmada.io_karmadas.yaml | 74 +++++++++++++++++++ operator/config/deploy/karmada-operator.yaml | 33 +++++++++ operator/config/samples/karmada.yaml | 1 + operator/pkg/apis/operator/v1alpha1/type.go | 4 +- .../v1alpha1/zz_generated.deepcopy.go | 2 +- operator/pkg/controlplane/etcd/etcd.go | 31 ++++++-- operator/pkg/controlplane/etcd/mainfests.go | 12 ++- operator/pkg/init.go | 17 ++++- 8 files changed, 160 insertions(+), 14 deletions(-) create mode 100644 operator/config/deploy/karmada-operator.yaml diff --git a/operator/config/crds/operator.karmada.io_karmadas.yaml b/operator/config/crds/operator.karmada.io_karmadas.yaml index 8eb2b5743..f2f195807 100644 --- a/operator/config/crds/operator.karmada.io_karmadas.yaml +++ b/operator/config/crds/operator.karmada.io_karmadas.yaml @@ -191,6 +191,15 @@ spec: the built-in etcd instance Local and External are mutually exclusive properties: + annotations: + additionalProperties: + type: string + description: 'Annotations is an unstructured key value + map stored with a resource that may be set by external + tools to store and retrieve arbitrary metadata. They + are not queryable and should be preserved when modifying + objects. More info: http://kubernetes.io/docs/user-guide/annotations' + type: object imageRepository: description: ImageRepository sets the container registry to pull images from. if not set, the ImageRepository @@ -202,12 +211,77 @@ spec: change automatically the version of the above components during upgrades. type: string + labels: + additionalProperties: + type: string + description: 'Map of string keys and values that can be + used to organize and categorize (scope and select) objects. + May match selectors of replication controllers and services. + More info: http://kubernetes.io/docs/user-guide/labels' + type: object peerCertSANs: description: PeerCertSANs sets extra Subject Alternative Names for the etcd peer signing cert. items: type: string type: array + replicas: + description: Number of desired pods. This is a pointer + to distinguish between explicit zero and not specified. + Defaults to 1. + format: int32 + type: integer + resources: + description: 'Compute Resources required by this component. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + properties: + claims: + description: "Claims lists the names of resources, + defined in spec.resourceClaims, that are used by + this container. \n This is an alpha field and requires + enabling the DynamicResourceAllocation feature gate. + \n This field is immutable." + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one + entry in pod.spec.resourceClaims of the Pod + where this field is used. It makes that resource + available inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is omitted + for a container, it defaults to Limits if that is + explicitly specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object serverCertSANs: description: ServerCertSANs sets extra Subject Alternative Names for the etcd server signing cert. diff --git a/operator/config/deploy/karmada-operator.yaml b/operator/config/deploy/karmada-operator.yaml new file mode 100644 index 000000000..2f4a5c8f0 --- /dev/null +++ b/operator/config/deploy/karmada-operator.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: karmada-operator + namespace: karmada-system + labels: + karmada-app: karmada-operator +spec: + replicas: 1 + selector: + matchLabels: + karmada-app: karmada-operator + template: + metadata: + labels: + karmada-app: karmada-operator + spec: + containers: + - name: karmada-operator + image: docker.io/karmada/karmada-operator:latest + imagePullPolicy: IfNotPresent + command: + - /bin/karmada-operator + - --kubeconfig=/etc/config + - --v=4 + volumeMounts: + - name: kubeconfig + mountPath: /etc/config + subPath: config + volumes: + - name: kubeconfig + secret: + secretName: my-kubeconfig diff --git a/operator/config/samples/karmada.yaml b/operator/config/samples/karmada.yaml index 0e0742131..b86de4756 100644 --- a/operator/config/samples/karmada.yaml +++ b/operator/config/samples/karmada.yaml @@ -9,6 +9,7 @@ spec: local: imageRepository: registry.k8s.io/etcd imageTag: 3.5.3-0 + replicas: 1 volumeData: emptyDir: {} karmadaAPIServer: diff --git a/operator/pkg/apis/operator/v1alpha1/type.go b/operator/pkg/apis/operator/v1alpha1/type.go index 8c1e83cff..32c940d1b 100644 --- a/operator/pkg/apis/operator/v1alpha1/type.go +++ b/operator/pkg/apis/operator/v1alpha1/type.go @@ -147,8 +147,8 @@ type Etcd struct { // LocalEtcd describes that operator should run an etcd cluster in a host cluster. type LocalEtcd struct { - // ImageMeta allows to customize the container used for etcd - Image `json:",inline"` + // CommonSettings holds common settings to etcd. + CommonSettings `json:",inline"` // VolumeData describes the settings of etcd data store. // We will support 3 modes: emtydir, hostPath, PVC. default by hostPath. diff --git a/operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go b/operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go index b4d66e3f8..7c3ecbb18 100644 --- a/operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go @@ -611,7 +611,7 @@ func (in *KubeControllerManager) DeepCopy() *KubeControllerManager { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LocalEtcd) DeepCopyInto(out *LocalEtcd) { *out = *in - out.Image = in.Image + in.CommonSettings.DeepCopyInto(&out.CommonSettings) if in.VolumeData != nil { in, out := &in.VolumeData, &out.VolumeData *out = new(VolumeData) diff --git a/operator/pkg/controlplane/etcd/etcd.go b/operator/pkg/controlplane/etcd/etcd.go index 14dcf3731..f073db741 100644 --- a/operator/pkg/controlplane/etcd/etcd.go +++ b/operator/pkg/controlplane/etcd/etcd.go @@ -2,13 +2,13 @@ package etcd import ( "fmt" + "strings" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" kuberuntime "k8s.io/apimachinery/pkg/runtime" clientset "k8s.io/client-go/kubernetes" clientsetscheme "k8s.io/client-go/kubernetes/scheme" - "k8s.io/utils/pointer" operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1" "github.com/karmada-io/karmada/operator/pkg/constants" @@ -25,11 +25,29 @@ func EnsureKarmadaEtcd(client clientset.Interface, cfg *operatorv1alpha1.LocalEt } func installKarmadaEtcd(client clientset.Interface, name, namespace string, cfg *operatorv1alpha1.LocalEtcd) error { + // if the number of etcd is greater than one, we need to concatenate the peerURL for each member cluster. + // memberName is podName generated by etcd statefuleset: ${statefulsetName}-index + // memberPeerURL uses the etcd peer headless service name: ${podName}.${serviceName}.${namespace}.svc.cluster.local:2380 + initialClusters := make([]string, *cfg.Replicas) + for index := range initialClusters { + memberName := fmt.Sprintf("%s-%d", util.KarmadaEtcdName(name), index) + + // build etcd member cluster peer url + memberPeerURL := fmt.Sprintf("http://%s.%s.%s.svc.cluster.local:%v", + memberName, + util.KarmadaEtcdName(name), + namespace, + constants.EtcdListenPeerPort, + ) + + initialClusters[index] = fmt.Sprintf("%s=%s", memberName, memberPeerURL) + } + etcdStatefuleSetBytes, err := util.ParseTemplate(KarmadaEtcdStatefulSet, struct { - StatefulSetName, Namespace, Image string - EtcdClientService, CertsSecretName, EtcdPeerServiceName string - Replicas *int32 - EtcdListenClientPort, EtcdListenPeerPort int32 + StatefulSetName, Namespace, Image string + EtcdClientService, CertsSecretName string + EtcdPeerServiceName, InitialCluster string + Replicas, EtcdListenClientPort, EtcdListenPeerPort int32 }{ StatefulSetName: util.KarmadaEtcdName(name), Namespace: namespace, @@ -37,7 +55,8 @@ func installKarmadaEtcd(client clientset.Interface, name, namespace string, cfg EtcdClientService: util.KarmadaEtcdClientName(name), CertsSecretName: util.EtcdCertSecretName(name), EtcdPeerServiceName: util.KarmadaEtcdName(name), - Replicas: pointer.Int32(1), + InitialCluster: strings.Join(initialClusters, ","), + Replicas: *cfg.Replicas, EtcdListenClientPort: constants.EtcdListenClientPort, EtcdListenPeerPort: constants.EtcdListenPeerPort, }) diff --git a/operator/pkg/controlplane/etcd/mainfests.go b/operator/pkg/controlplane/etcd/mainfests.go index 414de35a4..7eea3dbde 100644 --- a/operator/pkg/controlplane/etcd/mainfests.go +++ b/operator/pkg/controlplane/etcd/mainfests.go @@ -32,11 +32,11 @@ spec: imagePullPolicy: IfNotPresent command: - /usr/local/bin/etcd - - --name={{ .StatefulSetName }}0 + - --name=$(KARMADA_ETCD_NAME) - --listen-client-urls= https://0.0.0.0:{{ .EtcdListenClientPort }} - --listen-peer-urls=http://0.0.0.0:{{ .EtcdListenPeerPort }} - --advertise-client-urls=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenClientPort }} - - --initial-cluster={{ .StatefulSetName }}0=http://{{ .StatefulSetName }}-0.{{ .EtcdPeerServiceName }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenPeerPort }} + - --initial-cluster={{ .InitialCluster }} - --initial-cluster-state=new - --client-cert-auth=true - --trusted-ca-file=/etc/karmada/pki/etcd/etcd-ca.crt @@ -45,6 +45,12 @@ spec: - --data-dir=/var/lib/etcd - --snapshot-count=10000 - --log-level=debug + env: + - name: KARMADA_ETCD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name livenessProbe: exec: command: @@ -118,6 +124,8 @@ spec: port: {{ .EtcdListenPeerPort }} protocol: TCP targetPort: {{ .EtcdListenPeerPort }} + selector: + karmada-app: etcd type: ClusterIP ` ) diff --git a/operator/pkg/init.go b/operator/pkg/init.go index 07a290707..9bab4c7f3 100644 --- a/operator/pkg/init.go +++ b/operator/pkg/init.go @@ -108,6 +108,13 @@ func NewInitJob(opt *InitOptions) *workflow.Job { } } + if opt.Components.Etcd.Local != nil && opt.Components.Etcd.Local.CommonSettings.Replicas != nil { + replicas := *opt.Components.Etcd.Local.CommonSettings.Replicas + if (replicas % 2) == 0 { + klog.Warningf("invalid etcd replicas %d, expected an odd number", replicas) + } + } + // TODO: Verify whether important values of initData is valid return &initData{ @@ -213,10 +220,14 @@ func defaultComponents() *operatorv1alpha1.KarmadaComponents { return &operatorv1alpha1.KarmadaComponents{ Etcd: &operatorv1alpha1.Etcd{ Local: &operatorv1alpha1.LocalEtcd{ - Image: operatorv1alpha1.Image{ - ImageRepository: fmt.Sprintf("%s/%s", constants.KubeDefaultRepository, constants.Etcd), - ImageTag: constants.EtcdDefaultVersion, + CommonSettings: operatorv1alpha1.CommonSettings{ + Image: operatorv1alpha1.Image{ + ImageRepository: fmt.Sprintf("%s/%s", constants.KubeDefaultRepository, constants.Etcd), + ImageTag: constants.EtcdDefaultVersion, + }, + Replicas: pointer.Int32(1), }, + VolumeData: &operatorv1alpha1.VolumeData{ EmptyDir: &corev1.EmptyDirVolumeSource{}, },