karmada operator: support etcd high availability
Signed-off-by: calvin <wen.chen@daocloud.io>
This commit is contained in:
calvin
calvin0327
parent
35d6638263
commit
c3429f940f
|
|
@ -191,6 +191,15 @@ spec:
|
|||
the built-in etcd instance Local and External are mutually
|
||||
exclusive
|
||||
properties:
|
||||
annotations:
|
||||
additionalProperties:
|
||||
type: string
|
||||
description: 'Annotations is an unstructured key value
|
||||
map stored with a resource that may be set by external
|
||||
tools to store and retrieve arbitrary metadata. They
|
||||
are not queryable and should be preserved when modifying
|
||||
objects. More info: http://kubernetes.io/docs/user-guide/annotations'
|
||||
type: object
|
||||
imageRepository:
|
||||
description: ImageRepository sets the container registry
|
||||
to pull images from. if not set, the ImageRepository
|
||||
|
|
@ -202,12 +211,77 @@ spec:
|
|||
change automatically the version of the above components
|
||||
during upgrades.
|
||||
type: string
|
||||
labels:
|
||||
additionalProperties:
|
||||
type: string
|
||||
description: 'Map of string keys and values that can be
|
||||
used to organize and categorize (scope and select) objects.
|
||||
May match selectors of replication controllers and services.
|
||||
More info: http://kubernetes.io/docs/user-guide/labels'
|
||||
type: object
|
||||
peerCertSANs:
|
||||
description: PeerCertSANs sets extra Subject Alternative
|
||||
Names for the etcd peer signing cert.
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
replicas:
|
||||
description: Number of desired pods. This is a pointer
|
||||
to distinguish between explicit zero and not specified.
|
||||
Defaults to 1.
|
||||
format: int32
|
||||
type: integer
|
||||
resources:
|
||||
description: 'Compute Resources required by this component.
|
||||
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
|
||||
properties:
|
||||
claims:
|
||||
description: "Claims lists the names of resources,
|
||||
defined in spec.resourceClaims, that are used by
|
||||
this container. \n This is an alpha field and requires
|
||||
enabling the DynamicResourceAllocation feature gate.
|
||||
\n This field is immutable."
|
||||
items:
|
||||
description: ResourceClaim references one entry
|
||||
in PodSpec.ResourceClaims.
|
||||
properties:
|
||||
name:
|
||||
description: Name must match the name of one
|
||||
entry in pod.spec.resourceClaims of the Pod
|
||||
where this field is used. It makes that resource
|
||||
available inside a container.
|
||||
type: string
|
||||
required:
|
||||
- name
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-map-keys:
|
||||
- name
|
||||
x-kubernetes-list-type: map
|
||||
limits:
|
||||
additionalProperties:
|
||||
anyOf:
|
||||
- type: integer
|
||||
- type: string
|
||||
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
|
||||
x-kubernetes-int-or-string: true
|
||||
description: 'Limits describes the maximum amount
|
||||
of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
|
||||
type: object
|
||||
requests:
|
||||
additionalProperties:
|
||||
anyOf:
|
||||
- type: integer
|
||||
- type: string
|
||||
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
|
||||
x-kubernetes-int-or-string: true
|
||||
description: 'Requests describes the minimum amount
|
||||
of compute resources required. If Requests is omitted
|
||||
for a container, it defaults to Limits if that is
|
||||
explicitly specified, otherwise to an implementation-defined
|
||||
value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
|
||||
type: object
|
||||
type: object
|
||||
serverCertSANs:
|
||||
description: ServerCertSANs sets extra Subject Alternative
|
||||
Names for the etcd server signing cert.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: karmada-operator
|
||||
namespace: karmada-system
|
||||
labels:
|
||||
karmada-app: karmada-operator
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
karmada-app: karmada-operator
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
karmada-app: karmada-operator
|
||||
spec:
|
||||
containers:
|
||||
- name: karmada-operator
|
||||
image: docker.io/karmada/karmada-operator:latest
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/karmada-operator
|
||||
- --kubeconfig=/etc/config
|
||||
- --v=4
|
||||
volumeMounts:
|
||||
- name: kubeconfig
|
||||
mountPath: /etc/config
|
||||
subPath: config
|
||||
volumes:
|
||||
- name: kubeconfig
|
||||
secret:
|
||||
secretName: my-kubeconfig
|
||||
|
|
@ -9,6 +9,7 @@ spec:
|
|||
local:
|
||||
imageRepository: registry.k8s.io/etcd
|
||||
imageTag: 3.5.3-0
|
||||
replicas: 1
|
||||
volumeData:
|
||||
emptyDir: {}
|
||||
karmadaAPIServer:
|
||||
|
|
|
|||
|
|
@ -147,8 +147,8 @@ type Etcd struct {
|
|||
|
||||
// LocalEtcd describes that operator should run an etcd cluster in a host cluster.
|
||||
type LocalEtcd struct {
|
||||
// ImageMeta allows to customize the container used for etcd
|
||||
Image `json:",inline"`
|
||||
// CommonSettings holds common settings to etcd.
|
||||
CommonSettings `json:",inline"`
|
||||
|
||||
// VolumeData describes the settings of etcd data store.
|
||||
// We will support 3 modes: emtydir, hostPath, PVC. default by hostPath.
|
||||
|
|
|
|||
|
|
@ -611,7 +611,7 @@ func (in *KubeControllerManager) DeepCopy() *KubeControllerManager {
|
|||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *LocalEtcd) DeepCopyInto(out *LocalEtcd) {
|
||||
*out = *in
|
||||
out.Image = in.Image
|
||||
in.CommonSettings.DeepCopyInto(&out.CommonSettings)
|
||||
if in.VolumeData != nil {
|
||||
in, out := &in.VolumeData, &out.VolumeData
|
||||
*out = new(VolumeData)
|
||||
|
|
|
|||
|
|
@ -2,13 +2,13 @@ package etcd
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
kuberuntime "k8s.io/apimachinery/pkg/runtime"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
clientsetscheme "k8s.io/client-go/kubernetes/scheme"
|
||||
"k8s.io/utils/pointer"
|
||||
|
||||
operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
|
||||
"github.com/karmada-io/karmada/operator/pkg/constants"
|
||||
|
|
@ -25,11 +25,29 @@ func EnsureKarmadaEtcd(client clientset.Interface, cfg *operatorv1alpha1.LocalEt
|
|||
}
|
||||
|
||||
func installKarmadaEtcd(client clientset.Interface, name, namespace string, cfg *operatorv1alpha1.LocalEtcd) error {
|
||||
// if the number of etcd is greater than one, we need to concatenate the peerURL for each member cluster.
|
||||
// memberName is podName generated by etcd statefuleset: ${statefulsetName}-index
|
||||
// memberPeerURL uses the etcd peer headless service name: ${podName}.${serviceName}.${namespace}.svc.cluster.local:2380
|
||||
initialClusters := make([]string, *cfg.Replicas)
|
||||
for index := range initialClusters {
|
||||
memberName := fmt.Sprintf("%s-%d", util.KarmadaEtcdName(name), index)
|
||||
|
||||
// build etcd member cluster peer url
|
||||
memberPeerURL := fmt.Sprintf("http://%s.%s.%s.svc.cluster.local:%v",
|
||||
memberName,
|
||||
util.KarmadaEtcdName(name),
|
||||
namespace,
|
||||
constants.EtcdListenPeerPort,
|
||||
)
|
||||
|
||||
initialClusters[index] = fmt.Sprintf("%s=%s", memberName, memberPeerURL)
|
||||
}
|
||||
|
||||
etcdStatefuleSetBytes, err := util.ParseTemplate(KarmadaEtcdStatefulSet, struct {
|
||||
StatefulSetName, Namespace, Image string
|
||||
EtcdClientService, CertsSecretName, EtcdPeerServiceName string
|
||||
Replicas *int32
|
||||
EtcdListenClientPort, EtcdListenPeerPort int32
|
||||
StatefulSetName, Namespace, Image string
|
||||
EtcdClientService, CertsSecretName string
|
||||
EtcdPeerServiceName, InitialCluster string
|
||||
Replicas, EtcdListenClientPort, EtcdListenPeerPort int32
|
||||
}{
|
||||
StatefulSetName: util.KarmadaEtcdName(name),
|
||||
Namespace: namespace,
|
||||
|
|
@ -37,7 +55,8 @@ func installKarmadaEtcd(client clientset.Interface, name, namespace string, cfg
|
|||
EtcdClientService: util.KarmadaEtcdClientName(name),
|
||||
CertsSecretName: util.EtcdCertSecretName(name),
|
||||
EtcdPeerServiceName: util.KarmadaEtcdName(name),
|
||||
Replicas: pointer.Int32(1),
|
||||
InitialCluster: strings.Join(initialClusters, ","),
|
||||
Replicas: *cfg.Replicas,
|
||||
EtcdListenClientPort: constants.EtcdListenClientPort,
|
||||
EtcdListenPeerPort: constants.EtcdListenPeerPort,
|
||||
})
|
||||
|
|
|
|||
|
|
@ -32,11 +32,11 @@ spec:
|
|||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /usr/local/bin/etcd
|
||||
- --name={{ .StatefulSetName }}0
|
||||
- --name=$(KARMADA_ETCD_NAME)
|
||||
- --listen-client-urls= https://0.0.0.0:{{ .EtcdListenClientPort }}
|
||||
- --listen-peer-urls=http://0.0.0.0:{{ .EtcdListenPeerPort }}
|
||||
- --advertise-client-urls=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenClientPort }}
|
||||
- --initial-cluster={{ .StatefulSetName }}0=http://{{ .StatefulSetName }}-0.{{ .EtcdPeerServiceName }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenPeerPort }}
|
||||
- --initial-cluster={{ .InitialCluster }}
|
||||
- --initial-cluster-state=new
|
||||
- --client-cert-auth=true
|
||||
- --trusted-ca-file=/etc/karmada/pki/etcd/etcd-ca.crt
|
||||
|
|
@ -45,6 +45,12 @@ spec:
|
|||
- --data-dir=/var/lib/etcd
|
||||
- --snapshot-count=10000
|
||||
- --log-level=debug
|
||||
env:
|
||||
- name: KARMADA_ETCD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: metadata.name
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
|
|
@ -118,6 +124,8 @@ spec:
|
|||
port: {{ .EtcdListenPeerPort }}
|
||||
protocol: TCP
|
||||
targetPort: {{ .EtcdListenPeerPort }}
|
||||
selector:
|
||||
karmada-app: etcd
|
||||
type: ClusterIP
|
||||
`
|
||||
)
|
||||
|
|
|
|||
|
|
@ -108,6 +108,13 @@ func NewInitJob(opt *InitOptions) *workflow.Job {
|
|||
}
|
||||
}
|
||||
|
||||
if opt.Components.Etcd.Local != nil && opt.Components.Etcd.Local.CommonSettings.Replicas != nil {
|
||||
replicas := *opt.Components.Etcd.Local.CommonSettings.Replicas
|
||||
if (replicas % 2) == 0 {
|
||||
klog.Warningf("invalid etcd replicas %d, expected an odd number", replicas)
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Verify whether important values of initData is valid
|
||||
|
||||
return &initData{
|
||||
|
|
@ -213,10 +220,14 @@ func defaultComponents() *operatorv1alpha1.KarmadaComponents {
|
|||
return &operatorv1alpha1.KarmadaComponents{
|
||||
Etcd: &operatorv1alpha1.Etcd{
|
||||
Local: &operatorv1alpha1.LocalEtcd{
|
||||
Image: operatorv1alpha1.Image{
|
||||
ImageRepository: fmt.Sprintf("%s/%s", constants.KubeDefaultRepository, constants.Etcd),
|
||||
ImageTag: constants.EtcdDefaultVersion,
|
||||
CommonSettings: operatorv1alpha1.CommonSettings{
|
||||
Image: operatorv1alpha1.Image{
|
||||
ImageRepository: fmt.Sprintf("%s/%s", constants.KubeDefaultRepository, constants.Etcd),
|
||||
ImageTag: constants.EtcdDefaultVersion,
|
||||
},
|
||||
Replicas: pointer.Int32(1),
|
||||
},
|
||||
|
||||
VolumeData: &operatorv1alpha1.VolumeData{
|
||||
EmptyDir: &corev1.EmptyDirVolumeSource{},
|
||||
},
|
||||
|
|
|
|||
Loading…
Reference in New Issue