From 8fa6be1b034182cd23ea73b9585d9eeaee80bcdc Mon Sep 17 00:00:00 2001 From: Otto Sulin Date: Tue, 2 Mar 2021 15:20:11 +0200 Subject: [PATCH] Add etcd-manager discoveryPollInterval option --- k8s/crds/kops.k8s.io_clusters.yaml | 4 + pkg/apis/kops/cluster.go | 2 + pkg/apis/kops/v1alpha2/cluster.go | 2 + .../kops/v1alpha2/zz_generated.conversion.go | 2 + .../kops/v1alpha2/zz_generated.deepcopy.go | 5 + pkg/apis/kops/zz_generated.deepcopy.go | 5 + pkg/model/components/etcdmanager/model.go | 29 +-- .../components/etcdmanager/model_test.go | 1 + .../tests/pollinterval/cluster.yaml | 89 +++++++++ .../etcdmanager/tests/pollinterval/tasks.yaml | 189 ++++++++++++++++++ 10 files changed, 316 insertions(+), 12 deletions(-) create mode 100644 pkg/model/components/etcdmanager/tests/pollinterval/cluster.yaml create mode 100644 pkg/model/components/etcdmanager/tests/pollinterval/tasks.yaml diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index d83fddd454..5d5a3b85de 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -930,6 +930,10 @@ spec: manager: description: Manager describes the manager configuration properties: + discoveryPollInterval: + description: DiscoveryPollInterval which is used for discovering + other cluster members. The default is 60 seconds. + type: string env: description: Env allows users to pass in env variables to the etcd-manager container. Variables starting with ETCD_ diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index 4d8a8ea570..464ee02252 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -532,6 +532,8 @@ type EtcdManagerSpec struct { // This allows etcd setting to be overwriten. No config validation is done. // A list of etcd config ENV vars can be found at https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/configuration.md Env []EnvVar `json:"env,omitempty"` + // DiscoveryPollInterval which is used for discovering other cluster members. The default is 60 seconds. + DiscoveryPollInterval *string `json:"discoveryPollInterval,omitempty"` // LogLevel allows the klog library verbose log level to be set for etcd-manager. The default is 6. // https://github.com/google/glog#verbose-logging LogLevel *int32 `json:"logLevel,omitempty"` diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index 89a6e00317..bef0512f5a 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -529,6 +529,8 @@ type EtcdManagerSpec struct { // This allows etcd setting to be configured/overwriten. No config validation is done. // A list of etcd config ENV vars can be found at https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/configuration.md Env []EnvVar `json:"env,omitempty"` + // DiscoveryPollInterval which is used for discovering other cluster members. The default is 60 seconds. + DiscoveryPollInterval *string `json:"discoveryPollInterval,omitempty"` // LogLevel allows the klog library verbose log level to be set for etcd-manager. The default is 6. // https://github.com/google/glog#verbose-logging LogLevel *int32 `json:"logLevel,omitempty"` diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 9c165e45f3..5598490765 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -3303,6 +3303,7 @@ func autoConvert_v1alpha2_EtcdManagerSpec_To_kops_EtcdManagerSpec(in *EtcdManage } else { out.Env = nil } + out.DiscoveryPollInterval = in.DiscoveryPollInterval out.LogLevel = in.LogLevel return nil } @@ -3325,6 +3326,7 @@ func autoConvert_kops_EtcdManagerSpec_To_v1alpha2_EtcdManagerSpec(in *kops.EtcdM } else { out.Env = nil } + out.DiscoveryPollInterval = in.DiscoveryPollInterval out.LogLevel = in.LogLevel return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index bc5d769979..49318be853 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -1523,6 +1523,11 @@ func (in *EtcdManagerSpec) DeepCopyInto(out *EtcdManagerSpec) { *out = make([]EnvVar, len(*in)) copy(*out, *in) } + if in.DiscoveryPollInterval != nil { + in, out := &in.DiscoveryPollInterval, &out.DiscoveryPollInterval + *out = new(string) + **out = **in + } if in.LogLevel != nil { in, out := &in.LogLevel, &out.LogLevel *out = new(int32) diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 6a8a8d6c8c..e49e30b4fe 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -1673,6 +1673,11 @@ func (in *EtcdManagerSpec) DeepCopyInto(out *EtcdManagerSpec) { *out = make([]EnvVar, len(*in)) copy(*out, *in) } + if in.DiscoveryPollInterval != nil { + in, out := &in.DiscoveryPollInterval, &out.DiscoveryPollInterval + *out = new(string) + **out = **in + } if in.LogLevel != nil { in, out := &in.LogLevel, &out.LogLevel *out = new(int32) diff --git a/pkg/model/components/etcdmanager/model.go b/pkg/model/components/etcdmanager/model.go index 8b87a4c221..71e235126e 100644 --- a/pkg/model/components/etcdmanager/model.go +++ b/pkg/model/components/etcdmanager/model.go @@ -338,6 +338,10 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec) (*v1.Pod config.LogLevel = int(*etcdCluster.Manager.LogLevel) } + if etcdCluster.Manager != nil && etcdCluster.Manager.DiscoveryPollInterval != nil { + config.DiscoveryPollInterval = etcdCluster.Manager.DiscoveryPollInterval + } + { scheme := "https" @@ -545,16 +549,17 @@ type config struct { // EtcdInsecure can be used to turn off tls for etcd itself (compare with Insecure) EtcdInsecure bool `flag:"etcd-insecure"` - Address string `flag:"address"` - PeerUrls string `flag:"peer-urls"` - GrpcPort int `flag:"grpc-port"` - ClientUrls string `flag:"client-urls"` - QuarantineClientUrls string `flag:"quarantine-client-urls"` - ClusterName string `flag:"cluster-name"` - BackupStore string `flag:"backup-store"` - DataDir string `flag:"data-dir"` - VolumeProvider string `flag:"volume-provider"` - VolumeTag []string `flag:"volume-tag,repeat"` - VolumeNameTag string `flag:"volume-name-tag"` - DNSSuffix string `flag:"dns-suffix"` + Address string `flag:"address"` + PeerUrls string `flag:"peer-urls"` + GrpcPort int `flag:"grpc-port"` + ClientUrls string `flag:"client-urls"` + DiscoveryPollInterval *string `flag:"discovery-poll-interval"` + QuarantineClientUrls string `flag:"quarantine-client-urls"` + ClusterName string `flag:"cluster-name"` + BackupStore string `flag:"backup-store"` + DataDir string `flag:"data-dir"` + VolumeProvider string `flag:"volume-provider"` + VolumeTag []string `flag:"volume-tag,repeat"` + VolumeNameTag string `flag:"volume-name-tag"` + DNSSuffix string `flag:"dns-suffix"` } diff --git a/pkg/model/components/etcdmanager/model_test.go b/pkg/model/components/etcdmanager/model_test.go index f95fccff7a..a1ae1eab86 100644 --- a/pkg/model/components/etcdmanager/model_test.go +++ b/pkg/model/components/etcdmanager/model_test.go @@ -31,6 +31,7 @@ import ( func Test_RunEtcdManagerBuilder(t *testing.T) { tests := []string{ "tests/minimal", + "tests/pollinterval", "tests/proxy", "tests/old_versions_mount_hosts", "tests/overwrite_settings", diff --git a/pkg/model/components/etcdmanager/tests/pollinterval/cluster.yaml b/pkg/model/components/etcdmanager/tests/pollinterval/cluster.yaml new file mode 100644 index 0000000000..1fa1976919 --- /dev/null +++ b/pkg/model/components/etcdmanager/tests/pollinterval/cluster.yaml @@ -0,0 +1,89 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2016-12-10T22:42:27Z" + name: minimal.example.com +spec: + kubernetesApiAccess: + - 0.0.0.0/0 + channel: stable + cloudProvider: aws + configBase: memfs://clusters.example.com/minimal.example.com + etcdClusters: + - cpuRequest: 200m + etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + memoryRequest: 100Mi + name: main + manager: + discoveryPollInterval: 75s + provider: Manager + backups: + backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-main + - cpuRequest: 100m + etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + memoryRequest: 100Mi + name: events + manager: + discoveryPollInterval: 75s + provider: Manager + backups: + backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-events + kubernetesVersion: v1.17.0 + masterInternalName: api.internal.minimal.example.com + masterPublicName: api.minimal.example.com + networkCIDR: 172.20.0.0/16 + networking: + kubenet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + topology: + masters: public + nodes: public + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Public + zone: us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-10T22:42:28Z" + name: nodes + labels: + kops.k8s.io/cluster: minimal.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 + machineType: t2.medium + maxSize: 2 + minSize: 2 + role: Node + subnets: + - us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-10T22:42:28Z" + name: master-us-test-1a + labels: + kops.k8s.io/cluster: minimal.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 + machineType: m3.medium + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test-1a diff --git a/pkg/model/components/etcdmanager/tests/pollinterval/tasks.yaml b/pkg/model/components/etcdmanager/tests/pollinterval/tasks.yaml new file mode 100644 index 0000000000..064dd59ba1 --- /dev/null +++ b/pkg/model/components/etcdmanager/tests/pollinterval/tasks.yaml @@ -0,0 +1,189 @@ +Lifecycle: null +Name: etcd-clients-ca +Signer: null +alternateNames: null +oldFormat: false +subject: cn=etcd-clients-ca +type: ca +--- +Lifecycle: null +Name: etcd-manager-ca-events +Signer: null +alternateNames: null +oldFormat: false +subject: cn=etcd-manager-ca-events +type: ca +--- +Lifecycle: null +Name: etcd-manager-ca-main +Signer: null +alternateNames: null +oldFormat: false +subject: cn=etcd-manager-ca-main +type: ca +--- +Lifecycle: null +Name: etcd-peers-ca-events +Signer: null +alternateNames: null +oldFormat: false +subject: cn=etcd-peers-ca-events +type: ca +--- +Lifecycle: null +Name: etcd-peers-ca-main +Signer: null +alternateNames: null +oldFormat: false +subject: cn=etcd-peers-ca-main +type: ca +--- +Base: memfs://clusters.example.com/minimal.example.com/backups/etcd-events +Contents: |- + { + "memberCount": 1 + } +Lifecycle: null +Location: /control/etcd-cluster-spec +Name: etcd-cluster-spec-events +Public: null +--- +Base: memfs://clusters.example.com/minimal.example.com/backups/etcd-main +Contents: |- + { + "memberCount": 1 + } +Lifecycle: null +Location: /control/etcd-cluster-spec +Name: etcd-cluster-spec-main +Public: null +--- +Base: null +Contents: | + apiVersion: v1 + kind: Pod + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + creationTimestamp: null + labels: + k8s-app: etcd-manager-events + name: etcd-manager-events + namespace: kube-system + spec: + containers: + - command: + - /bin/sh + - -c + - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /etcd-manager + --backup-store=memfs://clusters.example.com/minimal.example.com/backups/etcd-events + --client-urls=https://__name__:4002 --cluster-name=etcd-events --containerized=true + --discovery-poll-interval=75s --dns-suffix=.internal.minimal.example.com --etcd-insecure=true + --grpc-port=3997 --insecure=false --peer-urls=https://__name__:2381 --quarantine-client-urls=https://__name__:3995 + --v=6 --volume-name-tag=k8s.io/etcd/events --volume-provider=aws --volume-tag=k8s.io/etcd/events + --volume-tag=k8s.io/role/master=1 --volume-tag=kubernetes.io/cluster/minimal.example.com=owned + > /tmp/pipe 2>&1 + image: kopeio/etcd-manager:3.0.20210228 + name: etcd-manager + resources: + requests: + cpu: 100m + memory: 100Mi + securityContext: + privileged: true + volumeMounts: + - mountPath: /rootfs + name: rootfs + - mountPath: /etc/kubernetes/pki/etcd-manager + name: pki + - mountPath: /var/log/etcd.log + name: varlogetcd + hostNetwork: true + hostPID: true + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: / + type: Directory + name: rootfs + - hostPath: + path: /etc/kubernetes/pki/etcd-manager-events + type: DirectoryOrCreate + name: pki + - hostPath: + path: /var/log/etcd-events.log + type: FileOrCreate + name: varlogetcd + status: {} +Lifecycle: null +Location: manifests/etcd/events.yaml +Name: manifests-etcdmanager-events +Public: null +--- +Base: null +Contents: | + apiVersion: v1 + kind: Pod + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + creationTimestamp: null + labels: + k8s-app: etcd-manager-main + name: etcd-manager-main + namespace: kube-system + spec: + containers: + - command: + - /bin/sh + - -c + - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /etcd-manager + --backup-store=memfs://clusters.example.com/minimal.example.com/backups/etcd-main + --client-urls=https://__name__:4001 --cluster-name=etcd --containerized=true + --discovery-poll-interval=75s --dns-suffix=.internal.minimal.example.com --etcd-insecure=true + --grpc-port=3996 --insecure=false --peer-urls=https://__name__:2380 --quarantine-client-urls=https://__name__:3994 + --v=6 --volume-name-tag=k8s.io/etcd/main --volume-provider=aws --volume-tag=k8s.io/etcd/main + --volume-tag=k8s.io/role/master=1 --volume-tag=kubernetes.io/cluster/minimal.example.com=owned + > /tmp/pipe 2>&1 + image: kopeio/etcd-manager:3.0.20210228 + name: etcd-manager + resources: + requests: + cpu: 200m + memory: 100Mi + securityContext: + privileged: true + volumeMounts: + - mountPath: /rootfs + name: rootfs + - mountPath: /etc/kubernetes/pki/etcd-manager + name: pki + - mountPath: /var/log/etcd.log + name: varlogetcd + hostNetwork: true + hostPID: true + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: / + type: Directory + name: rootfs + - hostPath: + path: /etc/kubernetes/pki/etcd-manager-main + type: DirectoryOrCreate + name: pki + - hostPath: + path: /var/log/etcd.log + type: FileOrCreate + name: varlogetcd + status: {} +Lifecycle: null +Location: manifests/etcd/main.yaml +Name: manifests-etcdmanager-main +Public: null