kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3675 from mschurenko/add-etcd-settings-to-nodeup-and-protokube 

Automatic merge from submit-queue.

Add etcd settings to nodeup and protokube

This adds etcd settings for leader timeout and heartbeat interval to protokube. It uses the additions to `EtcdClusterSpec` that were given the go ahead in https://github.com/kubernetes/kops/pull/3663

@chrislovecnm 
@robinpercy
This commit is contained in:
Kubernetes Submit Queue 2017-10-20 20:10:00 -07:00 committed by GitHub
parent 3315d29bdb 298747e9ae
commit 794e12dc52
16 changed files with 279 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
nodeup/pkg/model/convenience.go
View File

@ -28,6 +28,7 @@ import (
"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
"github.com/golang/glog"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// s is a helper that builds a *string from a string value
@ -171,3 +172,8 @@ func addHostPathMapping(pod *v1.Pod, container *v1.Container, name, path string)
return &container.VolumeMounts[len(container.VolumeMounts)-1]
}
// convEtcdSettingsToMs converts etcd settings to a string rep of int milliseconds
func convEtcdSettingsToMs(dur *metav1.Duration) string {
return strconv.FormatInt(dur.Nanoseconds()/1000000, 10)
}

57
nodeup/pkg/model/protokube.go
View File

@ -192,22 +192,24 @@ type ProtokubeFlags struct {
Channels []string `json:"channels,omitempty" flag:"channels"`
Cloud *string `json:"cloud,omitempty" flag:"cloud"`
// ClusterID flag is required only for vSphere cloud type, to pass cluster id information to protokube. AWS and GCE workflows ignore this flag.
ClusterID *string `json:"cluster-id,omitempty" flag:"cluster-id"`
Containerized *bool `json:"containerized,omitempty" flag:"containerized"`
DNSInternalSuffix *string `json:"dnsInternalSuffix,omitempty" flag:"dns-internal-suffix"`
DNSProvider *string `json:"dnsProvider,omitempty" flag:"dns"`
DNSServer *string `json:"dns-server,omitempty" flag:"dns-server"`
EtcdImage *string `json:"etcd-image,omitempty" flag:"etcd-image"`
InitializeRBAC *bool `json:"initializeRBAC,omitempty" flag:"initialize-rbac"`
LogLevel *int32 `json:"logLevel,omitempty" flag:"v"`
Master *bool `json:"master,omitempty" flag:"master"`
PeerTLSCaFile *string `json:"peer-ca,omitempty" flag:"peer-ca"`
PeerTLSCertFile *string `json:"peer-cert,omitempty" flag:"peer-cert"`
PeerTLSKeyFile *string `json:"peer-key,omitempty" flag:"peer-key"`
TLSCAFile *string `json:"tls-ca,omitempty" flag:"tls-ca"`
TLSCertFile *string `json:"tls-cert,omitempty" flag:"tls-cert"`
TLSKeyFile *string `json:"tls-key,omitempty" flag:"tls-key"`
Zone []string `json:"zone,omitempty" flag:"zone"`
ClusterID *string `json:"cluster-id,omitempty" flag:"cluster-id"`
Containerized *bool `json:"containerized,omitempty" flag:"containerized"`
DNSInternalSuffix *string `json:"dnsInternalSuffix,omitempty" flag:"dns-internal-suffix"`
DNSProvider *string `json:"dnsProvider,omitempty" flag:"dns"`
DNSServer *string `json:"dns-server,omitempty" flag:"dns-server"`
EtcdImage *string `json:"etcd-image,omitempty" flag:"etcd-image"`
EtcdLeaderElectionTimeout *string `json:"etcd-election-timeout,omitempty" flag:"etcd-election-timeout"`
EtcdHearbeatInterval *string `json:"etcd-heartbeat-interval,omitempty" flag:"etcd-heartbeat-interval"`
InitializeRBAC *bool `json:"initializeRBAC,omitempty" flag:"initialize-rbac"`
LogLevel *int32 `json:"logLevel,omitempty" flag:"v"`
Master *bool `json:"master,omitempty" flag:"master"`
PeerTLSCaFile *string `json:"peer-ca,omitempty" flag:"peer-ca"`
PeerTLSCertFile *string `json:"peer-cert,omitempty" flag:"peer-cert"`
PeerTLSKeyFile *string `json:"peer-key,omitempty" flag:"peer-key"`
TLSCAFile *string `json:"tls-ca,omitempty" flag:"tls-ca"`
TLSCertFile *string `json:"tls-cert,omitempty" flag:"tls-cert"`
TLSKeyFile *string `json:"tls-key,omitempty" flag:"tls-key"`
Zone []string `json:"zone,omitempty" flag:"zone"`
}
// ProtokubeFlags is responsible for building the command line flags for protokube
@ -216,12 +218,25 @@ func (t *ProtokubeBuilder) ProtokubeFlags(k8sVersion semver.Version) *ProtokubeF
// lets keep that for another PR and allow the version change
imageVersion := t.Cluster.Spec.EtcdClusters[0].Version
var leaderElectionTimeout string
var heartbeatInterval string
if v := t.Cluster.Spec.EtcdClusters[0].LeaderElectionTimeout; v != nil {
leaderElectionTimeout = convEtcdSettingsToMs(v)
}
if v := t.Cluster.Spec.EtcdClusters[0].HeartbeatInterval; v != nil {
heartbeatInterval = convEtcdSettingsToMs(v)
}
f := &ProtokubeFlags{
Channels: t.NodeupConfig.Channels,
Containerized: fi.Bool(true),
EtcdImage: s(fmt.Sprintf("gcr.io/google_containers/etcd:%s", imageVersion)),
LogLevel: fi.Int32(4),
Master: b(t.IsMaster),
Channels: t.NodeupConfig.Channels,
Containerized: fi.Bool(true),
EtcdImage: s(fmt.Sprintf("gcr.io/google_containers/etcd:%s", imageVersion)),
EtcdLeaderElectionTimeout: s(leaderElectionTimeout),
EtcdHearbeatInterval: s(heartbeatInterval),
LogLevel: fi.Int32(4),
Master: b(t.IsMaster),
}
// initialize rbac on Kubernetes >= 1.6 and master

4
pkg/apis/kops/cluster.go
View File

@ -292,6 +292,10 @@ type EtcdClusterSpec struct {
EnableEtcdTLS bool `json:"enableEtcdTLS,omitempty"`
// Version is the version of etcd to run i.e. 2.1.2, 3.0.17 etcd
Version string `json:"version,omitempty"`
// LeaderElectionTimeout is the time (in milliseconds) for an etcd leader election timeout
LeaderElectionTimeout *metav1.Duration `json:"leaderElectionTimeout,omitempty"`
// HeartbeatInterval is the time (in milliseconds) for an etcd heartbeat interval
HeartbeatInterval *metav1.Duration `json:"heartbeatInterval,omitempty"`
}
// EtcdMemberSpec is a specification for a etcd member

4
pkg/apis/kops/v1alpha1/cluster.go
View File

@ -291,6 +291,10 @@ type EtcdClusterSpec struct {
EnableEtcdTLS bool `json:"enableEtcdTLS,omitempty"`
// Version is the version of etcd to run i.e. 2.1.2, 3.0.17 etcd
Version string `json:"version,omitempty"`
// LeaderElectionTimeout is the time (in milliseconds) for an etcd leader election timeout
LeaderElectionTimeout *metav1.Duration `json:"leaderElectionTimeout,omitempty"`
// HeartbeatInterval is the time (in milliseconds) for an etcd heartbeat interval
HeartbeatInterval *metav1.Duration `json:"heartbeatInterval,omitempty"`
}
// EtcdMemberSpec is a specification for a etcd member

4
pkg/apis/kops/v1alpha1/zz_generated.conversion.go
View File

@ -1179,6 +1179,8 @@ func autoConvert_v1alpha1_EtcdClusterSpec_To_kops_EtcdClusterSpec(in *EtcdCluste
}
out.EnableEtcdTLS = in.EnableEtcdTLS
out.Version = in.Version
out.LeaderElectionTimeout = in.LeaderElectionTimeout
out.HeartbeatInterval = in.HeartbeatInterval
return nil
}
@ -1203,6 +1205,8 @@ func autoConvert_kops_EtcdClusterSpec_To_v1alpha1_EtcdClusterSpec(in *kops.EtcdC
}
out.EnableEtcdTLS = in.EnableEtcdTLS
out.Version = in.Version
out.LeaderElectionTimeout = in.LeaderElectionTimeout
out.HeartbeatInterval = in.HeartbeatInterval
return nil
}

18
pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go
View File

@ -1233,6 +1233,24 @@ func (in *EtcdClusterSpec) DeepCopyInto(out *EtcdClusterSpec) {
}
}
}
if in.LeaderElectionTimeout != nil {
in, out := &in.LeaderElectionTimeout, &out.LeaderElectionTimeout
if *in == nil {
*out = nil
} else {
*out = new(v1.Duration)
**out = **in
}
}
if in.HeartbeatInterval != nil {
in, out := &in.HeartbeatInterval, &out.HeartbeatInterval
if *in == nil {
*out = nil
} else {
*out = new(v1.Duration)
**out = **in
}
}
return
}

4
pkg/apis/kops/v1alpha2/cluster.go
View File

@ -289,6 +289,10 @@ type EtcdClusterSpec struct {
EnableEtcdTLS bool `json:"enableEtcdTLS,omitempty"`
// Version is the version of etcd to run i.e. 2.1.2, 3.0.17 etcd
Version string `json:"version,omitempty"`
// LeaderElectionTimeout is the time (in milliseconds) for an etcd leader election timeout
LeaderElectionTimeout *metav1.Duration `json:"leaderElectionTimeout,omitempty"`
// HeartbeatInterval is the time (in milliseconds) for an etcd heartbeat interval
HeartbeatInterval *metav1.Duration `json:"heartbeatInterval,omitempty"`
}
// EtcdMemberSpec is a specification for a etcd member

4
pkg/apis/kops/v1alpha2/zz_generated.conversion.go
View File

@ -1278,6 +1278,8 @@ func autoConvert_v1alpha2_EtcdClusterSpec_To_kops_EtcdClusterSpec(in *EtcdCluste
}
out.EnableEtcdTLS = in.EnableEtcdTLS
out.Version = in.Version
out.LeaderElectionTimeout = in.LeaderElectionTimeout
out.HeartbeatInterval = in.HeartbeatInterval
return nil
}
@ -1302,6 +1304,8 @@ func autoConvert_kops_EtcdClusterSpec_To_v1alpha2_EtcdClusterSpec(in *kops.EtcdC
}
out.EnableEtcdTLS = in.EnableEtcdTLS
out.Version = in.Version
out.LeaderElectionTimeout = in.LeaderElectionTimeout
out.HeartbeatInterval = in.HeartbeatInterval
return nil
}

18
pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go
View File

@ -1243,6 +1243,24 @@ func (in *EtcdClusterSpec) DeepCopyInto(out *EtcdClusterSpec) {
}
}
}
if in.LeaderElectionTimeout != nil {
in, out := &in.LeaderElectionTimeout, &out.LeaderElectionTimeout
if *in == nil {
*out = nil
} else {
*out = new(v1.Duration)
**out = **in
}
}
if in.HeartbeatInterval != nil {
in, out := &in.HeartbeatInterval, &out.HeartbeatInterval
if *in == nil {
*out = nil
} else {
*out = new(v1.Duration)
**out = **in
}
}
return
}

18
pkg/apis/kops/zz_generated.deepcopy.go
View File

@ -1402,6 +1402,24 @@ func (in *EtcdClusterSpec) DeepCopyInto(out *EtcdClusterSpec) {
}
}
}
if in.LeaderElectionTimeout != nil {
in, out := &in.LeaderElectionTimeout, &out.LeaderElectionTimeout
if *in == nil {
*out = nil
} else {
*out = new(v1.Duration)
**out = **in
}
}
if in.HeartbeatInterval != nil {
in, out := &in.HeartbeatInterval, &out.HeartbeatInterval
if *in == nil {
*out = nil
} else {
*out = new(v1.Duration)
**out = **in
}
}
return
}

39
protokube/cmd/protokube/main.go
View File

@ -61,7 +61,8 @@ func run() error {
var zones []string
var applyTaints, initializeRBAC, containerized, master bool
var cloud, clusterID, dnsServer, dnsProviderID, dnsInternalSuffix, gossipSecret, gossipListen string
var flagChannels, tlsCert, tlsKey, tlsCA, peerCert, peerKey, peerCA, etcdImageSource string
var flagChannels, tlsCert, tlsKey, tlsCA, peerCert, peerKey, peerCA string
var etcdImageSource, etcdElectionTimeout, etcdHeartbeatInterval string
flag.BoolVar(&applyTaints, "apply-taints", applyTaints, "Apply taints to nodes based on the role")
flag.BoolVar(&containerized, "containerized", containerized, "Set if we are running containerized.")
@ -82,6 +83,8 @@ func run() error {
flags.StringSliceVarP(&zones, "zone", "z", []string{}, "Configure permitted zones and their mappings")
flags.StringVar(&dnsProviderID, "dns", "aws-route53", "DNS provider we should use (aws-route53, google-clouddns, coredns)")
flags.StringVar(&etcdImageSource, "etcd-image", "gcr.io/google_containers/etcd:2.2.1", "Etcd Source Container Registry")
flags.StringVar(&etcdElectionTimeout, "etcd-election-timeout", etcdElectionTimeout, "time in ms for an election to timeout")
flags.StringVar(&etcdHeartbeatInterval, "etcd-heartbeat-interval", etcdHeartbeatInterval, "time in ms of a heartbeat interval")
flags.StringVar(&gossipSecret, "gossip-secret", gossipSecret, "Secret to use to secure gossip")
// Trick to avoid 'logging before flag.Parse' warning
@ -282,22 +285,24 @@ func run() error {
}
k := &protokube.KubeBoot{
ApplyTaints: applyTaints,
Channels: channels,
DNS: dnsProvider,
EtcdImageSource: etcdImageSource,
InitializeRBAC: initializeRBAC,
InternalDNSSuffix: dnsInternalSuffix,
InternalIP: internalIP,
Kubernetes: protokube.NewKubernetesContext(),
Master: master,
ModelDir: modelDir,
PeerCA: peerCA,
PeerCert: peerCert,
PeerKey: peerKey,
TLSCA: tlsCA,
TLSCert: tlsCert,
TLSKey: tlsKey,
ApplyTaints: applyTaints,
Channels: channels,
DNS: dnsProvider,
EtcdImageSource: etcdImageSource,
EtcdElectionTimeout: etcdElectionTimeout,
EtcdHeartbeatInterval: etcdHeartbeatInterval,
InitializeRBAC: initializeRBAC,
InternalDNSSuffix: dnsInternalSuffix,
InternalIP: internalIP,
Kubernetes: protokube.NewKubernetesContext(),
Master: master,
ModelDir: modelDir,
PeerCA: peerCA,
PeerCert: peerCert,
PeerKey: peerKey,
TLSCA: tlsCA,
TLSCert: tlsCert,
TLSKey: tlsKey,
}
k.Init(volumes)

36
protokube/pkg/protokube/etcd_cluster.go
View File

@ -73,6 +73,10 @@ type EtcdCluster struct {
PeerCert string
// PeerKey is the path to a peer ca for etcd
PeerKey string
// ElectionTimeout is the leader election timeout
ElectionTimeout string
// HeartbeatInterval is the heartbeat interval
HeartbeatInterval string
}
// EtcdNode is a definition for the etcd node
@ -97,21 +101,23 @@ func newEtcdController(kubeBoot *KubeBoot, v *Volume, spec *etcd.EtcdClusterSpec
cluster := &EtcdCluster{
// @TODO we need to deprecate this port and use 2379, but that would be a breaking change
ClientPort: 4001,
ClusterName: "etcd-" + spec.ClusterKey,
CPURequest: resource.MustParse("200m"),
DataDirName: "data-" + spec.ClusterKey,
ImageSource: kubeBoot.EtcdImageSource,
TLSCA: kubeBoot.TLSCA,
TLSCert: kubeBoot.TLSCert,
TLSKey: kubeBoot.TLSKey,
PeerCA: kubeBoot.PeerCA,
PeerCert: kubeBoot.PeerCert,
PeerKey: kubeBoot.PeerKey,
PeerPort: 2380,
PodName: "etcd-server-" + spec.ClusterKey,
Spec: spec,
VolumeMountPath: v.Mountpoint,
ClientPort: 4001,
ClusterName: "etcd-" + spec.ClusterKey,
CPURequest: resource.MustParse("200m"),
DataDirName: "data-" + spec.ClusterKey,
ImageSource: kubeBoot.EtcdImageSource,
TLSCA: kubeBoot.TLSCA,
TLSCert: kubeBoot.TLSCert,
TLSKey: kubeBoot.TLSKey,
PeerCA: kubeBoot.PeerCA,
PeerCert: kubeBoot.PeerCert,
PeerKey: kubeBoot.PeerKey,
PeerPort: 2380,
PodName: "etcd-server-" + spec.ClusterKey,
Spec: spec,
VolumeMountPath: v.Mountpoint,
ElectionTimeout: kubeBoot.EtcdElectionTimeout,
HeartbeatInterval: kubeBoot.EtcdHeartbeatInterval,
}
// We used to build this through text files ... it turns out to just be more complicated than code!

10
protokube/pkg/protokube/etcd_manifest.go
View File

@ -48,7 +48,7 @@ func BuildEtcdManifest(c *EtcdCluster) *v1.Pod {
"/bin/sh", "-c", "/usr/local/bin/etcd 2>&1 | /bin/tee -a /var/log/etcd.log",
},
}
// build the the environment variables for etcd service
// build the environment variables for etcd service
container.Env = buildEtcdEnvironmentOptions(c)
container.LivenessProbe = &v1.Probe{
@ -170,6 +170,14 @@ func buildEtcdEnvironmentOptions(c *EtcdCluster) []v1.EnvVar {
{Name: "ETCD_INITIAL_CLUSTER_STATE", Value: "new"},
{Name: "ETCD_INITIAL_CLUSTER_TOKEN", Value: c.ClusterToken}}...)
// add timeout/hearbeat settings
if notEmpty(c.ElectionTimeout) {
options = append(options, v1.EnvVar{Name: "ETCD_ELECTION_TIMEOUT", Value: c.ElectionTimeout})
}
if notEmpty(c.HeartbeatInterval) {
options = append(options, v1.EnvVar{Name: "ETCD_HEARTBEAT_INTERVAL", Value: c.HeartbeatInterval})
}
// @check if we are using peer certificates
if notEmpty(c.PeerCA) {
options = append(options, []v1.EnvVar{

4
protokube/pkg/protokube/kube_boot.go
View File

@ -50,6 +50,10 @@ type KubeBoot struct {
ModelDir string
// Etcd container registry location.
EtcdImageSource string
// EtcdElectionTimeout is is the leader election timeout
EtcdElectionTimeout string
// EtcdHeartbeatInterval is the heartbeat interval
EtcdHeartbeatInterval string
// TLSCA is the path to a client ca for etcd
TLSCA string
// TLSCert is the path to a tls certificate for etcd

1
protokube/tests/integration/build_etcd_manifest/integration_test.go
View File

@ -34,6 +34,7 @@ func TestBuildEtcdManifest(t *testing.T) {
}{
{TestFile: "non_tls.yaml"},
{TestFile: "tls.yaml"},
{TestFile: "etcd_env_vars.yaml"},
}
for i, x := range cs {
cluster, expected := loadTestIntegration(t, path.Join("main", x.TestFile))

106
protokube/tests/integration/build_etcd_manifest/main/etcd_env_vars.yaml Normal file
View File

@ -0,0 +1,106 @@
clientPort: 4001
clusterName: etcd-main
clusterToken: token-main
cpuRequest: "200m"
dataDirName: data-main
imageSource: gcr.io/google_containers/etcd:2.2.1
logFile: /var/log/etcd.log
peerPort: 2380
podName: etcd-server-main
volumeMountPath: /mnt/main
electionTimeout: "1000"
heartbeatInterval: "100"
me:
name: node0
internalName: node0.internal
nodes:
- name: node0
internalName: node0.internal
- name: node1
internalName: node1.internal
- name: node2
internalName: node2.internal
spec: {}
---
apiVersion: v1
kind: Pod
metadata:
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ""
creationTimestamp: null
labels:
k8s-app: etcd-server-main
name: etcd-server-main
namespace: kube-system
spec:
containers:
- command:
- /bin/sh
- -c
- /usr/local/bin/etcd 2>&1 | /bin/tee -a /var/log/etcd.log
env:
- name: ETCD_NAME
value: node0
- name: ETCD_DATA_DIR
value: /var/etcd/data-main
- name: ETCD_LISTEN_PEER_URLS
value: http://0.0.0.0:2380
- name: ETCD_LISTEN_CLIENT_URLS
value: http://0.0.0.0:4001
- name: ETCD_ADVERTISE_CLIENT_URLS
value: http://node0.internal:4001
- name: ETCD_INITIAL_ADVERTISE_PEER_URLS
value: http://node0.internal:2380
- name: ETCD_INITIAL_CLUSTER_STATE
value: new
- name: ETCD_INITIAL_CLUSTER_TOKEN
value: token-main
- name: ETCD_ELECTION_TIMEOUT
value: "1000"
- name: ETCD_HEARTBEAT_INTERVAL
value: "100"
- name: ETCD_INITIAL_CLUSTER
value: node0=http://node0.internal:2380,node1=http://node1.internal:2380,node2=http://node2.internal:2380
image: gcr.io/google_containers/etcd:2.2.1
livenessProbe:
httpGet:
host: 127.0.0.1
path: /health
port: 4001
scheme: HTTP
initialDelaySeconds: 15
timeoutSeconds: 15
name: etcd-container
ports:
- containerPort: 2380
hostPort: 2380
name: serverport
- containerPort: 4001
hostPort: 4001
name: clientport
resources:
requests:
cpu: 200m
volumeMounts:
- mountPath: /var/etcd/data-main
name: varetcdata
- mountPath: /var/log/etcd.log
name: varlogetcd
- mountPath: /etc/hosts
name: hosts
readOnly: true
hostNetwork: true
tolerations:
- key: CriticalAddonsOnly
operator: Exists
volumes:
- hostPath:
path: /mnt/main/var/etcd/data-main
name: varetcdata
- hostPath:
path: /var/log/etcd.log
name: varlogetcd
- hostPath:
path: /etc/hosts
name: hosts
status: {}