Merge pull request #4465 from justinsb/etcd_backups

Initial support for standalone etcd-manager backups

nodeup/pkg/model/protokube.go

@@ -201,6 +201,8 @@ type ProtokubeFlags struct {
 	DNSInternalSuffix         *string  `json:"dnsInternalSuffix,omitempty" flag:"dns-internal-suffix"`
 	DNSProvider               *string  `json:"dnsProvider,omitempty" flag:"dns"`
 	DNSServer                 *string  `json:"dns-server,omitempty" flag:"dns-server"`
+	EtcdBackupImage           string   `json:"etcd-backup-image,omitempty" flag:"etcd-backup-image"`
+	EtcdBackupStore           string   `json:"etcd-backup-store,omitempty" flag:"etcd-backup-store"`
 	EtcdImage                 *string  `json:"etcd-image,omitempty" flag:"etcd-image"`
 	EtcdLeaderElectionTimeout *string  `json:"etcd-election-timeout,omitempty" flag:"etcd-election-timeout"`
 	EtcdHearbeatInterval      *string  `json:"etcd-heartbeat-interval,omitempty" flag:"etcd-heartbeat-interval"`
@@ -242,6 +244,18 @@ func (t *ProtokubeBuilder) ProtokubeFlags(k8sVersion semver.Version) (*Protokube
 		Master:                    b(t.IsMaster),
 	}
 
+	for _, e := range t.Cluster.Spec.EtcdClusters {
+		if e.Backups != nil {
+			if f.EtcdBackupImage == "" {
+				f.EtcdBackupImage = e.Backups.Image
+			}
+
+			if f.EtcdBackupStore == "" {
+				f.EtcdBackupStore = e.Backups.BackupStore
+			}
+		}
+	}
+
 	// TODO this is dupicate code with etcd model
 	image := fmt.Sprintf("k8s.gcr.io/etcd:%s", imageVersion)
 	// override image if set as API value

pkg/apis/kops/cluster.go

@@ -316,6 +316,16 @@ type EtcdClusterSpec struct {
 	HeartbeatInterval *metav1.Duration `json:"heartbeatInterval,omitempty"`
 	// Image is the etcd docker image to use. Setting this will ignore the Version specified.
 	Image string `json:"image,omitempty"`
+	// Backups describes how we do backups of etcd
+	Backups *EtcdBackupSpec `json:"backups,omitempty"`
+}
+
+// EtcdBackupSpec describes how we want to do backups of etcd
+type EtcdBackupSpec struct {
+	// BackupStore is the VFS path where we will read/write backup data
+	BackupStore string `json:"backupStore,omitempty"`
+	// Image is the etcd backup manager image to use.  Setting this will create a sidecar container in the etcd pod with the specified image.
+	Image string `json:"image,omitempty"`
 }
 
 // EtcdMemberSpec is a specification for a etcd member

pkg/apis/kops/v1alpha1/cluster.go

@@ -315,6 +315,16 @@ type EtcdClusterSpec struct {
 	HeartbeatInterval *metav1.Duration `json:"heartbeatInterval,omitempty"`
 	// Image is the etcd docker image to use. Setting this will ignore the Version specified.
 	Image string `json:"image,omitempty"`
+	// Backups describes how we do backups of etcd
+	Backups *EtcdBackupSpec `json:"backups,omitempty"`
+}
+
+// EtcdBackupSpec describes how we want to do backups of etcd
+type EtcdBackupSpec struct {
+	// BackupStore is the VFS path where we will read/write backup data
+	BackupStore string `json:"backupStore,omitempty"`
+	// Image is the etcd backup manager image to use.  Setting this will create a sidecar container in the etcd pod with the specified image.
+	Image string `json:"image,omitempty"`
 }
 
 // EtcdMemberSpec is a specification for a etcd member

pkg/apis/kops/v1alpha1/zz_generated.conversion.go

@@ -73,6 +73,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
 		Convert_kops_DockerConfig_To_v1alpha1_DockerConfig,
 		Convert_v1alpha1_EgressProxySpec_To_kops_EgressProxySpec,
 		Convert_kops_EgressProxySpec_To_v1alpha1_EgressProxySpec,
+		Convert_v1alpha1_EtcdBackupSpec_To_kops_EtcdBackupSpec,
+		Convert_kops_EtcdBackupSpec_To_v1alpha1_EtcdBackupSpec,
 		Convert_v1alpha1_EtcdClusterSpec_To_kops_EtcdClusterSpec,
 		Convert_kops_EtcdClusterSpec_To_v1alpha1_EtcdClusterSpec,
 		Convert_v1alpha1_EtcdMemberSpec_To_kops_EtcdMemberSpec,
@@ -1207,6 +1209,28 @@ func Convert_kops_EgressProxySpec_To_v1alpha1_EgressProxySpec(in *kops.EgressPro
 	return autoConvert_kops_EgressProxySpec_To_v1alpha1_EgressProxySpec(in, out, s)
 }
 
+func autoConvert_v1alpha1_EtcdBackupSpec_To_kops_EtcdBackupSpec(in *EtcdBackupSpec, out *kops.EtcdBackupSpec, s conversion.Scope) error {
+	out.BackupStore = in.BackupStore
+	out.Image = in.Image
+	return nil
+}
+
+// Convert_v1alpha1_EtcdBackupSpec_To_kops_EtcdBackupSpec is an autogenerated conversion function.
+func Convert_v1alpha1_EtcdBackupSpec_To_kops_EtcdBackupSpec(in *EtcdBackupSpec, out *kops.EtcdBackupSpec, s conversion.Scope) error {
+	return autoConvert_v1alpha1_EtcdBackupSpec_To_kops_EtcdBackupSpec(in, out, s)
+}
+
+func autoConvert_kops_EtcdBackupSpec_To_v1alpha1_EtcdBackupSpec(in *kops.EtcdBackupSpec, out *EtcdBackupSpec, s conversion.Scope) error {
+	out.BackupStore = in.BackupStore
+	out.Image = in.Image
+	return nil
+}
+
+// Convert_kops_EtcdBackupSpec_To_v1alpha1_EtcdBackupSpec is an autogenerated conversion function.
+func Convert_kops_EtcdBackupSpec_To_v1alpha1_EtcdBackupSpec(in *kops.EtcdBackupSpec, out *EtcdBackupSpec, s conversion.Scope) error {
+	return autoConvert_kops_EtcdBackupSpec_To_v1alpha1_EtcdBackupSpec(in, out, s)
+}
+
 func autoConvert_v1alpha1_EtcdClusterSpec_To_kops_EtcdClusterSpec(in *EtcdClusterSpec, out *kops.EtcdClusterSpec, s conversion.Scope) error {
 	out.Name = in.Name
 	if in.Members != nil {
@@ -1226,6 +1250,15 @@ func autoConvert_v1alpha1_EtcdClusterSpec_To_kops_EtcdClusterSpec(in *EtcdCluste
 	out.LeaderElectionTimeout = in.LeaderElectionTimeout
 	out.HeartbeatInterval = in.HeartbeatInterval
 	out.Image = in.Image
+	if in.Backups != nil {
+		in, out := &in.Backups, &out.Backups
+		*out = new(kops.EtcdBackupSpec)
+		if err := Convert_v1alpha1_EtcdBackupSpec_To_kops_EtcdBackupSpec(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Backups = nil
+	}
 	return nil
 }
 
@@ -1253,6 +1286,15 @@ func autoConvert_kops_EtcdClusterSpec_To_v1alpha1_EtcdClusterSpec(in *kops.EtcdC
 	out.LeaderElectionTimeout = in.LeaderElectionTimeout
 	out.HeartbeatInterval = in.HeartbeatInterval
 	out.Image = in.Image
+	if in.Backups != nil {
+		in, out := &in.Backups, &out.Backups
+		*out = new(EtcdBackupSpec)
+		if err := Convert_kops_EtcdBackupSpec_To_v1alpha1_EtcdBackupSpec(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Backups = nil
+	}
 	return nil
 }
 

pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go

@@ -1021,6 +1021,22 @@ func (in *EgressProxySpec) DeepCopy() *EgressProxySpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EtcdBackupSpec) DeepCopyInto(out *EtcdBackupSpec) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdBackupSpec.
+func (in *EtcdBackupSpec) DeepCopy() *EtcdBackupSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(EtcdBackupSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *EtcdClusterSpec) DeepCopyInto(out *EtcdClusterSpec) {
 	*out = *in
@@ -1054,6 +1070,15 @@ func (in *EtcdClusterSpec) DeepCopyInto(out *EtcdClusterSpec) {
 			**out = **in
 		}
 	}
+	if in.Backups != nil {
+		in, out := &in.Backups, &out.Backups
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(EtcdBackupSpec)
+			**out = **in
+		}
+	}
 	return
 }
 

pkg/apis/kops/v1alpha2/cluster.go

@@ -313,6 +313,16 @@ type EtcdClusterSpec struct {
 	HeartbeatInterval *metav1.Duration `json:"heartbeatInterval,omitempty"`
 	// Image is the etcd docker image to use. Setting this will ignore the Version specified.
 	Image string `json:"image,omitempty"`
+	// Backups describes how we do backups of etcd
+	Backups *EtcdBackupSpec `json:"backups,omitempty"`
+}
+
+// EtcdBackupSpec describes how we want to do backups of etcd
+type EtcdBackupSpec struct {
+	// BackupStore is the VFS path where we will read/write backup data
+	BackupStore string `json:"backupStore,omitempty"`
+	// Image is the etcd backup manager image to use.  Setting this will create a sidecar container in the etcd pod with the specified image.
+	Image string `json:"image,omitempty"`
 }
 
 // EtcdMemberSpec is a specification for a etcd member

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -77,6 +77,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
 		Convert_kops_DockerConfig_To_v1alpha2_DockerConfig,
 		Convert_v1alpha2_EgressProxySpec_To_kops_EgressProxySpec,
 		Convert_kops_EgressProxySpec_To_v1alpha2_EgressProxySpec,
+		Convert_v1alpha2_EtcdBackupSpec_To_kops_EtcdBackupSpec,
+		Convert_kops_EtcdBackupSpec_To_v1alpha2_EtcdBackupSpec,
 		Convert_v1alpha2_EtcdClusterSpec_To_kops_EtcdClusterSpec,
 		Convert_kops_EtcdClusterSpec_To_v1alpha2_EtcdClusterSpec,
 		Convert_v1alpha2_EtcdMemberSpec_To_kops_EtcdMemberSpec,
@@ -1306,6 +1308,28 @@ func Convert_kops_EgressProxySpec_To_v1alpha2_EgressProxySpec(in *kops.EgressPro
 	return autoConvert_kops_EgressProxySpec_To_v1alpha2_EgressProxySpec(in, out, s)
 }
 
+func autoConvert_v1alpha2_EtcdBackupSpec_To_kops_EtcdBackupSpec(in *EtcdBackupSpec, out *kops.EtcdBackupSpec, s conversion.Scope) error {
+	out.BackupStore = in.BackupStore
+	out.Image = in.Image
+	return nil
+}
+
+// Convert_v1alpha2_EtcdBackupSpec_To_kops_EtcdBackupSpec is an autogenerated conversion function.
+func Convert_v1alpha2_EtcdBackupSpec_To_kops_EtcdBackupSpec(in *EtcdBackupSpec, out *kops.EtcdBackupSpec, s conversion.Scope) error {
+	return autoConvert_v1alpha2_EtcdBackupSpec_To_kops_EtcdBackupSpec(in, out, s)
+}
+
+func autoConvert_kops_EtcdBackupSpec_To_v1alpha2_EtcdBackupSpec(in *kops.EtcdBackupSpec, out *EtcdBackupSpec, s conversion.Scope) error {
+	out.BackupStore = in.BackupStore
+	out.Image = in.Image
+	return nil
+}
+
+// Convert_kops_EtcdBackupSpec_To_v1alpha2_EtcdBackupSpec is an autogenerated conversion function.
+func Convert_kops_EtcdBackupSpec_To_v1alpha2_EtcdBackupSpec(in *kops.EtcdBackupSpec, out *EtcdBackupSpec, s conversion.Scope) error {
+	return autoConvert_kops_EtcdBackupSpec_To_v1alpha2_EtcdBackupSpec(in, out, s)
+}
+
 func autoConvert_v1alpha2_EtcdClusterSpec_To_kops_EtcdClusterSpec(in *EtcdClusterSpec, out *kops.EtcdClusterSpec, s conversion.Scope) error {
 	out.Name = in.Name
 	if in.Members != nil {
@@ -1325,6 +1349,15 @@ func autoConvert_v1alpha2_EtcdClusterSpec_To_kops_EtcdClusterSpec(in *EtcdCluste
 	out.LeaderElectionTimeout = in.LeaderElectionTimeout
 	out.HeartbeatInterval = in.HeartbeatInterval
 	out.Image = in.Image
+	if in.Backups != nil {
+		in, out := &in.Backups, &out.Backups
+		*out = new(kops.EtcdBackupSpec)
+		if err := Convert_v1alpha2_EtcdBackupSpec_To_kops_EtcdBackupSpec(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Backups = nil
+	}
 	return nil
 }
 
@@ -1352,6 +1385,15 @@ func autoConvert_kops_EtcdClusterSpec_To_v1alpha2_EtcdClusterSpec(in *kops.EtcdC
 	out.LeaderElectionTimeout = in.LeaderElectionTimeout
 	out.HeartbeatInterval = in.HeartbeatInterval
 	out.Image = in.Image
+	if in.Backups != nil {
+		in, out := &in.Backups, &out.Backups
+		*out = new(EtcdBackupSpec)
+		if err := Convert_kops_EtcdBackupSpec_To_v1alpha2_EtcdBackupSpec(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Backups = nil
+	}
 	return nil
 }
 

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -1015,6 +1015,22 @@ func (in *EgressProxySpec) DeepCopy() *EgressProxySpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EtcdBackupSpec) DeepCopyInto(out *EtcdBackupSpec) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdBackupSpec.
+func (in *EtcdBackupSpec) DeepCopy() *EtcdBackupSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(EtcdBackupSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *EtcdClusterSpec) DeepCopyInto(out *EtcdClusterSpec) {
 	*out = *in
@@ -1048,6 +1064,15 @@ func (in *EtcdClusterSpec) DeepCopyInto(out *EtcdClusterSpec) {
 			**out = **in
 		}
 	}
+	if in.Backups != nil {
+		in, out := &in.Backups, &out.Backups
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(EtcdBackupSpec)
+			**out = **in
+		}
+	}
 	return
 }
 

pkg/apis/kops/zz_generated.deepcopy.go

@@ -1134,6 +1134,22 @@ func (in *EgressProxySpec) DeepCopy() *EgressProxySpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EtcdBackupSpec) DeepCopyInto(out *EtcdBackupSpec) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdBackupSpec.
+func (in *EtcdBackupSpec) DeepCopy() *EtcdBackupSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(EtcdBackupSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *EtcdClusterSpec) DeepCopyInto(out *EtcdClusterSpec) {
 	*out = *in
@@ -1167,6 +1183,15 @@ func (in *EtcdClusterSpec) DeepCopyInto(out *EtcdClusterSpec) {
 			**out = **in
 		}
 	}
+	if in.Backups != nil {
+		in, out := &in.Backups, &out.Backups
+		if *in == nil {
+			*out = nil
+		} else {
+			*out = new(EtcdBackupSpec)
+			**out = **in
+		}
+	}
 	return
 }
 

pkg/model/components/etcd.go

@@ -23,6 +23,8 @@ import (
 	"k8s.io/kops/upup/pkg/fi/loader"
 )
 
+const DefaultBackupImage = "kopeio/etcd-backup:1.0.20180220"
+
 // EtcdOptionsBuilder adds options for etcd to the model
 type EtcdOptionsBuilder struct {
 	Context *OptionsContext
@@ -43,16 +45,35 @@ func (b *EtcdOptionsBuilder) BuildOptions(o interface{}) error {
 		}
 	}
 
-	// default to gcr.io
-	image := fmt.Sprintf("k8s.gcr.io/etcd:%s", spec.EtcdClusters[0].Version)
+	// remap image
+	for _, c := range spec.EtcdClusters {
+		image := c.Image
+		if image == "" {
+			image = fmt.Sprintf("k8s.gcr.io/etcd:%s", c.Version)
+		}
 
-	// override image if set as API value
-	if spec.EtcdClusters[0].Image != "" {
-		image = spec.EtcdClusters[0].Image
+		image, err := b.Context.AssetBuilder.RemapImage(image)
+		if err != nil {
+			return fmt.Errorf("unable to remap container %q: %v", image, err)
+		}
+		c.Image = image
 	}
-	image, err := b.Context.AssetBuilder.RemapImage(image)
-	if err != nil {
-		return fmt.Errorf("unable to remap container %q: %v", image, err)
+
+	// remap backup manager images
+	for _, c := range spec.EtcdClusters {
+		if c.Backups == nil {
+			continue
+		}
+		image := c.Backups.Image
+		if image == "" {
+			image = fmt.Sprintf(DefaultBackupImage)
+		}
+
+		image, err := b.Context.AssetBuilder.RemapImage(image)
+		if err != nil {
+			return fmt.Errorf("unable to remap container %q: %v", image, err)
+		}
+		c.Backups.Image = image
 	}
 
 	return nil

pkg/model/iam/BUILD.bazel

@@ -12,6 +12,7 @@ go_library(
         "//upup/pkg/fi/cloudup/awstasks:go_default_library",
         "//util/pkg/vfs:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
     ],
 )
 

pkg/model/iam/iam_builder.go

@@ -33,6 +33,7 @@ import (
 	"strings"
 
 	"github.com/golang/glog"
+	"k8s.io/apimachinery/pkg/util/sets"
 
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/util/stringorslice"
@@ -396,6 +397,40 @@ func (b *PolicyBuilder) AddS3Permissions(p *Policy) (*Policy, error) {
 		}
 	}
 
+	// On the master, grant IAM permissions to the backup store, if it is configured
+	if b.Role == kops.InstanceGroupRoleMaster {
+		backupStores := sets.NewString()
+		for _, c := range b.Cluster.Spec.EtcdClusters {
+			if c.Backups == nil || c.Backups.BackupStore == "" || backupStores.Has(c.Backups.BackupStore) {
+				continue
+			}
+			backupStore := c.Backups.BackupStore
+
+			vfsPath, err := vfs.Context.BuildVfsPath(backupStore)
+			if err != nil {
+				return nil, fmt.Errorf("cannot parse VFS path %q: %v", backupStore, err)
+			}
+
+			if s3Path, ok := vfsPath.(*vfs.S3Path); ok {
+				iamS3Path := s3Path.Bucket() + "/" + s3Path.Key()
+				iamS3Path = strings.TrimSuffix(iamS3Path, "/")
+
+				p.Statement = append(p.Statement, &Statement{
+					Sid:    "kopsEtcdBackups",
+					Effect: StatementEffectAllow,
+					Action: stringorslice.Slice([]string{"s3:GetObject", "s3:DeleteObject", "s3:PutObject"}),
+					Resource: stringorslice.Of(
+						strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/*"}, ""),
+					),
+				})
+			} else {
+				glog.Warningf("unknown backup store, can't apply IAM policy: %q", backupStore)
+			}
+
+			backupStores.Insert(backupStore)
+		}
+	}
+
 	return p, nil
 }
 

protokube/cmd/protokube/main.go

@@ -62,7 +62,7 @@ func run() error {
 	var applyTaints, initializeRBAC, containerized, master bool
 	var cloud, clusterID, dnsServer, dnsProviderID, dnsInternalSuffix, gossipSecret, gossipListen string
 	var flagChannels, tlsCert, tlsKey, tlsCA, peerCert, peerKey, peerCA string
-	var etcdImageSource, etcdElectionTimeout, etcdHeartbeatInterval string
+	var etcdBackupImage, etcdBackupStore, etcdImageSource, etcdElectionTimeout, etcdHeartbeatInterval string
 
 	flag.BoolVar(&applyTaints, "apply-taints", applyTaints, "Apply taints to nodes based on the role")
 	flag.BoolVar(&containerized, "containerized", containerized, "Set if we are running containerized.")
@@ -82,6 +82,8 @@ func run() error {
 	flag.StringVar(&tlsKey, "tls-key", tlsKey, "Path to a file containing the private key for etcd server")
 	flags.StringSliceVarP(&zones, "zone", "z", []string{}, "Configure permitted zones and their mappings")
 	flags.StringVar(&dnsProviderID, "dns", "aws-route53", "DNS provider we should use (aws-route53, google-clouddns, coredns)")
+	flags.StringVar(&etcdBackupImage, "etcd-backup-image", "", "Set to override the image for (experimental) etcd backups")
+	flags.StringVar(&etcdBackupStore, "etcd-backup-store", "", "Set to enable (experimental) etcd backups")
 	flags.StringVar(&etcdImageSource, "etcd-image", "k8s.gcr.io/etcd:2.2.1", "Etcd Source Container Registry")
 	flags.StringVar(&etcdElectionTimeout, "etcd-election-timeout", etcdElectionTimeout, "time in ms for an election to timeout")
 	flags.StringVar(&etcdHeartbeatInterval, "etcd-heartbeat-interval", etcdHeartbeatInterval, "time in ms of a heartbeat interval")
@@ -295,6 +297,8 @@ func run() error {
 		ApplyTaints:           applyTaints,
 		Channels:              channels,
 		DNS:                   dnsProvider,
+		EtcdBackupImage:       etcdBackupImage,
+		EtcdBackupStore:       etcdBackupStore,
 		EtcdImageSource:       etcdImageSource,
 		EtcdElectionTimeout:   etcdElectionTimeout,
 		EtcdHeartbeatInterval: etcdHeartbeatInterval,

protokube/pkg/etcd/cluster_spec.go

@@ -23,7 +23,7 @@ import (
 
 // EtcdClusterSpec is configuration for the etcd cluster
 type EtcdClusterSpec struct {
-	// ClusterKey is the initial cluster key
+	// ClusterKey is a key that identifies the etcd cluster (main or events)
 	ClusterKey string `json:"clusterKey,omitempty"`
 	// NodeName is my nodename in the cluster
 	NodeName string `json:"nodeName,omitempty"`

protokube/pkg/protokube/etcd_cluster.go

@@ -47,9 +47,9 @@ type EtcdCluster struct {
 	ImageSource string
 	// LogFile is the location of the logfile
 	LogFile string
-	// Me represents myself
+	// Me is the node that we will be in the cluster
 	Me *EtcdNode
-	// Nodes is a list of nodes in the cluster
+	// Nodes is a list of nodes in the cluster (including the self-node, Me)
 	Nodes []*EtcdNode
 	// PeerPort is the port for peers to connect
 	PeerPort int
@@ -77,6 +77,10 @@ type EtcdCluster struct {
 	ElectionTimeout string
 	// HeartbeatInterval is the heartbeat interval
 	HeartbeatInterval string
+	// BackupImage is the image to use for backing up etcd
+	BackupImage string
+	// BackupStore is a VFS path for backing up etcd
+	BackupStore string
 }
 
 // EtcdNode is a definition for the etcd node
@@ -134,6 +138,9 @@ func newEtcdController(kubeBoot *KubeBoot, v *Volume, spec *etcd.EtcdClusterSpec
 		return nil, fmt.Errorf("unknown etcd cluster key %q", spec.ClusterKey)
 	}
 
+	cluster.BackupImage = kubeBoot.EtcdBackupImage
+	cluster.BackupStore = kubeBoot.EtcdBackupStore
+
 	k.cluster = cluster
 
 	return k, nil

protokube/pkg/protokube/etcd_manifest.go

@@ -142,6 +142,11 @@ func BuildEtcdManifest(c *EtcdCluster) *v1.Pod {
 		pod.Spec.Containers = append(pod.Spec.Containers, container)
 	}
 
+	if c.BackupStore != "" && c.BackupImage != "" {
+		backupContainer := buildEtcdBackupManagerContainer(c)
+		pod.Spec.Containers = append(pod.Spec.Containers, *backupContainer)
+	}
+
 	kubemanifest.MarkPodAsCritical(pod)
 
 	return pod
@@ -233,3 +238,42 @@ func buildCertificateDirectories(c *EtcdCluster) []string {
 func notEmpty(v string) bool {
 	return v != ""
 }
+
+// buildEtcdBackupManagerContainer builds a container for the standalone etcd backup manager
+func buildEtcdBackupManagerContainer(c *EtcdCluster) *v1.Container {
+	command := []string{"/etcd-backup"}
+	command = append(command, "--backup-store", c.BackupStore)
+	command = append(command, "--cluster-name", c.ClusterName)
+	command = append(command, "--data-dir", "/var/etcd/"+c.DataDirName)
+
+	container := v1.Container{
+		Name:    "etcd-backup",
+		Image:   c.BackupImage,
+		Command: command,
+	}
+
+	// TODO: TLS options
+	// TODO: Liveness probe?
+
+	// volume should already have been registered
+	container.VolumeMounts = append(container.VolumeMounts, v1.VolumeMount{
+		Name:      "varetcdata",
+		MountPath: "/var/etcd/" + c.DataDirName,
+		ReadOnly:  false,
+	})
+
+	if c.isTLS() {
+		for _, dirname := range buildCertificateDirectories(c) {
+			normalized := strings.Replace(dirname, "/", "", -1)
+
+			// pod volume already registered for etcd container above
+			container.VolumeMounts = append(container.VolumeMounts, v1.VolumeMount{
+				Name:      normalized,
+				MountPath: dirname,
+				ReadOnly:  true,
+			})
+		}
+	}
+
+	return &container
+}

protokube/pkg/protokube/kube_boot.go

@@ -48,6 +48,10 @@ type KubeBoot struct {
 	DNS DNSProvider
 	// ModelDir is the model directory
 	ModelDir string
+	// EtcdBackupImage is the image to use for backing up etcd
+	EtcdBackupImage string
+	// EtcdBackupStore is the VFS path to which we should backup etcd
+	EtcdBackupStore string
 	// Etcd container registry location.
 	EtcdImageSource string
 	// EtcdElectionTimeout is is the leader election timeout

tests/integration/update_cluster/additional_user-data/cloudformation.json.extracted.yaml

@@ -159,8 +159,10 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1amastersadditionaluserda
   encryptionConfig: null
   etcdClusters:
     events:
+      image: k8s.gcr.io/etcd:2.2.1
       version: 2.2.1
     main:
+      image: k8s.gcr.io/etcd:2.2.1
       version: 2.2.1
   kubeAPIServer:
     address: 127.0.0.1

tests/integration/update_cluster/minimal-cloudformation/cloudformation.json.extracted.yaml

@@ -150,8 +150,10 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1amastersminimalexampleco
   encryptionConfig: null
   etcdClusters:
     events:
+      image: k8s.gcr.io/etcd:2.2.1
       version: 2.2.1
     main:
+      image: k8s.gcr.io/etcd:2.2.1
       version: 2.2.1
   kubeAPIServer:
     address: 127.0.0.1