Create etcd-manager config for each instance group

cmd/kops/integration_test.go

@@ -487,7 +487,7 @@ func TestPrivateCiliumAdvanced(t *testing.T) {
 	newIntegrationTest("privateciliumadvanced.example.com", "privateciliumadvanced").
 		withPrivate().
 		withCiliumEtcd().
-		withManagedFiles("etcd-cluster-spec-cilium", "manifests-etcdmanager-cilium").
+		withManagedFiles("etcd-cluster-spec-cilium", "manifests-etcdmanager-cilium-master-us-test-1a").
 		withAddons(ciliumAddon, dnsControllerAddon).
 		runTestTerraformAWS(t)
 	newIntegrationTest("privateciliumadvanced.example.com", "privateciliumadvanced").
@@ -1202,8 +1202,6 @@ func (i *integrationTest) runTestTerraformAWS(t *testing.T) {
 		"aws_s3_object_etcd-cluster-spec-events_content",
 		"aws_s3_object_etcd-cluster-spec-main_content",
 		"aws_s3_object_kops-version.txt_content",
-		"aws_s3_object_manifests-etcdmanager-events_content",
-		"aws_s3_object_manifests-etcdmanager-main_content",
 		"aws_s3_object_manifests-static-kube-apiserver-healthcheck_content",
 		"aws_s3_object_nodeupconfig-nodes_content",
 		"aws_s3_object_"+i.clusterName+"-addons-bootstrap_content",
@@ -1234,6 +1232,8 @@ func (i *integrationTest) runTestTerraformAWS(t *testing.T) {
 	for j := 0; j < i.zones; j++ {
 		zone := "us-test-1" + string([]byte{byte('a') + byte(j)})
 		expectedFilenames = append(expectedFilenames,
+			"aws_s3_object_manifests-etcdmanager-events-master-"+zone+"_content",
+			"aws_s3_object_manifests-etcdmanager-main-master-"+zone+"_content",
 			"aws_s3_object_nodeupconfig-master-"+zone+"_content",
 			"aws_launch_template_master-"+zone+".masters."+i.clusterName+"_user_data")
 	}
@@ -1333,8 +1333,6 @@ func (i *integrationTest) runTestTerraformGCE(t *testing.T) {
 		"aws_s3_object_etcd-cluster-spec-events_content",
 		"aws_s3_object_etcd-cluster-spec-main_content",
 		"aws_s3_object_kops-version.txt_content",
-		"aws_s3_object_manifests-etcdmanager-events_content",
-		"aws_s3_object_manifests-etcdmanager-main_content",
 		"aws_s3_object_manifests-static-kube-apiserver-healthcheck_content",
 		"aws_s3_object_nodeupconfig-nodes_content",
 		"aws_s3_object_"+i.clusterName+"-addons-bootstrap_content",
@@ -1349,6 +1347,8 @@ func (i *integrationTest) runTestTerraformGCE(t *testing.T) {
 		zone := "us-test1-" + string([]byte{byte('a') + byte(j)})
 		prefix := "google_compute_instance_template_master-" + zone + "-" + gce.SafeClusterName(i.clusterName) + "_metadata_"
 
+		expectedFilenames = append(expectedFilenames, "aws_s3_object_manifests-etcdmanager-events-master-"+zone+"_content")
+		expectedFilenames = append(expectedFilenames, "aws_s3_object_manifests-etcdmanager-main-master-"+zone+"_content")
 		expectedFilenames = append(expectedFilenames, "aws_s3_object_nodeupconfig-master-"+zone+"_content")
 		expectedFilenames = append(expectedFilenames, prefix+"startup-script")
 		expectedFilenames = append(expectedFilenames, prefix+"ssh-keys")

pkg/model/components/etcdmanager/model.go

@@ -57,9 +57,6 @@ var _ fi.ModelBuilder = &EtcdManagerBuilder{}
 // Build creates the tasks
 func (b *EtcdManagerBuilder) Build(c *fi.ModelBuilderContext) error {
 	for _, etcdCluster := range b.Cluster.Spec.EtcdClusters {
-		name := etcdCluster.Name
-		version := etcdCluster.Version
-
 		backupStore := ""
 		if etcdCluster.Backups != nil {
 			backupStore = etcdCluster.Backups.BackupStore
@@ -68,25 +65,29 @@ func (b *EtcdManagerBuilder) Build(c *fi.ModelBuilderContext) error {
 			return fmt.Errorf("backupStore must be set for use with etcd-manager")
 		}
 
-		manifest, err := b.buildManifest(etcdCluster)
+		for _, member := range etcdCluster.Members {
-		if err != nil {
+			instanceGroupName := fi.StringValue(member.InstanceGroup)
-			return err
+			manifest, err := b.buildManifest(etcdCluster, instanceGroupName)
-		}
+			if err != nil {
+				return err
+			}
 
-		manifestYAML, err := k8scodecs.ToVersionedYaml(manifest)
+			manifestYAML, err := k8scodecs.ToVersionedYaml(manifest)
-		if err != nil {
+			if err != nil {
-			return fmt.Errorf("error marshaling manifest to yaml: %v", err)
+				return fmt.Errorf("error marshaling manifest to yaml: %v", err)
-		}
+			}
 
-		c.AddTask(&fitasks.ManagedFile{
+			name := fmt.Sprintf("%s-%s", etcdCluster.Name, instanceGroupName)
-			Contents:  fi.NewBytesResource(manifestYAML),
+			c.AddTask(&fitasks.ManagedFile{
-			Lifecycle: b.Lifecycle,
+				Contents:  fi.NewBytesResource(manifestYAML),
-			Location:  fi.String("manifests/etcd/" + name + ".yaml"),
+				Lifecycle: b.Lifecycle,
-			Name:      fi.String("manifests-etcdmanager-" + name),
+				Location:  fi.String("manifests/etcd/" + name + ".yaml"),
-		})
+				Name:      fi.String("manifests-etcdmanager-" + name),
+			})
+		}
 
 		info := &etcdClusterSpec{
-			EtcdVersion: version,
+			EtcdVersion: etcdCluster.Version,
 			MemberCount: int32(len(etcdCluster.Members)),
 		}
 
@@ -108,7 +109,7 @@ func (b *EtcdManagerBuilder) Build(c *fi.ModelBuilderContext) error {
 			Base:      fi.String(backupStore),
 			// TODO: We need this to match the backup base (currently)
 			Location: fi.String(location + "/control/etcd-cluster-spec"),
-			Name:     fi.String("etcd-cluster-spec-" + name),
+			Name:     fi.String("etcd-cluster-spec-" + etcdCluster.Name),
 		})
 
 		// We create a CA keypair to enable secure communication
@@ -166,8 +167,8 @@ type etcdClusterSpec struct {
 	EtcdVersion string `json:"etcdVersion,omitempty"`
 }
 
-func (b *EtcdManagerBuilder) buildManifest(etcdCluster kops.EtcdClusterSpec) (*v1.Pod, error) {
+func (b *EtcdManagerBuilder) buildManifest(etcdCluster kops.EtcdClusterSpec, instanceGroupName string) (*v1.Pod, error) {
-	return b.buildPod(etcdCluster)
+	return b.buildPod(etcdCluster, instanceGroupName)
 }
 
 // Until we introduce the bundle, we hard-code the manifest
@@ -214,7 +215,7 @@ spec:
 `
 
 // buildPod creates the pod spec, based on the EtcdClusterSpec
-func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec) (*v1.Pod, error) {
+func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instanceGroupName string) (*v1.Pod, error) {
 	var pod *v1.Pod
 	var container *v1.Container
 

upup/pkg/fi/cloudup/apply_cluster.go

@@ -1135,7 +1135,7 @@ type nodeUpConfigBuilder struct {
 	channels                   []string
 	configBase                 vfs.Path
 	cluster                    *kops.Cluster
-	etcdManifests              map[kops.InstanceGroupRole][]string
+	etcdManifests              map[string][]string
 	images                     map[kops.InstanceGroupRole]map[architectures.Architecture][]*nodeup.Image
 	protokubeAsset             map[architectures.Architecture][]*mirrors.MirroredAsset
 	channelsAsset              map[architectures.Architecture][]*mirrors.MirroredAsset
@@ -1156,7 +1156,7 @@ func newNodeUpConfigBuilder(cluster *kops.Cluster, assetBuilder *assets.AssetBui
 		channels = append(channels, cluster.Spec.Addons[i].Manifest)
 	}
 
-	etcdManifests := map[kops.InstanceGroupRole][]string{}
+	etcdManifests := map[string][]string{}
 	images := map[kops.InstanceGroupRole]map[architectures.Architecture][]*nodeup.Image{}
 	protokubeAsset := map[architectures.Architecture][]*mirrors.MirroredAsset{}
 	channelsAsset := map[architectures.Architecture][]*mirrors.MirroredAsset{}
@@ -1266,8 +1266,11 @@ func newNodeUpConfigBuilder(cluster *kops.Cluster, assetBuilder *assets.AssetBui
 
 		if isMaster {
 			for _, etcdCluster := range cluster.Spec.EtcdClusters {
-				p := configBase.Join("manifests/etcd/" + etcdCluster.Name + ".yaml").Path()
+				for _, member := range etcdCluster.Members {
-				etcdManifests[role] = append(etcdManifests[role], p)
+					instanceGroup := fi.StringValue(member.InstanceGroup)
+					etcdManifest := fmt.Sprintf("manifests/etcd/%s-%s.yaml", etcdCluster.Name, instanceGroup)
+					etcdManifests[instanceGroup] = append(etcdManifests[instanceGroup], configBase.Join(etcdManifest).Path())
+				}
 			}
 		}
 	}
@@ -1451,7 +1454,7 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAddit
 
 	config.Images = n.images[role]
 	config.Channels = n.channels
-	config.EtcdManifests = n.etcdManifests[role]
+	config.EtcdManifests = n.etcdManifests[ig.Name]
 
 	if ig.Spec.Containerd != nil || cluster.Spec.ContainerRuntime == "containerd" {
 		config.ContainerdConfig = n.buildContainerdConfig(ig)