diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index 6336b3ca53..a3f848d5d3 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -487,7 +487,7 @@ func TestPrivateCiliumAdvanced(t *testing.T) { newIntegrationTest("privateciliumadvanced.example.com", "privateciliumadvanced"). withPrivate(). withCiliumEtcd(). - withManagedFiles("etcd-cluster-spec-cilium", "manifests-etcdmanager-cilium"). + withManagedFiles("etcd-cluster-spec-cilium", "manifests-etcdmanager-cilium-master-us-test-1a"). withAddons(ciliumAddon, dnsControllerAddon). runTestTerraformAWS(t) newIntegrationTest("privateciliumadvanced.example.com", "privateciliumadvanced"). @@ -1202,8 +1202,6 @@ func (i *integrationTest) runTestTerraformAWS(t *testing.T) { "aws_s3_object_etcd-cluster-spec-events_content", "aws_s3_object_etcd-cluster-spec-main_content", "aws_s3_object_kops-version.txt_content", - "aws_s3_object_manifests-etcdmanager-events_content", - "aws_s3_object_manifests-etcdmanager-main_content", "aws_s3_object_manifests-static-kube-apiserver-healthcheck_content", "aws_s3_object_nodeupconfig-nodes_content", "aws_s3_object_"+i.clusterName+"-addons-bootstrap_content", @@ -1234,6 +1232,8 @@ func (i *integrationTest) runTestTerraformAWS(t *testing.T) { for j := 0; j < i.zones; j++ { zone := "us-test-1" + string([]byte{byte('a') + byte(j)}) expectedFilenames = append(expectedFilenames, + "aws_s3_object_manifests-etcdmanager-events-master-"+zone+"_content", + "aws_s3_object_manifests-etcdmanager-main-master-"+zone+"_content", "aws_s3_object_nodeupconfig-master-"+zone+"_content", "aws_launch_template_master-"+zone+".masters."+i.clusterName+"_user_data") } @@ -1333,8 +1333,6 @@ func (i *integrationTest) runTestTerraformGCE(t *testing.T) { "aws_s3_object_etcd-cluster-spec-events_content", "aws_s3_object_etcd-cluster-spec-main_content", "aws_s3_object_kops-version.txt_content", - "aws_s3_object_manifests-etcdmanager-events_content", - "aws_s3_object_manifests-etcdmanager-main_content", "aws_s3_object_manifests-static-kube-apiserver-healthcheck_content", "aws_s3_object_nodeupconfig-nodes_content", "aws_s3_object_"+i.clusterName+"-addons-bootstrap_content", @@ -1349,6 +1347,8 @@ func (i *integrationTest) runTestTerraformGCE(t *testing.T) { zone := "us-test1-" + string([]byte{byte('a') + byte(j)}) prefix := "google_compute_instance_template_master-" + zone + "-" + gce.SafeClusterName(i.clusterName) + "_metadata_" + expectedFilenames = append(expectedFilenames, "aws_s3_object_manifests-etcdmanager-events-master-"+zone+"_content") + expectedFilenames = append(expectedFilenames, "aws_s3_object_manifests-etcdmanager-main-master-"+zone+"_content") expectedFilenames = append(expectedFilenames, "aws_s3_object_nodeupconfig-master-"+zone+"_content") expectedFilenames = append(expectedFilenames, prefix+"startup-script") expectedFilenames = append(expectedFilenames, prefix+"ssh-keys") diff --git a/pkg/model/components/etcdmanager/model.go b/pkg/model/components/etcdmanager/model.go index 2576a70c3f..8fd9290e1e 100644 --- a/pkg/model/components/etcdmanager/model.go +++ b/pkg/model/components/etcdmanager/model.go @@ -57,9 +57,6 @@ var _ fi.ModelBuilder = &EtcdManagerBuilder{} // Build creates the tasks func (b *EtcdManagerBuilder) Build(c *fi.ModelBuilderContext) error { for _, etcdCluster := range b.Cluster.Spec.EtcdClusters { - name := etcdCluster.Name - version := etcdCluster.Version - backupStore := "" if etcdCluster.Backups != nil { backupStore = etcdCluster.Backups.BackupStore @@ -68,25 +65,29 @@ func (b *EtcdManagerBuilder) Build(c *fi.ModelBuilderContext) error { return fmt.Errorf("backupStore must be set for use with etcd-manager") } - manifest, err := b.buildManifest(etcdCluster) - if err != nil { - return err - } + for _, member := range etcdCluster.Members { + instanceGroupName := fi.StringValue(member.InstanceGroup) + manifest, err := b.buildManifest(etcdCluster, instanceGroupName) + if err != nil { + return err + } - manifestYAML, err := k8scodecs.ToVersionedYaml(manifest) - if err != nil { - return fmt.Errorf("error marshaling manifest to yaml: %v", err) - } + manifestYAML, err := k8scodecs.ToVersionedYaml(manifest) + if err != nil { + return fmt.Errorf("error marshaling manifest to yaml: %v", err) + } - c.AddTask(&fitasks.ManagedFile{ - Contents: fi.NewBytesResource(manifestYAML), - Lifecycle: b.Lifecycle, - Location: fi.String("manifests/etcd/" + name + ".yaml"), - Name: fi.String("manifests-etcdmanager-" + name), - }) + name := fmt.Sprintf("%s-%s", etcdCluster.Name, instanceGroupName) + c.AddTask(&fitasks.ManagedFile{ + Contents: fi.NewBytesResource(manifestYAML), + Lifecycle: b.Lifecycle, + Location: fi.String("manifests/etcd/" + name + ".yaml"), + Name: fi.String("manifests-etcdmanager-" + name), + }) + } info := &etcdClusterSpec{ - EtcdVersion: version, + EtcdVersion: etcdCluster.Version, MemberCount: int32(len(etcdCluster.Members)), } @@ -108,7 +109,7 @@ func (b *EtcdManagerBuilder) Build(c *fi.ModelBuilderContext) error { Base: fi.String(backupStore), // TODO: We need this to match the backup base (currently) Location: fi.String(location + "/control/etcd-cluster-spec"), - Name: fi.String("etcd-cluster-spec-" + name), + Name: fi.String("etcd-cluster-spec-" + etcdCluster.Name), }) // We create a CA keypair to enable secure communication @@ -166,8 +167,8 @@ type etcdClusterSpec struct { EtcdVersion string `json:"etcdVersion,omitempty"` } -func (b *EtcdManagerBuilder) buildManifest(etcdCluster kops.EtcdClusterSpec) (*v1.Pod, error) { - return b.buildPod(etcdCluster) +func (b *EtcdManagerBuilder) buildManifest(etcdCluster kops.EtcdClusterSpec, instanceGroupName string) (*v1.Pod, error) { + return b.buildPod(etcdCluster, instanceGroupName) } // Until we introduce the bundle, we hard-code the manifest @@ -214,7 +215,7 @@ spec: ` // buildPod creates the pod spec, based on the EtcdClusterSpec -func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec) (*v1.Pod, error) { +func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instanceGroupName string) (*v1.Pod, error) { var pod *v1.Pod var container *v1.Container diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index 5371a1183d..be823258bb 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -1135,7 +1135,7 @@ type nodeUpConfigBuilder struct { channels []string configBase vfs.Path cluster *kops.Cluster - etcdManifests map[kops.InstanceGroupRole][]string + etcdManifests map[string][]string images map[kops.InstanceGroupRole]map[architectures.Architecture][]*nodeup.Image protokubeAsset map[architectures.Architecture][]*mirrors.MirroredAsset channelsAsset map[architectures.Architecture][]*mirrors.MirroredAsset @@ -1156,7 +1156,7 @@ func newNodeUpConfigBuilder(cluster *kops.Cluster, assetBuilder *assets.AssetBui channels = append(channels, cluster.Spec.Addons[i].Manifest) } - etcdManifests := map[kops.InstanceGroupRole][]string{} + etcdManifests := map[string][]string{} images := map[kops.InstanceGroupRole]map[architectures.Architecture][]*nodeup.Image{} protokubeAsset := map[architectures.Architecture][]*mirrors.MirroredAsset{} channelsAsset := map[architectures.Architecture][]*mirrors.MirroredAsset{} @@ -1266,8 +1266,11 @@ func newNodeUpConfigBuilder(cluster *kops.Cluster, assetBuilder *assets.AssetBui if isMaster { for _, etcdCluster := range cluster.Spec.EtcdClusters { - p := configBase.Join("manifests/etcd/" + etcdCluster.Name + ".yaml").Path() - etcdManifests[role] = append(etcdManifests[role], p) + for _, member := range etcdCluster.Members { + instanceGroup := fi.StringValue(member.InstanceGroup) + etcdManifest := fmt.Sprintf("manifests/etcd/%s-%s.yaml", etcdCluster.Name, instanceGroup) + etcdManifests[instanceGroup] = append(etcdManifests[instanceGroup], configBase.Join(etcdManifest).Path()) + } } } } @@ -1451,7 +1454,7 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAddit config.Images = n.images[role] config.Channels = n.channels - config.EtcdManifests = n.etcdManifests[role] + config.EtcdManifests = n.etcdManifests[ig.Name] if ig.Spec.Containerd != nil || cluster.Spec.ContainerRuntime == "containerd" { config.ContainerdConfig = n.buildContainerdConfig(ig)