diff --git a/cloudmock/aws/mockautoscaling/BUILD.bazel b/cloudmock/aws/mockautoscaling/BUILD.bazel index 37e2c943a2..7809d50b6f 100644 --- a/cloudmock/aws/mockautoscaling/BUILD.bazel +++ b/cloudmock/aws/mockautoscaling/BUILD.bazel @@ -7,7 +7,6 @@ go_library( "attach.go", "ec2shim.go", "group.go", - "launchconfigurations.go", "tags.go", ], importpath = "k8s.io/kops/cloudmock/aws/mockautoscaling", diff --git a/cloudmock/aws/mockautoscaling/launchconfigurations.go b/cloudmock/aws/mockautoscaling/launchconfigurations.go deleted file mode 100644 index b77d682f9a..0000000000 --- a/cloudmock/aws/mockautoscaling/launchconfigurations.go +++ /dev/null @@ -1,143 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mockautoscaling - -import ( - "fmt" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/autoscaling" - "k8s.io/klog/v2" -) - -func (m *MockAutoscaling) DescribeLaunchConfigurations(*autoscaling.DescribeLaunchConfigurationsInput) (*autoscaling.DescribeLaunchConfigurationsOutput, error) { - klog.Fatalf("Not implemented") - return nil, nil -} -func (m *MockAutoscaling) DescribeLaunchConfigurationsWithContext(aws.Context, *autoscaling.DescribeLaunchConfigurationsInput, ...request.Option) (*autoscaling.DescribeLaunchConfigurationsOutput, error) { - klog.Fatalf("Not implemented") - return nil, nil -} -func (m *MockAutoscaling) DescribeLaunchConfigurationsRequest(*autoscaling.DescribeLaunchConfigurationsInput) (*request.Request, *autoscaling.DescribeLaunchConfigurationsOutput) { - klog.Fatalf("Not implemented") - return nil, nil -} - -func (m *MockAutoscaling) DescribeLaunchConfigurationsPages(request *autoscaling.DescribeLaunchConfigurationsInput, callback func(*autoscaling.DescribeLaunchConfigurationsOutput, bool) bool) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - if request.LaunchConfigurationNames != nil { - klog.Fatalf("LaunchConfigurationNames not implemented") - } - if request.MaxRecords != nil { - klog.Fatalf("MaxRecords not implemented") - } - if request.NextToken != nil { - klog.Fatalf("NextToken not implemented") - } - - // For the mock, we just send everything in one page - page := &autoscaling.DescribeLaunchConfigurationsOutput{} - - for _, lc := range m.LaunchConfigurations { - page.LaunchConfigurations = append(page.LaunchConfigurations, lc) - } - - callback(page, false) - - return nil -} -func (m *MockAutoscaling) DescribeLaunchConfigurationsPagesWithContext(aws.Context, *autoscaling.DescribeLaunchConfigurationsInput, func(*autoscaling.DescribeLaunchConfigurationsOutput, bool) bool, ...request.Option) error { - klog.Fatalf("Not implemented") - return nil -} - -func (m *MockAutoscaling) CreateLaunchConfiguration(request *autoscaling.CreateLaunchConfigurationInput) (*autoscaling.CreateLaunchConfigurationOutput, error) { - m.mutex.Lock() - defer m.mutex.Unlock() - - klog.Infof("CreateLaunchConfiguration: %v", request) - - createdTime := time.Now().UTC() - lc := &autoscaling.LaunchConfiguration{ - AssociatePublicIpAddress: request.AssociatePublicIpAddress, - BlockDeviceMappings: request.BlockDeviceMappings, - ClassicLinkVPCId: request.ClassicLinkVPCId, - ClassicLinkVPCSecurityGroups: request.ClassicLinkVPCSecurityGroups, - CreatedTime: &createdTime, - EbsOptimized: request.EbsOptimized, - IamInstanceProfile: request.IamInstanceProfile, - ImageId: request.ImageId, - InstanceMonitoring: request.InstanceMonitoring, - InstanceType: request.InstanceType, - KernelId: request.KernelId, - KeyName: request.KeyName, - // LaunchConfigurationARN: request.LaunchConfigurationARN, - LaunchConfigurationName: request.LaunchConfigurationName, - PlacementTenancy: request.PlacementTenancy, - RamdiskId: request.RamdiskId, - SecurityGroups: request.SecurityGroups, - SpotPrice: request.SpotPrice, - UserData: request.UserData, - } - - if m.LaunchConfigurations == nil { - m.LaunchConfigurations = make(map[string]*autoscaling.LaunchConfiguration) - } - if m.LaunchConfigurations[*lc.LaunchConfigurationName] != nil { - return nil, fmt.Errorf("duplicate LaunchConfigurationName %s", *lc.LaunchConfigurationName) - } - m.LaunchConfigurations[*lc.LaunchConfigurationName] = lc - - return &autoscaling.CreateLaunchConfigurationOutput{}, nil -} -func (m *MockAutoscaling) CreateLaunchConfigurationWithContext(aws.Context, *autoscaling.CreateLaunchConfigurationInput, ...request.Option) (*autoscaling.CreateLaunchConfigurationOutput, error) { - klog.Fatalf("Not implemented") - return nil, nil -} -func (m *MockAutoscaling) CreateLaunchConfigurationRequest(*autoscaling.CreateLaunchConfigurationInput) (*request.Request, *autoscaling.CreateLaunchConfigurationOutput) { - klog.Fatalf("Not implemented") - return nil, nil -} - -func (m *MockAutoscaling) DeleteLaunchConfiguration(request *autoscaling.DeleteLaunchConfigurationInput) (*autoscaling.DeleteLaunchConfigurationOutput, error) { - m.mutex.Lock() - defer m.mutex.Unlock() - - klog.Infof("DeleteLaunchConfiguration: %v", request) - - id := aws.StringValue(request.LaunchConfigurationName) - o := m.LaunchConfigurations[id] - if o == nil { - return nil, fmt.Errorf("LaunchConfiguration %q not found", id) - } - delete(m.LaunchConfigurations, id) - - return &autoscaling.DeleteLaunchConfigurationOutput{}, nil -} - -func (m *MockAutoscaling) DeleteLaunchConfigurationWithContext(aws.Context, *autoscaling.DeleteLaunchConfigurationInput, ...request.Option) (*autoscaling.DeleteLaunchConfigurationOutput, error) { - klog.Fatalf("Not implemented") - return nil, nil -} -func (m *MockAutoscaling) DeleteLaunchConfigurationRequest(*autoscaling.DeleteLaunchConfigurationInput) (*request.Request, *autoscaling.DeleteLaunchConfigurationOutput) { - klog.Fatalf("Not implemented") - return nil, nil -} diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index 4ef8ca5601..14a75a065e 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -60,7 +60,6 @@ type integrationTest struct { expectPolicies bool // expectServiceAccountRoles is true if we expect to assign per-ServiceAccount IAM roles (instead of just using the node roles) expectServiceAccountRoles bool - launchConfiguration bool lifecycleOverrides []string sshKey bool // caKey is true if we should use a provided ca.crt & ca.key as our CA @@ -128,11 +127,6 @@ func (i *integrationTest) withPrivate() *integrationTest { return i } -func (i *integrationTest) withLaunchConfiguration() *integrationTest { - i.launchConfiguration = true - return i -} - func (i *integrationTest) withBastionUserData() *integrationTest { i.bastionUserData = true return i @@ -347,18 +341,6 @@ func TestDockerCustom(t *testing.T) { newIntegrationTest("docker.example.com", "docker-custom").runTestCloudformation(t) } -// TestLaunchConfigurationASG tests ASGs using launch configurations instead of launch templates -func TestLaunchConfigurationASG(t *testing.T) { - featureflag.ParseFlags("-EnableLaunchTemplates") - unsetFeatureFlags := func() { - featureflag.ParseFlags("+EnableLaunchTemplates") - } - defer unsetFeatureFlags() - - newIntegrationTest("launchtemplates.example.com", "launch_templates").withZones(3).withLaunchConfiguration().runTestTerraformAWS(t) - newIntegrationTest("launchtemplates.example.com", "launch_templates").withZones(3).withLaunchConfiguration().runTestCloudformation(t) -} - // TestPublicJWKS runs a simple configuration, but with UseServiceAccountIAM and PublicJWKS enabled func TestPublicJWKS(t *testing.T) { featureflag.ParseFlags("+UseServiceAccountIAM,+PublicJWKS") @@ -536,22 +518,15 @@ func (i *integrationTest) runTestTerraformAWS(t *testing.T) { expectedFilenames := []string{} - if i.launchConfiguration { - expectedFilenames = append(expectedFilenames, "aws_launch_configuration_nodes."+i.clusterName+"_user_data") - } else { - expectedFilenames = append(expectedFilenames, "aws_launch_template_nodes."+i.clusterName+"_user_data") - } + expectedFilenames = append(expectedFilenames, "aws_launch_template_nodes."+i.clusterName+"_user_data") + if i.sshKey { expectedFilenames = append(expectedFilenames, "aws_key_pair_kubernetes."+i.clusterName+"-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key") } for j := 0; j < i.zones; j++ { zone := "us-test-1" + string([]byte{byte('a') + byte(j)}) - if featureflag.EnableLaunchTemplates.Enabled() { - expectedFilenames = append(expectedFilenames, "aws_launch_template_master-"+zone+".masters."+i.clusterName+"_user_data") - } else { - expectedFilenames = append(expectedFilenames, "aws_launch_configuration_master-"+zone+".masters."+i.clusterName+"_user_data") - } + expectedFilenames = append(expectedFilenames, "aws_launch_template_master-"+zone+".masters."+i.clusterName+"_user_data") } if i.expectPolicies { @@ -611,12 +586,12 @@ func (i *integrationTest) runTestPhase(t *testing.T, phase cloudup.Phase) { } } else if phase == cloudup.PhaseCluster { expectedFilenames = []string{ - "aws_launch_configuration_nodes." + i.clusterName + "_user_data", + "aws_launch_template_nodes." + i.clusterName + "_user_data", } for j := 0; j < i.zones; j++ { zone := "us-test-1" + string([]byte{byte('a') + byte(j)}) - s := "aws_launch_configuration_master-" + zone + ".masters." + i.clusterName + "_user_data" + s := "aws_launch_template_master-" + zone + ".masters." + i.clusterName + "_user_data" expectedFilenames = append(expectedFilenames, s) } } diff --git a/k8s/crds/kops.k8s.io_instancegroups.yaml b/k8s/crds/kops.k8s.io_instancegroups.yaml index c0111901d3..2c167fbff5 100644 --- a/k8s/crds/kops.k8s.io_instancegroups.yaml +++ b/k8s/crds/kops.k8s.io_instancegroups.yaml @@ -751,11 +751,8 @@ spec: x-kubernetes-int-or-string: true type: object rootVolumeDeleteOnTermination: - description: 'RootVolumeDeleteOnTermination configures root volume - retention policy upon instance termination. The root volume is deleted - by default. Cluster deletion does not remove retained root volumes. - NOTE: This setting applies only to the Launch Configuration and - does not affect Launch Templates.' + description: RootVolumeDeleteOnTermination is deprecated as of kOps + 1.21 and has no effect type: boolean rootVolumeEncryption: description: RootVolumeEncryption enables EBS root volume encryption diff --git a/pkg/apis/kops/instancegroup.go b/pkg/apis/kops/instancegroup.go index 702b0ed42a..8ce6b468be 100644 --- a/pkg/apis/kops/instancegroup.go +++ b/pkg/apis/kops/instancegroup.go @@ -106,9 +106,7 @@ type InstanceGroupSpec struct { RootVolumeThroughput *int32 `json:"rootVolumeThroughput,omitempty"` // RootVolumeOptimization enables EBS optimization for an instance RootVolumeOptimization *bool `json:"rootVolumeOptimization,omitempty"` - // RootVolumeDeleteOnTermination configures root volume retention policy upon instance termination. - // The root volume is deleted by default. Cluster deletion does not remove retained root volumes. - // NOTE: This setting applies only to the Launch Configuration and does not affect Launch Templates. + // RootVolumeDeleteOnTermination is deprecated as of kOps 1.21 and has no effect RootVolumeDeleteOnTermination *bool `json:"rootVolumeDeleteOnTermination,omitempty"` // RootVolumeEncryption enables EBS root volume encryption for an instance RootVolumeEncryption *bool `json:"rootVolumeEncryption,omitempty"` diff --git a/pkg/apis/kops/v1alpha2/instancegroup.go b/pkg/apis/kops/v1alpha2/instancegroup.go index d49fcf77da..553e5b426e 100644 --- a/pkg/apis/kops/v1alpha2/instancegroup.go +++ b/pkg/apis/kops/v1alpha2/instancegroup.go @@ -103,9 +103,7 @@ type InstanceGroupSpec struct { RootVolumeThroughput *int32 `json:"rootVolumeThroughput,omitempty"` // RootVolumeOptimization enables EBS optimization for an instance RootVolumeOptimization *bool `json:"rootVolumeOptimization,omitempty"` - // RootVolumeDeleteOnTermination configures root volume retention policy upon instance termination. - // The root volume is deleted by default. Cluster deletion does not remove retained root volumes. - // NOTE: This setting applies only to the Launch Configuration and does not affect Launch Templates. + // RootVolumeDeleteOnTermination is deprecated as of kOps 1.21 and has no effect RootVolumeDeleteOnTermination *bool `json:"rootVolumeDeleteOnTermination,omitempty"` // RootVolumeEncryption enables EBS root volume encryption for an instance RootVolumeEncryption *bool `json:"rootVolumeEncryption,omitempty"` diff --git a/pkg/featureflag/featureflag.go b/pkg/featureflag/featureflag.go index 3e936f2987..7a66b72a63 100644 --- a/pkg/featureflag/featureflag.go +++ b/pkg/featureflag/featureflag.go @@ -52,8 +52,6 @@ var ( CacheNodeidentityInfo = New("CacheNodeidentityInfo", Bool(false)) // DNSPreCreate controls whether we pre-create DNS records. DNSPreCreate = New("DNSPreCreate", Bool(true)) - // EnableLaunchTemplates indicates we wish to switch to using launch templates rather than launchconfigurations - EnableLaunchTemplates = New("EnableLaunchTemplates", Bool(true)) //EnableExternalCloudController toggles the use of cloud-controller-manager introduced in v1.7 EnableExternalCloudController = New("EnableExternalCloudController", Bool(false)) // EnableExternalDNS enables external DNS diff --git a/pkg/model/awsmodel/autoscalinggroup.go b/pkg/model/awsmodel/autoscalinggroup.go index 80d7619cb8..83f0a82ca1 100644 --- a/pkg/model/awsmodel/autoscalinggroup.go +++ b/pkg/model/awsmodel/autoscalinggroup.go @@ -72,15 +72,7 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { } } - // @check if his instancegroup is backed by a fleet and override with a launch template - task, err := func() (fi.Task, error) { - switch UseLaunchTemplate(ig) { - case true: - return b.buildLaunchTemplateTask(c, name, ig) - default: - return b.buildLaunchConfigurationTask(c, name, ig) - } - }() + task, err := b.buildLaunchTemplateTask(c, name, ig) if err != nil { return err } @@ -91,14 +83,8 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { if err != nil { return err } - switch UseLaunchTemplate(ig) { - case true: - tsk.LaunchTemplate = task.(*awstasks.LaunchTemplate) - default: - tsk.LaunchConfiguration = task.(*awstasks.LaunchConfiguration) - } + tsk.LaunchTemplate = task c.AddTask(tsk) - } return nil @@ -106,7 +92,7 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { // buildLaunchTemplateTask is responsible for creating the template task into the aws model func (b *AutoscalingGroupModelBuilder) buildLaunchTemplateTask(c *fi.ModelBuilderContext, name string, ig *kops.InstanceGroup) (*awstasks.LaunchTemplate, error) { - lc, err := b.buildLaunchConfigurationTask(c, name, ig) + lc, err := b.buildLaunchTemplateHelper(c, name, ig) if err != nil { return nil, err } @@ -116,9 +102,6 @@ func (b *AutoscalingGroupModelBuilder) buildLaunchTemplateTask(c *fi.ModelBuilde return nil, fmt.Errorf("error building cloud tags: %v", err) } - // @TODO check if there any a better way of doing this .. initially I had a type LaunchTemplate which included - // LaunchConfiguration as an anonymous field, bit given up the task dependency walker works this caused issues, due - // to the creation of a implicit dependency lt := &awstasks.LaunchTemplate{ Name: fi.String(name), Lifecycle: b.Lifecycle, @@ -145,8 +128,8 @@ func (b *AutoscalingGroupModelBuilder) buildLaunchTemplateTask(c *fi.ModelBuilde // rather than the LaunchTemplate or else it returns this error: // You cannot use a launch template that is set to request Spot Instances (InstanceMarketOptions) // when you configure an Auto Scaling group with a mixed instances policy. - if ig.Spec.MixedInstancesPolicy == nil { - lt.SpotPrice = fi.String(lc.SpotPrice) + if ig.Spec.MixedInstancesPolicy == nil && ig.Spec.MaxPrice != nil { + lt.SpotPrice = ig.Spec.MaxPrice } else { lt.SpotPrice = fi.String("") } @@ -186,8 +169,8 @@ func (b *AutoscalingGroupModelBuilder) buildLaunchTemplateTask(c *fi.ModelBuilde return lt, nil } -// buildLaunchConfigurationTask is responsible for building a launch configuration task into the model -func (b *AutoscalingGroupModelBuilder) buildLaunchConfigurationTask(c *fi.ModelBuilderContext, name string, ig *kops.InstanceGroup) (*awstasks.LaunchConfiguration, error) { +// buildLaunchTemplateHelper is responsible for building a launch configuration task into the model +func (b *AutoscalingGroupModelBuilder) buildLaunchTemplateHelper(c *fi.ModelBuilderContext, name string, ig *kops.InstanceGroup) (*awstasks.LaunchTemplate, error) { // @step: lets add the root volume settings volumeSize, err := defaults.DefaultInstanceGroupVolumeSize(ig.Spec.Role) if err != nil { @@ -202,11 +185,6 @@ func (b *AutoscalingGroupModelBuilder) buildLaunchConfigurationTask(c *fi.ModelB volumeType = DefaultLegacyVolumeType } - rootVolumeDeleteOnTermination := DefaultVolumeDeleteOnTermination - if ig.Spec.RootVolumeDeleteOnTermination != nil { - rootVolumeDeleteOnTermination = fi.BoolValue(ig.Spec.RootVolumeDeleteOnTermination) - } - rootVolumeEncryption := DefaultVolumeEncryption if ig.Spec.RootVolumeEncryption != nil { rootVolumeEncryption = fi.BoolValue(ig.Spec.RootVolumeEncryption) @@ -229,19 +207,18 @@ func (b *AutoscalingGroupModelBuilder) buildLaunchConfigurationTask(c *fi.ModelB return nil, fmt.Errorf("unable to find IAM profile link for instance group %q: %w", ig.ObjectMeta.Name, err) } - t := &awstasks.LaunchConfiguration{ - Name: fi.String(name), - Lifecycle: b.Lifecycle, - IAMInstanceProfile: link, - ImageID: fi.String(ig.Spec.Image), - InstanceMonitoring: ig.Spec.DetailedInstanceMonitoring, - InstanceType: fi.String(strings.Split(ig.Spec.MachineType, ",")[0]), - RootVolumeDeleteOnTermination: fi.Bool(rootVolumeDeleteOnTermination), - RootVolumeOptimization: ig.Spec.RootVolumeOptimization, - RootVolumeSize: fi.Int64(int64(volumeSize)), - RootVolumeType: fi.String(volumeType), - RootVolumeEncryption: fi.Bool(rootVolumeEncryption), - SecurityGroups: []*awstasks.SecurityGroup{sgLink}, + t := &awstasks.LaunchTemplate{ + Name: fi.String(name), + Lifecycle: b.Lifecycle, + IAMInstanceProfile: link, + ImageID: fi.String(ig.Spec.Image), + InstanceMonitoring: ig.Spec.DetailedInstanceMonitoring, + InstanceType: fi.String(strings.Split(ig.Spec.MachineType, ",")[0]), + RootVolumeOptimization: ig.Spec.RootVolumeOptimization, + RootVolumeSize: fi.Int64(int64(volumeSize)), + RootVolumeType: fi.String(volumeType), + RootVolumeEncryption: fi.Bool(rootVolumeEncryption), + SecurityGroups: []*awstasks.SecurityGroup{sgLink}, } t.HTTPTokens = fi.String(ec2.LaunchTemplateHttpTokensStateOptional) @@ -346,12 +323,6 @@ func (b *AutoscalingGroupModelBuilder) buildLaunchConfigurationTask(c *fi.ModelB return nil, err } - // @step: set up instance spot pricing - if fi.StringValue(ig.Spec.MaxPrice) != "" { - spotPrice := fi.StringValue(ig.Spec.MaxPrice) - t.SpotPrice = spotPrice - } - // @step: check the subnets are ok and pull together an array for us subnets, err := b.GatherSubnets(ig) if err != nil { diff --git a/pkg/model/awsmodel/context.go b/pkg/model/awsmodel/context.go index 1a0e6db0bd..097a1f3051 100644 --- a/pkg/model/awsmodel/context.go +++ b/pkg/model/awsmodel/context.go @@ -17,8 +17,6 @@ limitations under the License. package awsmodel import ( - "k8s.io/kops/pkg/apis/kops" - "k8s.io/kops/pkg/featureflag" "k8s.io/kops/pkg/model" ) @@ -26,17 +24,3 @@ import ( type AWSModelContext struct { *model.KopsModelContext } - -// UseMixedInstancePolicies indicates if we are using mixed instance policies -func UseMixedInstancePolicies(ig *kops.InstanceGroup) bool { - return ig.Spec.MixedInstancesPolicy != nil -} - -// UseLaunchTemplate checks if we need to use a launch template rather than configuration -func UseLaunchTemplate(ig *kops.InstanceGroup) bool { - if featureflag.EnableLaunchTemplates.Enabled() { - return true - } - - return UseMixedInstancePolicies(ig) -} diff --git a/pkg/resources/aws/aws.go b/pkg/resources/aws/aws.go index da260fae12..58b0481d3a 100644 --- a/pkg/resources/aws/aws.go +++ b/pkg/resources/aws/aws.go @@ -141,15 +141,11 @@ func ListResourcesAWS(cloud awsup.AWSCloud, clusterName string) (map[string]*res id := strings.TrimPrefix(k, "security-group:") securityGroups.Insert(id) } - lcs, err := FindAutoScalingLaunchConfigurations(cloud, securityGroups) - if err != nil { - return nil, err - } lts, err := FindAutoScalingLaunchTemplates(cloud, clusterName) if err != nil { return nil, err } - for _, t := range append(lcs, lts...) { + for _, t := range lts { resourceTrackers[t.Type+":"+t.ID] = t } } @@ -1234,50 +1230,6 @@ func FindAutoScalingLaunchTemplates(cloud fi.Cloud, clusterName string) ([]*reso return list, nil } -// FindAutoScalingLaunchConfigurations finds all launch configurations which has a reference to the security groups -func FindAutoScalingLaunchConfigurations(cloud fi.Cloud, securityGroups sets.String) ([]*resources.Resource, error) { - c := cloud.(awsup.AWSCloud) - - klog.V(2).Infof("Finding all Autoscaling LaunchConfigurations by security group") - var resourceTrackers []*resources.Resource - - request := &autoscaling.DescribeLaunchConfigurationsInput{} - err := c.Autoscaling().DescribeLaunchConfigurationsPages(request, func(p *autoscaling.DescribeLaunchConfigurationsOutput, lastPage bool) bool { - for _, t := range p.LaunchConfigurations { - found := false - for _, sg := range t.SecurityGroups { - if securityGroups.Has(aws.StringValue(sg)) { - found = true - break - } - } - if !found { - continue - } - - resourceTracker := &resources.Resource{ - Name: aws.StringValue(t.LaunchConfigurationName), - ID: aws.StringValue(t.LaunchConfigurationName), - Type: TypeAutoscalingLaunchConfig, - Deleter: DeleteAutoscalingLaunchConfiguration, - } - - var blocks []string - //blocks = append(blocks, TypeAutoscalingLaunchConfig + ":" + aws.StringValue(asg.LaunchConfigurationName)) - - resourceTracker.Blocks = blocks - - resourceTrackers = append(resourceTrackers, resourceTracker) - } - return true - }) - if err != nil { - return nil, fmt.Errorf("error listing autoscaling LaunchConfigurations: %v", err) - } - - return resourceTrackers, nil -} - func FindNatGateways(cloud fi.Cloud, routeTables map[string]*resources.Resource, clusterName string) ([]*resources.Resource, error) { if len(routeTables) == 0 { return nil, nil @@ -1394,21 +1346,6 @@ func DeleteAutoScalingGroupLaunchTemplate(cloud fi.Cloud, r *resources.Resource) return nil } -func DeleteAutoscalingLaunchConfiguration(cloud fi.Cloud, r *resources.Resource) error { - c := cloud.(awsup.AWSCloud) - - id := r.ID - klog.V(2).Infof("Deleting autoscaling LaunchConfiguration %q", id) - request := &autoscaling.DeleteLaunchConfigurationInput{ - LaunchConfigurationName: &id, - } - _, err := c.Autoscaling().DeleteLaunchConfiguration(request) - if err != nil { - return fmt.Errorf("error deleting autoscaling LaunchConfiguration %q: %v", id, err) - } - return nil -} - func DeleteELB(cloud fi.Cloud, r *resources.Resource) error { c := cloud.(awsup.AWSCloud) diff --git a/tests/integration/update_cluster/complex/in-legacy-v1alpha2.yaml b/tests/integration/update_cluster/complex/in-legacy-v1alpha2.yaml index da7bc1a9ac..c214e5be38 100644 --- a/tests/integration/update_cluster/complex/in-legacy-v1alpha2.yaml +++ b/tests/integration/update_cluster/complex/in-legacy-v1alpha2.yaml @@ -107,7 +107,6 @@ spec: subnets: - us-test-1a detailedInstanceMonitoring: true - rootVolumeDeleteOnTermination: false rootVolumeEncryption: true cpuCredits: standard volumes: diff --git a/tests/integration/update_cluster/complex/in-v1alpha2.yaml b/tests/integration/update_cluster/complex/in-v1alpha2.yaml index dbad8e572c..fb7da4081b 100644 --- a/tests/integration/update_cluster/complex/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/complex/in-v1alpha2.yaml @@ -107,7 +107,6 @@ spec: subnets: - us-test-1a detailedInstanceMonitoring: true - rootVolumeDeleteOnTermination: false rootVolumeEncryption: true cpuCredits: standard volumes: diff --git a/tests/integration/update_cluster/launch_templates/cloudformation.json b/tests/integration/update_cluster/launch_templates/cloudformation.json deleted file mode 100644 index 861f057388..0000000000 --- a/tests/integration/update_cluster/launch_templates/cloudformation.json +++ /dev/null @@ -1,1392 +0,0 @@ -{ - "Resources": { - "AWSAutoScalingAutoScalingGroupmasterustest1amasterslaunchtemplatesexamplecom": { - "Type": "AWS::AutoScaling::AutoScalingGroup", - "Properties": { - "AutoScalingGroupName": "master-us-test-1a.masters.launchtemplates.example.com", - "LaunchConfigurationName": { - "Ref": "AWSAutoScalingLaunchConfigurationmasterustest1amasterslaunchtemplatesexamplecom" - }, - "MaxSize": "1", - "MinSize": "1", - "VPCZoneIdentifier": [ - { - "Ref": "AWSEC2Subnetustest1alaunchtemplatesexamplecom" - } - ], - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "launchtemplates.example.com", - "PropagateAtLaunch": true - }, - { - "Key": "Name", - "Value": "master-us-test-1a.masters.launchtemplates.example.com", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki", - "Value": "", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", - "Value": "master", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane", - "Value": "", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master", - "Value": "", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers", - "Value": "", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/role/master", - "Value": "1", - "PropagateAtLaunch": true - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "master-us-test-1a", - "PropagateAtLaunch": true - }, - { - "Key": "kubernetes.io/cluster/launchtemplates.example.com", - "Value": "owned", - "PropagateAtLaunch": true - } - ], - "MetricsCollection": [ - { - "Granularity": "1Minute", - "Metrics": [ - "GroupDesiredCapacity", - "GroupInServiceInstances", - "GroupMaxSize", - "GroupMinSize", - "GroupPendingInstances", - "GroupStandbyInstances", - "GroupTerminatingInstances", - "GroupTotalInstances" - ] - } - ] - } - }, - "AWSAutoScalingAutoScalingGroupmasterustest1bmasterslaunchtemplatesexamplecom": { - "Type": "AWS::AutoScaling::AutoScalingGroup", - "Properties": { - "AutoScalingGroupName": "master-us-test-1b.masters.launchtemplates.example.com", - "LaunchConfigurationName": { - "Ref": "AWSAutoScalingLaunchConfigurationmasterustest1bmasterslaunchtemplatesexamplecom" - }, - "MaxSize": "1", - "MinSize": "1", - "VPCZoneIdentifier": [ - { - "Ref": "AWSEC2Subnetustest1blaunchtemplatesexamplecom" - } - ], - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "launchtemplates.example.com", - "PropagateAtLaunch": true - }, - { - "Key": "Name", - "Value": "master-us-test-1b.masters.launchtemplates.example.com", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki", - "Value": "", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", - "Value": "master", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane", - "Value": "", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master", - "Value": "", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers", - "Value": "", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/role/master", - "Value": "1", - "PropagateAtLaunch": true - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "master-us-test-1b", - "PropagateAtLaunch": true - }, - { - "Key": "kubernetes.io/cluster/launchtemplates.example.com", - "Value": "owned", - "PropagateAtLaunch": true - } - ], - "MetricsCollection": [ - { - "Granularity": "1Minute", - "Metrics": [ - "GroupDesiredCapacity", - "GroupInServiceInstances", - "GroupMaxSize", - "GroupMinSize", - "GroupPendingInstances", - "GroupStandbyInstances", - "GroupTerminatingInstances", - "GroupTotalInstances" - ] - } - ] - } - }, - "AWSAutoScalingAutoScalingGroupmasterustest1cmasterslaunchtemplatesexamplecom": { - "Type": "AWS::AutoScaling::AutoScalingGroup", - "Properties": { - "AutoScalingGroupName": "master-us-test-1c.masters.launchtemplates.example.com", - "LaunchConfigurationName": { - "Ref": "AWSAutoScalingLaunchConfigurationmasterustest1cmasterslaunchtemplatesexamplecom" - }, - "MaxSize": "1", - "MinSize": "1", - "VPCZoneIdentifier": [ - { - "Ref": "AWSEC2Subnetustest1claunchtemplatesexamplecom" - } - ], - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "launchtemplates.example.com", - "PropagateAtLaunch": true - }, - { - "Key": "Name", - "Value": "master-us-test-1c.masters.launchtemplates.example.com", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki", - "Value": "", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", - "Value": "master", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane", - "Value": "", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master", - "Value": "", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers", - "Value": "", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/role/master", - "Value": "1", - "PropagateAtLaunch": true - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "master-us-test-1c", - "PropagateAtLaunch": true - }, - { - "Key": "kubernetes.io/cluster/launchtemplates.example.com", - "Value": "owned", - "PropagateAtLaunch": true - } - ], - "MetricsCollection": [ - { - "Granularity": "1Minute", - "Metrics": [ - "GroupDesiredCapacity", - "GroupInServiceInstances", - "GroupMaxSize", - "GroupMinSize", - "GroupPendingInstances", - "GroupStandbyInstances", - "GroupTerminatingInstances", - "GroupTotalInstances" - ] - } - ] - } - }, - "AWSAutoScalingAutoScalingGroupnodeslaunchtemplatesexamplecom": { - "Type": "AWS::AutoScaling::AutoScalingGroup", - "Properties": { - "AutoScalingGroupName": "nodes.launchtemplates.example.com", - "LaunchConfigurationName": { - "Ref": "AWSAutoScalingLaunchConfigurationnodeslaunchtemplatesexamplecom" - }, - "MaxSize": "2", - "MinSize": "2", - "VPCZoneIdentifier": [ - { - "Ref": "AWSEC2Subnetustest1blaunchtemplatesexamplecom" - } - ], - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "launchtemplates.example.com", - "PropagateAtLaunch": true - }, - { - "Key": "Name", - "Value": "nodes.launchtemplates.example.com", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", - "Value": "node", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node", - "Value": "", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/role/node", - "Value": "1", - "PropagateAtLaunch": true - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "nodes", - "PropagateAtLaunch": true - }, - { - "Key": "kubernetes.io/cluster/launchtemplates.example.com", - "Value": "owned", - "PropagateAtLaunch": true - } - ], - "MetricsCollection": [ - { - "Granularity": "1Minute", - "Metrics": [ - "GroupDesiredCapacity", - "GroupInServiceInstances", - "GroupMaxSize", - "GroupMinSize", - "GroupPendingInstances", - "GroupStandbyInstances", - "GroupTerminatingInstances", - "GroupTotalInstances" - ] - } - ] - } - }, - "AWSAutoScalingLaunchConfigurationmasterustest1amasterslaunchtemplatesexamplecom": { - "Type": "AWS::AutoScaling::LaunchConfiguration", - "Properties": { - "AssociatePublicIpAddress": true, - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/xvda", - "Ebs": { - "VolumeType": "gp2", - "VolumeSize": 64, - "DeleteOnTermination": true, - "Encrypted": true - } - } - ], - "IamInstanceProfile": { - "Ref": "AWSIAMInstanceProfilemasterslaunchtemplatesexamplecom" - }, - "ImageId": "ami-12345678", - "InstanceType": "t3.medium", - "KeyName": "kubernetes.launchtemplates.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", - "SecurityGroups": [ - { - "Ref": "AWSEC2SecurityGroupmasterslaunchtemplatesexamplecom" - } - ], - "UserData": "extracted", - "InstanceMonitoring": false - } - }, - "AWSAutoScalingLaunchConfigurationmasterustest1bmasterslaunchtemplatesexamplecom": { - "Type": "AWS::AutoScaling::LaunchConfiguration", - "Properties": { - "AssociatePublicIpAddress": true, - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/xvda", - "Ebs": { - "VolumeType": "gp2", - "VolumeSize": 64, - "DeleteOnTermination": true, - "Encrypted": true - } - } - ], - "IamInstanceProfile": { - "Ref": "AWSIAMInstanceProfilemasterslaunchtemplatesexamplecom" - }, - "ImageId": "ami-12345678", - "InstanceType": "t3.medium", - "KeyName": "kubernetes.launchtemplates.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", - "SecurityGroups": [ - { - "Ref": "AWSEC2SecurityGroupmasterslaunchtemplatesexamplecom" - } - ], - "UserData": "extracted", - "InstanceMonitoring": false - } - }, - "AWSAutoScalingLaunchConfigurationmasterustest1cmasterslaunchtemplatesexamplecom": { - "Type": "AWS::AutoScaling::LaunchConfiguration", - "Properties": { - "AssociatePublicIpAddress": true, - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/xvda", - "Ebs": { - "VolumeType": "gp2", - "VolumeSize": 64, - "DeleteOnTermination": true, - "Encrypted": true - } - } - ], - "IamInstanceProfile": { - "Ref": "AWSIAMInstanceProfilemasterslaunchtemplatesexamplecom" - }, - "ImageId": "ami-12345678", - "InstanceType": "t3.medium", - "KeyName": "kubernetes.launchtemplates.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", - "SecurityGroups": [ - { - "Ref": "AWSEC2SecurityGroupmasterslaunchtemplatesexamplecom" - } - ], - "UserData": "extracted", - "InstanceMonitoring": false - } - }, - "AWSAutoScalingLaunchConfigurationnodeslaunchtemplatesexamplecom": { - "Type": "AWS::AutoScaling::LaunchConfiguration", - "Properties": { - "AssociatePublicIpAddress": true, - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/xvda", - "Ebs": { - "VolumeType": "gp2", - "VolumeSize": 128, - "DeleteOnTermination": true, - "Encrypted": true - } - } - ], - "IamInstanceProfile": { - "Ref": "AWSIAMInstanceProfilenodeslaunchtemplatesexamplecom" - }, - "ImageId": "ami-12345678", - "InstanceType": "t3.medium", - "KeyName": "kubernetes.launchtemplates.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", - "SecurityGroups": [ - { - "Ref": "AWSEC2SecurityGroupnodeslaunchtemplatesexamplecom" - } - ], - "SpotPrice": "0.1", - "UserData": "extracted", - "InstanceMonitoring": false - } - }, - "AWSEC2DHCPOptionslaunchtemplatesexamplecom": { - "Type": "AWS::EC2::DHCPOptions", - "Properties": { - "DomainName": "us-test-1.compute.internal", - "DomainNameServers": [ - "AmazonProvidedDNS" - ], - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "launchtemplates.example.com" - }, - { - "Key": "Name", - "Value": "launchtemplates.example.com" - }, - { - "Key": "kubernetes.io/cluster/launchtemplates.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2InternetGatewaylaunchtemplatesexamplecom": { - "Type": "AWS::EC2::InternetGateway", - "Properties": { - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "launchtemplates.example.com" - }, - { - "Key": "Name", - "Value": "launchtemplates.example.com" - }, - { - "Key": "kubernetes.io/cluster/launchtemplates.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2Route00000": { - "Type": "AWS::EC2::Route", - "Properties": { - "RouteTableId": { - "Ref": "AWSEC2RouteTablelaunchtemplatesexamplecom" - }, - "DestinationCidrBlock": "0.0.0.0/0", - "GatewayId": { - "Ref": "AWSEC2InternetGatewaylaunchtemplatesexamplecom" - } - } - }, - "AWSEC2RouteTablelaunchtemplatesexamplecom": { - "Type": "AWS::EC2::RouteTable", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPClaunchtemplatesexamplecom" - }, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "launchtemplates.example.com" - }, - { - "Key": "Name", - "Value": "launchtemplates.example.com" - }, - { - "Key": "kubernetes.io/cluster/launchtemplates.example.com", - "Value": "owned" - }, - { - "Key": "kubernetes.io/kops/role", - "Value": "public" - } - ] - } - }, - "AWSEC2SecurityGroupEgressfrommasterslaunchtemplatesexamplecomegressall0to000000": { - "Type": "AWS::EC2::SecurityGroupEgress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmasterslaunchtemplatesexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1", - "CidrIp": "0.0.0.0/0" - } - }, - "AWSEC2SecurityGroupEgressfromnodeslaunchtemplatesexamplecomegressall0to000000": { - "Type": "AWS::EC2::SecurityGroupEgress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupnodeslaunchtemplatesexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1", - "CidrIp": "0.0.0.0/0" - } - }, - "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22masterslaunchtemplatesexamplecom": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmasterslaunchtemplatesexamplecom" - }, - "FromPort": 22, - "ToPort": 22, - "IpProtocol": "tcp", - "CidrIp": "0.0.0.0/0" - } - }, - "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22nodeslaunchtemplatesexamplecom": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupnodeslaunchtemplatesexamplecom" - }, - "FromPort": 22, - "ToPort": 22, - "IpProtocol": "tcp", - "CidrIp": "0.0.0.0/0" - } - }, - "AWSEC2SecurityGroupIngressfrom00000ingresstcp443to443masterslaunchtemplatesexamplecom": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmasterslaunchtemplatesexamplecom" - }, - "FromPort": 443, - "ToPort": 443, - "IpProtocol": "tcp", - "CidrIp": "0.0.0.0/0" - } - }, - "AWSEC2SecurityGroupIngressfrommasterslaunchtemplatesexamplecomingressall0to0masterslaunchtemplatesexamplecom": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmasterslaunchtemplatesexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupmasterslaunchtemplatesexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1" - } - }, - "AWSEC2SecurityGroupIngressfrommasterslaunchtemplatesexamplecomingressall0to0nodeslaunchtemplatesexamplecom": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupnodeslaunchtemplatesexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupmasterslaunchtemplatesexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1" - } - }, - "AWSEC2SecurityGroupIngressfromnodeslaunchtemplatesexamplecomingressall0to0nodeslaunchtemplatesexamplecom": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupnodeslaunchtemplatesexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodeslaunchtemplatesexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1" - } - }, - "AWSEC2SecurityGroupIngressfromnodeslaunchtemplatesexamplecomingresstcp1to2379masterslaunchtemplatesexamplecom": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmasterslaunchtemplatesexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodeslaunchtemplatesexamplecom" - }, - "FromPort": 1, - "ToPort": 2379, - "IpProtocol": "tcp" - } - }, - "AWSEC2SecurityGroupIngressfromnodeslaunchtemplatesexamplecomingresstcp2382to4000masterslaunchtemplatesexamplecom": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmasterslaunchtemplatesexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodeslaunchtemplatesexamplecom" - }, - "FromPort": 2382, - "ToPort": 4000, - "IpProtocol": "tcp" - } - }, - "AWSEC2SecurityGroupIngressfromnodeslaunchtemplatesexamplecomingresstcp4003to65535masterslaunchtemplatesexamplecom": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmasterslaunchtemplatesexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodeslaunchtemplatesexamplecom" - }, - "FromPort": 4003, - "ToPort": 65535, - "IpProtocol": "tcp" - } - }, - "AWSEC2SecurityGroupIngressfromnodeslaunchtemplatesexamplecomingressudp1to65535masterslaunchtemplatesexamplecom": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmasterslaunchtemplatesexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodeslaunchtemplatesexamplecom" - }, - "FromPort": 1, - "ToPort": 65535, - "IpProtocol": "udp" - } - }, - "AWSEC2SecurityGroupmasterslaunchtemplatesexamplecom": { - "Type": "AWS::EC2::SecurityGroup", - "Properties": { - "GroupName": "masters.launchtemplates.example.com", - "VpcId": { - "Ref": "AWSEC2VPClaunchtemplatesexamplecom" - }, - "GroupDescription": "Security group for masters", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "launchtemplates.example.com" - }, - { - "Key": "Name", - "Value": "masters.launchtemplates.example.com" - }, - { - "Key": "kubernetes.io/cluster/launchtemplates.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2SecurityGroupnodeslaunchtemplatesexamplecom": { - "Type": "AWS::EC2::SecurityGroup", - "Properties": { - "GroupName": "nodes.launchtemplates.example.com", - "VpcId": { - "Ref": "AWSEC2VPClaunchtemplatesexamplecom" - }, - "GroupDescription": "Security group for nodes", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "launchtemplates.example.com" - }, - { - "Key": "Name", - "Value": "nodes.launchtemplates.example.com" - }, - { - "Key": "kubernetes.io/cluster/launchtemplates.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2SubnetRouteTableAssociationustest1alaunchtemplatesexamplecom": { - "Type": "AWS::EC2::SubnetRouteTableAssociation", - "Properties": { - "SubnetId": { - "Ref": "AWSEC2Subnetustest1alaunchtemplatesexamplecom" - }, - "RouteTableId": { - "Ref": "AWSEC2RouteTablelaunchtemplatesexamplecom" - } - } - }, - "AWSEC2SubnetRouteTableAssociationustest1blaunchtemplatesexamplecom": { - "Type": "AWS::EC2::SubnetRouteTableAssociation", - "Properties": { - "SubnetId": { - "Ref": "AWSEC2Subnetustest1blaunchtemplatesexamplecom" - }, - "RouteTableId": { - "Ref": "AWSEC2RouteTablelaunchtemplatesexamplecom" - } - } - }, - "AWSEC2SubnetRouteTableAssociationustest1claunchtemplatesexamplecom": { - "Type": "AWS::EC2::SubnetRouteTableAssociation", - "Properties": { - "SubnetId": { - "Ref": "AWSEC2Subnetustest1claunchtemplatesexamplecom" - }, - "RouteTableId": { - "Ref": "AWSEC2RouteTablelaunchtemplatesexamplecom" - } - } - }, - "AWSEC2Subnetustest1alaunchtemplatesexamplecom": { - "Type": "AWS::EC2::Subnet", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPClaunchtemplatesexamplecom" - }, - "CidrBlock": "10.0.1.0/24", - "AvailabilityZone": "us-test-1a", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "launchtemplates.example.com" - }, - { - "Key": "Name", - "Value": "us-test-1a.launchtemplates.example.com" - }, - { - "Key": "SubnetType", - "Value": "Public" - }, - { - "Key": "kubernetes.io/cluster/launchtemplates.example.com", - "Value": "owned" - }, - { - "Key": "kubernetes.io/role/elb", - "Value": "1" - } - ] - } - }, - "AWSEC2Subnetustest1blaunchtemplatesexamplecom": { - "Type": "AWS::EC2::Subnet", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPClaunchtemplatesexamplecom" - }, - "CidrBlock": "10.0.2.0/24", - "AvailabilityZone": "us-test-1b", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "launchtemplates.example.com" - }, - { - "Key": "Name", - "Value": "us-test-1b.launchtemplates.example.com" - }, - { - "Key": "SubnetType", - "Value": "Public" - }, - { - "Key": "kubernetes.io/cluster/launchtemplates.example.com", - "Value": "owned" - }, - { - "Key": "kubernetes.io/role/elb", - "Value": "1" - } - ] - } - }, - "AWSEC2Subnetustest1claunchtemplatesexamplecom": { - "Type": "AWS::EC2::Subnet", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPClaunchtemplatesexamplecom" - }, - "CidrBlock": "10.0.3.0/24", - "AvailabilityZone": "us-test-1c", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "launchtemplates.example.com" - }, - { - "Key": "Name", - "Value": "us-test-1c.launchtemplates.example.com" - }, - { - "Key": "SubnetType", - "Value": "Public" - }, - { - "Key": "kubernetes.io/cluster/launchtemplates.example.com", - "Value": "owned" - }, - { - "Key": "kubernetes.io/role/elb", - "Value": "1" - } - ] - } - }, - "AWSEC2VPCDHCPOptionsAssociationlaunchtemplatesexamplecom": { - "Type": "AWS::EC2::VPCDHCPOptionsAssociation", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPClaunchtemplatesexamplecom" - }, - "DhcpOptionsId": { - "Ref": "AWSEC2DHCPOptionslaunchtemplatesexamplecom" - } - } - }, - "AWSEC2VPCGatewayAttachmentlaunchtemplatesexamplecom": { - "Type": "AWS::EC2::VPCGatewayAttachment", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPClaunchtemplatesexamplecom" - }, - "InternetGatewayId": { - "Ref": "AWSEC2InternetGatewaylaunchtemplatesexamplecom" - } - } - }, - "AWSEC2VPClaunchtemplatesexamplecom": { - "Type": "AWS::EC2::VPC", - "Properties": { - "CidrBlock": "10.0.0.0/16", - "EnableDnsHostnames": true, - "EnableDnsSupport": true, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "launchtemplates.example.com" - }, - { - "Key": "Name", - "Value": "launchtemplates.example.com" - }, - { - "Key": "kubernetes.io/cluster/launchtemplates.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2Volumeustest1aetcdeventslaunchtemplatesexamplecom": { - "Type": "AWS::EC2::Volume", - "Properties": { - "AvailabilityZone": "us-test-1a", - "Size": 20, - "VolumeType": "gp3", - "Iops": 3000, - "Throughput": 125, - "Encrypted": false, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "launchtemplates.example.com" - }, - { - "Key": "Name", - "Value": "us-test-1a.etcd-events.launchtemplates.example.com" - }, - { - "Key": "k8s.io/etcd/events", - "Value": "us-test-1a/us-test-1a,us-test-1b,us-test-1c" - }, - { - "Key": "k8s.io/role/master", - "Value": "1" - }, - { - "Key": "kubernetes.io/cluster/launchtemplates.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2Volumeustest1aetcdmainlaunchtemplatesexamplecom": { - "Type": "AWS::EC2::Volume", - "Properties": { - "AvailabilityZone": "us-test-1a", - "Size": 20, - "VolumeType": "gp3", - "Iops": 3000, - "Throughput": 125, - "Encrypted": false, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "launchtemplates.example.com" - }, - { - "Key": "Name", - "Value": "us-test-1a.etcd-main.launchtemplates.example.com" - }, - { - "Key": "k8s.io/etcd/main", - "Value": "us-test-1a/us-test-1a,us-test-1b,us-test-1c" - }, - { - "Key": "k8s.io/role/master", - "Value": "1" - }, - { - "Key": "kubernetes.io/cluster/launchtemplates.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2Volumeustest1betcdeventslaunchtemplatesexamplecom": { - "Type": "AWS::EC2::Volume", - "Properties": { - "AvailabilityZone": "us-test-1b", - "Size": 20, - "VolumeType": "gp3", - "Iops": 3000, - "Throughput": 125, - "Encrypted": false, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "launchtemplates.example.com" - }, - { - "Key": "Name", - "Value": "us-test-1b.etcd-events.launchtemplates.example.com" - }, - { - "Key": "k8s.io/etcd/events", - "Value": "us-test-1b/us-test-1a,us-test-1b,us-test-1c" - }, - { - "Key": "k8s.io/role/master", - "Value": "1" - }, - { - "Key": "kubernetes.io/cluster/launchtemplates.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2Volumeustest1betcdmainlaunchtemplatesexamplecom": { - "Type": "AWS::EC2::Volume", - "Properties": { - "AvailabilityZone": "us-test-1b", - "Size": 20, - "VolumeType": "gp3", - "Iops": 3000, - "Throughput": 125, - "Encrypted": false, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "launchtemplates.example.com" - }, - { - "Key": "Name", - "Value": "us-test-1b.etcd-main.launchtemplates.example.com" - }, - { - "Key": "k8s.io/etcd/main", - "Value": "us-test-1b/us-test-1a,us-test-1b,us-test-1c" - }, - { - "Key": "k8s.io/role/master", - "Value": "1" - }, - { - "Key": "kubernetes.io/cluster/launchtemplates.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2Volumeustest1cetcdeventslaunchtemplatesexamplecom": { - "Type": "AWS::EC2::Volume", - "Properties": { - "AvailabilityZone": "us-test-1c", - "Size": 20, - "VolumeType": "gp3", - "Iops": 3000, - "Throughput": 125, - "Encrypted": false, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "launchtemplates.example.com" - }, - { - "Key": "Name", - "Value": "us-test-1c.etcd-events.launchtemplates.example.com" - }, - { - "Key": "k8s.io/etcd/events", - "Value": "us-test-1c/us-test-1a,us-test-1b,us-test-1c" - }, - { - "Key": "k8s.io/role/master", - "Value": "1" - }, - { - "Key": "kubernetes.io/cluster/launchtemplates.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2Volumeustest1cetcdmainlaunchtemplatesexamplecom": { - "Type": "AWS::EC2::Volume", - "Properties": { - "AvailabilityZone": "us-test-1c", - "Size": 20, - "VolumeType": "gp3", - "Iops": 3000, - "Throughput": 125, - "Encrypted": false, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "launchtemplates.example.com" - }, - { - "Key": "Name", - "Value": "us-test-1c.etcd-main.launchtemplates.example.com" - }, - { - "Key": "k8s.io/etcd/main", - "Value": "us-test-1c/us-test-1a,us-test-1b,us-test-1c" - }, - { - "Key": "k8s.io/role/master", - "Value": "1" - }, - { - "Key": "kubernetes.io/cluster/launchtemplates.example.com", - "Value": "owned" - } - ] - } - }, - "AWSIAMInstanceProfilemasterslaunchtemplatesexamplecom": { - "Type": "AWS::IAM::InstanceProfile", - "Properties": { - "InstanceProfileName": "masters.launchtemplates.example.com", - "Roles": [ - { - "Ref": "AWSIAMRolemasterslaunchtemplatesexamplecom" - } - ] - } - }, - "AWSIAMInstanceProfilenodeslaunchtemplatesexamplecom": { - "Type": "AWS::IAM::InstanceProfile", - "Properties": { - "InstanceProfileName": "nodes.launchtemplates.example.com", - "Roles": [ - { - "Ref": "AWSIAMRolenodeslaunchtemplatesexamplecom" - } - ] - } - }, - "AWSIAMPolicymasterslaunchtemplatesexamplecom": { - "Type": "AWS::IAM::Policy", - "Properties": { - "PolicyName": "masters.launchtemplates.example.com", - "Roles": [ - { - "Ref": "AWSIAMRolemasterslaunchtemplatesexamplecom" - } - ], - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "ec2:DescribeAccountAttributes", - "ec2:DescribeInstances", - "ec2:DescribeInternetGateways", - "ec2:DescribeRegions", - "ec2:DescribeRouteTables", - "ec2:DescribeSecurityGroups", - "ec2:DescribeSubnets", - "ec2:DescribeVolumes" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "ec2:CreateSecurityGroup", - "ec2:CreateTags", - "ec2:CreateVolume", - "ec2:DescribeVolumesModifications", - "ec2:ModifyInstanceAttribute", - "ec2:ModifyVolume" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "ec2:AttachVolume", - "ec2:AuthorizeSecurityGroupIngress", - "ec2:CreateRoute", - "ec2:DeleteRoute", - "ec2:DeleteSecurityGroup", - "ec2:DeleteVolume", - "ec2:DetachVolume", - "ec2:RevokeSecurityGroupIngress" - ], - "Condition": { - "StringEquals": { - "ec2:ResourceTag/KubernetesCluster": "launchtemplates.example.com" - } - }, - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:DescribeLaunchConfigurations", - "autoscaling:DescribeTags", - "ec2:DescribeLaunchTemplateVersions" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "autoscaling:SetDesiredCapacity", - "autoscaling:TerminateInstanceInAutoScalingGroup", - "autoscaling:UpdateAutoScalingGroup" - ], - "Condition": { - "StringEquals": { - "autoscaling:ResourceTag/KubernetesCluster": "launchtemplates.example.com" - } - }, - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "elasticloadbalancing:AddTags", - "elasticloadbalancing:AttachLoadBalancerToSubnets", - "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", - "elasticloadbalancing:CreateLoadBalancer", - "elasticloadbalancing:CreateLoadBalancerPolicy", - "elasticloadbalancing:CreateLoadBalancerListeners", - "elasticloadbalancing:ConfigureHealthCheck", - "elasticloadbalancing:DeleteLoadBalancer", - "elasticloadbalancing:DeleteLoadBalancerListeners", - "elasticloadbalancing:DescribeLoadBalancers", - "elasticloadbalancing:DescribeLoadBalancerAttributes", - "elasticloadbalancing:DetachLoadBalancerFromSubnets", - "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", - "elasticloadbalancing:ModifyLoadBalancerAttributes", - "elasticloadbalancing:RegisterInstancesWithLoadBalancer", - "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "ec2:DescribeVpcs", - "elasticloadbalancing:AddTags", - "elasticloadbalancing:CreateListener", - "elasticloadbalancing:CreateTargetGroup", - "elasticloadbalancing:DeleteListener", - "elasticloadbalancing:DeleteTargetGroup", - "elasticloadbalancing:DeregisterTargets", - "elasticloadbalancing:DescribeListeners", - "elasticloadbalancing:DescribeLoadBalancerPolicies", - "elasticloadbalancing:DescribeTargetGroups", - "elasticloadbalancing:DescribeTargetHealth", - "elasticloadbalancing:ModifyListener", - "elasticloadbalancing:ModifyTargetGroup", - "elasticloadbalancing:RegisterTargets", - "elasticloadbalancing:SetLoadBalancerPoliciesOfListener" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "iam:ListServerCertificates", - "iam:GetServerCertificate" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "route53:ChangeResourceRecordSets", - "route53:ListResourceRecordSets", - "route53:GetHostedZone" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" - ] - }, - { - "Action": [ - "route53:GetChange" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:route53:::change/*" - ] - }, - { - "Action": [ - "route53:ListHostedZones" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - } - ], - "Version": "2012-10-17" - } - } - }, - "AWSIAMPolicynodeslaunchtemplatesexamplecom": { - "Type": "AWS::IAM::Policy", - "Properties": { - "PolicyName": "nodes.launchtemplates.example.com", - "Roles": [ - { - "Ref": "AWSIAMRolenodeslaunchtemplatesexamplecom" - } - ], - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "ec2:DescribeInstances", - "ec2:DescribeRegions" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - } - ], - "Version": "2012-10-17" - } - } - }, - "AWSIAMRolemasterslaunchtemplatesexamplecom": { - "Type": "AWS::IAM::Role", - "Properties": { - "RoleName": "masters.launchtemplates.example.com", - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "ec2.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - }, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "launchtemplates.example.com" - }, - { - "Key": "Name", - "Value": "masters.launchtemplates.example.com" - }, - { - "Key": "kubernetes.io/cluster/launchtemplates.example.com", - "Value": "owned" - } - ] - } - }, - "AWSIAMRolenodeslaunchtemplatesexamplecom": { - "Type": "AWS::IAM::Role", - "Properties": { - "RoleName": "nodes.launchtemplates.example.com", - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "ec2.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - }, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "launchtemplates.example.com" - }, - { - "Key": "Name", - "Value": "nodes.launchtemplates.example.com" - }, - { - "Key": "kubernetes.io/cluster/launchtemplates.example.com", - "Value": "owned" - } - ] - } - } - } -} diff --git a/tests/integration/update_cluster/launch_templates/data/aws_iam_role_masters.launchtemplates.example.com_policy b/tests/integration/update_cluster/launch_templates/data/aws_iam_role_masters.launchtemplates.example.com_policy deleted file mode 100644 index 66d5de1d5a..0000000000 --- a/tests/integration/update_cluster/launch_templates/data/aws_iam_role_masters.launchtemplates.example.com_policy +++ /dev/null @@ -1,10 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { "Service": "ec2.amazonaws.com"}, - "Action": "sts:AssumeRole" - } - ] -} diff --git a/tests/integration/update_cluster/launch_templates/data/aws_iam_role_nodes.launchtemplates.example.com_policy b/tests/integration/update_cluster/launch_templates/data/aws_iam_role_nodes.launchtemplates.example.com_policy deleted file mode 100644 index 66d5de1d5a..0000000000 --- a/tests/integration/update_cluster/launch_templates/data/aws_iam_role_nodes.launchtemplates.example.com_policy +++ /dev/null @@ -1,10 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { "Service": "ec2.amazonaws.com"}, - "Action": "sts:AssumeRole" - } - ] -} diff --git a/tests/integration/update_cluster/launch_templates/data/aws_iam_role_policy_masters.launchtemplates.example.com_policy b/tests/integration/update_cluster/launch_templates/data/aws_iam_role_policy_masters.launchtemplates.example.com_policy deleted file mode 100644 index bd93af43a0..0000000000 --- a/tests/integration/update_cluster/launch_templates/data/aws_iam_role_policy_masters.launchtemplates.example.com_policy +++ /dev/null @@ -1,170 +0,0 @@ -{ - "Statement": [ - { - "Action": [ - "ec2:DescribeAccountAttributes", - "ec2:DescribeInstances", - "ec2:DescribeInternetGateways", - "ec2:DescribeRegions", - "ec2:DescribeRouteTables", - "ec2:DescribeSecurityGroups", - "ec2:DescribeSubnets", - "ec2:DescribeVolumes" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "ec2:CreateSecurityGroup", - "ec2:CreateTags", - "ec2:CreateVolume", - "ec2:DescribeVolumesModifications", - "ec2:ModifyInstanceAttribute", - "ec2:ModifyVolume" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "ec2:AttachVolume", - "ec2:AuthorizeSecurityGroupIngress", - "ec2:CreateRoute", - "ec2:DeleteRoute", - "ec2:DeleteSecurityGroup", - "ec2:DeleteVolume", - "ec2:DetachVolume", - "ec2:RevokeSecurityGroupIngress" - ], - "Condition": { - "StringEquals": { - "ec2:ResourceTag/KubernetesCluster": "launchtemplates.example.com" - } - }, - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:DescribeLaunchConfigurations", - "autoscaling:DescribeTags", - "ec2:DescribeLaunchTemplateVersions" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "autoscaling:SetDesiredCapacity", - "autoscaling:TerminateInstanceInAutoScalingGroup", - "autoscaling:UpdateAutoScalingGroup" - ], - "Condition": { - "StringEquals": { - "autoscaling:ResourceTag/KubernetesCluster": "launchtemplates.example.com" - } - }, - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "elasticloadbalancing:AddTags", - "elasticloadbalancing:AttachLoadBalancerToSubnets", - "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", - "elasticloadbalancing:CreateLoadBalancer", - "elasticloadbalancing:CreateLoadBalancerPolicy", - "elasticloadbalancing:CreateLoadBalancerListeners", - "elasticloadbalancing:ConfigureHealthCheck", - "elasticloadbalancing:DeleteLoadBalancer", - "elasticloadbalancing:DeleteLoadBalancerListeners", - "elasticloadbalancing:DescribeLoadBalancers", - "elasticloadbalancing:DescribeLoadBalancerAttributes", - "elasticloadbalancing:DetachLoadBalancerFromSubnets", - "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", - "elasticloadbalancing:ModifyLoadBalancerAttributes", - "elasticloadbalancing:RegisterInstancesWithLoadBalancer", - "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "ec2:DescribeVpcs", - "elasticloadbalancing:AddTags", - "elasticloadbalancing:CreateListener", - "elasticloadbalancing:CreateTargetGroup", - "elasticloadbalancing:DeleteListener", - "elasticloadbalancing:DeleteTargetGroup", - "elasticloadbalancing:DeregisterTargets", - "elasticloadbalancing:DescribeListeners", - "elasticloadbalancing:DescribeLoadBalancerPolicies", - "elasticloadbalancing:DescribeTargetGroups", - "elasticloadbalancing:DescribeTargetHealth", - "elasticloadbalancing:ModifyListener", - "elasticloadbalancing:ModifyTargetGroup", - "elasticloadbalancing:RegisterTargets", - "elasticloadbalancing:SetLoadBalancerPoliciesOfListener" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "iam:ListServerCertificates", - "iam:GetServerCertificate" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "route53:ChangeResourceRecordSets", - "route53:ListResourceRecordSets", - "route53:GetHostedZone" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" - ] - }, - { - "Action": [ - "route53:GetChange" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:route53:::change/*" - ] - }, - { - "Action": [ - "route53:ListHostedZones" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - } - ], - "Version": "2012-10-17" -} diff --git a/tests/integration/update_cluster/launch_templates/data/aws_iam_role_policy_nodes.launchtemplates.example.com_policy b/tests/integration/update_cluster/launch_templates/data/aws_iam_role_policy_nodes.launchtemplates.example.com_policy deleted file mode 100644 index 49749a010d..0000000000 --- a/tests/integration/update_cluster/launch_templates/data/aws_iam_role_policy_nodes.launchtemplates.example.com_policy +++ /dev/null @@ -1,15 +0,0 @@ -{ - "Statement": [ - { - "Action": [ - "ec2:DescribeInstances", - "ec2:DescribeRegions" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - } - ], - "Version": "2012-10-17" -} diff --git a/tests/integration/update_cluster/launch_templates/data/aws_key_pair_kubernetes.launchtemplates.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key b/tests/integration/update_cluster/launch_templates/data/aws_key_pair_kubernetes.launchtemplates.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key deleted file mode 100644 index 81cb012783..0000000000 --- a/tests/integration/update_cluster/launch_templates/data/aws_key_pair_kubernetes.launchtemplates.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/launch_templates/data/aws_launch_configuration_nodes.launchtemplates.example.com_user_data b/tests/integration/update_cluster/launch_templates/data/aws_launch_configuration_nodes.launchtemplates.example.com_user_data deleted file mode 100644 index 8d1196f445..0000000000 --- a/tests/integration/update_cluster/launch_templates/data/aws_launch_configuration_nodes.launchtemplates.example.com_user_data +++ /dev/null @@ -1,232 +0,0 @@ -#!/bin/bash -set -o errexit -set -o nounset -set -o pipefail - -NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/nodeup -NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 -NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/nodeup -NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 - -export AWS_REGION=us-test-1 - - - - -sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true - - -function ensure-install-dir() { - INSTALL_DIR="/opt/kops" - # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec - if [[ -d /var/lib/toolbox ]]; then - INSTALL_DIR="/var/lib/toolbox/kops" - fi - mkdir -p ${INSTALL_DIR}/bin - mkdir -p ${INSTALL_DIR}/conf - cd ${INSTALL_DIR} -} - -# Retry a download until we get it. args: name, sha, url1, url2... -download-or-bust() { - local -r file="$1" - local -r hash="$2" - shift 2 - - urls=( $* ) - while true; do - for url in "${urls[@]}"; do - commands=( - "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - ) - for cmd in "${commands[@]}"; do - echo "Attempting download with: ${cmd} {url}" - if ! (${cmd} "${url}"); then - echo "== Download failed with ${cmd} ==" - continue - fi - if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then - echo "== Hash validation of ${url} failed. Retrying. ==" - rm -f "${file}" - else - if [[ -n "${hash}" ]]; then - echo "== Downloaded ${url} (SHA1 = ${hash}) ==" - else - echo "== Downloaded ${url} ==" - fi - return - fi - done - done - - echo "All downloads failed; sleeping before retrying" - sleep 60 - done -} - -validate-hash() { - local -r file="$1" - local -r expected="$2" - local actual - - actual=$(sha256sum ${file} | awk '{ print $1 }') || true - if [[ "${actual}" != "${expected}" ]]; then - echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" - return 1 - fi -} - -function split-commas() { - echo $1 | tr "," "\n" -} - -function try-download-release() { - local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) - if [[ -n "${NODEUP_HASH:-}" ]]; then - local -r nodeup_hash="${NODEUP_HASH}" - else - # TODO: Remove? - echo "Downloading sha256 (not found in env)" - download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" - local -r nodeup_hash=$(cat nodeup.sha256) - fi - - echo "Downloading nodeup (${nodeup_urls[@]})" - download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" - - chmod +x nodeup -} - -function download-release() { - case "$(uname -m)" in - x86_64*|i?86_64*|amd64*) - NODEUP_URL="${NODEUP_URL_AMD64}" - NODEUP_HASH="${NODEUP_HASH_AMD64}" - ;; - aarch64*|arm64*) - NODEUP_URL="${NODEUP_URL_ARM64}" - NODEUP_HASH="${NODEUP_HASH_ARM64}" - ;; - *) - echo "Unsupported host arch: $(uname -m)" >&2 - exit 1 - ;; - esac - - # In case of failure checking integrity of release, retry. - cd ${INSTALL_DIR}/bin - until try-download-release; do - sleep 15 - echo "Couldn't download release. Retrying..." - done - - echo "Running nodeup" - # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 - ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) -} - -#################################################################################### - -/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" - -echo "== nodeup node config starting ==" -ensure-install-dir - -cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' -cloudConfig: - manageStorageClasses: true -containerRuntime: containerd -containerd: - configOverride: | - version = 2 - - [plugins] - - [plugins."io.containerd.grpc.v1.cri"] - - [plugins."io.containerd.grpc.v1.cri".containerd] - - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - logLevel: info - version: 1.4.3 -docker: - skipInstall: true -kubeProxy: - clusterCIDR: 100.96.0.0/11 - cpuRequest: 100m - hostnameOverride: '@aws' - image: k8s.gcr.io/kube-proxy:v1.20.0 - logLevel: 2 -kubelet: - anonymousAuth: false - cgroupDriver: systemd - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginName: cni - nonMasqueradeCIDR: 100.64.0.0/10 - podManifestPath: /etc/kubernetes/manifests - -__EOF_CLUSTER_SPEC - -cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' -{} - -__EOF_IG_SPEC - -cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' -Assets: - amd64: - - ff2422571c4c1e9696e367f5f25466b96fb6e501f28aed29f414b1524a52dea0@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubelet - - a5895007f331f08d2e082eb12458764949559f30bcc5beae26c38f3e2724262c@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubectl - - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz - - 2697a342e3477c211ab48313e259fd7e32ad1f5ded19320e6a559f50a82bff3d@https://github.com/containerd/containerd/releases/download/v1.4.3/cri-containerd-cni-1.4.3-linux-amd64.tar.gz - arm64: - - 47ab6c4273fc3bb0cb8ec9517271d915890c5a6b0e54b2991e7a8fbbe77b06e4@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubelet - - 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubectl - - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz - - 6e3f80e8451ecbe7b3559247721c3e226be6b228acaadee7e13683f80c20e81c@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.0.tgz -ClusterName: launchtemplates.example.com -ConfigBase: memfs://clusters.example.com/launchtemplates.example.com -InstanceGroupName: nodes -InstanceGroupRole: Node -KubeletConfig: - anonymousAuth: false - cgroupDriver: systemd - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginName: cni - nodeLabels: - kubernetes.io/role: node - node-role.kubernetes.io/node: "" - nonMasqueradeCIDR: 100.64.0.0/10 - podManifestPath: /etc/kubernetes/manifests -channels: -- memfs://clusters.example.com/launchtemplates.example.com/addons/bootstrap-channel.yaml - -__EOF_KUBE_ENV - -download-release -echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/launch_templates/id_rsa.pub b/tests/integration/update_cluster/launch_templates/id_rsa.pub deleted file mode 100755 index 81cb012783..0000000000 --- a/tests/integration/update_cluster/launch_templates/id_rsa.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/launch_templates/in-v1alpha2.yaml b/tests/integration/update_cluster/launch_templates/in-v1alpha2.yaml deleted file mode 100644 index d5e38a9804..0000000000 --- a/tests/integration/update_cluster/launch_templates/in-v1alpha2.yaml +++ /dev/null @@ -1,135 +0,0 @@ -apiVersion: kops.k8s.io/v1alpha2 -kind: Cluster -metadata: - creationTimestamp: "2016-12-10T22:42:27Z" - name: launchtemplates.example.com -spec: - kubernetesApiAccess: - - 0.0.0.0/0 - channel: stable - cloudProvider: aws - configBase: memfs://clusters.example.com/launchtemplates.example.com - etcdClusters: - - etcdMembers: - - instanceGroup: master-us-test-1a - name: us-test-1a - - instanceGroup: master-us-test-1b - name: us-test-1b - - instanceGroup: master-us-test-1c - name: us-test-1c - name: main - - etcdMembers: - - instanceGroup: master-us-test-1a - name: us-test-1a - - instanceGroup: master-us-test-1b - name: us-test-1b - - instanceGroup: master-us-test-1c - name: us-test-1c - name: events - iam: {} - kubelet: - anonymousAuth: false - kubernetesVersion: v1.20.0 - masterInternalName: api.internal.launchtemplates.example.com - masterPublicName: api.launchtemplates.example.com - networkCIDR: 10.0.0.0/16 - networking: - cni: {} - nonMasqueradeCIDR: 100.64.0.0/10 - sshAccess: - - 0.0.0.0/0 - topology: - masters: public - nodes: public - subnets: - - cidr: 10.0.1.0/24 - name: us-test-1a - type: Public - zone: us-test-1a - - cidr: 10.0.2.0/24 - name: us-test-1b - type: Public - zone: us-test-1b - - cidr: 10.0.3.0/24 - name: us-test-1c - type: Public - zone: us-test-1c - ---- - -apiVersion: kops.k8s.io/v1alpha2 -kind: InstanceGroup -metadata: - creationTimestamp: "2016-12-10T22:42:28Z" - name: nodes - labels: - kops.k8s.io/cluster: launchtemplates.example.com -spec: - associatePublicIp: true - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: t3.medium - maxSize: 2 - minSize: 2 - role: Node - instanceProtection: true - maxPrice: "0.1" - spotDurationInMinutes: 120 - instanceInterruptionBehavior: "hibernate" - subnets: - - us-test-1b ---- - -apiVersion: kops.k8s.io/v1alpha2 -kind: InstanceGroup -metadata: - creationTimestamp: "2016-12-10T22:42:28Z" - name: master-us-test-1a - labels: - kops.k8s.io/cluster: launchtemplates.example.com -spec: - associatePublicIp: true - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: t3.medium - maxSize: 1 - minSize: 1 - role: Master - subnets: - - us-test-1a - ---- - -apiVersion: kops.k8s.io/v1alpha2 -kind: InstanceGroup -metadata: - creationTimestamp: "2016-12-10T22:42:28Z" - name: master-us-test-1b - labels: - kops.k8s.io/cluster: launchtemplates.example.com -spec: - associatePublicIp: true - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: t3.medium - maxSize: 1 - minSize: 1 - role: Master - subnets: - - us-test-1b - ---- - -apiVersion: kops.k8s.io/v1alpha2 -kind: InstanceGroup -metadata: - creationTimestamp: "2016-12-10T22:42:28Z" - name: master-us-test-1c - labels: - kops.k8s.io/cluster: launchtemplates.example.com -spec: - associatePublicIp: true - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: t3.medium - maxSize: 1 - minSize: 1 - role: Master - subnets: - - us-test-1c diff --git a/tests/integration/update_cluster/launch_templates/kubernetes.tf b/tests/integration/update_cluster/launch_templates/kubernetes.tf deleted file mode 100644 index 2059a896bb..0000000000 --- a/tests/integration/update_cluster/launch_templates/kubernetes.tf +++ /dev/null @@ -1,795 +0,0 @@ -locals { - cluster_name = "launchtemplates.example.com" - master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-launchtemplates-example-com.id, aws_autoscaling_group.master-us-test-1b-masters-launchtemplates-example-com.id, aws_autoscaling_group.master-us-test-1c-masters-launchtemplates-example-com.id] - master_security_group_ids = [aws_security_group.masters-launchtemplates-example-com.id] - masters_role_arn = aws_iam_role.masters-launchtemplates-example-com.arn - masters_role_name = aws_iam_role.masters-launchtemplates-example-com.name - node_autoscaling_group_ids = [aws_autoscaling_group.nodes-launchtemplates-example-com.id] - node_security_group_ids = [aws_security_group.nodes-launchtemplates-example-com.id] - node_subnet_ids = [aws_subnet.us-test-1b-launchtemplates-example-com.id] - nodes_role_arn = aws_iam_role.nodes-launchtemplates-example-com.arn - nodes_role_name = aws_iam_role.nodes-launchtemplates-example-com.name - region = "us-test-1" - route_table_public_id = aws_route_table.launchtemplates-example-com.id - subnet_us-test-1a_id = aws_subnet.us-test-1a-launchtemplates-example-com.id - subnet_us-test-1b_id = aws_subnet.us-test-1b-launchtemplates-example-com.id - subnet_us-test-1c_id = aws_subnet.us-test-1c-launchtemplates-example-com.id - vpc_cidr_block = aws_vpc.launchtemplates-example-com.cidr_block - vpc_id = aws_vpc.launchtemplates-example-com.id -} - -output "cluster_name" { - value = "launchtemplates.example.com" -} - -output "master_autoscaling_group_ids" { - value = [aws_autoscaling_group.master-us-test-1a-masters-launchtemplates-example-com.id, aws_autoscaling_group.master-us-test-1b-masters-launchtemplates-example-com.id, aws_autoscaling_group.master-us-test-1c-masters-launchtemplates-example-com.id] -} - -output "master_security_group_ids" { - value = [aws_security_group.masters-launchtemplates-example-com.id] -} - -output "masters_role_arn" { - value = aws_iam_role.masters-launchtemplates-example-com.arn -} - -output "masters_role_name" { - value = aws_iam_role.masters-launchtemplates-example-com.name -} - -output "node_autoscaling_group_ids" { - value = [aws_autoscaling_group.nodes-launchtemplates-example-com.id] -} - -output "node_security_group_ids" { - value = [aws_security_group.nodes-launchtemplates-example-com.id] -} - -output "node_subnet_ids" { - value = [aws_subnet.us-test-1b-launchtemplates-example-com.id] -} - -output "nodes_role_arn" { - value = aws_iam_role.nodes-launchtemplates-example-com.arn -} - -output "nodes_role_name" { - value = aws_iam_role.nodes-launchtemplates-example-com.name -} - -output "region" { - value = "us-test-1" -} - -output "route_table_public_id" { - value = aws_route_table.launchtemplates-example-com.id -} - -output "subnet_us-test-1a_id" { - value = aws_subnet.us-test-1a-launchtemplates-example-com.id -} - -output "subnet_us-test-1b_id" { - value = aws_subnet.us-test-1b-launchtemplates-example-com.id -} - -output "subnet_us-test-1c_id" { - value = aws_subnet.us-test-1c-launchtemplates-example-com.id -} - -output "vpc_cidr_block" { - value = aws_vpc.launchtemplates-example-com.cidr_block -} - -output "vpc_id" { - value = aws_vpc.launchtemplates-example-com.id -} - -provider "aws" { - region = "us-test-1" -} - -resource "aws_autoscaling_group" "master-us-test-1a-masters-launchtemplates-example-com" { - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] - launch_configuration = aws_launch_configuration.master-us-test-1a-masters-launchtemplates-example-com.id - max_size = 1 - metrics_granularity = "1Minute" - min_size = 1 - name = "master-us-test-1a.masters.launchtemplates.example.com" - tag { - key = "KubernetesCluster" - propagate_at_launch = true - value = "launchtemplates.example.com" - } - tag { - key = "Name" - propagate_at_launch = true - value = "master-us-test-1a.masters.launchtemplates.example.com" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" - propagate_at_launch = true - value = "" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" - propagate_at_launch = true - value = "master" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" - propagate_at_launch = true - value = "" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" - propagate_at_launch = true - value = "" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" - propagate_at_launch = true - value = "" - } - tag { - key = "k8s.io/role/master" - propagate_at_launch = true - value = "1" - } - tag { - key = "kops.k8s.io/instancegroup" - propagate_at_launch = true - value = "master-us-test-1a" - } - tag { - key = "kubernetes.io/cluster/launchtemplates.example.com" - propagate_at_launch = true - value = "owned" - } - vpc_zone_identifier = [aws_subnet.us-test-1a-launchtemplates-example-com.id] -} - -resource "aws_autoscaling_group" "master-us-test-1b-masters-launchtemplates-example-com" { - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] - launch_configuration = aws_launch_configuration.master-us-test-1b-masters-launchtemplates-example-com.id - max_size = 1 - metrics_granularity = "1Minute" - min_size = 1 - name = "master-us-test-1b.masters.launchtemplates.example.com" - tag { - key = "KubernetesCluster" - propagate_at_launch = true - value = "launchtemplates.example.com" - } - tag { - key = "Name" - propagate_at_launch = true - value = "master-us-test-1b.masters.launchtemplates.example.com" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" - propagate_at_launch = true - value = "" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" - propagate_at_launch = true - value = "master" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" - propagate_at_launch = true - value = "" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" - propagate_at_launch = true - value = "" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" - propagate_at_launch = true - value = "" - } - tag { - key = "k8s.io/role/master" - propagate_at_launch = true - value = "1" - } - tag { - key = "kops.k8s.io/instancegroup" - propagate_at_launch = true - value = "master-us-test-1b" - } - tag { - key = "kubernetes.io/cluster/launchtemplates.example.com" - propagate_at_launch = true - value = "owned" - } - vpc_zone_identifier = [aws_subnet.us-test-1b-launchtemplates-example-com.id] -} - -resource "aws_autoscaling_group" "master-us-test-1c-masters-launchtemplates-example-com" { - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] - launch_configuration = aws_launch_configuration.master-us-test-1c-masters-launchtemplates-example-com.id - max_size = 1 - metrics_granularity = "1Minute" - min_size = 1 - name = "master-us-test-1c.masters.launchtemplates.example.com" - tag { - key = "KubernetesCluster" - propagate_at_launch = true - value = "launchtemplates.example.com" - } - tag { - key = "Name" - propagate_at_launch = true - value = "master-us-test-1c.masters.launchtemplates.example.com" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" - propagate_at_launch = true - value = "" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" - propagate_at_launch = true - value = "master" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" - propagate_at_launch = true - value = "" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" - propagate_at_launch = true - value = "" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" - propagate_at_launch = true - value = "" - } - tag { - key = "k8s.io/role/master" - propagate_at_launch = true - value = "1" - } - tag { - key = "kops.k8s.io/instancegroup" - propagate_at_launch = true - value = "master-us-test-1c" - } - tag { - key = "kubernetes.io/cluster/launchtemplates.example.com" - propagate_at_launch = true - value = "owned" - } - vpc_zone_identifier = [aws_subnet.us-test-1c-launchtemplates-example-com.id] -} - -resource "aws_autoscaling_group" "nodes-launchtemplates-example-com" { - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] - launch_configuration = aws_launch_configuration.nodes-launchtemplates-example-com.id - max_size = 2 - metrics_granularity = "1Minute" - min_size = 2 - name = "nodes.launchtemplates.example.com" - protect_from_scale_in = true - tag { - key = "KubernetesCluster" - propagate_at_launch = true - value = "launchtemplates.example.com" - } - tag { - key = "Name" - propagate_at_launch = true - value = "nodes.launchtemplates.example.com" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" - propagate_at_launch = true - value = "node" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" - propagate_at_launch = true - value = "" - } - tag { - key = "k8s.io/role/node" - propagate_at_launch = true - value = "1" - } - tag { - key = "kops.k8s.io/instancegroup" - propagate_at_launch = true - value = "nodes" - } - tag { - key = "kubernetes.io/cluster/launchtemplates.example.com" - propagate_at_launch = true - value = "owned" - } - vpc_zone_identifier = [aws_subnet.us-test-1b-launchtemplates-example-com.id] -} - -resource "aws_ebs_volume" "us-test-1a-etcd-events-launchtemplates-example-com" { - availability_zone = "us-test-1a" - encrypted = false - iops = 3000 - size = 20 - tags = { - "KubernetesCluster" = "launchtemplates.example.com" - "Name" = "us-test-1a.etcd-events.launchtemplates.example.com" - "k8s.io/etcd/events" = "us-test-1a/us-test-1a,us-test-1b,us-test-1c" - "k8s.io/role/master" = "1" - "kubernetes.io/cluster/launchtemplates.example.com" = "owned" - } - throughput = 125 - type = "gp3" -} - -resource "aws_ebs_volume" "us-test-1a-etcd-main-launchtemplates-example-com" { - availability_zone = "us-test-1a" - encrypted = false - iops = 3000 - size = 20 - tags = { - "KubernetesCluster" = "launchtemplates.example.com" - "Name" = "us-test-1a.etcd-main.launchtemplates.example.com" - "k8s.io/etcd/main" = "us-test-1a/us-test-1a,us-test-1b,us-test-1c" - "k8s.io/role/master" = "1" - "kubernetes.io/cluster/launchtemplates.example.com" = "owned" - } - throughput = 125 - type = "gp3" -} - -resource "aws_ebs_volume" "us-test-1b-etcd-events-launchtemplates-example-com" { - availability_zone = "us-test-1b" - encrypted = false - iops = 3000 - size = 20 - tags = { - "KubernetesCluster" = "launchtemplates.example.com" - "Name" = "us-test-1b.etcd-events.launchtemplates.example.com" - "k8s.io/etcd/events" = "us-test-1b/us-test-1a,us-test-1b,us-test-1c" - "k8s.io/role/master" = "1" - "kubernetes.io/cluster/launchtemplates.example.com" = "owned" - } - throughput = 125 - type = "gp3" -} - -resource "aws_ebs_volume" "us-test-1b-etcd-main-launchtemplates-example-com" { - availability_zone = "us-test-1b" - encrypted = false - iops = 3000 - size = 20 - tags = { - "KubernetesCluster" = "launchtemplates.example.com" - "Name" = "us-test-1b.etcd-main.launchtemplates.example.com" - "k8s.io/etcd/main" = "us-test-1b/us-test-1a,us-test-1b,us-test-1c" - "k8s.io/role/master" = "1" - "kubernetes.io/cluster/launchtemplates.example.com" = "owned" - } - throughput = 125 - type = "gp3" -} - -resource "aws_ebs_volume" "us-test-1c-etcd-events-launchtemplates-example-com" { - availability_zone = "us-test-1c" - encrypted = false - iops = 3000 - size = 20 - tags = { - "KubernetesCluster" = "launchtemplates.example.com" - "Name" = "us-test-1c.etcd-events.launchtemplates.example.com" - "k8s.io/etcd/events" = "us-test-1c/us-test-1a,us-test-1b,us-test-1c" - "k8s.io/role/master" = "1" - "kubernetes.io/cluster/launchtemplates.example.com" = "owned" - } - throughput = 125 - type = "gp3" -} - -resource "aws_ebs_volume" "us-test-1c-etcd-main-launchtemplates-example-com" { - availability_zone = "us-test-1c" - encrypted = false - iops = 3000 - size = 20 - tags = { - "KubernetesCluster" = "launchtemplates.example.com" - "Name" = "us-test-1c.etcd-main.launchtemplates.example.com" - "k8s.io/etcd/main" = "us-test-1c/us-test-1a,us-test-1b,us-test-1c" - "k8s.io/role/master" = "1" - "kubernetes.io/cluster/launchtemplates.example.com" = "owned" - } - throughput = 125 - type = "gp3" -} - -resource "aws_iam_instance_profile" "masters-launchtemplates-example-com" { - name = "masters.launchtemplates.example.com" - role = aws_iam_role.masters-launchtemplates-example-com.name -} - -resource "aws_iam_instance_profile" "nodes-launchtemplates-example-com" { - name = "nodes.launchtemplates.example.com" - role = aws_iam_role.nodes-launchtemplates-example-com.name -} - -resource "aws_iam_role_policy" "masters-launchtemplates-example-com" { - name = "masters.launchtemplates.example.com" - policy = file("${path.module}/data/aws_iam_role_policy_masters.launchtemplates.example.com_policy") - role = aws_iam_role.masters-launchtemplates-example-com.name -} - -resource "aws_iam_role_policy" "nodes-launchtemplates-example-com" { - name = "nodes.launchtemplates.example.com" - policy = file("${path.module}/data/aws_iam_role_policy_nodes.launchtemplates.example.com_policy") - role = aws_iam_role.nodes-launchtemplates-example-com.name -} - -resource "aws_iam_role" "masters-launchtemplates-example-com" { - assume_role_policy = file("${path.module}/data/aws_iam_role_masters.launchtemplates.example.com_policy") - name = "masters.launchtemplates.example.com" - tags = { - "KubernetesCluster" = "launchtemplates.example.com" - "Name" = "masters.launchtemplates.example.com" - "kubernetes.io/cluster/launchtemplates.example.com" = "owned" - } -} - -resource "aws_iam_role" "nodes-launchtemplates-example-com" { - assume_role_policy = file("${path.module}/data/aws_iam_role_nodes.launchtemplates.example.com_policy") - name = "nodes.launchtemplates.example.com" - tags = { - "KubernetesCluster" = "launchtemplates.example.com" - "Name" = "nodes.launchtemplates.example.com" - "kubernetes.io/cluster/launchtemplates.example.com" = "owned" - } -} - -resource "aws_internet_gateway" "launchtemplates-example-com" { - tags = { - "KubernetesCluster" = "launchtemplates.example.com" - "Name" = "launchtemplates.example.com" - "kubernetes.io/cluster/launchtemplates.example.com" = "owned" - } - vpc_id = aws_vpc.launchtemplates-example-com.id -} - -resource "aws_key_pair" "kubernetes-launchtemplates-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" { - key_name = "kubernetes.launchtemplates.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57" - public_key = file("${path.module}/data/aws_key_pair_kubernetes.launchtemplates.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key") - tags = { - "KubernetesCluster" = "launchtemplates.example.com" - "Name" = "launchtemplates.example.com" - "kubernetes.io/cluster/launchtemplates.example.com" = "owned" - } -} - -resource "aws_launch_configuration" "master-us-test-1a-masters-launchtemplates-example-com" { - associate_public_ip_address = true - enable_monitoring = false - iam_instance_profile = aws_iam_instance_profile.masters-launchtemplates-example-com.id - image_id = "ami-12345678" - instance_type = "t3.medium" - key_name = aws_key_pair.kubernetes-launchtemplates-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id - lifecycle { - create_before_destroy = true - } - name_prefix = "master-us-test-1a.masters.launchtemplates.example.com-" - root_block_device { - delete_on_termination = true - encrypted = true - volume_size = 64 - volume_type = "gp2" - } - security_groups = [aws_security_group.masters-launchtemplates-example-com.id] - user_data = file("${path.module}/data/aws_launch_configuration_master-us-test-1a.masters.launchtemplates.example.com_user_data") -} - -resource "aws_launch_configuration" "master-us-test-1b-masters-launchtemplates-example-com" { - associate_public_ip_address = true - enable_monitoring = false - iam_instance_profile = aws_iam_instance_profile.masters-launchtemplates-example-com.id - image_id = "ami-12345678" - instance_type = "t3.medium" - key_name = aws_key_pair.kubernetes-launchtemplates-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id - lifecycle { - create_before_destroy = true - } - name_prefix = "master-us-test-1b.masters.launchtemplates.example.com-" - root_block_device { - delete_on_termination = true - encrypted = true - volume_size = 64 - volume_type = "gp2" - } - security_groups = [aws_security_group.masters-launchtemplates-example-com.id] - user_data = file("${path.module}/data/aws_launch_configuration_master-us-test-1b.masters.launchtemplates.example.com_user_data") -} - -resource "aws_launch_configuration" "master-us-test-1c-masters-launchtemplates-example-com" { - associate_public_ip_address = true - enable_monitoring = false - iam_instance_profile = aws_iam_instance_profile.masters-launchtemplates-example-com.id - image_id = "ami-12345678" - instance_type = "t3.medium" - key_name = aws_key_pair.kubernetes-launchtemplates-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id - lifecycle { - create_before_destroy = true - } - name_prefix = "master-us-test-1c.masters.launchtemplates.example.com-" - root_block_device { - delete_on_termination = true - encrypted = true - volume_size = 64 - volume_type = "gp2" - } - security_groups = [aws_security_group.masters-launchtemplates-example-com.id] - user_data = file("${path.module}/data/aws_launch_configuration_master-us-test-1c.masters.launchtemplates.example.com_user_data") -} - -resource "aws_launch_configuration" "nodes-launchtemplates-example-com" { - associate_public_ip_address = true - enable_monitoring = false - iam_instance_profile = aws_iam_instance_profile.nodes-launchtemplates-example-com.id - image_id = "ami-12345678" - instance_type = "t3.medium" - key_name = aws_key_pair.kubernetes-launchtemplates-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id - lifecycle { - create_before_destroy = true - } - name_prefix = "nodes.launchtemplates.example.com-" - root_block_device { - delete_on_termination = true - encrypted = true - volume_size = 128 - volume_type = "gp2" - } - security_groups = [aws_security_group.nodes-launchtemplates-example-com.id] - spot_price = "0.1" - user_data = file("${path.module}/data/aws_launch_configuration_nodes.launchtemplates.example.com_user_data") -} - -resource "aws_route_table_association" "us-test-1a-launchtemplates-example-com" { - route_table_id = aws_route_table.launchtemplates-example-com.id - subnet_id = aws_subnet.us-test-1a-launchtemplates-example-com.id -} - -resource "aws_route_table_association" "us-test-1b-launchtemplates-example-com" { - route_table_id = aws_route_table.launchtemplates-example-com.id - subnet_id = aws_subnet.us-test-1b-launchtemplates-example-com.id -} - -resource "aws_route_table_association" "us-test-1c-launchtemplates-example-com" { - route_table_id = aws_route_table.launchtemplates-example-com.id - subnet_id = aws_subnet.us-test-1c-launchtemplates-example-com.id -} - -resource "aws_route_table" "launchtemplates-example-com" { - tags = { - "KubernetesCluster" = "launchtemplates.example.com" - "Name" = "launchtemplates.example.com" - "kubernetes.io/cluster/launchtemplates.example.com" = "owned" - "kubernetes.io/kops/role" = "public" - } - vpc_id = aws_vpc.launchtemplates-example-com.id -} - -resource "aws_route" "route-0-0-0-0--0" { - destination_cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.launchtemplates-example-com.id - route_table_id = aws_route_table.launchtemplates-example-com.id -} - -resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-masters-launchtemplates-example-com" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.masters-launchtemplates-example-com.id - to_port = 22 - type = "ingress" -} - -resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-nodes-launchtemplates-example-com" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.nodes-launchtemplates-example-com.id - to_port = 22 - type = "ingress" -} - -resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-masters-launchtemplates-example-com" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 443 - protocol = "tcp" - security_group_id = aws_security_group.masters-launchtemplates-example-com.id - to_port = 443 - type = "ingress" -} - -resource "aws_security_group_rule" "from-masters-launchtemplates-example-com-egress-all-0to0-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.masters-launchtemplates-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-masters-launchtemplates-example-com-ingress-all-0to0-masters-launchtemplates-example-com" { - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.masters-launchtemplates-example-com.id - source_security_group_id = aws_security_group.masters-launchtemplates-example-com.id - to_port = 0 - type = "ingress" -} - -resource "aws_security_group_rule" "from-masters-launchtemplates-example-com-ingress-all-0to0-nodes-launchtemplates-example-com" { - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.nodes-launchtemplates-example-com.id - source_security_group_id = aws_security_group.masters-launchtemplates-example-com.id - to_port = 0 - type = "ingress" -} - -resource "aws_security_group_rule" "from-nodes-launchtemplates-example-com-egress-all-0to0-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.nodes-launchtemplates-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-nodes-launchtemplates-example-com-ingress-all-0to0-nodes-launchtemplates-example-com" { - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.nodes-launchtemplates-example-com.id - source_security_group_id = aws_security_group.nodes-launchtemplates-example-com.id - to_port = 0 - type = "ingress" -} - -resource "aws_security_group_rule" "from-nodes-launchtemplates-example-com-ingress-tcp-1to2379-masters-launchtemplates-example-com" { - from_port = 1 - protocol = "tcp" - security_group_id = aws_security_group.masters-launchtemplates-example-com.id - source_security_group_id = aws_security_group.nodes-launchtemplates-example-com.id - to_port = 2379 - type = "ingress" -} - -resource "aws_security_group_rule" "from-nodes-launchtemplates-example-com-ingress-tcp-2382to4000-masters-launchtemplates-example-com" { - from_port = 2382 - protocol = "tcp" - security_group_id = aws_security_group.masters-launchtemplates-example-com.id - source_security_group_id = aws_security_group.nodes-launchtemplates-example-com.id - to_port = 4000 - type = "ingress" -} - -resource "aws_security_group_rule" "from-nodes-launchtemplates-example-com-ingress-tcp-4003to65535-masters-launchtemplates-example-com" { - from_port = 4003 - protocol = "tcp" - security_group_id = aws_security_group.masters-launchtemplates-example-com.id - source_security_group_id = aws_security_group.nodes-launchtemplates-example-com.id - to_port = 65535 - type = "ingress" -} - -resource "aws_security_group_rule" "from-nodes-launchtemplates-example-com-ingress-udp-1to65535-masters-launchtemplates-example-com" { - from_port = 1 - protocol = "udp" - security_group_id = aws_security_group.masters-launchtemplates-example-com.id - source_security_group_id = aws_security_group.nodes-launchtemplates-example-com.id - to_port = 65535 - type = "ingress" -} - -resource "aws_security_group" "masters-launchtemplates-example-com" { - description = "Security group for masters" - name = "masters.launchtemplates.example.com" - tags = { - "KubernetesCluster" = "launchtemplates.example.com" - "Name" = "masters.launchtemplates.example.com" - "kubernetes.io/cluster/launchtemplates.example.com" = "owned" - } - vpc_id = aws_vpc.launchtemplates-example-com.id -} - -resource "aws_security_group" "nodes-launchtemplates-example-com" { - description = "Security group for nodes" - name = "nodes.launchtemplates.example.com" - tags = { - "KubernetesCluster" = "launchtemplates.example.com" - "Name" = "nodes.launchtemplates.example.com" - "kubernetes.io/cluster/launchtemplates.example.com" = "owned" - } - vpc_id = aws_vpc.launchtemplates-example-com.id -} - -resource "aws_subnet" "us-test-1a-launchtemplates-example-com" { - availability_zone = "us-test-1a" - cidr_block = "10.0.1.0/24" - tags = { - "KubernetesCluster" = "launchtemplates.example.com" - "Name" = "us-test-1a.launchtemplates.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/launchtemplates.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - } - vpc_id = aws_vpc.launchtemplates-example-com.id -} - -resource "aws_subnet" "us-test-1b-launchtemplates-example-com" { - availability_zone = "us-test-1b" - cidr_block = "10.0.2.0/24" - tags = { - "KubernetesCluster" = "launchtemplates.example.com" - "Name" = "us-test-1b.launchtemplates.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/launchtemplates.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - } - vpc_id = aws_vpc.launchtemplates-example-com.id -} - -resource "aws_subnet" "us-test-1c-launchtemplates-example-com" { - availability_zone = "us-test-1c" - cidr_block = "10.0.3.0/24" - tags = { - "KubernetesCluster" = "launchtemplates.example.com" - "Name" = "us-test-1c.launchtemplates.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/launchtemplates.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - } - vpc_id = aws_vpc.launchtemplates-example-com.id -} - -resource "aws_vpc_dhcp_options_association" "launchtemplates-example-com" { - dhcp_options_id = aws_vpc_dhcp_options.launchtemplates-example-com.id - vpc_id = aws_vpc.launchtemplates-example-com.id -} - -resource "aws_vpc_dhcp_options" "launchtemplates-example-com" { - domain_name = "us-test-1.compute.internal" - domain_name_servers = ["AmazonProvidedDNS"] - tags = { - "KubernetesCluster" = "launchtemplates.example.com" - "Name" = "launchtemplates.example.com" - "kubernetes.io/cluster/launchtemplates.example.com" = "owned" - } -} - -resource "aws_vpc" "launchtemplates-example-com" { - cidr_block = "10.0.0.0/16" - enable_dns_hostnames = true - enable_dns_support = true - tags = { - "KubernetesCluster" = "launchtemplates.example.com" - "Name" = "launchtemplates.example.com" - "kubernetes.io/cluster/launchtemplates.example.com" = "owned" - } -} - -terraform { - required_version = ">= 0.12.26" - required_providers { - aws = { - "source" = "hashicorp/aws" - "version" = ">= 2.46.0" - } - } -} diff --git a/upup/pkg/fi/cloudup/awstasks/BUILD.bazel b/upup/pkg/fi/cloudup/awstasks/BUILD.bazel index cdd8ff64e1..8823d93815 100644 --- a/upup/pkg/fi/cloudup/awstasks/BUILD.bazel +++ b/upup/pkg/fi/cloudup/awstasks/BUILD.bazel @@ -39,8 +39,6 @@ go_library( "instance_volume_attachment.go", "internetgateway.go", "internetgateway_fitask.go", - "launchconfiguration.go", - "launchconfiguration_fitask.go", "launchtemplate.go", "launchtemplate_fitask.go", "launchtemplate_target_api.go", @@ -98,7 +96,6 @@ go_library( "//vendor/github.com/aws/aws-sdk-go/service/elbv2:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/iam:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/route53:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", ], @@ -111,7 +108,6 @@ go_test( "ebsvolume_test.go", "elastic_ip_test.go", "internetgateway_test.go", - "launchconfiguration_test.go", "launchtemplate_target_cloudformation_test.go", "launchtemplate_target_terraform_test.go", "render_test.go", @@ -121,7 +117,6 @@ go_test( ], embed = [":go_default_library"], deps = [ - "//cloudmock/aws/mockautoscaling:go_default_library", "//cloudmock/aws/mockec2:go_default_library", "//pkg/apis/kops:go_default_library", "//pkg/assets:go_default_library", diff --git a/upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go b/upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go index 1c7b2764de..e3fc9dd01c 100644 --- a/upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go +++ b/upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go @@ -47,8 +47,6 @@ type AutoscalingGroup struct { Granularity *string // InstanceProtection makes new instances in an autoscaling group protected from scale in InstanceProtection *bool - // LaunchConfiguration is the launch configuration for the autoscaling group - LaunchConfiguration *LaunchConfiguration // LaunchTemplate is the launch template for the asg LaunchTemplate *LaunchTemplate // LoadBalancers is a list of elastic load balancer names to add to the autoscaling group @@ -199,9 +197,6 @@ func (e *AutoscalingGroup) Find(c *fi.Context) (*AutoscalingGroup, error) { } } - if g.LaunchConfigurationName != nil { - actual.LaunchConfiguration = &LaunchConfiguration{ID: g.LaunchConfigurationName} - } if g.LaunchTemplate != nil { actual.LaunchTemplate = &LaunchTemplate{ Name: g.LaunchTemplate.LaunchTemplateName, @@ -356,10 +351,8 @@ func (v *AutoscalingGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Autos request.TargetGroupARNs = append(request.TargetGroupARNs, tg.ARN) } - // @check are we using a launch configuration, mixed instances policy, or launch template - if e.LaunchConfiguration != nil { - request.LaunchConfigurationName = e.LaunchConfiguration.ID - } else if e.UseMixedInstancesPolicy() { + // @check are we using mixed instances policy, or launch template + if e.UseMixedInstancesPolicy() { request.MixedInstancesPolicy = &autoscaling.MixedInstancesPolicy{ InstancesDistribution: &autoscaling.InstancesDistribution{ OnDemandPercentageAboveBaseCapacity: e.MixedOnDemandAboveBase, @@ -387,7 +380,7 @@ func (v *AutoscalingGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Autos Version: aws.String("$Latest"), } } else { - return fmt.Errorf("could not find one of launch template, mixed instances policy, or launch configuration") + return fmt.Errorf("could not find one of launch template or mixed instances policy") } // @step: attempt to create the autoscaling group for us @@ -434,10 +427,6 @@ func (v *AutoscalingGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Autos AutoScalingGroupName: e.Name, } - if changes.LaunchConfiguration != nil { - request.LaunchConfigurationName = e.LaunchConfiguration.ID - changes.LaunchConfiguration = nil - } setup := func(req *autoscaling.UpdateAutoScalingGroupInput) *autoscaling.MixedInstancesPolicy { if req.MixedInstancesPolicy == nil { req.MixedInstancesPolicy = &autoscaling.MixedInstancesPolicy{ @@ -912,9 +901,7 @@ func (_ *AutoscalingGroup) RenderTerraform(t *terraform.TerraformTarget, a, e, c } terraform.SortLiterals(tf.TargetGroupARNs) - if e.LaunchConfiguration != nil { - tf.LaunchConfigurationName = e.LaunchConfiguration.TerraformLink() - } else if e.UseMixedInstancesPolicy() { + if e.UseMixedInstancesPolicy() { // Temporary warning until https://github.com/terraform-providers/terraform-provider-aws/issues/9750 is resolved if e.MixedSpotAllocationStrategy == fi.String("capacity-optimized") { fmt.Print("Terraform does not currently support a capacity optimized strategy - please see https://github.com/terraform-providers/terraform-provider-aws/issues/9750") @@ -968,17 +955,7 @@ func (_ *AutoscalingGroup) RenderTerraform(t *terraform.TerraformTarget, a, e, c } } - if e.LaunchConfiguration != nil { - // Create TF output variable with security group ids - // This is in the launch configuration, but the ASG has the information about the instance group type - if role != "" { - for _, sg := range e.LaunchConfiguration.SecurityGroups { - if err := t.AddOutputVariableArray(role+"_security_group_ids", sg.TerraformLink()); err != nil { - return err - } - } - } - } else if e.LaunchTemplate != nil && role != "" { + if e.LaunchTemplate != nil && role != "" { for _, sg := range e.LaunchTemplate.SecurityGroups { if err := t.AddOutputVariableArray(role+"_security_group_ids", sg.TerraformLink()); err != nil { return err @@ -1094,9 +1071,7 @@ func (_ *AutoscalingGroup) RenderCloudformation(t *cloudformation.Cloudformation }, } - if e.LaunchConfiguration != nil { - cf.LaunchConfigurationName = e.LaunchConfiguration.CloudformationLink() - } else if e.UseMixedInstancesPolicy() { + if e.UseMixedInstancesPolicy() { cf.MixedInstancesPolicy = &cloudformationMixedInstancesPolicy{ LaunchTemplate: &cloudformationAutoscalingLaunchTemplate{ LaunchTemplateSpecification: &cloudformationAutoscalingLaunchTemplateSpecification{ diff --git a/upup/pkg/fi/cloudup/awstasks/autoscalinggroup_test.go b/upup/pkg/fi/cloudup/awstasks/autoscalinggroup_test.go index f7cfeacb79..faf8511a41 100644 --- a/upup/pkg/fi/cloudup/awstasks/autoscalinggroup_test.go +++ b/upup/pkg/fi/cloudup/awstasks/autoscalinggroup_test.go @@ -201,12 +201,12 @@ func TestAutoscalingGroupTerraformRender(t *testing.T) { cases := []*renderTest{ { Resource: &AutoscalingGroup{ - Name: fi.String("test"), - Granularity: fi.String("5min"), - LaunchConfiguration: &LaunchConfiguration{Name: fi.String("test_lc")}, - MaxSize: fi.Int64(10), - Metrics: []string{"test"}, - MinSize: fi.Int64(1), + Name: fi.String("test"), + Granularity: fi.String("5min"), + LaunchTemplate: &LaunchTemplate{Name: fi.String("test_lc")}, + MaxSize: fi.Int64(10), + Metrics: []string{"test"}, + MinSize: fi.Int64(1), Subnets: []*Subnet{ { Name: fi.String("test-sg"), @@ -223,12 +223,15 @@ func TestAutoscalingGroupTerraformRender(t *testing.T) { } resource "aws_autoscaling_group" "test" { - enabled_metrics = ["test"] - launch_configuration = aws_launch_configuration.test_lc.id - max_size = 10 - metrics_granularity = "5min" - min_size = 1 - name = "test" + enabled_metrics = ["test"] + launch_template { + id = aws_launch_template.test_lc.id + version = aws_launch_template.test_lc.latest_version + } + max_size = 10 + metrics_granularity = "5min" + min_size = 1 + name = "test" tag { key = "cluster" propagate_at_launch = true diff --git a/upup/pkg/fi/cloudup/awstasks/launchconfiguration.go b/upup/pkg/fi/cloudup/awstasks/launchconfiguration.go deleted file mode 100644 index df3bc8a679..0000000000 --- a/upup/pkg/fi/cloudup/awstasks/launchconfiguration.go +++ /dev/null @@ -1,794 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package awstasks - -import ( - "encoding/base64" - "fmt" - "math" - "sort" - "strings" - - "k8s.io/kops/pkg/apis/kops" - "k8s.io/kops/pkg/featureflag" - "k8s.io/kops/upup/pkg/fi" - "k8s.io/kops/upup/pkg/fi/cloudup/awsup" - "k8s.io/kops/upup/pkg/fi/cloudup/cloudformation" - "k8s.io/kops/upup/pkg/fi/cloudup/terraform" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/autoscaling" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/klog/v2" -) - -// defaultRetainLaunchConfigurationCount is the number of launch configurations (matching the name prefix) that we should -// keep, we delete older ones -var defaultRetainLaunchConfigurationCount = 3 - -// RetainLaunchConfigurationCount returns the number of launch configurations to keep -func RetainLaunchConfigurationCount() int { - if featureflag.KeepLaunchConfigurations.Enabled() { - return math.MaxInt32 - } - return defaultRetainLaunchConfigurationCount -} - -// LaunchConfiguration is the specification for a launch configuration -// +kops:fitask -type LaunchConfiguration struct { - // Name is the name of the configuration - Name *string - // Lifecycle is the resource lifecycle - Lifecycle *fi.Lifecycle - - // AssociatePublicIP indicates if a public ip address is assigned to instabces - AssociatePublicIP *bool - // BlockDeviceMappings is a block device mappings - BlockDeviceMappings []*BlockDeviceMapping - // HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for instance metadata requests. - HTTPPutResponseHopLimit *int64 - // HTTPTokens is the state of token usage for your instance metadata requests. - HTTPTokens *string - // IAMInstanceProfile is the IAM profile to assign to the nodes - IAMInstanceProfile *IAMInstanceProfile - // ID is the launch configuration name - ID *string - // ImageID is the AMI to use for the instances - ImageID *string - // InstanceMonitoring indicates if monitoring is enabled - InstanceMonitoring *bool - // InstanceType is the machine type to use - InstanceType *string - // RootVolumeDeleteOnTermination states if the root volume will be deleted after instance termination - RootVolumeDeleteOnTermination *bool - // RootVolumeIops is the provisioned IOPS when the volume type is io1, io2 or gp3 - RootVolumeIops *int64 - // RootVolumeOptimization enables EBS optimization for an instance - RootVolumeOptimization *bool - // RootVolumeSize is the size of the EBS root volume to use, in GB - RootVolumeSize *int64 - // RootVolumeType is the type of the EBS root volume to use (e.g. gp2) - RootVolumeType *string - // RootVolumeEncryption enables EBS root volume encryption for an instance - RootVolumeEncryption *bool - // SSHKey is the ssh key for the instances - SSHKey *SSHKey - // SecurityGroups is a list of security group associated - SecurityGroups []*SecurityGroup - // SpotPrice is set to the spot-price bid if this is a spot pricing request - SpotPrice string - // Tenancy. Can be either default or dedicated. - Tenancy *string - // UserData is the user data configuration - UserData fi.Resource -} - -var _ fi.CompareWithID = &LaunchConfiguration{} -var _ fi.ProducesDeletions = &LaunchConfiguration{} - -func (e *LaunchConfiguration) CompareWithID() *string { - return e.ID -} - -// findLaunchConfigurations returns matching LaunchConfigurations, sorted by CreatedTime (ascending) -func (e *LaunchConfiguration) findLaunchConfigurations(c *fi.Context) ([]*autoscaling.LaunchConfiguration, error) { - cloud := c.Cloud.(awsup.AWSCloud) - - request := &autoscaling.DescribeLaunchConfigurationsInput{} - - prefix := *e.Name + "-" - - var configurations []*autoscaling.LaunchConfiguration - err := cloud.Autoscaling().DescribeLaunchConfigurationsPages(request, func(page *autoscaling.DescribeLaunchConfigurationsOutput, lastPage bool) bool { - for _, l := range page.LaunchConfigurations { - name := aws.StringValue(l.LaunchConfigurationName) - if strings.HasPrefix(name, prefix) { - configurations = append(configurations, l) - } - } - return true - }) - if err != nil { - return nil, fmt.Errorf("error listing AutoscalingLaunchConfigurations: %v", err) - } - - sort.Slice(configurations, func(i, j int) bool { - ti := configurations[i].CreatedTime - tj := configurations[j].CreatedTime - if tj == nil { - return true - } - if ti == nil { - return false - } - return ti.UnixNano() < tj.UnixNano() - }) - - return configurations, nil -} - -// Find is responsible for finding the launch configuration -func (e *LaunchConfiguration) Find(c *fi.Context) (*LaunchConfiguration, error) { - cloud := c.Cloud.(awsup.AWSCloud) - - configurations, err := e.findLaunchConfigurations(c) - if err != nil { - return nil, err - } - - if len(configurations) == 0 { - return nil, nil - } - - // We pick up the latest launch configuration - // (TODO: this might not actually be attached to the AutoScalingGroup, if something went wrong previously) - lc := configurations[len(configurations)-1] - - klog.V(2).Infof("found existing AutoscalingLaunchConfiguration: %q", *lc.LaunchConfigurationName) - - actual := &LaunchConfiguration{ - Name: e.Name, - AssociatePublicIP: lc.AssociatePublicIpAddress, - ID: lc.LaunchConfigurationName, - ImageID: lc.ImageId, - InstanceMonitoring: lc.InstanceMonitoring.Enabled, - InstanceType: lc.InstanceType, - RootVolumeOptimization: lc.EbsOptimized, - SpotPrice: aws.StringValue(lc.SpotPrice), - Tenancy: lc.PlacementTenancy, - } - - // Only assign keyName if the existing launch config has one - // lc.KeyName comes back as an empty string when there is no key assigned - if lc.KeyName != nil && *lc.KeyName != "" { - actual.SSHKey = &SSHKey{Name: lc.KeyName} - } - - if lc.IamInstanceProfile != nil { - actual.IAMInstanceProfile = &IAMInstanceProfile{Name: lc.IamInstanceProfile} - } - - securityGroups := []*SecurityGroup{} - for _, sgID := range lc.SecurityGroups { - securityGroups = append(securityGroups, &SecurityGroup{ID: sgID}) - } - sort.Sort(OrderSecurityGroupsById(securityGroups)) - - actual.SecurityGroups = securityGroups - - // @step: get the image is order to find out the root device name as using the index - // is not variable, under conditions they move - image, err := cloud.ResolveImage(fi.StringValue(e.ImageID)) - if err != nil { - return nil, err - } - - // Find the root volume - for _, b := range lc.BlockDeviceMappings { - if b.Ebs == nil { - continue - } - if b.DeviceName != nil && fi.StringValue(b.DeviceName) == fi.StringValue(image.RootDeviceName) { - actual.RootVolumeSize = b.Ebs.VolumeSize - actual.RootVolumeType = b.Ebs.VolumeType - actual.RootVolumeIops = b.Ebs.Iops - actual.RootVolumeEncryption = b.Ebs.Encrypted - actual.RootVolumeDeleteOnTermination = b.Ebs.DeleteOnTermination - } else { - _, d := BlockDeviceMappingFromAutoscaling(b) - actual.BlockDeviceMappings = append(actual.BlockDeviceMappings, d) - } - } - - if lc.UserData != nil { - userData, err := base64.StdEncoding.DecodeString(aws.StringValue(lc.UserData)) - if err != nil { - return nil, fmt.Errorf("error decoding UserData: %v", err) - } - actual.UserData = fi.NewStringResource(string(userData)) - } - - // Avoid spurious changes on ImageId - if e.ImageID != nil && actual.ImageID != nil && *actual.ImageID != *e.ImageID { - image, err := cloud.ResolveImage(*e.ImageID) - if err != nil { - klog.Warningf("unable to resolve image: %q: %v", *e.ImageID, err) - } else if image == nil { - klog.Warningf("unable to resolve image: %q: not found", *e.ImageID) - } else if aws.StringValue(image.ImageId) == *actual.ImageID { - klog.V(4).Infof("Returning matching ImageId as expected name: %q -> %q", *actual.ImageID, *e.ImageID) - actual.ImageID = e.ImageID - } - } - - // Avoid spurious changes - actual.Lifecycle = e.Lifecycle - - if e.ID == nil { - e.ID = actual.ID - } - - return actual, nil -} - -func (e *LaunchConfiguration) Run(c *fi.Context) error { - // TODO: Make Normalize a standard method - e.Normalize() - - if e.SSHKey == nil && !useSSHKey(c.Cluster) { - e.SSHKey = &SSHKey{} - } - - return fi.DefaultDeltaRunMethod(e, c) -} - -func (e *LaunchConfiguration) Normalize() { - // We need to sort our arrays consistently, so we don't get spurious changes - sort.Stable(OrderSecurityGroupsById(e.SecurityGroups)) -} - -func (s *LaunchConfiguration) CheckChanges(a, e, changes *LaunchConfiguration) error { - if e.ImageID == nil { - return fi.RequiredField("ImageID") - } - if e.InstanceType == nil { - return fi.RequiredField("InstanceType") - } - - if a != nil { - if e.Name == nil { - return fi.RequiredField("Name") - } - } - return nil -} - -// RenderAWS is responsible for creating the launchconfiguration via api -func (_ *LaunchConfiguration) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *LaunchConfiguration) error { - launchConfigurationName := *e.Name + "-" + fi.BuildTimestampString() - - klog.V(2).Infof("Creating AutoscalingLaunchConfiguration with Name:%q", launchConfigurationName) - - if e.ImageID == nil { - return fi.RequiredField("ImageID") - } - - image, err := t.Cloud.ResolveImage(*e.ImageID) - if err != nil { - return err - } - - request := &autoscaling.CreateLaunchConfigurationInput{ - AssociatePublicIpAddress: e.AssociatePublicIP, - EbsOptimized: e.RootVolumeOptimization, - ImageId: image.ImageId, - InstanceType: e.InstanceType, - LaunchConfigurationName: &launchConfigurationName, - } - - request.MetadataOptions = &autoscaling.InstanceMetadataOptions{ - HttpPutResponseHopLimit: e.HTTPPutResponseHopLimit, - HttpTokens: e.HTTPTokens, - } - - if e.SSHKey != nil { - request.KeyName = e.SSHKey.Name - } - - if e.Tenancy != nil { - request.PlacementTenancy = e.Tenancy - } - - securityGroupIDs := []*string{} - for _, sg := range e.SecurityGroups { - securityGroupIDs = append(securityGroupIDs, sg.ID) - } - - request.SecurityGroups = securityGroupIDs - request.AssociatePublicIpAddress = e.AssociatePublicIP - if e.SpotPrice != "" { - request.SpotPrice = aws.String(e.SpotPrice) - } - - // Build up the actual block device mappings - { - rootDevices, err := e.buildRootDevice(t.Cloud) - if err != nil { - return err - } - ephemeralDevices, err := buildEphemeralDevices(t.Cloud, fi.StringValue(e.InstanceType)) - if err != nil { - return err - } - additionalDevices, err := buildAdditionalDevices(e.BlockDeviceMappings) - if err != nil { - return err - } - - // @step: add all the devices to the block device mappings - for _, x := range []map[string]*BlockDeviceMapping{rootDevices, ephemeralDevices, additionalDevices} { - for name, device := range x { - request.BlockDeviceMappings = append(request.BlockDeviceMappings, device.ToAutoscaling(name)) - } - } - } - - if e.UserData != nil { - d, err := fi.ResourceAsBytes(e.UserData) - if err != nil { - return fmt.Errorf("error rendering AutoScalingLaunchConfiguration UserData: %v", err) - } - request.UserData = aws.String(base64.StdEncoding.EncodeToString(d)) - } - if e.IAMInstanceProfile != nil { - request.IamInstanceProfile = e.IAMInstanceProfile.Name - } - if e.InstanceMonitoring != nil { - request.InstanceMonitoring = &autoscaling.InstanceMonitoring{Enabled: e.InstanceMonitoring} - } else { - request.InstanceMonitoring = &autoscaling.InstanceMonitoring{Enabled: fi.Bool(false)} - } - - if _, err = t.Cloud.Autoscaling().CreateLaunchConfiguration(request); err != nil { - code := awsup.AWSErrorCode(err) - message := awsup.AWSErrorMessage(err) - if code == "ValidationError" && strings.Contains(message, "Invalid IamInstanceProfile") { - klog.V(4).Infof("error creating LaunchConfiguration: %s", message) - return fi.NewTryAgainLaterError("waiting for the IAM Instance Profile to be propagated") - } - return fmt.Errorf("error creating LaunchConfiguration: %s", message) - } - - e.ID = fi.String(launchConfigurationName) - - return nil // No tags on a launch configuration -} - -// buildRootDevice is responsible for retrieving a boot device mapping from the image name -func (t *LaunchConfiguration) buildRootDevice(cloud awsup.AWSCloud) (map[string]*BlockDeviceMapping, error) { - image := fi.StringValue(t.ImageID) - - // @step: resolve the image ami - img, err := cloud.ResolveImage(image) - if err != nil { - return nil, fmt.Errorf("unable to resolve image: %q: %v", image, err) - } else if img == nil { - return nil, fmt.Errorf("unable to resolve image: %q: not found", image) - } - - bm := make(map[string]*BlockDeviceMapping) - - bm[aws.StringValue(img.RootDeviceName)] = &BlockDeviceMapping{ - EbsDeleteOnTermination: t.RootVolumeDeleteOnTermination, - EbsVolumeSize: t.RootVolumeSize, - EbsVolumeType: t.RootVolumeType, - EbsVolumeIops: t.RootVolumeIops, - EbsEncrypted: t.RootVolumeEncryption, - } - - return bm, nil -} - -type terraformLaunchConfiguration struct { - NamePrefix *string `json:"name_prefix,omitempty" cty:"name_prefix"` - ImageID *string `json:"image_id,omitempty" cty:"image_id"` - InstanceType *string `json:"instance_type,omitempty" cty:"instance_type"` - KeyName *terraform.Literal `json:"key_name,omitempty" cty:"key_name"` - IAMInstanceProfile *terraform.Literal `json:"iam_instance_profile,omitempty" cty:"iam_instance_profile"` - SecurityGroups []*terraform.Literal `json:"security_groups,omitempty" cty:"security_groups"` - AssociatePublicIpAddress *bool `json:"associate_public_ip_address,omitempty" cty:"associate_public_ip_address"` - UserData *terraform.Literal `json:"user_data,omitempty" cty:"user_data"` - RootBlockDevice *terraformBlockDevice `json:"root_block_device,omitempty" cty:"root_block_device"` - EBSOptimized *bool `json:"ebs_optimized,omitempty" cty:"ebs_optimized"` - EBSBlockDevice []*terraformBlockDevice `json:"ebs_block_device,omitempty" cty:"ebs_block_device"` - EphemeralBlockDevice []*terraformBlockDevice `json:"ephemeral_block_device,omitempty" cty:"ephemeral_block_device"` - Lifecycle *terraform.Lifecycle `json:"lifecycle,omitempty" cty:"lifecycle"` - SpotPrice *string `json:"spot_price,omitempty" cty:"spot_price"` - PlacementTenancy *string `json:"placement_tenancy,omitempty" cty:"placement_tenancy"` - InstanceMonitoring *bool `json:"enable_monitoring,omitempty" cty:"enable_monitoring"` -} - -type terraformBlockDevice struct { - // For ephemeral devices - DeviceName *string `json:"device_name,omitempty" cty:"device_name"` - VirtualName *string `json:"virtual_name,omitempty" cty:"virtual_name"` - - // For root - VolumeType *string `json:"volume_type,omitempty" cty:"volume_type"` - VolumeSize *int64 `json:"volume_size,omitempty" cty:"volume_size"` - Iops *int64 `json:"iops,omitempty" cty:"iops"` - // Encryption - Encrypted *bool `json:"encrypted,omitempty" cty:"encrypted"` - // Termination - DeleteOnTermination *bool `json:"delete_on_termination,omitempty" cty:"delete_on_termination"` -} - -// RenderTerraform is responsible for rendering the terraform json -func (_ *LaunchConfiguration) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *LaunchConfiguration) error { - cloud := t.Cloud.(awsup.AWSCloud) - - if e.ImageID == nil { - return fi.RequiredField("ImageID") - } - image, err := cloud.ResolveImage(*e.ImageID) - if err != nil { - return err - } - - tf := &terraformLaunchConfiguration{ - NamePrefix: fi.String(*e.Name + "-"), - ImageID: image.ImageId, - InstanceType: e.InstanceType, - } - - if e.SpotPrice != "" { - tf.SpotPrice = aws.String(e.SpotPrice) - } - - if e.SSHKey != nil { - tf.KeyName = e.SSHKey.TerraformLink() - } - - if e.Tenancy != nil { - tf.PlacementTenancy = e.Tenancy - } - - for _, sg := range e.SecurityGroups { - tf.SecurityGroups = append(tf.SecurityGroups, sg.TerraformLink()) - } - - tf.AssociatePublicIpAddress = e.AssociatePublicIP - tf.EBSOptimized = e.RootVolumeOptimization - - { - rootDevices, err := e.buildRootDevice(cloud) - if err != nil { - return err - } - ephemeralDevices, err := buildEphemeralDevices(cloud, fi.StringValue(e.InstanceType)) - if err != nil { - return err - } - additionalDevices, err := buildAdditionalDevices(e.BlockDeviceMappings) - if err != nil { - return err - } - - if len(rootDevices) != 0 { - if len(rootDevices) != 1 { - return fmt.Errorf("unexpectedly found multiple root devices") - } - - for _, bdm := range rootDevices { - tf.RootBlockDevice = &terraformBlockDevice{ - VolumeType: bdm.EbsVolumeType, - VolumeSize: bdm.EbsVolumeSize, - Iops: bdm.EbsVolumeIops, - Encrypted: bdm.EbsEncrypted, - DeleteOnTermination: bdm.EbsDeleteOnTermination, - } - } - } - - if len(ephemeralDevices) != 0 { - tf.EphemeralBlockDevice = []*terraformBlockDevice{} - for _, deviceName := range sets.StringKeySet(ephemeralDevices).List() { - bdm := ephemeralDevices[deviceName] - tf.EphemeralBlockDevice = append(tf.EphemeralBlockDevice, &terraformBlockDevice{ - VirtualName: bdm.VirtualName, - DeviceName: fi.String(deviceName), - }) - } - } - - if len(additionalDevices) != 0 { - tf.EBSBlockDevice = []*terraformBlockDevice{} - for _, deviceName := range sets.StringKeySet(additionalDevices).List() { - bdm := additionalDevices[deviceName] - tf.EBSBlockDevice = append(tf.EBSBlockDevice, &terraformBlockDevice{ - DeleteOnTermination: bdm.EbsDeleteOnTermination, - DeviceName: fi.String(deviceName), - Encrypted: bdm.EbsEncrypted, - VolumeSize: bdm.EbsVolumeSize, - VolumeType: bdm.EbsVolumeType, - }) - } - } - } - - if e.UserData != nil { - userData, err := fi.ResourceAsString(e.UserData) - if err != nil { - return err - } - if userData != "" { - tf.UserData, err = t.AddFile("aws_launch_configuration", *e.Name, "user_data", e.UserData, false) - if err != nil { - return err - } - } - } - if e.IAMInstanceProfile != nil { - tf.IAMInstanceProfile = e.IAMInstanceProfile.TerraformLink() - } - if e.InstanceMonitoring != nil { - tf.InstanceMonitoring = e.InstanceMonitoring - } else { - tf.InstanceMonitoring = fi.Bool(false) - } - // So that we can update configurations - tf.Lifecycle = &terraform.Lifecycle{CreateBeforeDestroy: fi.Bool(true)} - - return t.RenderResource("aws_launch_configuration", fi.StringValue(e.Name), tf) -} - -// TerraformLink returns the terraform reference -func (e *LaunchConfiguration) TerraformLink() *terraform.Literal { - return terraform.LiteralProperty("aws_launch_configuration", fi.StringValue(e.Name), "id") -} - -type cloudformationLaunchConfiguration struct { - AssociatePublicIpAddress *bool `json:"AssociatePublicIpAddress,omitempty"` - BlockDeviceMappings []*cloudformationBlockDevice `json:"BlockDeviceMappings,omitempty"` - EBSOptimized *bool `json:"EbsOptimized,omitempty"` - IAMInstanceProfile *cloudformation.Literal `json:"IamInstanceProfile,omitempty"` - ImageID *string `json:"ImageId,omitempty"` - InstanceType *string `json:"InstanceType,omitempty"` - KeyName *string `json:"KeyName,omitempty"` - SecurityGroups []*cloudformation.Literal `json:"SecurityGroups,omitempty"` - SpotPrice *string `json:"SpotPrice,omitempty"` - UserData *string `json:"UserData,omitempty"` - PlacementTenancy *string `json:"PlacementTenancy,omitempty"` - InstanceMonitoring *bool `json:"InstanceMonitoring,omitempty"` -} - -type cloudformationBlockDevice struct { - // For ephemeral devices - DeviceName *string `json:"DeviceName,omitempty"` - VirtualName *string `json:"VirtualName,omitempty"` - - // For root - Ebs *cloudformationBlockDeviceEBS `json:"Ebs,omitempty"` -} - -type cloudformationBlockDeviceEBS struct { - VolumeType *string `json:"VolumeType,omitempty"` - VolumeSize *int64 `json:"VolumeSize,omitempty"` - Iops *int64 `json:"Iops,omitempty"` - DeleteOnTermination *bool `json:"DeleteOnTermination,omitempty"` - Encrypted *bool `json:"Encrypted,omitempty"` -} - -// RenderCloudformation is responsible for rendering the cloudformation template -func (_ *LaunchConfiguration) RenderCloudformation(t *cloudformation.CloudformationTarget, a, e, changes *LaunchConfiguration) error { - cloud := t.Cloud.(awsup.AWSCloud) - - if e.ImageID == nil { - return fi.RequiredField("ImageID") - } - image, err := cloud.ResolveImage(*e.ImageID) - if err != nil { - return err - } - - cf := &cloudformationLaunchConfiguration{ - //NamePrefix: fi.String(*e.Name + "-"), - ImageID: image.ImageId, - InstanceType: e.InstanceType, - } - - if e.SpotPrice != "" { - cf.SpotPrice = aws.String(e.SpotPrice) - } - - if e.SSHKey != nil && !e.SSHKey.NoSSHKey() { - if e.SSHKey.Name == nil { - return fmt.Errorf("SSHKey Name not set") - } - cf.KeyName = e.SSHKey.Name - } - - if e.Tenancy != nil { - cf.PlacementTenancy = e.Tenancy - } - - for _, sg := range e.SecurityGroups { - cf.SecurityGroups = append(cf.SecurityGroups, sg.CloudformationLink()) - } - cf.AssociatePublicIpAddress = e.AssociatePublicIP - - cf.EBSOptimized = e.RootVolumeOptimization - - { - rootDevices, err := e.buildRootDevice(cloud) - if err != nil { - return err - } - ephemeralDevices, err := buildEphemeralDevices(cloud, fi.StringValue(e.InstanceType)) - if err != nil { - return err - } - additionalDevices, err := buildAdditionalDevices(e.BlockDeviceMappings) - if err != nil { - return err - } - - if len(rootDevices) != 0 { - if len(rootDevices) != 1 { - return fmt.Errorf("unexpectedly found multiple root devices") - } - - for deviceName, bdm := range rootDevices { - d := &cloudformationBlockDevice{ - DeviceName: fi.String(deviceName), - Ebs: &cloudformationBlockDeviceEBS{ - VolumeType: bdm.EbsVolumeType, - VolumeSize: bdm.EbsVolumeSize, - Iops: bdm.EbsVolumeIops, - Encrypted: bdm.EbsEncrypted, - DeleteOnTermination: bdm.EbsDeleteOnTermination, - }, - } - cf.BlockDeviceMappings = append(cf.BlockDeviceMappings, d) - } - } - - if len(ephemeralDevices) != 0 { - for deviceName, bdm := range ephemeralDevices { - cf.BlockDeviceMappings = append(cf.BlockDeviceMappings, &cloudformationBlockDevice{ - VirtualName: bdm.VirtualName, - DeviceName: fi.String(deviceName), - }) - } - } - - if len(additionalDevices) != 0 { - for deviceName, bdm := range additionalDevices { - d := &cloudformationBlockDevice{ - DeviceName: fi.String(deviceName), - Ebs: &cloudformationBlockDeviceEBS{ - VolumeType: bdm.EbsVolumeType, - VolumeSize: bdm.EbsVolumeSize, - DeleteOnTermination: bdm.EbsDeleteOnTermination, - Encrypted: bdm.EbsEncrypted, - }, - } - cf.BlockDeviceMappings = append(cf.BlockDeviceMappings, d) - } - } - } - - if e.UserData != nil { - d, err := fi.ResourceAsBytes(e.UserData) - if err != nil { - return fmt.Errorf("error rendering AutoScalingLaunchConfiguration UserData: %v", err) - } - cf.UserData = aws.String(base64.StdEncoding.EncodeToString(d)) - } - - if e.IAMInstanceProfile != nil { - cf.IAMInstanceProfile = e.IAMInstanceProfile.CloudformationLink() - } - - if e.InstanceMonitoring != nil { - cf.InstanceMonitoring = e.InstanceMonitoring - } else { - cf.InstanceMonitoring = fi.Bool(false) - } - // So that we can update configurations - //tf.Lifecycle = &cloudformation.Lifecycle{CreateBeforeDestroy: fi.Bool(true)} - - return t.RenderResource("AWS::AutoScaling::LaunchConfiguration", *e.Name, cf) -} - -func (e *LaunchConfiguration) CloudformationLink() *cloudformation.Literal { - return cloudformation.Ref("AWS::AutoScaling::LaunchConfiguration", *e.Name) -} - -// deleteLaunchConfiguration tracks a LaunchConfiguration that we're going to delete -// It implements fi.Deletion -type deleteLaunchConfiguration struct { - lc *autoscaling.LaunchConfiguration -} - -var _ fi.Deletion = &deleteLaunchConfiguration{} - -func (d *deleteLaunchConfiguration) TaskName() string { - return "LaunchConfiguration" -} - -func (d *deleteLaunchConfiguration) Item() string { - return aws.StringValue(d.lc.LaunchConfigurationName) -} - -func (d *deleteLaunchConfiguration) Delete(t fi.Target) error { - klog.V(2).Infof("deleting launch configuration %v", d) - - awsTarget, ok := t.(*awsup.AWSAPITarget) - if !ok { - return fmt.Errorf("unexpected target type for deletion: %T", t) - } - - request := &autoscaling.DeleteLaunchConfigurationInput{ - LaunchConfigurationName: d.lc.LaunchConfigurationName, - } - - name := aws.StringValue(request.LaunchConfigurationName) - klog.V(2).Infof("Calling autoscaling DeleteLaunchConfiguration for %s", name) - _, err := awsTarget.Cloud.Autoscaling().DeleteLaunchConfiguration(request) - if err != nil { - return fmt.Errorf("error deleting autoscaling LaunchConfiguration %s: %v", name, err) - } - - return nil -} - -func (d *deleteLaunchConfiguration) String() string { - return d.TaskName() + "-" + d.Item() -} - -func (e *LaunchConfiguration) FindDeletions(c *fi.Context) ([]fi.Deletion, error) { - var removals []fi.Deletion - - configurations, err := e.findLaunchConfigurations(c) - if err != nil { - return nil, err - } - - if len(configurations) <= RetainLaunchConfigurationCount() { - return nil, nil - } - - configurations = configurations[:len(configurations)-RetainLaunchConfigurationCount()] - - for _, configuration := range configurations { - removals = append(removals, &deleteLaunchConfiguration{lc: configuration}) - } - - klog.V(2).Infof("will delete launch configurations: %v", removals) - - return removals, nil -} - -func useSSHKey(c *kops.Cluster) bool { - if c != nil { - sshKeyName := c.Spec.SSHKeyName - return sshKeyName != nil && *sshKeyName != "" - } - return true -} diff --git a/upup/pkg/fi/cloudup/awstasks/launchconfiguration_fitask.go b/upup/pkg/fi/cloudup/awstasks/launchconfiguration_fitask.go deleted file mode 100644 index 614c7a91eb..0000000000 --- a/upup/pkg/fi/cloudup/awstasks/launchconfiguration_fitask.go +++ /dev/null @@ -1,51 +0,0 @@ -// +build !ignore_autogenerated - -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by fitask. DO NOT EDIT. - -package awstasks - -import ( - "k8s.io/kops/upup/pkg/fi" -) - -// LaunchConfiguration - -var _ fi.HasLifecycle = &LaunchConfiguration{} - -// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle -func (o *LaunchConfiguration) GetLifecycle() *fi.Lifecycle { - return o.Lifecycle -} - -// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle -func (o *LaunchConfiguration) SetLifecycle(lifecycle fi.Lifecycle) { - o.Lifecycle = &lifecycle -} - -var _ fi.HasName = &LaunchConfiguration{} - -// GetName returns the Name of the object, implementing fi.HasName -func (o *LaunchConfiguration) GetName() *string { - return o.Name -} - -// String is the stringer function for the task, producing readable output using fi.TaskAsString -func (o *LaunchConfiguration) String() string { - return fi.TaskAsString(o) -} diff --git a/upup/pkg/fi/cloudup/awstasks/launchconfiguration_test.go b/upup/pkg/fi/cloudup/awstasks/launchconfiguration_test.go deleted file mode 100644 index 96c74f7d2d..0000000000 --- a/upup/pkg/fi/cloudup/awstasks/launchconfiguration_test.go +++ /dev/null @@ -1,120 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package awstasks - -import ( - "strconv" - "testing" - "time" - - "k8s.io/kops/cloudmock/aws/mockautoscaling" - "k8s.io/kops/cloudmock/aws/mockec2" - "k8s.io/kops/upup/pkg/fi" - "k8s.io/kops/upup/pkg/fi/cloudup/awsup" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" -) - -func TestLaunchConfigurationGarbageCollection(t *testing.T) { - cloud := awsup.BuildMockAWSCloud("us-east-1", "abc") - mockEC2 := &mockec2.MockEC2{} - cloud.MockEC2 = mockEC2 - as := &mockautoscaling.MockAutoscaling{} - cloud.MockAutoscaling = as - - mockEC2.Images = append(mockEC2.Images, &ec2.Image{ - CreationDate: aws.String("2016-10-21T20:07:19.000Z"), - ImageId: aws.String("ami-12345678"), - Name: aws.String("k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21"), - OwnerId: aws.String(awsup.WellKnownAccountKopeio), - RootDeviceName: aws.String("/dev/xvda"), - }) - - // We define a function so we can rebuild the tasks, because we modify in-place when running - buildTasks := func(spotPrice string) map[string]fi.Task { - lc := &LaunchConfiguration{ - Name: s("lc1"), - SpotPrice: spotPrice, - ImageID: s("ami-12345678"), - InstanceType: s("m3.medium"), - SecurityGroups: []*SecurityGroup{}, - UserData: fi.NewStringResource(""), - } - - return map[string]fi.Task{ - "lc1": lc, - } - } - - // We change the launch configuration 5 times, verifying that new launch configurations are created, - // and that older ones are eventually GCed - for i := 0; i < 5; i++ { - spotPrice := strconv.Itoa(i + 1) - { - allTasks := buildTasks(spotPrice) - lc1 := allTasks["lc1"].(*LaunchConfiguration) - - target := &awsup.AWSAPITarget{ - Cloud: cloud, - } - - context, err := fi.NewContext(target, nil, cloud, nil, nil, nil, true, allTasks) - if err != nil { - t.Fatalf("error building context: %v", err) - } - defer context.Close() - - // We use a longer deadline because we know we often need to - // retry here, because we create different versions of - // launchconfigurations using the timestamp, but only to - // per-second granularity. This normally works out because we - // retry for O(minutes), so after a few retries the clock has - // advanced. But if we use too short a deadline in our tests we - // don't get this behaviour. - options := testRunTasksOptions - options.MaxTaskDuration = 5 * time.Second - if err := context.RunTasks(options); err != nil { - t.Fatalf("unexpected error during Run: %v", err) - } - - if fi.StringValue(lc1.ID) == "" { - t.Fatalf("ID not set after create") - } - - expectedCount := i + 1 - if expectedCount > RetainLaunchConfigurationCount() { - expectedCount = RetainLaunchConfigurationCount() - } - if len(as.LaunchConfigurations) != expectedCount { - t.Fatalf("Expected exactly %d LaunchConfigurations; found %v", expectedCount, as.LaunchConfigurations) - } - - // TODO: verify that we retained the N latest - - actual := as.LaunchConfigurations[*lc1.ID] - if aws.StringValue(actual.SpotPrice) != spotPrice { - t.Fatalf("Unexpected spotPrice: expected=%v actual=%v", spotPrice, aws.StringValue(actual.SpotPrice)) - } - } - - { - allTasks := buildTasks(spotPrice) - checkNoChanges(t, cloud, allTasks) - } - } -}