lifecycle integ test, docs, & small cleanup

cloudmock/aws/mockautoscaling/api.go

@@ -31,6 +31,7 @@ type MockAutoscaling struct {
 	Groups               map[string]*autoscaling.Group
 	WarmPoolInstances    map[string][]*autoscaling.Instance
 	LaunchConfigurations map[string]*autoscaling.LaunchConfiguration
+	LifecycleHooks       map[string]*autoscaling.LifecycleHook
 }
 
 var _ autoscalingiface.AutoScalingAPI = &MockAutoscaling{}

cloudmock/aws/mockautoscaling/group.go

@@ -338,5 +338,39 @@ func (m *MockAutoscaling) DeleteAutoScalingGroupRequest(*autoscaling.DeleteAutoS
 }
 
 func (m *MockAutoscaling) PutLifecycleHook(input *autoscaling.PutLifecycleHookInput) (*autoscaling.PutLifecycleHookOutput, error) {
-	panic("Not implemented")
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	hook := &autoscaling.LifecycleHook{
+		AutoScalingGroupName:  input.AutoScalingGroupName,
+		DefaultResult:         input.DefaultResult,
+		GlobalTimeout:         input.HeartbeatTimeout,
+		HeartbeatTimeout:      input.HeartbeatTimeout,
+		LifecycleHookName:     input.LifecycleHookName,
+		LifecycleTransition:   input.LifecycleTransition,
+		NotificationMetadata:  input.NotificationMetadata,
+		NotificationTargetARN: input.NotificationTargetARN,
+		RoleARN:               input.RoleARN,
+	}
+
+	if m.LifecycleHooks == nil {
+		m.LifecycleHooks = make(map[string]*autoscaling.LifecycleHook)
+	}
+	m.LifecycleHooks[*hook.AutoScalingGroupName] = hook
+
+	return &autoscaling.PutLifecycleHookOutput{}, nil
+}
+
+func (m *MockAutoscaling) DescribeLifecycleHooks(input *autoscaling.DescribeLifecycleHooksInput) (*autoscaling.DescribeLifecycleHooksOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	name := *input.AutoScalingGroupName
+	response := &autoscaling.DescribeLifecycleHooksOutput{}
+
+	hook := m.LifecycleHooks[name]
+	if hook == nil {
+		return response, nil
+	}
+	response.LifecycleHooks = []*autoscaling.LifecycleHook{hook}
+	return response, nil
 }

cloudmock/aws/mockeventbridge/api.go

@@ -17,42 +17,99 @@ limitations under the License.
 package mockeventbridge
 
 import (
+	"sync"
+
 	"github.com/aws/aws-sdk-go/service/eventbridge"
 	"github.com/aws/aws-sdk-go/service/eventbridge/eventbridgeiface"
 )
 
 type MockEventBridge struct {
 	eventbridgeiface.EventBridgeAPI
+	mutex sync.Mutex
 
-	Rules []*eventbridge.Rule
+	Rules         map[string]*eventbridge.Rule
+	TagsByArn     map[string][]*eventbridge.Tag
+	TargetsByRule map[string][]*eventbridge.Target
 }
 
 var _ eventbridgeiface.EventBridgeAPI = &MockEventBridge{}
 
-func (c *MockEventBridge) ListTargetsByRule(*eventbridge.ListTargetsByRuleInput) (*eventbridge.ListTargetsByRuleOutput, error) {
+func (m *MockEventBridge) PutRule(input *eventbridge.PutRuleInput) (*eventbridge.PutRuleOutput, error) {
-	panic("Not implemented")
+	m.mutex.Lock()
-}
+	defer m.mutex.Unlock()
 
-func (c *MockEventBridge) RemoveTargets(*eventbridge.RemoveTargetsInput) (*eventbridge.RemoveTargetsOutput, error) {
+	name := *input.Name
-	panic("Not implemented")
+	arn := "arn:aws:events:us-east-1:012345678901:rule/" + name
-}
 
-func (c *MockEventBridge) DeleteRule(*eventbridge.DeleteRuleInput) (*eventbridge.DeleteRuleOutput, error) {
+	rule := &eventbridge.Rule{
-	panic("Not implemented")
+		Arn:          &arn,
-}
+		EventPattern: input.EventPattern,
-
-func (c *MockEventBridge) ListRules(*eventbridge.ListRulesInput) (*eventbridge.ListRulesOutput, error) {
-	response := &eventbridge.ListRulesOutput{
-		Rules: c.Rules,
 	}
+	if m.Rules == nil {
+		m.Rules = make(map[string]*eventbridge.Rule)
+	}
+	if m.TagsByArn == nil {
+		m.TagsByArn = make(map[string][]*eventbridge.Tag)
+	}
+	m.Rules[name] = rule
+	m.TagsByArn[arn] = input.Tags
 
+	response := &eventbridge.PutRuleOutput{
+		RuleArn: &arn,
+	}
 	return response, nil
 }
 
-func (c *MockEventBridge) PutRule(*eventbridge.PutRuleInput) (*eventbridge.PutRuleOutput, error) {
+func (m *MockEventBridge) ListRules(input *eventbridge.ListRulesInput) (*eventbridge.ListRulesOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	response := &eventbridge.ListRulesOutput{}
+
+	rule := m.Rules[*input.NamePrefix]
+	if rule == nil {
+		return response, nil
+	}
+	response.Rules = []*eventbridge.Rule{rule}
+	return response, nil
+}
+
+func (m *MockEventBridge) DeleteRule(*eventbridge.DeleteRuleInput) (*eventbridge.DeleteRuleOutput, error) {
 	panic("Not implemented")
 }
 
-func (c *MockEventBridge) PutTargets(*eventbridge.PutTargetsInput) (*eventbridge.PutTargetsOutput, error) {
+func (m *MockEventBridge) ListTagsForResource(input *eventbridge.ListTagsForResourceInput) (*eventbridge.ListTagsForResourceOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	response := &eventbridge.ListTagsForResourceOutput{
+		Tags: m.TagsByArn[*input.ResourceARN],
+	}
+	return response, nil
+}
+
+func (m *MockEventBridge) PutTargets(input *eventbridge.PutTargetsInput) (*eventbridge.PutTargetsOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	if m.TargetsByRule == nil {
+		m.TargetsByRule = make(map[string][]*eventbridge.Target)
+	}
+	m.TargetsByRule[*input.Rule] = input.Targets
+
+	return &eventbridge.PutTargetsOutput{}, nil
+}
+
+func (m *MockEventBridge) ListTargetsByRule(input *eventbridge.ListTargetsByRuleInput) (*eventbridge.ListTargetsByRuleOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	response := &eventbridge.ListTargetsByRuleOutput{
+		Targets: m.TargetsByRule[*input.Rule],
+	}
+	return response, nil
+}
+
+func (m *MockEventBridge) RemoveTargets(*eventbridge.RemoveTargetsInput) (*eventbridge.RemoveTargetsOutput, error) {
 	panic("Not implemented")
 }

cloudmock/aws/mocksqs/api.go

@@ -17,34 +17,93 @@ limitations under the License.
 package mocksqs
 
 import (
+	"sync"
+
 	"github.com/aws/aws-sdk-go/service/sqs"
 	"github.com/aws/aws-sdk-go/service/sqs/sqsiface"
 )
 
 type MockSQS struct {
 	sqsiface.SQSAPI
+	mutex sync.Mutex
 
-	QueueUrls []*string
+	Queues map[string]mockQueue
+}
+
+type mockQueue struct {
+	url        *string
+	attributes map[string]*string
+	tags       map[string]*string
 }
 
 var _ sqsiface.SQSAPI = &MockSQS{}
 
-func (c *MockSQS) DeleteQueue(*sqs.DeleteQueueInput) (*sqs.DeleteQueueOutput, error) {
+func (m *MockSQS) CreateQueue(input *sqs.CreateQueueInput) (*sqs.CreateQueueOutput, error) {
-	panic("Not implemented")
+	m.mutex.Lock()
-}
+	defer m.mutex.Unlock()
 
-func (c *MockSQS) ListQueues(*sqs.ListQueuesInput) (*sqs.ListQueuesOutput, error) {
+	name := *input.QueueName
-	response := &sqs.ListQueuesOutput{
+	url := "https://sqs.us-east-1.amazonaws.com/123456789123/" + name
-		QueueUrls: c.QueueUrls,
+
+	if m.Queues == nil {
+		m.Queues = make(map[string]mockQueue)
+	}
+	queue := mockQueue{
+		url:        &url,
+		attributes: input.Attributes,
+		tags:       input.Tags,
 	}
 
+	m.Queues[name] = queue
+
+	response := &sqs.CreateQueueOutput{
+		QueueUrl: &url,
+	}
 	return response, nil
 }
 
-func (c *MockSQS) ListQueueTags(*sqs.ListQueueTagsInput) (*sqs.ListQueueTagsOutput, error) {
+func (m *MockSQS) ListQueues(input *sqs.ListQueuesInput) (*sqs.ListQueuesOutput, error) {
-	panic("Not implemented")
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	response := &sqs.ListQueuesOutput{}
+
+	if queue, ok := m.Queues[*input.QueueNamePrefix]; ok {
+		response.QueueUrls = []*string{queue.url}
+	}
+	return response, nil
 }
 
-func (c *MockSQS) CreateQueue(*sqs.CreateQueueInput) (*sqs.CreateQueueOutput, error) {
+func (m *MockSQS) GetQueueAttributes(input *sqs.GetQueueAttributesInput) (*sqs.GetQueueAttributesOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	response := &sqs.GetQueueAttributesOutput{}
+
+	for _, v := range m.Queues {
+		if *v.url == *input.QueueUrl {
+			response.Attributes = v.attributes
+			return response, nil
+		}
+	}
+	return response, nil
+}
+
+func (m *MockSQS) ListQueueTags(input *sqs.ListQueueTagsInput) (*sqs.ListQueueTagsOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	response := &sqs.ListQueueTagsOutput{}
+
+	for _, v := range m.Queues {
+		if *v.url == *input.QueueUrl {
+			response.Tags = v.tags
+			return response, nil
+		}
+	}
+	return response, nil
+}
+
+func (m *MockSQS) DeleteQueue(*sqs.DeleteQueueInput) (*sqs.DeleteQueueOutput, error) {
 	panic("Not implemented")
 }

cmd/kops/BUILD.bazel

@@ -170,7 +170,6 @@ go_test(
         "//pkg/featureflag:go_default_library",
         "//pkg/jsonutils:go_default_library",
         "//pkg/kopscodecs:go_default_library",
-        "//pkg/model:go_default_library",
         "//pkg/testutils:go_default_library",
         "//pkg/testutils/golden:go_default_library",
         "//upup/pkg/fi:go_default_library",

cmd/kops/integration_test.go

@@ -36,8 +36,6 @@ import (
 	"testing"
 	"time"
 
-	"k8s.io/kops/pkg/model"
-
 	"k8s.io/kops/cmd/kops/util"
 	"k8s.io/kops/pkg/featureflag"
 	"k8s.io/kops/pkg/jsonutils"
@@ -396,9 +394,10 @@ func TestAPIServerNodes(t *testing.T) {
 	newIntegrationTest("minimal.example.com", "apiservernodes").runTestCloudformation(t)
 }
 
+// TestNTHQueueProcessor tests the output for resources required by NTH Queue Processor mode
 func TestNTHQueueProcessor(t *testing.T) {
-	newIntegrationTest("queueprocessor.example.com", "nodeterminationhandler_sqs_resources").withNTH().runTestTerraformAWS(t)
+	newIntegrationTest("nthsqsresources.example.com", "nth_sqs_resources").withNTH().runTestTerraformAWS(t)
-	newIntegrationTest("queueprocessor.example.com", "nodeterminationhandler_sqs_resources").runTestCloudformation(t)
+	newIntegrationTest("nthsqsresources.example.com", "nth_sqs_resources").runTestCloudformation(t)
 }
 
 func (i *integrationTest) runTest(t *testing.T, h *testutils.IntegrationTestHarness, expectedDataFilenames []string, tfFileName string, expectedTfFileName string, phase *cloudup.Phase) {
@@ -598,7 +597,7 @@ func (i *integrationTest) runTestTerraformAWS(t *testing.T) {
 				"aws_cloudwatch_event_rule_" + i.clusterName + "-ASGLifecycle_event_pattern",
 				"aws_cloudwatch_event_rule_" + i.clusterName + "-RebalanceRecommendation_event_pattern",
 				"aws_cloudwatch_event_rule_" + i.clusterName + "-SpotInterruption_event_pattern",
-				"aws_sqs_queue_" + model.QueueNamePrefix(i.clusterName) + "-nth_policy",
+				"aws_sqs_queue_" + strings.Replace(i.clusterName, ".", "-", -1) + "-nth_policy",
 			}...)
 		}
 	}

cmd/kops/lifecycle_integration_test.go

@@ -153,6 +153,14 @@ func TestLifecyclePrivateSharedIP(t *testing.T) {
 	})
 }
 
+// TestLifecycleNodeTerminationHandlerQueueProcessor runs the test on a cluster with requisite resources for NTH Queue Processor
+func TestLifecycleNodeTerminationHandlerQueueProcessor(t *testing.T) {
+	runLifecycleTestAWS(&LifecycleTestOptions{
+		t:      t,
+		SrcDir: "nth_sqs_resources",
+	})
+}
+
 func runLifecycleTest(h *testutils.IntegrationTestHarness, o *LifecycleTestOptions, cloud *awsup.MockAWSCloud) {
 	ctx := context.Background()
 

docs/addons.md

@@ -127,12 +127,15 @@ spec:
 
 {{ kops_feature_table(kops_added_default='1.19') }}
 
-Node Termination Handler ensures that the Kubernetes control plane responds appropriately to events that can cause your EC2 instance to become unavailable, such as EC2 maintenance events, EC2 Spot interruptions, ASG Scale-In, ASG AZ Rebalance, and EC2 Instance Termination via the API or Console. If not handled, your application code may not stop gracefully, take longer to recover full availability, or accidentally schedule work to nodes that are going down.
+[Node Termination Handler](https://github.com/aws/aws-node-termination-handler) ensures that the Kubernetes control plane responds appropriately to events that can cause your EC2 instance to become unavailable, such as EC2 maintenance events, EC2 Spot interruptions, and EC2 instance rebalance recommendations. If not handled, your application code may not stop gracefully, take longer to recover full availability, or accidentally schedule work to nodes that are going down.
+
+If `enableSqsTerminationDraining` is enabled Node Termination Handler will operate in Queue Processor mode. In addition to the events mentioned above, Queue Processor mode allows Node Termination Handler to take care of ASG Scale-In, AZ-Rebalance, Unhealthy Instances, EC2 Instance Termination via the API or Console, and more. kOps will provision the necessary infrastructure: an SQS queue, EventBridge rules, and ASG Lifecycle hooks.
 
 ```yaml
 spec:
   nodeTerminationHandler:
     enabled: true
+    enableSqsTerminationDraining: true
 ```
 
 ## Static addons

pkg/model/awsmodel/nodeterminationhandler.go

@@ -111,7 +111,6 @@ func (b *NodeTerminationHandlerBuilder) configureASG(c *fi.ModelBuilderContext,
 		DefaultResult:       aws.String("CONTINUE"),
 		HeartbeatTimeout:    aws.Int64(DefaultMessageRetentionPeriod),
 		LifecycleTransition: aws.String("autoscaling:EC2_INSTANCE_TERMINATING"),
-		Tags:                tags,
 	}
 
 	c.AddTask(lifecyleTask)

pkg/model/names.go

@@ -268,5 +268,5 @@ func (b *KopsModelContext) InstanceName(ig *kops.InstanceGroup, suffix string) s
 
 func QueueNamePrefix(clusterName string) string {
 	// periods aren't allowed in queue name
-	return strings.Replace(clusterName, ".", "-", -1)
+	return strings.ReplaceAll(clusterName, ".", "-")
 }

pkg/resources/aws/BUILD.bazel

@@ -21,7 +21,6 @@ go_library(
     deps = [
         "//pkg/dns:go_default_library",
         "//pkg/featureflag:go_default_library",
-        "//pkg/model:go_default_library",
         "//pkg/resources:go_default_library",
         "//pkg/resources/spotinst:go_default_library",
         "//upup/pkg/fi:go_default_library",

pkg/resources/aws/eventbridge.go

@@ -46,7 +46,7 @@ func DeleteEventBridgeRule(cloud fi.Cloud, r *resources.Resource) error {
 		Rule: aws.String(r.Name),
 	})
 	if err != nil {
-		return fmt.Errorf("error listing targets for EventBridge Rule %q: %v", r.Name, err)
+		return fmt.Errorf("error listing targets for EventBridge rule %q: %v", r.Name, err)
 	}
 
 	var ids []*string
@@ -54,22 +54,22 @@ func DeleteEventBridgeRule(cloud fi.Cloud, r *resources.Resource) error {
 		ids = append(ids, target.Id)
 	}
 
-	klog.V(2).Infof("Removing EventBridge Targets for Rule %q", r.Name)
+	klog.V(2).Infof("Removing EventBridge Targets for rule %q", r.Name)
 	_, err = c.EventBridge().RemoveTargets(&eventbridge.RemoveTargetsInput{
 		Ids:  ids,
 		Rule: aws.String(r.Name),
 	})
 	if err != nil {
-		return fmt.Errorf("error removing targets for EventBridge Rule %q: %v", r.Name, err)
+		return fmt.Errorf("error removing targets for EventBridge rule %q: %v", r.Name, err)
 	}
 
-	klog.V(2).Infof("Deleting EventBridge Rule %q", r.Name)
+	klog.V(2).Infof("Deleting EventBridge rule %q", r.Name)
 	request := &eventbridge.DeleteRuleInput{
 		Name: aws.String(r.Name),
 	}
 	_, err = c.EventBridge().DeleteRule(request)
 	if err != nil {
-		return fmt.Errorf("error deleting EventBridge Rule %q: %v", r.Name, err)
+		return fmt.Errorf("error deleting EventBridge rule %q: %v", r.Name, err)
 	}
 	return nil
 }
@@ -77,9 +77,9 @@ func DeleteEventBridgeRule(cloud fi.Cloud, r *resources.Resource) error {
 func ListEventBridgeRules(cloud fi.Cloud, clusterName string) ([]*resources.Resource, error) {
 	c := cloud.(awsup.AWSCloud)
 
-	klog.V(2).Infof("Listing EventBridge Rules")
+	klog.V(2).Infof("Listing EventBridge rules")
 
-	// Rule names start with the cluster name so that we can search for them
+	// rule names start with the cluster name so that we can search for them
 	request := &eventbridge.ListRulesInput{
 		EventBusName: nil,
 		Limit:        nil,
@@ -87,7 +87,7 @@ func ListEventBridgeRules(cloud fi.Cloud, clusterName string) ([]*resources.Reso
 	}
 	response, err := c.EventBridge().ListRules(request)
 	if err != nil {
-		return nil, fmt.Errorf("error listing SQS queues: %v", err)
+		return nil, fmt.Errorf("error listing Eventbridge rules: %v", err)
 	}
 	if response == nil || len(response.Rules) == 0 {
 		return nil, nil

pkg/resources/aws/sqs.go

@@ -18,11 +18,10 @@ package aws
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/aws/aws-sdk-go/service/sqs"
 	"k8s.io/klog/v2"
-	"k8s.io/kops/pkg/model"
-
 	"k8s.io/kops/pkg/resources"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
@@ -59,7 +58,7 @@ func ListSQSQueues(cloud fi.Cloud, clusterName string) ([]*resources.Resource, e
 	c := cloud.(awsup.AWSCloud)
 
 	klog.V(2).Infof("Listing SQS queues")
-	queuePrefix := model.QueueNamePrefix(clusterName)
+	queuePrefix := strings.ReplaceAll(clusterName, ".", "-")
 
 	request := &sqs.ListQueuesInput{
 		QueueNamePrefix: &queuePrefix,

tests/integration/update_cluster/nodeterminationhandler_sqs_resources/kubernetes.tf

@@ -1,732 +0,0 @@
-locals {
-  cluster_name                 = "queueprocessor.example.com"
-  master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-queueprocessor-example-com.id]
-  master_security_group_ids    = [aws_security_group.masters-queueprocessor-example-com.id]
-  masters_role_arn             = aws_iam_role.masters-queueprocessor-example-com.arn
-  masters_role_name            = aws_iam_role.masters-queueprocessor-example-com.name
-  node_autoscaling_group_ids   = [aws_autoscaling_group.nodes-queueprocessor-example-com.id]
-  node_security_group_ids      = [aws_security_group.nodes-queueprocessor-example-com.id]
-  node_subnet_ids              = [aws_subnet.us-test-1a-queueprocessor-example-com.id]
-  nodes_role_arn               = aws_iam_role.nodes-queueprocessor-example-com.arn
-  nodes_role_name              = aws_iam_role.nodes-queueprocessor-example-com.name
-  region                       = "us-test-1"
-  route_table_public_id        = aws_route_table.queueprocessor-example-com.id
-  subnet_us-test-1a_id         = aws_subnet.us-test-1a-queueprocessor-example-com.id
-  vpc_cidr_block               = aws_vpc.queueprocessor-example-com.cidr_block
-  vpc_id                       = aws_vpc.queueprocessor-example-com.id
-}
-
-output "cluster_name" {
-  value = "queueprocessor.example.com"
-}
-
-output "master_autoscaling_group_ids" {
-  value = [aws_autoscaling_group.master-us-test-1a-masters-queueprocessor-example-com.id]
-}
-
-output "master_security_group_ids" {
-  value = [aws_security_group.masters-queueprocessor-example-com.id]
-}
-
-output "masters_role_arn" {
-  value = aws_iam_role.masters-queueprocessor-example-com.arn
-}
-
-output "masters_role_name" {
-  value = aws_iam_role.masters-queueprocessor-example-com.name
-}
-
-output "node_autoscaling_group_ids" {
-  value = [aws_autoscaling_group.nodes-queueprocessor-example-com.id]
-}
-
-output "node_security_group_ids" {
-  value = [aws_security_group.nodes-queueprocessor-example-com.id]
-}
-
-output "node_subnet_ids" {
-  value = [aws_subnet.us-test-1a-queueprocessor-example-com.id]
-}
-
-output "nodes_role_arn" {
-  value = aws_iam_role.nodes-queueprocessor-example-com.arn
-}
-
-output "nodes_role_name" {
-  value = aws_iam_role.nodes-queueprocessor-example-com.name
-}
-
-output "region" {
-  value = "us-test-1"
-}
-
-output "route_table_public_id" {
-  value = aws_route_table.queueprocessor-example-com.id
-}
-
-output "subnet_us-test-1a_id" {
-  value = aws_subnet.us-test-1a-queueprocessor-example-com.id
-}
-
-output "vpc_cidr_block" {
-  value = aws_vpc.queueprocessor-example-com.cidr_block
-}
-
-output "vpc_id" {
-  value = aws_vpc.queueprocessor-example-com.id
-}
-
-provider "aws" {
-  region = "us-test-1"
-}
-
-resource "aws_autoscaling_group" "master-us-test-1a-masters-queueprocessor-example-com" {
-  enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
-  launch_template {
-    id      = aws_launch_template.master-us-test-1a-masters-queueprocessor-example-com.id
-    version = aws_launch_template.master-us-test-1a-masters-queueprocessor-example-com.latest_version
-  }
-  max_size            = 1
-  metrics_granularity = "1Minute"
-  min_size            = 1
-  name                = "master-us-test-1a.masters.queueprocessor.example.com"
-  tag {
-    key                 = "KubernetesCluster"
-    propagate_at_launch = true
-    value               = "queueprocessor.example.com"
-  }
-  tag {
-    key                 = "Name"
-    propagate_at_launch = true
-    value               = "master-us-test-1a.masters.queueprocessor.example.com"
-  }
-  tag {
-    key                 = "aws-node-termination-handler/managed"
-    propagate_at_launch = true
-    value               = "true"
-  }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"
-    propagate_at_launch = true
-    value               = ""
-  }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"
-    propagate_at_launch = true
-    value               = "master"
-  }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"
-    propagate_at_launch = true
-    value               = ""
-  }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"
-    propagate_at_launch = true
-    value               = ""
-  }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers"
-    propagate_at_launch = true
-    value               = ""
-  }
-  tag {
-    key                 = "k8s.io/role/master"
-    propagate_at_launch = true
-    value               = "1"
-  }
-  tag {
-    key                 = "kops.k8s.io/instancegroup"
-    propagate_at_launch = true
-    value               = "master-us-test-1a"
-  }
-  tag {
-    key                 = "kubernetes.io/cluster/queueprocessor.example.com"
-    propagate_at_launch = true
-    value               = "owned"
-  }
-  vpc_zone_identifier = [aws_subnet.us-test-1a-queueprocessor-example-com.id]
-}
-
-resource "aws_autoscaling_group" "nodes-queueprocessor-example-com" {
-  enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
-  launch_template {
-    id      = aws_launch_template.nodes-queueprocessor-example-com.id
-    version = aws_launch_template.nodes-queueprocessor-example-com.latest_version
-  }
-  max_size            = 2
-  metrics_granularity = "1Minute"
-  min_size            = 2
-  name                = "nodes.queueprocessor.example.com"
-  tag {
-    key                 = "KubernetesCluster"
-    propagate_at_launch = true
-    value               = "queueprocessor.example.com"
-  }
-  tag {
-    key                 = "Name"
-    propagate_at_launch = true
-    value               = "nodes.queueprocessor.example.com"
-  }
-  tag {
-    key                 = "aws-node-termination-handler/managed"
-    propagate_at_launch = true
-    value               = "true"
-  }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"
-    propagate_at_launch = true
-    value               = "node"
-  }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
-    propagate_at_launch = true
-    value               = ""
-  }
-  tag {
-    key                 = "k8s.io/role/node"
-    propagate_at_launch = true
-    value               = "1"
-  }
-  tag {
-    key                 = "kops.k8s.io/instancegroup"
-    propagate_at_launch = true
-    value               = "nodes"
-  }
-  tag {
-    key                 = "kubernetes.io/cluster/queueprocessor.example.com"
-    propagate_at_launch = true
-    value               = "owned"
-  }
-  vpc_zone_identifier = [aws_subnet.us-test-1a-queueprocessor-example-com.id]
-}
-
-resource "aws_autoscaling_lifecycle_hook" "master-us-test-1a-NTHLifecycleHook" {
-  autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-queueprocessor-example-com.id
-  default_result         = "CONTINUE"
-  heartbeat_timeout      = 300
-  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
-  name                   = "master-us-test-1a-NTHLifecycleHook"
-}
-
-resource "aws_autoscaling_lifecycle_hook" "nodes-NTHLifecycleHook" {
-  autoscaling_group_name = aws_autoscaling_group.nodes-queueprocessor-example-com.id
-  default_result         = "CONTINUE"
-  heartbeat_timeout      = 300
-  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
-  name                   = "nodes-NTHLifecycleHook"
-}
-
-resource "aws_cloudwatch_event_rule" "queueprocessor-example-com-ASGLifecycle" {
-  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_queueprocessor.example.com-ASGLifecycle_event_pattern")
-  name          = "queueprocessor.example.com-ASGLifecycle"
-  tags = {
-    "KubernetesCluster"                                = "queueprocessor.example.com"
-    "Name"                                             = "queueprocessor.example.com-ASGLifecycle"
-    "kubernetes.io/cluster/queueprocessor.example.com" = "owned"
-  }
-}
-
-resource "aws_cloudwatch_event_rule" "queueprocessor-example-com-RebalanceRecommendation" {
-  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_queueprocessor.example.com-RebalanceRecommendation_event_pattern")
-  name          = "queueprocessor.example.com-RebalanceRecommendation"
-  tags = {
-    "KubernetesCluster"                                = "queueprocessor.example.com"
-    "Name"                                             = "queueprocessor.example.com-RebalanceRecommendation"
-    "kubernetes.io/cluster/queueprocessor.example.com" = "owned"
-  }
-}
-
-resource "aws_cloudwatch_event_rule" "queueprocessor-example-com-SpotInterruption" {
-  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_queueprocessor.example.com-SpotInterruption_event_pattern")
-  name          = "queueprocessor.example.com-SpotInterruption"
-  tags = {
-    "KubernetesCluster"                                = "queueprocessor.example.com"
-    "Name"                                             = "queueprocessor.example.com-SpotInterruption"
-    "kubernetes.io/cluster/queueprocessor.example.com" = "owned"
-  }
-}
-
-resource "aws_cloudwatch_event_target" "queueprocessor-example-com-ASGLifecycle-Target" {
-  arn  = "arn:aws:sqs:us-test-1:123456789012:queueprocessor-example-com-nth"
-  rule = aws_cloudwatch_event_rule.queueprocessor-example-com-ASGLifecycle.id
-}
-
-resource "aws_cloudwatch_event_target" "queueprocessor-example-com-RebalanceRecommendation-Target" {
-  arn  = "arn:aws:sqs:us-test-1:123456789012:queueprocessor-example-com-nth"
-  rule = aws_cloudwatch_event_rule.queueprocessor-example-com-RebalanceRecommendation.id
-}
-
-resource "aws_cloudwatch_event_target" "queueprocessor-example-com-SpotInterruption-Target" {
-  arn  = "arn:aws:sqs:us-test-1:123456789012:queueprocessor-example-com-nth"
-  rule = aws_cloudwatch_event_rule.queueprocessor-example-com-SpotInterruption.id
-}
-
-resource "aws_ebs_volume" "us-test-1a-etcd-events-queueprocessor-example-com" {
-  availability_zone = "us-test-1a"
-  encrypted         = false
-  iops              = 3000
-  size              = 20
-  tags = {
-    "KubernetesCluster"                                = "queueprocessor.example.com"
-    "Name"                                             = "us-test-1a.etcd-events.queueprocessor.example.com"
-    "k8s.io/etcd/events"                               = "us-test-1a/us-test-1a"
-    "k8s.io/role/master"                               = "1"
-    "kubernetes.io/cluster/queueprocessor.example.com" = "owned"
-  }
-  throughput = 125
-  type       = "gp3"
-}
-
-resource "aws_ebs_volume" "us-test-1a-etcd-main-queueprocessor-example-com" {
-  availability_zone = "us-test-1a"
-  encrypted         = false
-  iops              = 3000
-  size              = 20
-  tags = {
-    "KubernetesCluster"                                = "queueprocessor.example.com"
-    "Name"                                             = "us-test-1a.etcd-main.queueprocessor.example.com"
-    "k8s.io/etcd/main"                                 = "us-test-1a/us-test-1a"
-    "k8s.io/role/master"                               = "1"
-    "kubernetes.io/cluster/queueprocessor.example.com" = "owned"
-  }
-  throughput = 125
-  type       = "gp3"
-}
-
-resource "aws_iam_instance_profile" "masters-queueprocessor-example-com" {
-  name = "masters.queueprocessor.example.com"
-  role = aws_iam_role.masters-queueprocessor-example-com.name
-  tags = {
-    "KubernetesCluster"                                = "queueprocessor.example.com"
-    "Name"                                             = "masters.queueprocessor.example.com"
-    "kubernetes.io/cluster/queueprocessor.example.com" = "owned"
-  }
-}
-
-resource "aws_iam_instance_profile" "nodes-queueprocessor-example-com" {
-  name = "nodes.queueprocessor.example.com"
-  role = aws_iam_role.nodes-queueprocessor-example-com.name
-  tags = {
-    "KubernetesCluster"                                = "queueprocessor.example.com"
-    "Name"                                             = "nodes.queueprocessor.example.com"
-    "kubernetes.io/cluster/queueprocessor.example.com" = "owned"
-  }
-}
-
-resource "aws_iam_role_policy" "masters-queueprocessor-example-com" {
-  name   = "masters.queueprocessor.example.com"
-  policy = file("${path.module}/data/aws_iam_role_policy_masters.queueprocessor.example.com_policy")
-  role   = aws_iam_role.masters-queueprocessor-example-com.name
-}
-
-resource "aws_iam_role_policy" "nodes-queueprocessor-example-com" {
-  name   = "nodes.queueprocessor.example.com"
-  policy = file("${path.module}/data/aws_iam_role_policy_nodes.queueprocessor.example.com_policy")
-  role   = aws_iam_role.nodes-queueprocessor-example-com.name
-}
-
-resource "aws_iam_role" "masters-queueprocessor-example-com" {
-  assume_role_policy = file("${path.module}/data/aws_iam_role_masters.queueprocessor.example.com_policy")
-  name               = "masters.queueprocessor.example.com"
-  tags = {
-    "KubernetesCluster"                                = "queueprocessor.example.com"
-    "Name"                                             = "masters.queueprocessor.example.com"
-    "kubernetes.io/cluster/queueprocessor.example.com" = "owned"
-  }
-}
-
-resource "aws_iam_role" "nodes-queueprocessor-example-com" {
-  assume_role_policy = file("${path.module}/data/aws_iam_role_nodes.queueprocessor.example.com_policy")
-  name               = "nodes.queueprocessor.example.com"
-  tags = {
-    "KubernetesCluster"                                = "queueprocessor.example.com"
-    "Name"                                             = "nodes.queueprocessor.example.com"
-    "kubernetes.io/cluster/queueprocessor.example.com" = "owned"
-  }
-}
-
-resource "aws_internet_gateway" "queueprocessor-example-com" {
-  tags = {
-    "KubernetesCluster"                                = "queueprocessor.example.com"
-    "Name"                                             = "queueprocessor.example.com"
-    "kubernetes.io/cluster/queueprocessor.example.com" = "owned"
-  }
-  vpc_id = aws_vpc.queueprocessor-example-com.id
-}
-
-resource "aws_key_pair" "kubernetes-queueprocessor-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" {
-  key_name   = "kubernetes.queueprocessor.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57"
-  public_key = file("${path.module}/data/aws_key_pair_kubernetes.queueprocessor.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key")
-  tags = {
-    "KubernetesCluster"                                = "queueprocessor.example.com"
-    "Name"                                             = "queueprocessor.example.com"
-    "kubernetes.io/cluster/queueprocessor.example.com" = "owned"
-  }
-}
-
-resource "aws_launch_template" "master-us-test-1a-masters-queueprocessor-example-com" {
-  block_device_mappings {
-    device_name = "/dev/xvda"
-    ebs {
-      delete_on_termination = true
-      encrypted             = true
-      iops                  = 3000
-      throughput            = 125
-      volume_size           = 64
-      volume_type           = "gp3"
-    }
-  }
-  block_device_mappings {
-    device_name  = "/dev/sdc"
-    virtual_name = "ephemeral0"
-  }
-  iam_instance_profile {
-    name = aws_iam_instance_profile.masters-queueprocessor-example-com.id
-  }
-  image_id      = "ami-12345678"
-  instance_type = "m3.medium"
-  key_name      = aws_key_pair.kubernetes-queueprocessor-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id
-  lifecycle {
-    create_before_destroy = true
-  }
-  metadata_options {
-    http_endpoint               = "enabled"
-    http_put_response_hop_limit = 1
-    http_tokens                 = "optional"
-  }
-  name = "master-us-test-1a.masters.queueprocessor.example.com"
-  network_interfaces {
-    associate_public_ip_address = true
-    delete_on_termination       = true
-    security_groups             = [aws_security_group.masters-queueprocessor-example-com.id]
-  }
-  tag_specifications {
-    resource_type = "instance"
-    tags = {
-      "KubernetesCluster"                                                                                     = "queueprocessor.example.com"
-      "Name"                                                                                                  = "master-us-test-1a.masters.queueprocessor.example.com"
-      "aws-node-termination-handler/managed"                                                                  = "true"
-      "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
-      "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
-      "k8s.io/role/master"                                                                                    = "1"
-      "kops.k8s.io/instancegroup"                                                                             = "master-us-test-1a"
-      "kubernetes.io/cluster/queueprocessor.example.com"                                                      = "owned"
-    }
-  }
-  tag_specifications {
-    resource_type = "volume"
-    tags = {
-      "KubernetesCluster"                                                                                     = "queueprocessor.example.com"
-      "Name"                                                                                                  = "master-us-test-1a.masters.queueprocessor.example.com"
-      "aws-node-termination-handler/managed"                                                                  = "true"
-      "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
-      "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
-      "k8s.io/role/master"                                                                                    = "1"
-      "kops.k8s.io/instancegroup"                                                                             = "master-us-test-1a"
-      "kubernetes.io/cluster/queueprocessor.example.com"                                                      = "owned"
-    }
-  }
-  tags = {
-    "KubernetesCluster"                                                                                     = "queueprocessor.example.com"
-    "Name"                                                                                                  = "master-us-test-1a.masters.queueprocessor.example.com"
-    "aws-node-termination-handler/managed"                                                                  = "true"
-    "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
-    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
-    "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
-    "k8s.io/role/master"                                                                                    = "1"
-    "kops.k8s.io/instancegroup"                                                                             = "master-us-test-1a"
-    "kubernetes.io/cluster/queueprocessor.example.com"                                                      = "owned"
-  }
-  user_data = filebase64("${path.module}/data/aws_launch_template_master-us-test-1a.masters.queueprocessor.example.com_user_data")
-}
-
-resource "aws_launch_template" "nodes-queueprocessor-example-com" {
-  block_device_mappings {
-    device_name = "/dev/xvda"
-    ebs {
-      delete_on_termination = true
-      encrypted             = true
-      iops                  = 3000
-      throughput            = 125
-      volume_size           = 128
-      volume_type           = "gp3"
-    }
-  }
-  iam_instance_profile {
-    name = aws_iam_instance_profile.nodes-queueprocessor-example-com.id
-  }
-  image_id      = "ami-12345678"
-  instance_type = "t2.medium"
-  key_name      = aws_key_pair.kubernetes-queueprocessor-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id
-  lifecycle {
-    create_before_destroy = true
-  }
-  metadata_options {
-    http_endpoint               = "enabled"
-    http_put_response_hop_limit = 1
-    http_tokens                 = "optional"
-  }
-  name = "nodes.queueprocessor.example.com"
-  network_interfaces {
-    associate_public_ip_address = true
-    delete_on_termination       = true
-    security_groups             = [aws_security_group.nodes-queueprocessor-example-com.id]
-  }
-  tag_specifications {
-    resource_type = "instance"
-    tags = {
-      "KubernetesCluster"                                                          = "queueprocessor.example.com"
-      "Name"                                                                       = "nodes.queueprocessor.example.com"
-      "aws-node-termination-handler/managed"                                       = "true"
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
-      "k8s.io/role/node"                                                           = "1"
-      "kops.k8s.io/instancegroup"                                                  = "nodes"
-      "kubernetes.io/cluster/queueprocessor.example.com"                           = "owned"
-    }
-  }
-  tag_specifications {
-    resource_type = "volume"
-    tags = {
-      "KubernetesCluster"                                                          = "queueprocessor.example.com"
-      "Name"                                                                       = "nodes.queueprocessor.example.com"
-      "aws-node-termination-handler/managed"                                       = "true"
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
-      "k8s.io/role/node"                                                           = "1"
-      "kops.k8s.io/instancegroup"                                                  = "nodes"
-      "kubernetes.io/cluster/queueprocessor.example.com"                           = "owned"
-    }
-  }
-  tags = {
-    "KubernetesCluster"                                                          = "queueprocessor.example.com"
-    "Name"                                                                       = "nodes.queueprocessor.example.com"
-    "aws-node-termination-handler/managed"                                       = "true"
-    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
-    "k8s.io/role/node"                                                           = "1"
-    "kops.k8s.io/instancegroup"                                                  = "nodes"
-    "kubernetes.io/cluster/queueprocessor.example.com"                           = "owned"
-  }
-  user_data = filebase64("${path.module}/data/aws_launch_template_nodes.queueprocessor.example.com_user_data")
-}
-
-resource "aws_route_table_association" "us-test-1a-queueprocessor-example-com" {
-  route_table_id = aws_route_table.queueprocessor-example-com.id
-  subnet_id      = aws_subnet.us-test-1a-queueprocessor-example-com.id
-}
-
-resource "aws_route_table" "queueprocessor-example-com" {
-  tags = {
-    "KubernetesCluster"                                = "queueprocessor.example.com"
-    "Name"                                             = "queueprocessor.example.com"
-    "kubernetes.io/cluster/queueprocessor.example.com" = "owned"
-    "kubernetes.io/kops/role"                          = "public"
-  }
-  vpc_id = aws_vpc.queueprocessor-example-com.id
-}
-
-resource "aws_route" "route-0-0-0-0--0" {
-  destination_cidr_block = "0.0.0.0/0"
-  gateway_id             = aws_internet_gateway.queueprocessor-example-com.id
-  route_table_id         = aws_route_table.queueprocessor-example-com.id
-}
-
-resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-masters-queueprocessor-example-com" {
-  cidr_blocks       = ["0.0.0.0/0"]
-  from_port         = 22
-  protocol          = "tcp"
-  security_group_id = aws_security_group.masters-queueprocessor-example-com.id
-  to_port           = 22
-  type              = "ingress"
-}
-
-resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-nodes-queueprocessor-example-com" {
-  cidr_blocks       = ["0.0.0.0/0"]
-  from_port         = 22
-  protocol          = "tcp"
-  security_group_id = aws_security_group.nodes-queueprocessor-example-com.id
-  to_port           = 22
-  type              = "ingress"
-}
-
-resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-masters-queueprocessor-example-com" {
-  cidr_blocks       = ["0.0.0.0/0"]
-  from_port         = 443
-  protocol          = "tcp"
-  security_group_id = aws_security_group.masters-queueprocessor-example-com.id
-  to_port           = 443
-  type              = "ingress"
-}
-
-resource "aws_security_group_rule" "from-masters-queueprocessor-example-com-egress-all-0to0-0-0-0-0--0" {
-  cidr_blocks       = ["0.0.0.0/0"]
-  from_port         = 0
-  protocol          = "-1"
-  security_group_id = aws_security_group.masters-queueprocessor-example-com.id
-  to_port           = 0
-  type              = "egress"
-}
-
-resource "aws_security_group_rule" "from-masters-queueprocessor-example-com-ingress-all-0to0-masters-queueprocessor-example-com" {
-  from_port                = 0
-  protocol                 = "-1"
-  security_group_id        = aws_security_group.masters-queueprocessor-example-com.id
-  source_security_group_id = aws_security_group.masters-queueprocessor-example-com.id
-  to_port                  = 0
-  type                     = "ingress"
-}
-
-resource "aws_security_group_rule" "from-masters-queueprocessor-example-com-ingress-all-0to0-nodes-queueprocessor-example-com" {
-  from_port                = 0
-  protocol                 = "-1"
-  security_group_id        = aws_security_group.nodes-queueprocessor-example-com.id
-  source_security_group_id = aws_security_group.masters-queueprocessor-example-com.id
-  to_port                  = 0
-  type                     = "ingress"
-}
-
-resource "aws_security_group_rule" "from-nodes-queueprocessor-example-com-egress-all-0to0-0-0-0-0--0" {
-  cidr_blocks       = ["0.0.0.0/0"]
-  from_port         = 0
-  protocol          = "-1"
-  security_group_id = aws_security_group.nodes-queueprocessor-example-com.id
-  to_port           = 0
-  type              = "egress"
-}
-
-resource "aws_security_group_rule" "from-nodes-queueprocessor-example-com-ingress-all-0to0-nodes-queueprocessor-example-com" {
-  from_port                = 0
-  protocol                 = "-1"
-  security_group_id        = aws_security_group.nodes-queueprocessor-example-com.id
-  source_security_group_id = aws_security_group.nodes-queueprocessor-example-com.id
-  to_port                  = 0
-  type                     = "ingress"
-}
-
-resource "aws_security_group_rule" "from-nodes-queueprocessor-example-com-ingress-tcp-1to2379-masters-queueprocessor-example-com" {
-  from_port                = 1
-  protocol                 = "tcp"
-  security_group_id        = aws_security_group.masters-queueprocessor-example-com.id
-  source_security_group_id = aws_security_group.nodes-queueprocessor-example-com.id
-  to_port                  = 2379
-  type                     = "ingress"
-}
-
-resource "aws_security_group_rule" "from-nodes-queueprocessor-example-com-ingress-tcp-2382to4000-masters-queueprocessor-example-com" {
-  from_port                = 2382
-  protocol                 = "tcp"
-  security_group_id        = aws_security_group.masters-queueprocessor-example-com.id
-  source_security_group_id = aws_security_group.nodes-queueprocessor-example-com.id
-  to_port                  = 4000
-  type                     = "ingress"
-}
-
-resource "aws_security_group_rule" "from-nodes-queueprocessor-example-com-ingress-tcp-4003to65535-masters-queueprocessor-example-com" {
-  from_port                = 4003
-  protocol                 = "tcp"
-  security_group_id        = aws_security_group.masters-queueprocessor-example-com.id
-  source_security_group_id = aws_security_group.nodes-queueprocessor-example-com.id
-  to_port                  = 65535
-  type                     = "ingress"
-}
-
-resource "aws_security_group_rule" "from-nodes-queueprocessor-example-com-ingress-udp-1to65535-masters-queueprocessor-example-com" {
-  from_port                = 1
-  protocol                 = "udp"
-  security_group_id        = aws_security_group.masters-queueprocessor-example-com.id
-  source_security_group_id = aws_security_group.nodes-queueprocessor-example-com.id
-  to_port                  = 65535
-  type                     = "ingress"
-}
-
-resource "aws_security_group" "masters-queueprocessor-example-com" {
-  description = "Security group for masters"
-  name        = "masters.queueprocessor.example.com"
-  tags = {
-    "KubernetesCluster"                                = "queueprocessor.example.com"
-    "Name"                                             = "masters.queueprocessor.example.com"
-    "kubernetes.io/cluster/queueprocessor.example.com" = "owned"
-  }
-  vpc_id = aws_vpc.queueprocessor-example-com.id
-}
-
-resource "aws_security_group" "nodes-queueprocessor-example-com" {
-  description = "Security group for nodes"
-  name        = "nodes.queueprocessor.example.com"
-  tags = {
-    "KubernetesCluster"                                = "queueprocessor.example.com"
-    "Name"                                             = "nodes.queueprocessor.example.com"
-    "kubernetes.io/cluster/queueprocessor.example.com" = "owned"
-  }
-  vpc_id = aws_vpc.queueprocessor-example-com.id
-}
-
-resource "aws_sqs_queue" "queueprocessor-example-com-nth" {
-  message_retention_seconds = 300
-  name                      = "queueprocessor-example-com-nth"
-  policy                    = file("${path.module}/data/aws_sqs_queue_queueprocessor-example-com-nth_policy")
-  tags = {
-    "KubernetesCluster"                                = "queueprocessor.example.com"
-    "Name"                                             = "queueprocessor-example-com-nth"
-    "kubernetes.io/cluster/queueprocessor.example.com" = "owned"
-  }
-}
-
-resource "aws_subnet" "us-test-1a-queueprocessor-example-com" {
-  availability_zone = "us-test-1a"
-  cidr_block        = "172.20.32.0/19"
-  tags = {
-    "KubernetesCluster"                                = "queueprocessor.example.com"
-    "Name"                                             = "us-test-1a.queueprocessor.example.com"
-    "SubnetType"                                       = "Public"
-    "kubernetes.io/cluster/queueprocessor.example.com" = "owned"
-    "kubernetes.io/role/elb"                           = "1"
-  }
-  vpc_id = aws_vpc.queueprocessor-example-com.id
-}
-
-resource "aws_vpc_dhcp_options_association" "queueprocessor-example-com" {
-  dhcp_options_id = aws_vpc_dhcp_options.queueprocessor-example-com.id
-  vpc_id          = aws_vpc.queueprocessor-example-com.id
-}
-
-resource "aws_vpc_dhcp_options" "queueprocessor-example-com" {
-  domain_name         = "us-test-1.compute.internal"
-  domain_name_servers = ["AmazonProvidedDNS"]
-  tags = {
-    "KubernetesCluster"                                = "queueprocessor.example.com"
-    "Name"                                             = "queueprocessor.example.com"
-    "kubernetes.io/cluster/queueprocessor.example.com" = "owned"
-  }
-}
-
-resource "aws_vpc" "queueprocessor-example-com" {
-  cidr_block           = "172.20.0.0/16"
-  enable_dns_hostnames = true
-  enable_dns_support   = true
-  tags = {
-    "KubernetesCluster"                                = "queueprocessor.example.com"
-    "Name"                                             = "queueprocessor.example.com"
-    "kubernetes.io/cluster/queueprocessor.example.com" = "owned"
-  }
-}
-
-terraform {
-  required_version = ">= 0.12.26"
-  required_providers {
-    aws = {
-      "source"  = "hashicorp/aws"
-      "version" = ">= 3.34.0"
-    }
-  }
-}

tests/integration/update_cluster/nth_sqs_resources/cloudformation.json

@@ -1,16 +1,16 @@
 {
   "Resources": {
-    "AWSAutoScalingAutoScalingGroupmasterustest1amastersqueueprocessorexamplecom": {
+    "AWSAutoScalingAutoScalingGroupmasterustest1amastersnthsqsresourcesexamplecom": {
       "Type": "AWS::AutoScaling::AutoScalingGroup",
       "Properties": {
-        "AutoScalingGroupName": "master-us-test-1a.masters.queueprocessor.example.com",
+        "AutoScalingGroupName": "master-us-test-1a.masters.nthsqsresources.example.com",
         "LaunchTemplate": {
           "LaunchTemplateId": {
-            "Ref": "AWSEC2LaunchTemplatemasterustest1amastersqueueprocessorexamplecom"
+            "Ref": "AWSEC2LaunchTemplatemasterustest1amastersnthsqsresourcesexamplecom"
           },
           "Version": {
             "Fn::GetAtt": [
-              "AWSEC2LaunchTemplatemasterustest1amastersqueueprocessorexamplecom",
+              "AWSEC2LaunchTemplatemasterustest1amastersnthsqsresourcesexamplecom",
               "LatestVersionNumber"
             ]
           }
@@ -19,18 +19,18 @@
         "MinSize": "1",
         "VPCZoneIdentifier": [
           {
-            "Ref": "AWSEC2Subnetustest1aqueueprocessorexamplecom"
+            "Ref": "AWSEC2Subnetustest1anthsqsresourcesexamplecom"
           }
         ],
         "Tags": [
           {
             "Key": "KubernetesCluster",
-            "Value": "queueprocessor.example.com",
+            "Value": "nthsqsresources.example.com",
             "PropagateAtLaunch": true
           },
           {
             "Key": "Name",
-            "Value": "master-us-test-1a.masters.queueprocessor.example.com",
+            "Value": "master-us-test-1a.masters.nthsqsresources.example.com",
             "PropagateAtLaunch": true
           },
           {
@@ -74,7 +74,7 @@
             "PropagateAtLaunch": true
           },
           {
-            "Key": "kubernetes.io/cluster/queueprocessor.example.com",
+            "Key": "kubernetes.io/cluster/nthsqsresources.example.com",
             "Value": "owned",
             "PropagateAtLaunch": true
           }
@@ -96,17 +96,17 @@
         ]
       }
     },
-    "AWSAutoScalingAutoScalingGroupnodesqueueprocessorexamplecom": {
+    "AWSAutoScalingAutoScalingGroupnodesnthsqsresourcesexamplecom": {
       "Type": "AWS::AutoScaling::AutoScalingGroup",
       "Properties": {
-        "AutoScalingGroupName": "nodes.queueprocessor.example.com",
+        "AutoScalingGroupName": "nodes.nthsqsresources.example.com",
         "LaunchTemplate": {
           "LaunchTemplateId": {
-            "Ref": "AWSEC2LaunchTemplatenodesqueueprocessorexamplecom"
+            "Ref": "AWSEC2LaunchTemplatenodesnthsqsresourcesexamplecom"
           },
           "Version": {
             "Fn::GetAtt": [
-              "AWSEC2LaunchTemplatenodesqueueprocessorexamplecom",
+              "AWSEC2LaunchTemplatenodesnthsqsresourcesexamplecom",
               "LatestVersionNumber"
             ]
           }
@@ -115,18 +115,18 @@
         "MinSize": "2",
         "VPCZoneIdentifier": [
           {
-            "Ref": "AWSEC2Subnetustest1aqueueprocessorexamplecom"
+            "Ref": "AWSEC2Subnetustest1anthsqsresourcesexamplecom"
           }
         ],
         "Tags": [
           {
             "Key": "KubernetesCluster",
-            "Value": "queueprocessor.example.com",
+            "Value": "nthsqsresources.example.com",
             "PropagateAtLaunch": true
           },
           {
             "Key": "Name",
-            "Value": "nodes.queueprocessor.example.com",
+            "Value": "nodes.nthsqsresources.example.com",
             "PropagateAtLaunch": true
           },
           {
@@ -155,7 +155,7 @@
             "PropagateAtLaunch": true
           },
           {
-            "Key": "kubernetes.io/cluster/queueprocessor.example.com",
+            "Key": "kubernetes.io/cluster/nthsqsresources.example.com",
             "Value": "owned",
             "PropagateAtLaunch": true
           }
@@ -182,7 +182,7 @@
       "Properties": {
         "LifecycleHookName": "master-us-test-1a-NTHLifecycleHook",
         "AutoScalingGroupName": {
-          "Ref": "AWSAutoScalingAutoScalingGroupmasterustest1amastersqueueprocessorexamplecom"
+          "Ref": "AWSAutoScalingAutoScalingGroupmasterustest1amastersnthsqsresourcesexamplecom"
         },
         "DefaultResult": "CONTINUE",
         "HeartbeatTimeout": 300,
@@ -194,14 +194,14 @@
       "Properties": {
         "LifecycleHookName": "nodes-NTHLifecycleHook",
         "AutoScalingGroupName": {
-          "Ref": "AWSAutoScalingAutoScalingGroupnodesqueueprocessorexamplecom"
+          "Ref": "AWSAutoScalingAutoScalingGroupnodesnthsqsresourcesexamplecom"
         },
         "DefaultResult": "CONTINUE",
         "HeartbeatTimeout": 300,
         "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING"
       }
     },
-    "AWSEC2DHCPOptionsqueueprocessorexamplecom": {
+    "AWSEC2DHCPOptionsnthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::DHCPOptions",
       "Properties": {
         "DomainName": "us-test-1.compute.internal",
@@ -211,42 +211,42 @@
         "Tags": [
           {
             "Key": "KubernetesCluster",
-            "Value": "queueprocessor.example.com"
+            "Value": "nthsqsresources.example.com"
           },
           {
             "Key": "Name",
-            "Value": "queueprocessor.example.com"
+            "Value": "nthsqsresources.example.com"
           },
           {
-            "Key": "kubernetes.io/cluster/queueprocessor.example.com",
+            "Key": "kubernetes.io/cluster/nthsqsresources.example.com",
             "Value": "owned"
           }
         ]
       }
     },
-    "AWSEC2InternetGatewayqueueprocessorexamplecom": {
+    "AWSEC2InternetGatewaynthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::InternetGateway",
       "Properties": {
         "Tags": [
           {
             "Key": "KubernetesCluster",
-            "Value": "queueprocessor.example.com"
+            "Value": "nthsqsresources.example.com"
           },
           {
             "Key": "Name",
-            "Value": "queueprocessor.example.com"
+            "Value": "nthsqsresources.example.com"
           },
           {
-            "Key": "kubernetes.io/cluster/queueprocessor.example.com",
+            "Key": "kubernetes.io/cluster/nthsqsresources.example.com",
             "Value": "owned"
           }
         ]
       }
     },
-    "AWSEC2LaunchTemplatemasterustest1amastersqueueprocessorexamplecom": {
+    "AWSEC2LaunchTemplatemasterustest1amastersnthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::LaunchTemplate",
       "Properties": {
-        "LaunchTemplateName": "master-us-test-1a.masters.queueprocessor.example.com",
+        "LaunchTemplateName": "master-us-test-1a.masters.nthsqsresources.example.com",
         "LaunchTemplateData": {
           "BlockDeviceMappings": [
             {
@@ -267,12 +267,12 @@
           ],
           "IamInstanceProfile": {
             "Name": {
-              "Ref": "AWSIAMInstanceProfilemastersqueueprocessorexamplecom"
+              "Ref": "AWSIAMInstanceProfilemastersnthsqsresourcesexamplecom"
             }
           },
           "ImageId": "ami-12345678",
           "InstanceType": "m3.medium",
-          "KeyName": "kubernetes.queueprocessor.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57",
+          "KeyName": "kubernetes.nthsqsresources.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57",
           "MetadataOptions": {
             "HttpPutResponseHopLimit": 1,
             "HttpTokens": "optional"
@@ -284,7 +284,7 @@
               "DeviceIndex": 0,
               "Groups": [
                 {
-                  "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom"
+                  "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom"
                 }
               ]
             }
@@ -295,11 +295,11 @@
               "Tags": [
                 {
                   "Key": "KubernetesCluster",
-                  "Value": "queueprocessor.example.com"
+                  "Value": "nthsqsresources.example.com"
                 },
                 {
                   "Key": "Name",
-                  "Value": "master-us-test-1a.masters.queueprocessor.example.com"
+                  "Value": "master-us-test-1a.masters.nthsqsresources.example.com"
                 },
                 {
                   "Key": "aws-node-termination-handler/managed",
@@ -334,7 +334,7 @@
                   "Value": "master-us-test-1a"
                 },
                 {
-                  "Key": "kubernetes.io/cluster/queueprocessor.example.com",
+                  "Key": "kubernetes.io/cluster/nthsqsresources.example.com",
                   "Value": "owned"
                 }
               ]
@@ -344,11 +344,11 @@
               "Tags": [
                 {
                   "Key": "KubernetesCluster",
-                  "Value": "queueprocessor.example.com"
+                  "Value": "nthsqsresources.example.com"
                 },
                 {
                   "Key": "Name",
-                  "Value": "master-us-test-1a.masters.queueprocessor.example.com"
+                  "Value": "master-us-test-1a.masters.nthsqsresources.example.com"
                 },
                 {
                   "Key": "aws-node-termination-handler/managed",
@@ -383,7 +383,7 @@
                   "Value": "master-us-test-1a"
                 },
                 {
-                  "Key": "kubernetes.io/cluster/queueprocessor.example.com",
+                  "Key": "kubernetes.io/cluster/nthsqsresources.example.com",
                   "Value": "owned"
                 }
               ]
@@ -393,10 +393,10 @@
         }
       }
     },
-    "AWSEC2LaunchTemplatenodesqueueprocessorexamplecom": {
+    "AWSEC2LaunchTemplatenodesnthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::LaunchTemplate",
       "Properties": {
-        "LaunchTemplateName": "nodes.queueprocessor.example.com",
+        "LaunchTemplateName": "nodes.nthsqsresources.example.com",
         "LaunchTemplateData": {
           "BlockDeviceMappings": [
             {
@@ -413,12 +413,12 @@
           ],
           "IamInstanceProfile": {
             "Name": {
-              "Ref": "AWSIAMInstanceProfilenodesqueueprocessorexamplecom"
+              "Ref": "AWSIAMInstanceProfilenodesnthsqsresourcesexamplecom"
             }
           },
           "ImageId": "ami-12345678",
           "InstanceType": "t2.medium",
-          "KeyName": "kubernetes.queueprocessor.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57",
+          "KeyName": "kubernetes.nthsqsresources.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57",
           "MetadataOptions": {
             "HttpPutResponseHopLimit": 1,
             "HttpTokens": "optional"
@@ -430,7 +430,7 @@
               "DeviceIndex": 0,
               "Groups": [
                 {
-                  "Ref": "AWSEC2SecurityGroupnodesqueueprocessorexamplecom"
+                  "Ref": "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom"
                 }
               ]
             }
@@ -441,11 +441,11 @@
               "Tags": [
                 {
                   "Key": "KubernetesCluster",
-                  "Value": "queueprocessor.example.com"
+                  "Value": "nthsqsresources.example.com"
                 },
                 {
                   "Key": "Name",
-                  "Value": "nodes.queueprocessor.example.com"
+                  "Value": "nodes.nthsqsresources.example.com"
                 },
                 {
                   "Key": "aws-node-termination-handler/managed",
@@ -468,7 +468,7 @@
                   "Value": "nodes"
                 },
                 {
-                  "Key": "kubernetes.io/cluster/queueprocessor.example.com",
+                  "Key": "kubernetes.io/cluster/nthsqsresources.example.com",
                   "Value": "owned"
                 }
               ]
@@ -478,11 +478,11 @@
               "Tags": [
                 {
                   "Key": "KubernetesCluster",
-                  "Value": "queueprocessor.example.com"
+                  "Value": "nthsqsresources.example.com"
                 },
                 {
                   "Key": "Name",
-                  "Value": "nodes.queueprocessor.example.com"
+                  "Value": "nodes.nthsqsresources.example.com"
                 },
                 {
                   "Key": "aws-node-termination-handler/managed",
@@ -505,7 +505,7 @@
                   "Value": "nodes"
                 },
                 {
-                  "Key": "kubernetes.io/cluster/queueprocessor.example.com",
+                  "Key": "kubernetes.io/cluster/nthsqsresources.example.com",
                   "Value": "owned"
                 }
               ]
@@ -519,31 +519,31 @@
       "Type": "AWS::EC2::Route",
       "Properties": {
         "RouteTableId": {
-          "Ref": "AWSEC2RouteTablequeueprocessorexamplecom"
+          "Ref": "AWSEC2RouteTablenthsqsresourcesexamplecom"
         },
         "DestinationCidrBlock": "0.0.0.0/0",
         "GatewayId": {
-          "Ref": "AWSEC2InternetGatewayqueueprocessorexamplecom"
+          "Ref": "AWSEC2InternetGatewaynthsqsresourcesexamplecom"
         }
       }
     },
-    "AWSEC2RouteTablequeueprocessorexamplecom": {
+    "AWSEC2RouteTablenthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::RouteTable",
       "Properties": {
         "VpcId": {
-          "Ref": "AWSEC2VPCqueueprocessorexamplecom"
+          "Ref": "AWSEC2VPCnthsqsresourcesexamplecom"
         },
         "Tags": [
           {
             "Key": "KubernetesCluster",
-            "Value": "queueprocessor.example.com"
+            "Value": "nthsqsresources.example.com"
           },
           {
             "Key": "Name",
-            "Value": "queueprocessor.example.com"
+            "Value": "nthsqsresources.example.com"
           },
           {
-            "Key": "kubernetes.io/cluster/queueprocessor.example.com",
+            "Key": "kubernetes.io/cluster/nthsqsresources.example.com",
             "Value": "owned"
           },
           {
@@ -553,11 +553,11 @@
         ]
       }
     },
-    "AWSEC2SecurityGroupEgressfrommastersqueueprocessorexamplecomegressall0to000000": {
+    "AWSEC2SecurityGroupEgressfrommastersnthsqsresourcesexamplecomegressall0to000000": {
       "Type": "AWS::EC2::SecurityGroupEgress",
       "Properties": {
         "GroupId": {
-          "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom"
+          "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom"
         },
         "FromPort": 0,
         "ToPort": 0,
@@ -565,11 +565,11 @@
         "CidrIp": "0.0.0.0/0"
       }
     },
-    "AWSEC2SecurityGroupEgressfromnodesqueueprocessorexamplecomegressall0to000000": {
+    "AWSEC2SecurityGroupEgressfromnodesnthsqsresourcesexamplecomegressall0to000000": {
       "Type": "AWS::EC2::SecurityGroupEgress",
       "Properties": {
         "GroupId": {
-          "Ref": "AWSEC2SecurityGroupnodesqueueprocessorexamplecom"
+          "Ref": "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom"
         },
         "FromPort": 0,
         "ToPort": 0,
@@ -577,11 +577,11 @@
         "CidrIp": "0.0.0.0/0"
       }
     },
-    "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22mastersqueueprocessorexamplecom": {
+    "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22mastersnthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::SecurityGroupIngress",
       "Properties": {
         "GroupId": {
-          "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom"
+          "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom"
         },
         "FromPort": 22,
         "ToPort": 22,
@@ -589,11 +589,11 @@
         "CidrIp": "0.0.0.0/0"
       }
     },
-    "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22nodesqueueprocessorexamplecom": {
+    "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22nodesnthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::SecurityGroupIngress",
       "Properties": {
         "GroupId": {
-          "Ref": "AWSEC2SecurityGroupnodesqueueprocessorexamplecom"
+          "Ref": "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom"
         },
         "FromPort": 22,
         "ToPort": 22,
@@ -601,11 +601,11 @@
         "CidrIp": "0.0.0.0/0"
       }
     },
-    "AWSEC2SecurityGroupIngressfrom00000ingresstcp443to443mastersqueueprocessorexamplecom": {
+    "AWSEC2SecurityGroupIngressfrom00000ingresstcp443to443mastersnthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::SecurityGroupIngress",
       "Properties": {
         "GroupId": {
-          "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom"
+          "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom"
         },
         "FromPort": 443,
         "ToPort": 443,
@@ -613,186 +613,186 @@
         "CidrIp": "0.0.0.0/0"
       }
     },
-    "AWSEC2SecurityGroupIngressfrommastersqueueprocessorexamplecomingressall0to0mastersqueueprocessorexamplecom": {
+    "AWSEC2SecurityGroupIngressfrommastersnthsqsresourcesexamplecomingressall0to0mastersnthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::SecurityGroupIngress",
       "Properties": {
         "GroupId": {
-          "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom"
+          "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom"
         },
         "SourceSecurityGroupId": {
-          "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom"
+          "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom"
         },
         "FromPort": 0,
         "ToPort": 0,
         "IpProtocol": "-1"
       }
     },
-    "AWSEC2SecurityGroupIngressfrommastersqueueprocessorexamplecomingressall0to0nodesqueueprocessorexamplecom": {
+    "AWSEC2SecurityGroupIngressfrommastersnthsqsresourcesexamplecomingressall0to0nodesnthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::SecurityGroupIngress",
       "Properties": {
         "GroupId": {
-          "Ref": "AWSEC2SecurityGroupnodesqueueprocessorexamplecom"
+          "Ref": "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom"
         },
         "SourceSecurityGroupId": {
-          "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom"
+          "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom"
         },
         "FromPort": 0,
         "ToPort": 0,
         "IpProtocol": "-1"
       }
     },
-    "AWSEC2SecurityGroupIngressfromnodesqueueprocessorexamplecomingressall0to0nodesqueueprocessorexamplecom": {
+    "AWSEC2SecurityGroupIngressfromnodesnthsqsresourcesexamplecomingressall0to0nodesnthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::SecurityGroupIngress",
       "Properties": {
         "GroupId": {
-          "Ref": "AWSEC2SecurityGroupnodesqueueprocessorexamplecom"
+          "Ref": "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom"
         },
         "SourceSecurityGroupId": {
-          "Ref": "AWSEC2SecurityGroupnodesqueueprocessorexamplecom"
+          "Ref": "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom"
         },
         "FromPort": 0,
         "ToPort": 0,
         "IpProtocol": "-1"
       }
     },
-    "AWSEC2SecurityGroupIngressfromnodesqueueprocessorexamplecomingresstcp1to2379mastersqueueprocessorexamplecom": {
+    "AWSEC2SecurityGroupIngressfromnodesnthsqsresourcesexamplecomingresstcp1to2379mastersnthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::SecurityGroupIngress",
       "Properties": {
         "GroupId": {
-          "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom"
+          "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom"
         },
         "SourceSecurityGroupId": {
-          "Ref": "AWSEC2SecurityGroupnodesqueueprocessorexamplecom"
+          "Ref": "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom"
         },
         "FromPort": 1,
         "ToPort": 2379,
         "IpProtocol": "tcp"
       }
     },
-    "AWSEC2SecurityGroupIngressfromnodesqueueprocessorexamplecomingresstcp2382to4000mastersqueueprocessorexamplecom": {
+    "AWSEC2SecurityGroupIngressfromnodesnthsqsresourcesexamplecomingresstcp2382to4000mastersnthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::SecurityGroupIngress",
       "Properties": {
         "GroupId": {
-          "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom"
+          "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom"
         },
         "SourceSecurityGroupId": {
-          "Ref": "AWSEC2SecurityGroupnodesqueueprocessorexamplecom"
+          "Ref": "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom"
         },
         "FromPort": 2382,
         "ToPort": 4000,
         "IpProtocol": "tcp"
       }
     },
-    "AWSEC2SecurityGroupIngressfromnodesqueueprocessorexamplecomingresstcp4003to65535mastersqueueprocessorexamplecom": {
+    "AWSEC2SecurityGroupIngressfromnodesnthsqsresourcesexamplecomingresstcp4003to65535mastersnthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::SecurityGroupIngress",
       "Properties": {
         "GroupId": {
-          "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom"
+          "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom"
         },
         "SourceSecurityGroupId": {
-          "Ref": "AWSEC2SecurityGroupnodesqueueprocessorexamplecom"
+          "Ref": "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom"
         },
         "FromPort": 4003,
         "ToPort": 65535,
         "IpProtocol": "tcp"
       }
     },
-    "AWSEC2SecurityGroupIngressfromnodesqueueprocessorexamplecomingressudp1to65535mastersqueueprocessorexamplecom": {
+    "AWSEC2SecurityGroupIngressfromnodesnthsqsresourcesexamplecomingressudp1to65535mastersnthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::SecurityGroupIngress",
       "Properties": {
         "GroupId": {
-          "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom"
+          "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom"
         },
         "SourceSecurityGroupId": {
-          "Ref": "AWSEC2SecurityGroupnodesqueueprocessorexamplecom"
+          "Ref": "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom"
         },
         "FromPort": 1,
         "ToPort": 65535,
         "IpProtocol": "udp"
       }
     },
-    "AWSEC2SecurityGroupmastersqueueprocessorexamplecom": {
+    "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::SecurityGroup",
       "Properties": {
-        "GroupName": "masters.queueprocessor.example.com",
+        "GroupName": "masters.nthsqsresources.example.com",
         "VpcId": {
-          "Ref": "AWSEC2VPCqueueprocessorexamplecom"
+          "Ref": "AWSEC2VPCnthsqsresourcesexamplecom"
         },
         "GroupDescription": "Security group for masters",
         "Tags": [
           {
             "Key": "KubernetesCluster",
-            "Value": "queueprocessor.example.com"
+            "Value": "nthsqsresources.example.com"
           },
           {
             "Key": "Name",
-            "Value": "masters.queueprocessor.example.com"
+            "Value": "masters.nthsqsresources.example.com"
           },
           {
-            "Key": "kubernetes.io/cluster/queueprocessor.example.com",
+            "Key": "kubernetes.io/cluster/nthsqsresources.example.com",
             "Value": "owned"
           }
         ]
       }
     },
-    "AWSEC2SecurityGroupnodesqueueprocessorexamplecom": {
+    "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::SecurityGroup",
       "Properties": {
-        "GroupName": "nodes.queueprocessor.example.com",
+        "GroupName": "nodes.nthsqsresources.example.com",
         "VpcId": {
-          "Ref": "AWSEC2VPCqueueprocessorexamplecom"
+          "Ref": "AWSEC2VPCnthsqsresourcesexamplecom"
         },
         "GroupDescription": "Security group for nodes",
         "Tags": [
           {
             "Key": "KubernetesCluster",
-            "Value": "queueprocessor.example.com"
+            "Value": "nthsqsresources.example.com"
           },
           {
             "Key": "Name",
-            "Value": "nodes.queueprocessor.example.com"
+            "Value": "nodes.nthsqsresources.example.com"
           },
           {
-            "Key": "kubernetes.io/cluster/queueprocessor.example.com",
+            "Key": "kubernetes.io/cluster/nthsqsresources.example.com",
             "Value": "owned"
           }
         ]
       }
     },
-    "AWSEC2SubnetRouteTableAssociationustest1aqueueprocessorexamplecom": {
+    "AWSEC2SubnetRouteTableAssociationustest1anthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::SubnetRouteTableAssociation",
       "Properties": {
         "SubnetId": {
-          "Ref": "AWSEC2Subnetustest1aqueueprocessorexamplecom"
+          "Ref": "AWSEC2Subnetustest1anthsqsresourcesexamplecom"
         },
         "RouteTableId": {
-          "Ref": "AWSEC2RouteTablequeueprocessorexamplecom"
+          "Ref": "AWSEC2RouteTablenthsqsresourcesexamplecom"
         }
       }
     },
-    "AWSEC2Subnetustest1aqueueprocessorexamplecom": {
+    "AWSEC2Subnetustest1anthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::Subnet",
       "Properties": {
         "VpcId": {
-          "Ref": "AWSEC2VPCqueueprocessorexamplecom"
+          "Ref": "AWSEC2VPCnthsqsresourcesexamplecom"
         },
         "CidrBlock": "172.20.32.0/19",
         "AvailabilityZone": "us-test-1a",
         "Tags": [
           {
             "Key": "KubernetesCluster",
-            "Value": "queueprocessor.example.com"
+            "Value": "nthsqsresources.example.com"
           },
           {
             "Key": "Name",
-            "Value": "us-test-1a.queueprocessor.example.com"
+            "Value": "us-test-1a.nthsqsresources.example.com"
           },
           {
             "Key": "SubnetType",
             "Value": "Public"
           },
           {
-            "Key": "kubernetes.io/cluster/queueprocessor.example.com",
+            "Key": "kubernetes.io/cluster/nthsqsresources.example.com",
             "Value": "owned"
           },
           {
@@ -802,29 +802,29 @@
         ]
       }
     },
-    "AWSEC2VPCDHCPOptionsAssociationqueueprocessorexamplecom": {
+    "AWSEC2VPCDHCPOptionsAssociationnthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::VPCDHCPOptionsAssociation",
       "Properties": {
         "VpcId": {
-          "Ref": "AWSEC2VPCqueueprocessorexamplecom"
+          "Ref": "AWSEC2VPCnthsqsresourcesexamplecom"
         },
         "DhcpOptionsId": {
-          "Ref": "AWSEC2DHCPOptionsqueueprocessorexamplecom"
+          "Ref": "AWSEC2DHCPOptionsnthsqsresourcesexamplecom"
         }
       }
     },
-    "AWSEC2VPCGatewayAttachmentqueueprocessorexamplecom": {
+    "AWSEC2VPCGatewayAttachmentnthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::VPCGatewayAttachment",
       "Properties": {
         "VpcId": {
-          "Ref": "AWSEC2VPCqueueprocessorexamplecom"
+          "Ref": "AWSEC2VPCnthsqsresourcesexamplecom"
         },
         "InternetGatewayId": {
-          "Ref": "AWSEC2InternetGatewayqueueprocessorexamplecom"
+          "Ref": "AWSEC2InternetGatewaynthsqsresourcesexamplecom"
         }
       }
     },
-    "AWSEC2VPCqueueprocessorexamplecom": {
+    "AWSEC2VPCnthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::VPC",
       "Properties": {
         "CidrBlock": "172.20.0.0/16",
@@ -833,20 +833,20 @@
         "Tags": [
           {
             "Key": "KubernetesCluster",
-            "Value": "queueprocessor.example.com"
+            "Value": "nthsqsresources.example.com"
           },
           {
             "Key": "Name",
-            "Value": "queueprocessor.example.com"
+            "Value": "nthsqsresources.example.com"
           },
           {
-            "Key": "kubernetes.io/cluster/queueprocessor.example.com",
+            "Key": "kubernetes.io/cluster/nthsqsresources.example.com",
             "Value": "owned"
           }
         ]
       }
     },
-    "AWSEC2Volumeustest1aetcdeventsqueueprocessorexamplecom": {
+    "AWSEC2Volumeustest1aetcdeventsnthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::Volume",
       "Properties": {
         "AvailabilityZone": "us-test-1a",
@@ -858,11 +858,11 @@
         "Tags": [
           {
             "Key": "KubernetesCluster",
-            "Value": "queueprocessor.example.com"
+            "Value": "nthsqsresources.example.com"
           },
           {
             "Key": "Name",
-            "Value": "us-test-1a.etcd-events.queueprocessor.example.com"
+            "Value": "us-test-1a.etcd-events.nthsqsresources.example.com"
           },
           {
             "Key": "k8s.io/etcd/events",
@@ -873,13 +873,13 @@
             "Value": "1"
           },
           {
-            "Key": "kubernetes.io/cluster/queueprocessor.example.com",
+            "Key": "kubernetes.io/cluster/nthsqsresources.example.com",
             "Value": "owned"
           }
         ]
       }
     },
-    "AWSEC2Volumeustest1aetcdmainqueueprocessorexamplecom": {
+    "AWSEC2Volumeustest1aetcdmainnthsqsresourcesexamplecom": {
       "Type": "AWS::EC2::Volume",
       "Properties": {
         "AvailabilityZone": "us-test-1a",
@@ -891,11 +891,11 @@
         "Tags": [
           {
             "Key": "KubernetesCluster",
-            "Value": "queueprocessor.example.com"
+            "Value": "nthsqsresources.example.com"
           },
           {
             "Key": "Name",
-            "Value": "us-test-1a.etcd-main.queueprocessor.example.com"
+            "Value": "us-test-1a.etcd-main.nthsqsresources.example.com"
           },
           {
             "Key": "k8s.io/etcd/main",
@@ -906,16 +906,16 @@
             "Value": "1"
           },
           {
-            "Key": "kubernetes.io/cluster/queueprocessor.example.com",
+            "Key": "kubernetes.io/cluster/nthsqsresources.example.com",
             "Value": "owned"
           }
         ]
       }
     },
-    "AWSEventsRulequeueprocessorexamplecomASGLifecycle": {
+    "AWSEventsRulenthsqsresourcesexamplecomASGLifecycle": {
       "Type": "AWS::Events::Rule",
       "Properties": {
-        "Name": "queueprocessor.example.com-ASGLifecycle",
+        "Name": "nthsqsresources.example.com-ASGLifecycle",
         "EventPattern": {
           "detail-type": [
             "EC2 Instance-terminate Lifecycle Action"
@@ -927,15 +927,15 @@
         "Targets": [
           {
             "Id": "1",
-            "Arn": "arn:aws:sqs:us-test-1:123456789012:queueprocessor-example-com-nth"
+            "Arn": "arn:aws:sqs:us-test-1:123456789012:nthsqsresources-example-com-nth"
           }
         ]
       }
     },
-    "AWSEventsRulequeueprocessorexamplecomRebalanceRecommendation": {
+    "AWSEventsRulenthsqsresourcesexamplecomRebalanceRecommendation": {
       "Type": "AWS::Events::Rule",
       "Properties": {
-        "Name": "queueprocessor.example.com-RebalanceRecommendation",
+        "Name": "nthsqsresources.example.com-RebalanceRecommendation",
         "EventPattern": {
           "detail-type": [
             "EC2 Instance Rebalance Recommendation"
@@ -947,15 +947,15 @@
         "Targets": [
           {
             "Id": "1",
-            "Arn": "arn:aws:sqs:us-test-1:123456789012:queueprocessor-example-com-nth"
+            "Arn": "arn:aws:sqs:us-test-1:123456789012:nthsqsresources-example-com-nth"
           }
         ]
       }
     },
-    "AWSEventsRulequeueprocessorexamplecomSpotInterruption": {
+    "AWSEventsRulenthsqsresourcesexamplecomSpotInterruption": {
       "Type": "AWS::Events::Rule",
       "Properties": {
-        "Name": "queueprocessor.example.com-SpotInterruption",
+        "Name": "nthsqsresources.example.com-SpotInterruption",
         "EventPattern": {
           "detail-type": [
             "EC2 Spot Instance Interruption Warning"
@@ -967,40 +967,40 @@
         "Targets": [
           {
             "Id": "1",
-            "Arn": "arn:aws:sqs:us-test-1:123456789012:queueprocessor-example-com-nth"
+            "Arn": "arn:aws:sqs:us-test-1:123456789012:nthsqsresources-example-com-nth"
           }
         ]
       }
     },
-    "AWSIAMInstanceProfilemastersqueueprocessorexamplecom": {
+    "AWSIAMInstanceProfilemastersnthsqsresourcesexamplecom": {
       "Type": "AWS::IAM::InstanceProfile",
       "Properties": {
-        "InstanceProfileName": "masters.queueprocessor.example.com",
+        "InstanceProfileName": "masters.nthsqsresources.example.com",
         "Roles": [
           {
-            "Ref": "AWSIAMRolemastersqueueprocessorexamplecom"
+            "Ref": "AWSIAMRolemastersnthsqsresourcesexamplecom"
           }
         ]
       }
     },
-    "AWSIAMInstanceProfilenodesqueueprocessorexamplecom": {
+    "AWSIAMInstanceProfilenodesnthsqsresourcesexamplecom": {
       "Type": "AWS::IAM::InstanceProfile",
       "Properties": {
-        "InstanceProfileName": "nodes.queueprocessor.example.com",
+        "InstanceProfileName": "nodes.nthsqsresources.example.com",
         "Roles": [
           {
-            "Ref": "AWSIAMRolenodesqueueprocessorexamplecom"
+            "Ref": "AWSIAMRolenodesnthsqsresourcesexamplecom"
           }
         ]
       }
     },
-    "AWSIAMPolicymastersqueueprocessorexamplecom": {
+    "AWSIAMPolicymastersnthsqsresourcesexamplecom": {
       "Type": "AWS::IAM::Policy",
       "Properties": {
-        "PolicyName": "masters.queueprocessor.example.com",
+        "PolicyName": "masters.nthsqsresources.example.com",
         "Roles": [
           {
-            "Ref": "AWSIAMRolemastersqueueprocessorexamplecom"
+            "Ref": "AWSIAMRolemastersnthsqsresourcesexamplecom"
           }
         ],
         "PolicyDocument": {
@@ -1048,7 +1048,7 @@
               ],
               "Condition": {
                 "StringEquals": {
-                  "ec2:ResourceTag/KubernetesCluster": "queueprocessor.example.com"
+                  "ec2:ResourceTag/KubernetesCluster": "nthsqsresources.example.com"
                 }
               },
               "Effect": "Allow",
@@ -1076,7 +1076,7 @@
               ],
               "Condition": {
                 "StringEquals": {
-                  "autoscaling:ResourceTag/KubernetesCluster": "queueprocessor.example.com"
+                  "autoscaling:ResourceTag/KubernetesCluster": "nthsqsresources.example.com"
                 }
               },
               "Effect": "Allow",
@@ -1187,13 +1187,13 @@
         }
       }
     },
-    "AWSIAMPolicynodesqueueprocessorexamplecom": {
+    "AWSIAMPolicynodesnthsqsresourcesexamplecom": {
       "Type": "AWS::IAM::Policy",
       "Properties": {
-        "PolicyName": "nodes.queueprocessor.example.com",
+        "PolicyName": "nodes.nthsqsresources.example.com",
         "Roles": [
           {
-            "Ref": "AWSIAMRolenodesqueueprocessorexamplecom"
+            "Ref": "AWSIAMRolenodesnthsqsresourcesexamplecom"
           }
         ],
         "PolicyDocument": {
@@ -1213,10 +1213,10 @@
         }
       }
     },
-    "AWSIAMRolemastersqueueprocessorexamplecom": {
+    "AWSIAMRolemastersnthsqsresourcesexamplecom": {
       "Type": "AWS::IAM::Role",
       "Properties": {
-        "RoleName": "masters.queueprocessor.example.com",
+        "RoleName": "masters.nthsqsresources.example.com",
         "AssumeRolePolicyDocument": {
           "Statement": [
             {
@@ -1232,23 +1232,23 @@
         "Tags": [
           {
             "Key": "KubernetesCluster",
-            "Value": "queueprocessor.example.com"
+            "Value": "nthsqsresources.example.com"
           },
           {
             "Key": "Name",
-            "Value": "masters.queueprocessor.example.com"
+            "Value": "masters.nthsqsresources.example.com"
           },
           {
-            "Key": "kubernetes.io/cluster/queueprocessor.example.com",
+            "Key": "kubernetes.io/cluster/nthsqsresources.example.com",
             "Value": "owned"
           }
         ]
       }
     },
-    "AWSIAMRolenodesqueueprocessorexamplecom": {
+    "AWSIAMRolenodesnthsqsresourcesexamplecom": {
       "Type": "AWS::IAM::Role",
       "Properties": {
-        "RoleName": "nodes.queueprocessor.example.com",
+        "RoleName": "nodes.nthsqsresources.example.com",
         "AssumeRolePolicyDocument": {
           "Statement": [
             {
@@ -1264,25 +1264,25 @@
         "Tags": [
           {
             "Key": "KubernetesCluster",
-            "Value": "queueprocessor.example.com"
+            "Value": "nthsqsresources.example.com"
           },
           {
             "Key": "Name",
-            "Value": "nodes.queueprocessor.example.com"
+            "Value": "nodes.nthsqsresources.example.com"
           },
           {
-            "Key": "kubernetes.io/cluster/queueprocessor.example.com",
+            "Key": "kubernetes.io/cluster/nthsqsresources.example.com",
             "Value": "owned"
           }
         ]
       }
     },
-    "AWSSQSQueuePolicyqueueprocessorexamplecomnthPolicy": {
+    "AWSSQSQueuePolicynthsqsresourcesexamplecomnthPolicy": {
       "Type": "AWS::SQS::QueuePolicy",
       "Properties": {
         "Queues": [
           {
-            "Ref": "AWSSQSQueuequeueprocessorexamplecomnth"
+            "Ref": "AWSSQSQueuenthsqsresourcesexamplecomnth"
           }
         ],
         "PolicyDocument": {
@@ -1297,7 +1297,7 @@
                 ]
               },
               "Resource": [
-                "arn:aws:sqs:us-test-1:123456789012:queueprocessor-example-com-nth"
+                "arn:aws:sqs:us-test-1:123456789012:nthsqsresources-example-com-nth"
               ]
             }
           ],
@@ -1305,22 +1305,22 @@
         }
       }
     },
-    "AWSSQSQueuequeueprocessorexamplecomnth": {
+    "AWSSQSQueuenthsqsresourcesexamplecomnth": {
       "Type": "AWS::SQS::Queue",
       "Properties": {
-        "QueueName": "queueprocessor-example-com-nth",
+        "QueueName": "nthsqsresources-example-com-nth",
         "MessageRetentionPeriod": 300,
         "Tags": [
           {
             "Key": "KubernetesCluster",
-            "Value": "queueprocessor.example.com"
+            "Value": "nthsqsresources.example.com"
           },
           {
             "Key": "Name",
-            "Value": "queueprocessor-example-com-nth"
+            "Value": "nthsqsresources-example-com-nth"
           },
           {
-            "Key": "kubernetes.io/cluster/queueprocessor.example.com",
+            "Key": "kubernetes.io/cluster/nthsqsresources.example.com",
             "Value": "owned"
           }
         ]

tests/integration/update_cluster/nth_sqs_resources/cloudformation.json.extracted.yaml

@@ -1,4 +1,4 @@
-Resources.AWSEC2LaunchTemplatemasterustest1amastersqueueprocessorexamplecom.Properties.LaunchTemplateData.UserData: |
+Resources.AWSEC2LaunchTemplatemasterustest1amastersnthsqsresourcesexamplecom.Properties.LaunchTemplateData.UserData: |
   #!/bin/bash
   set -o errexit
   set -o nounset
@@ -206,8 +206,8 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersqueueprocessorexamplecom.Prop
     requestheaderUsernameHeaders:
     - X-Remote-User
     securePort: 443
-    serviceAccountIssuer: https://api.internal.queueprocessor.example.com
+    serviceAccountIssuer: https://api.internal.nthsqsresources.example.com
-    serviceAccountJWKSURI: https://api.internal.queueprocessor.example.com/openid/v1/jwks
+    serviceAccountJWKSURI: https://api.internal.nthsqsresources.example.com/openid/v1/jwks
     serviceClusterIPRange: 100.64.0.0/13
     storageBackend: etcd3
   kubeControllerManager:
@@ -215,7 +215,7 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersqueueprocessorexamplecom.Prop
     attachDetachReconcileSyncPeriod: 1m0s
     cloudProvider: aws
     clusterCIDR: 100.96.0.0/11
-    clusterName: queueprocessor.example.com
+    clusterName: nthsqsresources.example.com
     configureCloudRoutes: false
     image: k8s.gcr.io/kube-controller-manager:v1.20.0
     leaderElection:
@@ -288,8 +288,8 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersqueueprocessorexamplecom.Prop
     - 6e3f80e8451ecbe7b3559247721c3e226be6b228acaadee7e13683f80c20e81c@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.0.tgz
     - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/protokube
     - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/channels
-  ClusterName: queueprocessor.example.com
+  ClusterName: nthsqsresources.example.com
-  ConfigBase: memfs://clusters.example.com/queueprocessor.example.com
+  ConfigBase: memfs://clusters.example.com/nthsqsresources.example.com
   InstanceGroupName: master-us-test-1a
   InstanceGroupRole: Master
   KubeletConfig:
@@ -315,10 +315,10 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersqueueprocessorexamplecom.Prop
     podManifestPath: /etc/kubernetes/manifests
     registerSchedulable: false
   channels:
-  - memfs://clusters.example.com/queueprocessor.example.com/addons/bootstrap-channel.yaml
+  - memfs://clusters.example.com/nthsqsresources.example.com/addons/bootstrap-channel.yaml
   etcdManifests:
-  - memfs://clusters.example.com/queueprocessor.example.com/manifests/etcd/main.yaml
+  - memfs://clusters.example.com/nthsqsresources.example.com/manifests/etcd/main.yaml
-  - memfs://clusters.example.com/queueprocessor.example.com/manifests/etcd/events.yaml
+  - memfs://clusters.example.com/nthsqsresources.example.com/manifests/etcd/events.yaml
   staticManifests:
   - key: kube-apiserver-healthcheck
     path: manifests/static/kube-apiserver-healthcheck.yaml
@@ -327,7 +327,7 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersqueueprocessorexamplecom.Prop
 
   download-release
   echo "== nodeup node config done =="
-Resources.AWSEC2LaunchTemplatenodesqueueprocessorexamplecom.Properties.LaunchTemplateData.UserData: |
+Resources.AWSEC2LaunchTemplatenodesnthsqsresourcesexamplecom.Properties.LaunchTemplateData.UserData: |
   #!/bin/bash
   set -o errexit
   set -o nounset
@@ -531,8 +531,8 @@ Resources.AWSEC2LaunchTemplatenodesqueueprocessorexamplecom.Properties.LaunchTem
     - 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubectl
     - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
     - 6e3f80e8451ecbe7b3559247721c3e226be6b228acaadee7e13683f80c20e81c@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.0.tgz
-  ClusterName: queueprocessor.example.com
+  ClusterName: nthsqsresources.example.com
-  ConfigBase: memfs://clusters.example.com/queueprocessor.example.com
+  ConfigBase: memfs://clusters.example.com/nthsqsresources.example.com
   InstanceGroupName: nodes
   InstanceGroupRole: Node
   KubeletConfig:
@@ -554,7 +554,7 @@ Resources.AWSEC2LaunchTemplatenodesqueueprocessorexamplecom.Properties.LaunchTem
     nonMasqueradeCIDR: 100.64.0.0/10
     podManifestPath: /etc/kubernetes/manifests
   channels:
-  - memfs://clusters.example.com/queueprocessor.example.com/addons/bootstrap-channel.yaml
+  - memfs://clusters.example.com/nthsqsresources.example.com/addons/bootstrap-channel.yaml
 
   __EOF_KUBE_ENV
 

tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_queueprocessor.example.com-ASGLifecycle_event_pattern

@@ -0,0 +1 @@
+{"source":["aws.autoscaling"],"detail-type":["EC2 Instance-terminate Lifecycle Action"]}

tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_queueprocessor.example.com-RebalanceRecommendation_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Instance Rebalance Recommendation"]}

tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_queueprocessor.example.com-SpotInterruption_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Spot Instance Interruption Warning"]}

tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_nodes.nthsqsresources.example.com_policy

@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}

tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_nodes.queueprocessor.example.com_policy

@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}

tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_policy_masters.nthsqsresources.example.com_policy

@@ -0,0 +1,182 @@
+{
+  "Statement": [
+    {
+      "Action": [
+        "ec2:DescribeAccountAttributes",
+        "ec2:DescribeInstances",
+        "ec2:DescribeInternetGateways",
+        "ec2:DescribeRegions",
+        "ec2:DescribeRouteTables",
+        "ec2:DescribeSecurityGroups",
+        "ec2:DescribeSubnets",
+        "ec2:DescribeVolumes"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:CreateSecurityGroup",
+        "ec2:CreateTags",
+        "ec2:CreateVolume",
+        "ec2:DescribeVolumesModifications",
+        "ec2:ModifyInstanceAttribute",
+        "ec2:ModifyVolume"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:AttachVolume",
+        "ec2:AuthorizeSecurityGroupIngress",
+        "ec2:CreateRoute",
+        "ec2:DeleteRoute",
+        "ec2:DeleteSecurityGroup",
+        "ec2:DeleteVolume",
+        "ec2:DetachVolume",
+        "ec2:RevokeSecurityGroupIngress"
+      ],
+      "Condition": {
+        "StringEquals": {
+          "ec2:ResourceTag/KubernetesCluster": "nthsqsresources.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "autoscaling:DescribeAutoScalingGroups",
+        "autoscaling:DescribeLaunchConfigurations",
+        "autoscaling:DescribeTags",
+        "ec2:DescribeLaunchTemplateVersions"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "autoscaling:SetDesiredCapacity",
+        "autoscaling:TerminateInstanceInAutoScalingGroup",
+        "autoscaling:UpdateAutoScalingGroup"
+      ],
+      "Condition": {
+        "StringEquals": {
+          "autoscaling:ResourceTag/KubernetesCluster": "nthsqsresources.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "elasticloadbalancing:AddTags",
+        "elasticloadbalancing:AttachLoadBalancerToSubnets",
+        "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer",
+        "elasticloadbalancing:CreateLoadBalancer",
+        "elasticloadbalancing:CreateLoadBalancerPolicy",
+        "elasticloadbalancing:CreateLoadBalancerListeners",
+        "elasticloadbalancing:ConfigureHealthCheck",
+        "elasticloadbalancing:DeleteLoadBalancer",
+        "elasticloadbalancing:DeleteLoadBalancerListeners",
+        "elasticloadbalancing:DescribeLoadBalancers",
+        "elasticloadbalancing:DescribeLoadBalancerAttributes",
+        "elasticloadbalancing:DetachLoadBalancerFromSubnets",
+        "elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
+        "elasticloadbalancing:ModifyLoadBalancerAttributes",
+        "elasticloadbalancing:RegisterInstancesWithLoadBalancer",
+        "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:DescribeVpcs",
+        "elasticloadbalancing:AddTags",
+        "elasticloadbalancing:CreateListener",
+        "elasticloadbalancing:CreateTargetGroup",
+        "elasticloadbalancing:DeleteListener",
+        "elasticloadbalancing:DeleteTargetGroup",
+        "elasticloadbalancing:DeregisterTargets",
+        "elasticloadbalancing:DescribeListeners",
+        "elasticloadbalancing:DescribeLoadBalancerPolicies",
+        "elasticloadbalancing:DescribeTargetGroups",
+        "elasticloadbalancing:DescribeTargetHealth",
+        "elasticloadbalancing:ModifyListener",
+        "elasticloadbalancing:ModifyTargetGroup",
+        "elasticloadbalancing:RegisterTargets",
+        "elasticloadbalancing:SetLoadBalancerPoliciesOfListener"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "iam:ListServerCertificates",
+        "iam:GetServerCertificate"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "route53:ChangeResourceRecordSets",
+        "route53:ListResourceRecordSets",
+        "route53:GetHostedZone"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO"
+      ]
+    },
+    {
+      "Action": [
+        "route53:GetChange"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws:route53:::change/*"
+      ]
+    },
+    {
+      "Action": [
+        "route53:ListHostedZones"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "autoscaling:CompleteLifecycleAction",
+        "autoscaling:DescribeAutoScalingInstances",
+        "sqs:DeleteMessage",
+        "sqs:ReceiveMessage"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_policy_nodes.queueprocessor.example.com_policy

@@ -0,0 +1,15 @@
+{
+  "Statement": [
+    {
+      "Action": [
+        "ec2:DescribeInstances",
+        "ec2:DescribeRegions"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_master-us-test-1a.masters.nthsqsresources.example.com_user_data

@@ -0,0 +1,328 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/nodeup
+NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924
+NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/nodeup
+NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865
+
+export AWS_REGION=us-test-1
+
+
+
+
+sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true
+
+
+function ensure-install-dir() {
+  INSTALL_DIR="/opt/kops"
+  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+  if [[ -d /var/lib/toolbox ]]; then
+    INSTALL_DIR="/var/lib/toolbox/kops"
+  fi
+  mkdir -p ${INSTALL_DIR}/bin
+  mkdir -p ${INSTALL_DIR}/conf
+  cd ${INSTALL_DIR}
+}
+
+# Retry a download until we get it. args: name, sha, url1, url2...
+download-or-bust() {
+  local -r file="$1"
+  local -r hash="$2"
+  shift 2
+
+  urls=( $* )
+  while true; do
+    for url in "${urls[@]}"; do
+      commands=(
+        "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+        "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+      )
+      for cmd in "${commands[@]}"; do
+        echo "Attempting download with: ${cmd} {url}"
+        if ! (${cmd} "${url}"); then
+          echo "== Download failed with ${cmd} =="
+          continue
+        fi
+        if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
+          echo "== Hash validation of ${url} failed. Retrying. =="
+          rm -f "${file}"
+        else
+          if [[ -n "${hash}" ]]; then
+            echo "== Downloaded ${url} (SHA1 = ${hash}) =="
+          else
+            echo "== Downloaded ${url} =="
+          fi
+          return
+        fi
+      done
+    done
+
+    echo "All downloads failed; sleeping before retrying"
+    sleep 60
+  done
+}
+
+validate-hash() {
+  local -r file="$1"
+  local -r expected="$2"
+  local actual
+
+  actual=$(sha256sum ${file} | awk '{ print $1 }') || true
+  if [[ "${actual}" != "${expected}" ]]; then
+    echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} =="
+    return 1
+  fi
+}
+
+function split-commas() {
+  echo $1 | tr "," "\n"
+}
+
+function try-download-release() {
+  local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
+  if [[ -n "${NODEUP_HASH:-}" ]]; then
+    local -r nodeup_hash="${NODEUP_HASH}"
+  else
+  # TODO: Remove?
+    echo "Downloading sha256 (not found in env)"
+    download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}"
+    local -r nodeup_hash=$(cat nodeup.sha256)
+  fi
+
+  echo "Downloading nodeup (${nodeup_urls[@]})"
+  download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}"
+
+  chmod +x nodeup
+}
+
+function download-release() {
+  case "$(uname -m)" in
+  x86_64*|i?86_64*|amd64*)
+    NODEUP_URL="${NODEUP_URL_AMD64}"
+    NODEUP_HASH="${NODEUP_HASH_AMD64}"
+    ;;
+  aarch64*|arm64*)
+    NODEUP_URL="${NODEUP_URL_ARM64}"
+    NODEUP_HASH="${NODEUP_HASH_ARM64}"
+    ;;
+  *)
+    echo "Unsupported host arch: $(uname -m)" >&2
+    exit 1
+    ;;
+  esac
+
+  # In case of failure checking integrity of release, retry.
+  cd ${INSTALL_DIR}/bin
+  until try-download-release; do
+    sleep 15
+    echo "Couldn't download release. Retrying..."
+  done
+
+  echo "Running nodeup"
+  # We can't run in the foreground because of https://github.com/docker/docker/issues/23793
+  ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8  )
+}
+
+####################################################################################
+
+/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
+
+echo "== nodeup node config starting =="
+ensure-install-dir
+
+cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
+cloudConfig:
+  manageStorageClasses: true
+containerRuntime: containerd
+containerd:
+  configOverride: |
+    version = 2
+
+    [plugins]
+
+      [plugins."io.containerd.grpc.v1.cri"]
+
+        [plugins."io.containerd.grpc.v1.cri".containerd]
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+              runtime_type = "io.containerd.runc.v2"
+
+              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+                SystemdCgroup = true
+  logLevel: info
+  version: 1.4.4
+docker:
+  skipInstall: true
+encryptionConfig: null
+etcdClusters:
+  events:
+    version: 3.4.13
+  main:
+    version: 3.4.13
+kubeAPIServer:
+  allowPrivileged: true
+  anonymousAuth: false
+  apiAudiences:
+  - kubernetes.svc.default
+  apiServerCount: 1
+  authorizationMode: AlwaysAllow
+  bindAddress: 0.0.0.0
+  cloudProvider: aws
+  enableAdmissionPlugins:
+  - NamespaceLifecycle
+  - LimitRanger
+  - ServiceAccount
+  - PersistentVolumeLabel
+  - DefaultStorageClass
+  - DefaultTolerationSeconds
+  - MutatingAdmissionWebhook
+  - ValidatingAdmissionWebhook
+  - NodeRestriction
+  - ResourceQuota
+  etcdServers:
+  - http://127.0.0.1:4001
+  etcdServersOverrides:
+  - /events#http://127.0.0.1:4002
+  image: k8s.gcr.io/kube-apiserver:v1.20.0
+  kubeletPreferredAddressTypes:
+  - InternalIP
+  - Hostname
+  - ExternalIP
+  logLevel: 2
+  requestheaderAllowedNames:
+  - aggregator
+  requestheaderExtraHeaderPrefixes:
+  - X-Remote-Extra-
+  requestheaderGroupHeaders:
+  - X-Remote-Group
+  requestheaderUsernameHeaders:
+  - X-Remote-User
+  securePort: 443
+  serviceAccountIssuer: https://api.internal.nthsqsresources.example.com
+  serviceAccountJWKSURI: https://api.internal.nthsqsresources.example.com/openid/v1/jwks
+  serviceClusterIPRange: 100.64.0.0/13
+  storageBackend: etcd3
+kubeControllerManager:
+  allocateNodeCIDRs: true
+  attachDetachReconcileSyncPeriod: 1m0s
+  cloudProvider: aws
+  clusterCIDR: 100.96.0.0/11
+  clusterName: nthsqsresources.example.com
+  configureCloudRoutes: false
+  image: k8s.gcr.io/kube-controller-manager:v1.20.0
+  leaderElection:
+    leaderElect: true
+  logLevel: 2
+  useServiceAccountCredentials: true
+kubeProxy:
+  clusterCIDR: 100.96.0.0/11
+  cpuRequest: 100m
+  hostnameOverride: '@aws'
+  image: k8s.gcr.io/kube-proxy:v1.20.0
+  logLevel: 2
+kubeScheduler:
+  image: k8s.gcr.io/kube-scheduler:v1.20.0
+  leaderElection:
+    leaderElect: true
+  logLevel: 2
+kubelet:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginName: cni
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podManifestPath: /etc/kubernetes/manifests
+masterKubelet:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginName: cni
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podManifestPath: /etc/kubernetes/manifests
+  registerSchedulable: false
+
+__EOF_CLUSTER_SPEC
+
+cat > conf/ig_spec.yaml << '__EOF_IG_SPEC'
+{}
+
+__EOF_IG_SPEC
+
+cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
+Assets:
+  amd64:
+  - ff2422571c4c1e9696e367f5f25466b96fb6e501f28aed29f414b1524a52dea0@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubelet
+  - a5895007f331f08d2e082eb12458764949559f30bcc5beae26c38f3e2724262c@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubectl
+  - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
+  - 96641849cb78a0a119223a427dfdc1ade88412ef791a14193212c8c8e29d447b@https://github.com/containerd/containerd/releases/download/v1.4.4/cri-containerd-cni-1.4.4-linux-amd64.tar.gz
+  - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/protokube
+  - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/channels
+  arm64:
+  - 47ab6c4273fc3bb0cb8ec9517271d915890c5a6b0e54b2991e7a8fbbe77b06e4@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubelet
+  - 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubectl
+  - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
+  - 6e3f80e8451ecbe7b3559247721c3e226be6b228acaadee7e13683f80c20e81c@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.0.tgz
+  - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/protokube
+  - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/channels
+ClusterName: nthsqsresources.example.com
+ConfigBase: memfs://clusters.example.com/nthsqsresources.example.com
+InstanceGroupName: master-us-test-1a
+InstanceGroupRole: Master
+KubeletConfig:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginName: cni
+  nodeLabels:
+    kops.k8s.io/kops-controller-pki: ""
+    kubernetes.io/role: master
+    node-role.kubernetes.io/control-plane: ""
+    node-role.kubernetes.io/master: ""
+    node.kubernetes.io/exclude-from-external-load-balancers: ""
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podManifestPath: /etc/kubernetes/manifests
+  registerSchedulable: false
+channels:
+- memfs://clusters.example.com/nthsqsresources.example.com/addons/bootstrap-channel.yaml
+etcdManifests:
+- memfs://clusters.example.com/nthsqsresources.example.com/manifests/etcd/main.yaml
+- memfs://clusters.example.com/nthsqsresources.example.com/manifests/etcd/events.yaml
+staticManifests:
+- key: kube-apiserver-healthcheck
+  path: manifests/static/kube-apiserver-healthcheck.yaml
+
+__EOF_KUBE_ENV
+
+download-release
+echo "== nodeup node config done =="

tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_nodes.nthsqsresources.example.com_user_data

@@ -0,0 +1,232 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/nodeup
+NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924
+NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/nodeup
+NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865
+
+export AWS_REGION=us-test-1
+
+
+
+
+sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true
+
+
+function ensure-install-dir() {
+  INSTALL_DIR="/opt/kops"
+  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+  if [[ -d /var/lib/toolbox ]]; then
+    INSTALL_DIR="/var/lib/toolbox/kops"
+  fi
+  mkdir -p ${INSTALL_DIR}/bin
+  mkdir -p ${INSTALL_DIR}/conf
+  cd ${INSTALL_DIR}
+}
+
+# Retry a download until we get it. args: name, sha, url1, url2...
+download-or-bust() {
+  local -r file="$1"
+  local -r hash="$2"
+  shift 2
+
+  urls=( $* )
+  while true; do
+    for url in "${urls[@]}"; do
+      commands=(
+        "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+        "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+      )
+      for cmd in "${commands[@]}"; do
+        echo "Attempting download with: ${cmd} {url}"
+        if ! (${cmd} "${url}"); then
+          echo "== Download failed with ${cmd} =="
+          continue
+        fi
+        if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
+          echo "== Hash validation of ${url} failed. Retrying. =="
+          rm -f "${file}"
+        else
+          if [[ -n "${hash}" ]]; then
+            echo "== Downloaded ${url} (SHA1 = ${hash}) =="
+          else
+            echo "== Downloaded ${url} =="
+          fi
+          return
+        fi
+      done
+    done
+
+    echo "All downloads failed; sleeping before retrying"
+    sleep 60
+  done
+}
+
+validate-hash() {
+  local -r file="$1"
+  local -r expected="$2"
+  local actual
+
+  actual=$(sha256sum ${file} | awk '{ print $1 }') || true
+  if [[ "${actual}" != "${expected}" ]]; then
+    echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} =="
+    return 1
+  fi
+}
+
+function split-commas() {
+  echo $1 | tr "," "\n"
+}
+
+function try-download-release() {
+  local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
+  if [[ -n "${NODEUP_HASH:-}" ]]; then
+    local -r nodeup_hash="${NODEUP_HASH}"
+  else
+  # TODO: Remove?
+    echo "Downloading sha256 (not found in env)"
+    download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}"
+    local -r nodeup_hash=$(cat nodeup.sha256)
+  fi
+
+  echo "Downloading nodeup (${nodeup_urls[@]})"
+  download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}"
+
+  chmod +x nodeup
+}
+
+function download-release() {
+  case "$(uname -m)" in
+  x86_64*|i?86_64*|amd64*)
+    NODEUP_URL="${NODEUP_URL_AMD64}"
+    NODEUP_HASH="${NODEUP_HASH_AMD64}"
+    ;;
+  aarch64*|arm64*)
+    NODEUP_URL="${NODEUP_URL_ARM64}"
+    NODEUP_HASH="${NODEUP_HASH_ARM64}"
+    ;;
+  *)
+    echo "Unsupported host arch: $(uname -m)" >&2
+    exit 1
+    ;;
+  esac
+
+  # In case of failure checking integrity of release, retry.
+  cd ${INSTALL_DIR}/bin
+  until try-download-release; do
+    sleep 15
+    echo "Couldn't download release. Retrying..."
+  done
+
+  echo "Running nodeup"
+  # We can't run in the foreground because of https://github.com/docker/docker/issues/23793
+  ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8  )
+}
+
+####################################################################################
+
+/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
+
+echo "== nodeup node config starting =="
+ensure-install-dir
+
+cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
+cloudConfig:
+  manageStorageClasses: true
+containerRuntime: containerd
+containerd:
+  configOverride: |
+    version = 2
+
+    [plugins]
+
+      [plugins."io.containerd.grpc.v1.cri"]
+
+        [plugins."io.containerd.grpc.v1.cri".containerd]
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+              runtime_type = "io.containerd.runc.v2"
+
+              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+                SystemdCgroup = true
+  logLevel: info
+  version: 1.4.4
+docker:
+  skipInstall: true
+kubeProxy:
+  clusterCIDR: 100.96.0.0/11
+  cpuRequest: 100m
+  hostnameOverride: '@aws'
+  image: k8s.gcr.io/kube-proxy:v1.20.0
+  logLevel: 2
+kubelet:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginName: cni
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podManifestPath: /etc/kubernetes/manifests
+
+__EOF_CLUSTER_SPEC
+
+cat > conf/ig_spec.yaml << '__EOF_IG_SPEC'
+{}
+
+__EOF_IG_SPEC
+
+cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
+Assets:
+  amd64:
+  - ff2422571c4c1e9696e367f5f25466b96fb6e501f28aed29f414b1524a52dea0@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubelet
+  - a5895007f331f08d2e082eb12458764949559f30bcc5beae26c38f3e2724262c@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubectl
+  - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
+  - 96641849cb78a0a119223a427dfdc1ade88412ef791a14193212c8c8e29d447b@https://github.com/containerd/containerd/releases/download/v1.4.4/cri-containerd-cni-1.4.4-linux-amd64.tar.gz
+  arm64:
+  - 47ab6c4273fc3bb0cb8ec9517271d915890c5a6b0e54b2991e7a8fbbe77b06e4@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubelet
+  - 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubectl
+  - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
+  - 6e3f80e8451ecbe7b3559247721c3e226be6b228acaadee7e13683f80c20e81c@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.0.tgz
+ClusterName: nthsqsresources.example.com
+ConfigBase: memfs://clusters.example.com/nthsqsresources.example.com
+InstanceGroupName: nodes
+InstanceGroupRole: Node
+KubeletConfig:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginName: cni
+  nodeLabels:
+    kubernetes.io/role: node
+    node-role.kubernetes.io/node: ""
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podManifestPath: /etc/kubernetes/manifests
+channels:
+- memfs://clusters.example.com/nthsqsresources.example.com/addons/bootstrap-channel.yaml
+
+__EOF_KUBE_ENV
+
+download-release
+echo "== nodeup node config done =="

tests/integration/update_cluster/nth_sqs_resources/data/aws_sqs_queue_nthsqsresources-example-com-nth_policy

@@ -0,0 +1,13 @@
+{
+		"Version": "2012-10-17",
+		"Statement": [{                     
+			"Effect": "Allow",
+			"Principal": {
+				"Service": ["events.amazonaws.com", "sqs.amazonaws.com"]
+			},
+			"Action": "sqs:SendMessage",
+			"Resource": [
+				"arn:aws:sqs:us-test-1:123456789012:nthsqsresources-example-com-nth"
+			]
+		}]
+	}

tests/integration/update_cluster/nth_sqs_resources/id_rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==

tests/integration/update_cluster/nth_sqs_resources/in-v1alpha2.yaml

@@ -2,13 +2,13 @@ apiVersion: kops.k8s.io/v1alpha2
 kind: Cluster
 metadata:
   creationTimestamp: "2016-12-10T22:42:27Z"
-  name: queueprocessor.example.com
+  name: nthsqsresources.example.com
 spec:
   kubernetesApiAccess:
     - 0.0.0.0/0
   channel: stable
   cloudProvider: aws
-  configBase: memfs://clusters.example.com/queueprocessor.example.com
+  configBase: memfs://clusters.example.com/nthsqsresources.example.com
   etcdClusters:
     - etcdMembers:
         - instanceGroup: master-us-test-1a
@@ -22,8 +22,8 @@ spec:
   kubelet:
     anonymousAuth: false
   kubernetesVersion: v1.20.0
-  masterInternalName: api.internal.queueprocessor.example.com
+  masterInternalName: api.internal.nthsqsresources.example.com
-  masterPublicName: api.queueprocessor.example.com
+  masterPublicName: api.nthsqsresources.example.com
   networkCIDR: 172.20.0.0/16
   networking:
     cni: {}
@@ -50,7 +50,7 @@ metadata:
   creationTimestamp: "2016-12-10T22:42:28Z"
   name: nodes
   labels:
-    kops.k8s.io/cluster: queueprocessor.example.com
+    kops.k8s.io/cluster: nthsqsresources.example.com
 spec:
   associatePublicIp: true
   image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21
@@ -69,7 +69,7 @@ metadata:
   creationTimestamp: "2016-12-10T22:42:28Z"
   name: master-us-test-1a
   labels:
-    kops.k8s.io/cluster: queueprocessor.example.com
+    kops.k8s.io/cluster: nthsqsresources.example.com
 spec:
   associatePublicIp: true
   image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21

tests/integration/update_cluster/nth_sqs_resources/kubernetes.tf

@@ -0,0 +1,732 @@
+locals {
+  cluster_name                 = "nthsqsresources.example.com"
+  master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-nthsqsresources-example-com.id]
+  master_security_group_ids    = [aws_security_group.masters-nthsqsresources-example-com.id]
+  masters_role_arn             = aws_iam_role.masters-nthsqsresources-example-com.arn
+  masters_role_name            = aws_iam_role.masters-nthsqsresources-example-com.name
+  node_autoscaling_group_ids   = [aws_autoscaling_group.nodes-nthsqsresources-example-com.id]
+  node_security_group_ids      = [aws_security_group.nodes-nthsqsresources-example-com.id]
+  node_subnet_ids              = [aws_subnet.us-test-1a-nthsqsresources-example-com.id]
+  nodes_role_arn               = aws_iam_role.nodes-nthsqsresources-example-com.arn
+  nodes_role_name              = aws_iam_role.nodes-nthsqsresources-example-com.name
+  region                       = "us-test-1"
+  route_table_public_id        = aws_route_table.nthsqsresources-example-com.id
+  subnet_us-test-1a_id         = aws_subnet.us-test-1a-nthsqsresources-example-com.id
+  vpc_cidr_block               = aws_vpc.nthsqsresources-example-com.cidr_block
+  vpc_id                       = aws_vpc.nthsqsresources-example-com.id
+}
+
+output "cluster_name" {
+  value = "nthsqsresources.example.com"
+}
+
+output "master_autoscaling_group_ids" {
+  value = [aws_autoscaling_group.master-us-test-1a-masters-nthsqsresources-example-com.id]
+}
+
+output "master_security_group_ids" {
+  value = [aws_security_group.masters-nthsqsresources-example-com.id]
+}
+
+output "masters_role_arn" {
+  value = aws_iam_role.masters-nthsqsresources-example-com.arn
+}
+
+output "masters_role_name" {
+  value = aws_iam_role.masters-nthsqsresources-example-com.name
+}
+
+output "node_autoscaling_group_ids" {
+  value = [aws_autoscaling_group.nodes-nthsqsresources-example-com.id]
+}
+
+output "node_security_group_ids" {
+  value = [aws_security_group.nodes-nthsqsresources-example-com.id]
+}
+
+output "node_subnet_ids" {
+  value = [aws_subnet.us-test-1a-nthsqsresources-example-com.id]
+}
+
+output "nodes_role_arn" {
+  value = aws_iam_role.nodes-nthsqsresources-example-com.arn
+}
+
+output "nodes_role_name" {
+  value = aws_iam_role.nodes-nthsqsresources-example-com.name
+}
+
+output "region" {
+  value = "us-test-1"
+}
+
+output "route_table_public_id" {
+  value = aws_route_table.nthsqsresources-example-com.id
+}
+
+output "subnet_us-test-1a_id" {
+  value = aws_subnet.us-test-1a-nthsqsresources-example-com.id
+}
+
+output "vpc_cidr_block" {
+  value = aws_vpc.nthsqsresources-example-com.cidr_block
+}
+
+output "vpc_id" {
+  value = aws_vpc.nthsqsresources-example-com.id
+}
+
+provider "aws" {
+  region = "us-test-1"
+}
+
+resource "aws_autoscaling_group" "master-us-test-1a-masters-nthsqsresources-example-com" {
+  enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
+  launch_template {
+    id      = aws_launch_template.master-us-test-1a-masters-nthsqsresources-example-com.id
+    version = aws_launch_template.master-us-test-1a-masters-nthsqsresources-example-com.latest_version
+  }
+  max_size            = 1
+  metrics_granularity = "1Minute"
+  min_size            = 1
+  name                = "master-us-test-1a.masters.nthsqsresources.example.com"
+  tag {
+    key                 = "KubernetesCluster"
+    propagate_at_launch = true
+    value               = "nthsqsresources.example.com"
+  }
+  tag {
+    key                 = "Name"
+    propagate_at_launch = true
+    value               = "master-us-test-1a.masters.nthsqsresources.example.com"
+  }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = "true"
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"
+    propagate_at_launch = true
+    value               = ""
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"
+    propagate_at_launch = true
+    value               = "master"
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"
+    propagate_at_launch = true
+    value               = ""
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"
+    propagate_at_launch = true
+    value               = ""
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers"
+    propagate_at_launch = true
+    value               = ""
+  }
+  tag {
+    key                 = "k8s.io/role/master"
+    propagate_at_launch = true
+    value               = "1"
+  }
+  tag {
+    key                 = "kops.k8s.io/instancegroup"
+    propagate_at_launch = true
+    value               = "master-us-test-1a"
+  }
+  tag {
+    key                 = "kubernetes.io/cluster/nthsqsresources.example.com"
+    propagate_at_launch = true
+    value               = "owned"
+  }
+  vpc_zone_identifier = [aws_subnet.us-test-1a-nthsqsresources-example-com.id]
+}
+
+resource "aws_autoscaling_group" "nodes-nthsqsresources-example-com" {
+  enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
+  launch_template {
+    id      = aws_launch_template.nodes-nthsqsresources-example-com.id
+    version = aws_launch_template.nodes-nthsqsresources-example-com.latest_version
+  }
+  max_size            = 2
+  metrics_granularity = "1Minute"
+  min_size            = 2
+  name                = "nodes.nthsqsresources.example.com"
+  tag {
+    key                 = "KubernetesCluster"
+    propagate_at_launch = true
+    value               = "nthsqsresources.example.com"
+  }
+  tag {
+    key                 = "Name"
+    propagate_at_launch = true
+    value               = "nodes.nthsqsresources.example.com"
+  }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = "true"
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"
+    propagate_at_launch = true
+    value               = "node"
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
+    propagate_at_launch = true
+    value               = ""
+  }
+  tag {
+    key                 = "k8s.io/role/node"
+    propagate_at_launch = true
+    value               = "1"
+  }
+  tag {
+    key                 = "kops.k8s.io/instancegroup"
+    propagate_at_launch = true
+    value               = "nodes"
+  }
+  tag {
+    key                 = "kubernetes.io/cluster/nthsqsresources.example.com"
+    propagate_at_launch = true
+    value               = "owned"
+  }
+  vpc_zone_identifier = [aws_subnet.us-test-1a-nthsqsresources-example-com.id]
+}
+
+resource "aws_autoscaling_lifecycle_hook" "master-us-test-1a-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-nthsqsresources-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "master-us-test-1a-NTHLifecycleHook"
+}
+
+resource "aws_autoscaling_lifecycle_hook" "nodes-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.nodes-nthsqsresources-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "nodes-NTHLifecycleHook"
+}
+
+resource "aws_cloudwatch_event_rule" "nthsqsresources-example-com-ASGLifecycle" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_nthsqsresources.example.com-ASGLifecycle_event_pattern")
+  name          = "nthsqsresources.example.com-ASGLifecycle"
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nthsqsresources.example.com-ASGLifecycle"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "nthsqsresources-example-com-RebalanceRecommendation" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_nthsqsresources.example.com-RebalanceRecommendation_event_pattern")
+  name          = "nthsqsresources.example.com-RebalanceRecommendation"
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nthsqsresources.example.com-RebalanceRecommendation"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "nthsqsresources-example-com-SpotInterruption" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_nthsqsresources.example.com-SpotInterruption_event_pattern")
+  name          = "nthsqsresources.example.com-SpotInterruption"
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nthsqsresources.example.com-SpotInterruption"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_target" "nthsqsresources-example-com-ASGLifecycle-Target" {
+  arn  = "arn:aws:sqs:us-test-1:123456789012:nthsqsresources-example-com-nth"
+  rule = aws_cloudwatch_event_rule.nthsqsresources-example-com-ASGLifecycle.id
+}
+
+resource "aws_cloudwatch_event_target" "nthsqsresources-example-com-RebalanceRecommendation-Target" {
+  arn  = "arn:aws:sqs:us-test-1:123456789012:nthsqsresources-example-com-nth"
+  rule = aws_cloudwatch_event_rule.nthsqsresources-example-com-RebalanceRecommendation.id
+}
+
+resource "aws_cloudwatch_event_target" "nthsqsresources-example-com-SpotInterruption-Target" {
+  arn  = "arn:aws:sqs:us-test-1:123456789012:nthsqsresources-example-com-nth"
+  rule = aws_cloudwatch_event_rule.nthsqsresources-example-com-SpotInterruption.id
+}
+
+resource "aws_ebs_volume" "us-test-1a-etcd-events-nthsqsresources-example-com" {
+  availability_zone = "us-test-1a"
+  encrypted         = false
+  iops              = 3000
+  size              = 20
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "us-test-1a.etcd-events.nthsqsresources.example.com"
+    "k8s.io/etcd/events"                                = "us-test-1a/us-test-1a"
+    "k8s.io/role/master"                                = "1"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+  throughput = 125
+  type       = "gp3"
+}
+
+resource "aws_ebs_volume" "us-test-1a-etcd-main-nthsqsresources-example-com" {
+  availability_zone = "us-test-1a"
+  encrypted         = false
+  iops              = 3000
+  size              = 20
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "us-test-1a.etcd-main.nthsqsresources.example.com"
+    "k8s.io/etcd/main"                                  = "us-test-1a/us-test-1a"
+    "k8s.io/role/master"                                = "1"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+  throughput = 125
+  type       = "gp3"
+}
+
+resource "aws_iam_instance_profile" "masters-nthsqsresources-example-com" {
+  name = "masters.nthsqsresources.example.com"
+  role = aws_iam_role.masters-nthsqsresources-example-com.name
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "masters.nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+resource "aws_iam_instance_profile" "nodes-nthsqsresources-example-com" {
+  name = "nodes.nthsqsresources.example.com"
+  role = aws_iam_role.nodes-nthsqsresources-example-com.name
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nodes.nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+resource "aws_iam_role_policy" "masters-nthsqsresources-example-com" {
+  name   = "masters.nthsqsresources.example.com"
+  policy = file("${path.module}/data/aws_iam_role_policy_masters.nthsqsresources.example.com_policy")
+  role   = aws_iam_role.masters-nthsqsresources-example-com.name
+}
+
+resource "aws_iam_role_policy" "nodes-nthsqsresources-example-com" {
+  name   = "nodes.nthsqsresources.example.com"
+  policy = file("${path.module}/data/aws_iam_role_policy_nodes.nthsqsresources.example.com_policy")
+  role   = aws_iam_role.nodes-nthsqsresources-example-com.name
+}
+
+resource "aws_iam_role" "masters-nthsqsresources-example-com" {
+  assume_role_policy = file("${path.module}/data/aws_iam_role_masters.nthsqsresources.example.com_policy")
+  name               = "masters.nthsqsresources.example.com"
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "masters.nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+resource "aws_iam_role" "nodes-nthsqsresources-example-com" {
+  assume_role_policy = file("${path.module}/data/aws_iam_role_nodes.nthsqsresources.example.com_policy")
+  name               = "nodes.nthsqsresources.example.com"
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nodes.nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+resource "aws_internet_gateway" "nthsqsresources-example-com" {
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+  vpc_id = aws_vpc.nthsqsresources-example-com.id
+}
+
+resource "aws_key_pair" "kubernetes-nthsqsresources-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" {
+  key_name   = "kubernetes.nthsqsresources.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57"
+  public_key = file("${path.module}/data/aws_key_pair_kubernetes.nthsqsresources.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key")
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+resource "aws_launch_template" "master-us-test-1a-masters-nthsqsresources-example-com" {
+  block_device_mappings {
+    device_name = "/dev/xvda"
+    ebs {
+      delete_on_termination = true
+      encrypted             = true
+      iops                  = 3000
+      throughput            = 125
+      volume_size           = 64
+      volume_type           = "gp3"
+    }
+  }
+  block_device_mappings {
+    device_name  = "/dev/sdc"
+    virtual_name = "ephemeral0"
+  }
+  iam_instance_profile {
+    name = aws_iam_instance_profile.masters-nthsqsresources-example-com.id
+  }
+  image_id      = "ami-12345678"
+  instance_type = "m3.medium"
+  key_name      = aws_key_pair.kubernetes-nthsqsresources-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id
+  lifecycle {
+    create_before_destroy = true
+  }
+  metadata_options {
+    http_endpoint               = "enabled"
+    http_put_response_hop_limit = 1
+    http_tokens                 = "optional"
+  }
+  name = "master-us-test-1a.masters.nthsqsresources.example.com"
+  network_interfaces {
+    associate_public_ip_address = true
+    delete_on_termination       = true
+    security_groups             = [aws_security_group.masters-nthsqsresources-example-com.id]
+  }
+  tag_specifications {
+    resource_type = "instance"
+    tags = {
+      "KubernetesCluster"                                                                                     = "nthsqsresources.example.com"
+      "Name"                                                                                                  = "master-us-test-1a.masters.nthsqsresources.example.com"
+      "aws-node-termination-handler/managed"                                                                  = "true"
+      "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
+      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
+      "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
+      "k8s.io/role/master"                                                                                    = "1"
+      "kops.k8s.io/instancegroup"                                                                             = "master-us-test-1a"
+      "kubernetes.io/cluster/nthsqsresources.example.com"                                                     = "owned"
+    }
+  }
+  tag_specifications {
+    resource_type = "volume"
+    tags = {
+      "KubernetesCluster"                                                                                     = "nthsqsresources.example.com"
+      "Name"                                                                                                  = "master-us-test-1a.masters.nthsqsresources.example.com"
+      "aws-node-termination-handler/managed"                                                                  = "true"
+      "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
+      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
+      "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
+      "k8s.io/role/master"                                                                                    = "1"
+      "kops.k8s.io/instancegroup"                                                                             = "master-us-test-1a"
+      "kubernetes.io/cluster/nthsqsresources.example.com"                                                     = "owned"
+    }
+  }
+  tags = {
+    "KubernetesCluster"                                                                                     = "nthsqsresources.example.com"
+    "Name"                                                                                                  = "master-us-test-1a.masters.nthsqsresources.example.com"
+    "aws-node-termination-handler/managed"                                                                  = "true"
+    "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
+    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
+    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
+    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
+    "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
+    "k8s.io/role/master"                                                                                    = "1"
+    "kops.k8s.io/instancegroup"                                                                             = "master-us-test-1a"
+    "kubernetes.io/cluster/nthsqsresources.example.com"                                                     = "owned"
+  }
+  user_data = filebase64("${path.module}/data/aws_launch_template_master-us-test-1a.masters.nthsqsresources.example.com_user_data")
+}
+
+resource "aws_launch_template" "nodes-nthsqsresources-example-com" {
+  block_device_mappings {
+    device_name = "/dev/xvda"
+    ebs {
+      delete_on_termination = true
+      encrypted             = true
+      iops                  = 3000
+      throughput            = 125
+      volume_size           = 128
+      volume_type           = "gp3"
+    }
+  }
+  iam_instance_profile {
+    name = aws_iam_instance_profile.nodes-nthsqsresources-example-com.id
+  }
+  image_id      = "ami-12345678"
+  instance_type = "t2.medium"
+  key_name      = aws_key_pair.kubernetes-nthsqsresources-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id
+  lifecycle {
+    create_before_destroy = true
+  }
+  metadata_options {
+    http_endpoint               = "enabled"
+    http_put_response_hop_limit = 1
+    http_tokens                 = "optional"
+  }
+  name = "nodes.nthsqsresources.example.com"
+  network_interfaces {
+    associate_public_ip_address = true
+    delete_on_termination       = true
+    security_groups             = [aws_security_group.nodes-nthsqsresources-example-com.id]
+  }
+  tag_specifications {
+    resource_type = "instance"
+    tags = {
+      "KubernetesCluster"                                                          = "nthsqsresources.example.com"
+      "Name"                                                                       = "nodes.nthsqsresources.example.com"
+      "aws-node-termination-handler/managed"                                       = "true"
+      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
+      "k8s.io/role/node"                                                           = "1"
+      "kops.k8s.io/instancegroup"                                                  = "nodes"
+      "kubernetes.io/cluster/nthsqsresources.example.com"                          = "owned"
+    }
+  }
+  tag_specifications {
+    resource_type = "volume"
+    tags = {
+      "KubernetesCluster"                                                          = "nthsqsresources.example.com"
+      "Name"                                                                       = "nodes.nthsqsresources.example.com"
+      "aws-node-termination-handler/managed"                                       = "true"
+      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
+      "k8s.io/role/node"                                                           = "1"
+      "kops.k8s.io/instancegroup"                                                  = "nodes"
+      "kubernetes.io/cluster/nthsqsresources.example.com"                          = "owned"
+    }
+  }
+  tags = {
+    "KubernetesCluster"                                                          = "nthsqsresources.example.com"
+    "Name"                                                                       = "nodes.nthsqsresources.example.com"
+    "aws-node-termination-handler/managed"                                       = "true"
+    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
+    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
+    "k8s.io/role/node"                                                           = "1"
+    "kops.k8s.io/instancegroup"                                                  = "nodes"
+    "kubernetes.io/cluster/nthsqsresources.example.com"                          = "owned"
+  }
+  user_data = filebase64("${path.module}/data/aws_launch_template_nodes.nthsqsresources.example.com_user_data")
+}
+
+resource "aws_route_table_association" "us-test-1a-nthsqsresources-example-com" {
+  route_table_id = aws_route_table.nthsqsresources-example-com.id
+  subnet_id      = aws_subnet.us-test-1a-nthsqsresources-example-com.id
+}
+
+resource "aws_route_table" "nthsqsresources-example-com" {
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+    "kubernetes.io/kops/role"                           = "public"
+  }
+  vpc_id = aws_vpc.nthsqsresources-example-com.id
+}
+
+resource "aws_route" "route-0-0-0-0--0" {
+  destination_cidr_block = "0.0.0.0/0"
+  gateway_id             = aws_internet_gateway.nthsqsresources-example-com.id
+  route_table_id         = aws_route_table.nthsqsresources-example-com.id
+}
+
+resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-masters-nthsqsresources-example-com" {
+  cidr_blocks       = ["0.0.0.0/0"]
+  from_port         = 22
+  protocol          = "tcp"
+  security_group_id = aws_security_group.masters-nthsqsresources-example-com.id
+  to_port           = 22
+  type              = "ingress"
+}
+
+resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-nodes-nthsqsresources-example-com" {
+  cidr_blocks       = ["0.0.0.0/0"]
+  from_port         = 22
+  protocol          = "tcp"
+  security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id
+  to_port           = 22
+  type              = "ingress"
+}
+
+resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-masters-nthsqsresources-example-com" {
+  cidr_blocks       = ["0.0.0.0/0"]
+  from_port         = 443
+  protocol          = "tcp"
+  security_group_id = aws_security_group.masters-nthsqsresources-example-com.id
+  to_port           = 443
+  type              = "ingress"
+}
+
+resource "aws_security_group_rule" "from-masters-nthsqsresources-example-com-egress-all-0to0-0-0-0-0--0" {
+  cidr_blocks       = ["0.0.0.0/0"]
+  from_port         = 0
+  protocol          = "-1"
+  security_group_id = aws_security_group.masters-nthsqsresources-example-com.id
+  to_port           = 0
+  type              = "egress"
+}
+
+resource "aws_security_group_rule" "from-masters-nthsqsresources-example-com-ingress-all-0to0-masters-nthsqsresources-example-com" {
+  from_port                = 0
+  protocol                 = "-1"
+  security_group_id        = aws_security_group.masters-nthsqsresources-example-com.id
+  source_security_group_id = aws_security_group.masters-nthsqsresources-example-com.id
+  to_port                  = 0
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "from-masters-nthsqsresources-example-com-ingress-all-0to0-nodes-nthsqsresources-example-com" {
+  from_port                = 0
+  protocol                 = "-1"
+  security_group_id        = aws_security_group.nodes-nthsqsresources-example-com.id
+  source_security_group_id = aws_security_group.masters-nthsqsresources-example-com.id
+  to_port                  = 0
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "from-nodes-nthsqsresources-example-com-egress-all-0to0-0-0-0-0--0" {
+  cidr_blocks       = ["0.0.0.0/0"]
+  from_port         = 0
+  protocol          = "-1"
+  security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id
+  to_port           = 0
+  type              = "egress"
+}
+
+resource "aws_security_group_rule" "from-nodes-nthsqsresources-example-com-ingress-all-0to0-nodes-nthsqsresources-example-com" {
+  from_port                = 0
+  protocol                 = "-1"
+  security_group_id        = aws_security_group.nodes-nthsqsresources-example-com.id
+  source_security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id
+  to_port                  = 0
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "from-nodes-nthsqsresources-example-com-ingress-tcp-1to2379-masters-nthsqsresources-example-com" {
+  from_port                = 1
+  protocol                 = "tcp"
+  security_group_id        = aws_security_group.masters-nthsqsresources-example-com.id
+  source_security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id
+  to_port                  = 2379
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "from-nodes-nthsqsresources-example-com-ingress-tcp-2382to4000-masters-nthsqsresources-example-com" {
+  from_port                = 2382
+  protocol                 = "tcp"
+  security_group_id        = aws_security_group.masters-nthsqsresources-example-com.id
+  source_security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id
+  to_port                  = 4000
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "from-nodes-nthsqsresources-example-com-ingress-tcp-4003to65535-masters-nthsqsresources-example-com" {
+  from_port                = 4003
+  protocol                 = "tcp"
+  security_group_id        = aws_security_group.masters-nthsqsresources-example-com.id
+  source_security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id
+  to_port                  = 65535
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "from-nodes-nthsqsresources-example-com-ingress-udp-1to65535-masters-nthsqsresources-example-com" {
+  from_port                = 1
+  protocol                 = "udp"
+  security_group_id        = aws_security_group.masters-nthsqsresources-example-com.id
+  source_security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id
+  to_port                  = 65535
+  type                     = "ingress"
+}
+
+resource "aws_security_group" "masters-nthsqsresources-example-com" {
+  description = "Security group for masters"
+  name        = "masters.nthsqsresources.example.com"
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "masters.nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+  vpc_id = aws_vpc.nthsqsresources-example-com.id
+}
+
+resource "aws_security_group" "nodes-nthsqsresources-example-com" {
+  description = "Security group for nodes"
+  name        = "nodes.nthsqsresources.example.com"
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nodes.nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+  vpc_id = aws_vpc.nthsqsresources-example-com.id
+}
+
+resource "aws_sqs_queue" "nthsqsresources-example-com-nth" {
+  message_retention_seconds = 300
+  name                      = "nthsqsresources-example-com-nth"
+  policy                    = file("${path.module}/data/aws_sqs_queue_nthsqsresources-example-com-nth_policy")
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nthsqsresources-example-com-nth"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+resource "aws_subnet" "us-test-1a-nthsqsresources-example-com" {
+  availability_zone = "us-test-1a"
+  cidr_block        = "172.20.32.0/19"
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "us-test-1a.nthsqsresources.example.com"
+    "SubnetType"                                        = "Public"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+    "kubernetes.io/role/elb"                            = "1"
+  }
+  vpc_id = aws_vpc.nthsqsresources-example-com.id
+}
+
+resource "aws_vpc_dhcp_options_association" "nthsqsresources-example-com" {
+  dhcp_options_id = aws_vpc_dhcp_options.nthsqsresources-example-com.id
+  vpc_id          = aws_vpc.nthsqsresources-example-com.id
+}
+
+resource "aws_vpc_dhcp_options" "nthsqsresources-example-com" {
+  domain_name         = "us-test-1.compute.internal"
+  domain_name_servers = ["AmazonProvidedDNS"]
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+resource "aws_vpc" "nthsqsresources-example-com" {
+  cidr_block           = "172.20.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+terraform {
+  required_version = ">= 0.12.26"
+  required_providers {
+    aws = {
+      "source"  = "hashicorp/aws"
+      "version" = ">= 3.34.0"
+    }
+  }
+}

upup/pkg/fi/cloudup/awstasks/autoscalinggroup_lifecyclehook.go

@@ -37,8 +37,6 @@ type AutoscalingLifecycleHook struct {
 	DefaultResult       *string
 	HeartbeatTimeout    *int64
 	LifecycleTransition *string
-
-	Tags map[string]string
 }
 
 var _ fi.CompareWithID = &AutoscalingLifecycleHook{}
@@ -68,10 +66,13 @@ func (h *AutoscalingLifecycleHook) Find(c *fi.Context) (*AutoscalingLifecycleHoo
 
 	hook := response.LifecycleHooks[0]
 	actual := &AutoscalingLifecycleHook{
-		ID:               hook.AutoScalingGroupName,
+		ID:                  hook.LifecycleHookName,
-		Name:             h.Name,
+		Name:                hook.LifecycleHookName,
-		Lifecycle:        h.Lifecycle,
+		Lifecycle:           h.Lifecycle,
-		AutoscalingGroup: h.AutoscalingGroup,
+		AutoscalingGroup:    h.AutoscalingGroup,
+		DefaultResult:       hook.DefaultResult,
+		HeartbeatTimeout:    hook.HeartbeatTimeout,
+		LifecycleTransition: hook.LifecycleTransition,
 	}
 
 	return actual, nil

upup/pkg/fi/cloudup/awstasks/eventbridgerule.go

@@ -36,7 +36,7 @@ type EventBridgeRule struct {
 	Lifecycle *fi.Lifecycle
 
 	EventPattern *string
-	TargetArn    *string
+	TargetArn    *string // required for cloudformation rendering
 
 	Tags map[string]string
 }
@@ -69,11 +69,19 @@ func (eb *EventBridgeRule) Find(c *fi.Context) (*EventBridgeRule, error) {
 	}
 
 	rule := response.Rules[0]
+
+	tagResponse, err := cloud.EventBridge().ListTagsForResource(&eventbridge.ListTagsForResourceInput{ResourceARN: rule.Arn})
+	if err != nil {
+		return nil, fmt.Errorf("error listing tags for EventBridge rule: %v", err)
+	}
+
 	actual := &EventBridgeRule{
 		ID:           eb.ID,
 		Name:         eb.Name,
 		Lifecycle:    eb.Lifecycle,
 		EventPattern: rule.EventPattern,
+		TargetArn:    eb.TargetArn,
+		Tags:         mapEventBridgeTagsToMap(tagResponse.Tags),
 	}
 	return actual, nil
 }

upup/pkg/fi/cloudup/awstasks/sqs.go

@@ -56,35 +56,48 @@ func (q *SQS) Find(c *fi.Context) (*SQS, error) {
 		return nil, nil
 	}
 
-	request := &sqs.ListQueuesInput{
+	response, err := cloud.SQS().ListQueues(&sqs.ListQueuesInput{
 		MaxResults:      aws.Int64(2),
 		QueueNamePrefix: q.Name,
-	}
+	})
-	response, err := cloud.SQS().ListQueues(request)
 	if err != nil {
 		return nil, fmt.Errorf("error listing SQS queues: %v", err)
 	}
 	if response == nil || len(response.QueueUrls) == 0 {
 		return nil, nil
 	}
-
 	if len(response.QueueUrls) != 1 {
 		return nil, fmt.Errorf("found multiple SQS queues matching queue name")
 	}
+	url := response.QueueUrls[0]
+
+	attributes, err := cloud.SQS().GetQueueAttributes(&sqs.GetQueueAttributesInput{
+		AttributeNames: []*string{s("MessageRetentionPeriod"), s("Policy")},
+		QueueUrl:       url,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error getting SQS queue attributes: %v", err)
+	}
+	policy := fi.NewStringResource(*attributes.Attributes["Policy"])
+	period, err := strconv.Atoi(*attributes.Attributes["MessageRetentionPeriod"])
+	if err != nil {
+		return nil, fmt.Errorf("error coverting MessageRetentionPeriod to int: %v", err)
+	}
 
 	tags, err := cloud.SQS().ListQueueTags(&sqs.ListQueueTagsInput{
-		QueueUrl: q.URL,
+		QueueUrl: url,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("error listing SQS queue tags: %v", err)
 	}
 
 	actual := &SQS{
-		Name:      q.Name,
+		Name:                   q.Name,
-		URL:       response.QueueUrls[0],
+		URL:                    url,
-		Tags:      intersectSQSTags(tags.Tags, q.Tags),
+		Lifecycle:              q.Lifecycle,
-		Lifecycle: q.Lifecycle,
+		Policy:                 policy,
-		Policy:    q.Policy,
+		MessageRetentionPeriod: period,
+		Tags:                   intersectSQSTags(tags.Tags, q.Tags),
 	}
 
 	return actual, nil

upup/pkg/fi/cloudup/awstasks/tags.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/eventbridge"
 	"github.com/aws/aws-sdk-go/service/iam"
 )
 
@@ -66,6 +67,20 @@ func mapToIAMTags(tags map[string]string) []*iam.Tag {
 	return m
 }
 
+func mapEventBridgeTagsToMap(tags []*eventbridge.Tag) map[string]string {
+	if tags == nil {
+		return nil
+	}
+	m := make(map[string]string)
+	for _, t := range tags {
+		if strings.HasPrefix(aws.StringValue(t.Key), "aws:cloudformation:") {
+			continue
+		}
+		m[aws.StringValue(t.Key)] = aws.StringValue(t.Value)
+	}
+	return m
+}
+
 func findNameTag(tags []*ec2.Tag) *string {
 	for _, tag := range tags {
 		if aws.StringValue(tag.Key) == "Name" {

upup/pkg/fi/cloudup/template_functions.go

@@ -216,7 +216,7 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS
 	}
 
 	dest["UseServiceAccountIAM"] = tf.UseServiceAccountIAM
- 
+
 	if cluster.Spec.NodeTerminationHandler != nil {
 		dest["DefaultQueueName"] = func() string {
 			s := strings.Replace(tf.ClusterName(), ".", "-", -1)

vendor/github.com/aws/aws-sdk-go/service/eventbridge/api.go

@@ -3799,6 +3799,8 @@ func (c *EventBridge) PutTargetsRequest(input *PutTargetsInput) (req *request.Re
 //
 //    * Custom/SaaS HTTPS APIs via EventBridge API Destinations
 //
+//    * Amazon SageMaker Model Building Pipelines
+//
 // Creating rules with built-in targets is supported only in the AWS Management
 // Console. The built-in targets are EC2 CreateSnapshot API call, EC2 RebootInstances
 // API call, EC2 StopInstances API call, and EC2 TerminateInstances API call.
@@ -12619,6 +12621,109 @@ func (s *RunCommandTarget) SetValues(v []*string) *RunCommandTarget {
 	return s
 }
 
+// Name/Value pair of a parameter to start execution of a SageMaker Model Building
+// Pipeline.
+type SageMakerPipelineParameter struct {
+	_ struct{} `type:"structure"`
+
+	// Name of parameter to start execution of a SageMaker Model Building Pipeline.
+	//
+	// Name is a required field
+	Name *string `min:"1" type:"string" required:"true"`
+
+	// Value of parameter to start execution of a SageMaker Model Building Pipeline.
+	//
+	// Value is a required field
+	Value *string `type:"string" required:"true"`
+}
+
+// String returns the string representation
+func (s SageMakerPipelineParameter) String() string {
+	return awsutil.Prettify(s)
+}
+
+// GoString returns the string representation
+func (s SageMakerPipelineParameter) GoString() string {
+	return s.String()
+}
+
+// Validate inspects the fields of the type to determine if they are valid.
+func (s *SageMakerPipelineParameter) Validate() error {
+	invalidParams := request.ErrInvalidParams{Context: "SageMakerPipelineParameter"}
+	if s.Name == nil {
+		invalidParams.Add(request.NewErrParamRequired("Name"))
+	}
+	if s.Name != nil && len(*s.Name) < 1 {
+		invalidParams.Add(request.NewErrParamMinLen("Name", 1))
+	}
+	if s.Value == nil {
+		invalidParams.Add(request.NewErrParamRequired("Value"))
+	}
+
+	if invalidParams.Len() > 0 {
+		return invalidParams
+	}
+	return nil
+}
+
+// SetName sets the Name field's value.
+func (s *SageMakerPipelineParameter) SetName(v string) *SageMakerPipelineParameter {
+	s.Name = &v
+	return s
+}
+
+// SetValue sets the Value field's value.
+func (s *SageMakerPipelineParameter) SetValue(v string) *SageMakerPipelineParameter {
+	s.Value = &v
+	return s
+}
+
+// These are custom parameters to use when the target is a SageMaker Model Building
+// Pipeline that starts based on EventBridge events.
+type SageMakerPipelineParameters struct {
+	_ struct{} `type:"structure"`
+
+	// List of Parameter names and values for SageMaker Model Building Pipeline
+	// execution.
+	PipelineParameterList []*SageMakerPipelineParameter `type:"list"`
+}
+
+// String returns the string representation
+func (s SageMakerPipelineParameters) String() string {
+	return awsutil.Prettify(s)
+}
+
+// GoString returns the string representation
+func (s SageMakerPipelineParameters) GoString() string {
+	return s.String()
+}
+
+// Validate inspects the fields of the type to determine if they are valid.
+func (s *SageMakerPipelineParameters) Validate() error {
+	invalidParams := request.ErrInvalidParams{Context: "SageMakerPipelineParameters"}
+	if s.PipelineParameterList != nil {
+		for i, v := range s.PipelineParameterList {
+			if v == nil {
+				continue
+			}
+			if err := v.Validate(); err != nil {
+				invalidParams.AddNested(fmt.Sprintf("%s[%v]", "PipelineParameterList", i), err.(request.ErrInvalidParams))
+			}
+		}
+	}
+
+	if invalidParams.Len() > 0 {
+		return invalidParams
+	}
+	return nil
+}
+
+// SetPipelineParameterList sets the PipelineParameterList field's value.
+func (s *SageMakerPipelineParameters) SetPipelineParameterList(v []*SageMakerPipelineParameter) *SageMakerPipelineParameters {
+	s.PipelineParameterList = v
+	return s
+}
+
 // This structure includes the custom parameter to be used when the target is
 // an SQS FIFO queue.
 type SqsParameters struct {
@@ -13036,6 +13141,14 @@ type Target struct {
 	// Parameters used when you are using the rule to invoke Amazon EC2 Run Command.
 	RunCommandParameters *RunCommandParameters `type:"structure"`
 
+	// Contains the SageMaker Model Building Pipeline parameters to start execution
+	// of a SageMaker Model Building Pipeline.
+	//
+	// If you specify a SageMaker Model Building Pipeline as a target, you can use
+	// this to specify parameters to start a pipeline execution based on EventBridge
+	// events.
+	SageMakerPipelineParameters *SageMakerPipelineParameters `type:"structure"`
+
 	// Contains the message group ID to use when the target is a FIFO queue.
 	//
 	// If you specify an SQS FIFO queue as a target, the queue must have content-based
@@ -13111,6 +13224,11 @@ func (s *Target) Validate() error {
 			invalidParams.AddNested("RunCommandParameters", err.(request.ErrInvalidParams))
 		}
 	}
+	if s.SageMakerPipelineParameters != nil {
+		if err := s.SageMakerPipelineParameters.Validate(); err != nil {
+			invalidParams.AddNested("SageMakerPipelineParameters", err.(request.ErrInvalidParams))
+		}
+	}
 
 	if invalidParams.Len() > 0 {
 		return invalidParams
@@ -13202,6 +13320,12 @@ func (s *Target) SetRunCommandParameters(v *RunCommandParameters) *Target {
 	return s
 }
 
+// SetSageMakerPipelineParameters sets the SageMakerPipelineParameters field's value.
+func (s *Target) SetSageMakerPipelineParameters(v *SageMakerPipelineParameters) *Target {
+	s.SageMakerPipelineParameters = v
+	return s
+}
+
 // SetSqsParameters sets the SqsParameters field's value.
 func (s *Target) SetSqsParameters(v *SqsParameters) *Target {
 	s.SqsParameters = v

vendor/github.com/aws/aws-sdk-go/service/sqs/api.go

@@ -4617,9 +4617,9 @@ type SendMessageBatchResultEntry struct {
 	// about MD5, see RFC1321 (https://www.ietf.org/rfc/rfc1321.txt).
 	MD5OfMessageAttributes *string `type:"string"`
 
-	// An MD5 digest of the non-URL-encoded message attribute string. You can use
+	// An MD5 digest of the non-URL-encoded message body string. You can use this
-	// this attribute to verify that Amazon SQS received the message correctly.
+	// attribute to verify that Amazon SQS received the message correctly. Amazon
-	// Amazon SQS URL-decodes the message before creating the MD5 digest. For information
+	// SQS URL-decodes the message before creating the MD5 digest. For information
 	// about MD5, see RFC1321 (https://www.ietf.org/rfc/rfc1321.txt).
 	//
 	// MD5OfMessageBody is a required field
@@ -4912,9 +4912,9 @@ type SendMessageOutput struct {
 	// about MD5, see RFC1321 (https://www.ietf.org/rfc/rfc1321.txt).
 	MD5OfMessageAttributes *string `type:"string"`
 
-	// An MD5 digest of the non-URL-encoded message attribute string. You can use
+	// An MD5 digest of the non-URL-encoded message body string. You can use this
-	// this attribute to verify that Amazon SQS received the message correctly.
+	// attribute to verify that Amazon SQS received the message correctly. Amazon
-	// Amazon SQS URL-decodes the message before creating the MD5 digest. For information
+	// SQS URL-decodes the message before creating the MD5 digest. For information
 	// about MD5, see RFC1321 (https://www.ietf.org/rfc/rfc1321.txt).
 	MD5OfMessageBody *string `type:"string"`
 

vendor/github.com/aws/aws-sdk-go/service/sqs/doc.go

@@ -24,7 +24,7 @@
 //
 //    * Handle error responses
 //
-// Additional Information
+// Additional information
 //
 //    * Amazon SQS Product Page (http://aws.amazon.com/sqs/)
 //