Merge pull request #10995 from haugenj/release-1.19

Add NTH Queue Processor Mode

cloudmock/aws/mockautoscaling/api.go

@@ -31,6 +31,7 @@ type MockAutoscaling struct {
 	Groups               map[string]*autoscaling.Group
 	WarmPoolInstances    map[string][]*autoscaling.Instance
 	LaunchConfigurations map[string]*autoscaling.LaunchConfiguration
+	LifecycleHooks       map[string]*autoscaling.LifecycleHook
 }
 
 var _ autoscalingiface.AutoScalingAPI = &MockAutoscaling{}

cloudmock/aws/mockautoscaling/group.go

@@ -336,3 +336,41 @@ func (m *MockAutoscaling) DeleteAutoScalingGroupRequest(*autoscaling.DeleteAutoS
 	klog.Fatalf("Not implemented")
 	return nil, nil
 }
+
+func (m *MockAutoscaling) PutLifecycleHook(input *autoscaling.PutLifecycleHookInput) (*autoscaling.PutLifecycleHookOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	hook := &autoscaling.LifecycleHook{
+		AutoScalingGroupName:  input.AutoScalingGroupName,
+		DefaultResult:         input.DefaultResult,
+		GlobalTimeout:         input.HeartbeatTimeout,
+		HeartbeatTimeout:      input.HeartbeatTimeout,
+		LifecycleHookName:     input.LifecycleHookName,
+		LifecycleTransition:   input.LifecycleTransition,
+		NotificationMetadata:  input.NotificationMetadata,
+		NotificationTargetARN: input.NotificationTargetARN,
+		RoleARN:               input.RoleARN,
+	}
+
+	if m.LifecycleHooks == nil {
+		m.LifecycleHooks = make(map[string]*autoscaling.LifecycleHook)
+	}
+	m.LifecycleHooks[*hook.AutoScalingGroupName] = hook
+
+	return &autoscaling.PutLifecycleHookOutput{}, nil
+}
+
+func (m *MockAutoscaling) DescribeLifecycleHooks(input *autoscaling.DescribeLifecycleHooksInput) (*autoscaling.DescribeLifecycleHooksOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	name := *input.AutoScalingGroupName
+	response := &autoscaling.DescribeLifecycleHooksOutput{}
+
+	hook := m.LifecycleHooks[name]
+	if hook == nil {
+		return response, nil
+	}
+	response.LifecycleHooks = []*autoscaling.LifecycleHook{hook}
+	return response, nil
+}

cloudmock/aws/mockeventbridge/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["api.go"],
+    importpath = "k8s.io/kops/cloudmock/aws/mockeventbridge",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/aws/aws-sdk-go/service/eventbridge:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/eventbridge/eventbridgeiface:go_default_library",
+    ],
+)

cloudmock/aws/mockeventbridge/api.go

@@ -0,0 +1,115 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mockeventbridge
+
+import (
+	"sync"
+
+	"github.com/aws/aws-sdk-go/service/eventbridge"
+	"github.com/aws/aws-sdk-go/service/eventbridge/eventbridgeiface"
+)
+
+type MockEventBridge struct {
+	eventbridgeiface.EventBridgeAPI
+	mutex sync.Mutex
+
+	Rules         map[string]*eventbridge.Rule
+	TagsByArn     map[string][]*eventbridge.Tag
+	TargetsByRule map[string][]*eventbridge.Target
+}
+
+var _ eventbridgeiface.EventBridgeAPI = &MockEventBridge{}
+
+func (m *MockEventBridge) PutRule(input *eventbridge.PutRuleInput) (*eventbridge.PutRuleOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	name := *input.Name
+	arn := "arn:aws:events:us-east-1:012345678901:rule/" + name
+
+	rule := &eventbridge.Rule{
+		Arn:          &arn,
+		EventPattern: input.EventPattern,
+	}
+	if m.Rules == nil {
+		m.Rules = make(map[string]*eventbridge.Rule)
+	}
+	if m.TagsByArn == nil {
+		m.TagsByArn = make(map[string][]*eventbridge.Tag)
+	}
+	m.Rules[name] = rule
+	m.TagsByArn[arn] = input.Tags
+
+	response := &eventbridge.PutRuleOutput{
+		RuleArn: &arn,
+	}
+	return response, nil
+}
+
+func (m *MockEventBridge) ListRules(input *eventbridge.ListRulesInput) (*eventbridge.ListRulesOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	response := &eventbridge.ListRulesOutput{}
+
+	rule := m.Rules[*input.NamePrefix]
+	if rule == nil {
+		return response, nil
+	}
+	response.Rules = []*eventbridge.Rule{rule}
+	return response, nil
+}
+
+func (m *MockEventBridge) DeleteRule(*eventbridge.DeleteRuleInput) (*eventbridge.DeleteRuleOutput, error) {
+	panic("Not implemented")
+}
+
+func (m *MockEventBridge) ListTagsForResource(input *eventbridge.ListTagsForResourceInput) (*eventbridge.ListTagsForResourceOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	response := &eventbridge.ListTagsForResourceOutput{
+		Tags: m.TagsByArn[*input.ResourceARN],
+	}
+	return response, nil
+}
+
+func (m *MockEventBridge) PutTargets(input *eventbridge.PutTargetsInput) (*eventbridge.PutTargetsOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	if m.TargetsByRule == nil {
+		m.TargetsByRule = make(map[string][]*eventbridge.Target)
+	}
+	m.TargetsByRule[*input.Rule] = input.Targets
+
+	return &eventbridge.PutTargetsOutput{}, nil
+}
+
+func (m *MockEventBridge) ListTargetsByRule(input *eventbridge.ListTargetsByRuleInput) (*eventbridge.ListTargetsByRuleOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	response := &eventbridge.ListTargetsByRuleOutput{
+		Targets: m.TargetsByRule[*input.Rule],
+	}
+	return response, nil
+}
+
+func (m *MockEventBridge) RemoveTargets(*eventbridge.RemoveTargetsInput) (*eventbridge.RemoveTargetsOutput, error) {
+	panic("Not implemented")
+}

cloudmock/aws/mocksqs/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["api.go"],
+    importpath = "k8s.io/kops/cloudmock/aws/mocksqs",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/aws/aws-sdk-go/service/sqs:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/sqs/sqsiface:go_default_library",
+    ],
+)

cloudmock/aws/mocksqs/api.go

@@ -0,0 +1,109 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mocksqs
+
+import (
+	"sync"
+
+	"github.com/aws/aws-sdk-go/service/sqs"
+	"github.com/aws/aws-sdk-go/service/sqs/sqsiface"
+)
+
+type MockSQS struct {
+	sqsiface.SQSAPI
+	mutex sync.Mutex
+
+	Queues map[string]mockQueue
+}
+
+type mockQueue struct {
+	url        *string
+	attributes map[string]*string
+	tags       map[string]*string
+}
+
+var _ sqsiface.SQSAPI = &MockSQS{}
+
+func (m *MockSQS) CreateQueue(input *sqs.CreateQueueInput) (*sqs.CreateQueueOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	name := *input.QueueName
+	url := "https://sqs.us-east-1.amazonaws.com/123456789123/" + name
+
+	if m.Queues == nil {
+		m.Queues = make(map[string]mockQueue)
+	}
+	queue := mockQueue{
+		url:        &url,
+		attributes: input.Attributes,
+		tags:       input.Tags,
+	}
+
+	m.Queues[name] = queue
+
+	response := &sqs.CreateQueueOutput{
+		QueueUrl: &url,
+	}
+	return response, nil
+}
+
+func (m *MockSQS) ListQueues(input *sqs.ListQueuesInput) (*sqs.ListQueuesOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	response := &sqs.ListQueuesOutput{}
+
+	if queue, ok := m.Queues[*input.QueueNamePrefix]; ok {
+		response.QueueUrls = []*string{queue.url}
+	}
+	return response, nil
+}
+
+func (m *MockSQS) GetQueueAttributes(input *sqs.GetQueueAttributesInput) (*sqs.GetQueueAttributesOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	response := &sqs.GetQueueAttributesOutput{}
+
+	for _, v := range m.Queues {
+		if *v.url == *input.QueueUrl {
+			response.Attributes = v.attributes
+			return response, nil
+		}
+	}
+	return response, nil
+}
+
+func (m *MockSQS) ListQueueTags(input *sqs.ListQueueTagsInput) (*sqs.ListQueueTagsOutput, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	response := &sqs.ListQueueTagsOutput{}
+
+	for _, v := range m.Queues {
+		if *v.url == *input.QueueUrl {
+			response.Tags = v.tags
+			return response, nil
+		}
+	}
+	return response, nil
+}
+
+func (m *MockSQS) DeleteQueue(*sqs.DeleteQueueInput) (*sqs.DeleteQueueOutput, error) {
+	panic("Not implemented")
+}

cmd/kops/integration_test.go

@@ -67,6 +67,8 @@ type integrationTest struct {
 	caKey           bool
 	jsonOutput      bool
 	bastionUserData bool
+	// nth is true if we should check for files created by nth queue processor add on
+	nth bool
 }
 
 func newIntegrationTest(clusterName, srcDir string) *integrationTest {
@@ -135,6 +137,11 @@ func (i *integrationTest) withBastionUserData() *integrationTest {
 	return i
 }
 
+func (i *integrationTest) withNTH() *integrationTest {
+	i.nth = true
+	return i
+}
+
 // TestMinimal runs the test on a minimum configuration, similar to kops create cluster minimal.example.com --zones us-west-1a
 func TestMinimal(t *testing.T) {
 	newIntegrationTest("minimal.example.com", "minimal").runTestTerraformAWS(t)
@@ -387,6 +394,12 @@ func TestAPIServerNodes(t *testing.T) {
 	newIntegrationTest("minimal.example.com", "apiservernodes").runTestCloudformation(t)
 }
 
+// TestNTHQueueProcessor tests the output for resources required by NTH Queue Processor mode
+func TestNTHQueueProcessor(t *testing.T) {
+	newIntegrationTest("nthsqsresources.example.com", "nth_sqs_resources").withNTH().runTestTerraformAWS(t)
+	newIntegrationTest("nthsqsresources.example.com", "nth_sqs_resources").runTestCloudformation(t)
+}
+
 func (i *integrationTest) runTest(t *testing.T, h *testutils.IntegrationTestHarness, expectedDataFilenames []string, tfFileName string, expectedTfFileName string, phase *cloudup.Phase) {
 	ctx := context.Background()
 
@@ -579,6 +592,14 @@ func (i *integrationTest) runTestTerraformAWS(t *testing.T) {
 				expectedFilenames = append(expectedFilenames, "aws_launch_template_bastion."+i.clusterName+"_user_data")
 			}
 		}
+		if i.nth {
+			expectedFilenames = append(expectedFilenames, []string{
+				"aws_cloudwatch_event_rule_" + i.clusterName + "-ASGLifecycle_event_pattern",
+				"aws_cloudwatch_event_rule_" + i.clusterName + "-RebalanceRecommendation_event_pattern",
+				"aws_cloudwatch_event_rule_" + i.clusterName + "-SpotInterruption_event_pattern",
+				"aws_sqs_queue_" + strings.Replace(i.clusterName, ".", "-", -1) + "-nth_policy",
+			}...)
+		}
 	}
 
 	expectedFilenames = append(expectedFilenames, i.expectServiceAccountRolePolicies...)

cmd/kops/lifecycle_integration_test.go

@@ -153,6 +153,14 @@ func TestLifecyclePrivateSharedIP(t *testing.T) {
 	})
 }
 
+// TestLifecycleNodeTerminationHandlerQueueProcessor runs the test on a cluster with requisite resources for NTH Queue Processor
+func TestLifecycleNodeTerminationHandlerQueueProcessor(t *testing.T) {
+	runLifecycleTestAWS(&LifecycleTestOptions{
+		t:      t,
+		SrcDir: "nth_sqs_resources",
+	})
+}
+
 func runLifecycleTest(h *testutils.IntegrationTestHarness, o *LifecycleTestOptions, cloud *awsup.MockAWSCloud) {
 	ctx := context.Background()
 

docs/addons.md

@@ -127,14 +127,39 @@ spec:
 
 {{ kops_feature_table(kops_added_default='1.19') }}
 
-Node Termination Handler ensures that the Kubernetes control plane responds appropriately to events that can cause your EC2 instance to become unavailable, such as EC2 maintenance events, EC2 Spot interruptions, ASG Scale-In, ASG AZ Rebalance, and EC2 Instance Termination via the API or Console. If not handled, your application code may not stop gracefully, take longer to recover full availability, or accidentally schedule work to nodes that are going down.
+[Node Termination Handler](https://github.com/aws/aws-node-termination-handler) ensures that the Kubernetes control plane responds appropriately to events that can cause your EC2 instance to become unavailable, such as EC2 maintenance events, EC2 Spot interruptions, and EC2 instance rebalance recommendations. If not handled, your application code may not stop gracefully, take longer to recover full availability, or accidentally schedule work to nodes that are going down.
 
 ```yaml
 spec:
   nodeTerminationHandler:
     enabled: true
+    enableSQSTerminationDraining: true
+    managedASGTag: "aws-node-termination-handler/managed"
 ```
 
+If `enableSQSTerminationDraining` is true Node Termination Handler will operate in Queue Processor mode. In addition to the events mentioned above, Queue Processor mode allows Node Termination Handler to take care of ASG Scale-In, AZ-Rebalance, Unhealthy Instances, EC2 Instance Termination via the API or Console, and more. kOps will provision the necessary infrastructure: an SQS queue, EventBridge rules, and ASG Lifecycle hooks. `managedASGTag` can be configured with Queue Processor mode to distinguish resource ownership between multiple clusters.
+
+The kOps CLI requires additional IAM permissions to create the requisite EventBridge rules and SQS queue:
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "events:PutEvents",
+                "events:PutTargets",
+                "sqs:CreateQueue"
+            ],
+            "Resource": "*"
+        }
+    ]
+}
+```
+
+**Warning: If you switch between the two operating modes on an existing cluster, the old resources have to be manually deleted. For IMDS to Queue Processor, this means deleting the k8s nth daemonset. For Queue Processor to IMDS, this means deleting the k8s nth deployment and the AWS resources: the SQS queue, EventBridge rules, and ASG Lifecycle hooks.**
+
 ## Static addons
 
 The command `kops create cluster` does not support specifying addons to be added to the cluster when it is created. Instead they can be added after cluster creation using kubectl. Alternatively when creating a cluster from a yaml manifest, addons can be specified using `spec.addons`.

k8s/crds/kops.k8s.io_clusters.yaml

@@ -3906,6 +3906,10 @@ spec:
                 description: NodeTerminationHandler determines the cluster autoscaler
                   configuration.
                 properties:
+                  enableSQSTerminationDraining:
+                    description: EnableSQSTerminationDraining enables queue-processor
+                      mode which drains nodes when an SQS termination event is received.
+                    type: boolean
                   enableScheduledEventDraining:
                     description: 'EnableScheduledEventDraining makes node termination
                       handler drain nodes before the maintenance window starts for
@@ -3920,6 +3924,10 @@ spec:
                     description: 'Enabled enables the node termination handler. Default:
                       true'
                     type: boolean
+                  managedASGTag:
+                    description: ManagedASGTag is the tag used to determine which
+                      nodes NTH can take action on
+                    type: string
                   prometheusEnable:
                     description: EnablePrometheusMetrics enables the "/metrics" endpoint.
                     type: boolean

pkg/apis/kops/cluster.go

@@ -161,7 +161,7 @@ type ClusterSpec struct {
 	ExternalDNS                    *ExternalDNSConfig            `json:"externalDns,omitempty"`
 	NTP                            *NTPConfig                    `json:"ntp,omitempty"`
 
-	// NodeTerminationHandler determines the cluster autoscaler configuration.
+	// NodeTerminationHandler determines the node termination handler configuration.
 	NodeTerminationHandler *NodeTerminationHandlerConfig `json:"nodeTerminationHandler,omitempty"`
 	// MetricsServer determines the metrics server configuration.
 	MetricsServer *MetricsServerConfig `json:"metricsServer,omitempty"`

pkg/apis/kops/componentconfig.go

@@ -854,6 +854,12 @@ type NodeTerminationHandlerConfig struct {
 
 	// EnablePrometheusMetrics enables the "/metrics" endpoint.
 	EnablePrometheusMetrics *bool `json:"prometheusEnable,omitempty"`
+
+	// EnableSQSTerminationDraining enables queue-processor mode which drains nodes when an SQS termination event is received.
+	EnableSQSTerminationDraining *bool `json:"enableSQSTerminationDraining,omitempty"`
+
+	// ManagedASGTag is the tag used to determine which nodes NTH can take action on
+	ManagedASGTag *string `json:"managedASGTag,omitempty"`
 }
 
 // ClusterAutoscalerConfig determines the cluster autoscaler configuration.

pkg/apis/kops/v1alpha2/componentconfig.go

@@ -853,6 +853,12 @@ type NodeTerminationHandlerConfig struct {
 
 	// EnablePrometheusMetrics enables the "/metrics" endpoint.
 	EnablePrometheusMetrics *bool `json:"prometheusEnable,omitempty"`
+
+	// EnableSQSTerminationDraining enables queue-processor mode which drains nodes when an SQS termination event is received.
+	EnableSQSTerminationDraining *bool `json:"enableSQSTerminationDraining,omitempty"`
+
+	// ManagedASGTag is the tag used to determine which nodes NTH can take action on
+	ManagedASGTag *string `json:"managedASGTag,omitempty"`
 }
 
 // ClusterAutoscalerConfig determines the cluster autoscaler configuration.

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -5761,6 +5761,8 @@ func autoConvert_v1alpha2_NodeTerminationHandlerConfig_To_kops_NodeTerminationHa
 	out.EnableSpotInterruptionDraining = in.EnableSpotInterruptionDraining
 	out.EnableScheduledEventDraining = in.EnableScheduledEventDraining
 	out.EnablePrometheusMetrics = in.EnablePrometheusMetrics
+	out.EnableSQSTerminationDraining = in.EnableSQSTerminationDraining
+	out.ManagedASGTag = in.ManagedASGTag
 	return nil
 }
 
@@ -5774,6 +5776,8 @@ func autoConvert_kops_NodeTerminationHandlerConfig_To_v1alpha2_NodeTerminationHa
 	out.EnableSpotInterruptionDraining = in.EnableSpotInterruptionDraining
 	out.EnableScheduledEventDraining = in.EnableScheduledEventDraining
 	out.EnablePrometheusMetrics = in.EnablePrometheusMetrics
+	out.EnableSQSTerminationDraining = in.EnableSQSTerminationDraining
+	out.ManagedASGTag = in.ManagedASGTag
 	return nil
 }
 

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -3894,6 +3894,16 @@ func (in *NodeTerminationHandlerConfig) DeepCopyInto(out *NodeTerminationHandler
 		*out = new(bool)
 		**out = **in
 	}
+	if in.EnableSQSTerminationDraining != nil {
+		in, out := &in.EnableSQSTerminationDraining, &out.EnableSQSTerminationDraining
+		*out = new(bool)
+		**out = **in
+	}
+	if in.ManagedASGTag != nil {
+		in, out := &in.ManagedASGTag, &out.ManagedASGTag
+		*out = new(string)
+		**out = **in
+	}
 	return
 }
 

pkg/apis/kops/zz_generated.deepcopy.go

@@ -4092,6 +4092,16 @@ func (in *NodeTerminationHandlerConfig) DeepCopyInto(out *NodeTerminationHandler
 		*out = new(bool)
 		**out = **in
 	}
+	if in.EnableSQSTerminationDraining != nil {
+		in, out := &in.EnableSQSTerminationDraining, &out.EnableSQSTerminationDraining
+		*out = new(bool)
+		**out = **in
+	}
+	if in.ManagedASGTag != nil {
+		in, out := &in.ManagedASGTag, &out.ManagedASGTag
+		*out = new(string)
+		**out = **in
+	}
 	return
 }
 

pkg/model/awsmodel/BUILD.bazel

@@ -9,6 +9,7 @@ go_library(
         "context.go",
         "dns.go",
         "external_access.go",
+        "nodeterminationhandler.go",
         "oidc_provider.go",
         "spotinst.go",
     ],
@@ -26,6 +27,7 @@ go_library(
         "//upup/pkg/fi/cloudup/awsup:go_default_library",
         "//upup/pkg/fi/cloudup/spotinsttasks:go_default_library",
         "//upup/pkg/fi/fitasks:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
         "//vendor/gopkg.in/square/go-jose.v2:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",

pkg/model/awsmodel/nodeterminationhandler.go

@@ -0,0 +1,173 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package awsmodel
+
+import (
+	"strings"
+
+	"k8s.io/kops/pkg/model"
+
+	"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+const (
+	NTHTemplate = `{
+		"Version": "2012-10-17",
+		"Statement": [{                     
+			"Effect": "Allow",
+			"Principal": {
+				"Service": ["events.amazonaws.com", "sqs.amazonaws.com"]
+			},
+			"Action": "sqs:SendMessage",
+			"Resource": [
+				"arn:aws:sqs:{{ AWS_REGION }}:{{ ACCOUNT_ID }}:{{ SQS_QUEUE_NAME }}"
+			]
+		}]
+	}`
+	DefaultMessageRetentionPeriod = 300
+)
+
+type event struct {
+	name    string
+	pattern string
+}
+
+var (
+	_ fi.ModelBuilder = &NodeTerminationHandlerBuilder{}
+
+	events = []event{
+		{
+			name:    "ASGLifecycle",
+			pattern: `{"source":["aws.autoscaling"],"detail-type":["EC2 Instance-terminate Lifecycle Action"]}`,
+		},
+		{
+			name:    "SpotInterruption",
+			pattern: `{"source": ["aws.ec2"],"detail-type": ["EC2 Spot Instance Interruption Warning"]}`,
+		},
+		{
+			name:    "RebalanceRecommendation",
+			pattern: `{"source": ["aws.ec2"],"detail-type": ["EC2 Instance Rebalance Recommendation"]}`,
+		},
+	}
+)
+
+type NodeTerminationHandlerBuilder struct {
+	*AWSModelContext
+
+	Lifecycle *fi.Lifecycle
+}
+
+func (b *NodeTerminationHandlerBuilder) Build(c *fi.ModelBuilderContext) error {
+
+	for _, ig := range b.InstanceGroups {
+		err := b.configureASG(c, ig)
+		if err != nil {
+			return err
+		}
+	}
+
+	err := b.buildSQSQueue(c)
+	if err != nil {
+		return err
+	}
+
+	err = b.buildEventBridgeRules(c)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (b *NodeTerminationHandlerBuilder) configureASG(c *fi.ModelBuilderContext, ig *kops.InstanceGroup) error {
+	name := ig.Name + "-NTHLifecycleHook"
+
+	lifecyleTask := &awstasks.AutoscalingLifecycleHook{
+		ID:                  aws.String(name),
+		Name:                aws.String(name),
+		Lifecycle:           b.Lifecycle,
+		AutoscalingGroup:    b.LinkToAutoscalingGroup(ig),
+		DefaultResult:       aws.String("CONTINUE"),
+		HeartbeatTimeout:    aws.Int64(DefaultMessageRetentionPeriod),
+		LifecycleTransition: aws.String("autoscaling:EC2_INSTANCE_TERMINATING"),
+	}
+
+	c.AddTask(lifecyleTask)
+
+	return nil
+}
+
+func (b *NodeTerminationHandlerBuilder) buildSQSQueue(c *fi.ModelBuilderContext) error {
+	queueName := model.QueueNamePrefix(b.ClusterName()) + "-nth"
+	policy := strings.ReplaceAll(NTHTemplate, "{{ AWS_REGION }}", b.Region)
+	policy = strings.ReplaceAll(policy, "{{ ACCOUNT_ID }}", b.AWSAccountID)
+	policy = strings.ReplaceAll(policy, "{{ SQS_QUEUE_NAME }}", queueName)
+
+	task := &awstasks.SQS{
+		Name:                   aws.String(queueName),
+		Lifecycle:              b.Lifecycle,
+		Policy:                 fi.NewStringResource(policy),
+		MessageRetentionPeriod: DefaultMessageRetentionPeriod,
+		Tags:                   b.CloudTags(queueName, false),
+	}
+
+	c.AddTask(task)
+
+	return nil
+}
+
+func (b *NodeTerminationHandlerBuilder) buildEventBridgeRules(c *fi.ModelBuilderContext) error {
+	clusterName := b.ClusterName()
+	queueName := model.QueueNamePrefix(clusterName) + "-nth"
+	region := b.Region
+	accountID := b.AWSAccountID
+	targetArn := "arn:aws:sqs:" + region + ":" + accountID + ":" + queueName
+
+	for _, event := range events {
+		// build rule
+		ruleName := aws.String(clusterName + "-" + event.name)
+		pattern := event.pattern
+
+		ruleTask := &awstasks.EventBridgeRule{
+			Name:      ruleName,
+			Lifecycle: b.Lifecycle,
+			Tags:      b.CloudTags(*ruleName, false),
+
+			EventPattern: &pattern,
+			TargetArn:    &targetArn,
+		}
+
+		c.AddTask(ruleTask)
+
+		// build target
+		targetTask := &awstasks.EventBridgeTarget{
+			Name:      aws.String(*ruleName + "-Target"),
+			Lifecycle: b.Lifecycle,
+
+			Rule:      ruleTask,
+			TargetArn: &targetArn,
+		}
+
+		c.AddTask(targetTask)
+	}
+
+	return nil
+}

pkg/model/components/nodeterminationhandler.go

@@ -48,5 +48,14 @@ func (b *NodeTerminationHandlerOptionsBuilder) BuildOptions(o interface{}) error
 	if nth.EnablePrometheusMetrics == nil {
 		nth.EnablePrometheusMetrics = fi.Bool(false)
 	}
+
+	if nth.EnableSQSTerminationDraining == nil {
+		nth.EnableSQSTerminationDraining = fi.Bool(false)
+	}
+
+	if nth.ManagedASGTag == nil {
+		nth.ManagedASGTag = fi.String("aws-node-termination-handler/managed")
+	}
+
 	return nil
 }

pkg/model/context.go

@@ -143,6 +143,12 @@ func (m *KopsModelContext) CloudTagsForInstanceGroup(ig *kops.InstanceGroup) (ma
 		labels[k] = v
 	}
 
+	// Apply NTH Labels
+	nth := m.Cluster.Spec.NodeTerminationHandler
+	if nth != nil && fi.BoolValue(nth.Enabled) && fi.BoolValue(nth.EnableSQSTerminationDraining) {
+		labels[fi.StringValue(nth.ManagedASGTag)] = ""
+	}
+
 	// Apply labels for cluster autoscaler node labels
 	for k, v := range nodelabels.BuildNodeLabels(m.Cluster, ig) {
 		labels[nodeidentityaws.ClusterAutoscalerNodeTemplateLabel+k] = v

pkg/model/iam/iam_builder.go

@@ -337,6 +337,11 @@ func (r *NodeRoleMaster) BuildAWSPolicy(b *PolicyBuilder) (*Policy, error) {
 		addCalicoSrcDstCheckPermissions(p)
 	}
 
+	nth := b.Cluster.Spec.NodeTerminationHandler
+	if nth != nil && fi.BoolValue(nth.Enabled) && fi.BoolValue(nth.EnableSQSTerminationDraining) {
+		addNodeTerminationHandlerSQSPermissions(p, resource)
+	}
+
 	return p, nil
 }
 
@@ -1143,6 +1148,21 @@ func addAmazonVPCCNIPermissions(p *Policy, resource stringorslice.StringOrSlice,
 	)
 }
 
+func addNodeTerminationHandlerSQSPermissions(p *Policy, resource stringorslice.StringOrSlice) {
+	p.Statement = append(p.Statement,
+		&Statement{
+			Effect: StatementEffectAllow,
+			Action: stringorslice.Slice([]string{
+				"autoscaling:CompleteLifecycleAction",
+				"autoscaling:DescribeAutoScalingInstances",
+				"sqs:DeleteMessage",
+				"sqs:ReceiveMessage",
+			}),
+			Resource: resource,
+		},
+	)
+}
+
 func createResource(b *PolicyBuilder) stringorslice.StringOrSlice {
 	var resource stringorslice.StringOrSlice
 	if b.ResourceARN != nil {

pkg/model/names.go

@@ -265,3 +265,8 @@ func (b *KopsModelContext) LinkToPrivateRouteTableInZone(zoneName string) *awsta
 func (b *KopsModelContext) InstanceName(ig *kops.InstanceGroup, suffix string) string {
 	return b.AutoscalingGroupName(ig) + suffix
 }
+
+func QueueNamePrefix(clusterName string) string {
+	// periods aren't allowed in queue name
+	return strings.ReplaceAll(clusterName, ".", "-")
+}

pkg/resources/aws/BUILD.bazel

@@ -6,10 +6,12 @@ go_library(
         "aws.go",
         "elasticip.go",
         "errors.go",
+        "eventbridge.go",
         "filters.go",
         "natgateway.go",
         "routetable.go",
         "securitygroup.go",
+        "sqs.go",
         "subnet.go",
         "tags.go",
         "vpc.go",
@@ -29,8 +31,10 @@ go_library(
         "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/elb:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/elbv2:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/eventbridge:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/iam:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/route53:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/sqs:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/klog/v2:go_default_library",
     ],

pkg/resources/aws/aws.go

@@ -83,6 +83,11 @@ func ListResourcesAWS(cloud awsup.AWSCloud, clusterName string) (map[string]*res
 		ListIAMInstanceProfiles,
 		ListIAMRoles,
 		ListIAMOIDCProviders,
+
+		// SQS
+		ListSQSQueues,
+		// EventBridge
+		ListEventBridgeRules,
 	}
 
 	if featureflag.Spotinst.Enabled() {

pkg/resources/aws/eventbridge.go

@@ -0,0 +1,112 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/eventbridge"
+	"k8s.io/klog/v2"
+
+	"k8s.io/kops/pkg/resources"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
+)
+
+func DumpEventBridgeRule(op *resources.DumpOperation, r *resources.Resource) error {
+	data := make(map[string]interface{})
+	data["id"] = r.ID
+	data["name"] = r.Name
+	data["type"] = r.Type
+	data["raw"] = r.Obj
+	op.Dump.Resources = append(op.Dump.Resources, data)
+
+	return nil
+}
+
+func DeleteEventBridgeRule(cloud fi.Cloud, r *resources.Resource) error {
+	c := cloud.(awsup.AWSCloud)
+
+	targets, err := c.EventBridge().ListTargetsByRule(&eventbridge.ListTargetsByRuleInput{
+		Rule: aws.String(r.Name),
+	})
+	if err != nil {
+		return fmt.Errorf("error listing targets for EventBridge rule %q: %v", r.Name, err)
+	}
+
+	var ids []*string
+	for _, target := range targets.Targets {
+		ids = append(ids, target.Id)
+	}
+
+	klog.V(2).Infof("Removing EventBridge Targets for rule %q", r.Name)
+	_, err = c.EventBridge().RemoveTargets(&eventbridge.RemoveTargetsInput{
+		Ids:  ids,
+		Rule: aws.String(r.Name),
+	})
+	if err != nil {
+		return fmt.Errorf("error removing targets for EventBridge rule %q: %v", r.Name, err)
+	}
+
+	klog.V(2).Infof("Deleting EventBridge rule %q", r.Name)
+	request := &eventbridge.DeleteRuleInput{
+		Name: aws.String(r.Name),
+	}
+	_, err = c.EventBridge().DeleteRule(request)
+	if err != nil {
+		return fmt.Errorf("error deleting EventBridge rule %q: %v", r.Name, err)
+	}
+	return nil
+}
+
+func ListEventBridgeRules(cloud fi.Cloud, clusterName string) ([]*resources.Resource, error) {
+	c := cloud.(awsup.AWSCloud)
+
+	klog.V(2).Infof("Listing EventBridge rules")
+
+	// rule names start with the cluster name so that we can search for them
+	request := &eventbridge.ListRulesInput{
+		EventBusName: nil,
+		Limit:        nil,
+		NamePrefix:   aws.String(clusterName),
+	}
+	response, err := c.EventBridge().ListRules(request)
+	if err != nil {
+		return nil, fmt.Errorf("error listing Eventbridge rules: %v", err)
+	}
+	if response == nil || len(response.Rules) == 0 {
+		return nil, nil
+	}
+
+	var resourceTrackers []*resources.Resource
+
+	for _, rule := range response.Rules {
+		resourceTracker := &resources.Resource{
+			Name:    *rule.Name,
+			ID:      *rule.Name,
+			Type:    "eventbridge",
+			Deleter: DeleteEventBridgeRule,
+			Dumper:  DumpEventBridgeRule,
+			Obj:     rule,
+		}
+
+		resourceTrackers = append(resourceTrackers, resourceTracker)
+	}
+
+	return resourceTrackers, nil
+}

pkg/resources/aws/sqs.go

@@ -0,0 +1,90 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/service/sqs"
+	"k8s.io/klog/v2"
+	"k8s.io/kops/pkg/resources"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
+)
+
+func DumpSQSQueue(op *resources.DumpOperation, r *resources.Resource) error {
+	data := make(map[string]interface{})
+	data["id"] = r.ID
+	data["name"] = r.Name
+	data["type"] = r.Type
+	data["raw"] = r.Obj
+	op.Dump.Resources = append(op.Dump.Resources, data)
+
+	return nil
+}
+
+func DeleteSQSQueue(cloud fi.Cloud, r *resources.Resource) error {
+	c := cloud.(awsup.AWSCloud)
+
+	url := r.ID
+
+	klog.V(2).Infof("Deleting SQS queue %q", url)
+	request := &sqs.DeleteQueueInput{
+		QueueUrl: &url,
+	}
+	_, err := c.SQS().DeleteQueue(request)
+	if err != nil {
+		return fmt.Errorf("error deleting SQS queue %q: %v", url, err)
+	}
+	return nil
+}
+
+func ListSQSQueues(cloud fi.Cloud, clusterName string) ([]*resources.Resource, error) {
+	c := cloud.(awsup.AWSCloud)
+
+	klog.V(2).Infof("Listing SQS queues")
+	queuePrefix := strings.ReplaceAll(clusterName, ".", "-")
+
+	request := &sqs.ListQueuesInput{
+		QueueNamePrefix: &queuePrefix,
+	}
+	response, err := c.SQS().ListQueues(request)
+	if err != nil {
+		return nil, fmt.Errorf("error listing SQS queues: %v", err)
+	}
+	if response == nil || len(response.QueueUrls) == 0 {
+		return nil, nil
+	}
+
+	var resourceTrackers []*resources.Resource
+
+	for _, queueUrl := range response.QueueUrls {
+		resourceTracker := &resources.Resource{
+			Name:    *queueUrl,
+			ID:      *queueUrl,
+			Type:    "sqs",
+			Deleter: DeleteSQSQueue,
+			Dumper:  DumpSQSQueue,
+			Obj:     queueUrl,
+		}
+
+		resourceTrackers = append(resourceTrackers, resourceTracker)
+	}
+
+	return resourceTrackers, nil
+}

pkg/testutils/BUILD.bazel

@@ -15,8 +15,10 @@ go_library(
         "//cloudmock/aws/mockec2:go_default_library",
         "//cloudmock/aws/mockelb:go_default_library",
         "//cloudmock/aws/mockelbv2:go_default_library",
+        "//cloudmock/aws/mockeventbridge:go_default_library",
         "//cloudmock/aws/mockiam:go_default_library",
         "//cloudmock/aws/mockroute53:go_default_library",
+        "//cloudmock/aws/mocksqs:go_default_library",
         "//cloudmock/openstack/mockblockstorage:go_default_library",
         "//cloudmock/openstack/mockcompute:go_default_library",
         "//cloudmock/openstack/mockdns:go_default_library",

pkg/testutils/integrationtestharness.go

@@ -23,6 +23,9 @@ import (
 	"path/filepath"
 	"testing"
 
+	"k8s.io/kops/cloudmock/aws/mockeventbridge"
+	"k8s.io/kops/cloudmock/aws/mocksqs"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/elbv2"
@@ -140,6 +143,10 @@ func (h *IntegrationTestHarness) SetupMockAWS() *awsup.MockAWSCloud {
 	cloud.MockIAM = mockIAM
 	mockAutoscaling := &mockautoscaling.MockAutoscaling{}
 	cloud.MockAutoscaling = mockAutoscaling
+	mockSQS := &mocksqs.MockSQS{}
+	cloud.MockSQS = mockSQS
+	mockEventBridge := &mockeventbridge.MockEventBridge{}
+	cloud.MockEventBridge = mockEventBridge
 
 	mockRoute53.MockCreateZone(&route53.HostedZone{
 		Id:   aws.String("/hostedzone/Z1AFAKE1ZON3YO"),

tests/integration/update_cluster/nth_sqs_resources/cloudformation.json.extracted.yaml

@@ -0,0 +1,562 @@
+Resources.AWSEC2LaunchTemplatemasterustest1amastersnthsqsresourcesexamplecom.Properties.LaunchTemplateData.UserData: |
+  #!/bin/bash
+  set -o errexit
+  set -o nounset
+  set -o pipefail
+
+  NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/nodeup
+  NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924
+  NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/nodeup
+  NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865
+
+  export AWS_REGION=us-test-1
+
+
+
+
+  sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true
+
+
+  function ensure-install-dir() {
+    INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
+    mkdir -p ${INSTALL_DIR}/bin
+    mkdir -p ${INSTALL_DIR}/conf
+    cd ${INSTALL_DIR}
+  }
+
+  # Retry a download until we get it. args: name, sha, url1, url2...
+  download-or-bust() {
+    local -r file="$1"
+    local -r hash="$2"
+    shift 2
+
+    urls=( $* )
+    while true; do
+      for url in "${urls[@]}"; do
+        commands=(
+          "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+          "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+          "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+          "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+        )
+        for cmd in "${commands[@]}"; do
+          echo "Attempting download with: ${cmd} {url}"
+          if ! (${cmd} "${url}"); then
+            echo "== Download failed with ${cmd} =="
+            continue
+          fi
+          if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
+            echo "== Hash validation of ${url} failed. Retrying. =="
+            rm -f "${file}"
+          else
+            if [[ -n "${hash}" ]]; then
+              echo "== Downloaded ${url} (SHA1 = ${hash}) =="
+            else
+              echo "== Downloaded ${url} =="
+            fi
+            return
+          fi
+        done
+      done
+
+      echo "All downloads failed; sleeping before retrying"
+      sleep 60
+    done
+  }
+
+  validate-hash() {
+    local -r file="$1"
+    local -r expected="$2"
+    local actual
+
+    actual=$(sha256sum ${file} | awk '{ print $1 }') || true
+    if [[ "${actual}" != "${expected}" ]]; then
+      echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} =="
+      return 1
+    fi
+  }
+
+  function split-commas() {
+    echo $1 | tr "," "\n"
+  }
+
+  function try-download-release() {
+    local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
+    if [[ -n "${NODEUP_HASH:-}" ]]; then
+      local -r nodeup_hash="${NODEUP_HASH}"
+    else
+    # TODO: Remove?
+      echo "Downloading sha256 (not found in env)"
+      download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}"
+      local -r nodeup_hash=$(cat nodeup.sha256)
+    fi
+
+    echo "Downloading nodeup (${nodeup_urls[@]})"
+    download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}"
+
+    chmod +x nodeup
+  }
+
+  function download-release() {
+    case "$(uname -m)" in
+    x86_64*|i?86_64*|amd64*)
+      NODEUP_URL="${NODEUP_URL_AMD64}"
+      NODEUP_HASH="${NODEUP_HASH_AMD64}"
+      ;;
+    aarch64*|arm64*)
+      NODEUP_URL="${NODEUP_URL_ARM64}"
+      NODEUP_HASH="${NODEUP_HASH_ARM64}"
+      ;;
+    *)
+      echo "Unsupported host arch: $(uname -m)" >&2
+      exit 1
+      ;;
+    esac
+
+    # In case of failure checking integrity of release, retry.
+    cd ${INSTALL_DIR}/bin
+    until try-download-release; do
+      sleep 15
+      echo "Couldn't download release. Retrying..."
+    done
+
+    echo "Running nodeup"
+    # We can't run in the foreground because of https://github.com/docker/docker/issues/23793
+    ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8  )
+  }
+
+  ####################################################################################
+
+  /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
+
+  echo "== nodeup node config starting =="
+  ensure-install-dir
+
+  cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
+  cloudConfig:
+    manageStorageClasses: true
+  containerRuntime: containerd
+  containerd:
+    configOverride: |
+      version = 2
+
+      [plugins]
+
+        [plugins."io.containerd.grpc.v1.cri"]
+
+          [plugins."io.containerd.grpc.v1.cri".containerd]
+
+            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+                runtime_type = "io.containerd.runc.v2"
+
+                [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+                  SystemdCgroup = true
+    logLevel: info
+    version: 1.4.4
+  docker:
+    skipInstall: true
+  encryptionConfig: null
+  etcdClusters:
+    events:
+      version: 3.4.13
+    main:
+      version: 3.4.13
+  kubeAPIServer:
+    allowPrivileged: true
+    anonymousAuth: false
+    apiAudiences:
+    - kubernetes.svc.default
+    apiServerCount: 1
+    authorizationMode: AlwaysAllow
+    bindAddress: 0.0.0.0
+    cloudProvider: aws
+    enableAdmissionPlugins:
+    - NamespaceLifecycle
+    - LimitRanger
+    - ServiceAccount
+    - PersistentVolumeLabel
+    - DefaultStorageClass
+    - DefaultTolerationSeconds
+    - MutatingAdmissionWebhook
+    - ValidatingAdmissionWebhook
+    - NodeRestriction
+    - ResourceQuota
+    etcdServers:
+    - http://127.0.0.1:4001
+    etcdServersOverrides:
+    - /events#http://127.0.0.1:4002
+    image: k8s.gcr.io/kube-apiserver:v1.20.0
+    kubeletPreferredAddressTypes:
+    - InternalIP
+    - Hostname
+    - ExternalIP
+    logLevel: 2
+    requestheaderAllowedNames:
+    - aggregator
+    requestheaderExtraHeaderPrefixes:
+    - X-Remote-Extra-
+    requestheaderGroupHeaders:
+    - X-Remote-Group
+    requestheaderUsernameHeaders:
+    - X-Remote-User
+    securePort: 443
+    serviceAccountIssuer: https://api.internal.nthsqsresources.example.com
+    serviceAccountJWKSURI: https://api.internal.nthsqsresources.example.com/openid/v1/jwks
+    serviceClusterIPRange: 100.64.0.0/13
+    storageBackend: etcd3
+  kubeControllerManager:
+    allocateNodeCIDRs: true
+    attachDetachReconcileSyncPeriod: 1m0s
+    cloudProvider: aws
+    clusterCIDR: 100.96.0.0/11
+    clusterName: nthsqsresources.example.com
+    configureCloudRoutes: false
+    image: k8s.gcr.io/kube-controller-manager:v1.20.0
+    leaderElection:
+      leaderElect: true
+    logLevel: 2
+    useServiceAccountCredentials: true
+  kubeProxy:
+    clusterCIDR: 100.96.0.0/11
+    cpuRequest: 100m
+    hostnameOverride: '@aws'
+    image: k8s.gcr.io/kube-proxy:v1.20.0
+    logLevel: 2
+  kubeScheduler:
+    image: k8s.gcr.io/kube-scheduler:v1.20.0
+    leaderElection:
+      leaderElect: true
+    logLevel: 2
+  kubelet:
+    anonymousAuth: false
+    cgroupDriver: systemd
+    cgroupRoot: /
+    cloudProvider: aws
+    clusterDNS: 100.64.0.10
+    clusterDomain: cluster.local
+    enableDebuggingHandlers: true
+    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    hostnameOverride: '@aws'
+    kubeconfigPath: /var/lib/kubelet/kubeconfig
+    logLevel: 2
+    networkPluginName: cni
+    nonMasqueradeCIDR: 100.64.0.0/10
+    podManifestPath: /etc/kubernetes/manifests
+  masterKubelet:
+    anonymousAuth: false
+    cgroupDriver: systemd
+    cgroupRoot: /
+    cloudProvider: aws
+    clusterDNS: 100.64.0.10
+    clusterDomain: cluster.local
+    enableDebuggingHandlers: true
+    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    hostnameOverride: '@aws'
+    kubeconfigPath: /var/lib/kubelet/kubeconfig
+    logLevel: 2
+    networkPluginName: cni
+    nonMasqueradeCIDR: 100.64.0.0/10
+    podManifestPath: /etc/kubernetes/manifests
+    registerSchedulable: false
+
+  __EOF_CLUSTER_SPEC
+
+  cat > conf/ig_spec.yaml << '__EOF_IG_SPEC'
+  {}
+
+  __EOF_IG_SPEC
+
+  cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
+  Assets:
+    amd64:
+    - ff2422571c4c1e9696e367f5f25466b96fb6e501f28aed29f414b1524a52dea0@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubelet
+    - a5895007f331f08d2e082eb12458764949559f30bcc5beae26c38f3e2724262c@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubectl
+    - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
+    - 96641849cb78a0a119223a427dfdc1ade88412ef791a14193212c8c8e29d447b@https://github.com/containerd/containerd/releases/download/v1.4.4/cri-containerd-cni-1.4.4-linux-amd64.tar.gz
+    - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/protokube
+    - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/channels
+    arm64:
+    - 47ab6c4273fc3bb0cb8ec9517271d915890c5a6b0e54b2991e7a8fbbe77b06e4@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubelet
+    - 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubectl
+    - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
+    - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz
+    - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/protokube
+    - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/channels
+  ClusterName: nthsqsresources.example.com
+  ConfigBase: memfs://clusters.example.com/nthsqsresources.example.com
+  InstanceGroupName: master-us-test-1a
+  InstanceGroupRole: Master
+  KubeletConfig:
+    anonymousAuth: false
+    cgroupDriver: systemd
+    cgroupRoot: /
+    cloudProvider: aws
+    clusterDNS: 100.64.0.10
+    clusterDomain: cluster.local
+    enableDebuggingHandlers: true
+    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    hostnameOverride: '@aws'
+    kubeconfigPath: /var/lib/kubelet/kubeconfig
+    logLevel: 2
+    networkPluginName: cni
+    nodeLabels:
+      kops.k8s.io/kops-controller-pki: ""
+      kubernetes.io/role: master
+      node-role.kubernetes.io/control-plane: ""
+      node-role.kubernetes.io/master: ""
+      node.kubernetes.io/exclude-from-external-load-balancers: ""
+    nonMasqueradeCIDR: 100.64.0.0/10
+    podManifestPath: /etc/kubernetes/manifests
+    registerSchedulable: false
+  channels:
+  - memfs://clusters.example.com/nthsqsresources.example.com/addons/bootstrap-channel.yaml
+  etcdManifests:
+  - memfs://clusters.example.com/nthsqsresources.example.com/manifests/etcd/main.yaml
+  - memfs://clusters.example.com/nthsqsresources.example.com/manifests/etcd/events.yaml
+  staticManifests:
+  - key: kube-apiserver-healthcheck
+    path: manifests/static/kube-apiserver-healthcheck.yaml
+
+  __EOF_KUBE_ENV
+
+  download-release
+  echo "== nodeup node config done =="
+Resources.AWSEC2LaunchTemplatenodesnthsqsresourcesexamplecom.Properties.LaunchTemplateData.UserData: |
+  #!/bin/bash
+  set -o errexit
+  set -o nounset
+  set -o pipefail
+
+  NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/nodeup
+  NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924
+  NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/nodeup
+  NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865
+
+  export AWS_REGION=us-test-1
+
+
+
+
+  sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true
+
+
+  function ensure-install-dir() {
+    INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
+    mkdir -p ${INSTALL_DIR}/bin
+    mkdir -p ${INSTALL_DIR}/conf
+    cd ${INSTALL_DIR}
+  }
+
+  # Retry a download until we get it. args: name, sha, url1, url2...
+  download-or-bust() {
+    local -r file="$1"
+    local -r hash="$2"
+    shift 2
+
+    urls=( $* )
+    while true; do
+      for url in "${urls[@]}"; do
+        commands=(
+          "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+          "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+          "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+          "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+        )
+        for cmd in "${commands[@]}"; do
+          echo "Attempting download with: ${cmd} {url}"
+          if ! (${cmd} "${url}"); then
+            echo "== Download failed with ${cmd} =="
+            continue
+          fi
+          if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
+            echo "== Hash validation of ${url} failed. Retrying. =="
+            rm -f "${file}"
+          else
+            if [[ -n "${hash}" ]]; then
+              echo "== Downloaded ${url} (SHA1 = ${hash}) =="
+            else
+              echo "== Downloaded ${url} =="
+            fi
+            return
+          fi
+        done
+      done
+
+      echo "All downloads failed; sleeping before retrying"
+      sleep 60
+    done
+  }
+
+  validate-hash() {
+    local -r file="$1"
+    local -r expected="$2"
+    local actual
+
+    actual=$(sha256sum ${file} | awk '{ print $1 }') || true
+    if [[ "${actual}" != "${expected}" ]]; then
+      echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} =="
+      return 1
+    fi
+  }
+
+  function split-commas() {
+    echo $1 | tr "," "\n"
+  }
+
+  function try-download-release() {
+    local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
+    if [[ -n "${NODEUP_HASH:-}" ]]; then
+      local -r nodeup_hash="${NODEUP_HASH}"
+    else
+    # TODO: Remove?
+      echo "Downloading sha256 (not found in env)"
+      download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}"
+      local -r nodeup_hash=$(cat nodeup.sha256)
+    fi
+
+    echo "Downloading nodeup (${nodeup_urls[@]})"
+    download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}"
+
+    chmod +x nodeup
+  }
+
+  function download-release() {
+    case "$(uname -m)" in
+    x86_64*|i?86_64*|amd64*)
+      NODEUP_URL="${NODEUP_URL_AMD64}"
+      NODEUP_HASH="${NODEUP_HASH_AMD64}"
+      ;;
+    aarch64*|arm64*)
+      NODEUP_URL="${NODEUP_URL_ARM64}"
+      NODEUP_HASH="${NODEUP_HASH_ARM64}"
+      ;;
+    *)
+      echo "Unsupported host arch: $(uname -m)" >&2
+      exit 1
+      ;;
+    esac
+
+    # In case of failure checking integrity of release, retry.
+    cd ${INSTALL_DIR}/bin
+    until try-download-release; do
+      sleep 15
+      echo "Couldn't download release. Retrying..."
+    done
+
+    echo "Running nodeup"
+    # We can't run in the foreground because of https://github.com/docker/docker/issues/23793
+    ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8  )
+  }
+
+  ####################################################################################
+
+  /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
+
+  echo "== nodeup node config starting =="
+  ensure-install-dir
+
+  cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
+  cloudConfig:
+    manageStorageClasses: true
+  containerRuntime: containerd
+  containerd:
+    configOverride: |
+      version = 2
+
+      [plugins]
+
+        [plugins."io.containerd.grpc.v1.cri"]
+
+          [plugins."io.containerd.grpc.v1.cri".containerd]
+
+            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+                runtime_type = "io.containerd.runc.v2"
+
+                [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+                  SystemdCgroup = true
+    logLevel: info
+    version: 1.4.4
+  docker:
+    skipInstall: true
+  kubeProxy:
+    clusterCIDR: 100.96.0.0/11
+    cpuRequest: 100m
+    hostnameOverride: '@aws'
+    image: k8s.gcr.io/kube-proxy:v1.20.0
+    logLevel: 2
+  kubelet:
+    anonymousAuth: false
+    cgroupDriver: systemd
+    cgroupRoot: /
+    cloudProvider: aws
+    clusterDNS: 100.64.0.10
+    clusterDomain: cluster.local
+    enableDebuggingHandlers: true
+    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    hostnameOverride: '@aws'
+    kubeconfigPath: /var/lib/kubelet/kubeconfig
+    logLevel: 2
+    networkPluginName: cni
+    nonMasqueradeCIDR: 100.64.0.0/10
+    podManifestPath: /etc/kubernetes/manifests
+
+  __EOF_CLUSTER_SPEC
+
+  cat > conf/ig_spec.yaml << '__EOF_IG_SPEC'
+  {}
+
+  __EOF_IG_SPEC
+
+  cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
+  Assets:
+    amd64:
+    - ff2422571c4c1e9696e367f5f25466b96fb6e501f28aed29f414b1524a52dea0@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubelet
+    - a5895007f331f08d2e082eb12458764949559f30bcc5beae26c38f3e2724262c@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubectl
+    - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
+    - 96641849cb78a0a119223a427dfdc1ade88412ef791a14193212c8c8e29d447b@https://github.com/containerd/containerd/releases/download/v1.4.4/cri-containerd-cni-1.4.4-linux-amd64.tar.gz
+    arm64:
+    - 47ab6c4273fc3bb0cb8ec9517271d915890c5a6b0e54b2991e7a8fbbe77b06e4@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubelet
+    - 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubectl
+    - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
+    - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz
+  ClusterName: nthsqsresources.example.com
+  ConfigBase: memfs://clusters.example.com/nthsqsresources.example.com
+  InstanceGroupName: nodes
+  InstanceGroupRole: Node
+  KubeletConfig:
+    anonymousAuth: false
+    cgroupDriver: systemd
+    cgroupRoot: /
+    cloudProvider: aws
+    clusterDNS: 100.64.0.10
+    clusterDomain: cluster.local
+    enableDebuggingHandlers: true
+    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    hostnameOverride: '@aws'
+    kubeconfigPath: /var/lib/kubelet/kubeconfig
+    logLevel: 2
+    networkPluginName: cni
+    nodeLabels:
+      kubernetes.io/role: node
+      node-role.kubernetes.io/node: ""
+    nonMasqueradeCIDR: 100.64.0.0/10
+    podManifestPath: /etc/kubernetes/manifests
+  channels:
+  - memfs://clusters.example.com/nthsqsresources.example.com/addons/bootstrap-channel.yaml
+
+  __EOF_KUBE_ENV
+
+  download-release
+  echo "== nodeup node config done =="

tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_nthsqsresources.example.com-ASGLifecycle_event_pattern

@@ -0,0 +1 @@
+{"source":["aws.autoscaling"],"detail-type":["EC2 Instance-terminate Lifecycle Action"]}

tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_nthsqsresources.example.com-RebalanceRecommendation_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Instance Rebalance Recommendation"]}

tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_nthsqsresources.example.com-SpotInterruption_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Spot Instance Interruption Warning"]}

tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_queueprocessor.example.com-ASGLifecycle_event_pattern

@@ -0,0 +1 @@
+{"source":["aws.autoscaling"],"detail-type":["EC2 Instance-terminate Lifecycle Action"]}

tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_queueprocessor.example.com-RebalanceRecommendation_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Instance Rebalance Recommendation"]}

tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_queueprocessor.example.com-SpotInterruption_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Spot Instance Interruption Warning"]}

tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_masters.nthsqsresources.example.com_policy

@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}

tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_masters.queueprocessor.example.com_policy

@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}

tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_nodes.nthsqsresources.example.com_policy

@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}

tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_nodes.queueprocessor.example.com_policy

@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}

tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_policy_masters.nthsqsresources.example.com_policy

@@ -0,0 +1,197 @@
+{
+  "Statement": [
+    {
+      "Action": [
+        "ec2:DescribeAccountAttributes",
+        "ec2:DescribeInstances",
+        "ec2:DescribeInternetGateways",
+        "ec2:DescribeRegions",
+        "ec2:DescribeRouteTables",
+        "ec2:DescribeSecurityGroups",
+        "ec2:DescribeSubnets",
+        "ec2:DescribeVolumes"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:CreateSecurityGroup",
+        "ec2:CreateTags",
+        "ec2:CreateVolume",
+        "ec2:DescribeVolumesModifications",
+        "ec2:ModifyInstanceAttribute",
+        "ec2:ModifyVolume"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:AttachVolume",
+        "ec2:AuthorizeSecurityGroupIngress",
+        "ec2:CreateRoute",
+        "ec2:DeleteRoute",
+        "ec2:DeleteSecurityGroup",
+        "ec2:DeleteVolume",
+        "ec2:DetachVolume",
+        "ec2:RevokeSecurityGroupIngress"
+      ],
+      "Condition": {
+        "StringEquals": {
+          "ec2:ResourceTag/KubernetesCluster": "nthsqsresources.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "autoscaling:DescribeAutoScalingGroups",
+        "autoscaling:DescribeLaunchConfigurations",
+        "autoscaling:DescribeTags",
+        "ec2:DescribeLaunchTemplateVersions"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "autoscaling:SetDesiredCapacity",
+        "autoscaling:TerminateInstanceInAutoScalingGroup",
+        "autoscaling:UpdateAutoScalingGroup"
+      ],
+      "Condition": {
+        "StringEquals": {
+          "autoscaling:ResourceTag/KubernetesCluster": "nthsqsresources.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "autoscaling:CompleteLifecycleAction",
+        "autoscaling:DescribeAutoScalingInstances"
+      ],
+      "Condition": {
+        "StringEquals": {
+          "autoscaling:ResourceTag/KubernetesCluster": "nthsqsresources.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "elasticloadbalancing:AddTags",
+        "elasticloadbalancing:AttachLoadBalancerToSubnets",
+        "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer",
+        "elasticloadbalancing:CreateLoadBalancer",
+        "elasticloadbalancing:CreateLoadBalancerPolicy",
+        "elasticloadbalancing:CreateLoadBalancerListeners",
+        "elasticloadbalancing:ConfigureHealthCheck",
+        "elasticloadbalancing:DeleteLoadBalancer",
+        "elasticloadbalancing:DeleteLoadBalancerListeners",
+        "elasticloadbalancing:DescribeLoadBalancers",
+        "elasticloadbalancing:DescribeLoadBalancerAttributes",
+        "elasticloadbalancing:DetachLoadBalancerFromSubnets",
+        "elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
+        "elasticloadbalancing:ModifyLoadBalancerAttributes",
+        "elasticloadbalancing:RegisterInstancesWithLoadBalancer",
+        "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:DescribeVpcs",
+        "elasticloadbalancing:AddTags",
+        "elasticloadbalancing:CreateListener",
+        "elasticloadbalancing:CreateTargetGroup",
+        "elasticloadbalancing:DeleteListener",
+        "elasticloadbalancing:DeleteTargetGroup",
+        "elasticloadbalancing:DeregisterTargets",
+        "elasticloadbalancing:DescribeListeners",
+        "elasticloadbalancing:DescribeLoadBalancerPolicies",
+        "elasticloadbalancing:DescribeTargetGroups",
+        "elasticloadbalancing:DescribeTargetHealth",
+        "elasticloadbalancing:ModifyListener",
+        "elasticloadbalancing:ModifyTargetGroup",
+        "elasticloadbalancing:RegisterTargets",
+        "elasticloadbalancing:SetLoadBalancerPoliciesOfListener"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "iam:ListServerCertificates",
+        "iam:GetServerCertificate"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "route53:ChangeResourceRecordSets",
+        "route53:ListResourceRecordSets",
+        "route53:GetHostedZone"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO"
+      ]
+    },
+    {
+      "Action": [
+        "route53:GetChange"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws:route53:::change/*"
+      ]
+    },
+    {
+      "Action": [
+        "route53:ListHostedZones"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "autoscaling:CompleteLifecycleAction",
+        "autoscaling:DescribeAutoScalingInstances",
+        "sqs:DeleteMessage",
+        "sqs:ReceiveMessage"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_policy_masters.queueprocessor.example.com_policy

@@ -0,0 +1,182 @@
+{
+  "Statement": [
+    {
+      "Action": [
+        "ec2:DescribeAccountAttributes",
+        "ec2:DescribeInstances",
+        "ec2:DescribeInternetGateways",
+        "ec2:DescribeRegions",
+        "ec2:DescribeRouteTables",
+        "ec2:DescribeSecurityGroups",
+        "ec2:DescribeSubnets",
+        "ec2:DescribeVolumes"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:CreateSecurityGroup",
+        "ec2:CreateTags",
+        "ec2:CreateVolume",
+        "ec2:DescribeVolumesModifications",
+        "ec2:ModifyInstanceAttribute",
+        "ec2:ModifyVolume"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:AttachVolume",
+        "ec2:AuthorizeSecurityGroupIngress",
+        "ec2:CreateRoute",
+        "ec2:DeleteRoute",
+        "ec2:DeleteSecurityGroup",
+        "ec2:DeleteVolume",
+        "ec2:DetachVolume",
+        "ec2:RevokeSecurityGroupIngress"
+      ],
+      "Condition": {
+        "StringEquals": {
+          "ec2:ResourceTag/KubernetesCluster": "queueprocessor.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "autoscaling:DescribeAutoScalingGroups",
+        "autoscaling:DescribeLaunchConfigurations",
+        "autoscaling:DescribeTags",
+        "ec2:DescribeLaunchTemplateVersions"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "autoscaling:SetDesiredCapacity",
+        "autoscaling:TerminateInstanceInAutoScalingGroup",
+        "autoscaling:UpdateAutoScalingGroup"
+      ],
+      "Condition": {
+        "StringEquals": {
+          "autoscaling:ResourceTag/KubernetesCluster": "queueprocessor.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "elasticloadbalancing:AddTags",
+        "elasticloadbalancing:AttachLoadBalancerToSubnets",
+        "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer",
+        "elasticloadbalancing:CreateLoadBalancer",
+        "elasticloadbalancing:CreateLoadBalancerPolicy",
+        "elasticloadbalancing:CreateLoadBalancerListeners",
+        "elasticloadbalancing:ConfigureHealthCheck",
+        "elasticloadbalancing:DeleteLoadBalancer",
+        "elasticloadbalancing:DeleteLoadBalancerListeners",
+        "elasticloadbalancing:DescribeLoadBalancers",
+        "elasticloadbalancing:DescribeLoadBalancerAttributes",
+        "elasticloadbalancing:DetachLoadBalancerFromSubnets",
+        "elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
+        "elasticloadbalancing:ModifyLoadBalancerAttributes",
+        "elasticloadbalancing:RegisterInstancesWithLoadBalancer",
+        "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:DescribeVpcs",
+        "elasticloadbalancing:AddTags",
+        "elasticloadbalancing:CreateListener",
+        "elasticloadbalancing:CreateTargetGroup",
+        "elasticloadbalancing:DeleteListener",
+        "elasticloadbalancing:DeleteTargetGroup",
+        "elasticloadbalancing:DeregisterTargets",
+        "elasticloadbalancing:DescribeListeners",
+        "elasticloadbalancing:DescribeLoadBalancerPolicies",
+        "elasticloadbalancing:DescribeTargetGroups",
+        "elasticloadbalancing:DescribeTargetHealth",
+        "elasticloadbalancing:ModifyListener",
+        "elasticloadbalancing:ModifyTargetGroup",
+        "elasticloadbalancing:RegisterTargets",
+        "elasticloadbalancing:SetLoadBalancerPoliciesOfListener"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "iam:ListServerCertificates",
+        "iam:GetServerCertificate"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "route53:ChangeResourceRecordSets",
+        "route53:ListResourceRecordSets",
+        "route53:GetHostedZone"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO"
+      ]
+    },
+    {
+      "Action": [
+        "route53:GetChange"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws:route53:::change/*"
+      ]
+    },
+    {
+      "Action": [
+        "route53:ListHostedZones"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": [
+        "autoscaling:CompleteLifecycleAction",
+        "autoscaling:DescribeAutoScalingInstances",
+        "sqs:DeleteMessage",
+        "sqs:ReceiveMessage"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_policy_nodes.nthsqsresources.example.com_policy

@@ -0,0 +1,22 @@
+{
+  "Statement": [
+    {
+      "Action": [
+        "ec2:DescribeInstances",
+        "ec2:DescribeRegions"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Action": "autoscaling:DescribeAutoScalingInstances",
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_policy_nodes.queueprocessor.example.com_policy

@@ -0,0 +1,15 @@
+{
+  "Statement": [
+    {
+      "Action": [
+        "ec2:DescribeInstances",
+        "ec2:DescribeRegions"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "*"
+      ]
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/nth_sqs_resources/data/aws_key_pair_kubernetes.nthsqsresources.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==

tests/integration/update_cluster/nth_sqs_resources/data/aws_key_pair_kubernetes.queueprocessor.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==

tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_master-us-test-1a.masters.nthsqsresources.example.com_user_data

@@ -0,0 +1,328 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/nodeup
+NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924
+NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/nodeup
+NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865
+
+export AWS_REGION=us-test-1
+
+
+
+
+sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true
+
+
+function ensure-install-dir() {
+  INSTALL_DIR="/opt/kops"
+  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+  if [[ -d /var/lib/toolbox ]]; then
+    INSTALL_DIR="/var/lib/toolbox/kops"
+  fi
+  mkdir -p ${INSTALL_DIR}/bin
+  mkdir -p ${INSTALL_DIR}/conf
+  cd ${INSTALL_DIR}
+}
+
+# Retry a download until we get it. args: name, sha, url1, url2...
+download-or-bust() {
+  local -r file="$1"
+  local -r hash="$2"
+  shift 2
+
+  urls=( $* )
+  while true; do
+    for url in "${urls[@]}"; do
+      commands=(
+        "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+        "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+      )
+      for cmd in "${commands[@]}"; do
+        echo "Attempting download with: ${cmd} {url}"
+        if ! (${cmd} "${url}"); then
+          echo "== Download failed with ${cmd} =="
+          continue
+        fi
+        if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
+          echo "== Hash validation of ${url} failed. Retrying. =="
+          rm -f "${file}"
+        else
+          if [[ -n "${hash}" ]]; then
+            echo "== Downloaded ${url} (SHA1 = ${hash}) =="
+          else
+            echo "== Downloaded ${url} =="
+          fi
+          return
+        fi
+      done
+    done
+
+    echo "All downloads failed; sleeping before retrying"
+    sleep 60
+  done
+}
+
+validate-hash() {
+  local -r file="$1"
+  local -r expected="$2"
+  local actual
+
+  actual=$(sha256sum ${file} | awk '{ print $1 }') || true
+  if [[ "${actual}" != "${expected}" ]]; then
+    echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} =="
+    return 1
+  fi
+}
+
+function split-commas() {
+  echo $1 | tr "," "\n"
+}
+
+function try-download-release() {
+  local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
+  if [[ -n "${NODEUP_HASH:-}" ]]; then
+    local -r nodeup_hash="${NODEUP_HASH}"
+  else
+  # TODO: Remove?
+    echo "Downloading sha256 (not found in env)"
+    download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}"
+    local -r nodeup_hash=$(cat nodeup.sha256)
+  fi
+
+  echo "Downloading nodeup (${nodeup_urls[@]})"
+  download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}"
+
+  chmod +x nodeup
+}
+
+function download-release() {
+  case "$(uname -m)" in
+  x86_64*|i?86_64*|amd64*)
+    NODEUP_URL="${NODEUP_URL_AMD64}"
+    NODEUP_HASH="${NODEUP_HASH_AMD64}"
+    ;;
+  aarch64*|arm64*)
+    NODEUP_URL="${NODEUP_URL_ARM64}"
+    NODEUP_HASH="${NODEUP_HASH_ARM64}"
+    ;;
+  *)
+    echo "Unsupported host arch: $(uname -m)" >&2
+    exit 1
+    ;;
+  esac
+
+  # In case of failure checking integrity of release, retry.
+  cd ${INSTALL_DIR}/bin
+  until try-download-release; do
+    sleep 15
+    echo "Couldn't download release. Retrying..."
+  done
+
+  echo "Running nodeup"
+  # We can't run in the foreground because of https://github.com/docker/docker/issues/23793
+  ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8  )
+}
+
+####################################################################################
+
+/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
+
+echo "== nodeup node config starting =="
+ensure-install-dir
+
+cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
+cloudConfig:
+  manageStorageClasses: true
+containerRuntime: containerd
+containerd:
+  configOverride: |
+    version = 2
+
+    [plugins]
+
+      [plugins."io.containerd.grpc.v1.cri"]
+
+        [plugins."io.containerd.grpc.v1.cri".containerd]
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+              runtime_type = "io.containerd.runc.v2"
+
+              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+                SystemdCgroup = true
+  logLevel: info
+  version: 1.4.4
+docker:
+  skipInstall: true
+encryptionConfig: null
+etcdClusters:
+  events:
+    version: 3.4.13
+  main:
+    version: 3.4.13
+kubeAPIServer:
+  allowPrivileged: true
+  anonymousAuth: false
+  apiAudiences:
+  - kubernetes.svc.default
+  apiServerCount: 1
+  authorizationMode: AlwaysAllow
+  bindAddress: 0.0.0.0
+  cloudProvider: aws
+  enableAdmissionPlugins:
+  - NamespaceLifecycle
+  - LimitRanger
+  - ServiceAccount
+  - PersistentVolumeLabel
+  - DefaultStorageClass
+  - DefaultTolerationSeconds
+  - MutatingAdmissionWebhook
+  - ValidatingAdmissionWebhook
+  - NodeRestriction
+  - ResourceQuota
+  etcdServers:
+  - http://127.0.0.1:4001
+  etcdServersOverrides:
+  - /events#http://127.0.0.1:4002
+  image: k8s.gcr.io/kube-apiserver:v1.20.0
+  kubeletPreferredAddressTypes:
+  - InternalIP
+  - Hostname
+  - ExternalIP
+  logLevel: 2
+  requestheaderAllowedNames:
+  - aggregator
+  requestheaderExtraHeaderPrefixes:
+  - X-Remote-Extra-
+  requestheaderGroupHeaders:
+  - X-Remote-Group
+  requestheaderUsernameHeaders:
+  - X-Remote-User
+  securePort: 443
+  serviceAccountIssuer: https://api.internal.nthsqsresources.example.com
+  serviceAccountJWKSURI: https://api.internal.nthsqsresources.example.com/openid/v1/jwks
+  serviceClusterIPRange: 100.64.0.0/13
+  storageBackend: etcd3
+kubeControllerManager:
+  allocateNodeCIDRs: true
+  attachDetachReconcileSyncPeriod: 1m0s
+  cloudProvider: aws
+  clusterCIDR: 100.96.0.0/11
+  clusterName: nthsqsresources.example.com
+  configureCloudRoutes: false
+  image: k8s.gcr.io/kube-controller-manager:v1.20.0
+  leaderElection:
+    leaderElect: true
+  logLevel: 2
+  useServiceAccountCredentials: true
+kubeProxy:
+  clusterCIDR: 100.96.0.0/11
+  cpuRequest: 100m
+  hostnameOverride: '@aws'
+  image: k8s.gcr.io/kube-proxy:v1.20.0
+  logLevel: 2
+kubeScheduler:
+  image: k8s.gcr.io/kube-scheduler:v1.20.0
+  leaderElection:
+    leaderElect: true
+  logLevel: 2
+kubelet:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginName: cni
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podManifestPath: /etc/kubernetes/manifests
+masterKubelet:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginName: cni
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podManifestPath: /etc/kubernetes/manifests
+  registerSchedulable: false
+
+__EOF_CLUSTER_SPEC
+
+cat > conf/ig_spec.yaml << '__EOF_IG_SPEC'
+{}
+
+__EOF_IG_SPEC
+
+cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
+Assets:
+  amd64:
+  - ff2422571c4c1e9696e367f5f25466b96fb6e501f28aed29f414b1524a52dea0@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubelet
+  - a5895007f331f08d2e082eb12458764949559f30bcc5beae26c38f3e2724262c@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubectl
+  - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
+  - 96641849cb78a0a119223a427dfdc1ade88412ef791a14193212c8c8e29d447b@https://github.com/containerd/containerd/releases/download/v1.4.4/cri-containerd-cni-1.4.4-linux-amd64.tar.gz
+  - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/protokube
+  - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/channels
+  arm64:
+  - 47ab6c4273fc3bb0cb8ec9517271d915890c5a6b0e54b2991e7a8fbbe77b06e4@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubelet
+  - 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubectl
+  - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
+  - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz
+  - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/protokube
+  - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/channels
+ClusterName: nthsqsresources.example.com
+ConfigBase: memfs://clusters.example.com/nthsqsresources.example.com
+InstanceGroupName: master-us-test-1a
+InstanceGroupRole: Master
+KubeletConfig:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginName: cni
+  nodeLabels:
+    kops.k8s.io/kops-controller-pki: ""
+    kubernetes.io/role: master
+    node-role.kubernetes.io/control-plane: ""
+    node-role.kubernetes.io/master: ""
+    node.kubernetes.io/exclude-from-external-load-balancers: ""
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podManifestPath: /etc/kubernetes/manifests
+  registerSchedulable: false
+channels:
+- memfs://clusters.example.com/nthsqsresources.example.com/addons/bootstrap-channel.yaml
+etcdManifests:
+- memfs://clusters.example.com/nthsqsresources.example.com/manifests/etcd/main.yaml
+- memfs://clusters.example.com/nthsqsresources.example.com/manifests/etcd/events.yaml
+staticManifests:
+- key: kube-apiserver-healthcheck
+  path: manifests/static/kube-apiserver-healthcheck.yaml
+
+__EOF_KUBE_ENV
+
+download-release
+echo "== nodeup node config done =="

tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_master-us-test-1a.masters.queueprocessor.example.com_user_data

@@ -0,0 +1,328 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/nodeup
+NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924
+NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/nodeup
+NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865
+
+export AWS_REGION=us-test-1
+
+
+
+
+sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true
+
+
+function ensure-install-dir() {
+  INSTALL_DIR="/opt/kops"
+  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+  if [[ -d /var/lib/toolbox ]]; then
+    INSTALL_DIR="/var/lib/toolbox/kops"
+  fi
+  mkdir -p ${INSTALL_DIR}/bin
+  mkdir -p ${INSTALL_DIR}/conf
+  cd ${INSTALL_DIR}
+}
+
+# Retry a download until we get it. args: name, sha, url1, url2...
+download-or-bust() {
+  local -r file="$1"
+  local -r hash="$2"
+  shift 2
+
+  urls=( $* )
+  while true; do
+    for url in "${urls[@]}"; do
+      commands=(
+        "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+        "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+      )
+      for cmd in "${commands[@]}"; do
+        echo "Attempting download with: ${cmd} {url}"
+        if ! (${cmd} "${url}"); then
+          echo "== Download failed with ${cmd} =="
+          continue
+        fi
+        if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
+          echo "== Hash validation of ${url} failed. Retrying. =="
+          rm -f "${file}"
+        else
+          if [[ -n "${hash}" ]]; then
+            echo "== Downloaded ${url} (SHA1 = ${hash}) =="
+          else
+            echo "== Downloaded ${url} =="
+          fi
+          return
+        fi
+      done
+    done
+
+    echo "All downloads failed; sleeping before retrying"
+    sleep 60
+  done
+}
+
+validate-hash() {
+  local -r file="$1"
+  local -r expected="$2"
+  local actual
+
+  actual=$(sha256sum ${file} | awk '{ print $1 }') || true
+  if [[ "${actual}" != "${expected}" ]]; then
+    echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} =="
+    return 1
+  fi
+}
+
+function split-commas() {
+  echo $1 | tr "," "\n"
+}
+
+function try-download-release() {
+  local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
+  if [[ -n "${NODEUP_HASH:-}" ]]; then
+    local -r nodeup_hash="${NODEUP_HASH}"
+  else
+  # TODO: Remove?
+    echo "Downloading sha256 (not found in env)"
+    download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}"
+    local -r nodeup_hash=$(cat nodeup.sha256)
+  fi
+
+  echo "Downloading nodeup (${nodeup_urls[@]})"
+  download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}"
+
+  chmod +x nodeup
+}
+
+function download-release() {
+  case "$(uname -m)" in
+  x86_64*|i?86_64*|amd64*)
+    NODEUP_URL="${NODEUP_URL_AMD64}"
+    NODEUP_HASH="${NODEUP_HASH_AMD64}"
+    ;;
+  aarch64*|arm64*)
+    NODEUP_URL="${NODEUP_URL_ARM64}"
+    NODEUP_HASH="${NODEUP_HASH_ARM64}"
+    ;;
+  *)
+    echo "Unsupported host arch: $(uname -m)" >&2
+    exit 1
+    ;;
+  esac
+
+  # In case of failure checking integrity of release, retry.
+  cd ${INSTALL_DIR}/bin
+  until try-download-release; do
+    sleep 15
+    echo "Couldn't download release. Retrying..."
+  done
+
+  echo "Running nodeup"
+  # We can't run in the foreground because of https://github.com/docker/docker/issues/23793
+  ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8  )
+}
+
+####################################################################################
+
+/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
+
+echo "== nodeup node config starting =="
+ensure-install-dir
+
+cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
+cloudConfig:
+  manageStorageClasses: true
+containerRuntime: containerd
+containerd:
+  configOverride: |
+    version = 2
+
+    [plugins]
+
+      [plugins."io.containerd.grpc.v1.cri"]
+
+        [plugins."io.containerd.grpc.v1.cri".containerd]
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+              runtime_type = "io.containerd.runc.v2"
+
+              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+                SystemdCgroup = true
+  logLevel: info
+  version: 1.4.4
+docker:
+  skipInstall: true
+encryptionConfig: null
+etcdClusters:
+  events:
+    version: 3.4.13
+  main:
+    version: 3.4.13
+kubeAPIServer:
+  allowPrivileged: true
+  anonymousAuth: false
+  apiAudiences:
+  - kubernetes.svc.default
+  apiServerCount: 1
+  authorizationMode: AlwaysAllow
+  bindAddress: 0.0.0.0
+  cloudProvider: aws
+  enableAdmissionPlugins:
+  - NamespaceLifecycle
+  - LimitRanger
+  - ServiceAccount
+  - PersistentVolumeLabel
+  - DefaultStorageClass
+  - DefaultTolerationSeconds
+  - MutatingAdmissionWebhook
+  - ValidatingAdmissionWebhook
+  - NodeRestriction
+  - ResourceQuota
+  etcdServers:
+  - http://127.0.0.1:4001
+  etcdServersOverrides:
+  - /events#http://127.0.0.1:4002
+  image: k8s.gcr.io/kube-apiserver:v1.20.0
+  kubeletPreferredAddressTypes:
+  - InternalIP
+  - Hostname
+  - ExternalIP
+  logLevel: 2
+  requestheaderAllowedNames:
+  - aggregator
+  requestheaderExtraHeaderPrefixes:
+  - X-Remote-Extra-
+  requestheaderGroupHeaders:
+  - X-Remote-Group
+  requestheaderUsernameHeaders:
+  - X-Remote-User
+  securePort: 443
+  serviceAccountIssuer: https://api.internal.queueprocessor.example.com
+  serviceAccountJWKSURI: https://api.internal.queueprocessor.example.com/openid/v1/jwks
+  serviceClusterIPRange: 100.64.0.0/13
+  storageBackend: etcd3
+kubeControllerManager:
+  allocateNodeCIDRs: true
+  attachDetachReconcileSyncPeriod: 1m0s
+  cloudProvider: aws
+  clusterCIDR: 100.96.0.0/11
+  clusterName: queueprocessor.example.com
+  configureCloudRoutes: false
+  image: k8s.gcr.io/kube-controller-manager:v1.20.0
+  leaderElection:
+    leaderElect: true
+  logLevel: 2
+  useServiceAccountCredentials: true
+kubeProxy:
+  clusterCIDR: 100.96.0.0/11
+  cpuRequest: 100m
+  hostnameOverride: '@aws'
+  image: k8s.gcr.io/kube-proxy:v1.20.0
+  logLevel: 2
+kubeScheduler:
+  image: k8s.gcr.io/kube-scheduler:v1.20.0
+  leaderElection:
+    leaderElect: true
+  logLevel: 2
+kubelet:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginName: cni
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podManifestPath: /etc/kubernetes/manifests
+masterKubelet:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginName: cni
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podManifestPath: /etc/kubernetes/manifests
+  registerSchedulable: false
+
+__EOF_CLUSTER_SPEC
+
+cat > conf/ig_spec.yaml << '__EOF_IG_SPEC'
+{}
+
+__EOF_IG_SPEC
+
+cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
+Assets:
+  amd64:
+  - ff2422571c4c1e9696e367f5f25466b96fb6e501f28aed29f414b1524a52dea0@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubelet
+  - a5895007f331f08d2e082eb12458764949559f30bcc5beae26c38f3e2724262c@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubectl
+  - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
+  - 96641849cb78a0a119223a427dfdc1ade88412ef791a14193212c8c8e29d447b@https://github.com/containerd/containerd/releases/download/v1.4.4/cri-containerd-cni-1.4.4-linux-amd64.tar.gz
+  - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/protokube
+  - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/channels
+  arm64:
+  - 47ab6c4273fc3bb0cb8ec9517271d915890c5a6b0e54b2991e7a8fbbe77b06e4@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubelet
+  - 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubectl
+  - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
+  - 6e3f80e8451ecbe7b3559247721c3e226be6b228acaadee7e13683f80c20e81c@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.0.tgz
+  - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/protokube
+  - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/channels
+ClusterName: queueprocessor.example.com
+ConfigBase: memfs://clusters.example.com/queueprocessor.example.com
+InstanceGroupName: master-us-test-1a
+InstanceGroupRole: Master
+KubeletConfig:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginName: cni
+  nodeLabels:
+    kops.k8s.io/kops-controller-pki: ""
+    kubernetes.io/role: master
+    node-role.kubernetes.io/control-plane: ""
+    node-role.kubernetes.io/master: ""
+    node.kubernetes.io/exclude-from-external-load-balancers: ""
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podManifestPath: /etc/kubernetes/manifests
+  registerSchedulable: false
+channels:
+- memfs://clusters.example.com/queueprocessor.example.com/addons/bootstrap-channel.yaml
+etcdManifests:
+- memfs://clusters.example.com/queueprocessor.example.com/manifests/etcd/main.yaml
+- memfs://clusters.example.com/queueprocessor.example.com/manifests/etcd/events.yaml
+staticManifests:
+- key: kube-apiserver-healthcheck
+  path: manifests/static/kube-apiserver-healthcheck.yaml
+
+__EOF_KUBE_ENV
+
+download-release
+echo "== nodeup node config done =="

tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_nodes.nthsqsresources.example.com_user_data

@@ -0,0 +1,232 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/nodeup
+NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924
+NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/nodeup
+NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865
+
+export AWS_REGION=us-test-1
+
+
+
+
+sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true
+
+
+function ensure-install-dir() {
+  INSTALL_DIR="/opt/kops"
+  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+  if [[ -d /var/lib/toolbox ]]; then
+    INSTALL_DIR="/var/lib/toolbox/kops"
+  fi
+  mkdir -p ${INSTALL_DIR}/bin
+  mkdir -p ${INSTALL_DIR}/conf
+  cd ${INSTALL_DIR}
+}
+
+# Retry a download until we get it. args: name, sha, url1, url2...
+download-or-bust() {
+  local -r file="$1"
+  local -r hash="$2"
+  shift 2
+
+  urls=( $* )
+  while true; do
+    for url in "${urls[@]}"; do
+      commands=(
+        "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+        "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+      )
+      for cmd in "${commands[@]}"; do
+        echo "Attempting download with: ${cmd} {url}"
+        if ! (${cmd} "${url}"); then
+          echo "== Download failed with ${cmd} =="
+          continue
+        fi
+        if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
+          echo "== Hash validation of ${url} failed. Retrying. =="
+          rm -f "${file}"
+        else
+          if [[ -n "${hash}" ]]; then
+            echo "== Downloaded ${url} (SHA1 = ${hash}) =="
+          else
+            echo "== Downloaded ${url} =="
+          fi
+          return
+        fi
+      done
+    done
+
+    echo "All downloads failed; sleeping before retrying"
+    sleep 60
+  done
+}
+
+validate-hash() {
+  local -r file="$1"
+  local -r expected="$2"
+  local actual
+
+  actual=$(sha256sum ${file} | awk '{ print $1 }') || true
+  if [[ "${actual}" != "${expected}" ]]; then
+    echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} =="
+    return 1
+  fi
+}
+
+function split-commas() {
+  echo $1 | tr "," "\n"
+}
+
+function try-download-release() {
+  local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
+  if [[ -n "${NODEUP_HASH:-}" ]]; then
+    local -r nodeup_hash="${NODEUP_HASH}"
+  else
+  # TODO: Remove?
+    echo "Downloading sha256 (not found in env)"
+    download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}"
+    local -r nodeup_hash=$(cat nodeup.sha256)
+  fi
+
+  echo "Downloading nodeup (${nodeup_urls[@]})"
+  download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}"
+
+  chmod +x nodeup
+}
+
+function download-release() {
+  case "$(uname -m)" in
+  x86_64*|i?86_64*|amd64*)
+    NODEUP_URL="${NODEUP_URL_AMD64}"
+    NODEUP_HASH="${NODEUP_HASH_AMD64}"
+    ;;
+  aarch64*|arm64*)
+    NODEUP_URL="${NODEUP_URL_ARM64}"
+    NODEUP_HASH="${NODEUP_HASH_ARM64}"
+    ;;
+  *)
+    echo "Unsupported host arch: $(uname -m)" >&2
+    exit 1
+    ;;
+  esac
+
+  # In case of failure checking integrity of release, retry.
+  cd ${INSTALL_DIR}/bin
+  until try-download-release; do
+    sleep 15
+    echo "Couldn't download release. Retrying..."
+  done
+
+  echo "Running nodeup"
+  # We can't run in the foreground because of https://github.com/docker/docker/issues/23793
+  ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8  )
+}
+
+####################################################################################
+
+/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
+
+echo "== nodeup node config starting =="
+ensure-install-dir
+
+cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
+cloudConfig:
+  manageStorageClasses: true
+containerRuntime: containerd
+containerd:
+  configOverride: |
+    version = 2
+
+    [plugins]
+
+      [plugins."io.containerd.grpc.v1.cri"]
+
+        [plugins."io.containerd.grpc.v1.cri".containerd]
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+              runtime_type = "io.containerd.runc.v2"
+
+              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+                SystemdCgroup = true
+  logLevel: info
+  version: 1.4.4
+docker:
+  skipInstall: true
+kubeProxy:
+  clusterCIDR: 100.96.0.0/11
+  cpuRequest: 100m
+  hostnameOverride: '@aws'
+  image: k8s.gcr.io/kube-proxy:v1.20.0
+  logLevel: 2
+kubelet:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginName: cni
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podManifestPath: /etc/kubernetes/manifests
+
+__EOF_CLUSTER_SPEC
+
+cat > conf/ig_spec.yaml << '__EOF_IG_SPEC'
+{}
+
+__EOF_IG_SPEC
+
+cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
+Assets:
+  amd64:
+  - ff2422571c4c1e9696e367f5f25466b96fb6e501f28aed29f414b1524a52dea0@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubelet
+  - a5895007f331f08d2e082eb12458764949559f30bcc5beae26c38f3e2724262c@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubectl
+  - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
+  - 96641849cb78a0a119223a427dfdc1ade88412ef791a14193212c8c8e29d447b@https://github.com/containerd/containerd/releases/download/v1.4.4/cri-containerd-cni-1.4.4-linux-amd64.tar.gz
+  arm64:
+  - 47ab6c4273fc3bb0cb8ec9517271d915890c5a6b0e54b2991e7a8fbbe77b06e4@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubelet
+  - 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubectl
+  - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
+  - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz
+ClusterName: nthsqsresources.example.com
+ConfigBase: memfs://clusters.example.com/nthsqsresources.example.com
+InstanceGroupName: nodes
+InstanceGroupRole: Node
+KubeletConfig:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginName: cni
+  nodeLabels:
+    kubernetes.io/role: node
+    node-role.kubernetes.io/node: ""
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podManifestPath: /etc/kubernetes/manifests
+channels:
+- memfs://clusters.example.com/nthsqsresources.example.com/addons/bootstrap-channel.yaml
+
+__EOF_KUBE_ENV
+
+download-release
+echo "== nodeup node config done =="

tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_nodes.queueprocessor.example.com_user_data

@@ -0,0 +1,232 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/nodeup
+NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924
+NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/nodeup
+NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865
+
+export AWS_REGION=us-test-1
+
+
+
+
+sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true
+
+
+function ensure-install-dir() {
+  INSTALL_DIR="/opt/kops"
+  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+  if [[ -d /var/lib/toolbox ]]; then
+    INSTALL_DIR="/var/lib/toolbox/kops"
+  fi
+  mkdir -p ${INSTALL_DIR}/bin
+  mkdir -p ${INSTALL_DIR}/conf
+  cd ${INSTALL_DIR}
+}
+
+# Retry a download until we get it. args: name, sha, url1, url2...
+download-or-bust() {
+  local -r file="$1"
+  local -r hash="$2"
+  shift 2
+
+  urls=( $* )
+  while true; do
+    for url in "${urls[@]}"; do
+      commands=(
+        "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+        "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+        "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+      )
+      for cmd in "${commands[@]}"; do
+        echo "Attempting download with: ${cmd} {url}"
+        if ! (${cmd} "${url}"); then
+          echo "== Download failed with ${cmd} =="
+          continue
+        fi
+        if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
+          echo "== Hash validation of ${url} failed. Retrying. =="
+          rm -f "${file}"
+        else
+          if [[ -n "${hash}" ]]; then
+            echo "== Downloaded ${url} (SHA1 = ${hash}) =="
+          else
+            echo "== Downloaded ${url} =="
+          fi
+          return
+        fi
+      done
+    done
+
+    echo "All downloads failed; sleeping before retrying"
+    sleep 60
+  done
+}
+
+validate-hash() {
+  local -r file="$1"
+  local -r expected="$2"
+  local actual
+
+  actual=$(sha256sum ${file} | awk '{ print $1 }') || true
+  if [[ "${actual}" != "${expected}" ]]; then
+    echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} =="
+    return 1
+  fi
+}
+
+function split-commas() {
+  echo $1 | tr "," "\n"
+}
+
+function try-download-release() {
+  local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
+  if [[ -n "${NODEUP_HASH:-}" ]]; then
+    local -r nodeup_hash="${NODEUP_HASH}"
+  else
+  # TODO: Remove?
+    echo "Downloading sha256 (not found in env)"
+    download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}"
+    local -r nodeup_hash=$(cat nodeup.sha256)
+  fi
+
+  echo "Downloading nodeup (${nodeup_urls[@]})"
+  download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}"
+
+  chmod +x nodeup
+}
+
+function download-release() {
+  case "$(uname -m)" in
+  x86_64*|i?86_64*|amd64*)
+    NODEUP_URL="${NODEUP_URL_AMD64}"
+    NODEUP_HASH="${NODEUP_HASH_AMD64}"
+    ;;
+  aarch64*|arm64*)
+    NODEUP_URL="${NODEUP_URL_ARM64}"
+    NODEUP_HASH="${NODEUP_HASH_ARM64}"
+    ;;
+  *)
+    echo "Unsupported host arch: $(uname -m)" >&2
+    exit 1
+    ;;
+  esac
+
+  # In case of failure checking integrity of release, retry.
+  cd ${INSTALL_DIR}/bin
+  until try-download-release; do
+    sleep 15
+    echo "Couldn't download release. Retrying..."
+  done
+
+  echo "Running nodeup"
+  # We can't run in the foreground because of https://github.com/docker/docker/issues/23793
+  ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8  )
+}
+
+####################################################################################
+
+/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
+
+echo "== nodeup node config starting =="
+ensure-install-dir
+
+cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
+cloudConfig:
+  manageStorageClasses: true
+containerRuntime: containerd
+containerd:
+  configOverride: |
+    version = 2
+
+    [plugins]
+
+      [plugins."io.containerd.grpc.v1.cri"]
+
+        [plugins."io.containerd.grpc.v1.cri".containerd]
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+              runtime_type = "io.containerd.runc.v2"
+
+              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+                SystemdCgroup = true
+  logLevel: info
+  version: 1.4.4
+docker:
+  skipInstall: true
+kubeProxy:
+  clusterCIDR: 100.96.0.0/11
+  cpuRequest: 100m
+  hostnameOverride: '@aws'
+  image: k8s.gcr.io/kube-proxy:v1.20.0
+  logLevel: 2
+kubelet:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginName: cni
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podManifestPath: /etc/kubernetes/manifests
+
+__EOF_CLUSTER_SPEC
+
+cat > conf/ig_spec.yaml << '__EOF_IG_SPEC'
+{}
+
+__EOF_IG_SPEC
+
+cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
+Assets:
+  amd64:
+  - ff2422571c4c1e9696e367f5f25466b96fb6e501f28aed29f414b1524a52dea0@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubelet
+  - a5895007f331f08d2e082eb12458764949559f30bcc5beae26c38f3e2724262c@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubectl
+  - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
+  - 96641849cb78a0a119223a427dfdc1ade88412ef791a14193212c8c8e29d447b@https://github.com/containerd/containerd/releases/download/v1.4.4/cri-containerd-cni-1.4.4-linux-amd64.tar.gz
+  arm64:
+  - 47ab6c4273fc3bb0cb8ec9517271d915890c5a6b0e54b2991e7a8fbbe77b06e4@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubelet
+  - 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubectl
+  - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
+  - 6e3f80e8451ecbe7b3559247721c3e226be6b228acaadee7e13683f80c20e81c@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.0.tgz
+ClusterName: queueprocessor.example.com
+ConfigBase: memfs://clusters.example.com/queueprocessor.example.com
+InstanceGroupName: nodes
+InstanceGroupRole: Node
+KubeletConfig:
+  anonymousAuth: false
+  cgroupDriver: systemd
+  cgroupRoot: /
+  cloudProvider: aws
+  clusterDNS: 100.64.0.10
+  clusterDomain: cluster.local
+  enableDebuggingHandlers: true
+  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  hostnameOverride: '@aws'
+  kubeconfigPath: /var/lib/kubelet/kubeconfig
+  logLevel: 2
+  networkPluginName: cni
+  nodeLabels:
+    kubernetes.io/role: node
+    node-role.kubernetes.io/node: ""
+  nonMasqueradeCIDR: 100.64.0.0/10
+  podManifestPath: /etc/kubernetes/manifests
+channels:
+- memfs://clusters.example.com/queueprocessor.example.com/addons/bootstrap-channel.yaml
+
+__EOF_KUBE_ENV
+
+download-release
+echo "== nodeup node config done =="

tests/integration/update_cluster/nth_sqs_resources/data/aws_sqs_queue_nthsqsresources-example-com-nth_policy

@@ -0,0 +1,13 @@
+{
+		"Version": "2012-10-17",
+		"Statement": [{                     
+			"Effect": "Allow",
+			"Principal": {
+				"Service": ["events.amazonaws.com", "sqs.amazonaws.com"]
+			},
+			"Action": "sqs:SendMessage",
+			"Resource": [
+				"arn:aws:sqs:us-test-1:123456789012:nthsqsresources-example-com-nth"
+			]
+		}]
+	}

tests/integration/update_cluster/nth_sqs_resources/data/aws_sqs_queue_queueprocessor-example-com-nth_policy

@@ -0,0 +1,13 @@
+{
+		"Version": "2012-10-17",
+		"Statement": [{                     
+			"Effect": "Allow",
+			"Principal": {
+				"Service": ["events.amazonaws.com", "sqs.amazonaws.com"]
+			},
+			"Action": "sqs:SendMessage",
+			"Resource": [
+				"arn:aws:sqs:us-test-1:123456789012:queueprocessor-example-com-nth"
+			]
+		}]
+	}

tests/integration/update_cluster/nth_sqs_resources/id_rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==

tests/integration/update_cluster/nth_sqs_resources/in-v1alpha2.yaml

@@ -0,0 +1,81 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  creationTimestamp: "2016-12-10T22:42:27Z"
+  name: nthsqsresources.example.com
+spec:
+  kubernetesApiAccess:
+    - 0.0.0.0/0
+  channel: stable
+  cloudProvider: aws
+  configBase: memfs://clusters.example.com/nthsqsresources.example.com
+  etcdClusters:
+    - etcdMembers:
+        - instanceGroup: master-us-test-1a
+          name: us-test-1a
+      name: main
+    - etcdMembers:
+        - instanceGroup: master-us-test-1a
+          name: us-test-1a
+      name: events
+  iam: {}
+  kubelet:
+    anonymousAuth: false
+  kubernetesVersion: v1.20.0
+  masterInternalName: api.internal.nthsqsresources.example.com
+  masterPublicName: api.nthsqsresources.example.com
+  networkCIDR: 172.20.0.0/16
+  networking:
+    cni: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  nodeTerminationHandler:
+    enabled: true
+    enableSQSTerminationDraining: true
+  sshAccess:
+    - 0.0.0.0/0
+  topology:
+    masters: public
+    nodes: public
+  subnets:
+    - cidr: 172.20.32.0/19
+      name: us-test-1a
+      type: Public
+      zone: us-test-1a
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2016-12-10T22:42:28Z"
+  name: nodes
+  labels:
+    kops.k8s.io/cluster: nthsqsresources.example.com
+spec:
+  associatePublicIp: true
+  image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21
+  machineType: t2.medium
+  maxSize: 2
+  minSize: 2
+  role: Node
+  subnets:
+    - us-test-1a
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2016-12-10T22:42:28Z"
+  name: master-us-test-1a
+  labels:
+    kops.k8s.io/cluster: nthsqsresources.example.com
+spec:
+  associatePublicIp: true
+  image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21
+  machineType: m3.medium
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+    - us-test-1a

tests/integration/update_cluster/nth_sqs_resources/kubernetes.tf

@@ -0,0 +1,732 @@
+locals {
+  cluster_name                 = "nthsqsresources.example.com"
+  master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-nthsqsresources-example-com.id]
+  master_security_group_ids    = [aws_security_group.masters-nthsqsresources-example-com.id]
+  masters_role_arn             = aws_iam_role.masters-nthsqsresources-example-com.arn
+  masters_role_name            = aws_iam_role.masters-nthsqsresources-example-com.name
+  node_autoscaling_group_ids   = [aws_autoscaling_group.nodes-nthsqsresources-example-com.id]
+  node_security_group_ids      = [aws_security_group.nodes-nthsqsresources-example-com.id]
+  node_subnet_ids              = [aws_subnet.us-test-1a-nthsqsresources-example-com.id]
+  nodes_role_arn               = aws_iam_role.nodes-nthsqsresources-example-com.arn
+  nodes_role_name              = aws_iam_role.nodes-nthsqsresources-example-com.name
+  region                       = "us-test-1"
+  route_table_public_id        = aws_route_table.nthsqsresources-example-com.id
+  subnet_us-test-1a_id         = aws_subnet.us-test-1a-nthsqsresources-example-com.id
+  vpc_cidr_block               = aws_vpc.nthsqsresources-example-com.cidr_block
+  vpc_id                       = aws_vpc.nthsqsresources-example-com.id
+}
+
+output "cluster_name" {
+  value = "nthsqsresources.example.com"
+}
+
+output "master_autoscaling_group_ids" {
+  value = [aws_autoscaling_group.master-us-test-1a-masters-nthsqsresources-example-com.id]
+}
+
+output "master_security_group_ids" {
+  value = [aws_security_group.masters-nthsqsresources-example-com.id]
+}
+
+output "masters_role_arn" {
+  value = aws_iam_role.masters-nthsqsresources-example-com.arn
+}
+
+output "masters_role_name" {
+  value = aws_iam_role.masters-nthsqsresources-example-com.name
+}
+
+output "node_autoscaling_group_ids" {
+  value = [aws_autoscaling_group.nodes-nthsqsresources-example-com.id]
+}
+
+output "node_security_group_ids" {
+  value = [aws_security_group.nodes-nthsqsresources-example-com.id]
+}
+
+output "node_subnet_ids" {
+  value = [aws_subnet.us-test-1a-nthsqsresources-example-com.id]
+}
+
+output "nodes_role_arn" {
+  value = aws_iam_role.nodes-nthsqsresources-example-com.arn
+}
+
+output "nodes_role_name" {
+  value = aws_iam_role.nodes-nthsqsresources-example-com.name
+}
+
+output "region" {
+  value = "us-test-1"
+}
+
+output "route_table_public_id" {
+  value = aws_route_table.nthsqsresources-example-com.id
+}
+
+output "subnet_us-test-1a_id" {
+  value = aws_subnet.us-test-1a-nthsqsresources-example-com.id
+}
+
+output "vpc_cidr_block" {
+  value = aws_vpc.nthsqsresources-example-com.cidr_block
+}
+
+output "vpc_id" {
+  value = aws_vpc.nthsqsresources-example-com.id
+}
+
+provider "aws" {
+  region = "us-test-1"
+}
+
+resource "aws_autoscaling_group" "master-us-test-1a-masters-nthsqsresources-example-com" {
+  enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
+  launch_template {
+    id      = aws_launch_template.master-us-test-1a-masters-nthsqsresources-example-com.id
+    version = aws_launch_template.master-us-test-1a-masters-nthsqsresources-example-com.latest_version
+  }
+  max_size            = 1
+  metrics_granularity = "1Minute"
+  min_size            = 1
+  name                = "master-us-test-1a.masters.nthsqsresources.example.com"
+  tag {
+    key                 = "KubernetesCluster"
+    propagate_at_launch = true
+    value               = "nthsqsresources.example.com"
+  }
+  tag {
+    key                 = "Name"
+    propagate_at_launch = true
+    value               = "master-us-test-1a.masters.nthsqsresources.example.com"
+  }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"
+    propagate_at_launch = true
+    value               = ""
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"
+    propagate_at_launch = true
+    value               = "master"
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"
+    propagate_at_launch = true
+    value               = ""
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"
+    propagate_at_launch = true
+    value               = ""
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers"
+    propagate_at_launch = true
+    value               = ""
+  }
+  tag {
+    key                 = "k8s.io/role/master"
+    propagate_at_launch = true
+    value               = "1"
+  }
+  tag {
+    key                 = "kops.k8s.io/instancegroup"
+    propagate_at_launch = true
+    value               = "master-us-test-1a"
+  }
+  tag {
+    key                 = "kubernetes.io/cluster/nthsqsresources.example.com"
+    propagate_at_launch = true
+    value               = "owned"
+  }
+  vpc_zone_identifier = [aws_subnet.us-test-1a-nthsqsresources-example-com.id]
+}
+
+resource "aws_autoscaling_group" "nodes-nthsqsresources-example-com" {
+  enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
+  launch_template {
+    id      = aws_launch_template.nodes-nthsqsresources-example-com.id
+    version = aws_launch_template.nodes-nthsqsresources-example-com.latest_version
+  }
+  max_size            = 2
+  metrics_granularity = "1Minute"
+  min_size            = 2
+  name                = "nodes.nthsqsresources.example.com"
+  tag {
+    key                 = "KubernetesCluster"
+    propagate_at_launch = true
+    value               = "nthsqsresources.example.com"
+  }
+  tag {
+    key                 = "Name"
+    propagate_at_launch = true
+    value               = "nodes.nthsqsresources.example.com"
+  }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"
+    propagate_at_launch = true
+    value               = "node"
+  }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
+    propagate_at_launch = true
+    value               = ""
+  }
+  tag {
+    key                 = "k8s.io/role/node"
+    propagate_at_launch = true
+    value               = "1"
+  }
+  tag {
+    key                 = "kops.k8s.io/instancegroup"
+    propagate_at_launch = true
+    value               = "nodes"
+  }
+  tag {
+    key                 = "kubernetes.io/cluster/nthsqsresources.example.com"
+    propagate_at_launch = true
+    value               = "owned"
+  }
+  vpc_zone_identifier = [aws_subnet.us-test-1a-nthsqsresources-example-com.id]
+}
+
+resource "aws_autoscaling_lifecycle_hook" "master-us-test-1a-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-nthsqsresources-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "master-us-test-1a-NTHLifecycleHook"
+}
+
+resource "aws_autoscaling_lifecycle_hook" "nodes-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.nodes-nthsqsresources-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "nodes-NTHLifecycleHook"
+}
+
+resource "aws_cloudwatch_event_rule" "nthsqsresources-example-com-ASGLifecycle" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_nthsqsresources.example.com-ASGLifecycle_event_pattern")
+  name          = "nthsqsresources.example.com-ASGLifecycle"
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nthsqsresources.example.com-ASGLifecycle"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "nthsqsresources-example-com-RebalanceRecommendation" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_nthsqsresources.example.com-RebalanceRecommendation_event_pattern")
+  name          = "nthsqsresources.example.com-RebalanceRecommendation"
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nthsqsresources.example.com-RebalanceRecommendation"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "nthsqsresources-example-com-SpotInterruption" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_nthsqsresources.example.com-SpotInterruption_event_pattern")
+  name          = "nthsqsresources.example.com-SpotInterruption"
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nthsqsresources.example.com-SpotInterruption"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_target" "nthsqsresources-example-com-ASGLifecycle-Target" {
+  arn  = "arn:aws:sqs:us-test-1:123456789012:nthsqsresources-example-com-nth"
+  rule = aws_cloudwatch_event_rule.nthsqsresources-example-com-ASGLifecycle.id
+}
+
+resource "aws_cloudwatch_event_target" "nthsqsresources-example-com-RebalanceRecommendation-Target" {
+  arn  = "arn:aws:sqs:us-test-1:123456789012:nthsqsresources-example-com-nth"
+  rule = aws_cloudwatch_event_rule.nthsqsresources-example-com-RebalanceRecommendation.id
+}
+
+resource "aws_cloudwatch_event_target" "nthsqsresources-example-com-SpotInterruption-Target" {
+  arn  = "arn:aws:sqs:us-test-1:123456789012:nthsqsresources-example-com-nth"
+  rule = aws_cloudwatch_event_rule.nthsqsresources-example-com-SpotInterruption.id
+}
+
+resource "aws_ebs_volume" "us-test-1a-etcd-events-nthsqsresources-example-com" {
+  availability_zone = "us-test-1a"
+  encrypted         = false
+  iops              = 3000
+  size              = 20
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "us-test-1a.etcd-events.nthsqsresources.example.com"
+    "k8s.io/etcd/events"                                = "us-test-1a/us-test-1a"
+    "k8s.io/role/master"                                = "1"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+  throughput = 125
+  type       = "gp3"
+}
+
+resource "aws_ebs_volume" "us-test-1a-etcd-main-nthsqsresources-example-com" {
+  availability_zone = "us-test-1a"
+  encrypted         = false
+  iops              = 3000
+  size              = 20
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "us-test-1a.etcd-main.nthsqsresources.example.com"
+    "k8s.io/etcd/main"                                  = "us-test-1a/us-test-1a"
+    "k8s.io/role/master"                                = "1"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+  throughput = 125
+  type       = "gp3"
+}
+
+resource "aws_iam_instance_profile" "masters-nthsqsresources-example-com" {
+  name = "masters.nthsqsresources.example.com"
+  role = aws_iam_role.masters-nthsqsresources-example-com.name
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "masters.nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+resource "aws_iam_instance_profile" "nodes-nthsqsresources-example-com" {
+  name = "nodes.nthsqsresources.example.com"
+  role = aws_iam_role.nodes-nthsqsresources-example-com.name
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nodes.nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+resource "aws_iam_role_policy" "masters-nthsqsresources-example-com" {
+  name   = "masters.nthsqsresources.example.com"
+  policy = file("${path.module}/data/aws_iam_role_policy_masters.nthsqsresources.example.com_policy")
+  role   = aws_iam_role.masters-nthsqsresources-example-com.name
+}
+
+resource "aws_iam_role_policy" "nodes-nthsqsresources-example-com" {
+  name   = "nodes.nthsqsresources.example.com"
+  policy = file("${path.module}/data/aws_iam_role_policy_nodes.nthsqsresources.example.com_policy")
+  role   = aws_iam_role.nodes-nthsqsresources-example-com.name
+}
+
+resource "aws_iam_role" "masters-nthsqsresources-example-com" {
+  assume_role_policy = file("${path.module}/data/aws_iam_role_masters.nthsqsresources.example.com_policy")
+  name               = "masters.nthsqsresources.example.com"
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "masters.nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+resource "aws_iam_role" "nodes-nthsqsresources-example-com" {
+  assume_role_policy = file("${path.module}/data/aws_iam_role_nodes.nthsqsresources.example.com_policy")
+  name               = "nodes.nthsqsresources.example.com"
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nodes.nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+resource "aws_internet_gateway" "nthsqsresources-example-com" {
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+  vpc_id = aws_vpc.nthsqsresources-example-com.id
+}
+
+resource "aws_key_pair" "kubernetes-nthsqsresources-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" {
+  key_name   = "kubernetes.nthsqsresources.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57"
+  public_key = file("${path.module}/data/aws_key_pair_kubernetes.nthsqsresources.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key")
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+resource "aws_launch_template" "master-us-test-1a-masters-nthsqsresources-example-com" {
+  block_device_mappings {
+    device_name = "/dev/xvda"
+    ebs {
+      delete_on_termination = true
+      encrypted             = true
+      iops                  = 3000
+      throughput            = 125
+      volume_size           = 64
+      volume_type           = "gp3"
+    }
+  }
+  block_device_mappings {
+    device_name  = "/dev/sdc"
+    virtual_name = "ephemeral0"
+  }
+  iam_instance_profile {
+    name = aws_iam_instance_profile.masters-nthsqsresources-example-com.id
+  }
+  image_id      = "ami-12345678"
+  instance_type = "m3.medium"
+  key_name      = aws_key_pair.kubernetes-nthsqsresources-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id
+  lifecycle {
+    create_before_destroy = true
+  }
+  metadata_options {
+    http_endpoint               = "enabled"
+    http_put_response_hop_limit = 1
+    http_tokens                 = "optional"
+  }
+  name = "master-us-test-1a.masters.nthsqsresources.example.com"
+  network_interfaces {
+    associate_public_ip_address = true
+    delete_on_termination       = true
+    security_groups             = [aws_security_group.masters-nthsqsresources-example-com.id]
+  }
+  tag_specifications {
+    resource_type = "instance"
+    tags = {
+      "KubernetesCluster"                                                                                     = "nthsqsresources.example.com"
+      "Name"                                                                                                  = "master-us-test-1a.masters.nthsqsresources.example.com"
+      "aws-node-termination-handler/managed"                                                                  = ""
+      "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
+      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
+      "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
+      "k8s.io/role/master"                                                                                    = "1"
+      "kops.k8s.io/instancegroup"                                                                             = "master-us-test-1a"
+      "kubernetes.io/cluster/nthsqsresources.example.com"                                                     = "owned"
+    }
+  }
+  tag_specifications {
+    resource_type = "volume"
+    tags = {
+      "KubernetesCluster"                                                                                     = "nthsqsresources.example.com"
+      "Name"                                                                                                  = "master-us-test-1a.masters.nthsqsresources.example.com"
+      "aws-node-termination-handler/managed"                                                                  = ""
+      "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
+      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
+      "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
+      "k8s.io/role/master"                                                                                    = "1"
+      "kops.k8s.io/instancegroup"                                                                             = "master-us-test-1a"
+      "kubernetes.io/cluster/nthsqsresources.example.com"                                                     = "owned"
+    }
+  }
+  tags = {
+    "KubernetesCluster"                                                                                     = "nthsqsresources.example.com"
+    "Name"                                                                                                  = "master-us-test-1a.masters.nthsqsresources.example.com"
+    "aws-node-termination-handler/managed"                                                                  = ""
+    "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
+    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
+    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
+    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
+    "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
+    "k8s.io/role/master"                                                                                    = "1"
+    "kops.k8s.io/instancegroup"                                                                             = "master-us-test-1a"
+    "kubernetes.io/cluster/nthsqsresources.example.com"                                                     = "owned"
+  }
+  user_data = filebase64("${path.module}/data/aws_launch_template_master-us-test-1a.masters.nthsqsresources.example.com_user_data")
+}
+
+resource "aws_launch_template" "nodes-nthsqsresources-example-com" {
+  block_device_mappings {
+    device_name = "/dev/xvda"
+    ebs {
+      delete_on_termination = true
+      encrypted             = true
+      iops                  = 3000
+      throughput            = 125
+      volume_size           = 128
+      volume_type           = "gp3"
+    }
+  }
+  iam_instance_profile {
+    name = aws_iam_instance_profile.nodes-nthsqsresources-example-com.id
+  }
+  image_id      = "ami-12345678"
+  instance_type = "t2.medium"
+  key_name      = aws_key_pair.kubernetes-nthsqsresources-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id
+  lifecycle {
+    create_before_destroy = true
+  }
+  metadata_options {
+    http_endpoint               = "enabled"
+    http_put_response_hop_limit = 1
+    http_tokens                 = "optional"
+  }
+  name = "nodes.nthsqsresources.example.com"
+  network_interfaces {
+    associate_public_ip_address = true
+    delete_on_termination       = true
+    security_groups             = [aws_security_group.nodes-nthsqsresources-example-com.id]
+  }
+  tag_specifications {
+    resource_type = "instance"
+    tags = {
+      "KubernetesCluster"                                                          = "nthsqsresources.example.com"
+      "Name"                                                                       = "nodes.nthsqsresources.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
+      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
+      "k8s.io/role/node"                                                           = "1"
+      "kops.k8s.io/instancegroup"                                                  = "nodes"
+      "kubernetes.io/cluster/nthsqsresources.example.com"                          = "owned"
+    }
+  }
+  tag_specifications {
+    resource_type = "volume"
+    tags = {
+      "KubernetesCluster"                                                          = "nthsqsresources.example.com"
+      "Name"                                                                       = "nodes.nthsqsresources.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
+      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
+      "k8s.io/role/node"                                                           = "1"
+      "kops.k8s.io/instancegroup"                                                  = "nodes"
+      "kubernetes.io/cluster/nthsqsresources.example.com"                          = "owned"
+    }
+  }
+  tags = {
+    "KubernetesCluster"                                                          = "nthsqsresources.example.com"
+    "Name"                                                                       = "nodes.nthsqsresources.example.com"
+    "aws-node-termination-handler/managed"                                       = ""
+    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
+    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
+    "k8s.io/role/node"                                                           = "1"
+    "kops.k8s.io/instancegroup"                                                  = "nodes"
+    "kubernetes.io/cluster/nthsqsresources.example.com"                          = "owned"
+  }
+  user_data = filebase64("${path.module}/data/aws_launch_template_nodes.nthsqsresources.example.com_user_data")
+}
+
+resource "aws_route_table_association" "us-test-1a-nthsqsresources-example-com" {
+  route_table_id = aws_route_table.nthsqsresources-example-com.id
+  subnet_id      = aws_subnet.us-test-1a-nthsqsresources-example-com.id
+}
+
+resource "aws_route_table" "nthsqsresources-example-com" {
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+    "kubernetes.io/kops/role"                           = "public"
+  }
+  vpc_id = aws_vpc.nthsqsresources-example-com.id
+}
+
+resource "aws_route" "route-0-0-0-0--0" {
+  destination_cidr_block = "0.0.0.0/0"
+  gateway_id             = aws_internet_gateway.nthsqsresources-example-com.id
+  route_table_id         = aws_route_table.nthsqsresources-example-com.id
+}
+
+resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-masters-nthsqsresources-example-com" {
+  cidr_blocks       = ["0.0.0.0/0"]
+  from_port         = 22
+  protocol          = "tcp"
+  security_group_id = aws_security_group.masters-nthsqsresources-example-com.id
+  to_port           = 22
+  type              = "ingress"
+}
+
+resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-nodes-nthsqsresources-example-com" {
+  cidr_blocks       = ["0.0.0.0/0"]
+  from_port         = 22
+  protocol          = "tcp"
+  security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id
+  to_port           = 22
+  type              = "ingress"
+}
+
+resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-masters-nthsqsresources-example-com" {
+  cidr_blocks       = ["0.0.0.0/0"]
+  from_port         = 443
+  protocol          = "tcp"
+  security_group_id = aws_security_group.masters-nthsqsresources-example-com.id
+  to_port           = 443
+  type              = "ingress"
+}
+
+resource "aws_security_group_rule" "from-masters-nthsqsresources-example-com-egress-all-0to0-0-0-0-0--0" {
+  cidr_blocks       = ["0.0.0.0/0"]
+  from_port         = 0
+  protocol          = "-1"
+  security_group_id = aws_security_group.masters-nthsqsresources-example-com.id
+  to_port           = 0
+  type              = "egress"
+}
+
+resource "aws_security_group_rule" "from-masters-nthsqsresources-example-com-ingress-all-0to0-masters-nthsqsresources-example-com" {
+  from_port                = 0
+  protocol                 = "-1"
+  security_group_id        = aws_security_group.masters-nthsqsresources-example-com.id
+  source_security_group_id = aws_security_group.masters-nthsqsresources-example-com.id
+  to_port                  = 0
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "from-masters-nthsqsresources-example-com-ingress-all-0to0-nodes-nthsqsresources-example-com" {
+  from_port                = 0
+  protocol                 = "-1"
+  security_group_id        = aws_security_group.nodes-nthsqsresources-example-com.id
+  source_security_group_id = aws_security_group.masters-nthsqsresources-example-com.id
+  to_port                  = 0
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "from-nodes-nthsqsresources-example-com-egress-all-0to0-0-0-0-0--0" {
+  cidr_blocks       = ["0.0.0.0/0"]
+  from_port         = 0
+  protocol          = "-1"
+  security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id
+  to_port           = 0
+  type              = "egress"
+}
+
+resource "aws_security_group_rule" "from-nodes-nthsqsresources-example-com-ingress-all-0to0-nodes-nthsqsresources-example-com" {
+  from_port                = 0
+  protocol                 = "-1"
+  security_group_id        = aws_security_group.nodes-nthsqsresources-example-com.id
+  source_security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id
+  to_port                  = 0
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "from-nodes-nthsqsresources-example-com-ingress-tcp-1to2379-masters-nthsqsresources-example-com" {
+  from_port                = 1
+  protocol                 = "tcp"
+  security_group_id        = aws_security_group.masters-nthsqsresources-example-com.id
+  source_security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id
+  to_port                  = 2379
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "from-nodes-nthsqsresources-example-com-ingress-tcp-2382to4000-masters-nthsqsresources-example-com" {
+  from_port                = 2382
+  protocol                 = "tcp"
+  security_group_id        = aws_security_group.masters-nthsqsresources-example-com.id
+  source_security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id
+  to_port                  = 4000
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "from-nodes-nthsqsresources-example-com-ingress-tcp-4003to65535-masters-nthsqsresources-example-com" {
+  from_port                = 4003
+  protocol                 = "tcp"
+  security_group_id        = aws_security_group.masters-nthsqsresources-example-com.id
+  source_security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id
+  to_port                  = 65535
+  type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "from-nodes-nthsqsresources-example-com-ingress-udp-1to65535-masters-nthsqsresources-example-com" {
+  from_port                = 1
+  protocol                 = "udp"
+  security_group_id        = aws_security_group.masters-nthsqsresources-example-com.id
+  source_security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id
+  to_port                  = 65535
+  type                     = "ingress"
+}
+
+resource "aws_security_group" "masters-nthsqsresources-example-com" {
+  description = "Security group for masters"
+  name        = "masters.nthsqsresources.example.com"
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "masters.nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+  vpc_id = aws_vpc.nthsqsresources-example-com.id
+}
+
+resource "aws_security_group" "nodes-nthsqsresources-example-com" {
+  description = "Security group for nodes"
+  name        = "nodes.nthsqsresources.example.com"
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nodes.nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+  vpc_id = aws_vpc.nthsqsresources-example-com.id
+}
+
+resource "aws_sqs_queue" "nthsqsresources-example-com-nth" {
+  message_retention_seconds = 300
+  name                      = "nthsqsresources-example-com-nth"
+  policy                    = file("${path.module}/data/aws_sqs_queue_nthsqsresources-example-com-nth_policy")
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nthsqsresources-example-com-nth"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+resource "aws_subnet" "us-test-1a-nthsqsresources-example-com" {
+  availability_zone = "us-test-1a"
+  cidr_block        = "172.20.32.0/19"
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "us-test-1a.nthsqsresources.example.com"
+    "SubnetType"                                        = "Public"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+    "kubernetes.io/role/elb"                            = "1"
+  }
+  vpc_id = aws_vpc.nthsqsresources-example-com.id
+}
+
+resource "aws_vpc_dhcp_options_association" "nthsqsresources-example-com" {
+  dhcp_options_id = aws_vpc_dhcp_options.nthsqsresources-example-com.id
+  vpc_id          = aws_vpc.nthsqsresources-example-com.id
+}
+
+resource "aws_vpc_dhcp_options" "nthsqsresources-example-com" {
+  domain_name         = "us-test-1.compute.internal"
+  domain_name_servers = ["AmazonProvidedDNS"]
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+resource "aws_vpc" "nthsqsresources-example-com" {
+  cidr_block           = "172.20.0.0/16"
+  enable_dns_hostnames = true
+  enable_dns_support   = true
+  tags = {
+    "KubernetesCluster"                                 = "nthsqsresources.example.com"
+    "Name"                                              = "nthsqsresources.example.com"
+    "kubernetes.io/cluster/nthsqsresources.example.com" = "owned"
+  }
+}
+
+terraform {
+  required_version = ">= 0.12.26"
+  required_providers {
+    aws = {
+      "source"  = "hashicorp/aws"
+      "version" = ">= 3.34.0"
+    }
+  }
+}

upup/models/cloudup/resources/addons/node-termination-handler.aws/k8s-1.11.yaml.template

@@ -66,6 +66,148 @@ roleRef:
   kind: ClusterRole
   name: aws-node-termination-handler
   apiGroup: rbac.authorization.k8s.io
+{{ if EnableSQSTerminationDraining }}
+---
+# Source: aws-node-termination-handler/templates/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: aws-node-termination-handler
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/instance: aws-node-termination-handler
+    k8s-app: aws-node-termination-handler
+    app.kubernetes.io/version: "1.12.1"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: aws-node-termination-handler
+      app.kubernetes.io/instance: aws-node-termination-handler
+      kubernetes.io/os: linux
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: aws-node-termination-handler
+        app.kubernetes.io/instance: aws-node-termination-handler
+        k8s-app: aws-node-termination-handler
+        kubernetes.io/os: linux
+    spec:
+      priorityClassName: "system-node-critical"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: "kubernetes.io/os"
+                    operator: In
+                    values:
+                      - linux
+                  - key: "kubernetes.io/arch"
+                    operator: In
+                    values:
+                      - amd64
+                      - arm64
+                      - arm
+      serviceAccountName: aws-node-termination-handler
+      hostNetwork: false
+      dnsPolicy: ""
+      securityContext:
+        fsGroup: 1000
+      containers:
+        - name: aws-node-termination-handler
+          image: public.ecr.aws/aws-ec2/aws-node-termination-handler:v1.12.1
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 1000
+            runAsGroup: 1000
+            allowPrivilegeEscalation: false
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: SPOT_POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: DELETE_LOCAL_DATA
+              value: ""
+            - name: IGNORE_DAEMON_SETS
+              value: ""
+            - name: POD_TERMINATION_GRACE_PERIOD
+              value: ""
+            - name: INSTANCE_METADATA_URL
+              value: ""
+            - name: NODE_TERMINATION_GRACE_PERIOD
+              value: ""
+            - name: WEBHOOK_URL
+              value: ""
+            - name: WEBHOOK_HEADERS
+              value: ""
+            - name: WEBHOOK_TEMPLATE
+              value: ""
+            - name: DRY_RUN
+              value: "false"
+            - name: METADATA_TRIES
+              value: "3"
+            - name: CORDON_ONLY
+              value: "false"
+            - name: TAINT_NODE
+              value: "false"
+            - name: JSON_LOGGING
+              value: "false"
+            - name: LOG_LEVEL
+              value: "info"
+            - name: WEBHOOK_PROXY
+              value: ""
+            - name: ENABLE_PROMETHEUS_SERVER
+              value: "{{ .EnablePrometheusMetrics }}"
+            - name: ENABLE_SPOT_INTERRUPTION_DRAINING
+              value: "false"
+            - name: ENABLE_SCHEDULED_EVENT_DRAINING
+              value: "false"
+            - name: ENABLE_REBALANCE_MONITORING
+              value: "false"
+            - name: ENABLE_SQS_TERMINATION_DRAINING
+              value: "true"
+            - name: QUEUE_URL
+              value: "{{ DefaultQueueName }}"
+            - name: PROMETHEUS_SERVER_PORT
+              value: "9092"
+            - name: AWS_REGION
+              value: ""
+            - name: AWS_ENDPOINT
+              value: ""
+            - name: CHECK_ASG_TAG_BEFORE_DRAINING
+              value: "true"
+            - name: MANAGED_ASG_TAG
+              value: "{{ .ManagedASGTag }}"
+            - name: WORKERS
+              value: "10"
+          resources:
+            limits:
+              cpu: 100m
+              memory: 128Mi
+            requests:
+              cpu: 50m
+              memory: 64Mi
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+        - operator: Exists
+{{ else }}
 ---
 # Source: aws-node-termination-handler/templates/daemonset.linux.yaml
 apiVersion: apps/v1
@@ -176,4 +318,5 @@ spec:
         kubernetes.io/os: linux
       tolerations:
         - operator: Exists
+{{ end }}
 {{ end }}

upup/pkg/fi/cloudup/apply_cluster.go

@@ -580,6 +580,14 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error {
 				l.Builders = append(l.Builders, awsModelBuilder)
 			}
 
+			nth := c.Cluster.Spec.NodeTerminationHandler
+			if nth != nil && fi.BoolValue(nth.Enabled) && fi.BoolValue(nth.EnableSQSTerminationDraining) {
+				l.Builders = append(l.Builders, &awsmodel.NodeTerminationHandlerBuilder{
+					AWSModelContext: awsModelContext,
+					Lifecycle:       &clusterLifecycle,
+				})
+			}
+
 		case kops.CloudProviderDO:
 			doModelContext := &domodel.DOModelContext{
 				KopsModelContext: modelContext,

upup/pkg/fi/cloudup/awstasks/BUILD.bazel

@@ -5,6 +5,8 @@ go_library(
     srcs = [
         "autoscalinggroup.go",
         "autoscalinggroup_fitask.go",
+        "autoscalinggroup_lifecyclehook.go",
+        "autoscalinglifecyclehook_fitask.go",
         "block_device_mappings.go",
         "classic_load_balancer.go",
         "classic_loadbalancer_attributes.go",
@@ -22,6 +24,10 @@ go_library(
         "ebsvolume_fitask.go",
         "elastic_ip.go",
         "elasticip_fitask.go",
+        "eventbridgerule.go",
+        "eventbridgerule_fitask.go",
+        "eventbridgetarget.go",
+        "eventbridgetarget_fitask.go",
         "helper.go",
         "iaminstanceprofile.go",
         "iaminstanceprofile_fitask.go",
@@ -59,6 +65,8 @@ go_library(
         "securitygroup_fitask.go",
         "securitygrouprule.go",
         "securitygrouprule_fitask.go",
+        "sqs.go",
+        "sqs_fitask.go",
         "sshkey.go",
         "sshkey_fitask.go",
         "subnet.go",
@@ -97,8 +105,10 @@ go_library(
         "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/elb:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/elbv2:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/eventbridge:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/iam:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/route53:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/sqs:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
         "//vendor/k8s.io/klog/v2:go_default_library",
     ],

upup/pkg/fi/cloudup/awstasks/autoscalinggroup_lifecyclehook.go

@@ -0,0 +1,154 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package awstasks
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/service/autoscaling"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
+	"k8s.io/kops/upup/pkg/fi/cloudup/cloudformation"
+	"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
+)
+
+// +kops:fitask
+type AutoscalingLifecycleHook struct {
+	ID        *string
+	Name      *string
+	Lifecycle *fi.Lifecycle
+
+	AutoscalingGroup    *AutoscalingGroup
+	DefaultResult       *string
+	HeartbeatTimeout    *int64
+	LifecycleTransition *string
+}
+
+var _ fi.CompareWithID = &AutoscalingLifecycleHook{}
+
+func (h *AutoscalingLifecycleHook) CompareWithID() *string {
+	return h.Name
+}
+
+func (h *AutoscalingLifecycleHook) Find(c *fi.Context) (*AutoscalingLifecycleHook, error) {
+	cloud := c.Cloud.(awsup.AWSCloud)
+
+	request := &autoscaling.DescribeLifecycleHooksInput{
+		AutoScalingGroupName: h.AutoscalingGroup.Name,
+		LifecycleHookNames:   []*string{h.Name},
+	}
+
+	response, err := cloud.Autoscaling().DescribeLifecycleHooks(request)
+	if err != nil {
+		return nil, fmt.Errorf("error listing ASG Lifecycle Hooks: %v", err)
+	}
+	if response == nil || len(response.LifecycleHooks) == 0 {
+		return nil, nil
+	}
+	if len(response.LifecycleHooks) > 1 {
+		return nil, fmt.Errorf("found multiple ASG Lifecycle Hooks with the same name")
+	}
+
+	hook := response.LifecycleHooks[0]
+	actual := &AutoscalingLifecycleHook{
+		ID:                  hook.LifecycleHookName,
+		Name:                hook.LifecycleHookName,
+		Lifecycle:           h.Lifecycle,
+		AutoscalingGroup:    h.AutoscalingGroup,
+		DefaultResult:       hook.DefaultResult,
+		HeartbeatTimeout:    hook.HeartbeatTimeout,
+		LifecycleTransition: hook.LifecycleTransition,
+	}
+
+	return actual, nil
+}
+
+func (h *AutoscalingLifecycleHook) Run(c *fi.Context) error {
+	return fi.DefaultDeltaRunMethod(h, c)
+}
+
+func (_ *AutoscalingLifecycleHook) CheckChanges(a, e, changes *AutoscalingLifecycleHook) error {
+	if a == nil {
+		if e.Name == nil {
+			return field.Required(field.NewPath("Name"), "")
+		}
+		if e.AutoscalingGroup == nil {
+			return field.Required(field.NewPath("AutoScalingGroupName"), "")
+		}
+	}
+
+	return nil
+}
+
+func (h *AutoscalingLifecycleHook) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *AutoscalingLifecycleHook) error {
+	if a == nil {
+		request := &autoscaling.PutLifecycleHookInput{
+			AutoScalingGroupName: e.AutoscalingGroup.Name,
+			DefaultResult:        h.DefaultResult,
+			HeartbeatTimeout:     h.HeartbeatTimeout,
+			LifecycleHookName:    e.Name,
+			LifecycleTransition:  h.LifecycleTransition,
+		}
+		_, err := t.Cloud.Autoscaling().PutLifecycleHook(request)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+type terraformASGLifecycleHook struct {
+	Name                 *string            `json:"name" cty:"name"`
+	AutoScalingGroupName *terraform.Literal `json:"autoscaling_group_name" cty:"autoscaling_group_name"`
+	DefaultResult        *string            `json:"default_result" cty:"default_result"`
+	HeartbeatTimeout     *int64             `json:"heartbeat_timeout" cty:"heartbeat_timeout"`
+	LifecycleTransition  *string            `json:"lifecycle_transition" cty:"lifecycle_transition"`
+}
+
+func (_ *AutoscalingLifecycleHook) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *AutoscalingLifecycleHook) error {
+	tf := &terraformASGLifecycleHook{
+		Name:                 e.Name,
+		AutoScalingGroupName: e.AutoscalingGroup.TerraformLink(),
+		DefaultResult:        e.DefaultResult,
+		HeartbeatTimeout:     e.HeartbeatTimeout,
+		LifecycleTransition:  e.LifecycleTransition,
+	}
+
+	return t.RenderResource("aws_autoscaling_lifecycle_hook", *e.Name, tf)
+}
+
+type cloudformationASGLifecycleHook struct {
+	LifecycleHookName    *string
+	AutoScalingGroupName *cloudformation.Literal
+	DefaultResult        *string
+	HeartbeatTimeout     *int64
+	LifecycleTransition  *string
+}
+
+func (_ *AutoscalingLifecycleHook) RenderCloudformation(t *cloudformation.CloudformationTarget, a, e, changes *AutoscalingLifecycleHook) error {
+	tf := &cloudformationASGLifecycleHook{
+		LifecycleHookName:    e.Name,
+		AutoScalingGroupName: e.AutoscalingGroup.CloudformationLink(),
+		DefaultResult:        e.DefaultResult,
+		HeartbeatTimeout:     e.HeartbeatTimeout,
+		LifecycleTransition:  e.LifecycleTransition,
+	}
+
+	return t.RenderResource("AWS::AutoScaling::LifecycleHook", *e.Name, tf)
+}

upup/pkg/fi/cloudup/awstasks/autoscalinglifecyclehook_fitask.go

@@ -0,0 +1,51 @@
+// +build !ignore_autogenerated
+
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by fitask. DO NOT EDIT.
+
+package awstasks
+
+import (
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+// AutoscalingLifecycleHook
+
+var _ fi.HasLifecycle = &AutoscalingLifecycleHook{}
+
+// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle
+func (o *AutoscalingLifecycleHook) GetLifecycle() *fi.Lifecycle {
+	return o.Lifecycle
+}
+
+// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle
+func (o *AutoscalingLifecycleHook) SetLifecycle(lifecycle fi.Lifecycle) {
+	o.Lifecycle = &lifecycle
+}
+
+var _ fi.HasName = &AutoscalingLifecycleHook{}
+
+// GetName returns the Name of the object, implementing fi.HasName
+func (o *AutoscalingLifecycleHook) GetName() *string {
+	return o.Name
+}
+
+// String is the stringer function for the task, producing readable output using fi.TaskAsString
+func (o *AutoscalingLifecycleHook) String() string {
+	return fi.TaskAsString(o)
+}

upup/pkg/fi/cloudup/awstasks/eventbridgerule.go

@@ -0,0 +1,188 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package awstasks
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/eventbridge"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
+	"k8s.io/kops/upup/pkg/fi/cloudup/cloudformation"
+	"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
+)
+
+// +kops:fitask
+type EventBridgeRule struct {
+	ID        *string
+	Name      *string
+	Lifecycle *fi.Lifecycle
+
+	EventPattern *string
+	TargetArn    *string // required for cloudformation rendering
+
+	Tags map[string]string
+}
+
+var _ fi.CompareWithID = &EventBridgeRule{}
+
+func (eb *EventBridgeRule) CompareWithID() *string {
+	return eb.Name
+}
+
+func (eb *EventBridgeRule) Find(c *fi.Context) (*EventBridgeRule, error) {
+	cloud := c.Cloud.(awsup.AWSCloud)
+
+	if eb.Name == nil {
+		return nil, nil
+	}
+
+	request := &eventbridge.ListRulesInput{
+		NamePrefix: eb.Name,
+	}
+	response, err := cloud.EventBridge().ListRules(request)
+	if err != nil {
+		return nil, fmt.Errorf("error listing EventBridge rules: %v", err)
+	}
+	if response == nil || len(response.Rules) == 0 {
+		return nil, nil
+	}
+	if len(response.Rules) > 1 {
+		return nil, fmt.Errorf("found multiple EventBridge rules with the same name")
+	}
+
+	rule := response.Rules[0]
+
+	tagResponse, err := cloud.EventBridge().ListTagsForResource(&eventbridge.ListTagsForResourceInput{ResourceARN: rule.Arn})
+	if err != nil {
+		return nil, fmt.Errorf("error listing tags for EventBridge rule: %v", err)
+	}
+
+	actual := &EventBridgeRule{
+		ID:           eb.ID,
+		Name:         eb.Name,
+		Lifecycle:    eb.Lifecycle,
+		EventPattern: rule.EventPattern,
+		TargetArn:    eb.TargetArn,
+		Tags:         mapEventBridgeTagsToMap(tagResponse.Tags),
+	}
+	return actual, nil
+}
+
+func (eb *EventBridgeRule) Run(c *fi.Context) error {
+	return fi.DefaultDeltaRunMethod(eb, c)
+}
+
+func (_ *EventBridgeRule) CheckChanges(a, e, changes *EventBridgeRule) error {
+	if a == nil {
+		if e.Name == nil {
+			return field.Required(field.NewPath("Name"), "")
+		}
+	}
+
+	return nil
+}
+
+func (eb *EventBridgeRule) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *EventBridgeRule) error {
+	if a == nil {
+		var tags []*eventbridge.Tag
+		for k, v := range eb.Tags {
+			tags = append(tags, &eventbridge.Tag{
+				Key:   aws.String(k),
+				Value: aws.String(v),
+			})
+		}
+
+		request := &eventbridge.PutRuleInput{
+			Name:         eb.Name,
+			EventPattern: e.EventPattern,
+			Tags:         tags,
+		}
+
+		_, err := t.Cloud.EventBridge().PutRule(request)
+		if err != nil {
+			return fmt.Errorf("error creating EventBridge rule: %v", err)
+		}
+	}
+
+	return nil
+}
+
+type terraformEventBridgeRule struct {
+	Name         *string            `json:"name" cty:"name"`
+	EventPattern *terraform.Literal `json:"event_pattern" cty:"event_pattern"`
+	Tags         map[string]string  `json:"tags,omitempty" cty:"tags"`
+}
+
+func (_ *EventBridgeRule) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *EventBridgeRule) error {
+	m, err := t.AddFile("aws_cloudwatch_event_rule", *e.Name, "event_pattern", fi.NewStringResource(*e.EventPattern), false)
+	if err != nil {
+		return err
+	}
+
+	tf := &terraformEventBridgeRule{
+		Name:         e.Name,
+		EventPattern: m,
+		Tags:         e.Tags,
+	}
+
+	return t.RenderResource("aws_cloudwatch_event_rule", *e.Name, tf)
+}
+
+func (eb *EventBridgeRule) TerraformLink() *terraform.Literal {
+	return terraform.LiteralProperty("aws_cloudwatch_event_rule", fi.StringValue(eb.Name), "id")
+}
+
+type cloudformationTarget struct {
+	Id  *string
+	Arn *string
+}
+
+type cloudformationEventBridgeRule struct {
+	Name         *string                `json:"Name"`
+	EventPattern map[string]interface{} `json:"EventPattern"`
+	Targets      []cloudformationTarget `json:"Targets"`
+}
+
+func (_ *EventBridgeRule) RenderCloudformation(t *cloudformation.CloudformationTarget, a, e, changes *EventBridgeRule) error {
+	// convert event pattern string into a json struct
+	jsonString, err := fi.ResourceAsBytes(fi.NewStringResource(*e.EventPattern))
+	if err != nil {
+		return err
+	}
+	data := make(map[string]interface{})
+	err = json.Unmarshal(jsonString, &data)
+	if err != nil {
+		return fmt.Errorf("error parsing SQS PolicyDocument: %v", err)
+	}
+
+	target := &cloudformationTarget{
+		Id:  s("1"),
+		Arn: e.TargetArn,
+	}
+
+	cf := &cloudformationEventBridgeRule{
+		Name:         e.Name,
+		EventPattern: data,
+		Targets:      []cloudformationTarget{*target},
+	}
+
+	return t.RenderResource("AWS::Events::Rule", *e.Name, cf)
+}

upup/pkg/fi/cloudup/awstasks/eventbridgerule_fitask.go

@@ -0,0 +1,51 @@
+// +build !ignore_autogenerated
+
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by fitask. DO NOT EDIT.
+
+package awstasks
+
+import (
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+// EventBridgeRule
+
+var _ fi.HasLifecycle = &EventBridgeRule{}
+
+// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle
+func (o *EventBridgeRule) GetLifecycle() *fi.Lifecycle {
+	return o.Lifecycle
+}
+
+// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle
+func (o *EventBridgeRule) SetLifecycle(lifecycle fi.Lifecycle) {
+	o.Lifecycle = &lifecycle
+}
+
+var _ fi.HasName = &EventBridgeRule{}
+
+// GetName returns the Name of the object, implementing fi.HasName
+func (o *EventBridgeRule) GetName() *string {
+	return o.Name
+}
+
+// String is the stringer function for the task, producing readable output using fi.TaskAsString
+func (o *EventBridgeRule) String() string {
+	return fi.TaskAsString(o)
+}

upup/pkg/fi/cloudup/awstasks/eventbridgetarget.go

@@ -0,0 +1,146 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package awstasks
+
+import (
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/util/validation/field"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/eventbridge"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
+	"k8s.io/kops/upup/pkg/fi/cloudup/cloudformation"
+	"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
+)
+
+// +kops:fitask
+type EventBridgeTarget struct {
+	ID        *string
+	Name      *string
+	Lifecycle *fi.Lifecycle
+
+	Rule      *EventBridgeRule
+	TargetArn *string
+}
+
+var _ fi.CompareWithID = &EventBridgeTarget{}
+
+func (eb *EventBridgeTarget) CompareWithID() *string {
+	return eb.Name
+}
+
+func (eb *EventBridgeTarget) Find(c *fi.Context) (*EventBridgeTarget, error) {
+	cloud := c.Cloud.(awsup.AWSCloud)
+
+	if eb.Rule == nil || eb.TargetArn == nil {
+		return nil, nil
+	}
+
+	// find the rule the target is attached to
+	rule, err := eb.Rule.Find(c)
+	if err != nil {
+		return nil, err
+	}
+	if rule == nil {
+		return nil, nil
+	}
+
+	request := &eventbridge.ListTargetsByRuleInput{
+		Rule: eb.Rule.Name,
+	}
+
+	response, err := cloud.EventBridge().ListTargetsByRule(request)
+	if err != nil {
+		return nil, fmt.Errorf("error listing EventBridge targets: %v", err)
+	}
+	if response == nil || len(response.Targets) == 0 {
+		return nil, nil
+	}
+	for _, target := range response.Targets {
+		if *target.Arn == *eb.TargetArn {
+			actual := &EventBridgeTarget{
+				ID:        target.Id,
+				Name:      eb.Name,
+				Lifecycle: eb.Lifecycle,
+				Rule:      eb.Rule,
+				TargetArn: eb.TargetArn,
+			}
+			return actual, nil
+		}
+	}
+
+	return nil, nil
+}
+
+func (eb *EventBridgeTarget) Run(c *fi.Context) error {
+	return fi.DefaultDeltaRunMethod(eb, c)
+}
+
+func (_ *EventBridgeTarget) CheckChanges(a, e, changes *EventBridgeTarget) error {
+	if a == nil {
+		if e.Rule == nil {
+			return field.Required(field.NewPath("Rule"), "")
+		}
+		if e.TargetArn == nil {
+			return field.Required(field.NewPath("TargetArn"), "")
+		}
+	}
+
+	return nil
+}
+
+func (eb *EventBridgeTarget) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *EventBridgeTarget) error {
+	if a == nil {
+		target := &eventbridge.Target{
+			Arn: eb.TargetArn,
+			Id:  aws.String("1"),
+		}
+
+		request := &eventbridge.PutTargetsInput{
+			Rule:    eb.Rule.Name,
+			Targets: []*eventbridge.Target{target},
+		}
+
+		_, err := t.Cloud.EventBridge().PutTargets(request)
+		if err != nil {
+			return fmt.Errorf("error creating EventBridge target: %v", err)
+		}
+	}
+
+	return nil
+}
+
+type terraformEventBridgeTarget struct {
+	RuleName  *terraform.Literal `json:"rule" cty:"rule"`
+	TargetArn *string            `json:"arn" cty:"arn"`
+}
+
+func (_ *EventBridgeTarget) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *EventBridgeTarget) error {
+	tf := &terraformEventBridgeTarget{
+		RuleName:  e.Rule.TerraformLink(),
+		TargetArn: e.TargetArn,
+	}
+
+	return t.RenderResource("aws_cloudwatch_event_target", *e.Name, tf)
+}
+
+func (_ *EventBridgeTarget) RenderCloudformation(t *cloudformation.CloudformationTarget, a, e, changes *EventBridgeTarget) error {
+	// There is no Cloudformation EventBridge Target resource. Instead it's included in Cloudformation's EventBridge Rule resource
+	return nil
+}

upup/pkg/fi/cloudup/awstasks/eventbridgetarget_fitask.go

@@ -0,0 +1,51 @@
+// +build !ignore_autogenerated
+
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by fitask. DO NOT EDIT.
+
+package awstasks
+
+import (
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+// EventBridgeTarget
+
+var _ fi.HasLifecycle = &EventBridgeTarget{}
+
+// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle
+func (o *EventBridgeTarget) GetLifecycle() *fi.Lifecycle {
+	return o.Lifecycle
+}
+
+// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle
+func (o *EventBridgeTarget) SetLifecycle(lifecycle fi.Lifecycle) {
+	o.Lifecycle = &lifecycle
+}
+
+var _ fi.HasName = &EventBridgeTarget{}
+
+// GetName returns the Name of the object, implementing fi.HasName
+func (o *EventBridgeTarget) GetName() *string {
+	return o.Name
+}
+
+// String is the stringer function for the task, producing readable output using fi.TaskAsString
+func (o *EventBridgeTarget) String() string {
+	return fi.TaskAsString(o)
+}

upup/pkg/fi/cloudup/awstasks/sqs.go

@@ -0,0 +1,246 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package awstasks
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	"k8s.io/apimachinery/pkg/util/validation/field"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/sqs"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
+	"k8s.io/kops/upup/pkg/fi/cloudup/cloudformation"
+	"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
+)
+
+// +kops:fitask
+type SQS struct {
+	Name      *string
+	Lifecycle *fi.Lifecycle
+
+	URL                    *string
+	MessageRetentionPeriod int
+	Policy                 fi.Resource // "inline" IAM policy
+
+	Tags map[string]string
+}
+
+var _ fi.CompareWithID = &SQS{}
+
+func (q *SQS) CompareWithID() *string {
+	return q.URL
+}
+
+func (q *SQS) Find(c *fi.Context) (*SQS, error) {
+	cloud := c.Cloud.(awsup.AWSCloud)
+
+	if q.Name == nil {
+		return nil, nil
+	}
+
+	response, err := cloud.SQS().ListQueues(&sqs.ListQueuesInput{
+		MaxResults:      aws.Int64(2),
+		QueueNamePrefix: q.Name,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error listing SQS queues: %v", err)
+	}
+	if response == nil || len(response.QueueUrls) == 0 {
+		return nil, nil
+	}
+	if len(response.QueueUrls) != 1 {
+		return nil, fmt.Errorf("found multiple SQS queues matching queue name")
+	}
+	url := response.QueueUrls[0]
+
+	attributes, err := cloud.SQS().GetQueueAttributes(&sqs.GetQueueAttributesInput{
+		AttributeNames: []*string{s("MessageRetentionPeriod"), s("Policy")},
+		QueueUrl:       url,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error getting SQS queue attributes: %v", err)
+	}
+	policy := fi.NewStringResource(*attributes.Attributes["Policy"])
+	period, err := strconv.Atoi(*attributes.Attributes["MessageRetentionPeriod"])
+	if err != nil {
+		return nil, fmt.Errorf("error coverting MessageRetentionPeriod to int: %v", err)
+	}
+
+	tags, err := cloud.SQS().ListQueueTags(&sqs.ListQueueTagsInput{
+		QueueUrl: url,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error listing SQS queue tags: %v", err)
+	}
+
+	actual := &SQS{
+		Name:                   q.Name,
+		URL:                    url,
+		Lifecycle:              q.Lifecycle,
+		Policy:                 policy,
+		MessageRetentionPeriod: period,
+		Tags:                   intersectSQSTags(tags.Tags, q.Tags),
+	}
+
+	return actual, nil
+}
+
+func (q *SQS) Run(c *fi.Context) error {
+	return fi.DefaultDeltaRunMethod(q, c)
+}
+
+func (q *SQS) CheckChanges(a, e, changes *SQS) error {
+	if a == nil {
+		if e.Name == nil {
+			return field.Required(field.NewPath("Name"), "")
+		}
+	}
+	if a != nil {
+		if changes.URL != nil {
+			return fi.CannotChangeField("URL")
+		}
+	}
+	return nil
+}
+
+func (q *SQS) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *SQS) error {
+	policy, err := fi.ResourceAsString(e.Policy)
+	if err != nil {
+		return fmt.Errorf("error rendering RolePolicyDocument: %v", err)
+	}
+
+	if a == nil {
+		request := &sqs.CreateQueueInput{
+			Attributes: map[string]*string{
+				"MessageRetentionPeriod": s(strconv.Itoa(q.MessageRetentionPeriod)),
+				"Policy":                 s(policy),
+			},
+			QueueName: q.Name,
+			Tags:      convertTagsToPointers(q.Tags),
+		}
+		response, err := t.Cloud.SQS().CreateQueue(request)
+		if err != nil {
+			return fmt.Errorf("error creating SQS queue: %v", err)
+		}
+
+		q.URL = response.QueueUrl
+	}
+
+	return nil
+}
+
+type terraformSQSQueue struct {
+	Name                    *string            `json:"name" cty:"name"`
+	MessageRetentionSeconds int                `json:"message_retention_seconds" cty:"message_retention_seconds"`
+	Policy                  *terraform.Literal `json:"policy" cty:"policy"`
+	Tags                    map[string]string  `json:"tags" cty:"tags"`
+}
+
+func (_ *SQS) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *SQS) error {
+	p, err := t.AddFile("aws_sqs_queue", *e.Name, "policy", e.Policy, false)
+	if err != nil {
+		return err
+	}
+
+	tf := &terraformSQSQueue{
+		Name:                    e.Name,
+		MessageRetentionSeconds: e.MessageRetentionPeriod,
+		Policy:                  p,
+		Tags:                    e.Tags,
+	}
+
+	return t.RenderResource("aws_sqs_queue", *e.Name, tf)
+}
+
+type cloudformationSQSQueue struct {
+	QueueName              *string             `json:"QueueName"`
+	MessageRetentionPeriod int                 `json:"MessageRetentionPeriod"`
+	Tags                   []cloudformationTag `json:"Tags,omitempty"`
+}
+
+type cloudformationSQSQueuePolicy struct {
+	Queues         []*cloudformation.Literal `json:"Queues"`
+	PolicyDocument map[string]interface{}    `json:"PolicyDocument"`
+	Tags           []cloudformationTag       `json:"Tags,omitempty"`
+}
+
+func (_ *SQS) RenderCloudformation(t *cloudformation.CloudformationTarget, a, e, changes *SQS) error {
+	cfQueue := &cloudformationSQSQueue{
+		QueueName:              e.Name,
+		MessageRetentionPeriod: e.MessageRetentionPeriod,
+		Tags:                   buildCloudformationTags(e.Tags),
+	}
+
+	err := t.RenderResource("AWS::SQS::Queue", *e.Name, cfQueue)
+	if err != nil {
+		return err
+	}
+
+	// convert Policy string into json
+	jsonString, err := fi.ResourceAsBytes(e.Policy)
+	if err != nil {
+		return err
+	}
+	data := make(map[string]interface{})
+	err = json.Unmarshal(jsonString, &data)
+	if err != nil {
+		return fmt.Errorf("error parsing SQS PolicyDocument: %v", err)
+	}
+
+	cfQueueRef := cloudformation.Ref("AWS::SQS::Queue", fi.StringValue(e.Name))
+
+	cfQueuePolicy := &cloudformationSQSQueuePolicy{
+		Queues:         []*cloudformation.Literal{cfQueueRef},
+		PolicyDocument: data,
+	}
+	return t.RenderResource("AWS::SQS::QueuePolicy", *e.Name+"Policy", cfQueuePolicy)
+}
+
+// change tags to format required by CreateQueue
+func convertTagsToPointers(tags map[string]string) map[string]*string {
+	newTags := map[string]*string{}
+	for k, v := range tags {
+		vv := v
+		newTags[k] = &vv
+	}
+
+	return newTags
+}
+
+// intersectSQSTags does the same thing as intersectTags, but takes different input because SQS tags are listed differently
+func intersectSQSTags(tags map[string]*string, desired map[string]string) map[string]string {
+	if tags == nil {
+		return nil
+	}
+	actual := make(map[string]string)
+	for k, v := range tags {
+		vv := aws.StringValue(v)
+
+		if _, found := desired[k]; found {
+			actual[k] = vv
+		}
+	}
+	if len(actual) == 0 && desired == nil {
+		// Avoid problems with comparison between nil & {}
+		return nil
+	}
+	return actual
+}

upup/pkg/fi/cloudup/awstasks/sqs_fitask.go

@@ -0,0 +1,51 @@
+// +build !ignore_autogenerated
+
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by fitask. DO NOT EDIT.
+
+package awstasks
+
+import (
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+// SQS
+
+var _ fi.HasLifecycle = &SQS{}
+
+// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle
+func (o *SQS) GetLifecycle() *fi.Lifecycle {
+	return o.Lifecycle
+}
+
+// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle
+func (o *SQS) SetLifecycle(lifecycle fi.Lifecycle) {
+	o.Lifecycle = &lifecycle
+}
+
+var _ fi.HasName = &SQS{}
+
+// GetName returns the Name of the object, implementing fi.HasName
+func (o *SQS) GetName() *string {
+	return o.Name
+}
+
+// String is the stringer function for the task, producing readable output using fi.TaskAsString
+func (o *SQS) String() string {
+	return fi.TaskAsString(o)
+}

upup/pkg/fi/cloudup/awstasks/tags.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/eventbridge"
 	"github.com/aws/aws-sdk-go/service/iam"
 )
 
@@ -66,6 +67,20 @@ func mapToIAMTags(tags map[string]string) []*iam.Tag {
 	return m
 }
 
+func mapEventBridgeTagsToMap(tags []*eventbridge.Tag) map[string]string {
+	if tags == nil {
+		return nil
+	}
+	m := make(map[string]string)
+	for _, t := range tags {
+		if strings.HasPrefix(aws.StringValue(t.Key), "aws:cloudformation:") {
+			continue
+		}
+		m[aws.StringValue(t.Key)] = aws.StringValue(t.Value)
+	}
+	return m
+}
+
 func findNameTag(tags []*ec2.Tag) *string {
 	for _, tag := range tags {
 		if aws.StringValue(tag.Key) == "Name" {

upup/pkg/fi/cloudup/awsup/BUILD.bazel

@@ -45,10 +45,14 @@ go_library(
         "//vendor/github.com/aws/aws-sdk-go/service/elb/elbiface:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/elbv2:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/elbv2/elbv2iface:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/eventbridge:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/eventbridge/eventbridgeiface:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/iam:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/iam/iamiface:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/route53:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/route53/route53iface:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/sqs:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/sqs/sqsiface:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/service/sts:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",

upup/pkg/fi/cloudup/awsup/aws_cloud.go

@@ -23,6 +23,11 @@ import (
 	"sync"
 	"time"
 
+	"github.com/aws/aws-sdk-go/service/eventbridge"
+	"github.com/aws/aws-sdk-go/service/eventbridge/eventbridgeiface"
+	"github.com/aws/aws-sdk-go/service/sqs"
+	"github.com/aws/aws-sdk-go/service/sqs/sqsiface"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/arn"
 	"github.com/aws/aws-sdk-go/aws/awserr"
@@ -114,6 +119,9 @@ type AWSCloud interface {
 	Route53() route53iface.Route53API
 	Spotinst() spotinst.Cloud
 
+	SQS() sqsiface.SQSAPI
+	EventBridge() eventbridgeiface.EventBridgeAPI
+
 	// TODO: Document and rationalize these tags/filters methods
 	AddTags(name *string, tags map[string]string)
 	BuildFilters(name *string) []*ec2.Filter
@@ -181,6 +189,8 @@ type awsCloudImplementation struct {
 	route53     *route53.Route53
 	spotinst    spotinst.Cloud
 	sts         *sts.STS
+	sqs         *sqs.SQS
+	eventbridge *eventbridge.EventBridge
 
 	region string
 
@@ -314,6 +324,22 @@ func NewAWSCloud(region string, tags map[string]string) (AWSCloud, error) {
 			}
 		}
 
+		sess, err = session.NewSession(config)
+		if err != nil {
+			return c, err
+		}
+		c.sqs = sqs.New(sess, config)
+		c.sqs.Handlers.Send.PushFront(requestLogger)
+		c.addHandlers(region, &c.sqs.Handlers)
+
+		sess, err = session.NewSession(config)
+		if err != nil {
+			return c, err
+		}
+		c.eventbridge = eventbridge.New(sess, config)
+		c.eventbridge.Handlers.Send.PushFront(requestLogger)
+		c.addHandlers(region, &c.eventbridge.Handlers)
+
 		awsCloudInstances[region] = c
 		raw = c
 	}
@@ -1608,6 +1634,14 @@ func (c *awsCloudImplementation) Spotinst() spotinst.Cloud {
 	return c.spotinst
 }
 
+func (c *awsCloudImplementation) SQS() sqsiface.SQSAPI {
+	return c.sqs
+}
+
+func (c *awsCloudImplementation) EventBridge() eventbridgeiface.EventBridgeAPI {
+	return c.eventbridge
+}
+
 func (c *awsCloudImplementation) FindVPCInfo(vpcID string) (*fi.VPCInfo, error) {
 	return findVPCInfo(c, vpcID)
 }

upup/pkg/fi/cloudup/awsup/mock_aws_cloud.go

@@ -19,6 +19,9 @@ package awsup
 import (
 	"fmt"
 
+	"github.com/aws/aws-sdk-go/service/eventbridge/eventbridgeiface"
+	"github.com/aws/aws-sdk-go/service/sqs/sqsiface"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/autoscaling/autoscalingiface"
 	"github.com/aws/aws-sdk-go/service/cloudformation"
@@ -80,6 +83,8 @@ type MockCloud struct {
 	MockELB            elbiface.ELBAPI
 	MockELBV2          elbv2iface.ELBV2API
 	MockSpotinst       spotinst.Cloud
+	MockSQS            sqsiface.SQSAPI
+	MockEventBridge    eventbridgeiface.EventBridgeAPI
 }
 
 func (c *MockAWSCloud) DeleteGroup(g *cloudinstances.CloudInstanceGroup) error {
@@ -261,6 +266,20 @@ func (c *MockAWSCloud) Spotinst() spotinst.Cloud {
 	return c.MockSpotinst
 }
 
+func (c *MockAWSCloud) SQS() sqsiface.SQSAPI {
+	if c.MockSQS == nil {
+		klog.Fatalf("MockSQS not set")
+	}
+	return c.MockSQS
+}
+
+func (c *MockAWSCloud) EventBridge() eventbridgeiface.EventBridgeAPI {
+	if c.MockEventBridge == nil {
+		klog.Fatalf("MockEventBridgess not set")
+	}
+	return c.MockEventBridge
+}
+
 func (c *MockAWSCloud) FindVPCInfo(id string) (*fi.VPCInfo, error) {
 	return findVPCInfo(c, id)
 }

upup/pkg/fi/cloudup/template_functions.go

@@ -217,6 +217,20 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS
 
 	dest["UseServiceAccountIAM"] = tf.UseServiceAccountIAM
 
+	if cluster.Spec.NodeTerminationHandler != nil {
+		dest["DefaultQueueName"] = func() string {
+			s := strings.Replace(tf.ClusterName(), ".", "-", -1)
+			domain := ".amazonaws.com/"
+			if strings.Contains(tf.Region, "cn-") {
+				domain = ".amazonaws.com.cn/"
+			}
+			url := "https://sqs." + tf.Region + domain + tf.AWSAccountID + "/" + s + "-nth"
+			return url
+		}
+
+		dest["EnableSQSTerminationDraining"] = func() bool { return *cluster.Spec.NodeTerminationHandler.EnableSQSTerminationDraining }
+	}
+
 	return nil
 }
 

vendor/github.com/aws/aws-sdk-go/service/eventbridge/BUILD.bazel

@@ -0,0 +1,24 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "api.go",
+        "doc.go",
+        "errors.go",
+        "service.go",
+    ],
+    importmap = "k8s.io/kops/vendor/github.com/aws/aws-sdk-go/service/eventbridge",
+    importpath = "github.com/aws/aws-sdk-go/service/eventbridge",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/awsutil:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/client:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/client/metadata:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/request:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/signer/v4:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/private/protocol:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/private/protocol/jsonrpc:go_default_library",
+    ],
+)

vendor/github.com/aws/aws-sdk-go/service/eventbridge/doc.go

@@ -0,0 +1,46 @@
+// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT.
+
+// Package eventbridge provides the client and types for making API
+// requests to Amazon EventBridge.
+//
+// Amazon EventBridge helps you to respond to state changes in your AWS resources.
+// When your resources change state, they automatically send events into an
+// event stream. You can create rules that match selected events in the stream
+// and route them to targets to take action. You can also use rules to take
+// action on a predetermined schedule. For example, you can configure rules
+// to:
+//
+//    * Automatically invoke an AWS Lambda function to update DNS entries when
+//    an event notifies you that Amazon EC2 instance enters the running state.
+//
+//    * Direct specific API records from AWS CloudTrail to an Amazon Kinesis
+//    data stream for detailed analysis of potential security or availability
+//    risks.
+//
+//    * Periodically invoke a built-in target to create a snapshot of an Amazon
+//    EBS volume.
+//
+// For more information about the features of Amazon EventBridge, see the Amazon
+// EventBridge User Guide (https://docs.aws.amazon.com/eventbridge/latest/userguide).
+//
+// See https://docs.aws.amazon.com/goto/WebAPI/eventbridge-2015-10-07 for more information on this service.
+//
+// See eventbridge package documentation for more information.
+// https://docs.aws.amazon.com/sdk-for-go/api/service/eventbridge/
+//
+// Using the Client
+//
+// To contact Amazon EventBridge with the SDK use the New function to create
+// a new service client. With that client you can make API requests to the service.
+// These clients are safe to use concurrently.
+//
+// See the SDK's documentation for more information on how to use the SDK.
+// https://docs.aws.amazon.com/sdk-for-go/api/
+//
+// See aws.Config documentation for more information on configuring SDK clients.
+// https://docs.aws.amazon.com/sdk-for-go/api/aws/#Config
+//
+// See the Amazon EventBridge client EventBridge for more
+// information on creating client for this service.
+// https://docs.aws.amazon.com/sdk-for-go/api/service/eventbridge/#New
+package eventbridge

vendor/github.com/aws/aws-sdk-go/service/eventbridge/errors.go

@@ -0,0 +1,97 @@
+// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT.
+
+package eventbridge
+
+import (
+	"github.com/aws/aws-sdk-go/private/protocol"
+)
+
+const (
+
+	// ErrCodeConcurrentModificationException for service response error code
+	// "ConcurrentModificationException".
+	//
+	// There is concurrent modification on a rule, target, archive, or replay.
+	ErrCodeConcurrentModificationException = "ConcurrentModificationException"
+
+	// ErrCodeIllegalStatusException for service response error code
+	// "IllegalStatusException".
+	//
+	// An error occurred because a replay can be canceled only when the state is
+	// Running or Starting.
+	ErrCodeIllegalStatusException = "IllegalStatusException"
+
+	// ErrCodeInternalException for service response error code
+	// "InternalException".
+	//
+	// This exception occurs due to unexpected causes.
+	ErrCodeInternalException = "InternalException"
+
+	// ErrCodeInvalidEventPatternException for service response error code
+	// "InvalidEventPatternException".
+	//
+	// The event pattern is not valid.
+	ErrCodeInvalidEventPatternException = "InvalidEventPatternException"
+
+	// ErrCodeInvalidStateException for service response error code
+	// "InvalidStateException".
+	//
+	// The specified state is not a valid state for an event source.
+	ErrCodeInvalidStateException = "InvalidStateException"
+
+	// ErrCodeLimitExceededException for service response error code
+	// "LimitExceededException".
+	//
+	// The request failed because it attempted to create resource beyond the allowed
+	// service quota.
+	ErrCodeLimitExceededException = "LimitExceededException"
+
+	// ErrCodeManagedRuleException for service response error code
+	// "ManagedRuleException".
+	//
+	// This rule was created by an AWS service on behalf of your account. It is
+	// managed by that service. If you see this error in response to DeleteRule
+	// or RemoveTargets, you can use the Force parameter in those calls to delete
+	// the rule or remove targets from the rule. You cannot modify these managed
+	// rules by using DisableRule, EnableRule, PutTargets, PutRule, TagResource,
+	// or UntagResource.
+	ErrCodeManagedRuleException = "ManagedRuleException"
+
+	// ErrCodeOperationDisabledException for service response error code
+	// "OperationDisabledException".
+	//
+	// The operation you are attempting is not available in this region.
+	ErrCodeOperationDisabledException = "OperationDisabledException"
+
+	// ErrCodePolicyLengthExceededException for service response error code
+	// "PolicyLengthExceededException".
+	//
+	// The event bus policy is too long. For more information, see the limits.
+	ErrCodePolicyLengthExceededException = "PolicyLengthExceededException"
+
+	// ErrCodeResourceAlreadyExistsException for service response error code
+	// "ResourceAlreadyExistsException".
+	//
+	// The resource you are trying to create already exists.
+	ErrCodeResourceAlreadyExistsException = "ResourceAlreadyExistsException"
+
+	// ErrCodeResourceNotFoundException for service response error code
+	// "ResourceNotFoundException".
+	//
+	// An entity that you specified does not exist.
+	ErrCodeResourceNotFoundException = "ResourceNotFoundException"
+)
+
+var exceptionFromCode = map[string]func(protocol.ResponseMetadata) error{
+	"ConcurrentModificationException": newErrorConcurrentModificationException,
+	"IllegalStatusException":          newErrorIllegalStatusException,
+	"InternalException":               newErrorInternalException,
+	"InvalidEventPatternException":    newErrorInvalidEventPatternException,
+	"InvalidStateException":           newErrorInvalidStateException,
+	"LimitExceededException":          newErrorLimitExceededException,
+	"ManagedRuleException":            newErrorManagedRuleException,
+	"OperationDisabledException":      newErrorOperationDisabledException,
+	"PolicyLengthExceededException":   newErrorPolicyLengthExceededException,
+	"ResourceAlreadyExistsException":  newErrorResourceAlreadyExistsException,
+	"ResourceNotFoundException":       newErrorResourceNotFoundException,
+}

vendor/github.com/aws/aws-sdk-go/service/eventbridge/eventbridgeiface/BUILD.bazel

@@ -0,0 +1,14 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["interface.go"],
+    importmap = "k8s.io/kops/vendor/github.com/aws/aws-sdk-go/service/eventbridge/eventbridgeiface",
+    importpath = "github.com/aws/aws-sdk-go/service/eventbridge/eventbridgeiface",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/request:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/eventbridge:go_default_library",
+    ],
+)

vendor/github.com/aws/aws-sdk-go/service/eventbridge/eventbridgeiface/interface.go

@@ -0,0 +1,268 @@
+// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT.
+
+// Package eventbridgeiface provides an interface to enable mocking the Amazon EventBridge service client
+// for testing your code.
+//
+// It is important to note that this interface will have breaking changes
+// when the service model is updated and adds new API operations, paginators,
+// and waiters.
+package eventbridgeiface
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/service/eventbridge"
+)
+
+// EventBridgeAPI provides an interface to enable mocking the
+// eventbridge.EventBridge service client's API operation,
+// paginators, and waiters. This make unit testing your code that calls out
+// to the SDK's service client's calls easier.
+//
+// The best way to use this interface is so the SDK's service client's calls
+// can be stubbed out for unit testing your code with the SDK without needing
+// to inject custom request handlers into the SDK's request pipeline.
+//
+//    // myFunc uses an SDK service client to make a request to
+//    // Amazon EventBridge.
+//    func myFunc(svc eventbridgeiface.EventBridgeAPI) bool {
+//        // Make svc.ActivateEventSource request
+//    }
+//
+//    func main() {
+//        sess := session.New()
+//        svc := eventbridge.New(sess)
+//
+//        myFunc(svc)
+//    }
+//
+// In your _test.go file:
+//
+//    // Define a mock struct to be used in your unit tests of myFunc.
+//    type mockEventBridgeClient struct {
+//        eventbridgeiface.EventBridgeAPI
+//    }
+//    func (m *mockEventBridgeClient) ActivateEventSource(input *eventbridge.ActivateEventSourceInput) (*eventbridge.ActivateEventSourceOutput, error) {
+//        // mock response/functionality
+//    }
+//
+//    func TestMyFunc(t *testing.T) {
+//        // Setup Test
+//        mockSvc := &mockEventBridgeClient{}
+//
+//        myfunc(mockSvc)
+//
+//        // Verify myFunc's functionality
+//    }
+//
+// It is important to note that this interface will have breaking changes
+// when the service model is updated and adds new API operations, paginators,
+// and waiters. Its suggested to use the pattern above for testing, or using
+// tooling to generate mocks to satisfy the interfaces.
+type EventBridgeAPI interface {
+	ActivateEventSource(*eventbridge.ActivateEventSourceInput) (*eventbridge.ActivateEventSourceOutput, error)
+	ActivateEventSourceWithContext(aws.Context, *eventbridge.ActivateEventSourceInput, ...request.Option) (*eventbridge.ActivateEventSourceOutput, error)
+	ActivateEventSourceRequest(*eventbridge.ActivateEventSourceInput) (*request.Request, *eventbridge.ActivateEventSourceOutput)
+
+	CancelReplay(*eventbridge.CancelReplayInput) (*eventbridge.CancelReplayOutput, error)
+	CancelReplayWithContext(aws.Context, *eventbridge.CancelReplayInput, ...request.Option) (*eventbridge.CancelReplayOutput, error)
+	CancelReplayRequest(*eventbridge.CancelReplayInput) (*request.Request, *eventbridge.CancelReplayOutput)
+
+	CreateApiDestination(*eventbridge.CreateApiDestinationInput) (*eventbridge.CreateApiDestinationOutput, error)
+	CreateApiDestinationWithContext(aws.Context, *eventbridge.CreateApiDestinationInput, ...request.Option) (*eventbridge.CreateApiDestinationOutput, error)
+	CreateApiDestinationRequest(*eventbridge.CreateApiDestinationInput) (*request.Request, *eventbridge.CreateApiDestinationOutput)
+
+	CreateArchive(*eventbridge.CreateArchiveInput) (*eventbridge.CreateArchiveOutput, error)
+	CreateArchiveWithContext(aws.Context, *eventbridge.CreateArchiveInput, ...request.Option) (*eventbridge.CreateArchiveOutput, error)
+	CreateArchiveRequest(*eventbridge.CreateArchiveInput) (*request.Request, *eventbridge.CreateArchiveOutput)
+
+	CreateConnection(*eventbridge.CreateConnectionInput) (*eventbridge.CreateConnectionOutput, error)
+	CreateConnectionWithContext(aws.Context, *eventbridge.CreateConnectionInput, ...request.Option) (*eventbridge.CreateConnectionOutput, error)
+	CreateConnectionRequest(*eventbridge.CreateConnectionInput) (*request.Request, *eventbridge.CreateConnectionOutput)
+
+	CreateEventBus(*eventbridge.CreateEventBusInput) (*eventbridge.CreateEventBusOutput, error)
+	CreateEventBusWithContext(aws.Context, *eventbridge.CreateEventBusInput, ...request.Option) (*eventbridge.CreateEventBusOutput, error)
+	CreateEventBusRequest(*eventbridge.CreateEventBusInput) (*request.Request, *eventbridge.CreateEventBusOutput)
+
+	CreatePartnerEventSource(*eventbridge.CreatePartnerEventSourceInput) (*eventbridge.CreatePartnerEventSourceOutput, error)
+	CreatePartnerEventSourceWithContext(aws.Context, *eventbridge.CreatePartnerEventSourceInput, ...request.Option) (*eventbridge.CreatePartnerEventSourceOutput, error)
+	CreatePartnerEventSourceRequest(*eventbridge.CreatePartnerEventSourceInput) (*request.Request, *eventbridge.CreatePartnerEventSourceOutput)
+
+	DeactivateEventSource(*eventbridge.DeactivateEventSourceInput) (*eventbridge.DeactivateEventSourceOutput, error)
+	DeactivateEventSourceWithContext(aws.Context, *eventbridge.DeactivateEventSourceInput, ...request.Option) (*eventbridge.DeactivateEventSourceOutput, error)
+	DeactivateEventSourceRequest(*eventbridge.DeactivateEventSourceInput) (*request.Request, *eventbridge.DeactivateEventSourceOutput)
+
+	DeauthorizeConnection(*eventbridge.DeauthorizeConnectionInput) (*eventbridge.DeauthorizeConnectionOutput, error)
+	DeauthorizeConnectionWithContext(aws.Context, *eventbridge.DeauthorizeConnectionInput, ...request.Option) (*eventbridge.DeauthorizeConnectionOutput, error)
+	DeauthorizeConnectionRequest(*eventbridge.DeauthorizeConnectionInput) (*request.Request, *eventbridge.DeauthorizeConnectionOutput)
+
+	DeleteApiDestination(*eventbridge.DeleteApiDestinationInput) (*eventbridge.DeleteApiDestinationOutput, error)
+	DeleteApiDestinationWithContext(aws.Context, *eventbridge.DeleteApiDestinationInput, ...request.Option) (*eventbridge.DeleteApiDestinationOutput, error)
+	DeleteApiDestinationRequest(*eventbridge.DeleteApiDestinationInput) (*request.Request, *eventbridge.DeleteApiDestinationOutput)
+
+	DeleteArchive(*eventbridge.DeleteArchiveInput) (*eventbridge.DeleteArchiveOutput, error)
+	DeleteArchiveWithContext(aws.Context, *eventbridge.DeleteArchiveInput, ...request.Option) (*eventbridge.DeleteArchiveOutput, error)
+	DeleteArchiveRequest(*eventbridge.DeleteArchiveInput) (*request.Request, *eventbridge.DeleteArchiveOutput)
+
+	DeleteConnection(*eventbridge.DeleteConnectionInput) (*eventbridge.DeleteConnectionOutput, error)
+	DeleteConnectionWithContext(aws.Context, *eventbridge.DeleteConnectionInput, ...request.Option) (*eventbridge.DeleteConnectionOutput, error)
+	DeleteConnectionRequest(*eventbridge.DeleteConnectionInput) (*request.Request, *eventbridge.DeleteConnectionOutput)
+
+	DeleteEventBus(*eventbridge.DeleteEventBusInput) (*eventbridge.DeleteEventBusOutput, error)
+	DeleteEventBusWithContext(aws.Context, *eventbridge.DeleteEventBusInput, ...request.Option) (*eventbridge.DeleteEventBusOutput, error)
+	DeleteEventBusRequest(*eventbridge.DeleteEventBusInput) (*request.Request, *eventbridge.DeleteEventBusOutput)
+
+	DeletePartnerEventSource(*eventbridge.DeletePartnerEventSourceInput) (*eventbridge.DeletePartnerEventSourceOutput, error)
+	DeletePartnerEventSourceWithContext(aws.Context, *eventbridge.DeletePartnerEventSourceInput, ...request.Option) (*eventbridge.DeletePartnerEventSourceOutput, error)
+	DeletePartnerEventSourceRequest(*eventbridge.DeletePartnerEventSourceInput) (*request.Request, *eventbridge.DeletePartnerEventSourceOutput)
+
+	DeleteRule(*eventbridge.DeleteRuleInput) (*eventbridge.DeleteRuleOutput, error)
+	DeleteRuleWithContext(aws.Context, *eventbridge.DeleteRuleInput, ...request.Option) (*eventbridge.DeleteRuleOutput, error)
+	DeleteRuleRequest(*eventbridge.DeleteRuleInput) (*request.Request, *eventbridge.DeleteRuleOutput)
+
+	DescribeApiDestination(*eventbridge.DescribeApiDestinationInput) (*eventbridge.DescribeApiDestinationOutput, error)
+	DescribeApiDestinationWithContext(aws.Context, *eventbridge.DescribeApiDestinationInput, ...request.Option) (*eventbridge.DescribeApiDestinationOutput, error)
+	DescribeApiDestinationRequest(*eventbridge.DescribeApiDestinationInput) (*request.Request, *eventbridge.DescribeApiDestinationOutput)
+
+	DescribeArchive(*eventbridge.DescribeArchiveInput) (*eventbridge.DescribeArchiveOutput, error)
+	DescribeArchiveWithContext(aws.Context, *eventbridge.DescribeArchiveInput, ...request.Option) (*eventbridge.DescribeArchiveOutput, error)
+	DescribeArchiveRequest(*eventbridge.DescribeArchiveInput) (*request.Request, *eventbridge.DescribeArchiveOutput)
+
+	DescribeConnection(*eventbridge.DescribeConnectionInput) (*eventbridge.DescribeConnectionOutput, error)
+	DescribeConnectionWithContext(aws.Context, *eventbridge.DescribeConnectionInput, ...request.Option) (*eventbridge.DescribeConnectionOutput, error)
+	DescribeConnectionRequest(*eventbridge.DescribeConnectionInput) (*request.Request, *eventbridge.DescribeConnectionOutput)
+
+	DescribeEventBus(*eventbridge.DescribeEventBusInput) (*eventbridge.DescribeEventBusOutput, error)
+	DescribeEventBusWithContext(aws.Context, *eventbridge.DescribeEventBusInput, ...request.Option) (*eventbridge.DescribeEventBusOutput, error)
+	DescribeEventBusRequest(*eventbridge.DescribeEventBusInput) (*request.Request, *eventbridge.DescribeEventBusOutput)
+
+	DescribeEventSource(*eventbridge.DescribeEventSourceInput) (*eventbridge.DescribeEventSourceOutput, error)
+	DescribeEventSourceWithContext(aws.Context, *eventbridge.DescribeEventSourceInput, ...request.Option) (*eventbridge.DescribeEventSourceOutput, error)
+	DescribeEventSourceRequest(*eventbridge.DescribeEventSourceInput) (*request.Request, *eventbridge.DescribeEventSourceOutput)
+
+	DescribePartnerEventSource(*eventbridge.DescribePartnerEventSourceInput) (*eventbridge.DescribePartnerEventSourceOutput, error)
+	DescribePartnerEventSourceWithContext(aws.Context, *eventbridge.DescribePartnerEventSourceInput, ...request.Option) (*eventbridge.DescribePartnerEventSourceOutput, error)
+	DescribePartnerEventSourceRequest(*eventbridge.DescribePartnerEventSourceInput) (*request.Request, *eventbridge.DescribePartnerEventSourceOutput)
+
+	DescribeReplay(*eventbridge.DescribeReplayInput) (*eventbridge.DescribeReplayOutput, error)
+	DescribeReplayWithContext(aws.Context, *eventbridge.DescribeReplayInput, ...request.Option) (*eventbridge.DescribeReplayOutput, error)
+	DescribeReplayRequest(*eventbridge.DescribeReplayInput) (*request.Request, *eventbridge.DescribeReplayOutput)
+
+	DescribeRule(*eventbridge.DescribeRuleInput) (*eventbridge.DescribeRuleOutput, error)
+	DescribeRuleWithContext(aws.Context, *eventbridge.DescribeRuleInput, ...request.Option) (*eventbridge.DescribeRuleOutput, error)
+	DescribeRuleRequest(*eventbridge.DescribeRuleInput) (*request.Request, *eventbridge.DescribeRuleOutput)
+
+	DisableRule(*eventbridge.DisableRuleInput) (*eventbridge.DisableRuleOutput, error)
+	DisableRuleWithContext(aws.Context, *eventbridge.DisableRuleInput, ...request.Option) (*eventbridge.DisableRuleOutput, error)
+	DisableRuleRequest(*eventbridge.DisableRuleInput) (*request.Request, *eventbridge.DisableRuleOutput)
+
+	EnableRule(*eventbridge.EnableRuleInput) (*eventbridge.EnableRuleOutput, error)
+	EnableRuleWithContext(aws.Context, *eventbridge.EnableRuleInput, ...request.Option) (*eventbridge.EnableRuleOutput, error)
+	EnableRuleRequest(*eventbridge.EnableRuleInput) (*request.Request, *eventbridge.EnableRuleOutput)
+
+	ListApiDestinations(*eventbridge.ListApiDestinationsInput) (*eventbridge.ListApiDestinationsOutput, error)
+	ListApiDestinationsWithContext(aws.Context, *eventbridge.ListApiDestinationsInput, ...request.Option) (*eventbridge.ListApiDestinationsOutput, error)
+	ListApiDestinationsRequest(*eventbridge.ListApiDestinationsInput) (*request.Request, *eventbridge.ListApiDestinationsOutput)
+
+	ListArchives(*eventbridge.ListArchivesInput) (*eventbridge.ListArchivesOutput, error)
+	ListArchivesWithContext(aws.Context, *eventbridge.ListArchivesInput, ...request.Option) (*eventbridge.ListArchivesOutput, error)
+	ListArchivesRequest(*eventbridge.ListArchivesInput) (*request.Request, *eventbridge.ListArchivesOutput)
+
+	ListConnections(*eventbridge.ListConnectionsInput) (*eventbridge.ListConnectionsOutput, error)
+	ListConnectionsWithContext(aws.Context, *eventbridge.ListConnectionsInput, ...request.Option) (*eventbridge.ListConnectionsOutput, error)
+	ListConnectionsRequest(*eventbridge.ListConnectionsInput) (*request.Request, *eventbridge.ListConnectionsOutput)
+
+	ListEventBuses(*eventbridge.ListEventBusesInput) (*eventbridge.ListEventBusesOutput, error)
+	ListEventBusesWithContext(aws.Context, *eventbridge.ListEventBusesInput, ...request.Option) (*eventbridge.ListEventBusesOutput, error)
+	ListEventBusesRequest(*eventbridge.ListEventBusesInput) (*request.Request, *eventbridge.ListEventBusesOutput)
+
+	ListEventSources(*eventbridge.ListEventSourcesInput) (*eventbridge.ListEventSourcesOutput, error)
+	ListEventSourcesWithContext(aws.Context, *eventbridge.ListEventSourcesInput, ...request.Option) (*eventbridge.ListEventSourcesOutput, error)
+	ListEventSourcesRequest(*eventbridge.ListEventSourcesInput) (*request.Request, *eventbridge.ListEventSourcesOutput)
+
+	ListPartnerEventSourceAccounts(*eventbridge.ListPartnerEventSourceAccountsInput) (*eventbridge.ListPartnerEventSourceAccountsOutput, error)
+	ListPartnerEventSourceAccountsWithContext(aws.Context, *eventbridge.ListPartnerEventSourceAccountsInput, ...request.Option) (*eventbridge.ListPartnerEventSourceAccountsOutput, error)
+	ListPartnerEventSourceAccountsRequest(*eventbridge.ListPartnerEventSourceAccountsInput) (*request.Request, *eventbridge.ListPartnerEventSourceAccountsOutput)
+
+	ListPartnerEventSources(*eventbridge.ListPartnerEventSourcesInput) (*eventbridge.ListPartnerEventSourcesOutput, error)
+	ListPartnerEventSourcesWithContext(aws.Context, *eventbridge.ListPartnerEventSourcesInput, ...request.Option) (*eventbridge.ListPartnerEventSourcesOutput, error)
+	ListPartnerEventSourcesRequest(*eventbridge.ListPartnerEventSourcesInput) (*request.Request, *eventbridge.ListPartnerEventSourcesOutput)
+
+	ListReplays(*eventbridge.ListReplaysInput) (*eventbridge.ListReplaysOutput, error)
+	ListReplaysWithContext(aws.Context, *eventbridge.ListReplaysInput, ...request.Option) (*eventbridge.ListReplaysOutput, error)
+	ListReplaysRequest(*eventbridge.ListReplaysInput) (*request.Request, *eventbridge.ListReplaysOutput)
+
+	ListRuleNamesByTarget(*eventbridge.ListRuleNamesByTargetInput) (*eventbridge.ListRuleNamesByTargetOutput, error)
+	ListRuleNamesByTargetWithContext(aws.Context, *eventbridge.ListRuleNamesByTargetInput, ...request.Option) (*eventbridge.ListRuleNamesByTargetOutput, error)
+	ListRuleNamesByTargetRequest(*eventbridge.ListRuleNamesByTargetInput) (*request.Request, *eventbridge.ListRuleNamesByTargetOutput)
+
+	ListRules(*eventbridge.ListRulesInput) (*eventbridge.ListRulesOutput, error)
+	ListRulesWithContext(aws.Context, *eventbridge.ListRulesInput, ...request.Option) (*eventbridge.ListRulesOutput, error)
+	ListRulesRequest(*eventbridge.ListRulesInput) (*request.Request, *eventbridge.ListRulesOutput)
+
+	ListTagsForResource(*eventbridge.ListTagsForResourceInput) (*eventbridge.ListTagsForResourceOutput, error)
+	ListTagsForResourceWithContext(aws.Context, *eventbridge.ListTagsForResourceInput, ...request.Option) (*eventbridge.ListTagsForResourceOutput, error)
+	ListTagsForResourceRequest(*eventbridge.ListTagsForResourceInput) (*request.Request, *eventbridge.ListTagsForResourceOutput)
+
+	ListTargetsByRule(*eventbridge.ListTargetsByRuleInput) (*eventbridge.ListTargetsByRuleOutput, error)
+	ListTargetsByRuleWithContext(aws.Context, *eventbridge.ListTargetsByRuleInput, ...request.Option) (*eventbridge.ListTargetsByRuleOutput, error)
+	ListTargetsByRuleRequest(*eventbridge.ListTargetsByRuleInput) (*request.Request, *eventbridge.ListTargetsByRuleOutput)
+
+	PutEvents(*eventbridge.PutEventsInput) (*eventbridge.PutEventsOutput, error)
+	PutEventsWithContext(aws.Context, *eventbridge.PutEventsInput, ...request.Option) (*eventbridge.PutEventsOutput, error)
+	PutEventsRequest(*eventbridge.PutEventsInput) (*request.Request, *eventbridge.PutEventsOutput)
+
+	PutPartnerEvents(*eventbridge.PutPartnerEventsInput) (*eventbridge.PutPartnerEventsOutput, error)
+	PutPartnerEventsWithContext(aws.Context, *eventbridge.PutPartnerEventsInput, ...request.Option) (*eventbridge.PutPartnerEventsOutput, error)
+	PutPartnerEventsRequest(*eventbridge.PutPartnerEventsInput) (*request.Request, *eventbridge.PutPartnerEventsOutput)
+
+	PutPermission(*eventbridge.PutPermissionInput) (*eventbridge.PutPermissionOutput, error)
+	PutPermissionWithContext(aws.Context, *eventbridge.PutPermissionInput, ...request.Option) (*eventbridge.PutPermissionOutput, error)
+	PutPermissionRequest(*eventbridge.PutPermissionInput) (*request.Request, *eventbridge.PutPermissionOutput)
+
+	PutRule(*eventbridge.PutRuleInput) (*eventbridge.PutRuleOutput, error)
+	PutRuleWithContext(aws.Context, *eventbridge.PutRuleInput, ...request.Option) (*eventbridge.PutRuleOutput, error)
+	PutRuleRequest(*eventbridge.PutRuleInput) (*request.Request, *eventbridge.PutRuleOutput)
+
+	PutTargets(*eventbridge.PutTargetsInput) (*eventbridge.PutTargetsOutput, error)
+	PutTargetsWithContext(aws.Context, *eventbridge.PutTargetsInput, ...request.Option) (*eventbridge.PutTargetsOutput, error)
+	PutTargetsRequest(*eventbridge.PutTargetsInput) (*request.Request, *eventbridge.PutTargetsOutput)
+
+	RemovePermission(*eventbridge.RemovePermissionInput) (*eventbridge.RemovePermissionOutput, error)
+	RemovePermissionWithContext(aws.Context, *eventbridge.RemovePermissionInput, ...request.Option) (*eventbridge.RemovePermissionOutput, error)
+	RemovePermissionRequest(*eventbridge.RemovePermissionInput) (*request.Request, *eventbridge.RemovePermissionOutput)
+
+	RemoveTargets(*eventbridge.RemoveTargetsInput) (*eventbridge.RemoveTargetsOutput, error)
+	RemoveTargetsWithContext(aws.Context, *eventbridge.RemoveTargetsInput, ...request.Option) (*eventbridge.RemoveTargetsOutput, error)
+	RemoveTargetsRequest(*eventbridge.RemoveTargetsInput) (*request.Request, *eventbridge.RemoveTargetsOutput)
+
+	StartReplay(*eventbridge.StartReplayInput) (*eventbridge.StartReplayOutput, error)
+	StartReplayWithContext(aws.Context, *eventbridge.StartReplayInput, ...request.Option) (*eventbridge.StartReplayOutput, error)
+	StartReplayRequest(*eventbridge.StartReplayInput) (*request.Request, *eventbridge.StartReplayOutput)
+
+	TagResource(*eventbridge.TagResourceInput) (*eventbridge.TagResourceOutput, error)
+	TagResourceWithContext(aws.Context, *eventbridge.TagResourceInput, ...request.Option) (*eventbridge.TagResourceOutput, error)
+	TagResourceRequest(*eventbridge.TagResourceInput) (*request.Request, *eventbridge.TagResourceOutput)
+
+	TestEventPattern(*eventbridge.TestEventPatternInput) (*eventbridge.TestEventPatternOutput, error)
+	TestEventPatternWithContext(aws.Context, *eventbridge.TestEventPatternInput, ...request.Option) (*eventbridge.TestEventPatternOutput, error)
+	TestEventPatternRequest(*eventbridge.TestEventPatternInput) (*request.Request, *eventbridge.TestEventPatternOutput)
+
+	UntagResource(*eventbridge.UntagResourceInput) (*eventbridge.UntagResourceOutput, error)
+	UntagResourceWithContext(aws.Context, *eventbridge.UntagResourceInput, ...request.Option) (*eventbridge.UntagResourceOutput, error)
+	UntagResourceRequest(*eventbridge.UntagResourceInput) (*request.Request, *eventbridge.UntagResourceOutput)
+
+	UpdateApiDestination(*eventbridge.UpdateApiDestinationInput) (*eventbridge.UpdateApiDestinationOutput, error)
+	UpdateApiDestinationWithContext(aws.Context, *eventbridge.UpdateApiDestinationInput, ...request.Option) (*eventbridge.UpdateApiDestinationOutput, error)
+	UpdateApiDestinationRequest(*eventbridge.UpdateApiDestinationInput) (*request.Request, *eventbridge.UpdateApiDestinationOutput)
+
+	UpdateArchive(*eventbridge.UpdateArchiveInput) (*eventbridge.UpdateArchiveOutput, error)
+	UpdateArchiveWithContext(aws.Context, *eventbridge.UpdateArchiveInput, ...request.Option) (*eventbridge.UpdateArchiveOutput, error)
+	UpdateArchiveRequest(*eventbridge.UpdateArchiveInput) (*request.Request, *eventbridge.UpdateArchiveOutput)
+
+	UpdateConnection(*eventbridge.UpdateConnectionInput) (*eventbridge.UpdateConnectionOutput, error)
+	UpdateConnectionWithContext(aws.Context, *eventbridge.UpdateConnectionInput, ...request.Option) (*eventbridge.UpdateConnectionOutput, error)
+	UpdateConnectionRequest(*eventbridge.UpdateConnectionInput) (*request.Request, *eventbridge.UpdateConnectionOutput)
+}
+
+var _ EventBridgeAPI = (*eventbridge.EventBridge)(nil)

vendor/github.com/aws/aws-sdk-go/service/eventbridge/service.go

@@ -0,0 +1,103 @@
+// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT.
+
+package eventbridge
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/client"
+	"github.com/aws/aws-sdk-go/aws/client/metadata"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/aws/signer/v4"
+	"github.com/aws/aws-sdk-go/private/protocol"
+	"github.com/aws/aws-sdk-go/private/protocol/jsonrpc"
+)
+
+// EventBridge provides the API operation methods for making requests to
+// Amazon EventBridge. See this package's package overview docs
+// for details on the service.
+//
+// EventBridge methods are safe to use concurrently. It is not safe to
+// modify mutate any of the struct's properties though.
+type EventBridge struct {
+	*client.Client
+}
+
+// Used for custom client initialization logic
+var initClient func(*client.Client)
+
+// Used for custom request initialization logic
+var initRequest func(*request.Request)
+
+// Service information constants
+const (
+	ServiceName = "EventBridge" // Name of service.
+	EndpointsID = "events"      // ID to lookup a service endpoint with.
+	ServiceID   = "EventBridge" // ServiceID is a unique identifier of a specific service.
+)
+
+// New creates a new instance of the EventBridge client with a session.
+// If additional configuration is needed for the client instance use the optional
+// aws.Config parameter to add your extra config.
+//
+// Example:
+//     mySession := session.Must(session.NewSession())
+//
+//     // Create a EventBridge client from just a session.
+//     svc := eventbridge.New(mySession)
+//
+//     // Create a EventBridge client with additional configuration
+//     svc := eventbridge.New(mySession, aws.NewConfig().WithRegion("us-west-2"))
+func New(p client.ConfigProvider, cfgs ...*aws.Config) *EventBridge {
+	c := p.ClientConfig(EndpointsID, cfgs...)
+	return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName)
+}
+
+// newClient creates, initializes and returns a new service client instance.
+func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint, signingRegion, signingName string) *EventBridge {
+	svc := &EventBridge{
+		Client: client.New(
+			cfg,
+			metadata.ClientInfo{
+				ServiceName:   ServiceName,
+				ServiceID:     ServiceID,
+				SigningName:   signingName,
+				SigningRegion: signingRegion,
+				PartitionID:   partitionID,
+				Endpoint:      endpoint,
+				APIVersion:    "2015-10-07",
+				JSONVersion:   "1.1",
+				TargetPrefix:  "AWSEvents",
+			},
+			handlers,
+		),
+	}
+
+	// Handlers
+	svc.Handlers.Sign.PushBackNamed(v4.SignRequestHandler)
+	svc.Handlers.Build.PushBackNamed(jsonrpc.BuildHandler)
+	svc.Handlers.Unmarshal.PushBackNamed(jsonrpc.UnmarshalHandler)
+	svc.Handlers.UnmarshalMeta.PushBackNamed(jsonrpc.UnmarshalMetaHandler)
+	svc.Handlers.UnmarshalError.PushBackNamed(
+		protocol.NewUnmarshalErrorHandler(jsonrpc.NewUnmarshalTypedError(exceptionFromCode)).NamedHandler(),
+	)
+
+	// Run custom client initialization if present
+	if initClient != nil {
+		initClient(svc.Client)
+	}
+
+	return svc
+}
+
+// newRequest creates a new request for a EventBridge operation and runs any
+// custom request initialization.
+func (c *EventBridge) newRequest(op *request.Operation, params, data interface{}) *request.Request {
+	req := c.NewRequest(op, params, data)
+
+	// Run custom request initialization if present
+	if initRequest != nil {
+		initRequest(req)
+	}
+
+	return req
+}

vendor/github.com/aws/aws-sdk-go/service/sqs/BUILD.bazel

@@ -0,0 +1,27 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "api.go",
+        "checksums.go",
+        "customizations.go",
+        "doc.go",
+        "errors.go",
+        "service.go",
+    ],
+    importmap = "k8s.io/kops/vendor/github.com/aws/aws-sdk-go/service/sqs",
+    importpath = "github.com/aws/aws-sdk-go/service/sqs",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/awserr:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/awsutil:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/client:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/client/metadata:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/request:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/signer/v4:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/private/protocol:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/private/protocol/query:go_default_library",
+    ],
+)

vendor/github.com/aws/aws-sdk-go/service/sqs/checksums.go

@@ -0,0 +1,114 @@
+package sqs
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/request"
+)
+
+var (
+	errChecksumMissingBody = fmt.Errorf("cannot compute checksum. missing body")
+	errChecksumMissingMD5  = fmt.Errorf("cannot verify checksum. missing response MD5")
+)
+
+func setupChecksumValidation(r *request.Request) {
+	if aws.BoolValue(r.Config.DisableComputeChecksums) {
+		return
+	}
+
+	switch r.Operation.Name {
+	case opSendMessage:
+		r.Handlers.Unmarshal.PushBack(verifySendMessage)
+	case opSendMessageBatch:
+		r.Handlers.Unmarshal.PushBack(verifySendMessageBatch)
+	case opReceiveMessage:
+		r.Handlers.Unmarshal.PushBack(verifyReceiveMessage)
+	}
+}
+
+func verifySendMessage(r *request.Request) {
+	if r.DataFilled() && r.ParamsFilled() {
+		in := r.Params.(*SendMessageInput)
+		out := r.Data.(*SendMessageOutput)
+		err := checksumsMatch(in.MessageBody, out.MD5OfMessageBody)
+		if err != nil {
+			setChecksumError(r, err.Error())
+		}
+	}
+}
+
+func verifySendMessageBatch(r *request.Request) {
+	if r.DataFilled() && r.ParamsFilled() {
+		entries := map[string]*SendMessageBatchResultEntry{}
+		ids := []string{}
+
+		out := r.Data.(*SendMessageBatchOutput)
+		for _, entry := range out.Successful {
+			entries[*entry.Id] = entry
+		}
+
+		in := r.Params.(*SendMessageBatchInput)
+		for _, entry := range in.Entries {
+			if e, ok := entries[*entry.Id]; ok {
+				if err := checksumsMatch(entry.MessageBody, e.MD5OfMessageBody); err != nil {
+					ids = append(ids, *e.MessageId)
+				}
+			}
+		}
+		if len(ids) > 0 {
+			setChecksumError(r, "invalid messages: %s", strings.Join(ids, ", "))
+		}
+	}
+}
+
+func verifyReceiveMessage(r *request.Request) {
+	if r.DataFilled() && r.ParamsFilled() {
+		ids := []string{}
+		out := r.Data.(*ReceiveMessageOutput)
+		for i, msg := range out.Messages {
+			err := checksumsMatch(msg.Body, msg.MD5OfBody)
+			if err != nil {
+				if msg.MessageId == nil {
+					if r.Config.Logger != nil {
+						r.Config.Logger.Log(fmt.Sprintf(
+							"WARN: SQS.ReceiveMessage failed checksum request id: %s, message %d has no message ID.",
+							r.RequestID, i,
+						))
+					}
+					continue
+				}
+
+				ids = append(ids, *msg.MessageId)
+			}
+		}
+		if len(ids) > 0 {
+			setChecksumError(r, "invalid messages: %s", strings.Join(ids, ", "))
+		}
+	}
+}
+
+func checksumsMatch(body, expectedMD5 *string) error {
+	if body == nil {
+		return errChecksumMissingBody
+	} else if expectedMD5 == nil {
+		return errChecksumMissingMD5
+	}
+
+	msum := md5.Sum([]byte(*body))
+	sum := hex.EncodeToString(msum[:])
+	if sum != *expectedMD5 {
+		return fmt.Errorf("expected MD5 checksum '%s', got '%s'", *expectedMD5, sum)
+	}
+
+	return nil
+}
+
+func setChecksumError(r *request.Request, format string, args ...interface{}) {
+	r.Retryable = aws.Bool(true)
+	r.Error = awserr.New("InvalidChecksum", fmt.Sprintf(format, args...), nil)
+}

vendor/github.com/aws/aws-sdk-go/service/sqs/customizations.go

@@ -0,0 +1,9 @@
+package sqs
+
+import "github.com/aws/aws-sdk-go/aws/request"
+
+func init() {
+	initRequest = func(r *request.Request) {
+		setupChecksumValidation(r)
+	}
+}

vendor/github.com/aws/aws-sdk-go/service/sqs/doc.go

@@ -0,0 +1,59 @@
+// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT.
+
+// Package sqs provides the client and types for making API
+// requests to Amazon Simple Queue Service.
+//
+// Welcome to the Amazon Simple Queue Service API Reference.
+//
+// Amazon Simple Queue Service (Amazon SQS) is a reliable, highly-scalable hosted
+// queue for storing messages as they travel between applications or microservices.
+// Amazon SQS moves data between distributed application components and helps
+// you decouple these components.
+//
+// For information on the permissions you need to use this API, see Identity
+// and access management (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-authentication-and-access-control.html)
+// in the Amazon Simple Queue Service Developer Guide.
+//
+// You can use AWS SDKs (http://aws.amazon.com/tools/#sdk) to access Amazon
+// SQS using your favorite programming language. The SDKs perform tasks such
+// as the following automatically:
+//
+//    * Cryptographically sign your service requests
+//
+//    * Retry requests
+//
+//    * Handle error responses
+//
+// Additional information
+//
+//    * Amazon SQS Product Page (http://aws.amazon.com/sqs/)
+//
+//    * Amazon Simple Queue Service Developer Guide Making API Requests (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-making-api-requests.html)
+//    Amazon SQS Message Attributes (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-metadata.html#sqs-message-attributes)
+//    Amazon SQS Dead-Letter Queues (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-dead-letter-queues.html)
+//
+//    * Amazon SQS in the AWS CLI Command Reference (http://docs.aws.amazon.com/cli/latest/reference/sqs/index.html)
+//
+//    * Amazon Web Services General Reference Regions and Endpoints (https://docs.aws.amazon.com/general/latest/gr/rande.html#sqs_region)
+//
+// See https://docs.aws.amazon.com/goto/WebAPI/sqs-2012-11-05 for more information on this service.
+//
+// See sqs package documentation for more information.
+// https://docs.aws.amazon.com/sdk-for-go/api/service/sqs/
+//
+// Using the Client
+//
+// To contact Amazon Simple Queue Service with the SDK use the New function to create
+// a new service client. With that client you can make API requests to the service.
+// These clients are safe to use concurrently.
+//
+// See the SDK's documentation for more information on how to use the SDK.
+// https://docs.aws.amazon.com/sdk-for-go/api/
+//
+// See aws.Config documentation for more information on configuring SDK clients.
+// https://docs.aws.amazon.com/sdk-for-go/api/aws/#Config
+//
+// See the Amazon Simple Queue Service client SQS for more
+// information on creating client for this service.
+// https://docs.aws.amazon.com/sdk-for-go/api/service/sqs/#New
+package sqs

vendor/github.com/aws/aws-sdk-go/service/sqs/errors.go

@@ -0,0 +1,110 @@
+// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT.
+
+package sqs
+
+const (
+
+	// ErrCodeBatchEntryIdsNotDistinct for service response error code
+	// "AWS.SimpleQueueService.BatchEntryIdsNotDistinct".
+	//
+	// Two or more batch entries in the request have the same Id.
+	ErrCodeBatchEntryIdsNotDistinct = "AWS.SimpleQueueService.BatchEntryIdsNotDistinct"
+
+	// ErrCodeBatchRequestTooLong for service response error code
+	// "AWS.SimpleQueueService.BatchRequestTooLong".
+	//
+	// The length of all the messages put together is more than the limit.
+	ErrCodeBatchRequestTooLong = "AWS.SimpleQueueService.BatchRequestTooLong"
+
+	// ErrCodeEmptyBatchRequest for service response error code
+	// "AWS.SimpleQueueService.EmptyBatchRequest".
+	//
+	// The batch request doesn't contain any entries.
+	ErrCodeEmptyBatchRequest = "AWS.SimpleQueueService.EmptyBatchRequest"
+
+	// ErrCodeInvalidAttributeName for service response error code
+	// "InvalidAttributeName".
+	//
+	// The specified attribute doesn't exist.
+	ErrCodeInvalidAttributeName = "InvalidAttributeName"
+
+	// ErrCodeInvalidBatchEntryId for service response error code
+	// "AWS.SimpleQueueService.InvalidBatchEntryId".
+	//
+	// The Id of a batch entry in a batch request doesn't abide by the specification.
+	ErrCodeInvalidBatchEntryId = "AWS.SimpleQueueService.InvalidBatchEntryId"
+
+	// ErrCodeInvalidIdFormat for service response error code
+	// "InvalidIdFormat".
+	//
+	// The specified receipt handle isn't valid for the current version.
+	ErrCodeInvalidIdFormat = "InvalidIdFormat"
+
+	// ErrCodeInvalidMessageContents for service response error code
+	// "InvalidMessageContents".
+	//
+	// The message contains characters outside the allowed set.
+	ErrCodeInvalidMessageContents = "InvalidMessageContents"
+
+	// ErrCodeMessageNotInflight for service response error code
+	// "AWS.SimpleQueueService.MessageNotInflight".
+	//
+	// The specified message isn't in flight.
+	ErrCodeMessageNotInflight = "AWS.SimpleQueueService.MessageNotInflight"
+
+	// ErrCodeOverLimit for service response error code
+	// "OverLimit".
+	//
+	// The specified action violates a limit. For example, ReceiveMessage returns
+	// this error if the maximum number of inflight messages is reached and AddPermission
+	// returns this error if the maximum number of permissions for the queue is
+	// reached.
+	ErrCodeOverLimit = "OverLimit"
+
+	// ErrCodePurgeQueueInProgress for service response error code
+	// "AWS.SimpleQueueService.PurgeQueueInProgress".
+	//
+	// Indicates that the specified queue previously received a PurgeQueue request
+	// within the last 60 seconds (the time it can take to delete the messages in
+	// the queue).
+	ErrCodePurgeQueueInProgress = "AWS.SimpleQueueService.PurgeQueueInProgress"
+
+	// ErrCodeQueueDeletedRecently for service response error code
+	// "AWS.SimpleQueueService.QueueDeletedRecently".
+	//
+	// You must wait 60 seconds after deleting a queue before you can create another
+	// queue with the same name.
+	ErrCodeQueueDeletedRecently = "AWS.SimpleQueueService.QueueDeletedRecently"
+
+	// ErrCodeQueueDoesNotExist for service response error code
+	// "AWS.SimpleQueueService.NonExistentQueue".
+	//
+	// The specified queue doesn't exist.
+	ErrCodeQueueDoesNotExist = "AWS.SimpleQueueService.NonExistentQueue"
+
+	// ErrCodeQueueNameExists for service response error code
+	// "QueueAlreadyExists".
+	//
+	// A queue with this name already exists. Amazon SQS returns this error only
+	// if the request includes attributes whose values differ from those of the
+	// existing queue.
+	ErrCodeQueueNameExists = "QueueAlreadyExists"
+
+	// ErrCodeReceiptHandleIsInvalid for service response error code
+	// "ReceiptHandleIsInvalid".
+	//
+	// The specified receipt handle isn't valid.
+	ErrCodeReceiptHandleIsInvalid = "ReceiptHandleIsInvalid"
+
+	// ErrCodeTooManyEntriesInBatchRequest for service response error code
+	// "AWS.SimpleQueueService.TooManyEntriesInBatchRequest".
+	//
+	// The batch request contains more entries than permissible.
+	ErrCodeTooManyEntriesInBatchRequest = "AWS.SimpleQueueService.TooManyEntriesInBatchRequest"
+
+	// ErrCodeUnsupportedOperation for service response error code
+	// "AWS.SimpleQueueService.UnsupportedOperation".
+	//
+	// Error code 400. Unsupported operation.
+	ErrCodeUnsupportedOperation = "AWS.SimpleQueueService.UnsupportedOperation"
+)

vendor/github.com/aws/aws-sdk-go/service/sqs/service.go

@@ -0,0 +1,98 @@
+// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT.
+
+package sqs
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/client"
+	"github.com/aws/aws-sdk-go/aws/client/metadata"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/aws/signer/v4"
+	"github.com/aws/aws-sdk-go/private/protocol/query"
+)
+
+// SQS provides the API operation methods for making requests to
+// Amazon Simple Queue Service. See this package's package overview docs
+// for details on the service.
+//
+// SQS methods are safe to use concurrently. It is not safe to
+// modify mutate any of the struct's properties though.
+type SQS struct {
+	*client.Client
+}
+
+// Used for custom client initialization logic
+var initClient func(*client.Client)
+
+// Used for custom request initialization logic
+var initRequest func(*request.Request)
+
+// Service information constants
+const (
+	ServiceName = "sqs"       // Name of service.
+	EndpointsID = ServiceName // ID to lookup a service endpoint with.
+	ServiceID   = "SQS"       // ServiceID is a unique identifier of a specific service.
+)
+
+// New creates a new instance of the SQS client with a session.
+// If additional configuration is needed for the client instance use the optional
+// aws.Config parameter to add your extra config.
+//
+// Example:
+//     mySession := session.Must(session.NewSession())
+//
+//     // Create a SQS client from just a session.
+//     svc := sqs.New(mySession)
+//
+//     // Create a SQS client with additional configuration
+//     svc := sqs.New(mySession, aws.NewConfig().WithRegion("us-west-2"))
+func New(p client.ConfigProvider, cfgs ...*aws.Config) *SQS {
+	c := p.ClientConfig(EndpointsID, cfgs...)
+	return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName)
+}
+
+// newClient creates, initializes and returns a new service client instance.
+func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint, signingRegion, signingName string) *SQS {
+	svc := &SQS{
+		Client: client.New(
+			cfg,
+			metadata.ClientInfo{
+				ServiceName:   ServiceName,
+				ServiceID:     ServiceID,
+				SigningName:   signingName,
+				SigningRegion: signingRegion,
+				PartitionID:   partitionID,
+				Endpoint:      endpoint,
+				APIVersion:    "2012-11-05",
+			},
+			handlers,
+		),
+	}
+
+	// Handlers
+	svc.Handlers.Sign.PushBackNamed(v4.SignRequestHandler)
+	svc.Handlers.Build.PushBackNamed(query.BuildHandler)
+	svc.Handlers.Unmarshal.PushBackNamed(query.UnmarshalHandler)
+	svc.Handlers.UnmarshalMeta.PushBackNamed(query.UnmarshalMetaHandler)
+	svc.Handlers.UnmarshalError.PushBackNamed(query.UnmarshalErrorHandler)
+
+	// Run custom client initialization if present
+	if initClient != nil {
+		initClient(svc.Client)
+	}
+
+	return svc
+}
+
+// newRequest creates a new request for a SQS operation and runs any
+// custom request initialization.
+func (c *SQS) newRequest(op *request.Operation, params, data interface{}) *request.Request {
+	req := c.NewRequest(op, params, data)
+
+	// Run custom request initialization if present
+	if initRequest != nil {
+		initRequest(req)
+	}
+
+	return req
+}

vendor/github.com/aws/aws-sdk-go/service/sqs/sqsiface/BUILD.bazel

@@ -0,0 +1,14 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["interface.go"],
+    importmap = "k8s.io/kops/vendor/github.com/aws/aws-sdk-go/service/sqs/sqsiface",
+    importpath = "github.com/aws/aws-sdk-go/service/sqs/sqsiface",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/request:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/sqs:go_default_library",
+    ],
+)

vendor/github.com/aws/aws-sdk-go/service/sqs/sqsiface/interface.go

@@ -0,0 +1,150 @@
+// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT.
+
+// Package sqsiface provides an interface to enable mocking the Amazon Simple Queue Service service client
+// for testing your code.
+//
+// It is important to note that this interface will have breaking changes
+// when the service model is updated and adds new API operations, paginators,
+// and waiters.
+package sqsiface
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/service/sqs"
+)
+
+// SQSAPI provides an interface to enable mocking the
+// sqs.SQS service client's API operation,
+// paginators, and waiters. This make unit testing your code that calls out
+// to the SDK's service client's calls easier.
+//
+// The best way to use this interface is so the SDK's service client's calls
+// can be stubbed out for unit testing your code with the SDK without needing
+// to inject custom request handlers into the SDK's request pipeline.
+//
+//    // myFunc uses an SDK service client to make a request to
+//    // Amazon Simple Queue Service.
+//    func myFunc(svc sqsiface.SQSAPI) bool {
+//        // Make svc.AddPermission request
+//    }
+//
+//    func main() {
+//        sess := session.New()
+//        svc := sqs.New(sess)
+//
+//        myFunc(svc)
+//    }
+//
+// In your _test.go file:
+//
+//    // Define a mock struct to be used in your unit tests of myFunc.
+//    type mockSQSClient struct {
+//        sqsiface.SQSAPI
+//    }
+//    func (m *mockSQSClient) AddPermission(input *sqs.AddPermissionInput) (*sqs.AddPermissionOutput, error) {
+//        // mock response/functionality
+//    }
+//
+//    func TestMyFunc(t *testing.T) {
+//        // Setup Test
+//        mockSvc := &mockSQSClient{}
+//
+//        myfunc(mockSvc)
+//
+//        // Verify myFunc's functionality
+//    }
+//
+// It is important to note that this interface will have breaking changes
+// when the service model is updated and adds new API operations, paginators,
+// and waiters. Its suggested to use the pattern above for testing, or using
+// tooling to generate mocks to satisfy the interfaces.
+type SQSAPI interface {
+	AddPermission(*sqs.AddPermissionInput) (*sqs.AddPermissionOutput, error)
+	AddPermissionWithContext(aws.Context, *sqs.AddPermissionInput, ...request.Option) (*sqs.AddPermissionOutput, error)
+	AddPermissionRequest(*sqs.AddPermissionInput) (*request.Request, *sqs.AddPermissionOutput)
+
+	ChangeMessageVisibility(*sqs.ChangeMessageVisibilityInput) (*sqs.ChangeMessageVisibilityOutput, error)
+	ChangeMessageVisibilityWithContext(aws.Context, *sqs.ChangeMessageVisibilityInput, ...request.Option) (*sqs.ChangeMessageVisibilityOutput, error)
+	ChangeMessageVisibilityRequest(*sqs.ChangeMessageVisibilityInput) (*request.Request, *sqs.ChangeMessageVisibilityOutput)
+
+	ChangeMessageVisibilityBatch(*sqs.ChangeMessageVisibilityBatchInput) (*sqs.ChangeMessageVisibilityBatchOutput, error)
+	ChangeMessageVisibilityBatchWithContext(aws.Context, *sqs.ChangeMessageVisibilityBatchInput, ...request.Option) (*sqs.ChangeMessageVisibilityBatchOutput, error)
+	ChangeMessageVisibilityBatchRequest(*sqs.ChangeMessageVisibilityBatchInput) (*request.Request, *sqs.ChangeMessageVisibilityBatchOutput)
+
+	CreateQueue(*sqs.CreateQueueInput) (*sqs.CreateQueueOutput, error)
+	CreateQueueWithContext(aws.Context, *sqs.CreateQueueInput, ...request.Option) (*sqs.CreateQueueOutput, error)
+	CreateQueueRequest(*sqs.CreateQueueInput) (*request.Request, *sqs.CreateQueueOutput)
+
+	DeleteMessage(*sqs.DeleteMessageInput) (*sqs.DeleteMessageOutput, error)
+	DeleteMessageWithContext(aws.Context, *sqs.DeleteMessageInput, ...request.Option) (*sqs.DeleteMessageOutput, error)
+	DeleteMessageRequest(*sqs.DeleteMessageInput) (*request.Request, *sqs.DeleteMessageOutput)
+
+	DeleteMessageBatch(*sqs.DeleteMessageBatchInput) (*sqs.DeleteMessageBatchOutput, error)
+	DeleteMessageBatchWithContext(aws.Context, *sqs.DeleteMessageBatchInput, ...request.Option) (*sqs.DeleteMessageBatchOutput, error)
+	DeleteMessageBatchRequest(*sqs.DeleteMessageBatchInput) (*request.Request, *sqs.DeleteMessageBatchOutput)
+
+	DeleteQueue(*sqs.DeleteQueueInput) (*sqs.DeleteQueueOutput, error)
+	DeleteQueueWithContext(aws.Context, *sqs.DeleteQueueInput, ...request.Option) (*sqs.DeleteQueueOutput, error)
+	DeleteQueueRequest(*sqs.DeleteQueueInput) (*request.Request, *sqs.DeleteQueueOutput)
+
+	GetQueueAttributes(*sqs.GetQueueAttributesInput) (*sqs.GetQueueAttributesOutput, error)
+	GetQueueAttributesWithContext(aws.Context, *sqs.GetQueueAttributesInput, ...request.Option) (*sqs.GetQueueAttributesOutput, error)
+	GetQueueAttributesRequest(*sqs.GetQueueAttributesInput) (*request.Request, *sqs.GetQueueAttributesOutput)
+
+	GetQueueUrl(*sqs.GetQueueUrlInput) (*sqs.GetQueueUrlOutput, error)
+	GetQueueUrlWithContext(aws.Context, *sqs.GetQueueUrlInput, ...request.Option) (*sqs.GetQueueUrlOutput, error)
+	GetQueueUrlRequest(*sqs.GetQueueUrlInput) (*request.Request, *sqs.GetQueueUrlOutput)
+
+	ListDeadLetterSourceQueues(*sqs.ListDeadLetterSourceQueuesInput) (*sqs.ListDeadLetterSourceQueuesOutput, error)
+	ListDeadLetterSourceQueuesWithContext(aws.Context, *sqs.ListDeadLetterSourceQueuesInput, ...request.Option) (*sqs.ListDeadLetterSourceQueuesOutput, error)
+	ListDeadLetterSourceQueuesRequest(*sqs.ListDeadLetterSourceQueuesInput) (*request.Request, *sqs.ListDeadLetterSourceQueuesOutput)
+
+	ListDeadLetterSourceQueuesPages(*sqs.ListDeadLetterSourceQueuesInput, func(*sqs.ListDeadLetterSourceQueuesOutput, bool) bool) error
+	ListDeadLetterSourceQueuesPagesWithContext(aws.Context, *sqs.ListDeadLetterSourceQueuesInput, func(*sqs.ListDeadLetterSourceQueuesOutput, bool) bool, ...request.Option) error
+
+	ListQueueTags(*sqs.ListQueueTagsInput) (*sqs.ListQueueTagsOutput, error)
+	ListQueueTagsWithContext(aws.Context, *sqs.ListQueueTagsInput, ...request.Option) (*sqs.ListQueueTagsOutput, error)
+	ListQueueTagsRequest(*sqs.ListQueueTagsInput) (*request.Request, *sqs.ListQueueTagsOutput)
+
+	ListQueues(*sqs.ListQueuesInput) (*sqs.ListQueuesOutput, error)
+	ListQueuesWithContext(aws.Context, *sqs.ListQueuesInput, ...request.Option) (*sqs.ListQueuesOutput, error)
+	ListQueuesRequest(*sqs.ListQueuesInput) (*request.Request, *sqs.ListQueuesOutput)
+
+	ListQueuesPages(*sqs.ListQueuesInput, func(*sqs.ListQueuesOutput, bool) bool) error
+	ListQueuesPagesWithContext(aws.Context, *sqs.ListQueuesInput, func(*sqs.ListQueuesOutput, bool) bool, ...request.Option) error
+
+	PurgeQueue(*sqs.PurgeQueueInput) (*sqs.PurgeQueueOutput, error)
+	PurgeQueueWithContext(aws.Context, *sqs.PurgeQueueInput, ...request.Option) (*sqs.PurgeQueueOutput, error)
+	PurgeQueueRequest(*sqs.PurgeQueueInput) (*request.Request, *sqs.PurgeQueueOutput)
+
+	ReceiveMessage(*sqs.ReceiveMessageInput) (*sqs.ReceiveMessageOutput, error)
+	ReceiveMessageWithContext(aws.Context, *sqs.ReceiveMessageInput, ...request.Option) (*sqs.ReceiveMessageOutput, error)
+	ReceiveMessageRequest(*sqs.ReceiveMessageInput) (*request.Request, *sqs.ReceiveMessageOutput)
+
+	RemovePermission(*sqs.RemovePermissionInput) (*sqs.RemovePermissionOutput, error)
+	RemovePermissionWithContext(aws.Context, *sqs.RemovePermissionInput, ...request.Option) (*sqs.RemovePermissionOutput, error)
+	RemovePermissionRequest(*sqs.RemovePermissionInput) (*request.Request, *sqs.RemovePermissionOutput)
+
+	SendMessage(*sqs.SendMessageInput) (*sqs.SendMessageOutput, error)
+	SendMessageWithContext(aws.Context, *sqs.SendMessageInput, ...request.Option) (*sqs.SendMessageOutput, error)
+	SendMessageRequest(*sqs.SendMessageInput) (*request.Request, *sqs.SendMessageOutput)
+
+	SendMessageBatch(*sqs.SendMessageBatchInput) (*sqs.SendMessageBatchOutput, error)
+	SendMessageBatchWithContext(aws.Context, *sqs.SendMessageBatchInput, ...request.Option) (*sqs.SendMessageBatchOutput, error)
+	SendMessageBatchRequest(*sqs.SendMessageBatchInput) (*request.Request, *sqs.SendMessageBatchOutput)
+
+	SetQueueAttributes(*sqs.SetQueueAttributesInput) (*sqs.SetQueueAttributesOutput, error)
+	SetQueueAttributesWithContext(aws.Context, *sqs.SetQueueAttributesInput, ...request.Option) (*sqs.SetQueueAttributesOutput, error)
+	SetQueueAttributesRequest(*sqs.SetQueueAttributesInput) (*request.Request, *sqs.SetQueueAttributesOutput)
+
+	TagQueue(*sqs.TagQueueInput) (*sqs.TagQueueOutput, error)
+	TagQueueWithContext(aws.Context, *sqs.TagQueueInput, ...request.Option) (*sqs.TagQueueOutput, error)
+	TagQueueRequest(*sqs.TagQueueInput) (*request.Request, *sqs.TagQueueOutput)
+
+	UntagQueue(*sqs.UntagQueueInput) (*sqs.UntagQueueOutput, error)
+	UntagQueueWithContext(aws.Context, *sqs.UntagQueueInput, ...request.Option) (*sqs.UntagQueueOutput, error)
+	UntagQueueRequest(*sqs.UntagQueueInput) (*request.Request, *sqs.UntagQueueOutput)
+}
+
+var _ SQSAPI = (*sqs.SQS)(nil)

vendor/modules.txt

@@ -166,12 +166,16 @@ github.com/aws/aws-sdk-go/service/elb
 github.com/aws/aws-sdk-go/service/elb/elbiface
 github.com/aws/aws-sdk-go/service/elbv2
 github.com/aws/aws-sdk-go/service/elbv2/elbv2iface
+github.com/aws/aws-sdk-go/service/eventbridge
+github.com/aws/aws-sdk-go/service/eventbridge/eventbridgeiface
 github.com/aws/aws-sdk-go/service/iam
 github.com/aws/aws-sdk-go/service/iam/iamiface
 github.com/aws/aws-sdk-go/service/kms
 github.com/aws/aws-sdk-go/service/route53
 github.com/aws/aws-sdk-go/service/route53/route53iface
 github.com/aws/aws-sdk-go/service/s3
+github.com/aws/aws-sdk-go/service/sqs
+github.com/aws/aws-sdk-go/service/sqs/sqsiface
 github.com/aws/aws-sdk-go/service/sso
 github.com/aws/aws-sdk-go/service/sso/ssoiface
 github.com/aws/aws-sdk-go/service/sts