diff --git a/cloudmock/aws/mockautoscaling/api.go b/cloudmock/aws/mockautoscaling/api.go index cd7f0e540d..a0a034b824 100644 --- a/cloudmock/aws/mockautoscaling/api.go +++ b/cloudmock/aws/mockautoscaling/api.go @@ -31,6 +31,7 @@ type MockAutoscaling struct { Groups map[string]*autoscaling.Group WarmPoolInstances map[string][]*autoscaling.Instance LaunchConfigurations map[string]*autoscaling.LaunchConfiguration + LifecycleHooks map[string]*autoscaling.LifecycleHook } var _ autoscalingiface.AutoScalingAPI = &MockAutoscaling{} diff --git a/cloudmock/aws/mockautoscaling/group.go b/cloudmock/aws/mockautoscaling/group.go index cf509bd5be..c02288dbba 100644 --- a/cloudmock/aws/mockautoscaling/group.go +++ b/cloudmock/aws/mockautoscaling/group.go @@ -338,5 +338,39 @@ func (m *MockAutoscaling) DeleteAutoScalingGroupRequest(*autoscaling.DeleteAutoS } func (m *MockAutoscaling) PutLifecycleHook(input *autoscaling.PutLifecycleHookInput) (*autoscaling.PutLifecycleHookOutput, error) { - panic("Not implemented") + m.mutex.Lock() + defer m.mutex.Unlock() + hook := &autoscaling.LifecycleHook{ + AutoScalingGroupName: input.AutoScalingGroupName, + DefaultResult: input.DefaultResult, + GlobalTimeout: input.HeartbeatTimeout, + HeartbeatTimeout: input.HeartbeatTimeout, + LifecycleHookName: input.LifecycleHookName, + LifecycleTransition: input.LifecycleTransition, + NotificationMetadata: input.NotificationMetadata, + NotificationTargetARN: input.NotificationTargetARN, + RoleARN: input.RoleARN, + } + + if m.LifecycleHooks == nil { + m.LifecycleHooks = make(map[string]*autoscaling.LifecycleHook) + } + m.LifecycleHooks[*hook.AutoScalingGroupName] = hook + + return &autoscaling.PutLifecycleHookOutput{}, nil +} + +func (m *MockAutoscaling) DescribeLifecycleHooks(input *autoscaling.DescribeLifecycleHooksInput) (*autoscaling.DescribeLifecycleHooksOutput, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + + name := *input.AutoScalingGroupName + response := &autoscaling.DescribeLifecycleHooksOutput{} + + hook := m.LifecycleHooks[name] + if hook == nil { + return response, nil + } + response.LifecycleHooks = []*autoscaling.LifecycleHook{hook} + return response, nil } diff --git a/cloudmock/aws/mockeventbridge/api.go b/cloudmock/aws/mockeventbridge/api.go index 338f4f1952..ebd354e4e7 100644 --- a/cloudmock/aws/mockeventbridge/api.go +++ b/cloudmock/aws/mockeventbridge/api.go @@ -17,42 +17,99 @@ limitations under the License. package mockeventbridge import ( + "sync" + "github.com/aws/aws-sdk-go/service/eventbridge" "github.com/aws/aws-sdk-go/service/eventbridge/eventbridgeiface" ) type MockEventBridge struct { eventbridgeiface.EventBridgeAPI + mutex sync.Mutex - Rules []*eventbridge.Rule + Rules map[string]*eventbridge.Rule + TagsByArn map[string][]*eventbridge.Tag + TargetsByRule map[string][]*eventbridge.Target } var _ eventbridgeiface.EventBridgeAPI = &MockEventBridge{} -func (c *MockEventBridge) ListTargetsByRule(*eventbridge.ListTargetsByRuleInput) (*eventbridge.ListTargetsByRuleOutput, error) { - panic("Not implemented") -} +func (m *MockEventBridge) PutRule(input *eventbridge.PutRuleInput) (*eventbridge.PutRuleOutput, error) { + m.mutex.Lock() + defer m.mutex.Unlock() -func (c *MockEventBridge) RemoveTargets(*eventbridge.RemoveTargetsInput) (*eventbridge.RemoveTargetsOutput, error) { - panic("Not implemented") -} + name := *input.Name + arn := "arn:aws:events:us-east-1:012345678901:rule/" + name -func (c *MockEventBridge) DeleteRule(*eventbridge.DeleteRuleInput) (*eventbridge.DeleteRuleOutput, error) { - panic("Not implemented") -} - -func (c *MockEventBridge) ListRules(*eventbridge.ListRulesInput) (*eventbridge.ListRulesOutput, error) { - response := &eventbridge.ListRulesOutput{ - Rules: c.Rules, + rule := &eventbridge.Rule{ + Arn: &arn, + EventPattern: input.EventPattern, } + if m.Rules == nil { + m.Rules = make(map[string]*eventbridge.Rule) + } + if m.TagsByArn == nil { + m.TagsByArn = make(map[string][]*eventbridge.Tag) + } + m.Rules[name] = rule + m.TagsByArn[arn] = input.Tags + response := &eventbridge.PutRuleOutput{ + RuleArn: &arn, + } return response, nil } -func (c *MockEventBridge) PutRule(*eventbridge.PutRuleInput) (*eventbridge.PutRuleOutput, error) { +func (m *MockEventBridge) ListRules(input *eventbridge.ListRulesInput) (*eventbridge.ListRulesOutput, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + + response := &eventbridge.ListRulesOutput{} + + rule := m.Rules[*input.NamePrefix] + if rule == nil { + return response, nil + } + response.Rules = []*eventbridge.Rule{rule} + return response, nil +} + +func (m *MockEventBridge) DeleteRule(*eventbridge.DeleteRuleInput) (*eventbridge.DeleteRuleOutput, error) { panic("Not implemented") } -func (c *MockEventBridge) PutTargets(*eventbridge.PutTargetsInput) (*eventbridge.PutTargetsOutput, error) { +func (m *MockEventBridge) ListTagsForResource(input *eventbridge.ListTagsForResourceInput) (*eventbridge.ListTagsForResourceOutput, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + + response := &eventbridge.ListTagsForResourceOutput{ + Tags: m.TagsByArn[*input.ResourceARN], + } + return response, nil +} + +func (m *MockEventBridge) PutTargets(input *eventbridge.PutTargetsInput) (*eventbridge.PutTargetsOutput, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + + if m.TargetsByRule == nil { + m.TargetsByRule = make(map[string][]*eventbridge.Target) + } + m.TargetsByRule[*input.Rule] = input.Targets + + return &eventbridge.PutTargetsOutput{}, nil +} + +func (m *MockEventBridge) ListTargetsByRule(input *eventbridge.ListTargetsByRuleInput) (*eventbridge.ListTargetsByRuleOutput, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + + response := &eventbridge.ListTargetsByRuleOutput{ + Targets: m.TargetsByRule[*input.Rule], + } + return response, nil +} + +func (m *MockEventBridge) RemoveTargets(*eventbridge.RemoveTargetsInput) (*eventbridge.RemoveTargetsOutput, error) { panic("Not implemented") } diff --git a/cloudmock/aws/mocksqs/api.go b/cloudmock/aws/mocksqs/api.go index b933106ea0..906cc6f482 100644 --- a/cloudmock/aws/mocksqs/api.go +++ b/cloudmock/aws/mocksqs/api.go @@ -17,34 +17,93 @@ limitations under the License. package mocksqs import ( + "sync" + "github.com/aws/aws-sdk-go/service/sqs" "github.com/aws/aws-sdk-go/service/sqs/sqsiface" ) type MockSQS struct { sqsiface.SQSAPI + mutex sync.Mutex - QueueUrls []*string + Queues map[string]mockQueue +} + +type mockQueue struct { + url *string + attributes map[string]*string + tags map[string]*string } var _ sqsiface.SQSAPI = &MockSQS{} -func (c *MockSQS) DeleteQueue(*sqs.DeleteQueueInput) (*sqs.DeleteQueueOutput, error) { - panic("Not implemented") -} +func (m *MockSQS) CreateQueue(input *sqs.CreateQueueInput) (*sqs.CreateQueueOutput, error) { + m.mutex.Lock() + defer m.mutex.Unlock() -func (c *MockSQS) ListQueues(*sqs.ListQueuesInput) (*sqs.ListQueuesOutput, error) { - response := &sqs.ListQueuesOutput{ - QueueUrls: c.QueueUrls, + name := *input.QueueName + url := "https://sqs.us-east-1.amazonaws.com/123456789123/" + name + + if m.Queues == nil { + m.Queues = make(map[string]mockQueue) + } + queue := mockQueue{ + url: &url, + attributes: input.Attributes, + tags: input.Tags, } + m.Queues[name] = queue + + response := &sqs.CreateQueueOutput{ + QueueUrl: &url, + } return response, nil } -func (c *MockSQS) ListQueueTags(*sqs.ListQueueTagsInput) (*sqs.ListQueueTagsOutput, error) { - panic("Not implemented") +func (m *MockSQS) ListQueues(input *sqs.ListQueuesInput) (*sqs.ListQueuesOutput, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + + response := &sqs.ListQueuesOutput{} + + if queue, ok := m.Queues[*input.QueueNamePrefix]; ok { + response.QueueUrls = []*string{queue.url} + } + return response, nil } -func (c *MockSQS) CreateQueue(*sqs.CreateQueueInput) (*sqs.CreateQueueOutput, error) { +func (m *MockSQS) GetQueueAttributes(input *sqs.GetQueueAttributesInput) (*sqs.GetQueueAttributesOutput, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + + response := &sqs.GetQueueAttributesOutput{} + + for _, v := range m.Queues { + if *v.url == *input.QueueUrl { + response.Attributes = v.attributes + return response, nil + } + } + return response, nil +} + +func (m *MockSQS) ListQueueTags(input *sqs.ListQueueTagsInput) (*sqs.ListQueueTagsOutput, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + + response := &sqs.ListQueueTagsOutput{} + + for _, v := range m.Queues { + if *v.url == *input.QueueUrl { + response.Tags = v.tags + return response, nil + } + } + return response, nil +} + +func (m *MockSQS) DeleteQueue(*sqs.DeleteQueueInput) (*sqs.DeleteQueueOutput, error) { panic("Not implemented") } diff --git a/cmd/kops/BUILD.bazel b/cmd/kops/BUILD.bazel index 5aec4bb285..e5730205a0 100644 --- a/cmd/kops/BUILD.bazel +++ b/cmd/kops/BUILD.bazel @@ -170,7 +170,6 @@ go_test( "//pkg/featureflag:go_default_library", "//pkg/jsonutils:go_default_library", "//pkg/kopscodecs:go_default_library", - "//pkg/model:go_default_library", "//pkg/testutils:go_default_library", "//pkg/testutils/golden:go_default_library", "//upup/pkg/fi:go_default_library", diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index b8561eadf8..b9a5df3295 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -36,8 +36,6 @@ import ( "testing" "time" - "k8s.io/kops/pkg/model" - "k8s.io/kops/cmd/kops/util" "k8s.io/kops/pkg/featureflag" "k8s.io/kops/pkg/jsonutils" @@ -396,9 +394,10 @@ func TestAPIServerNodes(t *testing.T) { newIntegrationTest("minimal.example.com", "apiservernodes").runTestCloudformation(t) } +// TestNTHQueueProcessor tests the output for resources required by NTH Queue Processor mode func TestNTHQueueProcessor(t *testing.T) { - newIntegrationTest("queueprocessor.example.com", "nodeterminationhandler_sqs_resources").withNTH().runTestTerraformAWS(t) - newIntegrationTest("queueprocessor.example.com", "nodeterminationhandler_sqs_resources").runTestCloudformation(t) + newIntegrationTest("nthsqsresources.example.com", "nth_sqs_resources").withNTH().runTestTerraformAWS(t) + newIntegrationTest("nthsqsresources.example.com", "nth_sqs_resources").runTestCloudformation(t) } func (i *integrationTest) runTest(t *testing.T, h *testutils.IntegrationTestHarness, expectedDataFilenames []string, tfFileName string, expectedTfFileName string, phase *cloudup.Phase) { @@ -598,7 +597,7 @@ func (i *integrationTest) runTestTerraformAWS(t *testing.T) { "aws_cloudwatch_event_rule_" + i.clusterName + "-ASGLifecycle_event_pattern", "aws_cloudwatch_event_rule_" + i.clusterName + "-RebalanceRecommendation_event_pattern", "aws_cloudwatch_event_rule_" + i.clusterName + "-SpotInterruption_event_pattern", - "aws_sqs_queue_" + model.QueueNamePrefix(i.clusterName) + "-nth_policy", + "aws_sqs_queue_" + strings.Replace(i.clusterName, ".", "-", -1) + "-nth_policy", }...) } } diff --git a/cmd/kops/lifecycle_integration_test.go b/cmd/kops/lifecycle_integration_test.go index 0e8632c068..fdbc112a74 100644 --- a/cmd/kops/lifecycle_integration_test.go +++ b/cmd/kops/lifecycle_integration_test.go @@ -153,6 +153,14 @@ func TestLifecyclePrivateSharedIP(t *testing.T) { }) } +// TestLifecycleNodeTerminationHandlerQueueProcessor runs the test on a cluster with requisite resources for NTH Queue Processor +func TestLifecycleNodeTerminationHandlerQueueProcessor(t *testing.T) { + runLifecycleTestAWS(&LifecycleTestOptions{ + t: t, + SrcDir: "nth_sqs_resources", + }) +} + func runLifecycleTest(h *testutils.IntegrationTestHarness, o *LifecycleTestOptions, cloud *awsup.MockAWSCloud) { ctx := context.Background() diff --git a/docs/addons.md b/docs/addons.md index fd46e5f12b..2ee678bc35 100644 --- a/docs/addons.md +++ b/docs/addons.md @@ -127,12 +127,15 @@ spec: {{ kops_feature_table(kops_added_default='1.19') }} -Node Termination Handler ensures that the Kubernetes control plane responds appropriately to events that can cause your EC2 instance to become unavailable, such as EC2 maintenance events, EC2 Spot interruptions, ASG Scale-In, ASG AZ Rebalance, and EC2 Instance Termination via the API or Console. If not handled, your application code may not stop gracefully, take longer to recover full availability, or accidentally schedule work to nodes that are going down. +[Node Termination Handler](https://github.com/aws/aws-node-termination-handler) ensures that the Kubernetes control plane responds appropriately to events that can cause your EC2 instance to become unavailable, such as EC2 maintenance events, EC2 Spot interruptions, and EC2 instance rebalance recommendations. If not handled, your application code may not stop gracefully, take longer to recover full availability, or accidentally schedule work to nodes that are going down. + +If `enableSqsTerminationDraining` is enabled Node Termination Handler will operate in Queue Processor mode. In addition to the events mentioned above, Queue Processor mode allows Node Termination Handler to take care of ASG Scale-In, AZ-Rebalance, Unhealthy Instances, EC2 Instance Termination via the API or Console, and more. kOps will provision the necessary infrastructure: an SQS queue, EventBridge rules, and ASG Lifecycle hooks. ```yaml spec: nodeTerminationHandler: enabled: true + enableSqsTerminationDraining: true ``` ## Static addons diff --git a/pkg/model/awsmodel/nodeterminationhandler.go b/pkg/model/awsmodel/nodeterminationhandler.go index 0691c8cdd5..9b70a75080 100644 --- a/pkg/model/awsmodel/nodeterminationhandler.go +++ b/pkg/model/awsmodel/nodeterminationhandler.go @@ -111,7 +111,6 @@ func (b *NodeTerminationHandlerBuilder) configureASG(c *fi.ModelBuilderContext, DefaultResult: aws.String("CONTINUE"), HeartbeatTimeout: aws.Int64(DefaultMessageRetentionPeriod), LifecycleTransition: aws.String("autoscaling:EC2_INSTANCE_TERMINATING"), - Tags: tags, } c.AddTask(lifecyleTask) diff --git a/pkg/model/names.go b/pkg/model/names.go index 51bbc0fda6..4ce1bed9d8 100644 --- a/pkg/model/names.go +++ b/pkg/model/names.go @@ -268,5 +268,5 @@ func (b *KopsModelContext) InstanceName(ig *kops.InstanceGroup, suffix string) s func QueueNamePrefix(clusterName string) string { // periods aren't allowed in queue name - return strings.Replace(clusterName, ".", "-", -1) + return strings.ReplaceAll(clusterName, ".", "-") } diff --git a/pkg/resources/aws/BUILD.bazel b/pkg/resources/aws/BUILD.bazel index f7120921d1..39ed4d8bc0 100644 --- a/pkg/resources/aws/BUILD.bazel +++ b/pkg/resources/aws/BUILD.bazel @@ -21,7 +21,6 @@ go_library( deps = [ "//pkg/dns:go_default_library", "//pkg/featureflag:go_default_library", - "//pkg/model:go_default_library", "//pkg/resources:go_default_library", "//pkg/resources/spotinst:go_default_library", "//upup/pkg/fi:go_default_library", diff --git a/pkg/resources/aws/eventbridge.go b/pkg/resources/aws/eventbridge.go index 3fedf668fc..aa3c2e2e90 100644 --- a/pkg/resources/aws/eventbridge.go +++ b/pkg/resources/aws/eventbridge.go @@ -46,7 +46,7 @@ func DeleteEventBridgeRule(cloud fi.Cloud, r *resources.Resource) error { Rule: aws.String(r.Name), }) if err != nil { - return fmt.Errorf("error listing targets for EventBridge Rule %q: %v", r.Name, err) + return fmt.Errorf("error listing targets for EventBridge rule %q: %v", r.Name, err) } var ids []*string @@ -54,22 +54,22 @@ func DeleteEventBridgeRule(cloud fi.Cloud, r *resources.Resource) error { ids = append(ids, target.Id) } - klog.V(2).Infof("Removing EventBridge Targets for Rule %q", r.Name) + klog.V(2).Infof("Removing EventBridge Targets for rule %q", r.Name) _, err = c.EventBridge().RemoveTargets(&eventbridge.RemoveTargetsInput{ Ids: ids, Rule: aws.String(r.Name), }) if err != nil { - return fmt.Errorf("error removing targets for EventBridge Rule %q: %v", r.Name, err) + return fmt.Errorf("error removing targets for EventBridge rule %q: %v", r.Name, err) } - klog.V(2).Infof("Deleting EventBridge Rule %q", r.Name) + klog.V(2).Infof("Deleting EventBridge rule %q", r.Name) request := &eventbridge.DeleteRuleInput{ Name: aws.String(r.Name), } _, err = c.EventBridge().DeleteRule(request) if err != nil { - return fmt.Errorf("error deleting EventBridge Rule %q: %v", r.Name, err) + return fmt.Errorf("error deleting EventBridge rule %q: %v", r.Name, err) } return nil } @@ -77,9 +77,9 @@ func DeleteEventBridgeRule(cloud fi.Cloud, r *resources.Resource) error { func ListEventBridgeRules(cloud fi.Cloud, clusterName string) ([]*resources.Resource, error) { c := cloud.(awsup.AWSCloud) - klog.V(2).Infof("Listing EventBridge Rules") + klog.V(2).Infof("Listing EventBridge rules") - // Rule names start with the cluster name so that we can search for them + // rule names start with the cluster name so that we can search for them request := &eventbridge.ListRulesInput{ EventBusName: nil, Limit: nil, @@ -87,7 +87,7 @@ func ListEventBridgeRules(cloud fi.Cloud, clusterName string) ([]*resources.Reso } response, err := c.EventBridge().ListRules(request) if err != nil { - return nil, fmt.Errorf("error listing SQS queues: %v", err) + return nil, fmt.Errorf("error listing Eventbridge rules: %v", err) } if response == nil || len(response.Rules) == 0 { return nil, nil diff --git a/pkg/resources/aws/sqs.go b/pkg/resources/aws/sqs.go index 4a6a784963..9578db6a8b 100644 --- a/pkg/resources/aws/sqs.go +++ b/pkg/resources/aws/sqs.go @@ -18,11 +18,10 @@ package aws import ( "fmt" + "strings" "github.com/aws/aws-sdk-go/service/sqs" "k8s.io/klog/v2" - "k8s.io/kops/pkg/model" - "k8s.io/kops/pkg/resources" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" @@ -59,7 +58,7 @@ func ListSQSQueues(cloud fi.Cloud, clusterName string) ([]*resources.Resource, e c := cloud.(awsup.AWSCloud) klog.V(2).Infof("Listing SQS queues") - queuePrefix := model.QueueNamePrefix(clusterName) + queuePrefix := strings.ReplaceAll(clusterName, ".", "-") request := &sqs.ListQueuesInput{ QueueNamePrefix: &queuePrefix, diff --git a/tests/integration/update_cluster/nodeterminationhandler_sqs_resources/kubernetes.tf b/tests/integration/update_cluster/nodeterminationhandler_sqs_resources/kubernetes.tf deleted file mode 100644 index 18dbf04a90..0000000000 --- a/tests/integration/update_cluster/nodeterminationhandler_sqs_resources/kubernetes.tf +++ /dev/null @@ -1,732 +0,0 @@ -locals { - cluster_name = "queueprocessor.example.com" - master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-queueprocessor-example-com.id] - master_security_group_ids = [aws_security_group.masters-queueprocessor-example-com.id] - masters_role_arn = aws_iam_role.masters-queueprocessor-example-com.arn - masters_role_name = aws_iam_role.masters-queueprocessor-example-com.name - node_autoscaling_group_ids = [aws_autoscaling_group.nodes-queueprocessor-example-com.id] - node_security_group_ids = [aws_security_group.nodes-queueprocessor-example-com.id] - node_subnet_ids = [aws_subnet.us-test-1a-queueprocessor-example-com.id] - nodes_role_arn = aws_iam_role.nodes-queueprocessor-example-com.arn - nodes_role_name = aws_iam_role.nodes-queueprocessor-example-com.name - region = "us-test-1" - route_table_public_id = aws_route_table.queueprocessor-example-com.id - subnet_us-test-1a_id = aws_subnet.us-test-1a-queueprocessor-example-com.id - vpc_cidr_block = aws_vpc.queueprocessor-example-com.cidr_block - vpc_id = aws_vpc.queueprocessor-example-com.id -} - -output "cluster_name" { - value = "queueprocessor.example.com" -} - -output "master_autoscaling_group_ids" { - value = [aws_autoscaling_group.master-us-test-1a-masters-queueprocessor-example-com.id] -} - -output "master_security_group_ids" { - value = [aws_security_group.masters-queueprocessor-example-com.id] -} - -output "masters_role_arn" { - value = aws_iam_role.masters-queueprocessor-example-com.arn -} - -output "masters_role_name" { - value = aws_iam_role.masters-queueprocessor-example-com.name -} - -output "node_autoscaling_group_ids" { - value = [aws_autoscaling_group.nodes-queueprocessor-example-com.id] -} - -output "node_security_group_ids" { - value = [aws_security_group.nodes-queueprocessor-example-com.id] -} - -output "node_subnet_ids" { - value = [aws_subnet.us-test-1a-queueprocessor-example-com.id] -} - -output "nodes_role_arn" { - value = aws_iam_role.nodes-queueprocessor-example-com.arn -} - -output "nodes_role_name" { - value = aws_iam_role.nodes-queueprocessor-example-com.name -} - -output "region" { - value = "us-test-1" -} - -output "route_table_public_id" { - value = aws_route_table.queueprocessor-example-com.id -} - -output "subnet_us-test-1a_id" { - value = aws_subnet.us-test-1a-queueprocessor-example-com.id -} - -output "vpc_cidr_block" { - value = aws_vpc.queueprocessor-example-com.cidr_block -} - -output "vpc_id" { - value = aws_vpc.queueprocessor-example-com.id -} - -provider "aws" { - region = "us-test-1" -} - -resource "aws_autoscaling_group" "master-us-test-1a-masters-queueprocessor-example-com" { - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] - launch_template { - id = aws_launch_template.master-us-test-1a-masters-queueprocessor-example-com.id - version = aws_launch_template.master-us-test-1a-masters-queueprocessor-example-com.latest_version - } - max_size = 1 - metrics_granularity = "1Minute" - min_size = 1 - name = "master-us-test-1a.masters.queueprocessor.example.com" - tag { - key = "KubernetesCluster" - propagate_at_launch = true - value = "queueprocessor.example.com" - } - tag { - key = "Name" - propagate_at_launch = true - value = "master-us-test-1a.masters.queueprocessor.example.com" - } - tag { - key = "aws-node-termination-handler/managed" - propagate_at_launch = true - value = "true" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" - propagate_at_launch = true - value = "" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" - propagate_at_launch = true - value = "master" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" - propagate_at_launch = true - value = "" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" - propagate_at_launch = true - value = "" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" - propagate_at_launch = true - value = "" - } - tag { - key = "k8s.io/role/master" - propagate_at_launch = true - value = "1" - } - tag { - key = "kops.k8s.io/instancegroup" - propagate_at_launch = true - value = "master-us-test-1a" - } - tag { - key = "kubernetes.io/cluster/queueprocessor.example.com" - propagate_at_launch = true - value = "owned" - } - vpc_zone_identifier = [aws_subnet.us-test-1a-queueprocessor-example-com.id] -} - -resource "aws_autoscaling_group" "nodes-queueprocessor-example-com" { - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] - launch_template { - id = aws_launch_template.nodes-queueprocessor-example-com.id - version = aws_launch_template.nodes-queueprocessor-example-com.latest_version - } - max_size = 2 - metrics_granularity = "1Minute" - min_size = 2 - name = "nodes.queueprocessor.example.com" - tag { - key = "KubernetesCluster" - propagate_at_launch = true - value = "queueprocessor.example.com" - } - tag { - key = "Name" - propagate_at_launch = true - value = "nodes.queueprocessor.example.com" - } - tag { - key = "aws-node-termination-handler/managed" - propagate_at_launch = true - value = "true" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" - propagate_at_launch = true - value = "node" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" - propagate_at_launch = true - value = "" - } - tag { - key = "k8s.io/role/node" - propagate_at_launch = true - value = "1" - } - tag { - key = "kops.k8s.io/instancegroup" - propagate_at_launch = true - value = "nodes" - } - tag { - key = "kubernetes.io/cluster/queueprocessor.example.com" - propagate_at_launch = true - value = "owned" - } - vpc_zone_identifier = [aws_subnet.us-test-1a-queueprocessor-example-com.id] -} - -resource "aws_autoscaling_lifecycle_hook" "master-us-test-1a-NTHLifecycleHook" { - autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-queueprocessor-example-com.id - default_result = "CONTINUE" - heartbeat_timeout = 300 - lifecycle_transition = "autoscaling:EC2_INSTANCE_TERMINATING" - name = "master-us-test-1a-NTHLifecycleHook" -} - -resource "aws_autoscaling_lifecycle_hook" "nodes-NTHLifecycleHook" { - autoscaling_group_name = aws_autoscaling_group.nodes-queueprocessor-example-com.id - default_result = "CONTINUE" - heartbeat_timeout = 300 - lifecycle_transition = "autoscaling:EC2_INSTANCE_TERMINATING" - name = "nodes-NTHLifecycleHook" -} - -resource "aws_cloudwatch_event_rule" "queueprocessor-example-com-ASGLifecycle" { - event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_queueprocessor.example.com-ASGLifecycle_event_pattern") - name = "queueprocessor.example.com-ASGLifecycle" - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "queueprocessor.example.com-ASGLifecycle" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } -} - -resource "aws_cloudwatch_event_rule" "queueprocessor-example-com-RebalanceRecommendation" { - event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_queueprocessor.example.com-RebalanceRecommendation_event_pattern") - name = "queueprocessor.example.com-RebalanceRecommendation" - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "queueprocessor.example.com-RebalanceRecommendation" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } -} - -resource "aws_cloudwatch_event_rule" "queueprocessor-example-com-SpotInterruption" { - event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_queueprocessor.example.com-SpotInterruption_event_pattern") - name = "queueprocessor.example.com-SpotInterruption" - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "queueprocessor.example.com-SpotInterruption" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } -} - -resource "aws_cloudwatch_event_target" "queueprocessor-example-com-ASGLifecycle-Target" { - arn = "arn:aws:sqs:us-test-1:123456789012:queueprocessor-example-com-nth" - rule = aws_cloudwatch_event_rule.queueprocessor-example-com-ASGLifecycle.id -} - -resource "aws_cloudwatch_event_target" "queueprocessor-example-com-RebalanceRecommendation-Target" { - arn = "arn:aws:sqs:us-test-1:123456789012:queueprocessor-example-com-nth" - rule = aws_cloudwatch_event_rule.queueprocessor-example-com-RebalanceRecommendation.id -} - -resource "aws_cloudwatch_event_target" "queueprocessor-example-com-SpotInterruption-Target" { - arn = "arn:aws:sqs:us-test-1:123456789012:queueprocessor-example-com-nth" - rule = aws_cloudwatch_event_rule.queueprocessor-example-com-SpotInterruption.id -} - -resource "aws_ebs_volume" "us-test-1a-etcd-events-queueprocessor-example-com" { - availability_zone = "us-test-1a" - encrypted = false - iops = 3000 - size = 20 - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "us-test-1a.etcd-events.queueprocessor.example.com" - "k8s.io/etcd/events" = "us-test-1a/us-test-1a" - "k8s.io/role/master" = "1" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } - throughput = 125 - type = "gp3" -} - -resource "aws_ebs_volume" "us-test-1a-etcd-main-queueprocessor-example-com" { - availability_zone = "us-test-1a" - encrypted = false - iops = 3000 - size = 20 - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "us-test-1a.etcd-main.queueprocessor.example.com" - "k8s.io/etcd/main" = "us-test-1a/us-test-1a" - "k8s.io/role/master" = "1" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } - throughput = 125 - type = "gp3" -} - -resource "aws_iam_instance_profile" "masters-queueprocessor-example-com" { - name = "masters.queueprocessor.example.com" - role = aws_iam_role.masters-queueprocessor-example-com.name - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "masters.queueprocessor.example.com" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } -} - -resource "aws_iam_instance_profile" "nodes-queueprocessor-example-com" { - name = "nodes.queueprocessor.example.com" - role = aws_iam_role.nodes-queueprocessor-example-com.name - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "nodes.queueprocessor.example.com" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } -} - -resource "aws_iam_role_policy" "masters-queueprocessor-example-com" { - name = "masters.queueprocessor.example.com" - policy = file("${path.module}/data/aws_iam_role_policy_masters.queueprocessor.example.com_policy") - role = aws_iam_role.masters-queueprocessor-example-com.name -} - -resource "aws_iam_role_policy" "nodes-queueprocessor-example-com" { - name = "nodes.queueprocessor.example.com" - policy = file("${path.module}/data/aws_iam_role_policy_nodes.queueprocessor.example.com_policy") - role = aws_iam_role.nodes-queueprocessor-example-com.name -} - -resource "aws_iam_role" "masters-queueprocessor-example-com" { - assume_role_policy = file("${path.module}/data/aws_iam_role_masters.queueprocessor.example.com_policy") - name = "masters.queueprocessor.example.com" - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "masters.queueprocessor.example.com" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } -} - -resource "aws_iam_role" "nodes-queueprocessor-example-com" { - assume_role_policy = file("${path.module}/data/aws_iam_role_nodes.queueprocessor.example.com_policy") - name = "nodes.queueprocessor.example.com" - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "nodes.queueprocessor.example.com" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } -} - -resource "aws_internet_gateway" "queueprocessor-example-com" { - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "queueprocessor.example.com" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } - vpc_id = aws_vpc.queueprocessor-example-com.id -} - -resource "aws_key_pair" "kubernetes-queueprocessor-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" { - key_name = "kubernetes.queueprocessor.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57" - public_key = file("${path.module}/data/aws_key_pair_kubernetes.queueprocessor.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key") - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "queueprocessor.example.com" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } -} - -resource "aws_launch_template" "master-us-test-1a-masters-queueprocessor-example-com" { - block_device_mappings { - device_name = "/dev/xvda" - ebs { - delete_on_termination = true - encrypted = true - iops = 3000 - throughput = 125 - volume_size = 64 - volume_type = "gp3" - } - } - block_device_mappings { - device_name = "/dev/sdc" - virtual_name = "ephemeral0" - } - iam_instance_profile { - name = aws_iam_instance_profile.masters-queueprocessor-example-com.id - } - image_id = "ami-12345678" - instance_type = "m3.medium" - key_name = aws_key_pair.kubernetes-queueprocessor-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id - lifecycle { - create_before_destroy = true - } - metadata_options { - http_endpoint = "enabled" - http_put_response_hop_limit = 1 - http_tokens = "optional" - } - name = "master-us-test-1a.masters.queueprocessor.example.com" - network_interfaces { - associate_public_ip_address = true - delete_on_termination = true - security_groups = [aws_security_group.masters-queueprocessor-example-com.id] - } - tag_specifications { - resource_type = "instance" - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "master-us-test-1a.masters.queueprocessor.example.com" - "aws-node-termination-handler/managed" = "true" - "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "master" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" = "" - "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" - "k8s.io/role/master" = "1" - "kops.k8s.io/instancegroup" = "master-us-test-1a" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } - } - tag_specifications { - resource_type = "volume" - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "master-us-test-1a.masters.queueprocessor.example.com" - "aws-node-termination-handler/managed" = "true" - "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "master" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" = "" - "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" - "k8s.io/role/master" = "1" - "kops.k8s.io/instancegroup" = "master-us-test-1a" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } - } - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "master-us-test-1a.masters.queueprocessor.example.com" - "aws-node-termination-handler/managed" = "true" - "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "master" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" = "" - "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" - "k8s.io/role/master" = "1" - "kops.k8s.io/instancegroup" = "master-us-test-1a" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } - user_data = filebase64("${path.module}/data/aws_launch_template_master-us-test-1a.masters.queueprocessor.example.com_user_data") -} - -resource "aws_launch_template" "nodes-queueprocessor-example-com" { - block_device_mappings { - device_name = "/dev/xvda" - ebs { - delete_on_termination = true - encrypted = true - iops = 3000 - throughput = 125 - volume_size = 128 - volume_type = "gp3" - } - } - iam_instance_profile { - name = aws_iam_instance_profile.nodes-queueprocessor-example-com.id - } - image_id = "ami-12345678" - instance_type = "t2.medium" - key_name = aws_key_pair.kubernetes-queueprocessor-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id - lifecycle { - create_before_destroy = true - } - metadata_options { - http_endpoint = "enabled" - http_put_response_hop_limit = 1 - http_tokens = "optional" - } - name = "nodes.queueprocessor.example.com" - network_interfaces { - associate_public_ip_address = true - delete_on_termination = true - security_groups = [aws_security_group.nodes-queueprocessor-example-com.id] - } - tag_specifications { - resource_type = "instance" - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "nodes.queueprocessor.example.com" - "aws-node-termination-handler/managed" = "true" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "nodes" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } - } - tag_specifications { - resource_type = "volume" - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "nodes.queueprocessor.example.com" - "aws-node-termination-handler/managed" = "true" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "nodes" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } - } - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "nodes.queueprocessor.example.com" - "aws-node-termination-handler/managed" = "true" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "nodes" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } - user_data = filebase64("${path.module}/data/aws_launch_template_nodes.queueprocessor.example.com_user_data") -} - -resource "aws_route_table_association" "us-test-1a-queueprocessor-example-com" { - route_table_id = aws_route_table.queueprocessor-example-com.id - subnet_id = aws_subnet.us-test-1a-queueprocessor-example-com.id -} - -resource "aws_route_table" "queueprocessor-example-com" { - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "queueprocessor.example.com" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - "kubernetes.io/kops/role" = "public" - } - vpc_id = aws_vpc.queueprocessor-example-com.id -} - -resource "aws_route" "route-0-0-0-0--0" { - destination_cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.queueprocessor-example-com.id - route_table_id = aws_route_table.queueprocessor-example-com.id -} - -resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-masters-queueprocessor-example-com" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.masters-queueprocessor-example-com.id - to_port = 22 - type = "ingress" -} - -resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-nodes-queueprocessor-example-com" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.nodes-queueprocessor-example-com.id - to_port = 22 - type = "ingress" -} - -resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-masters-queueprocessor-example-com" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 443 - protocol = "tcp" - security_group_id = aws_security_group.masters-queueprocessor-example-com.id - to_port = 443 - type = "ingress" -} - -resource "aws_security_group_rule" "from-masters-queueprocessor-example-com-egress-all-0to0-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.masters-queueprocessor-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-masters-queueprocessor-example-com-ingress-all-0to0-masters-queueprocessor-example-com" { - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.masters-queueprocessor-example-com.id - source_security_group_id = aws_security_group.masters-queueprocessor-example-com.id - to_port = 0 - type = "ingress" -} - -resource "aws_security_group_rule" "from-masters-queueprocessor-example-com-ingress-all-0to0-nodes-queueprocessor-example-com" { - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.nodes-queueprocessor-example-com.id - source_security_group_id = aws_security_group.masters-queueprocessor-example-com.id - to_port = 0 - type = "ingress" -} - -resource "aws_security_group_rule" "from-nodes-queueprocessor-example-com-egress-all-0to0-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.nodes-queueprocessor-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "from-nodes-queueprocessor-example-com-ingress-all-0to0-nodes-queueprocessor-example-com" { - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.nodes-queueprocessor-example-com.id - source_security_group_id = aws_security_group.nodes-queueprocessor-example-com.id - to_port = 0 - type = "ingress" -} - -resource "aws_security_group_rule" "from-nodes-queueprocessor-example-com-ingress-tcp-1to2379-masters-queueprocessor-example-com" { - from_port = 1 - protocol = "tcp" - security_group_id = aws_security_group.masters-queueprocessor-example-com.id - source_security_group_id = aws_security_group.nodes-queueprocessor-example-com.id - to_port = 2379 - type = "ingress" -} - -resource "aws_security_group_rule" "from-nodes-queueprocessor-example-com-ingress-tcp-2382to4000-masters-queueprocessor-example-com" { - from_port = 2382 - protocol = "tcp" - security_group_id = aws_security_group.masters-queueprocessor-example-com.id - source_security_group_id = aws_security_group.nodes-queueprocessor-example-com.id - to_port = 4000 - type = "ingress" -} - -resource "aws_security_group_rule" "from-nodes-queueprocessor-example-com-ingress-tcp-4003to65535-masters-queueprocessor-example-com" { - from_port = 4003 - protocol = "tcp" - security_group_id = aws_security_group.masters-queueprocessor-example-com.id - source_security_group_id = aws_security_group.nodes-queueprocessor-example-com.id - to_port = 65535 - type = "ingress" -} - -resource "aws_security_group_rule" "from-nodes-queueprocessor-example-com-ingress-udp-1to65535-masters-queueprocessor-example-com" { - from_port = 1 - protocol = "udp" - security_group_id = aws_security_group.masters-queueprocessor-example-com.id - source_security_group_id = aws_security_group.nodes-queueprocessor-example-com.id - to_port = 65535 - type = "ingress" -} - -resource "aws_security_group" "masters-queueprocessor-example-com" { - description = "Security group for masters" - name = "masters.queueprocessor.example.com" - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "masters.queueprocessor.example.com" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } - vpc_id = aws_vpc.queueprocessor-example-com.id -} - -resource "aws_security_group" "nodes-queueprocessor-example-com" { - description = "Security group for nodes" - name = "nodes.queueprocessor.example.com" - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "nodes.queueprocessor.example.com" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } - vpc_id = aws_vpc.queueprocessor-example-com.id -} - -resource "aws_sqs_queue" "queueprocessor-example-com-nth" { - message_retention_seconds = 300 - name = "queueprocessor-example-com-nth" - policy = file("${path.module}/data/aws_sqs_queue_queueprocessor-example-com-nth_policy") - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "queueprocessor-example-com-nth" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } -} - -resource "aws_subnet" "us-test-1a-queueprocessor-example-com" { - availability_zone = "us-test-1a" - cidr_block = "172.20.32.0/19" - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "us-test-1a.queueprocessor.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - } - vpc_id = aws_vpc.queueprocessor-example-com.id -} - -resource "aws_vpc_dhcp_options_association" "queueprocessor-example-com" { - dhcp_options_id = aws_vpc_dhcp_options.queueprocessor-example-com.id - vpc_id = aws_vpc.queueprocessor-example-com.id -} - -resource "aws_vpc_dhcp_options" "queueprocessor-example-com" { - domain_name = "us-test-1.compute.internal" - domain_name_servers = ["AmazonProvidedDNS"] - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "queueprocessor.example.com" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } -} - -resource "aws_vpc" "queueprocessor-example-com" { - cidr_block = "172.20.0.0/16" - enable_dns_hostnames = true - enable_dns_support = true - tags = { - "KubernetesCluster" = "queueprocessor.example.com" - "Name" = "queueprocessor.example.com" - "kubernetes.io/cluster/queueprocessor.example.com" = "owned" - } -} - -terraform { - required_version = ">= 0.12.26" - required_providers { - aws = { - "source" = "hashicorp/aws" - "version" = ">= 3.34.0" - } - } -} diff --git a/tests/integration/update_cluster/nodeterminationhandler_sqs_resources/cloudformation.json b/tests/integration/update_cluster/nth_sqs_resources/cloudformation.json similarity index 73% rename from tests/integration/update_cluster/nodeterminationhandler_sqs_resources/cloudformation.json rename to tests/integration/update_cluster/nth_sqs_resources/cloudformation.json index d76bacecfe..3f38e79dc7 100644 --- a/tests/integration/update_cluster/nodeterminationhandler_sqs_resources/cloudformation.json +++ b/tests/integration/update_cluster/nth_sqs_resources/cloudformation.json @@ -1,16 +1,16 @@ { "Resources": { - "AWSAutoScalingAutoScalingGroupmasterustest1amastersqueueprocessorexamplecom": { + "AWSAutoScalingAutoScalingGroupmasterustest1amastersnthsqsresourcesexamplecom": { "Type": "AWS::AutoScaling::AutoScalingGroup", "Properties": { - "AutoScalingGroupName": "master-us-test-1a.masters.queueprocessor.example.com", + "AutoScalingGroupName": "master-us-test-1a.masters.nthsqsresources.example.com", "LaunchTemplate": { "LaunchTemplateId": { - "Ref": "AWSEC2LaunchTemplatemasterustest1amastersqueueprocessorexamplecom" + "Ref": "AWSEC2LaunchTemplatemasterustest1amastersnthsqsresourcesexamplecom" }, "Version": { "Fn::GetAtt": [ - "AWSEC2LaunchTemplatemasterustest1amastersqueueprocessorexamplecom", + "AWSEC2LaunchTemplatemasterustest1amastersnthsqsresourcesexamplecom", "LatestVersionNumber" ] } @@ -19,18 +19,18 @@ "MinSize": "1", "VPCZoneIdentifier": [ { - "Ref": "AWSEC2Subnetustest1aqueueprocessorexamplecom" + "Ref": "AWSEC2Subnetustest1anthsqsresourcesexamplecom" } ], "Tags": [ { "Key": "KubernetesCluster", - "Value": "queueprocessor.example.com", + "Value": "nthsqsresources.example.com", "PropagateAtLaunch": true }, { "Key": "Name", - "Value": "master-us-test-1a.masters.queueprocessor.example.com", + "Value": "master-us-test-1a.masters.nthsqsresources.example.com", "PropagateAtLaunch": true }, { @@ -74,7 +74,7 @@ "PropagateAtLaunch": true }, { - "Key": "kubernetes.io/cluster/queueprocessor.example.com", + "Key": "kubernetes.io/cluster/nthsqsresources.example.com", "Value": "owned", "PropagateAtLaunch": true } @@ -96,17 +96,17 @@ ] } }, - "AWSAutoScalingAutoScalingGroupnodesqueueprocessorexamplecom": { + "AWSAutoScalingAutoScalingGroupnodesnthsqsresourcesexamplecom": { "Type": "AWS::AutoScaling::AutoScalingGroup", "Properties": { - "AutoScalingGroupName": "nodes.queueprocessor.example.com", + "AutoScalingGroupName": "nodes.nthsqsresources.example.com", "LaunchTemplate": { "LaunchTemplateId": { - "Ref": "AWSEC2LaunchTemplatenodesqueueprocessorexamplecom" + "Ref": "AWSEC2LaunchTemplatenodesnthsqsresourcesexamplecom" }, "Version": { "Fn::GetAtt": [ - "AWSEC2LaunchTemplatenodesqueueprocessorexamplecom", + "AWSEC2LaunchTemplatenodesnthsqsresourcesexamplecom", "LatestVersionNumber" ] } @@ -115,18 +115,18 @@ "MinSize": "2", "VPCZoneIdentifier": [ { - "Ref": "AWSEC2Subnetustest1aqueueprocessorexamplecom" + "Ref": "AWSEC2Subnetustest1anthsqsresourcesexamplecom" } ], "Tags": [ { "Key": "KubernetesCluster", - "Value": "queueprocessor.example.com", + "Value": "nthsqsresources.example.com", "PropagateAtLaunch": true }, { "Key": "Name", - "Value": "nodes.queueprocessor.example.com", + "Value": "nodes.nthsqsresources.example.com", "PropagateAtLaunch": true }, { @@ -155,7 +155,7 @@ "PropagateAtLaunch": true }, { - "Key": "kubernetes.io/cluster/queueprocessor.example.com", + "Key": "kubernetes.io/cluster/nthsqsresources.example.com", "Value": "owned", "PropagateAtLaunch": true } @@ -182,7 +182,7 @@ "Properties": { "LifecycleHookName": "master-us-test-1a-NTHLifecycleHook", "AutoScalingGroupName": { - "Ref": "AWSAutoScalingAutoScalingGroupmasterustest1amastersqueueprocessorexamplecom" + "Ref": "AWSAutoScalingAutoScalingGroupmasterustest1amastersnthsqsresourcesexamplecom" }, "DefaultResult": "CONTINUE", "HeartbeatTimeout": 300, @@ -194,14 +194,14 @@ "Properties": { "LifecycleHookName": "nodes-NTHLifecycleHook", "AutoScalingGroupName": { - "Ref": "AWSAutoScalingAutoScalingGroupnodesqueueprocessorexamplecom" + "Ref": "AWSAutoScalingAutoScalingGroupnodesnthsqsresourcesexamplecom" }, "DefaultResult": "CONTINUE", "HeartbeatTimeout": 300, "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING" } }, - "AWSEC2DHCPOptionsqueueprocessorexamplecom": { + "AWSEC2DHCPOptionsnthsqsresourcesexamplecom": { "Type": "AWS::EC2::DHCPOptions", "Properties": { "DomainName": "us-test-1.compute.internal", @@ -211,42 +211,42 @@ "Tags": [ { "Key": "KubernetesCluster", - "Value": "queueprocessor.example.com" + "Value": "nthsqsresources.example.com" }, { "Key": "Name", - "Value": "queueprocessor.example.com" + "Value": "nthsqsresources.example.com" }, { - "Key": "kubernetes.io/cluster/queueprocessor.example.com", + "Key": "kubernetes.io/cluster/nthsqsresources.example.com", "Value": "owned" } ] } }, - "AWSEC2InternetGatewayqueueprocessorexamplecom": { + "AWSEC2InternetGatewaynthsqsresourcesexamplecom": { "Type": "AWS::EC2::InternetGateway", "Properties": { "Tags": [ { "Key": "KubernetesCluster", - "Value": "queueprocessor.example.com" + "Value": "nthsqsresources.example.com" }, { "Key": "Name", - "Value": "queueprocessor.example.com" + "Value": "nthsqsresources.example.com" }, { - "Key": "kubernetes.io/cluster/queueprocessor.example.com", + "Key": "kubernetes.io/cluster/nthsqsresources.example.com", "Value": "owned" } ] } }, - "AWSEC2LaunchTemplatemasterustest1amastersqueueprocessorexamplecom": { + "AWSEC2LaunchTemplatemasterustest1amastersnthsqsresourcesexamplecom": { "Type": "AWS::EC2::LaunchTemplate", "Properties": { - "LaunchTemplateName": "master-us-test-1a.masters.queueprocessor.example.com", + "LaunchTemplateName": "master-us-test-1a.masters.nthsqsresources.example.com", "LaunchTemplateData": { "BlockDeviceMappings": [ { @@ -267,12 +267,12 @@ ], "IamInstanceProfile": { "Name": { - "Ref": "AWSIAMInstanceProfilemastersqueueprocessorexamplecom" + "Ref": "AWSIAMInstanceProfilemastersnthsqsresourcesexamplecom" } }, "ImageId": "ami-12345678", "InstanceType": "m3.medium", - "KeyName": "kubernetes.queueprocessor.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", + "KeyName": "kubernetes.nthsqsresources.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", "MetadataOptions": { "HttpPutResponseHopLimit": 1, "HttpTokens": "optional" @@ -284,7 +284,7 @@ "DeviceIndex": 0, "Groups": [ { - "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom" + "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom" } ] } @@ -295,11 +295,11 @@ "Tags": [ { "Key": "KubernetesCluster", - "Value": "queueprocessor.example.com" + "Value": "nthsqsresources.example.com" }, { "Key": "Name", - "Value": "master-us-test-1a.masters.queueprocessor.example.com" + "Value": "master-us-test-1a.masters.nthsqsresources.example.com" }, { "Key": "aws-node-termination-handler/managed", @@ -334,7 +334,7 @@ "Value": "master-us-test-1a" }, { - "Key": "kubernetes.io/cluster/queueprocessor.example.com", + "Key": "kubernetes.io/cluster/nthsqsresources.example.com", "Value": "owned" } ] @@ -344,11 +344,11 @@ "Tags": [ { "Key": "KubernetesCluster", - "Value": "queueprocessor.example.com" + "Value": "nthsqsresources.example.com" }, { "Key": "Name", - "Value": "master-us-test-1a.masters.queueprocessor.example.com" + "Value": "master-us-test-1a.masters.nthsqsresources.example.com" }, { "Key": "aws-node-termination-handler/managed", @@ -383,7 +383,7 @@ "Value": "master-us-test-1a" }, { - "Key": "kubernetes.io/cluster/queueprocessor.example.com", + "Key": "kubernetes.io/cluster/nthsqsresources.example.com", "Value": "owned" } ] @@ -393,10 +393,10 @@ } } }, - "AWSEC2LaunchTemplatenodesqueueprocessorexamplecom": { + "AWSEC2LaunchTemplatenodesnthsqsresourcesexamplecom": { "Type": "AWS::EC2::LaunchTemplate", "Properties": { - "LaunchTemplateName": "nodes.queueprocessor.example.com", + "LaunchTemplateName": "nodes.nthsqsresources.example.com", "LaunchTemplateData": { "BlockDeviceMappings": [ { @@ -413,12 +413,12 @@ ], "IamInstanceProfile": { "Name": { - "Ref": "AWSIAMInstanceProfilenodesqueueprocessorexamplecom" + "Ref": "AWSIAMInstanceProfilenodesnthsqsresourcesexamplecom" } }, "ImageId": "ami-12345678", "InstanceType": "t2.medium", - "KeyName": "kubernetes.queueprocessor.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", + "KeyName": "kubernetes.nthsqsresources.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", "MetadataOptions": { "HttpPutResponseHopLimit": 1, "HttpTokens": "optional" @@ -430,7 +430,7 @@ "DeviceIndex": 0, "Groups": [ { - "Ref": "AWSEC2SecurityGroupnodesqueueprocessorexamplecom" + "Ref": "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom" } ] } @@ -441,11 +441,11 @@ "Tags": [ { "Key": "KubernetesCluster", - "Value": "queueprocessor.example.com" + "Value": "nthsqsresources.example.com" }, { "Key": "Name", - "Value": "nodes.queueprocessor.example.com" + "Value": "nodes.nthsqsresources.example.com" }, { "Key": "aws-node-termination-handler/managed", @@ -468,7 +468,7 @@ "Value": "nodes" }, { - "Key": "kubernetes.io/cluster/queueprocessor.example.com", + "Key": "kubernetes.io/cluster/nthsqsresources.example.com", "Value": "owned" } ] @@ -478,11 +478,11 @@ "Tags": [ { "Key": "KubernetesCluster", - "Value": "queueprocessor.example.com" + "Value": "nthsqsresources.example.com" }, { "Key": "Name", - "Value": "nodes.queueprocessor.example.com" + "Value": "nodes.nthsqsresources.example.com" }, { "Key": "aws-node-termination-handler/managed", @@ -505,7 +505,7 @@ "Value": "nodes" }, { - "Key": "kubernetes.io/cluster/queueprocessor.example.com", + "Key": "kubernetes.io/cluster/nthsqsresources.example.com", "Value": "owned" } ] @@ -519,31 +519,31 @@ "Type": "AWS::EC2::Route", "Properties": { "RouteTableId": { - "Ref": "AWSEC2RouteTablequeueprocessorexamplecom" + "Ref": "AWSEC2RouteTablenthsqsresourcesexamplecom" }, "DestinationCidrBlock": "0.0.0.0/0", "GatewayId": { - "Ref": "AWSEC2InternetGatewayqueueprocessorexamplecom" + "Ref": "AWSEC2InternetGatewaynthsqsresourcesexamplecom" } } }, - "AWSEC2RouteTablequeueprocessorexamplecom": { + "AWSEC2RouteTablenthsqsresourcesexamplecom": { "Type": "AWS::EC2::RouteTable", "Properties": { "VpcId": { - "Ref": "AWSEC2VPCqueueprocessorexamplecom" + "Ref": "AWSEC2VPCnthsqsresourcesexamplecom" }, "Tags": [ { "Key": "KubernetesCluster", - "Value": "queueprocessor.example.com" + "Value": "nthsqsresources.example.com" }, { "Key": "Name", - "Value": "queueprocessor.example.com" + "Value": "nthsqsresources.example.com" }, { - "Key": "kubernetes.io/cluster/queueprocessor.example.com", + "Key": "kubernetes.io/cluster/nthsqsresources.example.com", "Value": "owned" }, { @@ -553,11 +553,11 @@ ] } }, - "AWSEC2SecurityGroupEgressfrommastersqueueprocessorexamplecomegressall0to000000": { + "AWSEC2SecurityGroupEgressfrommastersnthsqsresourcesexamplecomegressall0to000000": { "Type": "AWS::EC2::SecurityGroupEgress", "Properties": { "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom" + "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom" }, "FromPort": 0, "ToPort": 0, @@ -565,11 +565,11 @@ "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupEgressfromnodesqueueprocessorexamplecomegressall0to000000": { + "AWSEC2SecurityGroupEgressfromnodesnthsqsresourcesexamplecomegressall0to000000": { "Type": "AWS::EC2::SecurityGroupEgress", "Properties": { "GroupId": { - "Ref": "AWSEC2SecurityGroupnodesqueueprocessorexamplecom" + "Ref": "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom" }, "FromPort": 0, "ToPort": 0, @@ -577,11 +577,11 @@ "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22mastersqueueprocessorexamplecom": { + "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22mastersnthsqsresourcesexamplecom": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom" + "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom" }, "FromPort": 22, "ToPort": 22, @@ -589,11 +589,11 @@ "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22nodesqueueprocessorexamplecom": { + "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22nodesnthsqsresourcesexamplecom": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { - "Ref": "AWSEC2SecurityGroupnodesqueueprocessorexamplecom" + "Ref": "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom" }, "FromPort": 22, "ToPort": 22, @@ -601,11 +601,11 @@ "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupIngressfrom00000ingresstcp443to443mastersqueueprocessorexamplecom": { + "AWSEC2SecurityGroupIngressfrom00000ingresstcp443to443mastersnthsqsresourcesexamplecom": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom" + "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom" }, "FromPort": 443, "ToPort": 443, @@ -613,186 +613,186 @@ "CidrIp": "0.0.0.0/0" } }, - "AWSEC2SecurityGroupIngressfrommastersqueueprocessorexamplecomingressall0to0mastersqueueprocessorexamplecom": { + "AWSEC2SecurityGroupIngressfrommastersnthsqsresourcesexamplecomingressall0to0mastersnthsqsresourcesexamplecom": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom" + "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom" }, "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom" + "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom" }, "FromPort": 0, "ToPort": 0, "IpProtocol": "-1" } }, - "AWSEC2SecurityGroupIngressfrommastersqueueprocessorexamplecomingressall0to0nodesqueueprocessorexamplecom": { + "AWSEC2SecurityGroupIngressfrommastersnthsqsresourcesexamplecomingressall0to0nodesnthsqsresourcesexamplecom": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { - "Ref": "AWSEC2SecurityGroupnodesqueueprocessorexamplecom" + "Ref": "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom" }, "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom" + "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom" }, "FromPort": 0, "ToPort": 0, "IpProtocol": "-1" } }, - "AWSEC2SecurityGroupIngressfromnodesqueueprocessorexamplecomingressall0to0nodesqueueprocessorexamplecom": { + "AWSEC2SecurityGroupIngressfromnodesnthsqsresourcesexamplecomingressall0to0nodesnthsqsresourcesexamplecom": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { - "Ref": "AWSEC2SecurityGroupnodesqueueprocessorexamplecom" + "Ref": "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom" }, "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodesqueueprocessorexamplecom" + "Ref": "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom" }, "FromPort": 0, "ToPort": 0, "IpProtocol": "-1" } }, - "AWSEC2SecurityGroupIngressfromnodesqueueprocessorexamplecomingresstcp1to2379mastersqueueprocessorexamplecom": { + "AWSEC2SecurityGroupIngressfromnodesnthsqsresourcesexamplecomingresstcp1to2379mastersnthsqsresourcesexamplecom": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom" + "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom" }, "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodesqueueprocessorexamplecom" + "Ref": "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom" }, "FromPort": 1, "ToPort": 2379, "IpProtocol": "tcp" } }, - "AWSEC2SecurityGroupIngressfromnodesqueueprocessorexamplecomingresstcp2382to4000mastersqueueprocessorexamplecom": { + "AWSEC2SecurityGroupIngressfromnodesnthsqsresourcesexamplecomingresstcp2382to4000mastersnthsqsresourcesexamplecom": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom" + "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom" }, "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodesqueueprocessorexamplecom" + "Ref": "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom" }, "FromPort": 2382, "ToPort": 4000, "IpProtocol": "tcp" } }, - "AWSEC2SecurityGroupIngressfromnodesqueueprocessorexamplecomingresstcp4003to65535mastersqueueprocessorexamplecom": { + "AWSEC2SecurityGroupIngressfromnodesnthsqsresourcesexamplecomingresstcp4003to65535mastersnthsqsresourcesexamplecom": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom" + "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom" }, "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodesqueueprocessorexamplecom" + "Ref": "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom" }, "FromPort": 4003, "ToPort": 65535, "IpProtocol": "tcp" } }, - "AWSEC2SecurityGroupIngressfromnodesqueueprocessorexamplecomingressudp1to65535mastersqueueprocessorexamplecom": { + "AWSEC2SecurityGroupIngressfromnodesnthsqsresourcesexamplecomingressudp1to65535mastersnthsqsresourcesexamplecom": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersqueueprocessorexamplecom" + "Ref": "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom" }, "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodesqueueprocessorexamplecom" + "Ref": "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom" }, "FromPort": 1, "ToPort": 65535, "IpProtocol": "udp" } }, - "AWSEC2SecurityGroupmastersqueueprocessorexamplecom": { + "AWSEC2SecurityGroupmastersnthsqsresourcesexamplecom": { "Type": "AWS::EC2::SecurityGroup", "Properties": { - "GroupName": "masters.queueprocessor.example.com", + "GroupName": "masters.nthsqsresources.example.com", "VpcId": { - "Ref": "AWSEC2VPCqueueprocessorexamplecom" + "Ref": "AWSEC2VPCnthsqsresourcesexamplecom" }, "GroupDescription": "Security group for masters", "Tags": [ { "Key": "KubernetesCluster", - "Value": "queueprocessor.example.com" + "Value": "nthsqsresources.example.com" }, { "Key": "Name", - "Value": "masters.queueprocessor.example.com" + "Value": "masters.nthsqsresources.example.com" }, { - "Key": "kubernetes.io/cluster/queueprocessor.example.com", + "Key": "kubernetes.io/cluster/nthsqsresources.example.com", "Value": "owned" } ] } }, - "AWSEC2SecurityGroupnodesqueueprocessorexamplecom": { + "AWSEC2SecurityGroupnodesnthsqsresourcesexamplecom": { "Type": "AWS::EC2::SecurityGroup", "Properties": { - "GroupName": "nodes.queueprocessor.example.com", + "GroupName": "nodes.nthsqsresources.example.com", "VpcId": { - "Ref": "AWSEC2VPCqueueprocessorexamplecom" + "Ref": "AWSEC2VPCnthsqsresourcesexamplecom" }, "GroupDescription": "Security group for nodes", "Tags": [ { "Key": "KubernetesCluster", - "Value": "queueprocessor.example.com" + "Value": "nthsqsresources.example.com" }, { "Key": "Name", - "Value": "nodes.queueprocessor.example.com" + "Value": "nodes.nthsqsresources.example.com" }, { - "Key": "kubernetes.io/cluster/queueprocessor.example.com", + "Key": "kubernetes.io/cluster/nthsqsresources.example.com", "Value": "owned" } ] } }, - "AWSEC2SubnetRouteTableAssociationustest1aqueueprocessorexamplecom": { + "AWSEC2SubnetRouteTableAssociationustest1anthsqsresourcesexamplecom": { "Type": "AWS::EC2::SubnetRouteTableAssociation", "Properties": { "SubnetId": { - "Ref": "AWSEC2Subnetustest1aqueueprocessorexamplecom" + "Ref": "AWSEC2Subnetustest1anthsqsresourcesexamplecom" }, "RouteTableId": { - "Ref": "AWSEC2RouteTablequeueprocessorexamplecom" + "Ref": "AWSEC2RouteTablenthsqsresourcesexamplecom" } } }, - "AWSEC2Subnetustest1aqueueprocessorexamplecom": { + "AWSEC2Subnetustest1anthsqsresourcesexamplecom": { "Type": "AWS::EC2::Subnet", "Properties": { "VpcId": { - "Ref": "AWSEC2VPCqueueprocessorexamplecom" + "Ref": "AWSEC2VPCnthsqsresourcesexamplecom" }, "CidrBlock": "172.20.32.0/19", "AvailabilityZone": "us-test-1a", "Tags": [ { "Key": "KubernetesCluster", - "Value": "queueprocessor.example.com" + "Value": "nthsqsresources.example.com" }, { "Key": "Name", - "Value": "us-test-1a.queueprocessor.example.com" + "Value": "us-test-1a.nthsqsresources.example.com" }, { "Key": "SubnetType", "Value": "Public" }, { - "Key": "kubernetes.io/cluster/queueprocessor.example.com", + "Key": "kubernetes.io/cluster/nthsqsresources.example.com", "Value": "owned" }, { @@ -802,29 +802,29 @@ ] } }, - "AWSEC2VPCDHCPOptionsAssociationqueueprocessorexamplecom": { + "AWSEC2VPCDHCPOptionsAssociationnthsqsresourcesexamplecom": { "Type": "AWS::EC2::VPCDHCPOptionsAssociation", "Properties": { "VpcId": { - "Ref": "AWSEC2VPCqueueprocessorexamplecom" + "Ref": "AWSEC2VPCnthsqsresourcesexamplecom" }, "DhcpOptionsId": { - "Ref": "AWSEC2DHCPOptionsqueueprocessorexamplecom" + "Ref": "AWSEC2DHCPOptionsnthsqsresourcesexamplecom" } } }, - "AWSEC2VPCGatewayAttachmentqueueprocessorexamplecom": { + "AWSEC2VPCGatewayAttachmentnthsqsresourcesexamplecom": { "Type": "AWS::EC2::VPCGatewayAttachment", "Properties": { "VpcId": { - "Ref": "AWSEC2VPCqueueprocessorexamplecom" + "Ref": "AWSEC2VPCnthsqsresourcesexamplecom" }, "InternetGatewayId": { - "Ref": "AWSEC2InternetGatewayqueueprocessorexamplecom" + "Ref": "AWSEC2InternetGatewaynthsqsresourcesexamplecom" } } }, - "AWSEC2VPCqueueprocessorexamplecom": { + "AWSEC2VPCnthsqsresourcesexamplecom": { "Type": "AWS::EC2::VPC", "Properties": { "CidrBlock": "172.20.0.0/16", @@ -833,20 +833,20 @@ "Tags": [ { "Key": "KubernetesCluster", - "Value": "queueprocessor.example.com" + "Value": "nthsqsresources.example.com" }, { "Key": "Name", - "Value": "queueprocessor.example.com" + "Value": "nthsqsresources.example.com" }, { - "Key": "kubernetes.io/cluster/queueprocessor.example.com", + "Key": "kubernetes.io/cluster/nthsqsresources.example.com", "Value": "owned" } ] } }, - "AWSEC2Volumeustest1aetcdeventsqueueprocessorexamplecom": { + "AWSEC2Volumeustest1aetcdeventsnthsqsresourcesexamplecom": { "Type": "AWS::EC2::Volume", "Properties": { "AvailabilityZone": "us-test-1a", @@ -858,11 +858,11 @@ "Tags": [ { "Key": "KubernetesCluster", - "Value": "queueprocessor.example.com" + "Value": "nthsqsresources.example.com" }, { "Key": "Name", - "Value": "us-test-1a.etcd-events.queueprocessor.example.com" + "Value": "us-test-1a.etcd-events.nthsqsresources.example.com" }, { "Key": "k8s.io/etcd/events", @@ -873,13 +873,13 @@ "Value": "1" }, { - "Key": "kubernetes.io/cluster/queueprocessor.example.com", + "Key": "kubernetes.io/cluster/nthsqsresources.example.com", "Value": "owned" } ] } }, - "AWSEC2Volumeustest1aetcdmainqueueprocessorexamplecom": { + "AWSEC2Volumeustest1aetcdmainnthsqsresourcesexamplecom": { "Type": "AWS::EC2::Volume", "Properties": { "AvailabilityZone": "us-test-1a", @@ -891,11 +891,11 @@ "Tags": [ { "Key": "KubernetesCluster", - "Value": "queueprocessor.example.com" + "Value": "nthsqsresources.example.com" }, { "Key": "Name", - "Value": "us-test-1a.etcd-main.queueprocessor.example.com" + "Value": "us-test-1a.etcd-main.nthsqsresources.example.com" }, { "Key": "k8s.io/etcd/main", @@ -906,16 +906,16 @@ "Value": "1" }, { - "Key": "kubernetes.io/cluster/queueprocessor.example.com", + "Key": "kubernetes.io/cluster/nthsqsresources.example.com", "Value": "owned" } ] } }, - "AWSEventsRulequeueprocessorexamplecomASGLifecycle": { + "AWSEventsRulenthsqsresourcesexamplecomASGLifecycle": { "Type": "AWS::Events::Rule", "Properties": { - "Name": "queueprocessor.example.com-ASGLifecycle", + "Name": "nthsqsresources.example.com-ASGLifecycle", "EventPattern": { "detail-type": [ "EC2 Instance-terminate Lifecycle Action" @@ -927,15 +927,15 @@ "Targets": [ { "Id": "1", - "Arn": "arn:aws:sqs:us-test-1:123456789012:queueprocessor-example-com-nth" + "Arn": "arn:aws:sqs:us-test-1:123456789012:nthsqsresources-example-com-nth" } ] } }, - "AWSEventsRulequeueprocessorexamplecomRebalanceRecommendation": { + "AWSEventsRulenthsqsresourcesexamplecomRebalanceRecommendation": { "Type": "AWS::Events::Rule", "Properties": { - "Name": "queueprocessor.example.com-RebalanceRecommendation", + "Name": "nthsqsresources.example.com-RebalanceRecommendation", "EventPattern": { "detail-type": [ "EC2 Instance Rebalance Recommendation" @@ -947,15 +947,15 @@ "Targets": [ { "Id": "1", - "Arn": "arn:aws:sqs:us-test-1:123456789012:queueprocessor-example-com-nth" + "Arn": "arn:aws:sqs:us-test-1:123456789012:nthsqsresources-example-com-nth" } ] } }, - "AWSEventsRulequeueprocessorexamplecomSpotInterruption": { + "AWSEventsRulenthsqsresourcesexamplecomSpotInterruption": { "Type": "AWS::Events::Rule", "Properties": { - "Name": "queueprocessor.example.com-SpotInterruption", + "Name": "nthsqsresources.example.com-SpotInterruption", "EventPattern": { "detail-type": [ "EC2 Spot Instance Interruption Warning" @@ -967,40 +967,40 @@ "Targets": [ { "Id": "1", - "Arn": "arn:aws:sqs:us-test-1:123456789012:queueprocessor-example-com-nth" + "Arn": "arn:aws:sqs:us-test-1:123456789012:nthsqsresources-example-com-nth" } ] } }, - "AWSIAMInstanceProfilemastersqueueprocessorexamplecom": { + "AWSIAMInstanceProfilemastersnthsqsresourcesexamplecom": { "Type": "AWS::IAM::InstanceProfile", "Properties": { - "InstanceProfileName": "masters.queueprocessor.example.com", + "InstanceProfileName": "masters.nthsqsresources.example.com", "Roles": [ { - "Ref": "AWSIAMRolemastersqueueprocessorexamplecom" + "Ref": "AWSIAMRolemastersnthsqsresourcesexamplecom" } ] } }, - "AWSIAMInstanceProfilenodesqueueprocessorexamplecom": { + "AWSIAMInstanceProfilenodesnthsqsresourcesexamplecom": { "Type": "AWS::IAM::InstanceProfile", "Properties": { - "InstanceProfileName": "nodes.queueprocessor.example.com", + "InstanceProfileName": "nodes.nthsqsresources.example.com", "Roles": [ { - "Ref": "AWSIAMRolenodesqueueprocessorexamplecom" + "Ref": "AWSIAMRolenodesnthsqsresourcesexamplecom" } ] } }, - "AWSIAMPolicymastersqueueprocessorexamplecom": { + "AWSIAMPolicymastersnthsqsresourcesexamplecom": { "Type": "AWS::IAM::Policy", "Properties": { - "PolicyName": "masters.queueprocessor.example.com", + "PolicyName": "masters.nthsqsresources.example.com", "Roles": [ { - "Ref": "AWSIAMRolemastersqueueprocessorexamplecom" + "Ref": "AWSIAMRolemastersnthsqsresourcesexamplecom" } ], "PolicyDocument": { @@ -1048,7 +1048,7 @@ ], "Condition": { "StringEquals": { - "ec2:ResourceTag/KubernetesCluster": "queueprocessor.example.com" + "ec2:ResourceTag/KubernetesCluster": "nthsqsresources.example.com" } }, "Effect": "Allow", @@ -1076,7 +1076,7 @@ ], "Condition": { "StringEquals": { - "autoscaling:ResourceTag/KubernetesCluster": "queueprocessor.example.com" + "autoscaling:ResourceTag/KubernetesCluster": "nthsqsresources.example.com" } }, "Effect": "Allow", @@ -1187,13 +1187,13 @@ } } }, - "AWSIAMPolicynodesqueueprocessorexamplecom": { + "AWSIAMPolicynodesnthsqsresourcesexamplecom": { "Type": "AWS::IAM::Policy", "Properties": { - "PolicyName": "nodes.queueprocessor.example.com", + "PolicyName": "nodes.nthsqsresources.example.com", "Roles": [ { - "Ref": "AWSIAMRolenodesqueueprocessorexamplecom" + "Ref": "AWSIAMRolenodesnthsqsresourcesexamplecom" } ], "PolicyDocument": { @@ -1213,10 +1213,10 @@ } } }, - "AWSIAMRolemastersqueueprocessorexamplecom": { + "AWSIAMRolemastersnthsqsresourcesexamplecom": { "Type": "AWS::IAM::Role", "Properties": { - "RoleName": "masters.queueprocessor.example.com", + "RoleName": "masters.nthsqsresources.example.com", "AssumeRolePolicyDocument": { "Statement": [ { @@ -1232,23 +1232,23 @@ "Tags": [ { "Key": "KubernetesCluster", - "Value": "queueprocessor.example.com" + "Value": "nthsqsresources.example.com" }, { "Key": "Name", - "Value": "masters.queueprocessor.example.com" + "Value": "masters.nthsqsresources.example.com" }, { - "Key": "kubernetes.io/cluster/queueprocessor.example.com", + "Key": "kubernetes.io/cluster/nthsqsresources.example.com", "Value": "owned" } ] } }, - "AWSIAMRolenodesqueueprocessorexamplecom": { + "AWSIAMRolenodesnthsqsresourcesexamplecom": { "Type": "AWS::IAM::Role", "Properties": { - "RoleName": "nodes.queueprocessor.example.com", + "RoleName": "nodes.nthsqsresources.example.com", "AssumeRolePolicyDocument": { "Statement": [ { @@ -1264,25 +1264,25 @@ "Tags": [ { "Key": "KubernetesCluster", - "Value": "queueprocessor.example.com" + "Value": "nthsqsresources.example.com" }, { "Key": "Name", - "Value": "nodes.queueprocessor.example.com" + "Value": "nodes.nthsqsresources.example.com" }, { - "Key": "kubernetes.io/cluster/queueprocessor.example.com", + "Key": "kubernetes.io/cluster/nthsqsresources.example.com", "Value": "owned" } ] } }, - "AWSSQSQueuePolicyqueueprocessorexamplecomnthPolicy": { + "AWSSQSQueuePolicynthsqsresourcesexamplecomnthPolicy": { "Type": "AWS::SQS::QueuePolicy", "Properties": { "Queues": [ { - "Ref": "AWSSQSQueuequeueprocessorexamplecomnth" + "Ref": "AWSSQSQueuenthsqsresourcesexamplecomnth" } ], "PolicyDocument": { @@ -1297,7 +1297,7 @@ ] }, "Resource": [ - "arn:aws:sqs:us-test-1:123456789012:queueprocessor-example-com-nth" + "arn:aws:sqs:us-test-1:123456789012:nthsqsresources-example-com-nth" ] } ], @@ -1305,22 +1305,22 @@ } } }, - "AWSSQSQueuequeueprocessorexamplecomnth": { + "AWSSQSQueuenthsqsresourcesexamplecomnth": { "Type": "AWS::SQS::Queue", "Properties": { - "QueueName": "queueprocessor-example-com-nth", + "QueueName": "nthsqsresources-example-com-nth", "MessageRetentionPeriod": 300, "Tags": [ { "Key": "KubernetesCluster", - "Value": "queueprocessor.example.com" + "Value": "nthsqsresources.example.com" }, { "Key": "Name", - "Value": "queueprocessor-example-com-nth" + "Value": "nthsqsresources-example-com-nth" }, { - "Key": "kubernetes.io/cluster/queueprocessor.example.com", + "Key": "kubernetes.io/cluster/nthsqsresources.example.com", "Value": "owned" } ] diff --git a/tests/integration/update_cluster/nodeterminationhandler_sqs_resources/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/nth_sqs_resources/cloudformation.json.extracted.yaml similarity index 95% rename from tests/integration/update_cluster/nodeterminationhandler_sqs_resources/cloudformation.json.extracted.yaml rename to tests/integration/update_cluster/nth_sqs_resources/cloudformation.json.extracted.yaml index 211b8ad486..2fd7a34145 100644 --- a/tests/integration/update_cluster/nodeterminationhandler_sqs_resources/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/nth_sqs_resources/cloudformation.json.extracted.yaml @@ -1,4 +1,4 @@ -Resources.AWSEC2LaunchTemplatemasterustest1amastersqueueprocessorexamplecom.Properties.LaunchTemplateData.UserData: | +Resources.AWSEC2LaunchTemplatemasterustest1amastersnthsqsresourcesexamplecom.Properties.LaunchTemplateData.UserData: | #!/bin/bash set -o errexit set -o nounset @@ -206,8 +206,8 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersqueueprocessorexamplecom.Prop requestheaderUsernameHeaders: - X-Remote-User securePort: 443 - serviceAccountIssuer: https://api.internal.queueprocessor.example.com - serviceAccountJWKSURI: https://api.internal.queueprocessor.example.com/openid/v1/jwks + serviceAccountIssuer: https://api.internal.nthsqsresources.example.com + serviceAccountJWKSURI: https://api.internal.nthsqsresources.example.com/openid/v1/jwks serviceClusterIPRange: 100.64.0.0/13 storageBackend: etcd3 kubeControllerManager: @@ -215,7 +215,7 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersqueueprocessorexamplecom.Prop attachDetachReconcileSyncPeriod: 1m0s cloudProvider: aws clusterCIDR: 100.96.0.0/11 - clusterName: queueprocessor.example.com + clusterName: nthsqsresources.example.com configureCloudRoutes: false image: k8s.gcr.io/kube-controller-manager:v1.20.0 leaderElection: @@ -288,8 +288,8 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersqueueprocessorexamplecom.Prop - 6e3f80e8451ecbe7b3559247721c3e226be6b228acaadee7e13683f80c20e81c@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.0.tgz - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/protokube - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/channels - ClusterName: queueprocessor.example.com - ConfigBase: memfs://clusters.example.com/queueprocessor.example.com + ClusterName: nthsqsresources.example.com + ConfigBase: memfs://clusters.example.com/nthsqsresources.example.com InstanceGroupName: master-us-test-1a InstanceGroupRole: Master KubeletConfig: @@ -315,10 +315,10 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersqueueprocessorexamplecom.Prop podManifestPath: /etc/kubernetes/manifests registerSchedulable: false channels: - - memfs://clusters.example.com/queueprocessor.example.com/addons/bootstrap-channel.yaml + - memfs://clusters.example.com/nthsqsresources.example.com/addons/bootstrap-channel.yaml etcdManifests: - - memfs://clusters.example.com/queueprocessor.example.com/manifests/etcd/main.yaml - - memfs://clusters.example.com/queueprocessor.example.com/manifests/etcd/events.yaml + - memfs://clusters.example.com/nthsqsresources.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/nthsqsresources.example.com/manifests/etcd/events.yaml staticManifests: - key: kube-apiserver-healthcheck path: manifests/static/kube-apiserver-healthcheck.yaml @@ -327,7 +327,7 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersqueueprocessorexamplecom.Prop download-release echo "== nodeup node config done ==" -Resources.AWSEC2LaunchTemplatenodesqueueprocessorexamplecom.Properties.LaunchTemplateData.UserData: | +Resources.AWSEC2LaunchTemplatenodesnthsqsresourcesexamplecom.Properties.LaunchTemplateData.UserData: | #!/bin/bash set -o errexit set -o nounset @@ -531,8 +531,8 @@ Resources.AWSEC2LaunchTemplatenodesqueueprocessorexamplecom.Properties.LaunchTem - 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubectl - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz - 6e3f80e8451ecbe7b3559247721c3e226be6b228acaadee7e13683f80c20e81c@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.0.tgz - ClusterName: queueprocessor.example.com - ConfigBase: memfs://clusters.example.com/queueprocessor.example.com + ClusterName: nthsqsresources.example.com + ConfigBase: memfs://clusters.example.com/nthsqsresources.example.com InstanceGroupName: nodes InstanceGroupRole: Node KubeletConfig: @@ -554,7 +554,7 @@ Resources.AWSEC2LaunchTemplatenodesqueueprocessorexamplecom.Properties.LaunchTem nonMasqueradeCIDR: 100.64.0.0/10 podManifestPath: /etc/kubernetes/manifests channels: - - memfs://clusters.example.com/queueprocessor.example.com/addons/bootstrap-channel.yaml + - memfs://clusters.example.com/nthsqsresources.example.com/addons/bootstrap-channel.yaml __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_cloudwatch_event_rule_queueprocessor.example.com-ASGLifecycle_event_pattern b/tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_nthsqsresources.example.com-ASGLifecycle_event_pattern similarity index 100% rename from tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_cloudwatch_event_rule_queueprocessor.example.com-ASGLifecycle_event_pattern rename to tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_nthsqsresources.example.com-ASGLifecycle_event_pattern diff --git a/tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_cloudwatch_event_rule_queueprocessor.example.com-RebalanceRecommendation_event_pattern b/tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_nthsqsresources.example.com-RebalanceRecommendation_event_pattern similarity index 100% rename from tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_cloudwatch_event_rule_queueprocessor.example.com-RebalanceRecommendation_event_pattern rename to tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_nthsqsresources.example.com-RebalanceRecommendation_event_pattern diff --git a/tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_cloudwatch_event_rule_queueprocessor.example.com-SpotInterruption_event_pattern b/tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_nthsqsresources.example.com-SpotInterruption_event_pattern similarity index 100% rename from tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_cloudwatch_event_rule_queueprocessor.example.com-SpotInterruption_event_pattern rename to tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_nthsqsresources.example.com-SpotInterruption_event_pattern diff --git a/tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_queueprocessor.example.com-ASGLifecycle_event_pattern b/tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_queueprocessor.example.com-ASGLifecycle_event_pattern new file mode 100644 index 0000000000..c8db9dbe9c --- /dev/null +++ b/tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_queueprocessor.example.com-ASGLifecycle_event_pattern @@ -0,0 +1 @@ +{"source":["aws.autoscaling"],"detail-type":["EC2 Instance-terminate Lifecycle Action"]} diff --git a/tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_queueprocessor.example.com-RebalanceRecommendation_event_pattern b/tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_queueprocessor.example.com-RebalanceRecommendation_event_pattern new file mode 100644 index 0000000000..226b0ac52d --- /dev/null +++ b/tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_queueprocessor.example.com-RebalanceRecommendation_event_pattern @@ -0,0 +1 @@ +{"source": ["aws.ec2"],"detail-type": ["EC2 Instance Rebalance Recommendation"]} diff --git a/tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_queueprocessor.example.com-SpotInterruption_event_pattern b/tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_queueprocessor.example.com-SpotInterruption_event_pattern new file mode 100644 index 0000000000..2d0e83b416 --- /dev/null +++ b/tests/integration/update_cluster/nth_sqs_resources/data/aws_cloudwatch_event_rule_queueprocessor.example.com-SpotInterruption_event_pattern @@ -0,0 +1 @@ +{"source": ["aws.ec2"],"detail-type": ["EC2 Spot Instance Interruption Warning"]} diff --git a/tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_iam_role_masters.queueprocessor.example.com_policy b/tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_masters.nthsqsresources.example.com_policy similarity index 100% rename from tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_iam_role_masters.queueprocessor.example.com_policy rename to tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_masters.nthsqsresources.example.com_policy diff --git a/tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_iam_role_nodes.queueprocessor.example.com_policy b/tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_masters.queueprocessor.example.com_policy similarity index 100% rename from tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_iam_role_nodes.queueprocessor.example.com_policy rename to tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_masters.queueprocessor.example.com_policy diff --git a/tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_nodes.nthsqsresources.example.com_policy b/tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_nodes.nthsqsresources.example.com_policy new file mode 100644 index 0000000000..66d5de1d5a --- /dev/null +++ b/tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_nodes.nthsqsresources.example.com_policy @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { "Service": "ec2.amazonaws.com"}, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_nodes.queueprocessor.example.com_policy b/tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_nodes.queueprocessor.example.com_policy new file mode 100644 index 0000000000..66d5de1d5a --- /dev/null +++ b/tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_nodes.queueprocessor.example.com_policy @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { "Service": "ec2.amazonaws.com"}, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_policy_masters.nthsqsresources.example.com_policy b/tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_policy_masters.nthsqsresources.example.com_policy new file mode 100644 index 0000000000..3911ebc4e7 --- /dev/null +++ b/tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_policy_masters.nthsqsresources.example.com_policy @@ -0,0 +1,182 @@ +{ + "Statement": [ + { + "Action": [ + "ec2:DescribeAccountAttributes", + "ec2:DescribeInstances", + "ec2:DescribeInternetGateways", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVolumes" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:DescribeVolumesModifications", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateRoute", + "ec2:DeleteRoute", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:RevokeSecurityGroupIngress" + ], + "Condition": { + "StringEquals": { + "ec2:ResourceTag/KubernetesCluster": "nthsqsresources.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeLaunchTemplateVersions" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "autoscaling:UpdateAutoScalingGroup" + ], + "Condition": { + "StringEquals": { + "autoscaling:ResourceTag/KubernetesCluster": "nthsqsresources.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "ec2:DescribeVpcs", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DeregisterTargets", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "iam:ListServerCertificates", + "iam:GetServerCertificate" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "route53:ChangeResourceRecordSets", + "route53:ListResourceRecordSets", + "route53:GetHostedZone" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" + ] + }, + { + "Action": [ + "route53:GetChange" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:route53:::change/*" + ] + }, + { + "Action": [ + "route53:ListHostedZones" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "autoscaling:CompleteLifecycleAction", + "autoscaling:DescribeAutoScalingInstances", + "sqs:DeleteMessage", + "sqs:ReceiveMessage" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_iam_role_policy_masters.queueprocessor.example.com_policy b/tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_policy_masters.queueprocessor.example.com_policy similarity index 100% rename from tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_iam_role_policy_masters.queueprocessor.example.com_policy rename to tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_policy_masters.queueprocessor.example.com_policy diff --git a/tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_iam_role_policy_nodes.queueprocessor.example.com_policy b/tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_policy_nodes.nthsqsresources.example.com_policy similarity index 100% rename from tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_iam_role_policy_nodes.queueprocessor.example.com_policy rename to tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_policy_nodes.nthsqsresources.example.com_policy diff --git a/tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_policy_nodes.queueprocessor.example.com_policy b/tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_policy_nodes.queueprocessor.example.com_policy new file mode 100644 index 0000000000..49749a010d --- /dev/null +++ b/tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_policy_nodes.queueprocessor.example.com_policy @@ -0,0 +1,15 @@ +{ + "Statement": [ + { + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeRegions" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_key_pair_kubernetes.queueprocessor.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key b/tests/integration/update_cluster/nth_sqs_resources/data/aws_key_pair_kubernetes.nthsqsresources.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key similarity index 100% rename from tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_key_pair_kubernetes.queueprocessor.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key rename to tests/integration/update_cluster/nth_sqs_resources/data/aws_key_pair_kubernetes.nthsqsresources.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key diff --git a/tests/integration/update_cluster/nodeterminationhandler_sqs_resources/id_rsa.pub b/tests/integration/update_cluster/nth_sqs_resources/data/aws_key_pair_kubernetes.queueprocessor.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key old mode 100755 new mode 100644 similarity index 100% rename from tests/integration/update_cluster/nodeterminationhandler_sqs_resources/id_rsa.pub rename to tests/integration/update_cluster/nth_sqs_resources/data/aws_key_pair_kubernetes.queueprocessor.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key diff --git a/tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_master-us-test-1a.masters.nthsqsresources.example.com_user_data b/tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_master-us-test-1a.masters.nthsqsresources.example.com_user_data new file mode 100644 index 0000000000..26311f33a6 --- /dev/null +++ b/tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_master-us-test-1a.masters.nthsqsresources.example.com_user_data @@ -0,0 +1,328 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/nodeup +NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 +NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/nodeup +NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + +export AWS_REGION=us-test-1 + + + + +sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: + manageStorageClasses: true +containerRuntime: containerd +containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.4 +docker: + skipInstall: true +encryptionConfig: null +etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.20.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.nthsqsresources.example.com + serviceAccountJWKSURI: https://api.internal.nthsqsresources.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: nthsqsresources.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.20.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.20.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.20.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +{} + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: + amd64: + - ff2422571c4c1e9696e367f5f25466b96fb6e501f28aed29f414b1524a52dea0@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubelet + - a5895007f331f08d2e082eb12458764949559f30bcc5beae26c38f3e2724262c@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 96641849cb78a0a119223a427dfdc1ade88412ef791a14193212c8c8e29d447b@https://github.com/containerd/containerd/releases/download/v1.4.4/cri-containerd-cni-1.4.4-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/protokube + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/channels + arm64: + - 47ab6c4273fc3bb0cb8ec9517271d915890c5a6b0e54b2991e7a8fbbe77b06e4@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubelet + - 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 6e3f80e8451ecbe7b3559247721c3e226be6b228acaadee7e13683f80c20e81c@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.0.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/protokube + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/channels +ClusterName: nthsqsresources.example.com +ConfigBase: memfs://clusters.example.com/nthsqsresources.example.com +InstanceGroupName: master-us-test-1a +InstanceGroupRole: Master +KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false +channels: +- memfs://clusters.example.com/nthsqsresources.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/nthsqsresources.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/nthsqsresources.example.com/manifests/etcd/events.yaml +staticManifests: +- key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_launch_template_master-us-test-1a.masters.queueprocessor.example.com_user_data b/tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_master-us-test-1a.masters.queueprocessor.example.com_user_data similarity index 100% rename from tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_launch_template_master-us-test-1a.masters.queueprocessor.example.com_user_data rename to tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_master-us-test-1a.masters.queueprocessor.example.com_user_data diff --git a/tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_nodes.nthsqsresources.example.com_user_data b/tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_nodes.nthsqsresources.example.com_user_data new file mode 100644 index 0000000000..a0525c9634 --- /dev/null +++ b/tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_nodes.nthsqsresources.example.com_user_data @@ -0,0 +1,232 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/amd64/nodeup +NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 +NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.21.0-alpha.1/linux/arm64/nodeup +NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + +export AWS_REGION=us-test-1 + + + + +sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: + manageStorageClasses: true +containerRuntime: containerd +containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.4 +docker: + skipInstall: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.20.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +{} + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: + amd64: + - ff2422571c4c1e9696e367f5f25466b96fb6e501f28aed29f414b1524a52dea0@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubelet + - a5895007f331f08d2e082eb12458764949559f30bcc5beae26c38f3e2724262c@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 96641849cb78a0a119223a427dfdc1ade88412ef791a14193212c8c8e29d447b@https://github.com/containerd/containerd/releases/download/v1.4.4/cri-containerd-cni-1.4.4-linux-amd64.tar.gz + arm64: + - 47ab6c4273fc3bb0cb8ec9517271d915890c5a6b0e54b2991e7a8fbbe77b06e4@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubelet + - 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 6e3f80e8451ecbe7b3559247721c3e226be6b228acaadee7e13683f80c20e81c@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.0.tgz +ClusterName: nthsqsresources.example.com +ConfigBase: memfs://clusters.example.com/nthsqsresources.example.com +InstanceGroupName: nodes +InstanceGroupRole: Node +KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests +channels: +- memfs://clusters.example.com/nthsqsresources.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_launch_template_nodes.queueprocessor.example.com_user_data b/tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_nodes.queueprocessor.example.com_user_data similarity index 100% rename from tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_launch_template_nodes.queueprocessor.example.com_user_data rename to tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_nodes.queueprocessor.example.com_user_data diff --git a/tests/integration/update_cluster/nth_sqs_resources/data/aws_sqs_queue_nthsqsresources-example-com-nth_policy b/tests/integration/update_cluster/nth_sqs_resources/data/aws_sqs_queue_nthsqsresources-example-com-nth_policy new file mode 100644 index 0000000000..8a1835212c --- /dev/null +++ b/tests/integration/update_cluster/nth_sqs_resources/data/aws_sqs_queue_nthsqsresources-example-com-nth_policy @@ -0,0 +1,13 @@ +{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": { + "Service": ["events.amazonaws.com", "sqs.amazonaws.com"] + }, + "Action": "sqs:SendMessage", + "Resource": [ + "arn:aws:sqs:us-test-1:123456789012:nthsqsresources-example-com-nth" + ] + }] + } diff --git a/tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_sqs_queue_queueprocessor-example-com-nth_policy b/tests/integration/update_cluster/nth_sqs_resources/data/aws_sqs_queue_queueprocessor-example-com-nth_policy similarity index 100% rename from tests/integration/update_cluster/nodeterminationhandler_sqs_resources/data/aws_sqs_queue_queueprocessor-example-com-nth_policy rename to tests/integration/update_cluster/nth_sqs_resources/data/aws_sqs_queue_queueprocessor-example-com-nth_policy diff --git a/tests/integration/update_cluster/nth_sqs_resources/id_rsa.pub b/tests/integration/update_cluster/nth_sqs_resources/id_rsa.pub new file mode 100755 index 0000000000..81cb012783 --- /dev/null +++ b/tests/integration/update_cluster/nth_sqs_resources/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/nodeterminationhandler_sqs_resources/in-v1alpha2.yaml b/tests/integration/update_cluster/nth_sqs_resources/in-v1alpha2.yaml similarity index 82% rename from tests/integration/update_cluster/nodeterminationhandler_sqs_resources/in-v1alpha2.yaml rename to tests/integration/update_cluster/nth_sqs_resources/in-v1alpha2.yaml index 68f1857e13..b91c177428 100644 --- a/tests/integration/update_cluster/nodeterminationhandler_sqs_resources/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/nth_sqs_resources/in-v1alpha2.yaml @@ -2,13 +2,13 @@ apiVersion: kops.k8s.io/v1alpha2 kind: Cluster metadata: creationTimestamp: "2016-12-10T22:42:27Z" - name: queueprocessor.example.com + name: nthsqsresources.example.com spec: kubernetesApiAccess: - 0.0.0.0/0 channel: stable cloudProvider: aws - configBase: memfs://clusters.example.com/queueprocessor.example.com + configBase: memfs://clusters.example.com/nthsqsresources.example.com etcdClusters: - etcdMembers: - instanceGroup: master-us-test-1a @@ -22,8 +22,8 @@ spec: kubelet: anonymousAuth: false kubernetesVersion: v1.20.0 - masterInternalName: api.internal.queueprocessor.example.com - masterPublicName: api.queueprocessor.example.com + masterInternalName: api.internal.nthsqsresources.example.com + masterPublicName: api.nthsqsresources.example.com networkCIDR: 172.20.0.0/16 networking: cni: {} @@ -50,7 +50,7 @@ metadata: creationTimestamp: "2016-12-10T22:42:28Z" name: nodes labels: - kops.k8s.io/cluster: queueprocessor.example.com + kops.k8s.io/cluster: nthsqsresources.example.com spec: associatePublicIp: true image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 @@ -69,7 +69,7 @@ metadata: creationTimestamp: "2016-12-10T22:42:28Z" name: master-us-test-1a labels: - kops.k8s.io/cluster: queueprocessor.example.com + kops.k8s.io/cluster: nthsqsresources.example.com spec: associatePublicIp: true image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 diff --git a/tests/integration/update_cluster/nth_sqs_resources/kubernetes.tf b/tests/integration/update_cluster/nth_sqs_resources/kubernetes.tf new file mode 100644 index 0000000000..75c6944b40 --- /dev/null +++ b/tests/integration/update_cluster/nth_sqs_resources/kubernetes.tf @@ -0,0 +1,732 @@ +locals { + cluster_name = "nthsqsresources.example.com" + master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-nthsqsresources-example-com.id] + master_security_group_ids = [aws_security_group.masters-nthsqsresources-example-com.id] + masters_role_arn = aws_iam_role.masters-nthsqsresources-example-com.arn + masters_role_name = aws_iam_role.masters-nthsqsresources-example-com.name + node_autoscaling_group_ids = [aws_autoscaling_group.nodes-nthsqsresources-example-com.id] + node_security_group_ids = [aws_security_group.nodes-nthsqsresources-example-com.id] + node_subnet_ids = [aws_subnet.us-test-1a-nthsqsresources-example-com.id] + nodes_role_arn = aws_iam_role.nodes-nthsqsresources-example-com.arn + nodes_role_name = aws_iam_role.nodes-nthsqsresources-example-com.name + region = "us-test-1" + route_table_public_id = aws_route_table.nthsqsresources-example-com.id + subnet_us-test-1a_id = aws_subnet.us-test-1a-nthsqsresources-example-com.id + vpc_cidr_block = aws_vpc.nthsqsresources-example-com.cidr_block + vpc_id = aws_vpc.nthsqsresources-example-com.id +} + +output "cluster_name" { + value = "nthsqsresources.example.com" +} + +output "master_autoscaling_group_ids" { + value = [aws_autoscaling_group.master-us-test-1a-masters-nthsqsresources-example-com.id] +} + +output "master_security_group_ids" { + value = [aws_security_group.masters-nthsqsresources-example-com.id] +} + +output "masters_role_arn" { + value = aws_iam_role.masters-nthsqsresources-example-com.arn +} + +output "masters_role_name" { + value = aws_iam_role.masters-nthsqsresources-example-com.name +} + +output "node_autoscaling_group_ids" { + value = [aws_autoscaling_group.nodes-nthsqsresources-example-com.id] +} + +output "node_security_group_ids" { + value = [aws_security_group.nodes-nthsqsresources-example-com.id] +} + +output "node_subnet_ids" { + value = [aws_subnet.us-test-1a-nthsqsresources-example-com.id] +} + +output "nodes_role_arn" { + value = aws_iam_role.nodes-nthsqsresources-example-com.arn +} + +output "nodes_role_name" { + value = aws_iam_role.nodes-nthsqsresources-example-com.name +} + +output "region" { + value = "us-test-1" +} + +output "route_table_public_id" { + value = aws_route_table.nthsqsresources-example-com.id +} + +output "subnet_us-test-1a_id" { + value = aws_subnet.us-test-1a-nthsqsresources-example-com.id +} + +output "vpc_cidr_block" { + value = aws_vpc.nthsqsresources-example-com.cidr_block +} + +output "vpc_id" { + value = aws_vpc.nthsqsresources-example-com.id +} + +provider "aws" { + region = "us-test-1" +} + +resource "aws_autoscaling_group" "master-us-test-1a-masters-nthsqsresources-example-com" { + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + launch_template { + id = aws_launch_template.master-us-test-1a-masters-nthsqsresources-example-com.id + version = aws_launch_template.master-us-test-1a-masters-nthsqsresources-example-com.latest_version + } + max_size = 1 + metrics_granularity = "1Minute" + min_size = 1 + name = "master-us-test-1a.masters.nthsqsresources.example.com" + tag { + key = "KubernetesCluster" + propagate_at_launch = true + value = "nthsqsresources.example.com" + } + tag { + key = "Name" + propagate_at_launch = true + value = "master-us-test-1a.masters.nthsqsresources.example.com" + } + tag { + key = "aws-node-termination-handler/managed" + propagate_at_launch = true + value = "true" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" + propagate_at_launch = true + value = "master" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/role/master" + propagate_at_launch = true + value = "1" + } + tag { + key = "kops.k8s.io/instancegroup" + propagate_at_launch = true + value = "master-us-test-1a" + } + tag { + key = "kubernetes.io/cluster/nthsqsresources.example.com" + propagate_at_launch = true + value = "owned" + } + vpc_zone_identifier = [aws_subnet.us-test-1a-nthsqsresources-example-com.id] +} + +resource "aws_autoscaling_group" "nodes-nthsqsresources-example-com" { + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + launch_template { + id = aws_launch_template.nodes-nthsqsresources-example-com.id + version = aws_launch_template.nodes-nthsqsresources-example-com.latest_version + } + max_size = 2 + metrics_granularity = "1Minute" + min_size = 2 + name = "nodes.nthsqsresources.example.com" + tag { + key = "KubernetesCluster" + propagate_at_launch = true + value = "nthsqsresources.example.com" + } + tag { + key = "Name" + propagate_at_launch = true + value = "nodes.nthsqsresources.example.com" + } + tag { + key = "aws-node-termination-handler/managed" + propagate_at_launch = true + value = "true" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" + propagate_at_launch = true + value = "node" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/role/node" + propagate_at_launch = true + value = "1" + } + tag { + key = "kops.k8s.io/instancegroup" + propagate_at_launch = true + value = "nodes" + } + tag { + key = "kubernetes.io/cluster/nthsqsresources.example.com" + propagate_at_launch = true + value = "owned" + } + vpc_zone_identifier = [aws_subnet.us-test-1a-nthsqsresources-example-com.id] +} + +resource "aws_autoscaling_lifecycle_hook" "master-us-test-1a-NTHLifecycleHook" { + autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-nthsqsresources-example-com.id + default_result = "CONTINUE" + heartbeat_timeout = 300 + lifecycle_transition = "autoscaling:EC2_INSTANCE_TERMINATING" + name = "master-us-test-1a-NTHLifecycleHook" +} + +resource "aws_autoscaling_lifecycle_hook" "nodes-NTHLifecycleHook" { + autoscaling_group_name = aws_autoscaling_group.nodes-nthsqsresources-example-com.id + default_result = "CONTINUE" + heartbeat_timeout = 300 + lifecycle_transition = "autoscaling:EC2_INSTANCE_TERMINATING" + name = "nodes-NTHLifecycleHook" +} + +resource "aws_cloudwatch_event_rule" "nthsqsresources-example-com-ASGLifecycle" { + event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_nthsqsresources.example.com-ASGLifecycle_event_pattern") + name = "nthsqsresources.example.com-ASGLifecycle" + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "nthsqsresources.example.com-ASGLifecycle" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } +} + +resource "aws_cloudwatch_event_rule" "nthsqsresources-example-com-RebalanceRecommendation" { + event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_nthsqsresources.example.com-RebalanceRecommendation_event_pattern") + name = "nthsqsresources.example.com-RebalanceRecommendation" + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "nthsqsresources.example.com-RebalanceRecommendation" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } +} + +resource "aws_cloudwatch_event_rule" "nthsqsresources-example-com-SpotInterruption" { + event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_nthsqsresources.example.com-SpotInterruption_event_pattern") + name = "nthsqsresources.example.com-SpotInterruption" + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "nthsqsresources.example.com-SpotInterruption" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } +} + +resource "aws_cloudwatch_event_target" "nthsqsresources-example-com-ASGLifecycle-Target" { + arn = "arn:aws:sqs:us-test-1:123456789012:nthsqsresources-example-com-nth" + rule = aws_cloudwatch_event_rule.nthsqsresources-example-com-ASGLifecycle.id +} + +resource "aws_cloudwatch_event_target" "nthsqsresources-example-com-RebalanceRecommendation-Target" { + arn = "arn:aws:sqs:us-test-1:123456789012:nthsqsresources-example-com-nth" + rule = aws_cloudwatch_event_rule.nthsqsresources-example-com-RebalanceRecommendation.id +} + +resource "aws_cloudwatch_event_target" "nthsqsresources-example-com-SpotInterruption-Target" { + arn = "arn:aws:sqs:us-test-1:123456789012:nthsqsresources-example-com-nth" + rule = aws_cloudwatch_event_rule.nthsqsresources-example-com-SpotInterruption.id +} + +resource "aws_ebs_volume" "us-test-1a-etcd-events-nthsqsresources-example-com" { + availability_zone = "us-test-1a" + encrypted = false + iops = 3000 + size = 20 + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "us-test-1a.etcd-events.nthsqsresources.example.com" + "k8s.io/etcd/events" = "us-test-1a/us-test-1a" + "k8s.io/role/master" = "1" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } + throughput = 125 + type = "gp3" +} + +resource "aws_ebs_volume" "us-test-1a-etcd-main-nthsqsresources-example-com" { + availability_zone = "us-test-1a" + encrypted = false + iops = 3000 + size = 20 + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "us-test-1a.etcd-main.nthsqsresources.example.com" + "k8s.io/etcd/main" = "us-test-1a/us-test-1a" + "k8s.io/role/master" = "1" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } + throughput = 125 + type = "gp3" +} + +resource "aws_iam_instance_profile" "masters-nthsqsresources-example-com" { + name = "masters.nthsqsresources.example.com" + role = aws_iam_role.masters-nthsqsresources-example-com.name + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "masters.nthsqsresources.example.com" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } +} + +resource "aws_iam_instance_profile" "nodes-nthsqsresources-example-com" { + name = "nodes.nthsqsresources.example.com" + role = aws_iam_role.nodes-nthsqsresources-example-com.name + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "nodes.nthsqsresources.example.com" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } +} + +resource "aws_iam_role_policy" "masters-nthsqsresources-example-com" { + name = "masters.nthsqsresources.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_masters.nthsqsresources.example.com_policy") + role = aws_iam_role.masters-nthsqsresources-example-com.name +} + +resource "aws_iam_role_policy" "nodes-nthsqsresources-example-com" { + name = "nodes.nthsqsresources.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_nodes.nthsqsresources.example.com_policy") + role = aws_iam_role.nodes-nthsqsresources-example-com.name +} + +resource "aws_iam_role" "masters-nthsqsresources-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_masters.nthsqsresources.example.com_policy") + name = "masters.nthsqsresources.example.com" + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "masters.nthsqsresources.example.com" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } +} + +resource "aws_iam_role" "nodes-nthsqsresources-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_nodes.nthsqsresources.example.com_policy") + name = "nodes.nthsqsresources.example.com" + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "nodes.nthsqsresources.example.com" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } +} + +resource "aws_internet_gateway" "nthsqsresources-example-com" { + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "nthsqsresources.example.com" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } + vpc_id = aws_vpc.nthsqsresources-example-com.id +} + +resource "aws_key_pair" "kubernetes-nthsqsresources-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" { + key_name = "kubernetes.nthsqsresources.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57" + public_key = file("${path.module}/data/aws_key_pair_kubernetes.nthsqsresources.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key") + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "nthsqsresources.example.com" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } +} + +resource "aws_launch_template" "master-us-test-1a-masters-nthsqsresources-example-com" { + block_device_mappings { + device_name = "/dev/xvda" + ebs { + delete_on_termination = true + encrypted = true + iops = 3000 + throughput = 125 + volume_size = 64 + volume_type = "gp3" + } + } + block_device_mappings { + device_name = "/dev/sdc" + virtual_name = "ephemeral0" + } + iam_instance_profile { + name = aws_iam_instance_profile.masters-nthsqsresources-example-com.id + } + image_id = "ami-12345678" + instance_type = "m3.medium" + key_name = aws_key_pair.kubernetes-nthsqsresources-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id + lifecycle { + create_before_destroy = true + } + metadata_options { + http_endpoint = "enabled" + http_put_response_hop_limit = 1 + http_tokens = "optional" + } + name = "master-us-test-1a.masters.nthsqsresources.example.com" + network_interfaces { + associate_public_ip_address = true + delete_on_termination = true + security_groups = [aws_security_group.masters-nthsqsresources-example-com.id] + } + tag_specifications { + resource_type = "instance" + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "master-us-test-1a.masters.nthsqsresources.example.com" + "aws-node-termination-handler/managed" = "true" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "master" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" = "" + "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" + "k8s.io/role/master" = "1" + "kops.k8s.io/instancegroup" = "master-us-test-1a" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } + } + tag_specifications { + resource_type = "volume" + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "master-us-test-1a.masters.nthsqsresources.example.com" + "aws-node-termination-handler/managed" = "true" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "master" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" = "" + "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" + "k8s.io/role/master" = "1" + "kops.k8s.io/instancegroup" = "master-us-test-1a" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } + } + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "master-us-test-1a.masters.nthsqsresources.example.com" + "aws-node-termination-handler/managed" = "true" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "master" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" = "" + "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" + "k8s.io/role/master" = "1" + "kops.k8s.io/instancegroup" = "master-us-test-1a" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } + user_data = filebase64("${path.module}/data/aws_launch_template_master-us-test-1a.masters.nthsqsresources.example.com_user_data") +} + +resource "aws_launch_template" "nodes-nthsqsresources-example-com" { + block_device_mappings { + device_name = "/dev/xvda" + ebs { + delete_on_termination = true + encrypted = true + iops = 3000 + throughput = 125 + volume_size = 128 + volume_type = "gp3" + } + } + iam_instance_profile { + name = aws_iam_instance_profile.nodes-nthsqsresources-example-com.id + } + image_id = "ami-12345678" + instance_type = "t2.medium" + key_name = aws_key_pair.kubernetes-nthsqsresources-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id + lifecycle { + create_before_destroy = true + } + metadata_options { + http_endpoint = "enabled" + http_put_response_hop_limit = 1 + http_tokens = "optional" + } + name = "nodes.nthsqsresources.example.com" + network_interfaces { + associate_public_ip_address = true + delete_on_termination = true + security_groups = [aws_security_group.nodes-nthsqsresources-example-com.id] + } + tag_specifications { + resource_type = "instance" + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "nodes.nthsqsresources.example.com" + "aws-node-termination-handler/managed" = "true" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "nodes" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } + } + tag_specifications { + resource_type = "volume" + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "nodes.nthsqsresources.example.com" + "aws-node-termination-handler/managed" = "true" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "nodes" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } + } + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "nodes.nthsqsresources.example.com" + "aws-node-termination-handler/managed" = "true" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "nodes" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } + user_data = filebase64("${path.module}/data/aws_launch_template_nodes.nthsqsresources.example.com_user_data") +} + +resource "aws_route_table_association" "us-test-1a-nthsqsresources-example-com" { + route_table_id = aws_route_table.nthsqsresources-example-com.id + subnet_id = aws_subnet.us-test-1a-nthsqsresources-example-com.id +} + +resource "aws_route_table" "nthsqsresources-example-com" { + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "nthsqsresources.example.com" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + "kubernetes.io/kops/role" = "public" + } + vpc_id = aws_vpc.nthsqsresources-example-com.id +} + +resource "aws_route" "route-0-0-0-0--0" { + destination_cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.nthsqsresources-example-com.id + route_table_id = aws_route_table.nthsqsresources-example-com.id +} + +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-masters-nthsqsresources-example-com" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.masters-nthsqsresources-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-nodes-nthsqsresources-example-com" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-masters-nthsqsresources-example-com" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 443 + protocol = "tcp" + security_group_id = aws_security_group.masters-nthsqsresources-example-com.id + to_port = 443 + type = "ingress" +} + +resource "aws_security_group_rule" "from-masters-nthsqsresources-example-com-egress-all-0to0-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.masters-nthsqsresources-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-masters-nthsqsresources-example-com-ingress-all-0to0-masters-nthsqsresources-example-com" { + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.masters-nthsqsresources-example-com.id + source_security_group_id = aws_security_group.masters-nthsqsresources-example-com.id + to_port = 0 + type = "ingress" +} + +resource "aws_security_group_rule" "from-masters-nthsqsresources-example-com-ingress-all-0to0-nodes-nthsqsresources-example-com" { + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id + source_security_group_id = aws_security_group.masters-nthsqsresources-example-com.id + to_port = 0 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-nthsqsresources-example-com-egress-all-0to0-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-nodes-nthsqsresources-example-com-ingress-all-0to0-nodes-nthsqsresources-example-com" { + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id + source_security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id + to_port = 0 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-nthsqsresources-example-com-ingress-tcp-1to2379-masters-nthsqsresources-example-com" { + from_port = 1 + protocol = "tcp" + security_group_id = aws_security_group.masters-nthsqsresources-example-com.id + source_security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id + to_port = 2379 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-nthsqsresources-example-com-ingress-tcp-2382to4000-masters-nthsqsresources-example-com" { + from_port = 2382 + protocol = "tcp" + security_group_id = aws_security_group.masters-nthsqsresources-example-com.id + source_security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id + to_port = 4000 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-nthsqsresources-example-com-ingress-tcp-4003to65535-masters-nthsqsresources-example-com" { + from_port = 4003 + protocol = "tcp" + security_group_id = aws_security_group.masters-nthsqsresources-example-com.id + source_security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id + to_port = 65535 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-nthsqsresources-example-com-ingress-udp-1to65535-masters-nthsqsresources-example-com" { + from_port = 1 + protocol = "udp" + security_group_id = aws_security_group.masters-nthsqsresources-example-com.id + source_security_group_id = aws_security_group.nodes-nthsqsresources-example-com.id + to_port = 65535 + type = "ingress" +} + +resource "aws_security_group" "masters-nthsqsresources-example-com" { + description = "Security group for masters" + name = "masters.nthsqsresources.example.com" + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "masters.nthsqsresources.example.com" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } + vpc_id = aws_vpc.nthsqsresources-example-com.id +} + +resource "aws_security_group" "nodes-nthsqsresources-example-com" { + description = "Security group for nodes" + name = "nodes.nthsqsresources.example.com" + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "nodes.nthsqsresources.example.com" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } + vpc_id = aws_vpc.nthsqsresources-example-com.id +} + +resource "aws_sqs_queue" "nthsqsresources-example-com-nth" { + message_retention_seconds = 300 + name = "nthsqsresources-example-com-nth" + policy = file("${path.module}/data/aws_sqs_queue_nthsqsresources-example-com-nth_policy") + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "nthsqsresources-example-com-nth" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } +} + +resource "aws_subnet" "us-test-1a-nthsqsresources-example-com" { + availability_zone = "us-test-1a" + cidr_block = "172.20.32.0/19" + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "us-test-1a.nthsqsresources.example.com" + "SubnetType" = "Public" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + } + vpc_id = aws_vpc.nthsqsresources-example-com.id +} + +resource "aws_vpc_dhcp_options_association" "nthsqsresources-example-com" { + dhcp_options_id = aws_vpc_dhcp_options.nthsqsresources-example-com.id + vpc_id = aws_vpc.nthsqsresources-example-com.id +} + +resource "aws_vpc_dhcp_options" "nthsqsresources-example-com" { + domain_name = "us-test-1.compute.internal" + domain_name_servers = ["AmazonProvidedDNS"] + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "nthsqsresources.example.com" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } +} + +resource "aws_vpc" "nthsqsresources-example-com" { + cidr_block = "172.20.0.0/16" + enable_dns_hostnames = true + enable_dns_support = true + tags = { + "KubernetesCluster" = "nthsqsresources.example.com" + "Name" = "nthsqsresources.example.com" + "kubernetes.io/cluster/nthsqsresources.example.com" = "owned" + } +} + +terraform { + required_version = ">= 0.12.26" + required_providers { + aws = { + "source" = "hashicorp/aws" + "version" = ">= 3.34.0" + } + } +} diff --git a/upup/pkg/fi/cloudup/awstasks/autoscalinggroup_lifecyclehook.go b/upup/pkg/fi/cloudup/awstasks/autoscalinggroup_lifecyclehook.go index 304f92517f..ff69872716 100644 --- a/upup/pkg/fi/cloudup/awstasks/autoscalinggroup_lifecyclehook.go +++ b/upup/pkg/fi/cloudup/awstasks/autoscalinggroup_lifecyclehook.go @@ -37,8 +37,6 @@ type AutoscalingLifecycleHook struct { DefaultResult *string HeartbeatTimeout *int64 LifecycleTransition *string - - Tags map[string]string } var _ fi.CompareWithID = &AutoscalingLifecycleHook{} @@ -68,10 +66,13 @@ func (h *AutoscalingLifecycleHook) Find(c *fi.Context) (*AutoscalingLifecycleHoo hook := response.LifecycleHooks[0] actual := &AutoscalingLifecycleHook{ - ID: hook.AutoScalingGroupName, - Name: h.Name, - Lifecycle: h.Lifecycle, - AutoscalingGroup: h.AutoscalingGroup, + ID: hook.LifecycleHookName, + Name: hook.LifecycleHookName, + Lifecycle: h.Lifecycle, + AutoscalingGroup: h.AutoscalingGroup, + DefaultResult: hook.DefaultResult, + HeartbeatTimeout: hook.HeartbeatTimeout, + LifecycleTransition: hook.LifecycleTransition, } return actual, nil diff --git a/upup/pkg/fi/cloudup/awstasks/eventbridgerule.go b/upup/pkg/fi/cloudup/awstasks/eventbridgerule.go index abefce3c12..1f5f61e87b 100644 --- a/upup/pkg/fi/cloudup/awstasks/eventbridgerule.go +++ b/upup/pkg/fi/cloudup/awstasks/eventbridgerule.go @@ -36,7 +36,7 @@ type EventBridgeRule struct { Lifecycle *fi.Lifecycle EventPattern *string - TargetArn *string + TargetArn *string // required for cloudformation rendering Tags map[string]string } @@ -69,11 +69,19 @@ func (eb *EventBridgeRule) Find(c *fi.Context) (*EventBridgeRule, error) { } rule := response.Rules[0] + + tagResponse, err := cloud.EventBridge().ListTagsForResource(&eventbridge.ListTagsForResourceInput{ResourceARN: rule.Arn}) + if err != nil { + return nil, fmt.Errorf("error listing tags for EventBridge rule: %v", err) + } + actual := &EventBridgeRule{ ID: eb.ID, Name: eb.Name, Lifecycle: eb.Lifecycle, EventPattern: rule.EventPattern, + TargetArn: eb.TargetArn, + Tags: mapEventBridgeTagsToMap(tagResponse.Tags), } return actual, nil } diff --git a/upup/pkg/fi/cloudup/awstasks/sqs.go b/upup/pkg/fi/cloudup/awstasks/sqs.go index 7a062f4d32..3547a84ba0 100644 --- a/upup/pkg/fi/cloudup/awstasks/sqs.go +++ b/upup/pkg/fi/cloudup/awstasks/sqs.go @@ -56,35 +56,48 @@ func (q *SQS) Find(c *fi.Context) (*SQS, error) { return nil, nil } - request := &sqs.ListQueuesInput{ + response, err := cloud.SQS().ListQueues(&sqs.ListQueuesInput{ MaxResults: aws.Int64(2), QueueNamePrefix: q.Name, - } - response, err := cloud.SQS().ListQueues(request) + }) if err != nil { return nil, fmt.Errorf("error listing SQS queues: %v", err) } if response == nil || len(response.QueueUrls) == 0 { return nil, nil } - if len(response.QueueUrls) != 1 { return nil, fmt.Errorf("found multiple SQS queues matching queue name") } + url := response.QueueUrls[0] + + attributes, err := cloud.SQS().GetQueueAttributes(&sqs.GetQueueAttributesInput{ + AttributeNames: []*string{s("MessageRetentionPeriod"), s("Policy")}, + QueueUrl: url, + }) + if err != nil { + return nil, fmt.Errorf("error getting SQS queue attributes: %v", err) + } + policy := fi.NewStringResource(*attributes.Attributes["Policy"]) + period, err := strconv.Atoi(*attributes.Attributes["MessageRetentionPeriod"]) + if err != nil { + return nil, fmt.Errorf("error coverting MessageRetentionPeriod to int: %v", err) + } tags, err := cloud.SQS().ListQueueTags(&sqs.ListQueueTagsInput{ - QueueUrl: q.URL, + QueueUrl: url, }) if err != nil { return nil, fmt.Errorf("error listing SQS queue tags: %v", err) } actual := &SQS{ - Name: q.Name, - URL: response.QueueUrls[0], - Tags: intersectSQSTags(tags.Tags, q.Tags), - Lifecycle: q.Lifecycle, - Policy: q.Policy, + Name: q.Name, + URL: url, + Lifecycle: q.Lifecycle, + Policy: policy, + MessageRetentionPeriod: period, + Tags: intersectSQSTags(tags.Tags, q.Tags), } return actual, nil diff --git a/upup/pkg/fi/cloudup/awstasks/tags.go b/upup/pkg/fi/cloudup/awstasks/tags.go index 3627c92c01..e75d2c8826 100644 --- a/upup/pkg/fi/cloudup/awstasks/tags.go +++ b/upup/pkg/fi/cloudup/awstasks/tags.go @@ -21,6 +21,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/eventbridge" "github.com/aws/aws-sdk-go/service/iam" ) @@ -66,6 +67,20 @@ func mapToIAMTags(tags map[string]string) []*iam.Tag { return m } +func mapEventBridgeTagsToMap(tags []*eventbridge.Tag) map[string]string { + if tags == nil { + return nil + } + m := make(map[string]string) + for _, t := range tags { + if strings.HasPrefix(aws.StringValue(t.Key), "aws:cloudformation:") { + continue + } + m[aws.StringValue(t.Key)] = aws.StringValue(t.Value) + } + return m +} + func findNameTag(tags []*ec2.Tag) *string { for _, tag := range tags { if aws.StringValue(tag.Key) == "Name" { diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 6908381848..e76ec74560 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -216,7 +216,7 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS } dest["UseServiceAccountIAM"] = tf.UseServiceAccountIAM - + if cluster.Spec.NodeTerminationHandler != nil { dest["DefaultQueueName"] = func() string { s := strings.Replace(tf.ClusterName(), ".", "-", -1) diff --git a/vendor/github.com/aws/aws-sdk-go/service/eventbridge/api.go b/vendor/github.com/aws/aws-sdk-go/service/eventbridge/api.go index a2359f3be7..e83771867b 100644 --- a/vendor/github.com/aws/aws-sdk-go/service/eventbridge/api.go +++ b/vendor/github.com/aws/aws-sdk-go/service/eventbridge/api.go @@ -3799,6 +3799,8 @@ func (c *EventBridge) PutTargetsRequest(input *PutTargetsInput) (req *request.Re // // * Custom/SaaS HTTPS APIs via EventBridge API Destinations // +// * Amazon SageMaker Model Building Pipelines +// // Creating rules with built-in targets is supported only in the AWS Management // Console. The built-in targets are EC2 CreateSnapshot API call, EC2 RebootInstances // API call, EC2 StopInstances API call, and EC2 TerminateInstances API call. @@ -12619,6 +12621,109 @@ func (s *RunCommandTarget) SetValues(v []*string) *RunCommandTarget { return s } +// Name/Value pair of a parameter to start execution of a SageMaker Model Building +// Pipeline. +type SageMakerPipelineParameter struct { + _ struct{} `type:"structure"` + + // Name of parameter to start execution of a SageMaker Model Building Pipeline. + // + // Name is a required field + Name *string `min:"1" type:"string" required:"true"` + + // Value of parameter to start execution of a SageMaker Model Building Pipeline. + // + // Value is a required field + Value *string `type:"string" required:"true"` +} + +// String returns the string representation +func (s SageMakerPipelineParameter) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s SageMakerPipelineParameter) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *SageMakerPipelineParameter) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "SageMakerPipelineParameter"} + if s.Name == nil { + invalidParams.Add(request.NewErrParamRequired("Name")) + } + if s.Name != nil && len(*s.Name) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Name", 1)) + } + if s.Value == nil { + invalidParams.Add(request.NewErrParamRequired("Value")) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetName sets the Name field's value. +func (s *SageMakerPipelineParameter) SetName(v string) *SageMakerPipelineParameter { + s.Name = &v + return s +} + +// SetValue sets the Value field's value. +func (s *SageMakerPipelineParameter) SetValue(v string) *SageMakerPipelineParameter { + s.Value = &v + return s +} + +// These are custom parameters to use when the target is a SageMaker Model Building +// Pipeline that starts based on EventBridge events. +type SageMakerPipelineParameters struct { + _ struct{} `type:"structure"` + + // List of Parameter names and values for SageMaker Model Building Pipeline + // execution. + PipelineParameterList []*SageMakerPipelineParameter `type:"list"` +} + +// String returns the string representation +func (s SageMakerPipelineParameters) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s SageMakerPipelineParameters) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *SageMakerPipelineParameters) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "SageMakerPipelineParameters"} + if s.PipelineParameterList != nil { + for i, v := range s.PipelineParameterList { + if v == nil { + continue + } + if err := v.Validate(); err != nil { + invalidParams.AddNested(fmt.Sprintf("%s[%v]", "PipelineParameterList", i), err.(request.ErrInvalidParams)) + } + } + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetPipelineParameterList sets the PipelineParameterList field's value. +func (s *SageMakerPipelineParameters) SetPipelineParameterList(v []*SageMakerPipelineParameter) *SageMakerPipelineParameters { + s.PipelineParameterList = v + return s +} + // This structure includes the custom parameter to be used when the target is // an SQS FIFO queue. type SqsParameters struct { @@ -13036,6 +13141,14 @@ type Target struct { // Parameters used when you are using the rule to invoke Amazon EC2 Run Command. RunCommandParameters *RunCommandParameters `type:"structure"` + // Contains the SageMaker Model Building Pipeline parameters to start execution + // of a SageMaker Model Building Pipeline. + // + // If you specify a SageMaker Model Building Pipeline as a target, you can use + // this to specify parameters to start a pipeline execution based on EventBridge + // events. + SageMakerPipelineParameters *SageMakerPipelineParameters `type:"structure"` + // Contains the message group ID to use when the target is a FIFO queue. // // If you specify an SQS FIFO queue as a target, the queue must have content-based @@ -13111,6 +13224,11 @@ func (s *Target) Validate() error { invalidParams.AddNested("RunCommandParameters", err.(request.ErrInvalidParams)) } } + if s.SageMakerPipelineParameters != nil { + if err := s.SageMakerPipelineParameters.Validate(); err != nil { + invalidParams.AddNested("SageMakerPipelineParameters", err.(request.ErrInvalidParams)) + } + } if invalidParams.Len() > 0 { return invalidParams @@ -13202,6 +13320,12 @@ func (s *Target) SetRunCommandParameters(v *RunCommandParameters) *Target { return s } +// SetSageMakerPipelineParameters sets the SageMakerPipelineParameters field's value. +func (s *Target) SetSageMakerPipelineParameters(v *SageMakerPipelineParameters) *Target { + s.SageMakerPipelineParameters = v + return s +} + // SetSqsParameters sets the SqsParameters field's value. func (s *Target) SetSqsParameters(v *SqsParameters) *Target { s.SqsParameters = v diff --git a/vendor/github.com/aws/aws-sdk-go/service/sqs/api.go b/vendor/github.com/aws/aws-sdk-go/service/sqs/api.go index e1885a8f2c..38ef4185f4 100644 --- a/vendor/github.com/aws/aws-sdk-go/service/sqs/api.go +++ b/vendor/github.com/aws/aws-sdk-go/service/sqs/api.go @@ -4617,9 +4617,9 @@ type SendMessageBatchResultEntry struct { // about MD5, see RFC1321 (https://www.ietf.org/rfc/rfc1321.txt). MD5OfMessageAttributes *string `type:"string"` - // An MD5 digest of the non-URL-encoded message attribute string. You can use - // this attribute to verify that Amazon SQS received the message correctly. - // Amazon SQS URL-decodes the message before creating the MD5 digest. For information + // An MD5 digest of the non-URL-encoded message body string. You can use this + // attribute to verify that Amazon SQS received the message correctly. Amazon + // SQS URL-decodes the message before creating the MD5 digest. For information // about MD5, see RFC1321 (https://www.ietf.org/rfc/rfc1321.txt). // // MD5OfMessageBody is a required field @@ -4912,9 +4912,9 @@ type SendMessageOutput struct { // about MD5, see RFC1321 (https://www.ietf.org/rfc/rfc1321.txt). MD5OfMessageAttributes *string `type:"string"` - // An MD5 digest of the non-URL-encoded message attribute string. You can use - // this attribute to verify that Amazon SQS received the message correctly. - // Amazon SQS URL-decodes the message before creating the MD5 digest. For information + // An MD5 digest of the non-URL-encoded message body string. You can use this + // attribute to verify that Amazon SQS received the message correctly. Amazon + // SQS URL-decodes the message before creating the MD5 digest. For information // about MD5, see RFC1321 (https://www.ietf.org/rfc/rfc1321.txt). MD5OfMessageBody *string `type:"string"` diff --git a/vendor/github.com/aws/aws-sdk-go/service/sqs/doc.go b/vendor/github.com/aws/aws-sdk-go/service/sqs/doc.go index 523b40ee38..854208bcc6 100644 --- a/vendor/github.com/aws/aws-sdk-go/service/sqs/doc.go +++ b/vendor/github.com/aws/aws-sdk-go/service/sqs/doc.go @@ -24,7 +24,7 @@ // // * Handle error responses // -// Additional Information +// Additional information // // * Amazon SQS Product Page (http://aws.amazon.com/sqs/) //