From 75ccf45eb73aae69c348ed710a20665dd732c0af Mon Sep 17 00:00:00 2001 From: Peter Rifel Date: Thu, 11 Jun 2020 16:36:10 -0500 Subject: [PATCH] Fold multiple integration test cases into the complex cluster test Each integration test cluster adds many LoC and some overhead in running the integration tests. Since many of the tests are only testing a specific feature, it would be simpler to combine all of the non-mutually exclusive features into the complex cluster. --- cmd/kops/integration_test.go | 32 +- .../additional_cidr/cloudformation.json | 1055 ----------------- .../cloudformation.json.extracted.yaml | 537 --------- ..._masters.additionalcidr.example.com_policy | 10 - ...le_nodes.additionalcidr.example.com_policy | 10 - ..._masters.additionalcidr.example.com_policy | 102 -- ...cy_nodes.additionalcidr.example.com_policy | 68 -- ...4a6ed9aa889b9e2c39cd663eb9c7157_public_key | 1 - ...sters.additionalcidr.example.com_user_data | 314 ----- ...sters.additionalcidr.example.com_user_data | 314 ----- ...sters.additionalcidr.example.com_user_data | 314 ----- ...nodes.additionalcidr.example.com_user_data | 221 ---- .../update_cluster/additional_cidr/id_rsa.pub | 1 - .../additional_cidr/in-v1alpha2.yaml | 85 -- .../additional_cidr/in-v1alpha3.yaml | 133 --- .../additional_cidr/kubernetes.tf | 808 ------------- .../additional_user-data/cloudformation.json | 1028 ---------------- .../cloudformation.json.extracted.yaml | 577 --------- .../additional_user-data/id_rsa.pub | 1 - .../additional_user-data/in-v1alpha2.yaml | 99 -- ..._role_masters.crosszone.example.com_policy | 10 - ...am_role_nodes.crosszone.example.com_policy | 10 - ...olicy_masters.crosszone.example.com_policy | 102 -- ..._policy_nodes.crosszone.example.com_policy | 68 -- ...4a6ed9aa889b9e2c39cd663eb9c7157_public_key | 1 - ...1a.masters.crosszone.example.com_user_data | 315 ----- ...late_nodes.crosszone.example.com_user_data | 221 ---- .../api_elb_cross_zone/id_rsa.pub | 1 - .../api_elb_cross_zone/in-v1alpha2.yaml | 98 -- .../api_elb_cross_zone/kubernetes.tf | 691 ----------- .../complex/cloudformation.json | 77 +- .../cloudformation.json.extracted.yaml | 40 + ...t-1a.masters.complex.example.com_user_data | 20 + ...mplate_nodes.complex.example.com_user_data | 20 + .../complex/in-legacy-v1alpha2.yaml | 22 +- .../update_cluster/complex/in-v1alpha2.yaml | 22 +- .../update_cluster/complex/kubernetes.tf | 66 +- .../cloudformation.json | 1001 ---------------- .../cloudformation.json.extracted.yaml | 541 --------- .../nosshkey-cloudformation/in-v1alpha2.yaml | 80 -- ...m_role_masters.nosshkey.example.com_policy | 10 - ...iam_role_nodes.nosshkey.example.com_policy | 10 - ...policy_masters.nosshkey.example.com_policy | 102 -- ...e_policy_nodes.nosshkey.example.com_policy | 68 -- ...-1a.masters.nosshkey.example.com_user_data | 318 ----- ...plate_nodes.nosshkey.example.com_user_data | 222 ---- .../update_cluster/nosshkey/in-v1alpha2.yaml | 98 -- .../update_cluster/nosshkey/kubernetes.tf | 684 ----------- ..._masters.restrictaccess.example.com_policy | 10 - ...le_nodes.restrictaccess.example.com_policy | 10 - ..._masters.restrictaccess.example.com_policy | 102 -- ...cy_nodes.restrictaccess.example.com_policy | 68 -- ...4a6ed9aa889b9e2c39cd663eb9c7157_public_key | 1 - ...sters.restrictaccess.example.com_user_data | 314 ----- ...nodes.restrictaccess.example.com_user_data | 221 ---- .../update_cluster/restrict_access/id_rsa.pub | 1 - .../restrict_access/in-v1alpha2.yaml | 81 -- .../restrict_access/kubernetes.tf | 550 --------- 58 files changed, 239 insertions(+), 11747 deletions(-) delete mode 100644 tests/integration/update_cluster/additional_cidr/cloudformation.json delete mode 100644 tests/integration/update_cluster/additional_cidr/cloudformation.json.extracted.yaml delete mode 100644 tests/integration/update_cluster/additional_cidr/data/aws_iam_role_masters.additionalcidr.example.com_policy delete mode 100644 tests/integration/update_cluster/additional_cidr/data/aws_iam_role_nodes.additionalcidr.example.com_policy delete mode 100644 tests/integration/update_cluster/additional_cidr/data/aws_iam_role_policy_masters.additionalcidr.example.com_policy delete mode 100644 tests/integration/update_cluster/additional_cidr/data/aws_iam_role_policy_nodes.additionalcidr.example.com_policy delete mode 100644 tests/integration/update_cluster/additional_cidr/data/aws_key_pair_kubernetes.additionalcidr.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key delete mode 100644 tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1a.masters.additionalcidr.example.com_user_data delete mode 100644 tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1b.masters.additionalcidr.example.com_user_data delete mode 100644 tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1c.masters.additionalcidr.example.com_user_data delete mode 100644 tests/integration/update_cluster/additional_cidr/data/aws_launch_template_nodes.additionalcidr.example.com_user_data delete mode 100755 tests/integration/update_cluster/additional_cidr/id_rsa.pub delete mode 100644 tests/integration/update_cluster/additional_cidr/in-v1alpha2.yaml delete mode 100644 tests/integration/update_cluster/additional_cidr/in-v1alpha3.yaml delete mode 100644 tests/integration/update_cluster/additional_cidr/kubernetes.tf delete mode 100644 tests/integration/update_cluster/additional_user-data/cloudformation.json delete mode 100644 tests/integration/update_cluster/additional_user-data/cloudformation.json.extracted.yaml delete mode 100755 tests/integration/update_cluster/additional_user-data/id_rsa.pub delete mode 100644 tests/integration/update_cluster/additional_user-data/in-v1alpha2.yaml delete mode 100644 tests/integration/update_cluster/api_elb_cross_zone/data/aws_iam_role_masters.crosszone.example.com_policy delete mode 100644 tests/integration/update_cluster/api_elb_cross_zone/data/aws_iam_role_nodes.crosszone.example.com_policy delete mode 100644 tests/integration/update_cluster/api_elb_cross_zone/data/aws_iam_role_policy_masters.crosszone.example.com_policy delete mode 100644 tests/integration/update_cluster/api_elb_cross_zone/data/aws_iam_role_policy_nodes.crosszone.example.com_policy delete mode 100644 tests/integration/update_cluster/api_elb_cross_zone/data/aws_key_pair_kubernetes.crosszone.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key delete mode 100644 tests/integration/update_cluster/api_elb_cross_zone/data/aws_launch_template_master-us-test-1a.masters.crosszone.example.com_user_data delete mode 100644 tests/integration/update_cluster/api_elb_cross_zone/data/aws_launch_template_nodes.crosszone.example.com_user_data delete mode 100755 tests/integration/update_cluster/api_elb_cross_zone/id_rsa.pub delete mode 100644 tests/integration/update_cluster/api_elb_cross_zone/in-v1alpha2.yaml delete mode 100644 tests/integration/update_cluster/api_elb_cross_zone/kubernetes.tf delete mode 100644 tests/integration/update_cluster/nosshkey-cloudformation/cloudformation.json delete mode 100644 tests/integration/update_cluster/nosshkey-cloudformation/cloudformation.json.extracted.yaml delete mode 100644 tests/integration/update_cluster/nosshkey-cloudformation/in-v1alpha2.yaml delete mode 100644 tests/integration/update_cluster/nosshkey/data/aws_iam_role_masters.nosshkey.example.com_policy delete mode 100644 tests/integration/update_cluster/nosshkey/data/aws_iam_role_nodes.nosshkey.example.com_policy delete mode 100644 tests/integration/update_cluster/nosshkey/data/aws_iam_role_policy_masters.nosshkey.example.com_policy delete mode 100644 tests/integration/update_cluster/nosshkey/data/aws_iam_role_policy_nodes.nosshkey.example.com_policy delete mode 100644 tests/integration/update_cluster/nosshkey/data/aws_launch_template_master-us-test-1a.masters.nosshkey.example.com_user_data delete mode 100644 tests/integration/update_cluster/nosshkey/data/aws_launch_template_nodes.nosshkey.example.com_user_data delete mode 100644 tests/integration/update_cluster/nosshkey/in-v1alpha2.yaml delete mode 100644 tests/integration/update_cluster/nosshkey/kubernetes.tf delete mode 100644 tests/integration/update_cluster/restrict_access/data/aws_iam_role_masters.restrictaccess.example.com_policy delete mode 100644 tests/integration/update_cluster/restrict_access/data/aws_iam_role_nodes.restrictaccess.example.com_policy delete mode 100644 tests/integration/update_cluster/restrict_access/data/aws_iam_role_policy_masters.restrictaccess.example.com_policy delete mode 100644 tests/integration/update_cluster/restrict_access/data/aws_iam_role_policy_nodes.restrictaccess.example.com_policy delete mode 100644 tests/integration/update_cluster/restrict_access/data/aws_key_pair_kubernetes.restrictaccess.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key delete mode 100644 tests/integration/update_cluster/restrict_access/data/aws_launch_template_master-us-test-1a.masters.restrictaccess.example.com_user_data delete mode 100644 tests/integration/update_cluster/restrict_access/data/aws_launch_template_nodes.restrictaccess.example.com_user_data delete mode 100755 tests/integration/update_cluster/restrict_access/id_rsa.pub delete mode 100644 tests/integration/update_cluster/restrict_access/in-v1alpha2.yaml delete mode 100644 tests/integration/update_cluster/restrict_access/kubernetes.tf diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index 008b5396a5..257a40ae98 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -131,11 +131,6 @@ func TestMinimalGCE(t *testing.T) { newIntegrationTest("minimal-gce.example.com", "minimal_gce").runTestTerraformGCE(t) } -// TestRestrictAccess runs the test on a simple SG configuration, similar to kops create cluster minimal.example.com --ssh-access=$(IPS) --admin-access=$(IPS) --master-count=3 -func TestRestrictAccess(t *testing.T) { - newIntegrationTest("restrictaccess.example.com", "restrict_access").runTestTerraformAWS(t) -} - // TestHA runs the test on a simple HA configuration, similar to kops create cluster minimal.example.com --zones us-west-1a,us-west-1b,us-west-1c --master-count=3 func TestHA(t *testing.T) { newIntegrationTest("ha.example.com", "ha").withZones(3).runTestTerraformAWS(t) @@ -149,9 +144,9 @@ func TestHighAvailabilityGCE(t *testing.T) { // TestComplex runs the test on a more complex configuration, intended to hit more of the edge cases func TestComplex(t *testing.T) { - newIntegrationTest("complex.example.com", "complex").runTestTerraformAWS(t) - newIntegrationTest("complex.example.com", "complex").runTestCloudformation(t) - newIntegrationTest("complex.example.com", "complex").withVersion("legacy-v1alpha2").runTestTerraformAWS(t) + newIntegrationTest("complex.example.com", "complex").withoutSSHKey().runTestTerraformAWS(t) + newIntegrationTest("complex.example.com", "complex").withoutSSHKey().runTestCloudformation(t) + newIntegrationTest("complex.example.com", "complex").withoutSSHKey().withVersion("legacy-v1alpha2").runTestTerraformAWS(t) } // TestExternalPolicies tests external policies output @@ -159,16 +154,6 @@ func TestExternalPolicies(t *testing.T) { newIntegrationTest("externalpolicies.example.com", "externalpolicies").runTestTerraformAWS(t) } -func TestNoSSHKey(t *testing.T) { - newIntegrationTest("nosshkey.example.com", "nosshkey").withoutSSHKey().runTestTerraformAWS(t) - newIntegrationTest("nosshkey.example.com", "nosshkey-cloudformation").withoutSSHKey().runTestCloudformation(t) -} - -// TestCrossZone tests that the cross zone setting on the API ELB is set properly -func TestCrossZone(t *testing.T) { - newIntegrationTest("crosszone.example.com", "api_elb_cross_zone").runTestTerraformAWS(t) -} - // TestMinimalCloudformation runs the test on a minimum configuration, similar to kops create cluster minimal.example.com --zones us-west-1a func TestMinimalCloudformation(t *testing.T) { newIntegrationTest("minimal.example.com", "minimal-cloudformation").runTestCloudformation(t) @@ -185,11 +170,6 @@ func TestExistingSG(t *testing.T) { newIntegrationTest("existingsg.example.com", "existing_sg").withZones(3).runTestTerraformAWS(t) } -// TestAdditionalUserData runs the test on passing additional user-data to an instance at bootstrap. -func TestAdditionalUserData(t *testing.T) { - newIntegrationTest("additionaluserdata.example.com", "additional_user-data").runTestCloudformation(t) -} - // TestBastionAdditionalUserData runs the test on passing additional user-data to a bastion instance group func TestBastionAdditionalUserData(t *testing.T) { newIntegrationTest("bastionuserdata.example.com", "bastionadditional_user-data").withPrivate().withBastionUserData().runTestTerraformAWS(t) @@ -286,12 +266,6 @@ func TestExistingIAM(t *testing.T) { newIntegrationTest("existing-iam.example.com", "existing_iam").withZones(3).withoutPolicies().withLifecycleOverrides(lifecycleOverrides).runTestTerraformAWS(t) } -// TestAdditionalCIDR runs the test on a configuration with a shared VPC -func TestAdditionalCIDR(t *testing.T) { - newIntegrationTest("additionalcidr.example.com", "additional_cidr").withVersion("v1alpha3").withZones(3).runTestTerraformAWS(t) - newIntegrationTest("additionalcidr.example.com", "additional_cidr").runTestCloudformation(t) -} - // TestPhaseNetwork tests the output of tf for the network phase func TestPhaseNetwork(t *testing.T) { newIntegrationTest("lifecyclephases.example.com", "lifecycle_phases").runTestPhase(t, cloudup.PhaseNetwork) diff --git a/tests/integration/update_cluster/additional_cidr/cloudformation.json b/tests/integration/update_cluster/additional_cidr/cloudformation.json deleted file mode 100644 index d6a81b0c34..0000000000 --- a/tests/integration/update_cluster/additional_cidr/cloudformation.json +++ /dev/null @@ -1,1055 +0,0 @@ -{ - "Resources": { - "AWSAutoScalingAutoScalingGroupmasterustest1bmastersadditionalcidrexamplecom": { - "Type": "AWS::AutoScaling::AutoScalingGroup", - "Properties": { - "AutoScalingGroupName": "master-us-test-1b.masters.additionalcidr.example.com", - "LaunchTemplate": { - "LaunchTemplateId": { - "Ref": "AWSEC2LaunchTemplatemasterustest1bmastersadditionalcidrexamplecom" - }, - "Version": { - "Fn::GetAtt": [ - "AWSEC2LaunchTemplatemasterustest1bmastersadditionalcidrexamplecom", - "LatestVersionNumber" - ] - } - }, - "MaxSize": 1, - "MinSize": 1, - "VPCZoneIdentifier": [ - { - "Ref": "AWSEC2Subnetustest1badditionalcidrexamplecom" - } - ], - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionalcidr.example.com", - "PropagateAtLaunch": true - }, - { - "Key": "Name", - "Value": "master-us-test-1b.masters.additionalcidr.example.com", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/role/master", - "Value": "1", - "PropagateAtLaunch": true - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "master-us-test-1b", - "PropagateAtLaunch": true - }, - { - "Key": "kubernetes.io/cluster/additionalcidr.example.com", - "Value": "owned", - "PropagateAtLaunch": true - } - ], - "MetricsCollection": [ - { - "Granularity": "1Minute", - "Metrics": [ - "GroupDesiredCapacity", - "GroupInServiceInstances", - "GroupMaxSize", - "GroupMinSize", - "GroupPendingInstances", - "GroupStandbyInstances", - "GroupTerminatingInstances", - "GroupTotalInstances" - ] - } - ] - } - }, - "AWSAutoScalingAutoScalingGroupnodesadditionalcidrexamplecom": { - "Type": "AWS::AutoScaling::AutoScalingGroup", - "Properties": { - "AutoScalingGroupName": "nodes.additionalcidr.example.com", - "LaunchTemplate": { - "LaunchTemplateId": { - "Ref": "AWSEC2LaunchTemplatenodesadditionalcidrexamplecom" - }, - "Version": { - "Fn::GetAtt": [ - "AWSEC2LaunchTemplatenodesadditionalcidrexamplecom", - "LatestVersionNumber" - ] - } - }, - "MaxSize": 2, - "MinSize": 2, - "VPCZoneIdentifier": [ - { - "Ref": "AWSEC2Subnetustest1badditionalcidrexamplecom" - } - ], - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionalcidr.example.com", - "PropagateAtLaunch": true - }, - { - "Key": "Name", - "Value": "nodes.additionalcidr.example.com", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/role/node", - "Value": "1", - "PropagateAtLaunch": true - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "nodes", - "PropagateAtLaunch": true - }, - { - "Key": "kubernetes.io/cluster/additionalcidr.example.com", - "Value": "owned", - "PropagateAtLaunch": true - } - ], - "MetricsCollection": [ - { - "Granularity": "1Minute", - "Metrics": [ - "GroupDesiredCapacity", - "GroupInServiceInstances", - "GroupMaxSize", - "GroupMinSize", - "GroupPendingInstances", - "GroupStandbyInstances", - "GroupTerminatingInstances", - "GroupTotalInstances" - ] - } - ] - } - }, - "AWSEC2DHCPOptionsadditionalcidrexamplecom": { - "Type": "AWS::EC2::DHCPOptions", - "Properties": { - "DomainName": "us-test-1.compute.internal", - "DomainNameServers": [ - "AmazonProvidedDNS" - ], - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionalcidr.example.com" - }, - { - "Key": "Name", - "Value": "additionalcidr.example.com" - }, - { - "Key": "kubernetes.io/cluster/additionalcidr.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2InternetGatewayadditionalcidrexamplecom": { - "Type": "AWS::EC2::InternetGateway", - "Properties": { - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionalcidr.example.com" - }, - { - "Key": "Name", - "Value": "additionalcidr.example.com" - }, - { - "Key": "kubernetes.io/cluster/additionalcidr.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2LaunchTemplatemasterustest1bmastersadditionalcidrexamplecom": { - "Type": "AWS::EC2::LaunchTemplate", - "Properties": { - "LaunchTemplateName": "master-us-test-1b.masters.additionalcidr.example.com", - "LaunchTemplateData": { - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/xvda", - "Ebs": { - "VolumeType": "gp2", - "VolumeSize": 64, - "DeleteOnTermination": true - } - }, - { - "DeviceName": "/dev/sdc", - "VirtualName": "ephemeral0" - } - ], - "IamInstanceProfile": { - "Name": { - "Ref": "AWSIAMInstanceProfilemastersadditionalcidrexamplecom" - } - }, - "ImageId": "ami-12345678", - "InstanceType": "m3.medium", - "KeyName": "kubernetes.additionalcidr.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", - "NetworkInterfaces": [ - { - "AssociatePublicIpAddress": true, - "DeleteOnTermination": true, - "DeviceIndex": 0, - "Groups": [ - { - "Ref": "AWSEC2SecurityGroupmastersadditionalcidrexamplecom" - } - ] - } - ], - "TagSpecifications": [ - { - "ResourceType": "instance", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionalcidr.example.com" - }, - { - "Key": "Name", - "Value": "master-us-test-1b.masters.additionalcidr.example.com" - }, - { - "Key": "k8s.io/role/master", - "Value": "1" - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "master-us-test-1b" - }, - { - "Key": "kubernetes.io/cluster/additionalcidr.example.com", - "Value": "owned" - } - ] - }, - { - "ResourceType": "volume", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionalcidr.example.com" - }, - { - "Key": "Name", - "Value": "master-us-test-1b.masters.additionalcidr.example.com" - }, - { - "Key": "k8s.io/role/master", - "Value": "1" - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "master-us-test-1b" - }, - { - "Key": "kubernetes.io/cluster/additionalcidr.example.com", - "Value": "owned" - } - ] - } - ], - "UserData": "extracted" - } - } - }, - "AWSEC2LaunchTemplatenodesadditionalcidrexamplecom": { - "Type": "AWS::EC2::LaunchTemplate", - "Properties": { - "LaunchTemplateName": "nodes.additionalcidr.example.com", - "LaunchTemplateData": { - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/xvda", - "Ebs": { - "VolumeType": "gp2", - "VolumeSize": 128, - "DeleteOnTermination": true - } - } - ], - "IamInstanceProfile": { - "Name": { - "Ref": "AWSIAMInstanceProfilenodesadditionalcidrexamplecom" - } - }, - "ImageId": "ami-12345678", - "InstanceType": "t2.medium", - "KeyName": "kubernetes.additionalcidr.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", - "NetworkInterfaces": [ - { - "AssociatePublicIpAddress": true, - "DeleteOnTermination": true, - "DeviceIndex": 0, - "Groups": [ - { - "Ref": "AWSEC2SecurityGroupnodesadditionalcidrexamplecom" - } - ] - } - ], - "TagSpecifications": [ - { - "ResourceType": "instance", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionalcidr.example.com" - }, - { - "Key": "Name", - "Value": "nodes.additionalcidr.example.com" - }, - { - "Key": "k8s.io/role/node", - "Value": "1" - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "nodes" - }, - { - "Key": "kubernetes.io/cluster/additionalcidr.example.com", - "Value": "owned" - } - ] - }, - { - "ResourceType": "volume", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionalcidr.example.com" - }, - { - "Key": "Name", - "Value": "nodes.additionalcidr.example.com" - }, - { - "Key": "k8s.io/role/node", - "Value": "1" - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "nodes" - }, - { - "Key": "kubernetes.io/cluster/additionalcidr.example.com", - "Value": "owned" - } - ] - } - ], - "UserData": "extracted" - } - } - }, - "AWSEC2Route00000": { - "Type": "AWS::EC2::Route", - "Properties": { - "RouteTableId": { - "Ref": "AWSEC2RouteTableadditionalcidrexamplecom" - }, - "DestinationCidrBlock": "0.0.0.0/0", - "GatewayId": { - "Ref": "AWSEC2InternetGatewayadditionalcidrexamplecom" - } - } - }, - "AWSEC2RouteTableadditionalcidrexamplecom": { - "Type": "AWS::EC2::RouteTable", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPCadditionalcidrexamplecom" - }, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionalcidr.example.com" - }, - { - "Key": "Name", - "Value": "additionalcidr.example.com" - }, - { - "Key": "kubernetes.io/cluster/additionalcidr.example.com", - "Value": "owned" - }, - { - "Key": "kubernetes.io/kops/role", - "Value": "public" - } - ] - } - }, - "AWSEC2SecurityGroupEgressmasteregress": { - "Type": "AWS::EC2::SecurityGroupEgress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersadditionalcidrexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1", - "CidrIp": "0.0.0.0/0" - } - }, - "AWSEC2SecurityGroupEgressnodeegress": { - "Type": "AWS::EC2::SecurityGroupEgress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupnodesadditionalcidrexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1", - "CidrIp": "0.0.0.0/0" - } - }, - "AWSEC2SecurityGroupIngressallmastertomaster": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersadditionalcidrexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupmastersadditionalcidrexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1" - } - }, - "AWSEC2SecurityGroupIngressallmastertonode": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupnodesadditionalcidrexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupmastersadditionalcidrexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1" - } - }, - "AWSEC2SecurityGroupIngressallnodetonode": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupnodesadditionalcidrexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodesadditionalcidrexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1" - } - }, - "AWSEC2SecurityGroupIngresshttpsexternaltomaster00000": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersadditionalcidrexamplecom" - }, - "FromPort": 443, - "ToPort": 443, - "IpProtocol": "tcp", - "CidrIp": "0.0.0.0/0" - } - }, - "AWSEC2SecurityGroupIngressnodetomastertcp12379": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersadditionalcidrexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodesadditionalcidrexamplecom" - }, - "FromPort": 1, - "ToPort": 2379, - "IpProtocol": "tcp" - } - }, - "AWSEC2SecurityGroupIngressnodetomastertcp23824000": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersadditionalcidrexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodesadditionalcidrexamplecom" - }, - "FromPort": 2382, - "ToPort": 4000, - "IpProtocol": "tcp" - } - }, - "AWSEC2SecurityGroupIngressnodetomastertcp400365535": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersadditionalcidrexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodesadditionalcidrexamplecom" - }, - "FromPort": 4003, - "ToPort": 65535, - "IpProtocol": "tcp" - } - }, - "AWSEC2SecurityGroupIngressnodetomasterudp165535": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersadditionalcidrexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodesadditionalcidrexamplecom" - }, - "FromPort": 1, - "ToPort": 65535, - "IpProtocol": "udp" - } - }, - "AWSEC2SecurityGroupIngresssshexternaltomaster00000": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersadditionalcidrexamplecom" - }, - "FromPort": 22, - "ToPort": 22, - "IpProtocol": "tcp", - "CidrIp": "0.0.0.0/0" - } - }, - "AWSEC2SecurityGroupIngresssshexternaltonode00000": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupnodesadditionalcidrexamplecom" - }, - "FromPort": 22, - "ToPort": 22, - "IpProtocol": "tcp", - "CidrIp": "0.0.0.0/0" - } - }, - "AWSEC2SecurityGroupmastersadditionalcidrexamplecom": { - "Type": "AWS::EC2::SecurityGroup", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPCadditionalcidrexamplecom" - }, - "GroupDescription": "Security group for masters", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionalcidr.example.com" - }, - { - "Key": "Name", - "Value": "masters.additionalcidr.example.com" - }, - { - "Key": "kubernetes.io/cluster/additionalcidr.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2SecurityGroupnodesadditionalcidrexamplecom": { - "Type": "AWS::EC2::SecurityGroup", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPCadditionalcidrexamplecom" - }, - "GroupDescription": "Security group for nodes", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionalcidr.example.com" - }, - { - "Key": "Name", - "Value": "nodes.additionalcidr.example.com" - }, - { - "Key": "kubernetes.io/cluster/additionalcidr.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2SubnetRouteTableAssociationustest1aadditionalcidrexamplecom": { - "Type": "AWS::EC2::SubnetRouteTableAssociation", - "Properties": { - "SubnetId": { - "Ref": "AWSEC2Subnetustest1aadditionalcidrexamplecom" - }, - "RouteTableId": { - "Ref": "AWSEC2RouteTableadditionalcidrexamplecom" - } - } - }, - "AWSEC2SubnetRouteTableAssociationustest1badditionalcidrexamplecom": { - "Type": "AWS::EC2::SubnetRouteTableAssociation", - "Properties": { - "SubnetId": { - "Ref": "AWSEC2Subnetustest1badditionalcidrexamplecom" - }, - "RouteTableId": { - "Ref": "AWSEC2RouteTableadditionalcidrexamplecom" - } - } - }, - "AWSEC2Subnetustest1aadditionalcidrexamplecom": { - "Type": "AWS::EC2::Subnet", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPCadditionalcidrexamplecom" - }, - "CidrBlock": "10.0.1.0/24", - "AvailabilityZone": "us-test-1a", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionalcidr.example.com" - }, - { - "Key": "Name", - "Value": "us-test-1a.additionalcidr.example.com" - }, - { - "Key": "SubnetType", - "Value": "Public" - }, - { - "Key": "kubernetes.io/cluster/additionalcidr.example.com", - "Value": "owned" - }, - { - "Key": "kubernetes.io/role/elb", - "Value": "1" - } - ] - } - }, - "AWSEC2Subnetustest1badditionalcidrexamplecom": { - "Type": "AWS::EC2::Subnet", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPCadditionalcidrexamplecom" - }, - "CidrBlock": "10.1.1.0/24", - "AvailabilityZone": "us-test-1b", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionalcidr.example.com" - }, - { - "Key": "Name", - "Value": "us-test-1b.additionalcidr.example.com" - }, - { - "Key": "SubnetType", - "Value": "Public" - }, - { - "Key": "kubernetes.io/cluster/additionalcidr.example.com", - "Value": "owned" - }, - { - "Key": "kubernetes.io/role/elb", - "Value": "1" - } - ] - } - }, - "AWSEC2VPCCidrBlock1010016": { - "Type": "AWS::EC2::VPCCidrBlock", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPCadditionalcidrexamplecom" - }, - "CidrBlock": "10.1.0.0/16" - } - }, - "AWSEC2VPCDHCPOptionsAssociationadditionalcidrexamplecom": { - "Type": "AWS::EC2::VPCDHCPOptionsAssociation", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPCadditionalcidrexamplecom" - }, - "DhcpOptionsId": { - "Ref": "AWSEC2DHCPOptionsadditionalcidrexamplecom" - } - } - }, - "AWSEC2VPCGatewayAttachmentadditionalcidrexamplecom": { - "Type": "AWS::EC2::VPCGatewayAttachment", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPCadditionalcidrexamplecom" - }, - "InternetGatewayId": { - "Ref": "AWSEC2InternetGatewayadditionalcidrexamplecom" - } - } - }, - "AWSEC2VPCadditionalcidrexamplecom": { - "Type": "AWS::EC2::VPC", - "Properties": { - "CidrBlock": "10.0.0.0/16", - "EnableDnsHostnames": true, - "EnableDnsSupport": true, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionalcidr.example.com" - }, - { - "Key": "Name", - "Value": "additionalcidr.example.com" - }, - { - "Key": "kubernetes.io/cluster/additionalcidr.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2Volumeustest1betcdeventsadditionalcidrexamplecom": { - "Type": "AWS::EC2::Volume", - "Properties": { - "AvailabilityZone": "us-test-1b", - "Size": 20, - "VolumeType": "gp2", - "Encrypted": false, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionalcidr.example.com" - }, - { - "Key": "Name", - "Value": "us-test-1b.etcd-events.additionalcidr.example.com" - }, - { - "Key": "k8s.io/etcd/events", - "Value": "us-test-1b/us-test-1b" - }, - { - "Key": "k8s.io/role/master", - "Value": "1" - }, - { - "Key": "kubernetes.io/cluster/additionalcidr.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2Volumeustest1betcdmainadditionalcidrexamplecom": { - "Type": "AWS::EC2::Volume", - "Properties": { - "AvailabilityZone": "us-test-1b", - "Size": 20, - "VolumeType": "gp2", - "Encrypted": false, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionalcidr.example.com" - }, - { - "Key": "Name", - "Value": "us-test-1b.etcd-main.additionalcidr.example.com" - }, - { - "Key": "k8s.io/etcd/main", - "Value": "us-test-1b/us-test-1b" - }, - { - "Key": "k8s.io/role/master", - "Value": "1" - }, - { - "Key": "kubernetes.io/cluster/additionalcidr.example.com", - "Value": "owned" - } - ] - } - }, - "AWSIAMInstanceProfilemastersadditionalcidrexamplecom": { - "Type": "AWS::IAM::InstanceProfile", - "Properties": { - "Roles": [ - { - "Ref": "AWSIAMRolemastersadditionalcidrexamplecom" - } - ] - } - }, - "AWSIAMInstanceProfilenodesadditionalcidrexamplecom": { - "Type": "AWS::IAM::InstanceProfile", - "Properties": { - "Roles": [ - { - "Ref": "AWSIAMRolenodesadditionalcidrexamplecom" - } - ] - } - }, - "AWSIAMPolicymastersadditionalcidrexamplecom": { - "Type": "AWS::IAM::Policy", - "Properties": { - "PolicyName": "masters.additionalcidr.example.com", - "Roles": [ - { - "Ref": "AWSIAMRolemastersadditionalcidrexamplecom" - } - ], - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "ec2:*" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:DescribeAutoScalingInstances", - "autoscaling:DescribeLaunchConfigurations", - "autoscaling:DescribeTags", - "autoscaling:SetDesiredCapacity", - "autoscaling:TerminateInstanceInAutoScalingGroup", - "autoscaling:UpdateAutoScalingGroup", - "ec2:DescribeLaunchTemplateVersions" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "elasticloadbalancing:*" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "iam:ListServerCertificates", - "iam:GetServerCertificate" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "route53:ChangeResourceRecordSets", - "route53:ListResourceRecordSets", - "route53:GetHostedZone" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" - ] - }, - { - "Action": [ - "route53:GetChange" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:route53:::change/*" - ] - }, - { - "Action": [ - "route53:ListHostedZones" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "route53:ListHostedZones" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "ecr:GetAuthorizationToken", - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:GetRepositoryPolicy", - "ecr:DescribeRepositories", - "ecr:ListImages", - "ecr:BatchGetImage" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - } - ], - "Version": "2012-10-17" - } - } - }, - "AWSIAMPolicynodesadditionalcidrexamplecom": { - "Type": "AWS::IAM::Policy", - "Properties": { - "PolicyName": "nodes.additionalcidr.example.com", - "Roles": [ - { - "Ref": "AWSIAMRolenodesadditionalcidrexamplecom" - } - ], - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "ec2:DescribeInstances", - "ec2:DescribeRegions" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "route53:ChangeResourceRecordSets", - "route53:ListResourceRecordSets", - "route53:GetHostedZone" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" - ] - }, - { - "Action": [ - "route53:GetChange" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:route53:::change/*" - ] - }, - { - "Action": [ - "route53:ListHostedZones" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "route53:ListHostedZones" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "ecr:GetAuthorizationToken", - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:GetRepositoryPolicy", - "ecr:DescribeRepositories", - "ecr:ListImages", - "ecr:BatchGetImage" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - } - ], - "Version": "2012-10-17" - } - } - }, - "AWSIAMRolemastersadditionalcidrexamplecom": { - "Type": "AWS::IAM::Role", - "Properties": { - "RoleName": "masters.additionalcidr.example.com", - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "ec2.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - } - } - }, - "AWSIAMRolenodesadditionalcidrexamplecom": { - "Type": "AWS::IAM::Role", - "Properties": { - "RoleName": "nodes.additionalcidr.example.com", - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "ec2.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - } - } - } - } -} diff --git a/tests/integration/update_cluster/additional_cidr/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/additional_cidr/cloudformation.json.extracted.yaml deleted file mode 100644 index 3e377e7cfa..0000000000 --- a/tests/integration/update_cluster/additional_cidr/cloudformation.json.extracted.yaml +++ /dev/null @@ -1,537 +0,0 @@ -Resources.AWSEC2LaunchTemplatemasterustest1bmastersadditionalcidrexamplecom.Properties.LaunchTemplateData.UserData: | - #!/bin/bash - # Copyright 2016 The Kubernetes Authors All rights reserved. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - set -o errexit - set -o nounset - set -o pipefail - - NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup - NODEUP_HASH_AMD64=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 - NODEUP_URL_ARM64= - NODEUP_HASH_ARM64= - - export AWS_REGION=us-test-1 - - - - - function ensure-install-dir() { - INSTALL_DIR="/opt/kops" - # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec - if [[ -d /var/lib/toolbox ]]; then - INSTALL_DIR="/var/lib/toolbox/kops" - fi - mkdir -p ${INSTALL_DIR}/bin - mkdir -p ${INSTALL_DIR}/conf - cd ${INSTALL_DIR} - } - - # Retry a download until we get it. args: name, sha, url1, url2... - download-or-bust() { - local -r file="$1" - local -r hash="$2" - shift 2 - - urls=( $* ) - while true; do - for url in "${urls[@]}"; do - commands=( - "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - ) - for cmd in "${commands[@]}"; do - echo "Attempting download with: ${cmd} {url}" - if ! (${cmd} "${url}"); then - echo "== Download failed with ${cmd} ==" - continue - fi - if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then - echo "== Hash validation of ${url} failed. Retrying. ==" - rm -f "${file}" - else - if [[ -n "${hash}" ]]; then - echo "== Downloaded ${url} (SHA1 = ${hash}) ==" - else - echo "== Downloaded ${url} ==" - fi - return - fi - done - done - - echo "All downloads failed; sleeping before retrying" - sleep 60 - done - } - - validate-hash() { - local -r file="$1" - local -r expected="$2" - local actual - - actual=$(sha256sum ${file} | awk '{ print $1 }') || true - if [[ "${actual}" != "${expected}" ]]; then - echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" - return 1 - fi - } - - function split-commas() { - echo $1 | tr "," "\n" - } - - function try-download-release() { - local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) - if [[ -n "${NODEUP_HASH:-}" ]]; then - local -r nodeup_hash="${NODEUP_HASH}" - else - # TODO: Remove? - echo "Downloading sha256 (not found in env)" - download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" - local -r nodeup_hash=$(cat nodeup.sha256) - fi - - echo "Downloading nodeup (${nodeup_urls[@]})" - download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" - - chmod +x nodeup - } - - function download-release() { - case "$(uname -m)" in - x86_64*|i?86_64*|amd64*) - NODEUP_URL="${NODEUP_URL_AMD64}" - NODEUP_HASH="${NODEUP_HASH_AMD64}" - ;; - aarch64*|arm64*) - NODEUP_URL="${NODEUP_URL_ARM64}" - NODEUP_HASH="${NODEUP_HASH_ARM64}" - ;; - *) - echo "Unsupported host arch: $(uname -m)" >&2 - exit 1 - ;; - esac - - # In case of failure checking integrity of release, retry. - cd ${INSTALL_DIR}/bin - until try-download-release; do - sleep 15 - echo "Couldn't download release. Retrying..." - done - - echo "Running nodeup" - # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 - ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) - } - - #################################################################################### - - /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" - - echo "== nodeup node config starting ==" - ensure-install-dir - - cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' - cloudConfig: null - containerRuntime: docker - containerd: - skipInstall: true - docker: - ipMasq: false - ipTables: false - logDriver: json-file - logLevel: info - logOpt: - - max-size=10m - - max-file=5 - storage: overlay2,overlay,aufs - version: 18.06.3 - encryptionConfig: null - etcdClusters: - events: - version: 3.3.10 - main: - version: 3.3.10 - kubeAPIServer: - allowPrivileged: true - anonymousAuth: false - apiServerCount: 1 - authorizationMode: AlwaysAllow - bindAddress: 0.0.0.0 - cloudProvider: aws - enableAdmissionPlugins: - - NamespaceLifecycle - - LimitRanger - - ServiceAccount - - PersistentVolumeLabel - - DefaultStorageClass - - DefaultTolerationSeconds - - MutatingAdmissionWebhook - - ValidatingAdmissionWebhook - - NodeRestriction - - ResourceQuota - etcdServers: - - http://127.0.0.1:4001 - etcdServersOverrides: - - /events#http://127.0.0.1:4002 - image: k8s.gcr.io/kube-apiserver:v1.14.0 - insecureBindAddress: 127.0.0.1 - insecurePort: 8080 - kubeletPreferredAddressTypes: - - InternalIP - - Hostname - - ExternalIP - logLevel: 2 - requestheaderAllowedNames: - - aggregator - requestheaderExtraHeaderPrefixes: - - X-Remote-Extra- - requestheaderGroupHeaders: - - X-Remote-Group - requestheaderUsernameHeaders: - - X-Remote-User - securePort: 443 - serviceClusterIPRange: 100.64.0.0/13 - storageBackend: etcd3 - kubeControllerManager: - allocateNodeCIDRs: true - attachDetachReconcileSyncPeriod: 1m0s - cloudProvider: aws - clusterCIDR: 100.96.0.0/11 - clusterName: additionalcidr.example.com - configureCloudRoutes: true - image: k8s.gcr.io/kube-controller-manager:v1.14.0 - leaderElection: - leaderElect: true - logLevel: 2 - useServiceAccountCredentials: true - kubeProxy: - clusterCIDR: 100.96.0.0/11 - cpuRequest: 100m - hostnameOverride: '@aws' - image: k8s.gcr.io/kube-proxy:v1.14.0 - logLevel: 2 - kubeScheduler: - image: k8s.gcr.io/kube-scheduler:v1.14.0 - leaderElection: - leaderElect: true - logLevel: 2 - kubelet: - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests - masterKubelet: - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests - registerSchedulable: false - - __EOF_CLUSTER_SPEC - - cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' - kubelet: null - nodeLabels: null - taints: null - - __EOF_IG_SPEC - - cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' - Assets: - amd64: - - c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet - - 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl - - 3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz - arm64: - - df38e04576026393055ccc77c0dce73612996561@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubelet - - 01c2b6b43d36b6bfafc80a3737391c19ebfb8ad5@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubectl - - 7fec91af78e9548df306f0ec43bea527c8c10cc3a9682c33e971c8522a7fcded@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-arm64-v0.7.5.tgz - ClusterName: additionalcidr.example.com - ConfigBase: memfs://clusters.example.com/additionalcidr.example.com - InstanceGroupName: master-us-test-1b - InstanceGroupRole: Master - Tags: - - _automatic_upgrades - - _aws - channels: - - memfs://clusters.example.com/additionalcidr.example.com/addons/bootstrap-channel.yaml - etcdManifests: - - memfs://clusters.example.com/additionalcidr.example.com/manifests/etcd/main.yaml - - memfs://clusters.example.com/additionalcidr.example.com/manifests/etcd/events.yaml - protokubeImage: - hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b - name: protokube:1.15.0 - sources: - - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz - - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz - - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz - - __EOF_KUBE_ENV - - download-release - echo "== nodeup node config done ==" -Resources.AWSEC2LaunchTemplatenodesadditionalcidrexamplecom.Properties.LaunchTemplateData.UserData: | - #!/bin/bash - # Copyright 2016 The Kubernetes Authors All rights reserved. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - set -o errexit - set -o nounset - set -o pipefail - - NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup - NODEUP_HASH_AMD64=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 - NODEUP_URL_ARM64= - NODEUP_HASH_ARM64= - - export AWS_REGION=us-test-1 - - - - - function ensure-install-dir() { - INSTALL_DIR="/opt/kops" - # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec - if [[ -d /var/lib/toolbox ]]; then - INSTALL_DIR="/var/lib/toolbox/kops" - fi - mkdir -p ${INSTALL_DIR}/bin - mkdir -p ${INSTALL_DIR}/conf - cd ${INSTALL_DIR} - } - - # Retry a download until we get it. args: name, sha, url1, url2... - download-or-bust() { - local -r file="$1" - local -r hash="$2" - shift 2 - - urls=( $* ) - while true; do - for url in "${urls[@]}"; do - commands=( - "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - ) - for cmd in "${commands[@]}"; do - echo "Attempting download with: ${cmd} {url}" - if ! (${cmd} "${url}"); then - echo "== Download failed with ${cmd} ==" - continue - fi - if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then - echo "== Hash validation of ${url} failed. Retrying. ==" - rm -f "${file}" - else - if [[ -n "${hash}" ]]; then - echo "== Downloaded ${url} (SHA1 = ${hash}) ==" - else - echo "== Downloaded ${url} ==" - fi - return - fi - done - done - - echo "All downloads failed; sleeping before retrying" - sleep 60 - done - } - - validate-hash() { - local -r file="$1" - local -r expected="$2" - local actual - - actual=$(sha256sum ${file} | awk '{ print $1 }') || true - if [[ "${actual}" != "${expected}" ]]; then - echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" - return 1 - fi - } - - function split-commas() { - echo $1 | tr "," "\n" - } - - function try-download-release() { - local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) - if [[ -n "${NODEUP_HASH:-}" ]]; then - local -r nodeup_hash="${NODEUP_HASH}" - else - # TODO: Remove? - echo "Downloading sha256 (not found in env)" - download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" - local -r nodeup_hash=$(cat nodeup.sha256) - fi - - echo "Downloading nodeup (${nodeup_urls[@]})" - download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" - - chmod +x nodeup - } - - function download-release() { - case "$(uname -m)" in - x86_64*|i?86_64*|amd64*) - NODEUP_URL="${NODEUP_URL_AMD64}" - NODEUP_HASH="${NODEUP_HASH_AMD64}" - ;; - aarch64*|arm64*) - NODEUP_URL="${NODEUP_URL_ARM64}" - NODEUP_HASH="${NODEUP_HASH_ARM64}" - ;; - *) - echo "Unsupported host arch: $(uname -m)" >&2 - exit 1 - ;; - esac - - # In case of failure checking integrity of release, retry. - cd ${INSTALL_DIR}/bin - until try-download-release; do - sleep 15 - echo "Couldn't download release. Retrying..." - done - - echo "Running nodeup" - # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 - ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) - } - - #################################################################################### - - /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" - - echo "== nodeup node config starting ==" - ensure-install-dir - - cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' - cloudConfig: null - containerRuntime: docker - containerd: - skipInstall: true - docker: - ipMasq: false - ipTables: false - logDriver: json-file - logLevel: info - logOpt: - - max-size=10m - - max-file=5 - storage: overlay2,overlay,aufs - version: 18.06.3 - kubeProxy: - clusterCIDR: 100.96.0.0/11 - cpuRequest: 100m - hostnameOverride: '@aws' - image: k8s.gcr.io/kube-proxy:v1.14.0 - logLevel: 2 - kubelet: - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests - - __EOF_CLUSTER_SPEC - - cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' - kubelet: null - nodeLabels: null - taints: null - - __EOF_IG_SPEC - - cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' - Assets: - amd64: - - c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet - - 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl - - 3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz - arm64: - - df38e04576026393055ccc77c0dce73612996561@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubelet - - 01c2b6b43d36b6bfafc80a3737391c19ebfb8ad5@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubectl - - 7fec91af78e9548df306f0ec43bea527c8c10cc3a9682c33e971c8522a7fcded@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-arm64-v0.7.5.tgz - ClusterName: additionalcidr.example.com - ConfigBase: memfs://clusters.example.com/additionalcidr.example.com - InstanceGroupName: nodes - InstanceGroupRole: Node - Tags: - - _automatic_upgrades - - _aws - channels: - - memfs://clusters.example.com/additionalcidr.example.com/addons/bootstrap-channel.yaml - - __EOF_KUBE_ENV - - download-release - echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/additional_cidr/data/aws_iam_role_masters.additionalcidr.example.com_policy b/tests/integration/update_cluster/additional_cidr/data/aws_iam_role_masters.additionalcidr.example.com_policy deleted file mode 100644 index 66d5de1d5a..0000000000 --- a/tests/integration/update_cluster/additional_cidr/data/aws_iam_role_masters.additionalcidr.example.com_policy +++ /dev/null @@ -1,10 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { "Service": "ec2.amazonaws.com"}, - "Action": "sts:AssumeRole" - } - ] -} diff --git a/tests/integration/update_cluster/additional_cidr/data/aws_iam_role_nodes.additionalcidr.example.com_policy b/tests/integration/update_cluster/additional_cidr/data/aws_iam_role_nodes.additionalcidr.example.com_policy deleted file mode 100644 index 66d5de1d5a..0000000000 --- a/tests/integration/update_cluster/additional_cidr/data/aws_iam_role_nodes.additionalcidr.example.com_policy +++ /dev/null @@ -1,10 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { "Service": "ec2.amazonaws.com"}, - "Action": "sts:AssumeRole" - } - ] -} diff --git a/tests/integration/update_cluster/additional_cidr/data/aws_iam_role_policy_masters.additionalcidr.example.com_policy b/tests/integration/update_cluster/additional_cidr/data/aws_iam_role_policy_masters.additionalcidr.example.com_policy deleted file mode 100644 index 340dff1ef9..0000000000 --- a/tests/integration/update_cluster/additional_cidr/data/aws_iam_role_policy_masters.additionalcidr.example.com_policy +++ /dev/null @@ -1,102 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "ec2:*" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:DescribeAutoScalingInstances", - "autoscaling:DescribeLaunchConfigurations", - "autoscaling:DescribeTags", - "autoscaling:SetDesiredCapacity", - "autoscaling:TerminateInstanceInAutoScalingGroup", - "autoscaling:UpdateAutoScalingGroup", - "ec2:DescribeLaunchTemplateVersions" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "elasticloadbalancing:*" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "iam:ListServerCertificates", - "iam:GetServerCertificate" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ChangeResourceRecordSets", - "route53:ListResourceRecordSets", - "route53:GetHostedZone" - ], - "Resource": [ - "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:GetChange" - ], - "Resource": [ - "arn:aws:route53:::change/*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ListHostedZones" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ListHostedZones" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "ecr:GetAuthorizationToken", - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:GetRepositoryPolicy", - "ecr:DescribeRepositories", - "ecr:ListImages", - "ecr:BatchGetImage" - ], - "Resource": [ - "*" - ] - } - ] -} diff --git a/tests/integration/update_cluster/additional_cidr/data/aws_iam_role_policy_nodes.additionalcidr.example.com_policy b/tests/integration/update_cluster/additional_cidr/data/aws_iam_role_policy_nodes.additionalcidr.example.com_policy deleted file mode 100644 index ef2600b497..0000000000 --- a/tests/integration/update_cluster/additional_cidr/data/aws_iam_role_policy_nodes.additionalcidr.example.com_policy +++ /dev/null @@ -1,68 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "ec2:DescribeInstances", - "ec2:DescribeRegions" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ChangeResourceRecordSets", - "route53:ListResourceRecordSets", - "route53:GetHostedZone" - ], - "Resource": [ - "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:GetChange" - ], - "Resource": [ - "arn:aws:route53:::change/*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ListHostedZones" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ListHostedZones" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "ecr:GetAuthorizationToken", - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:GetRepositoryPolicy", - "ecr:DescribeRepositories", - "ecr:ListImages", - "ecr:BatchGetImage" - ], - "Resource": [ - "*" - ] - } - ] -} diff --git a/tests/integration/update_cluster/additional_cidr/data/aws_key_pair_kubernetes.additionalcidr.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key b/tests/integration/update_cluster/additional_cidr/data/aws_key_pair_kubernetes.additionalcidr.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key deleted file mode 100644 index 81cb012783..0000000000 --- a/tests/integration/update_cluster/additional_cidr/data/aws_key_pair_kubernetes.additionalcidr.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1a.masters.additionalcidr.example.com_user_data b/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1a.masters.additionalcidr.example.com_user_data deleted file mode 100644 index 73c6c28216..0000000000 --- a/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1a.masters.additionalcidr.example.com_user_data +++ /dev/null @@ -1,314 +0,0 @@ -#!/bin/bash -# Copyright 2016 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup -NODEUP_HASH_AMD64=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 -NODEUP_URL_ARM64= -NODEUP_HASH_ARM64= - -export AWS_REGION=us-test-1 - - - - -function ensure-install-dir() { - INSTALL_DIR="/opt/kops" - # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec - if [[ -d /var/lib/toolbox ]]; then - INSTALL_DIR="/var/lib/toolbox/kops" - fi - mkdir -p ${INSTALL_DIR}/bin - mkdir -p ${INSTALL_DIR}/conf - cd ${INSTALL_DIR} -} - -# Retry a download until we get it. args: name, sha, url1, url2... -download-or-bust() { - local -r file="$1" - local -r hash="$2" - shift 2 - - urls=( $* ) - while true; do - for url in "${urls[@]}"; do - commands=( - "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - ) - for cmd in "${commands[@]}"; do - echo "Attempting download with: ${cmd} {url}" - if ! (${cmd} "${url}"); then - echo "== Download failed with ${cmd} ==" - continue - fi - if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then - echo "== Hash validation of ${url} failed. Retrying. ==" - rm -f "${file}" - else - if [[ -n "${hash}" ]]; then - echo "== Downloaded ${url} (SHA1 = ${hash}) ==" - else - echo "== Downloaded ${url} ==" - fi - return - fi - done - done - - echo "All downloads failed; sleeping before retrying" - sleep 60 - done -} - -validate-hash() { - local -r file="$1" - local -r expected="$2" - local actual - - actual=$(sha256sum ${file} | awk '{ print $1 }') || true - if [[ "${actual}" != "${expected}" ]]; then - echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" - return 1 - fi -} - -function split-commas() { - echo $1 | tr "," "\n" -} - -function try-download-release() { - local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) - if [[ -n "${NODEUP_HASH:-}" ]]; then - local -r nodeup_hash="${NODEUP_HASH}" - else - # TODO: Remove? - echo "Downloading sha256 (not found in env)" - download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" - local -r nodeup_hash=$(cat nodeup.sha256) - fi - - echo "Downloading nodeup (${nodeup_urls[@]})" - download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" - - chmod +x nodeup -} - -function download-release() { - case "$(uname -m)" in - x86_64*|i?86_64*|amd64*) - NODEUP_URL="${NODEUP_URL_AMD64}" - NODEUP_HASH="${NODEUP_HASH_AMD64}" - ;; - aarch64*|arm64*) - NODEUP_URL="${NODEUP_URL_ARM64}" - NODEUP_HASH="${NODEUP_HASH_ARM64}" - ;; - *) - echo "Unsupported host arch: $(uname -m)" >&2 - exit 1 - ;; - esac - - # In case of failure checking integrity of release, retry. - cd ${INSTALL_DIR}/bin - until try-download-release; do - sleep 15 - echo "Couldn't download release. Retrying..." - done - - echo "Running nodeup" - # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 - ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) -} - -#################################################################################### - -/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" - -echo "== nodeup node config starting ==" -ensure-install-dir - -cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' -cloudConfig: null -containerRuntime: docker -containerd: - skipInstall: true -docker: - ipMasq: false - ipTables: false - logDriver: json-file - logLevel: info - logOpt: - - max-size=10m - - max-file=5 - storage: overlay2,overlay,aufs - version: 18.06.3 -encryptionConfig: null -etcdClusters: - events: - version: 3.3.10 - main: - version: 3.3.10 -kubeAPIServer: - allowPrivileged: true - anonymousAuth: false - apiServerCount: 3 - authorizationMode: AlwaysAllow - bindAddress: 0.0.0.0 - cloudProvider: aws - enableAdmissionPlugins: - - NamespaceLifecycle - - LimitRanger - - ServiceAccount - - PersistentVolumeLabel - - DefaultStorageClass - - DefaultTolerationSeconds - - MutatingAdmissionWebhook - - ValidatingAdmissionWebhook - - NodeRestriction - - ResourceQuota - etcdServers: - - http://127.0.0.1:4001 - etcdServersOverrides: - - /events#http://127.0.0.1:4002 - image: k8s.gcr.io/kube-apiserver:v1.14.0 - insecureBindAddress: 127.0.0.1 - insecurePort: 8080 - kubeletPreferredAddressTypes: - - InternalIP - - Hostname - - ExternalIP - logLevel: 2 - requestheaderAllowedNames: - - aggregator - requestheaderExtraHeaderPrefixes: - - X-Remote-Extra- - requestheaderGroupHeaders: - - X-Remote-Group - requestheaderUsernameHeaders: - - X-Remote-User - securePort: 443 - serviceClusterIPRange: 100.64.0.0/13 - storageBackend: etcd3 -kubeControllerManager: - allocateNodeCIDRs: true - attachDetachReconcileSyncPeriod: 1m0s - cloudProvider: aws - clusterCIDR: 100.96.0.0/11 - clusterName: additionalcidr.example.com - configureCloudRoutes: true - image: k8s.gcr.io/kube-controller-manager:v1.14.0 - leaderElection: - leaderElect: true - logLevel: 2 - useServiceAccountCredentials: true -kubeProxy: - clusterCIDR: 100.96.0.0/11 - cpuRequest: 100m - hostnameOverride: '@aws' - image: k8s.gcr.io/kube-proxy:v1.14.0 - logLevel: 2 -kubeScheduler: - image: k8s.gcr.io/kube-scheduler:v1.14.0 - leaderElection: - leaderElect: true - logLevel: 2 -kubelet: - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests -masterKubelet: - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests - registerSchedulable: false - -__EOF_CLUSTER_SPEC - -cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' -kubelet: null -nodeLabels: null -taints: null - -__EOF_IG_SPEC - -cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' -Assets: - amd64: - - c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet - - 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl - - 3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz - arm64: - - df38e04576026393055ccc77c0dce73612996561@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubelet - - 01c2b6b43d36b6bfafc80a3737391c19ebfb8ad5@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubectl - - 7fec91af78e9548df306f0ec43bea527c8c10cc3a9682c33e971c8522a7fcded@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-arm64-v0.7.5.tgz -ClusterName: additionalcidr.example.com -ConfigBase: memfs://clusters.example.com/additionalcidr.example.com -InstanceGroupName: master-us-test-1a -InstanceGroupRole: Master -Tags: -- _automatic_upgrades -- _aws -channels: -- memfs://clusters.example.com/additionalcidr.example.com/addons/bootstrap-channel.yaml -etcdManifests: -- memfs://clusters.example.com/additionalcidr.example.com/manifests/etcd/main.yaml -- memfs://clusters.example.com/additionalcidr.example.com/manifests/etcd/events.yaml -protokubeImage: - hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b - name: protokube:1.15.0 - sources: - - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz - - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz - - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz - -__EOF_KUBE_ENV - -download-release -echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1b.masters.additionalcidr.example.com_user_data b/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1b.masters.additionalcidr.example.com_user_data deleted file mode 100644 index ba3ac37cd3..0000000000 --- a/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1b.masters.additionalcidr.example.com_user_data +++ /dev/null @@ -1,314 +0,0 @@ -#!/bin/bash -# Copyright 2016 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup -NODEUP_HASH_AMD64=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 -NODEUP_URL_ARM64= -NODEUP_HASH_ARM64= - -export AWS_REGION=us-test-1 - - - - -function ensure-install-dir() { - INSTALL_DIR="/opt/kops" - # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec - if [[ -d /var/lib/toolbox ]]; then - INSTALL_DIR="/var/lib/toolbox/kops" - fi - mkdir -p ${INSTALL_DIR}/bin - mkdir -p ${INSTALL_DIR}/conf - cd ${INSTALL_DIR} -} - -# Retry a download until we get it. args: name, sha, url1, url2... -download-or-bust() { - local -r file="$1" - local -r hash="$2" - shift 2 - - urls=( $* ) - while true; do - for url in "${urls[@]}"; do - commands=( - "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - ) - for cmd in "${commands[@]}"; do - echo "Attempting download with: ${cmd} {url}" - if ! (${cmd} "${url}"); then - echo "== Download failed with ${cmd} ==" - continue - fi - if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then - echo "== Hash validation of ${url} failed. Retrying. ==" - rm -f "${file}" - else - if [[ -n "${hash}" ]]; then - echo "== Downloaded ${url} (SHA1 = ${hash}) ==" - else - echo "== Downloaded ${url} ==" - fi - return - fi - done - done - - echo "All downloads failed; sleeping before retrying" - sleep 60 - done -} - -validate-hash() { - local -r file="$1" - local -r expected="$2" - local actual - - actual=$(sha256sum ${file} | awk '{ print $1 }') || true - if [[ "${actual}" != "${expected}" ]]; then - echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" - return 1 - fi -} - -function split-commas() { - echo $1 | tr "," "\n" -} - -function try-download-release() { - local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) - if [[ -n "${NODEUP_HASH:-}" ]]; then - local -r nodeup_hash="${NODEUP_HASH}" - else - # TODO: Remove? - echo "Downloading sha256 (not found in env)" - download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" - local -r nodeup_hash=$(cat nodeup.sha256) - fi - - echo "Downloading nodeup (${nodeup_urls[@]})" - download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" - - chmod +x nodeup -} - -function download-release() { - case "$(uname -m)" in - x86_64*|i?86_64*|amd64*) - NODEUP_URL="${NODEUP_URL_AMD64}" - NODEUP_HASH="${NODEUP_HASH_AMD64}" - ;; - aarch64*|arm64*) - NODEUP_URL="${NODEUP_URL_ARM64}" - NODEUP_HASH="${NODEUP_HASH_ARM64}" - ;; - *) - echo "Unsupported host arch: $(uname -m)" >&2 - exit 1 - ;; - esac - - # In case of failure checking integrity of release, retry. - cd ${INSTALL_DIR}/bin - until try-download-release; do - sleep 15 - echo "Couldn't download release. Retrying..." - done - - echo "Running nodeup" - # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 - ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) -} - -#################################################################################### - -/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" - -echo "== nodeup node config starting ==" -ensure-install-dir - -cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' -cloudConfig: null -containerRuntime: docker -containerd: - skipInstall: true -docker: - ipMasq: false - ipTables: false - logDriver: json-file - logLevel: info - logOpt: - - max-size=10m - - max-file=5 - storage: overlay2,overlay,aufs - version: 18.06.3 -encryptionConfig: null -etcdClusters: - events: - version: 3.3.10 - main: - version: 3.3.10 -kubeAPIServer: - allowPrivileged: true - anonymousAuth: false - apiServerCount: 3 - authorizationMode: AlwaysAllow - bindAddress: 0.0.0.0 - cloudProvider: aws - enableAdmissionPlugins: - - NamespaceLifecycle - - LimitRanger - - ServiceAccount - - PersistentVolumeLabel - - DefaultStorageClass - - DefaultTolerationSeconds - - MutatingAdmissionWebhook - - ValidatingAdmissionWebhook - - NodeRestriction - - ResourceQuota - etcdServers: - - http://127.0.0.1:4001 - etcdServersOverrides: - - /events#http://127.0.0.1:4002 - image: k8s.gcr.io/kube-apiserver:v1.14.0 - insecureBindAddress: 127.0.0.1 - insecurePort: 8080 - kubeletPreferredAddressTypes: - - InternalIP - - Hostname - - ExternalIP - logLevel: 2 - requestheaderAllowedNames: - - aggregator - requestheaderExtraHeaderPrefixes: - - X-Remote-Extra- - requestheaderGroupHeaders: - - X-Remote-Group - requestheaderUsernameHeaders: - - X-Remote-User - securePort: 443 - serviceClusterIPRange: 100.64.0.0/13 - storageBackend: etcd3 -kubeControllerManager: - allocateNodeCIDRs: true - attachDetachReconcileSyncPeriod: 1m0s - cloudProvider: aws - clusterCIDR: 100.96.0.0/11 - clusterName: additionalcidr.example.com - configureCloudRoutes: true - image: k8s.gcr.io/kube-controller-manager:v1.14.0 - leaderElection: - leaderElect: true - logLevel: 2 - useServiceAccountCredentials: true -kubeProxy: - clusterCIDR: 100.96.0.0/11 - cpuRequest: 100m - hostnameOverride: '@aws' - image: k8s.gcr.io/kube-proxy:v1.14.0 - logLevel: 2 -kubeScheduler: - image: k8s.gcr.io/kube-scheduler:v1.14.0 - leaderElection: - leaderElect: true - logLevel: 2 -kubelet: - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests -masterKubelet: - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests - registerSchedulable: false - -__EOF_CLUSTER_SPEC - -cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' -kubelet: null -nodeLabels: null -taints: null - -__EOF_IG_SPEC - -cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' -Assets: - amd64: - - c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet - - 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl - - 3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz - arm64: - - df38e04576026393055ccc77c0dce73612996561@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubelet - - 01c2b6b43d36b6bfafc80a3737391c19ebfb8ad5@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubectl - - 7fec91af78e9548df306f0ec43bea527c8c10cc3a9682c33e971c8522a7fcded@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-arm64-v0.7.5.tgz -ClusterName: additionalcidr.example.com -ConfigBase: memfs://clusters.example.com/additionalcidr.example.com -InstanceGroupName: master-us-test-1b -InstanceGroupRole: Master -Tags: -- _automatic_upgrades -- _aws -channels: -- memfs://clusters.example.com/additionalcidr.example.com/addons/bootstrap-channel.yaml -etcdManifests: -- memfs://clusters.example.com/additionalcidr.example.com/manifests/etcd/main.yaml -- memfs://clusters.example.com/additionalcidr.example.com/manifests/etcd/events.yaml -protokubeImage: - hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b - name: protokube:1.15.0 - sources: - - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz - - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz - - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz - -__EOF_KUBE_ENV - -download-release -echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1c.masters.additionalcidr.example.com_user_data b/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1c.masters.additionalcidr.example.com_user_data deleted file mode 100644 index 50e0d3a7f0..0000000000 --- a/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1c.masters.additionalcidr.example.com_user_data +++ /dev/null @@ -1,314 +0,0 @@ -#!/bin/bash -# Copyright 2016 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup -NODEUP_HASH_AMD64=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 -NODEUP_URL_ARM64= -NODEUP_HASH_ARM64= - -export AWS_REGION=us-test-1 - - - - -function ensure-install-dir() { - INSTALL_DIR="/opt/kops" - # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec - if [[ -d /var/lib/toolbox ]]; then - INSTALL_DIR="/var/lib/toolbox/kops" - fi - mkdir -p ${INSTALL_DIR}/bin - mkdir -p ${INSTALL_DIR}/conf - cd ${INSTALL_DIR} -} - -# Retry a download until we get it. args: name, sha, url1, url2... -download-or-bust() { - local -r file="$1" - local -r hash="$2" - shift 2 - - urls=( $* ) - while true; do - for url in "${urls[@]}"; do - commands=( - "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - ) - for cmd in "${commands[@]}"; do - echo "Attempting download with: ${cmd} {url}" - if ! (${cmd} "${url}"); then - echo "== Download failed with ${cmd} ==" - continue - fi - if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then - echo "== Hash validation of ${url} failed. Retrying. ==" - rm -f "${file}" - else - if [[ -n "${hash}" ]]; then - echo "== Downloaded ${url} (SHA1 = ${hash}) ==" - else - echo "== Downloaded ${url} ==" - fi - return - fi - done - done - - echo "All downloads failed; sleeping before retrying" - sleep 60 - done -} - -validate-hash() { - local -r file="$1" - local -r expected="$2" - local actual - - actual=$(sha256sum ${file} | awk '{ print $1 }') || true - if [[ "${actual}" != "${expected}" ]]; then - echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" - return 1 - fi -} - -function split-commas() { - echo $1 | tr "," "\n" -} - -function try-download-release() { - local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) - if [[ -n "${NODEUP_HASH:-}" ]]; then - local -r nodeup_hash="${NODEUP_HASH}" - else - # TODO: Remove? - echo "Downloading sha256 (not found in env)" - download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" - local -r nodeup_hash=$(cat nodeup.sha256) - fi - - echo "Downloading nodeup (${nodeup_urls[@]})" - download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" - - chmod +x nodeup -} - -function download-release() { - case "$(uname -m)" in - x86_64*|i?86_64*|amd64*) - NODEUP_URL="${NODEUP_URL_AMD64}" - NODEUP_HASH="${NODEUP_HASH_AMD64}" - ;; - aarch64*|arm64*) - NODEUP_URL="${NODEUP_URL_ARM64}" - NODEUP_HASH="${NODEUP_HASH_ARM64}" - ;; - *) - echo "Unsupported host arch: $(uname -m)" >&2 - exit 1 - ;; - esac - - # In case of failure checking integrity of release, retry. - cd ${INSTALL_DIR}/bin - until try-download-release; do - sleep 15 - echo "Couldn't download release. Retrying..." - done - - echo "Running nodeup" - # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 - ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) -} - -#################################################################################### - -/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" - -echo "== nodeup node config starting ==" -ensure-install-dir - -cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' -cloudConfig: null -containerRuntime: docker -containerd: - skipInstall: true -docker: - ipMasq: false - ipTables: false - logDriver: json-file - logLevel: info - logOpt: - - max-size=10m - - max-file=5 - storage: overlay2,overlay,aufs - version: 18.06.3 -encryptionConfig: null -etcdClusters: - events: - version: 3.3.10 - main: - version: 3.3.10 -kubeAPIServer: - allowPrivileged: true - anonymousAuth: false - apiServerCount: 3 - authorizationMode: AlwaysAllow - bindAddress: 0.0.0.0 - cloudProvider: aws - enableAdmissionPlugins: - - NamespaceLifecycle - - LimitRanger - - ServiceAccount - - PersistentVolumeLabel - - DefaultStorageClass - - DefaultTolerationSeconds - - MutatingAdmissionWebhook - - ValidatingAdmissionWebhook - - NodeRestriction - - ResourceQuota - etcdServers: - - http://127.0.0.1:4001 - etcdServersOverrides: - - /events#http://127.0.0.1:4002 - image: k8s.gcr.io/kube-apiserver:v1.14.0 - insecureBindAddress: 127.0.0.1 - insecurePort: 8080 - kubeletPreferredAddressTypes: - - InternalIP - - Hostname - - ExternalIP - logLevel: 2 - requestheaderAllowedNames: - - aggregator - requestheaderExtraHeaderPrefixes: - - X-Remote-Extra- - requestheaderGroupHeaders: - - X-Remote-Group - requestheaderUsernameHeaders: - - X-Remote-User - securePort: 443 - serviceClusterIPRange: 100.64.0.0/13 - storageBackend: etcd3 -kubeControllerManager: - allocateNodeCIDRs: true - attachDetachReconcileSyncPeriod: 1m0s - cloudProvider: aws - clusterCIDR: 100.96.0.0/11 - clusterName: additionalcidr.example.com - configureCloudRoutes: true - image: k8s.gcr.io/kube-controller-manager:v1.14.0 - leaderElection: - leaderElect: true - logLevel: 2 - useServiceAccountCredentials: true -kubeProxy: - clusterCIDR: 100.96.0.0/11 - cpuRequest: 100m - hostnameOverride: '@aws' - image: k8s.gcr.io/kube-proxy:v1.14.0 - logLevel: 2 -kubeScheduler: - image: k8s.gcr.io/kube-scheduler:v1.14.0 - leaderElection: - leaderElect: true - logLevel: 2 -kubelet: - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests -masterKubelet: - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests - registerSchedulable: false - -__EOF_CLUSTER_SPEC - -cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' -kubelet: null -nodeLabels: null -taints: null - -__EOF_IG_SPEC - -cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' -Assets: - amd64: - - c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet - - 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl - - 3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz - arm64: - - df38e04576026393055ccc77c0dce73612996561@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubelet - - 01c2b6b43d36b6bfafc80a3737391c19ebfb8ad5@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubectl - - 7fec91af78e9548df306f0ec43bea527c8c10cc3a9682c33e971c8522a7fcded@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-arm64-v0.7.5.tgz -ClusterName: additionalcidr.example.com -ConfigBase: memfs://clusters.example.com/additionalcidr.example.com -InstanceGroupName: master-us-test-1c -InstanceGroupRole: Master -Tags: -- _automatic_upgrades -- _aws -channels: -- memfs://clusters.example.com/additionalcidr.example.com/addons/bootstrap-channel.yaml -etcdManifests: -- memfs://clusters.example.com/additionalcidr.example.com/manifests/etcd/main.yaml -- memfs://clusters.example.com/additionalcidr.example.com/manifests/etcd/events.yaml -protokubeImage: - hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b - name: protokube:1.15.0 - sources: - - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz - - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz - - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz - -__EOF_KUBE_ENV - -download-release -echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_nodes.additionalcidr.example.com_user_data b/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_nodes.additionalcidr.example.com_user_data deleted file mode 100644 index 9e0f50ca15..0000000000 --- a/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_nodes.additionalcidr.example.com_user_data +++ /dev/null @@ -1,221 +0,0 @@ -#!/bin/bash -# Copyright 2016 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup -NODEUP_HASH_AMD64=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 -NODEUP_URL_ARM64= -NODEUP_HASH_ARM64= - -export AWS_REGION=us-test-1 - - - - -function ensure-install-dir() { - INSTALL_DIR="/opt/kops" - # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec - if [[ -d /var/lib/toolbox ]]; then - INSTALL_DIR="/var/lib/toolbox/kops" - fi - mkdir -p ${INSTALL_DIR}/bin - mkdir -p ${INSTALL_DIR}/conf - cd ${INSTALL_DIR} -} - -# Retry a download until we get it. args: name, sha, url1, url2... -download-or-bust() { - local -r file="$1" - local -r hash="$2" - shift 2 - - urls=( $* ) - while true; do - for url in "${urls[@]}"; do - commands=( - "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - ) - for cmd in "${commands[@]}"; do - echo "Attempting download with: ${cmd} {url}" - if ! (${cmd} "${url}"); then - echo "== Download failed with ${cmd} ==" - continue - fi - if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then - echo "== Hash validation of ${url} failed. Retrying. ==" - rm -f "${file}" - else - if [[ -n "${hash}" ]]; then - echo "== Downloaded ${url} (SHA1 = ${hash}) ==" - else - echo "== Downloaded ${url} ==" - fi - return - fi - done - done - - echo "All downloads failed; sleeping before retrying" - sleep 60 - done -} - -validate-hash() { - local -r file="$1" - local -r expected="$2" - local actual - - actual=$(sha256sum ${file} | awk '{ print $1 }') || true - if [[ "${actual}" != "${expected}" ]]; then - echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" - return 1 - fi -} - -function split-commas() { - echo $1 | tr "," "\n" -} - -function try-download-release() { - local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) - if [[ -n "${NODEUP_HASH:-}" ]]; then - local -r nodeup_hash="${NODEUP_HASH}" - else - # TODO: Remove? - echo "Downloading sha256 (not found in env)" - download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" - local -r nodeup_hash=$(cat nodeup.sha256) - fi - - echo "Downloading nodeup (${nodeup_urls[@]})" - download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" - - chmod +x nodeup -} - -function download-release() { - case "$(uname -m)" in - x86_64*|i?86_64*|amd64*) - NODEUP_URL="${NODEUP_URL_AMD64}" - NODEUP_HASH="${NODEUP_HASH_AMD64}" - ;; - aarch64*|arm64*) - NODEUP_URL="${NODEUP_URL_ARM64}" - NODEUP_HASH="${NODEUP_HASH_ARM64}" - ;; - *) - echo "Unsupported host arch: $(uname -m)" >&2 - exit 1 - ;; - esac - - # In case of failure checking integrity of release, retry. - cd ${INSTALL_DIR}/bin - until try-download-release; do - sleep 15 - echo "Couldn't download release. Retrying..." - done - - echo "Running nodeup" - # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 - ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) -} - -#################################################################################### - -/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" - -echo "== nodeup node config starting ==" -ensure-install-dir - -cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' -cloudConfig: null -containerRuntime: docker -containerd: - skipInstall: true -docker: - ipMasq: false - ipTables: false - logDriver: json-file - logLevel: info - logOpt: - - max-size=10m - - max-file=5 - storage: overlay2,overlay,aufs - version: 18.06.3 -kubeProxy: - clusterCIDR: 100.96.0.0/11 - cpuRequest: 100m - hostnameOverride: '@aws' - image: k8s.gcr.io/kube-proxy:v1.14.0 - logLevel: 2 -kubelet: - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests - -__EOF_CLUSTER_SPEC - -cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' -kubelet: null -nodeLabels: null -taints: null - -__EOF_IG_SPEC - -cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' -Assets: - amd64: - - c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet - - 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl - - 3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz - arm64: - - df38e04576026393055ccc77c0dce73612996561@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubelet - - 01c2b6b43d36b6bfafc80a3737391c19ebfb8ad5@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubectl - - 7fec91af78e9548df306f0ec43bea527c8c10cc3a9682c33e971c8522a7fcded@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-arm64-v0.7.5.tgz -ClusterName: additionalcidr.example.com -ConfigBase: memfs://clusters.example.com/additionalcidr.example.com -InstanceGroupName: nodes -InstanceGroupRole: Node -Tags: -- _automatic_upgrades -- _aws -channels: -- memfs://clusters.example.com/additionalcidr.example.com/addons/bootstrap-channel.yaml - -__EOF_KUBE_ENV - -download-release -echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/additional_cidr/id_rsa.pub b/tests/integration/update_cluster/additional_cidr/id_rsa.pub deleted file mode 100755 index 81cb012783..0000000000 --- a/tests/integration/update_cluster/additional_cidr/id_rsa.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/additional_cidr/in-v1alpha2.yaml b/tests/integration/update_cluster/additional_cidr/in-v1alpha2.yaml deleted file mode 100644 index fdc2d68c3e..0000000000 --- a/tests/integration/update_cluster/additional_cidr/in-v1alpha2.yaml +++ /dev/null @@ -1,85 +0,0 @@ -apiVersion: kops.k8s.io/v1alpha2 -kind: Cluster -metadata: - creationTimestamp: "2016-12-10T22:42:27Z" - name: additionalcidr.example.com -spec: - kubernetesApiAccess: - - 0.0.0.0/0 - channel: stable - cloudProvider: aws - configBase: memfs://clusters.example.com/additionalcidr.example.com - etcdClusters: - - etcdMembers: - - instanceGroup: master-us-test-1b - name: us-test-1b - name: main - - etcdMembers: - - instanceGroup: master-us-test-1b - name: us-test-1b - name: events - kubelet: - anonymousAuth: false - kubernetesVersion: v1.14.0 - masterInternalName: api.internal.additionalcidr.example.com - masterPublicName: api.additionalcidr.example.com - networkCIDR: 10.0.0.0/16 - additionalNetworkCIDRs: - - 10.1.0.0/16 - networking: - kubenet: {} - nonMasqueradeCIDR: 100.64.0.0/10 - sshAccess: - - 0.0.0.0/0 - topology: - masters: public - nodes: public - subnets: - - cidr: 10.0.1.0/24 - name: us-test-1a - type: Public - zone: us-test-1a - - cidr: 10.1.1.0/24 - name: us-test-1b - type: Public - zone: us-test-1b - ---- - -apiVersion: kops.k8s.io/v1alpha2 -kind: InstanceGroup -metadata: - creationTimestamp: "2016-12-10T22:42:28Z" - name: nodes - labels: - kops.k8s.io/cluster: additionalcidr.example.com -spec: - associatePublicIp: true - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: t2.medium - maxSize: 2 - minSize: 2 - role: Node - subnets: - - us-test-1b - ---- - -apiVersion: kops.k8s.io/v1alpha2 -kind: InstanceGroup -metadata: - creationTimestamp: "2016-12-10T22:42:28Z" - name: master-us-test-1b - labels: - kops.k8s.io/cluster: additionalcidr.example.com -spec: - associatePublicIp: true - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: m3.medium - maxSize: 1 - minSize: 1 - role: Master - subnets: - - us-test-1b - - diff --git a/tests/integration/update_cluster/additional_cidr/in-v1alpha3.yaml b/tests/integration/update_cluster/additional_cidr/in-v1alpha3.yaml deleted file mode 100644 index 680c12d962..0000000000 --- a/tests/integration/update_cluster/additional_cidr/in-v1alpha3.yaml +++ /dev/null @@ -1,133 +0,0 @@ -apiVersion: kops.k8s.io/v1alpha2 -kind: Cluster -metadata: - creationTimestamp: "2016-12-10T22:42:27Z" - name: additionalcidr.example.com -spec: - kubernetesApiAccess: - - 0.0.0.0/0 - channel: stable - cloudProvider: aws - configBase: memfs://clusters.example.com/additionalcidr.example.com - etcdClusters: - - etcdMembers: - - instanceGroup: master-us-test-1a - name: us-test-1a - - instanceGroup: master-us-test-1b - name: us-test-1b - - instanceGroup: master-us-test-1c - name: us-test-1c - name: main - - etcdMembers: - - instanceGroup: master-us-test-1a - name: us-test-1a - - instanceGroup: master-us-test-1b - name: us-test-1b - - instanceGroup: master-us-test-1c - name: us-test-1c - name: events - kubelet: - anonymousAuth: false - kubernetesVersion: v1.14.0 - masterInternalName: api.internal.additionalcidr.example.com - masterPublicName: api.additionalcidr.example.com - networkCIDR: 10.0.0.0/16 - additionalNetworkCIDRs: - - 10.1.0.0/16 - networking: - kubenet: {} - nonMasqueradeCIDR: 100.64.0.0/10 - sshAccess: - - 0.0.0.0/0 - topology: - masters: public - nodes: public - subnets: - - cidr: 10.0.1.0/24 - name: us-test-1a - type: Public - zone: us-test-1a - - cidr: 10.1.1.0/24 - name: us-test-1b - type: Public - zone: us-test-1b - - cidr: 10.1.2.0/24 - name: us-test-1c - type: Public - zone: us-test-1c - ---- - -apiVersion: kops.k8s.io/v1alpha2 -kind: InstanceGroup -metadata: - creationTimestamp: "2016-12-10T22:42:28Z" - name: nodes - labels: - kops.k8s.io/cluster: additionalcidr.example.com -spec: - associatePublicIp: true - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: t2.medium - maxSize: 2 - minSize: 2 - role: Node - subnets: - - us-test-1b - ---- - -apiVersion: kops.k8s.io/v1alpha2 -kind: InstanceGroup -metadata: - creationTimestamp: "2016-12-10T22:42:28Z" - name: master-us-test-1a - labels: - kops.k8s.io/cluster: additionalcidr.example.com -spec: - associatePublicIp: true - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: m3.medium - maxSize: 1 - minSize: 1 - role: Master - subnets: - - us-test-1a - ---- - -apiVersion: kops.k8s.io/v1alpha2 -kind: InstanceGroup -metadata: - creationTimestamp: "2016-12-10T22:42:28Z" - name: master-us-test-1b - labels: - kops.k8s.io/cluster: additionalcidr.example.com -spec: - associatePublicIp: true - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: m3.medium - maxSize: 1 - minSize: 1 - role: Master - subnets: - - us-test-1b - ---- - -apiVersion: kops.k8s.io/v1alpha2 -kind: InstanceGroup -metadata: - creationTimestamp: "2016-12-10T22:42:28Z" - name: master-us-test-1c - labels: - kops.k8s.io/cluster: additionalcidr.example.com -spec: - associatePublicIp: true - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: m3.medium - maxSize: 1 - minSize: 1 - role: Master - subnets: - - us-test-1c diff --git a/tests/integration/update_cluster/additional_cidr/kubernetes.tf b/tests/integration/update_cluster/additional_cidr/kubernetes.tf deleted file mode 100644 index ad44bb7183..0000000000 --- a/tests/integration/update_cluster/additional_cidr/kubernetes.tf +++ /dev/null @@ -1,808 +0,0 @@ -locals { - cluster_name = "additionalcidr.example.com" - master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-additionalcidr-example-com.id, aws_autoscaling_group.master-us-test-1b-masters-additionalcidr-example-com.id, aws_autoscaling_group.master-us-test-1c-masters-additionalcidr-example-com.id] - master_security_group_ids = [aws_security_group.masters-additionalcidr-example-com.id, aws_security_group.masters-additionalcidr-example-com.id, aws_security_group.masters-additionalcidr-example-com.id] - masters_role_arn = aws_iam_role.masters-additionalcidr-example-com.arn - masters_role_name = aws_iam_role.masters-additionalcidr-example-com.name - node_autoscaling_group_ids = [aws_autoscaling_group.nodes-additionalcidr-example-com.id] - node_security_group_ids = [aws_security_group.nodes-additionalcidr-example-com.id] - node_subnet_ids = [aws_subnet.us-test-1b-additionalcidr-example-com.id] - nodes_role_arn = aws_iam_role.nodes-additionalcidr-example-com.arn - nodes_role_name = aws_iam_role.nodes-additionalcidr-example-com.name - region = "us-test-1" - route_table_public_id = aws_route_table.additionalcidr-example-com.id - subnet_us-test-1a_id = aws_subnet.us-test-1a-additionalcidr-example-com.id - subnet_us-test-1b_id = aws_subnet.us-test-1b-additionalcidr-example-com.id - subnet_us-test-1c_id = aws_subnet.us-test-1c-additionalcidr-example-com.id - vpc_cidr_block = aws_vpc.additionalcidr-example-com.cidr_block - vpc_id = aws_vpc.additionalcidr-example-com.id -} - -output "cluster_name" { - value = "additionalcidr.example.com" -} - -output "master_autoscaling_group_ids" { - value = [aws_autoscaling_group.master-us-test-1a-masters-additionalcidr-example-com.id, aws_autoscaling_group.master-us-test-1b-masters-additionalcidr-example-com.id, aws_autoscaling_group.master-us-test-1c-masters-additionalcidr-example-com.id] -} - -output "master_security_group_ids" { - value = [aws_security_group.masters-additionalcidr-example-com.id, aws_security_group.masters-additionalcidr-example-com.id, aws_security_group.masters-additionalcidr-example-com.id] -} - -output "masters_role_arn" { - value = aws_iam_role.masters-additionalcidr-example-com.arn -} - -output "masters_role_name" { - value = aws_iam_role.masters-additionalcidr-example-com.name -} - -output "node_autoscaling_group_ids" { - value = [aws_autoscaling_group.nodes-additionalcidr-example-com.id] -} - -output "node_security_group_ids" { - value = [aws_security_group.nodes-additionalcidr-example-com.id] -} - -output "node_subnet_ids" { - value = [aws_subnet.us-test-1b-additionalcidr-example-com.id] -} - -output "nodes_role_arn" { - value = aws_iam_role.nodes-additionalcidr-example-com.arn -} - -output "nodes_role_name" { - value = aws_iam_role.nodes-additionalcidr-example-com.name -} - -output "region" { - value = "us-test-1" -} - -output "route_table_public_id" { - value = aws_route_table.additionalcidr-example-com.id -} - -output "subnet_us-test-1a_id" { - value = aws_subnet.us-test-1a-additionalcidr-example-com.id -} - -output "subnet_us-test-1b_id" { - value = aws_subnet.us-test-1b-additionalcidr-example-com.id -} - -output "subnet_us-test-1c_id" { - value = aws_subnet.us-test-1c-additionalcidr-example-com.id -} - -output "vpc_cidr_block" { - value = aws_vpc.additionalcidr-example-com.cidr_block -} - -output "vpc_id" { - value = aws_vpc.additionalcidr-example-com.id -} - -provider "aws" { - region = "us-test-1" -} - -resource "aws_autoscaling_group" "master-us-test-1a-masters-additionalcidr-example-com" { - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] - launch_template { - id = aws_launch_template.master-us-test-1a-masters-additionalcidr-example-com.id - version = aws_launch_template.master-us-test-1a-masters-additionalcidr-example-com.latest_version - } - max_size = 1 - metrics_granularity = "1Minute" - min_size = 1 - name = "master-us-test-1a.masters.additionalcidr.example.com" - tag { - key = "KubernetesCluster" - propagate_at_launch = true - value = "additionalcidr.example.com" - } - tag { - key = "Name" - propagate_at_launch = true - value = "master-us-test-1a.masters.additionalcidr.example.com" - } - tag { - key = "k8s.io/role/master" - propagate_at_launch = true - value = "1" - } - tag { - key = "kops.k8s.io/instancegroup" - propagate_at_launch = true - value = "master-us-test-1a" - } - tag { - key = "kubernetes.io/cluster/additionalcidr.example.com" - propagate_at_launch = true - value = "owned" - } - vpc_zone_identifier = [aws_subnet.us-test-1a-additionalcidr-example-com.id] -} - -resource "aws_autoscaling_group" "master-us-test-1b-masters-additionalcidr-example-com" { - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] - launch_template { - id = aws_launch_template.master-us-test-1b-masters-additionalcidr-example-com.id - version = aws_launch_template.master-us-test-1b-masters-additionalcidr-example-com.latest_version - } - max_size = 1 - metrics_granularity = "1Minute" - min_size = 1 - name = "master-us-test-1b.masters.additionalcidr.example.com" - tag { - key = "KubernetesCluster" - propagate_at_launch = true - value = "additionalcidr.example.com" - } - tag { - key = "Name" - propagate_at_launch = true - value = "master-us-test-1b.masters.additionalcidr.example.com" - } - tag { - key = "k8s.io/role/master" - propagate_at_launch = true - value = "1" - } - tag { - key = "kops.k8s.io/instancegroup" - propagate_at_launch = true - value = "master-us-test-1b" - } - tag { - key = "kubernetes.io/cluster/additionalcidr.example.com" - propagate_at_launch = true - value = "owned" - } - vpc_zone_identifier = [aws_subnet.us-test-1b-additionalcidr-example-com.id] -} - -resource "aws_autoscaling_group" "master-us-test-1c-masters-additionalcidr-example-com" { - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] - launch_template { - id = aws_launch_template.master-us-test-1c-masters-additionalcidr-example-com.id - version = aws_launch_template.master-us-test-1c-masters-additionalcidr-example-com.latest_version - } - max_size = 1 - metrics_granularity = "1Minute" - min_size = 1 - name = "master-us-test-1c.masters.additionalcidr.example.com" - tag { - key = "KubernetesCluster" - propagate_at_launch = true - value = "additionalcidr.example.com" - } - tag { - key = "Name" - propagate_at_launch = true - value = "master-us-test-1c.masters.additionalcidr.example.com" - } - tag { - key = "k8s.io/role/master" - propagate_at_launch = true - value = "1" - } - tag { - key = "kops.k8s.io/instancegroup" - propagate_at_launch = true - value = "master-us-test-1c" - } - tag { - key = "kubernetes.io/cluster/additionalcidr.example.com" - propagate_at_launch = true - value = "owned" - } - vpc_zone_identifier = [aws_subnet.us-test-1c-additionalcidr-example-com.id] -} - -resource "aws_autoscaling_group" "nodes-additionalcidr-example-com" { - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] - launch_template { - id = aws_launch_template.nodes-additionalcidr-example-com.id - version = aws_launch_template.nodes-additionalcidr-example-com.latest_version - } - max_size = 2 - metrics_granularity = "1Minute" - min_size = 2 - name = "nodes.additionalcidr.example.com" - tag { - key = "KubernetesCluster" - propagate_at_launch = true - value = "additionalcidr.example.com" - } - tag { - key = "Name" - propagate_at_launch = true - value = "nodes.additionalcidr.example.com" - } - tag { - key = "k8s.io/role/node" - propagate_at_launch = true - value = "1" - } - tag { - key = "kops.k8s.io/instancegroup" - propagate_at_launch = true - value = "nodes" - } - tag { - key = "kubernetes.io/cluster/additionalcidr.example.com" - propagate_at_launch = true - value = "owned" - } - vpc_zone_identifier = [aws_subnet.us-test-1b-additionalcidr-example-com.id] -} - -resource "aws_ebs_volume" "us-test-1a-etcd-events-additionalcidr-example-com" { - availability_zone = "us-test-1a" - encrypted = false - size = 20 - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "us-test-1a.etcd-events.additionalcidr.example.com" - "k8s.io/etcd/events" = "us-test-1a/us-test-1a,us-test-1b,us-test-1c" - "k8s.io/role/master" = "1" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - } - type = "gp2" -} - -resource "aws_ebs_volume" "us-test-1a-etcd-main-additionalcidr-example-com" { - availability_zone = "us-test-1a" - encrypted = false - size = 20 - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "us-test-1a.etcd-main.additionalcidr.example.com" - "k8s.io/etcd/main" = "us-test-1a/us-test-1a,us-test-1b,us-test-1c" - "k8s.io/role/master" = "1" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - } - type = "gp2" -} - -resource "aws_ebs_volume" "us-test-1b-etcd-events-additionalcidr-example-com" { - availability_zone = "us-test-1b" - encrypted = false - size = 20 - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "us-test-1b.etcd-events.additionalcidr.example.com" - "k8s.io/etcd/events" = "us-test-1b/us-test-1a,us-test-1b,us-test-1c" - "k8s.io/role/master" = "1" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - } - type = "gp2" -} - -resource "aws_ebs_volume" "us-test-1b-etcd-main-additionalcidr-example-com" { - availability_zone = "us-test-1b" - encrypted = false - size = 20 - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "us-test-1b.etcd-main.additionalcidr.example.com" - "k8s.io/etcd/main" = "us-test-1b/us-test-1a,us-test-1b,us-test-1c" - "k8s.io/role/master" = "1" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - } - type = "gp2" -} - -resource "aws_ebs_volume" "us-test-1c-etcd-events-additionalcidr-example-com" { - availability_zone = "us-test-1c" - encrypted = false - size = 20 - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "us-test-1c.etcd-events.additionalcidr.example.com" - "k8s.io/etcd/events" = "us-test-1c/us-test-1a,us-test-1b,us-test-1c" - "k8s.io/role/master" = "1" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - } - type = "gp2" -} - -resource "aws_ebs_volume" "us-test-1c-etcd-main-additionalcidr-example-com" { - availability_zone = "us-test-1c" - encrypted = false - size = 20 - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "us-test-1c.etcd-main.additionalcidr.example.com" - "k8s.io/etcd/main" = "us-test-1c/us-test-1a,us-test-1b,us-test-1c" - "k8s.io/role/master" = "1" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - } - type = "gp2" -} - -resource "aws_iam_instance_profile" "masters-additionalcidr-example-com" { - name = "masters.additionalcidr.example.com" - role = aws_iam_role.masters-additionalcidr-example-com.name -} - -resource "aws_iam_instance_profile" "nodes-additionalcidr-example-com" { - name = "nodes.additionalcidr.example.com" - role = aws_iam_role.nodes-additionalcidr-example-com.name -} - -resource "aws_iam_role_policy" "masters-additionalcidr-example-com" { - name = "masters.additionalcidr.example.com" - policy = file("${path.module}/data/aws_iam_role_policy_masters.additionalcidr.example.com_policy") - role = aws_iam_role.masters-additionalcidr-example-com.name -} - -resource "aws_iam_role_policy" "nodes-additionalcidr-example-com" { - name = "nodes.additionalcidr.example.com" - policy = file("${path.module}/data/aws_iam_role_policy_nodes.additionalcidr.example.com_policy") - role = aws_iam_role.nodes-additionalcidr-example-com.name -} - -resource "aws_iam_role" "masters-additionalcidr-example-com" { - assume_role_policy = file("${path.module}/data/aws_iam_role_masters.additionalcidr.example.com_policy") - name = "masters.additionalcidr.example.com" -} - -resource "aws_iam_role" "nodes-additionalcidr-example-com" { - assume_role_policy = file("${path.module}/data/aws_iam_role_nodes.additionalcidr.example.com_policy") - name = "nodes.additionalcidr.example.com" -} - -resource "aws_internet_gateway" "additionalcidr-example-com" { - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "additionalcidr.example.com" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - } - vpc_id = aws_vpc.additionalcidr-example-com.id -} - -resource "aws_key_pair" "kubernetes-additionalcidr-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" { - key_name = "kubernetes.additionalcidr.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57" - public_key = file("${path.module}/data/aws_key_pair_kubernetes.additionalcidr.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key") -} - -resource "aws_launch_template" "master-us-test-1a-masters-additionalcidr-example-com" { - block_device_mappings { - device_name = "/dev/xvda" - ebs { - delete_on_termination = true - volume_size = 64 - volume_type = "gp2" - } - } - block_device_mappings { - device_name = "/dev/sdc" - virtual_name = "ephemeral0" - } - iam_instance_profile { - name = aws_iam_instance_profile.masters-additionalcidr-example-com.id - } - image_id = "ami-12345678" - instance_type = "m3.medium" - key_name = aws_key_pair.kubernetes-additionalcidr-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id - lifecycle { - create_before_destroy = true - } - name_prefix = "master-us-test-1a.masters.additionalcidr.example.com-" - network_interfaces { - associate_public_ip_address = true - delete_on_termination = true - security_groups = [aws_security_group.masters-additionalcidr-example-com.id] - } - tag_specifications { - resource_type = "instance" - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "master-us-test-1a.masters.additionalcidr.example.com" - "k8s.io/role/master" = "1" - "kops.k8s.io/instancegroup" = "master-us-test-1a" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - } - } - tag_specifications { - resource_type = "volume" - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "master-us-test-1a.masters.additionalcidr.example.com" - "k8s.io/role/master" = "1" - "kops.k8s.io/instancegroup" = "master-us-test-1a" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - } - } - user_data = file("${path.module}/data/aws_launch_template_master-us-test-1a.masters.additionalcidr.example.com_user_data") -} - -resource "aws_launch_template" "master-us-test-1b-masters-additionalcidr-example-com" { - block_device_mappings { - device_name = "/dev/xvda" - ebs { - delete_on_termination = true - volume_size = 64 - volume_type = "gp2" - } - } - block_device_mappings { - device_name = "/dev/sdc" - virtual_name = "ephemeral0" - } - iam_instance_profile { - name = aws_iam_instance_profile.masters-additionalcidr-example-com.id - } - image_id = "ami-12345678" - instance_type = "m3.medium" - key_name = aws_key_pair.kubernetes-additionalcidr-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id - lifecycle { - create_before_destroy = true - } - name_prefix = "master-us-test-1b.masters.additionalcidr.example.com-" - network_interfaces { - associate_public_ip_address = true - delete_on_termination = true - security_groups = [aws_security_group.masters-additionalcidr-example-com.id] - } - tag_specifications { - resource_type = "instance" - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "master-us-test-1b.masters.additionalcidr.example.com" - "k8s.io/role/master" = "1" - "kops.k8s.io/instancegroup" = "master-us-test-1b" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - } - } - tag_specifications { - resource_type = "volume" - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "master-us-test-1b.masters.additionalcidr.example.com" - "k8s.io/role/master" = "1" - "kops.k8s.io/instancegroup" = "master-us-test-1b" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - } - } - user_data = file("${path.module}/data/aws_launch_template_master-us-test-1b.masters.additionalcidr.example.com_user_data") -} - -resource "aws_launch_template" "master-us-test-1c-masters-additionalcidr-example-com" { - block_device_mappings { - device_name = "/dev/xvda" - ebs { - delete_on_termination = true - volume_size = 64 - volume_type = "gp2" - } - } - block_device_mappings { - device_name = "/dev/sdc" - virtual_name = "ephemeral0" - } - iam_instance_profile { - name = aws_iam_instance_profile.masters-additionalcidr-example-com.id - } - image_id = "ami-12345678" - instance_type = "m3.medium" - key_name = aws_key_pair.kubernetes-additionalcidr-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id - lifecycle { - create_before_destroy = true - } - name_prefix = "master-us-test-1c.masters.additionalcidr.example.com-" - network_interfaces { - associate_public_ip_address = true - delete_on_termination = true - security_groups = [aws_security_group.masters-additionalcidr-example-com.id] - } - tag_specifications { - resource_type = "instance" - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "master-us-test-1c.masters.additionalcidr.example.com" - "k8s.io/role/master" = "1" - "kops.k8s.io/instancegroup" = "master-us-test-1c" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - } - } - tag_specifications { - resource_type = "volume" - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "master-us-test-1c.masters.additionalcidr.example.com" - "k8s.io/role/master" = "1" - "kops.k8s.io/instancegroup" = "master-us-test-1c" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - } - } - user_data = file("${path.module}/data/aws_launch_template_master-us-test-1c.masters.additionalcidr.example.com_user_data") -} - -resource "aws_launch_template" "nodes-additionalcidr-example-com" { - block_device_mappings { - device_name = "/dev/xvda" - ebs { - delete_on_termination = true - volume_size = 128 - volume_type = "gp2" - } - } - iam_instance_profile { - name = aws_iam_instance_profile.nodes-additionalcidr-example-com.id - } - image_id = "ami-12345678" - instance_type = "t2.medium" - key_name = aws_key_pair.kubernetes-additionalcidr-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id - lifecycle { - create_before_destroy = true - } - name_prefix = "nodes.additionalcidr.example.com-" - network_interfaces { - associate_public_ip_address = true - delete_on_termination = true - security_groups = [aws_security_group.nodes-additionalcidr-example-com.id] - } - tag_specifications { - resource_type = "instance" - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "nodes.additionalcidr.example.com" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "nodes" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - } - } - tag_specifications { - resource_type = "volume" - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "nodes.additionalcidr.example.com" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "nodes" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - } - } - user_data = file("${path.module}/data/aws_launch_template_nodes.additionalcidr.example.com_user_data") -} - -resource "aws_route_table_association" "us-test-1a-additionalcidr-example-com" { - route_table_id = aws_route_table.additionalcidr-example-com.id - subnet_id = aws_subnet.us-test-1a-additionalcidr-example-com.id -} - -resource "aws_route_table_association" "us-test-1b-additionalcidr-example-com" { - route_table_id = aws_route_table.additionalcidr-example-com.id - subnet_id = aws_subnet.us-test-1b-additionalcidr-example-com.id -} - -resource "aws_route_table_association" "us-test-1c-additionalcidr-example-com" { - route_table_id = aws_route_table.additionalcidr-example-com.id - subnet_id = aws_subnet.us-test-1c-additionalcidr-example-com.id -} - -resource "aws_route_table" "additionalcidr-example-com" { - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "additionalcidr.example.com" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - "kubernetes.io/kops/role" = "public" - } - vpc_id = aws_vpc.additionalcidr-example-com.id -} - -resource "aws_route" "route-0-0-0-0--0" { - destination_cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.additionalcidr-example-com.id - route_table_id = aws_route_table.additionalcidr-example-com.id -} - -resource "aws_security_group_rule" "all-master-to-master" { - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.masters-additionalcidr-example-com.id - source_security_group_id = aws_security_group.masters-additionalcidr-example-com.id - to_port = 0 - type = "ingress" -} - -resource "aws_security_group_rule" "all-master-to-node" { - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.nodes-additionalcidr-example-com.id - source_security_group_id = aws_security_group.masters-additionalcidr-example-com.id - to_port = 0 - type = "ingress" -} - -resource "aws_security_group_rule" "all-node-to-node" { - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.nodes-additionalcidr-example-com.id - source_security_group_id = aws_security_group.nodes-additionalcidr-example-com.id - to_port = 0 - type = "ingress" -} - -resource "aws_security_group_rule" "https-external-to-master-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 443 - protocol = "tcp" - security_group_id = aws_security_group.masters-additionalcidr-example-com.id - to_port = 443 - type = "ingress" -} - -resource "aws_security_group_rule" "master-egress" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.masters-additionalcidr-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "node-egress" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.nodes-additionalcidr-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "node-to-master-tcp-1-2379" { - from_port = 1 - protocol = "tcp" - security_group_id = aws_security_group.masters-additionalcidr-example-com.id - source_security_group_id = aws_security_group.nodes-additionalcidr-example-com.id - to_port = 2379 - type = "ingress" -} - -resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" { - from_port = 2382 - protocol = "tcp" - security_group_id = aws_security_group.masters-additionalcidr-example-com.id - source_security_group_id = aws_security_group.nodes-additionalcidr-example-com.id - to_port = 4000 - type = "ingress" -} - -resource "aws_security_group_rule" "node-to-master-tcp-4003-65535" { - from_port = 4003 - protocol = "tcp" - security_group_id = aws_security_group.masters-additionalcidr-example-com.id - source_security_group_id = aws_security_group.nodes-additionalcidr-example-com.id - to_port = 65535 - type = "ingress" -} - -resource "aws_security_group_rule" "node-to-master-udp-1-65535" { - from_port = 1 - protocol = "udp" - security_group_id = aws_security_group.masters-additionalcidr-example-com.id - source_security_group_id = aws_security_group.nodes-additionalcidr-example-com.id - to_port = 65535 - type = "ingress" -} - -resource "aws_security_group_rule" "ssh-external-to-master-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.masters-additionalcidr-example-com.id - to_port = 22 - type = "ingress" -} - -resource "aws_security_group_rule" "ssh-external-to-node-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.nodes-additionalcidr-example-com.id - to_port = 22 - type = "ingress" -} - -resource "aws_security_group" "masters-additionalcidr-example-com" { - description = "Security group for masters" - name = "masters.additionalcidr.example.com" - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "masters.additionalcidr.example.com" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - } - vpc_id = aws_vpc.additionalcidr-example-com.id -} - -resource "aws_security_group" "nodes-additionalcidr-example-com" { - description = "Security group for nodes" - name = "nodes.additionalcidr.example.com" - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "nodes.additionalcidr.example.com" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - } - vpc_id = aws_vpc.additionalcidr-example-com.id -} - -resource "aws_subnet" "us-test-1a-additionalcidr-example-com" { - availability_zone = "us-test-1a" - cidr_block = "10.0.1.0/24" - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "us-test-1a.additionalcidr.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - } - vpc_id = aws_vpc.additionalcidr-example-com.id -} - -resource "aws_subnet" "us-test-1b-additionalcidr-example-com" { - availability_zone = "us-test-1b" - cidr_block = "10.1.1.0/24" - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "us-test-1b.additionalcidr.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - } - vpc_id = aws_vpc.additionalcidr-example-com.id -} - -resource "aws_subnet" "us-test-1c-additionalcidr-example-com" { - availability_zone = "us-test-1c" - cidr_block = "10.1.2.0/24" - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "us-test-1c.additionalcidr.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - } - vpc_id = aws_vpc.additionalcidr-example-com.id -} - -resource "aws_vpc_dhcp_options_association" "additionalcidr-example-com" { - dhcp_options_id = aws_vpc_dhcp_options.additionalcidr-example-com.id - vpc_id = aws_vpc.additionalcidr-example-com.id -} - -resource "aws_vpc_dhcp_options" "additionalcidr-example-com" { - domain_name = "us-test-1.compute.internal" - domain_name_servers = ["AmazonProvidedDNS"] - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "additionalcidr.example.com" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - } -} - -resource "aws_vpc_ipv4_cidr_block_association" "cidr-10-1-0-0--16" { - cidr_block = "10.1.0.0/16" - vpc_id = aws_vpc.additionalcidr-example-com.id -} - -resource "aws_vpc" "additionalcidr-example-com" { - cidr_block = "10.0.0.0/16" - enable_dns_hostnames = true - enable_dns_support = true - tags = { - "KubernetesCluster" = "additionalcidr.example.com" - "Name" = "additionalcidr.example.com" - "kubernetes.io/cluster/additionalcidr.example.com" = "owned" - } -} - -terraform { - required_version = ">= 0.12.0" -} diff --git a/tests/integration/update_cluster/additional_user-data/cloudformation.json b/tests/integration/update_cluster/additional_user-data/cloudformation.json deleted file mode 100644 index 15309b3f1f..0000000000 --- a/tests/integration/update_cluster/additional_user-data/cloudformation.json +++ /dev/null @@ -1,1028 +0,0 @@ -{ - "Resources": { - "AWSAutoScalingAutoScalingGroupmasterustest1amastersadditionaluserdataexamplecom": { - "Type": "AWS::AutoScaling::AutoScalingGroup", - "Properties": { - "AutoScalingGroupName": "master-us-test-1a.masters.additionaluserdata.example.com", - "LaunchTemplate": { - "LaunchTemplateId": { - "Ref": "AWSEC2LaunchTemplatemasterustest1amastersadditionaluserdataexamplecom" - }, - "Version": { - "Fn::GetAtt": [ - "AWSEC2LaunchTemplatemasterustest1amastersadditionaluserdataexamplecom", - "LatestVersionNumber" - ] - } - }, - "MaxSize": 1, - "MinSize": 1, - "VPCZoneIdentifier": [ - { - "Ref": "AWSEC2Subnetustest1aadditionaluserdataexamplecom" - } - ], - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionaluserdata.example.com", - "PropagateAtLaunch": true - }, - { - "Key": "Name", - "Value": "master-us-test-1a.masters.additionaluserdata.example.com", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/role/master", - "Value": "1", - "PropagateAtLaunch": true - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "master-us-test-1a", - "PropagateAtLaunch": true - }, - { - "Key": "kubernetes.io/cluster/additionaluserdata.example.com", - "Value": "owned", - "PropagateAtLaunch": true - } - ], - "MetricsCollection": [ - { - "Granularity": "1Minute", - "Metrics": [ - "GroupDesiredCapacity", - "GroupInServiceInstances", - "GroupMaxSize", - "GroupMinSize", - "GroupPendingInstances", - "GroupStandbyInstances", - "GroupTerminatingInstances", - "GroupTotalInstances" - ] - } - ] - } - }, - "AWSAutoScalingAutoScalingGroupnodesadditionaluserdataexamplecom": { - "Type": "AWS::AutoScaling::AutoScalingGroup", - "Properties": { - "AutoScalingGroupName": "nodes.additionaluserdata.example.com", - "LaunchTemplate": { - "LaunchTemplateId": { - "Ref": "AWSEC2LaunchTemplatenodesadditionaluserdataexamplecom" - }, - "Version": { - "Fn::GetAtt": [ - "AWSEC2LaunchTemplatenodesadditionaluserdataexamplecom", - "LatestVersionNumber" - ] - } - }, - "MaxSize": 2, - "MinSize": 2, - "VPCZoneIdentifier": [ - { - "Ref": "AWSEC2Subnetustest1aadditionaluserdataexamplecom" - } - ], - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionaluserdata.example.com", - "PropagateAtLaunch": true - }, - { - "Key": "Name", - "Value": "nodes.additionaluserdata.example.com", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/role/node", - "Value": "1", - "PropagateAtLaunch": true - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "nodes", - "PropagateAtLaunch": true - }, - { - "Key": "kubernetes.io/cluster/additionaluserdata.example.com", - "Value": "owned", - "PropagateAtLaunch": true - } - ], - "MetricsCollection": [ - { - "Granularity": "1Minute", - "Metrics": [ - "GroupDesiredCapacity", - "GroupInServiceInstances", - "GroupMaxSize", - "GroupMinSize", - "GroupPendingInstances", - "GroupStandbyInstances", - "GroupTerminatingInstances", - "GroupTotalInstances" - ] - } - ] - } - }, - "AWSEC2DHCPOptionsadditionaluserdataexamplecom": { - "Type": "AWS::EC2::DHCPOptions", - "Properties": { - "DomainName": "us-test-1.compute.internal", - "DomainNameServers": [ - "AmazonProvidedDNS" - ], - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionaluserdata.example.com" - }, - { - "Key": "Name", - "Value": "additionaluserdata.example.com" - }, - { - "Key": "kubernetes.io/cluster/additionaluserdata.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2InternetGatewayadditionaluserdataexamplecom": { - "Type": "AWS::EC2::InternetGateway", - "Properties": { - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionaluserdata.example.com" - }, - { - "Key": "Name", - "Value": "additionaluserdata.example.com" - }, - { - "Key": "kubernetes.io/cluster/additionaluserdata.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2LaunchTemplatemasterustest1amastersadditionaluserdataexamplecom": { - "Type": "AWS::EC2::LaunchTemplate", - "Properties": { - "LaunchTemplateName": "master-us-test-1a.masters.additionaluserdata.example.com", - "LaunchTemplateData": { - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/xvda", - "Ebs": { - "VolumeType": "gp2", - "VolumeSize": 64, - "DeleteOnTermination": true - } - }, - { - "DeviceName": "/dev/sdc", - "VirtualName": "ephemeral0" - } - ], - "IamInstanceProfile": { - "Name": { - "Ref": "AWSIAMInstanceProfilemastersadditionaluserdataexamplecom" - } - }, - "ImageId": "ami-12345678", - "InstanceType": "m3.medium", - "KeyName": "kubernetes.additionaluserdata.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", - "NetworkInterfaces": [ - { - "AssociatePublicIpAddress": true, - "DeleteOnTermination": true, - "DeviceIndex": 0, - "Groups": [ - { - "Ref": "AWSEC2SecurityGroupmastersadditionaluserdataexamplecom" - } - ] - } - ], - "TagSpecifications": [ - { - "ResourceType": "instance", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionaluserdata.example.com" - }, - { - "Key": "Name", - "Value": "master-us-test-1a.masters.additionaluserdata.example.com" - }, - { - "Key": "k8s.io/role/master", - "Value": "1" - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "master-us-test-1a" - }, - { - "Key": "kubernetes.io/cluster/additionaluserdata.example.com", - "Value": "owned" - } - ] - }, - { - "ResourceType": "volume", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionaluserdata.example.com" - }, - { - "Key": "Name", - "Value": "master-us-test-1a.masters.additionaluserdata.example.com" - }, - { - "Key": "k8s.io/role/master", - "Value": "1" - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "master-us-test-1a" - }, - { - "Key": "kubernetes.io/cluster/additionaluserdata.example.com", - "Value": "owned" - } - ] - } - ], - "UserData": "extracted" - } - } - }, - "AWSEC2LaunchTemplatenodesadditionaluserdataexamplecom": { - "Type": "AWS::EC2::LaunchTemplate", - "Properties": { - "LaunchTemplateName": "nodes.additionaluserdata.example.com", - "LaunchTemplateData": { - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/xvda", - "Ebs": { - "VolumeType": "gp2", - "VolumeSize": 128, - "DeleteOnTermination": true - } - } - ], - "IamInstanceProfile": { - "Name": { - "Ref": "AWSIAMInstanceProfilenodesadditionaluserdataexamplecom" - } - }, - "ImageId": "ami-12345678", - "InstanceType": "t2.medium", - "KeyName": "kubernetes.additionaluserdata.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", - "NetworkInterfaces": [ - { - "AssociatePublicIpAddress": true, - "DeleteOnTermination": true, - "DeviceIndex": 0, - "Groups": [ - { - "Ref": "AWSEC2SecurityGroupnodesadditionaluserdataexamplecom" - } - ] - } - ], - "TagSpecifications": [ - { - "ResourceType": "instance", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionaluserdata.example.com" - }, - { - "Key": "Name", - "Value": "nodes.additionaluserdata.example.com" - }, - { - "Key": "k8s.io/role/node", - "Value": "1" - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "nodes" - }, - { - "Key": "kubernetes.io/cluster/additionaluserdata.example.com", - "Value": "owned" - } - ] - }, - { - "ResourceType": "volume", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionaluserdata.example.com" - }, - { - "Key": "Name", - "Value": "nodes.additionaluserdata.example.com" - }, - { - "Key": "k8s.io/role/node", - "Value": "1" - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "nodes" - }, - { - "Key": "kubernetes.io/cluster/additionaluserdata.example.com", - "Value": "owned" - } - ] - } - ], - "UserData": "extracted" - } - } - }, - "AWSEC2Route00000": { - "Type": "AWS::EC2::Route", - "Properties": { - "RouteTableId": { - "Ref": "AWSEC2RouteTableadditionaluserdataexamplecom" - }, - "DestinationCidrBlock": "0.0.0.0/0", - "GatewayId": { - "Ref": "AWSEC2InternetGatewayadditionaluserdataexamplecom" - } - } - }, - "AWSEC2RouteTableadditionaluserdataexamplecom": { - "Type": "AWS::EC2::RouteTable", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPCadditionaluserdataexamplecom" - }, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionaluserdata.example.com" - }, - { - "Key": "Name", - "Value": "additionaluserdata.example.com" - }, - { - "Key": "kubernetes.io/cluster/additionaluserdata.example.com", - "Value": "owned" - }, - { - "Key": "kubernetes.io/kops/role", - "Value": "public" - } - ] - } - }, - "AWSEC2SecurityGroupEgressmasteregress": { - "Type": "AWS::EC2::SecurityGroupEgress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersadditionaluserdataexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1", - "CidrIp": "0.0.0.0/0" - } - }, - "AWSEC2SecurityGroupEgressnodeegress": { - "Type": "AWS::EC2::SecurityGroupEgress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupnodesadditionaluserdataexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1", - "CidrIp": "0.0.0.0/0" - } - }, - "AWSEC2SecurityGroupIngressallmastertomaster": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersadditionaluserdataexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupmastersadditionaluserdataexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1" - } - }, - "AWSEC2SecurityGroupIngressallmastertonode": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupnodesadditionaluserdataexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupmastersadditionaluserdataexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1" - } - }, - "AWSEC2SecurityGroupIngressallnodetonode": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupnodesadditionaluserdataexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodesadditionaluserdataexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1" - } - }, - "AWSEC2SecurityGroupIngresshttpsexternaltomaster00000": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersadditionaluserdataexamplecom" - }, - "FromPort": 443, - "ToPort": 443, - "IpProtocol": "tcp", - "CidrIp": "0.0.0.0/0" - } - }, - "AWSEC2SecurityGroupIngressnodetomastertcp12379": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersadditionaluserdataexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodesadditionaluserdataexamplecom" - }, - "FromPort": 1, - "ToPort": 2379, - "IpProtocol": "tcp" - } - }, - "AWSEC2SecurityGroupIngressnodetomastertcp23824000": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersadditionaluserdataexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodesadditionaluserdataexamplecom" - }, - "FromPort": 2382, - "ToPort": 4000, - "IpProtocol": "tcp" - } - }, - "AWSEC2SecurityGroupIngressnodetomastertcp400365535": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersadditionaluserdataexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodesadditionaluserdataexamplecom" - }, - "FromPort": 4003, - "ToPort": 65535, - "IpProtocol": "tcp" - } - }, - "AWSEC2SecurityGroupIngressnodetomasterudp165535": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersadditionaluserdataexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodesadditionaluserdataexamplecom" - }, - "FromPort": 1, - "ToPort": 65535, - "IpProtocol": "udp" - } - }, - "AWSEC2SecurityGroupIngresssshexternaltomaster00000": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersadditionaluserdataexamplecom" - }, - "FromPort": 22, - "ToPort": 22, - "IpProtocol": "tcp", - "CidrIp": "0.0.0.0/0" - } - }, - "AWSEC2SecurityGroupIngresssshexternaltonode00000": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupnodesadditionaluserdataexamplecom" - }, - "FromPort": 22, - "ToPort": 22, - "IpProtocol": "tcp", - "CidrIp": "0.0.0.0/0" - } - }, - "AWSEC2SecurityGroupmastersadditionaluserdataexamplecom": { - "Type": "AWS::EC2::SecurityGroup", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPCadditionaluserdataexamplecom" - }, - "GroupDescription": "Security group for masters", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionaluserdata.example.com" - }, - { - "Key": "Name", - "Value": "masters.additionaluserdata.example.com" - }, - { - "Key": "kubernetes.io/cluster/additionaluserdata.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2SecurityGroupnodesadditionaluserdataexamplecom": { - "Type": "AWS::EC2::SecurityGroup", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPCadditionaluserdataexamplecom" - }, - "GroupDescription": "Security group for nodes", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionaluserdata.example.com" - }, - { - "Key": "Name", - "Value": "nodes.additionaluserdata.example.com" - }, - { - "Key": "kubernetes.io/cluster/additionaluserdata.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2SubnetRouteTableAssociationustest1aadditionaluserdataexamplecom": { - "Type": "AWS::EC2::SubnetRouteTableAssociation", - "Properties": { - "SubnetId": { - "Ref": "AWSEC2Subnetustest1aadditionaluserdataexamplecom" - }, - "RouteTableId": { - "Ref": "AWSEC2RouteTableadditionaluserdataexamplecom" - } - } - }, - "AWSEC2Subnetustest1aadditionaluserdataexamplecom": { - "Type": "AWS::EC2::Subnet", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPCadditionaluserdataexamplecom" - }, - "CidrBlock": "172.20.32.0/19", - "AvailabilityZone": "us-test-1a", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionaluserdata.example.com" - }, - { - "Key": "Name", - "Value": "us-test-1a.additionaluserdata.example.com" - }, - { - "Key": "SubnetType", - "Value": "Public" - }, - { - "Key": "kubernetes.io/cluster/additionaluserdata.example.com", - "Value": "owned" - }, - { - "Key": "kubernetes.io/role/elb", - "Value": "1" - } - ] - } - }, - "AWSEC2VPCDHCPOptionsAssociationadditionaluserdataexamplecom": { - "Type": "AWS::EC2::VPCDHCPOptionsAssociation", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPCadditionaluserdataexamplecom" - }, - "DhcpOptionsId": { - "Ref": "AWSEC2DHCPOptionsadditionaluserdataexamplecom" - } - } - }, - "AWSEC2VPCGatewayAttachmentadditionaluserdataexamplecom": { - "Type": "AWS::EC2::VPCGatewayAttachment", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPCadditionaluserdataexamplecom" - }, - "InternetGatewayId": { - "Ref": "AWSEC2InternetGatewayadditionaluserdataexamplecom" - } - } - }, - "AWSEC2VPCadditionaluserdataexamplecom": { - "Type": "AWS::EC2::VPC", - "Properties": { - "CidrBlock": "172.20.0.0/16", - "EnableDnsHostnames": true, - "EnableDnsSupport": true, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionaluserdata.example.com" - }, - { - "Key": "Name", - "Value": "additionaluserdata.example.com" - }, - { - "Key": "kubernetes.io/cluster/additionaluserdata.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2Volumeustest1aetcdeventsadditionaluserdataexamplecom": { - "Type": "AWS::EC2::Volume", - "Properties": { - "AvailabilityZone": "us-test-1a", - "Size": 20, - "VolumeType": "gp2", - "Encrypted": false, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionaluserdata.example.com" - }, - { - "Key": "Name", - "Value": "us-test-1a.etcd-events.additionaluserdata.example.com" - }, - { - "Key": "k8s.io/etcd/events", - "Value": "us-test-1a/us-test-1a" - }, - { - "Key": "k8s.io/role/master", - "Value": "1" - }, - { - "Key": "kubernetes.io/cluster/additionaluserdata.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2Volumeustest1aetcdmainadditionaluserdataexamplecom": { - "Type": "AWS::EC2::Volume", - "Properties": { - "AvailabilityZone": "us-test-1a", - "Size": 20, - "VolumeType": "gp2", - "Encrypted": false, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "additionaluserdata.example.com" - }, - { - "Key": "Name", - "Value": "us-test-1a.etcd-main.additionaluserdata.example.com" - }, - { - "Key": "k8s.io/etcd/main", - "Value": "us-test-1a/us-test-1a" - }, - { - "Key": "k8s.io/role/master", - "Value": "1" - }, - { - "Key": "kubernetes.io/cluster/additionaluserdata.example.com", - "Value": "owned" - } - ] - } - }, - "AWSIAMInstanceProfilemastersadditionaluserdataexamplecom": { - "Type": "AWS::IAM::InstanceProfile", - "Properties": { - "Roles": [ - { - "Ref": "AWSIAMRolemastersadditionaluserdataexamplecom" - } - ] - } - }, - "AWSIAMInstanceProfilenodesadditionaluserdataexamplecom": { - "Type": "AWS::IAM::InstanceProfile", - "Properties": { - "Roles": [ - { - "Ref": "AWSIAMRolenodesadditionaluserdataexamplecom" - } - ] - } - }, - "AWSIAMPolicyadditionalmastersadditionaluserdataexamplecom": { - "Type": "AWS::IAM::Policy", - "Properties": { - "PolicyName": "additional.masters.additionaluserdata.example.com", - "Roles": [ - { - "Ref": "AWSIAMRolemastersadditionaluserdataexamplecom" - } - ], - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "s3:GetObject" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:s3:::somebucket/someobject" - ] - } - ], - "Version": "2012-10-17" - } - } - }, - "AWSIAMPolicymastersadditionaluserdataexamplecom": { - "Type": "AWS::IAM::Policy", - "Properties": { - "PolicyName": "masters.additionaluserdata.example.com", - "Roles": [ - { - "Ref": "AWSIAMRolemastersadditionaluserdataexamplecom" - } - ], - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "ec2:*" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:DescribeAutoScalingInstances", - "autoscaling:DescribeLaunchConfigurations", - "autoscaling:DescribeTags", - "autoscaling:SetDesiredCapacity", - "autoscaling:TerminateInstanceInAutoScalingGroup", - "autoscaling:UpdateAutoScalingGroup", - "ec2:DescribeLaunchTemplateVersions" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "elasticloadbalancing:*" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "iam:ListServerCertificates", - "iam:GetServerCertificate" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "route53:ChangeResourceRecordSets", - "route53:ListResourceRecordSets", - "route53:GetHostedZone" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" - ] - }, - { - "Action": [ - "route53:GetChange" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:route53:::change/*" - ] - }, - { - "Action": [ - "route53:ListHostedZones" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "route53:ListHostedZones" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "ecr:GetAuthorizationToken", - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:GetRepositoryPolicy", - "ecr:DescribeRepositories", - "ecr:ListImages", - "ecr:BatchGetImage" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - } - ], - "Version": "2012-10-17" - } - } - }, - "AWSIAMPolicynodesadditionaluserdataexamplecom": { - "Type": "AWS::IAM::Policy", - "Properties": { - "PolicyName": "nodes.additionaluserdata.example.com", - "Roles": [ - { - "Ref": "AWSIAMRolenodesadditionaluserdataexamplecom" - } - ], - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "ec2:DescribeInstances", - "ec2:DescribeRegions" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "route53:ChangeResourceRecordSets", - "route53:ListResourceRecordSets", - "route53:GetHostedZone" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" - ] - }, - { - "Action": [ - "route53:GetChange" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:route53:::change/*" - ] - }, - { - "Action": [ - "route53:ListHostedZones" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "route53:ListHostedZones" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "ecr:GetAuthorizationToken", - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:GetRepositoryPolicy", - "ecr:DescribeRepositories", - "ecr:ListImages", - "ecr:BatchGetImage" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - } - ], - "Version": "2012-10-17" - } - } - }, - "AWSIAMRolemastersadditionaluserdataexamplecom": { - "Type": "AWS::IAM::Role", - "Properties": { - "RoleName": "masters.additionaluserdata.example.com", - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "ec2.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - } - } - }, - "AWSIAMRolenodesadditionaluserdataexamplecom": { - "Type": "AWS::IAM::Role", - "Properties": { - "RoleName": "nodes.additionaluserdata.example.com", - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "ec2.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - } - } - } - } -} diff --git a/tests/integration/update_cluster/additional_user-data/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/additional_user-data/cloudformation.json.extracted.yaml deleted file mode 100644 index fee350d2b2..0000000000 --- a/tests/integration/update_cluster/additional_user-data/cloudformation.json.extracted.yaml +++ /dev/null @@ -1,577 +0,0 @@ -Resources.AWSEC2LaunchTemplatemasterustest1amastersadditionaluserdataexamplecom.Properties.LaunchTemplateData.UserData: | - Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" - MIME-Version: 1.0 - - --MIMEBOUNDARY - Content-Disposition: attachment; filename="nodeup.sh" - Content-Transfer-Encoding: 7bit - Content-Type: text/x-shellscript - Mime-Version: 1.0 - - #!/bin/bash - # Copyright 2016 The Kubernetes Authors All rights reserved. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - set -o errexit - set -o nounset - set -o pipefail - - NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup - NODEUP_HASH_AMD64=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 - NODEUP_URL_ARM64= - NODEUP_HASH_ARM64= - - export AWS_REGION=us-test-1 - - - - - function ensure-install-dir() { - INSTALL_DIR="/opt/kops" - # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec - if [[ -d /var/lib/toolbox ]]; then - INSTALL_DIR="/var/lib/toolbox/kops" - fi - mkdir -p ${INSTALL_DIR}/bin - mkdir -p ${INSTALL_DIR}/conf - cd ${INSTALL_DIR} - } - - # Retry a download until we get it. args: name, sha, url1, url2... - download-or-bust() { - local -r file="$1" - local -r hash="$2" - shift 2 - - urls=( $* ) - while true; do - for url in "${urls[@]}"; do - commands=( - "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - ) - for cmd in "${commands[@]}"; do - echo "Attempting download with: ${cmd} {url}" - if ! (${cmd} "${url}"); then - echo "== Download failed with ${cmd} ==" - continue - fi - if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then - echo "== Hash validation of ${url} failed. Retrying. ==" - rm -f "${file}" - else - if [[ -n "${hash}" ]]; then - echo "== Downloaded ${url} (SHA1 = ${hash}) ==" - else - echo "== Downloaded ${url} ==" - fi - return - fi - done - done - - echo "All downloads failed; sleeping before retrying" - sleep 60 - done - } - - validate-hash() { - local -r file="$1" - local -r expected="$2" - local actual - - actual=$(sha256sum ${file} | awk '{ print $1 }') || true - if [[ "${actual}" != "${expected}" ]]; then - echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" - return 1 - fi - } - - function split-commas() { - echo $1 | tr "," "\n" - } - - function try-download-release() { - local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) - if [[ -n "${NODEUP_HASH:-}" ]]; then - local -r nodeup_hash="${NODEUP_HASH}" - else - # TODO: Remove? - echo "Downloading sha256 (not found in env)" - download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" - local -r nodeup_hash=$(cat nodeup.sha256) - fi - - echo "Downloading nodeup (${nodeup_urls[@]})" - download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" - - chmod +x nodeup - } - - function download-release() { - case "$(uname -m)" in - x86_64*|i?86_64*|amd64*) - NODEUP_URL="${NODEUP_URL_AMD64}" - NODEUP_HASH="${NODEUP_HASH_AMD64}" - ;; - aarch64*|arm64*) - NODEUP_URL="${NODEUP_URL_ARM64}" - NODEUP_HASH="${NODEUP_HASH_ARM64}" - ;; - *) - echo "Unsupported host arch: $(uname -m)" >&2 - exit 1 - ;; - esac - - # In case of failure checking integrity of release, retry. - cd ${INSTALL_DIR}/bin - until try-download-release; do - sleep 15 - echo "Couldn't download release. Retrying..." - done - - echo "Running nodeup" - # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 - ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) - } - - #################################################################################### - - /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" - - echo "== nodeup node config starting ==" - ensure-install-dir - - cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' - cloudConfig: null - containerRuntime: docker - containerd: - skipInstall: true - docker: - ipMasq: false - ipTables: false - logDriver: json-file - logLevel: info - logOpt: - - max-size=10m - - max-file=5 - storage: overlay2,overlay,aufs - version: 18.06.3 - encryptionConfig: null - etcdClusters: - events: - version: 3.3.10 - main: - version: 3.3.10 - kubeAPIServer: - allowPrivileged: true - anonymousAuth: false - apiServerCount: 1 - authorizationMode: AlwaysAllow - bindAddress: 0.0.0.0 - cloudProvider: aws - enableAdmissionPlugins: - - NamespaceLifecycle - - LimitRanger - - ServiceAccount - - PersistentVolumeLabel - - DefaultStorageClass - - DefaultTolerationSeconds - - MutatingAdmissionWebhook - - ValidatingAdmissionWebhook - - NodeRestriction - - ResourceQuota - etcdServers: - - http://127.0.0.1:4001 - etcdServersOverrides: - - /events#http://127.0.0.1:4002 - image: k8s.gcr.io/kube-apiserver:v1.14.0 - insecureBindAddress: 127.0.0.1 - insecurePort: 8080 - kubeletPreferredAddressTypes: - - InternalIP - - Hostname - - ExternalIP - logLevel: 2 - requestheaderAllowedNames: - - aggregator - requestheaderExtraHeaderPrefixes: - - X-Remote-Extra- - requestheaderGroupHeaders: - - X-Remote-Group - requestheaderUsernameHeaders: - - X-Remote-User - securePort: 443 - serviceClusterIPRange: 100.64.0.0/13 - storageBackend: etcd3 - kubeControllerManager: - allocateNodeCIDRs: true - attachDetachReconcileSyncPeriod: 1m0s - cloudProvider: aws - clusterCIDR: 100.96.0.0/11 - clusterName: additionaluserdata.example.com - configureCloudRoutes: true - image: k8s.gcr.io/kube-controller-manager:v1.14.0 - leaderElection: - leaderElect: true - logLevel: 2 - useServiceAccountCredentials: true - kubeProxy: - clusterCIDR: 100.96.0.0/11 - cpuRequest: 100m - hostnameOverride: '@aws' - image: k8s.gcr.io/kube-proxy:v1.14.0 - logLevel: 2 - kubeScheduler: - image: k8s.gcr.io/kube-scheduler:v1.14.0 - leaderElection: - leaderElect: true - logLevel: 2 - kubelet: - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests - masterKubelet: - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests - registerSchedulable: false - - __EOF_CLUSTER_SPEC - - cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' - kubelet: null - nodeLabels: null - taints: null - - __EOF_IG_SPEC - - cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' - Assets: - amd64: - - c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet - - 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl - - 3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz - arm64: - - df38e04576026393055ccc77c0dce73612996561@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubelet - - 01c2b6b43d36b6bfafc80a3737391c19ebfb8ad5@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubectl - - 7fec91af78e9548df306f0ec43bea527c8c10cc3a9682c33e971c8522a7fcded@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-arm64-v0.7.5.tgz - ClusterName: additionaluserdata.example.com - ConfigBase: memfs://clusters.example.com/additionaluserdata.example.com - InstanceGroupName: master-us-test-1a - InstanceGroupRole: Master - Tags: - - _automatic_upgrades - - _aws - channels: - - memfs://clusters.example.com/additionaluserdata.example.com/addons/bootstrap-channel.yaml - etcdManifests: - - memfs://clusters.example.com/additionaluserdata.example.com/manifests/etcd/main.yaml - - memfs://clusters.example.com/additionaluserdata.example.com/manifests/etcd/events.yaml - protokubeImage: - hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b - name: protokube:1.15.0 - sources: - - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz - - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz - - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz - - __EOF_KUBE_ENV - - download-release - echo "== nodeup node config done ==" - - --MIMEBOUNDARY - Content-Disposition: attachment; filename="myscript.sh" - Content-Transfer-Encoding: 7bit - Content-Type: text/x-shellscript - Mime-Version: 1.0 - - #!/bin/sh - echo "master: The time is now $(date -R)!" | tee /root/output.txt - - --MIMEBOUNDARY-- -Resources.AWSEC2LaunchTemplatenodesadditionaluserdataexamplecom.Properties.LaunchTemplateData.UserData: | - Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" - MIME-Version: 1.0 - - --MIMEBOUNDARY - Content-Disposition: attachment; filename="nodeup.sh" - Content-Transfer-Encoding: 7bit - Content-Type: text/x-shellscript - Mime-Version: 1.0 - - #!/bin/bash - # Copyright 2016 The Kubernetes Authors All rights reserved. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - set -o errexit - set -o nounset - set -o pipefail - - NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup - NODEUP_HASH_AMD64=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 - NODEUP_URL_ARM64= - NODEUP_HASH_ARM64= - - export AWS_REGION=us-test-1 - - - - - function ensure-install-dir() { - INSTALL_DIR="/opt/kops" - # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec - if [[ -d /var/lib/toolbox ]]; then - INSTALL_DIR="/var/lib/toolbox/kops" - fi - mkdir -p ${INSTALL_DIR}/bin - mkdir -p ${INSTALL_DIR}/conf - cd ${INSTALL_DIR} - } - - # Retry a download until we get it. args: name, sha, url1, url2... - download-or-bust() { - local -r file="$1" - local -r hash="$2" - shift 2 - - urls=( $* ) - while true; do - for url in "${urls[@]}"; do - commands=( - "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - ) - for cmd in "${commands[@]}"; do - echo "Attempting download with: ${cmd} {url}" - if ! (${cmd} "${url}"); then - echo "== Download failed with ${cmd} ==" - continue - fi - if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then - echo "== Hash validation of ${url} failed. Retrying. ==" - rm -f "${file}" - else - if [[ -n "${hash}" ]]; then - echo "== Downloaded ${url} (SHA1 = ${hash}) ==" - else - echo "== Downloaded ${url} ==" - fi - return - fi - done - done - - echo "All downloads failed; sleeping before retrying" - sleep 60 - done - } - - validate-hash() { - local -r file="$1" - local -r expected="$2" - local actual - - actual=$(sha256sum ${file} | awk '{ print $1 }') || true - if [[ "${actual}" != "${expected}" ]]; then - echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" - return 1 - fi - } - - function split-commas() { - echo $1 | tr "," "\n" - } - - function try-download-release() { - local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) - if [[ -n "${NODEUP_HASH:-}" ]]; then - local -r nodeup_hash="${NODEUP_HASH}" - else - # TODO: Remove? - echo "Downloading sha256 (not found in env)" - download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" - local -r nodeup_hash=$(cat nodeup.sha256) - fi - - echo "Downloading nodeup (${nodeup_urls[@]})" - download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" - - chmod +x nodeup - } - - function download-release() { - case "$(uname -m)" in - x86_64*|i?86_64*|amd64*) - NODEUP_URL="${NODEUP_URL_AMD64}" - NODEUP_HASH="${NODEUP_HASH_AMD64}" - ;; - aarch64*|arm64*) - NODEUP_URL="${NODEUP_URL_ARM64}" - NODEUP_HASH="${NODEUP_HASH_ARM64}" - ;; - *) - echo "Unsupported host arch: $(uname -m)" >&2 - exit 1 - ;; - esac - - # In case of failure checking integrity of release, retry. - cd ${INSTALL_DIR}/bin - until try-download-release; do - sleep 15 - echo "Couldn't download release. Retrying..." - done - - echo "Running nodeup" - # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 - ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) - } - - #################################################################################### - - /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" - - echo "== nodeup node config starting ==" - ensure-install-dir - - cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' - cloudConfig: null - containerRuntime: docker - containerd: - skipInstall: true - docker: - ipMasq: false - ipTables: false - logDriver: json-file - logLevel: info - logOpt: - - max-size=10m - - max-file=5 - storage: overlay2,overlay,aufs - version: 18.06.3 - kubeProxy: - clusterCIDR: 100.96.0.0/11 - cpuRequest: 100m - hostnameOverride: '@aws' - image: k8s.gcr.io/kube-proxy:v1.14.0 - logLevel: 2 - kubelet: - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests - - __EOF_CLUSTER_SPEC - - cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' - kubelet: null - nodeLabels: null - taints: null - - __EOF_IG_SPEC - - cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' - Assets: - amd64: - - c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet - - 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl - - 3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz - arm64: - - df38e04576026393055ccc77c0dce73612996561@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubelet - - 01c2b6b43d36b6bfafc80a3737391c19ebfb8ad5@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubectl - - 7fec91af78e9548df306f0ec43bea527c8c10cc3a9682c33e971c8522a7fcded@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-arm64-v0.7.5.tgz - ClusterName: additionaluserdata.example.com - ConfigBase: memfs://clusters.example.com/additionaluserdata.example.com - InstanceGroupName: nodes - InstanceGroupRole: Node - Tags: - - _automatic_upgrades - - _aws - channels: - - memfs://clusters.example.com/additionaluserdata.example.com/addons/bootstrap-channel.yaml - - __EOF_KUBE_ENV - - download-release - echo "== nodeup node config done ==" - - --MIMEBOUNDARY - Content-Disposition: attachment; filename="myscript.sh" - Content-Transfer-Encoding: 7bit - Content-Type: text/x-shellscript - Mime-Version: 1.0 - - #!/bin/sh - echo "nodes: The time is now $(date -R)!" | tee /root/output.txt - - --MIMEBOUNDARY-- diff --git a/tests/integration/update_cluster/additional_user-data/id_rsa.pub b/tests/integration/update_cluster/additional_user-data/id_rsa.pub deleted file mode 100755 index 81cb012783..0000000000 --- a/tests/integration/update_cluster/additional_user-data/id_rsa.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/additional_user-data/in-v1alpha2.yaml b/tests/integration/update_cluster/additional_user-data/in-v1alpha2.yaml deleted file mode 100644 index 1be6152eae..0000000000 --- a/tests/integration/update_cluster/additional_user-data/in-v1alpha2.yaml +++ /dev/null @@ -1,99 +0,0 @@ -apiVersion: kops.k8s.io/v1alpha2 -kind: Cluster -metadata: - creationTimestamp: "2016-12-10T22:42:27Z" - name: additionaluserdata.example.com -spec: - additionalPolicies: - master: | - [ - { - "Action": [ "s3:GetObject" ], - "Resource": [ "arn:aws:s3:::somebucket/someobject" ], - "Effect": "Allow" - } - ] - kubernetesApiAccess: - - 0.0.0.0/0 - channel: stable - cloudProvider: aws - configBase: memfs://clusters.example.com/additionaluserdata.example.com - etcdClusters: - - etcdMembers: - - instanceGroup: master-us-test-1a - name: us-test-1a - name: main - - etcdMembers: - - instanceGroup: master-us-test-1a - name: us-test-1a - name: events - kubelet: - anonymousAuth: false - kubernetesVersion: v1.14.0 - masterInternalName: api.internal.additionaluserdata.example.com - masterPublicName: api.additionaluserdata.example.com - networkCIDR: 172.20.0.0/16 - networking: - kubenet: {} - nonMasqueradeCIDR: 100.64.0.0/10 - sshAccess: - - 0.0.0.0/0 - topology: - masters: public - nodes: public - subnets: - - cidr: 172.20.32.0/19 - name: us-test-1a - type: Public - zone: us-test-1a - ---- - -apiVersion: kops.k8s.io/v1alpha2 -kind: InstanceGroup -metadata: - creationTimestamp: "2016-12-10T22:42:28Z" - name: nodes - labels: - kops.k8s.io/cluster: additionaluserdata.example.com -spec: - associatePublicIp: true - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: t2.medium - maxSize: 2 - minSize: 2 - role: Node - subnets: - - us-test-1a - additionalUserData: - - name: myscript.sh - type: text/x-shellscript - content: | - #!/bin/sh - echo "nodes: The time is now $(date -R)!" | tee /root/output.txt - ---- - -apiVersion: kops.k8s.io/v1alpha2 -kind: InstanceGroup -metadata: - creationTimestamp: "2016-12-10T22:42:28Z" - name: master-us-test-1a - labels: - kops.k8s.io/cluster: additionaluserdata.example.com -spec: - associatePublicIp: true - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: m3.medium - maxSize: 1 - minSize: 1 - role: Master - subnets: - - us-test-1a - additionalUserData: - - name: myscript.sh - type: text/x-shellscript - content: | - #!/bin/sh - echo "master: The time is now $(date -R)!" | tee /root/output.txt - diff --git a/tests/integration/update_cluster/api_elb_cross_zone/data/aws_iam_role_masters.crosszone.example.com_policy b/tests/integration/update_cluster/api_elb_cross_zone/data/aws_iam_role_masters.crosszone.example.com_policy deleted file mode 100644 index 66d5de1d5a..0000000000 --- a/tests/integration/update_cluster/api_elb_cross_zone/data/aws_iam_role_masters.crosszone.example.com_policy +++ /dev/null @@ -1,10 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { "Service": "ec2.amazonaws.com"}, - "Action": "sts:AssumeRole" - } - ] -} diff --git a/tests/integration/update_cluster/api_elb_cross_zone/data/aws_iam_role_nodes.crosszone.example.com_policy b/tests/integration/update_cluster/api_elb_cross_zone/data/aws_iam_role_nodes.crosszone.example.com_policy deleted file mode 100644 index 66d5de1d5a..0000000000 --- a/tests/integration/update_cluster/api_elb_cross_zone/data/aws_iam_role_nodes.crosszone.example.com_policy +++ /dev/null @@ -1,10 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { "Service": "ec2.amazonaws.com"}, - "Action": "sts:AssumeRole" - } - ] -} diff --git a/tests/integration/update_cluster/api_elb_cross_zone/data/aws_iam_role_policy_masters.crosszone.example.com_policy b/tests/integration/update_cluster/api_elb_cross_zone/data/aws_iam_role_policy_masters.crosszone.example.com_policy deleted file mode 100644 index 340dff1ef9..0000000000 --- a/tests/integration/update_cluster/api_elb_cross_zone/data/aws_iam_role_policy_masters.crosszone.example.com_policy +++ /dev/null @@ -1,102 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "ec2:*" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:DescribeAutoScalingInstances", - "autoscaling:DescribeLaunchConfigurations", - "autoscaling:DescribeTags", - "autoscaling:SetDesiredCapacity", - "autoscaling:TerminateInstanceInAutoScalingGroup", - "autoscaling:UpdateAutoScalingGroup", - "ec2:DescribeLaunchTemplateVersions" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "elasticloadbalancing:*" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "iam:ListServerCertificates", - "iam:GetServerCertificate" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ChangeResourceRecordSets", - "route53:ListResourceRecordSets", - "route53:GetHostedZone" - ], - "Resource": [ - "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:GetChange" - ], - "Resource": [ - "arn:aws:route53:::change/*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ListHostedZones" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ListHostedZones" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "ecr:GetAuthorizationToken", - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:GetRepositoryPolicy", - "ecr:DescribeRepositories", - "ecr:ListImages", - "ecr:BatchGetImage" - ], - "Resource": [ - "*" - ] - } - ] -} diff --git a/tests/integration/update_cluster/api_elb_cross_zone/data/aws_iam_role_policy_nodes.crosszone.example.com_policy b/tests/integration/update_cluster/api_elb_cross_zone/data/aws_iam_role_policy_nodes.crosszone.example.com_policy deleted file mode 100644 index ef2600b497..0000000000 --- a/tests/integration/update_cluster/api_elb_cross_zone/data/aws_iam_role_policy_nodes.crosszone.example.com_policy +++ /dev/null @@ -1,68 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "ec2:DescribeInstances", - "ec2:DescribeRegions" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ChangeResourceRecordSets", - "route53:ListResourceRecordSets", - "route53:GetHostedZone" - ], - "Resource": [ - "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:GetChange" - ], - "Resource": [ - "arn:aws:route53:::change/*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ListHostedZones" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ListHostedZones" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "ecr:GetAuthorizationToken", - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:GetRepositoryPolicy", - "ecr:DescribeRepositories", - "ecr:ListImages", - "ecr:BatchGetImage" - ], - "Resource": [ - "*" - ] - } - ] -} diff --git a/tests/integration/update_cluster/api_elb_cross_zone/data/aws_key_pair_kubernetes.crosszone.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key b/tests/integration/update_cluster/api_elb_cross_zone/data/aws_key_pair_kubernetes.crosszone.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key deleted file mode 100644 index 81cb012783..0000000000 --- a/tests/integration/update_cluster/api_elb_cross_zone/data/aws_key_pair_kubernetes.crosszone.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/api_elb_cross_zone/data/aws_launch_template_master-us-test-1a.masters.crosszone.example.com_user_data b/tests/integration/update_cluster/api_elb_cross_zone/data/aws_launch_template_master-us-test-1a.masters.crosszone.example.com_user_data deleted file mode 100644 index 1c59d179ae..0000000000 --- a/tests/integration/update_cluster/api_elb_cross_zone/data/aws_launch_template_master-us-test-1a.masters.crosszone.example.com_user_data +++ /dev/null @@ -1,315 +0,0 @@ -#!/bin/bash -# Copyright 2016 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup -NODEUP_HASH_AMD64=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 -NODEUP_URL_ARM64= -NODEUP_HASH_ARM64= - -export AWS_REGION=us-test-1 - - - - -function ensure-install-dir() { - INSTALL_DIR="/opt/kops" - # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec - if [[ -d /var/lib/toolbox ]]; then - INSTALL_DIR="/var/lib/toolbox/kops" - fi - mkdir -p ${INSTALL_DIR}/bin - mkdir -p ${INSTALL_DIR}/conf - cd ${INSTALL_DIR} -} - -# Retry a download until we get it. args: name, sha, url1, url2... -download-or-bust() { - local -r file="$1" - local -r hash="$2" - shift 2 - - urls=( $* ) - while true; do - for url in "${urls[@]}"; do - commands=( - "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - ) - for cmd in "${commands[@]}"; do - echo "Attempting download with: ${cmd} {url}" - if ! (${cmd} "${url}"); then - echo "== Download failed with ${cmd} ==" - continue - fi - if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then - echo "== Hash validation of ${url} failed. Retrying. ==" - rm -f "${file}" - else - if [[ -n "${hash}" ]]; then - echo "== Downloaded ${url} (SHA1 = ${hash}) ==" - else - echo "== Downloaded ${url} ==" - fi - return - fi - done - done - - echo "All downloads failed; sleeping before retrying" - sleep 60 - done -} - -validate-hash() { - local -r file="$1" - local -r expected="$2" - local actual - - actual=$(sha256sum ${file} | awk '{ print $1 }') || true - if [[ "${actual}" != "${expected}" ]]; then - echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" - return 1 - fi -} - -function split-commas() { - echo $1 | tr "," "\n" -} - -function try-download-release() { - local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) - if [[ -n "${NODEUP_HASH:-}" ]]; then - local -r nodeup_hash="${NODEUP_HASH}" - else - # TODO: Remove? - echo "Downloading sha256 (not found in env)" - download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" - local -r nodeup_hash=$(cat nodeup.sha256) - fi - - echo "Downloading nodeup (${nodeup_urls[@]})" - download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" - - chmod +x nodeup -} - -function download-release() { - case "$(uname -m)" in - x86_64*|i?86_64*|amd64*) - NODEUP_URL="${NODEUP_URL_AMD64}" - NODEUP_HASH="${NODEUP_HASH_AMD64}" - ;; - aarch64*|arm64*) - NODEUP_URL="${NODEUP_URL_ARM64}" - NODEUP_HASH="${NODEUP_HASH_ARM64}" - ;; - *) - echo "Unsupported host arch: $(uname -m)" >&2 - exit 1 - ;; - esac - - # In case of failure checking integrity of release, retry. - cd ${INSTALL_DIR}/bin - until try-download-release; do - sleep 15 - echo "Couldn't download release. Retrying..." - done - - echo "Running nodeup" - # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 - ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) -} - -#################################################################################### - -/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" - -echo "== nodeup node config starting ==" -ensure-install-dir - -cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' -cloudConfig: null -containerRuntime: docker -containerd: - skipInstall: true -docker: - ipMasq: false - ipTables: false - logDriver: json-file - logLevel: info - logOpt: - - max-size=10m - - max-file=5 - storage: overlay2,overlay,aufs - version: 18.06.3 -encryptionConfig: null -etcdClusters: - events: - version: 3.3.10 - main: - version: 3.3.10 -kubeAPIServer: - allowPrivileged: true - anonymousAuth: false - apiServerCount: 1 - authorizationMode: AlwaysAllow - bindAddress: 0.0.0.0 - cloudProvider: aws - enableAdmissionPlugins: - - NamespaceLifecycle - - LimitRanger - - ServiceAccount - - PersistentVolumeLabel - - DefaultStorageClass - - DefaultTolerationSeconds - - MutatingAdmissionWebhook - - ValidatingAdmissionWebhook - - NodeRestriction - - ResourceQuota - etcdServers: - - http://127.0.0.1:4001 - etcdServersOverrides: - - /events#http://127.0.0.1:4002 - image: k8s.gcr.io/kube-apiserver:v1.14.0 - insecureBindAddress: 127.0.0.1 - insecurePort: 8080 - kubeletPreferredAddressTypes: - - InternalIP - - Hostname - - ExternalIP - logLevel: 2 - requestheaderAllowedNames: - - aggregator - requestheaderExtraHeaderPrefixes: - - X-Remote-Extra- - requestheaderGroupHeaders: - - X-Remote-Group - requestheaderUsernameHeaders: - - X-Remote-User - securePort: 443 - serviceClusterIPRange: 100.64.0.0/13 - serviceNodePortRange: 28000-32767 - storageBackend: etcd3 -kubeControllerManager: - allocateNodeCIDRs: true - attachDetachReconcileSyncPeriod: 1m0s - cloudProvider: aws - clusterCIDR: 100.96.0.0/11 - clusterName: crosszone.example.com - configureCloudRoutes: true - image: k8s.gcr.io/kube-controller-manager:v1.14.0 - leaderElection: - leaderElect: true - logLevel: 2 - useServiceAccountCredentials: true -kubeProxy: - clusterCIDR: 100.96.0.0/11 - cpuRequest: 100m - hostnameOverride: '@aws' - image: k8s.gcr.io/kube-proxy:v1.14.0 - logLevel: 2 -kubeScheduler: - image: k8s.gcr.io/kube-scheduler:v1.14.0 - leaderElection: - leaderElect: true - logLevel: 2 -kubelet: - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests -masterKubelet: - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests - registerSchedulable: false - -__EOF_CLUSTER_SPEC - -cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' -kubelet: null -nodeLabels: null -taints: null - -__EOF_IG_SPEC - -cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' -Assets: - amd64: - - c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet - - 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl - - 3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz - arm64: - - df38e04576026393055ccc77c0dce73612996561@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubelet - - 01c2b6b43d36b6bfafc80a3737391c19ebfb8ad5@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubectl - - 7fec91af78e9548df306f0ec43bea527c8c10cc3a9682c33e971c8522a7fcded@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-arm64-v0.7.5.tgz -ClusterName: crosszone.example.com -ConfigBase: memfs://clusters.example.com/crosszone.example.com -InstanceGroupName: master-us-test-1a -InstanceGroupRole: Master -Tags: -- _automatic_upgrades -- _aws -channels: -- memfs://clusters.example.com/crosszone.example.com/addons/bootstrap-channel.yaml -etcdManifests: -- memfs://clusters.example.com/crosszone.example.com/manifests/etcd/main.yaml -- memfs://clusters.example.com/crosszone.example.com/manifests/etcd/events.yaml -protokubeImage: - hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b - name: protokube:1.15.0 - sources: - - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz - - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz - - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz - -__EOF_KUBE_ENV - -download-release -echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/api_elb_cross_zone/data/aws_launch_template_nodes.crosszone.example.com_user_data b/tests/integration/update_cluster/api_elb_cross_zone/data/aws_launch_template_nodes.crosszone.example.com_user_data deleted file mode 100644 index de43754079..0000000000 --- a/tests/integration/update_cluster/api_elb_cross_zone/data/aws_launch_template_nodes.crosszone.example.com_user_data +++ /dev/null @@ -1,221 +0,0 @@ -#!/bin/bash -# Copyright 2016 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup -NODEUP_HASH_AMD64=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 -NODEUP_URL_ARM64= -NODEUP_HASH_ARM64= - -export AWS_REGION=us-test-1 - - - - -function ensure-install-dir() { - INSTALL_DIR="/opt/kops" - # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec - if [[ -d /var/lib/toolbox ]]; then - INSTALL_DIR="/var/lib/toolbox/kops" - fi - mkdir -p ${INSTALL_DIR}/bin - mkdir -p ${INSTALL_DIR}/conf - cd ${INSTALL_DIR} -} - -# Retry a download until we get it. args: name, sha, url1, url2... -download-or-bust() { - local -r file="$1" - local -r hash="$2" - shift 2 - - urls=( $* ) - while true; do - for url in "${urls[@]}"; do - commands=( - "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - ) - for cmd in "${commands[@]}"; do - echo "Attempting download with: ${cmd} {url}" - if ! (${cmd} "${url}"); then - echo "== Download failed with ${cmd} ==" - continue - fi - if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then - echo "== Hash validation of ${url} failed. Retrying. ==" - rm -f "${file}" - else - if [[ -n "${hash}" ]]; then - echo "== Downloaded ${url} (SHA1 = ${hash}) ==" - else - echo "== Downloaded ${url} ==" - fi - return - fi - done - done - - echo "All downloads failed; sleeping before retrying" - sleep 60 - done -} - -validate-hash() { - local -r file="$1" - local -r expected="$2" - local actual - - actual=$(sha256sum ${file} | awk '{ print $1 }') || true - if [[ "${actual}" != "${expected}" ]]; then - echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" - return 1 - fi -} - -function split-commas() { - echo $1 | tr "," "\n" -} - -function try-download-release() { - local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) - if [[ -n "${NODEUP_HASH:-}" ]]; then - local -r nodeup_hash="${NODEUP_HASH}" - else - # TODO: Remove? - echo "Downloading sha256 (not found in env)" - download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" - local -r nodeup_hash=$(cat nodeup.sha256) - fi - - echo "Downloading nodeup (${nodeup_urls[@]})" - download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" - - chmod +x nodeup -} - -function download-release() { - case "$(uname -m)" in - x86_64*|i?86_64*|amd64*) - NODEUP_URL="${NODEUP_URL_AMD64}" - NODEUP_HASH="${NODEUP_HASH_AMD64}" - ;; - aarch64*|arm64*) - NODEUP_URL="${NODEUP_URL_ARM64}" - NODEUP_HASH="${NODEUP_HASH_ARM64}" - ;; - *) - echo "Unsupported host arch: $(uname -m)" >&2 - exit 1 - ;; - esac - - # In case of failure checking integrity of release, retry. - cd ${INSTALL_DIR}/bin - until try-download-release; do - sleep 15 - echo "Couldn't download release. Retrying..." - done - - echo "Running nodeup" - # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 - ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) -} - -#################################################################################### - -/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" - -echo "== nodeup node config starting ==" -ensure-install-dir - -cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' -cloudConfig: null -containerRuntime: docker -containerd: - skipInstall: true -docker: - ipMasq: false - ipTables: false - logDriver: json-file - logLevel: info - logOpt: - - max-size=10m - - max-file=5 - storage: overlay2,overlay,aufs - version: 18.06.3 -kubeProxy: - clusterCIDR: 100.96.0.0/11 - cpuRequest: 100m - hostnameOverride: '@aws' - image: k8s.gcr.io/kube-proxy:v1.14.0 - logLevel: 2 -kubelet: - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests - -__EOF_CLUSTER_SPEC - -cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' -kubelet: null -nodeLabels: null -taints: null - -__EOF_IG_SPEC - -cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' -Assets: - amd64: - - c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet - - 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl - - 3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz - arm64: - - df38e04576026393055ccc77c0dce73612996561@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubelet - - 01c2b6b43d36b6bfafc80a3737391c19ebfb8ad5@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubectl - - 7fec91af78e9548df306f0ec43bea527c8c10cc3a9682c33e971c8522a7fcded@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-arm64-v0.7.5.tgz -ClusterName: crosszone.example.com -ConfigBase: memfs://clusters.example.com/crosszone.example.com -InstanceGroupName: nodes -InstanceGroupRole: Node -Tags: -- _automatic_upgrades -- _aws -channels: -- memfs://clusters.example.com/crosszone.example.com/addons/bootstrap-channel.yaml - -__EOF_KUBE_ENV - -download-release -echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/api_elb_cross_zone/id_rsa.pub b/tests/integration/update_cluster/api_elb_cross_zone/id_rsa.pub deleted file mode 100755 index 81cb012783..0000000000 --- a/tests/integration/update_cluster/api_elb_cross_zone/id_rsa.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/api_elb_cross_zone/in-v1alpha2.yaml b/tests/integration/update_cluster/api_elb_cross_zone/in-v1alpha2.yaml deleted file mode 100644 index be1f4ab1de..0000000000 --- a/tests/integration/update_cluster/api_elb_cross_zone/in-v1alpha2.yaml +++ /dev/null @@ -1,98 +0,0 @@ -apiVersion: kops.k8s.io/v1alpha2 -kind: Cluster -metadata: - creationTimestamp: "2016-12-10T22:42:27Z" - name: crosszone.example.com -spec: - api: - loadBalancer: - type: Public - additionalSecurityGroups: - - sg-exampleid3 - - sg-exampleid4 - crossZoneLoadBalancing: true - kubernetesApiAccess: - - 0.0.0.0/0 - channel: stable - cloudProvider: aws - cloudLabels: - Owner: John Doe - foo/bar: fib+baz - configBase: memfs://clusters.example.com/crosszone.example.com - etcdClusters: - - etcdMembers: - - instanceGroup: master-us-test-1a - name: us-test-1a - name: main - - etcdMembers: - - instanceGroup: master-us-test-1a - name: us-test-1a - name: events - kubeAPIServer: - serviceNodePortRange: 28000-32767 - kubelet: - anonymousAuth: false - kubernetesVersion: v1.14.0 - masterInternalName: api.internal.crosszone.example.com - masterPublicName: api.crosszone.example.com - networkCIDR: 172.20.0.0/16 - networking: - kubenet: {} - nodePortAccess: - - 1.2.3.4/32 - - 10.20.30.0/24 - nonMasqueradeCIDR: 100.64.0.0/10 - sshAccess: - - 0.0.0.0/0 - topology: - masters: public - nodes: public - subnets: - - cidr: 172.20.32.0/19 - name: us-test-1a - type: Public - zone: us-test-1a - ---- - -apiVersion: kops.k8s.io/v1alpha2 -kind: InstanceGroup -metadata: - creationTimestamp: "2016-12-10T22:42:28Z" - name: nodes - labels: - kops.k8s.io/cluster: crosszone.example.com -spec: - additionalSecurityGroups: - - sg-exampleid3 - - sg-exampleid4 - associatePublicIp: true - suspendProcesses: - - AZRebalance - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: t2.medium - maxSize: 2 - minSize: 2 - role: Node - subnets: - - us-test-1a - detailedInstanceMonitoring: true - ---- - -apiVersion: kops.k8s.io/v1alpha2 -kind: InstanceGroup -metadata: - creationTimestamp: "2016-12-10T22:42:28Z" - name: master-us-test-1a - labels: - kops.k8s.io/cluster: crosszone.example.com -spec: - associatePublicIp: true - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: m3.medium - maxSize: 1 - minSize: 1 - role: Master - subnets: - - us-test-1a diff --git a/tests/integration/update_cluster/api_elb_cross_zone/kubernetes.tf b/tests/integration/update_cluster/api_elb_cross_zone/kubernetes.tf deleted file mode 100644 index 18765eb6da..0000000000 --- a/tests/integration/update_cluster/api_elb_cross_zone/kubernetes.tf +++ /dev/null @@ -1,691 +0,0 @@ -locals { - cluster_name = "crosszone.example.com" - master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-crosszone-example-com.id] - master_security_group_ids = [aws_security_group.masters-crosszone-example-com.id] - masters_role_arn = aws_iam_role.masters-crosszone-example-com.arn - masters_role_name = aws_iam_role.masters-crosszone-example-com.name - node_autoscaling_group_ids = [aws_autoscaling_group.nodes-crosszone-example-com.id] - node_security_group_ids = [aws_security_group.nodes-crosszone-example-com.id, "sg-exampleid3", "sg-exampleid4"] - node_subnet_ids = [aws_subnet.us-test-1a-crosszone-example-com.id] - nodes_role_arn = aws_iam_role.nodes-crosszone-example-com.arn - nodes_role_name = aws_iam_role.nodes-crosszone-example-com.name - region = "us-test-1" - route_table_public_id = aws_route_table.crosszone-example-com.id - subnet_us-test-1a_id = aws_subnet.us-test-1a-crosszone-example-com.id - vpc_cidr_block = aws_vpc.crosszone-example-com.cidr_block - vpc_id = aws_vpc.crosszone-example-com.id -} - -output "cluster_name" { - value = "crosszone.example.com" -} - -output "master_autoscaling_group_ids" { - value = [aws_autoscaling_group.master-us-test-1a-masters-crosszone-example-com.id] -} - -output "master_security_group_ids" { - value = [aws_security_group.masters-crosszone-example-com.id] -} - -output "masters_role_arn" { - value = aws_iam_role.masters-crosszone-example-com.arn -} - -output "masters_role_name" { - value = aws_iam_role.masters-crosszone-example-com.name -} - -output "node_autoscaling_group_ids" { - value = [aws_autoscaling_group.nodes-crosszone-example-com.id] -} - -output "node_security_group_ids" { - value = [aws_security_group.nodes-crosszone-example-com.id, "sg-exampleid3", "sg-exampleid4"] -} - -output "node_subnet_ids" { - value = [aws_subnet.us-test-1a-crosszone-example-com.id] -} - -output "nodes_role_arn" { - value = aws_iam_role.nodes-crosszone-example-com.arn -} - -output "nodes_role_name" { - value = aws_iam_role.nodes-crosszone-example-com.name -} - -output "region" { - value = "us-test-1" -} - -output "route_table_public_id" { - value = aws_route_table.crosszone-example-com.id -} - -output "subnet_us-test-1a_id" { - value = aws_subnet.us-test-1a-crosszone-example-com.id -} - -output "vpc_cidr_block" { - value = aws_vpc.crosszone-example-com.cidr_block -} - -output "vpc_id" { - value = aws_vpc.crosszone-example-com.id -} - -provider "aws" { - region = "us-test-1" -} - -resource "aws_autoscaling_attachment" "master-us-test-1a-masters-crosszone-example-com" { - autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-crosszone-example-com.id - elb = aws_elb.api-crosszone-example-com.id -} - -resource "aws_autoscaling_group" "master-us-test-1a-masters-crosszone-example-com" { - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] - launch_template { - id = aws_launch_template.master-us-test-1a-masters-crosszone-example-com.id - version = aws_launch_template.master-us-test-1a-masters-crosszone-example-com.latest_version - } - max_size = 1 - metrics_granularity = "1Minute" - min_size = 1 - name = "master-us-test-1a.masters.crosszone.example.com" - tag { - key = "KubernetesCluster" - propagate_at_launch = true - value = "crosszone.example.com" - } - tag { - key = "Name" - propagate_at_launch = true - value = "master-us-test-1a.masters.crosszone.example.com" - } - tag { - key = "Owner" - propagate_at_launch = true - value = "John Doe" - } - tag { - key = "foo/bar" - propagate_at_launch = true - value = "fib+baz" - } - tag { - key = "k8s.io/role/master" - propagate_at_launch = true - value = "1" - } - tag { - key = "kops.k8s.io/instancegroup" - propagate_at_launch = true - value = "master-us-test-1a" - } - tag { - key = "kubernetes.io/cluster/crosszone.example.com" - propagate_at_launch = true - value = "owned" - } - vpc_zone_identifier = [aws_subnet.us-test-1a-crosszone-example-com.id] -} - -resource "aws_autoscaling_group" "nodes-crosszone-example-com" { - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] - launch_template { - id = aws_launch_template.nodes-crosszone-example-com.id - version = aws_launch_template.nodes-crosszone-example-com.latest_version - } - max_size = 2 - metrics_granularity = "1Minute" - min_size = 2 - name = "nodes.crosszone.example.com" - suspended_processes = ["AZRebalance"] - tag { - key = "KubernetesCluster" - propagate_at_launch = true - value = "crosszone.example.com" - } - tag { - key = "Name" - propagate_at_launch = true - value = "nodes.crosszone.example.com" - } - tag { - key = "Owner" - propagate_at_launch = true - value = "John Doe" - } - tag { - key = "foo/bar" - propagate_at_launch = true - value = "fib+baz" - } - tag { - key = "k8s.io/role/node" - propagate_at_launch = true - value = "1" - } - tag { - key = "kops.k8s.io/instancegroup" - propagate_at_launch = true - value = "nodes" - } - tag { - key = "kubernetes.io/cluster/crosszone.example.com" - propagate_at_launch = true - value = "owned" - } - vpc_zone_identifier = [aws_subnet.us-test-1a-crosszone-example-com.id] -} - -resource "aws_ebs_volume" "us-test-1a-etcd-events-crosszone-example-com" { - availability_zone = "us-test-1a" - encrypted = false - size = 20 - tags = { - "KubernetesCluster" = "crosszone.example.com" - "Name" = "us-test-1a.etcd-events.crosszone.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "k8s.io/etcd/events" = "us-test-1a/us-test-1a" - "k8s.io/role/master" = "1" - "kubernetes.io/cluster/crosszone.example.com" = "owned" - } - type = "gp2" -} - -resource "aws_ebs_volume" "us-test-1a-etcd-main-crosszone-example-com" { - availability_zone = "us-test-1a" - encrypted = false - size = 20 - tags = { - "KubernetesCluster" = "crosszone.example.com" - "Name" = "us-test-1a.etcd-main.crosszone.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "k8s.io/etcd/main" = "us-test-1a/us-test-1a" - "k8s.io/role/master" = "1" - "kubernetes.io/cluster/crosszone.example.com" = "owned" - } - type = "gp2" -} - -resource "aws_elb" "api-crosszone-example-com" { - cross_zone_load_balancing = true - health_check { - healthy_threshold = 2 - interval = 10 - target = "SSL:443" - timeout = 5 - unhealthy_threshold = 2 - } - idle_timeout = 300 - listener { - instance_port = 443 - instance_protocol = "TCP" - lb_port = 443 - lb_protocol = "TCP" - ssl_certificate_id = "" - } - name = "api-crosszone-example-com-qhvtkl" - security_groups = [aws_security_group.api-elb-crosszone-example-com.id, "sg-exampleid3", "sg-exampleid4"] - subnets = [aws_subnet.us-test-1a-crosszone-example-com.id] - tags = { - "KubernetesCluster" = "crosszone.example.com" - "Name" = "api.crosszone.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "kubernetes.io/cluster/crosszone.example.com" = "owned" - } -} - -resource "aws_iam_instance_profile" "masters-crosszone-example-com" { - name = "masters.crosszone.example.com" - role = aws_iam_role.masters-crosszone-example-com.name -} - -resource "aws_iam_instance_profile" "nodes-crosszone-example-com" { - name = "nodes.crosszone.example.com" - role = aws_iam_role.nodes-crosszone-example-com.name -} - -resource "aws_iam_role_policy" "masters-crosszone-example-com" { - name = "masters.crosszone.example.com" - policy = file("${path.module}/data/aws_iam_role_policy_masters.crosszone.example.com_policy") - role = aws_iam_role.masters-crosszone-example-com.name -} - -resource "aws_iam_role_policy" "nodes-crosszone-example-com" { - name = "nodes.crosszone.example.com" - policy = file("${path.module}/data/aws_iam_role_policy_nodes.crosszone.example.com_policy") - role = aws_iam_role.nodes-crosszone-example-com.name -} - -resource "aws_iam_role" "masters-crosszone-example-com" { - assume_role_policy = file("${path.module}/data/aws_iam_role_masters.crosszone.example.com_policy") - name = "masters.crosszone.example.com" -} - -resource "aws_iam_role" "nodes-crosszone-example-com" { - assume_role_policy = file("${path.module}/data/aws_iam_role_nodes.crosszone.example.com_policy") - name = "nodes.crosszone.example.com" -} - -resource "aws_internet_gateway" "crosszone-example-com" { - tags = { - "KubernetesCluster" = "crosszone.example.com" - "Name" = "crosszone.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "kubernetes.io/cluster/crosszone.example.com" = "owned" - } - vpc_id = aws_vpc.crosszone-example-com.id -} - -resource "aws_key_pair" "kubernetes-crosszone-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" { - key_name = "kubernetes.crosszone.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57" - public_key = file("${path.module}/data/aws_key_pair_kubernetes.crosszone.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key") -} - -resource "aws_launch_template" "master-us-test-1a-masters-crosszone-example-com" { - block_device_mappings { - device_name = "/dev/xvda" - ebs { - delete_on_termination = true - volume_size = 64 - volume_type = "gp2" - } - } - block_device_mappings { - device_name = "/dev/sdc" - virtual_name = "ephemeral0" - } - iam_instance_profile { - name = aws_iam_instance_profile.masters-crosszone-example-com.id - } - image_id = "ami-12345678" - instance_type = "m3.medium" - key_name = aws_key_pair.kubernetes-crosszone-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id - lifecycle { - create_before_destroy = true - } - name_prefix = "master-us-test-1a.masters.crosszone.example.com-" - network_interfaces { - associate_public_ip_address = true - delete_on_termination = true - security_groups = [aws_security_group.masters-crosszone-example-com.id] - } - tag_specifications { - resource_type = "instance" - tags = { - "KubernetesCluster" = "crosszone.example.com" - "Name" = "master-us-test-1a.masters.crosszone.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "k8s.io/role/master" = "1" - "kops.k8s.io/instancegroup" = "master-us-test-1a" - "kubernetes.io/cluster/crosszone.example.com" = "owned" - } - } - tag_specifications { - resource_type = "volume" - tags = { - "KubernetesCluster" = "crosszone.example.com" - "Name" = "master-us-test-1a.masters.crosszone.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "k8s.io/role/master" = "1" - "kops.k8s.io/instancegroup" = "master-us-test-1a" - "kubernetes.io/cluster/crosszone.example.com" = "owned" - } - } - user_data = file("${path.module}/data/aws_launch_template_master-us-test-1a.masters.crosszone.example.com_user_data") -} - -resource "aws_launch_template" "nodes-crosszone-example-com" { - block_device_mappings { - device_name = "/dev/xvda" - ebs { - delete_on_termination = true - volume_size = 128 - volume_type = "gp2" - } - } - iam_instance_profile { - name = aws_iam_instance_profile.nodes-crosszone-example-com.id - } - image_id = "ami-12345678" - instance_type = "t2.medium" - key_name = aws_key_pair.kubernetes-crosszone-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id - lifecycle { - create_before_destroy = true - } - name_prefix = "nodes.crosszone.example.com-" - network_interfaces { - associate_public_ip_address = true - delete_on_termination = true - security_groups = [aws_security_group.nodes-crosszone-example-com.id, "sg-exampleid3", "sg-exampleid4"] - } - tag_specifications { - resource_type = "instance" - tags = { - "KubernetesCluster" = "crosszone.example.com" - "Name" = "nodes.crosszone.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "nodes" - "kubernetes.io/cluster/crosszone.example.com" = "owned" - } - } - tag_specifications { - resource_type = "volume" - tags = { - "KubernetesCluster" = "crosszone.example.com" - "Name" = "nodes.crosszone.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "nodes" - "kubernetes.io/cluster/crosszone.example.com" = "owned" - } - } - user_data = file("${path.module}/data/aws_launch_template_nodes.crosszone.example.com_user_data") -} - -resource "aws_route53_record" "api-crosszone-example-com" { - alias { - evaluate_target_health = false - name = aws_elb.api-crosszone-example-com.dns_name - zone_id = aws_elb.api-crosszone-example-com.zone_id - } - name = "api.crosszone.example.com" - type = "A" - zone_id = "/hostedzone/Z1AFAKE1ZON3YO" -} - -resource "aws_route_table_association" "us-test-1a-crosszone-example-com" { - route_table_id = aws_route_table.crosszone-example-com.id - subnet_id = aws_subnet.us-test-1a-crosszone-example-com.id -} - -resource "aws_route_table" "crosszone-example-com" { - tags = { - "KubernetesCluster" = "crosszone.example.com" - "Name" = "crosszone.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "kubernetes.io/cluster/crosszone.example.com" = "owned" - "kubernetes.io/kops/role" = "public" - } - vpc_id = aws_vpc.crosszone-example-com.id -} - -resource "aws_route" "route-0-0-0-0--0" { - destination_cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.crosszone-example-com.id - route_table_id = aws_route_table.crosszone-example-com.id -} - -resource "aws_security_group_rule" "all-master-to-master" { - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.masters-crosszone-example-com.id - source_security_group_id = aws_security_group.masters-crosszone-example-com.id - to_port = 0 - type = "ingress" -} - -resource "aws_security_group_rule" "all-master-to-node" { - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.nodes-crosszone-example-com.id - source_security_group_id = aws_security_group.masters-crosszone-example-com.id - to_port = 0 - type = "ingress" -} - -resource "aws_security_group_rule" "all-node-to-node" { - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.nodes-crosszone-example-com.id - source_security_group_id = aws_security_group.nodes-crosszone-example-com.id - to_port = 0 - type = "ingress" -} - -resource "aws_security_group_rule" "api-elb-egress" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.api-elb-crosszone-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "https-api-elb-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 443 - protocol = "tcp" - security_group_id = aws_security_group.api-elb-crosszone-example-com.id - to_port = 443 - type = "ingress" -} - -resource "aws_security_group_rule" "https-elb-to-master" { - from_port = 443 - protocol = "tcp" - security_group_id = aws_security_group.masters-crosszone-example-com.id - source_security_group_id = aws_security_group.api-elb-crosszone-example-com.id - to_port = 443 - type = "ingress" -} - -resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 3 - protocol = "icmp" - security_group_id = aws_security_group.api-elb-crosszone-example-com.id - to_port = 4 - type = "ingress" -} - -resource "aws_security_group_rule" "master-egress" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.masters-crosszone-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "node-egress" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.nodes-crosszone-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "node-to-master-tcp-1-2379" { - from_port = 1 - protocol = "tcp" - security_group_id = aws_security_group.masters-crosszone-example-com.id - source_security_group_id = aws_security_group.nodes-crosszone-example-com.id - to_port = 2379 - type = "ingress" -} - -resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" { - from_port = 2382 - protocol = "tcp" - security_group_id = aws_security_group.masters-crosszone-example-com.id - source_security_group_id = aws_security_group.nodes-crosszone-example-com.id - to_port = 4000 - type = "ingress" -} - -resource "aws_security_group_rule" "node-to-master-tcp-4003-65535" { - from_port = 4003 - protocol = "tcp" - security_group_id = aws_security_group.masters-crosszone-example-com.id - source_security_group_id = aws_security_group.nodes-crosszone-example-com.id - to_port = 65535 - type = "ingress" -} - -resource "aws_security_group_rule" "node-to-master-udp-1-65535" { - from_port = 1 - protocol = "udp" - security_group_id = aws_security_group.masters-crosszone-example-com.id - source_security_group_id = aws_security_group.nodes-crosszone-example-com.id - to_port = 65535 - type = "ingress" -} - -resource "aws_security_group_rule" "nodeport-tcp-external-to-node-1-2-3-4--32" { - cidr_blocks = ["1.2.3.4/32"] - from_port = 28000 - protocol = "tcp" - security_group_id = aws_security_group.nodes-crosszone-example-com.id - to_port = 32767 - type = "ingress" -} - -resource "aws_security_group_rule" "nodeport-tcp-external-to-node-10-20-30-0--24" { - cidr_blocks = ["10.20.30.0/24"] - from_port = 28000 - protocol = "tcp" - security_group_id = aws_security_group.nodes-crosszone-example-com.id - to_port = 32767 - type = "ingress" -} - -resource "aws_security_group_rule" "nodeport-udp-external-to-node-1-2-3-4--32" { - cidr_blocks = ["1.2.3.4/32"] - from_port = 28000 - protocol = "udp" - security_group_id = aws_security_group.nodes-crosszone-example-com.id - to_port = 32767 - type = "ingress" -} - -resource "aws_security_group_rule" "nodeport-udp-external-to-node-10-20-30-0--24" { - cidr_blocks = ["10.20.30.0/24"] - from_port = 28000 - protocol = "udp" - security_group_id = aws_security_group.nodes-crosszone-example-com.id - to_port = 32767 - type = "ingress" -} - -resource "aws_security_group_rule" "ssh-external-to-master-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.masters-crosszone-example-com.id - to_port = 22 - type = "ingress" -} - -resource "aws_security_group_rule" "ssh-external-to-node-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.nodes-crosszone-example-com.id - to_port = 22 - type = "ingress" -} - -resource "aws_security_group" "api-elb-crosszone-example-com" { - description = "Security group for api ELB" - name = "api-elb.crosszone.example.com" - tags = { - "KubernetesCluster" = "crosszone.example.com" - "Name" = "api-elb.crosszone.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "kubernetes.io/cluster/crosszone.example.com" = "owned" - } - vpc_id = aws_vpc.crosszone-example-com.id -} - -resource "aws_security_group" "masters-crosszone-example-com" { - description = "Security group for masters" - name = "masters.crosszone.example.com" - tags = { - "KubernetesCluster" = "crosszone.example.com" - "Name" = "masters.crosszone.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "kubernetes.io/cluster/crosszone.example.com" = "owned" - } - vpc_id = aws_vpc.crosszone-example-com.id -} - -resource "aws_security_group" "nodes-crosszone-example-com" { - description = "Security group for nodes" - name = "nodes.crosszone.example.com" - tags = { - "KubernetesCluster" = "crosszone.example.com" - "Name" = "nodes.crosszone.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "kubernetes.io/cluster/crosszone.example.com" = "owned" - } - vpc_id = aws_vpc.crosszone-example-com.id -} - -resource "aws_subnet" "us-test-1a-crosszone-example-com" { - availability_zone = "us-test-1a" - cidr_block = "172.20.32.0/19" - tags = { - "KubernetesCluster" = "crosszone.example.com" - "Name" = "us-test-1a.crosszone.example.com" - "Owner" = "John Doe" - "SubnetType" = "Public" - "foo/bar" = "fib+baz" - "kubernetes.io/cluster/crosszone.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - } - vpc_id = aws_vpc.crosszone-example-com.id -} - -resource "aws_vpc_dhcp_options_association" "crosszone-example-com" { - dhcp_options_id = aws_vpc_dhcp_options.crosszone-example-com.id - vpc_id = aws_vpc.crosszone-example-com.id -} - -resource "aws_vpc_dhcp_options" "crosszone-example-com" { - domain_name = "us-test-1.compute.internal" - domain_name_servers = ["AmazonProvidedDNS"] - tags = { - "KubernetesCluster" = "crosszone.example.com" - "Name" = "crosszone.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "kubernetes.io/cluster/crosszone.example.com" = "owned" - } -} - -resource "aws_vpc" "crosszone-example-com" { - cidr_block = "172.20.0.0/16" - enable_dns_hostnames = true - enable_dns_support = true - tags = { - "KubernetesCluster" = "crosszone.example.com" - "Name" = "crosszone.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "kubernetes.io/cluster/crosszone.example.com" = "owned" - } -} - -terraform { - required_version = ">= 0.12.0" -} diff --git a/tests/integration/update_cluster/complex/cloudformation.json b/tests/integration/update_cluster/complex/cloudformation.json index 305334c541..5a653e54a1 100644 --- a/tests/integration/update_cluster/complex/cloudformation.json +++ b/tests/integration/update_cluster/complex/cloudformation.json @@ -241,7 +241,6 @@ }, "ImageId": "ami-12345678", "InstanceType": "m3.medium", - "KeyName": "kubernetes.complex.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", "NetworkInterfaces": [ { "AssociatePublicIpAddress": true, @@ -356,7 +355,6 @@ }, "ImageId": "ami-12345678", "InstanceType": "t2.medium", - "KeyName": "kubernetes.complex.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", "NetworkInterfaces": [ { "AssociatePublicIpAddress": true, @@ -567,7 +565,7 @@ "IpProtocol": "-1" } }, - "AWSEC2SecurityGroupIngresshttpsapielb00000": { + "AWSEC2SecurityGroupIngresshttpsapielb111024": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { @@ -576,7 +574,19 @@ "FromPort": 443, "ToPort": 443, "IpProtocol": "tcp", - "CidrIp": "0.0.0.0/0" + "CidrIp": "1.1.1.0/24" + } + }, + "AWSEC2SecurityGroupIngresshttpsapielb20010850040": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupapielbcomplexexamplecom" + }, + "FromPort": 443, + "ToPort": 443, + "IpProtocol": "tcp", + "CidrIp": "2001:0:8500::/40" } }, "AWSEC2SecurityGroupIngresshttpselbtomaster": { @@ -593,7 +603,7 @@ "IpProtocol": "tcp" } }, - "AWSEC2SecurityGroupIngressicmppmtuapielb00000": { + "AWSEC2SecurityGroupIngressicmppmtuapielb111024": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { @@ -602,7 +612,19 @@ "FromPort": 3, "ToPort": 4, "IpProtocol": "icmp", - "CidrIp": "0.0.0.0/0" + "CidrIp": "1.1.1.0/24" + } + }, + "AWSEC2SecurityGroupIngressicmppmtuapielb20010850040": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupapielbcomplexexamplecom" + }, + "FromPort": 3, + "ToPort": 4, + "IpProtocol": "icmp", + "CidrIp": "2001:0:8500::/40" } }, "AWSEC2SecurityGroupIngressnodeporttcpexternaltonode102030024": { @@ -709,7 +731,7 @@ "IpProtocol": "udp" } }, - "AWSEC2SecurityGroupIngresssshexternaltomaster00000": { + "AWSEC2SecurityGroupIngresssshexternaltomaster111132": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { @@ -718,10 +740,22 @@ "FromPort": 22, "ToPort": 22, "IpProtocol": "tcp", - "CidrIp": "0.0.0.0/0" + "CidrIp": "1.1.1.1/32" } }, - "AWSEC2SecurityGroupIngresssshexternaltonode00000": { + "AWSEC2SecurityGroupIngresssshexternaltomaster2001085a348": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmasterscomplexexamplecom" + }, + "FromPort": 22, + "ToPort": 22, + "IpProtocol": "tcp", + "CidrIp": "2001:0:85a3::/48" + } + }, + "AWSEC2SecurityGroupIngresssshexternaltonode111132": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { "GroupId": { @@ -730,7 +764,19 @@ "FromPort": 22, "ToPort": 22, "IpProtocol": "tcp", - "CidrIp": "0.0.0.0/0" + "CidrIp": "1.1.1.1/32" + } + }, + "AWSEC2SecurityGroupIngresssshexternaltonode2001085a348": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupnodescomplexexamplecom" + }, + "FromPort": 22, + "ToPort": 22, + "IpProtocol": "tcp", + "CidrIp": "2001:0:85a3::/48" } }, "AWSEC2SecurityGroupapielbcomplexexamplecom": { @@ -877,6 +923,15 @@ ] } }, + "AWSEC2VPCCidrBlock1010016": { + "Type": "AWS::EC2::VPCCidrBlock", + "Properties": { + "VpcId": { + "Ref": "AWSEC2VPCcomplexexamplecom" + }, + "CidrBlock": "10.1.0.0/16" + } + }, "AWSEC2VPCDHCPOptionsAssociationcomplexexamplecom": { "Type": "AWS::EC2::VPCDHCPOptionsAssociation", "Properties": { @@ -1041,7 +1096,7 @@ "ConnectionSettings": { "IdleTimeout": 300 }, - "CrossZone": false, + "CrossZone": true, "Tags": [ { "Key": "KubernetesCluster", diff --git a/tests/integration/update_cluster/complex/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/complex/cloudformation.json.extracted.yaml index 629aef8eab..5517d40f68 100644 --- a/tests/integration/update_cluster/complex/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/complex/cloudformation.json.extracted.yaml @@ -1,4 +1,13 @@ Resources.AWSEC2LaunchTemplatemasterustest1amasterscomplexexamplecom.Properties.LaunchTemplateData.UserData: | + Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" + MIME-Version: 1.0 + + --MIMEBOUNDARY + Content-Disposition: attachment; filename="nodeup.sh" + Content-Transfer-Encoding: 7bit + Content-Type: text/x-shellscript + Mime-Version: 1.0 + #!/bin/bash # Copyright 2016 The Kubernetes Authors All rights reserved. # @@ -315,7 +324,27 @@ Resources.AWSEC2LaunchTemplatemasterustest1amasterscomplexexamplecom.Properties. download-release echo "== nodeup node config done ==" + + --MIMEBOUNDARY + Content-Disposition: attachment; filename="myscript.sh" + Content-Transfer-Encoding: 7bit + Content-Type: text/x-shellscript + Mime-Version: 1.0 + + #!/bin/sh + echo "nodes: The time is now $(date -R)!" | tee /root/output.txt + + --MIMEBOUNDARY-- Resources.AWSEC2LaunchTemplatenodescomplexexamplecom.Properties.LaunchTemplateData.UserData: | + Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" + MIME-Version: 1.0 + + --MIMEBOUNDARY + Content-Disposition: attachment; filename="nodeup.sh" + Content-Transfer-Encoding: 7bit + Content-Type: text/x-shellscript + Mime-Version: 1.0 + #!/bin/bash # Copyright 2016 The Kubernetes Authors All rights reserved. # @@ -537,3 +566,14 @@ Resources.AWSEC2LaunchTemplatenodescomplexexamplecom.Properties.LaunchTemplateDa download-release echo "== nodeup node config done ==" + + --MIMEBOUNDARY + Content-Disposition: attachment; filename="myscript.sh" + Content-Transfer-Encoding: 7bit + Content-Type: text/x-shellscript + Mime-Version: 1.0 + + #!/bin/sh + echo "nodes: The time is now $(date -R)!" | tee /root/output.txt + + --MIMEBOUNDARY-- diff --git a/tests/integration/update_cluster/complex/data/aws_launch_template_master-us-test-1a.masters.complex.example.com_user_data b/tests/integration/update_cluster/complex/data/aws_launch_template_master-us-test-1a.masters.complex.example.com_user_data index 27b9556c3e..a444bc8d5a 100644 --- a/tests/integration/update_cluster/complex/data/aws_launch_template_master-us-test-1a.masters.complex.example.com_user_data +++ b/tests/integration/update_cluster/complex/data/aws_launch_template_master-us-test-1a.masters.complex.example.com_user_data @@ -1,3 +1,12 @@ +Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" +MIME-Version: 1.0 + +--MIMEBOUNDARY +Content-Disposition: attachment; filename="nodeup.sh" +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript +Mime-Version: 1.0 + #!/bin/bash # Copyright 2016 The Kubernetes Authors All rights reserved. # @@ -314,3 +323,14 @@ __EOF_KUBE_ENV download-release echo "== nodeup node config done ==" + +--MIMEBOUNDARY +Content-Disposition: attachment; filename="myscript.sh" +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript +Mime-Version: 1.0 + +#!/bin/sh +echo "nodes: The time is now $(date -R)!" | tee /root/output.txt + +--MIMEBOUNDARY-- diff --git a/tests/integration/update_cluster/complex/data/aws_launch_template_nodes.complex.example.com_user_data b/tests/integration/update_cluster/complex/data/aws_launch_template_nodes.complex.example.com_user_data index 0fe40f44ed..ded8ba0628 100644 --- a/tests/integration/update_cluster/complex/data/aws_launch_template_nodes.complex.example.com_user_data +++ b/tests/integration/update_cluster/complex/data/aws_launch_template_nodes.complex.example.com_user_data @@ -1,3 +1,12 @@ +Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" +MIME-Version: 1.0 + +--MIMEBOUNDARY +Content-Disposition: attachment; filename="nodeup.sh" +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript +Mime-Version: 1.0 + #!/bin/bash # Copyright 2016 The Kubernetes Authors All rights reserved. # @@ -219,3 +228,14 @@ __EOF_KUBE_ENV download-release echo "== nodeup node config done ==" + +--MIMEBOUNDARY +Content-Disposition: attachment; filename="myscript.sh" +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript +Mime-Version: 1.0 + +#!/bin/sh +echo "nodes: The time is now $(date -R)!" | tee /root/output.txt + +--MIMEBOUNDARY-- diff --git a/tests/integration/update_cluster/complex/in-legacy-v1alpha2.yaml b/tests/integration/update_cluster/complex/in-legacy-v1alpha2.yaml index 2de2d20f33..e8e5de2a8a 100644 --- a/tests/integration/update_cluster/complex/in-legacy-v1alpha2.yaml +++ b/tests/integration/update_cluster/complex/in-legacy-v1alpha2.yaml @@ -10,8 +10,10 @@ spec: additionalSecurityGroups: - sg-exampleid3 - sg-exampleid4 + crossZoneLoadBalancing: true kubernetesApiAccess: - - 0.0.0.0/0 + - 1.1.1.0/24 + - 2001:0:8500::/40 channel: stable cloudProvider: aws cloudLabels: @@ -36,6 +38,8 @@ spec: masterInternalName: api.internal.complex.example.com masterPublicName: api.complex.example.com networkCIDR: 172.20.0.0/16 + additionalNetworkCIDRs: + - 10.1.0.0/16 networking: kubenet: {} nodePortAccess: @@ -43,7 +47,9 @@ spec: - 10.20.30.0/24 nonMasqueradeCIDR: 100.64.0.0/10 sshAccess: - - 0.0.0.0/0 + - 1.1.1.1/32 + - 2001:0:85a3::/48 + sshKeyName: "" topology: masters: public nodes: public @@ -83,6 +89,12 @@ spec: deleteOnTermination: false size: 20 type: gp2 + additionalUserData: + - name: myscript.sh + type: text/x-shellscript + content: | + #!/bin/sh + echo "nodes: The time is now $(date -R)!" | tee /root/output.txt --- @@ -102,3 +114,9 @@ spec: role: Master subnets: - us-test-1a + additionalUserData: + - name: myscript.sh + type: text/x-shellscript + content: | + #!/bin/sh + echo "nodes: The time is now $(date -R)!" | tee /root/output.txt diff --git a/tests/integration/update_cluster/complex/in-v1alpha2.yaml b/tests/integration/update_cluster/complex/in-v1alpha2.yaml index f5bcaa7f30..09a1486221 100644 --- a/tests/integration/update_cluster/complex/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/complex/in-v1alpha2.yaml @@ -10,8 +10,10 @@ spec: additionalSecurityGroups: - sg-exampleid3 - sg-exampleid4 + crossZoneLoadBalancing: true kubernetesApiAccess: - - 0.0.0.0/0 + - 1.1.1.0/24 + - 2001:0:8500::/40 channel: stable cloudProvider: aws cloudLabels: @@ -36,6 +38,8 @@ spec: masterInternalName: api.internal.complex.example.com masterPublicName: api.complex.example.com networkCIDR: 172.20.0.0/16 + additionalNetworkCIDRs: + - 10.1.0.0/16 networking: kubenet: {} nodePortAccess: @@ -43,7 +47,9 @@ spec: - 10.20.30.0/24 nonMasqueradeCIDR: 100.64.0.0/10 sshAccess: - - 0.0.0.0/0 + - 1.1.1.1/32 + - 2001:0:85a3::/48 + sshKeyName: "" topology: masters: public nodes: public @@ -83,6 +89,12 @@ spec: deleteOnTermination: false size: 20 type: gp2 + additionalUserData: + - name: myscript.sh + type: text/x-shellscript + content: | + #!/bin/sh + echo "nodes: The time is now $(date -R)!" | tee /root/output.txt --- @@ -102,3 +114,9 @@ spec: role: Master subnets: - us-test-1a + additionalUserData: + - name: myscript.sh + type: text/x-shellscript + content: | + #!/bin/sh + echo "nodes: The time is now $(date -R)!" | tee /root/output.txt diff --git a/tests/integration/update_cluster/complex/kubernetes.tf b/tests/integration/update_cluster/complex/kubernetes.tf index bc39165329..9f9a1fc434 100644 --- a/tests/integration/update_cluster/complex/kubernetes.tf +++ b/tests/integration/update_cluster/complex/kubernetes.tf @@ -215,7 +215,7 @@ resource "aws_ebs_volume" "us-test-1a-etcd-main-complex-example-com" { } resource "aws_elb" "api-complex-example-com" { - cross_zone_load_balancing = false + cross_zone_load_balancing = true health_check { healthy_threshold = 2 interval = 10 @@ -286,11 +286,6 @@ resource "aws_internet_gateway" "complex-example-com" { vpc_id = aws_vpc.complex-example-com.id } -resource "aws_key_pair" "kubernetes-complex-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" { - key_name = "kubernetes.complex.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57" - public_key = file("${path.module}/data/aws_key_pair_kubernetes.complex.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key") -} - resource "aws_launch_template" "master-us-test-1a-masters-complex-example-com" { block_device_mappings { device_name = "/dev/xvda" @@ -309,7 +304,6 @@ resource "aws_launch_template" "master-us-test-1a-masters-complex-example-com" { } image_id = "ami-12345678" instance_type = "m3.medium" - key_name = aws_key_pair.kubernetes-complex-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id lifecycle { create_before_destroy = true } @@ -368,7 +362,6 @@ resource "aws_launch_template" "nodes-complex-example-com" { } image_id = "ami-12345678" instance_type = "t2.medium" - key_name = aws_key_pair.kubernetes-complex-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id lifecycle { create_before_destroy = true } @@ -475,8 +468,17 @@ resource "aws_security_group_rule" "api-elb-egress" { type = "egress" } -resource "aws_security_group_rule" "https-api-elb-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] +resource "aws_security_group_rule" "https-api-elb-1-1-1-0--24" { + cidr_blocks = ["1.1.1.0/24"] + from_port = 443 + protocol = "tcp" + security_group_id = aws_security_group.api-elb-complex-example-com.id + to_port = 443 + type = "ingress" +} + +resource "aws_security_group_rule" "https-api-elb-2001_0_8500__--40" { + cidr_blocks = ["2001:0:8500::/40"] from_port = 443 protocol = "tcp" security_group_id = aws_security_group.api-elb-complex-example-com.id @@ -493,8 +495,17 @@ resource "aws_security_group_rule" "https-elb-to-master" { type = "ingress" } -resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] +resource "aws_security_group_rule" "icmp-pmtu-api-elb-1-1-1-0--24" { + cidr_blocks = ["1.1.1.0/24"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.api-elb-complex-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-api-elb-2001_0_8500__--40" { + cidr_blocks = ["2001:0:8500::/40"] from_port = 3 protocol = "icmp" security_group_id = aws_security_group.api-elb-complex-example-com.id @@ -592,8 +603,8 @@ resource "aws_security_group_rule" "nodeport-udp-external-to-node-10-20-30-0--24 type = "ingress" } -resource "aws_security_group_rule" "ssh-external-to-master-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] +resource "aws_security_group_rule" "ssh-external-to-master-1-1-1-1--32" { + cidr_blocks = ["1.1.1.1/32"] from_port = 22 protocol = "tcp" security_group_id = aws_security_group.masters-complex-example-com.id @@ -601,8 +612,26 @@ resource "aws_security_group_rule" "ssh-external-to-master-0-0-0-0--0" { type = "ingress" } -resource "aws_security_group_rule" "ssh-external-to-node-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] +resource "aws_security_group_rule" "ssh-external-to-master-2001_0_85a3__--48" { + cidr_blocks = ["2001:0:85a3::/48"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.masters-complex-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "ssh-external-to-node-1-1-1-1--32" { + cidr_blocks = ["1.1.1.1/32"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.nodes-complex-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "ssh-external-to-node-2001_0_85a3__--48" { + cidr_blocks = ["2001:0:85a3::/48"] from_port = 22 protocol = "tcp" security_group_id = aws_security_group.nodes-complex-example-com.id @@ -681,6 +710,11 @@ resource "aws_vpc_dhcp_options" "complex-example-com" { } } +resource "aws_vpc_ipv4_cidr_block_association" "cidr-10-1-0-0--16" { + cidr_block = "10.1.0.0/16" + vpc_id = aws_vpc.complex-example-com.id +} + resource "aws_vpc" "complex-example-com" { cidr_block = "172.20.0.0/16" enable_dns_hostnames = true diff --git a/tests/integration/update_cluster/nosshkey-cloudformation/cloudformation.json b/tests/integration/update_cluster/nosshkey-cloudformation/cloudformation.json deleted file mode 100644 index 1df2f0eb4e..0000000000 --- a/tests/integration/update_cluster/nosshkey-cloudformation/cloudformation.json +++ /dev/null @@ -1,1001 +0,0 @@ -{ - "Resources": { - "AWSAutoScalingAutoScalingGroupmasterustest1amastersnosshkeyexamplecom": { - "Type": "AWS::AutoScaling::AutoScalingGroup", - "Properties": { - "AutoScalingGroupName": "master-us-test-1a.masters.nosshkey.example.com", - "LaunchTemplate": { - "LaunchTemplateId": { - "Ref": "AWSEC2LaunchTemplatemasterustest1amastersnosshkeyexamplecom" - }, - "Version": { - "Fn::GetAtt": [ - "AWSEC2LaunchTemplatemasterustest1amastersnosshkeyexamplecom", - "LatestVersionNumber" - ] - } - }, - "MaxSize": 1, - "MinSize": 1, - "VPCZoneIdentifier": [ - { - "Ref": "AWSEC2Subnetustest1anosshkeyexamplecom" - } - ], - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "nosshkey.example.com", - "PropagateAtLaunch": true - }, - { - "Key": "Name", - "Value": "master-us-test-1a.masters.nosshkey.example.com", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/role/master", - "Value": "1", - "PropagateAtLaunch": true - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "master-us-test-1a", - "PropagateAtLaunch": true - }, - { - "Key": "kubernetes.io/cluster/nosshkey.example.com", - "Value": "owned", - "PropagateAtLaunch": true - } - ], - "MetricsCollection": [ - { - "Granularity": "1Minute", - "Metrics": [ - "GroupDesiredCapacity", - "GroupInServiceInstances", - "GroupMaxSize", - "GroupMinSize", - "GroupPendingInstances", - "GroupStandbyInstances", - "GroupTerminatingInstances", - "GroupTotalInstances" - ] - } - ] - } - }, - "AWSAutoScalingAutoScalingGroupnodesnosshkeyexamplecom": { - "Type": "AWS::AutoScaling::AutoScalingGroup", - "Properties": { - "AutoScalingGroupName": "nodes.nosshkey.example.com", - "LaunchTemplate": { - "LaunchTemplateId": { - "Ref": "AWSEC2LaunchTemplatenodesnosshkeyexamplecom" - }, - "Version": { - "Fn::GetAtt": [ - "AWSEC2LaunchTemplatenodesnosshkeyexamplecom", - "LatestVersionNumber" - ] - } - }, - "MaxSize": 2, - "MinSize": 2, - "VPCZoneIdentifier": [ - { - "Ref": "AWSEC2Subnetustest1anosshkeyexamplecom" - } - ], - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "nosshkey.example.com", - "PropagateAtLaunch": true - }, - { - "Key": "Name", - "Value": "nodes.nosshkey.example.com", - "PropagateAtLaunch": true - }, - { - "Key": "k8s.io/role/node", - "Value": "1", - "PropagateAtLaunch": true - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "nodes", - "PropagateAtLaunch": true - }, - { - "Key": "kubernetes.io/cluster/nosshkey.example.com", - "Value": "owned", - "PropagateAtLaunch": true - } - ], - "MetricsCollection": [ - { - "Granularity": "1Minute", - "Metrics": [ - "GroupDesiredCapacity", - "GroupInServiceInstances", - "GroupMaxSize", - "GroupMinSize", - "GroupPendingInstances", - "GroupStandbyInstances", - "GroupTerminatingInstances", - "GroupTotalInstances" - ] - } - ] - } - }, - "AWSEC2DHCPOptionsnosshkeyexamplecom": { - "Type": "AWS::EC2::DHCPOptions", - "Properties": { - "DomainName": "us-test-1.compute.internal", - "DomainNameServers": [ - "AmazonProvidedDNS" - ], - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "nosshkey.example.com" - }, - { - "Key": "Name", - "Value": "nosshkey.example.com" - }, - { - "Key": "kubernetes.io/cluster/nosshkey.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2InternetGatewaynosshkeyexamplecom": { - "Type": "AWS::EC2::InternetGateway", - "Properties": { - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "nosshkey.example.com" - }, - { - "Key": "Name", - "Value": "nosshkey.example.com" - }, - { - "Key": "kubernetes.io/cluster/nosshkey.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2LaunchTemplatemasterustest1amastersnosshkeyexamplecom": { - "Type": "AWS::EC2::LaunchTemplate", - "Properties": { - "LaunchTemplateName": "master-us-test-1a.masters.nosshkey.example.com", - "LaunchTemplateData": { - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/xvda", - "Ebs": { - "VolumeType": "gp2", - "VolumeSize": 64, - "DeleteOnTermination": true - } - }, - { - "DeviceName": "/dev/sdc", - "VirtualName": "ephemeral0" - } - ], - "IamInstanceProfile": { - "Name": { - "Ref": "AWSIAMInstanceProfilemastersnosshkeyexamplecom" - } - }, - "ImageId": "ami-12345678", - "InstanceType": "m3.medium", - "NetworkInterfaces": [ - { - "AssociatePublicIpAddress": true, - "DeleteOnTermination": true, - "DeviceIndex": 0, - "Groups": [ - { - "Ref": "AWSEC2SecurityGroupmastersnosshkeyexamplecom" - } - ] - } - ], - "TagSpecifications": [ - { - "ResourceType": "instance", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "nosshkey.example.com" - }, - { - "Key": "Name", - "Value": "master-us-test-1a.masters.nosshkey.example.com" - }, - { - "Key": "k8s.io/role/master", - "Value": "1" - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "master-us-test-1a" - }, - { - "Key": "kubernetes.io/cluster/nosshkey.example.com", - "Value": "owned" - } - ] - }, - { - "ResourceType": "volume", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "nosshkey.example.com" - }, - { - "Key": "Name", - "Value": "master-us-test-1a.masters.nosshkey.example.com" - }, - { - "Key": "k8s.io/role/master", - "Value": "1" - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "master-us-test-1a" - }, - { - "Key": "kubernetes.io/cluster/nosshkey.example.com", - "Value": "owned" - } - ] - } - ], - "UserData": "extracted" - } - } - }, - "AWSEC2LaunchTemplatenodesnosshkeyexamplecom": { - "Type": "AWS::EC2::LaunchTemplate", - "Properties": { - "LaunchTemplateName": "nodes.nosshkey.example.com", - "LaunchTemplateData": { - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/xvda", - "Ebs": { - "VolumeType": "gp2", - "VolumeSize": 128, - "DeleteOnTermination": true - } - } - ], - "IamInstanceProfile": { - "Name": { - "Ref": "AWSIAMInstanceProfilenodesnosshkeyexamplecom" - } - }, - "ImageId": "ami-12345678", - "InstanceType": "t2.medium", - "NetworkInterfaces": [ - { - "AssociatePublicIpAddress": true, - "DeleteOnTermination": true, - "DeviceIndex": 0, - "Groups": [ - { - "Ref": "AWSEC2SecurityGroupnodesnosshkeyexamplecom" - } - ] - } - ], - "TagSpecifications": [ - { - "ResourceType": "instance", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "nosshkey.example.com" - }, - { - "Key": "Name", - "Value": "nodes.nosshkey.example.com" - }, - { - "Key": "k8s.io/role/node", - "Value": "1" - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "nodes" - }, - { - "Key": "kubernetes.io/cluster/nosshkey.example.com", - "Value": "owned" - } - ] - }, - { - "ResourceType": "volume", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "nosshkey.example.com" - }, - { - "Key": "Name", - "Value": "nodes.nosshkey.example.com" - }, - { - "Key": "k8s.io/role/node", - "Value": "1" - }, - { - "Key": "kops.k8s.io/instancegroup", - "Value": "nodes" - }, - { - "Key": "kubernetes.io/cluster/nosshkey.example.com", - "Value": "owned" - } - ] - } - ], - "UserData": "extracted" - } - } - }, - "AWSEC2Route00000": { - "Type": "AWS::EC2::Route", - "Properties": { - "RouteTableId": { - "Ref": "AWSEC2RouteTablenosshkeyexamplecom" - }, - "DestinationCidrBlock": "0.0.0.0/0", - "GatewayId": { - "Ref": "AWSEC2InternetGatewaynosshkeyexamplecom" - } - } - }, - "AWSEC2RouteTablenosshkeyexamplecom": { - "Type": "AWS::EC2::RouteTable", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPCnosshkeyexamplecom" - }, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "nosshkey.example.com" - }, - { - "Key": "Name", - "Value": "nosshkey.example.com" - }, - { - "Key": "kubernetes.io/cluster/nosshkey.example.com", - "Value": "owned" - }, - { - "Key": "kubernetes.io/kops/role", - "Value": "public" - } - ] - } - }, - "AWSEC2SecurityGroupEgressmasteregress": { - "Type": "AWS::EC2::SecurityGroupEgress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersnosshkeyexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1", - "CidrIp": "0.0.0.0/0" - } - }, - "AWSEC2SecurityGroupEgressnodeegress": { - "Type": "AWS::EC2::SecurityGroupEgress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupnodesnosshkeyexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1", - "CidrIp": "0.0.0.0/0" - } - }, - "AWSEC2SecurityGroupIngressallmastertomaster": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersnosshkeyexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupmastersnosshkeyexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1" - } - }, - "AWSEC2SecurityGroupIngressallmastertonode": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupnodesnosshkeyexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupmastersnosshkeyexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1" - } - }, - "AWSEC2SecurityGroupIngressallnodetonode": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupnodesnosshkeyexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodesnosshkeyexamplecom" - }, - "FromPort": 0, - "ToPort": 0, - "IpProtocol": "-1" - } - }, - "AWSEC2SecurityGroupIngresshttpsexternaltomaster00000": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersnosshkeyexamplecom" - }, - "FromPort": 443, - "ToPort": 443, - "IpProtocol": "tcp", - "CidrIp": "0.0.0.0/0" - } - }, - "AWSEC2SecurityGroupIngressnodetomastertcp12379": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersnosshkeyexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodesnosshkeyexamplecom" - }, - "FromPort": 1, - "ToPort": 2379, - "IpProtocol": "tcp" - } - }, - "AWSEC2SecurityGroupIngressnodetomastertcp23824000": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersnosshkeyexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodesnosshkeyexamplecom" - }, - "FromPort": 2382, - "ToPort": 4000, - "IpProtocol": "tcp" - } - }, - "AWSEC2SecurityGroupIngressnodetomastertcp400365535": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersnosshkeyexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodesnosshkeyexamplecom" - }, - "FromPort": 4003, - "ToPort": 65535, - "IpProtocol": "tcp" - } - }, - "AWSEC2SecurityGroupIngressnodetomasterudp165535": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersnosshkeyexamplecom" - }, - "SourceSecurityGroupId": { - "Ref": "AWSEC2SecurityGroupnodesnosshkeyexamplecom" - }, - "FromPort": 1, - "ToPort": 65535, - "IpProtocol": "udp" - } - }, - "AWSEC2SecurityGroupIngresssshexternaltomaster00000": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupmastersnosshkeyexamplecom" - }, - "FromPort": 22, - "ToPort": 22, - "IpProtocol": "tcp", - "CidrIp": "0.0.0.0/0" - } - }, - "AWSEC2SecurityGroupIngresssshexternaltonode00000": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "AWSEC2SecurityGroupnodesnosshkeyexamplecom" - }, - "FromPort": 22, - "ToPort": 22, - "IpProtocol": "tcp", - "CidrIp": "0.0.0.0/0" - } - }, - "AWSEC2SecurityGroupmastersnosshkeyexamplecom": { - "Type": "AWS::EC2::SecurityGroup", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPCnosshkeyexamplecom" - }, - "GroupDescription": "Security group for masters", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "nosshkey.example.com" - }, - { - "Key": "Name", - "Value": "masters.nosshkey.example.com" - }, - { - "Key": "kubernetes.io/cluster/nosshkey.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2SecurityGroupnodesnosshkeyexamplecom": { - "Type": "AWS::EC2::SecurityGroup", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPCnosshkeyexamplecom" - }, - "GroupDescription": "Security group for nodes", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "nosshkey.example.com" - }, - { - "Key": "Name", - "Value": "nodes.nosshkey.example.com" - }, - { - "Key": "kubernetes.io/cluster/nosshkey.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2SubnetRouteTableAssociationustest1anosshkeyexamplecom": { - "Type": "AWS::EC2::SubnetRouteTableAssociation", - "Properties": { - "SubnetId": { - "Ref": "AWSEC2Subnetustest1anosshkeyexamplecom" - }, - "RouteTableId": { - "Ref": "AWSEC2RouteTablenosshkeyexamplecom" - } - } - }, - "AWSEC2Subnetustest1anosshkeyexamplecom": { - "Type": "AWS::EC2::Subnet", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPCnosshkeyexamplecom" - }, - "CidrBlock": "172.20.32.0/19", - "AvailabilityZone": "us-test-1a", - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "nosshkey.example.com" - }, - { - "Key": "Name", - "Value": "us-test-1a.nosshkey.example.com" - }, - { - "Key": "SubnetType", - "Value": "Public" - }, - { - "Key": "kubernetes.io/cluster/nosshkey.example.com", - "Value": "owned" - }, - { - "Key": "kubernetes.io/role/elb", - "Value": "1" - } - ] - } - }, - "AWSEC2VPCDHCPOptionsAssociationnosshkeyexamplecom": { - "Type": "AWS::EC2::VPCDHCPOptionsAssociation", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPCnosshkeyexamplecom" - }, - "DhcpOptionsId": { - "Ref": "AWSEC2DHCPOptionsnosshkeyexamplecom" - } - } - }, - "AWSEC2VPCGatewayAttachmentnosshkeyexamplecom": { - "Type": "AWS::EC2::VPCGatewayAttachment", - "Properties": { - "VpcId": { - "Ref": "AWSEC2VPCnosshkeyexamplecom" - }, - "InternetGatewayId": { - "Ref": "AWSEC2InternetGatewaynosshkeyexamplecom" - } - } - }, - "AWSEC2VPCnosshkeyexamplecom": { - "Type": "AWS::EC2::VPC", - "Properties": { - "CidrBlock": "172.20.0.0/16", - "EnableDnsHostnames": true, - "EnableDnsSupport": true, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "nosshkey.example.com" - }, - { - "Key": "Name", - "Value": "nosshkey.example.com" - }, - { - "Key": "kubernetes.io/cluster/nosshkey.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2Volumeustest1aetcdeventsnosshkeyexamplecom": { - "Type": "AWS::EC2::Volume", - "Properties": { - "AvailabilityZone": "us-test-1a", - "Size": 20, - "VolumeType": "gp2", - "Encrypted": false, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "nosshkey.example.com" - }, - { - "Key": "Name", - "Value": "us-test-1a.etcd-events.nosshkey.example.com" - }, - { - "Key": "k8s.io/etcd/events", - "Value": "us-test-1a/us-test-1a" - }, - { - "Key": "k8s.io/role/master", - "Value": "1" - }, - { - "Key": "kubernetes.io/cluster/nosshkey.example.com", - "Value": "owned" - } - ] - } - }, - "AWSEC2Volumeustest1aetcdmainnosshkeyexamplecom": { - "Type": "AWS::EC2::Volume", - "Properties": { - "AvailabilityZone": "us-test-1a", - "Size": 20, - "VolumeType": "gp2", - "Encrypted": false, - "Tags": [ - { - "Key": "KubernetesCluster", - "Value": "nosshkey.example.com" - }, - { - "Key": "Name", - "Value": "us-test-1a.etcd-main.nosshkey.example.com" - }, - { - "Key": "k8s.io/etcd/main", - "Value": "us-test-1a/us-test-1a" - }, - { - "Key": "k8s.io/role/master", - "Value": "1" - }, - { - "Key": "kubernetes.io/cluster/nosshkey.example.com", - "Value": "owned" - } - ] - } - }, - "AWSIAMInstanceProfilemastersnosshkeyexamplecom": { - "Type": "AWS::IAM::InstanceProfile", - "Properties": { - "Roles": [ - { - "Ref": "AWSIAMRolemastersnosshkeyexamplecom" - } - ] - } - }, - "AWSIAMInstanceProfilenodesnosshkeyexamplecom": { - "Type": "AWS::IAM::InstanceProfile", - "Properties": { - "Roles": [ - { - "Ref": "AWSIAMRolenodesnosshkeyexamplecom" - } - ] - } - }, - "AWSIAMPolicymastersnosshkeyexamplecom": { - "Type": "AWS::IAM::Policy", - "Properties": { - "PolicyName": "masters.nosshkey.example.com", - "Roles": [ - { - "Ref": "AWSIAMRolemastersnosshkeyexamplecom" - } - ], - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "ec2:*" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:DescribeAutoScalingInstances", - "autoscaling:DescribeLaunchConfigurations", - "autoscaling:DescribeTags", - "autoscaling:SetDesiredCapacity", - "autoscaling:TerminateInstanceInAutoScalingGroup", - "autoscaling:UpdateAutoScalingGroup", - "ec2:DescribeLaunchTemplateVersions" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "elasticloadbalancing:*" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "iam:ListServerCertificates", - "iam:GetServerCertificate" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "route53:ChangeResourceRecordSets", - "route53:ListResourceRecordSets", - "route53:GetHostedZone" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" - ] - }, - { - "Action": [ - "route53:GetChange" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:route53:::change/*" - ] - }, - { - "Action": [ - "route53:ListHostedZones" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "route53:ListHostedZones" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "ecr:GetAuthorizationToken", - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:GetRepositoryPolicy", - "ecr:DescribeRepositories", - "ecr:ListImages", - "ecr:BatchGetImage" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - } - ], - "Version": "2012-10-17" - } - } - }, - "AWSIAMPolicynodesnosshkeyexamplecom": { - "Type": "AWS::IAM::Policy", - "Properties": { - "PolicyName": "nodes.nosshkey.example.com", - "Roles": [ - { - "Ref": "AWSIAMRolenodesnosshkeyexamplecom" - } - ], - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "ec2:DescribeInstances", - "ec2:DescribeRegions" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "route53:ChangeResourceRecordSets", - "route53:ListResourceRecordSets", - "route53:GetHostedZone" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" - ] - }, - { - "Action": [ - "route53:GetChange" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:route53:::change/*" - ] - }, - { - "Action": [ - "route53:ListHostedZones" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "route53:ListHostedZones" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - }, - { - "Action": [ - "ecr:GetAuthorizationToken", - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:GetRepositoryPolicy", - "ecr:DescribeRepositories", - "ecr:ListImages", - "ecr:BatchGetImage" - ], - "Effect": "Allow", - "Resource": [ - "*" - ] - } - ], - "Version": "2012-10-17" - } - } - }, - "AWSIAMRolemastersnosshkeyexamplecom": { - "Type": "AWS::IAM::Role", - "Properties": { - "RoleName": "masters.nosshkey.example.com", - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "ec2.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - } - } - }, - "AWSIAMRolenodesnosshkeyexamplecom": { - "Type": "AWS::IAM::Role", - "Properties": { - "RoleName": "nodes.nosshkey.example.com", - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "ec2.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - } - } - } - } -} diff --git a/tests/integration/update_cluster/nosshkey-cloudformation/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/nosshkey-cloudformation/cloudformation.json.extracted.yaml deleted file mode 100644 index 0a9f8b78e1..0000000000 --- a/tests/integration/update_cluster/nosshkey-cloudformation/cloudformation.json.extracted.yaml +++ /dev/null @@ -1,541 +0,0 @@ -Resources.AWSEC2LaunchTemplatemasterustest1amastersnosshkeyexamplecom.Properties.LaunchTemplateData.UserData: | - #!/bin/bash - # Copyright 2016 The Kubernetes Authors All rights reserved. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - set -o errexit - set -o nounset - set -o pipefail - - NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup - NODEUP_HASH_AMD64=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 - NODEUP_URL_ARM64= - NODEUP_HASH_ARM64= - - export AWS_REGION=us-test-1 - - - - - function ensure-install-dir() { - INSTALL_DIR="/opt/kops" - # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec - if [[ -d /var/lib/toolbox ]]; then - INSTALL_DIR="/var/lib/toolbox/kops" - fi - mkdir -p ${INSTALL_DIR}/bin - mkdir -p ${INSTALL_DIR}/conf - cd ${INSTALL_DIR} - } - - # Retry a download until we get it. args: name, sha, url1, url2... - download-or-bust() { - local -r file="$1" - local -r hash="$2" - shift 2 - - urls=( $* ) - while true; do - for url in "${urls[@]}"; do - commands=( - "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - ) - for cmd in "${commands[@]}"; do - echo "Attempting download with: ${cmd} {url}" - if ! (${cmd} "${url}"); then - echo "== Download failed with ${cmd} ==" - continue - fi - if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then - echo "== Hash validation of ${url} failed. Retrying. ==" - rm -f "${file}" - else - if [[ -n "${hash}" ]]; then - echo "== Downloaded ${url} (SHA1 = ${hash}) ==" - else - echo "== Downloaded ${url} ==" - fi - return - fi - done - done - - echo "All downloads failed; sleeping before retrying" - sleep 60 - done - } - - validate-hash() { - local -r file="$1" - local -r expected="$2" - local actual - - actual=$(sha256sum ${file} | awk '{ print $1 }') || true - if [[ "${actual}" != "${expected}" ]]; then - echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" - return 1 - fi - } - - function split-commas() { - echo $1 | tr "," "\n" - } - - function try-download-release() { - local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) - if [[ -n "${NODEUP_HASH:-}" ]]; then - local -r nodeup_hash="${NODEUP_HASH}" - else - # TODO: Remove? - echo "Downloading sha256 (not found in env)" - download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" - local -r nodeup_hash=$(cat nodeup.sha256) - fi - - echo "Downloading nodeup (${nodeup_urls[@]})" - download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" - - chmod +x nodeup - } - - function download-release() { - case "$(uname -m)" in - x86_64*|i?86_64*|amd64*) - NODEUP_URL="${NODEUP_URL_AMD64}" - NODEUP_HASH="${NODEUP_HASH_AMD64}" - ;; - aarch64*|arm64*) - NODEUP_URL="${NODEUP_URL_ARM64}" - NODEUP_HASH="${NODEUP_HASH_ARM64}" - ;; - *) - echo "Unsupported host arch: $(uname -m)" >&2 - exit 1 - ;; - esac - - # In case of failure checking integrity of release, retry. - cd ${INSTALL_DIR}/bin - until try-download-release; do - sleep 15 - echo "Couldn't download release. Retrying..." - done - - echo "Running nodeup" - # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 - ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) - } - - #################################################################################### - - /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" - - echo "== nodeup node config starting ==" - ensure-install-dir - - cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' - cloudConfig: null - containerRuntime: docker - containerd: - skipInstall: true - docker: - ipMasq: false - ipTables: false - logDriver: json-file - logLevel: info - logOpt: - - max-size=10m - - max-file=5 - storage: overlay2,overlay,aufs - version: 17.03.2 - encryptionConfig: null - etcdClusters: - events: - image: k8s.gcr.io/etcd:2.2.1 - version: 2.2.1 - main: - image: k8s.gcr.io/etcd:2.2.1 - version: 2.2.1 - kubeAPIServer: - allowPrivileged: true - anonymousAuth: false - apiServerCount: 1 - authorizationMode: AlwaysAllow - bindAddress: 0.0.0.0 - cloudProvider: aws - enableAdmissionPlugins: - - Initializers - - NamespaceLifecycle - - LimitRanger - - ServiceAccount - - PersistentVolumeLabel - - DefaultStorageClass - - DefaultTolerationSeconds - - MutatingAdmissionWebhook - - ValidatingAdmissionWebhook - - NodeRestriction - - ResourceQuota - etcdQuorumRead: false - etcdServers: - - http://127.0.0.1:4001 - etcdServersOverrides: - - /events#http://127.0.0.1:4002 - image: k8s.gcr.io/kube-apiserver:v1.11.10 - insecureBindAddress: 127.0.0.1 - insecurePort: 8080 - kubeletPreferredAddressTypes: - - InternalIP - - Hostname - - ExternalIP - logLevel: 2 - requestheaderAllowedNames: - - aggregator - requestheaderExtraHeaderPrefixes: - - X-Remote-Extra- - requestheaderGroupHeaders: - - X-Remote-Group - requestheaderUsernameHeaders: - - X-Remote-User - securePort: 443 - serviceClusterIPRange: 100.64.0.0/13 - storageBackend: etcd2 - kubeControllerManager: - allocateNodeCIDRs: true - attachDetachReconcileSyncPeriod: 1m0s - cloudProvider: aws - clusterCIDR: 100.96.0.0/11 - clusterName: nosshkey.example.com - configureCloudRoutes: true - image: k8s.gcr.io/kube-controller-manager:v1.11.10 - leaderElection: - leaderElect: true - logLevel: 2 - useServiceAccountCredentials: true - kubeProxy: - clusterCIDR: 100.96.0.0/11 - cpuRequest: 100m - hostnameOverride: '@aws' - image: k8s.gcr.io/kube-proxy:v1.11.10 - logLevel: 2 - kubeScheduler: - image: k8s.gcr.io/kube-scheduler:v1.11.10 - leaderElection: - leaderElect: true - logLevel: 2 - kubelet: - allowPrivileged: true - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests - masterKubelet: - allowPrivileged: true - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests - registerSchedulable: false - - __EOF_CLUSTER_SPEC - - cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' - kubelet: null - nodeLabels: null - taints: null - - __EOF_IG_SPEC - - cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' - Assets: - amd64: - - a1e5d2a7da4cabc29af0dda630564511a9b437d8@https://storage.googleapis.com/kubernetes-release/release/v1.11.10/bin/linux/amd64/kubelet - - c133f55152c76c33d9b41894dcd311064904503e@https://storage.googleapis.com/kubernetes-release/release/v1.11.10/bin/linux/amd64/kubectl - - 3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz - arm64: - - e895202b1ff6beaee15232f0c3118f2dc0cd4862@https://storage.googleapis.com/kubernetes-release/release/v1.11.10/bin/linux/arm64/kubelet - - b7b2bda2aff7a430a04c0e2b7a0ffe853ffb5288@https://storage.googleapis.com/kubernetes-release/release/v1.11.10/bin/linux/arm64/kubectl - - 7fec91af78e9548df306f0ec43bea527c8c10cc3a9682c33e971c8522a7fcded@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-arm64-v0.7.5.tgz - ClusterName: nosshkey.example.com - ConfigBase: memfs://clusters.example.com/nosshkey.example.com - InstanceGroupName: master-us-test-1a - InstanceGroupRole: Master - Tags: - - _automatic_upgrades - - _aws - channels: - - memfs://clusters.example.com/nosshkey.example.com/addons/bootstrap-channel.yaml - protokubeImage: - hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b - name: protokube:1.15.0 - sources: - - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz - - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz - - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz - - __EOF_KUBE_ENV - - download-release - echo "== nodeup node config done ==" -Resources.AWSEC2LaunchTemplatenodesnosshkeyexamplecom.Properties.LaunchTemplateData.UserData: | - #!/bin/bash - # Copyright 2016 The Kubernetes Authors All rights reserved. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - set -o errexit - set -o nounset - set -o pipefail - - NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup - NODEUP_HASH_AMD64=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 - NODEUP_URL_ARM64= - NODEUP_HASH_ARM64= - - export AWS_REGION=us-test-1 - - - - - function ensure-install-dir() { - INSTALL_DIR="/opt/kops" - # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec - if [[ -d /var/lib/toolbox ]]; then - INSTALL_DIR="/var/lib/toolbox/kops" - fi - mkdir -p ${INSTALL_DIR}/bin - mkdir -p ${INSTALL_DIR}/conf - cd ${INSTALL_DIR} - } - - # Retry a download until we get it. args: name, sha, url1, url2... - download-or-bust() { - local -r file="$1" - local -r hash="$2" - shift 2 - - urls=( $* ) - while true; do - for url in "${urls[@]}"; do - commands=( - "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - ) - for cmd in "${commands[@]}"; do - echo "Attempting download with: ${cmd} {url}" - if ! (${cmd} "${url}"); then - echo "== Download failed with ${cmd} ==" - continue - fi - if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then - echo "== Hash validation of ${url} failed. Retrying. ==" - rm -f "${file}" - else - if [[ -n "${hash}" ]]; then - echo "== Downloaded ${url} (SHA1 = ${hash}) ==" - else - echo "== Downloaded ${url} ==" - fi - return - fi - done - done - - echo "All downloads failed; sleeping before retrying" - sleep 60 - done - } - - validate-hash() { - local -r file="$1" - local -r expected="$2" - local actual - - actual=$(sha256sum ${file} | awk '{ print $1 }') || true - if [[ "${actual}" != "${expected}" ]]; then - echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" - return 1 - fi - } - - function split-commas() { - echo $1 | tr "," "\n" - } - - function try-download-release() { - local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) - if [[ -n "${NODEUP_HASH:-}" ]]; then - local -r nodeup_hash="${NODEUP_HASH}" - else - # TODO: Remove? - echo "Downloading sha256 (not found in env)" - download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" - local -r nodeup_hash=$(cat nodeup.sha256) - fi - - echo "Downloading nodeup (${nodeup_urls[@]})" - download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" - - chmod +x nodeup - } - - function download-release() { - case "$(uname -m)" in - x86_64*|i?86_64*|amd64*) - NODEUP_URL="${NODEUP_URL_AMD64}" - NODEUP_HASH="${NODEUP_HASH_AMD64}" - ;; - aarch64*|arm64*) - NODEUP_URL="${NODEUP_URL_ARM64}" - NODEUP_HASH="${NODEUP_HASH_ARM64}" - ;; - *) - echo "Unsupported host arch: $(uname -m)" >&2 - exit 1 - ;; - esac - - # In case of failure checking integrity of release, retry. - cd ${INSTALL_DIR}/bin - until try-download-release; do - sleep 15 - echo "Couldn't download release. Retrying..." - done - - echo "Running nodeup" - # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 - ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) - } - - #################################################################################### - - /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" - - echo "== nodeup node config starting ==" - ensure-install-dir - - cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' - cloudConfig: null - containerRuntime: docker - containerd: - skipInstall: true - docker: - ipMasq: false - ipTables: false - logDriver: json-file - logLevel: info - logOpt: - - max-size=10m - - max-file=5 - storage: overlay2,overlay,aufs - version: 17.03.2 - kubeProxy: - clusterCIDR: 100.96.0.0/11 - cpuRequest: 100m - hostnameOverride: '@aws' - image: k8s.gcr.io/kube-proxy:v1.11.10 - logLevel: 2 - kubelet: - allowPrivileged: true - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests - - __EOF_CLUSTER_SPEC - - cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' - kubelet: null - nodeLabels: null - taints: null - - __EOF_IG_SPEC - - cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' - Assets: - amd64: - - a1e5d2a7da4cabc29af0dda630564511a9b437d8@https://storage.googleapis.com/kubernetes-release/release/v1.11.10/bin/linux/amd64/kubelet - - c133f55152c76c33d9b41894dcd311064904503e@https://storage.googleapis.com/kubernetes-release/release/v1.11.10/bin/linux/amd64/kubectl - - 3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz - arm64: - - e895202b1ff6beaee15232f0c3118f2dc0cd4862@https://storage.googleapis.com/kubernetes-release/release/v1.11.10/bin/linux/arm64/kubelet - - b7b2bda2aff7a430a04c0e2b7a0ffe853ffb5288@https://storage.googleapis.com/kubernetes-release/release/v1.11.10/bin/linux/arm64/kubectl - - 7fec91af78e9548df306f0ec43bea527c8c10cc3a9682c33e971c8522a7fcded@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-arm64-v0.7.5.tgz - ClusterName: nosshkey.example.com - ConfigBase: memfs://clusters.example.com/nosshkey.example.com - InstanceGroupName: nodes - InstanceGroupRole: Node - Tags: - - _automatic_upgrades - - _aws - channels: - - memfs://clusters.example.com/nosshkey.example.com/addons/bootstrap-channel.yaml - - __EOF_KUBE_ENV - - download-release - echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/nosshkey-cloudformation/in-v1alpha2.yaml b/tests/integration/update_cluster/nosshkey-cloudformation/in-v1alpha2.yaml deleted file mode 100644 index 5d5a3dffeb..0000000000 --- a/tests/integration/update_cluster/nosshkey-cloudformation/in-v1alpha2.yaml +++ /dev/null @@ -1,80 +0,0 @@ -apiVersion: kops.k8s.io/v1alpha2 -kind: Cluster -metadata: - creationTimestamp: "2016-12-10T22:42:27Z" - name: nosshkey.example.com -spec: - kubernetesApiAccess: - - 0.0.0.0/0 - channel: stable - cloudProvider: aws - configBase: memfs://clusters.example.com/nosshkey.example.com - etcdClusters: - - etcdMembers: - - instanceGroup: master-us-test-1a - name: us-test-1a - name: main - - etcdMembers: - - instanceGroup: master-us-test-1a - name: us-test-1a - name: events - kubelet: - anonymousAuth: false - kubernetesVersion: v1.11.10 - masterInternalName: api.internal.nosshkey.example.com - masterPublicName: api.nosshkey.example.com - networkCIDR: 172.20.0.0/16 - networking: - kubenet: {} - nonMasqueradeCIDR: 100.64.0.0/10 - sshAccess: - - 0.0.0.0/0 - sshKeyName: "" - topology: - masters: public - nodes: public - subnets: - - cidr: 172.20.32.0/19 - name: us-test-1a - type: Public - zone: us-test-1a - ---- - -apiVersion: kops.k8s.io/v1alpha2 -kind: InstanceGroup -metadata: - creationTimestamp: "2016-12-10T22:42:28Z" - name: nodes - labels: - kops.k8s.io/cluster: nosshkey.example.com -spec: - associatePublicIp: true - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: t2.medium - maxSize: 2 - minSize: 2 - role: Node - subnets: - - us-test-1a - ---- - -apiVersion: kops.k8s.io/v1alpha2 -kind: InstanceGroup -metadata: - creationTimestamp: "2016-12-10T22:42:28Z" - name: master-us-test-1a - labels: - kops.k8s.io/cluster: nosshkey.example.com -spec: - associatePublicIp: true - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: m3.medium - maxSize: 1 - minSize: 1 - role: Master - subnets: - - us-test-1a - - diff --git a/tests/integration/update_cluster/nosshkey/data/aws_iam_role_masters.nosshkey.example.com_policy b/tests/integration/update_cluster/nosshkey/data/aws_iam_role_masters.nosshkey.example.com_policy deleted file mode 100644 index 66d5de1d5a..0000000000 --- a/tests/integration/update_cluster/nosshkey/data/aws_iam_role_masters.nosshkey.example.com_policy +++ /dev/null @@ -1,10 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { "Service": "ec2.amazonaws.com"}, - "Action": "sts:AssumeRole" - } - ] -} diff --git a/tests/integration/update_cluster/nosshkey/data/aws_iam_role_nodes.nosshkey.example.com_policy b/tests/integration/update_cluster/nosshkey/data/aws_iam_role_nodes.nosshkey.example.com_policy deleted file mode 100644 index 66d5de1d5a..0000000000 --- a/tests/integration/update_cluster/nosshkey/data/aws_iam_role_nodes.nosshkey.example.com_policy +++ /dev/null @@ -1,10 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { "Service": "ec2.amazonaws.com"}, - "Action": "sts:AssumeRole" - } - ] -} diff --git a/tests/integration/update_cluster/nosshkey/data/aws_iam_role_policy_masters.nosshkey.example.com_policy b/tests/integration/update_cluster/nosshkey/data/aws_iam_role_policy_masters.nosshkey.example.com_policy deleted file mode 100644 index 340dff1ef9..0000000000 --- a/tests/integration/update_cluster/nosshkey/data/aws_iam_role_policy_masters.nosshkey.example.com_policy +++ /dev/null @@ -1,102 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "ec2:*" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:DescribeAutoScalingInstances", - "autoscaling:DescribeLaunchConfigurations", - "autoscaling:DescribeTags", - "autoscaling:SetDesiredCapacity", - "autoscaling:TerminateInstanceInAutoScalingGroup", - "autoscaling:UpdateAutoScalingGroup", - "ec2:DescribeLaunchTemplateVersions" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "elasticloadbalancing:*" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "iam:ListServerCertificates", - "iam:GetServerCertificate" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ChangeResourceRecordSets", - "route53:ListResourceRecordSets", - "route53:GetHostedZone" - ], - "Resource": [ - "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:GetChange" - ], - "Resource": [ - "arn:aws:route53:::change/*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ListHostedZones" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ListHostedZones" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "ecr:GetAuthorizationToken", - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:GetRepositoryPolicy", - "ecr:DescribeRepositories", - "ecr:ListImages", - "ecr:BatchGetImage" - ], - "Resource": [ - "*" - ] - } - ] -} diff --git a/tests/integration/update_cluster/nosshkey/data/aws_iam_role_policy_nodes.nosshkey.example.com_policy b/tests/integration/update_cluster/nosshkey/data/aws_iam_role_policy_nodes.nosshkey.example.com_policy deleted file mode 100644 index ef2600b497..0000000000 --- a/tests/integration/update_cluster/nosshkey/data/aws_iam_role_policy_nodes.nosshkey.example.com_policy +++ /dev/null @@ -1,68 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "ec2:DescribeInstances", - "ec2:DescribeRegions" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ChangeResourceRecordSets", - "route53:ListResourceRecordSets", - "route53:GetHostedZone" - ], - "Resource": [ - "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:GetChange" - ], - "Resource": [ - "arn:aws:route53:::change/*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ListHostedZones" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ListHostedZones" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "ecr:GetAuthorizationToken", - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:GetRepositoryPolicy", - "ecr:DescribeRepositories", - "ecr:ListImages", - "ecr:BatchGetImage" - ], - "Resource": [ - "*" - ] - } - ] -} diff --git a/tests/integration/update_cluster/nosshkey/data/aws_launch_template_master-us-test-1a.masters.nosshkey.example.com_user_data b/tests/integration/update_cluster/nosshkey/data/aws_launch_template_master-us-test-1a.masters.nosshkey.example.com_user_data deleted file mode 100644 index 0198b63fcb..0000000000 --- a/tests/integration/update_cluster/nosshkey/data/aws_launch_template_master-us-test-1a.masters.nosshkey.example.com_user_data +++ /dev/null @@ -1,318 +0,0 @@ -#!/bin/bash -# Copyright 2016 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup -NODEUP_HASH_AMD64=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 -NODEUP_URL_ARM64= -NODEUP_HASH_ARM64= - -export AWS_REGION=us-test-1 - - - - -function ensure-install-dir() { - INSTALL_DIR="/opt/kops" - # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec - if [[ -d /var/lib/toolbox ]]; then - INSTALL_DIR="/var/lib/toolbox/kops" - fi - mkdir -p ${INSTALL_DIR}/bin - mkdir -p ${INSTALL_DIR}/conf - cd ${INSTALL_DIR} -} - -# Retry a download until we get it. args: name, sha, url1, url2... -download-or-bust() { - local -r file="$1" - local -r hash="$2" - shift 2 - - urls=( $* ) - while true; do - for url in "${urls[@]}"; do - commands=( - "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - ) - for cmd in "${commands[@]}"; do - echo "Attempting download with: ${cmd} {url}" - if ! (${cmd} "${url}"); then - echo "== Download failed with ${cmd} ==" - continue - fi - if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then - echo "== Hash validation of ${url} failed. Retrying. ==" - rm -f "${file}" - else - if [[ -n "${hash}" ]]; then - echo "== Downloaded ${url} (SHA1 = ${hash}) ==" - else - echo "== Downloaded ${url} ==" - fi - return - fi - done - done - - echo "All downloads failed; sleeping before retrying" - sleep 60 - done -} - -validate-hash() { - local -r file="$1" - local -r expected="$2" - local actual - - actual=$(sha256sum ${file} | awk '{ print $1 }') || true - if [[ "${actual}" != "${expected}" ]]; then - echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" - return 1 - fi -} - -function split-commas() { - echo $1 | tr "," "\n" -} - -function try-download-release() { - local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) - if [[ -n "${NODEUP_HASH:-}" ]]; then - local -r nodeup_hash="${NODEUP_HASH}" - else - # TODO: Remove? - echo "Downloading sha256 (not found in env)" - download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" - local -r nodeup_hash=$(cat nodeup.sha256) - fi - - echo "Downloading nodeup (${nodeup_urls[@]})" - download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" - - chmod +x nodeup -} - -function download-release() { - case "$(uname -m)" in - x86_64*|i?86_64*|amd64*) - NODEUP_URL="${NODEUP_URL_AMD64}" - NODEUP_HASH="${NODEUP_HASH_AMD64}" - ;; - aarch64*|arm64*) - NODEUP_URL="${NODEUP_URL_ARM64}" - NODEUP_HASH="${NODEUP_HASH_ARM64}" - ;; - *) - echo "Unsupported host arch: $(uname -m)" >&2 - exit 1 - ;; - esac - - # In case of failure checking integrity of release, retry. - cd ${INSTALL_DIR}/bin - until try-download-release; do - sleep 15 - echo "Couldn't download release. Retrying..." - done - - echo "Running nodeup" - # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 - ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) -} - -#################################################################################### - -/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" - -echo "== nodeup node config starting ==" -ensure-install-dir - -cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' -cloudConfig: null -containerRuntime: docker -containerd: - skipInstall: true -docker: - ipMasq: false - ipTables: false - logDriver: json-file - logLevel: info - logOpt: - - max-size=10m - - max-file=5 - storage: overlay2,overlay,aufs - version: 17.03.2 -encryptionConfig: null -etcdClusters: - events: - image: k8s.gcr.io/etcd:2.2.1 - version: 2.2.1 - main: - image: k8s.gcr.io/etcd:2.2.1 - version: 2.2.1 -kubeAPIServer: - allowPrivileged: true - anonymousAuth: false - apiServerCount: 1 - authorizationMode: AlwaysAllow - bindAddress: 0.0.0.0 - cloudProvider: aws - enableAdmissionPlugins: - - Initializers - - NamespaceLifecycle - - LimitRanger - - ServiceAccount - - PersistentVolumeLabel - - DefaultStorageClass - - DefaultTolerationSeconds - - MutatingAdmissionWebhook - - ValidatingAdmissionWebhook - - NodeRestriction - - ResourceQuota - etcdQuorumRead: false - etcdServers: - - http://127.0.0.1:4001 - etcdServersOverrides: - - /events#http://127.0.0.1:4002 - image: k8s.gcr.io/kube-apiserver:v1.11.10 - insecureBindAddress: 127.0.0.1 - insecurePort: 8080 - kubeletPreferredAddressTypes: - - InternalIP - - Hostname - - ExternalIP - logLevel: 2 - requestheaderAllowedNames: - - aggregator - requestheaderExtraHeaderPrefixes: - - X-Remote-Extra- - requestheaderGroupHeaders: - - X-Remote-Group - requestheaderUsernameHeaders: - - X-Remote-User - securePort: 443 - serviceClusterIPRange: 100.64.0.0/13 - serviceNodePortRange: 28000-32767 - storageBackend: etcd2 -kubeControllerManager: - allocateNodeCIDRs: true - attachDetachReconcileSyncPeriod: 1m0s - cloudProvider: aws - clusterCIDR: 100.96.0.0/11 - clusterName: nosshkey.example.com - configureCloudRoutes: true - image: k8s.gcr.io/kube-controller-manager:v1.11.10 - leaderElection: - leaderElect: true - logLevel: 2 - useServiceAccountCredentials: true -kubeProxy: - clusterCIDR: 100.96.0.0/11 - cpuRequest: 100m - hostnameOverride: '@aws' - image: k8s.gcr.io/kube-proxy:v1.11.10 - logLevel: 2 -kubeScheduler: - image: k8s.gcr.io/kube-scheduler:v1.11.10 - leaderElection: - leaderElect: true - logLevel: 2 -kubelet: - allowPrivileged: true - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests -masterKubelet: - allowPrivileged: true - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests - registerSchedulable: false - -__EOF_CLUSTER_SPEC - -cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' -kubelet: null -nodeLabels: null -taints: null - -__EOF_IG_SPEC - -cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' -Assets: - amd64: - - a1e5d2a7da4cabc29af0dda630564511a9b437d8@https://storage.googleapis.com/kubernetes-release/release/v1.11.10/bin/linux/amd64/kubelet - - c133f55152c76c33d9b41894dcd311064904503e@https://storage.googleapis.com/kubernetes-release/release/v1.11.10/bin/linux/amd64/kubectl - - 3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz - arm64: - - e895202b1ff6beaee15232f0c3118f2dc0cd4862@https://storage.googleapis.com/kubernetes-release/release/v1.11.10/bin/linux/arm64/kubelet - - b7b2bda2aff7a430a04c0e2b7a0ffe853ffb5288@https://storage.googleapis.com/kubernetes-release/release/v1.11.10/bin/linux/arm64/kubectl - - 7fec91af78e9548df306f0ec43bea527c8c10cc3a9682c33e971c8522a7fcded@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-arm64-v0.7.5.tgz -ClusterName: nosshkey.example.com -ConfigBase: memfs://clusters.example.com/nosshkey.example.com -InstanceGroupName: master-us-test-1a -InstanceGroupRole: Master -Tags: -- _automatic_upgrades -- _aws -channels: -- memfs://clusters.example.com/nosshkey.example.com/addons/bootstrap-channel.yaml -protokubeImage: - hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b - name: protokube:1.15.0 - sources: - - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz - - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz - - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz - -__EOF_KUBE_ENV - -download-release -echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/nosshkey/data/aws_launch_template_nodes.nosshkey.example.com_user_data b/tests/integration/update_cluster/nosshkey/data/aws_launch_template_nodes.nosshkey.example.com_user_data deleted file mode 100644 index 99ab9b9553..0000000000 --- a/tests/integration/update_cluster/nosshkey/data/aws_launch_template_nodes.nosshkey.example.com_user_data +++ /dev/null @@ -1,222 +0,0 @@ -#!/bin/bash -# Copyright 2016 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup -NODEUP_HASH_AMD64=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 -NODEUP_URL_ARM64= -NODEUP_HASH_ARM64= - -export AWS_REGION=us-test-1 - - - - -function ensure-install-dir() { - INSTALL_DIR="/opt/kops" - # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec - if [[ -d /var/lib/toolbox ]]; then - INSTALL_DIR="/var/lib/toolbox/kops" - fi - mkdir -p ${INSTALL_DIR}/bin - mkdir -p ${INSTALL_DIR}/conf - cd ${INSTALL_DIR} -} - -# Retry a download until we get it. args: name, sha, url1, url2... -download-or-bust() { - local -r file="$1" - local -r hash="$2" - shift 2 - - urls=( $* ) - while true; do - for url in "${urls[@]}"; do - commands=( - "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - ) - for cmd in "${commands[@]}"; do - echo "Attempting download with: ${cmd} {url}" - if ! (${cmd} "${url}"); then - echo "== Download failed with ${cmd} ==" - continue - fi - if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then - echo "== Hash validation of ${url} failed. Retrying. ==" - rm -f "${file}" - else - if [[ -n "${hash}" ]]; then - echo "== Downloaded ${url} (SHA1 = ${hash}) ==" - else - echo "== Downloaded ${url} ==" - fi - return - fi - done - done - - echo "All downloads failed; sleeping before retrying" - sleep 60 - done -} - -validate-hash() { - local -r file="$1" - local -r expected="$2" - local actual - - actual=$(sha256sum ${file} | awk '{ print $1 }') || true - if [[ "${actual}" != "${expected}" ]]; then - echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" - return 1 - fi -} - -function split-commas() { - echo $1 | tr "," "\n" -} - -function try-download-release() { - local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) - if [[ -n "${NODEUP_HASH:-}" ]]; then - local -r nodeup_hash="${NODEUP_HASH}" - else - # TODO: Remove? - echo "Downloading sha256 (not found in env)" - download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" - local -r nodeup_hash=$(cat nodeup.sha256) - fi - - echo "Downloading nodeup (${nodeup_urls[@]})" - download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" - - chmod +x nodeup -} - -function download-release() { - case "$(uname -m)" in - x86_64*|i?86_64*|amd64*) - NODEUP_URL="${NODEUP_URL_AMD64}" - NODEUP_HASH="${NODEUP_HASH_AMD64}" - ;; - aarch64*|arm64*) - NODEUP_URL="${NODEUP_URL_ARM64}" - NODEUP_HASH="${NODEUP_HASH_ARM64}" - ;; - *) - echo "Unsupported host arch: $(uname -m)" >&2 - exit 1 - ;; - esac - - # In case of failure checking integrity of release, retry. - cd ${INSTALL_DIR}/bin - until try-download-release; do - sleep 15 - echo "Couldn't download release. Retrying..." - done - - echo "Running nodeup" - # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 - ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) -} - -#################################################################################### - -/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" - -echo "== nodeup node config starting ==" -ensure-install-dir - -cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' -cloudConfig: null -containerRuntime: docker -containerd: - skipInstall: true -docker: - ipMasq: false - ipTables: false - logDriver: json-file - logLevel: info - logOpt: - - max-size=10m - - max-file=5 - storage: overlay2,overlay,aufs - version: 17.03.2 -kubeProxy: - clusterCIDR: 100.96.0.0/11 - cpuRequest: 100m - hostnameOverride: '@aws' - image: k8s.gcr.io/kube-proxy:v1.11.10 - logLevel: 2 -kubelet: - allowPrivileged: true - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests - -__EOF_CLUSTER_SPEC - -cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' -kubelet: null -nodeLabels: null -taints: null - -__EOF_IG_SPEC - -cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' -Assets: - amd64: - - a1e5d2a7da4cabc29af0dda630564511a9b437d8@https://storage.googleapis.com/kubernetes-release/release/v1.11.10/bin/linux/amd64/kubelet - - c133f55152c76c33d9b41894dcd311064904503e@https://storage.googleapis.com/kubernetes-release/release/v1.11.10/bin/linux/amd64/kubectl - - 3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz - arm64: - - e895202b1ff6beaee15232f0c3118f2dc0cd4862@https://storage.googleapis.com/kubernetes-release/release/v1.11.10/bin/linux/arm64/kubelet - - b7b2bda2aff7a430a04c0e2b7a0ffe853ffb5288@https://storage.googleapis.com/kubernetes-release/release/v1.11.10/bin/linux/arm64/kubectl - - 7fec91af78e9548df306f0ec43bea527c8c10cc3a9682c33e971c8522a7fcded@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-arm64-v0.7.5.tgz -ClusterName: nosshkey.example.com -ConfigBase: memfs://clusters.example.com/nosshkey.example.com -InstanceGroupName: nodes -InstanceGroupRole: Node -Tags: -- _automatic_upgrades -- _aws -channels: -- memfs://clusters.example.com/nosshkey.example.com/addons/bootstrap-channel.yaml - -__EOF_KUBE_ENV - -download-release -echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/nosshkey/in-v1alpha2.yaml b/tests/integration/update_cluster/nosshkey/in-v1alpha2.yaml deleted file mode 100644 index 24bdecabc5..0000000000 --- a/tests/integration/update_cluster/nosshkey/in-v1alpha2.yaml +++ /dev/null @@ -1,98 +0,0 @@ -apiVersion: kops.k8s.io/v1alpha2 -kind: Cluster -metadata: - creationTimestamp: "2016-12-10T22:42:27Z" - name: nosshkey.example.com -spec: - api: - loadBalancer: - type: Public - additionalSecurityGroups: - - sg-exampleid3 - - sg-exampleid4 - kubernetesApiAccess: - - 0.0.0.0/0 - channel: stable - cloudProvider: aws - cloudLabels: - Owner: John Doe - foo/bar: fib+baz - configBase: memfs://clusters.example.com/nosshkey.example.com - etcdClusters: - - etcdMembers: - - instanceGroup: master-us-test-1a - name: us-test-1a - name: main - - etcdMembers: - - instanceGroup: master-us-test-1a - name: us-test-1a - name: events - kubelet: - anonymousAuth: false - kubeAPIServer: - serviceNodePortRange: 28000-32767 - kubernetesVersion: v1.11.10 - masterInternalName: api.internal.nosshkey.example.com - masterPublicName: api.nosshkey.example.com - networkCIDR: 172.20.0.0/16 - networking: - kubenet: {} - nodePortAccess: - - 1.2.3.4/32 - - 10.20.30.0/24 - nonMasqueradeCIDR: 100.64.0.0/10 - sshAccess: - - 0.0.0.0/0 - sshKeyName: "" - topology: - masters: public - nodes: public - subnets: - - cidr: 172.20.32.0/19 - name: us-test-1a - type: Public - zone: us-test-1a - ---- - -apiVersion: kops.k8s.io/v1alpha2 -kind: InstanceGroup -metadata: - creationTimestamp: "2016-12-10T22:42:28Z" - name: nodes - labels: - kops.k8s.io/cluster: nosshkey.example.com -spec: - additionalSecurityGroups: - - sg-exampleid3 - - sg-exampleid4 - associatePublicIp: true - suspendProcesses: - - AZRebalance - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: t2.medium - maxSize: 2 - minSize: 2 - role: Node - subnets: - - us-test-1a - detailedInstanceMonitoring: true - ---- - -apiVersion: kops.k8s.io/v1alpha2 -kind: InstanceGroup -metadata: - creationTimestamp: "2016-12-10T22:42:28Z" - name: master-us-test-1a - labels: - kops.k8s.io/cluster: nosshkey.example.com -spec: - associatePublicIp: true - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: m3.medium - maxSize: 1 - minSize: 1 - role: Master - subnets: - - us-test-1a diff --git a/tests/integration/update_cluster/nosshkey/kubernetes.tf b/tests/integration/update_cluster/nosshkey/kubernetes.tf deleted file mode 100644 index 1796456d99..0000000000 --- a/tests/integration/update_cluster/nosshkey/kubernetes.tf +++ /dev/null @@ -1,684 +0,0 @@ -locals { - cluster_name = "nosshkey.example.com" - master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-nosshkey-example-com.id] - master_security_group_ids = [aws_security_group.masters-nosshkey-example-com.id] - masters_role_arn = aws_iam_role.masters-nosshkey-example-com.arn - masters_role_name = aws_iam_role.masters-nosshkey-example-com.name - node_autoscaling_group_ids = [aws_autoscaling_group.nodes-nosshkey-example-com.id] - node_security_group_ids = [aws_security_group.nodes-nosshkey-example-com.id, "sg-exampleid3", "sg-exampleid4"] - node_subnet_ids = [aws_subnet.us-test-1a-nosshkey-example-com.id] - nodes_role_arn = aws_iam_role.nodes-nosshkey-example-com.arn - nodes_role_name = aws_iam_role.nodes-nosshkey-example-com.name - region = "us-test-1" - route_table_public_id = aws_route_table.nosshkey-example-com.id - subnet_us-test-1a_id = aws_subnet.us-test-1a-nosshkey-example-com.id - vpc_cidr_block = aws_vpc.nosshkey-example-com.cidr_block - vpc_id = aws_vpc.nosshkey-example-com.id -} - -output "cluster_name" { - value = "nosshkey.example.com" -} - -output "master_autoscaling_group_ids" { - value = [aws_autoscaling_group.master-us-test-1a-masters-nosshkey-example-com.id] -} - -output "master_security_group_ids" { - value = [aws_security_group.masters-nosshkey-example-com.id] -} - -output "masters_role_arn" { - value = aws_iam_role.masters-nosshkey-example-com.arn -} - -output "masters_role_name" { - value = aws_iam_role.masters-nosshkey-example-com.name -} - -output "node_autoscaling_group_ids" { - value = [aws_autoscaling_group.nodes-nosshkey-example-com.id] -} - -output "node_security_group_ids" { - value = [aws_security_group.nodes-nosshkey-example-com.id, "sg-exampleid3", "sg-exampleid4"] -} - -output "node_subnet_ids" { - value = [aws_subnet.us-test-1a-nosshkey-example-com.id] -} - -output "nodes_role_arn" { - value = aws_iam_role.nodes-nosshkey-example-com.arn -} - -output "nodes_role_name" { - value = aws_iam_role.nodes-nosshkey-example-com.name -} - -output "region" { - value = "us-test-1" -} - -output "route_table_public_id" { - value = aws_route_table.nosshkey-example-com.id -} - -output "subnet_us-test-1a_id" { - value = aws_subnet.us-test-1a-nosshkey-example-com.id -} - -output "vpc_cidr_block" { - value = aws_vpc.nosshkey-example-com.cidr_block -} - -output "vpc_id" { - value = aws_vpc.nosshkey-example-com.id -} - -provider "aws" { - region = "us-test-1" -} - -resource "aws_autoscaling_attachment" "master-us-test-1a-masters-nosshkey-example-com" { - autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-nosshkey-example-com.id - elb = aws_elb.api-nosshkey-example-com.id -} - -resource "aws_autoscaling_group" "master-us-test-1a-masters-nosshkey-example-com" { - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] - launch_template { - id = aws_launch_template.master-us-test-1a-masters-nosshkey-example-com.id - version = aws_launch_template.master-us-test-1a-masters-nosshkey-example-com.latest_version - } - max_size = 1 - metrics_granularity = "1Minute" - min_size = 1 - name = "master-us-test-1a.masters.nosshkey.example.com" - tag { - key = "KubernetesCluster" - propagate_at_launch = true - value = "nosshkey.example.com" - } - tag { - key = "Name" - propagate_at_launch = true - value = "master-us-test-1a.masters.nosshkey.example.com" - } - tag { - key = "Owner" - propagate_at_launch = true - value = "John Doe" - } - tag { - key = "foo/bar" - propagate_at_launch = true - value = "fib+baz" - } - tag { - key = "k8s.io/role/master" - propagate_at_launch = true - value = "1" - } - tag { - key = "kops.k8s.io/instancegroup" - propagate_at_launch = true - value = "master-us-test-1a" - } - tag { - key = "kubernetes.io/cluster/nosshkey.example.com" - propagate_at_launch = true - value = "owned" - } - vpc_zone_identifier = [aws_subnet.us-test-1a-nosshkey-example-com.id] -} - -resource "aws_autoscaling_group" "nodes-nosshkey-example-com" { - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] - launch_template { - id = aws_launch_template.nodes-nosshkey-example-com.id - version = aws_launch_template.nodes-nosshkey-example-com.latest_version - } - max_size = 2 - metrics_granularity = "1Minute" - min_size = 2 - name = "nodes.nosshkey.example.com" - suspended_processes = ["AZRebalance"] - tag { - key = "KubernetesCluster" - propagate_at_launch = true - value = "nosshkey.example.com" - } - tag { - key = "Name" - propagate_at_launch = true - value = "nodes.nosshkey.example.com" - } - tag { - key = "Owner" - propagate_at_launch = true - value = "John Doe" - } - tag { - key = "foo/bar" - propagate_at_launch = true - value = "fib+baz" - } - tag { - key = "k8s.io/role/node" - propagate_at_launch = true - value = "1" - } - tag { - key = "kops.k8s.io/instancegroup" - propagate_at_launch = true - value = "nodes" - } - tag { - key = "kubernetes.io/cluster/nosshkey.example.com" - propagate_at_launch = true - value = "owned" - } - vpc_zone_identifier = [aws_subnet.us-test-1a-nosshkey-example-com.id] -} - -resource "aws_ebs_volume" "us-test-1a-etcd-events-nosshkey-example-com" { - availability_zone = "us-test-1a" - encrypted = false - size = 20 - tags = { - "KubernetesCluster" = "nosshkey.example.com" - "Name" = "us-test-1a.etcd-events.nosshkey.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "k8s.io/etcd/events" = "us-test-1a/us-test-1a" - "k8s.io/role/master" = "1" - "kubernetes.io/cluster/nosshkey.example.com" = "owned" - } - type = "gp2" -} - -resource "aws_ebs_volume" "us-test-1a-etcd-main-nosshkey-example-com" { - availability_zone = "us-test-1a" - encrypted = false - size = 20 - tags = { - "KubernetesCluster" = "nosshkey.example.com" - "Name" = "us-test-1a.etcd-main.nosshkey.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "k8s.io/etcd/main" = "us-test-1a/us-test-1a" - "k8s.io/role/master" = "1" - "kubernetes.io/cluster/nosshkey.example.com" = "owned" - } - type = "gp2" -} - -resource "aws_elb" "api-nosshkey-example-com" { - cross_zone_load_balancing = false - health_check { - healthy_threshold = 2 - interval = 10 - target = "SSL:443" - timeout = 5 - unhealthy_threshold = 2 - } - idle_timeout = 300 - listener { - instance_port = 443 - instance_protocol = "TCP" - lb_port = 443 - lb_protocol = "TCP" - ssl_certificate_id = "" - } - name = "api-nosshkey-example-com-bdulnp" - security_groups = [aws_security_group.api-elb-nosshkey-example-com.id, "sg-exampleid3", "sg-exampleid4"] - subnets = [aws_subnet.us-test-1a-nosshkey-example-com.id] - tags = { - "KubernetesCluster" = "nosshkey.example.com" - "Name" = "api.nosshkey.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "kubernetes.io/cluster/nosshkey.example.com" = "owned" - } -} - -resource "aws_iam_instance_profile" "masters-nosshkey-example-com" { - name = "masters.nosshkey.example.com" - role = aws_iam_role.masters-nosshkey-example-com.name -} - -resource "aws_iam_instance_profile" "nodes-nosshkey-example-com" { - name = "nodes.nosshkey.example.com" - role = aws_iam_role.nodes-nosshkey-example-com.name -} - -resource "aws_iam_role_policy" "masters-nosshkey-example-com" { - name = "masters.nosshkey.example.com" - policy = file("${path.module}/data/aws_iam_role_policy_masters.nosshkey.example.com_policy") - role = aws_iam_role.masters-nosshkey-example-com.name -} - -resource "aws_iam_role_policy" "nodes-nosshkey-example-com" { - name = "nodes.nosshkey.example.com" - policy = file("${path.module}/data/aws_iam_role_policy_nodes.nosshkey.example.com_policy") - role = aws_iam_role.nodes-nosshkey-example-com.name -} - -resource "aws_iam_role" "masters-nosshkey-example-com" { - assume_role_policy = file("${path.module}/data/aws_iam_role_masters.nosshkey.example.com_policy") - name = "masters.nosshkey.example.com" -} - -resource "aws_iam_role" "nodes-nosshkey-example-com" { - assume_role_policy = file("${path.module}/data/aws_iam_role_nodes.nosshkey.example.com_policy") - name = "nodes.nosshkey.example.com" -} - -resource "aws_internet_gateway" "nosshkey-example-com" { - tags = { - "KubernetesCluster" = "nosshkey.example.com" - "Name" = "nosshkey.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "kubernetes.io/cluster/nosshkey.example.com" = "owned" - } - vpc_id = aws_vpc.nosshkey-example-com.id -} - -resource "aws_launch_template" "master-us-test-1a-masters-nosshkey-example-com" { - block_device_mappings { - device_name = "/dev/xvda" - ebs { - delete_on_termination = true - volume_size = 64 - volume_type = "gp2" - } - } - block_device_mappings { - device_name = "/dev/sdc" - virtual_name = "ephemeral0" - } - iam_instance_profile { - name = aws_iam_instance_profile.masters-nosshkey-example-com.id - } - image_id = "ami-12345678" - instance_type = "m3.medium" - lifecycle { - create_before_destroy = true - } - name_prefix = "master-us-test-1a.masters.nosshkey.example.com-" - network_interfaces { - associate_public_ip_address = true - delete_on_termination = true - security_groups = [aws_security_group.masters-nosshkey-example-com.id] - } - tag_specifications { - resource_type = "instance" - tags = { - "KubernetesCluster" = "nosshkey.example.com" - "Name" = "master-us-test-1a.masters.nosshkey.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "k8s.io/role/master" = "1" - "kops.k8s.io/instancegroup" = "master-us-test-1a" - "kubernetes.io/cluster/nosshkey.example.com" = "owned" - } - } - tag_specifications { - resource_type = "volume" - tags = { - "KubernetesCluster" = "nosshkey.example.com" - "Name" = "master-us-test-1a.masters.nosshkey.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "k8s.io/role/master" = "1" - "kops.k8s.io/instancegroup" = "master-us-test-1a" - "kubernetes.io/cluster/nosshkey.example.com" = "owned" - } - } - user_data = file("${path.module}/data/aws_launch_template_master-us-test-1a.masters.nosshkey.example.com_user_data") -} - -resource "aws_launch_template" "nodes-nosshkey-example-com" { - block_device_mappings { - device_name = "/dev/xvda" - ebs { - delete_on_termination = true - volume_size = 128 - volume_type = "gp2" - } - } - iam_instance_profile { - name = aws_iam_instance_profile.nodes-nosshkey-example-com.id - } - image_id = "ami-12345678" - instance_type = "t2.medium" - lifecycle { - create_before_destroy = true - } - name_prefix = "nodes.nosshkey.example.com-" - network_interfaces { - associate_public_ip_address = true - delete_on_termination = true - security_groups = [aws_security_group.nodes-nosshkey-example-com.id, "sg-exampleid3", "sg-exampleid4"] - } - tag_specifications { - resource_type = "instance" - tags = { - "KubernetesCluster" = "nosshkey.example.com" - "Name" = "nodes.nosshkey.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "nodes" - "kubernetes.io/cluster/nosshkey.example.com" = "owned" - } - } - tag_specifications { - resource_type = "volume" - tags = { - "KubernetesCluster" = "nosshkey.example.com" - "Name" = "nodes.nosshkey.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "nodes" - "kubernetes.io/cluster/nosshkey.example.com" = "owned" - } - } - user_data = file("${path.module}/data/aws_launch_template_nodes.nosshkey.example.com_user_data") -} - -resource "aws_route53_record" "api-nosshkey-example-com" { - alias { - evaluate_target_health = false - name = aws_elb.api-nosshkey-example-com.dns_name - zone_id = aws_elb.api-nosshkey-example-com.zone_id - } - name = "api.nosshkey.example.com" - type = "A" - zone_id = "/hostedzone/Z1AFAKE1ZON3YO" -} - -resource "aws_route_table_association" "us-test-1a-nosshkey-example-com" { - route_table_id = aws_route_table.nosshkey-example-com.id - subnet_id = aws_subnet.us-test-1a-nosshkey-example-com.id -} - -resource "aws_route_table" "nosshkey-example-com" { - tags = { - "KubernetesCluster" = "nosshkey.example.com" - "Name" = "nosshkey.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "kubernetes.io/cluster/nosshkey.example.com" = "owned" - "kubernetes.io/kops/role" = "public" - } - vpc_id = aws_vpc.nosshkey-example-com.id -} - -resource "aws_route" "route-0-0-0-0--0" { - destination_cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.nosshkey-example-com.id - route_table_id = aws_route_table.nosshkey-example-com.id -} - -resource "aws_security_group_rule" "all-master-to-master" { - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.masters-nosshkey-example-com.id - source_security_group_id = aws_security_group.masters-nosshkey-example-com.id - to_port = 0 - type = "ingress" -} - -resource "aws_security_group_rule" "all-master-to-node" { - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.nodes-nosshkey-example-com.id - source_security_group_id = aws_security_group.masters-nosshkey-example-com.id - to_port = 0 - type = "ingress" -} - -resource "aws_security_group_rule" "all-node-to-node" { - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.nodes-nosshkey-example-com.id - source_security_group_id = aws_security_group.nodes-nosshkey-example-com.id - to_port = 0 - type = "ingress" -} - -resource "aws_security_group_rule" "api-elb-egress" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.api-elb-nosshkey-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "https-api-elb-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 443 - protocol = "tcp" - security_group_id = aws_security_group.api-elb-nosshkey-example-com.id - to_port = 443 - type = "ingress" -} - -resource "aws_security_group_rule" "https-elb-to-master" { - from_port = 443 - protocol = "tcp" - security_group_id = aws_security_group.masters-nosshkey-example-com.id - source_security_group_id = aws_security_group.api-elb-nosshkey-example-com.id - to_port = 443 - type = "ingress" -} - -resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 3 - protocol = "icmp" - security_group_id = aws_security_group.api-elb-nosshkey-example-com.id - to_port = 4 - type = "ingress" -} - -resource "aws_security_group_rule" "master-egress" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.masters-nosshkey-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "node-egress" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.nodes-nosshkey-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "node-to-master-tcp-1-2379" { - from_port = 1 - protocol = "tcp" - security_group_id = aws_security_group.masters-nosshkey-example-com.id - source_security_group_id = aws_security_group.nodes-nosshkey-example-com.id - to_port = 2379 - type = "ingress" -} - -resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" { - from_port = 2382 - protocol = "tcp" - security_group_id = aws_security_group.masters-nosshkey-example-com.id - source_security_group_id = aws_security_group.nodes-nosshkey-example-com.id - to_port = 4000 - type = "ingress" -} - -resource "aws_security_group_rule" "node-to-master-tcp-4003-65535" { - from_port = 4003 - protocol = "tcp" - security_group_id = aws_security_group.masters-nosshkey-example-com.id - source_security_group_id = aws_security_group.nodes-nosshkey-example-com.id - to_port = 65535 - type = "ingress" -} - -resource "aws_security_group_rule" "node-to-master-udp-1-65535" { - from_port = 1 - protocol = "udp" - security_group_id = aws_security_group.masters-nosshkey-example-com.id - source_security_group_id = aws_security_group.nodes-nosshkey-example-com.id - to_port = 65535 - type = "ingress" -} - -resource "aws_security_group_rule" "nodeport-tcp-external-to-node-1-2-3-4--32" { - cidr_blocks = ["1.2.3.4/32"] - from_port = 28000 - protocol = "tcp" - security_group_id = aws_security_group.nodes-nosshkey-example-com.id - to_port = 32767 - type = "ingress" -} - -resource "aws_security_group_rule" "nodeport-tcp-external-to-node-10-20-30-0--24" { - cidr_blocks = ["10.20.30.0/24"] - from_port = 28000 - protocol = "tcp" - security_group_id = aws_security_group.nodes-nosshkey-example-com.id - to_port = 32767 - type = "ingress" -} - -resource "aws_security_group_rule" "nodeport-udp-external-to-node-1-2-3-4--32" { - cidr_blocks = ["1.2.3.4/32"] - from_port = 28000 - protocol = "udp" - security_group_id = aws_security_group.nodes-nosshkey-example-com.id - to_port = 32767 - type = "ingress" -} - -resource "aws_security_group_rule" "nodeport-udp-external-to-node-10-20-30-0--24" { - cidr_blocks = ["10.20.30.0/24"] - from_port = 28000 - protocol = "udp" - security_group_id = aws_security_group.nodes-nosshkey-example-com.id - to_port = 32767 - type = "ingress" -} - -resource "aws_security_group_rule" "ssh-external-to-master-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.masters-nosshkey-example-com.id - to_port = 22 - type = "ingress" -} - -resource "aws_security_group_rule" "ssh-external-to-node-0-0-0-0--0" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.nodes-nosshkey-example-com.id - to_port = 22 - type = "ingress" -} - -resource "aws_security_group" "api-elb-nosshkey-example-com" { - description = "Security group for api ELB" - name = "api-elb.nosshkey.example.com" - tags = { - "KubernetesCluster" = "nosshkey.example.com" - "Name" = "api-elb.nosshkey.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "kubernetes.io/cluster/nosshkey.example.com" = "owned" - } - vpc_id = aws_vpc.nosshkey-example-com.id -} - -resource "aws_security_group" "masters-nosshkey-example-com" { - description = "Security group for masters" - name = "masters.nosshkey.example.com" - tags = { - "KubernetesCluster" = "nosshkey.example.com" - "Name" = "masters.nosshkey.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "kubernetes.io/cluster/nosshkey.example.com" = "owned" - } - vpc_id = aws_vpc.nosshkey-example-com.id -} - -resource "aws_security_group" "nodes-nosshkey-example-com" { - description = "Security group for nodes" - name = "nodes.nosshkey.example.com" - tags = { - "KubernetesCluster" = "nosshkey.example.com" - "Name" = "nodes.nosshkey.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "kubernetes.io/cluster/nosshkey.example.com" = "owned" - } - vpc_id = aws_vpc.nosshkey-example-com.id -} - -resource "aws_subnet" "us-test-1a-nosshkey-example-com" { - availability_zone = "us-test-1a" - cidr_block = "172.20.32.0/19" - tags = { - "KubernetesCluster" = "nosshkey.example.com" - "Name" = "us-test-1a.nosshkey.example.com" - "Owner" = "John Doe" - "SubnetType" = "Public" - "foo/bar" = "fib+baz" - "kubernetes.io/cluster/nosshkey.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - } - vpc_id = aws_vpc.nosshkey-example-com.id -} - -resource "aws_vpc_dhcp_options_association" "nosshkey-example-com" { - dhcp_options_id = aws_vpc_dhcp_options.nosshkey-example-com.id - vpc_id = aws_vpc.nosshkey-example-com.id -} - -resource "aws_vpc_dhcp_options" "nosshkey-example-com" { - domain_name = "us-test-1.compute.internal" - domain_name_servers = ["AmazonProvidedDNS"] - tags = { - "KubernetesCluster" = "nosshkey.example.com" - "Name" = "nosshkey.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "kubernetes.io/cluster/nosshkey.example.com" = "owned" - } -} - -resource "aws_vpc" "nosshkey-example-com" { - cidr_block = "172.20.0.0/16" - enable_dns_hostnames = true - enable_dns_support = true - tags = { - "KubernetesCluster" = "nosshkey.example.com" - "Name" = "nosshkey.example.com" - "Owner" = "John Doe" - "foo/bar" = "fib+baz" - "kubernetes.io/cluster/nosshkey.example.com" = "owned" - } -} - -terraform { - required_version = ">= 0.12.0" -} diff --git a/tests/integration/update_cluster/restrict_access/data/aws_iam_role_masters.restrictaccess.example.com_policy b/tests/integration/update_cluster/restrict_access/data/aws_iam_role_masters.restrictaccess.example.com_policy deleted file mode 100644 index 66d5de1d5a..0000000000 --- a/tests/integration/update_cluster/restrict_access/data/aws_iam_role_masters.restrictaccess.example.com_policy +++ /dev/null @@ -1,10 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { "Service": "ec2.amazonaws.com"}, - "Action": "sts:AssumeRole" - } - ] -} diff --git a/tests/integration/update_cluster/restrict_access/data/aws_iam_role_nodes.restrictaccess.example.com_policy b/tests/integration/update_cluster/restrict_access/data/aws_iam_role_nodes.restrictaccess.example.com_policy deleted file mode 100644 index 66d5de1d5a..0000000000 --- a/tests/integration/update_cluster/restrict_access/data/aws_iam_role_nodes.restrictaccess.example.com_policy +++ /dev/null @@ -1,10 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { "Service": "ec2.amazonaws.com"}, - "Action": "sts:AssumeRole" - } - ] -} diff --git a/tests/integration/update_cluster/restrict_access/data/aws_iam_role_policy_masters.restrictaccess.example.com_policy b/tests/integration/update_cluster/restrict_access/data/aws_iam_role_policy_masters.restrictaccess.example.com_policy deleted file mode 100644 index 340dff1ef9..0000000000 --- a/tests/integration/update_cluster/restrict_access/data/aws_iam_role_policy_masters.restrictaccess.example.com_policy +++ /dev/null @@ -1,102 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "ec2:*" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:DescribeAutoScalingInstances", - "autoscaling:DescribeLaunchConfigurations", - "autoscaling:DescribeTags", - "autoscaling:SetDesiredCapacity", - "autoscaling:TerminateInstanceInAutoScalingGroup", - "autoscaling:UpdateAutoScalingGroup", - "ec2:DescribeLaunchTemplateVersions" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "elasticloadbalancing:*" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "iam:ListServerCertificates", - "iam:GetServerCertificate" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ChangeResourceRecordSets", - "route53:ListResourceRecordSets", - "route53:GetHostedZone" - ], - "Resource": [ - "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:GetChange" - ], - "Resource": [ - "arn:aws:route53:::change/*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ListHostedZones" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ListHostedZones" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "ecr:GetAuthorizationToken", - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:GetRepositoryPolicy", - "ecr:DescribeRepositories", - "ecr:ListImages", - "ecr:BatchGetImage" - ], - "Resource": [ - "*" - ] - } - ] -} diff --git a/tests/integration/update_cluster/restrict_access/data/aws_iam_role_policy_nodes.restrictaccess.example.com_policy b/tests/integration/update_cluster/restrict_access/data/aws_iam_role_policy_nodes.restrictaccess.example.com_policy deleted file mode 100644 index ef2600b497..0000000000 --- a/tests/integration/update_cluster/restrict_access/data/aws_iam_role_policy_nodes.restrictaccess.example.com_policy +++ /dev/null @@ -1,68 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "ec2:DescribeInstances", - "ec2:DescribeRegions" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ChangeResourceRecordSets", - "route53:ListResourceRecordSets", - "route53:GetHostedZone" - ], - "Resource": [ - "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:GetChange" - ], - "Resource": [ - "arn:aws:route53:::change/*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ListHostedZones" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:ListHostedZones" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "ecr:GetAuthorizationToken", - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:GetRepositoryPolicy", - "ecr:DescribeRepositories", - "ecr:ListImages", - "ecr:BatchGetImage" - ], - "Resource": [ - "*" - ] - } - ] -} diff --git a/tests/integration/update_cluster/restrict_access/data/aws_key_pair_kubernetes.restrictaccess.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key b/tests/integration/update_cluster/restrict_access/data/aws_key_pair_kubernetes.restrictaccess.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key deleted file mode 100644 index 81cb012783..0000000000 --- a/tests/integration/update_cluster/restrict_access/data/aws_key_pair_kubernetes.restrictaccess.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/restrict_access/data/aws_launch_template_master-us-test-1a.masters.restrictaccess.example.com_user_data b/tests/integration/update_cluster/restrict_access/data/aws_launch_template_master-us-test-1a.masters.restrictaccess.example.com_user_data deleted file mode 100644 index d0a6c70140..0000000000 --- a/tests/integration/update_cluster/restrict_access/data/aws_launch_template_master-us-test-1a.masters.restrictaccess.example.com_user_data +++ /dev/null @@ -1,314 +0,0 @@ -#!/bin/bash -# Copyright 2016 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup -NODEUP_HASH_AMD64=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 -NODEUP_URL_ARM64= -NODEUP_HASH_ARM64= - -export AWS_REGION=us-test-1 - - - - -function ensure-install-dir() { - INSTALL_DIR="/opt/kops" - # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec - if [[ -d /var/lib/toolbox ]]; then - INSTALL_DIR="/var/lib/toolbox/kops" - fi - mkdir -p ${INSTALL_DIR}/bin - mkdir -p ${INSTALL_DIR}/conf - cd ${INSTALL_DIR} -} - -# Retry a download until we get it. args: name, sha, url1, url2... -download-or-bust() { - local -r file="$1" - local -r hash="$2" - shift 2 - - urls=( $* ) - while true; do - for url in "${urls[@]}"; do - commands=( - "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - ) - for cmd in "${commands[@]}"; do - echo "Attempting download with: ${cmd} {url}" - if ! (${cmd} "${url}"); then - echo "== Download failed with ${cmd} ==" - continue - fi - if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then - echo "== Hash validation of ${url} failed. Retrying. ==" - rm -f "${file}" - else - if [[ -n "${hash}" ]]; then - echo "== Downloaded ${url} (SHA1 = ${hash}) ==" - else - echo "== Downloaded ${url} ==" - fi - return - fi - done - done - - echo "All downloads failed; sleeping before retrying" - sleep 60 - done -} - -validate-hash() { - local -r file="$1" - local -r expected="$2" - local actual - - actual=$(sha256sum ${file} | awk '{ print $1 }') || true - if [[ "${actual}" != "${expected}" ]]; then - echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" - return 1 - fi -} - -function split-commas() { - echo $1 | tr "," "\n" -} - -function try-download-release() { - local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) - if [[ -n "${NODEUP_HASH:-}" ]]; then - local -r nodeup_hash="${NODEUP_HASH}" - else - # TODO: Remove? - echo "Downloading sha256 (not found in env)" - download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" - local -r nodeup_hash=$(cat nodeup.sha256) - fi - - echo "Downloading nodeup (${nodeup_urls[@]})" - download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" - - chmod +x nodeup -} - -function download-release() { - case "$(uname -m)" in - x86_64*|i?86_64*|amd64*) - NODEUP_URL="${NODEUP_URL_AMD64}" - NODEUP_HASH="${NODEUP_HASH_AMD64}" - ;; - aarch64*|arm64*) - NODEUP_URL="${NODEUP_URL_ARM64}" - NODEUP_HASH="${NODEUP_HASH_ARM64}" - ;; - *) - echo "Unsupported host arch: $(uname -m)" >&2 - exit 1 - ;; - esac - - # In case of failure checking integrity of release, retry. - cd ${INSTALL_DIR}/bin - until try-download-release; do - sleep 15 - echo "Couldn't download release. Retrying..." - done - - echo "Running nodeup" - # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 - ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) -} - -#################################################################################### - -/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" - -echo "== nodeup node config starting ==" -ensure-install-dir - -cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' -cloudConfig: null -containerRuntime: docker -containerd: - skipInstall: true -docker: - ipMasq: false - ipTables: false - logDriver: json-file - logLevel: info - logOpt: - - max-size=10m - - max-file=5 - storage: overlay2,overlay,aufs - version: 18.06.3 -encryptionConfig: null -etcdClusters: - events: - version: 3.3.10 - main: - version: 3.3.10 -kubeAPIServer: - allowPrivileged: true - anonymousAuth: false - apiServerCount: 1 - authorizationMode: AlwaysAllow - bindAddress: 0.0.0.0 - cloudProvider: aws - enableAdmissionPlugins: - - NamespaceLifecycle - - LimitRanger - - ServiceAccount - - PersistentVolumeLabel - - DefaultStorageClass - - DefaultTolerationSeconds - - MutatingAdmissionWebhook - - ValidatingAdmissionWebhook - - NodeRestriction - - ResourceQuota - etcdServers: - - http://127.0.0.1:4001 - etcdServersOverrides: - - /events#http://127.0.0.1:4002 - image: k8s.gcr.io/kube-apiserver:v1.14.0 - insecureBindAddress: 127.0.0.1 - insecurePort: 8080 - kubeletPreferredAddressTypes: - - InternalIP - - Hostname - - ExternalIP - logLevel: 2 - requestheaderAllowedNames: - - aggregator - requestheaderExtraHeaderPrefixes: - - X-Remote-Extra- - requestheaderGroupHeaders: - - X-Remote-Group - requestheaderUsernameHeaders: - - X-Remote-User - securePort: 443 - serviceClusterIPRange: 100.64.0.0/13 - storageBackend: etcd3 -kubeControllerManager: - allocateNodeCIDRs: true - attachDetachReconcileSyncPeriod: 1m0s - cloudProvider: aws - clusterCIDR: 100.96.0.0/11 - clusterName: restrictaccess.example.com - configureCloudRoutes: true - image: k8s.gcr.io/kube-controller-manager:v1.14.0 - leaderElection: - leaderElect: true - logLevel: 2 - useServiceAccountCredentials: true -kubeProxy: - clusterCIDR: 100.96.0.0/11 - cpuRequest: 100m - hostnameOverride: '@aws' - image: k8s.gcr.io/kube-proxy:v1.14.0 - logLevel: 2 -kubeScheduler: - image: k8s.gcr.io/kube-scheduler:v1.14.0 - leaderElection: - leaderElect: true - logLevel: 2 -kubelet: - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests -masterKubelet: - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests - registerSchedulable: false - -__EOF_CLUSTER_SPEC - -cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' -kubelet: null -nodeLabels: null -taints: null - -__EOF_IG_SPEC - -cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' -Assets: - amd64: - - c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet - - 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl - - 3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz - arm64: - - df38e04576026393055ccc77c0dce73612996561@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubelet - - 01c2b6b43d36b6bfafc80a3737391c19ebfb8ad5@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubectl - - 7fec91af78e9548df306f0ec43bea527c8c10cc3a9682c33e971c8522a7fcded@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-arm64-v0.7.5.tgz -ClusterName: restrictaccess.example.com -ConfigBase: memfs://clusters.example.com/restrictaccess.example.com -InstanceGroupName: master-us-test-1a -InstanceGroupRole: Master -Tags: -- _automatic_upgrades -- _aws -channels: -- memfs://clusters.example.com/restrictaccess.example.com/addons/bootstrap-channel.yaml -etcdManifests: -- memfs://clusters.example.com/restrictaccess.example.com/manifests/etcd/main.yaml -- memfs://clusters.example.com/restrictaccess.example.com/manifests/etcd/events.yaml -protokubeImage: - hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b - name: protokube:1.15.0 - sources: - - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz - - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz - - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz - -__EOF_KUBE_ENV - -download-release -echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/restrict_access/data/aws_launch_template_nodes.restrictaccess.example.com_user_data b/tests/integration/update_cluster/restrict_access/data/aws_launch_template_nodes.restrictaccess.example.com_user_data deleted file mode 100644 index 2dd515b8e6..0000000000 --- a/tests/integration/update_cluster/restrict_access/data/aws_launch_template_nodes.restrictaccess.example.com_user_data +++ /dev/null @@ -1,221 +0,0 @@ -#!/bin/bash -# Copyright 2016 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup -NODEUP_HASH_AMD64=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 -NODEUP_URL_ARM64= -NODEUP_HASH_ARM64= - -export AWS_REGION=us-test-1 - - - - -function ensure-install-dir() { - INSTALL_DIR="/opt/kops" - # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec - if [[ -d /var/lib/toolbox ]]; then - INSTALL_DIR="/var/lib/toolbox/kops" - fi - mkdir -p ${INSTALL_DIR}/bin - mkdir -p ${INSTALL_DIR}/conf - cd ${INSTALL_DIR} -} - -# Retry a download until we get it. args: name, sha, url1, url2... -download-or-bust() { - local -r file="$1" - local -r hash="$2" - shift 2 - - urls=( $* ) - while true; do - for url in "${urls[@]}"; do - commands=( - "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" - "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" - ) - for cmd in "${commands[@]}"; do - echo "Attempting download with: ${cmd} {url}" - if ! (${cmd} "${url}"); then - echo "== Download failed with ${cmd} ==" - continue - fi - if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then - echo "== Hash validation of ${url} failed. Retrying. ==" - rm -f "${file}" - else - if [[ -n "${hash}" ]]; then - echo "== Downloaded ${url} (SHA1 = ${hash}) ==" - else - echo "== Downloaded ${url} ==" - fi - return - fi - done - done - - echo "All downloads failed; sleeping before retrying" - sleep 60 - done -} - -validate-hash() { - local -r file="$1" - local -r expected="$2" - local actual - - actual=$(sha256sum ${file} | awk '{ print $1 }') || true - if [[ "${actual}" != "${expected}" ]]; then - echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" - return 1 - fi -} - -function split-commas() { - echo $1 | tr "," "\n" -} - -function try-download-release() { - local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) - if [[ -n "${NODEUP_HASH:-}" ]]; then - local -r nodeup_hash="${NODEUP_HASH}" - else - # TODO: Remove? - echo "Downloading sha256 (not found in env)" - download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" - local -r nodeup_hash=$(cat nodeup.sha256) - fi - - echo "Downloading nodeup (${nodeup_urls[@]})" - download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" - - chmod +x nodeup -} - -function download-release() { - case "$(uname -m)" in - x86_64*|i?86_64*|amd64*) - NODEUP_URL="${NODEUP_URL_AMD64}" - NODEUP_HASH="${NODEUP_HASH_AMD64}" - ;; - aarch64*|arm64*) - NODEUP_URL="${NODEUP_URL_ARM64}" - NODEUP_HASH="${NODEUP_HASH_ARM64}" - ;; - *) - echo "Unsupported host arch: $(uname -m)" >&2 - exit 1 - ;; - esac - - # In case of failure checking integrity of release, retry. - cd ${INSTALL_DIR}/bin - until try-download-release; do - sleep 15 - echo "Couldn't download release. Retrying..." - done - - echo "Running nodeup" - # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 - ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) -} - -#################################################################################### - -/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" - -echo "== nodeup node config starting ==" -ensure-install-dir - -cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' -cloudConfig: null -containerRuntime: docker -containerd: - skipInstall: true -docker: - ipMasq: false - ipTables: false - logDriver: json-file - logLevel: info - logOpt: - - max-size=10m - - max-file=5 - storage: overlay2,overlay,aufs - version: 18.06.3 -kubeProxy: - clusterCIDR: 100.96.0.0/11 - cpuRequest: 100m - hostnameOverride: '@aws' - image: k8s.gcr.io/kube-proxy:v1.14.0 - logLevel: 2 -kubelet: - anonymousAuth: false - cgroupRoot: / - cloudProvider: aws - clusterDNS: 100.64.0.10 - clusterDomain: cluster.local - enableDebuggingHandlers: true - evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% - featureGates: - ExperimentalCriticalPodAnnotation: "true" - hostnameOverride: '@aws' - kubeconfigPath: /var/lib/kubelet/kubeconfig - logLevel: 2 - networkPluginMTU: 9001 - networkPluginName: kubenet - nonMasqueradeCIDR: 100.64.0.0/10 - podInfraContainerImage: k8s.gcr.io/pause:3.2 - podManifestPath: /etc/kubernetes/manifests - -__EOF_CLUSTER_SPEC - -cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' -kubelet: null -nodeLabels: null -taints: null - -__EOF_IG_SPEC - -cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' -Assets: - amd64: - - c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet - - 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl - - 3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz - arm64: - - df38e04576026393055ccc77c0dce73612996561@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubelet - - 01c2b6b43d36b6bfafc80a3737391c19ebfb8ad5@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubectl - - 7fec91af78e9548df306f0ec43bea527c8c10cc3a9682c33e971c8522a7fcded@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-arm64-v0.7.5.tgz -ClusterName: restrictaccess.example.com -ConfigBase: memfs://clusters.example.com/restrictaccess.example.com -InstanceGroupName: nodes -InstanceGroupRole: Node -Tags: -- _automatic_upgrades -- _aws -channels: -- memfs://clusters.example.com/restrictaccess.example.com/addons/bootstrap-channel.yaml - -__EOF_KUBE_ENV - -download-release -echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/restrict_access/id_rsa.pub b/tests/integration/update_cluster/restrict_access/id_rsa.pub deleted file mode 100755 index 81cb012783..0000000000 --- a/tests/integration/update_cluster/restrict_access/id_rsa.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/restrict_access/in-v1alpha2.yaml b/tests/integration/update_cluster/restrict_access/in-v1alpha2.yaml deleted file mode 100644 index 0be25a9337..0000000000 --- a/tests/integration/update_cluster/restrict_access/in-v1alpha2.yaml +++ /dev/null @@ -1,81 +0,0 @@ -apiVersion: kops.k8s.io/v1alpha2 -kind: Cluster -metadata: - creationTimestamp: "2016-12-10T22:42:27Z" - name: restrictaccess.example.com -spec: - kubernetesApiAccess: - - 1.1.1.0/24 - - 2001:0:8500::/40 - channel: stable - cloudProvider: aws - configBase: memfs://clusters.example.com/restrictaccess.example.com - etcdClusters: - - etcdMembers: - - instanceGroup: master-us-test-1a - name: us-test-1a - name: main - - etcdMembers: - - instanceGroup: master-us-test-1a - name: us-test-1a - name: events - kubelet: - anonymousAuth: false - kubernetesVersion: v1.14.0 - masterInternalName: api.internal.restrictaccess.example.com - masterPublicName: api.restrictaccess.example.com - networkCIDR: 172.20.0.0/16 - networking: - kubenet: {} - nonMasqueradeCIDR: 100.64.0.0/10 - sshAccess: - - 1.1.1.1/32 - - 2001:0:85a3::/48 - topology: - masters: public - nodes: public - subnets: - - cidr: 172.20.32.0/19 - name: us-test-1a - type: Public - zone: us-test-1a - ---- - -apiVersion: kops.k8s.io/v1alpha2 -kind: InstanceGroup -metadata: - creationTimestamp: "2016-12-10T22:42:28Z" - name: nodes - labels: - kops.k8s.io/cluster: restrictaccess.example.com -spec: - associatePublicIp: true - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: t2.medium - maxSize: 2 - minSize: 2 - role: Node - subnets: - - us-test-1a - ---- - -apiVersion: kops.k8s.io/v1alpha2 -kind: InstanceGroup -metadata: - creationTimestamp: "2016-12-10T22:42:28Z" - name: master-us-test-1a - labels: - kops.k8s.io/cluster: restrictaccess.example.com -spec: - associatePublicIp: true - image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 - machineType: m3.medium - maxSize: 1 - minSize: 1 - role: Master - subnets: - - us-test-1a - - diff --git a/tests/integration/update_cluster/restrict_access/kubernetes.tf b/tests/integration/update_cluster/restrict_access/kubernetes.tf deleted file mode 100644 index 8bf5f791e1..0000000000 --- a/tests/integration/update_cluster/restrict_access/kubernetes.tf +++ /dev/null @@ -1,550 +0,0 @@ -locals { - cluster_name = "restrictaccess.example.com" - master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-restrictaccess-example-com.id] - master_security_group_ids = [aws_security_group.masters-restrictaccess-example-com.id] - masters_role_arn = aws_iam_role.masters-restrictaccess-example-com.arn - masters_role_name = aws_iam_role.masters-restrictaccess-example-com.name - node_autoscaling_group_ids = [aws_autoscaling_group.nodes-restrictaccess-example-com.id] - node_security_group_ids = [aws_security_group.nodes-restrictaccess-example-com.id] - node_subnet_ids = [aws_subnet.us-test-1a-restrictaccess-example-com.id] - nodes_role_arn = aws_iam_role.nodes-restrictaccess-example-com.arn - nodes_role_name = aws_iam_role.nodes-restrictaccess-example-com.name - region = "us-test-1" - route_table_public_id = aws_route_table.restrictaccess-example-com.id - subnet_us-test-1a_id = aws_subnet.us-test-1a-restrictaccess-example-com.id - vpc_cidr_block = aws_vpc.restrictaccess-example-com.cidr_block - vpc_id = aws_vpc.restrictaccess-example-com.id -} - -output "cluster_name" { - value = "restrictaccess.example.com" -} - -output "master_autoscaling_group_ids" { - value = [aws_autoscaling_group.master-us-test-1a-masters-restrictaccess-example-com.id] -} - -output "master_security_group_ids" { - value = [aws_security_group.masters-restrictaccess-example-com.id] -} - -output "masters_role_arn" { - value = aws_iam_role.masters-restrictaccess-example-com.arn -} - -output "masters_role_name" { - value = aws_iam_role.masters-restrictaccess-example-com.name -} - -output "node_autoscaling_group_ids" { - value = [aws_autoscaling_group.nodes-restrictaccess-example-com.id] -} - -output "node_security_group_ids" { - value = [aws_security_group.nodes-restrictaccess-example-com.id] -} - -output "node_subnet_ids" { - value = [aws_subnet.us-test-1a-restrictaccess-example-com.id] -} - -output "nodes_role_arn" { - value = aws_iam_role.nodes-restrictaccess-example-com.arn -} - -output "nodes_role_name" { - value = aws_iam_role.nodes-restrictaccess-example-com.name -} - -output "region" { - value = "us-test-1" -} - -output "route_table_public_id" { - value = aws_route_table.restrictaccess-example-com.id -} - -output "subnet_us-test-1a_id" { - value = aws_subnet.us-test-1a-restrictaccess-example-com.id -} - -output "vpc_cidr_block" { - value = aws_vpc.restrictaccess-example-com.cidr_block -} - -output "vpc_id" { - value = aws_vpc.restrictaccess-example-com.id -} - -provider "aws" { - region = "us-test-1" -} - -resource "aws_autoscaling_group" "master-us-test-1a-masters-restrictaccess-example-com" { - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] - launch_template { - id = aws_launch_template.master-us-test-1a-masters-restrictaccess-example-com.id - version = aws_launch_template.master-us-test-1a-masters-restrictaccess-example-com.latest_version - } - max_size = 1 - metrics_granularity = "1Minute" - min_size = 1 - name = "master-us-test-1a.masters.restrictaccess.example.com" - tag { - key = "KubernetesCluster" - propagate_at_launch = true - value = "restrictaccess.example.com" - } - tag { - key = "Name" - propagate_at_launch = true - value = "master-us-test-1a.masters.restrictaccess.example.com" - } - tag { - key = "k8s.io/role/master" - propagate_at_launch = true - value = "1" - } - tag { - key = "kops.k8s.io/instancegroup" - propagate_at_launch = true - value = "master-us-test-1a" - } - tag { - key = "kubernetes.io/cluster/restrictaccess.example.com" - propagate_at_launch = true - value = "owned" - } - vpc_zone_identifier = [aws_subnet.us-test-1a-restrictaccess-example-com.id] -} - -resource "aws_autoscaling_group" "nodes-restrictaccess-example-com" { - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] - launch_template { - id = aws_launch_template.nodes-restrictaccess-example-com.id - version = aws_launch_template.nodes-restrictaccess-example-com.latest_version - } - max_size = 2 - metrics_granularity = "1Minute" - min_size = 2 - name = "nodes.restrictaccess.example.com" - tag { - key = "KubernetesCluster" - propagate_at_launch = true - value = "restrictaccess.example.com" - } - tag { - key = "Name" - propagate_at_launch = true - value = "nodes.restrictaccess.example.com" - } - tag { - key = "k8s.io/role/node" - propagate_at_launch = true - value = "1" - } - tag { - key = "kops.k8s.io/instancegroup" - propagate_at_launch = true - value = "nodes" - } - tag { - key = "kubernetes.io/cluster/restrictaccess.example.com" - propagate_at_launch = true - value = "owned" - } - vpc_zone_identifier = [aws_subnet.us-test-1a-restrictaccess-example-com.id] -} - -resource "aws_ebs_volume" "us-test-1a-etcd-events-restrictaccess-example-com" { - availability_zone = "us-test-1a" - encrypted = false - size = 20 - tags = { - "KubernetesCluster" = "restrictaccess.example.com" - "Name" = "us-test-1a.etcd-events.restrictaccess.example.com" - "k8s.io/etcd/events" = "us-test-1a/us-test-1a" - "k8s.io/role/master" = "1" - "kubernetes.io/cluster/restrictaccess.example.com" = "owned" - } - type = "gp2" -} - -resource "aws_ebs_volume" "us-test-1a-etcd-main-restrictaccess-example-com" { - availability_zone = "us-test-1a" - encrypted = false - size = 20 - tags = { - "KubernetesCluster" = "restrictaccess.example.com" - "Name" = "us-test-1a.etcd-main.restrictaccess.example.com" - "k8s.io/etcd/main" = "us-test-1a/us-test-1a" - "k8s.io/role/master" = "1" - "kubernetes.io/cluster/restrictaccess.example.com" = "owned" - } - type = "gp2" -} - -resource "aws_iam_instance_profile" "masters-restrictaccess-example-com" { - name = "masters.restrictaccess.example.com" - role = aws_iam_role.masters-restrictaccess-example-com.name -} - -resource "aws_iam_instance_profile" "nodes-restrictaccess-example-com" { - name = "nodes.restrictaccess.example.com" - role = aws_iam_role.nodes-restrictaccess-example-com.name -} - -resource "aws_iam_role_policy" "masters-restrictaccess-example-com" { - name = "masters.restrictaccess.example.com" - policy = file("${path.module}/data/aws_iam_role_policy_masters.restrictaccess.example.com_policy") - role = aws_iam_role.masters-restrictaccess-example-com.name -} - -resource "aws_iam_role_policy" "nodes-restrictaccess-example-com" { - name = "nodes.restrictaccess.example.com" - policy = file("${path.module}/data/aws_iam_role_policy_nodes.restrictaccess.example.com_policy") - role = aws_iam_role.nodes-restrictaccess-example-com.name -} - -resource "aws_iam_role" "masters-restrictaccess-example-com" { - assume_role_policy = file("${path.module}/data/aws_iam_role_masters.restrictaccess.example.com_policy") - name = "masters.restrictaccess.example.com" -} - -resource "aws_iam_role" "nodes-restrictaccess-example-com" { - assume_role_policy = file("${path.module}/data/aws_iam_role_nodes.restrictaccess.example.com_policy") - name = "nodes.restrictaccess.example.com" -} - -resource "aws_internet_gateway" "restrictaccess-example-com" { - tags = { - "KubernetesCluster" = "restrictaccess.example.com" - "Name" = "restrictaccess.example.com" - "kubernetes.io/cluster/restrictaccess.example.com" = "owned" - } - vpc_id = aws_vpc.restrictaccess-example-com.id -} - -resource "aws_key_pair" "kubernetes-restrictaccess-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" { - key_name = "kubernetes.restrictaccess.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57" - public_key = file("${path.module}/data/aws_key_pair_kubernetes.restrictaccess.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key") -} - -resource "aws_launch_template" "master-us-test-1a-masters-restrictaccess-example-com" { - block_device_mappings { - device_name = "/dev/xvda" - ebs { - delete_on_termination = true - volume_size = 64 - volume_type = "gp2" - } - } - block_device_mappings { - device_name = "/dev/sdc" - virtual_name = "ephemeral0" - } - iam_instance_profile { - name = aws_iam_instance_profile.masters-restrictaccess-example-com.id - } - image_id = "ami-12345678" - instance_type = "m3.medium" - key_name = aws_key_pair.kubernetes-restrictaccess-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id - lifecycle { - create_before_destroy = true - } - name_prefix = "master-us-test-1a.masters.restrictaccess.example.com-" - network_interfaces { - associate_public_ip_address = true - delete_on_termination = true - security_groups = [aws_security_group.masters-restrictaccess-example-com.id] - } - tag_specifications { - resource_type = "instance" - tags = { - "KubernetesCluster" = "restrictaccess.example.com" - "Name" = "master-us-test-1a.masters.restrictaccess.example.com" - "k8s.io/role/master" = "1" - "kops.k8s.io/instancegroup" = "master-us-test-1a" - "kubernetes.io/cluster/restrictaccess.example.com" = "owned" - } - } - tag_specifications { - resource_type = "volume" - tags = { - "KubernetesCluster" = "restrictaccess.example.com" - "Name" = "master-us-test-1a.masters.restrictaccess.example.com" - "k8s.io/role/master" = "1" - "kops.k8s.io/instancegroup" = "master-us-test-1a" - "kubernetes.io/cluster/restrictaccess.example.com" = "owned" - } - } - user_data = file("${path.module}/data/aws_launch_template_master-us-test-1a.masters.restrictaccess.example.com_user_data") -} - -resource "aws_launch_template" "nodes-restrictaccess-example-com" { - block_device_mappings { - device_name = "/dev/xvda" - ebs { - delete_on_termination = true - volume_size = 128 - volume_type = "gp2" - } - } - iam_instance_profile { - name = aws_iam_instance_profile.nodes-restrictaccess-example-com.id - } - image_id = "ami-12345678" - instance_type = "t2.medium" - key_name = aws_key_pair.kubernetes-restrictaccess-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id - lifecycle { - create_before_destroy = true - } - name_prefix = "nodes.restrictaccess.example.com-" - network_interfaces { - associate_public_ip_address = true - delete_on_termination = true - security_groups = [aws_security_group.nodes-restrictaccess-example-com.id] - } - tag_specifications { - resource_type = "instance" - tags = { - "KubernetesCluster" = "restrictaccess.example.com" - "Name" = "nodes.restrictaccess.example.com" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "nodes" - "kubernetes.io/cluster/restrictaccess.example.com" = "owned" - } - } - tag_specifications { - resource_type = "volume" - tags = { - "KubernetesCluster" = "restrictaccess.example.com" - "Name" = "nodes.restrictaccess.example.com" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "nodes" - "kubernetes.io/cluster/restrictaccess.example.com" = "owned" - } - } - user_data = file("${path.module}/data/aws_launch_template_nodes.restrictaccess.example.com_user_data") -} - -resource "aws_route_table_association" "us-test-1a-restrictaccess-example-com" { - route_table_id = aws_route_table.restrictaccess-example-com.id - subnet_id = aws_subnet.us-test-1a-restrictaccess-example-com.id -} - -resource "aws_route_table" "restrictaccess-example-com" { - tags = { - "KubernetesCluster" = "restrictaccess.example.com" - "Name" = "restrictaccess.example.com" - "kubernetes.io/cluster/restrictaccess.example.com" = "owned" - "kubernetes.io/kops/role" = "public" - } - vpc_id = aws_vpc.restrictaccess-example-com.id -} - -resource "aws_route" "route-0-0-0-0--0" { - destination_cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.restrictaccess-example-com.id - route_table_id = aws_route_table.restrictaccess-example-com.id -} - -resource "aws_security_group_rule" "all-master-to-master" { - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.masters-restrictaccess-example-com.id - source_security_group_id = aws_security_group.masters-restrictaccess-example-com.id - to_port = 0 - type = "ingress" -} - -resource "aws_security_group_rule" "all-master-to-node" { - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.nodes-restrictaccess-example-com.id - source_security_group_id = aws_security_group.masters-restrictaccess-example-com.id - to_port = 0 - type = "ingress" -} - -resource "aws_security_group_rule" "all-node-to-node" { - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.nodes-restrictaccess-example-com.id - source_security_group_id = aws_security_group.nodes-restrictaccess-example-com.id - to_port = 0 - type = "ingress" -} - -resource "aws_security_group_rule" "https-external-to-master-1-1-1-0--24" { - cidr_blocks = ["1.1.1.0/24"] - from_port = 443 - protocol = "tcp" - security_group_id = aws_security_group.masters-restrictaccess-example-com.id - to_port = 443 - type = "ingress" -} - -resource "aws_security_group_rule" "https-external-to-master-2001_0_8500__--40" { - cidr_blocks = ["2001:0:8500::/40"] - from_port = 443 - protocol = "tcp" - security_group_id = aws_security_group.masters-restrictaccess-example-com.id - to_port = 443 - type = "ingress" -} - -resource "aws_security_group_rule" "master-egress" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.masters-restrictaccess-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "node-egress" { - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - protocol = "-1" - security_group_id = aws_security_group.nodes-restrictaccess-example-com.id - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "node-to-master-tcp-1-2379" { - from_port = 1 - protocol = "tcp" - security_group_id = aws_security_group.masters-restrictaccess-example-com.id - source_security_group_id = aws_security_group.nodes-restrictaccess-example-com.id - to_port = 2379 - type = "ingress" -} - -resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" { - from_port = 2382 - protocol = "tcp" - security_group_id = aws_security_group.masters-restrictaccess-example-com.id - source_security_group_id = aws_security_group.nodes-restrictaccess-example-com.id - to_port = 4000 - type = "ingress" -} - -resource "aws_security_group_rule" "node-to-master-tcp-4003-65535" { - from_port = 4003 - protocol = "tcp" - security_group_id = aws_security_group.masters-restrictaccess-example-com.id - source_security_group_id = aws_security_group.nodes-restrictaccess-example-com.id - to_port = 65535 - type = "ingress" -} - -resource "aws_security_group_rule" "node-to-master-udp-1-65535" { - from_port = 1 - protocol = "udp" - security_group_id = aws_security_group.masters-restrictaccess-example-com.id - source_security_group_id = aws_security_group.nodes-restrictaccess-example-com.id - to_port = 65535 - type = "ingress" -} - -resource "aws_security_group_rule" "ssh-external-to-master-1-1-1-1--32" { - cidr_blocks = ["1.1.1.1/32"] - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.masters-restrictaccess-example-com.id - to_port = 22 - type = "ingress" -} - -resource "aws_security_group_rule" "ssh-external-to-master-2001_0_85a3__--48" { - cidr_blocks = ["2001:0:85a3::/48"] - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.masters-restrictaccess-example-com.id - to_port = 22 - type = "ingress" -} - -resource "aws_security_group_rule" "ssh-external-to-node-1-1-1-1--32" { - cidr_blocks = ["1.1.1.1/32"] - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.nodes-restrictaccess-example-com.id - to_port = 22 - type = "ingress" -} - -resource "aws_security_group_rule" "ssh-external-to-node-2001_0_85a3__--48" { - cidr_blocks = ["2001:0:85a3::/48"] - from_port = 22 - protocol = "tcp" - security_group_id = aws_security_group.nodes-restrictaccess-example-com.id - to_port = 22 - type = "ingress" -} - -resource "aws_security_group" "masters-restrictaccess-example-com" { - description = "Security group for masters" - name = "masters.restrictaccess.example.com" - tags = { - "KubernetesCluster" = "restrictaccess.example.com" - "Name" = "masters.restrictaccess.example.com" - "kubernetes.io/cluster/restrictaccess.example.com" = "owned" - } - vpc_id = aws_vpc.restrictaccess-example-com.id -} - -resource "aws_security_group" "nodes-restrictaccess-example-com" { - description = "Security group for nodes" - name = "nodes.restrictaccess.example.com" - tags = { - "KubernetesCluster" = "restrictaccess.example.com" - "Name" = "nodes.restrictaccess.example.com" - "kubernetes.io/cluster/restrictaccess.example.com" = "owned" - } - vpc_id = aws_vpc.restrictaccess-example-com.id -} - -resource "aws_subnet" "us-test-1a-restrictaccess-example-com" { - availability_zone = "us-test-1a" - cidr_block = "172.20.32.0/19" - tags = { - "KubernetesCluster" = "restrictaccess.example.com" - "Name" = "us-test-1a.restrictaccess.example.com" - "SubnetType" = "Public" - "kubernetes.io/cluster/restrictaccess.example.com" = "owned" - "kubernetes.io/role/elb" = "1" - } - vpc_id = aws_vpc.restrictaccess-example-com.id -} - -resource "aws_vpc_dhcp_options_association" "restrictaccess-example-com" { - dhcp_options_id = aws_vpc_dhcp_options.restrictaccess-example-com.id - vpc_id = aws_vpc.restrictaccess-example-com.id -} - -resource "aws_vpc_dhcp_options" "restrictaccess-example-com" { - domain_name = "us-test-1.compute.internal" - domain_name_servers = ["AmazonProvidedDNS"] - tags = { - "KubernetesCluster" = "restrictaccess.example.com" - "Name" = "restrictaccess.example.com" - "kubernetes.io/cluster/restrictaccess.example.com" = "owned" - } -} - -resource "aws_vpc" "restrictaccess-example-com" { - cidr_block = "172.20.0.0/16" - enable_dns_hostnames = true - enable_dns_support = true - tags = { - "KubernetesCluster" = "restrictaccess.example.com" - "Name" = "restrictaccess.example.com" - "kubernetes.io/cluster/restrictaccess.example.com" = "owned" - } -} - -terraform { - required_version = ">= 0.12.0" -}