From 6b38b2c3a64613f2d207767ad72c1b2f6d7277f2 Mon Sep 17 00:00:00 2001 From: Fred Dubois Date: Thu, 15 Feb 2018 14:44:35 -0500 Subject: [PATCH] Add support for bastion aws user-data Fixes #4444 --- cmd/kops/integration_test.go | 50 +- pkg/apis/kops/instancegroup.go | 15 + pkg/model/bootstrapscript.go | 4 +- pkg/model/resources/nodeup.go | 9 +- ...tion.bastionuserdata.example.com_user_data | 13 + .../bastionadditional_user-data/id_rsa.pub | 1 + .../in-v1alpha2.yaml | 110 +++ .../bastionadditional_user-data/kubernetes.tf | 715 ++++++++++++++++++ 8 files changed, 909 insertions(+), 8 deletions(-) create mode 100644 tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_configuration_bastion.bastionuserdata.example.com_user_data create mode 100755 tests/integration/update_cluster/bastionadditional_user-data/id_rsa.pub create mode 100644 tests/integration/update_cluster/bastionadditional_user-data/in-v1alpha2.yaml create mode 100644 tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index 1891521eb9..6e064aec2a 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -84,6 +84,11 @@ func TestAdditionalUserData(t *testing.T) { runTestCloudformation(t, "additionaluserdata.example.com", "additional_user-data", "v1alpha2", false) } +// TestBastionAdditionalUserData runs the test on passing additional user-data to a bastion instance group +func TestBastionAdditionalUserData(t *testing.T) { + runTestAWS(t, "bastionuserdata.example.com", "bastionadditional_user-data", "v1alpha2", true, 1) +} + // TestMinimal_141 runs the test on a configuration from 1.4.1 release func TestMinimal_141(t *testing.T) { runTestAWS(t, "minimal-141.example.com", "minimal-141", "v1alpha0", false, 1) @@ -257,7 +262,8 @@ func runTest(t *testing.T, h *testutils.IntegrationTestHarness, clusterName stri // Compare data files if they are provided if len(expectedFilenames) > 0 { - files, err := ioutil.ReadDir(path.Join(h.TempDir, "out", "data")) + actualDataPath := path.Join(h.TempDir, "out", "data") + files, err := ioutil.ReadDir(actualDataPath) if err != nil { t.Fatalf("failed to read data dir: %v", err) } @@ -272,7 +278,41 @@ func runTest(t *testing.T, h *testutils.IntegrationTestHarness, clusterName stri t.Fatalf("unexpected data files. actual=%q, expected=%q", actualFilenames, expectedFilenames) } - // TODO: any verification of data files? + // Some tests might provide _some_ tf data files (not necessarilly all that + // are actually produced), validate that the provided expected data file + // contents match actual data file content + expectedDataPath := path.Join(srcDir, "data") + if _, err := os.Stat(expectedDataPath); err == nil { + expectedDataFiles, err := ioutil.ReadDir(expectedDataPath) + if err != nil { + t.Fatalf("failed to read expected data dir: %v", err) + } + for _, expectedDataFile := range expectedDataFiles { + dataFileName := expectedDataFile.Name() + expectedDataContent, err := + ioutil.ReadFile(path.Join(expectedDataPath, dataFileName)) + if err != nil { + t.Fatalf("failed to read expected data file: %v", err) + } + actualDataContent, err := + ioutil.ReadFile(path.Join(actualDataPath, dataFileName)) + if err != nil { + t.Fatalf("failed to read actual data file: %v", err) + } + if string(expectedDataContent) != string(actualDataContent) { + t.Fatalf( + "actual data file (%s) did not match the content of expected data file (%s). "+ + "NOTE: If outputs seem identical, check for end-of-line differences, "+ + "especially if the file is in multipart MIME format!"+ + "\nBEGIN_ACTUAL:\n%s\nEND_ACTUAL\nBEGIN_EXPECTED:\n%s\nEND_EXPECTED", + path.Join(actualDataPath, dataFileName), + path.Join(expectedDataPath, dataFileName), + actualDataContent, + expectedDataContent, + ) + } + } + } } } @@ -303,10 +343,14 @@ func runTestAWS(t *testing.T, clusterName string, srcDir string, version string, "aws_iam_role_bastions." + clusterName + "_policy", "aws_iam_role_policy_bastions." + clusterName + "_policy", - // bastions don't have any userdata + // bastions usually don't have any userdata // "aws_launch_configuration_bastions." + clusterName + "_user_data", }...) } + // Special case that tests a bastion with user-data + if srcDir == "bastionadditional_user-data" { + expectedFilenames = append(expectedFilenames, "aws_launch_configuration_bastion."+clusterName+"_user_data") + } runTest(t, h, clusterName, srcDir, version, private, zones, expectedFilenames, "", nil) } diff --git a/pkg/apis/kops/instancegroup.go b/pkg/apis/kops/instancegroup.go index 7a0cefc1f3..2b92f498bb 100644 --- a/pkg/apis/kops/instancegroup.go +++ b/pkg/apis/kops/instancegroup.go @@ -168,6 +168,21 @@ func (g *InstanceGroup) IsMaster() bool { } } +// IsBastion checks if instanceGroup is a bastion +func (g *InstanceGroup) IsBastion() bool { + switch g.Spec.Role { + case InstanceGroupRoleMaster: + return false + case InstanceGroupRoleNode: + return false + case InstanceGroupRoleBastion: + return true + default: + glog.Fatalf("Role not set in group %v", g) + return false + } +} + func (g *InstanceGroup) AddInstanceGroupNodeLabel() { if g.Spec.NodeLabels == nil { nodeLabels := make(map[string]string) diff --git a/pkg/model/bootstrapscript.go b/pkg/model/bootstrapscript.go index 65bbd0b8ff..21176d95cd 100644 --- a/pkg/model/bootstrapscript.go +++ b/pkg/model/bootstrapscript.go @@ -59,8 +59,8 @@ func (b *BootstrapScript) KubeEnv(ig *kops.InstanceGroup) (string, error) { // ResourceNodeUp generates and returns a nodeup (bootstrap) script from a // template file, substituting in specific env vars & cluster spec configuration func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup, cs *kops.ClusterSpec) (*fi.ResourceHolder, error) { - if ig.Spec.Role == kops.InstanceGroupRoleBastion { - // Bastions are just bare machines (currently), used as SSH jump-hosts + // Bastions can have AdditionalUserData, but if there isn't any skip this part + if ig.IsBastion() && len(ig.Spec.AdditionalUserData) == 0 { return nil, nil } diff --git a/pkg/model/resources/nodeup.go b/pkg/model/resources/nodeup.go index 4d9de3cc14..f64d226c21 100644 --- a/pkg/model/resources/nodeup.go +++ b/pkg/model/resources/nodeup.go @@ -203,9 +203,12 @@ func AWSNodeUpTemplate(ig *kops.InstanceGroup) (string, error) { writer.Write([]byte(fmt.Sprintf("Content-Type: multipart/mixed; boundary=\"%s\"\r\n", boundary))) writer.Write([]byte("MIME-Version: 1.0\r\n\r\n")) - err := writeUserDataPart(mimeWriter, "nodeup.sh", "text/x-shellscript", []byte(userDataTemplate)) - if err != nil { - return "", err + var err error + if !ig.IsBastion() { + err := writeUserDataPart(mimeWriter, "nodeup.sh", "text/x-shellscript", []byte(userDataTemplate)) + if err != nil { + return "", err + } } for _, UserDataInfo := range ig.Spec.AdditionalUserData { diff --git a/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_configuration_bastion.bastionuserdata.example.com_user_data b/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_configuration_bastion.bastionuserdata.example.com_user_data new file mode 100644 index 0000000000..7b0869f82f --- /dev/null +++ b/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_configuration_bastion.bastionuserdata.example.com_user_data @@ -0,0 +1,13 @@ +Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" +MIME-Version: 1.0 + +--MIMEBOUNDARY +Content-Disposition: attachment; filename="myscript.sh" +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript +Mime-Version: 1.0 + +#!/bin/sh +echo "Hello World, from the bastion! The time is now $(date -R)!" | tee /root/output.txt + +--MIMEBOUNDARY-- diff --git a/tests/integration/update_cluster/bastionadditional_user-data/id_rsa.pub b/tests/integration/update_cluster/bastionadditional_user-data/id_rsa.pub new file mode 100755 index 0000000000..81cb012783 --- /dev/null +++ b/tests/integration/update_cluster/bastionadditional_user-data/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/bastionadditional_user-data/in-v1alpha2.yaml b/tests/integration/update_cluster/bastionadditional_user-data/in-v1alpha2.yaml new file mode 100644 index 0000000000..b1a9d349e1 --- /dev/null +++ b/tests/integration/update_cluster/bastionadditional_user-data/in-v1alpha2.yaml @@ -0,0 +1,110 @@ +apiVersion: kops/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2016-12-12T04:13:14Z" + name: bastionuserdata.example.com +spec: + kubernetesApiAccess: + - 0.0.0.0/0 + channel: stable + cloudProvider: aws + configBase: memfs://clusters.example.com/bastionuserdata.example.com + etcdClusters: + - etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: main + - etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: events + kubernetesVersion: v1.8.0 + masterInternalName: api.internal.bastionuserdata.example.com + masterPublicName: api.bastionuserdata.example.com + networkCIDR: 172.20.0.0/16 + networking: + calico: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + topology: + masters: private + nodes: private + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Private + zone: us-test-1a + - cidr: 172.20.4.0/22 + name: utility-us-test-1a + type: Utility + zone: us-test-1a + +--- + +apiVersion: kops/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-12T04:13:15Z" + name: master-us-test-1a + labels: + kops.k8s.io/cluster: bastionuserdata.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 + machineType: m3.medium + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test-1a + +--- + +apiVersion: kops/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-12T04:13:15Z" + name: nodes + labels: + kops.k8s.io/cluster: bastionuserdata.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 + machineType: t2.medium + maxSize: 2 + minSize: 2 + role: Node + subnets: + - us-test-1a + additionalUserData: + - name: myscript.sh + type: text/x-shellscript + content: | + #!/bin/sh + echo "Hello World, from a node! The time is now $(date -R)!" | tee /root/output.txt + +--- + +apiVersion: kops/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-14T15:32:41Z" + name: bastion + labels: + kops.k8s.io/cluster: bastionuserdata.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 + machineType: t2.micro + maxSize: 1 + minSize: 1 + role: Bastion + subnets: + - utility-us-test-1a + additionalUserData: + - name: myscript.sh + type: text/x-shellscript + content: | + #!/bin/sh + echo "Hello World, from the bastion! The time is now $(date -R)!" | tee /root/output.txt diff --git a/tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf b/tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf new file mode 100644 index 0000000000..afc8d85ac7 --- /dev/null +++ b/tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf @@ -0,0 +1,715 @@ +output "bastion_security_group_ids" { + value = ["${aws_security_group.bastion-bastionuserdata-example-com.id}"] +} + +output "bastions_role_arn" { + value = "${aws_iam_role.bastions-bastionuserdata-example-com.arn}" +} + +output "bastions_role_name" { + value = "${aws_iam_role.bastions-bastionuserdata-example-com.name}" +} + +output "cluster_name" { + value = "bastionuserdata.example.com" +} + +output "master_security_group_ids" { + value = ["${aws_security_group.masters-bastionuserdata-example-com.id}"] +} + +output "masters_role_arn" { + value = "${aws_iam_role.masters-bastionuserdata-example-com.arn}" +} + +output "masters_role_name" { + value = "${aws_iam_role.masters-bastionuserdata-example-com.name}" +} + +output "node_security_group_ids" { + value = ["${aws_security_group.nodes-bastionuserdata-example-com.id}"] +} + +output "node_subnet_ids" { + value = ["${aws_subnet.us-test-1a-bastionuserdata-example-com.id}"] +} + +output "nodes_role_arn" { + value = "${aws_iam_role.nodes-bastionuserdata-example-com.arn}" +} + +output "nodes_role_name" { + value = "${aws_iam_role.nodes-bastionuserdata-example-com.name}" +} + +output "region" { + value = "us-test-1" +} + +output "vpc_id" { + value = "${aws_vpc.bastionuserdata-example-com.id}" +} + +provider "aws" { + region = "us-test-1" +} + +resource "aws_autoscaling_attachment" "bastion-bastionuserdata-example-com" { + elb = "${aws_elb.bastion-bastionuserdata-example-com.id}" + autoscaling_group_name = "${aws_autoscaling_group.bastion-bastionuserdata-example-com.id}" +} + +resource "aws_autoscaling_attachment" "master-us-test-1a-masters-bastionuserdata-example-com" { + elb = "${aws_elb.api-bastionuserdata-example-com.id}" + autoscaling_group_name = "${aws_autoscaling_group.master-us-test-1a-masters-bastionuserdata-example-com.id}" +} + +resource "aws_autoscaling_group" "bastion-bastionuserdata-example-com" { + name = "bastion.bastionuserdata.example.com" + launch_configuration = "${aws_launch_configuration.bastion-bastionuserdata-example-com.id}" + max_size = 1 + min_size = 1 + vpc_zone_identifier = ["${aws_subnet.utility-us-test-1a-bastionuserdata-example-com.id}"] + + tag = { + key = "KubernetesCluster" + value = "bastionuserdata.example.com" + propagate_at_launch = true + } + + tag = { + key = "Name" + value = "bastion.bastionuserdata.example.com" + propagate_at_launch = true + } + + tag = { + key = "k8s.io/role/bastion" + value = "1" + propagate_at_launch = true + } + + metrics_granularity = "1Minute" + enabled_metrics = ["GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity", "GroupInServiceInstances", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] +} + +resource "aws_autoscaling_group" "master-us-test-1a-masters-bastionuserdata-example-com" { + name = "master-us-test-1a.masters.bastionuserdata.example.com" + launch_configuration = "${aws_launch_configuration.master-us-test-1a-masters-bastionuserdata-example-com.id}" + max_size = 1 + min_size = 1 + vpc_zone_identifier = ["${aws_subnet.us-test-1a-bastionuserdata-example-com.id}"] + + tag = { + key = "KubernetesCluster" + value = "bastionuserdata.example.com" + propagate_at_launch = true + } + + tag = { + key = "Name" + value = "master-us-test-1a.masters.bastionuserdata.example.com" + propagate_at_launch = true + } + + tag = { + key = "k8s.io/role/master" + value = "1" + propagate_at_launch = true + } + + metrics_granularity = "1Minute" + enabled_metrics = ["GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity", "GroupInServiceInstances", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] +} + +resource "aws_autoscaling_group" "nodes-bastionuserdata-example-com" { + name = "nodes.bastionuserdata.example.com" + launch_configuration = "${aws_launch_configuration.nodes-bastionuserdata-example-com.id}" + max_size = 2 + min_size = 2 + vpc_zone_identifier = ["${aws_subnet.us-test-1a-bastionuserdata-example-com.id}"] + + tag = { + key = "KubernetesCluster" + value = "bastionuserdata.example.com" + propagate_at_launch = true + } + + tag = { + key = "Name" + value = "nodes.bastionuserdata.example.com" + propagate_at_launch = true + } + + tag = { + key = "k8s.io/role/node" + value = "1" + propagate_at_launch = true + } + + metrics_granularity = "1Minute" + enabled_metrics = ["GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity", "GroupInServiceInstances", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] +} + +resource "aws_ebs_volume" "us-test-1a-etcd-events-bastionuserdata-example-com" { + availability_zone = "us-test-1a" + size = 20 + type = "gp2" + encrypted = false + + tags = { + KubernetesCluster = "bastionuserdata.example.com" + Name = "us-test-1a.etcd-events.bastionuserdata.example.com" + "k8s.io/etcd/events" = "us-test-1a/us-test-1a" + "k8s.io/role/master" = "1" + } +} + +resource "aws_ebs_volume" "us-test-1a-etcd-main-bastionuserdata-example-com" { + availability_zone = "us-test-1a" + size = 20 + type = "gp2" + encrypted = false + + tags = { + KubernetesCluster = "bastionuserdata.example.com" + Name = "us-test-1a.etcd-main.bastionuserdata.example.com" + "k8s.io/etcd/main" = "us-test-1a/us-test-1a" + "k8s.io/role/master" = "1" + } +} + +resource "aws_eip" "us-test-1a-bastionuserdata-example-com" { + vpc = true +} + +resource "aws_elb" "api-bastionuserdata-example-com" { + name = "api-bastionuserdata-examp-qbgom9" + + listener = { + instance_port = 443 + instance_protocol = "TCP" + lb_port = 443 + lb_protocol = "TCP" + } + + security_groups = ["${aws_security_group.api-elb-bastionuserdata-example-com.id}"] + subnets = ["${aws_subnet.utility-us-test-1a-bastionuserdata-example-com.id}"] + + health_check = { + target = "SSL:443" + healthy_threshold = 2 + unhealthy_threshold = 2 + interval = 10 + timeout = 5 + } + + idle_timeout = 300 + + tags = { + KubernetesCluster = "bastionuserdata.example.com" + Name = "api.bastionuserdata.example.com" + } +} + +resource "aws_elb" "bastion-bastionuserdata-example-com" { + name = "bastion-bastionuserdata-e-4grhsv" + + listener = { + instance_port = 22 + instance_protocol = "TCP" + lb_port = 22 + lb_protocol = "TCP" + } + + security_groups = ["${aws_security_group.bastion-elb-bastionuserdata-example-com.id}"] + subnets = ["${aws_subnet.utility-us-test-1a-bastionuserdata-example-com.id}"] + + health_check = { + target = "TCP:22" + healthy_threshold = 2 + unhealthy_threshold = 2 + interval = 10 + timeout = 5 + } + + idle_timeout = 300 + + tags = { + KubernetesCluster = "bastionuserdata.example.com" + Name = "bastion.bastionuserdata.example.com" + } +} + +resource "aws_iam_instance_profile" "bastions-bastionuserdata-example-com" { + name = "bastions.bastionuserdata.example.com" + role = "${aws_iam_role.bastions-bastionuserdata-example-com.name}" +} + +resource "aws_iam_instance_profile" "masters-bastionuserdata-example-com" { + name = "masters.bastionuserdata.example.com" + role = "${aws_iam_role.masters-bastionuserdata-example-com.name}" +} + +resource "aws_iam_instance_profile" "nodes-bastionuserdata-example-com" { + name = "nodes.bastionuserdata.example.com" + role = "${aws_iam_role.nodes-bastionuserdata-example-com.name}" +} + +resource "aws_iam_role" "bastions-bastionuserdata-example-com" { + name = "bastions.bastionuserdata.example.com" + assume_role_policy = "${file("${path.module}/data/aws_iam_role_bastions.bastionuserdata.example.com_policy")}" +} + +resource "aws_iam_role" "masters-bastionuserdata-example-com" { + name = "masters.bastionuserdata.example.com" + assume_role_policy = "${file("${path.module}/data/aws_iam_role_masters.bastionuserdata.example.com_policy")}" +} + +resource "aws_iam_role" "nodes-bastionuserdata-example-com" { + name = "nodes.bastionuserdata.example.com" + assume_role_policy = "${file("${path.module}/data/aws_iam_role_nodes.bastionuserdata.example.com_policy")}" +} + +resource "aws_iam_role_policy" "bastions-bastionuserdata-example-com" { + name = "bastions.bastionuserdata.example.com" + role = "${aws_iam_role.bastions-bastionuserdata-example-com.name}" + policy = "${file("${path.module}/data/aws_iam_role_policy_bastions.bastionuserdata.example.com_policy")}" +} + +resource "aws_iam_role_policy" "masters-bastionuserdata-example-com" { + name = "masters.bastionuserdata.example.com" + role = "${aws_iam_role.masters-bastionuserdata-example-com.name}" + policy = "${file("${path.module}/data/aws_iam_role_policy_masters.bastionuserdata.example.com_policy")}" +} + +resource "aws_iam_role_policy" "nodes-bastionuserdata-example-com" { + name = "nodes.bastionuserdata.example.com" + role = "${aws_iam_role.nodes-bastionuserdata-example-com.name}" + policy = "${file("${path.module}/data/aws_iam_role_policy_nodes.bastionuserdata.example.com_policy")}" +} + +resource "aws_internet_gateway" "bastionuserdata-example-com" { + vpc_id = "${aws_vpc.bastionuserdata-example-com.id}" + + tags = { + KubernetesCluster = "bastionuserdata.example.com" + Name = "bastionuserdata.example.com" + } +} + +resource "aws_key_pair" "kubernetes-bastionuserdata-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" { + key_name = "kubernetes.bastionuserdata.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57" + public_key = "${file("${path.module}/data/aws_key_pair_kubernetes.bastionuserdata.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key")}" +} + +resource "aws_launch_configuration" "bastion-bastionuserdata-example-com" { + name_prefix = "bastion.bastionuserdata.example.com-" + image_id = "ami-12345678" + instance_type = "t2.micro" + key_name = "${aws_key_pair.kubernetes-bastionuserdata-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id}" + iam_instance_profile = "${aws_iam_instance_profile.bastions-bastionuserdata-example-com.id}" + security_groups = ["${aws_security_group.bastion-bastionuserdata-example-com.id}"] + associate_public_ip_address = true + user_data = "${file("${path.module}/data/aws_launch_configuration_bastion.bastionuserdata.example.com_user_data")}" + + root_block_device = { + volume_type = "gp2" + volume_size = 32 + delete_on_termination = true + } + + lifecycle = { + create_before_destroy = true + } +} + +resource "aws_launch_configuration" "master-us-test-1a-masters-bastionuserdata-example-com" { + name_prefix = "master-us-test-1a.masters.bastionuserdata.example.com-" + image_id = "ami-12345678" + instance_type = "m3.medium" + key_name = "${aws_key_pair.kubernetes-bastionuserdata-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id}" + iam_instance_profile = "${aws_iam_instance_profile.masters-bastionuserdata-example-com.id}" + security_groups = ["${aws_security_group.masters-bastionuserdata-example-com.id}"] + associate_public_ip_address = false + user_data = "${file("${path.module}/data/aws_launch_configuration_master-us-test-1a.masters.bastionuserdata.example.com_user_data")}" + + root_block_device = { + volume_type = "gp2" + volume_size = 64 + delete_on_termination = true + } + + ephemeral_block_device = { + device_name = "/dev/sdc" + virtual_name = "ephemeral0" + } + + lifecycle = { + create_before_destroy = true + } +} + +resource "aws_launch_configuration" "nodes-bastionuserdata-example-com" { + name_prefix = "nodes.bastionuserdata.example.com-" + image_id = "ami-12345678" + instance_type = "t2.medium" + key_name = "${aws_key_pair.kubernetes-bastionuserdata-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id}" + iam_instance_profile = "${aws_iam_instance_profile.nodes-bastionuserdata-example-com.id}" + security_groups = ["${aws_security_group.nodes-bastionuserdata-example-com.id}"] + associate_public_ip_address = false + user_data = "${file("${path.module}/data/aws_launch_configuration_nodes.bastionuserdata.example.com_user_data")}" + + root_block_device = { + volume_type = "gp2" + volume_size = 128 + delete_on_termination = true + } + + lifecycle = { + create_before_destroy = true + } +} + +resource "aws_nat_gateway" "us-test-1a-bastionuserdata-example-com" { + allocation_id = "${aws_eip.us-test-1a-bastionuserdata-example-com.id}" + subnet_id = "${aws_subnet.utility-us-test-1a-bastionuserdata-example-com.id}" +} + +resource "aws_route" "0-0-0-0--0" { + route_table_id = "${aws_route_table.bastionuserdata-example-com.id}" + destination_cidr_block = "0.0.0.0/0" + gateway_id = "${aws_internet_gateway.bastionuserdata-example-com.id}" +} + +resource "aws_route" "private-us-test-1a-0-0-0-0--0" { + route_table_id = "${aws_route_table.private-us-test-1a-bastionuserdata-example-com.id}" + destination_cidr_block = "0.0.0.0/0" + nat_gateway_id = "${aws_nat_gateway.us-test-1a-bastionuserdata-example-com.id}" +} + +resource "aws_route53_record" "api-bastionuserdata-example-com" { + name = "api.bastionuserdata.example.com" + type = "A" + + alias = { + name = "${aws_elb.api-bastionuserdata-example-com.dns_name}" + zone_id = "${aws_elb.api-bastionuserdata-example-com.zone_id}" + evaluate_target_health = false + } + + zone_id = "/hostedzone/Z1AFAKE1ZON3YO" +} + +resource "aws_route_table" "bastionuserdata-example-com" { + vpc_id = "${aws_vpc.bastionuserdata-example-com.id}" + + tags = { + KubernetesCluster = "bastionuserdata.example.com" + Name = "bastionuserdata.example.com" + } +} + +resource "aws_route_table" "private-us-test-1a-bastionuserdata-example-com" { + vpc_id = "${aws_vpc.bastionuserdata-example-com.id}" + + tags = { + KubernetesCluster = "bastionuserdata.example.com" + Name = "private-us-test-1a.bastionuserdata.example.com" + } +} + +resource "aws_route_table_association" "private-us-test-1a-bastionuserdata-example-com" { + subnet_id = "${aws_subnet.us-test-1a-bastionuserdata-example-com.id}" + route_table_id = "${aws_route_table.private-us-test-1a-bastionuserdata-example-com.id}" +} + +resource "aws_route_table_association" "utility-us-test-1a-bastionuserdata-example-com" { + subnet_id = "${aws_subnet.utility-us-test-1a-bastionuserdata-example-com.id}" + route_table_id = "${aws_route_table.bastionuserdata-example-com.id}" +} + +resource "aws_security_group" "api-elb-bastionuserdata-example-com" { + name = "api-elb.bastionuserdata.example.com" + vpc_id = "${aws_vpc.bastionuserdata-example-com.id}" + description = "Security group for api ELB" + + tags = { + KubernetesCluster = "bastionuserdata.example.com" + Name = "api-elb.bastionuserdata.example.com" + } +} + +resource "aws_security_group" "bastion-bastionuserdata-example-com" { + name = "bastion.bastionuserdata.example.com" + vpc_id = "${aws_vpc.bastionuserdata-example-com.id}" + description = "Security group for bastion" + + tags = { + KubernetesCluster = "bastionuserdata.example.com" + Name = "bastion.bastionuserdata.example.com" + } +} + +resource "aws_security_group" "bastion-elb-bastionuserdata-example-com" { + name = "bastion-elb.bastionuserdata.example.com" + vpc_id = "${aws_vpc.bastionuserdata-example-com.id}" + description = "Security group for bastion ELB" + + tags = { + KubernetesCluster = "bastionuserdata.example.com" + Name = "bastion-elb.bastionuserdata.example.com" + } +} + +resource "aws_security_group" "masters-bastionuserdata-example-com" { + name = "masters.bastionuserdata.example.com" + vpc_id = "${aws_vpc.bastionuserdata-example-com.id}" + description = "Security group for masters" + + tags = { + KubernetesCluster = "bastionuserdata.example.com" + Name = "masters.bastionuserdata.example.com" + } +} + +resource "aws_security_group" "nodes-bastionuserdata-example-com" { + name = "nodes.bastionuserdata.example.com" + vpc_id = "${aws_vpc.bastionuserdata-example-com.id}" + description = "Security group for nodes" + + tags = { + KubernetesCluster = "bastionuserdata.example.com" + Name = "nodes.bastionuserdata.example.com" + } +} + +resource "aws_security_group_rule" "all-master-to-master" { + type = "ingress" + security_group_id = "${aws_security_group.masters-bastionuserdata-example-com.id}" + source_security_group_id = "${aws_security_group.masters-bastionuserdata-example-com.id}" + from_port = 0 + to_port = 0 + protocol = "-1" +} + +resource "aws_security_group_rule" "all-master-to-node" { + type = "ingress" + security_group_id = "${aws_security_group.nodes-bastionuserdata-example-com.id}" + source_security_group_id = "${aws_security_group.masters-bastionuserdata-example-com.id}" + from_port = 0 + to_port = 0 + protocol = "-1" +} + +resource "aws_security_group_rule" "all-node-to-node" { + type = "ingress" + security_group_id = "${aws_security_group.nodes-bastionuserdata-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-bastionuserdata-example-com.id}" + from_port = 0 + to_port = 0 + protocol = "-1" +} + +resource "aws_security_group_rule" "api-elb-egress" { + type = "egress" + security_group_id = "${aws_security_group.api-elb-bastionuserdata-example-com.id}" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "bastion-egress" { + type = "egress" + security_group_id = "${aws_security_group.bastion-bastionuserdata-example-com.id}" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "bastion-elb-egress" { + type = "egress" + security_group_id = "${aws_security_group.bastion-elb-bastionuserdata-example-com.id}" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "bastion-to-master-ssh" { + type = "ingress" + security_group_id = "${aws_security_group.masters-bastionuserdata-example-com.id}" + source_security_group_id = "${aws_security_group.bastion-bastionuserdata-example-com.id}" + from_port = 22 + to_port = 22 + protocol = "tcp" +} + +resource "aws_security_group_rule" "bastion-to-node-ssh" { + type = "ingress" + security_group_id = "${aws_security_group.nodes-bastionuserdata-example-com.id}" + source_security_group_id = "${aws_security_group.bastion-bastionuserdata-example-com.id}" + from_port = 22 + to_port = 22 + protocol = "tcp" +} + +resource "aws_security_group_rule" "https-api-elb-0-0-0-0--0" { + type = "ingress" + security_group_id = "${aws_security_group.api-elb-bastionuserdata-example-com.id}" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "https-elb-to-master" { + type = "ingress" + security_group_id = "${aws_security_group.masters-bastionuserdata-example-com.id}" + source_security_group_id = "${aws_security_group.api-elb-bastionuserdata-example-com.id}" + from_port = 443 + to_port = 443 + protocol = "tcp" +} + +resource "aws_security_group_rule" "master-egress" { + type = "egress" + security_group_id = "${aws_security_group.masters-bastionuserdata-example-com.id}" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "node-egress" { + type = "egress" + security_group_id = "${aws_security_group.nodes-bastionuserdata-example-com.id}" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "node-to-master-protocol-ipip" { + type = "ingress" + security_group_id = "${aws_security_group.masters-bastionuserdata-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-bastionuserdata-example-com.id}" + from_port = 0 + to_port = 65535 + protocol = "4" +} + +resource "aws_security_group_rule" "node-to-master-tcp-1-2379" { + type = "ingress" + security_group_id = "${aws_security_group.masters-bastionuserdata-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-bastionuserdata-example-com.id}" + from_port = 1 + to_port = 2379 + protocol = "tcp" +} + +resource "aws_security_group_rule" "node-to-master-tcp-2382-4001" { + type = "ingress" + security_group_id = "${aws_security_group.masters-bastionuserdata-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-bastionuserdata-example-com.id}" + from_port = 2382 + to_port = 4001 + protocol = "tcp" +} + +resource "aws_security_group_rule" "node-to-master-tcp-4003-65535" { + type = "ingress" + security_group_id = "${aws_security_group.masters-bastionuserdata-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-bastionuserdata-example-com.id}" + from_port = 4003 + to_port = 65535 + protocol = "tcp" +} + +resource "aws_security_group_rule" "node-to-master-udp-1-65535" { + type = "ingress" + security_group_id = "${aws_security_group.masters-bastionuserdata-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-bastionuserdata-example-com.id}" + from_port = 1 + to_port = 65535 + protocol = "udp" +} + +resource "aws_security_group_rule" "ssh-elb-to-bastion" { + type = "ingress" + security_group_id = "${aws_security_group.bastion-bastionuserdata-example-com.id}" + source_security_group_id = "${aws_security_group.bastion-elb-bastionuserdata-example-com.id}" + from_port = 22 + to_port = 22 + protocol = "tcp" +} + +resource "aws_security_group_rule" "ssh-external-to-bastion-elb-0-0-0-0--0" { + type = "ingress" + security_group_id = "${aws_security_group.bastion-elb-bastionuserdata-example-com.id}" + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_subnet" "us-test-1a-bastionuserdata-example-com" { + vpc_id = "${aws_vpc.bastionuserdata-example-com.id}" + cidr_block = "172.20.32.0/19" + availability_zone = "us-test-1a" + + tags = { + KubernetesCluster = "bastionuserdata.example.com" + Name = "us-test-1a.bastionuserdata.example.com" + SubnetType = "Private" + "kubernetes.io/cluster/bastionuserdata.example.com" = "owned" + "kubernetes.io/role/internal-elb" = "1" + } +} + +resource "aws_subnet" "utility-us-test-1a-bastionuserdata-example-com" { + vpc_id = "${aws_vpc.bastionuserdata-example-com.id}" + cidr_block = "172.20.4.0/22" + availability_zone = "us-test-1a" + + tags = { + KubernetesCluster = "bastionuserdata.example.com" + Name = "utility-us-test-1a.bastionuserdata.example.com" + SubnetType = "Utility" + "kubernetes.io/cluster/bastionuserdata.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + } +} + +resource "aws_vpc" "bastionuserdata-example-com" { + cidr_block = "172.20.0.0/16" + enable_dns_hostnames = true + enable_dns_support = true + + tags = { + KubernetesCluster = "bastionuserdata.example.com" + Name = "bastionuserdata.example.com" + "kubernetes.io/cluster/bastionuserdata.example.com" = "owned" + } +} + +resource "aws_vpc_dhcp_options" "bastionuserdata-example-com" { + domain_name = "us-test-1.compute.internal" + domain_name_servers = ["AmazonProvidedDNS"] + + tags = { + KubernetesCluster = "bastionuserdata.example.com" + Name = "bastionuserdata.example.com" + } +} + +resource "aws_vpc_dhcp_options_association" "bastionuserdata-example-com" { + vpc_id = "${aws_vpc.bastionuserdata-example-com.id}" + dhcp_options_id = "${aws_vpc_dhcp_options.bastionuserdata-example-com.id}" +} + +terraform = { + required_version = ">= 0.9.3" +}