diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index cb74cf398e..a3d80df139 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -45,20 +45,20 @@ import ( // TestMinimal runs the test on a minimum configuration, similar to kops create cluster minimal.example.com --zones us-west-1a func TestMinimal(t *testing.T) { - runTest(t, "minimal.example.com", "../../tests/integration/minimal") + runTest(t, "minimal.example.com", "../../tests/integration/minimal", false) } // TestMinimal_141 runs the test on a configuration from 1.4.1 release func TestMinimal_141(t *testing.T) { - runTest(t, "minimal-141.example.com", "../../tests/integration/minimal-141") + runTest(t, "minimal-141.example.com", "../../tests/integration/minimal-141", false) } // TestPrivateWeave runs the test on a configuration with private topology, weave networking func TestPrivateWeave(t *testing.T) { - runTest(t, "privateweave.example.com", "../../tests/integration/privateweave") + runTest(t, "privateweave.example.com", "../../tests/integration/privateweave", true) } -func runTest(t *testing.T, clusterName string, srcDir string) { +func runTest(t *testing.T, clusterName string, srcDir string, private bool) { var stdout bytes.Buffer inputYAML := "in.yaml" @@ -169,6 +169,16 @@ func runTest(t *testing.T, clusterName string, srcDir string) { "aws_launch_configuration_master-us-test-1a.masters." + clusterName + "_user_data", "aws_launch_configuration_nodes." + clusterName + "_user_data", } + + if private { + expectedFilenames = append(expectedFilenames, []string{ + "aws_iam_role_bastions." + clusterName + "_policy", + "aws_iam_role_policy_bastions." + clusterName + "_policy", + + // bastions don't have any userdata + // "aws_launch_configuration_bastions." + clusterName + "_user_data", + }...) + } sort.Strings(expectedFilenames) if !reflect.DeepEqual(actualFilenames, expectedFilenames) { t.Fatalf("unexpected data files. actual=%q, expected=%q", actualFilenames, expectedFilenames) diff --git a/pkg/apis/kops/v1alpha2/upgrade.md b/pkg/apis/kops/v1alpha2/upgrade.md index 2e12fabe2e..5fd3bbaaee 100644 --- a/pkg/apis/kops/v1alpha2/upgrade.md +++ b/pkg/apis/kops/v1alpha2/upgrade.md @@ -6,6 +6,6 @@ Rename your private subnet. It will be called something like `private-us-east-1 Rename your route table It will be called something like `main-cluster.example.com`, rename it to `cluster.example.com` -Create an instance group for the bastions +Create an instance group for the bastions. A name of `bastion` will minimize changes. -`kops create ig bastions --role bastions` +`kops create ig bastion --role bastions` diff --git a/pkg/model/api_loadbalancer.go b/pkg/model/api_loadbalancer.go index db6d8df20f..6e104c9437 100644 --- a/pkg/model/api_loadbalancer.go +++ b/pkg/model/api_loadbalancer.go @@ -81,7 +81,7 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { t := &awstasks.SecurityGroup{ Name: s(b.ELBSecurityGroupName("api")), VPC: b.LinkToVPC(), - Description: s("Security group for ELB in front of API"), + Description: s("Security group for api ELB"), RemoveExtraRules: []string{"port=443"}, } c.AddTask(t) diff --git a/pkg/model/bastion.go b/pkg/model/bastion.go index 531f4baf41..d67435fc49 100644 --- a/pkg/model/bastion.go +++ b/pkg/model/bastion.go @@ -60,7 +60,7 @@ func (b *BastionModelBuilder) Build(c *fi.ModelBuilderContext) error { t := &awstasks.SecurityGroup{ Name: s(b.SecurityGroupName(kops.InstanceGroupRoleBastion)), VPC: b.LinkToVPC(), - Description: s("Security group for bastions"), + Description: s("Security group for bastion"), RemoveExtraRules: []string{"port=22"}, } c.AddTask(t) diff --git a/pkg/model/names.go b/pkg/model/names.go index 51d9ce2f9f..54a7b61c6d 100644 --- a/pkg/model/names.go +++ b/pkg/model/names.go @@ -26,7 +26,7 @@ import ( func (b *KopsModelContext) SecurityGroupName(role kops.InstanceGroupRole) string { switch role { case kops.InstanceGroupRoleBastion: - return "bastions." + b.ClusterName() + return "bastion." + b.ClusterName() case kops.InstanceGroupRoleNode: return "nodes." + b.ClusterName() diff --git a/tests/integration/privateweave/in.yaml b/tests/integration/privateweave/in.yaml index 04be4220c0..03dd6b5d10 100644 --- a/tests/integration/privateweave/in.yaml +++ b/tests/integration/privateweave/in.yaml @@ -82,7 +82,7 @@ apiVersion: kops/v1alpha2 kind: InstanceGroup metadata: creationTimestamp: "2016-12-14T15:32:41Z" - name: bastions + name: bastion labels: kops.k8s.io/cluster: privateweave.example.com spec: diff --git a/tests/integration/privateweave/kubernetes.tf b/tests/integration/privateweave/kubernetes.tf index 096958ac62..5a77c109a2 100644 --- a/tests/integration/privateweave/kubernetes.tf +++ b/tests/integration/privateweave/kubernetes.tf @@ -13,7 +13,7 @@ resource "aws_autoscaling_group" "bastion-privateweave-example-com" { launch_configuration = "${aws_launch_configuration.bastion-privateweave-example-com.id}" max_size = 1 min_size = 1 - vpc_zone_identifier = ["${aws_subnet.private-us-test-1a-privateweave-example-com.id}"] + vpc_zone_identifier = ["${aws_subnet.utility-us-test-1a-privateweave-example-com.id}"] tag = { key = "KubernetesCluster" value = "privateweave.example.com" @@ -24,6 +24,11 @@ resource "aws_autoscaling_group" "bastion-privateweave-example-com" { value = "bastion.privateweave.example.com" propagate_at_launch = true } + tag = { + key = "k8s.io/role/bastion" + value = "1" + propagate_at_launch = true + } } resource "aws_autoscaling_group" "master-us-test-1a-masters-privateweave-example-com" { @@ -31,7 +36,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-privateweave-example launch_configuration = "${aws_launch_configuration.master-us-test-1a-masters-privateweave-example-com.id}" max_size = 1 min_size = 1 - vpc_zone_identifier = ["${aws_subnet.private-us-test-1a-privateweave-example-com.id}"] + vpc_zone_identifier = ["${aws_subnet.us-test-1a-privateweave-example-com.id}"] tag = { key = "KubernetesCluster" value = "privateweave.example.com" @@ -54,7 +59,7 @@ resource "aws_autoscaling_group" "nodes-privateweave-example-com" { launch_configuration = "${aws_launch_configuration.nodes-privateweave-example-com.id}" max_size = 2 min_size = 2 - vpc_zone_identifier = ["${aws_subnet.private-us-test-1a-privateweave-example-com.id}"] + vpc_zone_identifier = ["${aws_subnet.us-test-1a-privateweave-example-com.id}"] tag = { key = "KubernetesCluster" value = "privateweave.example.com" @@ -131,9 +136,21 @@ resource "aws_elb" "bastion-privateweave-example-com" { } security_groups = ["${aws_security_group.bastion-elb-privateweave-example-com.id}"] subnets = ["${aws_subnet.utility-us-test-1a-privateweave-example-com.id}"] + health_check = { + target = "TCP:22" + healthy_threshold = 2 + unhealthy_threshold = 2 + interval = 10 + timeout = 5 + } idle_timeout = 120 } +resource "aws_iam_instance_profile" "bastions-privateweave-example-com" { + name = "bastions.privateweave.example.com" + roles = ["${aws_iam_role.bastions-privateweave-example-com.name}"] +} + resource "aws_iam_instance_profile" "masters-privateweave-example-com" { name = "masters.privateweave.example.com" roles = ["${aws_iam_role.masters-privateweave-example-com.name}"] @@ -144,6 +161,11 @@ resource "aws_iam_instance_profile" "nodes-privateweave-example-com" { roles = ["${aws_iam_role.nodes-privateweave-example-com.name}"] } +resource "aws_iam_role" "bastions-privateweave-example-com" { + name = "bastions.privateweave.example.com" + assume_role_policy = "${file("${path.module}/data/aws_iam_role_bastions.privateweave.example.com_policy")}" +} + resource "aws_iam_role" "masters-privateweave-example-com" { name = "masters.privateweave.example.com" assume_role_policy = "${file("${path.module}/data/aws_iam_role_masters.privateweave.example.com_policy")}" @@ -154,6 +176,12 @@ resource "aws_iam_role" "nodes-privateweave-example-com" { assume_role_policy = "${file("${path.module}/data/aws_iam_role_nodes.privateweave.example.com_policy")}" } +resource "aws_iam_role_policy" "bastions-privateweave-example-com" { + name = "bastions.privateweave.example.com" + role = "${aws_iam_role.bastions-privateweave-example-com.name}" + policy = "${file("${path.module}/data/aws_iam_role_policy_bastions.privateweave.example.com_policy")}" +} + resource "aws_iam_role_policy" "masters-privateweave-example-com" { name = "masters.privateweave.example.com" role = "${aws_iam_role.masters-privateweave-example-com.name}" @@ -182,9 +210,9 @@ resource "aws_key_pair" "kubernetes-privateweave-example-com-c4a6ed9aa889b9e2c39 resource "aws_launch_configuration" "bastion-privateweave-example-com" { name_prefix = "bastion.privateweave.example.com-" image_id = "ami-12345678" - instance_type = "t2.medium" + instance_type = "t2.micro" key_name = "${aws_key_pair.kubernetes-privateweave-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id}" - iam_instance_profile = "${aws_iam_instance_profile.masters-privateweave-example-com.id}" + iam_instance_profile = "${aws_iam_instance_profile.bastions-privateweave-example-com.id}" security_groups = ["${aws_security_group.bastion-privateweave-example-com.id}"] associate_public_ip_address = false root_block_device = { @@ -244,18 +272,18 @@ resource "aws_nat_gateway" "us-test-1a-privateweave-example-com" { subnet_id = "${aws_subnet.utility-us-test-1a-privateweave-example-com.id}" } -resource "aws_route" "private-us-test-1a-privateweave-example-com" { +resource "aws_route" "0-0-0-0--0" { + route_table_id = "${aws_route_table.privateweave-example-com.id}" + destination_cidr_block = "0.0.0.0/0" + gateway_id = "${aws_internet_gateway.privateweave-example-com.id}" +} + +resource "aws_route" "private-us-test-1a-0-0-0-0--0" { route_table_id = "${aws_route_table.private-us-test-1a-privateweave-example-com.id}" destination_cidr_block = "0.0.0.0/0" nat_gateway_id = "${aws_nat_gateway.us-test-1a-privateweave-example-com.id}" } -resource "aws_route" "wan" { - route_table_id = "${aws_route_table.main-privateweave-example-com.id}" - destination_cidr_block = "0.0.0.0/0" - gateway_id = "${aws_internet_gateway.privateweave-example-com.id}" -} - resource "aws_route53_record" "api-privateweave-example-com" { name = "api.privateweave.example.com" type = "A" @@ -267,14 +295,6 @@ resource "aws_route53_record" "api-privateweave-example-com" { zone_id = "/hostedzone/Z1AFAKE1ZON3YO" } -resource "aws_route_table" "main-privateweave-example-com" { - vpc_id = "${aws_vpc.privateweave-example-com.id}" - tags = { - KubernetesCluster = "privateweave.example.com" - Name = "main-privateweave.example.com" - } -} - resource "aws_route_table" "private-us-test-1a-privateweave-example-com" { vpc_id = "${aws_vpc.privateweave-example-com.id}" tags = { @@ -283,16 +303,24 @@ resource "aws_route_table" "private-us-test-1a-privateweave-example-com" { } } -resource "aws_route_table_association" "main-us-test-1a-privateweave-example-com" { - subnet_id = "${aws_subnet.utility-us-test-1a-privateweave-example-com.id}" - route_table_id = "${aws_route_table.main-privateweave-example-com.id}" +resource "aws_route_table" "privateweave-example-com" { + vpc_id = "${aws_vpc.privateweave-example-com.id}" + tags = { + KubernetesCluster = "privateweave.example.com" + Name = "privateweave.example.com" + } } resource "aws_route_table_association" "private-us-test-1a-privateweave-example-com" { - subnet_id = "${aws_subnet.private-us-test-1a-privateweave-example-com.id}" + subnet_id = "${aws_subnet.us-test-1a-privateweave-example-com.id}" route_table_id = "${aws_route_table.private-us-test-1a-privateweave-example-com.id}" } +resource "aws_route_table_association" "utility-us-test-1a-privateweave-example-com" { + subnet_id = "${aws_subnet.utility-us-test-1a-privateweave-example-com.id}" + route_table_id = "${aws_route_table.privateweave-example-com.id}" +} + resource "aws_security_group" "api-elb-privateweave-example-com" { name = "api-elb.privateweave.example.com" vpc_id = "${aws_vpc.privateweave-example-com.id}" @@ -344,6 +372,15 @@ resource "aws_security_group" "nodes-privateweave-example-com" { } resource "aws_security_group_rule" "all-bastion-to-master" { + type = "ingress" + security_group_id = "${aws_security_group.masters-privateweave-example-com.id}" + source_security_group_id = "${aws_security_group.bastion-privateweave-example-com.id}" + from_port = 0 + to_port = 0 + protocol = "-1" +} + +resource "aws_security_group_rule" "all-bastion-to-node" { type = "ingress" security_group_id = "${aws_security_group.nodes-privateweave-example-com.id}" source_security_group_id = "${aws_security_group.bastion-privateweave-example-com.id}" @@ -399,7 +436,7 @@ resource "aws_security_group_rule" "api-elb-egress" { resource "aws_security_group_rule" "bastion-egress" { type = "egress" - security_group_id = "${aws_security_group.nodes-privateweave-example-com.id}" + security_group_id = "${aws_security_group.bastion-privateweave-example-com.id}" from_port = 0 to_port = 0 protocol = "-1" @@ -415,16 +452,7 @@ resource "aws_security_group_rule" "bastion-elb-egress" { cidr_blocks = ["0.0.0.0/0"] } -resource "aws_security_group_rule" "bastion-to-master" { - type = "ingress" - security_group_id = "${aws_security_group.masters-privateweave-example-com.id}" - source_security_group_id = "${aws_security_group.bastion-privateweave-example-com.id}" - from_port = 0 - to_port = 0 - protocol = "-1" -} - -resource "aws_security_group_rule" "https-api-elb" { +resource "aws_security_group_rule" "https-api-elb-0-0-0-0--0" { type = "ingress" security_group_id = "${aws_security_group.api-elb-privateweave-example-com.id}" from_port = 443 @@ -433,7 +461,7 @@ resource "aws_security_group_rule" "https-api-elb" { cidr_blocks = ["0.0.0.0/0"] } -resource "aws_security_group_rule" "kube-proxy-api-elb" { +resource "aws_security_group_rule" "https-elb-to-master" { type = "ingress" security_group_id = "${aws_security_group.masters-privateweave-example-com.id}" source_security_group_id = "${aws_security_group.api-elb-privateweave-example-com.id}" @@ -460,7 +488,7 @@ resource "aws_security_group_rule" "node-egress" { cidr_blocks = ["0.0.0.0/0"] } -resource "aws_security_group_rule" "ssh-external-to-bastion" { +resource "aws_security_group_rule" "ssh-elb-to-bastion" { type = "ingress" security_group_id = "${aws_security_group.bastion-privateweave-example-com.id}" source_security_group_id = "${aws_security_group.bastion-elb-privateweave-example-com.id}" @@ -469,7 +497,7 @@ resource "aws_security_group_rule" "ssh-external-to-bastion" { protocol = "tcp" } -resource "aws_security_group_rule" "ssh-external-to-bastion-elb" { +resource "aws_security_group_rule" "ssh-external-to-bastion-elb-0-0-0-0--0" { type = "ingress" security_group_id = "${aws_security_group.bastion-elb-privateweave-example-com.id}" from_port = 22 @@ -478,13 +506,13 @@ resource "aws_security_group_rule" "ssh-external-to-bastion-elb" { cidr_blocks = ["0.0.0.0/0"] } -resource "aws_subnet" "private-us-test-1a-privateweave-example-com" { +resource "aws_subnet" "us-test-1a-privateweave-example-com" { vpc_id = "${aws_vpc.privateweave-example-com.id}" cidr_block = "172.20.4.0/22" availability_zone = "us-test-1a" tags = { KubernetesCluster = "privateweave.example.com" - Name = "private-us-test-1a.privateweave.example.com" + Name = "us-test-1a.privateweave.example.com" } }