diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index 3781b2d945..18c87f00e5 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -141,6 +141,11 @@ func TestPrivateKopeio(t *testing.T) { runTestAWS(t, "privatekopeio.example.com", "privatekopeio", "v1alpha2", true, 1, true, nil) } +// TestUnmanaged is a test where all the subnets opt-out of route management +func TestUnmanaged(t *testing.T) { + runTestAWS(t, "unmanaged.example.com", "unmanaged", "v1alpha2", true, 1, true, nil) +} + // TestPrivateSharedSubnet runs the test on a configuration with private topology & shared subnets func TestPrivateSharedSubnet(t *testing.T) { runTestAWS(t, "private-shared-subnet.example.com", "private-shared-subnet", "v1alpha2", true, 1, true, nil) diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index b00e39ad21..fea8a8c8ff 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -155,6 +155,18 @@ spec: zone: us-east-1a ``` +In the case that you don't use NAT gateways or internet gateways, you can use the "External" flag for egress to force kops to ignore egress for the subnet. This can be useful when other tools are used to manage egress for the subnet such as virtual private gateways. Please note that your cluster may need to have access to the internet upon creation, so egress must be available upon initializing a cluster. This is intended for use when egress is managed external to kops, typically with an existing cluster. + +``` +spec: + subnets: + - cidr: 10.20.64.0/21 + name: us-east-1a + egress: External + type: Private + zone: us-east-1a +``` + #### publicIP The IP of an existing EIP that you would like to attach to the NAT gateway. diff --git a/docs/run_in_existing_vpc.md b/docs/run_in_existing_vpc.md index a4daacdc4c..e1666633e2 100644 --- a/docs/run_in_existing_vpc.md +++ b/docs/run_in_existing_vpc.md @@ -1,7 +1,8 @@ ## Running in a shared VPC -When launching into a shared VPC, the VPC & the Internet Gateway will be reused. By default we create a new subnet per zone, -and a new route table, but you can also use a shared subnet (see [below](#shared-subnets)). +When launching into a shared VPC, the VPC & the Internet Gateway will be reused. If you are not using an internet gateway + or NAT gateway you can tell _kops_ to ignore egress. By default we create a new subnet per zone, and a new route table, + but you can also use a shared subnet (see [below](#shared-subnets)). 1. Use `kops create cluster` with the `--vpc` argument for your existing VPC: @@ -205,6 +206,33 @@ Please note: * kops won't create a route-table at all if we're not creating subnets. * In the example above the first subnet is using a shared NAT Gateway while the second one is using a shared NAT Instance + +### Externally Managed Egress + +If you are using an unsupported egress configuration in your VPC, _kops_ can be told to ignore egress by using a configuration like: + +```yaml +spec: + subnets: + - cidr: 10.20.64.0/21 + name: us-east-1a + egress: External + type: Private + zone: us-east-1a + - cidr: 10.20.96.0/21 + name: us-east-1b + egress: External + type: Private + zone: us-east-1a + - cidr: 10.20.32.0/21 + name: utility-us-east-1a + type: Utility + zone: us-east-1a + egress: External +``` + +This tells _kops_ that egress is being managed externally. This is preferable when using virtual private gateways +(currently unsupported) or using other configurations to handle egress routing. ### Proxy VPC Egress diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index c2a10c20b7..cd21c64402 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -442,6 +442,9 @@ const ( SubnetTypeUtility SubnetType = "Utility" ) +// EgressExternal means that egress configuration is done externally (preconfigured) +const EgressExternal = "External" + // ClusterSubnetSpec defines a subnet type ClusterSubnetSpec struct { // Name is the name of the subnet diff --git a/pkg/apis/kops/validation/legacy.go b/pkg/apis/kops/validation/legacy.go index 0f6b876a25..f85534ad6b 100644 --- a/pkg/apis/kops/validation/legacy.go +++ b/pkg/apis/kops/validation/legacy.go @@ -563,11 +563,14 @@ func ValidateCluster(c *kops.Cluster, strict bool) *field.Error { // Egress specification support { for i, s := range c.Spec.Subnets { - fieldSubnet := fieldSpec.Child("Subnets").Index(i) - if s.Egress != "" && !strings.HasPrefix(s.Egress, "nat-") && !strings.HasPrefix(s.Egress, "i-") { - return field.Invalid(fieldSubnet.Child("Egress"), s.Egress, "egress must be of type NAT Gateway or NAT EC2 Instance") + if s.Egress == "" { + continue } - if s.Egress != "" && !(s.Type == "Private") { + fieldSubnet := fieldSpec.Child("Subnets").Index(i) + if !strings.HasPrefix(s.Egress, "nat-") && !strings.HasPrefix(s.Egress, "i-") && s.Egress != kops.EgressExternal { + return field.Invalid(fieldSubnet.Child("Egress"), s.Egress, "egress must be of type NAT Gateway or NAT EC2 Instance or 'External'") + } + if s.Egress != kops.EgressExternal && s.Type != "Private" { return field.Invalid(fieldSubnet.Child("Egress"), s.Egress, "egress can only be specified for Private subnets") } } diff --git a/pkg/model/network.go b/pkg/model/network.go index b605fa63a7..d024a34c0f 100644 --- a/pkg/model/network.go +++ b/pkg/model/network.go @@ -40,6 +40,10 @@ type zoneInfo struct { PrivateSubnets []*kops.ClusterSubnetSpec } +func isUnmanaged(subnet *kops.ClusterSubnetSpec) bool { + return subnet.Egress == kops.EgressExternal +} + func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { sharedVPC := b.Cluster.SharedVPC() vpcName := b.ClusterName() @@ -120,6 +124,7 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { // TODO: would be good to create these as shared, to verify them } + allSubnetsUnmanaged := true allSubnetsShared := true allSubnetsSharedInZone := make(map[string]bool) for i := range b.Cluster.Spec.Subnets { @@ -134,11 +139,15 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { allSubnetsShared = false allSubnetsSharedInZone[subnetSpec.Zone] = false } + + if !isUnmanaged(subnetSpec) { + allSubnetsUnmanaged = false + } } // We always have a public route table, though for private networks it is only used for NGWs and ELBs var publicRouteTable *awstasks.RouteTable - { + if !allSubnetsUnmanaged { // The internet gateway is the main entry point to the cluster. igw := &awstasks.InternetGateway{ Name: s(b.ClusterName()), @@ -224,7 +233,7 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { switch subnetSpec.Type { case kops.SubnetTypePublic, kops.SubnetTypeUtility: - if !sharedSubnet { + if !sharedSubnet && !isUnmanaged(subnetSpec) { c.AddTask(&awstasks.RouteTableAssociation{ Name: s(subnetSpec.Name + "." + b.ClusterName()), Lifecycle: b.Lifecycle, @@ -236,7 +245,7 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { case kops.SubnetTypePrivate: // Private subnets get a Network Gateway, and their own route table to associate them with the network gateway - if !sharedSubnet { + if !sharedSubnet && !isUnmanaged(subnetSpec) { // Private Subnet Route Table Associations // // Map the Private subnet to the Private route table @@ -272,6 +281,17 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { egress := info.PrivateSubnets[0].Egress publicIP := info.PrivateSubnets[0].PublicIP + allUnmanaged := true + for _, subnetSpec := range info.PrivateSubnets { + if !isUnmanaged(subnetSpec) { + allUnmanaged = false + } + } + if allUnmanaged { + glog.V(4).Infof("skipping network configuration in zone %s - all subnets unmanaged", zone) + continue + } + // Verify we don't have mixed values for egress/publicIP - the code doesn't handle it for _, subnet := range info.PrivateSubnets { if subnet.Egress != egress { @@ -312,6 +332,8 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { c.AddTask(in) + } else if egress == "External" { + // Nothing to do here } else { return fmt.Errorf("kops currently only supports re-use of either NAT EC2 Instances or NAT Gateways. We will support more eventually! Please see https://github.com/kubernetes/kops/issues/1530") } diff --git a/tests/integration/update_cluster/unmanaged/id_rsa.pub b/tests/integration/update_cluster/unmanaged/id_rsa.pub new file mode 100755 index 0000000000..81cb012783 --- /dev/null +++ b/tests/integration/update_cluster/unmanaged/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/unmanaged/in-v1alpha2.yaml b/tests/integration/update_cluster/unmanaged/in-v1alpha2.yaml new file mode 100644 index 0000000000..e94446dbad --- /dev/null +++ b/tests/integration/update_cluster/unmanaged/in-v1alpha2.yaml @@ -0,0 +1,113 @@ +apiVersion: kops/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2016-12-12T04:13:14Z" + name: unmanaged.example.com +spec: + kubernetesApiAccess: + - 0.0.0.0/0 + channel: stable + cloudProvider: aws + configBase: memfs://clusters.example.com/unmanaged.example.com + etcdClusters: + - etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: main + - etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: events + kubernetesVersion: v1.8.2 + masterInternalName: api.internal.unmanaged.example.com + masterPublicName: api.unmanaged.example.com + networkID: vpc-12345678 + networkCIDR: 172.20.0.0/16 + networking: + weave: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + topology: + masters: private + nodes: private + subnets: + - cidr: 172.20.32.0/19 + egress: External + name: us-test-1a + type: Private + zone: us-test-1a + - cidr: 172.20.64.0/19 + egress: External + name: us-test-1b + type: Private + zone: us-test-1b + - cidr: 172.20.4.0/22 + egress: External + name: utility-us-test-1a + type: Utility + zone: us-test-1a + - cidr: 172.20.8.0/22 + egress: External + name: utility-us-test-1b + type: Utility + zone: us-test-1b + +--- + +apiVersion: kops/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-12T04:13:15Z" + name: master-us-test-1a + labels: + kops.k8s.io/cluster: unmanaged.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.5-debian-jessie-amd64-hvm-ebs-2017-01-09 + machineType: m3.medium + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test-1a + +--- + +apiVersion: kops/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-12T04:13:15Z" + name: nodes + labels: + kops.k8s.io/cluster: unmanaged.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.5-debian-jessie-amd64-hvm-ebs-2017-01-09 + machineType: t2.medium + maxSize: 2 + minSize: 2 + role: Node + subnets: + - us-test-1a + - us-test-1b + + +--- + +apiVersion: kops/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-14T15:32:41Z" + name: bastion + labels: + kops.k8s.io/cluster: unmanaged.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.5-debian-jessie-amd64-hvm-ebs-2017-01-09 + machineType: t2.micro + maxSize: 1 + minSize: 1 + role: Bastion + subnets: + - utility-us-test-1a diff --git a/tests/integration/update_cluster/unmanaged/kubernetes.tf b/tests/integration/update_cluster/unmanaged/kubernetes.tf new file mode 100644 index 0000000000..1ddd23f5d3 --- /dev/null +++ b/tests/integration/update_cluster/unmanaged/kubernetes.tf @@ -0,0 +1,721 @@ +locals = { + bastion_autoscaling_group_ids = ["${aws_autoscaling_group.bastion-unmanaged-example-com.id}"] + bastion_security_group_ids = ["${aws_security_group.bastion-unmanaged-example-com.id}"] + bastions_role_arn = "${aws_iam_role.bastions-unmanaged-example-com.arn}" + bastions_role_name = "${aws_iam_role.bastions-unmanaged-example-com.name}" + cluster_name = "unmanaged.example.com" + master_autoscaling_group_ids = ["${aws_autoscaling_group.master-us-test-1a-masters-unmanaged-example-com.id}"] + master_security_group_ids = ["${aws_security_group.masters-unmanaged-example-com.id}"] + masters_role_arn = "${aws_iam_role.masters-unmanaged-example-com.arn}" + masters_role_name = "${aws_iam_role.masters-unmanaged-example-com.name}" + node_autoscaling_group_ids = ["${aws_autoscaling_group.nodes-unmanaged-example-com.id}"] + node_security_group_ids = ["${aws_security_group.nodes-unmanaged-example-com.id}"] + node_subnet_ids = ["${aws_subnet.us-test-1a-unmanaged-example-com.id}", "${aws_subnet.us-test-1b-unmanaged-example-com.id}"] + nodes_role_arn = "${aws_iam_role.nodes-unmanaged-example-com.arn}" + nodes_role_name = "${aws_iam_role.nodes-unmanaged-example-com.name}" + region = "us-test-1" + subnet_us-test-1a_id = "${aws_subnet.us-test-1a-unmanaged-example-com.id}" + subnet_us-test-1b_id = "${aws_subnet.us-test-1b-unmanaged-example-com.id}" + subnet_utility-us-test-1a_id = "${aws_subnet.utility-us-test-1a-unmanaged-example-com.id}" + subnet_utility-us-test-1b_id = "${aws_subnet.utility-us-test-1b-unmanaged-example-com.id}" + vpc_id = "vpc-12345678" +} + +output "bastion_autoscaling_group_ids" { + value = ["${aws_autoscaling_group.bastion-unmanaged-example-com.id}"] +} + +output "bastion_security_group_ids" { + value = ["${aws_security_group.bastion-unmanaged-example-com.id}"] +} + +output "bastions_role_arn" { + value = "${aws_iam_role.bastions-unmanaged-example-com.arn}" +} + +output "bastions_role_name" { + value = "${aws_iam_role.bastions-unmanaged-example-com.name}" +} + +output "cluster_name" { + value = "unmanaged.example.com" +} + +output "master_autoscaling_group_ids" { + value = ["${aws_autoscaling_group.master-us-test-1a-masters-unmanaged-example-com.id}"] +} + +output "master_security_group_ids" { + value = ["${aws_security_group.masters-unmanaged-example-com.id}"] +} + +output "masters_role_arn" { + value = "${aws_iam_role.masters-unmanaged-example-com.arn}" +} + +output "masters_role_name" { + value = "${aws_iam_role.masters-unmanaged-example-com.name}" +} + +output "node_autoscaling_group_ids" { + value = ["${aws_autoscaling_group.nodes-unmanaged-example-com.id}"] +} + +output "node_security_group_ids" { + value = ["${aws_security_group.nodes-unmanaged-example-com.id}"] +} + +output "node_subnet_ids" { + value = ["${aws_subnet.us-test-1a-unmanaged-example-com.id}", "${aws_subnet.us-test-1b-unmanaged-example-com.id}"] +} + +output "nodes_role_arn" { + value = "${aws_iam_role.nodes-unmanaged-example-com.arn}" +} + +output "nodes_role_name" { + value = "${aws_iam_role.nodes-unmanaged-example-com.name}" +} + +output "region" { + value = "us-test-1" +} + +output "subnet_us-test-1a_id" { + value = "${aws_subnet.us-test-1a-unmanaged-example-com.id}" +} + +output "subnet_us-test-1b_id" { + value = "${aws_subnet.us-test-1b-unmanaged-example-com.id}" +} + +output "subnet_utility-us-test-1a_id" { + value = "${aws_subnet.utility-us-test-1a-unmanaged-example-com.id}" +} + +output "subnet_utility-us-test-1b_id" { + value = "${aws_subnet.utility-us-test-1b-unmanaged-example-com.id}" +} + +output "vpc_id" { + value = "vpc-12345678" +} + +provider "aws" { + region = "us-test-1" +} + +resource "aws_autoscaling_attachment" "bastion-unmanaged-example-com" { + elb = "${aws_elb.bastion-unmanaged-example-com.id}" + autoscaling_group_name = "${aws_autoscaling_group.bastion-unmanaged-example-com.id}" +} + +resource "aws_autoscaling_attachment" "master-us-test-1a-masters-unmanaged-example-com" { + elb = "${aws_elb.api-unmanaged-example-com.id}" + autoscaling_group_name = "${aws_autoscaling_group.master-us-test-1a-masters-unmanaged-example-com.id}" +} + +resource "aws_autoscaling_group" "bastion-unmanaged-example-com" { + name = "bastion.unmanaged.example.com" + launch_configuration = "${aws_launch_configuration.bastion-unmanaged-example-com.id}" + max_size = 1 + min_size = 1 + vpc_zone_identifier = ["${aws_subnet.utility-us-test-1a-unmanaged-example-com.id}"] + + tag = { + key = "KubernetesCluster" + value = "unmanaged.example.com" + propagate_at_launch = true + } + + tag = { + key = "Name" + value = "bastion.unmanaged.example.com" + propagate_at_launch = true + } + + tag = { + key = "k8s.io/role/bastion" + value = "1" + propagate_at_launch = true + } + + metrics_granularity = "1Minute" + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] +} + +resource "aws_autoscaling_group" "master-us-test-1a-masters-unmanaged-example-com" { + name = "master-us-test-1a.masters.unmanaged.example.com" + launch_configuration = "${aws_launch_configuration.master-us-test-1a-masters-unmanaged-example-com.id}" + max_size = 1 + min_size = 1 + vpc_zone_identifier = ["${aws_subnet.us-test-1a-unmanaged-example-com.id}"] + + tag = { + key = "KubernetesCluster" + value = "unmanaged.example.com" + propagate_at_launch = true + } + + tag = { + key = "Name" + value = "master-us-test-1a.masters.unmanaged.example.com" + propagate_at_launch = true + } + + tag = { + key = "k8s.io/role/master" + value = "1" + propagate_at_launch = true + } + + metrics_granularity = "1Minute" + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] +} + +resource "aws_autoscaling_group" "nodes-unmanaged-example-com" { + name = "nodes.unmanaged.example.com" + launch_configuration = "${aws_launch_configuration.nodes-unmanaged-example-com.id}" + max_size = 2 + min_size = 2 + vpc_zone_identifier = ["${aws_subnet.us-test-1a-unmanaged-example-com.id}", "${aws_subnet.us-test-1b-unmanaged-example-com.id}"] + + tag = { + key = "KubernetesCluster" + value = "unmanaged.example.com" + propagate_at_launch = true + } + + tag = { + key = "Name" + value = "nodes.unmanaged.example.com" + propagate_at_launch = true + } + + tag = { + key = "k8s.io/role/node" + value = "1" + propagate_at_launch = true + } + + metrics_granularity = "1Minute" + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] +} + +resource "aws_ebs_volume" "us-test-1a-etcd-events-unmanaged-example-com" { + availability_zone = "us-test-1a" + size = 20 + type = "gp2" + encrypted = false + + tags = { + KubernetesCluster = "unmanaged.example.com" + Name = "us-test-1a.etcd-events.unmanaged.example.com" + "k8s.io/etcd/events" = "us-test-1a/us-test-1a" + "k8s.io/role/master" = "1" + "kubernetes.io/cluster/unmanaged.example.com" = "owned" + } +} + +resource "aws_ebs_volume" "us-test-1a-etcd-main-unmanaged-example-com" { + availability_zone = "us-test-1a" + size = 20 + type = "gp2" + encrypted = false + + tags = { + KubernetesCluster = "unmanaged.example.com" + Name = "us-test-1a.etcd-main.unmanaged.example.com" + "k8s.io/etcd/main" = "us-test-1a/us-test-1a" + "k8s.io/role/master" = "1" + "kubernetes.io/cluster/unmanaged.example.com" = "owned" + } +} + +resource "aws_elb" "api-unmanaged-example-com" { + name = "api-unmanaged-example-com-t82m6f" + + listener = { + instance_port = 443 + instance_protocol = "TCP" + lb_port = 443 + lb_protocol = "TCP" + } + + security_groups = ["${aws_security_group.api-elb-unmanaged-example-com.id}"] + subnets = ["${aws_subnet.utility-us-test-1a-unmanaged-example-com.id}", "${aws_subnet.utility-us-test-1b-unmanaged-example-com.id}"] + + health_check = { + target = "SSL:443" + healthy_threshold = 2 + unhealthy_threshold = 2 + interval = 10 + timeout = 5 + } + + idle_timeout = 300 + + tags = { + KubernetesCluster = "unmanaged.example.com" + Name = "api.unmanaged.example.com" + } +} + +resource "aws_elb" "bastion-unmanaged-example-com" { + name = "bastion-unmanaged-example-d7bn3d" + + listener = { + instance_port = 22 + instance_protocol = "TCP" + lb_port = 22 + lb_protocol = "TCP" + } + + security_groups = ["${aws_security_group.bastion-elb-unmanaged-example-com.id}"] + subnets = ["${aws_subnet.utility-us-test-1a-unmanaged-example-com.id}"] + + health_check = { + target = "TCP:22" + healthy_threshold = 2 + unhealthy_threshold = 2 + interval = 10 + timeout = 5 + } + + idle_timeout = 300 + + tags = { + KubernetesCluster = "unmanaged.example.com" + Name = "bastion.unmanaged.example.com" + } +} + +resource "aws_iam_instance_profile" "bastions-unmanaged-example-com" { + name = "bastions.unmanaged.example.com" + role = "${aws_iam_role.bastions-unmanaged-example-com.name}" +} + +resource "aws_iam_instance_profile" "masters-unmanaged-example-com" { + name = "masters.unmanaged.example.com" + role = "${aws_iam_role.masters-unmanaged-example-com.name}" +} + +resource "aws_iam_instance_profile" "nodes-unmanaged-example-com" { + name = "nodes.unmanaged.example.com" + role = "${aws_iam_role.nodes-unmanaged-example-com.name}" +} + +resource "aws_iam_role" "bastions-unmanaged-example-com" { + name = "bastions.unmanaged.example.com" + assume_role_policy = "${file("${path.module}/data/aws_iam_role_bastions.unmanaged.example.com_policy")}" +} + +resource "aws_iam_role" "masters-unmanaged-example-com" { + name = "masters.unmanaged.example.com" + assume_role_policy = "${file("${path.module}/data/aws_iam_role_masters.unmanaged.example.com_policy")}" +} + +resource "aws_iam_role" "nodes-unmanaged-example-com" { + name = "nodes.unmanaged.example.com" + assume_role_policy = "${file("${path.module}/data/aws_iam_role_nodes.unmanaged.example.com_policy")}" +} + +resource "aws_iam_role_policy" "bastions-unmanaged-example-com" { + name = "bastions.unmanaged.example.com" + role = "${aws_iam_role.bastions-unmanaged-example-com.name}" + policy = "${file("${path.module}/data/aws_iam_role_policy_bastions.unmanaged.example.com_policy")}" +} + +resource "aws_iam_role_policy" "masters-unmanaged-example-com" { + name = "masters.unmanaged.example.com" + role = "${aws_iam_role.masters-unmanaged-example-com.name}" + policy = "${file("${path.module}/data/aws_iam_role_policy_masters.unmanaged.example.com_policy")}" +} + +resource "aws_iam_role_policy" "nodes-unmanaged-example-com" { + name = "nodes.unmanaged.example.com" + role = "${aws_iam_role.nodes-unmanaged-example-com.name}" + policy = "${file("${path.module}/data/aws_iam_role_policy_nodes.unmanaged.example.com_policy")}" +} + +resource "aws_key_pair" "kubernetes-unmanaged-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" { + key_name = "kubernetes.unmanaged.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57" + public_key = "${file("${path.module}/data/aws_key_pair_kubernetes.unmanaged.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key")}" +} + +resource "aws_launch_configuration" "bastion-unmanaged-example-com" { + name_prefix = "bastion.unmanaged.example.com-" + image_id = "ami-15000000" + instance_type = "t2.micro" + key_name = "${aws_key_pair.kubernetes-unmanaged-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id}" + iam_instance_profile = "${aws_iam_instance_profile.bastions-unmanaged-example-com.id}" + security_groups = ["${aws_security_group.bastion-unmanaged-example-com.id}"] + associate_public_ip_address = true + + root_block_device = { + volume_type = "gp2" + volume_size = 32 + delete_on_termination = true + } + + lifecycle = { + create_before_destroy = true + } + + enable_monitoring = false +} + +resource "aws_launch_configuration" "master-us-test-1a-masters-unmanaged-example-com" { + name_prefix = "master-us-test-1a.masters.unmanaged.example.com-" + image_id = "ami-15000000" + instance_type = "m3.medium" + key_name = "${aws_key_pair.kubernetes-unmanaged-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id}" + iam_instance_profile = "${aws_iam_instance_profile.masters-unmanaged-example-com.id}" + security_groups = ["${aws_security_group.masters-unmanaged-example-com.id}"] + associate_public_ip_address = false + user_data = "${file("${path.module}/data/aws_launch_configuration_master-us-test-1a.masters.unmanaged.example.com_user_data")}" + + root_block_device = { + volume_type = "gp2" + volume_size = 64 + delete_on_termination = true + } + + ephemeral_block_device = { + device_name = "/dev/sdc" + virtual_name = "ephemeral0" + } + + lifecycle = { + create_before_destroy = true + } + + enable_monitoring = false +} + +resource "aws_launch_configuration" "nodes-unmanaged-example-com" { + name_prefix = "nodes.unmanaged.example.com-" + image_id = "ami-15000000" + instance_type = "t2.medium" + key_name = "${aws_key_pair.kubernetes-unmanaged-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id}" + iam_instance_profile = "${aws_iam_instance_profile.nodes-unmanaged-example-com.id}" + security_groups = ["${aws_security_group.nodes-unmanaged-example-com.id}"] + associate_public_ip_address = false + user_data = "${file("${path.module}/data/aws_launch_configuration_nodes.unmanaged.example.com_user_data")}" + + root_block_device = { + volume_type = "gp2" + volume_size = 128 + delete_on_termination = true + } + + lifecycle = { + create_before_destroy = true + } + + enable_monitoring = false +} + +resource "aws_route53_record" "api-unmanaged-example-com" { + name = "api.unmanaged.example.com" + type = "A" + + alias = { + name = "${aws_elb.api-unmanaged-example-com.dns_name}" + zone_id = "${aws_elb.api-unmanaged-example-com.zone_id}" + evaluate_target_health = false + } + + zone_id = "/hostedzone/Z1AFAKE1ZON3YO" +} + +resource "aws_security_group" "api-elb-unmanaged-example-com" { + name = "api-elb.unmanaged.example.com" + vpc_id = "vpc-12345678" + description = "Security group for api ELB" + + tags = { + KubernetesCluster = "unmanaged.example.com" + Name = "api-elb.unmanaged.example.com" + "kubernetes.io/cluster/unmanaged.example.com" = "owned" + } +} + +resource "aws_security_group" "bastion-elb-unmanaged-example-com" { + name = "bastion-elb.unmanaged.example.com" + vpc_id = "vpc-12345678" + description = "Security group for bastion ELB" + + tags = { + KubernetesCluster = "unmanaged.example.com" + Name = "bastion-elb.unmanaged.example.com" + "kubernetes.io/cluster/unmanaged.example.com" = "owned" + } +} + +resource "aws_security_group" "bastion-unmanaged-example-com" { + name = "bastion.unmanaged.example.com" + vpc_id = "vpc-12345678" + description = "Security group for bastion" + + tags = { + KubernetesCluster = "unmanaged.example.com" + Name = "bastion.unmanaged.example.com" + "kubernetes.io/cluster/unmanaged.example.com" = "owned" + } +} + +resource "aws_security_group" "masters-unmanaged-example-com" { + name = "masters.unmanaged.example.com" + vpc_id = "vpc-12345678" + description = "Security group for masters" + + tags = { + KubernetesCluster = "unmanaged.example.com" + Name = "masters.unmanaged.example.com" + "kubernetes.io/cluster/unmanaged.example.com" = "owned" + } +} + +resource "aws_security_group" "nodes-unmanaged-example-com" { + name = "nodes.unmanaged.example.com" + vpc_id = "vpc-12345678" + description = "Security group for nodes" + + tags = { + KubernetesCluster = "unmanaged.example.com" + Name = "nodes.unmanaged.example.com" + "kubernetes.io/cluster/unmanaged.example.com" = "owned" + } +} + +resource "aws_security_group_rule" "all-master-to-master" { + type = "ingress" + security_group_id = "${aws_security_group.masters-unmanaged-example-com.id}" + source_security_group_id = "${aws_security_group.masters-unmanaged-example-com.id}" + from_port = 0 + to_port = 0 + protocol = "-1" +} + +resource "aws_security_group_rule" "all-master-to-node" { + type = "ingress" + security_group_id = "${aws_security_group.nodes-unmanaged-example-com.id}" + source_security_group_id = "${aws_security_group.masters-unmanaged-example-com.id}" + from_port = 0 + to_port = 0 + protocol = "-1" +} + +resource "aws_security_group_rule" "all-node-to-node" { + type = "ingress" + security_group_id = "${aws_security_group.nodes-unmanaged-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-unmanaged-example-com.id}" + from_port = 0 + to_port = 0 + protocol = "-1" +} + +resource "aws_security_group_rule" "api-elb-egress" { + type = "egress" + security_group_id = "${aws_security_group.api-elb-unmanaged-example-com.id}" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "bastion-egress" { + type = "egress" + security_group_id = "${aws_security_group.bastion-unmanaged-example-com.id}" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "bastion-elb-egress" { + type = "egress" + security_group_id = "${aws_security_group.bastion-elb-unmanaged-example-com.id}" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "bastion-to-master-ssh" { + type = "ingress" + security_group_id = "${aws_security_group.masters-unmanaged-example-com.id}" + source_security_group_id = "${aws_security_group.bastion-unmanaged-example-com.id}" + from_port = 22 + to_port = 22 + protocol = "tcp" +} + +resource "aws_security_group_rule" "bastion-to-node-ssh" { + type = "ingress" + security_group_id = "${aws_security_group.nodes-unmanaged-example-com.id}" + source_security_group_id = "${aws_security_group.bastion-unmanaged-example-com.id}" + from_port = 22 + to_port = 22 + protocol = "tcp" +} + +resource "aws_security_group_rule" "https-api-elb-0-0-0-0--0" { + type = "ingress" + security_group_id = "${aws_security_group.api-elb-unmanaged-example-com.id}" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "https-elb-to-master" { + type = "ingress" + security_group_id = "${aws_security_group.masters-unmanaged-example-com.id}" + source_security_group_id = "${aws_security_group.api-elb-unmanaged-example-com.id}" + from_port = 443 + to_port = 443 + protocol = "tcp" +} + +resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" { + type = "ingress" + security_group_id = "${aws_security_group.api-elb-unmanaged-example-com.id}" + from_port = 3 + to_port = 4 + protocol = "icmp" + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "master-egress" { + type = "egress" + security_group_id = "${aws_security_group.masters-unmanaged-example-com.id}" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "node-egress" { + type = "egress" + security_group_id = "${aws_security_group.nodes-unmanaged-example-com.id}" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "node-to-master-tcp-1-2379" { + type = "ingress" + security_group_id = "${aws_security_group.masters-unmanaged-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-unmanaged-example-com.id}" + from_port = 1 + to_port = 2379 + protocol = "tcp" +} + +resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" { + type = "ingress" + security_group_id = "${aws_security_group.masters-unmanaged-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-unmanaged-example-com.id}" + from_port = 2382 + to_port = 4000 + protocol = "tcp" +} + +resource "aws_security_group_rule" "node-to-master-tcp-4003-65535" { + type = "ingress" + security_group_id = "${aws_security_group.masters-unmanaged-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-unmanaged-example-com.id}" + from_port = 4003 + to_port = 65535 + protocol = "tcp" +} + +resource "aws_security_group_rule" "node-to-master-udp-1-65535" { + type = "ingress" + security_group_id = "${aws_security_group.masters-unmanaged-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-unmanaged-example-com.id}" + from_port = 1 + to_port = 65535 + protocol = "udp" +} + +resource "aws_security_group_rule" "ssh-elb-to-bastion" { + type = "ingress" + security_group_id = "${aws_security_group.bastion-unmanaged-example-com.id}" + source_security_group_id = "${aws_security_group.bastion-elb-unmanaged-example-com.id}" + from_port = 22 + to_port = 22 + protocol = "tcp" +} + +resource "aws_security_group_rule" "ssh-external-to-bastion-elb-0-0-0-0--0" { + type = "ingress" + security_group_id = "${aws_security_group.bastion-elb-unmanaged-example-com.id}" + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_subnet" "us-test-1a-unmanaged-example-com" { + vpc_id = "vpc-12345678" + cidr_block = "172.20.32.0/19" + availability_zone = "us-test-1a" + + tags = { + KubernetesCluster = "unmanaged.example.com" + Name = "us-test-1a.unmanaged.example.com" + SubnetType = "Private" + "kubernetes.io/cluster/unmanaged.example.com" = "owned" + "kubernetes.io/role/internal-elb" = "1" + } +} + +resource "aws_subnet" "us-test-1b-unmanaged-example-com" { + vpc_id = "vpc-12345678" + cidr_block = "172.20.64.0/19" + availability_zone = "us-test-1b" + + tags = { + KubernetesCluster = "unmanaged.example.com" + Name = "us-test-1b.unmanaged.example.com" + SubnetType = "Private" + "kubernetes.io/cluster/unmanaged.example.com" = "owned" + "kubernetes.io/role/internal-elb" = "1" + } +} + +resource "aws_subnet" "utility-us-test-1a-unmanaged-example-com" { + vpc_id = "vpc-12345678" + cidr_block = "172.20.4.0/22" + availability_zone = "us-test-1a" + + tags = { + KubernetesCluster = "unmanaged.example.com" + Name = "utility-us-test-1a.unmanaged.example.com" + SubnetType = "Utility" + "kubernetes.io/cluster/unmanaged.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + } +} + +resource "aws_subnet" "utility-us-test-1b-unmanaged-example-com" { + vpc_id = "vpc-12345678" + cidr_block = "172.20.8.0/22" + availability_zone = "us-test-1b" + + tags = { + KubernetesCluster = "unmanaged.example.com" + Name = "utility-us-test-1b.unmanaged.example.com" + SubnetType = "Utility" + "kubernetes.io/cluster/unmanaged.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + } +} + +terraform = { + required_version = ">= 0.9.3" +}