diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index 49a482ca9e..1226d061e8 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -23,6 +23,18 @@ spec: When configuring a LoadBalancer, you can also choose to have a public ELB or an internal (VPC only) ELB. The `type` field should be `Public` or `Internal`. +Also, you can add precreated additional security groups to the load balancer by setting `additionalSecurityGroups`. + +```yaml +spec: + api: + loadBalancer: + type: Public + additionalSecurityGroups: + - sg-xxxxxxxx + - sg-xxxxxxxx +``` + Additionally, you can increase idle timeout of the load balancer by setting its `idleTimeoutSeconds`. The default idle timeout is 5 minutes, with a maximum of 3600 seconds (60 minutes) being allowed by AWS. For more information see [configuring idle timeouts](http://docs.aws.amazon.com/elasticloadbalancing/latest/classic/config-idle-timeout.html). diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index cf0e3b3903..531b41b7c1 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -267,9 +267,11 @@ const ( LoadBalancerTypeInternal LoadBalancerType = "Internal" ) +// LoadBalancerAccessSpec provides configuration details related to API LoadBalancer and its access type LoadBalancerAccessSpec struct { - Type LoadBalancerType `json:"type,omitempty"` - IdleTimeoutSeconds *int64 `json:"idleTimeoutSeconds,omitempty"` + Type LoadBalancerType `json:"type,omitempty"` + IdleTimeoutSeconds *int64 `json:"idleTimeoutSeconds,omitempty"` + AdditionalSecurityGroups []string `json:"additionalSecurityGroups,omitempty"` } // KubeDNSConfig defines the kube dns configuration diff --git a/pkg/apis/kops/v1alpha1/cluster.go b/pkg/apis/kops/v1alpha1/cluster.go index a253490315..8ae40130f1 100644 --- a/pkg/apis/kops/v1alpha1/cluster.go +++ b/pkg/apis/kops/v1alpha1/cluster.go @@ -266,9 +266,11 @@ const ( LoadBalancerTypeInternal LoadBalancerType = "Internal" ) +// LoadBalancerAccessSpec provides configuration details related to API LoadBalancer and its access type LoadBalancerAccessSpec struct { - Type LoadBalancerType `json:"type,omitempty"` - IdleTimeoutSeconds *int64 `json:"idleTimeoutSeconds,omitempty"` + Type LoadBalancerType `json:"type,omitempty"` + IdleTimeoutSeconds *int64 `json:"idleTimeoutSeconds,omitempty"` + AdditionalSecurityGroups []string `json:"additionalSecurityGroups,omitempty"` } // KubeDNSConfig defines the kube dns configuration diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index d4fc92ee48..5504521f15 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -2283,6 +2283,7 @@ func Convert_kops_LeaderElectionConfiguration_To_v1alpha1_LeaderElectionConfigur func autoConvert_v1alpha1_LoadBalancerAccessSpec_To_kops_LoadBalancerAccessSpec(in *LoadBalancerAccessSpec, out *kops.LoadBalancerAccessSpec, s conversion.Scope) error { out.Type = kops.LoadBalancerType(in.Type) out.IdleTimeoutSeconds = in.IdleTimeoutSeconds + out.AdditionalSecurityGroups = in.AdditionalSecurityGroups return nil } @@ -2294,6 +2295,7 @@ func Convert_v1alpha1_LoadBalancerAccessSpec_To_kops_LoadBalancerAccessSpec(in * func autoConvert_kops_LoadBalancerAccessSpec_To_v1alpha1_LoadBalancerAccessSpec(in *kops.LoadBalancerAccessSpec, out *LoadBalancerAccessSpec, s conversion.Scope) error { out.Type = LoadBalancerType(in.Type) out.IdleTimeoutSeconds = in.IdleTimeoutSeconds + out.AdditionalSecurityGroups = in.AdditionalSecurityGroups return nil } diff --git a/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go index ee2355bad8..cf2eb7811b 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go @@ -2622,6 +2622,11 @@ func (in *LoadBalancerAccessSpec) DeepCopyInto(out *LoadBalancerAccessSpec) { **out = **in } } + if in.AdditionalSecurityGroups != nil { + in, out := &in.AdditionalSecurityGroups, &out.AdditionalSecurityGroups + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index ae8862aa56..d4bab81b22 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -267,9 +267,11 @@ const ( LoadBalancerTypeInternal LoadBalancerType = "Internal" ) +// LoadBalancerAccessSpec provides configuration details related to API LoadBalancer and its access type LoadBalancerAccessSpec struct { - Type LoadBalancerType `json:"type,omitempty"` - IdleTimeoutSeconds *int64 `json:"idleTimeoutSeconds,omitempty"` + Type LoadBalancerType `json:"type,omitempty"` + IdleTimeoutSeconds *int64 `json:"idleTimeoutSeconds,omitempty"` + AdditionalSecurityGroups []string `json:"additionalSecurityGroups,omitempty"` } type KubeDNSConfig struct { diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 797a9f633e..8b4d99d4e1 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -2545,6 +2545,7 @@ func Convert_kops_LeaderElectionConfiguration_To_v1alpha2_LeaderElectionConfigur func autoConvert_v1alpha2_LoadBalancerAccessSpec_To_kops_LoadBalancerAccessSpec(in *LoadBalancerAccessSpec, out *kops.LoadBalancerAccessSpec, s conversion.Scope) error { out.Type = kops.LoadBalancerType(in.Type) out.IdleTimeoutSeconds = in.IdleTimeoutSeconds + out.AdditionalSecurityGroups = in.AdditionalSecurityGroups return nil } @@ -2556,6 +2557,7 @@ func Convert_v1alpha2_LoadBalancerAccessSpec_To_kops_LoadBalancerAccessSpec(in * func autoConvert_kops_LoadBalancerAccessSpec_To_v1alpha2_LoadBalancerAccessSpec(in *kops.LoadBalancerAccessSpec, out *LoadBalancerAccessSpec, s conversion.Scope) error { out.Type = LoadBalancerType(in.Type) out.IdleTimeoutSeconds = in.IdleTimeoutSeconds + out.AdditionalSecurityGroups = in.AdditionalSecurityGroups return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index cf397562a7..e2eb0f5412 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -2748,6 +2748,11 @@ func (in *LoadBalancerAccessSpec) DeepCopyInto(out *LoadBalancerAccessSpec) { **out = **in } } + if in.AdditionalSecurityGroups != nil { + in, out := &in.AdditionalSecurityGroups, &out.AdditionalSecurityGroups + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/apis/kops/validation/aws.go b/pkg/apis/kops/validation/aws.go index 517582f292..90eb38f6f5 100644 --- a/pkg/apis/kops/validation/aws.go +++ b/pkg/apis/kops/validation/aws.go @@ -19,13 +19,22 @@ package validation import ( "strings" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" ) func awsValidateCluster(c *kops.Cluster) field.ErrorList { - return nil + allErrs := field.ErrorList{} + + if c.Spec.API != nil { + if c.Spec.API.LoadBalancer != nil { + allErrs = append(allErrs, awsValidateAdditionalSecurityGroups(field.NewPath("spec", "api", "loadBalancer", "additionalSecurityGroups"), c.Spec.API.LoadBalancer.AdditionalSecurityGroups)...) + } + } + + return allErrs } func awsValidateInstanceGroup(ig *kops.InstanceGroup) field.ErrorList { @@ -41,7 +50,12 @@ func awsValidateInstanceGroup(ig *kops.InstanceGroup) field.ErrorList { func awsValidateAdditionalSecurityGroups(fieldPath *field.Path, groups []string) field.ErrorList { allErrs := field.ErrorList{} + names := sets.NewString() for i, s := range groups { + if names.Has(s) { + allErrs = append(allErrs, field.Invalid(fieldPath.Index(i), s, "security groups with duplicate name found")) + } + names.Insert(s) if strings.TrimSpace(s) == "" { allErrs = append(allErrs, field.Invalid(fieldPath.Index(i), s, "security group cannot be empty, if specified")) continue diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 44466b322f..2e0b0364cc 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -2983,6 +2983,11 @@ func (in *LoadBalancerAccessSpec) DeepCopyInto(out *LoadBalancerAccessSpec) { **out = **in } } + if in.AdditionalSecurityGroups != nil { + in, out := &in.AdditionalSecurityGroups, &out.AdditionalSecurityGroups + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/model/awsmodel/api_loadbalancer.go b/pkg/model/awsmodel/api_loadbalancer.go index 3490ca35c0..d0e5e5d39f 100644 --- a/pkg/model/awsmodel/api_loadbalancer.go +++ b/pkg/model/awsmodel/api_loadbalancer.go @@ -184,6 +184,21 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { } } + // Add precreated additional security groups to the ELB + { + for _, id := range b.Cluster.Spec.API.LoadBalancer.AdditionalSecurityGroups { + t := &awstasks.SecurityGroup{ + Name: fi.String(id), + ID: fi.String(id), + Shared: fi.Bool(true), + } + if err := c.EnsureTask(t); err != nil { + return err + } + elb.SecurityGroups = append(elb.SecurityGroups, t) + } + } + // Allow HTTPS to the master instances from the ELB { t := &awstasks.SecurityGroupRule{ diff --git a/tests/integration/update_cluster/complex/in-v1alpha2.yaml b/tests/integration/update_cluster/complex/in-v1alpha2.yaml index 62b9c5bc26..0b8d201667 100644 --- a/tests/integration/update_cluster/complex/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/complex/in-v1alpha2.yaml @@ -4,6 +4,12 @@ metadata: creationTimestamp: "2016-12-10T22:42:27Z" name: complex.example.com spec: + api: + loadBalancer: + type: Public + additionalSecurityGroups: + - sg-exampleid3 + - sg-exampleid4 kubernetesApiAccess: - 0.0.0.0/0 channel: stable @@ -84,5 +90,3 @@ spec: role: Master subnets: - us-test-1a - - diff --git a/tests/integration/update_cluster/complex/kubernetes.tf b/tests/integration/update_cluster/complex/kubernetes.tf index cc77b45e4a..e8613f032e 100644 --- a/tests/integration/update_cluster/complex/kubernetes.tf +++ b/tests/integration/update_cluster/complex/kubernetes.tf @@ -42,6 +42,11 @@ provider "aws" { region = "us-test-1" } +resource "aws_autoscaling_attachment" "master-us-test-1a-masters-complex-example-com" { + elb = "${aws_elb.api-complex-example-com.id}" + autoscaling_group_name = "${aws_autoscaling_group.master-us-test-1a-masters-complex-example-com.id}" +} + resource "aws_autoscaling_group" "master-us-test-1a-masters-complex-example-com" { name = "master-us-test-1a.masters.complex.example.com" launch_configuration = "${aws_launch_configuration.master-us-test-1a-masters-complex-example-com.id}" @@ -150,6 +155,35 @@ resource "aws_ebs_volume" "us-test-1a-etcd-main-complex-example-com" { } } +resource "aws_elb" "api-complex-example-com" { + name = "api-complex-example-com-vd3t5n" + + listener = { + instance_port = 443 + instance_protocol = "TCP" + lb_port = 443 + lb_protocol = "TCP" + } + + security_groups = ["${aws_security_group.api-elb-complex-example-com.id}", "sg-exampleid3", "sg-exampleid4"] + subnets = ["${aws_subnet.us-test-1a-complex-example-com.id}"] + + health_check = { + target = "SSL:443" + healthy_threshold = 2 + unhealthy_threshold = 2 + interval = 10 + timeout = 5 + } + + idle_timeout = 300 + + tags = { + KubernetesCluster = "complex.example.com" + Name = "api.complex.example.com" + } +} + resource "aws_iam_instance_profile" "masters-complex-example-com" { name = "masters.complex.example.com" role = "${aws_iam_role.masters-complex-example-com.name}" @@ -249,6 +283,19 @@ resource "aws_route" "0-0-0-0--0" { gateway_id = "${aws_internet_gateway.complex-example-com.id}" } +resource "aws_route53_record" "api-complex-example-com" { + name = "api.complex.example.com" + type = "A" + + alias = { + name = "${aws_elb.api-complex-example-com.dns_name}" + zone_id = "${aws_elb.api-complex-example-com.zone_id}" + evaluate_target_health = false + } + + zone_id = "/hostedzone/Z1AFAKE1ZON3YO" +} + resource "aws_route_table" "complex-example-com" { vpc_id = "${aws_vpc.complex-example-com.id}" @@ -263,6 +310,17 @@ resource "aws_route_table_association" "us-test-1a-complex-example-com" { route_table_id = "${aws_route_table.complex-example-com.id}" } +resource "aws_security_group" "api-elb-complex-example-com" { + name = "api-elb.complex.example.com" + vpc_id = "${aws_vpc.complex-example-com.id}" + description = "Security group for api ELB" + + tags = { + KubernetesCluster = "complex.example.com" + Name = "api-elb.complex.example.com" + } +} + resource "aws_security_group" "masters-complex-example-com" { name = "masters.complex.example.com" vpc_id = "${aws_vpc.complex-example-com.id}" @@ -312,15 +370,33 @@ resource "aws_security_group_rule" "all-node-to-node" { protocol = "-1" } -resource "aws_security_group_rule" "https-external-to-master-0-0-0-0--0" { +resource "aws_security_group_rule" "api-elb-egress" { + type = "egress" + security_group_id = "${aws_security_group.api-elb-complex-example-com.id}" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "https-api-elb-0-0-0-0--0" { type = "ingress" - security_group_id = "${aws_security_group.masters-complex-example-com.id}" + security_group_id = "${aws_security_group.api-elb-complex-example-com.id}" from_port = 443 to_port = 443 protocol = "tcp" cidr_blocks = ["0.0.0.0/0"] } +resource "aws_security_group_rule" "https-elb-to-master" { + type = "ingress" + security_group_id = "${aws_security_group.masters-complex-example-com.id}" + source_security_group_id = "${aws_security_group.api-elb-complex-example-com.id}" + from_port = 443 + to_port = 443 + protocol = "tcp" +} + resource "aws_security_group_rule" "master-egress" { type = "egress" security_group_id = "${aws_security_group.masters-complex-example-com.id}"