Merge pull request #10157 from rifelpet/acm-nlb

Setup a second NLB listener when an AWS ACM certificate is used
This commit is contained in:
Kubernetes Prow Robot 2020-11-10 10:36:41 -08:00 committed by GitHub
commit e43efbe102
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 429 additions and 105 deletions

View File

@ -217,6 +217,12 @@ func validateClusterSpec(spec *kops.ClusterSpec, c *kops.Cluster, fieldPath *fie
if featureflag.Spotinst.Enabled() && spec.API.LoadBalancer.Class == kops.LoadBalancerClassNetwork { if featureflag.Spotinst.Enabled() && spec.API.LoadBalancer.Class == kops.LoadBalancerClassNetwork {
allErrs = append(allErrs, field.Forbidden(fieldPath, "cannot use NLB together with spotinst")) allErrs = append(allErrs, field.Forbidden(fieldPath, "cannot use NLB together with spotinst"))
} }
if spec.API.LoadBalancer.SSLCertificate != "" && spec.API.LoadBalancer.Class != kops.LoadBalancerClassNetwork && c.IsKubernetesGTE("1.19") {
allErrs = append(allErrs, field.Forbidden(fieldPath, "sslCertificate requires network loadbalancer for K8s 1.19+ see https://github.com/kubernetes/kops/blob/master/permalinks/acm_nlb.md"))
}
if spec.API.LoadBalancer.Class == kops.LoadBalancerClassNetwork && spec.API.LoadBalancer.UseForInternalApi && spec.API.LoadBalancer.Type == kops.LoadBalancerTypeInternal {
allErrs = append(allErrs, field.Forbidden(fieldPath, "useForInternalApi cannot be used with internal NLB due lack of hairpinning support"))
}
} }
return allErrs return allErrs

View File

@ -99,11 +99,17 @@ func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.Se
b := NewKubeconfigBuilder() b := NewKubeconfigBuilder()
// Use the secondary load balancer port if a certificate is on the primary listener
if admin != 0 && cluster.Spec.API != nil && cluster.Spec.API.LoadBalancer != nil && cluster.Spec.API.LoadBalancer.SSLCertificate != "" && cluster.Spec.API.LoadBalancer.Class == kops.LoadBalancerClassNetwork {
server = server + ":8443"
}
b.Context = clusterName b.Context = clusterName
b.Server = server b.Server = server
// add the CA Cert to the kubeconfig only if we didn't specify a SSL cert for the LB or are targeting the internal DNS name // add the CA Cert to the kubeconfig only if we didn't specify a certificate for the LB
if cluster.Spec.API == nil || cluster.Spec.API.LoadBalancer == nil || cluster.Spec.API.LoadBalancer.SSLCertificate == "" || internal { // or if we're using admin credentials and the secondary port
if cluster.Spec.API == nil || cluster.Spec.API.LoadBalancer == nil || cluster.Spec.API.LoadBalancer.SSLCertificate == "" || cluster.Spec.API.LoadBalancer.Class == kops.LoadBalancerClassNetwork || internal {
cert, _, _, err := keyStore.FindKeypair(fi.CertificateIDCA) cert, _, _, err := keyStore.FindKeypair(fi.CertificateIDCA)
if err != nil { if err != nil {
return nil, fmt.Errorf("error fetching CA keypair: %v", err) return nil, fmt.Errorf("error fetching CA keypair: %v", err)

View File

@ -71,10 +71,20 @@ func (f fakeKeyStore) MirrorTo(basedir vfs.Path) error {
} }
// build a generic minimal cluster // build a generic minimal cluster
func buildMinimalCluster(clusterName string, masterPublicName string) *kops.Cluster { func buildMinimalCluster(clusterName string, masterPublicName string, lbCert bool, nlb bool) *kops.Cluster {
cluster := testutils.BuildMinimalCluster(clusterName) cluster := testutils.BuildMinimalCluster(clusterName)
cluster.Spec.MasterPublicName = masterPublicName cluster.Spec.MasterPublicName = masterPublicName
cluster.Spec.MasterInternalName = fmt.Sprintf("internal.%v", masterPublicName) cluster.Spec.MasterInternalName = fmt.Sprintf("internal.%v", masterPublicName)
cluster.Spec.KubernetesVersion = "1.19.3"
cluster.Spec.API = &kops.AccessSpec{
LoadBalancer: &kops.LoadBalancerAccessSpec{},
}
if lbCert {
cluster.Spec.API.LoadBalancer.SSLCertificate = "cert-arn"
}
if nlb {
cluster.Spec.API.LoadBalancer.Class = kops.LoadBalancerClassNetwork
}
return cluster return cluster
} }
@ -107,9 +117,12 @@ func TestBuildKubecfg(t *testing.T) {
useKopsAuthenticationPlugin bool useKopsAuthenticationPlugin bool
} }
publiccluster := buildMinimalCluster("testcluster", "testcluster.test.com") publicCluster := buildMinimalCluster("testcluster", "testcluster.test.com", false, false)
emptyMasterPublicNameCluster := buildMinimalCluster("emptyMasterPublicNameCluster", "") emptyMasterPublicNameCluster := buildMinimalCluster("emptyMasterPublicNameCluster", "", false, false)
gossipCluster := buildMinimalCluster("testgossipcluster.k8s.local", "") gossipCluster := buildMinimalCluster("testgossipcluster.k8s.local", "", false, false)
certCluster := buildMinimalCluster("testcluster", "testcluster.test.com", true, false)
certNLBCluster := buildMinimalCluster("testcluster", "testcluster.test.com", true, true)
certGossipNLBCluster := buildMinimalCluster("testgossipcluster.k8s.local", "", true, true)
tests := []struct { tests := []struct {
name string name string
@ -121,7 +134,7 @@ func TestBuildKubecfg(t *testing.T) {
{ {
name: "Test Kube Config Data For Public DNS with admin", name: "Test Kube Config Data For Public DNS with admin",
args: args{ args: args{
cluster: publiccluster, cluster: publicCluster,
status: fakeStatusStore{}, status: fakeStatusStore{},
admin: DefaultKubecfgAdminLifetime, admin: DefaultKubecfgAdminLifetime,
user: "", user: "",
@ -134,10 +147,55 @@ func TestBuildKubecfg(t *testing.T) {
}, },
wantClientCert: true, wantClientCert: true,
}, },
{
name: "Test Kube Config Data For Public DNS with admin and secondary NLB port",
args: args{
cluster: certNLBCluster,
status: fakeStatusStore{},
admin: DefaultKubecfgAdminLifetime,
},
want: &KubeconfigBuilder{
Context: "testcluster",
Server: "https://testcluster.test.com:8443",
CACert: []byte(certData),
User: "testcluster",
},
wantClientCert: true,
},
{
name: "Test Kube Config Data For Public DNS with admin and CLB ACM Certificate",
args: args{
cluster: certCluster,
status: fakeStatusStore{},
admin: DefaultKubecfgAdminLifetime,
},
want: &KubeconfigBuilder{
Context: "testcluster",
Server: "https://testcluster.test.com",
CACert: nil,
User: "testcluster",
},
wantClientCert: true,
},
{
name: "Test Kube Config Data For Public DNS without admin and with ACM certificate",
args: args{
cluster: certNLBCluster,
status: fakeStatusStore{},
admin: 0,
},
want: &KubeconfigBuilder{
Context: "testcluster",
Server: "https://testcluster.test.com",
CACert: []byte(certData),
User: "testcluster",
},
wantClientCert: false,
},
{ {
name: "Test Kube Config Data For Public DNS without admin", name: "Test Kube Config Data For Public DNS without admin",
args: args{ args: args{
cluster: publiccluster, cluster: publicCluster,
status: fakeStatusStore{}, status: fakeStatusStore{},
admin: 0, admin: 0,
user: "myuser", user: "myuser",
@ -191,7 +249,7 @@ func TestBuildKubecfg(t *testing.T) {
{ {
name: "Public DNS with kops auth plugin", name: "Public DNS with kops auth plugin",
args: args{ args: args{
cluster: publiccluster, cluster: publicCluster,
status: fakeStatusStore{}, status: fakeStatusStore{},
admin: 0, admin: 0,
useKopsAuthenticationPlugin: true, useKopsAuthenticationPlugin: true,
@ -214,7 +272,7 @@ func TestBuildKubecfg(t *testing.T) {
{ {
name: "Test Kube Config Data For internal DNS name with admin", name: "Test Kube Config Data For internal DNS name with admin",
args: args{ args: args{
cluster: publiccluster, cluster: publicCluster,
status: fakeStatusStore{}, status: fakeStatusStore{},
admin: DefaultKubecfgAdminLifetime, admin: DefaultKubecfgAdminLifetime,
internal: true, internal: true,
@ -227,6 +285,29 @@ func TestBuildKubecfg(t *testing.T) {
}, },
wantClientCert: true, wantClientCert: true,
}, },
{
name: "Test Kube Config Data For Gossip cluster with admin and secondary NLB port",
args: args{
cluster: certGossipNLBCluster,
status: fakeStatusStore{
GetApiIngressStatusFn: func(cluster *kops.Cluster) ([]kops.ApiIngressStatus, error) {
return []kops.ApiIngressStatus{
{
Hostname: "nlbHostName",
},
}, nil
},
},
admin: DefaultKubecfgAdminLifetime,
},
want: &KubeconfigBuilder{
Context: "testgossipcluster.k8s.local",
Server: "https://nlbHostName:8443",
CACert: []byte(certData),
User: "testgossipcluster.k8s.local",
},
wantClientCert: true,
},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {

View File

@ -109,14 +109,21 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
"443": {InstancePort: 443}, "443": {InstancePort: 443},
} }
nlbListenerPort := "443" nlbListeners := []*awstasks.NetworkLoadBalancerListener{
nlbListeners := map[string]*awstasks.NetworkLoadBalancerListener{ {
nlbListenerPort: {Port: 443}, Port: 443,
TargetGroupName: b.NLBTargetGroupName("tcp"),
},
} }
if lbSpec.SSLCertificate != "" { if lbSpec.SSLCertificate != "" {
listeners["443"].SSLCertificateID = lbSpec.SSLCertificate listeners["443"].SSLCertificateID = lbSpec.SSLCertificate
nlbListeners["443"].SSLCertificateID = lbSpec.SSLCertificate nlbListeners[0].Port = 8443
nlbListeners = append(nlbListeners, &awstasks.NetworkLoadBalancerListener{
Port: 443,
TargetGroupName: b.NLBTargetGroupName("tls"),
SSLCertificateID: lbSpec.SSLCertificate,
})
} }
if lbSpec.SecurityGroupOverride != nil { if lbSpec.SecurityGroupOverride != nil {
@ -138,6 +145,7 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
LoadBalancerName: fi.String(loadBalancerName), LoadBalancerName: fi.String(loadBalancerName),
Subnets: elbSubnets, Subnets: elbSubnets,
Listeners: nlbListeners, Listeners: nlbListeners,
TargetGroups: make([]*awstasks.TargetGroup, 0),
Tags: tags, Tags: tags,
VPC: b.LinkToVPC(), VPC: b.LinkToVPC(),
@ -196,19 +204,18 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
c.AddTask(clb) c.AddTask(clb)
} else if b.APILoadBalancerClass() == kops.LoadBalancerClassNetwork { } else if b.APILoadBalancerClass() == kops.LoadBalancerClassNetwork {
targetGroupPort := fi.Int64(443) tcpGroupName := b.NLBTargetGroupName("tcp")
targetGroupName := b.NLBTargetGroupName("api") tcpGroupTags := b.CloudTags(tcpGroupName, false)
tags := b.CloudTags(targetGroupName, false)
// Override the returned name to be the expected NLB TG name // Override the returned name to be the expected NLB TG name
tags["Name"] = targetGroupName tcpGroupTags["Name"] = tcpGroupName
tg := &awstasks.TargetGroup{ tg := &awstasks.TargetGroup{
Name: fi.String(targetGroupName), Name: fi.String(tcpGroupName),
VPC: b.LinkToVPC(), VPC: b.LinkToVPC(),
Tags: tags, Tags: tcpGroupTags,
Protocol: fi.String("TCP"), Protocol: fi.String("TCP"),
Port: targetGroupPort, Port: fi.Int64(443),
HealthyThreshold: fi.Int64(2), HealthyThreshold: fi.Int64(2),
UnhealthyThreshold: fi.Int64(2), UnhealthyThreshold: fi.Int64(2),
Shared: fi.Bool(false), Shared: fi.Bool(false),
@ -216,8 +223,28 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
c.AddTask(tg) c.AddTask(tg)
nlb.TargetGroup = tg nlb.TargetGroups = append(nlb.TargetGroups, tg)
if lbSpec.SSLCertificate != "" {
tlsGroupName := b.NLBTargetGroupName("tls")
tlsGroupTags := b.CloudTags(tlsGroupName, false)
// Override the returned name to be the expected NLB TG name
tlsGroupTags["Name"] = tlsGroupName
secondaryTG := &awstasks.TargetGroup{
Name: fi.String(tlsGroupName),
VPC: b.LinkToVPC(),
Tags: tlsGroupTags,
Protocol: fi.String("TLS"),
Port: fi.Int64(443),
HealthyThreshold: fi.Int64(2),
UnhealthyThreshold: fi.Int64(2),
Shared: fi.Bool(false),
}
c.AddTask(secondaryTG)
nlb.TargetGroups = append(nlb.TargetGroups, secondaryTG)
}
sort.Stable(awstasks.OrderTargetGroupsByName(nlb.TargetGroups))
c.AddTask(nlb) c.AddTask(nlb)
} }
@ -291,7 +318,6 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
for _, masterGroup := range masterGroups { for _, masterGroup := range masterGroups {
t := &awstasks.SecurityGroupRule{ t := &awstasks.SecurityGroupRule{
// TODO: figure out how to only add this to one SG (GetSecurityGroups above returns multiple)
Name: fi.String(fmt.Sprintf("https-api-elb-%s", cidr)), Name: fi.String(fmt.Sprintf("https-api-elb-%s", cidr)),
Lifecycle: b.SecurityLifecycle, Lifecycle: b.SecurityLifecycle,
CIDR: fi.String(cidr), CIDR: fi.String(cidr),
@ -304,7 +330,6 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
// Allow ICMP traffic required for PMTU discovery // Allow ICMP traffic required for PMTU discovery
c.AddTask(&awstasks.SecurityGroupRule{ c.AddTask(&awstasks.SecurityGroupRule{
// TODO: figure out how to only add this to one SG (GetSecurityGroups above returns multiple)
Name: fi.String("icmp-pmtu-api-elb-" + cidr), Name: fi.String("icmp-pmtu-api-elb-" + cidr),
Lifecycle: b.SecurityLifecycle, Lifecycle: b.SecurityLifecycle,
CIDR: fi.String(cidr), CIDR: fi.String(cidr),
@ -313,6 +338,19 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
SecurityGroup: masterGroup.Task, SecurityGroup: masterGroup.Task,
ToPort: fi.Int64(4), ToPort: fi.Int64(4),
}) })
if b.Cluster.Spec.API != nil && b.Cluster.Spec.API.LoadBalancer != nil && b.Cluster.Spec.API.LoadBalancer.SSLCertificate != "" {
// Allow access to masters on secondary port through NLB
c.AddTask(&awstasks.SecurityGroupRule{
Name: fi.String(fmt.Sprintf("tcp-api-%s", cidr)),
Lifecycle: b.SecurityLifecycle,
CIDR: fi.String(cidr),
FromPort: fi.Int64(8443),
Protocol: fi.String("tcp"),
SecurityGroup: masterGroup.Task,
ToPort: fi.Int64(8443),
})
}
} }
} }
} }

View File

@ -366,13 +366,18 @@ func (b *AutoscalingGroupModelBuilder) buildAutoScalingGroupTask(c *fi.ModelBuil
t.InstanceProtection = ig.Spec.InstanceProtection t.InstanceProtection = ig.Spec.InstanceProtection
t.TargetGroups = []*awstasks.TargetGroup{}
// When Spotinst Elastigroups are used, there is no need to create // When Spotinst Elastigroups are used, there is no need to create
// a separate task for the attachment of the load balancer since this // a separate task for the attachment of the load balancer since this
// is already done as part of the Elastigroup's creation, if needed. // is already done as part of the Elastigroup's creation, if needed.
if !featureflag.Spotinst.Enabled() { if !featureflag.Spotinst.Enabled() {
if b.UseLoadBalancerForAPI() && ig.Spec.Role == kops.InstanceGroupRoleMaster { if b.UseLoadBalancerForAPI() && ig.Spec.Role == kops.InstanceGroupRoleMaster {
if b.UseNetworkLoadBalancer() { if b.UseNetworkLoadBalancer() {
t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("api")) t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("tcp"))
if b.Cluster.Spec.API.LoadBalancer.SSLCertificate != "" {
t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("tls"))
}
} else { } else {
t.LoadBalancers = append(t.LoadBalancers, b.LinkToCLB("api")) t.LoadBalancers = append(t.LoadBalancers, b.LinkToCLB("api"))
} }
@ -383,7 +388,7 @@ func (b *AutoscalingGroupModelBuilder) buildAutoScalingGroupTask(c *fi.ModelBuil
} }
} }
for _, extLB := range ig.Spec.ExternalLoadBalancers { for i, extLB := range ig.Spec.ExternalLoadBalancers {
if extLB.LoadBalancerName != nil { if extLB.LoadBalancerName != nil {
lb := &awstasks.ClassicLoadBalancer{ lb := &awstasks.ClassicLoadBalancer{
Name: extLB.LoadBalancerName, Name: extLB.LoadBalancerName,
@ -396,7 +401,7 @@ func (b *AutoscalingGroupModelBuilder) buildAutoScalingGroupTask(c *fi.ModelBuil
if extLB.TargetGroupARN != nil { if extLB.TargetGroupARN != nil {
tg := &awstasks.TargetGroup{ tg := &awstasks.TargetGroup{
Name: extLB.TargetGroupARN, Name: fi.String(fmt.Sprintf("external-tg-%d", i)),
ARN: extLB.TargetGroupARN, ARN: extLB.TargetGroupARN,
Shared: fi.Bool(true), Shared: fi.Bool(true),
} }

View File

@ -300,6 +300,7 @@ func (b *KopsModelContext) GetSecurityGroups(role kops.InstanceGroupRole) ([]Sec
"port=4002", // etcd events "port=4002", // etcd events
"port=4789", // VXLAN "port=4789", // VXLAN
"port=179", // Calico "port=179", // Calico
"port=8443", // k8s api secondary listener
// TODO: UDP vs TCP // TODO: UDP vs TCP
// TODO: Protocol 4 for calico // TODO: Protocol 4 for calico

View File

@ -105,7 +105,7 @@ func (b *KopsModelContext) LinkToNLB(prefix string) *awstasks.NetworkLoadBalance
} }
func (b *KopsModelContext) LinkToTargetGroup(prefix string) *awstasks.TargetGroup { func (b *KopsModelContext) LinkToTargetGroup(prefix string) *awstasks.TargetGroup {
name := b.NLBTargetGroupName(prefix) // TODO: this will need to change for the ACM cert bugfix since we'll have multiple TGs name := b.NLBTargetGroupName(prefix)
return &awstasks.TargetGroup{Name: &name} return &awstasks.TargetGroup{Name: &name}
} }

View File

@ -86,7 +86,10 @@
], ],
"TargetGroupARNs": [ "TargetGroupARNs": [
{ {
"Ref": "AWSElasticLoadBalancingV2TargetGroupapicomplexexamplecomvd3t5n" "Ref": "AWSElasticLoadBalancingV2TargetGrouptcpcomplexexamplecomvpjolq"
},
{
"Ref": "AWSElasticLoadBalancingV2TargetGrouptlscomplexexamplecom5nursn"
} }
] ]
} }
@ -850,6 +853,30 @@
"CidrIpv6": "2001:0:85a3::/48" "CidrIpv6": "2001:0:85a3::/48"
} }
}, },
"AWSEC2SecurityGroupIngresstcpapi111024": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"GroupId": {
"Ref": "AWSEC2SecurityGroupmasterscomplexexamplecom"
},
"FromPort": 8443,
"ToPort": 8443,
"IpProtocol": "tcp",
"CidrIp": "1.1.1.0/24"
}
},
"AWSEC2SecurityGroupIngresstcpapi20010850040": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"GroupId": {
"Ref": "AWSEC2SecurityGroupmasterscomplexexamplecom"
},
"FromPort": 8443,
"ToPort": 8443,
"IpProtocol": "tcp",
"CidrIpv6": "2001:0:8500::/40"
}
},
"AWSEC2SecurityGroupapielbcomplexexamplecom": { "AWSEC2SecurityGroupapielbcomplexexamplecom": {
"Type": "AWS::EC2::SecurityGroup", "Type": "AWS::EC2::SecurityGroup",
"Properties": { "Properties": {
@ -1148,11 +1175,16 @@
"AWSElasticLoadBalancingV2Listenerapicomplexexamplecom443": { "AWSElasticLoadBalancingV2Listenerapicomplexexamplecom443": {
"Type": "AWS::ElasticLoadBalancingV2::Listener", "Type": "AWS::ElasticLoadBalancingV2::Listener",
"Properties": { "Properties": {
"Certificates": [
{
"CertificateArn": "arn:aws:acm:us-test-1:000000000000:certificate/123456789012-1234-1234-1234-12345678"
}
],
"DefaultActions": [ "DefaultActions": [
{ {
"Type": "forward", "Type": "forward",
"TargetGroupArn": { "TargetGroupArn": {
"Ref": "AWSElasticLoadBalancingV2TargetGroupapicomplexexamplecomvd3t5n" "Ref": "AWSElasticLoadBalancingV2TargetGrouptcpcomplexexamplecomvpjolq"
} }
} }
], ],
@ -1160,6 +1192,24 @@
"Ref": "AWSElasticLoadBalancingV2LoadBalancerapicomplexexamplecom" "Ref": "AWSElasticLoadBalancingV2LoadBalancerapicomplexexamplecom"
}, },
"Port": 443, "Port": 443,
"Protocol": "TLS"
}
},
"AWSElasticLoadBalancingV2Listenerapicomplexexamplecom8443": {
"Type": "AWS::ElasticLoadBalancingV2::Listener",
"Properties": {
"DefaultActions": [
{
"Type": "forward",
"TargetGroupArn": {
"Ref": "AWSElasticLoadBalancingV2TargetGrouptlscomplexexamplecom5nursn"
}
}
],
"LoadBalancerArn": {
"Ref": "AWSElasticLoadBalancingV2LoadBalancerapicomplexexamplecom"
},
"Port": 8443,
"Protocol": "TCP" "Protocol": "TCP"
} }
}, },
@ -1198,10 +1248,10 @@
] ]
} }
}, },
"AWSElasticLoadBalancingV2TargetGroupapicomplexexamplecomvd3t5n": { "AWSElasticLoadBalancingV2TargetGrouptcpcomplexexamplecomvpjolq": {
"Type": "AWS::ElasticLoadBalancingV2::TargetGroup", "Type": "AWS::ElasticLoadBalancingV2::TargetGroup",
"Properties": { "Properties": {
"Name": "api-complex-example-com-vd3t5n", "Name": "tcp-complex-example-com-vpjolq",
"Port": 443, "Port": 443,
"Protocol": "TCP", "Protocol": "TCP",
"VpcId": { "VpcId": {
@ -1214,7 +1264,7 @@
}, },
{ {
"Key": "Name", "Key": "Name",
"Value": "api-complex-example-com-vd3t5n" "Value": "tcp-complex-example-com-vpjolq"
}, },
{ {
"Key": "Owner", "Key": "Owner",
@ -1234,6 +1284,42 @@
"UnhealthyThresholdCount": 2 "UnhealthyThresholdCount": 2
} }
}, },
"AWSElasticLoadBalancingV2TargetGrouptlscomplexexamplecom5nursn": {
"Type": "AWS::ElasticLoadBalancingV2::TargetGroup",
"Properties": {
"Name": "tls-complex-example-com-5nursn",
"Port": 443,
"Protocol": "TLS",
"VpcId": {
"Ref": "AWSEC2VPCcomplexexamplecom"
},
"Tags": [
{
"Key": "KubernetesCluster",
"Value": "complex.example.com"
},
{
"Key": "Name",
"Value": "tls-complex-example-com-5nursn"
},
{
"Key": "Owner",
"Value": "John Doe"
},
{
"Key": "foo/bar",
"Value": "fib+baz"
},
{
"Key": "kubernetes.io/cluster/complex.example.com",
"Value": "owned"
}
],
"HealthCheckProtocol": "TLS",
"HealthyThresholdCount": 2,
"UnhealthyThresholdCount": 2
}
},
"AWSIAMInstanceProfilemasterscomplexexamplecom": { "AWSIAMInstanceProfilemasterscomplexexamplecom": {
"Type": "AWS::IAM::InstanceProfile", "Type": "AWS::IAM::InstanceProfile",
"Properties": { "Properties": {

View File

@ -12,6 +12,7 @@ spec:
- sg-exampleid4 - sg-exampleid4
crossZoneLoadBalancing: true crossZoneLoadBalancing: true
class: Network class: Network
sslCertificate: arn:aws:acm:us-test-1:000000000000:certificate/123456789012-1234-1234-1234-12345678
kubernetesApiAccess: kubernetesApiAccess:
- 1.1.1.0/24 - 1.1.1.0/24
- 2001:0:8500::/40 - 2001:0:8500::/40

View File

@ -12,6 +12,7 @@ spec:
- sg-exampleid4 - sg-exampleid4
crossZoneLoadBalancing: true crossZoneLoadBalancing: true
class: Network class: Network
sslCertificate: arn:aws:acm:us-test-1:000000000000:certificate/123456789012-1234-1234-1234-12345678
kubernetesApiAccess: kubernetesApiAccess:
- 1.1.1.0/24 - 1.1.1.0/24
- 2001:0:8500::/40 - 2001:0:8500::/40

View File

@ -135,7 +135,7 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-complex-example-com"
propagate_at_launch = true propagate_at_launch = true
value = "owned" value = "owned"
} }
target_group_arns = [aws_lb_target_group.api-complex-example-com-vd3t5n.id] target_group_arns = [aws_lb_target_group.tcp-complex-example-com-vpjolq.id, aws_lb_target_group.tls-complex-example-com-5nursn.id]
vpc_zone_identifier = [aws_subnet.us-test-1a-complex-example-com.id] vpc_zone_identifier = [aws_subnet.us-test-1a-complex-example-com.id]
} }
@ -423,27 +423,57 @@ resource "aws_launch_template" "nodes-complex-example-com" {
} }
resource "aws_lb_listener" "api-complex-example-com-443" { resource "aws_lb_listener" "api-complex-example-com-443" {
certificate_arn = "arn:aws:acm:us-test-1:000000000000:certificate/123456789012-1234-1234-1234-12345678"
default_action { default_action {
target_group_arn = aws_lb_target_group.api-complex-example-com-vd3t5n.id target_group_arn = aws_lb_target_group.tcp-complex-example-com-vpjolq.id
type = "forward" type = "forward"
} }
load_balancer_arn = aws_lb.api-complex-example-com.id load_balancer_arn = aws_lb.api-complex-example-com.id
port = 443 port = 443
protocol = "TLS"
}
resource "aws_lb_listener" "api-complex-example-com-8443" {
default_action {
target_group_arn = aws_lb_target_group.tls-complex-example-com-5nursn.id
type = "forward"
}
load_balancer_arn = aws_lb.api-complex-example-com.id
port = 8443
protocol = "TCP" protocol = "TCP"
} }
resource "aws_lb_target_group" "api-complex-example-com-vd3t5n" { resource "aws_lb_target_group" "tcp-complex-example-com-vpjolq" {
health_check { health_check {
healthy_threshold = 2 healthy_threshold = 2
protocol = "TCP" protocol = "TCP"
unhealthy_threshold = 2 unhealthy_threshold = 2
} }
name = "api-complex-example-com-vd3t5n" name = "tcp-complex-example-com-vpjolq"
port = 443 port = 443
protocol = "TCP" protocol = "TCP"
tags = { tags = {
"KubernetesCluster" = "complex.example.com" "KubernetesCluster" = "complex.example.com"
"Name" = "api-complex-example-com-vd3t5n" "Name" = "tcp-complex-example-com-vpjolq"
"Owner" = "John Doe"
"foo/bar" = "fib+baz"
"kubernetes.io/cluster/complex.example.com" = "owned"
}
vpc_id = aws_vpc.complex-example-com.id
}
resource "aws_lb_target_group" "tls-complex-example-com-5nursn" {
health_check {
healthy_threshold = 2
protocol = "TCP"
unhealthy_threshold = 2
}
name = "tls-complex-example-com-5nursn"
port = 443
protocol = "TLS"
tags = {
"KubernetesCluster" = "complex.example.com"
"Name" = "tls-complex-example-com-5nursn"
"Owner" = "John Doe" "Owner" = "John Doe"
"foo/bar" = "fib+baz" "foo/bar" = "fib+baz"
"kubernetes.io/cluster/complex.example.com" = "owned" "kubernetes.io/cluster/complex.example.com" = "owned"
@ -716,6 +746,24 @@ resource "aws_security_group_rule" "ssh-external-to-node-2001_0_85a3__--48" {
type = "ingress" type = "ingress"
} }
resource "aws_security_group_rule" "tcp-api-1-1-1-0--24" {
cidr_blocks = ["1.1.1.0/24"]
from_port = 8443
protocol = "tcp"
security_group_id = aws_security_group.masters-complex-example-com.id
to_port = 8443
type = "ingress"
}
resource "aws_security_group_rule" "tcp-api-2001_0_8500__--40" {
cidr_blocks = ["2001:0:8500::/40"]
from_port = 8443
protocol = "tcp"
security_group_id = aws_security_group.masters-complex-example-com.id
to_port = 8443
type = "ingress"
}
resource "aws_security_group" "api-elb-complex-example-com" { resource "aws_security_group" "api-elb-complex-example-com" {
description = "Security group for api ELB" description = "Security group for api ELB"
name = "api-elb.complex.example.com" name = "api-elb.complex.example.com"

View File

@ -162,13 +162,13 @@ func (e *AutoscalingGroup) Find(c *fi.Context) (*AutoscalingGroup, error) {
} }
} }
actual.TargetGroups = []*TargetGroup{}
if len(g.TargetGroupARNs) > 0 { if len(g.TargetGroupARNs) > 0 {
targetGroups := make([]*TargetGroup, 0) actualTGs := make([]*TargetGroup, 0)
for _, tg := range g.TargetGroupARNs { for _, tg := range g.TargetGroupARNs {
targetGroups = append(actual.TargetGroups, &TargetGroup{ARN: aws.String(*tg)}) actualTGs = append(actualTGs, &TargetGroup{ARN: aws.String(*tg)})
} }
targetGroups, err := ReconcileTargetGroups(c.Cloud.(awsup.AWSCloud), actualTGs, e.TargetGroups)
targetGroups, err := ReconcileTargetGroups(c.Cloud.(awsup.AWSCloud), targetGroups, e.TargetGroups)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -645,7 +645,7 @@ func (v *AutoscalingGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Autos
} }
if detachTGRequest != nil { if detachTGRequest != nil {
if _, err := t.Cloud.Autoscaling().DetachLoadBalancerTargetGroups(detachTGRequest); err != nil { if _, err := t.Cloud.Autoscaling().DetachLoadBalancerTargetGroups(detachTGRequest); err != nil {
return fmt.Errorf("error attaching TargetGroups: %v", err) return fmt.Errorf("error detaching TargetGroups: %v", err)
} }
} }
if attachTGRequest != nil { if attachTGRequest != nil {

View File

@ -55,7 +55,7 @@ type NetworkLoadBalancer struct {
Subnets []*Subnet Subnets []*Subnet
Listeners map[string]*NetworkLoadBalancerListener Listeners []*NetworkLoadBalancerListener
Scheme *string Scheme *string
@ -66,8 +66,8 @@ type NetworkLoadBalancer struct {
Type *string Type *string
VPC *VPC VPC *VPC
TargetGroup *TargetGroup TargetGroups []*TargetGroup
} }
var _ fi.CompareWithID = &NetworkLoadBalancer{} var _ fi.CompareWithID = &NetworkLoadBalancer{}
@ -78,15 +78,25 @@ func (e *NetworkLoadBalancer) CompareWithID() *string {
type NetworkLoadBalancerListener struct { type NetworkLoadBalancerListener struct {
Port int Port int
TargetGroupName string
SSLCertificateID string SSLCertificateID string
} }
func (e *NetworkLoadBalancerListener) mapToAWS(targetGroupArn string, loadBalancerArn string) *elbv2.CreateListenerInput { func (e *NetworkLoadBalancerListener) mapToAWS(targetGroups []*TargetGroup, loadBalancerArn string) (*elbv2.CreateListenerInput, error) {
var tgARN string
for _, tg := range targetGroups {
if fi.StringValue(tg.Name) == e.TargetGroupName {
tgARN = fi.StringValue(tg.ARN)
}
}
if tgARN == "" {
return nil, fmt.Errorf("target group not found for NLB listener %+v", e)
}
l := &elbv2.CreateListenerInput{ l := &elbv2.CreateListenerInput{
DefaultActions: []*elbv2.Action{ DefaultActions: []*elbv2.Action{
{ {
TargetGroupArn: aws.String(targetGroupArn), TargetGroupArn: aws.String(tgARN),
Type: aws.String(elbv2.ActionTypeEnumForward), Type: aws.String(elbv2.ActionTypeEnumForward),
}, },
}, },
@ -104,7 +114,7 @@ func (e *NetworkLoadBalancerListener) mapToAWS(targetGroupArn string, loadBalanc
l.Protocol = aws.String(elbv2.ProtocolEnumTcp) l.Protocol = aws.String(elbv2.ProtocolEnumTcp)
} }
return l return l, nil
} }
var _ fi.HasDependencies = &NetworkLoadBalancerListener{} var _ fi.HasDependencies = &NetworkLoadBalancerListener{}
@ -113,6 +123,15 @@ func (e *NetworkLoadBalancerListener) GetDependencies(tasks map[string]fi.Task)
return nil return nil
} }
// OrderListenersByPort implements sort.Interface for []OrderListenersByPort, based on port number
type OrderListenersByPort []*NetworkLoadBalancerListener
func (a OrderListenersByPort) Len() int { return len(a) }
func (a OrderListenersByPort) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a OrderListenersByPort) Less(i, j int) bool {
return a[i].Port < a[j].Port
}
//The load balancer name 'api.renamenlbcluster.k8s.local' can only contain characters that are alphanumeric characters and hyphens(-)\n\tstatus code: 400, //The load balancer name 'api.renamenlbcluster.k8s.local' can only contain characters that are alphanumeric characters and hyphens(-)\n\tstatus code: 400,
func findNetworkLoadBalancerByLoadBalancerName(cloud awsup.AWSCloud, loadBalancerName string) (*elbv2.LoadBalancer, error) { func findNetworkLoadBalancerByLoadBalancerName(cloud awsup.AWSCloud, loadBalancerName string) (*elbv2.LoadBalancer, error) {
request := &elbv2.DescribeLoadBalancersInput{ request := &elbv2.DescribeLoadBalancersInput{
@ -343,36 +362,44 @@ func (e *NetworkLoadBalancer) Find(c *fi.Context) (*NetworkLoadBalancer, error)
return nil, fmt.Errorf("error querying for NLB listeners :%v", err) return nil, fmt.Errorf("error querying for NLB listeners :%v", err)
} }
actual.Listeners = make(map[string]*NetworkLoadBalancerListener) actual.Listeners = []*NetworkLoadBalancerListener{}
actual.TargetGroups = []*TargetGroup{}
for _, l := range response.Listeners { for _, l := range response.Listeners {
loadBalancerPort := strconv.FormatInt(aws.Int64Value(l.Port), 10)
actualListener := &NetworkLoadBalancerListener{} actualListener := &NetworkLoadBalancerListener{}
actualListener.Port = int(aws.Int64Value(l.Port)) actualListener.Port = int(aws.Int64Value(l.Port))
if len(l.Certificates) != 0 { if len(l.Certificates) != 0 {
actualListener.SSLCertificateID = aws.StringValue(l.Certificates[0].CertificateArn) // What if there is more then one certificate, can we just grab the default certificate? we don't set it as default, we only set the one. actualListener.SSLCertificateID = aws.StringValue(l.Certificates[0].CertificateArn) // What if there is more then one certificate, can we just grab the default certificate? we don't set it as default, we only set the one.
} }
actual.Listeners[loadBalancerPort] = actualListener
// This will need to be rearranged when we recognized multiple listeners and target groups per NLB // This will need to be rearranged when we recognized multiple listeners and target groups per NLB
if len(l.DefaultActions) > 0 { if len(l.DefaultActions) > 0 {
targetGroupARN := l.DefaultActions[0].TargetGroupArn targetGroupARN := l.DefaultActions[0].TargetGroupArn
if targetGroupARN != nil { if targetGroupARN != nil {
actual.TargetGroup = &TargetGroup{ARN: targetGroupARN} actual.TargetGroups = append(actual.TargetGroups, &TargetGroup{ARN: targetGroupARN})
cloud := c.Cloud.(awsup.AWSCloud)
descResp, err := cloud.ELBV2().DescribeTargetGroups(&elbv2.DescribeTargetGroupsInput{
TargetGroupArns: []*string{targetGroupARN},
})
if err != nil {
return nil, fmt.Errorf("error querying for NLB listener target groups: %v", err)
}
if len(descResp.TargetGroups) != 1 {
return nil, fmt.Errorf("unexpected DescribeTargetGroups response: %v", descResp)
}
actualListener.TargetGroupName = aws.StringValue(descResp.TargetGroups[0].TargetGroupName)
} }
} }
actual.Listeners = append(actual.Listeners, actualListener)
} }
// This will be cleaned up once the NLB task supports multiple target groups if len(actual.TargetGroups) > 0 {
if actual.TargetGroup != nil { targetGroups, err := ReconcileTargetGroups(c.Cloud.(awsup.AWSCloud), actual.TargetGroups, e.TargetGroups)
actualTGs := []*TargetGroup{actual.TargetGroup}
expectedTGs := []*TargetGroup{e.TargetGroup}
targetGroups, err := ReconcileTargetGroups(c.Cloud.(awsup.AWSCloud), actualTGs, expectedTGs)
if err != nil { if err != nil {
return nil, err return nil, err
} }
actual.TargetGroup = targetGroups[0] actual.TargetGroups = targetGroups
} }
sort.Stable(OrderTargetGroupsByName(actual.TargetGroups))
} }
@ -465,6 +492,8 @@ func (e *NetworkLoadBalancer) Run(c *fi.Context) error {
func (e *NetworkLoadBalancer) Normalize() { func (e *NetworkLoadBalancer) Normalize() {
// We need to sort our arrays consistently, so we don't get spurious changes // We need to sort our arrays consistently, so we don't get spurious changes
sort.Stable(OrderSubnetsById(e.Subnets)) sort.Stable(OrderSubnetsById(e.Subnets))
sort.Stable(OrderListenersByPort(e.Listeners))
sort.Stable(OrderTargetGroupsByName(e.TargetGroups))
} }
func (s *NetworkLoadBalancer) CheckChanges(a, e, changes *NetworkLoadBalancer) error { func (s *NetworkLoadBalancer) CheckChanges(a, e, changes *NetworkLoadBalancer) error {
@ -493,12 +522,18 @@ func (_ *NetworkLoadBalancer) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Ne
var loadBalancerName string var loadBalancerName string
var loadBalancerArn string var loadBalancerArn string
if len(e.Listeners) != len(e.TargetGroups) {
return fmt.Errorf("nlb listeners and target groups do not match: %v listeners vs %v target groups", len(e.Listeners), len(e.TargetGroups))
}
if a == nil { if a == nil {
if e.LoadBalancerName == nil { if e.LoadBalancerName == nil {
return fi.RequiredField("LoadBalancerName") return fi.RequiredField("LoadBalancerName")
} }
if e.TargetGroup.ARN == nil { for _, tg := range e.TargetGroups {
return fmt.Errorf("missing required target group ARN for NLB creation %v", e.TargetGroup) if tg.ARN == nil {
return fmt.Errorf("missing required target group ARN for NLB creation %v", tg)
}
} }
loadBalancerName = *e.LoadBalancerName loadBalancerName = *e.LoadBalancerName
@ -533,10 +568,13 @@ func (_ *NetworkLoadBalancer) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Ne
{ {
for _, listener := range e.Listeners { for _, listener := range e.Listeners {
createListenerInput := listener.mapToAWS(*e.TargetGroup.ARN, loadBalancerArn) createListenerInput, err := listener.mapToAWS(e.TargetGroups, loadBalancerArn)
if err != nil {
return err
}
klog.V(2).Infof("Creating Listener for NLB") klog.V(2).Infof("Creating Listener for NLB with port %v", listener.Port)
_, err := t.Cloud.ELBV2().CreateListener(createListenerInput) _, err = t.Cloud.ELBV2().CreateListener(createListenerInput)
if err != nil { if err != nil {
return fmt.Errorf("error creating listener for NLB: %v", err) return fmt.Errorf("error creating listener for NLB: %v", err)
} }
@ -606,10 +644,14 @@ func (_ *NetworkLoadBalancer) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Ne
} }
for _, listener := range changes.Listeners { for _, listener := range changes.Listeners {
awsListener := listener.mapToAWS(*e.TargetGroup.ARN, loadBalancerArn)
klog.V(2).Infof("Creating Listener for NLB") awsListener, err := listener.mapToAWS(e.TargetGroups, loadBalancerArn)
_, err := t.Cloud.ELBV2().CreateListener(awsListener) if err != nil {
return err
}
klog.V(2).Infof("Creating Listener for NLB with port %v", listener.Port)
_, err = t.Cloud.ELBV2().CreateListener(awsListener)
if err != nil { if err != nil {
return fmt.Errorf("error creating NLB listener: %v", err) return fmt.Errorf("error creating NLB listener: %v", err)
} }
@ -674,18 +716,14 @@ func (_ *NetworkLoadBalancer) RenderTerraform(t *terraform.TerraformTarget, a, e
return err return err
} }
for portStr, listener := range e.Listeners { for i, listener := range e.Listeners {
port, err := strconv.Atoi(portStr)
if err != nil {
return err
}
listenerTF := &terraformNetworkLoadBalancerListener{ listenerTF := &terraformNetworkLoadBalancerListener{
LoadBalancer: e.TerraformLink(), LoadBalancer: e.TerraformLink(),
Port: int64(port), Port: int64(listener.Port),
DefaultAction: []terraformNetworkLoadBalancerListenerAction{ DefaultAction: []terraformNetworkLoadBalancerListenerAction{
{ {
Type: elbv2.ActionTypeEnumForward, Type: elbv2.ActionTypeEnumForward,
TargetGroupARN: e.TargetGroup.TerraformLink(), TargetGroupARN: e.TargetGroups[i].TerraformLink(),
}, },
}, },
} }
@ -696,7 +734,7 @@ func (_ *NetworkLoadBalancer) RenderTerraform(t *terraform.TerraformTarget, a, e
listenerTF.Protocol = elbv2.ProtocolEnumTcp listenerTF.Protocol = elbv2.ProtocolEnumTcp
} }
err = t.RenderResource("aws_lb_listener", fmt.Sprintf("%v-%v", *e.Name, portStr), listenerTF) err = t.RenderResource("aws_lb_listener", fmt.Sprintf("%v-%v", *e.Name, listener.Port), listenerTF)
if err != nil { if err != nil {
return err return err
} }
@ -721,11 +759,15 @@ type cloudformationNetworkLoadBalancer struct {
} }
type cloudformationNetworkLoadBalancerListener struct { type cloudformationNetworkLoadBalancerListener struct {
Certificates []string `json:"Certificates,omitempty"` Certificates []cloudformationNetworkLoadBalancerListenerCertificate `json:"Certificates,omitempty"`
DefaultActions []cloudformationNetworkLoadBalancerListenerAction `json:"DefaultActions"` DefaultActions []cloudformationNetworkLoadBalancerListenerAction `json:"DefaultActions"`
LoadBalancerARN *cloudformation.Literal `json:"LoadBalancerArn"` LoadBalancerARN *cloudformation.Literal `json:"LoadBalancerArn"`
Port int64 `json:"Port"` Port int64 `json:"Port"`
Protocol string `json:"Protocol"` Protocol string `json:"Protocol"`
}
type cloudformationNetworkLoadBalancerListenerCertificate struct {
CertificateArn string `json:"CertificateArn"`
} }
type cloudformationNetworkLoadBalancerListenerAction struct { type cloudformationNetworkLoadBalancerListenerAction struct {
@ -753,29 +795,27 @@ func (_ *NetworkLoadBalancer) RenderCloudformation(t *cloudformation.Cloudformat
return err return err
} }
for portStr, listener := range e.Listeners { for i, listener := range e.Listeners {
port, err := strconv.Atoi(portStr)
if err != nil {
return err
}
listenerCF := &cloudformationNetworkLoadBalancerListener{ listenerCF := &cloudformationNetworkLoadBalancerListener{
LoadBalancerARN: e.CloudformationLink(), LoadBalancerARN: e.CloudformationLink(),
Port: int64(port), Port: int64(listener.Port),
DefaultActions: []cloudformationNetworkLoadBalancerListenerAction{ DefaultActions: []cloudformationNetworkLoadBalancerListenerAction{
{ {
Type: elbv2.ActionTypeEnumForward, Type: elbv2.ActionTypeEnumForward,
TargetGroupARN: e.TargetGroup.CloudformationLink(), TargetGroupARN: e.TargetGroups[i].CloudformationLink(),
}, },
}, },
} }
if listener.SSLCertificateID != "" { if listener.SSLCertificateID != "" {
listenerCF.Certificates = []string{listener.SSLCertificateID} listenerCF.Certificates = []cloudformationNetworkLoadBalancerListenerCertificate{
{CertificateArn: listener.SSLCertificateID},
}
listenerCF.Protocol = elbv2.ProtocolEnumTls listenerCF.Protocol = elbv2.ProtocolEnumTls
} else { } else {
listenerCF.Protocol = elbv2.ProtocolEnumTcp listenerCF.Protocol = elbv2.ProtocolEnumTcp
} }
err = t.RenderResource("AWS::ElasticLoadBalancingV2::Listener", fmt.Sprintf("%v-%v", *e.Name, portStr), listenerCF) err = t.RenderResource("AWS::ElasticLoadBalancingV2::Listener", fmt.Sprintf("%v-%v", *e.Name, listener.Port), listenerCF)
if err != nil { if err != nil {
return err return err
} }
@ -794,5 +834,4 @@ func (e *NetworkLoadBalancer) CloudformationAttrCanonicalHostedZoneNameID() *clo
func (e *NetworkLoadBalancer) CloudformationAttrDNSName() *cloudformation.Literal { func (e *NetworkLoadBalancer) CloudformationAttrDNSName() *cloudformation.Literal {
return cloudformation.GetAtt("AWS::ElasticLoadBalancingV2::LoadBalancer", *e.Name, "DNSName") return cloudformation.GetAtt("AWS::ElasticLoadBalancingV2::LoadBalancer", *e.Name, "DNSName")
} }

View File

@ -60,8 +60,7 @@ func (e *TargetGroup) Find(c *fi.Context) (*TargetGroup, error) {
request := &elbv2.DescribeTargetGroupsInput{} request := &elbv2.DescribeTargetGroupsInput{}
if e.ARN != nil { if e.ARN != nil {
request.TargetGroupArns = []*string{e.ARN} request.TargetGroupArns = []*string{e.ARN}
} } else if e.Name != nil {
if e.Name != nil {
request.Names = []*string{e.Name} request.Names = []*string{e.Name}
} }
@ -128,6 +127,9 @@ func FindTargetGroupByName(cloud awsup.AWSCloud, findName string) (*elbv2.Target
resp, err := cloud.ELBV2().DescribeTargetGroups(request) resp, err := cloud.ELBV2().DescribeTargetGroups(request)
if err != nil { if err != nil {
if aerr, ok := err.(awserr.Error); ok && aerr.Code() == elbv2.ErrCodeTargetGroupNotFoundException {
return nil, nil
}
return nil, fmt.Errorf("error describing TargetGroups: %v", err) return nil, fmt.Errorf("error describing TargetGroups: %v", err)
} }
if len(resp.TargetGroups) == 0 { if len(resp.TargetGroups) == 0 {
@ -192,6 +194,15 @@ func (_ *TargetGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *TargetGrou
return nil return nil
} }
// OrderTargetGroupsByName implements sort.Interface for []OrderTargetGroupsByName, based on port number
type OrderTargetGroupsByName []*TargetGroup
func (a OrderTargetGroupsByName) Len() int { return len(a) }
func (a OrderTargetGroupsByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a OrderTargetGroupsByName) Less(i, j int) bool {
return fi.StringValue(a[i].Name) < fi.StringValue(a[j].Name)
}
// pkg/model/awsmodel doesn't know the ARN of the API TargetGroup tasks that it passes to the master ASGs, // pkg/model/awsmodel doesn't know the ARN of the API TargetGroup tasks that it passes to the master ASGs,
// it only knows the ARN of external target groups passed through the InstanceGroupSpec. // it only knows the ARN of external target groups passed through the InstanceGroupSpec.
// We lookup the ARN for TargetGroup tasks that don't have it set in order to attach the LB to the ASG. // We lookup the ARN for TargetGroup tasks that don't have it set in order to attach the LB to the ASG.
@ -206,12 +217,12 @@ func (_ *TargetGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *TargetGrou
// Because we don't know whether any given ARN attached to an ASG is an API TargetGroup task or not, // Because we don't know whether any given ARN attached to an ASG is an API TargetGroup task or not,
// we have to find the API TargetGroup task, lookup its ARN, then compare that to the list of attached TargetGroups. // we have to find the API TargetGroup task, lookup its ARN, then compare that to the list of attached TargetGroups.
func ReconcileTargetGroups(cloud awsup.AWSCloud, actual []*TargetGroup, expected []*TargetGroup) ([]*TargetGroup, error) { func ReconcileTargetGroups(cloud awsup.AWSCloud, actual []*TargetGroup, expected []*TargetGroup) ([]*TargetGroup, error) {
var apiTGTask *TargetGroup apiTGTasks := make([]*TargetGroup, 0)
for _, tg := range expected { for _, tg := range expected {
// All external TargetGroups have their Shared field set to true. The API TargetGroups do not. // All external TargetGroups have their Shared field set to true. The API TargetGroups do not.
// Note that Shared is set by the kops model rather than AWS tags. // Note that Shared is set by the kops model rather than AWS tags.
if !fi.BoolValue(tg.Shared) { if !fi.BoolValue(tg.Shared) {
apiTGTask = tg apiTGTasks = append(apiTGTasks, tg)
} }
} }
@ -221,18 +232,19 @@ func ReconcileTargetGroups(cloud awsup.AWSCloud, actual []*TargetGroup, expected
return reconciled, nil return reconciled, nil
} }
var apiTG *elbv2.TargetGroup apiTGs := make(map[string]*TargetGroup)
var err error for _, task := range apiTGTasks {
if apiTGTask != nil { apiTG, err := FindTargetGroupByName(cloud, fi.StringValue(task.Name))
apiTG, err = FindTargetGroupByName(cloud, fi.StringValue(apiTGTask.Name))
if err != nil { if err != nil {
return nil, err return nil, err
} }
if apiTG != nil {
apiTGs[aws.StringValue(apiTG.TargetGroupArn)] = task
}
} }
for i := 0; i < len(actual); i++ { for _, tg := range actual {
tg := actual[i] if apiTask, ok := apiTGs[aws.StringValue(tg.ARN)]; ok {
if apiTG != nil && aws.StringValue(apiTG.TargetGroupArn) == aws.StringValue(tg.ARN) { reconciled = append(reconciled, apiTask)
reconciled = append(reconciled, apiTGTask)
} else { } else {
reconciled = append(reconciled, tg) reconciled = append(reconciled, tg)
} }
@ -275,7 +287,7 @@ func (_ *TargetGroup) RenderTerraform(t *terraform.TerraformTarget, a, e, change
HealthCheck: terraformTargetGroupHealthCheck{ HealthCheck: terraformTargetGroupHealthCheck{
HealthyThreshold: *e.HealthyThreshold, HealthyThreshold: *e.HealthyThreshold,
UnhealthyThreshold: *e.UnhealthyThreshold, UnhealthyThreshold: *e.UnhealthyThreshold,
Protocol: *e.Protocol, Protocol: elbv2.ProtocolEnumTcp,
}, },
} }