diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index 4ed2471ae2..8bb3538f3f 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -217,6 +217,12 @@ func validateClusterSpec(spec *kops.ClusterSpec, c *kops.Cluster, fieldPath *fie if featureflag.Spotinst.Enabled() && spec.API.LoadBalancer.Class == kops.LoadBalancerClassNetwork { allErrs = append(allErrs, field.Forbidden(fieldPath, "cannot use NLB together with spotinst")) } + if spec.API.LoadBalancer.SSLCertificate != "" && spec.API.LoadBalancer.Class != kops.LoadBalancerClassNetwork && c.IsKubernetesGTE("1.19") { + allErrs = append(allErrs, field.Forbidden(fieldPath, "sslCertificate requires network loadbalancer for K8s 1.19+ see ")) + } + if spec.API.LoadBalancer.Class == kops.LoadBalancerClassNetwork && spec.API.LoadBalancer.UseForInternalApi && spec.API.LoadBalancer.Type == kops.LoadBalancerTypeInternal { + allErrs = append(allErrs, field.Forbidden(fieldPath, "useForInternalApi cannot be used with internal NLB due lack of hairpinning support")) + } } return allErrs diff --git a/pkg/model/awsmodel/api_loadbalancer.go b/pkg/model/awsmodel/api_loadbalancer.go index 2e4439665c..47d507b98c 100644 --- a/pkg/model/awsmodel/api_loadbalancer.go +++ b/pkg/model/awsmodel/api_loadbalancer.go @@ -110,12 +110,22 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { } nlbListeners := []*awstasks.NetworkLoadBalancerListener{ - {Port: 443}, + { + Port: 443, + TargetGroupName: b.NLBTargetGroupName("api"), + }, } if lbSpec.SSLCertificate != "" { listeners["443"].SSLCertificateID = lbSpec.SSLCertificate nlbListeners[0].SSLCertificateID = lbSpec.SSLCertificate + nlbListeners = append(nlbListeners, &awstasks.NetworkLoadBalancerListener{ + Port: 8443, + TargetGroupName: b.NLBTargetGroupName("tcp"), + }) + klog.Info("Using ACM certificate for API ELB") + } else { + klog.Info("NOT using ACM certificate for API ELB") } if lbSpec.SecurityGroupOverride != nil { @@ -202,11 +212,15 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { // Override the returned name to be the expected NLB TG name primaryTags["Name"] = targetGroupName + protocol := fi.String("TCP") + if lbSpec.SSLCertificate != "" { + protocol = fi.String("TLS") + } tg := &awstasks.TargetGroup{ Name: fi.String(targetGroupName), VPC: b.LinkToVPC(), Tags: primaryTags, - Protocol: fi.String("TCP"), + Protocol: protocol, Port: fi.Int64(443), HealthyThreshold: fi.Int64(2), UnhealthyThreshold: fi.Int64(2), @@ -217,6 +231,26 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { nlb.TargetGroups = append(nlb.TargetGroups, tg) + if lbSpec.SSLCertificate != "" { + secondaryTags := b.CloudTags(targetGroupName, false) + secondaryName := b.NLBTargetGroupName("tcp") + + // Override the returned name to be the expected NLB TG name + secondaryTags["Name"] = secondaryName + secondaryTG := &awstasks.TargetGroup{ + Name: fi.String(secondaryName), + VPC: b.LinkToVPC(), + Tags: secondaryTags, + Protocol: fi.String("TCP"), + Port: fi.Int64(443), + HealthyThreshold: fi.Int64(2), + UnhealthyThreshold: fi.Int64(2), + Shared: fi.Bool(false), + } + c.AddTask(secondaryTG) + nlb.TargetGroups = append(nlb.TargetGroups, secondaryTG) + } + c.AddTask(nlb) } @@ -310,6 +344,19 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { SecurityGroup: masterGroup.Task, ToPort: fi.Int64(4), }) + + if b.Cluster.Spec.API != nil && b.Cluster.Spec.API.LoadBalancer != nil && b.Cluster.Spec.API.LoadBalancer.SSLCertificate != "" { + // Allow access to masters on secondary port through NLB + c.AddTask(&awstasks.SecurityGroupRule{ + Name: fi.String(fmt.Sprintf("tcp-api-%s", cidr)), + Lifecycle: b.SecurityLifecycle, + CIDR: fi.String(cidr), + FromPort: fi.Int64(8443), + Protocol: fi.String("tcp"), + SecurityGroup: masterGroup.Task, + ToPort: fi.Int64(8443), + }) + } } } } diff --git a/pkg/model/awsmodel/autoscalinggroup.go b/pkg/model/awsmodel/autoscalinggroup.go index 95b7608cc3..356ba8cc5e 100644 --- a/pkg/model/awsmodel/autoscalinggroup.go +++ b/pkg/model/awsmodel/autoscalinggroup.go @@ -371,6 +371,9 @@ func (b *AutoscalingGroupModelBuilder) buildAutoScalingGroupTask(c *fi.ModelBuil if b.UseLoadBalancerForAPI() && ig.Spec.Role == kops.InstanceGroupRoleMaster { if b.UseNetworkLoadBalancer() { t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("api")) + if b.Cluster.Spec.API.LoadBalancer.SSLCertificate != "" { + t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("tcp")) + } } else { t.LoadBalancers = append(t.LoadBalancers, b.LinkToCLB("api")) } diff --git a/pkg/model/firewall.go b/pkg/model/firewall.go index 2e834d7b0f..d331611cfa 100644 --- a/pkg/model/firewall.go +++ b/pkg/model/firewall.go @@ -418,6 +418,7 @@ func (b *KopsModelContext) GetSecurityGroups(role kops.InstanceGroupRole) ([]Sec "port=4002", // etcd events "port=4789", // VXLAN "port=179", // Calico + "port=8443", // k8s api secondary listener // TODO: UDP vs TCP // TODO: Protocol 4 for calico diff --git a/pkg/model/names.go b/pkg/model/names.go index fc46d232a3..80aaa67da4 100644 --- a/pkg/model/names.go +++ b/pkg/model/names.go @@ -105,7 +105,7 @@ func (b *KopsModelContext) LinkToNLB(prefix string) *awstasks.NetworkLoadBalance } func (b *KopsModelContext) LinkToTargetGroup(prefix string) *awstasks.TargetGroup { - name := b.NLBTargetGroupName(prefix) // TODO: this will need to change for the ACM cert bugfix since we'll have multiple TGs + name := b.NLBTargetGroupName(prefix) return &awstasks.TargetGroup{Name: &name} } diff --git a/upup/pkg/fi/cloudup/awstasks/network_load_balancer.go b/upup/pkg/fi/cloudup/awstasks/network_load_balancer.go index b658cd69c3..6535f42f12 100644 --- a/upup/pkg/fi/cloudup/awstasks/network_load_balancer.go +++ b/upup/pkg/fi/cloudup/awstasks/network_load_balancer.go @@ -78,15 +78,25 @@ func (e *NetworkLoadBalancer) CompareWithID() *string { type NetworkLoadBalancerListener struct { Port int + TargetGroupName string SSLCertificateID string } -func (e *NetworkLoadBalancerListener) mapToAWS(targetGroupArn string, loadBalancerArn string) *elbv2.CreateListenerInput { +func (e *NetworkLoadBalancerListener) mapToAWS(targetGroups []*TargetGroup, loadBalancerArn string) (*elbv2.CreateListenerInput, error) { + var tgARN string + for _, tg := range targetGroups { + if fi.StringValue(tg.Name) == e.TargetGroupName { + tgARN = fi.StringValue(tg.ARN) + } + } + if tgARN == "" { + return nil, fmt.Errorf("target group not found for NLB listener %+v", e) + } l := &elbv2.CreateListenerInput{ DefaultActions: []*elbv2.Action{ { - TargetGroupArn: aws.String(targetGroupArn), + TargetGroupArn: aws.String(tgARN), Type: aws.String(elbv2.ActionTypeEnumForward), }, }, @@ -104,7 +114,7 @@ func (e *NetworkLoadBalancerListener) mapToAWS(targetGroupArn string, loadBalanc l.Protocol = aws.String(elbv2.ProtocolEnumTcp) } - return l + return l, nil } var _ fi.HasDependencies = &NetworkLoadBalancerListener{} @@ -545,11 +555,14 @@ func (_ *NetworkLoadBalancer) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Ne } { - for i, listener := range e.Listeners { - createListenerInput := listener.mapToAWS(*e.TargetGroups[i].ARN, loadBalancerArn) + for _, listener := range e.Listeners { + createListenerInput, err := listener.mapToAWS(e.TargetGroups, loadBalancerArn) + if err != nil { + return err + } - klog.V(2).Infof("Creating Listener for NLB") - _, err := t.Cloud.ELBV2().CreateListener(createListenerInput) + klog.V(2).Infof("Creating Listener for NLB with port %v", listener.Port) + _, err = t.Cloud.ELBV2().CreateListener(createListenerInput) if err != nil { return fmt.Errorf("error creating listener for NLB: %v", err) } @@ -618,11 +631,15 @@ func (_ *NetworkLoadBalancer) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Ne } } - for i, listener := range changes.Listeners { - awsListener := listener.mapToAWS(*e.TargetGroups[i].ARN, loadBalancerArn) + for _, listener := range changes.Listeners { - klog.V(2).Infof("Creating Listener for NLB") - _, err := t.Cloud.ELBV2().CreateListener(awsListener) + awsListener, err := listener.mapToAWS(e.TargetGroups, loadBalancerArn) + if err != nil { + return err + } + + klog.V(2).Infof("Creating Listener for NLB with port %v", listener.Port) + _, err = t.Cloud.ELBV2().CreateListener(awsListener) if err != nil { return fmt.Errorf("error creating NLB listener: %v", err) }