Setup a second NLB listener on 8443 when sslCertificate is set

pkg/apis/kops/validation/validation.go

@@ -217,6 +217,12 @@ func validateClusterSpec(spec *kops.ClusterSpec, c *kops.Cluster, fieldPath *fie
 		if featureflag.Spotinst.Enabled() && spec.API.LoadBalancer.Class == kops.LoadBalancerClassNetwork {
 			allErrs = append(allErrs, field.Forbidden(fieldPath, "cannot use NLB together with spotinst"))
 		}
+		if spec.API.LoadBalancer.SSLCertificate != "" && spec.API.LoadBalancer.Class != kops.LoadBalancerClassNetwork && c.IsKubernetesGTE("1.19") {
+			allErrs = append(allErrs, field.Forbidden(fieldPath, "sslCertificate requires network loadbalancer for K8s 1.19+ see <TODO: permalink here>"))
+		}
+		if spec.API.LoadBalancer.Class == kops.LoadBalancerClassNetwork && spec.API.LoadBalancer.UseForInternalApi && spec.API.LoadBalancer.Type == kops.LoadBalancerTypeInternal {
+			allErrs = append(allErrs, field.Forbidden(fieldPath, "useForInternalApi cannot be used with internal NLB due lack of hairpinning support"))
+		}
 	}
 
 	return allErrs

pkg/model/awsmodel/api_loadbalancer.go

@@ -110,12 +110,22 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
 		}
 
 		nlbListeners := []*awstasks.NetworkLoadBalancerListener{
-			{Port: 443},
+			{
+				Port:            443,
+				TargetGroupName: b.NLBTargetGroupName("api"),
+			},
 		}
 
 		if lbSpec.SSLCertificate != "" {
 			listeners["443"].SSLCertificateID = lbSpec.SSLCertificate
 			nlbListeners[0].SSLCertificateID = lbSpec.SSLCertificate
+			nlbListeners = append(nlbListeners, &awstasks.NetworkLoadBalancerListener{
+				Port:            8443,
+				TargetGroupName: b.NLBTargetGroupName("tcp"),
+			})
+			klog.Info("Using ACM certificate for API ELB")
+		} else {
+			klog.Info("NOT using ACM certificate for API ELB")
 		}
 
 		if lbSpec.SecurityGroupOverride != nil {
@@ -202,11 +212,15 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
 			// Override the returned name to be the expected NLB TG name
 			primaryTags["Name"] = targetGroupName
 
+			protocol := fi.String("TCP")
+			if lbSpec.SSLCertificate != "" {
+				protocol = fi.String("TLS")
+			}
 			tg := &awstasks.TargetGroup{
 				Name:               fi.String(targetGroupName),
 				VPC:                b.LinkToVPC(),
 				Tags:               primaryTags,
-				Protocol:           fi.String("TCP"),
+				Protocol:           protocol,
 				Port:               fi.Int64(443),
 				HealthyThreshold:   fi.Int64(2),
 				UnhealthyThreshold: fi.Int64(2),
@@ -217,6 +231,26 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
 
 			nlb.TargetGroups = append(nlb.TargetGroups, tg)
 
+			if lbSpec.SSLCertificate != "" {
+				secondaryTags := b.CloudTags(targetGroupName, false)
+				secondaryName := b.NLBTargetGroupName("tcp")
+
+				// Override the returned name to be the expected NLB TG name
+				secondaryTags["Name"] = secondaryName
+				secondaryTG := &awstasks.TargetGroup{
+					Name:               fi.String(secondaryName),
+					VPC:                b.LinkToVPC(),
+					Tags:               secondaryTags,
+					Protocol:           fi.String("TCP"),
+					Port:               fi.Int64(443),
+					HealthyThreshold:   fi.Int64(2),
+					UnhealthyThreshold: fi.Int64(2),
+					Shared:             fi.Bool(false),
+				}
+				c.AddTask(secondaryTG)
+				nlb.TargetGroups = append(nlb.TargetGroups, secondaryTG)
+			}
+
 			c.AddTask(nlb)
 		}
 
@@ -310,6 +344,19 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
 					SecurityGroup: masterGroup.Task,
 					ToPort:        fi.Int64(4),
 				})
+
+				if b.Cluster.Spec.API != nil && b.Cluster.Spec.API.LoadBalancer != nil && b.Cluster.Spec.API.LoadBalancer.SSLCertificate != "" {
+					// Allow access to masters on secondary port through NLB
+					c.AddTask(&awstasks.SecurityGroupRule{
+						Name:          fi.String(fmt.Sprintf("tcp-api-%s", cidr)),
+						Lifecycle:     b.SecurityLifecycle,
+						CIDR:          fi.String(cidr),
+						FromPort:      fi.Int64(8443),
+						Protocol:      fi.String("tcp"),
+						SecurityGroup: masterGroup.Task,
+						ToPort:        fi.Int64(8443),
+					})
+				}
 			}
 		}
 	}

pkg/model/awsmodel/autoscalinggroup.go

@@ -371,6 +371,9 @@ func (b *AutoscalingGroupModelBuilder) buildAutoScalingGroupTask(c *fi.ModelBuil
 		if b.UseLoadBalancerForAPI() && ig.Spec.Role == kops.InstanceGroupRoleMaster {
 			if b.UseNetworkLoadBalancer() {
 				t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("api"))
+				if b.Cluster.Spec.API.LoadBalancer.SSLCertificate != "" {
+					t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("tcp"))
+				}
 			} else {
 				t.LoadBalancers = append(t.LoadBalancers, b.LinkToCLB("api"))
 			}

pkg/model/firewall.go

@@ -418,6 +418,7 @@ func (b *KopsModelContext) GetSecurityGroups(role kops.InstanceGroupRole) ([]Sec
 				"port=4002", // etcd events
 				"port=4789", // VXLAN
 				"port=179",  // Calico
+				"port=8443", // k8s api secondary listener
 
 				// TODO: UDP vs TCP
 				// TODO: Protocol 4 for calico

pkg/model/names.go

@@ -105,7 +105,7 @@ func (b *KopsModelContext) LinkToNLB(prefix string) *awstasks.NetworkLoadBalance
 }
 
 func (b *KopsModelContext) LinkToTargetGroup(prefix string) *awstasks.TargetGroup {
-	name := b.NLBTargetGroupName(prefix) // TODO: this will need to change for the ACM cert bugfix since we'll have multiple TGs
+	name := b.NLBTargetGroupName(prefix)
 	return &awstasks.TargetGroup{Name: &name}
 }
 

upup/pkg/fi/cloudup/awstasks/network_load_balancer.go

@@ -78,15 +78,25 @@ func (e *NetworkLoadBalancer) CompareWithID() *string {
 
 type NetworkLoadBalancerListener struct {
 	Port             int
+	TargetGroupName  string
 	SSLCertificateID string
 }
 
-func (e *NetworkLoadBalancerListener) mapToAWS(targetGroupArn string, loadBalancerArn string) *elbv2.CreateListenerInput {
+func (e *NetworkLoadBalancerListener) mapToAWS(targetGroups []*TargetGroup, loadBalancerArn string) (*elbv2.CreateListenerInput, error) {
+	var tgARN string
+	for _, tg := range targetGroups {
+		if fi.StringValue(tg.Name) == e.TargetGroupName {
+			tgARN = fi.StringValue(tg.ARN)
+		}
+	}
+	if tgARN == "" {
+		return nil, fmt.Errorf("target group not found for NLB listener %+v", e)
+	}
 
 	l := &elbv2.CreateListenerInput{
 		DefaultActions: []*elbv2.Action{
 			{
-				TargetGroupArn: aws.String(targetGroupArn),
+				TargetGroupArn: aws.String(tgARN),
 				Type:           aws.String(elbv2.ActionTypeEnumForward),
 			},
 		},
@@ -104,7 +114,7 @@ func (e *NetworkLoadBalancerListener) mapToAWS(targetGroupArn string, loadBalanc
 		l.Protocol = aws.String(elbv2.ProtocolEnumTcp)
 	}
 
-	return l
+	return l, nil
 }
 
 var _ fi.HasDependencies = &NetworkLoadBalancerListener{}
@@ -545,11 +555,14 @@ func (_ *NetworkLoadBalancer) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Ne
 		}
 
 		{
-			for i, listener := range e.Listeners {
+			for _, listener := range e.Listeners {
-				createListenerInput := listener.mapToAWS(*e.TargetGroups[i].ARN, loadBalancerArn)
+				createListenerInput, err := listener.mapToAWS(e.TargetGroups, loadBalancerArn)
+				if err != nil {
+					return err
+				}
 
-				klog.V(2).Infof("Creating Listener for NLB")
+				klog.V(2).Infof("Creating Listener for NLB with port %v", listener.Port)
-				_, err := t.Cloud.ELBV2().CreateListener(createListenerInput)
+				_, err = t.Cloud.ELBV2().CreateListener(createListenerInput)
 				if err != nil {
 					return fmt.Errorf("error creating listener for NLB: %v", err)
 				}
@@ -618,11 +631,15 @@ func (_ *NetworkLoadBalancer) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Ne
 				}
 			}
 
-			for i, listener := range changes.Listeners {
+			for _, listener := range changes.Listeners {
-				awsListener := listener.mapToAWS(*e.TargetGroups[i].ARN, loadBalancerArn)
 
-				klog.V(2).Infof("Creating Listener for NLB")
+				awsListener, err := listener.mapToAWS(e.TargetGroups, loadBalancerArn)
-				_, err := t.Cloud.ELBV2().CreateListener(awsListener)
+				if err != nil {
+					return err
+				}
+
+				klog.V(2).Infof("Creating Listener for NLB with port %v", listener.Port)
+				_, err = t.Cloud.ELBV2().CreateListener(awsListener)
 				if err != nil {
 					return fmt.Errorf("error creating NLB listener: %v", err)
 				}