aws: Create cluster without DNS or Gossip

nodeup/pkg/model/etc_hosts.go

@@ -51,6 +51,12 @@ func (b *EtcHostsBuilder) Build(c *fi.ModelBuilderContext) error {
 			Hostname:  b.Cluster.Spec.MasterInternalName,
 			Addresses: []string{b.BootConfig.APIServer},
 		})
+		if b.UseKopsControllerForNodeBootstrap() {
+			task.Records = append(task.Records, nodetasks.HostRecord{
+				Hostname:  "kops-controller.internal." + b.Cluster.Name,
+				Addresses: []string{b.BootConfig.APIServer},
+			})
+		}
 	}
 
 	if len(task.Records) != 0 {

pkg/apis/kops/validation/validation.go

@@ -427,9 +427,10 @@ func validateTopology(c *kops.Cluster, topology *kops.TopologySpec, fieldPath *f
 	}
 
 	if topology.DNS != nil {
+		cloud := c.Spec.GetCloudProvider()
 		value := string(topology.DNS.Type)
 		allErrs = append(allErrs, IsValidValue(fieldPath.Child("dns", "type"), &value, kops.SupportedDnsTypes)...)
-		if value == string(kops.DNSTypeNone) && c.Spec.GetCloudProvider() != kops.CloudProviderHetzner {
+		if value == string(kops.DNSTypeNone) && cloud != kops.CloudProviderHetzner && cloud != kops.CloudProviderAWS {
 			allErrs = append(allErrs, field.Invalid(fieldPath.Child("dns", "type"), &value, fmt.Sprintf("not supported for %q", c.Spec.GetCloudProvider())))
 		}
 	}

pkg/model/awsmodel/api_loadbalancer.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog/v2"
 	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/wellknownports"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"
 	"k8s.io/kops/upup/pkg/fi/utils"
@@ -141,6 +142,12 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
 				TargetGroupName: b.NLBTargetGroupName("tcp"),
 			},
 		}
+		if b.Cluster.UsesNoneDNS() {
+			nlbListeners = append(nlbListeners, &awstasks.NetworkLoadBalancerListener{
+				Port:            wellknownports.KopsControllerPort,
+				TargetGroupName: b.NLBTargetGroupName("kops-controller"),
+			})
+		}
 
 		if lbSpec.SSLCertificate != "" {
 			listeners["443"].SSLCertificateID = lbSpec.SSLCertificate
@@ -219,7 +226,9 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
 			Tags: tags,
 		}
 
-		if lbSpec.CrossZoneLoadBalancing == nil {
+		if b.Cluster.UsesNoneDNS() {
+			lbSpec.CrossZoneLoadBalancing = fi.Bool(true)
+		} else if lbSpec.CrossZoneLoadBalancing == nil {
 			lbSpec.CrossZoneLoadBalancing = fi.Bool(false)
 		}
 
@@ -265,17 +274,18 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
 			c.AddTask(clb)
 		} else if b.APILoadBalancerClass() == kops.LoadBalancerClassNetwork {
 
-			tcpGroupName := b.NLBTargetGroupName("tcp")
+			{
-			tcpGroupTags := b.CloudTags(tcpGroupName, false)
+				groupName := b.NLBTargetGroupName("tcp")
+				groupTags := b.CloudTags(groupName, false)
 
 				// Override the returned name to be the expected NLB TG name
-			tcpGroupTags["Name"] = tcpGroupName
+				groupTags["Name"] = groupName
 
 				tg := &awstasks.TargetGroup{
-				Name:               fi.String(tcpGroupName),
+					Name:               fi.String(groupName),
 					Lifecycle:          b.Lifecycle,
 					VPC:                b.LinkToVPC(),
-				Tags:               tcpGroupTags,
+					Tags:               groupTags,
 					Protocol:           fi.String("TCP"),
 					Port:               fi.Int64(443),
 					Interval:           fi.Int64(10),
@@ -287,6 +297,32 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
 				c.AddTask(tg)
 
 				nlb.TargetGroups = append(nlb.TargetGroups, tg)
+			}
+
+			if b.Cluster.UsesNoneDNS() {
+				groupName := b.NLBTargetGroupName("kops-controller")
+				groupTags := b.CloudTags(groupName, false)
+
+				// Override the returned name to be the expected NLB TG name
+				groupTags["Name"] = groupName
+
+				tg := &awstasks.TargetGroup{
+					Name:               fi.String(groupName),
+					Lifecycle:          b.Lifecycle,
+					VPC:                b.LinkToVPC(),
+					Tags:               groupTags,
+					Protocol:           fi.String("TCP"),
+					Port:               fi.Int64(wellknownports.KopsControllerPort),
+					Interval:           fi.Int64(10),
+					HealthyThreshold:   fi.Int64(2),
+					UnhealthyThreshold: fi.Int64(2),
+					Shared:             fi.Bool(false),
+				}
+
+				c.AddTask(tg)
+
+				nlb.TargetGroups = append(nlb.TargetGroups, tg)
+			}
 
 			if lbSpec.SSLCertificate != "" {
 				tlsGroupName := b.NLBTargetGroupName("tls")
@@ -521,6 +557,33 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
 		}
 	}
 
+	// Allow kops-controller to the master instances from the ELB
+	if b.Cluster.UsesNoneDNS() && b.APILoadBalancerClass() == kops.LoadBalancerClassNetwork {
+		for _, masterGroup := range masterGroups {
+			suffix := masterGroup.Suffix
+			c.AddTask(&awstasks.SecurityGroupRule{
+				Name:          fi.String(fmt.Sprintf("kops-controller-lb-to-master%s", suffix)),
+				Lifecycle:     b.SecurityLifecycle,
+				FromPort:      fi.Int64(wellknownports.KopsControllerPort),
+				Protocol:      fi.String("tcp"),
+				SecurityGroup: masterGroup.Task,
+				ToPort:        fi.Int64(wellknownports.KopsControllerPort),
+				CIDR:          fi.String(b.Cluster.Spec.NetworkCIDR),
+			})
+			for _, cidr := range b.Cluster.Spec.AdditionalNetworkCIDRs {
+				c.AddTask(&awstasks.SecurityGroupRule{
+					Name:          fi.String(fmt.Sprintf("kops-controller-lb-to-master%s-%s", suffix, cidr)),
+					Lifecycle:     b.SecurityLifecycle,
+					FromPort:      fi.Int64(wellknownports.KopsControllerPort),
+					Protocol:      fi.String("tcp"),
+					SecurityGroup: masterGroup.Task,
+					ToPort:        fi.Int64(wellknownports.KopsControllerPort),
+					CIDR:          fi.String(cidr),
+				})
+			}
+		}
+	}
+
 	if b.Cluster.IsGossip() || b.Cluster.UsesPrivateDNS() || b.Cluster.UsesNoneDNS() {
 		// Ensure the LB hostname is included in the TLS certificate,
 		// if we're not going to use an alias for it

pkg/model/awsmodel/autoscalinggroup.go

@@ -451,6 +451,9 @@ func (b *AutoscalingGroupModelBuilder) buildAutoScalingGroupTask(c *fi.ModelBuil
 		if b.UseLoadBalancerForAPI() && ig.HasAPIServer() {
 			if b.UseNetworkLoadBalancer() {
 				t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("tcp"))
+				if b.Cluster.UsesNoneDNS() {
+					t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("kops-controller"))
+				}
 				if b.Cluster.Spec.API.LoadBalancer.SSLCertificate != "" {
 					t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("tls"))
 				}

upup/pkg/fi/cloudup/awstasks/network_load_balancer.go

@@ -435,21 +435,36 @@ func (e *NetworkLoadBalancer) IsForAPIServer() bool {
 }
 
 func (e *NetworkLoadBalancer) FindAddresses(context *fi.Context) ([]string, error) {
-	cloud := context.Cloud.(awsup.AWSCloud)
+	var addresses []string
 
+	cloud := context.Cloud.(awsup.AWSCloud)
+	cluster := context.Cluster
+
+	{
 		lb, err := cloud.FindELBV2ByNameTag(e.Tags["Name"])
 		if err != nil {
-		return nil, err
+			return nil, fmt.Errorf("failed to find load balancer matching %q: %w", e.Tags["Name"], err)
+		}
+		if lb != nil && fi.StringValue(lb.DNSName) != "" {
+			addresses = append(addresses, fi.StringValue(lb.DNSName))
 		}
-	if lb == nil {
-		return nil, nil
 	}
 
-	lbDnsName := fi.StringValue(lb.DNSName)
+	if cluster.UsesNoneDNS() {
-	if lbDnsName == "" {
+		nis, err := cloud.FindELBV2NetworkInterfacesByName(fi.StringValue(e.VPC.ID), fi.StringValue(e.LoadBalancerName))
-		return nil, nil
+		if err != nil {
+			return nil, fmt.Errorf("failed to find network interfaces matching %q: %w", fi.StringValue(e.LoadBalancerName), err)
 		}
-	return []string{lbDnsName}, nil
+		for _, ni := range nis {
+			if fi.StringValue(ni.PrivateIpAddress) != "" {
+				addresses = append(addresses, fi.StringValue(ni.PrivateIpAddress))
+			}
+		}
+	}
+
+	sort.Strings(addresses)
+
+	return addresses, nil
 }
 
 func (e *NetworkLoadBalancer) Run(c *fi.Context) error {
@@ -536,6 +551,7 @@ func (_ *NetworkLoadBalancer) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Ne
 
 		loadBalancerName = *e.LoadBalancerName
 
+		{
 			request := &elbv2.CreateLoadBalancerInput{}
 			request.Name = e.LoadBalancerName
 			request.Scheme = e.Scheme
@@ -551,22 +567,37 @@ func (_ *NetworkLoadBalancer) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Ne
 				})
 			}
 
-		{
+			klog.V(2).Infof("Creating NLB %q", loadBalancerName)
-			klog.V(2).Infof("Creating NLB with Name:%q", loadBalancerName)
 
 			response, err := t.Cloud.ELBV2().CreateLoadBalancer(request)
 			if err != nil {
-				return fmt.Errorf("error creating NLB: %v", err)
+				return fmt.Errorf("error creating NLB %q: %w", loadBalancerName, err)
+			}
+			if len(response.LoadBalancers) != 1 {
+				return fmt.Errorf("error creating NLB %q: found %d", loadBalancerName, len(response.LoadBalancers))
 			}
 
-			if len(response.LoadBalancers) != 1 {
-				return fmt.Errorf("Either too many or too few NLBs were created, wanted to find %q", loadBalancerName)
-			} else {
 			lb := response.LoadBalancers[0]
 			e.DNSName = lb.DNSName
 			e.HostedZoneId = lb.CanonicalHostedZoneId
 			e.VPC = &VPC{ID: lb.VpcId}
 			loadBalancerArn = fi.StringValue(lb.LoadBalancerArn)
+
+		}
+
+		// Wait for all load balancer components to be created (including network interfaces needed for NoneDNS).
+		// Limiting this to clusters using NoneDNS because load balancer creation is quite slow.
+		for _, tg := range e.TargetGroups {
+			if strings.HasPrefix(fi.StringValue(tg.Name), "kops-controller") {
+				klog.Infof("Waiting for load balancer %q to be created...", loadBalancerName)
+				request := &elbv2.DescribeLoadBalancersInput{
+					Names: []*string{&loadBalancerName},
+				}
+				err := t.Cloud.ELBV2().WaitUntilLoadBalancerAvailable(request)
+				if err != nil {
+					return fmt.Errorf("error waiting for NLB %q: %w", loadBalancerName, err)
+				}
+				break
 			}
 		}
 

upup/pkg/fi/cloudup/awstasks/targetgroup.go

@@ -178,6 +178,7 @@ func (_ *TargetGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *TargetGrou
 			Port:                       e.Port,
 			Protocol:                   e.Protocol,
 			VpcId:                      e.VPC.ID,
+			HealthCheckIntervalSeconds: e.Interval,
 			HealthyThresholdCount:      e.HealthyThreshold,
 			UnhealthyThresholdCount:    e.UnhealthyThreshold,
 			Tags:                       awsup.ELBv2Tags(e.Tags),

upup/pkg/fi/cloudup/awsup/aws_cloud.go

@@ -163,6 +163,7 @@ type AWSCloud interface {
 	DescribeELBTags(loadBalancerNames []string) (map[string][]*elb.Tag, error)
 	FindELBV2ByNameTag(findNameTag string) (*elbv2.LoadBalancer, error)
 	DescribeELBV2Tags(loadBalancerNames []string) (map[string][]*elbv2.Tag, error)
+	FindELBV2NetworkInterfacesByName(vpcID string, loadBalancerName string) ([]*ec2.NetworkInterface, error)
 
 	// DescribeInstance is a helper that queries for the specified instance by id
 	DescribeInstance(instanceID string) (*ec2.Instance, error)
@@ -1871,6 +1872,35 @@ func findELBV2ByNameTag(c AWSCloud, findNameTag string) (*elbv2.LoadBalancer, er
 	return found[0], nil
 }
 
+func (c *awsCloudImplementation) FindELBV2NetworkInterfacesByName(vpcID string, loadBalancerName string) ([]*ec2.NetworkInterface, error) {
+	return findELBV2NetworkInterfaces(c, vpcID, loadBalancerName)
+}
+
+func findELBV2NetworkInterfaces(c AWSCloud, vpcID, lbName string) ([]*ec2.NetworkInterface, error) {
+	klog.V(2).Infof("Listing all NLB network interfaces")
+
+	request := &ec2.DescribeNetworkInterfacesInput{
+		Filters: []*ec2.Filter{
+			NewEC2Filter("vpc-id", vpcID),
+			NewEC2Filter("interface-type", "network_load_balancer"),
+		},
+	}
+
+	response, err := c.EC2().DescribeNetworkInterfaces(request)
+	if err != nil {
+		return nil, fmt.Errorf("error describing network interfaces: %w", err)
+	}
+
+	var found []*ec2.NetworkInterface
+	for _, ni := range response.NetworkInterfaces {
+		if strings.HasPrefix(aws.StringValue(ni.Description), "ELB net/"+lbName+"/") {
+			found = append(found, ni)
+		}
+	}
+
+	return found, nil
+}
+
 func (c *awsCloudImplementation) DescribeELBV2Tags(loadBalancerArns []string) (map[string][]*elbv2.Tag, error) {
 	return describeELBV2Tags(c, loadBalancerArns)
 }

upup/pkg/fi/cloudup/awsup/mock_aws_cloud.go

@@ -214,6 +214,10 @@ func (c *MockAWSCloud) DescribeELBV2Tags(loadBalancerArns []string) (map[string]
 	return describeELBV2Tags(c, loadBalancerArns)
 }
 
+func (c *MockAWSCloud) FindELBV2NetworkInterfacesByName(vpcID, loadBalancerName string) ([]*ec2.NetworkInterface, error) {
+	return nil, nil
+}
+
 func (c *MockAWSCloud) DescribeInstance(instanceID string) (*ec2.Instance, error) {
 	return nil, fmt.Errorf("MockAWSCloud DescribeInstance not implemented")
 }