diff --git a/nodeup/pkg/model/etc_hosts.go b/nodeup/pkg/model/etc_hosts.go index 9b460e1b3b..cc0f93874d 100644 --- a/nodeup/pkg/model/etc_hosts.go +++ b/nodeup/pkg/model/etc_hosts.go @@ -51,6 +51,12 @@ func (b *EtcHostsBuilder) Build(c *fi.ModelBuilderContext) error { Hostname: b.Cluster.Spec.MasterInternalName, Addresses: []string{b.BootConfig.APIServer}, }) + if b.UseKopsControllerForNodeBootstrap() { + task.Records = append(task.Records, nodetasks.HostRecord{ + Hostname: "kops-controller.internal." + b.Cluster.Name, + Addresses: []string{b.BootConfig.APIServer}, + }) + } } if len(task.Records) != 0 { diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index 37a8a2f7b9..8666aa567f 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -427,9 +427,10 @@ func validateTopology(c *kops.Cluster, topology *kops.TopologySpec, fieldPath *f } if topology.DNS != nil { + cloud := c.Spec.GetCloudProvider() value := string(topology.DNS.Type) allErrs = append(allErrs, IsValidValue(fieldPath.Child("dns", "type"), &value, kops.SupportedDnsTypes)...) - if value == string(kops.DNSTypeNone) && c.Spec.GetCloudProvider() != kops.CloudProviderHetzner { + if value == string(kops.DNSTypeNone) && cloud != kops.CloudProviderHetzner && cloud != kops.CloudProviderAWS { allErrs = append(allErrs, field.Invalid(fieldPath.Child("dns", "type"), &value, fmt.Sprintf("not supported for %q", c.Spec.GetCloudProvider()))) } } diff --git a/pkg/model/awsmodel/api_loadbalancer.go b/pkg/model/awsmodel/api_loadbalancer.go index 3828843e03..60d068ab1f 100644 --- a/pkg/model/awsmodel/api_loadbalancer.go +++ b/pkg/model/awsmodel/api_loadbalancer.go @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/wellknownports" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awstasks" "k8s.io/kops/upup/pkg/fi/utils" @@ -141,6 +142,12 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { TargetGroupName: b.NLBTargetGroupName("tcp"), }, } + if b.Cluster.UsesNoneDNS() { + nlbListeners = append(nlbListeners, &awstasks.NetworkLoadBalancerListener{ + Port: wellknownports.KopsControllerPort, + TargetGroupName: b.NLBTargetGroupName("kops-controller"), + }) + } if lbSpec.SSLCertificate != "" { listeners["443"].SSLCertificateID = lbSpec.SSLCertificate @@ -219,7 +226,9 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { Tags: tags, } - if lbSpec.CrossZoneLoadBalancing == nil { + if b.Cluster.UsesNoneDNS() { + lbSpec.CrossZoneLoadBalancing = fi.Bool(true) + } else if lbSpec.CrossZoneLoadBalancing == nil { lbSpec.CrossZoneLoadBalancing = fi.Bool(false) } @@ -265,28 +274,55 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { c.AddTask(clb) } else if b.APILoadBalancerClass() == kops.LoadBalancerClassNetwork { - tcpGroupName := b.NLBTargetGroupName("tcp") - tcpGroupTags := b.CloudTags(tcpGroupName, false) + { + groupName := b.NLBTargetGroupName("tcp") + groupTags := b.CloudTags(groupName, false) - // Override the returned name to be the expected NLB TG name - tcpGroupTags["Name"] = tcpGroupName + // Override the returned name to be the expected NLB TG name + groupTags["Name"] = groupName - tg := &awstasks.TargetGroup{ - Name: fi.String(tcpGroupName), - Lifecycle: b.Lifecycle, - VPC: b.LinkToVPC(), - Tags: tcpGroupTags, - Protocol: fi.String("TCP"), - Port: fi.Int64(443), - Interval: fi.Int64(10), - HealthyThreshold: fi.Int64(2), - UnhealthyThreshold: fi.Int64(2), - Shared: fi.Bool(false), + tg := &awstasks.TargetGroup{ + Name: fi.String(groupName), + Lifecycle: b.Lifecycle, + VPC: b.LinkToVPC(), + Tags: groupTags, + Protocol: fi.String("TCP"), + Port: fi.Int64(443), + Interval: fi.Int64(10), + HealthyThreshold: fi.Int64(2), + UnhealthyThreshold: fi.Int64(2), + Shared: fi.Bool(false), + } + + c.AddTask(tg) + + nlb.TargetGroups = append(nlb.TargetGroups, tg) } - c.AddTask(tg) + if b.Cluster.UsesNoneDNS() { + groupName := b.NLBTargetGroupName("kops-controller") + groupTags := b.CloudTags(groupName, false) - nlb.TargetGroups = append(nlb.TargetGroups, tg) + // Override the returned name to be the expected NLB TG name + groupTags["Name"] = groupName + + tg := &awstasks.TargetGroup{ + Name: fi.String(groupName), + Lifecycle: b.Lifecycle, + VPC: b.LinkToVPC(), + Tags: groupTags, + Protocol: fi.String("TCP"), + Port: fi.Int64(wellknownports.KopsControllerPort), + Interval: fi.Int64(10), + HealthyThreshold: fi.Int64(2), + UnhealthyThreshold: fi.Int64(2), + Shared: fi.Bool(false), + } + + c.AddTask(tg) + + nlb.TargetGroups = append(nlb.TargetGroups, tg) + } if lbSpec.SSLCertificate != "" { tlsGroupName := b.NLBTargetGroupName("tls") @@ -521,6 +557,33 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { } } + // Allow kops-controller to the master instances from the ELB + if b.Cluster.UsesNoneDNS() && b.APILoadBalancerClass() == kops.LoadBalancerClassNetwork { + for _, masterGroup := range masterGroups { + suffix := masterGroup.Suffix + c.AddTask(&awstasks.SecurityGroupRule{ + Name: fi.String(fmt.Sprintf("kops-controller-lb-to-master%s", suffix)), + Lifecycle: b.SecurityLifecycle, + FromPort: fi.Int64(wellknownports.KopsControllerPort), + Protocol: fi.String("tcp"), + SecurityGroup: masterGroup.Task, + ToPort: fi.Int64(wellknownports.KopsControllerPort), + CIDR: fi.String(b.Cluster.Spec.NetworkCIDR), + }) + for _, cidr := range b.Cluster.Spec.AdditionalNetworkCIDRs { + c.AddTask(&awstasks.SecurityGroupRule{ + Name: fi.String(fmt.Sprintf("kops-controller-lb-to-master%s-%s", suffix, cidr)), + Lifecycle: b.SecurityLifecycle, + FromPort: fi.Int64(wellknownports.KopsControllerPort), + Protocol: fi.String("tcp"), + SecurityGroup: masterGroup.Task, + ToPort: fi.Int64(wellknownports.KopsControllerPort), + CIDR: fi.String(cidr), + }) + } + } + } + if b.Cluster.IsGossip() || b.Cluster.UsesPrivateDNS() || b.Cluster.UsesNoneDNS() { // Ensure the LB hostname is included in the TLS certificate, // if we're not going to use an alias for it diff --git a/pkg/model/awsmodel/autoscalinggroup.go b/pkg/model/awsmodel/autoscalinggroup.go index 3f7978d4c2..62a8411bb0 100644 --- a/pkg/model/awsmodel/autoscalinggroup.go +++ b/pkg/model/awsmodel/autoscalinggroup.go @@ -451,6 +451,9 @@ func (b *AutoscalingGroupModelBuilder) buildAutoScalingGroupTask(c *fi.ModelBuil if b.UseLoadBalancerForAPI() && ig.HasAPIServer() { if b.UseNetworkLoadBalancer() { t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("tcp")) + if b.Cluster.UsesNoneDNS() { + t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("kops-controller")) + } if b.Cluster.Spec.API.LoadBalancer.SSLCertificate != "" { t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("tls")) } diff --git a/upup/pkg/fi/cloudup/awstasks/network_load_balancer.go b/upup/pkg/fi/cloudup/awstasks/network_load_balancer.go index 1f21fefca5..4ecddd7ae8 100644 --- a/upup/pkg/fi/cloudup/awstasks/network_load_balancer.go +++ b/upup/pkg/fi/cloudup/awstasks/network_load_balancer.go @@ -435,21 +435,36 @@ func (e *NetworkLoadBalancer) IsForAPIServer() bool { } func (e *NetworkLoadBalancer) FindAddresses(context *fi.Context) ([]string, error) { + var addresses []string + cloud := context.Cloud.(awsup.AWSCloud) + cluster := context.Cluster - lb, err := cloud.FindELBV2ByNameTag(e.Tags["Name"]) - if err != nil { - return nil, err - } - if lb == nil { - return nil, nil + { + lb, err := cloud.FindELBV2ByNameTag(e.Tags["Name"]) + if err != nil { + return nil, fmt.Errorf("failed to find load balancer matching %q: %w", e.Tags["Name"], err) + } + if lb != nil && fi.StringValue(lb.DNSName) != "" { + addresses = append(addresses, fi.StringValue(lb.DNSName)) + } } - lbDnsName := fi.StringValue(lb.DNSName) - if lbDnsName == "" { - return nil, nil + if cluster.UsesNoneDNS() { + nis, err := cloud.FindELBV2NetworkInterfacesByName(fi.StringValue(e.VPC.ID), fi.StringValue(e.LoadBalancerName)) + if err != nil { + return nil, fmt.Errorf("failed to find network interfaces matching %q: %w", fi.StringValue(e.LoadBalancerName), err) + } + for _, ni := range nis { + if fi.StringValue(ni.PrivateIpAddress) != "" { + addresses = append(addresses, fi.StringValue(ni.PrivateIpAddress)) + } + } } - return []string{lbDnsName}, nil + + sort.Strings(addresses) + + return addresses, nil } func (e *NetworkLoadBalancer) Run(c *fi.Context) error { @@ -536,37 +551,53 @@ func (_ *NetworkLoadBalancer) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Ne loadBalancerName = *e.LoadBalancerName - request := &elbv2.CreateLoadBalancerInput{} - request.Name = e.LoadBalancerName - request.Scheme = e.Scheme - request.Type = e.Type - request.IpAddressType = e.IpAddressType - request.Tags = awsup.ELBv2Tags(e.Tags) - - for _, subnetMapping := range e.SubnetMappings { - request.SubnetMappings = append(request.SubnetMappings, &elbv2.SubnetMapping{ - SubnetId: subnetMapping.Subnet.ID, - AllocationId: subnetMapping.AllocationID, - PrivateIPv4Address: subnetMapping.PrivateIPv4Address, - }) - } - { - klog.V(2).Infof("Creating NLB with Name:%q", loadBalancerName) + request := &elbv2.CreateLoadBalancerInput{} + request.Name = e.LoadBalancerName + request.Scheme = e.Scheme + request.Type = e.Type + request.IpAddressType = e.IpAddressType + request.Tags = awsup.ELBv2Tags(e.Tags) + + for _, subnetMapping := range e.SubnetMappings { + request.SubnetMappings = append(request.SubnetMappings, &elbv2.SubnetMapping{ + SubnetId: subnetMapping.Subnet.ID, + AllocationId: subnetMapping.AllocationID, + PrivateIPv4Address: subnetMapping.PrivateIPv4Address, + }) + } + + klog.V(2).Infof("Creating NLB %q", loadBalancerName) response, err := t.Cloud.ELBV2().CreateLoadBalancer(request) if err != nil { - return fmt.Errorf("error creating NLB: %v", err) + return fmt.Errorf("error creating NLB %q: %w", loadBalancerName, err) + } + if len(response.LoadBalancers) != 1 { + return fmt.Errorf("error creating NLB %q: found %d", loadBalancerName, len(response.LoadBalancers)) } - if len(response.LoadBalancers) != 1 { - return fmt.Errorf("Either too many or too few NLBs were created, wanted to find %q", loadBalancerName) - } else { - lb := response.LoadBalancers[0] - e.DNSName = lb.DNSName - e.HostedZoneId = lb.CanonicalHostedZoneId - e.VPC = &VPC{ID: lb.VpcId} - loadBalancerArn = fi.StringValue(lb.LoadBalancerArn) + lb := response.LoadBalancers[0] + e.DNSName = lb.DNSName + e.HostedZoneId = lb.CanonicalHostedZoneId + e.VPC = &VPC{ID: lb.VpcId} + loadBalancerArn = fi.StringValue(lb.LoadBalancerArn) + + } + + // Wait for all load balancer components to be created (including network interfaces needed for NoneDNS). + // Limiting this to clusters using NoneDNS because load balancer creation is quite slow. + for _, tg := range e.TargetGroups { + if strings.HasPrefix(fi.StringValue(tg.Name), "kops-controller") { + klog.Infof("Waiting for load balancer %q to be created...", loadBalancerName) + request := &elbv2.DescribeLoadBalancersInput{ + Names: []*string{&loadBalancerName}, + } + err := t.Cloud.ELBV2().WaitUntilLoadBalancerAvailable(request) + if err != nil { + return fmt.Errorf("error waiting for NLB %q: %w", loadBalancerName, err) + } + break } } diff --git a/upup/pkg/fi/cloudup/awstasks/targetgroup.go b/upup/pkg/fi/cloudup/awstasks/targetgroup.go index 5411b7141f..3dd2496a3f 100644 --- a/upup/pkg/fi/cloudup/awstasks/targetgroup.go +++ b/upup/pkg/fi/cloudup/awstasks/targetgroup.go @@ -174,13 +174,14 @@ func (_ *TargetGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *TargetGrou if a == nil { request := &elbv2.CreateTargetGroupInput{ - Name: e.Name, - Port: e.Port, - Protocol: e.Protocol, - VpcId: e.VPC.ID, - HealthyThresholdCount: e.HealthyThreshold, - UnhealthyThresholdCount: e.UnhealthyThreshold, - Tags: awsup.ELBv2Tags(e.Tags), + Name: e.Name, + Port: e.Port, + Protocol: e.Protocol, + VpcId: e.VPC.ID, + HealthCheckIntervalSeconds: e.Interval, + HealthyThresholdCount: e.HealthyThreshold, + UnhealthyThresholdCount: e.UnhealthyThreshold, + Tags: awsup.ELBv2Tags(e.Tags), } klog.V(2).Infof("Creating Target Group for NLB") diff --git a/upup/pkg/fi/cloudup/awsup/aws_cloud.go b/upup/pkg/fi/cloudup/awsup/aws_cloud.go index ae019cda44..b6c7e284dd 100644 --- a/upup/pkg/fi/cloudup/awsup/aws_cloud.go +++ b/upup/pkg/fi/cloudup/awsup/aws_cloud.go @@ -163,6 +163,7 @@ type AWSCloud interface { DescribeELBTags(loadBalancerNames []string) (map[string][]*elb.Tag, error) FindELBV2ByNameTag(findNameTag string) (*elbv2.LoadBalancer, error) DescribeELBV2Tags(loadBalancerNames []string) (map[string][]*elbv2.Tag, error) + FindELBV2NetworkInterfacesByName(vpcID string, loadBalancerName string) ([]*ec2.NetworkInterface, error) // DescribeInstance is a helper that queries for the specified instance by id DescribeInstance(instanceID string) (*ec2.Instance, error) @@ -1871,6 +1872,35 @@ func findELBV2ByNameTag(c AWSCloud, findNameTag string) (*elbv2.LoadBalancer, er return found[0], nil } +func (c *awsCloudImplementation) FindELBV2NetworkInterfacesByName(vpcID string, loadBalancerName string) ([]*ec2.NetworkInterface, error) { + return findELBV2NetworkInterfaces(c, vpcID, loadBalancerName) +} + +func findELBV2NetworkInterfaces(c AWSCloud, vpcID, lbName string) ([]*ec2.NetworkInterface, error) { + klog.V(2).Infof("Listing all NLB network interfaces") + + request := &ec2.DescribeNetworkInterfacesInput{ + Filters: []*ec2.Filter{ + NewEC2Filter("vpc-id", vpcID), + NewEC2Filter("interface-type", "network_load_balancer"), + }, + } + + response, err := c.EC2().DescribeNetworkInterfaces(request) + if err != nil { + return nil, fmt.Errorf("error describing network interfaces: %w", err) + } + + var found []*ec2.NetworkInterface + for _, ni := range response.NetworkInterfaces { + if strings.HasPrefix(aws.StringValue(ni.Description), "ELB net/"+lbName+"/") { + found = append(found, ni) + } + } + + return found, nil +} + func (c *awsCloudImplementation) DescribeELBV2Tags(loadBalancerArns []string) (map[string][]*elbv2.Tag, error) { return describeELBV2Tags(c, loadBalancerArns) } diff --git a/upup/pkg/fi/cloudup/awsup/mock_aws_cloud.go b/upup/pkg/fi/cloudup/awsup/mock_aws_cloud.go index 1a6dc8c6c8..947ea11c85 100644 --- a/upup/pkg/fi/cloudup/awsup/mock_aws_cloud.go +++ b/upup/pkg/fi/cloudup/awsup/mock_aws_cloud.go @@ -214,6 +214,10 @@ func (c *MockAWSCloud) DescribeELBV2Tags(loadBalancerArns []string) (map[string] return describeELBV2Tags(c, loadBalancerArns) } +func (c *MockAWSCloud) FindELBV2NetworkInterfacesByName(vpcID, loadBalancerName string) ([]*ec2.NetworkInterface, error) { + return nil, nil +} + func (c *MockAWSCloud) DescribeInstance(instanceID string) (*ec2.Instance, error) { return nil, fmt.Errorf("MockAWSCloud DescribeInstance not implemented") }