kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

aws: Create cluster without DNS or Gossip

This commit is contained in:
Ciprian Hacman 2022-10-25 17:16:34 +03:00
parent 231ce799bd
commit edb44610f7
8 changed files with 200 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
nodeup/pkg/model/etc_hosts.go
View File

@ -51,6 +51,12 @@ func (b *EtcHostsBuilder) Build(c *fi.ModelBuilderContext) error {
Hostname: b.Cluster.Spec.MasterInternalName,
Addresses: []string{b.BootConfig.APIServer},
})
if b.UseKopsControllerForNodeBootstrap() {
task.Records = append(task.Records, nodetasks.HostRecord{
Hostname: "kops-controller.internal." + b.Cluster.Name,
Addresses: []string{b.BootConfig.APIServer},
})
}
}
if len(task.Records) != 0 {

3
pkg/apis/kops/validation/validation.go
View File

@ -427,9 +427,10 @@ func validateTopology(c *kops.Cluster, topology *kops.TopologySpec, fieldPath *f
}
if topology.DNS != nil {
cloud := c.Spec.GetCloudProvider()
value := string(topology.DNS.Type)
allErrs = append(allErrs, IsValidValue(fieldPath.Child("dns", "type"), &value, kops.SupportedDnsTypes)...)
if value == string(kops.DNSTypeNone) && c.Spec.GetCloudProvider() != kops.CloudProviderHetzner {
if value == string(kops.DNSTypeNone) && cloud != kops.CloudProviderHetzner && cloud != kops.CloudProviderAWS {
allErrs = append(allErrs, field.Invalid(fieldPath.Child("dns", "type"), &value, fmt.Sprintf("not supported for %q", c.Spec.GetCloudProvider())))
}
}

99
pkg/model/awsmodel/api_loadbalancer.go
View File

@ -25,6 +25,7 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog/v2"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/wellknownports"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"
"k8s.io/kops/upup/pkg/fi/utils"
@ -141,6 +142,12 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
TargetGroupName: b.NLBTargetGroupName("tcp"),
},
}
if b.Cluster.UsesNoneDNS() {
nlbListeners = append(nlbListeners, &awstasks.NetworkLoadBalancerListener{
Port: wellknownports.KopsControllerPort,
TargetGroupName: b.NLBTargetGroupName("kops-controller"),
})
}
if lbSpec.SSLCertificate != "" {
listeners["443"].SSLCertificateID = lbSpec.SSLCertificate
@ -219,7 +226,9 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
Tags: tags,
}
if lbSpec.CrossZoneLoadBalancing == nil {
if b.Cluster.UsesNoneDNS() {
lbSpec.CrossZoneLoadBalancing = fi.Bool(true)
} else if lbSpec.CrossZoneLoadBalancing == nil {
lbSpec.CrossZoneLoadBalancing = fi.Bool(false)
}
@ -265,28 +274,55 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
c.AddTask(clb)
} else if b.APILoadBalancerClass() == kops.LoadBalancerClassNetwork {
tcpGroupName := b.NLBTargetGroupName("tcp")
tcpGroupTags := b.CloudTags(tcpGroupName, false)
{
groupName := b.NLBTargetGroupName("tcp")
groupTags := b.CloudTags(groupName, false)
// Override the returned name to be the expected NLB TG name
tcpGroupTags["Name"] = tcpGroupName
// Override the returned name to be the expected NLB TG name
groupTags["Name"] = groupName
tg := &awstasks.TargetGroup{
Name: fi.String(tcpGroupName),
Lifecycle: b.Lifecycle,
VPC: b.LinkToVPC(),
Tags: tcpGroupTags,
Protocol: fi.String("TCP"),
Port: fi.Int64(443),
Interval: fi.Int64(10),
HealthyThreshold: fi.Int64(2),
UnhealthyThreshold: fi.Int64(2),
Shared: fi.Bool(false),
tg := &awstasks.TargetGroup{
Name: fi.String(groupName),
Lifecycle: b.Lifecycle,
VPC: b.LinkToVPC(),
Tags: groupTags,
Protocol: fi.String("TCP"),
Port: fi.Int64(443),
Interval: fi.Int64(10),
HealthyThreshold: fi.Int64(2),
UnhealthyThreshold: fi.Int64(2),
Shared: fi.Bool(false),
}
c.AddTask(tg)
nlb.TargetGroups = append(nlb.TargetGroups, tg)
}
c.AddTask(tg)
if b.Cluster.UsesNoneDNS() {
groupName := b.NLBTargetGroupName("kops-controller")
groupTags := b.CloudTags(groupName, false)
nlb.TargetGroups = append(nlb.TargetGroups, tg)
// Override the returned name to be the expected NLB TG name
groupTags["Name"] = groupName
tg := &awstasks.TargetGroup{
Name: fi.String(groupName),
Lifecycle: b.Lifecycle,
VPC: b.LinkToVPC(),
Tags: groupTags,
Protocol: fi.String("TCP"),
Port: fi.Int64(wellknownports.KopsControllerPort),
Interval: fi.Int64(10),
HealthyThreshold: fi.Int64(2),
UnhealthyThreshold: fi.Int64(2),
Shared: fi.Bool(false),
}
c.AddTask(tg)
nlb.TargetGroups = append(nlb.TargetGroups, tg)
}
if lbSpec.SSLCertificate != "" {
tlsGroupName := b.NLBTargetGroupName("tls")
@ -521,6 +557,33 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
}
}
// Allow kops-controller to the master instances from the ELB
if b.Cluster.UsesNoneDNS() && b.APILoadBalancerClass() == kops.LoadBalancerClassNetwork {
for _, masterGroup := range masterGroups {
suffix := masterGroup.Suffix
c.AddTask(&awstasks.SecurityGroupRule{
Name: fi.String(fmt.Sprintf("kops-controller-lb-to-master%s", suffix)),
Lifecycle: b.SecurityLifecycle,
FromPort: fi.Int64(wellknownports.KopsControllerPort),
Protocol: fi.String("tcp"),
SecurityGroup: masterGroup.Task,
ToPort: fi.Int64(wellknownports.KopsControllerPort),
CIDR: fi.String(b.Cluster.Spec.NetworkCIDR),
})
for _, cidr := range b.Cluster.Spec.AdditionalNetworkCIDRs {
c.AddTask(&awstasks.SecurityGroupRule{
Name: fi.String(fmt.Sprintf("kops-controller-lb-to-master%s-%s", suffix, cidr)),
Lifecycle: b.SecurityLifecycle,
FromPort: fi.Int64(wellknownports.KopsControllerPort),
Protocol: fi.String("tcp"),
SecurityGroup: masterGroup.Task,
ToPort: fi.Int64(wellknownports.KopsControllerPort),
CIDR: fi.String(cidr),
})
}
}
}
if b.Cluster.IsGossip() || b.Cluster.UsesPrivateDNS() || b.Cluster.UsesNoneDNS() {
// Ensure the LB hostname is included in the TLS certificate,
// if we're not going to use an alias for it

3
pkg/model/awsmodel/autoscalinggroup.go
View File

@ -451,6 +451,9 @@ func (b *AutoscalingGroupModelBuilder) buildAutoScalingGroupTask(c *fi.ModelBuil
if b.UseLoadBalancerForAPI() && ig.HasAPIServer() {
if b.UseNetworkLoadBalancer() {
t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("tcp"))
if b.Cluster.UsesNoneDNS() {
t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("kops-controller"))
}
if b.Cluster.Spec.API.LoadBalancer.SSLCertificate != "" {
t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("tls"))
}

101
upup/pkg/fi/cloudup/awstasks/network_load_balancer.go
View File

@ -435,21 +435,36 @@ func (e *NetworkLoadBalancer) IsForAPIServer() bool {
}
func (e *NetworkLoadBalancer) FindAddresses(context *fi.Context) ([]string, error) {
var addresses []string
cloud := context.Cloud.(awsup.AWSCloud)
cluster := context.Cluster
lb, err := cloud.FindELBV2ByNameTag(e.Tags["Name"])
if err != nil {
return nil, err
}
if lb == nil {
return nil, nil
{
lb, err := cloud.FindELBV2ByNameTag(e.Tags["Name"])
if err != nil {
return nil, fmt.Errorf("failed to find load balancer matching %q: %w", e.Tags["Name"], err)
}
if lb != nil && fi.StringValue(lb.DNSName) != "" {
addresses = append(addresses, fi.StringValue(lb.DNSName))
}
}
lbDnsName := fi.StringValue(lb.DNSName)
if lbDnsName == "" {
return nil, nil
if cluster.UsesNoneDNS() {
nis, err := cloud.FindELBV2NetworkInterfacesByName(fi.StringValue(e.VPC.ID), fi.StringValue(e.LoadBalancerName))
if err != nil {
return nil, fmt.Errorf("failed to find network interfaces matching %q: %w", fi.StringValue(e.LoadBalancerName), err)
}
for _, ni := range nis {
if fi.StringValue(ni.PrivateIpAddress) != "" {
addresses = append(addresses, fi.StringValue(ni.PrivateIpAddress))
}
}
}
return []string{lbDnsName}, nil
sort.Strings(addresses)
return addresses, nil
}
func (e *NetworkLoadBalancer) Run(c *fi.Context) error {
@ -536,37 +551,53 @@ func (_ *NetworkLoadBalancer) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Ne
loadBalancerName = *e.LoadBalancerName
request := &elbv2.CreateLoadBalancerInput{}
request.Name = e.LoadBalancerName
request.Scheme = e.Scheme
request.Type = e.Type
request.IpAddressType = e.IpAddressType
request.Tags = awsup.ELBv2Tags(e.Tags)
for _, subnetMapping := range e.SubnetMappings {
request.SubnetMappings = append(request.SubnetMappings, &elbv2.SubnetMapping{
SubnetId: subnetMapping.Subnet.ID,
AllocationId: subnetMapping.AllocationID,
PrivateIPv4Address: subnetMapping.PrivateIPv4Address,
})
}
{
klog.V(2).Infof("Creating NLB with Name:%q", loadBalancerName)
request := &elbv2.CreateLoadBalancerInput{}
request.Name = e.LoadBalancerName
request.Scheme = e.Scheme
request.Type = e.Type
request.IpAddressType = e.IpAddressType
request.Tags = awsup.ELBv2Tags(e.Tags)
for _, subnetMapping := range e.SubnetMappings {
request.SubnetMappings = append(request.SubnetMappings, &elbv2.SubnetMapping{
SubnetId: subnetMapping.Subnet.ID,
AllocationId: subnetMapping.AllocationID,
PrivateIPv4Address: subnetMapping.PrivateIPv4Address,
})
}
klog.V(2).Infof("Creating NLB %q", loadBalancerName)
response, err := t.Cloud.ELBV2().CreateLoadBalancer(request)
if err != nil {
return fmt.Errorf("error creating NLB: %v", err)
return fmt.Errorf("error creating NLB %q: %w", loadBalancerName, err)
}
if len(response.LoadBalancers) != 1 {
return fmt.Errorf("error creating NLB %q: found %d", loadBalancerName, len(response.LoadBalancers))
}
if len(response.LoadBalancers) != 1 {
return fmt.Errorf("Either too many or too few NLBs were created, wanted to find %q", loadBalancerName)
} else {
lb := response.LoadBalancers[0]
e.DNSName = lb.DNSName
e.HostedZoneId = lb.CanonicalHostedZoneId
e.VPC = &VPC{ID: lb.VpcId}
loadBalancerArn = fi.StringValue(lb.LoadBalancerArn)
lb := response.LoadBalancers[0]
e.DNSName = lb.DNSName
e.HostedZoneId = lb.CanonicalHostedZoneId
e.VPC = &VPC{ID: lb.VpcId}
loadBalancerArn = fi.StringValue(lb.LoadBalancerArn)
}
// Wait for all load balancer components to be created (including network interfaces needed for NoneDNS).
// Limiting this to clusters using NoneDNS because load balancer creation is quite slow.
for _, tg := range e.TargetGroups {
if strings.HasPrefix(fi.StringValue(tg.Name), "kops-controller") {
klog.Infof("Waiting for load balancer %q to be created...", loadBalancerName)
request := &elbv2.DescribeLoadBalancersInput{
Names: []*string{&loadBalancerName},
}
err := t.Cloud.ELBV2().WaitUntilLoadBalancerAvailable(request)
if err != nil {
return fmt.Errorf("error waiting for NLB %q: %w", loadBalancerName, err)
}
break
}
}

15
upup/pkg/fi/cloudup/awstasks/targetgroup.go
View File

@ -174,13 +174,14 @@ func (_ *TargetGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *TargetGrou
if a == nil {
request := &elbv2.CreateTargetGroupInput{
Name: e.Name,
Port: e.Port,
Protocol: e.Protocol,
VpcId: e.VPC.ID,
HealthyThresholdCount: e.HealthyThreshold,
UnhealthyThresholdCount: e.UnhealthyThreshold,
Tags: awsup.ELBv2Tags(e.Tags),
Name: e.Name,
Port: e.Port,
Protocol: e.Protocol,
VpcId: e.VPC.ID,
HealthCheckIntervalSeconds: e.Interval,
HealthyThresholdCount: e.HealthyThreshold,
UnhealthyThresholdCount: e.UnhealthyThreshold,
Tags: awsup.ELBv2Tags(e.Tags),
}
klog.V(2).Infof("Creating Target Group for NLB")

30
upup/pkg/fi/cloudup/awsup/aws_cloud.go
View File

@ -163,6 +163,7 @@ type AWSCloud interface {
DescribeELBTags(loadBalancerNames []string) (map[string][]*elb.Tag, error)
FindELBV2ByNameTag(findNameTag string) (*elbv2.LoadBalancer, error)
DescribeELBV2Tags(loadBalancerNames []string) (map[string][]*elbv2.Tag, error)
FindELBV2NetworkInterfacesByName(vpcID string, loadBalancerName string) ([]*ec2.NetworkInterface, error)
// DescribeInstance is a helper that queries for the specified instance by id
DescribeInstance(instanceID string) (*ec2.Instance, error)
@ -1871,6 +1872,35 @@ func findELBV2ByNameTag(c AWSCloud, findNameTag string) (*elbv2.LoadBalancer, er
return found[0], nil
}
func (c *awsCloudImplementation) FindELBV2NetworkInterfacesByName(vpcID string, loadBalancerName string) ([]*ec2.NetworkInterface, error) {
return findELBV2NetworkInterfaces(c, vpcID, loadBalancerName)
}
func findELBV2NetworkInterfaces(c AWSCloud, vpcID, lbName string) ([]*ec2.NetworkInterface, error) {
klog.V(2).Infof("Listing all NLB network interfaces")
request := &ec2.DescribeNetworkInterfacesInput{
Filters: []*ec2.Filter{
NewEC2Filter("vpc-id", vpcID),
NewEC2Filter("interface-type", "network_load_balancer"),
},
}
response, err := c.EC2().DescribeNetworkInterfaces(request)
if err != nil {
return nil, fmt.Errorf("error describing network interfaces: %w", err)
}
var found []*ec2.NetworkInterface
for _, ni := range response.NetworkInterfaces {
if strings.HasPrefix(aws.StringValue(ni.Description), "ELB net/"+lbName+"/") {
found = append(found, ni)
}
}
return found, nil
}
func (c *awsCloudImplementation) DescribeELBV2Tags(loadBalancerArns []string) (map[string][]*elbv2.Tag, error) {
return describeELBV2Tags(c, loadBalancerArns)
}

4
upup/pkg/fi/cloudup/awsup/mock_aws_cloud.go
View File

@ -214,6 +214,10 @@ func (c *MockAWSCloud) DescribeELBV2Tags(loadBalancerArns []string) (map[string]
return describeELBV2Tags(c, loadBalancerArns)
}
func (c *MockAWSCloud) FindELBV2NetworkInterfacesByName(vpcID, loadBalancerName string) ([]*ec2.NetworkInterface, error) {
return nil, nil
}
func (c *MockAWSCloud) DescribeInstance(instanceID string) (*ec2.Instance, error) {
return nil, fmt.Errorf("MockAWSCloud DescribeInstance not implemented")
}