mirror of https://github.com/kubernetes/kops.git
416 lines
12 KiB
Go
416 lines
12 KiB
Go
/*
|
|
Copyright 2019 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package awsmodel
|
|
|
|
import (
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
|
|
"k8s.io/apimachinery/pkg/util/sets"
|
|
"k8s.io/klog/v2"
|
|
"k8s.io/kops/pkg/apis/kops"
|
|
"k8s.io/kops/upup/pkg/fi"
|
|
"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"
|
|
"k8s.io/kops/upup/pkg/fi/utils"
|
|
)
|
|
|
|
// BastionModelBuilder adds model objects to support bastions
|
|
//
|
|
// Bastion instances live in the utility subnets created in the private topology.
|
|
// All traffic goes through an ELB, and the ELB has port 22 open to SSHAccess.
|
|
// Bastion instances have access to all internal master and node instances.
|
|
|
|
type BastionModelBuilder struct {
|
|
*AWSModelContext
|
|
Lifecycle fi.Lifecycle
|
|
SecurityLifecycle fi.Lifecycle
|
|
}
|
|
|
|
var _ fi.CloudupModelBuilder = &BastionModelBuilder{}
|
|
|
|
func (b *BastionModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
|
|
var bastionInstanceGroups []*kops.InstanceGroup
|
|
for _, ig := range b.InstanceGroups {
|
|
if ig.Spec.Role == kops.InstanceGroupRoleBastion {
|
|
bastionInstanceGroups = append(bastionInstanceGroups, ig)
|
|
}
|
|
}
|
|
|
|
if len(bastionInstanceGroups) == 0 {
|
|
return nil
|
|
}
|
|
|
|
bastionGroups, err := b.GetSecurityGroups(kops.InstanceGroupRoleBastion)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
nodeGroups, err := b.GetSecurityGroups(kops.InstanceGroupRoleNode)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
masterGroups, err := b.GetSecurityGroups(kops.InstanceGroupRoleControlPlane)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Create security group for bastion instances
|
|
for _, bastionGroup := range bastionGroups {
|
|
bastionGroup.Task.Lifecycle = b.SecurityLifecycle
|
|
c.AddTask(bastionGroup.Task)
|
|
}
|
|
|
|
for _, src := range bastionGroups {
|
|
// Allow traffic from bastion instances to egress freely
|
|
{
|
|
t := &awstasks.SecurityGroupRule{
|
|
Name: fi.PtrTo("ipv4-bastion-egress" + src.Suffix),
|
|
Lifecycle: b.SecurityLifecycle,
|
|
SecurityGroup: src.Task,
|
|
Egress: fi.PtrTo(true),
|
|
CIDR: fi.PtrTo("0.0.0.0/0"),
|
|
}
|
|
AddDirectionalGroupRule(c, t)
|
|
}
|
|
{
|
|
t := &awstasks.SecurityGroupRule{
|
|
Name: fi.PtrTo("ipv6-bastion-egress" + src.Suffix),
|
|
Lifecycle: b.SecurityLifecycle,
|
|
SecurityGroup: src.Task,
|
|
Egress: fi.PtrTo(true),
|
|
IPv6CIDR: fi.PtrTo("::/0"),
|
|
}
|
|
AddDirectionalGroupRule(c, t)
|
|
}
|
|
}
|
|
|
|
var bastionLoadBalancerType kops.LoadBalancerType
|
|
{
|
|
// Check if we requested a public or internal NLB
|
|
if b.Cluster.Spec.Networking.Topology != nil && b.Cluster.Spec.Networking.Topology.Bastion != nil && b.Cluster.Spec.Networking.Topology.Bastion.LoadBalancer != nil {
|
|
if b.Cluster.Spec.Networking.Topology.Bastion.LoadBalancer.Type != "" {
|
|
switch b.Cluster.Spec.Networking.Topology.Bastion.LoadBalancer.Type {
|
|
case kops.LoadBalancerTypeInternal:
|
|
bastionLoadBalancerType = "Internal"
|
|
case kops.LoadBalancerTypePublic:
|
|
bastionLoadBalancerType = "Public"
|
|
default:
|
|
return fmt.Errorf("unhandled bastion LoadBalancer type %q", b.Cluster.Spec.Networking.Topology.Bastion.LoadBalancer.Type)
|
|
}
|
|
} else {
|
|
// Default to Public
|
|
b.Cluster.Spec.Networking.Topology.Bastion.LoadBalancer.Type = kops.LoadBalancerTypePublic
|
|
bastionLoadBalancerType = "Public"
|
|
}
|
|
} else {
|
|
// Default to Public
|
|
bastionLoadBalancerType = "Public"
|
|
}
|
|
}
|
|
|
|
// Allow bastion nodes to SSH to masters
|
|
for _, src := range bastionGroups {
|
|
for _, dest := range masterGroups {
|
|
t := &awstasks.SecurityGroupRule{
|
|
Name: fi.PtrTo("bastion-to-master-ssh" + JoinSuffixes(src, dest)),
|
|
Lifecycle: b.SecurityLifecycle,
|
|
SecurityGroup: dest.Task,
|
|
SourceGroup: src.Task,
|
|
Protocol: fi.PtrTo("tcp"),
|
|
FromPort: fi.PtrTo(int64(22)),
|
|
ToPort: fi.PtrTo(int64(22)),
|
|
}
|
|
AddDirectionalGroupRule(c, t)
|
|
}
|
|
}
|
|
|
|
// Allow bastion nodes to SSH to nodes
|
|
for _, src := range bastionGroups {
|
|
for _, dest := range nodeGroups {
|
|
t := &awstasks.SecurityGroupRule{
|
|
Name: fi.PtrTo("bastion-to-node-ssh" + JoinSuffixes(src, dest)),
|
|
Lifecycle: b.SecurityLifecycle,
|
|
SecurityGroup: dest.Task,
|
|
SourceGroup: src.Task,
|
|
Protocol: fi.PtrTo("tcp"),
|
|
FromPort: fi.PtrTo(int64(22)),
|
|
ToPort: fi.PtrTo(int64(22)),
|
|
}
|
|
AddDirectionalGroupRule(c, t)
|
|
}
|
|
}
|
|
|
|
var sshAllowedCIDRs []string
|
|
var nlbSubnetMappings []*awstasks.SubnetMapping
|
|
{
|
|
// Compute the subnets - only one per zone, and then break ties based on chooseBestSubnetForNLB
|
|
subnetsByZone := make(map[string][]*kops.ClusterSubnetSpec)
|
|
for i := range b.Cluster.Spec.Networking.Subnets {
|
|
subnet := &b.Cluster.Spec.Networking.Subnets[i]
|
|
|
|
switch subnet.Type {
|
|
case kops.SubnetTypePublic, kops.SubnetTypeUtility:
|
|
if bastionLoadBalancerType != kops.LoadBalancerTypePublic {
|
|
continue
|
|
}
|
|
|
|
case kops.SubnetTypeDualStack, kops.SubnetTypePrivate:
|
|
if bastionLoadBalancerType != kops.LoadBalancerTypeInternal {
|
|
continue
|
|
}
|
|
|
|
default:
|
|
return fmt.Errorf("subnet %q had unknown type %q", subnet.Name, subnet.Type)
|
|
}
|
|
|
|
subnetsByZone[subnet.Zone] = append(subnetsByZone[subnet.Zone], subnet)
|
|
}
|
|
|
|
for zone, subnets := range subnetsByZone {
|
|
for _, subnet := range subnets {
|
|
sshAllowedCIDRs = append(sshAllowedCIDRs, subnet.CIDR)
|
|
}
|
|
subnet := b.chooseBestSubnetForNLB(zone, subnets)
|
|
nlbSubnetMappings = append(nlbSubnetMappings, &awstasks.SubnetMapping{Subnet: b.LinkToSubnet(subnet)})
|
|
}
|
|
}
|
|
|
|
sshAllowedCIDRs = append(sshAllowedCIDRs, b.Cluster.Spec.SSHAccess...)
|
|
for _, cidr := range sshAllowedCIDRs {
|
|
// Allow incoming SSH traffic to bastions, through the NLB
|
|
// TODO: Could we get away without an NLB here? Tricky to fix if dns-controller breaks though...
|
|
for _, bastionGroup := range bastionGroups {
|
|
{
|
|
t := &awstasks.SecurityGroupRule{
|
|
Name: fi.PtrTo(fmt.Sprintf("ssh-nlb-%s", cidr)),
|
|
Lifecycle: b.SecurityLifecycle,
|
|
SecurityGroup: bastionGroup.Task,
|
|
Protocol: fi.PtrTo("tcp"),
|
|
FromPort: fi.PtrTo(int64(22)),
|
|
ToPort: fi.PtrTo(int64(22)),
|
|
}
|
|
t.SetCidrOrPrefix(cidr)
|
|
AddDirectionalGroupRule(c, t)
|
|
}
|
|
|
|
if strings.HasPrefix(cidr, "pl-") {
|
|
// In case of a prefix list we do not add a rule for ICMP traffic for PMTU discovery.
|
|
// This would require calling out to AWS to check whether the prefix list is IPv4 or IPv6.
|
|
} else if utils.IsIPv6CIDR(cidr) {
|
|
// Allow ICMP traffic required for PMTU discovery
|
|
t := &awstasks.SecurityGroupRule{
|
|
Name: fi.PtrTo("icmpv6-pmtu-ssh-nlb-" + cidr),
|
|
Lifecycle: b.SecurityLifecycle,
|
|
FromPort: fi.PtrTo(int64(-1)),
|
|
Protocol: fi.PtrTo("icmpv6"),
|
|
SecurityGroup: bastionGroup.Task,
|
|
ToPort: fi.PtrTo(int64(-1)),
|
|
}
|
|
t.SetCidrOrPrefix(cidr)
|
|
c.AddTask(t)
|
|
} else {
|
|
t := &awstasks.SecurityGroupRule{
|
|
Name: fi.PtrTo("icmp-pmtu-ssh-nlb-" + cidr),
|
|
Lifecycle: b.SecurityLifecycle,
|
|
FromPort: fi.PtrTo(int64(3)),
|
|
Protocol: fi.PtrTo("icmp"),
|
|
SecurityGroup: bastionGroup.Task,
|
|
ToPort: fi.PtrTo(int64(4)),
|
|
}
|
|
t.SetCidrOrPrefix(cidr)
|
|
c.AddTask(t)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Create NLB itself
|
|
var nlb *awstasks.NetworkLoadBalancer
|
|
{
|
|
loadBalancerName := b.LBName32("bastion")
|
|
|
|
tags := b.CloudTags(loadBalancerName, false)
|
|
for k, v := range b.Cluster.Spec.CloudLabels {
|
|
tags[k] = v
|
|
}
|
|
// Override the returned name to be the expected ELB name
|
|
tags["Name"] = "bastion." + b.ClusterName()
|
|
|
|
nlbListeners := []*awstasks.NetworkLoadBalancerListener{
|
|
{
|
|
Port: 22,
|
|
TargetGroupName: b.NLBTargetGroupName("bastion"),
|
|
},
|
|
}
|
|
nlb = &awstasks.NetworkLoadBalancer{
|
|
Name: fi.PtrTo(b.NLBName("bastion")),
|
|
Lifecycle: b.Lifecycle,
|
|
|
|
LoadBalancerName: fi.PtrTo(loadBalancerName),
|
|
CLBName: fi.PtrTo("bastion." + b.ClusterName()),
|
|
SubnetMappings: nlbSubnetMappings,
|
|
Listeners: nlbListeners,
|
|
TargetGroups: make([]*awstasks.TargetGroup, 0),
|
|
|
|
Tags: tags,
|
|
VPC: b.LinkToVPC(),
|
|
Type: fi.PtrTo("network"),
|
|
IpAddressType: fi.PtrTo("ipv4"),
|
|
}
|
|
if useIPv6ForBastion(b) {
|
|
nlb.IpAddressType = fi.PtrTo("dualstack")
|
|
}
|
|
// Set the NLB Scheme according to load balancer Type
|
|
switch bastionLoadBalancerType {
|
|
case kops.LoadBalancerTypeInternal:
|
|
nlb.Scheme = fi.PtrTo("internal")
|
|
case kops.LoadBalancerTypePublic:
|
|
nlb.Scheme = nil
|
|
default:
|
|
return fmt.Errorf("unhandled bastion LoadBalancer type %q", bastionLoadBalancerType)
|
|
}
|
|
|
|
sshGroupName := b.NLBTargetGroupName("bastion")
|
|
sshGroupTags := b.CloudTags(sshGroupName, false)
|
|
|
|
// Override the returned name to be the expected NLB TG name
|
|
sshGroupTags["Name"] = sshGroupName
|
|
|
|
groupAttrs := map[string]string{
|
|
awstasks.TargetGroupAttributeDeregistrationDelayConnectionTerminationEnabled: "true",
|
|
awstasks.TargetGroupAttributeDeregistrationDelayTimeoutSeconds: "30",
|
|
}
|
|
|
|
tg := &awstasks.TargetGroup{
|
|
Name: fi.PtrTo(sshGroupName),
|
|
Lifecycle: b.Lifecycle,
|
|
VPC: b.LinkToVPC(),
|
|
Tags: sshGroupTags,
|
|
Protocol: fi.PtrTo("TCP"),
|
|
Port: fi.PtrTo(int64(22)),
|
|
Attributes: groupAttrs,
|
|
Interval: fi.PtrTo(int64(10)),
|
|
HealthyThreshold: fi.PtrTo(int64(2)),
|
|
UnhealthyThreshold: fi.PtrTo(int64(2)),
|
|
Shared: fi.PtrTo(false),
|
|
}
|
|
|
|
c.AddTask(tg)
|
|
|
|
nlb.TargetGroups = append(nlb.TargetGroups, tg)
|
|
|
|
sort.Stable(awstasks.OrderTargetGroupsByName(nlb.TargetGroups))
|
|
c.AddTask(nlb)
|
|
}
|
|
|
|
publicName := ""
|
|
if b.Cluster.Spec.Networking.Topology != nil && b.Cluster.Spec.Networking.Topology.Bastion != nil {
|
|
publicName = b.Cluster.Spec.Networking.Topology.Bastion.PublicName
|
|
}
|
|
if publicName != "" {
|
|
// Here we implement the bastion CNAME logic
|
|
// By default bastions will create a CNAME that follows the `bastion-$clustername` formula
|
|
t := &awstasks.DNSName{
|
|
Name: fi.PtrTo(publicName),
|
|
Lifecycle: b.Lifecycle,
|
|
|
|
Zone: b.LinkToDNSZone(),
|
|
ResourceName: fi.PtrTo(publicName),
|
|
ResourceType: fi.PtrTo("A"),
|
|
TargetLoadBalancer: b.LinkToNLB("bastion"),
|
|
}
|
|
c.AddTask(t)
|
|
t = &awstasks.DNSName{
|
|
Name: fi.PtrTo(publicName + "-AAAA"),
|
|
Lifecycle: b.Lifecycle,
|
|
|
|
Zone: b.LinkToDNSZone(),
|
|
ResourceName: fi.PtrTo(publicName),
|
|
ResourceType: fi.PtrTo("AAAA"),
|
|
TargetLoadBalancer: b.LinkToNLB("bastion"),
|
|
}
|
|
c.AddTask(t)
|
|
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func useIPv6ForBastion(b *BastionModelBuilder) bool {
|
|
for _, ig := range b.InstanceGroups {
|
|
for _, igSubnetName := range ig.Spec.Subnets {
|
|
for _, clusterSubnet := range b.Cluster.Spec.Networking.Subnets {
|
|
if igSubnetName != clusterSubnet.Name {
|
|
continue
|
|
}
|
|
if clusterSubnet.IPv6CIDR != "" {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Choose between subnets in a zone.
|
|
// We have already applied the rules to match internal subnets to internal NLBs and vice-versa for public-facing NLBs.
|
|
// For internal NLBs: we prefer the master subnets
|
|
// For public facing NLBs: we prefer the utility subnets
|
|
func (b *BastionModelBuilder) chooseBestSubnetForNLB(zone string, subnets []*kops.ClusterSubnetSpec) *kops.ClusterSubnetSpec {
|
|
if len(subnets) == 0 {
|
|
return nil
|
|
}
|
|
if len(subnets) == 1 {
|
|
return subnets[0]
|
|
}
|
|
|
|
migSubnets := sets.NewString()
|
|
for _, ig := range b.MasterInstanceGroups() {
|
|
for _, subnet := range ig.Spec.Subnets {
|
|
migSubnets.Insert(subnet)
|
|
}
|
|
}
|
|
|
|
var scoredSubnets []*scoredSubnet
|
|
for _, subnet := range subnets {
|
|
score := 0
|
|
|
|
if migSubnets.Has(subnet.Name) {
|
|
score += 1
|
|
}
|
|
|
|
if subnet.Type == kops.SubnetTypeDualStack {
|
|
score += 2
|
|
}
|
|
|
|
if subnet.Type == kops.SubnetTypeUtility {
|
|
score += 3
|
|
}
|
|
|
|
scoredSubnets = append(scoredSubnets, &scoredSubnet{
|
|
score: score,
|
|
subnet: subnet,
|
|
})
|
|
}
|
|
|
|
sort.Sort(ByScoreDescending(scoredSubnets))
|
|
|
|
if scoredSubnets[0].score == scoredSubnets[1].score {
|
|
klog.V(2).Infof("Making arbitrary choice between subnets in zone %q to attach to NLB (%q vs %q)", zone, scoredSubnets[0].subnet.Name, scoredSubnets[1].subnet.Name)
|
|
}
|
|
|
|
return scoredSubnets[0].subnet
|
|
}
|