kops/pkg/model/openstackmodel/servergroup.go

/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package openstackmodel

import (
	"fmt"
	"regexp"
	"sort"
	"strings"

	"github.com/gophercloud/gophercloud/v2/openstack/networking/v2/ports"
	"k8s.io/klog/v2"
	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/model"
	"k8s.io/kops/pkg/truncate"
	"k8s.io/kops/pkg/wellknownports"
	"k8s.io/kops/pkg/wellknownservices"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
	"k8s.io/kops/upup/pkg/fi/cloudup/openstacktasks"
	"k8s.io/utils/net"
)

// ServerGroupModelBuilder configures server group objects
type ServerGroupModelBuilder struct {
	*OpenstackModelContext
	BootstrapScriptBuilder *model.BootstrapScriptBuilder
	Lifecycle              fi.Lifecycle
}

var _ fi.CloudupModelBuilder = &ServerGroupModelBuilder{}

// See https://specs.openstack.org/openstack/nova-specs/specs/newton/approved/lowercase-metadata-keys.html for details
var instanceMetadataNotAllowedCharacters = regexp.MustCompile("[^a-zA-Z0-9-_:. ]")

// Constants for truncating Tags
const MAX_TAG_LENGTH_OPENSTACK = 60

var TRUNCATE_OPT = truncate.TruncateStringOptions{
	MaxLength:     MAX_TAG_LENGTH_OPENSTACK,
	AlwaysAddHash: false,
	HashLength:    6,
}

func (b *ServerGroupModelBuilder) buildAllowedAddressPairs(annotations map[string]string) []ports.AddressPair {
	keyPrefix := openstack.OS_ANNOTATION + openstack.ALLOWED_ADDRESS_PAIR + "/"

	var allowedAddressPairs []ports.AddressPair
	for key := range annotations {
		if strings.HasPrefix(key, keyPrefix) {
			ipAddress, macAddress, _ := strings.Cut(annotations[key], ",")

			allowedAddressPair := ports.AddressPair{
				IPAddress: ipAddress,
			}
			if macAddress != "" {
				allowedAddressPair.MACAddress = macAddress
			}

			allowedAddressPairs = append(allowedAddressPairs, allowedAddressPair)
		}
	}

	sort.Slice(allowedAddressPairs, func(i, j int) bool {
		return allowedAddressPairs[i].IPAddress < allowedAddressPairs[j].IPAddress
	})

	return allowedAddressPairs
}

func (b *ServerGroupModelBuilder) buildInstances(c *fi.CloudupModelBuilderContext, sg *openstacktasks.ServerGroup, ig *kops.InstanceGroup) error {
	sshKeyNameFull, err := b.SSHKeyName()
	if err != nil {
		return err
	}

	sshKeyName := strings.ReplaceAll(sshKeyNameFull, ":", "_")

	igMeta := make(map[string]string)
	cloudTags, err := b.KopsModelContext.CloudTagsForInstanceGroup(ig)
	if err != nil {
		return fmt.Errorf("could not get cloud tags for instance group %s: %v", ig.Name, err)
	}
	for label, labelVal := range cloudTags {
		sanitizedLabel := strings.ToLower(
			instanceMetadataNotAllowedCharacters.ReplaceAllLiteralString(label, "_"),
		)
		igMeta[sanitizedLabel] = labelVal
	}
	if ig.Spec.Role != kops.InstanceGroupRoleBastion {
		// Bastion does not belong to the cluster and will not be running protokube.

		igMeta[openstack.TagClusterName] = b.ClusterName()
	}
	igMeta["k8s"] = b.ClusterName()
	netName, err := b.GetNetworkName()
	if err != nil {
		return err
	}
	igMeta[openstack.TagKopsNetwork] = netName
	igMeta[openstack.TagKopsInstanceGroup] = ig.Name
	igMeta[openstack.TagKopsRole] = string(ig.Spec.Role)
	igMeta[openstack.INSTANCE_GROUP_GENERATION] = fmt.Sprintf("%d", ig.GetGeneration())
	igMeta[openstack.CLUSTER_GENERATION] = fmt.Sprintf("%d", b.Cluster.GetGeneration())

	if e, ok := ig.ObjectMeta.Annotations[openstack.OS_ANNOTATION+openstack.BOOT_FROM_VOLUME]; ok {
		igMeta[openstack.BOOT_FROM_VOLUME] = e
	}

	if v, ok := ig.ObjectMeta.Annotations[openstack.OS_ANNOTATION+openstack.BOOT_VOLUME_SIZE]; ok {
		igMeta[openstack.BOOT_VOLUME_SIZE] = v
	}

	startupScript, err := b.BootstrapScriptBuilder.ResourceNodeUp(c, ig)
	if err != nil {
		return fmt.Errorf("could not create startup script for instance group %s: %v", ig.Name, err)
	}

	var securityGroups []*openstacktasks.SecurityGroup
	securityGroupName := b.SecurityGroupName(ig.Spec.Role)
	securityGroups = append(securityGroups, b.LinkToSecurityGroup(securityGroupName))

	if b.Cluster.Spec.CloudProvider.Openstack.Loadbalancer == nil && ig.Spec.Role == kops.InstanceGroupRoleControlPlane {
		securityGroups = append(securityGroups, b.LinkToSecurityGroup(b.APIResourceName()))
	}

	r := strings.NewReplacer("_", "-", ".", "-")
	groupName := r.Replace(strings.ToLower(ig.Name))
	// In the future, OpenStack will use Machine API to manage groups,
	// for now create d.InstanceGroups.Spec.MinSize amount of servers
	for i := int32(0); i < *ig.Spec.MinSize; i++ {
		// FIXME: Must ensure 63 or less characters
		// replace all dots and _ with -, this is needed to get external cloudprovider working
		iName := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s-%d.%s", ig.Name, i+1, b.ClusterName())), "_", "-")
		instanceName := fi.PtrTo(strings.ReplaceAll(iName, ".", "-"))

		var az *string
		var subnets []*openstacktasks.Subnet
		havePublicSubnet := false
		if len(ig.Spec.Subnets) > 0 {
			subnet := ig.Spec.Subnets[int(i)%len(ig.Spec.Subnets)]
			// bastion subnet name might contain a "utility-" prefix
			if ig.Spec.Role == kops.InstanceGroupRoleBastion {
				az = fi.PtrTo(strings.Replace(subnet, "utility-", "", 1))
			} else {
				az = fi.PtrTo(subnet)
			}

			subnetName, subnetType, err := b.findSubnetClusterSpec(subnet)
			if err != nil {
				return err
			}
			subnets = append(subnets, b.LinkToSubnet(s(subnetName)))
			if subnetType == kops.SubnetTypePublic || subnetType == kops.SubnetTypeUtility {
				havePublicSubnet = true
			}
		}
		if len(ig.Spec.Zones) > 0 {
			zone := ig.Spec.Zones[int(i)%len(ig.Spec.Zones)]
			az = fi.PtrTo(zone)
		}
		// Create instance port task
		portName := fmt.Sprintf("%s-%s", "port", *instanceName)
		portTagKopsName := strings.ReplaceAll(
			strings.ReplaceAll(
				strings.ToLower(
					fmt.Sprintf("port-%s-%d", ig.Name, i+1),
				),
				"_", "-",
			), ".", "-",
		)
		portTask := &openstacktasks.Port{
			Name:              fi.PtrTo(portName),
			InstanceGroupName: &groupName,
			Network:           b.LinkToNetwork(),
			Tags: []string{
				truncate.TruncateString(fmt.Sprintf("%s=%s", openstack.TagKopsInstanceGroup, groupName), TRUNCATE_OPT),
				truncate.TruncateString(fmt.Sprintf("%s=%s", openstack.TagKopsName, portTagKopsName), TRUNCATE_OPT),
				truncate.TruncateString(fmt.Sprintf("%s=%s", openstack.TagClusterName, b.ClusterName()), TRUNCATE_OPT),
			},
			SecurityGroups:           securityGroups,
			AdditionalSecurityGroups: ig.Spec.AdditionalSecurityGroups,
			Subnets:                  subnets,
			AllowedAddressPairs:      b.buildAllowedAddressPairs(ig.ObjectMeta.Annotations),
			Lifecycle:                b.Lifecycle,
		}
		c.AddTask(portTask)

		if b.Cluster.UsesNoneDNS() && ig.Spec.Role == kops.InstanceGroupRoleControlPlane {
			portTask.WellKnownServices = append(portTask.WellKnownServices, wellknownservices.KubeAPIServer)
		}

		metaWithName := make(map[string]string)
		for k, v := range igMeta {
			metaWithName[k] = v
		}
		metaWithName[openstack.TagKopsName] = fi.ValueOf(instanceName)
		instanceTask := &openstacktasks.Instance{
			Name:             instanceName,
			Lifecycle:        b.Lifecycle,
			GroupName:        s(groupName),
			Region:           fi.PtrTo(b.Cluster.Spec.Networking.Subnets[0].Region),
			Flavor:           fi.PtrTo(ig.Spec.MachineType),
			Image:            fi.PtrTo(ig.Spec.Image),
			SSHKey:           fi.PtrTo(sshKeyName),
			ServerGroup:      sg,
			Role:             fi.PtrTo(string(ig.Spec.Role)),
			Port:             portTask,
			UserData:         startupScript,
			Metadata:         metaWithName,
			SecurityGroups:   ig.Spec.AdditionalSecurityGroups,
			AvailabilityZone: az,
			ConfigDrive:      b.Cluster.Spec.CloudProvider.Openstack.Metadata.ConfigDrive,
		}
		c.AddTask(instanceTask)

		// Associate a floating IP to the instances if we have external network in router
		// and respective subnet is "Public" or "Utility".
		if b.Cluster.Spec.CloudProvider.Openstack.Router != nil {
			if ig.Spec.AssociatePublicIP != nil && !fi.ValueOf(ig.Spec.AssociatePublicIP) {
				continue
			}
			if havePublicSubnet || ig.Spec.Role == kops.InstanceGroupRoleBastion {
				t := &openstacktasks.FloatingIP{
					Name:      fi.PtrTo(fmt.Sprintf("%s-%s", "fip", *instanceTask.Name)),
					Lifecycle: b.Lifecycle,
				}
				c.AddTask(t)
				if ig.Spec.Role == kops.InstanceGroupRoleControlPlane {
					// Ensure the floating IP is included in the TLS certificate,
					// if we're not going to use an alias for it
					t.WellKnownServices = append(t.WellKnownServices, wellknownservices.KubeAPIServer, wellknownservices.KopsController)
				}
				instanceTask.FloatingIP = t
			}
		}
	}

	return nil
}

func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
	clusterName := b.ClusterName()

	sgs := make(map[string]*openstacktasks.ServerGroup)
	for _, ig := range b.InstanceGroups {
		klog.V(2).Infof("Found instance group with name %s and role %v.", ig.Name, ig.Spec.Role)
		affinityPolicies := []string{}
		if v, ok := ig.ObjectMeta.Annotations[openstack.OS_ANNOTATION+openstack.SERVER_GROUP_AFFINITY]; ok {
			affinityPolicies = append(affinityPolicies, v)
		} else {
			affinityPolicies = append(affinityPolicies, "anti-affinity")
		}

		sgName := fmt.Sprintf("%s-%s", clusterName, ig.Name)
		if name, ok := ig.ObjectMeta.Annotations[openstack.OS_ANNOTATION+openstack.SERVER_GROUP_NAME]; ok {
			sgName = fmt.Sprintf("%s-%s", clusterName, name)
		}

		sgTask, ok := sgs[sgName]
		if !ok {
			igMap := make(map[string]*int32)
			igMap[ig.Name] = ig.Spec.MaxSize
			sgTask = &openstacktasks.ServerGroup{
				Name:        s(sgName),
				ClusterName: s(clusterName),
				IGMap:       igMap,
				Policies:    affinityPolicies,
				Lifecycle:   b.Lifecycle,
			}
			sgs[sgName] = sgTask
		} else {
			sgTask.IGMap[ig.Name] = ig.Spec.MaxSize
		}

		err := b.buildInstances(c, sgTask, ig)
		if err != nil {
			return err
		}
	}

	for _, s := range sgs {
		c.AddTask(s)
	}

	if b.Cluster.Spec.CloudProvider.Openstack.Loadbalancer != nil {
		var lbSubnetName string
		var err error
		for _, sp := range b.Cluster.Spec.Networking.Subnets {
			if sp.Type == kops.SubnetTypeDualStack || sp.Type == kops.SubnetTypePrivate {
				lbSubnetName, err = b.findSubnetNameByID(sp.ID, sp.Name)
				if err != nil {
					return err
				}
				break
			}
		}
		if lbSubnetName == "" {
			return fmt.Errorf("could not find subnet for Kubernetes API loadbalancer")
		}

		lbTask := &openstacktasks.LB{
			Name:      fi.PtrTo(b.APIResourceName()),
			Subnet:    fi.PtrTo(lbSubnetName),
			Lifecycle: b.Lifecycle,
		}

		if b.Cluster.Spec.CloudProvider.Openstack.Loadbalancer.FlavorID != nil {
			lbTask.FlavorID = b.Cluster.Spec.CloudProvider.Openstack.Loadbalancer.FlavorID
		}

		useVIPACL := b.UseVIPACL()
		if !useVIPACL {
			lbTask.SecurityGroup = b.LinkToSecurityGroup(b.APIResourceName())
		}

		c.AddTask(lbTask)

		lbfipTask := &openstacktasks.FloatingIP{
			Name:      fi.PtrTo(fmt.Sprintf("%s-%s", "fip", *lbTask.Name)),
			LB:        lbTask,
			Lifecycle: b.Lifecycle,
		}
		c.AddTask(lbfipTask)

		lbfipTask.WellKnownServices = append(lbfipTask.WellKnownServices, wellknownservices.KubeAPIServer)

		poolTask := &openstacktasks.LBPool{
			Name:         fi.PtrTo(fmt.Sprintf("%s-https", fi.ValueOf(lbTask.Name))),
			Loadbalancer: lbTask,
			Lifecycle:    b.Lifecycle,
		}
		c.AddTask(poolTask)

		nameForResource := fi.ValueOf(lbTask.Name)
		listenerTask := &openstacktasks.LBListener{
			Name:      fi.PtrTo(nameForResource),
			Port:      fi.PtrTo(wellknownports.KubeAPIServer),
			Lifecycle: b.Lifecycle,
			Pool:      poolTask,
		}
		if useVIPACL {
			var AllowedCIDRs []string
			// currently kOps openstack supports only ipv4 addresses
			for _, CIDR := range b.Cluster.Spec.API.Access {
				if net.IsIPv4CIDRString(CIDR) {
					AllowedCIDRs = append(AllowedCIDRs, CIDR)
				}
			}
			sort.Strings(AllowedCIDRs)
			listenerTask.AllowedCIDRs = AllowedCIDRs
		}
		c.AddTask(listenerTask)

		monitorTask := &openstacktasks.PoolMonitor{
			Name:      fi.PtrTo(nameForResource),
			Pool:      poolTask,
			Lifecycle: b.Lifecycle,
		}
		c.AddTask(monitorTask)

		ifName, err := b.GetNetworkName()
		if err != nil {
			return err
		}

		for _, ig := range b.InstanceGroups {
			if ig.Spec.Role == kops.InstanceGroupRoleControlPlane {
				associateTask := &openstacktasks.PoolAssociation{
					Name:          fi.PtrTo(fmt.Sprintf("%s-%s", clusterName, ig.Name)),
					ServerPrefix:  fi.PtrTo(ig.Name),
					ClusterName:   s(clusterName),
					Pool:          poolTask,
					InterfaceName: fi.PtrTo(ifName),
					ProtocolPort:  fi.PtrTo(wellknownports.KubeAPIServer),
					Lifecycle:     b.Lifecycle,
					Weight:        fi.PtrTo(1),
				}
				c.AddTask(associateTask)
			}
		}

	}

	return nil
}