/* Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package openstackmodel import ( "fmt" "regexp" "sort" "strings" "k8s.io/klog/v2" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/dns" "k8s.io/kops/pkg/model" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/openstack" "k8s.io/kops/upup/pkg/fi/cloudup/openstacktasks" ) // ServerGroupModelBuilder configures server group objects type ServerGroupModelBuilder struct { *OpenstackModelContext BootstrapScriptBuilder *model.BootstrapScriptBuilder Lifecycle *fi.Lifecycle } var _ fi.ModelBuilder = &ServerGroupModelBuilder{} // See https://specs.openstack.org/openstack/nova-specs/specs/newton/approved/lowercase-metadata-keys.html for details var instanceMetadataNotAllowedCharacters = regexp.MustCompile("[^a-zA-Z0-9-_:. ]") func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg *openstacktasks.ServerGroup, ig *kops.InstanceGroup) error { sshKeyNameFull, err := b.SSHKeyName() if err != nil { return err } sshKeyName := strings.Replace(sshKeyNameFull, ":", "_", -1) igMeta := make(map[string]string) cloudTags, err := b.KopsModelContext.CloudTagsForInstanceGroup(ig) if err != nil { return fmt.Errorf("could not get cloud tags for instance group %s: %v", ig.Name, err) } for label, labelVal := range cloudTags { sanitizedLabel := strings.ToLower( instanceMetadataNotAllowedCharacters.ReplaceAllLiteralString(label, "_"), ) igMeta[sanitizedLabel] = labelVal } if ig.Spec.Role != kops.InstanceGroupRoleBastion { // Bastion does not belong to the cluster and will not be running protokube. igMeta[openstack.TagClusterName] = b.ClusterName() } igMeta["k8s"] = b.ClusterName() netName, err := b.GetNetworkName() if err != nil { return err } igMeta[openstack.TagKopsNetwork] = netName igMeta["KopsInstanceGroup"] = ig.Name igMeta["KopsRole"] = string(ig.Spec.Role) igMeta[openstack.INSTANCE_GROUP_GENERATION] = fmt.Sprintf("%d", ig.GetGeneration()) igMeta[openstack.CLUSTER_GENERATION] = fmt.Sprintf("%d", b.Cluster.GetGeneration()) if e, ok := ig.ObjectMeta.Annotations[openstack.OS_ANNOTATION+openstack.BOOT_FROM_VOLUME]; ok { igMeta[openstack.BOOT_FROM_VOLUME] = e } if v, ok := ig.ObjectMeta.Annotations[openstack.OS_ANNOTATION+openstack.BOOT_VOLUME_SIZE]; ok { igMeta[openstack.BOOT_VOLUME_SIZE] = v } startupScript, err := b.BootstrapScriptBuilder.ResourceNodeUp(c, ig) if err != nil { return fmt.Errorf("could not create startup script for instance group %s: %v", ig.Name, err) } var securityGroups []*openstacktasks.SecurityGroup securityGroupName := b.SecurityGroupName(ig.Spec.Role) securityGroups = append(securityGroups, b.LinkToSecurityGroup(securityGroupName)) if b.Cluster.Spec.CloudConfig.Openstack.Loadbalancer == nil && ig.Spec.Role == kops.InstanceGroupRoleMaster { securityGroups = append(securityGroups, b.LinkToSecurityGroup(b.Cluster.Spec.MasterPublicName)) } r := strings.NewReplacer("_", "-", ".", "-") groupName := r.Replace(strings.ToLower(ig.Name)) // In the future, OpenStack will use Machine API to manage groups, // for now create d.InstanceGroups.Spec.MinSize amount of servers for i := int32(0); i < *ig.Spec.MinSize; i++ { // FIXME: Must ensure 63 or less characters // replace all dots and _ with -, this is needed to get external cloudprovider working iName := strings.Replace(strings.ToLower(fmt.Sprintf("%s-%d.%s", ig.Name, i+1, b.ClusterName())), "_", "-", -1) instanceName := fi.String(strings.Replace(iName, ".", "-", -1)) var az *string var subnets []*openstacktasks.Subnet if len(ig.Spec.Subnets) > 0 { subnet := ig.Spec.Subnets[int(i)%len(ig.Spec.Subnets)] // bastion subnet name might contain a "utility-" prefix if ig.Spec.Role == kops.InstanceGroupRoleBastion { az = fi.String(strings.Replace(subnet, "utility-", "", 1)) } else { az = fi.String(subnet) } subnetName, err := b.findSubnetClusterSpec(subnet) if err != nil { return err } subnets = append(subnets, b.LinkToSubnet(s(subnetName))) } if len(ig.Spec.Zones) > 0 { zone := ig.Spec.Zones[int(i)%len(ig.Spec.Zones)] az = fi.String(zone) } // Create instance port task portTask := &openstacktasks.Port{ Name: fi.String(fmt.Sprintf("%s-%s", "port", *instanceName)), Network: b.LinkToNetwork(), Tag: s(b.ClusterName()), SecurityGroups: securityGroups, AdditionalSecurityGroups: ig.Spec.AdditionalSecurityGroups, Subnets: subnets, Lifecycle: b.Lifecycle, } c.AddTask(portTask) metaWithName := make(map[string]string) for k, v := range igMeta { metaWithName[k] = v } metaWithName[openstack.TagKopsName] = fi.StringValue(instanceName) instanceTask := &openstacktasks.Instance{ Name: instanceName, GroupName: s(groupName), Region: fi.String(b.Cluster.Spec.Subnets[0].Region), Flavor: fi.String(ig.Spec.MachineType), Image: fi.String(ig.Spec.Image), SSHKey: fi.String(sshKeyName), ServerGroup: sg, Role: fi.String(string(ig.Spec.Role)), Port: portTask, UserData: startupScript, Metadata: metaWithName, SecurityGroups: ig.Spec.AdditionalSecurityGroups, AvailabilityZone: az, } c.AddTask(instanceTask) // Associate a floating IP to the instances if we have external network in router // and respective topology is "public" if b.Cluster.Spec.CloudConfig.Openstack != nil && b.Cluster.Spec.CloudConfig.Openstack.Router != nil { if ig.Spec.AssociatePublicIP != nil && !fi.BoolValue(ig.Spec.AssociatePublicIP) { continue } switch ig.Spec.Role { case kops.InstanceGroupRoleBastion: t := &openstacktasks.FloatingIP{ Name: fi.String(fmt.Sprintf("%s-%s", "fip", *instanceTask.Name)), Lifecycle: b.Lifecycle, } c.AddTask(t) instanceTask.FloatingIP = t case kops.InstanceGroupRoleMaster: if b.Cluster.Spec.Topology == nil || b.Cluster.Spec.Topology.Masters != kops.TopologyPrivate { t := &openstacktasks.FloatingIP{ Name: fi.String(fmt.Sprintf("%s-%s", "fip", *instanceTask.Name)), Lifecycle: b.Lifecycle, } c.AddTask(t) b.associateFIPToKeypair(t) instanceTask.FloatingIP = t } default: if b.Cluster.Spec.Topology == nil || b.Cluster.Spec.Topology.Nodes != kops.TopologyPrivate { t := &openstacktasks.FloatingIP{ Name: fi.String(fmt.Sprintf("%s-%s", "fip", *instanceTask.Name)), Lifecycle: b.Lifecycle, } c.AddTask(t) instanceTask.FloatingIP = t } } } } return nil } func (b *ServerGroupModelBuilder) associateFIPToKeypair(fipTask *openstacktasks.FloatingIP) { // Ensure the floating IP is included in the TLS certificate, // if we're not going to use an alias for it fipTask.ForAPIServer = true } func (b *ServerGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { clusterName := b.ClusterName() var masters []*openstacktasks.ServerGroup for _, ig := range b.InstanceGroups { klog.V(2).Infof("Found instance group with name %s and role %v.", ig.Name, ig.Spec.Role) sgTask := &openstacktasks.ServerGroup{ Name: s(fmt.Sprintf("%s-%s", clusterName, ig.Name)), ClusterName: s(clusterName), IGName: s(ig.Name), Policies: []string{"anti-affinity"}, Lifecycle: b.Lifecycle, MaxSize: ig.Spec.MaxSize, } c.AddTask(sgTask) err := b.buildInstances(c, sgTask, ig) if err != nil { return err } if ig.Spec.Role == kops.InstanceGroupRoleMaster { masters = append(masters, sgTask) } } if b.Cluster.Spec.CloudConfig.Openstack.Loadbalancer != nil { var lbSubnetName string var err error for _, sp := range b.Cluster.Spec.Subnets { if sp.Type == kops.SubnetTypePrivate { lbSubnetName, err = b.findSubnetNameByID(sp.ProviderID, sp.Name) if err != nil { return err } break } } if lbSubnetName == "" { return fmt.Errorf("could not find subnet for master loadbalancer") } lbTask := &openstacktasks.LB{ Name: fi.String(b.Cluster.Spec.MasterPublicName), Subnet: fi.String(lbSubnetName), Lifecycle: b.Lifecycle, } useVIPACL := b.UseVIPACL() if !useVIPACL { lbTask.SecurityGroup = b.LinkToSecurityGroup(b.Cluster.Spec.MasterPublicName) } c.AddTask(lbTask) lbfipTask := &openstacktasks.FloatingIP{ Name: fi.String(fmt.Sprintf("%s-%s", "fip", *lbTask.Name)), LB: lbTask, Lifecycle: b.Lifecycle, } c.AddTask(lbfipTask) if dns.IsGossipHostname(b.Cluster.Name) || b.UsePrivateDNS() { b.associateFIPToKeypair(lbfipTask) } poolTask := &openstacktasks.LBPool{ Name: fi.String(fmt.Sprintf("%s-https", fi.StringValue(lbTask.Name))), Loadbalancer: lbTask, Lifecycle: b.Lifecycle, } c.AddTask(poolTask) listenerTask := &openstacktasks.LBListener{ Name: lbTask.Name, Lifecycle: b.Lifecycle, Pool: poolTask, } if useVIPACL { // sort for consistent comparison sort.Strings(b.Cluster.Spec.KubernetesAPIAccess) listenerTask.AllowedCIDRs = b.Cluster.Spec.KubernetesAPIAccess } c.AddTask(listenerTask) ifName, err := b.GetNetworkName() if err != nil { return err } for _, mastersg := range masters { associateTask := &openstacktasks.PoolAssociation{ Name: mastersg.Name, Pool: poolTask, ServerGroup: mastersg, InterfaceName: fi.String(ifName), ProtocolPort: fi.Int(443), Lifecycle: b.Lifecycle, } c.AddTask(associateTask) } } return nil }