kops/upup/pkg/fi/cloudup/azuretasks/vmscaleset.go

394 lines
12 KiB
Go

/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package azuretasks
import (
"context"
"encoding/base64"
"fmt"
"strings"
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-08-01/compute"
"github.com/Azure/go-autorest/autorest/to"
"k8s.io/klog/v2"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/azure"
)
// SubnetID contains the resource ID/names required to construct a subnet ID.
type SubnetID struct {
SubscriptionID string
ResourceGroupName string
VirtualNetworkName string
SubnetName string
}
// String returns the subnet ID in the path format.
func (s *SubnetID) String() string {
return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s",
s.SubscriptionID,
s.ResourceGroupName,
s.VirtualNetworkName,
s.SubnetName)
}
// ParseSubnetID parses a given subnet ID string and returns a SubnetID.
func ParseSubnetID(s string) (*SubnetID, error) {
l := strings.Split(s, "/")
if len(l) != 11 {
return nil, fmt.Errorf("malformed format of subnet ID: %s, %d", s, len(l))
}
return &SubnetID{
SubscriptionID: l[2],
ResourceGroupName: l[4],
VirtualNetworkName: l[8],
SubnetName: l[10],
}, nil
}
// loadBalancerID contains the resource ID/names required to construct a loadbalancer ID.
type loadBalancerID struct {
SubscriptionID string
ResourceGroupName string
LoadBalancerName string
}
// String returns the loadbalancer ID in the path format.
func (lb *loadBalancerID) String() string {
return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadbalancers/%s/backendAddressPools/LoadBalancerBackEnd",
lb.SubscriptionID,
lb.ResourceGroupName,
lb.LoadBalancerName,
)
}
// parseLoadBalancerID parses a given loadbalancer ID string and returns a loadBalancerID.
func parseLoadBalancerID(lb string) (*loadBalancerID, error) {
l := strings.Split(lb, "/")
if len(l) != 11 {
return nil, fmt.Errorf("malformed format of loadbalancer ID: %s, %d", lb, len(l))
}
return &loadBalancerID{
SubscriptionID: l[2],
ResourceGroupName: l[4],
LoadBalancerName: l[8],
}, nil
}
// VMScaleSet is an Azure VM Scale Set.
// +kops:fitask
type VMScaleSet struct {
Name *string
Lifecycle fi.Lifecycle
ResourceGroup *ResourceGroup
VirtualNetwork *VirtualNetwork
Subnet *Subnet
StorageProfile *VMScaleSetStorageProfile
// RequirePublicIP is set to true when VMs require public IPs.
RequirePublicIP *bool
// LoadBalancer is the Load Balancer object the VMs will use.
LoadBalancer *LoadBalancer
// SKUName specifies the SKU of of the VM Scale Set
SKUName *string
// Capacity specifies the number of virtual machines the VM Scale Set.
Capacity *int64
// ComputerNamePrefix is the prefix of each VM name of the form <prefix><base-36-instance-id>.
// See https://docs.microsoft.com/en-us/azure/virtual-machine-scale-sets/virtual-machine-scale-sets-instance-ids.
ComputerNamePrefix *string
// AdmnUser specifies the name of the administrative account.
AdminUser *string
SSHPublicKey *string
// UserData is the user data configuration
UserData fi.Resource
Tags map[string]*string
Zones []string
PrincipalID *string
}
var _ fi.CloudupTaskNormalize = &VMScaleSet{}
// VMScaleSetStorageProfile wraps *compute.VirtualMachineScaleSetStorageProfile
// and implements fi.HasDependencies.
//
// If we don't implement the interface and directly use
// compute.VirtualMachineScaleSetStorageProfile in VMScaleSet, the
// topological sort on VMScaleSet will fail as StorageProfile doesn't
// implement a proper interface.
type VMScaleSetStorageProfile struct {
*compute.VirtualMachineScaleSetStorageProfile
}
var _ fi.CloudupHasDependencies = &VMScaleSetStorageProfile{}
// GetDependencies returns a slice of tasks on which the tasks depends on.
func (p *VMScaleSetStorageProfile) GetDependencies(tasks map[string]fi.CloudupTask) []fi.CloudupTask {
return nil
}
var (
_ fi.CloudupTask = &VMScaleSet{}
_ fi.CompareWithID = &VMScaleSet{}
)
// CompareWithID returns the Name of the VM Scale Set.
func (s *VMScaleSet) CompareWithID() *string {
return s.Name
}
// Find discovers the VMScaleSet in the cloud provider.
func (s *VMScaleSet) Find(c *fi.CloudupContext) (*VMScaleSet, error) {
cloud := c.T.Cloud.(azure.AzureCloud)
found, err := cloud.VMScaleSet().Get(context.TODO(), *s.ResourceGroup.Name, *s.Name)
if err != nil && !strings.Contains(err.Error(), "ResourceNotFound") {
return nil, err
}
if found == nil {
return nil, nil
}
profile := found.VirtualMachineProfile
nwConfigs := *profile.NetworkProfile.NetworkInterfaceConfigurations
if len(nwConfigs) != 1 {
return nil, fmt.Errorf("unexpected number of network configs found for VM ScaleSet %s: %d", *s.Name, len(nwConfigs))
}
nwConfig := nwConfigs[0]
ipConfigs := *nwConfig.VirtualMachineScaleSetNetworkConfigurationProperties.IPConfigurations
if len(ipConfigs) != 1 {
return nil, fmt.Errorf("unexpected number of IP configs found for VM ScaleSet %s: %d", *s.Name, len(ipConfigs))
}
ipConfig := ipConfigs[0]
subnetID, err := ParseSubnetID(*ipConfig.Subnet.ID)
if err != nil {
return nil, fmt.Errorf("failed to parse subnet ID %s", *ipConfig.Subnet.ID)
}
var loadBalancerID *loadBalancerID
if ipConfig.LoadBalancerBackendAddressPools != nil {
for _, i := range *ipConfig.LoadBalancerBackendAddressPools {
if !strings.Contains(*i.ID, "api") {
continue
}
loadBalancerID, err = parseLoadBalancerID(*i.ID)
if err != nil {
return nil, fmt.Errorf("failed to parse loadbalancer ID %s", *ipConfig.Subnet.ID)
}
}
}
osProfile := profile.OsProfile
sshKeys := *osProfile.LinuxConfiguration.SSH.PublicKeys
if len(sshKeys) != 1 {
return nil, fmt.Errorf("unexpected number of SSH keys found for VM ScaleSet %s: %d", *s.Name, len(sshKeys))
}
userData, err := base64.StdEncoding.DecodeString(*profile.UserData)
if err != nil {
return nil, fmt.Errorf("failed to decode user data: %w", err)
}
vmss := &VMScaleSet{
Name: s.Name,
Lifecycle: s.Lifecycle,
ResourceGroup: &ResourceGroup{
Name: s.ResourceGroup.Name,
},
VirtualNetwork: &VirtualNetwork{
Name: to.StringPtr(subnetID.VirtualNetworkName),
},
Subnet: &Subnet{
Name: to.StringPtr(subnetID.SubnetName),
},
StorageProfile: &VMScaleSetStorageProfile{
VirtualMachineScaleSetStorageProfile: profile.StorageProfile,
},
RequirePublicIP: to.BoolPtr(ipConfig.PublicIPAddressConfiguration != nil),
SKUName: found.Sku.Name,
Capacity: found.Sku.Capacity,
ComputerNamePrefix: osProfile.ComputerNamePrefix,
AdminUser: osProfile.AdminUsername,
SSHPublicKey: sshKeys[0].KeyData,
UserData: fi.NewBytesResource(userData),
Tags: found.Tags,
PrincipalID: found.Identity.PrincipalID,
}
if loadBalancerID != nil {
vmss.LoadBalancer = &LoadBalancer{
Name: to.StringPtr(loadBalancerID.LoadBalancerName),
}
}
if found.Zones != nil {
vmss.Zones = *found.Zones
}
s.PrincipalID = found.Identity.PrincipalID
return vmss, nil
}
func (s *VMScaleSet) Normalize(c *fi.CloudupContext) error {
c.T.Cloud.(azure.AzureCloud).AddClusterTags(s.Tags)
return nil
}
// Run implements fi.Task.Run.
func (s *VMScaleSet) Run(c *fi.CloudupContext) error {
return fi.CloudupDefaultDeltaRunMethod(s, c)
}
// CheckChanges returns an error if a change is not allowed.
func (s *VMScaleSet) CheckChanges(a, e, changes *VMScaleSet) error {
if a == nil {
// Check if required fields are set when a new resource is created.
if e.Name == nil {
return fi.RequiredField("Name")
}
return nil
}
// Check if unchangeable fields won't be changed.
if changes.Name != nil {
return fi.CannotChangeField("Name")
}
return nil
}
// RenderAzure creates or updates a VM Scale Set.
func (s *VMScaleSet) RenderAzure(t *azure.AzureAPITarget, a, e, changes *VMScaleSet) error {
if a == nil {
klog.Infof("Creating a new VM Scale Set with name: %s", fi.ValueOf(e.Name))
} else {
klog.Infof("Updating a VM Scale Set with name: %s", fi.ValueOf(e.Name))
}
name := *e.Name
var customData *string
if e.UserData != nil {
d, err := fi.ResourceAsBytes(e.UserData)
if err != nil {
return fmt.Errorf("error rendering UserData: %s", err)
}
customData = to.StringPtr(base64.StdEncoding.EncodeToString(d))
}
osProfile := &compute.VirtualMachineScaleSetOSProfile{
ComputerNamePrefix: e.ComputerNamePrefix,
AdminUsername: e.AdminUser,
LinuxConfiguration: &compute.LinuxConfiguration{
SSH: &compute.SSHConfiguration{
PublicKeys: &[]compute.SSHPublicKey{
{
Path: to.StringPtr(fmt.Sprintf("/home/%s/.ssh/authorized_keys", *e.AdminUser)),
KeyData: to.StringPtr(*e.SSHPublicKey),
},
},
},
DisablePasswordAuthentication: to.BoolPtr(true),
},
}
subnetID := SubnetID{
SubscriptionID: t.Cloud.SubscriptionID(),
ResourceGroupName: *e.ResourceGroup.Name,
VirtualNetworkName: *e.VirtualNetwork.Name,
SubnetName: *e.Subnet.Name,
}
ipConfigProperties := &compute.VirtualMachineScaleSetIPConfigurationProperties{
Subnet: &compute.APIEntityReference{
ID: to.StringPtr(subnetID.String()),
},
Primary: to.BoolPtr(true),
PrivateIPAddressVersion: compute.IPv4,
}
if *e.RequirePublicIP {
ipConfigProperties.PublicIPAddressConfiguration = &compute.VirtualMachineScaleSetPublicIPAddressConfiguration{
Name: to.StringPtr(name + "-publicipconfig"),
VirtualMachineScaleSetPublicIPAddressConfigurationProperties: &compute.VirtualMachineScaleSetPublicIPAddressConfigurationProperties{
PublicIPAddressVersion: compute.IPv4,
},
}
}
if e.LoadBalancer != nil {
loadBalancerID := loadBalancerID{
SubscriptionID: t.Cloud.SubscriptionID(),
ResourceGroupName: *e.ResourceGroup.Name,
LoadBalancerName: *e.LoadBalancer.Name,
}
ipConfigProperties.LoadBalancerBackendAddressPools = &[]compute.SubResource{
{
ID: to.StringPtr(loadBalancerID.String()),
},
}
}
networkConfig := compute.VirtualMachineScaleSetNetworkConfiguration{
Name: to.StringPtr(name + "-netconfig"),
VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{
Primary: to.BoolPtr(true),
EnableIPForwarding: to.BoolPtr(true),
IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{
{
Name: to.StringPtr(name + "-ipconfig"),
VirtualMachineScaleSetIPConfigurationProperties: ipConfigProperties,
},
},
},
}
vmss := compute.VirtualMachineScaleSet{
Location: to.StringPtr(t.Cloud.Region()),
Sku: &compute.Sku{
Name: e.SKUName,
Capacity: e.Capacity,
},
VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{
UpgradePolicy: &compute.UpgradePolicy{
Mode: compute.UpgradeModeManual,
},
VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{
OsProfile: osProfile,
StorageProfile: e.StorageProfile.VirtualMachineScaleSetStorageProfile,
UserData: customData,
NetworkProfile: &compute.VirtualMachineScaleSetNetworkProfile{
NetworkInterfaceConfigurations: &[]compute.VirtualMachineScaleSetNetworkConfiguration{
networkConfig,
},
},
},
},
// Assign a system-assigned managed identity so that
// Azure creates an identity for VMs and provision
// its credentials on the VMs.
Identity: &compute.VirtualMachineScaleSetIdentity{
Type: compute.ResourceIdentityTypeSystemAssigned,
},
Tags: e.Tags,
Zones: &e.Zones,
}
result, err := t.Cloud.VMScaleSet().CreateOrUpdate(
context.TODO(),
*e.ResourceGroup.Name,
name,
vmss)
if err != nil {
return err
}
e.PrincipalID = result.Identity.PrincipalID
return nil
}