Merge pull request #16456 from hakman/azure-updates

azure: Various updates

pkg/apis/kops/cluster.go

@@ -878,9 +878,8 @@ func (c *Cluster) IsSharedAzureResourceGroup() bool {
 
 // AzureResourceGroupName returns the name of the resource group where the cluster is built.
 func (c *Cluster) AzureResourceGroupName() string {
-	r := c.Spec.CloudProvider.Azure.ResourceGroupName
-	if r != "" {
-		return r
+	if c.Spec.CloudProvider.Azure.ResourceGroupName != "" {
+		return c.Spec.CloudProvider.Azure.ResourceGroupName
 	}
 	return c.Name
 }
@@ -890,6 +889,14 @@ func (c *Cluster) IsSharedAzureRouteTable() bool {
 	return c.Spec.CloudProvider.Azure.RouteTableName != ""
 }
 
+// AzureRouteTableName returns the name of the route table used by the cluster.
+func (c *Cluster) AzureRouteTableName() string {
+	if c.Spec.CloudProvider.Azure.RouteTableName != "" {
+		return c.Spec.CloudProvider.Azure.RouteTableName
+	}
+	return c.Name
+}
+
 func (c *Cluster) PublishesDNSRecords() bool {
 	if c.UsesNoneDNS() || dns.IsGossipClusterName(c.Name) {
 		return false

pkg/apis/nodeup/config.go

@@ -276,7 +276,7 @@ func NewConfig(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) (*Confi
 		config.AzureSubscriptionID = cluster.Spec.CloudProvider.Azure.SubscriptionID
 		config.AzureTenantID = cluster.Spec.CloudProvider.Azure.TenantID
 		config.AzureResourceGroup = cluster.AzureResourceGroupName()
-		config.AzureRouteTableName = cluster.Spec.CloudProvider.Azure.RouteTableName
+		config.AzureRouteTableName = cluster.AzureRouteTableName()
 		config.Networking.NetworkID = cluster.Spec.Networking.NetworkID
 	}
 

pkg/instancegroups/instancegroups.go

@@ -476,7 +476,8 @@ func (c *RollingUpdateCluster) reconcileInstanceGroup() error {
 	if c.Cluster.Spec.GetCloudProvider() != api.CloudProviderOpenstack &&
 		c.Cluster.Spec.GetCloudProvider() != api.CloudProviderHetzner &&
 		c.Cluster.Spec.GetCloudProvider() != api.CloudProviderScaleway &&
-		c.Cluster.Spec.GetCloudProvider() != api.CloudProviderDO {
+		c.Cluster.Spec.GetCloudProvider() != api.CloudProviderDO &&
+		c.Cluster.Spec.GetCloudProvider() != api.CloudProviderAzure {
 		return nil
 	}
 	rto := fi.RunTasksOptions{}

pkg/model/azuremodel/context.go

@@ -64,7 +64,7 @@ func (c *AzureModelContext) LinkToAzureSubnet(spec *kops.ClusterSubnetSpec) *azu
 
 // NameForRouteTable returns the name of the Route Table object for the cluster.
 func (c *AzureModelContext) NameForRouteTable() string {
-	return c.Cluster.Spec.CloudProvider.Azure.RouteTableName
+	return c.Cluster.AzureRouteTableName()
 }
 
 // LinkToLoadBalancer returns the Load Balancer object for the cluster.

upup/pkg/fi/cloudup/azure/applicationsecuritygroup.go

@@ -18,8 +18,10 @@ package azure
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	network "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
 )
@@ -50,11 +52,19 @@ func (c *ApplicationSecurityGroupsClientImpl) CreateOrUpdate(ctx context.Context
 }
 
 func (c *ApplicationSecurityGroupsClientImpl) List(ctx context.Context, resourceGroupName string) ([]*network.ApplicationSecurityGroup, error) {
+	if resourceGroupName == "" {
+		return nil, nil
+	}
+
 	var l []*network.ApplicationSecurityGroup
 	pager := c.c.NewListPager(resourceGroupName, nil)
 	for pager.More() {
 		resp, err := pager.NextPage(ctx)
 		if err != nil {
+			var respErr *azcore.ResponseError
+			if errors.As(err, &respErr) && respErr.ErrorCode == "ResourceGroupNotFound" {
+				return nil, nil
+			}
 			return nil, fmt.Errorf("listing application security groups: %w", err)
 		}
 		l = append(l, resp.Value...)

upup/pkg/fi/cloudup/azure/azure_cloud.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	"k8s.io/klog/v2"
@@ -192,7 +193,9 @@ func (c *azureCloudImplementation) FindVNetInfo(id, resourceGroup string) (*fi.V
 }
 
 func (c *azureCloudImplementation) DeleteInstance(i *cloudinstances.CloudInstance) error {
-	return errors.New("DeleteInstance not implemented on azureCloud")
+	vmssName := i.CloudInstanceGroup.HumanName
+	instanceID := strings.TrimPrefix(i.ID, vmssName+"_")
+	return c.vmscaleSetVMsClient.Delete(context.TODO(), "my.k8s", vmssName, instanceID)
 }
 
 // DeregisterInstance drains a cloud instance and loadbalancers.

upup/pkg/fi/cloudup/azure/loadbalancer.go

@@ -18,8 +18,10 @@ package azure
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	network "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
@@ -52,11 +54,19 @@ func (c *loadBalancersClientImpl) CreateOrUpdate(ctx context.Context, resourceGr
 }
 
 func (c *loadBalancersClientImpl) List(ctx context.Context, resourceGroupName string) ([]*network.LoadBalancer, error) {
+	if resourceGroupName == "" {
+		return nil, nil
+	}
+
 	var l []*network.LoadBalancer
 	pager := c.c.NewListPager(resourceGroupName, nil)
 	for pager.More() {
 		resp, err := pager.NextPage(ctx)
 		if err != nil {
+			var respErr *azcore.ResponseError
+			if errors.As(err, &respErr) && respErr.ErrorCode == "ResourceGroupNotFound" {
+				return nil, nil
+			}
 			return nil, fmt.Errorf("listing load balancers: %w", err)
 		}
 		l = append(l, resp.Value...)

upup/pkg/fi/cloudup/azure/natgateway.go

@@ -18,8 +18,10 @@ package azure
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	network "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
 )
@@ -50,11 +52,19 @@ func (c *NatGatewaysClientImpl) CreateOrUpdate(ctx context.Context, resourceGrou
 }
 
 func (c *NatGatewaysClientImpl) List(ctx context.Context, resourceGroupName string) ([]*network.NatGateway, error) {
+	if resourceGroupName == "" {
+		return nil, nil
+	}
+
 	var l []*network.NatGateway
 	pager := c.c.NewListPager(resourceGroupName, nil)
 	for pager.More() {
 		resp, err := pager.NextPage(ctx)
 		if err != nil {
+			var respErr *azcore.ResponseError
+			if errors.As(err, &respErr) && respErr.ErrorCode == "ResourceGroupNotFound" {
+				return nil, nil
+			}
 			return nil, fmt.Errorf("listing nat gateways: %w", err)
 		}
 		l = append(l, resp.Value...)

upup/pkg/fi/cloudup/azure/networksecuritygroup.go

@@ -18,8 +18,10 @@ package azure
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	network "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
 )
@@ -50,11 +52,19 @@ func (c *NetworkSecurityGroupsClientImpl) CreateOrUpdate(ctx context.Context, re
 }
 
 func (c *NetworkSecurityGroupsClientImpl) List(ctx context.Context, resourceGroupName string) ([]*network.SecurityGroup, error) {
+	if resourceGroupName == "" {
+		return nil, nil
+	}
+
 	var l []*network.SecurityGroup
 	pager := c.c.NewListPager(resourceGroupName, nil)
 	for pager.More() {
 		resp, err := pager.NextPage(ctx)
 		if err != nil {
+			var respErr *azcore.ResponseError
+			if errors.As(err, &respErr) && respErr.ErrorCode == "ResourceGroupNotFound" {
+				return nil, nil
+			}
 			return nil, fmt.Errorf("listing network security groups: %w", err)
 		}
 		l = append(l, resp.Value...)

upup/pkg/fi/cloudup/azure/publicipaddress.go

@@ -18,8 +18,10 @@ package azure
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	network "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
 )
@@ -50,11 +52,19 @@ func (c *publicIPAddressesClientImpl) CreateOrUpdate(ctx context.Context, resour
 }
 
 func (c *publicIPAddressesClientImpl) List(ctx context.Context, resourceGroupName string) ([]*network.PublicIPAddress, error) {
+	if resourceGroupName == "" {
+		return nil, nil
+	}
+
 	var l []*network.PublicIPAddress
 	pager := c.c.NewListPager(resourceGroupName, nil)
 	for pager.More() {
 		resp, err := pager.NextPage(ctx)
 		if err != nil {
+			var respErr *azcore.ResponseError
+			if errors.As(err, &respErr) && respErr.ErrorCode == "ResourceGroupNotFound" {
+				return nil, nil
+			}
 			return nil, fmt.Errorf("listing public ip addresses: %w", err)
 		}
 		l = append(l, resp.Value...)

upup/pkg/fi/cloudup/azure/routetable.go

@@ -18,8 +18,10 @@ package azure
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	network "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
 )
@@ -50,11 +52,19 @@ func (c *routeTablesClientImpl) CreateOrUpdate(ctx context.Context, resourceGrou
 }
 
 func (c *routeTablesClientImpl) List(ctx context.Context, resourceGroupName string) ([]*network.RouteTable, error) {
+	if resourceGroupName == "" {
+		return nil, nil
+	}
+
 	var l []*network.RouteTable
 	pager := c.c.NewListPager(resourceGroupName, nil)
 	for pager.More() {
 		resp, err := pager.NextPage(ctx)
 		if err != nil {
+			var respErr *azcore.ResponseError
+			if errors.As(err, &respErr) && respErr.ErrorCode == "ResourceGroupNotFound" {
+				return nil, nil
+			}
 			return nil, fmt.Errorf("listing route tables: %w", err)
 		}
 		l = append(l, resp.Value...)

upup/pkg/fi/cloudup/azure/status_test.go

@@ -67,6 +67,17 @@ func (c *mockVMScaleSetVMsClient) List(ctx context.Context, resourceGroupName, v
 	return c.vms, nil
 }
 
+func (c *mockVMScaleSetVMsClient) Delete(ctx context.Context, resourceGroupName, vmssName, instanceID string) error {
+	var l []*compute.VirtualMachineScaleSetVM
+	for _, vm := range c.vms {
+		if *vm.ID != instanceID {
+			l = append(l, vm)
+		}
+	}
+	c.vms = l
+	return nil
+}
+
 func TestFindEtcdStatus(t *testing.T) {
 	clusterName := "my-cluster"
 	c := &azureCloudImplementation{

upup/pkg/fi/cloudup/azure/virtualnetwork.go

@@ -18,8 +18,10 @@ package azure
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	network "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork"
 )
@@ -50,11 +52,19 @@ func (c *virtualNetworksClientImpl) CreateOrUpdate(ctx context.Context, resource
 }
 
 func (c *virtualNetworksClientImpl) List(ctx context.Context, resourceGroupName string) ([]*network.VirtualNetwork, error) {
+	if resourceGroupName == "" {
+		return nil, nil
+	}
+
 	var l []*network.VirtualNetwork
 	pager := c.c.NewListPager(resourceGroupName, nil)
 	for pager.More() {
 		resp, err := pager.NextPage(ctx)
 		if err != nil {
+			var respErr *azcore.ResponseError
+			if errors.As(err, &respErr) && respErr.ErrorCode == "ResourceGroupNotFound" {
+				return nil, nil
+			}
 			return nil, fmt.Errorf("listing virtual networks: %w", err)
 		}
 		l = append(l, resp.Value...)

upup/pkg/fi/cloudup/azure/vmscaleset.go

@@ -18,8 +18,10 @@ package azure
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	compute "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute"
@@ -52,11 +54,19 @@ func (c *vmScaleSetsClientImpl) CreateOrUpdate(ctx context.Context, resourceGrou
 }
 
 func (c *vmScaleSetsClientImpl) List(ctx context.Context, resourceGroupName string) ([]*compute.VirtualMachineScaleSet, error) {
+	if resourceGroupName == "" {
+		return nil, nil
+	}
+
 	var l []*compute.VirtualMachineScaleSet
 	pager := c.c.NewListPager(resourceGroupName, nil)
 	for pager.More() {
 		resp, err := pager.NextPage(ctx)
 		if err != nil {
+			var respErr *azcore.ResponseError
+			if errors.As(err, &respErr) && respErr.ErrorCode == "ResourceGroupNotFound" {
+				return nil, nil
+			}
 			return nil, fmt.Errorf("listing VMSSs: %w", err)
 		}
 		l = append(l, resp.Value...)

upup/pkg/fi/cloudup/azure/vmscalesetvm.go

@@ -27,6 +27,7 @@ import (
 // VMScaleSetVMsClient is a client for managing VMs in VM Scale Sets.
 type VMScaleSetVMsClient interface {
 	List(ctx context.Context, resourceGroupName, vmssName string) ([]*compute.VirtualMachineScaleSetVM, error)
+	Delete(ctx context.Context, resourceGroupName, vmssName, instanceId string) error
 }
 
 type vmScaleSetVMsClientImpl struct {
@@ -48,6 +49,17 @@ func (c *vmScaleSetVMsClientImpl) List(ctx context.Context, resourceGroupName, v
 	return l, nil
 }
 
+func (c *vmScaleSetVMsClientImpl) Delete(ctx context.Context, resourceGroupName, vmssName, instanceId string) error {
+	future, err := c.c.BeginDelete(ctx, resourceGroupName, vmssName, instanceId, nil)
+	if err != nil {
+		return fmt.Errorf("deleting VMSS VM: %w", err)
+	}
+	if _, err = future.PollUntilDone(ctx, nil); err != nil {
+		return fmt.Errorf("waiting for VMSS VM deletion completion: %w", err)
+	}
+	return nil
+}
+
 func newVMScaleSetVMsClientImpl(subscriptionID string, cred *azidentity.DefaultAzureCredential) (*vmScaleSetVMsClientImpl, error) {
 	c, err := compute.NewVirtualMachineScaleSetVMsClient(subscriptionID, cred, nil)
 	if err != nil {

upup/pkg/fi/cloudup/azuretasks/subnet.go

@@ -61,7 +61,7 @@ func (s *Subnet) Find(c *fi.CloudupContext) (*Subnet, error) {
 	if err != nil {
 		var azErr *azcore.ResponseError
 		if errors.As(err, &azErr) {
-			if azErr.ErrorCode == "ResourceNotFound" {
+			if azErr.ErrorCode == "ResourceNotFound" || azErr.ErrorCode == "ResourceGroupNotFound" {
 				return nil, nil
 			} else {
 				return nil, azErr

upup/pkg/fi/cloudup/azuretasks/testing.go

@@ -463,6 +463,13 @@ func (c *MockVMScaleSetVMsClient) List(ctx context.Context, resourceGroupName, v
 	return l, nil
 }
 
+// Delete deletes a VM Scale Set VMs.
+func (c *MockVMScaleSetVMsClient) Delete(ctx context.Context, resourceGroupName, vmssName, instanceID string) error {
+	// Ignore resourceGroupName and vmssName for simplicity.
+	delete(c.VMs, instanceID)
+	return nil
+}
+
 // MockDisksClient is a mock implementation of disk client.
 type MockDisksClient struct {
 	Disks map[string]*compute.Disk