Merge pull request #10609 from t1cg/azureApiIngressStatus

feat: implement azure get api ingress status fn

docs/getting_started/azure.md

@@ -174,7 +174,7 @@ The command line flags prefixed with `--azure-` are for
 Azure-specific configurations.
 
 ```bash
-$ export KOPS_FEATURE_FLAGS=AlphaAllowAzure
+$ export KOPS_FEATURE_FLAGS=Azure
 
 $ kops create cluster \
   --cloud azure \

upup/pkg/fi/cloudup/azure/BUILD.bazel

@@ -7,6 +7,7 @@ go_library(
         "azure_cloud.go",
         "azure_utils.go",
         "disk.go",
+        "networkinterface.go",
         "resourcegroup.go",
         "roleassignment.go",
         "routetable.go",

upup/pkg/fi/cloudup/azure/azure_cloud.go

@@ -17,7 +17,9 @@ limitations under the License.
 package azure
 
 import (
+	"context"
 	"errors"
+	"fmt"
 
 	"github.com/Azure/go-autorest/autorest/azure/auth"
 	"k8s.io/kops/dnsprovider/pkg/dnsprovider"
@@ -52,20 +54,22 @@ type AzureCloud interface {
 	VMScaleSetVM() VMScaleSetVMsClient
 	Disk() DisksClient
 	RoleAssignment() RoleAssignmentsClient
+	NetworkInterface() NetworkInterfacesClient
 }
 
 type azureCloudImplementation struct {
-	subscriptionID        string
-	location              string
-	tags                  map[string]string
-	resourceGroupsClient  ResourceGroupsClient
-	vnetsClient           VirtualNetworksClient
-	subnetsClient         SubnetsClient
-	routeTablesClient     RouteTablesClient
-	vmscaleSetsClient     VMScaleSetsClient
-	vmscaleSetVMsClient   VMScaleSetVMsClient
-	disksClient           DisksClient
-	roleAssignmentsClient RoleAssignmentsClient
+	subscriptionID          string
+	location                string
+	tags                    map[string]string
+	resourceGroupsClient    ResourceGroupsClient
+	vnetsClient             VirtualNetworksClient
+	subnetsClient           SubnetsClient
+	routeTablesClient       RouteTablesClient
+	vmscaleSetsClient       VMScaleSetsClient
+	vmscaleSetVMsClient     VMScaleSetVMsClient
+	disksClient             DisksClient
+	roleAssignmentsClient   RoleAssignmentsClient
+	networkInterfacesClient NetworkInterfacesClient
 }
 
 var _ fi.Cloud = &azureCloudImplementation{}
@@ -78,17 +82,18 @@ func NewAzureCloud(subscriptionID, location string, tags map[string]string) (Azu
 	}
 
 	return &azureCloudImplementation{
-		subscriptionID:        subscriptionID,
-		location:              location,
-		tags:                  tags,
-		resourceGroupsClient:  newResourceGroupsClientImpl(subscriptionID, authorizer),
-		vnetsClient:           newVirtualNetworksClientImpl(subscriptionID, authorizer),
-		subnetsClient:         newSubnetsClientImpl(subscriptionID, authorizer),
-		routeTablesClient:     newRouteTablesClientImpl(subscriptionID, authorizer),
-		vmscaleSetsClient:     newVMScaleSetsClientImpl(subscriptionID, authorizer),
-		vmscaleSetVMsClient:   newVMScaleSetVMsClientImpl(subscriptionID, authorizer),
-		disksClient:           newDisksClientImpl(subscriptionID, authorizer),
-		roleAssignmentsClient: newRoleAssignmentsClientImpl(subscriptionID, authorizer),
+		subscriptionID:          subscriptionID,
+		location:                location,
+		tags:                    tags,
+		resourceGroupsClient:    newResourceGroupsClientImpl(subscriptionID, authorizer),
+		vnetsClient:             newVirtualNetworksClientImpl(subscriptionID, authorizer),
+		subnetsClient:           newSubnetsClientImpl(subscriptionID, authorizer),
+		routeTablesClient:       newRouteTablesClientImpl(subscriptionID, authorizer),
+		vmscaleSetsClient:       newVMScaleSetsClientImpl(subscriptionID, authorizer),
+		vmscaleSetVMsClient:     newVMScaleSetVMsClientImpl(subscriptionID, authorizer),
+		disksClient:             newDisksClientImpl(subscriptionID, authorizer),
+		roleAssignmentsClient:   newRoleAssignmentsClientImpl(subscriptionID, authorizer),
+		networkInterfacesClient: newNetworkInterfacesClientImpl(subscriptionID, authorizer),
 	}, nil
 }
 
@@ -128,9 +133,43 @@ func (c *azureCloudImplementation) AddClusterTags(tags map[string]*string) {
 }
 
 func (c *azureCloudImplementation) GetApiIngressStatus(cluster *kops.Cluster) ([]kops.ApiIngressStatus, error) {
-	// TODO(kenji): Implement this. Currently we return nil as we
-	// don't create any resources for ingress to the API server.
-	return nil, nil
+	var ingresses []kops.ApiIngressStatus
+	var rg string = cluster.AzureResourceGroupName()
+
+	// Get scale sets in cluster resource group and find masters scale set
+	scaleSets, err := c.vmscaleSetsClient.List(context.TODO(), rg)
+	if err != nil {
+		return nil, fmt.Errorf("error getting Cluster Master Scale Set for API Ingress Status: %s", err)
+	}
+	var vmssName string
+	for _, scaleSet := range scaleSets {
+		val, ok := scaleSet.Tags[TagClusterName]
+		val2, ok2 := scaleSet.Tags[TagNameRolePrefix+TagRoleMaster]
+		if ok && *val == cluster.Name && ok2 && *val2 == "1" {
+			vmssName = *scaleSet.Name
+			break
+		}
+	}
+	if vmssName == "" {
+		return nil, fmt.Errorf("error getting Master Scale Set Name for API Ingress Status")
+	}
+
+	// Get masters scale set network interfaces and append to api ingress status
+	nis, err := c.NetworkInterface().ListScaleSetsNetworkInterfaces(context.TODO(), rg, vmssName)
+	if err != nil {
+		return nil, fmt.Errorf("error getting Master Scale Set Network Interfaces for API Ingress Status: %s", err)
+	}
+	for _, ni := range nis {
+		if !*ni.Primary {
+			continue
+		}
+		for _, i := range *ni.IPConfigurations {
+			ingresses = append(ingresses, kops.ApiIngressStatus{
+				IP: *i.PrivateIPAddress,
+			})
+		}
+	}
+	return ingresses, nil
 }
 
 func (c *azureCloudImplementation) SubscriptionID() string {
@@ -168,3 +207,7 @@ func (c *azureCloudImplementation) Disk() DisksClient {
 func (c *azureCloudImplementation) RoleAssignment() RoleAssignmentsClient {
 	return c.roleAssignmentsClient
 }
+
+func (c *azureCloudImplementation) NetworkInterface() NetworkInterfacesClient {
+	return c.networkInterfacesClient
+}

upup/pkg/fi/cloudup/azure/networkinterface.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"context"
+
+	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-06-01/network"
+	"github.com/Azure/go-autorest/autorest"
+)
+
+// NetworkInterfacesClient is a client for managing Network Interfaces.
+type NetworkInterfacesClient interface {
+	ListScaleSetsNetworkInterfaces(ctx context.Context, resourceGroupName, vmssName string) ([]network.Interface, error)
+}
+
+type networkInterfacesClientImpl struct {
+	c *network.InterfacesClient
+}
+
+var _ NetworkInterfacesClient = &networkInterfacesClientImpl{}
+
+func (c *networkInterfacesClientImpl) ListScaleSetsNetworkInterfaces(ctx context.Context, resourceGroupName, vmssName string) ([]network.Interface, error) {
+	var l []network.Interface
+	for iter, err := c.c.ListVirtualMachineScaleSetNetworkInterfacesComplete(ctx, resourceGroupName, vmssName); iter.NotDone(); err = iter.Next() {
+		if err != nil {
+			return nil, err
+		}
+		l = append(l, iter.Value())
+	}
+	return l, nil
+}
+
+func newNetworkInterfacesClientImpl(subscriptionID string, authorizer autorest.Authorizer) *networkInterfacesClientImpl {
+	c := network.NewInterfacesClient(subscriptionID)
+	c.Authorizer = authorizer
+	return &networkInterfacesClientImpl{
+		c: &c,
+	}
+}

upup/pkg/fi/cloudup/azuretasks/testing.go

@@ -46,15 +46,16 @@ const (
 
 // MockAzureCloud is a mock implementation of AzureCloud.
 type MockAzureCloud struct {
-	Location              string
-	ResourceGroupsClient  *MockResourceGroupsClient
-	VirtualNetworksClient *MockVirtualNetworksClient
-	SubnetsClient         *MockSubnetsClient
-	RouteTablesClient     *MockRouteTablesClient
-	VMScaleSetsClient     *MockVMScaleSetsClient
-	VMScaleSetVMsClient   *MockVMScaleSetVMsClient
-	DisksClient           *MockDisksClient
-	RoleAssignmentsClient *MockRoleAssignmentsClient
+	Location                string
+	ResourceGroupsClient    *MockResourceGroupsClient
+	VirtualNetworksClient   *MockVirtualNetworksClient
+	SubnetsClient           *MockSubnetsClient
+	RouteTablesClient       *MockRouteTablesClient
+	VMScaleSetsClient       *MockVMScaleSetsClient
+	VMScaleSetVMsClient     *MockVMScaleSetVMsClient
+	DisksClient             *MockDisksClient
+	RoleAssignmentsClient   *MockRoleAssignmentsClient
+	NetworkInterfacesClient *MockNetworkInterfacesClient
 }
 
 var _ azure.AzureCloud = &MockAzureCloud{}
@@ -87,6 +88,9 @@ func NewMockAzureCloud(location string) *MockAzureCloud {
 		RoleAssignmentsClient: &MockRoleAssignmentsClient{
 			RAs: map[string]authz.RoleAssignment{},
 		},
+		NetworkInterfacesClient: &MockNetworkInterfacesClient{
+			NIs: map[string]network.Interface{},
+		},
 	}
 }
 
@@ -195,6 +199,11 @@ func (c *MockAzureCloud) RoleAssignment() azure.RoleAssignmentsClient {
 	return c.RoleAssignmentsClient
 }
 
+// NetworkInterface returns the network interface client.
+func (c *MockAzureCloud) NetworkInterface() azure.NetworkInterfacesClient {
+	return c.NetworkInterfacesClient
+}
+
 // MockResourceGroupsClient is a mock implementation of resource group client.
 type MockResourceGroupsClient struct {
 	RGs map[string]resources.Group
@@ -479,3 +488,20 @@ func (c *MockRoleAssignmentsClient) Delete(ctx context.Context, scope, raName st
 	delete(c.RAs, raName)
 	return nil
 }
+
+// MockNetworkInterfacesClient is a mock implementation of network interfaces client.
+type MockNetworkInterfacesClient struct {
+	NIs map[string]network.Interface
+}
+
+var _ azure.NetworkInterfacesClient = &MockNetworkInterfacesClient{}
+
+// List returns a slice of VM Scale Set Network Interfaces.
+func (c *MockNetworkInterfacesClient) ListScaleSetsNetworkInterfaces(ctx context.Context, resourceGroupName, vmssName string) ([]network.Interface, error) {
+	// Ignore resourceGroupName and vmssName for simplicity.
+	var l []network.Interface
+	for _, ni := range c.NIs {
+		l = append(l, ni)
+	}
+	return l, nil
+}