diff --git a/docs/getting_started/azure.md b/docs/getting_started/azure.md index 5dc8ce35fd..a8a2564508 100644 --- a/docs/getting_started/azure.md +++ b/docs/getting_started/azure.md @@ -174,7 +174,7 @@ The command line flags prefixed with `--azure-` are for Azure-specific configurations. ```bash -$ export KOPS_FEATURE_FLAGS=AlphaAllowAzure +$ export KOPS_FEATURE_FLAGS=Azure $ kops create cluster \ --cloud azure \ diff --git a/upup/pkg/fi/cloudup/azure/BUILD.bazel b/upup/pkg/fi/cloudup/azure/BUILD.bazel index 2655c3e96b..7b683b4778 100644 --- a/upup/pkg/fi/cloudup/azure/BUILD.bazel +++ b/upup/pkg/fi/cloudup/azure/BUILD.bazel @@ -7,6 +7,7 @@ go_library( "azure_cloud.go", "azure_utils.go", "disk.go", + "networkinterface.go", "resourcegroup.go", "roleassignment.go", "routetable.go", diff --git a/upup/pkg/fi/cloudup/azure/azure_cloud.go b/upup/pkg/fi/cloudup/azure/azure_cloud.go index ffe9b8d380..52938ff9ee 100644 --- a/upup/pkg/fi/cloudup/azure/azure_cloud.go +++ b/upup/pkg/fi/cloudup/azure/azure_cloud.go @@ -17,7 +17,9 @@ limitations under the License. package azure import ( + "context" "errors" + "fmt" "github.com/Azure/go-autorest/autorest/azure/auth" "k8s.io/kops/dnsprovider/pkg/dnsprovider" @@ -52,20 +54,22 @@ type AzureCloud interface { VMScaleSetVM() VMScaleSetVMsClient Disk() DisksClient RoleAssignment() RoleAssignmentsClient + NetworkInterface() NetworkInterfacesClient } type azureCloudImplementation struct { - subscriptionID string - location string - tags map[string]string - resourceGroupsClient ResourceGroupsClient - vnetsClient VirtualNetworksClient - subnetsClient SubnetsClient - routeTablesClient RouteTablesClient - vmscaleSetsClient VMScaleSetsClient - vmscaleSetVMsClient VMScaleSetVMsClient - disksClient DisksClient - roleAssignmentsClient RoleAssignmentsClient + subscriptionID string + location string + tags map[string]string + resourceGroupsClient ResourceGroupsClient + vnetsClient VirtualNetworksClient + subnetsClient SubnetsClient + routeTablesClient RouteTablesClient + vmscaleSetsClient VMScaleSetsClient + vmscaleSetVMsClient VMScaleSetVMsClient + disksClient DisksClient + roleAssignmentsClient RoleAssignmentsClient + networkInterfacesClient NetworkInterfacesClient } var _ fi.Cloud = &azureCloudImplementation{} @@ -78,17 +82,18 @@ func NewAzureCloud(subscriptionID, location string, tags map[string]string) (Azu } return &azureCloudImplementation{ - subscriptionID: subscriptionID, - location: location, - tags: tags, - resourceGroupsClient: newResourceGroupsClientImpl(subscriptionID, authorizer), - vnetsClient: newVirtualNetworksClientImpl(subscriptionID, authorizer), - subnetsClient: newSubnetsClientImpl(subscriptionID, authorizer), - routeTablesClient: newRouteTablesClientImpl(subscriptionID, authorizer), - vmscaleSetsClient: newVMScaleSetsClientImpl(subscriptionID, authorizer), - vmscaleSetVMsClient: newVMScaleSetVMsClientImpl(subscriptionID, authorizer), - disksClient: newDisksClientImpl(subscriptionID, authorizer), - roleAssignmentsClient: newRoleAssignmentsClientImpl(subscriptionID, authorizer), + subscriptionID: subscriptionID, + location: location, + tags: tags, + resourceGroupsClient: newResourceGroupsClientImpl(subscriptionID, authorizer), + vnetsClient: newVirtualNetworksClientImpl(subscriptionID, authorizer), + subnetsClient: newSubnetsClientImpl(subscriptionID, authorizer), + routeTablesClient: newRouteTablesClientImpl(subscriptionID, authorizer), + vmscaleSetsClient: newVMScaleSetsClientImpl(subscriptionID, authorizer), + vmscaleSetVMsClient: newVMScaleSetVMsClientImpl(subscriptionID, authorizer), + disksClient: newDisksClientImpl(subscriptionID, authorizer), + roleAssignmentsClient: newRoleAssignmentsClientImpl(subscriptionID, authorizer), + networkInterfacesClient: newNetworkInterfacesClientImpl(subscriptionID, authorizer), }, nil } @@ -128,9 +133,43 @@ func (c *azureCloudImplementation) AddClusterTags(tags map[string]*string) { } func (c *azureCloudImplementation) GetApiIngressStatus(cluster *kops.Cluster) ([]kops.ApiIngressStatus, error) { - // TODO(kenji): Implement this. Currently we return nil as we - // don't create any resources for ingress to the API server. - return nil, nil + var ingresses []kops.ApiIngressStatus + var rg string = cluster.AzureResourceGroupName() + + // Get scale sets in cluster resource group and find masters scale set + scaleSets, err := c.vmscaleSetsClient.List(context.TODO(), rg) + if err != nil { + return nil, fmt.Errorf("error getting Cluster Master Scale Set for API Ingress Status: %s", err) + } + var vmssName string + for _, scaleSet := range scaleSets { + val, ok := scaleSet.Tags[TagClusterName] + val2, ok2 := scaleSet.Tags[TagNameRolePrefix+TagRoleMaster] + if ok && *val == cluster.Name && ok2 && *val2 == "1" { + vmssName = *scaleSet.Name + break + } + } + if vmssName == "" { + return nil, fmt.Errorf("error getting Master Scale Set Name for API Ingress Status") + } + + // Get masters scale set network interfaces and append to api ingress status + nis, err := c.NetworkInterface().ListScaleSetsNetworkInterfaces(context.TODO(), rg, vmssName) + if err != nil { + return nil, fmt.Errorf("error getting Master Scale Set Network Interfaces for API Ingress Status: %s", err) + } + for _, ni := range nis { + if !*ni.Primary { + continue + } + for _, i := range *ni.IPConfigurations { + ingresses = append(ingresses, kops.ApiIngressStatus{ + IP: *i.PrivateIPAddress, + }) + } + } + return ingresses, nil } func (c *azureCloudImplementation) SubscriptionID() string { @@ -168,3 +207,7 @@ func (c *azureCloudImplementation) Disk() DisksClient { func (c *azureCloudImplementation) RoleAssignment() RoleAssignmentsClient { return c.roleAssignmentsClient } + +func (c *azureCloudImplementation) NetworkInterface() NetworkInterfacesClient { + return c.networkInterfacesClient +} diff --git a/upup/pkg/fi/cloudup/azure/networkinterface.go b/upup/pkg/fi/cloudup/azure/networkinterface.go new file mode 100644 index 0000000000..e0d5195c8b --- /dev/null +++ b/upup/pkg/fi/cloudup/azure/networkinterface.go @@ -0,0 +1,54 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-06-01/network" + "github.com/Azure/go-autorest/autorest" +) + +// NetworkInterfacesClient is a client for managing Network Interfaces. +type NetworkInterfacesClient interface { + ListScaleSetsNetworkInterfaces(ctx context.Context, resourceGroupName, vmssName string) ([]network.Interface, error) +} + +type networkInterfacesClientImpl struct { + c *network.InterfacesClient +} + +var _ NetworkInterfacesClient = &networkInterfacesClientImpl{} + +func (c *networkInterfacesClientImpl) ListScaleSetsNetworkInterfaces(ctx context.Context, resourceGroupName, vmssName string) ([]network.Interface, error) { + var l []network.Interface + for iter, err := c.c.ListVirtualMachineScaleSetNetworkInterfacesComplete(ctx, resourceGroupName, vmssName); iter.NotDone(); err = iter.Next() { + if err != nil { + return nil, err + } + l = append(l, iter.Value()) + } + return l, nil +} + +func newNetworkInterfacesClientImpl(subscriptionID string, authorizer autorest.Authorizer) *networkInterfacesClientImpl { + c := network.NewInterfacesClient(subscriptionID) + c.Authorizer = authorizer + return &networkInterfacesClientImpl{ + c: &c, + } +} diff --git a/upup/pkg/fi/cloudup/azuretasks/testing.go b/upup/pkg/fi/cloudup/azuretasks/testing.go index d9855c2195..1777513803 100644 --- a/upup/pkg/fi/cloudup/azuretasks/testing.go +++ b/upup/pkg/fi/cloudup/azuretasks/testing.go @@ -46,15 +46,16 @@ const ( // MockAzureCloud is a mock implementation of AzureCloud. type MockAzureCloud struct { - Location string - ResourceGroupsClient *MockResourceGroupsClient - VirtualNetworksClient *MockVirtualNetworksClient - SubnetsClient *MockSubnetsClient - RouteTablesClient *MockRouteTablesClient - VMScaleSetsClient *MockVMScaleSetsClient - VMScaleSetVMsClient *MockVMScaleSetVMsClient - DisksClient *MockDisksClient - RoleAssignmentsClient *MockRoleAssignmentsClient + Location string + ResourceGroupsClient *MockResourceGroupsClient + VirtualNetworksClient *MockVirtualNetworksClient + SubnetsClient *MockSubnetsClient + RouteTablesClient *MockRouteTablesClient + VMScaleSetsClient *MockVMScaleSetsClient + VMScaleSetVMsClient *MockVMScaleSetVMsClient + DisksClient *MockDisksClient + RoleAssignmentsClient *MockRoleAssignmentsClient + NetworkInterfacesClient *MockNetworkInterfacesClient } var _ azure.AzureCloud = &MockAzureCloud{} @@ -87,6 +88,9 @@ func NewMockAzureCloud(location string) *MockAzureCloud { RoleAssignmentsClient: &MockRoleAssignmentsClient{ RAs: map[string]authz.RoleAssignment{}, }, + NetworkInterfacesClient: &MockNetworkInterfacesClient{ + NIs: map[string]network.Interface{}, + }, } } @@ -195,6 +199,11 @@ func (c *MockAzureCloud) RoleAssignment() azure.RoleAssignmentsClient { return c.RoleAssignmentsClient } +// NetworkInterface returns the network interface client. +func (c *MockAzureCloud) NetworkInterface() azure.NetworkInterfacesClient { + return c.NetworkInterfacesClient +} + // MockResourceGroupsClient is a mock implementation of resource group client. type MockResourceGroupsClient struct { RGs map[string]resources.Group @@ -479,3 +488,20 @@ func (c *MockRoleAssignmentsClient) Delete(ctx context.Context, scope, raName st delete(c.RAs, raName) return nil } + +// MockNetworkInterfacesClient is a mock implementation of network interfaces client. +type MockNetworkInterfacesClient struct { + NIs map[string]network.Interface +} + +var _ azure.NetworkInterfacesClient = &MockNetworkInterfacesClient{} + +// List returns a slice of VM Scale Set Network Interfaces. +func (c *MockNetworkInterfacesClient) ListScaleSetsNetworkInterfaces(ctx context.Context, resourceGroupName, vmssName string) ([]network.Interface, error) { + // Ignore resourceGroupName and vmssName for simplicity. + var l []network.Interface + for _, ni := range c.NIs { + l = append(l, ni) + } + return l, nil +}