add support for azure public loadbalancer

This commit is contained in:
Collin Woodruff 2021-02-22 18:21:31 -05:00
parent 5e10d54563
commit ee7fc850ff
11 changed files with 552 additions and 25 deletions

View File

@ -48,16 +48,6 @@ func (b *APILoadBalancerModelBuilder) Build(c *fi.ModelBuilderContext) error {
return nil
}
switch lbSpec.Type {
case kops.LoadBalancerTypeInternal:
// OK
case kops.LoadBalancerTypePublic:
// TODO: Implement creating public ip and attach to public loadbalancer
return fmt.Errorf("only internal loadbalancer for API server is implemented in Azure")
default:
return fmt.Errorf("unhandled LoadBalancer type %q", lbSpec.Type)
}
// Create LoadBalancer for API ELB
lb := &azuretasks.LoadBalancer{
Name: fi.String(b.NameForLoadBalancer()),
@ -76,6 +66,15 @@ func (b *APILoadBalancerModelBuilder) Build(c *fi.ModelBuilderContext) error {
lb.Subnet = b.LinkToAzureSubnet(subnet)
case kops.LoadBalancerTypePublic:
lb.External = to.BoolPtr(true)
// Create Public IP Address for Public Loadbalacer
p := &azuretasks.PublicIPAddress{
Name: fi.String(b.NameForLoadBalancer()),
Lifecycle: b.Lifecycle,
ResourceGroup: b.LinkToResourceGroup(),
Tags: map[string]*string{},
}
c.AddTask(p)
default:
return fmt.Errorf("unknown load balancer Type: %q", lbSpec.Type)
}

View File

@ -40,6 +40,7 @@ const (
typeDisk = "Disk"
typeRoleAssignment = "RoleAssignment"
typeLoadBalancer = "LoadBalancer"
typePublicIPAddress = "PublicIPAddress"
)
// ListResourcesAzure lists all resources for the cluster by quering Azure.
@ -89,6 +90,7 @@ func (g *resourceGetter) listAll() ([]*resources.Resource, error) {
g.listVMScaleSetsAndRoleAssignments,
g.listDisks,
g.listLoadBalancers,
g.listPublicIPAddresses,
}
var resources []*resources.Resource
@ -408,11 +410,11 @@ func (g *resourceGetter) listLoadBalancers(ctx context.Context) ([]*resources.Re
var rs []*resources.Resource
for i := range loadBalancers {
rt := &loadBalancers[i]
if !g.isOwnedByCluster(rt.Tags) {
lb := &loadBalancers[i]
if !g.isOwnedByCluster(lb.Tags) {
continue
}
rs = append(rs, g.toLoadBalancerResource(rt))
rs = append(rs, g.toLoadBalancerResource(lb))
}
return rs, nil
}
@ -432,6 +434,38 @@ func (g *resourceGetter) deleteLoadBalancer(_ fi.Cloud, r *resources.Resource) e
return g.cloud.LoadBalancer().Delete(context.TODO(), g.resourceGroupName(), r.Name)
}
func (g *resourceGetter) listPublicIPAddresses(ctx context.Context) ([]*resources.Resource, error) {
publicIPAddresses, err := g.cloud.PublicIPAddress().List(ctx, g.resourceGroupName())
if err != nil {
return nil, err
}
var rs []*resources.Resource
for i := range publicIPAddresses {
p := &publicIPAddresses[i]
if !g.isOwnedByCluster(p.Tags) {
continue
}
rs = append(rs, g.toPublicIPAddressResource(p))
}
return rs, nil
}
func (g *resourceGetter) toPublicIPAddressResource(publicIPAddress *network.PublicIPAddress) *resources.Resource {
return &resources.Resource{
Obj: publicIPAddress,
Type: typePublicIPAddress,
ID: *publicIPAddress.Name,
Name: *publicIPAddress.Name,
Deleter: g.deletePublicIPAddress,
Blocks: []string{toKey(typeResourceGroup, g.resourceGroupName())},
}
}
func (g *resourceGetter) deletePublicIPAddress(_ fi.Cloud, r *resources.Resource) error {
return g.cloud.PublicIPAddress().Delete(context.TODO(), g.resourceGroupName(), r.Name)
}
// isOwnedByCluster returns true if the resource is owned by the cluster.
func (g *resourceGetter) isOwnedByCluster(tags map[string]*string) bool {
for k, v := range tags {

View File

@ -9,6 +9,7 @@ go_library(
"disk.go",
"loadbalancer.go",
"networkinterface.go",
"publicipaddress.go",
"resourcegroup.go",
"roleassignment.go",
"routetable.go",

View File

@ -56,6 +56,7 @@ type AzureCloud interface {
RoleAssignment() RoleAssignmentsClient
NetworkInterface() NetworkInterfacesClient
LoadBalancer() LoadBalancersClient
PublicIPAddress() PublicIPAddressesClient
}
type azureCloudImplementation struct {
@ -72,6 +73,7 @@ type azureCloudImplementation struct {
roleAssignmentsClient RoleAssignmentsClient
networkInterfacesClient NetworkInterfacesClient
loadBalancersClient LoadBalancersClient
publicIPAddressesClient PublicIPAddressesClient
}
var _ fi.Cloud = &azureCloudImplementation{}
@ -97,6 +99,7 @@ func NewAzureCloud(subscriptionID, location string, tags map[string]string) (Azu
roleAssignmentsClient: newRoleAssignmentsClientImpl(subscriptionID, authorizer),
networkInterfacesClient: newNetworkInterfacesClientImpl(subscriptionID, authorizer),
loadBalancersClient: newLoadBalancersClientImpl(subscriptionID, authorizer),
publicIPAddressesClient: newPublicIPAddressesClientImpl(subscriptionID, authorizer),
}, nil
}
@ -266,3 +269,7 @@ func (c *azureCloudImplementation) NetworkInterface() NetworkInterfacesClient {
func (c *azureCloudImplementation) LoadBalancer() LoadBalancersClient {
return c.loadBalancersClient
}
func (c *azureCloudImplementation) PublicIPAddress() PublicIPAddressesClient {
return c.publicIPAddressesClient
}

View File

@ -0,0 +1,73 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package azure
import (
"context"
"fmt"
"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-06-01/network"
"github.com/Azure/go-autorest/autorest"
)
// PublicIPAddressesClient is a client for public ip addresses.
type PublicIPAddressesClient interface {
CreateOrUpdate(ctx context.Context, resourceGroupName, publicIPAddressName string, parameters network.PublicIPAddress) error
List(ctx context.Context, resourceGroupName string) ([]network.PublicIPAddress, error)
Delete(ctx context.Context, resourceGroupName, publicIPAddressName string) error
}
type publicIPAddressesClientImpl struct {
c *network.PublicIPAddressesClient
}
var _ PublicIPAddressesClient = &publicIPAddressesClientImpl{}
func (c *publicIPAddressesClientImpl) CreateOrUpdate(ctx context.Context, resourceGroupName, publicIPAddressName string, parameters network.PublicIPAddress) error {
_, err := c.c.CreateOrUpdate(ctx, resourceGroupName, publicIPAddressName, parameters)
return err
}
func (c *publicIPAddressesClientImpl) List(ctx context.Context, resourceGroupName string) ([]network.PublicIPAddress, error) {
var l []network.PublicIPAddress
for iter, err := c.c.ListComplete(ctx, resourceGroupName); iter.NotDone(); err = iter.Next() {
if err != nil {
return nil, err
}
l = append(l, iter.Value())
}
return l, nil
}
func (c *publicIPAddressesClientImpl) Delete(ctx context.Context, resourceGroupName, publicIPAddressName string) error {
future, err := c.c.Delete(ctx, resourceGroupName, publicIPAddressName)
if err != nil {
return fmt.Errorf("error deleting public ip address: %s", err)
}
if err := future.WaitForCompletionRef(ctx, c.c.Client); err != nil {
return fmt.Errorf("error waiting for public ip address deletion completion: %s", err)
}
return nil
}
func newPublicIPAddressesClientImpl(subscriptionID string, authorizer autorest.Authorizer) *publicIPAddressesClientImpl {
c := network.NewPublicIPAddressesClient(subscriptionID)
c.Authorizer = authorizer
return &publicIPAddressesClientImpl{
c: &c,
}
}

View File

@ -7,6 +7,8 @@ go_library(
"disk_fitask.go",
"loadbalancer.go",
"loadbalancer_fitask.go",
"publicipaddress.go",
"publicipaddress_fitask.go",
"resourcegroup.go",
"resourcegroup_fitask.go",
"roleassignment.go",
@ -45,6 +47,7 @@ go_test(
srcs = [
"disk_test.go",
"loadbalancer_test.go",
"publicipaddress_test.go",
"resourcegroup_test.go",
"roleassignment_test.go",
"subnet_test.go",

View File

@ -131,10 +131,9 @@ func (*LoadBalancer) RenderAzure(t *azure.AzureAPITarget, a, e, changes *LoadBal
idPrefix := fmt.Sprintf("subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network", t.Cloud.SubscriptionID(), *e.ResourceGroup.Name)
feConfigProperties := &network.FrontendIPConfigurationPropertiesFormat{}
if *e.External {
// TODO: Implement public load balancer
// feConfigProperties.PublicIPAddress = &network.PublicIPAddress{
// ID: to.StringPtr(fmt.Sprintf("/%s/publicIPAddresses/%s", idPrefix, *e.PublicIPName)),
// }
feConfigProperties.PublicIPAddress = &network.PublicIPAddress{
ID: to.StringPtr(fmt.Sprintf("/%s/publicIPAddresses/%s", idPrefix, *e.Name)),
}
} else {
feConfigProperties.PrivateIPAllocationMethod = network.Dynamic
feConfigProperties.Subnet = &network.Subnet{

View File

@ -0,0 +1,123 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package azuretasks
import (
"context"
"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-06-01/network"
"github.com/Azure/go-autorest/autorest/to"
"k8s.io/klog/v2"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/azure"
)
//go:generate fitask -type=PublicIPAddress
// PublicIPAddress is an Azure Cloud Public IP Address
type PublicIPAddress struct {
Name *string
Lifecycle *fi.Lifecycle
ResourceGroup *ResourceGroup
Tags map[string]*string
}
var _ fi.Task = &PublicIPAddress{}
var _ fi.CompareWithID = &PublicIPAddress{}
// CompareWithID returns the Name of the Public IP Address
func (p *PublicIPAddress) CompareWithID() *string {
return p.Name
}
// Find discovers the Public IP Address in the cloud provider
func (p *PublicIPAddress) Find(c *fi.Context) (*PublicIPAddress, error) {
cloud := c.Cloud.(azure.AzureCloud)
l, err := cloud.PublicIPAddress().List(context.TODO(), *p.ResourceGroup.Name)
if err != nil {
return nil, err
}
var found *network.PublicIPAddress
for _, v := range l {
if *v.Name == *p.Name {
found = &v
break
}
}
if found == nil {
return nil, nil
}
return &PublicIPAddress{
Name: p.Name,
Lifecycle: p.Lifecycle,
ResourceGroup: &ResourceGroup{
Name: p.ResourceGroup.Name,
},
Tags: found.Tags,
}, nil
}
// Run implements fi.Task.Run.
func (p *PublicIPAddress) Run(c *fi.Context) error {
c.Cloud.(azure.AzureCloud).AddClusterTags(p.Tags)
return fi.DefaultDeltaRunMethod(p, c)
}
// CheckChanges returns an error if a change is not allowed.
func (*PublicIPAddress) CheckChanges(a, e, changes *PublicIPAddress) error {
if a == nil {
// Check if required fields are set when a new resource is created.
if e.Name == nil {
return fi.RequiredField("Name")
}
return nil
}
// Check if unchanegable fields won't be changed.
if changes.Name != nil {
return fi.CannotChangeField("Name")
}
return nil
}
// RenderAzure creates or updates a Public IP Address.
func (*PublicIPAddress) RenderAzure(t *azure.AzureAPITarget, a, e, changes *PublicIPAddress) error {
if a == nil {
klog.Infof("Creating a new Public IP Address with name: %s", fi.StringValue(e.Name))
} else {
klog.Infof("Updating a Public IP Address with name: %s", fi.StringValue(e.Name))
}
p := network.PublicIPAddress{
Location: to.StringPtr(t.Cloud.Region()),
Name: to.StringPtr(*e.Name),
PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{
PublicIPAddressVersion: network.IPv4,
PublicIPAllocationMethod: network.Dynamic,
},
Tags: e.Tags,
}
return t.Cloud.PublicIPAddress().CreateOrUpdate(
context.TODO(),
*e.ResourceGroup.Name,
*e.Name,
p)
}

View File

@ -0,0 +1,75 @@
/*
Copyright The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Code generated by ""fitask" -type=PublicIPAddress"; DO NOT EDIT
package azuretasks
import (
"encoding/json"
"k8s.io/kops/upup/pkg/fi"
)
// PublicIPAddress
// JSON marshaling boilerplate
type realPublicIPAddress PublicIPAddress
// UnmarshalJSON implements conversion to JSON, supporting an alternate specification of the object as a string
func (o *PublicIPAddress) UnmarshalJSON(data []byte) error {
var jsonName string
if err := json.Unmarshal(data, &jsonName); err == nil {
o.Name = &jsonName
return nil
}
var r realPublicIPAddress
if err := json.Unmarshal(data, &r); err != nil {
return err
}
*o = PublicIPAddress(r)
return nil
}
var _ fi.HasLifecycle = &PublicIPAddress{}
// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle
func (o *PublicIPAddress) GetLifecycle() *fi.Lifecycle {
return o.Lifecycle
}
// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle
func (o *PublicIPAddress) SetLifecycle(lifecycle fi.Lifecycle) {
o.Lifecycle = &lifecycle
}
var _ fi.HasName = &PublicIPAddress{}
// GetName returns the Name of the object, implementing fi.HasName
func (o *PublicIPAddress) GetName() *string {
return o.Name
}
// SetName sets the Name of the object, implementing fi.SetName
func (o *PublicIPAddress) SetName(name string) {
o.Name = &name
}
// String is the stringer function for the task, producing readable output using fi.TaskAsString
func (o *PublicIPAddress) String() string {
return fi.TaskAsString(o)
}

View File

@ -0,0 +1,168 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package azuretasks
import (
"context"
"fmt"
"reflect"
"testing"
"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-06-01/network"
"github.com/Azure/go-autorest/autorest/to"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/azure"
)
func newTestPublicIPAddress() *PublicIPAddress {
return &PublicIPAddress{
Name: to.StringPtr("publicIPAddress"),
ResourceGroup: &ResourceGroup{
Name: to.StringPtr("rg"),
},
Tags: map[string]*string{
testTagKey: to.StringPtr(testTagValue),
},
}
}
func TestPublicIPAddressRenderAzure(t *testing.T) {
cloud := NewMockAzureCloud("eastus")
apiTarget := azure.NewAzureAPITarget(cloud)
publicIPAddress := &PublicIPAddress{}
expected := newTestPublicIPAddress()
if err := publicIPAddress.RenderAzure(apiTarget, nil, expected, nil); err != nil {
t.Fatalf("unexpected error: %s", err)
}
actual := cloud.PublicIPAddressesClient.PubIPs[*expected.Name]
if a, e := *actual.Name, *expected.Name; a != e {
t.Errorf("unexpected Name: expected %s, but got %s", e, a)
}
if a, e := *actual.Location, cloud.Region(); a != e {
t.Fatalf("unexpected location: expected %s, but got %s", e, a)
}
}
func TestPublicIPAddressFind(t *testing.T) {
cloud := NewMockAzureCloud("eastus")
ctx := &fi.Context{
Cloud: cloud,
}
rg := &ResourceGroup{
Name: to.StringPtr("rg"),
}
publicIPAddress := &PublicIPAddress{
Name: to.StringPtr("publicIPAddress"),
ResourceGroup: &ResourceGroup{
Name: rg.Name,
},
}
// Find will return nothing if there is no public ip address created.
actual, err := publicIPAddress.Find(ctx)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if actual != nil {
t.Errorf("unexpected publicIPAddress found: %+v", actual)
}
// Create a public ip address.
publicIPAddressParameters := network.PublicIPAddress{
Location: to.StringPtr("eastus"),
Name: to.StringPtr("publicIPAddress"),
PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{
PublicIPAddressVersion: network.IPv4,
PublicIPAllocationMethod: network.Dynamic,
},
}
if err := cloud.PublicIPAddress().CreateOrUpdate(context.Background(), *rg.Name, *publicIPAddress.Name, publicIPAddressParameters); err != nil {
t.Fatalf("failed to create: %s", err)
}
// Find again.
actual, err = publicIPAddress.Find(ctx)
t.Log(actual)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if a, e := *actual.Name, *publicIPAddress.Name; a != e {
t.Errorf("unexpected publicIPAddress name: expected %s, but got %s", e, a)
}
if a, e := *actual.ResourceGroup.Name, *rg.Name; a != e {
t.Errorf("unexpected Resource Group name: expected %s, but got %s", e, a)
}
}
func TestPublicIPAddressRun(t *testing.T) {
cloud := NewMockAzureCloud("eastus")
ctx := &fi.Context{
Cloud: cloud,
Target: azure.NewAzureAPITarget(cloud),
}
lb := newTestPublicIPAddress()
err := lb.Run(ctx)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
e := map[string]*string{
azure.TagClusterName: to.StringPtr(testClusterName),
testTagKey: to.StringPtr(testTagValue),
}
if a := lb.Tags; !reflect.DeepEqual(a, e) {
t.Errorf("unexpected tags: expected %+v, but got %+v", e, a)
}
}
func TestPublicIPAddressCheckChanges(t *testing.T) {
testCases := []struct {
a, e, changes *PublicIPAddress
success bool
}{
{
a: nil,
e: &PublicIPAddress{Name: to.StringPtr("name")},
changes: nil,
success: true,
},
{
a: nil,
e: &PublicIPAddress{Name: nil},
changes: nil,
success: false,
},
{
a: &PublicIPAddress{Name: to.StringPtr("name")},
changes: &PublicIPAddress{Name: nil},
success: true,
},
{
a: &PublicIPAddress{Name: to.StringPtr("name")},
changes: &PublicIPAddress{Name: to.StringPtr("newName")},
success: false,
},
}
for i, tc := range testCases {
t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
publicIPAddress := PublicIPAddress{}
err := publicIPAddress.CheckChanges(tc.a, tc.e, tc.changes)
if tc.success != (err == nil) {
t.Errorf("expected success=%t, but got err=%v", tc.success, err)
}
})
}
}

View File

@ -57,6 +57,7 @@ type MockAzureCloud struct {
RoleAssignmentsClient *MockRoleAssignmentsClient
NetworkInterfacesClient *MockNetworkInterfacesClient
LoadBalancersClient *MockLoadBalancersClient
PublicIPAddressesClient *MockPublicIPAddressesClient
}
var _ azure.AzureCloud = &MockAzureCloud{}
@ -95,6 +96,9 @@ func NewMockAzureCloud(location string) *MockAzureCloud {
LoadBalancersClient: &MockLoadBalancersClient{
LBs: map[string]network.LoadBalancer{},
},
PublicIPAddressesClient: &MockPublicIPAddressesClient{
PubIPs: map[string]network.PublicIPAddress{},
},
}
}
@ -213,6 +217,11 @@ func (c *MockAzureCloud) LoadBalancer() azure.LoadBalancersClient {
return c.LoadBalancersClient
}
// PublicIPAddress returns the public ip address client.
func (c *MockAzureCloud) PublicIPAddress() azure.PublicIPAddressesClient {
return c.PublicIPAddressesClient
}
// MockResourceGroupsClient is a mock implementation of resource group client.
type MockResourceGroupsClient struct {
RGs map[string]resources.Group
@ -550,3 +559,39 @@ func (c *MockLoadBalancersClient) Delete(ctx context.Context, scope, lbName stri
delete(c.LBs, lbName)
return nil
}
// MockPublicIPAddressesClient is a mock implementation of role assignment client.
type MockPublicIPAddressesClient struct {
PubIPs map[string]network.PublicIPAddress
}
var _ azure.PublicIPAddressesClient = &MockPublicIPAddressesClient{}
// CreateOrUpdate creates a new public ip address.
func (c *MockPublicIPAddressesClient) CreateOrUpdate(ctx context.Context, resourceGroupName, publicIPAddressName string, parameters network.PublicIPAddress) error {
if _, ok := c.PubIPs[publicIPAddressName]; ok {
return nil
}
parameters.Name = &publicIPAddressName
c.PubIPs[publicIPAddressName] = parameters
return nil
}
// List returns a slice of public ip address.
func (c *MockPublicIPAddressesClient) List(ctx context.Context, resourceGroupName string) ([]network.PublicIPAddress, error) {
var l []network.PublicIPAddress
for _, lb := range c.PubIPs {
l = append(l, lb)
}
return l, nil
}
// Delete deletes a specified public ip address.
func (c *MockPublicIPAddressesClient) Delete(ctx context.Context, scope, publicIPAddressName string) error {
// Ignore scope for simplicity.
if _, ok := c.PubIPs[publicIPAddressName]; !ok {
return fmt.Errorf("%s does not exist", publicIPAddressName)
}
delete(c.PubIPs, publicIPAddressName)
return nil
}