From 7c53b35c9d248e68e582e5fcb76b4c15665810af Mon Sep 17 00:00:00 2001 From: "Derek Lemon (delemon)" Date: Tue, 21 May 2019 07:17:34 -0600 Subject: [PATCH 1/3] Delete floating IP's based on the subnet resource, to clean up dynamically added ones as well, allowing for subnet deletion Bazel updates Updating deletion logic for octavia clusters --- pkg/resources/openstack/BUILD.bazel | 3 +- pkg/resources/openstack/lb.go | 77 +++++++++---------- pkg/resources/openstack/network.go | 8 ++ pkg/resources/openstack/openstack.go | 3 - upup/pkg/fi/cloudup/openstack/BUILD.bazel | 1 + upup/pkg/fi/cloudup/openstack/cloud.go | 7 ++ upup/pkg/fi/cloudup/openstack/loadbalancer.go | 40 ++++++++++ 7 files changed, 92 insertions(+), 47 deletions(-) diff --git a/pkg/resources/openstack/BUILD.bazel b/pkg/resources/openstack/BUILD.bazel index 64d9649641..5ca6361777 100644 --- a/pkg/resources/openstack/BUILD.bazel +++ b/pkg/resources/openstack/BUILD.bazel @@ -25,9 +25,8 @@ go_library( "//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/servers:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack/dns/v2/recordsets:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack/dns/v2/zones:go_default_library", - "//vendor/github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/listeners:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers:go_default_library", - "//vendor/github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/pools:go_default_library", + "//vendor/github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/monitors:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups:go_default_library", diff --git a/pkg/resources/openstack/lb.go b/pkg/resources/openstack/lb.go index 1f2f5c0a9e..4583862cf1 100644 --- a/pkg/resources/openstack/lb.go +++ b/pkg/resources/openstack/lb.go @@ -17,11 +17,9 @@ limitations under the License. package openstack import ( - "strings" - - "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/listeners" "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers" - v2pools "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/pools" + "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/monitors" + "github.com/gophercloud/gophercloud/openstack/networking/v2/subnets" "k8s.io/klog" "k8s.io/kops/pkg/resources" "k8s.io/kops/upup/pkg/fi" @@ -29,15 +27,17 @@ import ( ) const ( - typeLB = "LoadBalancer" - typeLBL = "LBListener" - typeLBP = "LBPool" + typeLB = "LoadBalancer" + typeLBL = "LBListener" + typeLBP = "LBPool" + typeLBPM = "LBPoolMonitor" ) -func (os *clusterDiscoveryOS) ListLB() ([]*resources.Resource, error) { +func (os *clusterDiscoveryOS) DeleteSubnetLBs(subnet subnets.Subnet) ([]*resources.Resource, error) { var resourceTrackers []*resources.Resource + opts := loadbalancers.ListOpts{ - Name: "api." + os.clusterName, + VipSubnetID: subnet.ID, } lbs, err := os.osCloud.ListLBs(opts) if err != nil { @@ -57,25 +57,33 @@ func (os *clusterDiscoveryOS) ListLB() ([]*resources.Resource, error) { }, } resourceTrackers = append(resourceTrackers, resourceTracker) - } - return resourceTrackers, nil -} -func (os *clusterDiscoveryOS) ListLBPools() ([]*resources.Resource, error) { - var resourceTrackers []*resources.Resource + if os.osCloud.UseOctavia() { + klog.V(2).Info("skipping LB Children because using Octavia") + continue + } - if os.osCloud.UseOctavia() { - klog.V(2).Info("skipping ListLBPools because using Octavia") - return nil, nil - } + //Identify pools associated to this LB + for _, pool := range lb.Pools { - pools, err := os.osCloud.ListPools(v2pools.ListOpts{}) - if err != nil { - return nil, err - } + monitorList, err := os.cloud.(openstack.OpenstackCloud).ListMonitors(monitors.ListOpts{ + PoolID: pool.ID, + }) + if err != nil { + return nil, err + } + for _, monitor := range monitorList { + resourceTracker := &resources.Resource{ + Name: monitor.Name, + ID: monitor.ID, + Type: typeLBPM, + Deleter: func(cloud fi.Cloud, r *resources.Resource) error { + return cloud.(openstack.OpenstackCloud).DeleteMonitor(r.ID) + }, + } + resourceTrackers = append(resourceTrackers, resourceTracker) + } - for _, pool := range pools { - if strings.Contains(pool.Name, os.clusterName) { resourceTracker := &resources.Resource{ Name: pool.Name, ID: pool.ID, @@ -86,25 +94,9 @@ func (os *clusterDiscoveryOS) ListLBPools() ([]*resources.Resource, error) { } resourceTrackers = append(resourceTrackers, resourceTracker) } - } - return resourceTrackers, nil -} -func (os *clusterDiscoveryOS) ListLBListener() ([]*resources.Resource, error) { - var resourceTrackers []*resources.Resource - - if os.osCloud.UseOctavia() { - klog.V(2).Info("skipping ListLBListener because using Octavia") - return nil, nil - } - - listeners, err := os.osCloud.ListListeners(listeners.ListOpts{}) - if err != nil { - return nil, err - } - - for _, listener := range listeners { - if strings.Contains(listener.Name, os.clusterName) { + //Identify listeners associated to this LB + for _, listener := range lb.Listeners { resourceTracker := &resources.Resource{ Name: listener.Name, ID: listener.ID, @@ -116,5 +108,6 @@ func (os *clusterDiscoveryOS) ListLBListener() ([]*resources.Resource, error) { resourceTrackers = append(resourceTrackers, resourceTracker) } } + return resourceTrackers, nil } diff --git a/pkg/resources/openstack/network.go b/pkg/resources/openstack/network.go index 4e5225253f..ffbb76cc5c 100644 --- a/pkg/resources/openstack/network.go +++ b/pkg/resources/openstack/network.go @@ -97,6 +97,13 @@ func (os *clusterDiscoveryOS) ListNetwork() ([]*resources.Resource, error) { } resourceTrackers = append(resourceTrackers, resourceTracker) } + //associated load balancers + lbTrackers, err := os.DeleteSubnetLBs(subnet) + if err != nil { + return resourceTrackers, err + } + resourceTrackers = append(resourceTrackers, lbTrackers...) + resourceTracker := &resources.Resource{ Name: subnet.Name, ID: subnet.ID, @@ -106,6 +113,7 @@ func (os *clusterDiscoveryOS) ListNetwork() ([]*resources.Resource, error) { }, } resourceTrackers = append(resourceTrackers, resourceTracker) + } resourceTracker := &resources.Resource{ Name: network.Name, diff --git a/pkg/resources/openstack/openstack.go b/pkg/resources/openstack/openstack.go index 969de5374e..5c3fe05384 100644 --- a/pkg/resources/openstack/openstack.go +++ b/pkg/resources/openstack/openstack.go @@ -47,9 +47,6 @@ func ListResources(cloud openstack.OpenstackCloud, clusterName string) (map[stri os.ListInstances, os.ListServerGroups, os.ListVolumes, - os.ListLBListener, - os.ListLBPools, - os.ListLB, os.ListPorts, os.ListSecurityGroups, os.ListNetwork, diff --git a/upup/pkg/fi/cloudup/openstack/BUILD.bazel b/upup/pkg/fi/cloudup/openstack/BUILD.bazel index 003077022b..7508f1afa1 100644 --- a/upup/pkg/fi/cloudup/openstack/BUILD.bazel +++ b/upup/pkg/fi/cloudup/openstack/BUILD.bazel @@ -46,6 +46,7 @@ go_library( "//vendor/github.com/gophercloud/gophercloud/openstack/dns/v2/zones:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/listeners:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers:go_default_library", + "//vendor/github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/monitors:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/pools:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/external:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips:go_default_library", diff --git a/upup/pkg/fi/cloudup/openstack/cloud.go b/upup/pkg/fi/cloudup/openstack/cloud.go index 32b14038e7..1254940963 100644 --- a/upup/pkg/fi/cloudup/openstack/cloud.go +++ b/upup/pkg/fi/cloudup/openstack/cloud.go @@ -38,6 +38,7 @@ import ( "github.com/gophercloud/gophercloud/openstack/dns/v2/zones" "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/listeners" "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers" + "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/monitors" v2pools "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/pools" l3floatingip "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips" "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers" @@ -250,6 +251,12 @@ type OpenstackCloud interface { ListPools(v2pools.ListOpts) ([]v2pools.Pool, error) + // ListMonitors will list HealthMonitors matching the provided options + ListMonitors(monitors.ListOpts) ([]monitors.Monitor, error) + + // DeleteMonitor will delete a Pool resources Health Monitor + DeleteMonitor(monitorID string) error + // DeletePool will delete loadbalancer pool DeletePool(poolID string) error diff --git a/upup/pkg/fi/cloudup/openstack/loadbalancer.go b/upup/pkg/fi/cloudup/openstack/loadbalancer.go index 00d2b2f475..2b2078c13b 100644 --- a/upup/pkg/fi/cloudup/openstack/loadbalancer.go +++ b/upup/pkg/fi/cloudup/openstack/loadbalancer.go @@ -22,11 +22,51 @@ import ( "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/listeners" "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers" + "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/monitors" v2pools "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/pools" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/kops/util/pkg/vfs" ) +func (c *openstackCloud) ListMonitors(opts monitors.ListOpts) (monitorList []monitors.Monitor, err error) { + + done, err := vfs.RetryWithBackoff(readBackoff, func() (bool, error) { + allPages, err := monitors.List(c.LoadBalancerClient(), opts).AllPages() + if err != nil { + return false, fmt.Errorf("failed to list monitors: %s", err) + } + monitorList, err = monitors.ExtractMonitors(allPages) + if err != nil { + return false, fmt.Errorf("failed to extract monitor pages: %s", err) + } + return true, nil + }) + if !done { + if err == nil { + err = wait.ErrWaitTimeout + } + return monitorList, err + } + return monitorList, nil +} + +func (c *openstackCloud) DeleteMonitor(monitorID string) error { + done, err := vfs.RetryWithBackoff(writeBackoff, func() (bool, error) { + err := monitors.Delete(c.LoadBalancerClient(), monitorID).ExtractErr() + if err != nil && !isNotFound(err) { + return false, fmt.Errorf("error deleting pool: %v", err) + } + return true, nil + }) + if err != nil { + return err + } else if done { + return nil + } else { + return wait.ErrWaitTimeout + } +} + func (c *openstackCloud) DeletePool(poolID string) error { done, err := vfs.RetryWithBackoff(writeBackoff, func() (bool, error) { err := v2pools.Delete(c.LoadBalancerClient(), poolID).ExtractErr() From 7b433cfae733cc0b466ffa5def559485051bed36 Mon Sep 17 00:00:00 2001 From: "Derek Lemon (delemon)" Date: Wed, 29 May 2019 07:42:09 -0600 Subject: [PATCH 2/3] Openstack port resources derived from network --- pkg/resources/openstack/network.go | 8 +++++++ pkg/resources/openstack/openstack.go | 1 - pkg/resources/openstack/ports.go | 32 ++++++++++++++-------------- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/pkg/resources/openstack/network.go b/pkg/resources/openstack/network.go index ffbb76cc5c..f7abb3ddab 100644 --- a/pkg/resources/openstack/network.go +++ b/pkg/resources/openstack/network.go @@ -115,6 +115,14 @@ func (os *clusterDiscoveryOS) ListNetwork() ([]*resources.Resource, error) { resourceTrackers = append(resourceTrackers, resourceTracker) } + + // Ports + portTrackers, err := os.ListPorts(network) + if err != nil { + return resourceTrackers, err + } + resourceTrackers = append(resourceTrackers, portTrackers...) + resourceTracker := &resources.Resource{ Name: network.Name, ID: network.ID, diff --git a/pkg/resources/openstack/openstack.go b/pkg/resources/openstack/openstack.go index 5c3fe05384..6798396a56 100644 --- a/pkg/resources/openstack/openstack.go +++ b/pkg/resources/openstack/openstack.go @@ -47,7 +47,6 @@ func ListResources(cloud openstack.OpenstackCloud, clusterName string) (map[stri os.ListInstances, os.ListServerGroups, os.ListVolumes, - os.ListPorts, os.ListSecurityGroups, os.ListNetwork, os.ListDNSRecordsets, diff --git a/pkg/resources/openstack/ports.go b/pkg/resources/openstack/ports.go index 5a3b250e84..c473e81dbe 100644 --- a/pkg/resources/openstack/ports.go +++ b/pkg/resources/openstack/ports.go @@ -17,8 +17,7 @@ limitations under the License. package openstack import ( - "strings" - + "github.com/gophercloud/gophercloud/openstack/networking/v2/networks" "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" "k8s.io/kops/pkg/resources" "k8s.io/kops/upup/pkg/fi" @@ -26,30 +25,31 @@ import ( ) const ( - typePort = "Port" + typePort = "Port" + typeDynamicPort = "DynamicPort" ) -func (os *clusterDiscoveryOS) ListPorts() ([]*resources.Resource, error) { +func (os *clusterDiscoveryOS) ListPorts(network networks.Network) ([]*resources.Resource, error) { var resourceTrackers []*resources.Resource - ports, err := os.osCloud.ListPorts(ports.ListOpts{}) + ports, err := os.osCloud.ListPorts(ports.ListOpts{ + TenantID: network.ProjectID, + NetworkID: network.ID, + }) if err != nil { return nil, err } for _, port := range ports { - clusteReplaced := strings.Replace(os.clusterName, ".", "-", -1) - if strings.HasSuffix(port.Name, clusteReplaced) { - resourceTracker := &resources.Resource{ - Name: port.Name, - ID: port.ID, - Type: typePort, - Deleter: func(cloud fi.Cloud, r *resources.Resource) error { - return cloud.(openstack.OpenstackCloud).DeletePort(r.ID) - }, - } - resourceTrackers = append(resourceTrackers, resourceTracker) + resourceTracker := &resources.Resource{ + Name: port.Name, + ID: port.ID, + Type: typePort, + Deleter: func(cloud fi.Cloud, r *resources.Resource) error { + return cloud.(openstack.OpenstackCloud).DeletePort(r.ID) + }, } + resourceTrackers = append(resourceTrackers, resourceTracker) } return resourceTrackers, nil } From 9542e9d1d714c5030f381ca695787e87233eef98 Mon Sep 17 00:00:00 2001 From: "Derek Lemon (delemon)" Date: Fri, 31 May 2019 11:18:12 -0600 Subject: [PATCH 3/3] Removing Unused type --- pkg/resources/openstack/ports.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/resources/openstack/ports.go b/pkg/resources/openstack/ports.go index c473e81dbe..814db49b85 100644 --- a/pkg/resources/openstack/ports.go +++ b/pkg/resources/openstack/ports.go @@ -25,8 +25,7 @@ import ( ) const ( - typePort = "Port" - typeDynamicPort = "DynamicPort" + typePort = "Port" ) func (os *clusterDiscoveryOS) ListPorts(network networks.Network) ([]*resources.Resource, error) {