Merge pull request #12894 from zetaab/detectlbs

Cleanup GCE loadbalancers created by k8s

cloudmock/gce/mockcompute/BUILD.bazel

@@ -8,6 +8,7 @@ go_library(
         "disk.go",
         "firewall.go",
         "forwarding_rule.go",
+        "http_healthcheck.go",
         "instance_group_manager.go",
         "instance_template.go",
         "network.go",

cloudmock/gce/mockcompute/api.go

@@ -27,13 +27,14 @@ type MockClient struct {
 	projectClient *projectClient
 	zoneClient    *zoneClient
 
-	networkClient        *networkClient
-	subnetworkClient     *subnetworkClient
-	routeClient          *routeClient
-	forwardingRuleClient *forwardingRuleClient
-	addressClient        *addressClient
-	firewallClient       *firewallClient
-	routerClient         *routerClient
+	networkClient          *networkClient
+	subnetworkClient       *subnetworkClient
+	routeClient            *routeClient
+	forwardingRuleClient   *forwardingRuleClient
+	httpHealthChecksClient *httpHealthChecksClient
+	addressClient          *addressClient
+	firewallClient         *firewallClient
+	routerClient           *routerClient
 
 	instanceTemplateClient     *instanceTemplateClient
 	instanceGroupManagerClient *instanceGroupManagerClient
@@ -50,13 +51,14 @@ func NewMockClient(project string) *MockClient {
 		projectClient: newProjectClient(project),
 		zoneClient:    newZoneClient(project),
 
-		networkClient:        newNetworkClient(),
-		subnetworkClient:     newSubnetworkClient(),
-		routeClient:          newRouteClient(),
-		forwardingRuleClient: newForwardingRuleClient(),
-		addressClient:        newAddressClient(),
-		firewallClient:       newFirewallClient(),
-		routerClient:         newRouterClient(),
+		networkClient:          newNetworkClient(),
+		subnetworkClient:       newSubnetworkClient(),
+		routeClient:            newRouteClient(),
+		forwardingRuleClient:   newForwardingRuleClient(),
+		httpHealthChecksClient: newHttpHealthChecksClient(),
+		addressClient:          newAddressClient(),
+		firewallClient:         newFirewallClient(),
+		routerClient:           newRouterClient(),
 
 		instanceTemplateClient:     newInstanceTemplateClient(),
 		instanceGroupManagerClient: newInstanceGroupManagerClient(),
@@ -77,6 +79,7 @@ func (c *MockClient) AllResources() map[string]interface{} {
 		c.subnetworkClient.All,
 		c.routeClient.All,
 		c.forwardingRuleClient.All,
+		c.httpHealthChecksClient.All,
 		c.addressClient.All,
 		c.firewallClient.All,
 		c.routerClient.All,
@@ -124,6 +127,10 @@ func (c *MockClient) ForwardingRules() gce.ForwardingRuleClient {
 	return c.forwardingRuleClient
 }
 
+func (c *MockClient) HTTPHealthChecks() gce.HttpHealthChecksClient {
+	return c.httpHealthChecksClient
+}
+
 func (c *MockClient) Addresses() gce.AddressClient {
 	return c.addressClient
 }

cloudmock/gce/mockcompute/http_healthcheck.go

@@ -0,0 +1,107 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mockcompute
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	compute "google.golang.org/api/compute/v1"
+	"k8s.io/kops/upup/pkg/fi/cloudup/gce"
+)
+
+type httpHealthChecksClient struct {
+	// httpHealthchecks are httpHealthchecks keyed by project, and httpHealthcheck name.
+	httpHealthchecks map[string]map[string]*compute.HttpHealthCheck
+	sync.Mutex
+}
+
+var _ gce.HttpHealthChecksClient = &httpHealthChecksClient{}
+
+func newHttpHealthChecksClient() *httpHealthChecksClient {
+	return &httpHealthChecksClient{
+		httpHealthchecks: map[string]map[string]*compute.HttpHealthCheck{},
+	}
+}
+
+func (c *httpHealthChecksClient) All() map[string]interface{} {
+	c.Lock()
+	defer c.Unlock()
+	m := map[string]interface{}{}
+	for _, hcs := range c.httpHealthchecks {
+		for n, hc := range hcs {
+			m[n] = hc
+		}
+	}
+	return m
+}
+
+func (c *httpHealthChecksClient) Insert(project string, hc *compute.HttpHealthCheck) (*compute.Operation, error) {
+	c.Lock()
+	defer c.Unlock()
+	hcs, ok := c.httpHealthchecks[project]
+	if !ok {
+		hcs = map[string]*compute.HttpHealthCheck{}
+		c.httpHealthchecks[project] = hcs
+	}
+	hc.SelfLink = fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/global/httpHealthChecks/%s", project, hc.Name)
+	hcs[hc.Name] = hc
+	return doneOperation(), nil
+}
+
+func (c *httpHealthChecksClient) Delete(project, name string) (*compute.Operation, error) {
+	c.Lock()
+	defer c.Unlock()
+	hcs, ok := c.httpHealthchecks[project]
+	if !ok {
+		return nil, notFoundError()
+	}
+	if _, ok := hcs[name]; !ok {
+		return nil, notFoundError()
+	}
+	delete(hcs, name)
+	return doneOperation(), nil
+}
+
+func (c *httpHealthChecksClient) Get(project, name string) (*compute.HttpHealthCheck, error) {
+	c.Lock()
+	defer c.Unlock()
+	hcs, ok := c.httpHealthchecks[project]
+	if !ok {
+		return nil, notFoundError()
+	}
+	hc, ok := hcs[name]
+	if !ok {
+		return nil, notFoundError()
+	}
+	return hc, nil
+}
+
+func (c *httpHealthChecksClient) List(ctx context.Context, project string) ([]*compute.HttpHealthCheck, error) {
+	c.Lock()
+	defer c.Unlock()
+	hcs, ok := c.httpHealthchecks[project]
+	if !ok {
+		return nil, nil
+	}
+	var l []*compute.HttpHealthCheck
+	for _, hc := range hcs {
+		l = append(l, hc)
+	}
+	return l, nil
+}

pkg/resources/gce/gce.go

@@ -40,6 +40,7 @@ const (
 	typeTargetPool           = "TargetPool"
 	typeFirewallRule         = "FirewallRule"
 	typeForwardingRule       = "ForwardingRule"
+	typeHTTPHealthcheck      = "HTTP HealthCheck"
 	typeAddress              = "Address"
 	typeRoute                = "Route"
 	typeNetwork              = "Network"
@@ -503,7 +504,7 @@ func (d *clusterDiscoveryGCE) listFirewallRules() ([]*resources.Resource, error)
 	}
 
 	for _, fr := range frs {
-		if !d.matchesClusterNameMultipart(fr.Name, maxPrefixTokens) {
+		if !d.matchesClusterNameMultipart(fr.Name, maxPrefixTokens) && !strings.HasPrefix(fr.Name, "k8s-") {
 			continue
 		}
 
@@ -515,7 +516,59 @@ func (d *clusterDiscoveryGCE) listFirewallRules() ([]*resources.Resource, error)
 			}
 		}
 		if !foundMatchingTarget {
-			break
+			continue
+		}
+
+		// find the Kubernetes LoadBalancer
+		if strings.HasPrefix(fr.Name, "k8s-fw-") {
+			name := strings.ReplaceAll(fr.Name, "k8s-fw-", "")
+			fr, err := c.Compute().ForwardingRules().Get(c.Project(), c.Region(), name)
+			if err != nil {
+				return nil, fmt.Errorf("error get ForwardingRule: %v", err)
+			}
+
+			frResourceTracker := &resources.Resource{
+				Name:    fr.Name,
+				ID:      fr.Name,
+				Type:    typeForwardingRule,
+				Deleter: deleteForwardingRule,
+				Obj:     fr,
+			}
+			if fr.Target != "" {
+				frResourceTracker.Blocks = append(frResourceTracker.Blocks, typeTargetPool+":"+gce.LastComponent(fr.Target))
+			}
+			resourceTrackers = append(resourceTrackers, frResourceTracker)
+
+			tp, err := c.Compute().TargetPools().Get(c.Project(), c.Region(), name)
+			if err != nil {
+				return nil, fmt.Errorf("error get TargetPool: %v", err)
+			}
+
+			tpResourceTracker := &resources.Resource{
+				Name:    tp.Name,
+				ID:      tp.Name,
+				Type:    typeTargetPool,
+				Deleter: deleteTargetPool,
+				Obj:     tp,
+			}
+			resourceTrackers = append(resourceTrackers, tpResourceTracker)
+
+		}
+		// l4 level healthchecks
+		if strings.HasPrefix(fr.Name, "k8s-") && strings.HasSuffix(fr.Name, "-http-hc") {
+			name := strings.ReplaceAll(strings.ReplaceAll(fr.Name, "k8s-", ""), "-http-hc", "")
+			hc, err := c.Compute().HTTPHealthChecks().Get(c.Project(), name)
+			if err != nil {
+				return nil, fmt.Errorf("error get HTTPHealthCheck: %v", err)
+			}
+			hcResourceTracker := &resources.Resource{
+				Name:    hc.Name,
+				ID:      hc.Name,
+				Type:    typeHTTPHealthcheck,
+				Deleter: deleteHTTPHealthCheck,
+				Obj:     hc,
+			}
+			resourceTrackers = append(resourceTrackers, hcResourceTracker)
 		}
 
 		resourceTracker := &resources.Resource{
@@ -533,6 +586,29 @@ func (d *clusterDiscoveryGCE) listFirewallRules() ([]*resources.Resource, error)
 	return resourceTrackers, nil
 }
 
+// deleteHTTPHealthCheck is the helper function to delete a Resource for a HTTP health check object
+func deleteHTTPHealthCheck(cloud fi.Cloud, r *resources.Resource) error {
+	c := cloud.(gce.GCECloud)
+	t := r.Obj.(*compute.HttpHealthCheck)
+
+	klog.V(2).Infof("Deleting GCE HTTP HealthCheck %s", t.SelfLink)
+	u, err := gce.ParseGoogleCloudURL(t.SelfLink)
+	if err != nil {
+		return err
+	}
+
+	op, err := c.Compute().HTTPHealthChecks().Delete(u.Project, u.Name)
+	if err != nil {
+		if gce.IsNotFound(err) {
+			klog.Infof("HTTP HealthCheck not found, assuming deleted: %q", t.SelfLink)
+			return nil
+		}
+		return fmt.Errorf("error deleting HTTP HealthCheck %s: %v", t.SelfLink, err)
+	}
+
+	return c.WaitForOp(op)
+}
+
 // deleteFirewallRule is the helper function to delete a Resource for a Firewall object
 func deleteFirewallRule(cloud fi.Cloud, r *resources.Resource) error {
 	c := cloud.(gce.GCECloud)

upup/pkg/fi/cloudup/gce/compute.go

@@ -31,6 +31,7 @@ type ComputeClient interface {
 	Subnetworks() SubnetworkClient
 	Routes() RouteClient
 	ForwardingRules() ForwardingRuleClient
+	HTTPHealthChecks() HttpHealthChecksClient
 	Addresses() AddressClient
 	Firewalls() FirewallClient
 	Routers() RouterClient
@@ -99,6 +100,12 @@ func (c *computeClientImpl) ForwardingRules() ForwardingRuleClient {
 	}
 }
 
+func (c *computeClientImpl) HTTPHealthChecks() HttpHealthChecksClient {
+	return &httpHealthCheckClientImpl{
+		srv: c.srv.HttpHealthChecks,
+	}
+}
+
 func (c *computeClientImpl) Addresses() AddressClient {
 	return &addressClientImpl{
 		srv: c.srv.Addresses,
@@ -337,6 +344,42 @@ func (c *forwardingRuleClientImpl) List(ctx context.Context, project, region str
 	return frs, nil
 }
 
+type HttpHealthChecksClient interface {
+	Insert(project string, fr *compute.HttpHealthCheck) (*compute.Operation, error)
+	Delete(project, name string) (*compute.Operation, error)
+	Get(project, name string) (*compute.HttpHealthCheck, error)
+	List(ctx context.Context, project string) ([]*compute.HttpHealthCheck, error)
+}
+
+type httpHealthCheckClientImpl struct {
+	srv *compute.HttpHealthChecksService
+}
+
+var _ HttpHealthChecksClient = &httpHealthCheckClientImpl{}
+
+func (c *httpHealthCheckClientImpl) Insert(project string, fr *compute.HttpHealthCheck) (*compute.Operation, error) {
+	return c.srv.Insert(project, fr).Do()
+}
+
+func (c *httpHealthCheckClientImpl) Delete(project, name string) (*compute.Operation, error) {
+	return c.srv.Delete(project, name).Do()
+}
+
+func (c *httpHealthCheckClientImpl) Get(project, name string) (*compute.HttpHealthCheck, error) {
+	return c.srv.Get(project, name).Do()
+}
+
+func (c *httpHealthCheckClientImpl) List(ctx context.Context, project string) ([]*compute.HttpHealthCheck, error) {
+	var hcs []*compute.HttpHealthCheck
+	if err := c.srv.List(project).Pages(ctx, func(p *compute.HttpHealthCheckList) error {
+		hcs = append(hcs, p.Items...)
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+	return hcs, nil
+}
+
 type AddressClient interface {
 	Insert(project, region string, addr *compute.Address) (*compute.Operation, error)
 	Delete(project, region, name string) (*compute.Operation, error)