From d3fac0c1bee4263850e7a8ed9390c65a4a251887 Mon Sep 17 00:00:00 2001 From: Jesse Haka Date: Thu, 3 Feb 2022 20:15:37 +0200 Subject: [PATCH 1/5] GCP API health checks --- cloudmock/gce/mockcompute/target_pool.go | 23 ++++ pkg/model/gcemodel/api_loadbalancer.go | 16 +++ pkg/model/gcemodel/context.go | 8 ++ pkg/resources/gce/gce.go | 20 +++- upup/pkg/fi/cloudup/gce/compute.go | 5 + upup/pkg/fi/cloudup/gcetasks/BUILD.bazel | 4 + upup/pkg/fi/cloudup/gcetasks/healthcheck.go | 95 ++++++++++++++++ .../fi/cloudup/gcetasks/healthcheck_fitask.go | 52 +++++++++ .../fi/cloudup/gcetasks/poolhealthcheck.go | 107 ++++++++++++++++++ .../gcetasks/poolhealthcheck_fitask.go | 52 +++++++++ 10 files changed, 381 insertions(+), 1 deletion(-) create mode 100644 upup/pkg/fi/cloudup/gcetasks/healthcheck.go create mode 100644 upup/pkg/fi/cloudup/gcetasks/healthcheck_fitask.go create mode 100644 upup/pkg/fi/cloudup/gcetasks/poolhealthcheck.go create mode 100644 upup/pkg/fi/cloudup/gcetasks/poolhealthcheck_fitask.go diff --git a/cloudmock/gce/mockcompute/target_pool.go b/cloudmock/gce/mockcompute/target_pool.go index 3c13839728..44b67705dd 100644 --- a/cloudmock/gce/mockcompute/target_pool.go +++ b/cloudmock/gce/mockcompute/target_pool.go @@ -124,3 +124,26 @@ func (c *targetPoolClient) List(ctx context.Context, project, region string) ([] } return l, nil } + +func (c *targetPoolClient) List(ctx context.Context, project, region string) ([]*compute.TargetPool, error) { + c.Lock() + defer c.Unlock() + regions, ok := c.targetPools[project] + if !ok { + return nil, nil + } + tps, ok := regions[region] + if !ok { + return nil, nil + } + var l []*compute.TargetPool + for _, tp := range tps { + l = append(l, tp) + } + return l, nil +} + +func (c *targetPoolClient) AddHealthCheck(project, region, name string, req *compute.TargetPoolsAddHealthCheckRequest) (*compute.Operation, error) { + // TODO: AddHealthCheck test + return doneOperation(), nil +} diff --git a/pkg/model/gcemodel/api_loadbalancer.go b/pkg/model/gcemodel/api_loadbalancer.go index f1e7ba950e..b2d4a0542a 100644 --- a/pkg/model/gcemodel/api_loadbalancer.go +++ b/pkg/model/gcemodel/api_loadbalancer.go @@ -54,12 +54,28 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { return fmt.Errorf("unhandled LoadBalancer type %q", lbSpec.Type) } + healthCheck := &gcetasks.Healthcheck{ + Name: s(b.NameForHealthcheck("api")), + Port: i64(3990), + Lifecycle: b.Lifecycle, + } + + c.AddTask(healthCheck) + targetPool := &gcetasks.TargetPool{ Name: s(b.NameForTargetPool("api")), Lifecycle: b.Lifecycle, } c.AddTask(targetPool) + poolHealthCheck := &gcetasks.PoolHealthCheck{ + Name: s(b.NameForPoolHealthcheck("api")), + Healthcheck: healthCheck, + Pool: targetPool, + Lifecycle: b.Lifecycle, + } + c.AddTask(poolHealthCheck) + ipAddress := &gcetasks.Address{ Name: s(b.NameForIPAddress("api")), Lifecycle: b.Lifecycle, diff --git a/pkg/model/gcemodel/context.go b/pkg/model/gcemodel/context.go index 26c886b18d..15e1e97004 100644 --- a/pkg/model/gcemodel/context.go +++ b/pkg/model/gcemodel/context.go @@ -91,6 +91,14 @@ func (c *GCEModelContext) NameForIPAddress(id string) string { return c.SafeObjectName(id) } +func (c *GCEModelContext) NameForPoolHealthcheck(id string) string { + return c.SafeObjectName(id) +} + +func (c *GCEModelContext) NameForHealthcheck(id string) string { + return c.SafeObjectName(id) +} + func (c *GCEModelContext) NameForFirewallRule(id string) string { return c.SafeObjectName(id) } diff --git a/pkg/resources/gce/gce.go b/pkg/resources/gce/gce.go index bb7f82bac7..58e994a90c 100644 --- a/pkg/resources/gce/gce.go +++ b/pkg/resources/gce/gce.go @@ -401,6 +401,25 @@ func (d *clusterDiscoveryGCE) listTargetPools() ([]*resources.Resource, error) { klog.V(4).Infof("Found resource: %s", tp.SelfLink) resourceTrackers = append(resourceTrackers, resourceTracker) + + for _, healthCheckLink := range tp.HealthChecks { + healthCheckName := gce.LastComponent(healthCheckLink) + hc, err := c.Compute().HTTPHealthChecks().Get(c.Project(), healthCheckName) + if err != nil { + return nil, fmt.Errorf("error getting HTTPHealthCheck %q: %w", healthCheckName, err) + } + + healthCheckResource := &resources.Resource{ + Name: hc.Name, + ID: hc.Name, + Type: typeHTTPHealthcheck, + Deleter: deleteHTTPHealthCheck, + Obj: hc, + } + healthCheckResource.Blocked = append(healthCheckResource.Blocked, resourceTracker.Type+":"+resourceTracker.ID) + resourceTrackers = append(resourceTrackers, healthCheckResource) + } + } return resourceTrackers, nil @@ -607,7 +626,6 @@ nextFirewallRule: // l4 level healthchecks healthCheckName := gce.LastComponent(healthCheckLink) - if !strings.HasPrefix(healthCheckName, "k8s-") || !strings.Contains(healthCheckLink, "/httpHealthChecks/") { klog.Warningf("found non-k8s healthcheck %q in targetPool %q, assuming firewallRule %q is not a k8s rule", healthCheckLink, targetPoolName, firewallRule.Name) continue nextFirewallRule diff --git a/upup/pkg/fi/cloudup/gce/compute.go b/upup/pkg/fi/cloudup/gce/compute.go index 079f6ec08a..0767021e5b 100644 --- a/upup/pkg/fi/cloudup/gce/compute.go +++ b/upup/pkg/fi/cloudup/gce/compute.go @@ -658,6 +658,7 @@ type TargetPoolClient interface { Delete(project, region, name string) (*compute.Operation, error) Get(project, region, name string) (*compute.TargetPool, error) List(ctx context.Context, project, region string) ([]*compute.TargetPool, error) + AddHealthCheck(project, region, name string, req *compute.TargetPoolsAddHealthCheckRequest) (*compute.Operation, error) } type targetPoolClientImpl struct { @@ -678,6 +679,10 @@ func (c *targetPoolClientImpl) Get(project, region, name string) (*compute.Targe return c.srv.Get(project, region, name).Do() } +func (c *targetPoolClientImpl) AddHealthCheck(project, region, name string, req *compute.TargetPoolsAddHealthCheckRequest) (*compute.Operation, error) { + return c.srv.AddHealthCheck(project, region, name, req).Do() +} + func (c *targetPoolClientImpl) List(ctx context.Context, project, region string) ([]*compute.TargetPool, error) { var tps []*compute.TargetPool if err := c.srv.List(project, region).Pages(ctx, func(p *compute.TargetPoolList) error { diff --git a/upup/pkg/fi/cloudup/gcetasks/BUILD.bazel b/upup/pkg/fi/cloudup/gcetasks/BUILD.bazel index 0ded164cfd..9730dfd43a 100644 --- a/upup/pkg/fi/cloudup/gcetasks/BUILD.bazel +++ b/upup/pkg/fi/cloudup/gcetasks/BUILD.bazel @@ -12,6 +12,8 @@ go_library( "firewallrule_fitask.go", "forwardingrule.go", "forwardingrule_fitask.go", + "healthcheck.go", + "healthcheck_fitask.go", "instance.go", "instance_fitask.go", "instancegroupmanager.go", @@ -20,6 +22,8 @@ go_library( "instancetemplate_fitask.go", "network.go", "network_fitask.go", + "poolhealthcheck.go", + "poolhealthcheck_fitask.go", "projectiambinding.go", "projectiambinding_fitask.go", "router.go", diff --git a/upup/pkg/fi/cloudup/gcetasks/healthcheck.go b/upup/pkg/fi/cloudup/gcetasks/healthcheck.go new file mode 100644 index 0000000000..9867634134 --- /dev/null +++ b/upup/pkg/fi/cloudup/gcetasks/healthcheck.go @@ -0,0 +1,95 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gcetasks + +import ( + "fmt" + + compute "google.golang.org/api/compute/v1" + "k8s.io/klog/v2" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/gce" +) + +// Healthcheck represents a GCE Healthcheck +// +kops:fitask +type Healthcheck struct { + Name *string + Lifecycle fi.Lifecycle + + SelfLink string + Port *int64 +} + +var _ fi.CompareWithID = &Healthcheck{} + +func (e *Healthcheck) CompareWithID() *string { + return e.Name +} + +func (e *Healthcheck) Find(c *fi.Context) (*Healthcheck, error) { + cloud := c.Cloud.(gce.GCECloud) + name := fi.StringValue(e.Name) + r, err := cloud.Compute().HTTPHealthChecks().Get(cloud.Project(), name) + if err != nil { + if gce.IsNotFound(err) { + return nil, nil + } + return nil, fmt.Errorf("error getting HealthCheck %q: %v", name, err) + } + actual := &Healthcheck{ + Name: fi.String(r.Name), + Port: fi.Int64(r.Port), + SelfLink: r.SelfLink, + } + // System fields + actual.Lifecycle = e.Lifecycle + e.SelfLink = r.SelfLink + return actual, nil +} + +func (e *Healthcheck) Run(c *fi.Context) error { + return fi.DefaultDeltaRunMethod(e, c) +} + +func (_ *Healthcheck) CheckChanges(a, e, changes *Healthcheck) error { + if fi.StringValue(e.Name) == "" { + return fi.RequiredField("Name") + } + return nil +} + +func (h *Healthcheck) RenderGCE(t *gce.GCEAPITarget, a, e, changes *Healthcheck) error { + if a == nil { + o := &compute.HttpHealthCheck{ + Name: fi.StringValue(e.Name), + Port: fi.Int64Value(e.Port), + RequestPath: "/healthz", + } + + klog.V(4).Infof("Creating Healthcheck %q", o.Name) + r, err := t.Cloud.Compute().HTTPHealthChecks().Insert(t.Cloud.Project(), o) + if err != nil { + return fmt.Errorf("error creating Healthcheck %q: %v", o.Name, err) + } + if err := t.Cloud.WaitForOp(r); err != nil { + return fmt.Errorf("error creating Healthcheck: %v", err) + } + h.SelfLink = r.TargetLink + } + return nil +} diff --git a/upup/pkg/fi/cloudup/gcetasks/healthcheck_fitask.go b/upup/pkg/fi/cloudup/gcetasks/healthcheck_fitask.go new file mode 100644 index 0000000000..7a455b5240 --- /dev/null +++ b/upup/pkg/fi/cloudup/gcetasks/healthcheck_fitask.go @@ -0,0 +1,52 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by fitask. DO NOT EDIT. + +package gcetasks + +import ( + "k8s.io/kops/upup/pkg/fi" +) + +// Healthcheck + +var _ fi.HasLifecycle = &Healthcheck{} + +// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle +func (o *Healthcheck) GetLifecycle() fi.Lifecycle { + return o.Lifecycle +} + +// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle +func (o *Healthcheck) SetLifecycle(lifecycle fi.Lifecycle) { + o.Lifecycle = lifecycle +} + +var _ fi.HasName = &Healthcheck{} + +// GetName returns the Name of the object, implementing fi.HasName +func (o *Healthcheck) GetName() *string { + return o.Name +} + +// String is the stringer function for the task, producing readable output using fi.TaskAsString +func (o *Healthcheck) String() string { + return fi.TaskAsString(o) +} diff --git a/upup/pkg/fi/cloudup/gcetasks/poolhealthcheck.go b/upup/pkg/fi/cloudup/gcetasks/poolhealthcheck.go new file mode 100644 index 0000000000..4edb7b79e2 --- /dev/null +++ b/upup/pkg/fi/cloudup/gcetasks/poolhealthcheck.go @@ -0,0 +1,107 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gcetasks + +import ( + "fmt" + + compute "google.golang.org/api/compute/v1" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/gce" +) + +// PoolHealthCheck represents a GCE target pool HealthCheck +// +kops:fitask +type PoolHealthCheck struct { + Name *string + Lifecycle fi.Lifecycle + Healthcheck *Healthcheck + Pool *TargetPool +} + +var _ fi.CompareWithID = &PoolHealthCheck{} + +// GetDependencies returns the dependencies of the PoolHealthCheck task +func (_ *PoolHealthCheck) GetDependencies(tasks map[string]fi.Task) []fi.Task { + var deps []fi.Task + for _, task := range tasks { + if _, ok := task.(*Healthcheck); ok { + deps = append(deps, task) + } + if _, ok := task.(*TargetPool); ok { + deps = append(deps, task) + } + } + return deps +} + +func (e *PoolHealthCheck) CompareWithID() *string { + return e.Name +} + +func (e *PoolHealthCheck) Find(c *fi.Context) (*PoolHealthCheck, error) { + cloud := c.Cloud.(gce.GCECloud) + name := fi.StringValue(e.Pool.Name) + r, err := cloud.Compute().TargetPools().Get(cloud.Project(), cloud.Region(), name) + if err != nil { + if gce.IsNotFound(err) { + return nil, nil + } + return nil, fmt.Errorf("error getting TargetPool %q: %v", name, err) + } + for _, check := range r.HealthChecks { + if check == e.Healthcheck.SelfLink { + return &PoolHealthCheck{ + Name: e.Name, + Healthcheck: e.Healthcheck, + Pool: e.Pool, + Lifecycle: e.Lifecycle, + }, nil + } + } + return nil, nil +} + +func (e *PoolHealthCheck) Run(c *fi.Context) error { + return fi.DefaultDeltaRunMethod(e, c) +} + +func (_ *PoolHealthCheck) CheckChanges(a, e, changes *PoolHealthCheck) error { + return nil +} + +func (p *PoolHealthCheck) RenderGCE(t *gce.GCEAPITarget, a, e, changes *PoolHealthCheck) error { + if a == nil { + targetPool := fi.StringValue(p.Pool.Name) + req := &compute.TargetPoolsAddHealthCheckRequest{ + HealthChecks: []*compute.HealthCheckReference{ + { + HealthCheck: p.Healthcheck.SelfLink, + }, + }, + } + op, err := t.Cloud.Compute().TargetPools().AddHealthCheck(t.Cloud.Project(), t.Cloud.Region(), targetPool, req) + if err != nil { + return fmt.Errorf("error creating PoolHealthCheck: %v", err) + } + + if err := t.Cloud.WaitForOp(op); err != nil { + return fmt.Errorf("error creating PoolHealthCheck: %v", err) + } + } + return nil +} diff --git a/upup/pkg/fi/cloudup/gcetasks/poolhealthcheck_fitask.go b/upup/pkg/fi/cloudup/gcetasks/poolhealthcheck_fitask.go new file mode 100644 index 0000000000..189211bbeb --- /dev/null +++ b/upup/pkg/fi/cloudup/gcetasks/poolhealthcheck_fitask.go @@ -0,0 +1,52 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by fitask. DO NOT EDIT. + +package gcetasks + +import ( + "k8s.io/kops/upup/pkg/fi" +) + +// PoolHealthCheck + +var _ fi.HasLifecycle = &PoolHealthCheck{} + +// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle +func (o *PoolHealthCheck) GetLifecycle() fi.Lifecycle { + return o.Lifecycle +} + +// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle +func (o *PoolHealthCheck) SetLifecycle(lifecycle fi.Lifecycle) { + o.Lifecycle = lifecycle +} + +var _ fi.HasName = &PoolHealthCheck{} + +// GetName PoolHealthCheck the Name of the object, implementing fi.HasName +func (o *PoolHealthCheck) GetName() *string { + return o.Name +} + +// String is the stringer function for the task, producing readable output using fi.TaskAsString +func (o *PoolHealthCheck) String() string { + return fi.TaskAsString(o) +} From 6aeccc1c9eef940075d44f81e3064b5a55a36c14 Mon Sep 17 00:00:00 2001 From: Jesse Haka Date: Thu, 3 Feb 2022 21:06:09 +0200 Subject: [PATCH 2/5] remove list --- cloudmock/gce/mockcompute/target_pool.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/cloudmock/gce/mockcompute/target_pool.go b/cloudmock/gce/mockcompute/target_pool.go index 44b67705dd..7761f190c3 100644 --- a/cloudmock/gce/mockcompute/target_pool.go +++ b/cloudmock/gce/mockcompute/target_pool.go @@ -125,24 +125,6 @@ func (c *targetPoolClient) List(ctx context.Context, project, region string) ([] return l, nil } -func (c *targetPoolClient) List(ctx context.Context, project, region string) ([]*compute.TargetPool, error) { - c.Lock() - defer c.Unlock() - regions, ok := c.targetPools[project] - if !ok { - return nil, nil - } - tps, ok := regions[region] - if !ok { - return nil, nil - } - var l []*compute.TargetPool - for _, tp := range tps { - l = append(l, tp) - } - return l, nil -} - func (c *targetPoolClient) AddHealthCheck(project, region, name string, req *compute.TargetPoolsAddHealthCheckRequest) (*compute.Operation, error) { // TODO: AddHealthCheck test return doneOperation(), nil From 7c8f2cb41eb9dfcaecf035ab6c56831a3c5a5049 Mon Sep 17 00:00:00 2001 From: Jesse Haka Date: Thu, 3 Feb 2022 22:26:07 +0200 Subject: [PATCH 3/5] codegen --- upup/pkg/fi/cloudup/gcetasks/poolhealthcheck_fitask.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upup/pkg/fi/cloudup/gcetasks/poolhealthcheck_fitask.go b/upup/pkg/fi/cloudup/gcetasks/poolhealthcheck_fitask.go index 189211bbeb..58b4199ca0 100644 --- a/upup/pkg/fi/cloudup/gcetasks/poolhealthcheck_fitask.go +++ b/upup/pkg/fi/cloudup/gcetasks/poolhealthcheck_fitask.go @@ -41,7 +41,7 @@ func (o *PoolHealthCheck) SetLifecycle(lifecycle fi.Lifecycle) { var _ fi.HasName = &PoolHealthCheck{} -// GetName PoolHealthCheck the Name of the object, implementing fi.HasName +// GetName returns the Name of the object, implementing fi.HasName func (o *PoolHealthCheck) GetName() *string { return o.Name } From 180c3ae475f00ac1abe4d68adabcfa5f688abc1d Mon Sep 17 00:00:00 2001 From: Jesse Haka Date: Mon, 7 Feb 2022 21:32:05 +0200 Subject: [PATCH 4/5] Update pkg/model/gcemodel/api_loadbalancer.go Co-authored-by: Peter Rifel --- pkg/model/gcemodel/api_loadbalancer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/model/gcemodel/api_loadbalancer.go b/pkg/model/gcemodel/api_loadbalancer.go index b2d4a0542a..f4e9aa810f 100644 --- a/pkg/model/gcemodel/api_loadbalancer.go +++ b/pkg/model/gcemodel/api_loadbalancer.go @@ -56,7 +56,7 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { healthCheck := &gcetasks.Healthcheck{ Name: s(b.NameForHealthcheck("api")), - Port: i64(3990), + Port: i64(wellknownports.KubeAPIServerHealthCheck), Lifecycle: b.Lifecycle, } From 3e505a559ebf97bb42c1c922e0895b6f9a601eb1 Mon Sep 17 00:00:00 2001 From: Jesse Haka Date: Mon, 7 Feb 2022 21:35:01 +0200 Subject: [PATCH 5/5] add missing import --- pkg/model/gcemodel/api_loadbalancer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/model/gcemodel/api_loadbalancer.go b/pkg/model/gcemodel/api_loadbalancer.go index f4e9aa810f..0546251393 100644 --- a/pkg/model/gcemodel/api_loadbalancer.go +++ b/pkg/model/gcemodel/api_loadbalancer.go @@ -20,6 +20,7 @@ import ( "fmt" "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/wellknownports" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/gcetasks" )