diff --git a/pkg/commands/BUILD.bazel b/pkg/commands/BUILD.bazel index 6683aa7684..053b5e4117 100644 --- a/pkg/commands/BUILD.bazel +++ b/pkg/commands/BUILD.bazel @@ -18,6 +18,7 @@ go_library( "//pkg/assets:go_default_library", "//pkg/client/simple:go_default_library", "//pkg/featureflag:go_default_library", + "//pkg/resources/digitalocean:go_default_library", "//upup/pkg/fi/cloudup:go_default_library", "//upup/pkg/fi/cloudup/aliup:go_default_library", "//upup/pkg/fi/cloudup/awstasks:go_default_library", diff --git a/pkg/commands/status_discovery.go b/pkg/commands/status_discovery.go index 1d34528802..69b412de3d 100644 --- a/pkg/commands/status_discovery.go +++ b/pkg/commands/status_discovery.go @@ -21,6 +21,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/resources/digitalocean" "k8s.io/kops/upup/pkg/fi/cloudup" "k8s.io/kops/upup/pkg/fi/cloudup/aliup" "k8s.io/kops/upup/pkg/fi/cloudup/awstasks" @@ -77,6 +78,10 @@ func (s *CloudDiscoveryStatusStore) GetApiIngressStatus(cluster *kops.Cluster) ( return osCloud.GetApiIngressStatus(cluster) } + if doCloud, ok := cloud.(*digitalocean.Cloud); ok { + return doCloud.GetApiIngressStatus(cluster) + } + return nil, fmt.Errorf("API Ingress Status not implemented for %T", cloud) } diff --git a/pkg/model/domodel/BUILD.bazel b/pkg/model/domodel/BUILD.bazel index 64cc31bf4e..c6af73b155 100644 --- a/pkg/model/domodel/BUILD.bazel +++ b/pkg/model/domodel/BUILD.bazel @@ -3,15 +3,19 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", srcs = [ + "api_loadbalancer.go", "context.go", "droplets.go", ], importpath = "k8s.io/kops/pkg/model/domodel", visibility = ["//visibility:public"], deps = [ + "//pkg/apis/kops:go_default_library", + "//pkg/dns:go_default_library", "//pkg/model:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/cloudup/do:go_default_library", "//upup/pkg/fi/cloudup/dotasks:go_default_library", + "//upup/pkg/fi/fitasks:go_default_library", ], ) diff --git a/pkg/model/domodel/api_loadbalancer.go b/pkg/model/domodel/api_loadbalancer.go new file mode 100644 index 0000000000..58f578bd33 --- /dev/null +++ b/pkg/model/domodel/api_loadbalancer.go @@ -0,0 +1,89 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package domodel + +import ( + "errors" + "fmt" + "strings" + + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/dns" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/do" + "k8s.io/kops/upup/pkg/fi/cloudup/dotasks" + "k8s.io/kops/upup/pkg/fi/fitasks" +) + +// APILoadBalancerModelBuilder builds a LoadBalancer for accessing the API +type APILoadBalancerModelBuilder struct { + *DOModelContext + Lifecycle *fi.Lifecycle +} + +var _ fi.ModelBuilder = &APILoadBalancerModelBuilder{} + +func (b *APILoadBalancerModelBuilder) Build(c *fi.ModelBuilderContext) error { + // Configuration where a load balancer fronts the API + if !b.UseLoadBalancerForAPI() { + return nil + } + + lbSpec := b.Cluster.Spec.API.LoadBalancer + if lbSpec == nil { + // Skipping API LB creation; not requested in Spec + return nil + } + + switch lbSpec.Type { + case kops.LoadBalancerTypeInternal: + // OK + case kops.LoadBalancerTypePublic: + // OK + default: + return fmt.Errorf("unhandled LoadBalancer type %q", lbSpec.Type) + } + + clusterName := strings.Replace(b.ClusterName(), ".", "-", -1) + loadbalancerName := "api-" + clusterName + clusterMasterTag := do.TagKubernetesClusterMasterPrefix + ":" + clusterName + + // Create LoadBalancer for API LB + loadbalancer := &dotasks.LoadBalancer{ + Name: fi.String(loadbalancerName), + Region: fi.String(b.Cluster.Spec.Subnets[0].Region), + DropletTag: fi.String(clusterMasterTag), + Lifecycle: b.Lifecycle, + } + c.AddTask(loadbalancer) + + // Temporarily do not know the role of the following function + if dns.IsGossipHostname(b.Cluster.Name) || b.UsePrivateDNS() { + // Ensure the LB hostname is included in the TLS certificate, + // if we're not going to use an alias for it + // TODO: I don't love this technique for finding the task by name & modifying it + masterKeypairTask, found := c.Tasks["Keypair/master"] + if !found { + return errors.New("keypair/master task not found") + } + masterKeypair := masterKeypairTask.(*fitasks.Keypair) + masterKeypair.AlternateNameTasks = append(masterKeypair.AlternateNameTasks, loadbalancer) + } + + return nil + +} diff --git a/pkg/resources/digitalocean/cloud.go b/pkg/resources/digitalocean/cloud.go index 88e927f786..f6e02242c8 100644 --- a/pkg/resources/digitalocean/cloud.go +++ b/pkg/resources/digitalocean/cloud.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "os" + "strings" "github.com/digitalocean/godo" "golang.org/x/oauth2" @@ -128,7 +129,43 @@ func (c *Cloud) Droplets() godo.DropletsService { return c.Client.Droplets } +func (c *Cloud) LoadBalancers() godo.LoadBalancersService { + return c.Client.LoadBalancers +} + // FindVPCInfo is not implemented, it's only here to satisfy the fi.Cloud interface func (c *Cloud) FindVPCInfo(id string) (*fi.VPCInfo, error) { return nil, errors.New("not implemented") } + +func (c *Cloud) GetApiIngressStatus(cluster *kops.Cluster) ([]kops.ApiIngressStatus, error) { + var ingresses []kops.ApiIngressStatus + if cluster.Spec.MasterPublicName != "" { + // Note that this must match Digital Ocean's lb name + klog.V(2).Infof("Querying DO to find Loadbalancers for API (%q)", cluster.Name) + + loadBalancers, err := getAllLoadBalancers(c) + if err != nil { + return nil, fmt.Errorf("LoadBalancers.List returned error: %v", err) + } + + lbName := "api-" + strings.Replace(cluster.Name, ".", "-", -1) + + for _, lb := range loadBalancers { + if lb.Name == lbName { + klog.V(10).Infof("Matching LB name found for API (%q)", cluster.Name) + + if lb.Status != "active" { + return nil, fmt.Errorf("load-balancer is not yet active (current status: %s)", lb.Status) + } + + address := lb.IP + ingresses = append(ingresses, kops.ApiIngressStatus{IP: address}) + + return ingresses, nil + } + } + } + + return nil, nil +} diff --git a/pkg/resources/digitalocean/resources.go b/pkg/resources/digitalocean/resources.go index a144d01223..3954c0032f 100644 --- a/pkg/resources/digitalocean/resources.go +++ b/pkg/resources/digitalocean/resources.go @@ -33,9 +33,10 @@ import ( ) const ( - resourceTypeDroplet = "droplet" - resourceTypeVolume = "volume" - resourceTypeDNSRecord = "dns-record" + resourceTypeDroplet = "droplet" + resourceTypeVolume = "volume" + resourceTypeDNSRecord = "dns-record" + resourceTypeLoadBalancer = "loadbalancer" ) type listFn func(fi.Cloud, string) ([]*resources.Resource, error) @@ -47,6 +48,7 @@ func ListResources(cloud *Cloud, clusterName string) (map[string]*resources.Reso listVolumes, listDroplets, listDNS, + listLoadBalancers, } for _, fn := range listFunctions { @@ -265,6 +267,67 @@ func getAllRecordsByDomain(cloud *Cloud, domain string) ([]godo.DomainRecord, er return allRecords, nil } +func listLoadBalancers(cloud fi.Cloud, clusterName string) ([]*resources.Resource, error) { + c := cloud.(*Cloud) + var resourceTrackers []*resources.Resource + + clusterTag := "KubernetesCluster-Master:" + strings.Replace(clusterName, ".", "-", -1) + + lbs, err := getAllLoadBalancers(c) + if err != nil { + return nil, fmt.Errorf("failed to list lbs: %v", err) + } + + for _, lb := range lbs { + if strings.Contains(lb.Tag, clusterTag) { + resourceTracker := &resources.Resource{ + Name: lb.Name, + ID: lb.ID, + Type: resourceTypeLoadBalancer, + Deleter: deleteLoadBalancer, + Obj: lb, + } + + var blocks []string + for _, dropletID := range lb.DropletIDs { + blocks = append(blocks, "droplet:"+strconv.Itoa(dropletID)) + } + + resourceTracker.Blocks = blocks + resourceTrackers = append(resourceTrackers, resourceTracker) + } + } + + return resourceTrackers, nil +} + +func getAllLoadBalancers(cloud *Cloud) ([]godo.LoadBalancer, error) { + allLoadBalancers := []godo.LoadBalancer{} + + opt := &godo.ListOptions{} + for { + lbs, resp, err := cloud.LoadBalancers().List(context.TODO(), opt) + if err != nil { + return nil, err + } + + allLoadBalancers = append(allLoadBalancers, lbs...) + + if resp.Links == nil || resp.Links.IsLastPage() { + break + } + + page, err := resp.Links.CurrentPage() + if err != nil { + return nil, err + } + + opt.Page = page + 1 + } + + return allLoadBalancers, nil +} + func deleteDroplet(cloud fi.Cloud, t *resources.Resource) error { c := cloud.(*Cloud) @@ -315,6 +378,18 @@ func deleteRecord(cloud fi.Cloud, domain string, t *resources.Resource) error { return nil } +func deleteLoadBalancer(cloud fi.Cloud, t *resources.Resource) error { + c := cloud.(*Cloud) + lb := t.Obj.(godo.LoadBalancer) + _, err := c.Client.LoadBalancers.Delete(context.TODO(), lb.ID) + + if err != nil { + return fmt.Errorf("failed to delete load balancer with name %s %v", lb.Name, err) + } + + return nil +} + func waitForDetach(cloud *Cloud, action *godo.Action) error { timeout := time.After(10 * time.Second) ticker := time.NewTicker(500 * time.Millisecond) diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index eaa995c160..c75c2788d3 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -396,8 +396,9 @@ func (c *ApplyClusterCmd) Run() error { modelContext.SSHPublicKeys = sshPublicKeys l.AddTypes(map[string]interface{}{ - "volume": &dotasks.Volume{}, - "droplet": &dotasks.Droplet{}, + "volume": &dotasks.Volume{}, + "droplet": &dotasks.Droplet{}, + "loadbalancer": &dotasks.LoadBalancer{}, }) } case kops.CloudProviderAWS: @@ -643,8 +644,12 @@ func (c *ApplyClusterCmd) Run() error { &model.IAMModelBuilder{KopsModelContext: modelContext, Lifecycle: &securityLifecycle}, ) case kops.CloudProviderDO: + doModelContext := &domodel.DOModelContext{ + KopsModelContext: modelContext, + } l.Builders = append(l.Builders, &model.MasterVolumeBuilder{KopsModelContext: modelContext, Lifecycle: &clusterLifecycle}, + &domodel.APILoadBalancerModelBuilder{DOModelContext: doModelContext, Lifecycle: &securityLifecycle}, ) case kops.CloudProviderGCE: diff --git a/upup/pkg/fi/cloudup/dotasks/BUILD.bazel b/upup/pkg/fi/cloudup/dotasks/BUILD.bazel index c492878367..4de9ebf319 100644 --- a/upup/pkg/fi/cloudup/dotasks/BUILD.bazel +++ b/upup/pkg/fi/cloudup/dotasks/BUILD.bazel @@ -5,6 +5,8 @@ go_library( srcs = [ "droplet.go", "droplet_fitask.go", + "loadbalancer.go", + "loadbalancer_fitask.go", "volume.go", "volume_fitask.go", ], diff --git a/upup/pkg/fi/cloudup/dotasks/loadbalancer.go b/upup/pkg/fi/cloudup/dotasks/loadbalancer.go new file mode 100644 index 0000000000..e534fbc59e --- /dev/null +++ b/upup/pkg/fi/cloudup/dotasks/loadbalancer.go @@ -0,0 +1,182 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dotasks + +import ( + "context" + "errors" + "fmt" + "net" + "time" + + "github.com/digitalocean/godo" + + "k8s.io/klog" + "k8s.io/kops/pkg/resources/digitalocean" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/do" +) + +//go:generate fitask -type=LoadBalancer +type LoadBalancer struct { + Name *string + ID *string + Lifecycle *fi.Lifecycle + + Region *string + DropletTag *string + IPAddress *string +} + +var _ fi.CompareWithID = &LoadBalancer{} + +func (lb *LoadBalancer) CompareWithID() *string { + return lb.ID +} + +func (lb *LoadBalancer) Find(c *fi.Context) (*LoadBalancer, error) { + if fi.StringValue(lb.ID) == "" { + // Loadbalancer = nil if not found + return nil, nil + } + + cloud := c.Cloud.(*digitalocean.Cloud) + lbService := cloud.LoadBalancers() + loadbalancer, _, err := lbService.Get(context.TODO(), fi.StringValue(lb.ID)) + + if err != nil { + return nil, fmt.Errorf("load balancer service get request returned error %v", err) + } + + return &LoadBalancer{ + Name: fi.String(loadbalancer.Name), + ID: fi.String(loadbalancer.ID), + Lifecycle: lb.Lifecycle, + Region: fi.String(loadbalancer.Region.Slug), + }, nil +} + +func (lb *LoadBalancer) Run(c *fi.Context) error { + return fi.DefaultDeltaRunMethod(lb, c) +} + +func (_ *LoadBalancer) CheckChanges(a, e, changes *LoadBalancer) error { + if a != nil { + if changes.Name != nil { + return fi.CannotChangeField("Name") + } + if changes.ID != nil { + return fi.CannotChangeField("ID") + } + if changes.Region != nil { + return fi.CannotChangeField("Region") + } + } else { + if e.Name == nil { + return fi.RequiredField("Name") + } + if e.Region == nil { + return fi.RequiredField("Region") + } + } + return nil +} + +func (_ *LoadBalancer) RenderDO(t *do.DOAPITarget, a, e, changes *LoadBalancer) error { + + Rules := []godo.ForwardingRule{ + { + EntryProtocol: "https", + EntryPort: 443, + TargetProtocol: "https", + TargetPort: 443, + TlsPassthrough: true, + }, + { + EntryProtocol: "http", + EntryPort: 80, + TargetProtocol: "http", + TargetPort: 80, + }, + } + + HealthCheck := &godo.HealthCheck{ + Protocol: "tcp", + Port: 443, + Path: "", + CheckIntervalSeconds: 60, + ResponseTimeoutSeconds: 5, + UnhealthyThreshold: 3, + HealthyThreshold: 5, + } + + klog.V(10).Infof("Creating load balancer for DO") + + loadBalancerService := t.Cloud.LoadBalancers() + loadbalancer, _, err := loadBalancerService.Create(context.TODO(), &godo.LoadBalancerRequest{ + Name: fi.StringValue(e.Name), + Region: fi.StringValue(e.Region), + Tag: fi.StringValue(e.DropletTag), + ForwardingRules: Rules, + HealthCheck: HealthCheck, + }) + + if err != nil { + klog.Errorf("Error creating load balancer with Name=%s, Error=%v", fi.StringValue(e.Name), err) + return err + } + + e.ID = fi.String(loadbalancer.ID) + e.IPAddress = fi.String(loadbalancer.IP) // This will be empty on create, but will be filled later on FindIPAddress invokation. + + return nil +} + +func (lb *LoadBalancer) FindIPAddress(c *fi.Context) (*string, error) { + cloud := c.Cloud.(*digitalocean.Cloud) + loadBalancerService := cloud.LoadBalancers() + + klog.V(10).Infof("Find IP address for load balancer ID=%s", fi.StringValue(lb.ID)) + loadBalancer, _, err := loadBalancerService.Get(context.TODO(), fi.StringValue(lb.ID)) + if err != nil { + klog.Errorf("Error fetching load balancer with Name=%s", fi.StringValue(lb.Name)) + return nil, err + } + + address := loadBalancer.IP + + if isIPv4(address) { + klog.V(10).Infof("load balancer address=%s", address) + return &address, nil + } + + const lbWaitTime = 10 * time.Second + klog.Warningf("IP address for LB %s not yet available -- sleeping %s", fi.StringValue(lb.Name), lbWaitTime) + time.Sleep(lbWaitTime) + + return nil, errors.New("IP Address is still empty.") +} + +func isIPv4(host string) bool { + + ip := net.ParseIP(host) + if ip == nil { + return false + } + + return ip.To4() != nil +} diff --git a/upup/pkg/fi/cloudup/dotasks/loadbalancer_fitask.go b/upup/pkg/fi/cloudup/dotasks/loadbalancer_fitask.go new file mode 100644 index 0000000000..f1e0ce7b4b --- /dev/null +++ b/upup/pkg/fi/cloudup/dotasks/loadbalancer_fitask.go @@ -0,0 +1,75 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by ""fitask" -type=LoadBalancer"; DO NOT EDIT + +package dotasks + +import ( + "encoding/json" + + "k8s.io/kops/upup/pkg/fi" +) + +// LoadBalancer + +// JSON marshaling boilerplate +type realLoadBalancer LoadBalancer + +// UnmarshalJSON implements conversion to JSON, supporting an alternate specification of the object as a string +func (o *LoadBalancer) UnmarshalJSON(data []byte) error { + var jsonName string + if err := json.Unmarshal(data, &jsonName); err == nil { + o.Name = &jsonName + return nil + } + + var r realLoadBalancer + if err := json.Unmarshal(data, &r); err != nil { + return err + } + *o = LoadBalancer(r) + return nil +} + +var _ fi.HasLifecycle = &LoadBalancer{} + +// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle +func (o *LoadBalancer) GetLifecycle() *fi.Lifecycle { + return o.Lifecycle +} + +// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle +func (o *LoadBalancer) SetLifecycle(lifecycle fi.Lifecycle) { + o.Lifecycle = &lifecycle +} + +var _ fi.HasName = &LoadBalancer{} + +// GetName returns the Name of the object, implementing fi.HasName +func (o *LoadBalancer) GetName() *string { + return o.Name +} + +// SetName sets the Name of the object, implementing fi.SetName +func (o *LoadBalancer) SetName(name string) { + o.Name = &name +} + +// String is the stringer function for the task, producing readable output using fi.TaskAsString +func (o *LoadBalancer) String() string { + return fi.TaskAsString(o) +}