diff --git a/pkg/model/hetznermodel/servers.go b/pkg/model/hetznermodel/servers.go index 175a8601ae..c0bb6b7ac0 100644 --- a/pkg/model/hetznermodel/servers.go +++ b/pkg/model/hetznermodel/servers.go @@ -17,24 +17,22 @@ limitations under the License. package hetznermodel import ( - "strconv" - "k8s.io/kops/pkg/model" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/hetzner" "k8s.io/kops/upup/pkg/fi/cloudup/hetznertasks" ) -// ServerModelBuilder configures network objects -type ServerModelBuilder struct { +// ServerGroupModelBuilder configures server objects +type ServerGroupModelBuilder struct { *HetznerModelContext Lifecycle fi.Lifecycle BootstrapScriptBuilder *model.BootstrapScriptBuilder } -var _ fi.ModelBuilder = &ServerModelBuilder{} +var _ fi.ModelBuilder = &ServerGroupModelBuilder{} -func (b *ServerModelBuilder) Build(c *fi.ModelBuilderContext) error { +func (b *ServerGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { for _, ig := range b.InstanceGroups { igSize := fi.Int32Value(ig.Spec.MinSize) @@ -48,27 +46,23 @@ func (b *ServerModelBuilder) Build(c *fi.ModelBuilderContext) error { return err } - for i := 1; i <= int(igSize); i++ { - // hcloud-cloud-controller-manager requires hostname to be same as server name. - // This means server names should not contain the cluster name (which contains "." chars" - // https://github.com/hetznercloud/hcloud-cloud-controller-manager/blob/f7d624e83c2c3475c5606306214814250922cb8a/hcloud/util.go#L39 - name := ig.Name + "-" + strconv.Itoa(i) - server := hetznertasks.Server{ - Name: fi.String(name), - Lifecycle: b.Lifecycle, - SSHKey: b.LinkToSSHKey(), - Network: b.LinkToNetwork(), - Location: ig.Spec.Subnets[0], - Size: ig.Spec.MachineType, - Image: ig.Spec.Image, - EnableIPv4: true, - EnableIPv6: false, - UserData: userData, - Labels: labels, - } - - c.AddTask(&server) + serverGroup := hetznertasks.ServerGroup{ + Name: fi.String(ig.Name), + Lifecycle: b.Lifecycle, + SSHKey: b.LinkToSSHKey(), + Network: b.LinkToNetwork(), + Count: int(igSize), + Outdated: 0, + Location: ig.Spec.Subnets[0], + Size: ig.Spec.MachineType, + Image: ig.Spec.Image, + EnableIPv4: true, + EnableIPv6: false, + UserData: userData, + Labels: labels, } + + c.AddTask(&serverGroup) } return nil diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index 16d0fa74bf..a21740ea24 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -601,7 +601,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error { &hetznermodel.NetworkModelBuilder{HetznerModelContext: hetznerModelContext, Lifecycle: networkLifecycle}, &hetznermodel.ExternalAccessModelBuilder{HetznerModelContext: hetznerModelContext, Lifecycle: networkLifecycle}, &hetznermodel.LoadBalancerModelBuilder{HetznerModelContext: hetznerModelContext, Lifecycle: networkLifecycle}, - &hetznermodel.ServerModelBuilder{HetznerModelContext: hetznerModelContext, BootstrapScriptBuilder: bootstrapScriptBuilder, Lifecycle: clusterLifecycle}, + &hetznermodel.ServerGroupModelBuilder{HetznerModelContext: hetznerModelContext, BootstrapScriptBuilder: bootstrapScriptBuilder, Lifecycle: clusterLifecycle}, ) case kops.CloudProviderGCE: gceModelContext := &gcemodel.GCEModelContext{ diff --git a/upup/pkg/fi/cloudup/hetzner/cloud.go b/upup/pkg/fi/cloudup/hetzner/cloud.go index d3b00b4575..3b47564267 100644 --- a/upup/pkg/fi/cloudup/hetzner/cloud.go +++ b/upup/pkg/fi/cloudup/hetzner/cloud.go @@ -32,11 +32,12 @@ import ( ) const ( - TagKubernetesClusterName = "kops.k8s.io/cluster" - TagKubernetesFirewallRole = "kops.k8s.io/firewall-role" - TagKubernetesInstanceGroup = "kops.k8s.io/instance-group" - TagKubernetesInstanceRole = "kops.k8s.io/instance-role" - TagKubernetesVolumeRole = "kops.k8s.io/volume-role" + TagKubernetesClusterName = "kops.k8s.io/cluster" + TagKubernetesFirewallRole = "kops.k8s.io/firewall-role" + TagKubernetesInstanceGroup = "kops.k8s.io/instance-group" + TagKubernetesInstanceRole = "kops.k8s.io/instance-role" + TagKubernetesInstanceUserData = "kops.k8s.io/instance-userdata" + TagKubernetesVolumeRole = "kops.k8s.io/volume-role" ) // HetznerCloud exposes all the interfaces required to operate on Hetzner Cloud resources diff --git a/upup/pkg/fi/cloudup/hetznertasks/server.go b/upup/pkg/fi/cloudup/hetznertasks/server.go index 52f5c59f47..f3734cc59c 100644 --- a/upup/pkg/fi/cloudup/hetznertasks/server.go +++ b/upup/pkg/fi/cloudup/hetznertasks/server.go @@ -18,8 +18,11 @@ package hetznertasks import ( "context" + "crypto/sha256" + "encoding/base64" "fmt" - "strconv" + "math/rand" + "strings" "github.com/hetznercloud/hcloud-go/hcloud" "k8s.io/kops/upup/pkg/fi" @@ -27,13 +30,15 @@ import ( ) // +kops:fitask -type Server struct { +type ServerGroup struct { Name *string Lifecycle fi.Lifecycle SSHKey *SSHKey Network *Network - ID *int + Count int + Outdated int + Location string Size string Image string @@ -46,123 +51,131 @@ type Server struct { Labels map[string]string } -var _ fi.CompareWithID = &Server{} - -func (v *Server) CompareWithID() *string { - return fi.String(strconv.Itoa(fi.IntValue(v.ID))) -} - -func (v *Server) Find(c *fi.Context) (*Server, error) { +func (v *ServerGroup) Find(c *fi.Context) (*ServerGroup, error) { cloud := c.Cloud.(hetzner.HetznerCloud) client := cloud.ServerClient() - // TODO(hakman): Find using label selector - servers, err := client.All(context.TODO()) + labelSelector := []string{ + fmt.Sprintf("%s=%s", hetzner.TagKubernetesClusterName, c.Cluster.Name), + fmt.Sprintf("%s=%s", hetzner.TagKubernetesInstanceGroup, v.Labels[hetzner.TagKubernetesInstanceGroup]), + } + listOptions := hcloud.ListOpts{ + PerPage: 50, + LabelSelector: strings.Join(labelSelector, ","), + } + serverListOptions := hcloud.ServerListOpts{ListOpts: listOptions} + servers, err := client.AllWithOpts(context.TODO(), serverListOptions) if err != nil { return nil, err } - - for _, server := range servers { - if server.Name == fi.StringValue(v.Name) { - matches := &Server{ - Lifecycle: v.Lifecycle, - Name: fi.String(server.Name), - ID: fi.Int(server.ID), - Labels: server.Labels, - } - - if server.Datacenter != nil && server.Datacenter.Location != nil { - matches.Location = server.Datacenter.Location.Name - } - if server.ServerType != nil { - matches.Size = server.ServerType.Name - } - if server.Image != nil { - matches.Image = server.Image.Name - } - if server.PublicNet.IPv4.IP != nil { - matches.EnableIPv4 = true - } - if server.PublicNet.IPv6.IP != nil { - matches.EnableIPv4 = true - } - - // Ignore fields that are not returned by the Hetzner Cloud API - matches.SSHKey = v.SSHKey - matches.UserData = v.UserData - - // TODO: The API only returns the network ID, a new API call is required to get the network name - matches.Network = v.Network - - v.ID = matches.ID - return matches, nil - } + if len(servers) == 0 { + return nil, nil } - return nil, nil + // Calculate the user-data hash + userDataBytes, err := fi.ResourceAsBytes(v.UserData) + if err != nil { + return nil, err + } + userDataHash := safeBytesHash(userDataBytes) + + // Add the expected user-data hash label + v.Labels[hetzner.TagKubernetesInstanceUserData] = userDataHash + + actual := *v + actual.Count = 0 + + for _, server := range servers { + if server.Labels[hetzner.TagKubernetesInstanceUserData] != userDataHash { + actual.Outdated++ + continue + } + if server.Datacenter == nil || server.Datacenter.Location == nil || server.Datacenter.Location.Name != v.Location { + actual.Outdated++ + continue + } + if server.ServerType == nil || server.ServerType.Name != v.Size { + actual.Outdated++ + continue + } + if server.Image == nil || server.Image.Name != v.Image { + actual.Outdated++ + continue + } + if (server.PublicNet.IPv4.IP != nil) != v.EnableIPv4 { + actual.Outdated++ + continue + } + if (server.PublicNet.IPv6.IP != nil) != v.EnableIPv6 { + actual.Outdated++ + continue + } + + actual.Count++ + } + + return &actual, nil } -func (v *Server) Run(c *fi.Context) error { +func (v *ServerGroup) Run(c *fi.Context) error { return fi.DefaultDeltaRunMethod(v, c) } -func (_ *Server) CheckChanges(a, e, changes *Server) error { - if a != nil { - if changes.Name != nil { - return fi.CannotChangeField("Name") - } - if changes.ID != nil { - return fi.CannotChangeField("ID") - } - if changes.Location != "" { - return fi.CannotChangeField("Location") - } - if changes.Size != "" { - return fi.CannotChangeField("Size") - } - if changes.Image != "" { - return fi.CannotChangeField("Image") - } - if changes.UserData != nil { - return fi.CannotChangeField("UserData") - } - } else { - if e.Name == nil { - return fi.RequiredField("Name") - } - if e.Location == "" { - return fi.RequiredField("Location") - } - if e.Size == "" { - return fi.RequiredField("Size") - } - if e.Image == "" { - return fi.RequiredField("Image") - } - if e.UserData == nil { - return fi.RequiredField("UserData") - } +func (_ *ServerGroup) CheckChanges(a, e, changes *ServerGroup) error { + if e.Name == nil { + return fi.RequiredField("Name") + } + if e.Location == "" { + return fi.RequiredField("Location") + } + if e.Size == "" { + return fi.RequiredField("Size") + } + if e.Image == "" { + return fi.RequiredField("Image") + } + if e.UserData == nil { + return fi.RequiredField("UserData") } return nil } -func (_ *Server) RenderHetzner(t *hetzner.HetznerAPITarget, a, e, changes *Server) error { +func (_ *ServerGroup) RenderHetzner(t *hetzner.HetznerAPITarget, a, e, changes *ServerGroup) error { client := t.Cloud.ServerClient() - if a == nil { - if e.SSHKey == nil { - return fmt.Errorf("failed to find ssh key for server %q", fi.StringValue(e.Name)) - } - if e.Network == nil { - return fmt.Errorf("failed to find network for server %q", fi.StringValue(e.Name)) - } - userData, err := fi.ResourceAsString(e.UserData) - if err != nil { - return err - } + actualCount := 0 + if a != nil { + actualCount = a.Count + } + expectedCount := e.Count + + if actualCount >= expectedCount { + return nil + } + + if e.SSHKey == nil { + return fmt.Errorf("failed to find ssh key for server %q", fi.StringValue(e.Name)) + } + if e.Network == nil { + return fmt.Errorf("failed to find network for server %q", fi.StringValue(e.Name)) + } + + userData, err := fi.ResourceAsString(e.UserData) + if err != nil { + return err + } + userDataBytes, err := fi.ResourceAsBytes(e.UserData) + if err != nil { + return err + } + userDataHash := safeBytesHash(userDataBytes) + + for i := 1; i <= expectedCount-actualCount; i++ { + // Append a random/unique ID to the node name + name := fmt.Sprintf("%s-%x", e.Labels[hetzner.TagKubernetesInstanceGroup], rand.Int63()) opts := hcloud.ServerCreateOpts{ - Name: fi.StringValue(e.Name), + Name: name, StartAfterCreate: fi.Bool(true), SSHKeys: []*hcloud.SSHKey{ { @@ -191,28 +204,29 @@ func (_ *Server) RenderHetzner(t *hetzner.HetznerAPITarget, a, e, changes *Serve }, } + // Add the user-data hash label + opts.Labels[hetzner.TagKubernetesInstanceUserData] = userDataHash + _, _, err = client.Create(context.TODO(), opts) if err != nil { return err } - - } else { - server, _, err := client.Get(context.TODO(), strconv.Itoa(fi.IntValue(a.ID))) - if err != nil { - return err - } - - // Update the labels - if changes.Name != nil || len(changes.Labels) != 0 { - _, _, err := client.Update(context.TODO(), server, hcloud.ServerUpdateOpts{ - Name: fi.StringValue(e.Name), - Labels: e.Labels, - }) - if err != nil { - return err - } - } } return nil } + +func safeBytesHash(data []byte) string { + // Calculate the SHA256 checksum of the data + sum256 := sha256.Sum256(data) + + // Replace the unsupported chars with supported ones + safe256 := base64.StdEncoding.EncodeToString(sum256[:]) + safe256 = strings.ReplaceAll(safe256, "+", "-") + safe256 = strings.ReplaceAll(safe256, "/", "_") + + // Trim the unsupported "=" padding chars + safe256 = strings.TrimRight(safe256, "=") + + return fmt.Sprintf("sha256.%s", safe256) +} diff --git a/upup/pkg/fi/cloudup/hetznertasks/server_fitask.go b/upup/pkg/fi/cloudup/hetznertasks/servergroup_fitask.go similarity index 80% rename from upup/pkg/fi/cloudup/hetznertasks/server_fitask.go rename to upup/pkg/fi/cloudup/hetznertasks/servergroup_fitask.go index 7023da1fbd..10b5998421 100644 --- a/upup/pkg/fi/cloudup/hetznertasks/server_fitask.go +++ b/upup/pkg/fi/cloudup/hetznertasks/servergroup_fitask.go @@ -25,28 +25,28 @@ import ( "k8s.io/kops/upup/pkg/fi" ) -// Server +// ServerGroup -var _ fi.HasLifecycle = &Server{} +var _ fi.HasLifecycle = &ServerGroup{} // GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle -func (o *Server) GetLifecycle() fi.Lifecycle { +func (o *ServerGroup) GetLifecycle() fi.Lifecycle { return o.Lifecycle } // SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle -func (o *Server) SetLifecycle(lifecycle fi.Lifecycle) { +func (o *ServerGroup) SetLifecycle(lifecycle fi.Lifecycle) { o.Lifecycle = lifecycle } -var _ fi.HasName = &Server{} +var _ fi.HasName = &ServerGroup{} // GetName returns the Name of the object, implementing fi.HasName -func (o *Server) GetName() *string { +func (o *ServerGroup) GetName() *string { return o.Name } // String is the stringer function for the task, producing readable output using fi.TaskAsString -func (o *Server) String() string { +func (o *ServerGroup) String() string { return fi.TaskAsString(o) }