Merge pull request #14018 from hakman/hetzner_server_groups

Add server group management for Hetzner

pkg/model/hetznermodel/servers.go

@@ -17,24 +17,22 @@ limitations under the License.
 package hetznermodel
 
 import (
-	"strconv"
-
 	"k8s.io/kops/pkg/model"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
 	"k8s.io/kops/upup/pkg/fi/cloudup/hetznertasks"
 )
 
-// ServerModelBuilder configures network objects
+// ServerGroupModelBuilder configures server objects
-type ServerModelBuilder struct {
+type ServerGroupModelBuilder struct {
 	*HetznerModelContext
 	Lifecycle              fi.Lifecycle
 	BootstrapScriptBuilder *model.BootstrapScriptBuilder
 }
 
-var _ fi.ModelBuilder = &ServerModelBuilder{}
+var _ fi.ModelBuilder = &ServerGroupModelBuilder{}
 
-func (b *ServerModelBuilder) Build(c *fi.ModelBuilderContext) error {
+func (b *ServerGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
 	for _, ig := range b.InstanceGroups {
 		igSize := fi.Int32Value(ig.Spec.MinSize)
 
@@ -48,27 +46,23 @@ func (b *ServerModelBuilder) Build(c *fi.ModelBuilderContext) error {
 			return err
 		}
 
-		for i := 1; i <= int(igSize); i++ {
+		serverGroup := hetznertasks.ServerGroup{
-			// hcloud-cloud-controller-manager requires hostname to be same as server name.
+			Name:       fi.String(ig.Name),
-			// This means server names should not contain the cluster name (which contains "." chars"
+			Lifecycle:  b.Lifecycle,
-			// https://github.com/hetznercloud/hcloud-cloud-controller-manager/blob/f7d624e83c2c3475c5606306214814250922cb8a/hcloud/util.go#L39
+			SSHKey:     b.LinkToSSHKey(),
-			name := ig.Name + "-" + strconv.Itoa(i)
+			Network:    b.LinkToNetwork(),
-			server := hetznertasks.Server{
+			Count:      int(igSize),
-				Name:       fi.String(name),
+			Outdated:   0,
-				Lifecycle:  b.Lifecycle,
+			Location:   ig.Spec.Subnets[0],
-				SSHKey:     b.LinkToSSHKey(),
+			Size:       ig.Spec.MachineType,
-				Network:    b.LinkToNetwork(),
+			Image:      ig.Spec.Image,
-				Location:   ig.Spec.Subnets[0],
+			EnableIPv4: true,
-				Size:       ig.Spec.MachineType,
+			EnableIPv6: false,
-				Image:      ig.Spec.Image,
+			UserData:   userData,
-				EnableIPv4: true,
+			Labels:     labels,
-				EnableIPv6: false,
-				UserData:   userData,
-				Labels:     labels,
-			}
-
-			c.AddTask(&server)
 		}
+
+		c.AddTask(&serverGroup)
 	}
 
 	return nil

upup/pkg/fi/cloudup/apply_cluster.go

@@ -601,7 +601,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error {
 				&hetznermodel.NetworkModelBuilder{HetznerModelContext: hetznerModelContext, Lifecycle: networkLifecycle},
 				&hetznermodel.ExternalAccessModelBuilder{HetznerModelContext: hetznerModelContext, Lifecycle: networkLifecycle},
 				&hetznermodel.LoadBalancerModelBuilder{HetznerModelContext: hetznerModelContext, Lifecycle: networkLifecycle},
-				&hetznermodel.ServerModelBuilder{HetznerModelContext: hetznerModelContext, BootstrapScriptBuilder: bootstrapScriptBuilder, Lifecycle: clusterLifecycle},
+				&hetznermodel.ServerGroupModelBuilder{HetznerModelContext: hetznerModelContext, BootstrapScriptBuilder: bootstrapScriptBuilder, Lifecycle: clusterLifecycle},
 			)
 		case kops.CloudProviderGCE:
 			gceModelContext := &gcemodel.GCEModelContext{

upup/pkg/fi/cloudup/hetzner/cloud.go

@@ -32,11 +32,12 @@ import (
 )
 
 const (
-	TagKubernetesClusterName   = "kops.k8s.io/cluster"
+	TagKubernetesClusterName      = "kops.k8s.io/cluster"
-	TagKubernetesFirewallRole  = "kops.k8s.io/firewall-role"
+	TagKubernetesFirewallRole     = "kops.k8s.io/firewall-role"
-	TagKubernetesInstanceGroup = "kops.k8s.io/instance-group"
+	TagKubernetesInstanceGroup    = "kops.k8s.io/instance-group"
-	TagKubernetesInstanceRole  = "kops.k8s.io/instance-role"
+	TagKubernetesInstanceRole     = "kops.k8s.io/instance-role"
-	TagKubernetesVolumeRole    = "kops.k8s.io/volume-role"
+	TagKubernetesInstanceUserData = "kops.k8s.io/instance-userdata"
+	TagKubernetesVolumeRole       = "kops.k8s.io/volume-role"
 )
 
 // HetznerCloud exposes all the interfaces required to operate on Hetzner Cloud resources

upup/pkg/fi/cloudup/hetznertasks/server.go

@@ -18,8 +18,11 @@ package hetznertasks
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/base64"
 	"fmt"
-	"strconv"
+	"math/rand"
+	"strings"
 
 	"github.com/hetznercloud/hcloud-go/hcloud"
 	"k8s.io/kops/upup/pkg/fi"
@@ -27,13 +30,15 @@ import (
 )
 
 // +kops:fitask
-type Server struct {
+type ServerGroup struct {
 	Name      *string
 	Lifecycle fi.Lifecycle
 	SSHKey    *SSHKey
 	Network   *Network
 
-	ID       *int
+	Count    int
+	Outdated int
+
 	Location string
 	Size     string
 	Image    string
@@ -46,123 +51,131 @@ type Server struct {
 	Labels map[string]string
 }
 
-var _ fi.CompareWithID = &Server{}
+func (v *ServerGroup) Find(c *fi.Context) (*ServerGroup, error) {
-
-func (v *Server) CompareWithID() *string {
-	return fi.String(strconv.Itoa(fi.IntValue(v.ID)))
-}
-
-func (v *Server) Find(c *fi.Context) (*Server, error) {
 	cloud := c.Cloud.(hetzner.HetznerCloud)
 	client := cloud.ServerClient()
 
-	// TODO(hakman): Find using label selector
+	labelSelector := []string{
-	servers, err := client.All(context.TODO())
+		fmt.Sprintf("%s=%s", hetzner.TagKubernetesClusterName, c.Cluster.Name),
+		fmt.Sprintf("%s=%s", hetzner.TagKubernetesInstanceGroup, v.Labels[hetzner.TagKubernetesInstanceGroup]),
+	}
+	listOptions := hcloud.ListOpts{
+		PerPage:       50,
+		LabelSelector: strings.Join(labelSelector, ","),
+	}
+	serverListOptions := hcloud.ServerListOpts{ListOpts: listOptions}
+	servers, err := client.AllWithOpts(context.TODO(), serverListOptions)
 	if err != nil {
 		return nil, err
 	}
-
+	if len(servers) == 0 {
-	for _, server := range servers {
+		return nil, nil
-		if server.Name == fi.StringValue(v.Name) {
-			matches := &Server{
-				Lifecycle: v.Lifecycle,
-				Name:      fi.String(server.Name),
-				ID:        fi.Int(server.ID),
-				Labels:    server.Labels,
-			}
-
-			if server.Datacenter != nil && server.Datacenter.Location != nil {
-				matches.Location = server.Datacenter.Location.Name
-			}
-			if server.ServerType != nil {
-				matches.Size = server.ServerType.Name
-			}
-			if server.Image != nil {
-				matches.Image = server.Image.Name
-			}
-			if server.PublicNet.IPv4.IP != nil {
-				matches.EnableIPv4 = true
-			}
-			if server.PublicNet.IPv6.IP != nil {
-				matches.EnableIPv4 = true
-			}
-
-			// Ignore fields that are not returned by the Hetzner Cloud API
-			matches.SSHKey = v.SSHKey
-			matches.UserData = v.UserData
-
-			// TODO: The API only returns the network ID, a new API call is required to get the network name
-			matches.Network = v.Network
-
-			v.ID = matches.ID
-			return matches, nil
-		}
 	}
 
-	return nil, nil
+	// Calculate the user-data hash
+	userDataBytes, err := fi.ResourceAsBytes(v.UserData)
+	if err != nil {
+		return nil, err
+	}
+	userDataHash := safeBytesHash(userDataBytes)
+
+	// Add the expected user-data hash label
+	v.Labels[hetzner.TagKubernetesInstanceUserData] = userDataHash
+
+	actual := *v
+	actual.Count = 0
+
+	for _, server := range servers {
+		if server.Labels[hetzner.TagKubernetesInstanceUserData] != userDataHash {
+			actual.Outdated++
+			continue
+		}
+		if server.Datacenter == nil || server.Datacenter.Location == nil || server.Datacenter.Location.Name != v.Location {
+			actual.Outdated++
+			continue
+		}
+		if server.ServerType == nil || server.ServerType.Name != v.Size {
+			actual.Outdated++
+			continue
+		}
+		if server.Image == nil || server.Image.Name != v.Image {
+			actual.Outdated++
+			continue
+		}
+		if (server.PublicNet.IPv4.IP != nil) != v.EnableIPv4 {
+			actual.Outdated++
+			continue
+		}
+		if (server.PublicNet.IPv6.IP != nil) != v.EnableIPv6 {
+			actual.Outdated++
+			continue
+		}
+
+		actual.Count++
+	}
+
+	return &actual, nil
 }
 
-func (v *Server) Run(c *fi.Context) error {
+func (v *ServerGroup) Run(c *fi.Context) error {
 	return fi.DefaultDeltaRunMethod(v, c)
 }
 
-func (_ *Server) CheckChanges(a, e, changes *Server) error {
+func (_ *ServerGroup) CheckChanges(a, e, changes *ServerGroup) error {
-	if a != nil {
+	if e.Name == nil {
-		if changes.Name != nil {
+		return fi.RequiredField("Name")
-			return fi.CannotChangeField("Name")
+	}
-		}
+	if e.Location == "" {
-		if changes.ID != nil {
+		return fi.RequiredField("Location")
-			return fi.CannotChangeField("ID")
+	}
-		}
+	if e.Size == "" {
-		if changes.Location != "" {
+		return fi.RequiredField("Size")
-			return fi.CannotChangeField("Location")
+	}
-		}
+	if e.Image == "" {
-		if changes.Size != "" {
+		return fi.RequiredField("Image")
-			return fi.CannotChangeField("Size")
+	}
-		}
+	if e.UserData == nil {
-		if changes.Image != "" {
+		return fi.RequiredField("UserData")
-			return fi.CannotChangeField("Image")
-		}
-		if changes.UserData != nil {
-			return fi.CannotChangeField("UserData")
-		}
-	} else {
-		if e.Name == nil {
-			return fi.RequiredField("Name")
-		}
-		if e.Location == "" {
-			return fi.RequiredField("Location")
-		}
-		if e.Size == "" {
-			return fi.RequiredField("Size")
-		}
-		if e.Image == "" {
-			return fi.RequiredField("Image")
-		}
-		if e.UserData == nil {
-			return fi.RequiredField("UserData")
-		}
 	}
 	return nil
 }
 
-func (_ *Server) RenderHetzner(t *hetzner.HetznerAPITarget, a, e, changes *Server) error {
+func (_ *ServerGroup) RenderHetzner(t *hetzner.HetznerAPITarget, a, e, changes *ServerGroup) error {
 	client := t.Cloud.ServerClient()
-	if a == nil {
-		if e.SSHKey == nil {
-			return fmt.Errorf("failed to find ssh key for server %q", fi.StringValue(e.Name))
-		}
-		if e.Network == nil {
-			return fmt.Errorf("failed to find network for server %q", fi.StringValue(e.Name))
-		}
 
-		userData, err := fi.ResourceAsString(e.UserData)
+	actualCount := 0
-		if err != nil {
+	if a != nil {
-			return err
+		actualCount = a.Count
-		}
+	}
+	expectedCount := e.Count
+
+	if actualCount >= expectedCount {
+		return nil
+	}
+
+	if e.SSHKey == nil {
+		return fmt.Errorf("failed to find ssh key for server %q", fi.StringValue(e.Name))
+	}
+	if e.Network == nil {
+		return fmt.Errorf("failed to find network for server %q", fi.StringValue(e.Name))
+	}
+
+	userData, err := fi.ResourceAsString(e.UserData)
+	if err != nil {
+		return err
+	}
+	userDataBytes, err := fi.ResourceAsBytes(e.UserData)
+	if err != nil {
+		return err
+	}
+	userDataHash := safeBytesHash(userDataBytes)
+
+	for i := 1; i <= expectedCount-actualCount; i++ {
+		// Append a random/unique ID to the node name
+		name := fmt.Sprintf("%s-%x", e.Labels[hetzner.TagKubernetesInstanceGroup], rand.Int63())
 
 		opts := hcloud.ServerCreateOpts{
-			Name:             fi.StringValue(e.Name),
+			Name:             name,
 			StartAfterCreate: fi.Bool(true),
 			SSHKeys: []*hcloud.SSHKey{
 				{
@@ -191,28 +204,29 @@ func (_ *Server) RenderHetzner(t *hetzner.HetznerAPITarget, a, e, changes *Serve
 			},
 		}
 
+		// Add the user-data hash label
+		opts.Labels[hetzner.TagKubernetesInstanceUserData] = userDataHash
+
 		_, _, err = client.Create(context.TODO(), opts)
 		if err != nil {
 			return err
 		}
-
-	} else {
-		server, _, err := client.Get(context.TODO(), strconv.Itoa(fi.IntValue(a.ID)))
-		if err != nil {
-			return err
-		}
-
-		// Update the labels
-		if changes.Name != nil || len(changes.Labels) != 0 {
-			_, _, err := client.Update(context.TODO(), server, hcloud.ServerUpdateOpts{
-				Name:   fi.StringValue(e.Name),
-				Labels: e.Labels,
-			})
-			if err != nil {
-				return err
-			}
-		}
 	}
 
 	return nil
 }
+
+func safeBytesHash(data []byte) string {
+	// Calculate the SHA256 checksum of the data
+	sum256 := sha256.Sum256(data)
+
+	// Replace the unsupported chars with supported ones
+	safe256 := base64.StdEncoding.EncodeToString(sum256[:])
+	safe256 = strings.ReplaceAll(safe256, "+", "-")
+	safe256 = strings.ReplaceAll(safe256, "/", "_")
+
+	// Trim the unsupported "=" padding chars
+	safe256 = strings.TrimRight(safe256, "=")
+
+	return fmt.Sprintf("sha256.%s", safe256)
+}

upup/pkg/fi/cloudup/hetznertasks/servergroup_fitask.go

@@ -25,28 +25,28 @@ import (
 	"k8s.io/kops/upup/pkg/fi"
 )
 
-// Server
+// ServerGroup
 
-var _ fi.HasLifecycle = &Server{}
+var _ fi.HasLifecycle = &ServerGroup{}
 
 // GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle
-func (o *Server) GetLifecycle() fi.Lifecycle {
+func (o *ServerGroup) GetLifecycle() fi.Lifecycle {
 	return o.Lifecycle
 }
 
 // SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle
-func (o *Server) SetLifecycle(lifecycle fi.Lifecycle) {
+func (o *ServerGroup) SetLifecycle(lifecycle fi.Lifecycle) {
 	o.Lifecycle = lifecycle
 }
 
-var _ fi.HasName = &Server{}
+var _ fi.HasName = &ServerGroup{}
 
 // GetName returns the Name of the object, implementing fi.HasName
-func (o *Server) GetName() *string {
+func (o *ServerGroup) GetName() *string {
 	return o.Name
 }
 
 // String is the stringer function for the task, producing readable output using fi.TaskAsString
-func (o *Server) String() string {
+func (o *ServerGroup) String() string {
 	return fi.TaskAsString(o)
 }