Merge pull request #16739 from rifelpet/hetzner-labels

Propagate IG NodeLabels to k8s nodes in Hetzner

pkg/model/context.go

@@ -33,6 +33,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
+	"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
 	"k8s.io/kops/upup/pkg/fi/cloudup/scaleway"
 
 	"github.com/blang/semver/v4"
@@ -174,7 +175,12 @@ func (b *KopsModelContext) CloudTagsForInstanceGroup(ig *kops.InstanceGroup) (ma
 		return nil, fmt.Errorf("error building node labels: %w", err)
 	}
 	for k, v := range nodeLabels {
-		labels[nodeidentityaws.ClusterAutoscalerNodeTemplateLabel+k] = v
+		switch b.Cluster.GetCloudProvider() {
+		case kops.CloudProviderHetzner:
+			labels[hetzner.TagKubernetesNodeLabelPrefix+k] = v
+		default:
+			labels[nodeidentityaws.ClusterAutoscalerNodeTemplateLabel+k] = v
+		}
 	}
 
 	// Apply labels for cluster autoscaler node taints
@@ -185,27 +191,33 @@ func (b *KopsModelContext) CloudTagsForInstanceGroup(ig *kops.InstanceGroup) (ma
 		}
 	}
 
-	// The system tags take priority because the cluster likely breaks without them...
+	switch b.Cluster.GetCloudProvider() {
+	case kops.CloudProviderHetzner:
+		labels[hetzner.TagKubernetesInstanceRole] = string(ig.Spec.Role)
+		labels[hetzner.TagKubernetesClusterName] = b.ClusterName()
+		labels[hetzner.TagKubernetesInstanceGroup] = ig.Name
+	default:
+		// The system tags take priority because the cluster likely breaks without them...
 
-	if ig.Spec.Role == kops.InstanceGroupRoleControlPlane {
-		labels[awstasks.CloudTagInstanceGroupRolePrefix+"master"] = "1"
-		labels[awstasks.CloudTagInstanceGroupRolePrefix+kops.InstanceGroupRoleControlPlane.ToLowerString()] = "1"
+		if ig.Spec.Role == kops.InstanceGroupRoleControlPlane {
+			labels[awstasks.CloudTagInstanceGroupRolePrefix+"master"] = "1"
+			labels[awstasks.CloudTagInstanceGroupRolePrefix+kops.InstanceGroupRoleControlPlane.ToLowerString()] = "1"
+		}
+
+		if ig.Spec.Role == kops.InstanceGroupRoleAPIServer {
+			labels[awstasks.CloudTagInstanceGroupRolePrefix+strings.ToLower(string(kops.InstanceGroupRoleAPIServer))] = "1"
+		}
+
+		if ig.Spec.Role == kops.InstanceGroupRoleNode {
+			labels[awstasks.CloudTagInstanceGroupRolePrefix+strings.ToLower(string(kops.InstanceGroupRoleNode))] = "1"
+		}
+
+		if ig.Spec.Role == kops.InstanceGroupRoleBastion {
+			labels[awstasks.CloudTagInstanceGroupRolePrefix+strings.ToLower(string(kops.InstanceGroupRoleBastion))] = "1"
+		}
+
+		labels[nodeidentityaws.CloudTagInstanceGroupName] = ig.Name
 	}
-
-	if ig.Spec.Role == kops.InstanceGroupRoleAPIServer {
-		labels[awstasks.CloudTagInstanceGroupRolePrefix+strings.ToLower(string(kops.InstanceGroupRoleAPIServer))] = "1"
-	}
-
-	if ig.Spec.Role == kops.InstanceGroupRoleNode {
-		labels[awstasks.CloudTagInstanceGroupRolePrefix+strings.ToLower(string(kops.InstanceGroupRoleNode))] = "1"
-	}
-
-	if ig.Spec.Role == kops.InstanceGroupRoleBastion {
-		labels[awstasks.CloudTagInstanceGroupRolePrefix+strings.ToLower(string(kops.InstanceGroupRoleBastion))] = "1"
-	}
-
-	labels[nodeidentityaws.CloudTagInstanceGroupName] = ig.Name
-
 	return labels, nil
 }
 
@@ -259,6 +271,8 @@ func (b *KopsModelContext) CloudTags(name string, shared bool) map[string]string
 			}
 			tags[k] = v
 		}
+	case kops.CloudProviderHetzner:
+		tags[hetzner.TagKubernetesClusterName] = b.ClusterName()
 	}
 	return tags
 }

pkg/model/hetznermodel/servers.go

@@ -54,11 +54,10 @@ func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error
 
 	for _, ig := range b.InstanceGroups {
 		igSize := fi.ValueOf(ig.Spec.MinSize)
-
-		labels := make(map[string]string)
-		labels[hetzner.TagKubernetesClusterName] = b.ClusterName()
-		labels[hetzner.TagKubernetesInstanceGroup] = ig.Name
-		labels[hetzner.TagKubernetesInstanceRole] = string(ig.Spec.Role)
+		labels, err := b.CloudTagsForInstanceGroup(ig)
+		if err != nil {
+			return err
+		}
 
 		userData, err := b.BootstrapScriptBuilder.ResourceNodeUp(c, ig)
 		if err != nil {

pkg/nodeidentity/hetzner/identify.go

@@ -97,7 +97,8 @@ func (i *nodeIdentifier) IdentifyNode(ctx context.Context, node *corev1.Node) (*
 
 	labels := map[string]string{}
 	for key, value := range server.Labels {
-		if key == hetzner.TagKubernetesInstanceRole {
+		switch {
+		case key == hetzner.TagKubernetesInstanceRole:
 			switch kops.InstanceGroupRole(value) {
 			case kops.InstanceGroupRoleControlPlane:
 				labels[nodelabels.RoleLabelControlPlane20] = ""
@@ -108,6 +109,8 @@ func (i *nodeIdentifier) IdentifyNode(ctx context.Context, node *corev1.Node) (*
 			default:
 				klog.Warningf("Unknown node role %q for server %s(%d)", value, server.Name, server.ID)
 			}
+		case strings.HasPrefix(key, hetzner.TagKubernetesNodeLabelPrefix):
+			labels[strings.TrimPrefix(key, hetzner.TagKubernetesNodeLabelPrefix)] = value
 		}
 	}
 

tests/integration/update_cluster/minimal_hetzner/kubernetes.tf

@@ -235,9 +235,12 @@ resource "hcloud_server" "master-fsn1" {
   count = 1
   image = "ubuntu-20.04"
   labels = {
-    "kops.k8s.io/cluster"        = "minimal.example.com"
-    "kops.k8s.io/instance-group" = "master-fsn1"
-    "kops.k8s.io/instance-role"  = "ControlPlane"
+    "kops.k8s.io/cluster"                                                            = "minimal.example.com"
+    "kops.k8s.io/instance-group"                                                     = "master-fsn1"
+    "kops.k8s.io/instance-role"                                                      = "ControlPlane"
+    "node-label.kops.k8s.io.kops.k8s.io/kops-controller-pki"                         = ""
+    "node-label.kops.k8s.io.node-role.kubernetes.io/control-plane"                   = ""
+    "node-label.kops.k8s.io.node.kubernetes.io/exclude-from-external-load-balancers" = ""
   }
   location = "fsn1"
   name     = "master-fsn1-${count.index}"
@@ -257,9 +260,10 @@ resource "hcloud_server" "nodes-fsn1" {
   count = 1
   image = "ubuntu-20.04"
   labels = {
-    "kops.k8s.io/cluster"        = "minimal.example.com"
-    "kops.k8s.io/instance-group" = "nodes-fsn1"
-    "kops.k8s.io/instance-role"  = "Node"
+    "kops.k8s.io/cluster"                                 = "minimal.example.com"
+    "kops.k8s.io/instance-group"                          = "nodes-fsn1"
+    "kops.k8s.io/instance-role"                           = "Node"
+    "node-label.kops.k8s.io.node-role.kubernetes.io/node" = ""
   }
   location = "fsn1"
   name     = "nodes-fsn1-${count.index}"

upup/pkg/fi/cloudup/hetzner/cloud.go

@@ -41,6 +41,7 @@ const (
 	TagKubernetesInstanceUserData    = "kops.k8s.io/instance-userdata"
 	TagKubernetesInstanceNeedsUpdate = "kops.k8s.io/needs-update"
 	TagKubernetesVolumeRole          = "kops.k8s.io/volume-role"
+	TagKubernetesNodeLabelPrefix     = "node-label.kops.k8s.io."
 )
 
 // HetznerCloud exposes all the interfaces required to operate on Hetzner Cloud resources