Merge pull request #15515 from justinsb/strict_node_label_checking

node labeling: don't ignore unknown roles

cmd/kops-controller/controllers/legacy_node_controller.go

@@ -111,7 +111,10 @@ func (r *LegacyNodeReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		return ctrl.Result{}, fmt.Errorf("unable to load instance group object for node %s: %v", node.Name, err)
 	}
 
-	labels := nodelabels.BuildNodeLabels(cluster, ig)
+	labels, err := nodelabels.BuildNodeLabels(cluster, ig)
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("error building node labels for node %q: %w", node.Name, err)
+	}
 
 	lifecycle, err := r.getInstanceLifecycle(ctx, node)
 	if err != nil {

pkg/model/context.go

@@ -157,7 +157,11 @@ func (b *KopsModelContext) CloudTagsForInstanceGroup(ig *kops.InstanceGroup) (ma
 	}
 
 	// Apply labels for cluster autoscaler node labels
-	for k, v := range nodelabels.BuildNodeLabels(b.Cluster, ig) {
+	nodeLabels, err := nodelabels.BuildNodeLabels(b.Cluster, ig)
+	if err != nil {
+		return nil, fmt.Errorf("error building node labels: %w", err)
+	}
+	for k, v := range nodeLabels {
 		labels[nodeidentityaws.ClusterAutoscalerNodeTemplateLabel+k] = v
 	}
 

pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion-2.yaml

@@ -24,7 +24,6 @@ Metadata:
   cluster_generation: "0"
   ig_generation: "0"
   k8s: cluster
-  k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_node: ""
   k8s.io_role_bastion: "1"
   kops.k8s.io_instancegroup: bastion
 Name: bastion-1-cluster

pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion.yaml

@@ -37,7 +37,6 @@ Metadata:
   cluster_generation: "0"
   ig_generation: "0"
   k8s: cluster
-  k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_node: ""
   k8s.io_role_bastion: "1"
   kops.k8s.io_instancegroup: bastion
 Name: bastion-1-cluster

pkg/nodelabels/builder.go

@@ -17,6 +17,8 @@ limitations under the License.
 package nodelabels
 
 import (
+	"fmt"
+
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/featureflag"
 	"k8s.io/kops/util/pkg/reflectutils"
@@ -37,10 +39,22 @@ const (
 
 // BuildNodeLabels returns the node labels for the specified instance group
 // This moved from the kubelet to a central controller in kubernetes 1.16
-func BuildNodeLabels(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) map[string]string {
+func BuildNodeLabels(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) (map[string]string, error) {
-	isControlPlane := instanceGroup.Spec.Role == kops.InstanceGroupRoleControlPlane
+	isControlPlane := false
-
+	isAPIServer := false
-	isAPIServer := instanceGroup.Spec.Role == kops.InstanceGroupRoleAPIServer
+	isNode := false
+	switch instanceGroup.Spec.Role {
+	case kops.InstanceGroupRoleControlPlane:
+		isControlPlane = true
+	case kops.InstanceGroupRoleAPIServer:
+		isAPIServer = true
+	case kops.InstanceGroupRoleNode:
+		isNode = true
+	case kops.InstanceGroupRoleBastion:
+		// no labels to add
+	default:
+		return nil, fmt.Errorf("unhandled instanceGroup role %q", instanceGroup.Spec.Role)
+	}
 
 	// Merge KubeletConfig for NodeLabels
 	c := &kops.KubeletConfigSpec{}
@@ -70,7 +84,9 @@ func BuildNodeLabels(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) m
 		if cluster.IsKubernetesLT("1.24") {
 			nodeLabels[RoleLabelName15] = RoleAPIServerLabelValue15
 		}
-	} else {
+	}
+
+	if isNode {
 		if nodeLabels == nil {
 			nodeLabels = make(map[string]string)
 		}
@@ -104,7 +120,7 @@ func BuildNodeLabels(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) m
 		nodeLabels["karpenter.sh/provisioner-name"] = instanceGroup.ObjectMeta.Name
 	}
 
-	return nodeLabels
+	return nodeLabels, nil
 }
 
 // BuildMandatoryControlPlaneLabels returns the list of labels all CP nodes must have

pkg/nodelabels/builder_test.go

@@ -115,7 +115,10 @@ func TestBuildNodeLabels(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			out := BuildNodeLabels(test.cluster, test.ig)
+			out, err := BuildNodeLabels(test.cluster, test.ig)
+			if err != nil {
+				t.Fatalf("unexpected error from BuildNodeLabels: %v", err)
+			}
 			if !reflect.DeepEqual(out, test.expected) {
 				t.Fatalf("Actual result:\n%v\nExpect:\n%v", out, test.expected)
 			}

tests/integration/update_cluster/bastionadditional_user-data/data/aws_s3_object_nodeupconfig-bastion_content

@@ -37,8 +37,6 @@ KubeletConfig:
     InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  nodeLabels:
-    node-role.kubernetes.io/node: ""
   podInfraContainerImage: registry.k8s.io/pause:3.9
   podManifestPath: /etc/kubernetes/manifests
   protectKernelDefaults: true

tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf

@@ -147,11 +147,6 @@ resource "aws_autoscaling_group" "bastion-bastionuserdata-example-com" {
     propagate_at_launch = true
     value               = "bastion.bastionuserdata.example.com"
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
-    propagate_at_launch = true
-    value               = ""
-  }
   tag {
     key                 = "k8s.io/role/bastion"
     propagate_at_launch = true
@@ -487,7 +482,6 @@ resource "aws_launch_template" "bastion-bastionuserdata-example-com" {
     tags = {
       "KubernetesCluster"                                 = "bastionuserdata.example.com"
       "Name"                                              = "bastion.bastionuserdata.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                               = "1"
       "kops.k8s.io/instancegroup"                         = "bastion"
       "kubernetes.io/cluster/bastionuserdata.example.com" = "owned"
@@ -498,7 +492,6 @@ resource "aws_launch_template" "bastion-bastionuserdata-example-com" {
     tags = {
       "KubernetesCluster"                                 = "bastionuserdata.example.com"
       "Name"                                              = "bastion.bastionuserdata.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                               = "1"
       "kops.k8s.io/instancegroup"                         = "bastion"
       "kubernetes.io/cluster/bastionuserdata.example.com" = "owned"
@@ -507,7 +500,6 @@ resource "aws_launch_template" "bastion-bastionuserdata-example-com" {
   tags = {
     "KubernetesCluster"                                 = "bastionuserdata.example.com"
     "Name"                                              = "bastion.bastionuserdata.example.com"
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/bastion"                               = "1"
     "kops.k8s.io/instancegroup"                         = "bastion"
     "kubernetes.io/cluster/bastionuserdata.example.com" = "owned"

tests/integration/update_cluster/private-shared-ip/kubernetes.tf

@@ -147,11 +147,6 @@ resource "aws_autoscaling_group" "bastion-private-shared-ip-example-com" {
     propagate_at_launch = true
     value               = "bastion.private-shared-ip.example.com"
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
-    propagate_at_launch = true
-    value               = ""
-  }
   tag {
     key                 = "k8s.io/role/bastion"
     propagate_at_launch = true
@@ -469,7 +464,6 @@ resource "aws_launch_template" "bastion-private-shared-ip-example-com" {
     tags = {
       "KubernetesCluster"                                   = "private-shared-ip.example.com"
       "Name"                                                = "bastion.private-shared-ip.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                                 = "1"
       "kops.k8s.io/instancegroup"                           = "bastion"
       "kubernetes.io/cluster/private-shared-ip.example.com" = "owned"
@@ -480,7 +474,6 @@ resource "aws_launch_template" "bastion-private-shared-ip-example-com" {
     tags = {
       "KubernetesCluster"                                   = "private-shared-ip.example.com"
       "Name"                                                = "bastion.private-shared-ip.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                                 = "1"
       "kops.k8s.io/instancegroup"                           = "bastion"
       "kubernetes.io/cluster/private-shared-ip.example.com" = "owned"
@@ -489,7 +482,6 @@ resource "aws_launch_template" "bastion-private-shared-ip-example-com" {
   tags = {
     "KubernetesCluster"                                   = "private-shared-ip.example.com"
     "Name"                                                = "bastion.private-shared-ip.example.com"
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/bastion"                                 = "1"
     "kops.k8s.io/instancegroup"                           = "bastion"
     "kubernetes.io/cluster/private-shared-ip.example.com" = "owned"

tests/integration/update_cluster/private-shared-subnet/kubernetes.tf

@@ -142,11 +142,6 @@ resource "aws_autoscaling_group" "bastion-private-shared-subnet-example-com" {
     propagate_at_launch = true
     value               = "bastion.private-shared-subnet.example.com"
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
-    propagate_at_launch = true
-    value               = ""
-  }
   tag {
     key                 = "k8s.io/role/bastion"
     propagate_at_launch = true
@@ -464,7 +459,6 @@ resource "aws_launch_template" "bastion-private-shared-subnet-example-com" {
     tags = {
       "KubernetesCluster"                                       = "private-shared-subnet.example.com"
       "Name"                                                    = "bastion.private-shared-subnet.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                                     = "1"
       "kops.k8s.io/instancegroup"                               = "bastion"
       "kubernetes.io/cluster/private-shared-subnet.example.com" = "owned"
@@ -475,7 +469,6 @@ resource "aws_launch_template" "bastion-private-shared-subnet-example-com" {
     tags = {
       "KubernetesCluster"                                       = "private-shared-subnet.example.com"
       "Name"                                                    = "bastion.private-shared-subnet.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                                     = "1"
       "kops.k8s.io/instancegroup"                               = "bastion"
       "kubernetes.io/cluster/private-shared-subnet.example.com" = "owned"
@@ -484,7 +477,6 @@ resource "aws_launch_template" "bastion-private-shared-subnet-example-com" {
   tags = {
     "KubernetesCluster"                                       = "private-shared-subnet.example.com"
     "Name"                                                    = "bastion.private-shared-subnet.example.com"
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/bastion"                                     = "1"
     "kops.k8s.io/instancegroup"                               = "bastion"
     "kubernetes.io/cluster/private-shared-subnet.example.com" = "owned"

tests/integration/update_cluster/privatecalico/kubernetes.tf

@@ -147,11 +147,6 @@ resource "aws_autoscaling_group" "bastion-privatecalico-example-com" {
     propagate_at_launch = true
     value               = "bastion.privatecalico.example.com"
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
-    propagate_at_launch = true
-    value               = ""
-  }
   tag {
     key                 = "k8s.io/role/bastion"
     propagate_at_launch = true
@@ -487,7 +482,6 @@ resource "aws_launch_template" "bastion-privatecalico-example-com" {
     tags = {
       "KubernetesCluster"                               = "privatecalico.example.com"
       "Name"                                            = "bastion.privatecalico.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                             = "1"
       "kops.k8s.io/instancegroup"                       = "bastion"
       "kubernetes.io/cluster/privatecalico.example.com" = "owned"
@@ -498,7 +492,6 @@ resource "aws_launch_template" "bastion-privatecalico-example-com" {
     tags = {
       "KubernetesCluster"                               = "privatecalico.example.com"
       "Name"                                            = "bastion.privatecalico.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                             = "1"
       "kops.k8s.io/instancegroup"                       = "bastion"
       "kubernetes.io/cluster/privatecalico.example.com" = "owned"
@@ -507,7 +500,6 @@ resource "aws_launch_template" "bastion-privatecalico-example-com" {
   tags = {
     "KubernetesCluster"                               = "privatecalico.example.com"
     "Name"                                            = "bastion.privatecalico.example.com"
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/bastion"                             = "1"
     "kops.k8s.io/instancegroup"                       = "bastion"
     "kubernetes.io/cluster/privatecalico.example.com" = "owned"

tests/integration/update_cluster/privatecanal/kubernetes.tf

@@ -147,11 +147,6 @@ resource "aws_autoscaling_group" "bastion-privatecanal-example-com" {
     propagate_at_launch = true
     value               = "bastion.privatecanal.example.com"
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
-    propagate_at_launch = true
-    value               = ""
-  }
   tag {
     key                 = "k8s.io/role/bastion"
     propagate_at_launch = true
@@ -487,7 +482,6 @@ resource "aws_launch_template" "bastion-privatecanal-example-com" {
     tags = {
       "KubernetesCluster"                              = "privatecanal.example.com"
       "Name"                                           = "bastion.privatecanal.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                            = "1"
       "kops.k8s.io/instancegroup"                      = "bastion"
       "kubernetes.io/cluster/privatecanal.example.com" = "owned"
@@ -498,7 +492,6 @@ resource "aws_launch_template" "bastion-privatecanal-example-com" {
     tags = {
       "KubernetesCluster"                              = "privatecanal.example.com"
       "Name"                                           = "bastion.privatecanal.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                            = "1"
       "kops.k8s.io/instancegroup"                      = "bastion"
       "kubernetes.io/cluster/privatecanal.example.com" = "owned"
@@ -507,7 +500,6 @@ resource "aws_launch_template" "bastion-privatecanal-example-com" {
   tags = {
     "KubernetesCluster"                              = "privatecanal.example.com"
     "Name"                                           = "bastion.privatecanal.example.com"
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/bastion"                            = "1"
     "kops.k8s.io/instancegroup"                      = "bastion"
     "kubernetes.io/cluster/privatecanal.example.com" = "owned"

tests/integration/update_cluster/privatecilium-eni/kubernetes.tf

@@ -147,11 +147,6 @@ resource "aws_autoscaling_group" "bastion-privatecilium-example-com" {
     propagate_at_launch = true
     value               = "bastion.privatecilium.example.com"
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
-    propagate_at_launch = true
-    value               = ""
-  }
   tag {
     key                 = "k8s.io/role/bastion"
     propagate_at_launch = true
@@ -487,7 +482,6 @@ resource "aws_launch_template" "bastion-privatecilium-example-com" {
     tags = {
       "KubernetesCluster"                               = "privatecilium.example.com"
       "Name"                                            = "bastion.privatecilium.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                             = "1"
       "kops.k8s.io/instancegroup"                       = "bastion"
       "kubernetes.io/cluster/privatecilium.example.com" = "owned"
@@ -498,7 +492,6 @@ resource "aws_launch_template" "bastion-privatecilium-example-com" {
     tags = {
       "KubernetesCluster"                               = "privatecilium.example.com"
       "Name"                                            = "bastion.privatecilium.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                             = "1"
       "kops.k8s.io/instancegroup"                       = "bastion"
       "kubernetes.io/cluster/privatecilium.example.com" = "owned"
@@ -507,7 +500,6 @@ resource "aws_launch_template" "bastion-privatecilium-example-com" {
   tags = {
     "KubernetesCluster"                               = "privatecilium.example.com"
     "Name"                                            = "bastion.privatecilium.example.com"
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/bastion"                             = "1"
     "kops.k8s.io/instancegroup"                       = "bastion"
     "kubernetes.io/cluster/privatecilium.example.com" = "owned"

tests/integration/update_cluster/privatecilium/kubernetes.tf

@@ -147,11 +147,6 @@ resource "aws_autoscaling_group" "bastion-privatecilium-example-com" {
     propagate_at_launch = true
     value               = "bastion.privatecilium.example.com"
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
-    propagate_at_launch = true
-    value               = ""
-  }
   tag {
     key                 = "k8s.io/role/bastion"
     propagate_at_launch = true
@@ -487,7 +482,6 @@ resource "aws_launch_template" "bastion-privatecilium-example-com" {
     tags = {
       "KubernetesCluster"                               = "privatecilium.example.com"
       "Name"                                            = "bastion.privatecilium.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                             = "1"
       "kops.k8s.io/instancegroup"                       = "bastion"
       "kubernetes.io/cluster/privatecilium.example.com" = "owned"
@@ -498,7 +492,6 @@ resource "aws_launch_template" "bastion-privatecilium-example-com" {
     tags = {
       "KubernetesCluster"                               = "privatecilium.example.com"
       "Name"                                            = "bastion.privatecilium.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                             = "1"
       "kops.k8s.io/instancegroup"                       = "bastion"
       "kubernetes.io/cluster/privatecilium.example.com" = "owned"
@@ -507,7 +500,6 @@ resource "aws_launch_template" "bastion-privatecilium-example-com" {
   tags = {
     "KubernetesCluster"                               = "privatecilium.example.com"
     "Name"                                            = "bastion.privatecilium.example.com"
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/bastion"                             = "1"
     "kops.k8s.io/instancegroup"                       = "bastion"
     "kubernetes.io/cluster/privatecilium.example.com" = "owned"

tests/integration/update_cluster/privatecilium2/kubernetes.tf

@@ -147,11 +147,6 @@ resource "aws_autoscaling_group" "bastion-privatecilium-example-com" {
     propagate_at_launch = true
     value               = "bastion.privatecilium.example.com"
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
-    propagate_at_launch = true
-    value               = ""
-  }
   tag {
     key                 = "k8s.io/role/bastion"
     propagate_at_launch = true
@@ -487,7 +482,6 @@ resource "aws_launch_template" "bastion-privatecilium-example-com" {
     tags = {
       "KubernetesCluster"                               = "privatecilium.example.com"
       "Name"                                            = "bastion.privatecilium.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                             = "1"
       "kops.k8s.io/instancegroup"                       = "bastion"
       "kubernetes.io/cluster/privatecilium.example.com" = "owned"
@@ -498,7 +492,6 @@ resource "aws_launch_template" "bastion-privatecilium-example-com" {
     tags = {
       "KubernetesCluster"                               = "privatecilium.example.com"
       "Name"                                            = "bastion.privatecilium.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                             = "1"
       "kops.k8s.io/instancegroup"                       = "bastion"
       "kubernetes.io/cluster/privatecilium.example.com" = "owned"
@@ -507,7 +500,6 @@ resource "aws_launch_template" "bastion-privatecilium-example-com" {
   tags = {
     "KubernetesCluster"                               = "privatecilium.example.com"
     "Name"                                            = "bastion.privatecilium.example.com"
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/bastion"                             = "1"
     "kops.k8s.io/instancegroup"                       = "bastion"
     "kubernetes.io/cluster/privatecilium.example.com" = "owned"

tests/integration/update_cluster/privateciliumadvanced/kubernetes.tf

@@ -147,11 +147,6 @@ resource "aws_autoscaling_group" "bastion-privateciliumadvanced-example-com" {
     propagate_at_launch = true
     value               = "bastion.privateciliumadvanced.example.com"
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
-    propagate_at_launch = true
-    value               = ""
-  }
   tag {
     key                 = "k8s.io/role/bastion"
     propagate_at_launch = true
@@ -504,7 +499,6 @@ resource "aws_launch_template" "bastion-privateciliumadvanced-example-com" {
     tags = {
       "KubernetesCluster"                                       = "privateciliumadvanced.example.com"
       "Name"                                                    = "bastion.privateciliumadvanced.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                                     = "1"
       "kops.k8s.io/instancegroup"                               = "bastion"
       "kubernetes.io/cluster/privateciliumadvanced.example.com" = "owned"
@@ -515,7 +509,6 @@ resource "aws_launch_template" "bastion-privateciliumadvanced-example-com" {
     tags = {
       "KubernetesCluster"                                       = "privateciliumadvanced.example.com"
       "Name"                                                    = "bastion.privateciliumadvanced.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                                     = "1"
       "kops.k8s.io/instancegroup"                               = "bastion"
       "kubernetes.io/cluster/privateciliumadvanced.example.com" = "owned"
@@ -524,7 +517,6 @@ resource "aws_launch_template" "bastion-privateciliumadvanced-example-com" {
   tags = {
     "KubernetesCluster"                                       = "privateciliumadvanced.example.com"
     "Name"                                                    = "bastion.privateciliumadvanced.example.com"
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/bastion"                                     = "1"
     "kops.k8s.io/instancegroup"                               = "bastion"
     "kubernetes.io/cluster/privateciliumadvanced.example.com" = "owned"

tests/integration/update_cluster/privatedns1/kubernetes.tf

@@ -157,16 +157,6 @@ resource "aws_autoscaling_group" "bastion-privatedns1-example-com" {
     propagate_at_launch = true
     value               = "fib+baz"
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"
-    propagate_at_launch = true
-    value               = "node"
-  }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
-    propagate_at_launch = true
-    value               = ""
-  }
   tag {
     key                 = "k8s.io/role/bastion"
     propagate_at_launch = true
@@ -563,8 +553,6 @@ resource "aws_launch_template" "bastion-privatedns1-example-com" {
       "Name"                                          = "bastion.privatedns1.example.com"
       "Owner"                                         = "John Doe"
       "foo/bar"                                       = "fib+baz"
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                           = "1"
       "kops.k8s.io/instancegroup"                     = "bastion"
       "kubernetes.io/cluster/privatedns1.example.com" = "owned"
@@ -577,8 +565,6 @@ resource "aws_launch_template" "bastion-privatedns1-example-com" {
       "Name"                                          = "bastion.privatedns1.example.com"
       "Owner"                                         = "John Doe"
       "foo/bar"                                       = "fib+baz"
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                           = "1"
       "kops.k8s.io/instancegroup"                     = "bastion"
       "kubernetes.io/cluster/privatedns1.example.com" = "owned"
@@ -589,8 +575,6 @@ resource "aws_launch_template" "bastion-privatedns1-example-com" {
     "Name"                                          = "bastion.privatedns1.example.com"
     "Owner"                                         = "John Doe"
     "foo/bar"                                       = "fib+baz"
-    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/bastion"                           = "1"
     "kops.k8s.io/instancegroup"                     = "bastion"
     "kubernetes.io/cluster/privatedns1.example.com" = "owned"

tests/integration/update_cluster/privatedns2/kubernetes.tf

@@ -147,11 +147,6 @@ resource "aws_autoscaling_group" "bastion-privatedns2-example-com" {
     propagate_at_launch = true
     value               = "bastion.privatedns2.example.com"
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
-    propagate_at_launch = true
-    value               = ""
-  }
   tag {
     key                 = "k8s.io/role/bastion"
     propagate_at_launch = true
@@ -478,7 +473,6 @@ resource "aws_launch_template" "bastion-privatedns2-example-com" {
     tags = {
       "KubernetesCluster"                             = "privatedns2.example.com"
       "Name"                                          = "bastion.privatedns2.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                           = "1"
       "kops.k8s.io/instancegroup"                     = "bastion"
       "kubernetes.io/cluster/privatedns2.example.com" = "owned"
@@ -489,7 +483,6 @@ resource "aws_launch_template" "bastion-privatedns2-example-com" {
     tags = {
       "KubernetesCluster"                             = "privatedns2.example.com"
       "Name"                                          = "bastion.privatedns2.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                           = "1"
       "kops.k8s.io/instancegroup"                     = "bastion"
       "kubernetes.io/cluster/privatedns2.example.com" = "owned"
@@ -498,7 +491,6 @@ resource "aws_launch_template" "bastion-privatedns2-example-com" {
   tags = {
     "KubernetesCluster"                             = "privatedns2.example.com"
     "Name"                                          = "bastion.privatedns2.example.com"
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/bastion"                           = "1"
     "kops.k8s.io/instancegroup"                     = "bastion"
     "kubernetes.io/cluster/privatedns2.example.com" = "owned"

tests/integration/update_cluster/privateflannel/kubernetes.tf

@@ -147,11 +147,6 @@ resource "aws_autoscaling_group" "bastion-privateflannel-example-com" {
     propagate_at_launch = true
     value               = "bastion.privateflannel.example.com"
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
-    propagate_at_launch = true
-    value               = ""
-  }
   tag {
     key                 = "k8s.io/role/bastion"
     propagate_at_launch = true
@@ -487,7 +482,6 @@ resource "aws_launch_template" "bastion-privateflannel-example-com" {
     tags = {
       "KubernetesCluster"                                = "privateflannel.example.com"
       "Name"                                             = "bastion.privateflannel.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                              = "1"
       "kops.k8s.io/instancegroup"                        = "bastion"
       "kubernetes.io/cluster/privateflannel.example.com" = "owned"
@@ -498,7 +492,6 @@ resource "aws_launch_template" "bastion-privateflannel-example-com" {
     tags = {
       "KubernetesCluster"                                = "privateflannel.example.com"
       "Name"                                             = "bastion.privateflannel.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                              = "1"
       "kops.k8s.io/instancegroup"                        = "bastion"
       "kubernetes.io/cluster/privateflannel.example.com" = "owned"
@@ -507,7 +500,6 @@ resource "aws_launch_template" "bastion-privateflannel-example-com" {
   tags = {
     "KubernetesCluster"                                = "privateflannel.example.com"
     "Name"                                             = "bastion.privateflannel.example.com"
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/bastion"                              = "1"
     "kops.k8s.io/instancegroup"                        = "bastion"
     "kubernetes.io/cluster/privateflannel.example.com" = "owned"

tests/integration/update_cluster/privatekopeio/kubernetes.tf

@@ -162,11 +162,6 @@ resource "aws_autoscaling_group" "bastion-privatekopeio-example-com" {
     propagate_at_launch = true
     value               = "bastion.privatekopeio.example.com"
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
-    propagate_at_launch = true
-    value               = ""
-  }
   tag {
     key                 = "k8s.io/role/bastion"
     propagate_at_launch = true
@@ -493,7 +488,6 @@ resource "aws_launch_template" "bastion-privatekopeio-example-com" {
     tags = {
       "KubernetesCluster"                               = "privatekopeio.example.com"
       "Name"                                            = "bastion.privatekopeio.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                             = "1"
       "kops.k8s.io/instancegroup"                       = "bastion"
       "kubernetes.io/cluster/privatekopeio.example.com" = "owned"
@@ -504,7 +498,6 @@ resource "aws_launch_template" "bastion-privatekopeio-example-com" {
     tags = {
       "KubernetesCluster"                               = "privatekopeio.example.com"
       "Name"                                            = "bastion.privatekopeio.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                             = "1"
       "kops.k8s.io/instancegroup"                       = "bastion"
       "kubernetes.io/cluster/privatekopeio.example.com" = "owned"
@@ -513,7 +506,6 @@ resource "aws_launch_template" "bastion-privatekopeio-example-com" {
   tags = {
     "KubernetesCluster"                               = "privatekopeio.example.com"
     "Name"                                            = "bastion.privatekopeio.example.com"
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/bastion"                             = "1"
     "kops.k8s.io/instancegroup"                       = "bastion"
     "kubernetes.io/cluster/privatekopeio.example.com" = "owned"

tests/integration/update_cluster/privateweave/kubernetes.tf

@@ -147,16 +147,6 @@ resource "aws_autoscaling_group" "bastion-privateweave-example-com" {
     propagate_at_launch = true
     value               = "bastion.privateweave.example.com"
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"
-    propagate_at_launch = true
-    value               = "node"
-  }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
-    propagate_at_launch = true
-    value               = ""
-  }
   tag {
     key                 = "k8s.io/role/bastion"
     propagate_at_launch = true
@@ -507,8 +497,6 @@ resource "aws_launch_template" "bastion-privateweave-example-com" {
     tags = {
       "KubernetesCluster"                              = "privateweave.example.com"
       "Name"                                           = "bastion.privateweave.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                            = "1"
       "kops.k8s.io/instancegroup"                      = "bastion"
       "kubernetes.io/cluster/privateweave.example.com" = "owned"
@@ -519,8 +507,6 @@ resource "aws_launch_template" "bastion-privateweave-example-com" {
     tags = {
       "KubernetesCluster"                              = "privateweave.example.com"
       "Name"                                           = "bastion.privateweave.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                            = "1"
       "kops.k8s.io/instancegroup"                      = "bastion"
       "kubernetes.io/cluster/privateweave.example.com" = "owned"
@@ -529,8 +515,6 @@ resource "aws_launch_template" "bastion-privateweave-example-com" {
   tags = {
     "KubernetesCluster"                              = "privateweave.example.com"
     "Name"                                           = "bastion.privateweave.example.com"
-    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/bastion"                            = "1"
     "kops.k8s.io/instancegroup"                      = "bastion"
     "kubernetes.io/cluster/privateweave.example.com" = "owned"

tests/integration/update_cluster/unmanaged/kubernetes.tf

@@ -147,11 +147,6 @@ resource "aws_autoscaling_group" "bastion-unmanaged-example-com" {
     propagate_at_launch = true
     value               = "bastion.unmanaged.example.com"
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
-    propagate_at_launch = true
-    value               = ""
-  }
   tag {
     key                 = "k8s.io/role/bastion"
     propagate_at_launch = true
@@ -469,7 +464,6 @@ resource "aws_launch_template" "bastion-unmanaged-example-com" {
     tags = {
       "KubernetesCluster"                           = "unmanaged.example.com"
       "Name"                                        = "bastion.unmanaged.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                         = "1"
       "kops.k8s.io/instancegroup"                   = "bastion"
       "kubernetes.io/cluster/unmanaged.example.com" = "owned"
@@ -480,7 +474,6 @@ resource "aws_launch_template" "bastion-unmanaged-example-com" {
     tags = {
       "KubernetesCluster"                           = "unmanaged.example.com"
       "Name"                                        = "bastion.unmanaged.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                         = "1"
       "kops.k8s.io/instancegroup"                   = "bastion"
       "kubernetes.io/cluster/unmanaged.example.com" = "owned"
@@ -489,7 +482,6 @@ resource "aws_launch_template" "bastion-unmanaged-example-com" {
   tags = {
     "KubernetesCluster"                           = "unmanaged.example.com"
     "Name"                                        = "bastion.unmanaged.example.com"
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/bastion"                         = "1"
     "kops.k8s.io/instancegroup"                   = "bastion"
     "kubernetes.io/cluster/unmanaged.example.com" = "owned"

upup/pkg/fi/cloudup/populate_instancegroup_spec.go

@@ -258,7 +258,11 @@ func PopulateInstanceGroupSpec(cluster *kops.Cluster, input *kops.InstanceGroup,
 
 	// We include the NodeLabels in the userdata even for Kubernetes 1.16 and later so that
 	// rolling update will still replace nodes when they change.
-	igKubeletConfig.NodeLabels = nodelabels.BuildNodeLabels(cluster, ig)
+	nodeLabels, err := nodelabels.BuildNodeLabels(cluster, ig)
+	if err != nil {
+		return nil, fmt.Errorf("error building node labels: %w", err)
+	}
+	igKubeletConfig.NodeLabels = nodeLabels
 
 	useSecureKubelet := fi.ValueOf(igKubeletConfig.AnonymousAuth)