From 75d2b97754db464aff0a9cedf95f9771e3d7ea99 Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Tue, 7 Dec 2021 20:59:12 +0100 Subject: [PATCH] Set provider name on karpenter-managed node --- pkg/nodeidentity/aws/identify.go | 14 +++++- pkg/nodelabels/builder.go | 4 ++ ...penter-nodes.minimal.example.com_user_data | 2 +- ...bject_nodeupconfig-karpenter-nodes_content | 1 + .../update_cluster/karpenter/kubernetes.tf | 45 ++++++++++--------- 5 files changed, 42 insertions(+), 24 deletions(-) diff --git a/pkg/nodeidentity/aws/identify.go b/pkg/nodeidentity/aws/identify.go index 83beaa0620..13bcd6cc2e 100644 --- a/pkg/nodeidentity/aws/identify.go +++ b/pkg/nodeidentity/aws/identify.go @@ -31,6 +31,7 @@ import ( corev1 "k8s.io/api/core/v1" expirationcache "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" + "k8s.io/kops/pkg/nodeidentity" ) @@ -41,7 +42,8 @@ const ( // ClusterAutoscalerNodeTemplateLabel is the prefix used on node labels when copying to cloud tags. ClusterAutoscalerNodeTemplateLabel = "k8s.io/cluster-autoscaler/node-template/label/" // The expiration time of nodeidentity.Info cache. - cacheTTL = 60 * time.Minute + cacheTTL = 60 * time.Minute + KarpenterNodeLabel = "karpenter.sh/" ) // nodeIdentifier identifies a node from EC2 @@ -145,10 +147,18 @@ func (i *nodeIdentifier) IdentifyNode(ctx context.Context, node *corev1.Node) (* Labels: labels, } + isKarpenterManaged := false for _, tag := range instance.Tags { - if strings.HasPrefix(aws.StringValue(tag.Key), ClusterAutoscalerNodeTemplateLabel) { + key := aws.StringValue(tag.Key) + if strings.HasPrefix(key, ClusterAutoscalerNodeTemplateLabel) { info.Labels[strings.TrimPrefix(aws.StringValue(tag.Key), ClusterAutoscalerNodeTemplateLabel)] = aws.StringValue(tag.Value) } + if strings.HasPrefix(key, KarpenterNodeLabel) { + isKarpenterManaged = true + } + } + if isKarpenterManaged { + info.Labels["karpenter.sh/provisioner-name"] = info.Labels[CloudTagInstanceGroupName] } // If caching is enabled add the nodeidentity.Info to cache. diff --git a/pkg/nodelabels/builder.go b/pkg/nodelabels/builder.go index a6ea671959..f01d56e4b5 100644 --- a/pkg/nodelabels/builder.go +++ b/pkg/nodelabels/builder.go @@ -92,6 +92,10 @@ func BuildNodeLabels(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) m nodeLabels[k] = v } + if instanceGroup.Spec.InstanceManager == kops.InstanceManagerKarpenter { + nodeLabels["karpenter.sh/provisioner-name"] = instanceGroup.ObjectMeta.Name + } + return nodeLabels } diff --git a/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes.minimal.example.com_user_data b/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes.minimal.example.com_user_data index d3ff3ce9b2..ae9d9bc182 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes.minimal.example.com_user_data +++ b/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes.minimal.example.com_user_data @@ -160,7 +160,7 @@ CloudProvider: aws ConfigBase: memfs://clusters.example.com/minimal.example.com InstanceGroupName: karpenter-nodes InstanceGroupRole: Node -NodeupConfigHash: ehZK5PooPMXQw0YD3dy5oARwClEXIj8ymh6DR1XYbQ0= +NodeupConfigHash: 82WPA9hO1RdHX0zXhyshQojhW4qQug4izPAWZkgZkN4= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-karpenter-nodes_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-karpenter-nodes_content index 014862648a..7dcfc40cd3 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-karpenter-nodes_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-karpenter-nodes_content @@ -50,6 +50,7 @@ KubeletConfig: logLevel: 2 networkPluginName: cni nodeLabels: + karpenter.sh/provisioner-name: karpenter-nodes kubernetes.io/role: node node-role.kubernetes.io/node: "" nonMasqueradeCIDR: 100.64.0.0/10 diff --git a/tests/integration/update_cluster/karpenter/kubernetes.tf b/tests/integration/update_cluster/karpenter/kubernetes.tf index 6e38358a48..77a268e08c 100644 --- a/tests/integration/update_cluster/karpenter/kubernetes.tf +++ b/tests/integration/update_cluster/karpenter/kubernetes.tf @@ -414,35 +414,38 @@ resource "aws_launch_template" "karpenter-nodes-minimal-example-com" { tag_specifications { resource_type = "instance" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "karpenter-nodes.minimal.example.com" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "karpenter-nodes" - "kubernetes.io/cluster/minimal.example.com" = "owned" + "KubernetesCluster" = "minimal.example.com" + "Name" = "karpenter-nodes.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "karpenter-nodes" + "kubernetes.io/cluster/minimal.example.com" = "owned" } } tag_specifications { resource_type = "volume" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "karpenter-nodes.minimal.example.com" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "karpenter-nodes" - "kubernetes.io/cluster/minimal.example.com" = "owned" + "KubernetesCluster" = "minimal.example.com" + "Name" = "karpenter-nodes.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "karpenter-nodes" + "kubernetes.io/cluster/minimal.example.com" = "owned" } } tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "karpenter-nodes.minimal.example.com" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "karpenter-nodes" - "kubernetes.io/cluster/minimal.example.com" = "owned" + "KubernetesCluster" = "minimal.example.com" + "Name" = "karpenter-nodes.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "karpenter-nodes" + "kubernetes.io/cluster/minimal.example.com" = "owned" } user_data = filebase64("${path.module}/data/aws_launch_template_karpenter-nodes.minimal.example.com_user_data") }