Add irsa support for nth

pkg/model/components/addonmanifests/BUILD.bazel

@@ -15,6 +15,7 @@ go_library(
         "//pkg/model/components/addonmanifests/awsloadbalancercontroller:go_default_library",
         "//pkg/model/components/addonmanifests/clusterautoscaler:go_default_library",
         "//pkg/model/components/addonmanifests/dnscontroller:go_default_library",
+        "//pkg/model/components/addonmanifests/nodeterminationhandler:go_default_library",
         "//pkg/model/iam:go_default_library",
         "//upup/pkg/fi:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",

pkg/model/components/addonmanifests/nodeterminationhandler/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["iam.go"],
+    importpath = "k8s.io/kops/pkg/model/components/addonmanifests/nodeterminationhandler",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/model/iam:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+    ],
+)

pkg/model/components/addonmanifests/nodeterminationhandler/iam.go

@@ -0,0 +1,48 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nodeterminationhandler
+
+import (
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kops/pkg/model/iam"
+)
+
+// ServiceAccount represents the service-account used by the dns-controller.
+// It implements iam.Subject to get AWS IAM permissions.
+type ServiceAccount struct {
+}
+
+var _ iam.Subject = &ServiceAccount{}
+
+// BuildAWSPolicy generates a custom policy for a ServiceAccount IAM role.
+func (r *ServiceAccount) BuildAWSPolicy(b *iam.PolicyBuilder) (*iam.Policy, error) {
+
+	clusterName := b.Cluster.ObjectMeta.Name
+	p := iam.NewPolicy(clusterName)
+
+	iam.AddNodeTerminationHandlerSQSPermissions(p)
+
+	return p, nil
+}
+
+// ServiceAccount returns the kubernetes service account used.
+func (r *ServiceAccount) ServiceAccount() (types.NamespacedName, bool) {
+	return types.NamespacedName{
+		Namespace: "kube-system",
+		Name:      "aws-node-termination-handler",
+	}, true
+}

pkg/model/components/addonmanifests/remap.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/kops/pkg/model/components/addonmanifests/awsloadbalancercontroller"
 	"k8s.io/kops/pkg/model/components/addonmanifests/clusterautoscaler"
 	"k8s.io/kops/pkg/model/components/addonmanifests/dnscontroller"
+	"k8s.io/kops/pkg/model/components/addonmanifests/nodeterminationhandler"
 	"k8s.io/kops/pkg/model/iam"
 	"k8s.io/kops/upup/pkg/fi"
 )
@@ -123,6 +124,8 @@ func getWellknownServiceAccount(name string) iam.Subject {
 		return &clusterautoscaler.ServiceAccount{}
 	case "ebs-csi-controller-sa":
 		return &awsebscsidriver.ServiceAccount{}
+	case "aws-node-termination-handler":
+		return &nodeterminationhandler.ServiceAccount{}
 	default:
 		return nil
 	}

pkg/model/iam/iam_builder.go

@@ -339,6 +339,11 @@ func (r *NodeRoleMaster) BuildAWSPolicy(b *PolicyBuilder) (*Policy, error) {
 			AddAWSLoadbalancerControllerPermissions(p)
 		}
 		AddClusterAutoscalerPermissions(p)
+
+		nth := b.Cluster.Spec.NodeTerminationHandler
+		if nth != nil && fi.BoolValue(nth.Enabled) && fi.BoolValue(nth.EnableSQSTerminationDraining) {
+			AddNodeTerminationHandlerSQSPermissions(p)
+		}
 	}
 
 	if b.Cluster.Spec.IAM.AllowContainerRegistry {
@@ -361,11 +366,6 @@ func (r *NodeRoleMaster) BuildAWSPolicy(b *PolicyBuilder) (*Policy, error) {
 		addCalicoSrcDstCheckPermissions(p)
 	}
 
-	nth := b.Cluster.Spec.NodeTerminationHandler
-	if nth != nil && fi.BoolValue(nth.Enabled) && fi.BoolValue(nth.EnableSQSTerminationDraining) {
-		addNodeTerminationHandlerSQSPermissions(p)
-	}
-
 	if b.Cluster.Spec.SnapshotController != nil && fi.BoolValue(b.Cluster.Spec.SnapshotController.Enabled) {
 		addSnapshotPersmissions(p)
 	}
@@ -1170,11 +1170,17 @@ func addAmazonVPCCNIPermissions(p *Policy, iamPrefix string) {
 	)
 }
 
-func addNodeTerminationHandlerSQSPermissions(p *Policy) {
+func AddNodeTerminationHandlerSQSPermissions(p *Policy) {
 	p.unconditionalAction.Insert(
-		"autoscaling:CompleteLifecycleAction",
 		"autoscaling:DescribeAutoScalingInstances",
+		"autoscaling:DescribeTags",
+		"ec2:DescribeInstances",
+		// SQS permissions do not support conditions.
 		"sqs:DeleteMessage",
 		"sqs:ReceiveMessage",
 	)
+	p.clusterTaggedAction.Insert(
+		"autoscaling:CompleteLifecycleAction",
+	)
+
 }

tests/integration/update_cluster/nth_sqs_resources/cloudformation.json

@@ -1234,7 +1234,6 @@
             },
             {
               "Action": [
-                "autoscaling:CompleteLifecycleAction",
                 "autoscaling:DescribeAutoScalingGroups",
                 "autoscaling:DescribeAutoScalingInstances",
                 "autoscaling:DescribeLaunchConfigurations",
@@ -1269,6 +1268,7 @@
             },
             {
               "Action": [
+                "autoscaling:CompleteLifecycleAction",
                 "autoscaling:SetDesiredCapacity",
                 "autoscaling:TerminateInstanceInAutoScalingGroup",
                 "ec2:AttachVolume",

tests/integration/update_cluster/nth_sqs_resources/data/aws_iam_role_policy_masters.nthsqsresources.example.com_policy

@@ -172,7 +172,6 @@
     },
     {
       "Action": [
-        "autoscaling:CompleteLifecycleAction",
         "autoscaling:DescribeAutoScalingGroups",
         "autoscaling:DescribeAutoScalingInstances",
         "autoscaling:DescribeLaunchConfigurations",
@@ -207,6 +206,7 @@
     },
     {
       "Action": [
+        "autoscaling:CompleteLifecycleAction",
         "autoscaling:SetDesiredCapacity",
         "autoscaling:TerminateInstanceInAutoScalingGroup",
         "ec2:AttachVolume",

tests/integration/update_cluster/nth_sqs_resources/data/aws_s3_bucket_object_nthsqsresources.example.com-addons-bootstrap_content

@@ -41,7 +41,7 @@ spec:
       k8s-addon: dns-controller.addons.k8s.io
   - id: k8s-1.11
     manifest: node-termination-handler.aws/k8s-1.11.yaml
-    manifestHash: 57274d900239a8ba937e5887c4fa8b276165f7f0
+    manifestHash: 4959667b24bdb70ded01a59a2c502ef197b0d8f4
     name: node-termination-handler.aws
     selector:
       k8s-addon: node-termination-handler.aws

tests/integration/update_cluster/nth_sqs_resources/data/aws_s3_bucket_object_nthsqsresources.example.com-addons-node-termination-handler.aws-k8s-1.11_content

@@ -214,7 +214,7 @@ spec:
       hostNetwork: false
       nodeSelector:
         node-role.kubernetes.io/master: ""
-      priorityClassName: system-node-critical
+      priorityClassName: system-cluster-critical
       securityContext:
         fsGroup: 1000
       serviceAccountName: aws-node-termination-handler

upup/models/cloudup/resources/addons/node-termination-handler.aws/k8s-1.11.yaml.template

@@ -94,7 +94,7 @@ spec:
         k8s-app: aws-node-termination-handler
         kubernetes.io/os: linux
     spec:
-      priorityClassName: "system-node-critical"
+      priorityClassName: "system-cluster-critical"
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -200,10 +200,12 @@ spec:
             requests:
               cpu: {{ .CPURequest }}
               memory: {{ .MemoryRequest }}
+      {{ if not UseServiceAccountIAM }}
       nodeSelector:
         node-role.kubernetes.io/master: ""
       tolerations:
         - operator: Exists
+      {{ end }}
 {{ else }}
 ---
 # Source: aws-node-termination-handler/templates/daemonset.linux.yaml

upup/pkg/fi/cloudup/bootstrapchannelbuilder/BUILD.bazel

@@ -21,6 +21,7 @@ go_library(
         "//pkg/model/components/addonmanifests/awsloadbalancercontroller:go_default_library",
         "//pkg/model/components/addonmanifests/clusterautoscaler:go_default_library",
         "//pkg/model/components/addonmanifests/dnscontroller:go_default_library",
+        "//pkg/model/components/addonmanifests/nodeterminationhandler:go_default_library",
         "//pkg/model/iam:go_default_library",
         "//pkg/templates:go_default_library",
         "//pkg/wellknownoperators:go_default_library",

upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go

@@ -33,6 +33,7 @@ import (
 	"k8s.io/kops/pkg/model/components/addonmanifests/awsloadbalancercontroller"
 	"k8s.io/kops/pkg/model/components/addonmanifests/clusterautoscaler"
 	"k8s.io/kops/pkg/model/components/addonmanifests/dnscontroller"
+	"k8s.io/kops/pkg/model/components/addonmanifests/nodeterminationhandler"
 	"k8s.io/kops/pkg/model/iam"
 	"k8s.io/kops/pkg/templates"
 	"k8s.io/kops/pkg/wellknownoperators"
@@ -559,6 +560,10 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				Id:       id,
 			})
 		}
+
+		if b.UseServiceAccountIAM() {
+			serviceAccountRoles = append(serviceAccountRoles, &nodeterminationhandler.ServiceAccount{})
+		}
 	}
 
 	npd := b.Cluster.Spec.NodeProblemDetector