Merge pull request #15666 from johngmyers/nth

Enable NTH by default on AWS

cmd/kops/integration_test.go

@@ -81,6 +81,7 @@ func newIntegrationTest(clusterName, srcDir string) *integrationTest {
 		version:        "v1alpha2",
 		zones:          1,
 		expectPolicies: true,
+		nth:            true,
 		sshKey:         true,
 	}
 }
@@ -145,8 +146,8 @@ func (i *integrationTest) withDedicatedAPIServer() *integrationTest {
 	return i
 }
 
-func (i *integrationTest) withNTH() *integrationTest {
+func (i *integrationTest) withoutNTH() *integrationTest {
-	i.nth = true
+	i.nth = false
 	return i
 }
 
@@ -338,6 +339,7 @@ func TestMinimalGossipIRSA(t *testing.T) {
 		withOIDCDiscovery().
 		withServiceAccountRole("aws-cloud-controller-manager.kube-system", true).
 		withServiceAccountRole("ebs-csi-controller-sa.kube-system", true).
+		withServiceAccountRole("aws-node-termination-handler.kube-system", true).
 		withAddons(
 			awsEBSCSIAddon,
 			dnsControllerAddon,
@@ -752,6 +754,7 @@ func TestPrivateDns2(t *testing.T) {
 func TestDiscoveryFeatureGate(t *testing.T) {
 	newIntegrationTest("minimal.example.com", "public-jwks-apiserver").
 		withDefaultServiceAccountRoles24().
+		withServiceAccountRole("aws-node-termination-handler.kube-system", true).
 		withDefaultAddons24().
 		withOIDCDiscovery().
 		runTestTerraformAWS(t)
@@ -775,6 +778,7 @@ func TestAWSLBController(t *testing.T) {
 		withServiceAccountRole("dns-controller.kube-system", true).
 		withServiceAccountRole("aws-load-balancer-controller.kube-system", true).
 		withServiceAccountRole("aws-cloud-controller-manager.kube-system", true).
+		withServiceAccountRole("aws-node-termination-handler.kube-system", true).
 		withServiceAccountRole("ebs-csi-controller-sa.kube-system", true).
 		withAddons("aws-load-balancer-controller.addons.k8s.io-k8s-1.19",
 			"certmanager.io-k8s-1.16",
@@ -798,7 +802,6 @@ func TestManyAddons(t *testing.T) {
 			dnsControllerAddon,
 			awsCCMAddon,
 		).
-		withNTH().
 		runTestTerraformAWS(t)
 }
 
@@ -808,9 +811,9 @@ func TestManyAddonsCCMIRSA(t *testing.T) {
 		withServiceAccountRole("dns-controller.kube-system", true).
 		withServiceAccountRole("aws-load-balancer-controller.kube-system", true).
 		withServiceAccountRole("aws-cloud-controller-manager.kube-system", true).
+		withServiceAccountRole("aws-node-termination-handler.kube-system", true).
 		withServiceAccountRole("cluster-autoscaler.kube-system", true).
 		withServiceAccountRole("ebs-csi-controller-sa.kube-system", true).
-		withServiceAccountRole("aws-node-termination-handler.kube-system", true).
 		withAddons(
 			"aws-ebs-csi-driver.addons.k8s.io-k8s-1.17",
 			"aws-load-balancer-controller.addons.k8s.io-k8s-1.19",
@@ -822,7 +825,6 @@ func TestManyAddonsCCMIRSA(t *testing.T) {
 			metricsServerAddon,
 			dnsControllerAddon,
 		).
-		withNTH().
 		runTestTerraformAWS(t)
 }
 
@@ -847,7 +849,6 @@ func TestManyAddonsCCMIRSA23(t *testing.T) {
 			metricsServerAddon,
 			dnsControllerAddon,
 		).
-		withNTH().
 		runTestTerraformAWS(t)
 }
 
@@ -872,7 +873,6 @@ func TestManyAddonsCCMIRSA24(t *testing.T) {
 			metricsServerAddon,
 			dnsControllerAddon,
 		).
-		withNTH().
 		runTestTerraformAWS(t)
 }
 
@@ -897,7 +897,6 @@ func TestManyAddonsCCMIRSA25(t *testing.T) {
 			metricsServerAddon,
 			dnsControllerAddon,
 		).
-		withNTH().
 		runTestTerraformAWS(t)
 }
 
@@ -923,7 +922,6 @@ func TestManyAddonsCCMIRSA26(t *testing.T) {
 			metricsServerAddon,
 			dnsControllerAddon,
 		).
-		withNTH().
 		runTestTerraformAWS(t)
 }
 
@@ -954,7 +952,6 @@ func TestCCM(t *testing.T) {
 			dnsControllerAddon,
 			metricsServerAddon,
 		).
-		withNTH().
 		withNTHRebalance().
 		runTestTerraformAWS(t)
 }
@@ -978,6 +975,7 @@ func TestExternalDNSIRSA(t *testing.T) {
 			"external-dns.addons.k8s.io-k8s-1.19",
 		).
 		withServiceAccountRole("aws-cloud-controller-manager.kube-system", true).
+		withServiceAccountRole("aws-node-termination-handler.kube-system", true).
 		withServiceAccountRole("ebs-csi-controller-sa.kube-system", true).
 		withServiceAccountRole("external-dns.kube-system", true).
 		runTestTerraformAWS(t)
@@ -988,6 +986,7 @@ func TestKarpenter(t *testing.T) {
 		withOIDCDiscovery().
 		withDefaults24().
 		withAddons("karpenter.sh-k8s-1.19").
+		withServiceAccountRole("aws-node-termination-handler.kube-system", true).
 		withServiceAccountRole("karpenter.kube-system", true)
 	test.expectTerraformFilenames = append(test.expectTerraformFilenames,
 		"aws_launch_template_karpenter-nodes-single-machinetype.minimal.example.com_user_data",
@@ -1164,6 +1163,7 @@ func TestNTHIMDSProcessor(t *testing.T) {
 			awsCCMAddon,
 			"node-termination-handler.aws-k8s-1.11",
 		).
+		withoutNTH().
 		runTestTerraformAWS(t)
 }
 
@@ -1181,6 +1181,7 @@ func TestNTHIMDSProcessorIRSA(t *testing.T) {
 			awsCCMAddon,
 			"node-termination-handler.aws-k8s-1.11",
 		).
+		withoutNTH().
 		runTestTerraformAWS(t)
 }
 
@@ -1620,6 +1621,7 @@ func (i *integrationTest) runTestTerraformAWS(t *testing.T) {
 					"aws_launch_template_bastion."+i.clusterName+"_user_data")
 			}
 		}
+	}
 	if i.nth {
 		expectedFilenames = append(expectedFilenames, []string{
 			"aws_s3_object_" + i.clusterName + "-addons-node-termination-handler.aws-k8s-1.11_content",
@@ -1633,7 +1635,6 @@ func (i *integrationTest) runTestTerraformAWS(t *testing.T) {
 	if i.nthRebalance {
 		expectedFilenames = append(expectedFilenames, "aws_cloudwatch_event_rule_"+awsup.GetClusterName40(i.clusterName)+"-RebalanceRecommendation_event_pattern")
 	}
-	}
 	expectedFilenames = append(expectedFilenames, i.expectServiceAccountRolePolicies...)
 
 	i.runTest(t, ctx, h, expectedFilenames, "", "", nil)

docs/releases/1.28-NOTES.md

@@ -8,6 +8,8 @@ This is a document to gather the release notes prior to the release.
 
 ## AWS
 
+* Node Termination Handler is now enabled by default.
+
 ## GCP
 
 ## Openstack

pkg/model/components/nodeterminationhandler.go

@@ -32,9 +32,12 @@ var _ loader.OptionsBuilder = &NodeTerminationHandlerOptionsBuilder{}
 
 func (b *NodeTerminationHandlerOptionsBuilder) BuildOptions(o interface{}) error {
 	clusterSpec := o.(*kops.ClusterSpec)
-	if clusterSpec.CloudProvider.AWS == nil || clusterSpec.CloudProvider.AWS.NodeTerminationHandler == nil {
+	if clusterSpec.CloudProvider.AWS == nil {
 		return nil
 	}
+	if clusterSpec.CloudProvider.AWS.NodeTerminationHandler == nil {
+		clusterSpec.CloudProvider.AWS.NodeTerminationHandler = &kops.NodeTerminationHandlerSpec{}
+	}
 	nth := clusterSpec.CloudProvider.AWS.NodeTerminationHandler
 	if nth.Enabled == nil {
 		nth.Enabled = fi.PtrTo(true)

tests/integration/update_cluster/additionalobjects/data/aws_cloudwatch_event_rule_additionalobjects.example.com-ASGLifecycle_event_pattern

@@ -0,0 +1 @@
+{"source":["aws.autoscaling"],"detail-type":["EC2 Instance-terminate Lifecycle Action"]}

tests/integration/update_cluster/additionalobjects/data/aws_cloudwatch_event_rule_additionalobjects.example.com-InstanceScheduledChange_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.health"],"detail-type": ["AWS Health Event"],"detail": {"service": ["EC2"],"eventTypeCategory": ["scheduledChange"]}}

tests/integration/update_cluster/additionalobjects/data/aws_cloudwatch_event_rule_additionalobjects.example.com-InstanceStateChange_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Instance State-change Notification"]}

tests/integration/update_cluster/additionalobjects/data/aws_cloudwatch_event_rule_additionalobjects.example.com-SpotInterruption_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Spot Instance Interruption Warning"]}

tests/integration/update_cluster/additionalobjects/data/aws_iam_role_policy_masters.additionalobjects.example.com_policy

@@ -215,13 +215,16 @@
         "iam:GetServerCertificate",
         "iam:ListServerCertificates",
         "kms:DescribeKey",
-        "kms:GenerateRandom"
+        "kms:GenerateRandom",
+        "sqs:DeleteMessage",
+        "sqs:ReceiveMessage"
       ],
       "Effect": "Allow",
       "Resource": "*"
     },
     {
       "Action": [
+        "autoscaling:CompleteLifecycleAction",
         "autoscaling:SetDesiredCapacity",
         "autoscaling:TerminateInstanceInAutoScalingGroup",
         "ec2:AttachVolume",

tests/integration/update_cluster/additionalobjects/data/aws_s3_object_additionalobjects.example.com-addons-bootstrap_content

@@ -46,6 +46,57 @@ spec:
     selector:
       k8s-addon: dns-controller.addons.k8s.io
     version: 9.99.0
+  - id: k8s-1.11
+    manifest: node-termination-handler.aws/k8s-1.11.yaml
+    manifestHash: 85bcd6d9570c8124229981ec9bdd16fac9b4420e3fceca74a8884a25b15fc7a6
+    name: node-termination-handler.aws
+    prune:
+      kinds:
+      - kind: ConfigMap
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: Service
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: ServiceAccount
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: admissionregistration.k8s.io
+        kind: MutatingWebhookConfiguration
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: admissionregistration.k8s.io
+        kind: ValidatingWebhookConfiguration
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: DaemonSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: Deployment
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: apps
+        kind: StatefulSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: policy
+        kind: PodDisruptionBudget
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRole
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: Role
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: RoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+    selector:
+      k8s-addon: node-termination-handler.aws
+    version: 9.99.0
   - id: v1.15.0
     manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
     manifestHash: 4e2cda50cd5048133aad1b5e28becb60f4629d3f9e09c514a2757c27998b4200

tests/integration/update_cluster/additionalobjects/data/aws_s3_object_additionalobjects.example.com-addons-node-termination-handler.aws-k8s-1.11_content

@@ -0,0 +1,283 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+    k8s-app: aws-node-termination-handler
+  name: aws-node-termination-handler
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - list
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - pods/eviction
+  verbs:
+  - create
+- apiGroups:
+  - extensions
+  resources:
+  - daemonsets
+  verbs:
+  - get
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: aws-node-termination-handler
+subjects:
+- kind: ServiceAccount
+  name: aws-node-termination-handler
+  namespace: kube-system
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/component: deployment
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+    k8s-app: aws-node-termination-handler
+  name: aws-node-termination-handler
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: aws-node-termination-handler
+      app.kubernetes.io/name: aws-node-termination-handler
+      kubernetes.io/os: linux
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app.kubernetes.io/component: deployment
+        app.kubernetes.io/instance: aws-node-termination-handler
+        app.kubernetes.io/name: aws-node-termination-handler
+        k8s-app: aws-node-termination-handler
+        kops.k8s.io/managed-by: kops
+        kops.k8s.io/nth-mode: sqs
+        kubernetes.io/os: linux
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
+      containers:
+      - env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: ENABLE_PROBES_SERVER
+          value: "true"
+        - name: PROBES_SERVER_PORT
+          value: "8080"
+        - name: PROBES_SERVER_ENDPOINT
+          value: /healthz
+        - name: LOG_LEVEL
+          value: info
+        - name: JSON_LOGGING
+          value: "true"
+        - name: LOG_FORMAT_VERSION
+          value: "2"
+        - name: ENABLE_PROMETHEUS_SERVER
+          value: "false"
+        - name: PROMETHEUS_SERVER_PORT
+          value: "9092"
+        - name: CHECK_TAG_BEFORE_DRAINING
+          value: "true"
+        - name: MANAGED_TAG
+          value: aws-node-termination-handler/managed
+        - name: USE_PROVIDER_ID
+          value: "true"
+        - name: DRY_RUN
+          value: "false"
+        - name: CORDON_ONLY
+          value: "false"
+        - name: TAINT_NODE
+          value: "false"
+        - name: EXCLUDE_FROM_LOAD_BALANCERS
+          value: "true"
+        - name: DELETE_LOCAL_DATA
+          value: "true"
+        - name: IGNORE_DAEMON_SETS
+          value: "true"
+        - name: POD_TERMINATION_GRACE_PERIOD
+          value: "-1"
+        - name: NODE_TERMINATION_GRACE_PERIOD
+          value: "120"
+        - name: EMIT_KUBERNETES_EVENTS
+          value: "true"
+        - name: COMPLETE_LIFECYCLE_ACTION_DELAY_SECONDS
+          value: "-1"
+        - name: ENABLE_SQS_TERMINATION_DRAINING
+          value: "true"
+        - name: QUEUE_URL
+          value: https://sqs.us-test-1.amazonaws.com/123456789012/additionalobjects-example-com-nth
+        - name: WORKERS
+          value: "10"
+        image: public.ecr.aws/aws-ec2/aws-node-termination-handler:v1.18.3
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        name: aws-node-termination-handler
+        ports:
+        - containerPort: 8080
+          name: liveness-probe
+          protocol: TCP
+        - containerPort: 9092
+          name: metrics
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 50m
+            memory: 64Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsGroup: 1000
+          runAsNonRoot: true
+          runAsUser: 1000
+      hostNetwork: true
+      nodeSelector: null
+      priorityClassName: system-cluster-critical
+      securityContext:
+        fsGroup: 1000
+      serviceAccountName: aws-node-termination-handler
+      tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+      topologySpreadConstraints:
+      - labelSelector:
+          matchLabels:
+            app.kubernetes.io/instance: aws-node-termination-handler
+            app.kubernetes.io/name: aws-node-termination-handler
+            kops.k8s.io/nth-mode: sqs
+        maxSkew: 1
+        topologyKey: topology.kubernetes.io/zone
+        whenUnsatisfiable: ScheduleAnyway
+      - labelSelector:
+          matchLabels:
+            app.kubernetes.io/instance: aws-node-termination-handler
+            app.kubernetes.io/name: aws-node-termination-handler
+            kops.k8s.io/nth-mode: sqs
+        maxSkew: 1
+        topologyKey: kubernetes.io/hostname
+        whenUnsatisfiable: DoNotSchedule
+
+---
+
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+  namespace: kube-system
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: aws-node-termination-handler
+      app.kubernetes.io/name: aws-node-termination-handler
+      kops.k8s.io/nth-mode: sqs

tests/integration/update_cluster/additionalobjects/data/aws_s3_object_cluster-completed.spec_content

@@ -195,6 +195,18 @@ spec:
   networkCIDR: 172.20.0.0/16
   networking:
     cni: {}
+  nodeTerminationHandler:
+    cpuRequest: 50m
+    enableRebalanceDraining: false
+    enableRebalanceMonitoring: false
+    enableScheduledEventDraining: true
+    enableSpotInterruptionDraining: true
+    enabled: true
+    excludeFromLoadBalancers: true
+    managedASGTag: aws-node-termination-handler/managed
+    memoryRequest: 64Mi
+    prometheusEnable: false
+    version: v1.18.3
   nonMasqueradeCIDR: 100.64.0.0/10
   podCIDR: 100.96.0.0/11
   secretStore: memfs://tests/additionalobjects.example.com/secrets

tests/integration/update_cluster/additionalobjects/data/aws_sqs_queue_additionalobjects-example-com-nth_policy

@@ -0,0 +1,11 @@
+{
+		"Version": "2012-10-17",
+		"Statement": [{                     
+			"Effect": "Allow",
+			"Principal": {
+				"Service": ["events.amazonaws.com", "sqs.amazonaws.com"]
+			},
+			"Action": "sqs:SendMessage",
+			"Resource": "arn:aws-test:sqs:us-test-1:123456789012:additionalobjects-example-com-nth"
+		}]
+	}

tests/integration/update_cluster/additionalobjects/kubernetes.tf

@@ -117,6 +117,11 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-additionalobjects-ex
     propagate_at_launch = true
     value               = "master-us-test-1a.masters.additionalobjects.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/instancegroup"
     propagate_at_launch = true
@@ -192,6 +197,11 @@ resource "aws_autoscaling_group" "nodes-additionalobjects-example-com" {
     propagate_at_launch = true
     value               = "nodes.additionalobjects.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/instancegroup"
     propagate_at_launch = true
@@ -225,6 +235,82 @@ resource "aws_autoscaling_group" "nodes-additionalobjects-example-com" {
   vpc_zone_identifier = [aws_subnet.us-test-1a-additionalobjects-example-com.id]
 }
 
+resource "aws_autoscaling_lifecycle_hook" "master-us-test-1a-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-additionalobjects-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "master-us-test-1a-NTHLifecycleHook"
+}
+
+resource "aws_autoscaling_lifecycle_hook" "nodes-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.nodes-additionalobjects-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "nodes-NTHLifecycleHook"
+}
+
+resource "aws_cloudwatch_event_rule" "additionalobjects-example-com-ASGLifecycle" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_additionalobjects.example.com-ASGLifecycle_event_pattern")
+  name          = "additionalobjects.example.com-ASGLifecycle"
+  tags = {
+    "KubernetesCluster"                                   = "additionalobjects.example.com"
+    "Name"                                                = "additionalobjects.example.com-ASGLifecycle"
+    "kubernetes.io/cluster/additionalobjects.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "additionalobjects-example-com-InstanceScheduledChange" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_additionalobjects.example.com-InstanceScheduledChange_event_pattern")
+  name          = "additionalobjects.example.com-InstanceScheduledChange"
+  tags = {
+    "KubernetesCluster"                                   = "additionalobjects.example.com"
+    "Name"                                                = "additionalobjects.example.com-InstanceScheduledChange"
+    "kubernetes.io/cluster/additionalobjects.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "additionalobjects-example-com-InstanceStateChange" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_additionalobjects.example.com-InstanceStateChange_event_pattern")
+  name          = "additionalobjects.example.com-InstanceStateChange"
+  tags = {
+    "KubernetesCluster"                                   = "additionalobjects.example.com"
+    "Name"                                                = "additionalobjects.example.com-InstanceStateChange"
+    "kubernetes.io/cluster/additionalobjects.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "additionalobjects-example-com-SpotInterruption" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_additionalobjects.example.com-SpotInterruption_event_pattern")
+  name          = "additionalobjects.example.com-SpotInterruption"
+  tags = {
+    "KubernetesCluster"                                   = "additionalobjects.example.com"
+    "Name"                                                = "additionalobjects.example.com-SpotInterruption"
+    "kubernetes.io/cluster/additionalobjects.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_target" "additionalobjects-example-com-ASGLifecycle-Target" {
+  arn  = aws_sqs_queue.additionalobjects-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.additionalobjects-example-com-ASGLifecycle.id
+}
+
+resource "aws_cloudwatch_event_target" "additionalobjects-example-com-InstanceScheduledChange-Target" {
+  arn  = aws_sqs_queue.additionalobjects-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.additionalobjects-example-com-InstanceScheduledChange.id
+}
+
+resource "aws_cloudwatch_event_target" "additionalobjects-example-com-InstanceStateChange-Target" {
+  arn  = aws_sqs_queue.additionalobjects-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.additionalobjects-example-com-InstanceStateChange.id
+}
+
+resource "aws_cloudwatch_event_target" "additionalobjects-example-com-SpotInterruption-Target" {
+  arn  = aws_sqs_queue.additionalobjects-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.additionalobjects-example-com-SpotInterruption.id
+}
+
 resource "aws_ebs_volume" "a-etcd-events-additionalobjects-example-com" {
   availability_zone = "us-test-1a"
   encrypted         = true
@@ -376,6 +462,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-additionalobjects-exam
     tags = {
       "KubernetesCluster"                                                                                     = "additionalobjects.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.additionalobjects.example.com"
+      "aws-node-termination-handler/managed"                                                                  = ""
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/instancegroup"                               = "master-us-test-1a"
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
       "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
@@ -393,6 +480,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-additionalobjects-exam
     tags = {
       "KubernetesCluster"                                                                                     = "additionalobjects.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.additionalobjects.example.com"
+      "aws-node-termination-handler/managed"                                                                  = ""
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/instancegroup"                               = "master-us-test-1a"
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
       "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
@@ -408,6 +496,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-additionalobjects-exam
   tags = {
     "KubernetesCluster"                                                                                     = "additionalobjects.example.com"
     "Name"                                                                                                  = "master-us-test-1a.masters.additionalobjects.example.com"
+    "aws-node-termination-handler/managed"                                                                  = ""
     "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/instancegroup"                               = "master-us-test-1a"
     "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
     "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
@@ -464,6 +553,7 @@ resource "aws_launch_template" "nodes-additionalobjects-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "additionalobjects.example.com"
       "Name"                                                                       = "nodes.additionalobjects.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/instancegroup"    = "nodes-us-test-1a"
       "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
@@ -477,6 +567,7 @@ resource "aws_launch_template" "nodes-additionalobjects-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "additionalobjects.example.com"
       "Name"                                                                       = "nodes.additionalobjects.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/instancegroup"    = "nodes-us-test-1a"
       "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
@@ -488,6 +579,7 @@ resource "aws_launch_template" "nodes-additionalobjects-example-com" {
   tags = {
     "KubernetesCluster"                                                          = "additionalobjects.example.com"
     "Name"                                                                       = "nodes.additionalobjects.example.com"
+    "aws-node-termination-handler/managed"                                       = ""
     "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/instancegroup"    = "nodes-us-test-1a"
     "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
@@ -589,6 +681,14 @@ resource "aws_s3_object" "additionalobjects-example-com-addons-limit-range-addon
   server_side_encryption = "AES256"
 }
 
+resource "aws_s3_object" "additionalobjects-example-com-addons-node-termination-handler-aws-k8s-1-11" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_additionalobjects.example.com-addons-node-termination-handler.aws-k8s-1.11_content")
+  key                    = "tests/additionalobjects.example.com/addons/node-termination-handler.aws/k8s-1.11.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
 resource "aws_s3_object" "additionalobjects-example-com-addons-storage-aws-addons-k8s-io-v1-15-0" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_object_additionalobjects.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content")
@@ -844,6 +944,17 @@ resource "aws_security_group_rule" "from-nodes-additionalobjects-example-com-ing
   type                     = "ingress"
 }
 
+resource "aws_sqs_queue" "additionalobjects-example-com-nth" {
+  message_retention_seconds = 300
+  name                      = "additionalobjects-example-com-nth"
+  policy                    = file("${path.module}/data/aws_sqs_queue_additionalobjects-example-com-nth_policy")
+  tags = {
+    "KubernetesCluster"                                   = "additionalobjects.example.com"
+    "Name"                                                = "additionalobjects-example-com-nth"
+    "kubernetes.io/cluster/additionalobjects.example.com" = "owned"
+  }
+}
+
 resource "aws_subnet" "us-test-1a-additionalobjects-example-com" {
   availability_zone = "us-test-1a"
   cidr_block        = "172.20.32.0/19"

tests/integration/update_cluster/apiservernodes/data/aws_cloudwatch_event_rule_minimal.example.com-ASGLifecycle_event_pattern

@@ -0,0 +1 @@
+{"source":["aws.autoscaling"],"detail-type":["EC2 Instance-terminate Lifecycle Action"]}

tests/integration/update_cluster/apiservernodes/data/aws_cloudwatch_event_rule_minimal.example.com-InstanceScheduledChange_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.health"],"detail-type": ["AWS Health Event"],"detail": {"service": ["EC2"],"eventTypeCategory": ["scheduledChange"]}}

tests/integration/update_cluster/apiservernodes/data/aws_cloudwatch_event_rule_minimal.example.com-InstanceStateChange_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Instance State-change Notification"]}

tests/integration/update_cluster/apiservernodes/data/aws_cloudwatch_event_rule_minimal.example.com-SpotInterruption_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Spot Instance Interruption Warning"]}

tests/integration/update_cluster/apiservernodes/data/aws_iam_role_policy_masters.minimal.example.com_policy

@@ -192,13 +192,16 @@
         "iam:GetServerCertificate",
         "iam:ListServerCertificates",
         "kms:DescribeKey",
-        "kms:GenerateRandom"
+        "kms:GenerateRandom",
+        "sqs:DeleteMessage",
+        "sqs:ReceiveMessage"
       ],
       "Effect": "Allow",
       "Resource": "*"
     },
     {
       "Action": [
+        "autoscaling:CompleteLifecycleAction",
         "autoscaling:SetDesiredCapacity",
         "autoscaling:TerminateInstanceInAutoScalingGroup",
         "ec2:AttachVolume",

tests/integration/update_cluster/apiservernodes/data/aws_s3_object_cluster-completed.spec_content

@@ -187,6 +187,18 @@ spec:
   networkCIDR: 172.20.0.0/16
   networking:
     cni: {}
+  nodeTerminationHandler:
+    cpuRequest: 50m
+    enableRebalanceDraining: false
+    enableRebalanceMonitoring: false
+    enableScheduledEventDraining: true
+    enableSpotInterruptionDraining: true
+    enabled: true
+    excludeFromLoadBalancers: true
+    managedASGTag: aws-node-termination-handler/managed
+    memoryRequest: 64Mi
+    prometheusEnable: false
+    version: v1.18.3
   nonMasqueradeCIDR: 100.64.0.0/10
   podCIDR: 100.96.0.0/11
   secretStore: memfs://clusters.example.com/minimal.example.com/secrets

tests/integration/update_cluster/apiservernodes/data/aws_s3_object_minimal.example.com-addons-bootstrap_content

@@ -39,6 +39,57 @@ spec:
     selector:
       k8s-addon: dns-controller.addons.k8s.io
     version: 9.99.0
+  - id: k8s-1.11
+    manifest: node-termination-handler.aws/k8s-1.11.yaml
+    manifestHash: bd6955162f93335d20e8f1215411cb1e8388371f2f4a1ff6e29529175db262f9
+    name: node-termination-handler.aws
+    prune:
+      kinds:
+      - kind: ConfigMap
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: Service
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: ServiceAccount
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: admissionregistration.k8s.io
+        kind: MutatingWebhookConfiguration
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: admissionregistration.k8s.io
+        kind: ValidatingWebhookConfiguration
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: DaemonSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: Deployment
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: apps
+        kind: StatefulSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: policy
+        kind: PodDisruptionBudget
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRole
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: Role
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: RoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+    selector:
+      k8s-addon: node-termination-handler.aws
+    version: 9.99.0
   - id: v1.15.0
     manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
     manifestHash: 4e2cda50cd5048133aad1b5e28becb60f4629d3f9e09c514a2757c27998b4200

tests/integration/update_cluster/apiservernodes/data/aws_s3_object_minimal.example.com-addons-node-termination-handler.aws-k8s-1.11_content

@@ -0,0 +1,283 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+    k8s-app: aws-node-termination-handler
+  name: aws-node-termination-handler
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - list
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - pods/eviction
+  verbs:
+  - create
+- apiGroups:
+  - extensions
+  resources:
+  - daemonsets
+  verbs:
+  - get
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: aws-node-termination-handler
+subjects:
+- kind: ServiceAccount
+  name: aws-node-termination-handler
+  namespace: kube-system
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/component: deployment
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+    k8s-app: aws-node-termination-handler
+  name: aws-node-termination-handler
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: aws-node-termination-handler
+      app.kubernetes.io/name: aws-node-termination-handler
+      kubernetes.io/os: linux
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app.kubernetes.io/component: deployment
+        app.kubernetes.io/instance: aws-node-termination-handler
+        app.kubernetes.io/name: aws-node-termination-handler
+        k8s-app: aws-node-termination-handler
+        kops.k8s.io/managed-by: kops
+        kops.k8s.io/nth-mode: sqs
+        kubernetes.io/os: linux
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
+      containers:
+      - env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: ENABLE_PROBES_SERVER
+          value: "true"
+        - name: PROBES_SERVER_PORT
+          value: "8080"
+        - name: PROBES_SERVER_ENDPOINT
+          value: /healthz
+        - name: LOG_LEVEL
+          value: info
+        - name: JSON_LOGGING
+          value: "true"
+        - name: LOG_FORMAT_VERSION
+          value: "2"
+        - name: ENABLE_PROMETHEUS_SERVER
+          value: "false"
+        - name: PROMETHEUS_SERVER_PORT
+          value: "9092"
+        - name: CHECK_TAG_BEFORE_DRAINING
+          value: "true"
+        - name: MANAGED_TAG
+          value: aws-node-termination-handler/managed
+        - name: USE_PROVIDER_ID
+          value: "true"
+        - name: DRY_RUN
+          value: "false"
+        - name: CORDON_ONLY
+          value: "false"
+        - name: TAINT_NODE
+          value: "false"
+        - name: EXCLUDE_FROM_LOAD_BALANCERS
+          value: "true"
+        - name: DELETE_LOCAL_DATA
+          value: "true"
+        - name: IGNORE_DAEMON_SETS
+          value: "true"
+        - name: POD_TERMINATION_GRACE_PERIOD
+          value: "-1"
+        - name: NODE_TERMINATION_GRACE_PERIOD
+          value: "120"
+        - name: EMIT_KUBERNETES_EVENTS
+          value: "true"
+        - name: COMPLETE_LIFECYCLE_ACTION_DELAY_SECONDS
+          value: "-1"
+        - name: ENABLE_SQS_TERMINATION_DRAINING
+          value: "true"
+        - name: QUEUE_URL
+          value: https://sqs.us-test-1.amazonaws.com/123456789012/minimal-example-com-nth
+        - name: WORKERS
+          value: "10"
+        image: public.ecr.aws/aws-ec2/aws-node-termination-handler:v1.18.3
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        name: aws-node-termination-handler
+        ports:
+        - containerPort: 8080
+          name: liveness-probe
+          protocol: TCP
+        - containerPort: 9092
+          name: metrics
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 50m
+            memory: 64Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsGroup: 1000
+          runAsNonRoot: true
+          runAsUser: 1000
+      hostNetwork: true
+      nodeSelector: null
+      priorityClassName: system-cluster-critical
+      securityContext:
+        fsGroup: 1000
+      serviceAccountName: aws-node-termination-handler
+      tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+      topologySpreadConstraints:
+      - labelSelector:
+          matchLabels:
+            app.kubernetes.io/instance: aws-node-termination-handler
+            app.kubernetes.io/name: aws-node-termination-handler
+            kops.k8s.io/nth-mode: sqs
+        maxSkew: 1
+        topologyKey: topology.kubernetes.io/zone
+        whenUnsatisfiable: ScheduleAnyway
+      - labelSelector:
+          matchLabels:
+            app.kubernetes.io/instance: aws-node-termination-handler
+            app.kubernetes.io/name: aws-node-termination-handler
+            kops.k8s.io/nth-mode: sqs
+        maxSkew: 1
+        topologyKey: kubernetes.io/hostname
+        whenUnsatisfiable: DoNotSchedule
+
+---
+
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+  namespace: kube-system
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: aws-node-termination-handler
+      app.kubernetes.io/name: aws-node-termination-handler
+      kops.k8s.io/nth-mode: sqs

tests/integration/update_cluster/apiservernodes/data/aws_sqs_queue_minimal-example-com-nth_policy

@@ -0,0 +1,11 @@
+{
+		"Version": "2012-10-17",
+		"Statement": [{                     
+			"Effect": "Allow",
+			"Principal": {
+				"Service": ["events.amazonaws.com", "sqs.amazonaws.com"]
+			},
+			"Action": "sqs:SendMessage",
+			"Resource": "arn:aws-test:sqs:us-test-1:123456789012:minimal-example-com-nth"
+		}]
+	}

tests/integration/update_cluster/apiservernodes/kubernetes.tf

@@ -137,6 +137,11 @@ resource "aws_autoscaling_group" "apiserver-apiservers-minimal-example-com" {
     propagate_at_launch = true
     value               = "apiserver.apiservers.minimal.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/api-server"
     propagate_at_launch = true
@@ -182,6 +187,11 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com"
     propagate_at_launch = true
     value               = "master-us-test-1a.masters.minimal.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"
     propagate_at_launch = true
@@ -247,6 +257,11 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" {
     propagate_at_launch = true
     value               = "nodes.minimal.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
     propagate_at_launch = true
@@ -270,6 +285,90 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" {
   vpc_zone_identifier = [aws_subnet.us-test-1a-minimal-example-com.id]
 }
 
+resource "aws_autoscaling_lifecycle_hook" "apiserver-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.apiserver-apiservers-minimal-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "apiserver-NTHLifecycleHook"
+}
+
+resource "aws_autoscaling_lifecycle_hook" "master-us-test-1a-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "master-us-test-1a-NTHLifecycleHook"
+}
+
+resource "aws_autoscaling_lifecycle_hook" "nodes-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.nodes-minimal-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "nodes-NTHLifecycleHook"
+}
+
+resource "aws_cloudwatch_event_rule" "minimal-example-com-ASGLifecycle" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_minimal.example.com-ASGLifecycle_event_pattern")
+  name          = "minimal.example.com-ASGLifecycle"
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "minimal.example.com-ASGLifecycle"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "minimal-example-com-InstanceScheduledChange" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_minimal.example.com-InstanceScheduledChange_event_pattern")
+  name          = "minimal.example.com-InstanceScheduledChange"
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "minimal.example.com-InstanceScheduledChange"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "minimal-example-com-InstanceStateChange" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_minimal.example.com-InstanceStateChange_event_pattern")
+  name          = "minimal.example.com-InstanceStateChange"
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "minimal.example.com-InstanceStateChange"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "minimal-example-com-SpotInterruption" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_minimal.example.com-SpotInterruption_event_pattern")
+  name          = "minimal.example.com-SpotInterruption"
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "minimal.example.com-SpotInterruption"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_target" "minimal-example-com-ASGLifecycle-Target" {
+  arn  = aws_sqs_queue.minimal-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.minimal-example-com-ASGLifecycle.id
+}
+
+resource "aws_cloudwatch_event_target" "minimal-example-com-InstanceScheduledChange-Target" {
+  arn  = aws_sqs_queue.minimal-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.minimal-example-com-InstanceScheduledChange.id
+}
+
+resource "aws_cloudwatch_event_target" "minimal-example-com-InstanceStateChange-Target" {
+  arn  = aws_sqs_queue.minimal-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.minimal-example-com-InstanceStateChange.id
+}
+
+resource "aws_cloudwatch_event_target" "minimal-example-com-SpotInterruption-Target" {
+  arn  = aws_sqs_queue.minimal-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.minimal-example-com-SpotInterruption.id
+}
+
 resource "aws_ebs_volume" "us-test-1a-etcd-events-minimal-example-com" {
   availability_zone = "us-test-1a"
   encrypted         = false
@@ -443,6 +542,7 @@ resource "aws_launch_template" "apiserver-apiservers-minimal-example-com" {
     tags = {
       "KubernetesCluster"                                                                = "minimal.example.com"
       "Name"                                                                             = "apiserver.apiservers.minimal.example.com"
+      "aws-node-termination-handler/managed"                                             = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/api-server" = ""
       "k8s.io/role/apiserver"                                                            = "1"
       "kops.k8s.io/instancegroup"                                                        = "apiserver"
@@ -454,6 +554,7 @@ resource "aws_launch_template" "apiserver-apiservers-minimal-example-com" {
     tags = {
       "KubernetesCluster"                                                                = "minimal.example.com"
       "Name"                                                                             = "apiserver.apiservers.minimal.example.com"
+      "aws-node-termination-handler/managed"                                             = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/api-server" = ""
       "k8s.io/role/apiserver"                                                            = "1"
       "kops.k8s.io/instancegroup"                                                        = "apiserver"
@@ -463,6 +564,7 @@ resource "aws_launch_template" "apiserver-apiservers-minimal-example-com" {
   tags = {
     "KubernetesCluster"                                                                = "minimal.example.com"
     "Name"                                                                             = "apiserver.apiservers.minimal.example.com"
+    "aws-node-termination-handler/managed"                                             = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/api-server" = ""
     "k8s.io/role/apiserver"                                                            = "1"
     "kops.k8s.io/instancegroup"                                                        = "apiserver"
@@ -517,6 +619,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" {
     tags = {
       "KubernetesCluster"                                                                                     = "minimal.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.minimal.example.com"
+      "aws-node-termination-handler/managed"                                                                  = ""
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/api-server"                      = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
@@ -532,6 +635,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" {
     tags = {
       "KubernetesCluster"                                                                                     = "minimal.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.minimal.example.com"
+      "aws-node-termination-handler/managed"                                                                  = ""
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/api-server"                      = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
@@ -545,6 +649,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" {
   tags = {
     "KubernetesCluster"                                                                                     = "minimal.example.com"
     "Name"                                                                                                  = "master-us-test-1a.masters.minimal.example.com"
+    "aws-node-termination-handler/managed"                                                                  = ""
     "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/api-server"                      = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
@@ -599,6 +704,7 @@ resource "aws_launch_template" "nodes-minimal-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "minimal.example.com"
       "Name"                                                                       = "nodes.minimal.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -610,6 +716,7 @@ resource "aws_launch_template" "nodes-minimal-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "minimal.example.com"
       "Name"                                                                       = "nodes.minimal.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -619,6 +726,7 @@ resource "aws_launch_template" "nodes-minimal-example-com" {
   tags = {
     "KubernetesCluster"                                                          = "minimal.example.com"
     "Name"                                                                       = "nodes.minimal.example.com"
+    "aws-node-termination-handler/managed"                                       = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/node"                                                           = "1"
     "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -774,6 +882,14 @@ resource "aws_s3_object" "minimal-example-com-addons-limit-range-addons-k8s-io"
   server_side_encryption = "AES256"
 }
 
+resource "aws_s3_object" "minimal-example-com-addons-node-termination-handler-aws-k8s-1-11" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-node-termination-handler.aws-k8s-1.11_content")
+  key                    = "clusters.example.com/minimal.example.com/addons/node-termination-handler.aws/k8s-1.11.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
 resource "aws_s3_object" "minimal-example-com-addons-storage-aws-addons-k8s-io-v1-15-0" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content")
@@ -954,6 +1070,17 @@ resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-udp-1
   type                     = "ingress"
 }
 
+resource "aws_sqs_queue" "minimal-example-com-nth" {
+  message_retention_seconds = 300
+  name                      = "minimal-example-com-nth"
+  policy                    = file("${path.module}/data/aws_sqs_queue_minimal-example-com-nth_policy")
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "minimal-example-com-nth"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
 resource "aws_subnet" "us-test-1a-minimal-example-com" {
   availability_zone                           = "us-test-1a"
   cidr_block                                  = "172.20.32.0/19"

tests/integration/update_cluster/aws-lb-controller/data/aws_cloudwatch_event_rule_minimal.example.com-ASGLifecycle_event_pattern

@@ -0,0 +1 @@
+{"source":["aws.autoscaling"],"detail-type":["EC2 Instance-terminate Lifecycle Action"]}

tests/integration/update_cluster/aws-lb-controller/data/aws_cloudwatch_event_rule_minimal.example.com-InstanceScheduledChange_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.health"],"detail-type": ["AWS Health Event"],"detail": {"service": ["EC2"],"eventTypeCategory": ["scheduledChange"]}}

tests/integration/update_cluster/aws-lb-controller/data/aws_cloudwatch_event_rule_minimal.example.com-InstanceStateChange_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Instance State-change Notification"]}

tests/integration/update_cluster/aws-lb-controller/data/aws_cloudwatch_event_rule_minimal.example.com-SpotInterruption_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Spot Instance Interruption Warning"]}

tests/integration/update_cluster/aws-lb-controller/data/aws_iam_role_aws-node-termination-handler.kube-system.sa.minimal.example.com_policy

@@ -0,0 +1,17 @@
+{
+  "Statement": [
+    {
+      "Action": "sts:AssumeRoleWithWebIdentity",
+      "Condition": {
+        "StringEquals": {
+          "discovery.example.com/minimal.example.com:sub": "system:serviceaccount:kube-system:aws-node-termination-handler"
+        }
+      },
+      "Effect": "Allow",
+      "Principal": {
+        "Federated": "arn:aws-test:iam::123456789012:oidc-provider/discovery.example.com/minimal.example.com"
+      }
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/aws-lb-controller/data/aws_iam_role_policy_aws-node-termination-handler.kube-system.sa.minimal.example.com_policy

@@ -0,0 +1,26 @@
+{
+  "Statement": [
+    {
+      "Action": [
+        "autoscaling:DescribeAutoScalingInstances",
+        "autoscaling:DescribeTags",
+        "ec2:DescribeInstances",
+        "sqs:DeleteMessage",
+        "sqs:ReceiveMessage"
+      ],
+      "Effect": "Allow",
+      "Resource": "*"
+    },
+    {
+      "Action": "autoscaling:CompleteLifecycleAction",
+      "Condition": {
+        "StringEquals": {
+          "aws:ResourceTag/KubernetesCluster": "minimal.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": "*"
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/aws-lb-controller/data/aws_s3_object_cluster-completed.spec_content

@@ -197,6 +197,18 @@ spec:
   networkCIDR: 172.20.0.0/16
   networking:
     cni: {}
+  nodeTerminationHandler:
+    cpuRequest: 50m
+    enableRebalanceDraining: false
+    enableRebalanceMonitoring: false
+    enableScheduledEventDraining: true
+    enableSpotInterruptionDraining: true
+    enabled: true
+    excludeFromLoadBalancers: true
+    managedASGTag: aws-node-termination-handler/managed
+    memoryRequest: 64Mi
+    prometheusEnable: false
+    version: v1.18.3
   nonMasqueradeCIDR: 100.64.0.0/10
   podCIDR: 100.96.0.0/11
   secretStore: memfs://clusters.example.com/minimal.example.com/secrets

tests/integration/update_cluster/aws-lb-controller/data/aws_s3_object_minimal.example.com-addons-bootstrap_content

@@ -95,6 +95,57 @@ spec:
         - kube-system
     selector: null
     version: 9.99.0
+  - id: k8s-1.11
+    manifest: node-termination-handler.aws/k8s-1.11.yaml
+    manifestHash: 7035f67688131cea8befafa5b345137fd67adb4ea8d722b3cd5672e6d3540375
+    name: node-termination-handler.aws
+    prune:
+      kinds:
+      - kind: ConfigMap
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: Service
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: ServiceAccount
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: admissionregistration.k8s.io
+        kind: MutatingWebhookConfiguration
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: admissionregistration.k8s.io
+        kind: ValidatingWebhookConfiguration
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: DaemonSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: Deployment
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: apps
+        kind: StatefulSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: policy
+        kind: PodDisruptionBudget
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRole
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: Role
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: RoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+    selector:
+      k8s-addon: node-termination-handler.aws
+    version: 9.99.0
   - id: k8s-1.19
     manifest: aws-load-balancer-controller.addons.k8s.io/k8s-1.19.yaml
     manifestHash: 2ea476c06cd69a04a9b0da9d5d77a20876e247e9e6a4888743c126f39e325bf8

tests/integration/update_cluster/aws-lb-controller/data/aws_s3_object_minimal.example.com-addons-node-termination-handler.aws-k8s-1.11_content

@@ -0,0 +1,283 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+    k8s-app: aws-node-termination-handler
+  name: aws-node-termination-handler
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - list
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - pods/eviction
+  verbs:
+  - create
+- apiGroups:
+  - extensions
+  resources:
+  - daemonsets
+  verbs:
+  - get
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: aws-node-termination-handler
+subjects:
+- kind: ServiceAccount
+  name: aws-node-termination-handler
+  namespace: kube-system
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/component: deployment
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+    k8s-app: aws-node-termination-handler
+  name: aws-node-termination-handler
+  namespace: kube-system
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: aws-node-termination-handler
+      app.kubernetes.io/name: aws-node-termination-handler
+      kubernetes.io/os: linux
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app.kubernetes.io/component: deployment
+        app.kubernetes.io/instance: aws-node-termination-handler
+        app.kubernetes.io/name: aws-node-termination-handler
+        k8s-app: aws-node-termination-handler
+        kops.k8s.io/managed-by: kops
+        kops.k8s.io/nth-mode: sqs
+        kubernetes.io/os: linux
+    spec:
+      containers:
+      - env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: ENABLE_PROBES_SERVER
+          value: "true"
+        - name: PROBES_SERVER_PORT
+          value: "8080"
+        - name: PROBES_SERVER_ENDPOINT
+          value: /healthz
+        - name: LOG_LEVEL
+          value: info
+        - name: JSON_LOGGING
+          value: "true"
+        - name: LOG_FORMAT_VERSION
+          value: "2"
+        - name: ENABLE_PROMETHEUS_SERVER
+          value: "false"
+        - name: PROMETHEUS_SERVER_PORT
+          value: "9092"
+        - name: CHECK_TAG_BEFORE_DRAINING
+          value: "true"
+        - name: MANAGED_TAG
+          value: aws-node-termination-handler/managed
+        - name: USE_PROVIDER_ID
+          value: "true"
+        - name: DRY_RUN
+          value: "false"
+        - name: CORDON_ONLY
+          value: "false"
+        - name: TAINT_NODE
+          value: "false"
+        - name: EXCLUDE_FROM_LOAD_BALANCERS
+          value: "true"
+        - name: DELETE_LOCAL_DATA
+          value: "true"
+        - name: IGNORE_DAEMON_SETS
+          value: "true"
+        - name: POD_TERMINATION_GRACE_PERIOD
+          value: "-1"
+        - name: NODE_TERMINATION_GRACE_PERIOD
+          value: "120"
+        - name: EMIT_KUBERNETES_EVENTS
+          value: "true"
+        - name: COMPLETE_LIFECYCLE_ACTION_DELAY_SECONDS
+          value: "-1"
+        - name: ENABLE_SQS_TERMINATION_DRAINING
+          value: "true"
+        - name: QUEUE_URL
+          value: https://sqs.us-test-1.amazonaws.com/123456789012/minimal-example-com-nth
+        - name: WORKERS
+          value: "10"
+        - name: AWS_ROLE_ARN
+          value: arn:aws-test:iam::123456789012:role/aws-node-termination-handler.kube-system.sa.minimal.example.com
+        - name: AWS_WEB_IDENTITY_TOKEN_FILE
+          value: /var/run/secrets/amazonaws.com/token
+        image: public.ecr.aws/aws-ec2/aws-node-termination-handler:v1.18.3
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        name: aws-node-termination-handler
+        ports:
+        - containerPort: 8080
+          name: liveness-probe
+          protocol: TCP
+        - containerPort: 9092
+          name: metrics
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 50m
+            memory: 64Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsGroup: 1000
+          runAsNonRoot: true
+          runAsUser: 1000
+        volumeMounts:
+        - mountPath: /var/run/secrets/amazonaws.com/
+          name: token-amazonaws-com
+          readOnly: true
+      priorityClassName: system-cluster-critical
+      securityContext:
+        fsGroup: 1000
+      serviceAccountName: aws-node-termination-handler
+      topologySpreadConstraints:
+      - labelSelector:
+          matchLabels:
+            app.kubernetes.io/instance: aws-node-termination-handler
+            app.kubernetes.io/name: aws-node-termination-handler
+            kops.k8s.io/nth-mode: sqs
+        maxSkew: 1
+        topologyKey: topology.kubernetes.io/zone
+        whenUnsatisfiable: ScheduleAnyway
+      - labelSelector:
+          matchLabels:
+            app.kubernetes.io/instance: aws-node-termination-handler
+            app.kubernetes.io/name: aws-node-termination-handler
+            kops.k8s.io/nth-mode: sqs
+        maxSkew: 1
+        topologyKey: kubernetes.io/hostname
+        whenUnsatisfiable: DoNotSchedule
+      volumes:
+      - name: token-amazonaws-com
+        projected:
+          defaultMode: 420
+          sources:
+          - serviceAccountToken:
+              audience: amazonaws.com
+              expirationSeconds: 86400
+              path: token
+
+---
+
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+  namespace: kube-system
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: aws-node-termination-handler
+      app.kubernetes.io/name: aws-node-termination-handler
+      kops.k8s.io/nth-mode: sqs

tests/integration/update_cluster/aws-lb-controller/data/aws_sqs_queue_minimal-example-com-nth_policy

@@ -0,0 +1,11 @@
+{
+		"Version": "2012-10-17",
+		"Statement": [{                     
+			"Effect": "Allow",
+			"Principal": {
+				"Service": ["events.amazonaws.com", "sqs.amazonaws.com"]
+			},
+			"Action": "sqs:SendMessage",
+			"Resource": "arn:aws-test:sqs:us-test-1:123456789012:minimal-example-com-nth"
+		}]
+	}

tests/integration/update_cluster/aws-lb-controller/kubernetes.tf

@@ -6,6 +6,8 @@ locals {
   kube-system-aws-cloud-controller-manager_role_name = aws_iam_role.aws-cloud-controller-manager-kube-system-sa-minimal-example-com.name
   kube-system-aws-load-balancer-controller_role_arn  = aws_iam_role.aws-load-balancer-controller-kube-system-sa-minimal-example-com.arn
   kube-system-aws-load-balancer-controller_role_name = aws_iam_role.aws-load-balancer-controller-kube-system-sa-minimal-example-com.name
+  kube-system-aws-node-termination-handler_role_arn  = aws_iam_role.aws-node-termination-handler-kube-system-sa-minimal-example-com.arn
+  kube-system-aws-node-termination-handler_role_name = aws_iam_role.aws-node-termination-handler-kube-system-sa-minimal-example-com.name
   kube-system-dns-controller_role_arn                = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.arn
   kube-system-dns-controller_role_name               = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.name
   kube-system-ebs-csi-controller-sa_role_arn         = aws_iam_role.ebs-csi-controller-sa-kube-system-sa-minimal-example-com.arn
@@ -56,6 +58,14 @@ output "kube-system-aws-load-balancer-controller_role_name" {
   value = aws_iam_role.aws-load-balancer-controller-kube-system-sa-minimal-example-com.name
 }
 
+output "kube-system-aws-node-termination-handler_role_arn" {
+  value = aws_iam_role.aws-node-termination-handler-kube-system-sa-minimal-example-com.arn
+}
+
+output "kube-system-aws-node-termination-handler_role_name" {
+  value = aws_iam_role.aws-node-termination-handler-kube-system-sa-minimal-example-com.name
+}
+
 output "kube-system-dns-controller_role_arn" {
   value = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.arn
 }
@@ -167,6 +177,11 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com"
     propagate_at_launch = true
     value               = "master-us-test-1a.masters.minimal.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"
     propagate_at_launch = true
@@ -227,6 +242,11 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" {
     propagate_at_launch = true
     value               = "nodes.minimal.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
     propagate_at_launch = true
@@ -250,6 +270,82 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" {
   vpc_zone_identifier = [aws_subnet.us-test-1a-minimal-example-com.id]
 }
 
+resource "aws_autoscaling_lifecycle_hook" "master-us-test-1a-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "master-us-test-1a-NTHLifecycleHook"
+}
+
+resource "aws_autoscaling_lifecycle_hook" "nodes-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.nodes-minimal-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "nodes-NTHLifecycleHook"
+}
+
+resource "aws_cloudwatch_event_rule" "minimal-example-com-ASGLifecycle" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_minimal.example.com-ASGLifecycle_event_pattern")
+  name          = "minimal.example.com-ASGLifecycle"
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "minimal.example.com-ASGLifecycle"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "minimal-example-com-InstanceScheduledChange" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_minimal.example.com-InstanceScheduledChange_event_pattern")
+  name          = "minimal.example.com-InstanceScheduledChange"
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "minimal.example.com-InstanceScheduledChange"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "minimal-example-com-InstanceStateChange" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_minimal.example.com-InstanceStateChange_event_pattern")
+  name          = "minimal.example.com-InstanceStateChange"
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "minimal.example.com-InstanceStateChange"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "minimal-example-com-SpotInterruption" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_minimal.example.com-SpotInterruption_event_pattern")
+  name          = "minimal.example.com-SpotInterruption"
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "minimal.example.com-SpotInterruption"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_target" "minimal-example-com-ASGLifecycle-Target" {
+  arn  = aws_sqs_queue.minimal-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.minimal-example-com-ASGLifecycle.id
+}
+
+resource "aws_cloudwatch_event_target" "minimal-example-com-InstanceScheduledChange-Target" {
+  arn  = aws_sqs_queue.minimal-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.minimal-example-com-InstanceScheduledChange.id
+}
+
+resource "aws_cloudwatch_event_target" "minimal-example-com-InstanceStateChange-Target" {
+  arn  = aws_sqs_queue.minimal-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.minimal-example-com-InstanceStateChange.id
+}
+
+resource "aws_cloudwatch_event_target" "minimal-example-com-SpotInterruption-Target" {
+  arn  = aws_sqs_queue.minimal-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.minimal-example-com-SpotInterruption.id
+}
+
 resource "aws_ebs_volume" "us-test-1a-etcd-events-minimal-example-com" {
   availability_zone = "us-test-1a"
   encrypted         = false
@@ -339,6 +435,18 @@ resource "aws_iam_role" "aws-load-balancer-controller-kube-system-sa-minimal-exa
   }
 }
 
+resource "aws_iam_role" "aws-node-termination-handler-kube-system-sa-minimal-example-com" {
+  assume_role_policy = file("${path.module}/data/aws_iam_role_aws-node-termination-handler.kube-system.sa.minimal.example.com_policy")
+  name               = "aws-node-termination-handler.kube-system.sa.minimal.example.com"
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "aws-node-termination-handler.kube-system.sa.minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+    "service-account.kops.k8s.io/name"          = "aws-node-termination-handler"
+    "service-account.kops.k8s.io/namespace"     = "kube-system"
+  }
+}
+
 resource "aws_iam_role" "dns-controller-kube-system-sa-minimal-example-com" {
   assume_role_policy = file("${path.module}/data/aws_iam_role_dns-controller.kube-system.sa.minimal.example.com_policy")
   name               = "dns-controller.kube-system.sa.minimal.example.com"
@@ -395,6 +503,12 @@ resource "aws_iam_role_policy" "aws-load-balancer-controller-kube-system-sa-mini
   role   = aws_iam_role.aws-load-balancer-controller-kube-system-sa-minimal-example-com.name
 }
 
+resource "aws_iam_role_policy" "aws-node-termination-handler-kube-system-sa-minimal-example-com" {
+  name   = "aws-node-termination-handler.kube-system.sa.minimal.example.com"
+  policy = file("${path.module}/data/aws_iam_role_policy_aws-node-termination-handler.kube-system.sa.minimal.example.com_policy")
+  role   = aws_iam_role.aws-node-termination-handler-kube-system-sa-minimal-example-com.name
+}
+
 resource "aws_iam_role_policy" "dns-controller-kube-system-sa-minimal-example-com" {
   name   = "dns-controller.kube-system.sa.minimal.example.com"
   policy = file("${path.module}/data/aws_iam_role_policy_dns-controller.kube-system.sa.minimal.example.com_policy")
@@ -484,6 +598,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" {
     tags = {
       "KubernetesCluster"                                                                                     = "minimal.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.minimal.example.com"
+      "aws-node-termination-handler/managed"                                                                  = ""
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
       "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
@@ -498,6 +613,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" {
     tags = {
       "KubernetesCluster"                                                                                     = "minimal.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.minimal.example.com"
+      "aws-node-termination-handler/managed"                                                                  = ""
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
       "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
@@ -510,6 +626,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" {
   tags = {
     "KubernetesCluster"                                                                                     = "minimal.example.com"
     "Name"                                                                                                  = "master-us-test-1a.masters.minimal.example.com"
+    "aws-node-termination-handler/managed"                                                                  = ""
     "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
     "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
@@ -563,6 +680,7 @@ resource "aws_launch_template" "nodes-minimal-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "minimal.example.com"
       "Name"                                                                       = "nodes.minimal.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -574,6 +692,7 @@ resource "aws_launch_template" "nodes-minimal-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "minimal.example.com"
       "Name"                                                                       = "nodes.minimal.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -583,6 +702,7 @@ resource "aws_launch_template" "nodes-minimal-example-com" {
   tags = {
     "KubernetesCluster"                                                          = "minimal.example.com"
     "Name"                                                                       = "nodes.minimal.example.com"
+    "aws-node-termination-handler/managed"                                       = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/node"                                                           = "1"
     "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -770,6 +890,14 @@ resource "aws_s3_object" "minimal-example-com-addons-limit-range-addons-k8s-io"
   server_side_encryption = "AES256"
 }
 
+resource "aws_s3_object" "minimal-example-com-addons-node-termination-handler-aws-k8s-1-11" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-node-termination-handler.aws-k8s-1.11_content")
+  key                    = "clusters.example.com/minimal.example.com/addons/node-termination-handler.aws/k8s-1.11.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
 resource "aws_s3_object" "minimal-example-com-addons-storage-aws-addons-k8s-io-v1-15-0" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content")
@@ -942,6 +1070,17 @@ resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-udp-1
   type                     = "ingress"
 }
 
+resource "aws_sqs_queue" "minimal-example-com-nth" {
+  message_retention_seconds = 300
+  name                      = "minimal-example-com-nth"
+  policy                    = file("${path.module}/data/aws_sqs_queue_minimal-example-com-nth_policy")
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "minimal-example-com-nth"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
 resource "aws_subnet" "us-test-1a-minimal-example-com" {
   availability_zone                           = "us-test-1a"
   cidr_block                                  = "172.20.32.0/19"

tests/integration/update_cluster/bastionadditional_user-data/data/aws_cloudwatch_event_rule_bastionuserdata.example.com-ASGLifecycle_event_pattern

@@ -0,0 +1 @@
+{"source":["aws.autoscaling"],"detail-type":["EC2 Instance-terminate Lifecycle Action"]}

tests/integration/update_cluster/bastionadditional_user-data/data/aws_cloudwatch_event_rule_bastionuserdata.example.com-InstanceScheduledChange_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.health"],"detail-type": ["AWS Health Event"],"detail": {"service": ["EC2"],"eventTypeCategory": ["scheduledChange"]}}

tests/integration/update_cluster/bastionadditional_user-data/data/aws_cloudwatch_event_rule_bastionuserdata.example.com-InstanceStateChange_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Instance State-change Notification"]}

tests/integration/update_cluster/bastionadditional_user-data/data/aws_cloudwatch_event_rule_bastionuserdata.example.com-SpotInterruption_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Spot Instance Interruption Warning"]}

tests/integration/update_cluster/bastionadditional_user-data/data/aws_iam_role_policy_masters.bastionuserdata.example.com_policy

@@ -192,13 +192,16 @@
         "iam:GetServerCertificate",
         "iam:ListServerCertificates",
         "kms:DescribeKey",
-        "kms:GenerateRandom"
+        "kms:GenerateRandom",
+        "sqs:DeleteMessage",
+        "sqs:ReceiveMessage"
       ],
       "Effect": "Allow",
       "Resource": "*"
     },
     {
       "Action": [
+        "autoscaling:CompleteLifecycleAction",
         "autoscaling:SetDesiredCapacity",
         "autoscaling:TerminateInstanceInAutoScalingGroup",
         "ec2:AttachVolume",

tests/integration/update_cluster/bastionadditional_user-data/data/aws_s3_object_bastionuserdata.example.com-addons-bootstrap_content

@@ -39,6 +39,57 @@ spec:
     selector:
       k8s-addon: dns-controller.addons.k8s.io
     version: 9.99.0
+  - id: k8s-1.11
+    manifest: node-termination-handler.aws/k8s-1.11.yaml
+    manifestHash: 11eeca8af35742efc852033436383112fa509a1701fdf5b4255a5914b2c2cab3
+    name: node-termination-handler.aws
+    prune:
+      kinds:
+      - kind: ConfigMap
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: Service
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: ServiceAccount
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: admissionregistration.k8s.io
+        kind: MutatingWebhookConfiguration
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: admissionregistration.k8s.io
+        kind: ValidatingWebhookConfiguration
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: DaemonSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: Deployment
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: apps
+        kind: StatefulSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: policy
+        kind: PodDisruptionBudget
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRole
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: Role
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: RoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+    selector:
+      k8s-addon: node-termination-handler.aws
+    version: 9.99.0
   - id: v1.15.0
     manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
     manifestHash: 4e2cda50cd5048133aad1b5e28becb60f4629d3f9e09c514a2757c27998b4200

tests/integration/update_cluster/bastionadditional_user-data/data/aws_s3_object_bastionuserdata.example.com-addons-node-termination-handler.aws-k8s-1.11_content

@@ -0,0 +1,283 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+    k8s-app: aws-node-termination-handler
+  name: aws-node-termination-handler
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - list
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - pods/eviction
+  verbs:
+  - create
+- apiGroups:
+  - extensions
+  resources:
+  - daemonsets
+  verbs:
+  - get
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: aws-node-termination-handler
+subjects:
+- kind: ServiceAccount
+  name: aws-node-termination-handler
+  namespace: kube-system
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/component: deployment
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+    k8s-app: aws-node-termination-handler
+  name: aws-node-termination-handler
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: aws-node-termination-handler
+      app.kubernetes.io/name: aws-node-termination-handler
+      kubernetes.io/os: linux
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app.kubernetes.io/component: deployment
+        app.kubernetes.io/instance: aws-node-termination-handler
+        app.kubernetes.io/name: aws-node-termination-handler
+        k8s-app: aws-node-termination-handler
+        kops.k8s.io/managed-by: kops
+        kops.k8s.io/nth-mode: sqs
+        kubernetes.io/os: linux
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
+      containers:
+      - env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: ENABLE_PROBES_SERVER
+          value: "true"
+        - name: PROBES_SERVER_PORT
+          value: "8080"
+        - name: PROBES_SERVER_ENDPOINT
+          value: /healthz
+        - name: LOG_LEVEL
+          value: info
+        - name: JSON_LOGGING
+          value: "true"
+        - name: LOG_FORMAT_VERSION
+          value: "2"
+        - name: ENABLE_PROMETHEUS_SERVER
+          value: "false"
+        - name: PROMETHEUS_SERVER_PORT
+          value: "9092"
+        - name: CHECK_TAG_BEFORE_DRAINING
+          value: "true"
+        - name: MANAGED_TAG
+          value: aws-node-termination-handler/managed
+        - name: USE_PROVIDER_ID
+          value: "true"
+        - name: DRY_RUN
+          value: "false"
+        - name: CORDON_ONLY
+          value: "false"
+        - name: TAINT_NODE
+          value: "false"
+        - name: EXCLUDE_FROM_LOAD_BALANCERS
+          value: "true"
+        - name: DELETE_LOCAL_DATA
+          value: "true"
+        - name: IGNORE_DAEMON_SETS
+          value: "true"
+        - name: POD_TERMINATION_GRACE_PERIOD
+          value: "-1"
+        - name: NODE_TERMINATION_GRACE_PERIOD
+          value: "120"
+        - name: EMIT_KUBERNETES_EVENTS
+          value: "true"
+        - name: COMPLETE_LIFECYCLE_ACTION_DELAY_SECONDS
+          value: "-1"
+        - name: ENABLE_SQS_TERMINATION_DRAINING
+          value: "true"
+        - name: QUEUE_URL
+          value: https://sqs.us-test-1.amazonaws.com/123456789012/bastionuserdata-example-com-nth
+        - name: WORKERS
+          value: "10"
+        image: public.ecr.aws/aws-ec2/aws-node-termination-handler:v1.18.3
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        name: aws-node-termination-handler
+        ports:
+        - containerPort: 8080
+          name: liveness-probe
+          protocol: TCP
+        - containerPort: 9092
+          name: metrics
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 50m
+            memory: 64Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsGroup: 1000
+          runAsNonRoot: true
+          runAsUser: 1000
+      hostNetwork: true
+      nodeSelector: null
+      priorityClassName: system-cluster-critical
+      securityContext:
+        fsGroup: 1000
+      serviceAccountName: aws-node-termination-handler
+      tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+      topologySpreadConstraints:
+      - labelSelector:
+          matchLabels:
+            app.kubernetes.io/instance: aws-node-termination-handler
+            app.kubernetes.io/name: aws-node-termination-handler
+            kops.k8s.io/nth-mode: sqs
+        maxSkew: 1
+        topologyKey: topology.kubernetes.io/zone
+        whenUnsatisfiable: ScheduleAnyway
+      - labelSelector:
+          matchLabels:
+            app.kubernetes.io/instance: aws-node-termination-handler
+            app.kubernetes.io/name: aws-node-termination-handler
+            kops.k8s.io/nth-mode: sqs
+        maxSkew: 1
+        topologyKey: kubernetes.io/hostname
+        whenUnsatisfiable: DoNotSchedule
+
+---
+
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+  namespace: kube-system
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: aws-node-termination-handler
+      app.kubernetes.io/name: aws-node-termination-handler
+      kops.k8s.io/nth-mode: sqs

tests/integration/update_cluster/bastionadditional_user-data/data/aws_s3_object_cluster-completed.spec_content

@@ -194,6 +194,18 @@ spec:
   networkCIDR: 172.20.0.0/16
   networking:
     cni: {}
+  nodeTerminationHandler:
+    cpuRequest: 50m
+    enableRebalanceDraining: false
+    enableRebalanceMonitoring: false
+    enableScheduledEventDraining: true
+    enableSpotInterruptionDraining: true
+    enabled: true
+    excludeFromLoadBalancers: true
+    managedASGTag: aws-node-termination-handler/managed
+    memoryRequest: 64Mi
+    prometheusEnable: false
+    version: v1.18.3
   nonMasqueradeCIDR: 100.64.0.0/10
   podCIDR: 100.96.0.0/11
   secretStore: memfs://clusters.example.com/bastionuserdata.example.com/secrets

tests/integration/update_cluster/bastionadditional_user-data/data/aws_sqs_queue_bastionuserdata-example-com-nth_policy

@@ -0,0 +1,11 @@
+{
+		"Version": "2012-10-17",
+		"Statement": [{                     
+			"Effect": "Allow",
+			"Principal": {
+				"Service": ["events.amazonaws.com", "sqs.amazonaws.com"]
+			},
+			"Action": "sqs:SendMessage",
+			"Resource": "arn:aws-test:sqs:us-test-1:123456789012:bastionuserdata-example-com-nth"
+		}]
+	}

tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf

@@ -147,6 +147,11 @@ resource "aws_autoscaling_group" "bastion-bastionuserdata-example-com" {
     propagate_at_launch = true
     value               = "bastion.bastionuserdata.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/role/bastion"
     propagate_at_launch = true
@@ -189,6 +194,11 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-bastionuserdata-exam
     propagate_at_launch = true
     value               = "master-us-test-1a.masters.bastionuserdata.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"
     propagate_at_launch = true
@@ -249,6 +259,11 @@ resource "aws_autoscaling_group" "nodes-bastionuserdata-example-com" {
     propagate_at_launch = true
     value               = "nodes.bastionuserdata.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
     propagate_at_launch = true
@@ -272,6 +287,90 @@ resource "aws_autoscaling_group" "nodes-bastionuserdata-example-com" {
   vpc_zone_identifier = [aws_subnet.us-test-1a-bastionuserdata-example-com.id]
 }
 
+resource "aws_autoscaling_lifecycle_hook" "bastion-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.bastion-bastionuserdata-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "bastion-NTHLifecycleHook"
+}
+
+resource "aws_autoscaling_lifecycle_hook" "master-us-test-1a-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-bastionuserdata-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "master-us-test-1a-NTHLifecycleHook"
+}
+
+resource "aws_autoscaling_lifecycle_hook" "nodes-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.nodes-bastionuserdata-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "nodes-NTHLifecycleHook"
+}
+
+resource "aws_cloudwatch_event_rule" "bastionuserdata-example-com-ASGLifecycle" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_bastionuserdata.example.com-ASGLifecycle_event_pattern")
+  name          = "bastionuserdata.example.com-ASGLifecycle"
+  tags = {
+    "KubernetesCluster"                                 = "bastionuserdata.example.com"
+    "Name"                                              = "bastionuserdata.example.com-ASGLifecycle"
+    "kubernetes.io/cluster/bastionuserdata.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "bastionuserdata-example-com-InstanceScheduledChange" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_bastionuserdata.example.com-InstanceScheduledChange_event_pattern")
+  name          = "bastionuserdata.example.com-InstanceScheduledChange"
+  tags = {
+    "KubernetesCluster"                                 = "bastionuserdata.example.com"
+    "Name"                                              = "bastionuserdata.example.com-InstanceScheduledChange"
+    "kubernetes.io/cluster/bastionuserdata.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "bastionuserdata-example-com-InstanceStateChange" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_bastionuserdata.example.com-InstanceStateChange_event_pattern")
+  name          = "bastionuserdata.example.com-InstanceStateChange"
+  tags = {
+    "KubernetesCluster"                                 = "bastionuserdata.example.com"
+    "Name"                                              = "bastionuserdata.example.com-InstanceStateChange"
+    "kubernetes.io/cluster/bastionuserdata.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "bastionuserdata-example-com-SpotInterruption" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_bastionuserdata.example.com-SpotInterruption_event_pattern")
+  name          = "bastionuserdata.example.com-SpotInterruption"
+  tags = {
+    "KubernetesCluster"                                 = "bastionuserdata.example.com"
+    "Name"                                              = "bastionuserdata.example.com-SpotInterruption"
+    "kubernetes.io/cluster/bastionuserdata.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_target" "bastionuserdata-example-com-ASGLifecycle-Target" {
+  arn  = aws_sqs_queue.bastionuserdata-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.bastionuserdata-example-com-ASGLifecycle.id
+}
+
+resource "aws_cloudwatch_event_target" "bastionuserdata-example-com-InstanceScheduledChange-Target" {
+  arn  = aws_sqs_queue.bastionuserdata-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.bastionuserdata-example-com-InstanceScheduledChange.id
+}
+
+resource "aws_cloudwatch_event_target" "bastionuserdata-example-com-InstanceStateChange-Target" {
+  arn  = aws_sqs_queue.bastionuserdata-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.bastionuserdata-example-com-InstanceStateChange.id
+}
+
+resource "aws_cloudwatch_event_target" "bastionuserdata-example-com-SpotInterruption-Target" {
+  arn  = aws_sqs_queue.bastionuserdata-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.bastionuserdata-example-com-SpotInterruption.id
+}
+
 resource "aws_ebs_volume" "us-test-1a-etcd-events-bastionuserdata-example-com" {
   availability_zone = "us-test-1a"
   encrypted         = false
@@ -482,6 +581,7 @@ resource "aws_launch_template" "bastion-bastionuserdata-example-com" {
     tags = {
       "KubernetesCluster"                                 = "bastionuserdata.example.com"
       "Name"                                              = "bastion.bastionuserdata.example.com"
+      "aws-node-termination-handler/managed"              = ""
       "k8s.io/role/bastion"                               = "1"
       "kops.k8s.io/instancegroup"                         = "bastion"
       "kubernetes.io/cluster/bastionuserdata.example.com" = "owned"
@@ -492,6 +592,7 @@ resource "aws_launch_template" "bastion-bastionuserdata-example-com" {
     tags = {
       "KubernetesCluster"                                 = "bastionuserdata.example.com"
       "Name"                                              = "bastion.bastionuserdata.example.com"
+      "aws-node-termination-handler/managed"              = ""
       "k8s.io/role/bastion"                               = "1"
       "kops.k8s.io/instancegroup"                         = "bastion"
       "kubernetes.io/cluster/bastionuserdata.example.com" = "owned"
@@ -500,6 +601,7 @@ resource "aws_launch_template" "bastion-bastionuserdata-example-com" {
   tags = {
     "KubernetesCluster"                                 = "bastionuserdata.example.com"
     "Name"                                              = "bastion.bastionuserdata.example.com"
+    "aws-node-termination-handler/managed"              = ""
     "k8s.io/role/bastion"                               = "1"
     "kops.k8s.io/instancegroup"                         = "bastion"
     "kubernetes.io/cluster/bastionuserdata.example.com" = "owned"
@@ -553,6 +655,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-bastionuserdata-exampl
     tags = {
       "KubernetesCluster"                                                                                     = "bastionuserdata.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.bastionuserdata.example.com"
+      "aws-node-termination-handler/managed"                                                                  = ""
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
       "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
@@ -567,6 +670,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-bastionuserdata-exampl
     tags = {
       "KubernetesCluster"                                                                                     = "bastionuserdata.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.bastionuserdata.example.com"
+      "aws-node-termination-handler/managed"                                                                  = ""
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
       "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
@@ -579,6 +683,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-bastionuserdata-exampl
   tags = {
     "KubernetesCluster"                                                                                     = "bastionuserdata.example.com"
     "Name"                                                                                                  = "master-us-test-1a.masters.bastionuserdata.example.com"
+    "aws-node-termination-handler/managed"                                                                  = ""
     "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
     "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
@@ -632,6 +737,7 @@ resource "aws_launch_template" "nodes-bastionuserdata-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "bastionuserdata.example.com"
       "Name"                                                                       = "nodes.bastionuserdata.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -643,6 +749,7 @@ resource "aws_launch_template" "nodes-bastionuserdata-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "bastionuserdata.example.com"
       "Name"                                                                       = "nodes.bastionuserdata.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -652,6 +759,7 @@ resource "aws_launch_template" "nodes-bastionuserdata-example-com" {
   tags = {
     "KubernetesCluster"                                                          = "bastionuserdata.example.com"
     "Name"                                                                       = "nodes.bastionuserdata.example.com"
+    "aws-node-termination-handler/managed"                                       = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/node"                                                           = "1"
     "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -849,6 +957,14 @@ resource "aws_s3_object" "bastionuserdata-example-com-addons-limit-range-addons-
   server_side_encryption = "AES256"
 }
 
+resource "aws_s3_object" "bastionuserdata-example-com-addons-node-termination-handler-aws-k8s-1-11" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_bastionuserdata.example.com-addons-node-termination-handler.aws-k8s-1.11_content")
+  key                    = "clusters.example.com/bastionuserdata.example.com/addons/node-termination-handler.aws/k8s-1.11.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
 resource "aws_s3_object" "bastionuserdata-example-com-addons-storage-aws-addons-k8s-io-v1-15-0" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_object_bastionuserdata.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content")
@@ -1197,6 +1313,17 @@ resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-172-20-4-0--22" {
   type              = "ingress"
 }
 
+resource "aws_sqs_queue" "bastionuserdata-example-com-nth" {
+  message_retention_seconds = 300
+  name                      = "bastionuserdata-example-com-nth"
+  policy                    = file("${path.module}/data/aws_sqs_queue_bastionuserdata-example-com-nth_policy")
+  tags = {
+    "KubernetesCluster"                                 = "bastionuserdata.example.com"
+    "Name"                                              = "bastionuserdata-example-com-nth"
+    "kubernetes.io/cluster/bastionuserdata.example.com" = "owned"
+  }
+}
+
 resource "aws_subnet" "us-test-1a-bastionuserdata-example-com" {
   availability_zone                           = "us-test-1a"
   cidr_block                                  = "172.20.32.0/19"

tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_cloudwatch_event_rule_cas-priority-expander-custom.example.com-ASGLifecycle_event_pattern

@@ -0,0 +1 @@
+{"source":["aws.autoscaling"],"detail-type":["EC2 Instance-terminate Lifecycle Action"]}

tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_cloudwatch_event_rule_cas-priority-expander-custom.example.com-InstanceScheduledChange_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.health"],"detail-type": ["AWS Health Event"],"detail": {"service": ["EC2"],"eventTypeCategory": ["scheduledChange"]}}

tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_cloudwatch_event_rule_cas-priority-expander-custom.example.com-InstanceStateChange_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Instance State-change Notification"]}

tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_cloudwatch_event_rule_cas-priority-expander-custom.example.com-SpotInterruption_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Spot Instance Interruption Warning"]}

tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_iam_role_policy_masters.cas-priority-expander-custom.example.com_policy

@@ -192,13 +192,16 @@
         "iam:GetServerCertificate",
         "iam:ListServerCertificates",
         "kms:DescribeKey",
-        "kms:GenerateRandom"
+        "kms:GenerateRandom",
+        "sqs:DeleteMessage",
+        "sqs:ReceiveMessage"
       ],
       "Effect": "Allow",
       "Resource": "*"
     },
     {
       "Action": [
+        "autoscaling:CompleteLifecycleAction",
         "autoscaling:SetDesiredCapacity",
         "autoscaling:TerminateInstanceInAutoScalingGroup",
         "ec2:AttachVolume",

tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_s3_object_cas-priority-expander-custom.example.com-addons-bootstrap_content

@@ -46,6 +46,57 @@ spec:
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io
     version: 9.99.0
+  - id: k8s-1.11
+    manifest: node-termination-handler.aws/k8s-1.11.yaml
+    manifestHash: eadb4337be84c9287c574c2f1759b58d0050cb414487e92b204ceba9ab2b72e0
+    name: node-termination-handler.aws
+    prune:
+      kinds:
+      - kind: ConfigMap
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: Service
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: ServiceAccount
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: admissionregistration.k8s.io
+        kind: MutatingWebhookConfiguration
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: admissionregistration.k8s.io
+        kind: ValidatingWebhookConfiguration
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: DaemonSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: Deployment
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: apps
+        kind: StatefulSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: policy
+        kind: PodDisruptionBudget
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRole
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: Role
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: RoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+    selector:
+      k8s-addon: node-termination-handler.aws
+    version: 9.99.0
   - id: v1.15.0
     manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
     manifestHash: 4e2cda50cd5048133aad1b5e28becb60f4629d3f9e09c514a2757c27998b4200

tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_s3_object_cas-priority-expander-custom.example.com-addons-node-termination-handler.aws-k8s-1.11_content

@@ -0,0 +1,283 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+    k8s-app: aws-node-termination-handler
+  name: aws-node-termination-handler
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - list
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - pods/eviction
+  verbs:
+  - create
+- apiGroups:
+  - extensions
+  resources:
+  - daemonsets
+  verbs:
+  - get
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: aws-node-termination-handler
+subjects:
+- kind: ServiceAccount
+  name: aws-node-termination-handler
+  namespace: kube-system
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/component: deployment
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+    k8s-app: aws-node-termination-handler
+  name: aws-node-termination-handler
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: aws-node-termination-handler
+      app.kubernetes.io/name: aws-node-termination-handler
+      kubernetes.io/os: linux
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app.kubernetes.io/component: deployment
+        app.kubernetes.io/instance: aws-node-termination-handler
+        app.kubernetes.io/name: aws-node-termination-handler
+        k8s-app: aws-node-termination-handler
+        kops.k8s.io/managed-by: kops
+        kops.k8s.io/nth-mode: sqs
+        kubernetes.io/os: linux
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
+      containers:
+      - env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: ENABLE_PROBES_SERVER
+          value: "true"
+        - name: PROBES_SERVER_PORT
+          value: "8080"
+        - name: PROBES_SERVER_ENDPOINT
+          value: /healthz
+        - name: LOG_LEVEL
+          value: info
+        - name: JSON_LOGGING
+          value: "true"
+        - name: LOG_FORMAT_VERSION
+          value: "2"
+        - name: ENABLE_PROMETHEUS_SERVER
+          value: "false"
+        - name: PROMETHEUS_SERVER_PORT
+          value: "9092"
+        - name: CHECK_TAG_BEFORE_DRAINING
+          value: "true"
+        - name: MANAGED_TAG
+          value: aws-node-termination-handler/managed
+        - name: USE_PROVIDER_ID
+          value: "true"
+        - name: DRY_RUN
+          value: "false"
+        - name: CORDON_ONLY
+          value: "false"
+        - name: TAINT_NODE
+          value: "false"
+        - name: EXCLUDE_FROM_LOAD_BALANCERS
+          value: "true"
+        - name: DELETE_LOCAL_DATA
+          value: "true"
+        - name: IGNORE_DAEMON_SETS
+          value: "true"
+        - name: POD_TERMINATION_GRACE_PERIOD
+          value: "-1"
+        - name: NODE_TERMINATION_GRACE_PERIOD
+          value: "120"
+        - name: EMIT_KUBERNETES_EVENTS
+          value: "true"
+        - name: COMPLETE_LIFECYCLE_ACTION_DELAY_SECONDS
+          value: "-1"
+        - name: ENABLE_SQS_TERMINATION_DRAINING
+          value: "true"
+        - name: QUEUE_URL
+          value: https://sqs.us-test-1.amazonaws.com/123456789012/cas-priority-expander-custom-example-com-nth
+        - name: WORKERS
+          value: "10"
+        image: public.ecr.aws/aws-ec2/aws-node-termination-handler:v1.18.3
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        name: aws-node-termination-handler
+        ports:
+        - containerPort: 8080
+          name: liveness-probe
+          protocol: TCP
+        - containerPort: 9092
+          name: metrics
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 50m
+            memory: 64Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsGroup: 1000
+          runAsNonRoot: true
+          runAsUser: 1000
+      hostNetwork: true
+      nodeSelector: null
+      priorityClassName: system-cluster-critical
+      securityContext:
+        fsGroup: 1000
+      serviceAccountName: aws-node-termination-handler
+      tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+      topologySpreadConstraints:
+      - labelSelector:
+          matchLabels:
+            app.kubernetes.io/instance: aws-node-termination-handler
+            app.kubernetes.io/name: aws-node-termination-handler
+            kops.k8s.io/nth-mode: sqs
+        maxSkew: 1
+        topologyKey: topology.kubernetes.io/zone
+        whenUnsatisfiable: ScheduleAnyway
+      - labelSelector:
+          matchLabels:
+            app.kubernetes.io/instance: aws-node-termination-handler
+            app.kubernetes.io/name: aws-node-termination-handler
+            kops.k8s.io/nth-mode: sqs
+        maxSkew: 1
+        topologyKey: kubernetes.io/hostname
+        whenUnsatisfiable: DoNotSchedule
+
+---
+
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+  namespace: kube-system
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: aws-node-termination-handler
+      app.kubernetes.io/name: aws-node-termination-handler
+      kops.k8s.io/nth-mode: sqs

tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_s3_object_cluster-completed.spec_content

@@ -212,6 +212,18 @@ spec:
   networkCIDR: 172.20.0.0/16
   networking:
     cni: {}
+  nodeTerminationHandler:
+    cpuRequest: 50m
+    enableRebalanceDraining: false
+    enableRebalanceMonitoring: false
+    enableScheduledEventDraining: true
+    enableSpotInterruptionDraining: true
+    enabled: true
+    excludeFromLoadBalancers: true
+    managedASGTag: aws-node-termination-handler/managed
+    memoryRequest: 64Mi
+    prometheusEnable: false
+    version: v1.18.3
   nonMasqueradeCIDR: 100.64.0.0/10
   podCIDR: 100.96.0.0/11
   secretStore: memfs://clusters.example.com/cas-priority-expander-custom.example.com/secrets

tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/data/aws_sqs_queue_cas-priority-expander-custom-example-com-nth_policy

@@ -0,0 +1,11 @@
+{
+		"Version": "2012-10-17",
+		"Statement": [{                     
+			"Effect": "Allow",
+			"Principal": {
+				"Service": ["events.amazonaws.com", "sqs.amazonaws.com"]
+			},
+			"Action": "sqs:SendMessage",
+			"Resource": "arn:aws-test:sqs:us-test-1:123456789012:cas-priority-expander-custom-example-com-nth"
+		}]
+	}

tests/integration/update_cluster/cluster-autoscaler-priority-expander-custom/kubernetes.tf

@@ -117,6 +117,11 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-cas-priority-expande
     propagate_at_launch = true
     value               = "master-us-test-1a.masters.cas-priority-expander-custom.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"
     propagate_at_launch = true
@@ -177,6 +182,11 @@ resource "aws_autoscaling_group" "nodes-cas-priority-expander-custom-example-com
     propagate_at_launch = true
     value               = "nodes.cas-priority-expander-custom.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
     propagate_at_launch = true
@@ -222,6 +232,11 @@ resource "aws_autoscaling_group" "nodes-high-priority-cas-priority-expander-cust
     propagate_at_launch = true
     value               = "nodes-high-priority.cas-priority-expander-custom.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
     propagate_at_launch = true
@@ -267,6 +282,11 @@ resource "aws_autoscaling_group" "nodes-low-priority-cas-priority-expander-custo
     propagate_at_launch = true
     value               = "nodes-low-priority.cas-priority-expander-custom.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
     propagate_at_launch = true
@@ -290,6 +310,98 @@ resource "aws_autoscaling_group" "nodes-low-priority-cas-priority-expander-custo
   vpc_zone_identifier = [aws_subnet.us-test-1a-cas-priority-expander-custom-example-com.id]
 }
 
+resource "aws_autoscaling_lifecycle_hook" "master-us-test-1a-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-cas-priority-expander-custom-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "master-us-test-1a-NTHLifecycleHook"
+}
+
+resource "aws_autoscaling_lifecycle_hook" "nodes-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.nodes-cas-priority-expander-custom-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "nodes-NTHLifecycleHook"
+}
+
+resource "aws_autoscaling_lifecycle_hook" "nodes-high-priority-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.nodes-high-priority-cas-priority-expander-custom-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "nodes-high-priority-NTHLifecycleHook"
+}
+
+resource "aws_autoscaling_lifecycle_hook" "nodes-low-priority-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.nodes-low-priority-cas-priority-expander-custom-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "nodes-low-priority-NTHLifecycleHook"
+}
+
+resource "aws_cloudwatch_event_rule" "cas-priority-expander-custom-example-com-ASGLifecycle" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_cas-priority-expander-custom.example.com-ASGLifecycle_event_pattern")
+  name          = "cas-priority-expander-custom.example.com-ASGLifecycle"
+  tags = {
+    "KubernetesCluster"                                              = "cas-priority-expander-custom.example.com"
+    "Name"                                                           = "cas-priority-expander-custom.example.com-ASGLifecycle"
+    "kubernetes.io/cluster/cas-priority-expander-custom.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "cas-priority-expander-custom-example-com-InstanceScheduledChange" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_cas-priority-expander-custom.example.com-InstanceScheduledChange_event_pattern")
+  name          = "cas-priority-expander-custom.example.com-InstanceScheduledChange"
+  tags = {
+    "KubernetesCluster"                                              = "cas-priority-expander-custom.example.com"
+    "Name"                                                           = "cas-priority-expander-custom.example.com-InstanceScheduledChange"
+    "kubernetes.io/cluster/cas-priority-expander-custom.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "cas-priority-expander-custom-example-com-InstanceStateChange" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_cas-priority-expander-custom.example.com-InstanceStateChange_event_pattern")
+  name          = "cas-priority-expander-custom.example.com-InstanceStateChange"
+  tags = {
+    "KubernetesCluster"                                              = "cas-priority-expander-custom.example.com"
+    "Name"                                                           = "cas-priority-expander-custom.example.com-InstanceStateChange"
+    "kubernetes.io/cluster/cas-priority-expander-custom.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "cas-priority-expander-custom-example-com-SpotInterruption" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_cas-priority-expander-custom.example.com-SpotInterruption_event_pattern")
+  name          = "cas-priority-expander-custom.example.com-SpotInterruption"
+  tags = {
+    "KubernetesCluster"                                              = "cas-priority-expander-custom.example.com"
+    "Name"                                                           = "cas-priority-expander-custom.example.com-SpotInterruption"
+    "kubernetes.io/cluster/cas-priority-expander-custom.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_target" "cas-priority-expander-custom-example-com-ASGLifecycle-Target" {
+  arn  = aws_sqs_queue.cas-priority-expander-custom-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.cas-priority-expander-custom-example-com-ASGLifecycle.id
+}
+
+resource "aws_cloudwatch_event_target" "cas-priority-expander-custom-example-com-InstanceScheduledChange-Target" {
+  arn  = aws_sqs_queue.cas-priority-expander-custom-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.cas-priority-expander-custom-example-com-InstanceScheduledChange.id
+}
+
+resource "aws_cloudwatch_event_target" "cas-priority-expander-custom-example-com-InstanceStateChange-Target" {
+  arn  = aws_sqs_queue.cas-priority-expander-custom-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.cas-priority-expander-custom-example-com-InstanceStateChange.id
+}
+
+resource "aws_cloudwatch_event_target" "cas-priority-expander-custom-example-com-SpotInterruption-Target" {
+  arn  = aws_sqs_queue.cas-priority-expander-custom-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.cas-priority-expander-custom-example-com-SpotInterruption.id
+}
+
 resource "aws_ebs_volume" "us-test-1a-etcd-events-cas-priority-expander-custom-example-com" {
   availability_zone = "us-test-1a"
   encrypted         = false
@@ -441,6 +553,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-cas-priority-expander-
     tags = {
       "KubernetesCluster"                                                                                     = "cas-priority-expander-custom.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.cas-priority-expander-custom.example.com"
+      "aws-node-termination-handler/managed"                                                                  = ""
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
       "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
@@ -455,6 +568,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-cas-priority-expander-
     tags = {
       "KubernetesCluster"                                                                                     = "cas-priority-expander-custom.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.cas-priority-expander-custom.example.com"
+      "aws-node-termination-handler/managed"                                                                  = ""
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
       "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
@@ -467,6 +581,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-cas-priority-expander-
   tags = {
     "KubernetesCluster"                                                                                     = "cas-priority-expander-custom.example.com"
     "Name"                                                                                                  = "master-us-test-1a.masters.cas-priority-expander-custom.example.com"
+    "aws-node-termination-handler/managed"                                                                  = ""
     "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
     "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
@@ -520,6 +635,7 @@ resource "aws_launch_template" "nodes-cas-priority-expander-custom-example-com"
     tags = {
       "KubernetesCluster"                                                          = "cas-priority-expander-custom.example.com"
       "Name"                                                                       = "nodes.cas-priority-expander-custom.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -531,6 +647,7 @@ resource "aws_launch_template" "nodes-cas-priority-expander-custom-example-com"
     tags = {
       "KubernetesCluster"                                                          = "cas-priority-expander-custom.example.com"
       "Name"                                                                       = "nodes.cas-priority-expander-custom.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -540,6 +657,7 @@ resource "aws_launch_template" "nodes-cas-priority-expander-custom-example-com"
   tags = {
     "KubernetesCluster"                                                          = "cas-priority-expander-custom.example.com"
     "Name"                                                                       = "nodes.cas-priority-expander-custom.example.com"
+    "aws-node-termination-handler/managed"                                       = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/node"                                                           = "1"
     "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -590,6 +708,7 @@ resource "aws_launch_template" "nodes-high-priority-cas-priority-expander-custom
     tags = {
       "KubernetesCluster"                                                          = "cas-priority-expander-custom.example.com"
       "Name"                                                                       = "nodes-high-priority.cas-priority-expander-custom.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes-high-priority"
@@ -601,6 +720,7 @@ resource "aws_launch_template" "nodes-high-priority-cas-priority-expander-custom
     tags = {
       "KubernetesCluster"                                                          = "cas-priority-expander-custom.example.com"
       "Name"                                                                       = "nodes-high-priority.cas-priority-expander-custom.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes-high-priority"
@@ -610,6 +730,7 @@ resource "aws_launch_template" "nodes-high-priority-cas-priority-expander-custom
   tags = {
     "KubernetesCluster"                                                          = "cas-priority-expander-custom.example.com"
     "Name"                                                                       = "nodes-high-priority.cas-priority-expander-custom.example.com"
+    "aws-node-termination-handler/managed"                                       = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/node"                                                           = "1"
     "kops.k8s.io/instancegroup"                                                  = "nodes-high-priority"
@@ -660,6 +781,7 @@ resource "aws_launch_template" "nodes-low-priority-cas-priority-expander-custom-
     tags = {
       "KubernetesCluster"                                                          = "cas-priority-expander-custom.example.com"
       "Name"                                                                       = "nodes-low-priority.cas-priority-expander-custom.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes-low-priority"
@@ -671,6 +793,7 @@ resource "aws_launch_template" "nodes-low-priority-cas-priority-expander-custom-
     tags = {
       "KubernetesCluster"                                                          = "cas-priority-expander-custom.example.com"
       "Name"                                                                       = "nodes-low-priority.cas-priority-expander-custom.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes-low-priority"
@@ -680,6 +803,7 @@ resource "aws_launch_template" "nodes-low-priority-cas-priority-expander-custom-
   tags = {
     "KubernetesCluster"                                                          = "cas-priority-expander-custom.example.com"
     "Name"                                                                       = "nodes-low-priority.cas-priority-expander-custom.example.com"
+    "aws-node-termination-handler/managed"                                       = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/node"                                                           = "1"
     "kops.k8s.io/instancegroup"                                                  = "nodes-low-priority"
@@ -787,6 +911,14 @@ resource "aws_s3_object" "cas-priority-expander-custom-example-com-addons-limit-
   server_side_encryption = "AES256"
 }
 
+resource "aws_s3_object" "cas-priority-expander-custom-example-com-addons-node-termination-handler-aws-k8s-1-11" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_cas-priority-expander-custom.example.com-addons-node-termination-handler.aws-k8s-1.11_content")
+  key                    = "clusters.example.com/cas-priority-expander-custom.example.com/addons/node-termination-handler.aws/k8s-1.11.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
 resource "aws_s3_object" "cas-priority-expander-custom-example-com-addons-storage-aws-addons-k8s-io-v1-15-0" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_object_cas-priority-expander-custom.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content")
@@ -1031,6 +1163,17 @@ resource "aws_security_group_rule" "from-nodes-cas-priority-expander-custom-exam
   type                     = "ingress"
 }
 
+resource "aws_sqs_queue" "cas-priority-expander-custom-example-com-nth" {
+  message_retention_seconds = 300
+  name                      = "cas-priority-expander-custom-example-com-nth"
+  policy                    = file("${path.module}/data/aws_sqs_queue_cas-priority-expander-custom-example-com-nth_policy")
+  tags = {
+    "KubernetesCluster"                                              = "cas-priority-expander-custom.example.com"
+    "Name"                                                           = "cas-priority-expander-custom-example-com-nth"
+    "kubernetes.io/cluster/cas-priority-expander-custom.example.com" = "owned"
+  }
+}
+
 resource "aws_subnet" "us-test-1a-cas-priority-expander-custom-example-com" {
   availability_zone                           = "us-test-1a"
   cidr_block                                  = "172.20.32.0/19"

tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_cloudwatch_event_rule_cas-priority-expander.example.com-ASGLifecycle_event_pattern

@@ -0,0 +1 @@
+{"source":["aws.autoscaling"],"detail-type":["EC2 Instance-terminate Lifecycle Action"]}

tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_cloudwatch_event_rule_cas-priority-expander.example.com-InstanceScheduledChange_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.health"],"detail-type": ["AWS Health Event"],"detail": {"service": ["EC2"],"eventTypeCategory": ["scheduledChange"]}}

tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_cloudwatch_event_rule_cas-priority-expander.example.com-InstanceStateChange_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Instance State-change Notification"]}

tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_cloudwatch_event_rule_cas-priority-expander.example.com-SpotInterruption_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Spot Instance Interruption Warning"]}

tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_iam_role_policy_masters.cas-priority-expander.example.com_policy

@@ -192,13 +192,16 @@
         "iam:GetServerCertificate",
         "iam:ListServerCertificates",
         "kms:DescribeKey",
-        "kms:GenerateRandom"
+        "kms:GenerateRandom",
+        "sqs:DeleteMessage",
+        "sqs:ReceiveMessage"
       ],
       "Effect": "Allow",
       "Resource": "*"
     },
     {
       "Action": [
+        "autoscaling:CompleteLifecycleAction",
         "autoscaling:SetDesiredCapacity",
         "autoscaling:TerminateInstanceInAutoScalingGroup",
         "ec2:AttachVolume",

tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_s3_object_cas-priority-expander.example.com-addons-bootstrap_content

@@ -46,6 +46,57 @@ spec:
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io
     version: 9.99.0
+  - id: k8s-1.11
+    manifest: node-termination-handler.aws/k8s-1.11.yaml
+    manifestHash: 6c38974ac71212263abf762f9def698f23cb15bbd080dc65ff24ccd3aa87b621
+    name: node-termination-handler.aws
+    prune:
+      kinds:
+      - kind: ConfigMap
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: Service
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: ServiceAccount
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: admissionregistration.k8s.io
+        kind: MutatingWebhookConfiguration
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: admissionregistration.k8s.io
+        kind: ValidatingWebhookConfiguration
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: DaemonSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: Deployment
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: apps
+        kind: StatefulSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: policy
+        kind: PodDisruptionBudget
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRole
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: Role
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: RoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+    selector:
+      k8s-addon: node-termination-handler.aws
+    version: 9.99.0
   - id: v1.15.0
     manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
     manifestHash: 4e2cda50cd5048133aad1b5e28becb60f4629d3f9e09c514a2757c27998b4200

tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_s3_object_cas-priority-expander.example.com-addons-node-termination-handler.aws-k8s-1.11_content

@@ -0,0 +1,283 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+    k8s-app: aws-node-termination-handler
+  name: aws-node-termination-handler
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - list
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - pods/eviction
+  verbs:
+  - create
+- apiGroups:
+  - extensions
+  resources:
+  - daemonsets
+  verbs:
+  - get
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: aws-node-termination-handler
+subjects:
+- kind: ServiceAccount
+  name: aws-node-termination-handler
+  namespace: kube-system
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/component: deployment
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+    k8s-app: aws-node-termination-handler
+  name: aws-node-termination-handler
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: aws-node-termination-handler
+      app.kubernetes.io/name: aws-node-termination-handler
+      kubernetes.io/os: linux
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app.kubernetes.io/component: deployment
+        app.kubernetes.io/instance: aws-node-termination-handler
+        app.kubernetes.io/name: aws-node-termination-handler
+        k8s-app: aws-node-termination-handler
+        kops.k8s.io/managed-by: kops
+        kops.k8s.io/nth-mode: sqs
+        kubernetes.io/os: linux
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
+      containers:
+      - env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: ENABLE_PROBES_SERVER
+          value: "true"
+        - name: PROBES_SERVER_PORT
+          value: "8080"
+        - name: PROBES_SERVER_ENDPOINT
+          value: /healthz
+        - name: LOG_LEVEL
+          value: info
+        - name: JSON_LOGGING
+          value: "true"
+        - name: LOG_FORMAT_VERSION
+          value: "2"
+        - name: ENABLE_PROMETHEUS_SERVER
+          value: "false"
+        - name: PROMETHEUS_SERVER_PORT
+          value: "9092"
+        - name: CHECK_TAG_BEFORE_DRAINING
+          value: "true"
+        - name: MANAGED_TAG
+          value: aws-node-termination-handler/managed
+        - name: USE_PROVIDER_ID
+          value: "true"
+        - name: DRY_RUN
+          value: "false"
+        - name: CORDON_ONLY
+          value: "false"
+        - name: TAINT_NODE
+          value: "false"
+        - name: EXCLUDE_FROM_LOAD_BALANCERS
+          value: "true"
+        - name: DELETE_LOCAL_DATA
+          value: "true"
+        - name: IGNORE_DAEMON_SETS
+          value: "true"
+        - name: POD_TERMINATION_GRACE_PERIOD
+          value: "-1"
+        - name: NODE_TERMINATION_GRACE_PERIOD
+          value: "120"
+        - name: EMIT_KUBERNETES_EVENTS
+          value: "true"
+        - name: COMPLETE_LIFECYCLE_ACTION_DELAY_SECONDS
+          value: "-1"
+        - name: ENABLE_SQS_TERMINATION_DRAINING
+          value: "true"
+        - name: QUEUE_URL
+          value: https://sqs.us-test-1.amazonaws.com/123456789012/cas-priority-expander-example-com-nth
+        - name: WORKERS
+          value: "10"
+        image: public.ecr.aws/aws-ec2/aws-node-termination-handler:v1.18.3
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        name: aws-node-termination-handler
+        ports:
+        - containerPort: 8080
+          name: liveness-probe
+          protocol: TCP
+        - containerPort: 9092
+          name: metrics
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 50m
+            memory: 64Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsGroup: 1000
+          runAsNonRoot: true
+          runAsUser: 1000
+      hostNetwork: true
+      nodeSelector: null
+      priorityClassName: system-cluster-critical
+      securityContext:
+        fsGroup: 1000
+      serviceAccountName: aws-node-termination-handler
+      tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+      topologySpreadConstraints:
+      - labelSelector:
+          matchLabels:
+            app.kubernetes.io/instance: aws-node-termination-handler
+            app.kubernetes.io/name: aws-node-termination-handler
+            kops.k8s.io/nth-mode: sqs
+        maxSkew: 1
+        topologyKey: topology.kubernetes.io/zone
+        whenUnsatisfiable: ScheduleAnyway
+      - labelSelector:
+          matchLabels:
+            app.kubernetes.io/instance: aws-node-termination-handler
+            app.kubernetes.io/name: aws-node-termination-handler
+            kops.k8s.io/nth-mode: sqs
+        maxSkew: 1
+        topologyKey: kubernetes.io/hostname
+        whenUnsatisfiable: DoNotSchedule
+
+---
+
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+  namespace: kube-system
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: aws-node-termination-handler
+      app.kubernetes.io/name: aws-node-termination-handler
+      kops.k8s.io/nth-mode: sqs

tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_s3_object_cluster-completed.spec_content

@@ -205,6 +205,18 @@ spec:
   networkCIDR: 172.20.0.0/16
   networking:
     cni: {}
+  nodeTerminationHandler:
+    cpuRequest: 50m
+    enableRebalanceDraining: false
+    enableRebalanceMonitoring: false
+    enableScheduledEventDraining: true
+    enableSpotInterruptionDraining: true
+    enabled: true
+    excludeFromLoadBalancers: true
+    managedASGTag: aws-node-termination-handler/managed
+    memoryRequest: 64Mi
+    prometheusEnable: false
+    version: v1.18.3
   nonMasqueradeCIDR: 100.64.0.0/10
   podCIDR: 100.96.0.0/11
   secretStore: memfs://clusters.example.com/cas-priority-expander.example.com/secrets

tests/integration/update_cluster/cluster-autoscaler-priority-expander/data/aws_sqs_queue_cas-priority-expander-example-com-nth_policy

@@ -0,0 +1,11 @@
+{
+		"Version": "2012-10-17",
+		"Statement": [{                     
+			"Effect": "Allow",
+			"Principal": {
+				"Service": ["events.amazonaws.com", "sqs.amazonaws.com"]
+			},
+			"Action": "sqs:SendMessage",
+			"Resource": "arn:aws-test:sqs:us-test-1:123456789012:cas-priority-expander-example-com-nth"
+		}]
+	}

tests/integration/update_cluster/cluster-autoscaler-priority-expander/kubernetes.tf

@@ -117,6 +117,11 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-cas-priority-expande
     propagate_at_launch = true
     value               = "master-us-test-1a.masters.cas-priority-expander.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"
     propagate_at_launch = true
@@ -177,6 +182,11 @@ resource "aws_autoscaling_group" "nodes-cas-priority-expander-example-com" {
     propagate_at_launch = true
     value               = "nodes.cas-priority-expander.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
     propagate_at_launch = true
@@ -222,6 +232,11 @@ resource "aws_autoscaling_group" "nodes-high-priority-cas-priority-expander-exam
     propagate_at_launch = true
     value               = "nodes-high-priority.cas-priority-expander.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
     propagate_at_launch = true
@@ -267,6 +282,11 @@ resource "aws_autoscaling_group" "nodes-low-priority-cas-priority-expander-examp
     propagate_at_launch = true
     value               = "nodes-low-priority.cas-priority-expander.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
     propagate_at_launch = true
@@ -290,6 +310,98 @@ resource "aws_autoscaling_group" "nodes-low-priority-cas-priority-expander-examp
   vpc_zone_identifier = [aws_subnet.us-test-1a-cas-priority-expander-example-com.id]
 }
 
+resource "aws_autoscaling_lifecycle_hook" "master-us-test-1a-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-cas-priority-expander-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "master-us-test-1a-NTHLifecycleHook"
+}
+
+resource "aws_autoscaling_lifecycle_hook" "nodes-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.nodes-cas-priority-expander-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "nodes-NTHLifecycleHook"
+}
+
+resource "aws_autoscaling_lifecycle_hook" "nodes-high-priority-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.nodes-high-priority-cas-priority-expander-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "nodes-high-priority-NTHLifecycleHook"
+}
+
+resource "aws_autoscaling_lifecycle_hook" "nodes-low-priority-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.nodes-low-priority-cas-priority-expander-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "nodes-low-priority-NTHLifecycleHook"
+}
+
+resource "aws_cloudwatch_event_rule" "cas-priority-expander-example-com-ASGLifecycle" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_cas-priority-expander.example.com-ASGLifecycle_event_pattern")
+  name          = "cas-priority-expander.example.com-ASGLifecycle"
+  tags = {
+    "KubernetesCluster"                                       = "cas-priority-expander.example.com"
+    "Name"                                                    = "cas-priority-expander.example.com-ASGLifecycle"
+    "kubernetes.io/cluster/cas-priority-expander.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "cas-priority-expander-example-com-InstanceScheduledChange" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_cas-priority-expander.example.com-InstanceScheduledChange_event_pattern")
+  name          = "cas-priority-expander.example.com-InstanceScheduledChange"
+  tags = {
+    "KubernetesCluster"                                       = "cas-priority-expander.example.com"
+    "Name"                                                    = "cas-priority-expander.example.com-InstanceScheduledChange"
+    "kubernetes.io/cluster/cas-priority-expander.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "cas-priority-expander-example-com-InstanceStateChange" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_cas-priority-expander.example.com-InstanceStateChange_event_pattern")
+  name          = "cas-priority-expander.example.com-InstanceStateChange"
+  tags = {
+    "KubernetesCluster"                                       = "cas-priority-expander.example.com"
+    "Name"                                                    = "cas-priority-expander.example.com-InstanceStateChange"
+    "kubernetes.io/cluster/cas-priority-expander.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "cas-priority-expander-example-com-SpotInterruption" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_cas-priority-expander.example.com-SpotInterruption_event_pattern")
+  name          = "cas-priority-expander.example.com-SpotInterruption"
+  tags = {
+    "KubernetesCluster"                                       = "cas-priority-expander.example.com"
+    "Name"                                                    = "cas-priority-expander.example.com-SpotInterruption"
+    "kubernetes.io/cluster/cas-priority-expander.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_target" "cas-priority-expander-example-com-ASGLifecycle-Target" {
+  arn  = aws_sqs_queue.cas-priority-expander-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.cas-priority-expander-example-com-ASGLifecycle.id
+}
+
+resource "aws_cloudwatch_event_target" "cas-priority-expander-example-com-InstanceScheduledChange-Target" {
+  arn  = aws_sqs_queue.cas-priority-expander-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.cas-priority-expander-example-com-InstanceScheduledChange.id
+}
+
+resource "aws_cloudwatch_event_target" "cas-priority-expander-example-com-InstanceStateChange-Target" {
+  arn  = aws_sqs_queue.cas-priority-expander-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.cas-priority-expander-example-com-InstanceStateChange.id
+}
+
+resource "aws_cloudwatch_event_target" "cas-priority-expander-example-com-SpotInterruption-Target" {
+  arn  = aws_sqs_queue.cas-priority-expander-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.cas-priority-expander-example-com-SpotInterruption.id
+}
+
 resource "aws_ebs_volume" "us-test-1a-etcd-events-cas-priority-expander-example-com" {
   availability_zone = "us-test-1a"
   encrypted         = false
@@ -441,6 +553,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-cas-priority-expander-
     tags = {
       "KubernetesCluster"                                                                                     = "cas-priority-expander.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.cas-priority-expander.example.com"
+      "aws-node-termination-handler/managed"                                                                  = ""
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
       "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
@@ -455,6 +568,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-cas-priority-expander-
     tags = {
       "KubernetesCluster"                                                                                     = "cas-priority-expander.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.cas-priority-expander.example.com"
+      "aws-node-termination-handler/managed"                                                                  = ""
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
       "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
@@ -467,6 +581,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-cas-priority-expander-
   tags = {
     "KubernetesCluster"                                                                                     = "cas-priority-expander.example.com"
     "Name"                                                                                                  = "master-us-test-1a.masters.cas-priority-expander.example.com"
+    "aws-node-termination-handler/managed"                                                                  = ""
     "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
     "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
@@ -520,6 +635,7 @@ resource "aws_launch_template" "nodes-cas-priority-expander-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "cas-priority-expander.example.com"
       "Name"                                                                       = "nodes.cas-priority-expander.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -531,6 +647,7 @@ resource "aws_launch_template" "nodes-cas-priority-expander-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "cas-priority-expander.example.com"
       "Name"                                                                       = "nodes.cas-priority-expander.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -540,6 +657,7 @@ resource "aws_launch_template" "nodes-cas-priority-expander-example-com" {
   tags = {
     "KubernetesCluster"                                                          = "cas-priority-expander.example.com"
     "Name"                                                                       = "nodes.cas-priority-expander.example.com"
+    "aws-node-termination-handler/managed"                                       = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/node"                                                           = "1"
     "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -590,6 +708,7 @@ resource "aws_launch_template" "nodes-high-priority-cas-priority-expander-exampl
     tags = {
       "KubernetesCluster"                                                          = "cas-priority-expander.example.com"
       "Name"                                                                       = "nodes-high-priority.cas-priority-expander.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes-high-priority"
@@ -601,6 +720,7 @@ resource "aws_launch_template" "nodes-high-priority-cas-priority-expander-exampl
     tags = {
       "KubernetesCluster"                                                          = "cas-priority-expander.example.com"
       "Name"                                                                       = "nodes-high-priority.cas-priority-expander.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes-high-priority"
@@ -610,6 +730,7 @@ resource "aws_launch_template" "nodes-high-priority-cas-priority-expander-exampl
   tags = {
     "KubernetesCluster"                                                          = "cas-priority-expander.example.com"
     "Name"                                                                       = "nodes-high-priority.cas-priority-expander.example.com"
+    "aws-node-termination-handler/managed"                                       = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/node"                                                           = "1"
     "kops.k8s.io/instancegroup"                                                  = "nodes-high-priority"
@@ -660,6 +781,7 @@ resource "aws_launch_template" "nodes-low-priority-cas-priority-expander-example
     tags = {
       "KubernetesCluster"                                                          = "cas-priority-expander.example.com"
       "Name"                                                                       = "nodes-low-priority.cas-priority-expander.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes-low-priority"
@@ -671,6 +793,7 @@ resource "aws_launch_template" "nodes-low-priority-cas-priority-expander-example
     tags = {
       "KubernetesCluster"                                                          = "cas-priority-expander.example.com"
       "Name"                                                                       = "nodes-low-priority.cas-priority-expander.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes-low-priority"
@@ -680,6 +803,7 @@ resource "aws_launch_template" "nodes-low-priority-cas-priority-expander-example
   tags = {
     "KubernetesCluster"                                                          = "cas-priority-expander.example.com"
     "Name"                                                                       = "nodes-low-priority.cas-priority-expander.example.com"
+    "aws-node-termination-handler/managed"                                       = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/node"                                                           = "1"
     "kops.k8s.io/instancegroup"                                                  = "nodes-low-priority"
@@ -787,6 +911,14 @@ resource "aws_s3_object" "cas-priority-expander-example-com-addons-limit-range-a
   server_side_encryption = "AES256"
 }
 
+resource "aws_s3_object" "cas-priority-expander-example-com-addons-node-termination-handler-aws-k8s-1-11" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_cas-priority-expander.example.com-addons-node-termination-handler.aws-k8s-1.11_content")
+  key                    = "clusters.example.com/cas-priority-expander.example.com/addons/node-termination-handler.aws/k8s-1.11.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
 resource "aws_s3_object" "cas-priority-expander-example-com-addons-storage-aws-addons-k8s-io-v1-15-0" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_object_cas-priority-expander.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content")
@@ -1031,6 +1163,17 @@ resource "aws_security_group_rule" "from-nodes-cas-priority-expander-example-com
   type                     = "ingress"
 }
 
+resource "aws_sqs_queue" "cas-priority-expander-example-com-nth" {
+  message_retention_seconds = 300
+  name                      = "cas-priority-expander-example-com-nth"
+  policy                    = file("${path.module}/data/aws_sqs_queue_cas-priority-expander-example-com-nth_policy")
+  tags = {
+    "KubernetesCluster"                                       = "cas-priority-expander.example.com"
+    "Name"                                                    = "cas-priority-expander-example-com-nth"
+    "kubernetes.io/cluster/cas-priority-expander.example.com" = "owned"
+  }
+}
+
 resource "aws_subnet" "us-test-1a-cas-priority-expander-example-com" {
   availability_zone                           = "us-test-1a"
   cidr_block                                  = "172.20.32.0/19"

tests/integration/update_cluster/complex/data/aws_cloudwatch_event_rule_complex.example.com-ASGLifecycle_event_pattern

@@ -0,0 +1 @@
+{"source":["aws.autoscaling"],"detail-type":["EC2 Instance-terminate Lifecycle Action"]}

tests/integration/update_cluster/complex/data/aws_cloudwatch_event_rule_complex.example.com-InstanceScheduledChange_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.health"],"detail-type": ["AWS Health Event"],"detail": {"service": ["EC2"],"eventTypeCategory": ["scheduledChange"]}}

tests/integration/update_cluster/complex/data/aws_cloudwatch_event_rule_complex.example.com-InstanceStateChange_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Instance State-change Notification"]}

tests/integration/update_cluster/complex/data/aws_cloudwatch_event_rule_complex.example.com-SpotInterruption_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Spot Instance Interruption Warning"]}

tests/integration/update_cluster/complex/data/aws_iam_role_policy_masters.complex.example.com_policy

@@ -192,13 +192,16 @@
         "iam:GetServerCertificate",
         "iam:ListServerCertificates",
         "kms:DescribeKey",
-        "kms:GenerateRandom"
+        "kms:GenerateRandom",
+        "sqs:DeleteMessage",
+        "sqs:ReceiveMessage"
       ],
       "Effect": "Allow",
       "Resource": "*"
     },
     {
       "Action": [
+        "autoscaling:CompleteLifecycleAction",
         "autoscaling:SetDesiredCapacity",
         "autoscaling:TerminateInstanceInAutoScalingGroup",
         "ec2:AttachVolume",

tests/integration/update_cluster/complex/data/aws_s3_object_cluster-completed.spec_content

@@ -225,6 +225,18 @@ spec:
   nodePortAccess:
   - 1.2.3.4/32
   - 10.20.30.0/24
+  nodeTerminationHandler:
+    cpuRequest: 50m
+    enableRebalanceDraining: false
+    enableRebalanceMonitoring: false
+    enableScheduledEventDraining: true
+    enableSpotInterruptionDraining: true
+    enabled: true
+    excludeFromLoadBalancers: true
+    managedASGTag: aws-node-termination-handler/managed
+    memoryRequest: 64Mi
+    prometheusEnable: false
+    version: v1.18.3
   nonMasqueradeCIDR: 100.64.0.0/10
   podCIDR: 100.96.0.0/11
   secretStore: memfs://clusters.example.com/complex.example.com/secrets

tests/integration/update_cluster/complex/data/aws_s3_object_complex.example.com-addons-bootstrap_content

@@ -46,6 +46,57 @@ spec:
     selector:
       k8s-addon: dns-controller.addons.k8s.io
     version: 9.99.0
+  - id: k8s-1.11
+    manifest: node-termination-handler.aws/k8s-1.11.yaml
+    manifestHash: a1f4481c1a51c222571da402ce760831a435acb55565ff3eaa6833743233d3fe
+    name: node-termination-handler.aws
+    prune:
+      kinds:
+      - kind: ConfigMap
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: Service
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: ServiceAccount
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: admissionregistration.k8s.io
+        kind: MutatingWebhookConfiguration
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: admissionregistration.k8s.io
+        kind: ValidatingWebhookConfiguration
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: DaemonSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: Deployment
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: apps
+        kind: StatefulSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: policy
+        kind: PodDisruptionBudget
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRole
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: Role
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: RoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+    selector:
+      k8s-addon: node-termination-handler.aws
+    version: 9.99.0
   - id: v1.15.0
     manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
     manifestHash: 4e2cda50cd5048133aad1b5e28becb60f4629d3f9e09c514a2757c27998b4200

tests/integration/update_cluster/complex/data/aws_s3_object_complex.example.com-addons-node-termination-handler.aws-k8s-1.11_content

@@ -0,0 +1,283 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+    k8s-app: aws-node-termination-handler
+  name: aws-node-termination-handler
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - list
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - pods/eviction
+  verbs:
+  - create
+- apiGroups:
+  - extensions
+  resources:
+  - daemonsets
+  verbs:
+  - get
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: aws-node-termination-handler
+subjects:
+- kind: ServiceAccount
+  name: aws-node-termination-handler
+  namespace: kube-system
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/component: deployment
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+    k8s-app: aws-node-termination-handler
+  name: aws-node-termination-handler
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: aws-node-termination-handler
+      app.kubernetes.io/name: aws-node-termination-handler
+      kubernetes.io/os: linux
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app.kubernetes.io/component: deployment
+        app.kubernetes.io/instance: aws-node-termination-handler
+        app.kubernetes.io/name: aws-node-termination-handler
+        k8s-app: aws-node-termination-handler
+        kops.k8s.io/managed-by: kops
+        kops.k8s.io/nth-mode: sqs
+        kubernetes.io/os: linux
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
+      containers:
+      - env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: ENABLE_PROBES_SERVER
+          value: "true"
+        - name: PROBES_SERVER_PORT
+          value: "8080"
+        - name: PROBES_SERVER_ENDPOINT
+          value: /healthz
+        - name: LOG_LEVEL
+          value: info
+        - name: JSON_LOGGING
+          value: "true"
+        - name: LOG_FORMAT_VERSION
+          value: "2"
+        - name: ENABLE_PROMETHEUS_SERVER
+          value: "false"
+        - name: PROMETHEUS_SERVER_PORT
+          value: "9092"
+        - name: CHECK_TAG_BEFORE_DRAINING
+          value: "true"
+        - name: MANAGED_TAG
+          value: aws-node-termination-handler/managed
+        - name: USE_PROVIDER_ID
+          value: "true"
+        - name: DRY_RUN
+          value: "false"
+        - name: CORDON_ONLY
+          value: "false"
+        - name: TAINT_NODE
+          value: "false"
+        - name: EXCLUDE_FROM_LOAD_BALANCERS
+          value: "true"
+        - name: DELETE_LOCAL_DATA
+          value: "true"
+        - name: IGNORE_DAEMON_SETS
+          value: "true"
+        - name: POD_TERMINATION_GRACE_PERIOD
+          value: "-1"
+        - name: NODE_TERMINATION_GRACE_PERIOD
+          value: "120"
+        - name: EMIT_KUBERNETES_EVENTS
+          value: "true"
+        - name: COMPLETE_LIFECYCLE_ACTION_DELAY_SECONDS
+          value: "-1"
+        - name: ENABLE_SQS_TERMINATION_DRAINING
+          value: "true"
+        - name: QUEUE_URL
+          value: https://sqs.us-test-1.amazonaws.com/123456789012/complex-example-com-nth
+        - name: WORKERS
+          value: "10"
+        image: public.ecr.aws/aws-ec2/aws-node-termination-handler:v1.18.3
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        name: aws-node-termination-handler
+        ports:
+        - containerPort: 8080
+          name: liveness-probe
+          protocol: TCP
+        - containerPort: 9092
+          name: metrics
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 50m
+            memory: 64Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsGroup: 1000
+          runAsNonRoot: true
+          runAsUser: 1000
+      hostNetwork: true
+      nodeSelector: null
+      priorityClassName: system-cluster-critical
+      securityContext:
+        fsGroup: 1000
+      serviceAccountName: aws-node-termination-handler
+      tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+      topologySpreadConstraints:
+      - labelSelector:
+          matchLabels:
+            app.kubernetes.io/instance: aws-node-termination-handler
+            app.kubernetes.io/name: aws-node-termination-handler
+            kops.k8s.io/nth-mode: sqs
+        maxSkew: 1
+        topologyKey: topology.kubernetes.io/zone
+        whenUnsatisfiable: ScheduleAnyway
+      - labelSelector:
+          matchLabels:
+            app.kubernetes.io/instance: aws-node-termination-handler
+            app.kubernetes.io/name: aws-node-termination-handler
+            kops.k8s.io/nth-mode: sqs
+        maxSkew: 1
+        topologyKey: kubernetes.io/hostname
+        whenUnsatisfiable: DoNotSchedule
+
+---
+
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+  namespace: kube-system
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: aws-node-termination-handler
+      app.kubernetes.io/name: aws-node-termination-handler
+      kops.k8s.io/nth-mode: sqs

tests/integration/update_cluster/complex/data/aws_sqs_queue_complex-example-com-nth_policy

@@ -0,0 +1,11 @@
+{
+		"Version": "2012-10-17",
+		"Statement": [{                     
+			"Effect": "Allow",
+			"Principal": {
+				"Service": ["events.amazonaws.com", "sqs.amazonaws.com"]
+			},
+			"Action": "sqs:SendMessage",
+			"Resource": "arn:aws-test:sqs:us-test-1:123456789012:complex-example-com-nth"
+		}]
+	}

tests/integration/update_cluster/complex/kubernetes.tf

@@ -140,6 +140,11 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-complex-example-com"
     propagate_at_launch = true
     value               = "John Doe"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "foo/bar"
     propagate_at_launch = true
@@ -213,6 +218,11 @@ resource "aws_autoscaling_group" "nodes-complex-example-com" {
     propagate_at_launch = true
     value               = "John Doe"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "foo/bar"
     propagate_at_launch = true
@@ -241,6 +251,90 @@ resource "aws_autoscaling_group" "nodes-complex-example-com" {
   vpc_zone_identifier = [aws_subnet.us-test-1a-complex-example-com.id]
 }
 
+resource "aws_autoscaling_lifecycle_hook" "master-us-test-1a-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-complex-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "master-us-test-1a-NTHLifecycleHook"
+}
+
+resource "aws_autoscaling_lifecycle_hook" "nodes-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.nodes-complex-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "nodes-NTHLifecycleHook"
+}
+
+resource "aws_cloudwatch_event_rule" "complex-example-com-ASGLifecycle" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_complex.example.com-ASGLifecycle_event_pattern")
+  name          = "complex.example.com-ASGLifecycle"
+  tags = {
+    "KubernetesCluster"                         = "complex.example.com"
+    "Name"                                      = "complex.example.com-ASGLifecycle"
+    "Owner"                                     = "John Doe"
+    "foo/bar"                                   = "fib+baz"
+    "kubernetes.io/cluster/complex.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "complex-example-com-InstanceScheduledChange" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_complex.example.com-InstanceScheduledChange_event_pattern")
+  name          = "complex.example.com-InstanceScheduledChange"
+  tags = {
+    "KubernetesCluster"                         = "complex.example.com"
+    "Name"                                      = "complex.example.com-InstanceScheduledChange"
+    "Owner"                                     = "John Doe"
+    "foo/bar"                                   = "fib+baz"
+    "kubernetes.io/cluster/complex.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "complex-example-com-InstanceStateChange" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_complex.example.com-InstanceStateChange_event_pattern")
+  name          = "complex.example.com-InstanceStateChange"
+  tags = {
+    "KubernetesCluster"                         = "complex.example.com"
+    "Name"                                      = "complex.example.com-InstanceStateChange"
+    "Owner"                                     = "John Doe"
+    "foo/bar"                                   = "fib+baz"
+    "kubernetes.io/cluster/complex.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "complex-example-com-SpotInterruption" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_complex.example.com-SpotInterruption_event_pattern")
+  name          = "complex.example.com-SpotInterruption"
+  tags = {
+    "KubernetesCluster"                         = "complex.example.com"
+    "Name"                                      = "complex.example.com-SpotInterruption"
+    "Owner"                                     = "John Doe"
+    "foo/bar"                                   = "fib+baz"
+    "kubernetes.io/cluster/complex.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_target" "complex-example-com-ASGLifecycle-Target" {
+  arn  = aws_sqs_queue.complex-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.complex-example-com-ASGLifecycle.id
+}
+
+resource "aws_cloudwatch_event_target" "complex-example-com-InstanceScheduledChange-Target" {
+  arn  = aws_sqs_queue.complex-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.complex-example-com-InstanceScheduledChange.id
+}
+
+resource "aws_cloudwatch_event_target" "complex-example-com-InstanceStateChange-Target" {
+  arn  = aws_sqs_queue.complex-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.complex-example-com-InstanceStateChange.id
+}
+
+resource "aws_cloudwatch_event_target" "complex-example-com-SpotInterruption-Target" {
+  arn  = aws_sqs_queue.complex-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.complex-example-com-SpotInterruption.id
+}
+
 resource "aws_ebs_volume" "a-etcd-events-complex-example-com" {
   availability_zone = "us-test-1a"
   encrypted         = false
@@ -399,6 +493,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-complex-example-com" {
       "KubernetesCluster"                                                                                     = "complex.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.complex.example.com"
       "Owner"                                                                                                 = "John Doe"
+      "aws-node-termination-handler/managed"                                                                  = ""
       "foo/bar"                                                                                               = "fib+baz"
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
@@ -415,6 +510,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-complex-example-com" {
       "KubernetesCluster"                                                                                     = "complex.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.complex.example.com"
       "Owner"                                                                                                 = "John Doe"
+      "aws-node-termination-handler/managed"                                                                  = ""
       "foo/bar"                                                                                               = "fib+baz"
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
@@ -429,6 +525,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-complex-example-com" {
     "KubernetesCluster"                                                                                     = "complex.example.com"
     "Name"                                                                                                  = "master-us-test-1a.masters.complex.example.com"
     "Owner"                                                                                                 = "John Doe"
+    "aws-node-termination-handler/managed"                                                                  = ""
     "foo/bar"                                                                                               = "fib+baz"
     "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
@@ -496,6 +593,7 @@ resource "aws_launch_template" "nodes-complex-example-com" {
       "KubernetesCluster"                                                          = "complex.example.com"
       "Name"                                                                       = "nodes.complex.example.com"
       "Owner"                                                                      = "John Doe"
+      "aws-node-termination-handler/managed"                                       = ""
       "foo/bar"                                                                    = "fib+baz"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
@@ -509,6 +607,7 @@ resource "aws_launch_template" "nodes-complex-example-com" {
       "KubernetesCluster"                                                          = "complex.example.com"
       "Name"                                                                       = "nodes.complex.example.com"
       "Owner"                                                                      = "John Doe"
+      "aws-node-termination-handler/managed"                                       = ""
       "foo/bar"                                                                    = "fib+baz"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
@@ -520,6 +619,7 @@ resource "aws_launch_template" "nodes-complex-example-com" {
     "KubernetesCluster"                                                          = "complex.example.com"
     "Name"                                                                       = "nodes.complex.example.com"
     "Owner"                                                                      = "John Doe"
+    "aws-node-termination-handler/managed"                                       = ""
     "foo/bar"                                                                    = "fib+baz"
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/node"                                                           = "1"
@@ -790,6 +890,14 @@ resource "aws_s3_object" "complex-example-com-addons-limit-range-addons-k8s-io"
   server_side_encryption = "AES256"
 }
 
+resource "aws_s3_object" "complex-example-com-addons-node-termination-handler-aws-k8s-1-11" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_complex.example.com-addons-node-termination-handler.aws-k8s-1.11_content")
+  key                    = "clusters.example.com/complex.example.com/addons/node-termination-handler.aws/k8s-1.11.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
 resource "aws_s3_object" "complex-example-com-addons-storage-aws-addons-k8s-io-v1-15-0" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_object_complex.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content")
@@ -1144,6 +1252,19 @@ resource "aws_security_group_rule" "tcp-api-pl-44444444" {
   type              = "ingress"
 }
 
+resource "aws_sqs_queue" "complex-example-com-nth" {
+  message_retention_seconds = 300
+  name                      = "complex-example-com-nth"
+  policy                    = file("${path.module}/data/aws_sqs_queue_complex-example-com-nth_policy")
+  tags = {
+    "KubernetesCluster"                         = "complex.example.com"
+    "Name"                                      = "complex-example-com-nth"
+    "Owner"                                     = "John Doe"
+    "foo/bar"                                   = "fib+baz"
+    "kubernetes.io/cluster/complex.example.com" = "owned"
+  }
+}
+
 resource "aws_subnet" "us-east-1a-private-complex-example-com" {
   availability_zone                           = "us-test-1a"
   cidr_block                                  = "10.1.64.0/19"

tests/integration/update_cluster/compress/data/aws_cloudwatch_event_rule_compress.example.com-ASGLifecycle_event_pattern

@@ -0,0 +1 @@
+{"source":["aws.autoscaling"],"detail-type":["EC2 Instance-terminate Lifecycle Action"]}

tests/integration/update_cluster/compress/data/aws_cloudwatch_event_rule_compress.example.com-InstanceScheduledChange_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.health"],"detail-type": ["AWS Health Event"],"detail": {"service": ["EC2"],"eventTypeCategory": ["scheduledChange"]}}

tests/integration/update_cluster/compress/data/aws_cloudwatch_event_rule_compress.example.com-InstanceStateChange_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Instance State-change Notification"]}

tests/integration/update_cluster/compress/data/aws_cloudwatch_event_rule_compress.example.com-SpotInterruption_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Spot Instance Interruption Warning"]}

tests/integration/update_cluster/compress/data/aws_iam_role_policy_masters.compress.example.com_policy

@@ -192,13 +192,16 @@
         "iam:GetServerCertificate",
         "iam:ListServerCertificates",
         "kms:DescribeKey",
-        "kms:GenerateRandom"
+        "kms:GenerateRandom",
+        "sqs:DeleteMessage",
+        "sqs:ReceiveMessage"
       ],
       "Effect": "Allow",
       "Resource": "*"
     },
     {
       "Action": [
+        "autoscaling:CompleteLifecycleAction",
         "autoscaling:SetDesiredCapacity",
         "autoscaling:TerminateInstanceInAutoScalingGroup",
         "ec2:AttachVolume",

tests/integration/update_cluster/compress/data/aws_s3_object_cluster-completed.spec_content

@@ -192,6 +192,18 @@ spec:
   networkCIDR: 172.20.0.0/16
   networking:
     cni: {}
+  nodeTerminationHandler:
+    cpuRequest: 50m
+    enableRebalanceDraining: false
+    enableRebalanceMonitoring: false
+    enableScheduledEventDraining: true
+    enableSpotInterruptionDraining: true
+    enabled: true
+    excludeFromLoadBalancers: true
+    managedASGTag: aws-node-termination-handler/managed
+    memoryRequest: 64Mi
+    prometheusEnable: false
+    version: v1.18.3
   nonMasqueradeCIDR: 100.64.0.0/10
   podCIDR: 100.96.0.0/11
   secretStore: memfs://clusters.example.com/compress.example.com/secrets

tests/integration/update_cluster/compress/data/aws_s3_object_compress.example.com-addons-bootstrap_content

@@ -39,6 +39,57 @@ spec:
     selector:
       k8s-addon: dns-controller.addons.k8s.io
     version: 9.99.0
+  - id: k8s-1.11
+    manifest: node-termination-handler.aws/k8s-1.11.yaml
+    manifestHash: 8a6fa7b6d7e58a9787fe977560ac5a665e6378e88db2fa9ecacad9af02afbe30
+    name: node-termination-handler.aws
+    prune:
+      kinds:
+      - kind: ConfigMap
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: Service
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: ServiceAccount
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: admissionregistration.k8s.io
+        kind: MutatingWebhookConfiguration
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: admissionregistration.k8s.io
+        kind: ValidatingWebhookConfiguration
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: DaemonSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: Deployment
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: apps
+        kind: StatefulSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: policy
+        kind: PodDisruptionBudget
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRole
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: Role
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: RoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+    selector:
+      k8s-addon: node-termination-handler.aws
+    version: 9.99.0
   - id: v1.15.0
     manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
     manifestHash: 4e2cda50cd5048133aad1b5e28becb60f4629d3f9e09c514a2757c27998b4200

tests/integration/update_cluster/compress/data/aws_s3_object_compress.example.com-addons-node-termination-handler.aws-k8s-1.11_content

@@ -0,0 +1,283 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+    k8s-app: aws-node-termination-handler
+  name: aws-node-termination-handler
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - list
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - pods/eviction
+  verbs:
+  - create
+- apiGroups:
+  - extensions
+  resources:
+  - daemonsets
+  verbs:
+  - get
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: aws-node-termination-handler
+subjects:
+- kind: ServiceAccount
+  name: aws-node-termination-handler
+  namespace: kube-system
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/component: deployment
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+    k8s-app: aws-node-termination-handler
+  name: aws-node-termination-handler
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: aws-node-termination-handler
+      app.kubernetes.io/name: aws-node-termination-handler
+      kubernetes.io/os: linux
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app.kubernetes.io/component: deployment
+        app.kubernetes.io/instance: aws-node-termination-handler
+        app.kubernetes.io/name: aws-node-termination-handler
+        k8s-app: aws-node-termination-handler
+        kops.k8s.io/managed-by: kops
+        kops.k8s.io/nth-mode: sqs
+        kubernetes.io/os: linux
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
+      containers:
+      - env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: ENABLE_PROBES_SERVER
+          value: "true"
+        - name: PROBES_SERVER_PORT
+          value: "8080"
+        - name: PROBES_SERVER_ENDPOINT
+          value: /healthz
+        - name: LOG_LEVEL
+          value: info
+        - name: JSON_LOGGING
+          value: "true"
+        - name: LOG_FORMAT_VERSION
+          value: "2"
+        - name: ENABLE_PROMETHEUS_SERVER
+          value: "false"
+        - name: PROMETHEUS_SERVER_PORT
+          value: "9092"
+        - name: CHECK_TAG_BEFORE_DRAINING
+          value: "true"
+        - name: MANAGED_TAG
+          value: aws-node-termination-handler/managed
+        - name: USE_PROVIDER_ID
+          value: "true"
+        - name: DRY_RUN
+          value: "false"
+        - name: CORDON_ONLY
+          value: "false"
+        - name: TAINT_NODE
+          value: "false"
+        - name: EXCLUDE_FROM_LOAD_BALANCERS
+          value: "true"
+        - name: DELETE_LOCAL_DATA
+          value: "true"
+        - name: IGNORE_DAEMON_SETS
+          value: "true"
+        - name: POD_TERMINATION_GRACE_PERIOD
+          value: "-1"
+        - name: NODE_TERMINATION_GRACE_PERIOD
+          value: "120"
+        - name: EMIT_KUBERNETES_EVENTS
+          value: "true"
+        - name: COMPLETE_LIFECYCLE_ACTION_DELAY_SECONDS
+          value: "-1"
+        - name: ENABLE_SQS_TERMINATION_DRAINING
+          value: "true"
+        - name: QUEUE_URL
+          value: https://sqs.us-test-1.amazonaws.com/123456789012/compress-example-com-nth
+        - name: WORKERS
+          value: "10"
+        image: public.ecr.aws/aws-ec2/aws-node-termination-handler:v1.18.3
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        name: aws-node-termination-handler
+        ports:
+        - containerPort: 8080
+          name: liveness-probe
+          protocol: TCP
+        - containerPort: 9092
+          name: metrics
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 50m
+            memory: 64Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsGroup: 1000
+          runAsNonRoot: true
+          runAsUser: 1000
+      hostNetwork: true
+      nodeSelector: null
+      priorityClassName: system-cluster-critical
+      securityContext:
+        fsGroup: 1000
+      serviceAccountName: aws-node-termination-handler
+      tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+      topologySpreadConstraints:
+      - labelSelector:
+          matchLabels:
+            app.kubernetes.io/instance: aws-node-termination-handler
+            app.kubernetes.io/name: aws-node-termination-handler
+            kops.k8s.io/nth-mode: sqs
+        maxSkew: 1
+        topologyKey: topology.kubernetes.io/zone
+        whenUnsatisfiable: ScheduleAnyway
+      - labelSelector:
+          matchLabels:
+            app.kubernetes.io/instance: aws-node-termination-handler
+            app.kubernetes.io/name: aws-node-termination-handler
+            kops.k8s.io/nth-mode: sqs
+        maxSkew: 1
+        topologyKey: kubernetes.io/hostname
+        whenUnsatisfiable: DoNotSchedule
+
+---
+
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+  namespace: kube-system
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: aws-node-termination-handler
+      app.kubernetes.io/name: aws-node-termination-handler
+      kops.k8s.io/nth-mode: sqs

tests/integration/update_cluster/compress/data/aws_sqs_queue_compress-example-com-nth_policy

@@ -0,0 +1,11 @@
+{
+		"Version": "2012-10-17",
+		"Statement": [{                     
+			"Effect": "Allow",
+			"Principal": {
+				"Service": ["events.amazonaws.com", "sqs.amazonaws.com"]
+			},
+			"Action": "sqs:SendMessage",
+			"Resource": "arn:aws-test:sqs:us-test-1:123456789012:compress-example-com-nth"
+		}]
+	}

tests/integration/update_cluster/compress/kubernetes.tf

@@ -117,6 +117,11 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-compress-example-com
     propagate_at_launch = true
     value               = "master-us-test-1a.masters.compress.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"
     propagate_at_launch = true
@@ -177,6 +182,11 @@ resource "aws_autoscaling_group" "nodes-compress-example-com" {
     propagate_at_launch = true
     value               = "nodes.compress.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
     propagate_at_launch = true
@@ -200,6 +210,82 @@ resource "aws_autoscaling_group" "nodes-compress-example-com" {
   vpc_zone_identifier = [aws_subnet.us-test-1a-compress-example-com.id]
 }
 
+resource "aws_autoscaling_lifecycle_hook" "master-us-test-1a-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-compress-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "master-us-test-1a-NTHLifecycleHook"
+}
+
+resource "aws_autoscaling_lifecycle_hook" "nodes-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.nodes-compress-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "nodes-NTHLifecycleHook"
+}
+
+resource "aws_cloudwatch_event_rule" "compress-example-com-ASGLifecycle" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_compress.example.com-ASGLifecycle_event_pattern")
+  name          = "compress.example.com-ASGLifecycle"
+  tags = {
+    "KubernetesCluster"                          = "compress.example.com"
+    "Name"                                       = "compress.example.com-ASGLifecycle"
+    "kubernetes.io/cluster/compress.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "compress-example-com-InstanceScheduledChange" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_compress.example.com-InstanceScheduledChange_event_pattern")
+  name          = "compress.example.com-InstanceScheduledChange"
+  tags = {
+    "KubernetesCluster"                          = "compress.example.com"
+    "Name"                                       = "compress.example.com-InstanceScheduledChange"
+    "kubernetes.io/cluster/compress.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "compress-example-com-InstanceStateChange" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_compress.example.com-InstanceStateChange_event_pattern")
+  name          = "compress.example.com-InstanceStateChange"
+  tags = {
+    "KubernetesCluster"                          = "compress.example.com"
+    "Name"                                       = "compress.example.com-InstanceStateChange"
+    "kubernetes.io/cluster/compress.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "compress-example-com-SpotInterruption" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_compress.example.com-SpotInterruption_event_pattern")
+  name          = "compress.example.com-SpotInterruption"
+  tags = {
+    "KubernetesCluster"                          = "compress.example.com"
+    "Name"                                       = "compress.example.com-SpotInterruption"
+    "kubernetes.io/cluster/compress.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_target" "compress-example-com-ASGLifecycle-Target" {
+  arn  = aws_sqs_queue.compress-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.compress-example-com-ASGLifecycle.id
+}
+
+resource "aws_cloudwatch_event_target" "compress-example-com-InstanceScheduledChange-Target" {
+  arn  = aws_sqs_queue.compress-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.compress-example-com-InstanceScheduledChange.id
+}
+
+resource "aws_cloudwatch_event_target" "compress-example-com-InstanceStateChange-Target" {
+  arn  = aws_sqs_queue.compress-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.compress-example-com-InstanceStateChange.id
+}
+
+resource "aws_cloudwatch_event_target" "compress-example-com-SpotInterruption-Target" {
+  arn  = aws_sqs_queue.compress-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.compress-example-com-SpotInterruption.id
+}
+
 resource "aws_ebs_volume" "us-test-1a-etcd-events-compress-example-com" {
   availability_zone = "us-test-1a"
   encrypted         = false
@@ -340,6 +426,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-compress-example-com"
     tags = {
       "KubernetesCluster"                                                                                     = "compress.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.compress.example.com"
+      "aws-node-termination-handler/managed"                                                                  = ""
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
       "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
@@ -354,6 +441,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-compress-example-com"
     tags = {
       "KubernetesCluster"                                                                                     = "compress.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.compress.example.com"
+      "aws-node-termination-handler/managed"                                                                  = ""
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
       "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
@@ -366,6 +454,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-compress-example-com"
   tags = {
     "KubernetesCluster"                                                                                     = "compress.example.com"
     "Name"                                                                                                  = "master-us-test-1a.masters.compress.example.com"
+    "aws-node-termination-handler/managed"                                                                  = ""
     "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
     "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
@@ -418,6 +507,7 @@ resource "aws_launch_template" "nodes-compress-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "compress.example.com"
       "Name"                                                                       = "nodes.compress.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -429,6 +519,7 @@ resource "aws_launch_template" "nodes-compress-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "compress.example.com"
       "Name"                                                                       = "nodes.compress.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -438,6 +529,7 @@ resource "aws_launch_template" "nodes-compress-example-com" {
   tags = {
     "KubernetesCluster"                                                          = "compress.example.com"
     "Name"                                                                       = "nodes.compress.example.com"
+    "aws-node-termination-handler/managed"                                       = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/node"                                                           = "1"
     "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -545,6 +637,14 @@ resource "aws_s3_object" "compress-example-com-addons-limit-range-addons-k8s-io"
   server_side_encryption = "AES256"
 }
 
+resource "aws_s3_object" "compress-example-com-addons-node-termination-handler-aws-k8s-1-11" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_compress.example.com-addons-node-termination-handler.aws-k8s-1.11_content")
+  key                    = "clusters.example.com/compress.example.com/addons/node-termination-handler.aws/k8s-1.11.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
 resource "aws_s3_object" "compress-example-com-addons-storage-aws-addons-k8s-io-v1-15-0" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_object_compress.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content")
@@ -765,6 +865,17 @@ resource "aws_security_group_rule" "from-nodes-compress-example-com-ingress-udp-
   type                     = "ingress"
 }
 
+resource "aws_sqs_queue" "compress-example-com-nth" {
+  message_retention_seconds = 300
+  name                      = "compress-example-com-nth"
+  policy                    = file("${path.module}/data/aws_sqs_queue_compress-example-com-nth_policy")
+  tags = {
+    "KubernetesCluster"                          = "compress.example.com"
+    "Name"                                       = "compress-example-com-nth"
+    "kubernetes.io/cluster/compress.example.com" = "owned"
+  }
+}
+
 resource "aws_subnet" "us-test-1a-compress-example-com" {
   availability_zone                           = "us-test-1a"
   cidr_block                                  = "172.20.32.0/19"

tests/integration/update_cluster/containerd-custom/data/aws_cloudwatch_event_rule_containerd.example.com-ASGLifecycle_event_pattern

@@ -0,0 +1 @@
+{"source":["aws.autoscaling"],"detail-type":["EC2 Instance-terminate Lifecycle Action"]}

tests/integration/update_cluster/containerd-custom/data/aws_cloudwatch_event_rule_containerd.example.com-InstanceScheduledChange_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.health"],"detail-type": ["AWS Health Event"],"detail": {"service": ["EC2"],"eventTypeCategory": ["scheduledChange"]}}

tests/integration/update_cluster/containerd-custom/data/aws_cloudwatch_event_rule_containerd.example.com-InstanceStateChange_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Instance State-change Notification"]}

tests/integration/update_cluster/containerd-custom/data/aws_cloudwatch_event_rule_containerd.example.com-SpotInterruption_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Spot Instance Interruption Warning"]}

tests/integration/update_cluster/containerd-custom/data/aws_iam_role_policy_masters.containerd.example.com_policy

@@ -192,13 +192,16 @@
         "iam:GetServerCertificate",
         "iam:ListServerCertificates",
         "kms:DescribeKey",
-        "kms:GenerateRandom"
+        "kms:GenerateRandom",
+        "sqs:DeleteMessage",
+        "sqs:ReceiveMessage"
       ],
       "Effect": "Allow",
       "Resource": "*"
     },
     {
       "Action": [
+        "autoscaling:CompleteLifecycleAction",
         "autoscaling:SetDesiredCapacity",
         "autoscaling:TerminateInstanceInAutoScalingGroup",
         "ec2:AttachVolume",

tests/integration/update_cluster/containerd-custom/data/aws_s3_object_cluster-completed.spec_content

@@ -201,6 +201,18 @@ spec:
   networkCIDR: 172.20.0.0/16
   networking:
     cni: {}
+  nodeTerminationHandler:
+    cpuRequest: 50m
+    enableRebalanceDraining: false
+    enableRebalanceMonitoring: false
+    enableScheduledEventDraining: true
+    enableSpotInterruptionDraining: true
+    enabled: true
+    excludeFromLoadBalancers: true
+    managedASGTag: aws-node-termination-handler/managed
+    memoryRequest: 64Mi
+    prometheusEnable: false
+    version: v1.18.3
   nonMasqueradeCIDR: 100.64.0.0/10
   podCIDR: 100.96.0.0/11
   secretStore: memfs://clusters.example.com/containerd.example.com/secrets

tests/integration/update_cluster/containerd-custom/data/aws_s3_object_containerd.example.com-addons-bootstrap_content

@@ -39,6 +39,57 @@ spec:
     selector:
       k8s-addon: dns-controller.addons.k8s.io
     version: 9.99.0
+  - id: k8s-1.11
+    manifest: node-termination-handler.aws/k8s-1.11.yaml
+    manifestHash: c7b1a78363bdc90729afd95a2a79531de23575917be79a08b53633a21a11d259
+    name: node-termination-handler.aws
+    prune:
+      kinds:
+      - kind: ConfigMap
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: Service
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: ServiceAccount
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: admissionregistration.k8s.io
+        kind: MutatingWebhookConfiguration
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: admissionregistration.k8s.io
+        kind: ValidatingWebhookConfiguration
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: DaemonSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: Deployment
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: apps
+        kind: StatefulSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: policy
+        kind: PodDisruptionBudget
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRole
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: Role
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: RoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+    selector:
+      k8s-addon: node-termination-handler.aws
+    version: 9.99.0
   - id: v1.15.0
     manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
     manifestHash: 4e2cda50cd5048133aad1b5e28becb60f4629d3f9e09c514a2757c27998b4200

tests/integration/update_cluster/containerd-custom/data/aws_s3_object_containerd.example.com-addons-node-termination-handler.aws-k8s-1.11_content

@@ -0,0 +1,283 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+    k8s-app: aws-node-termination-handler
+  name: aws-node-termination-handler
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - list
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - pods/eviction
+  verbs:
+  - create
+- apiGroups:
+  - extensions
+  resources:
+  - daemonsets
+  verbs:
+  - get
+- apiGroups:
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: aws-node-termination-handler
+subjects:
+- kind: ServiceAccount
+  name: aws-node-termination-handler
+  namespace: kube-system
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/component: deployment
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    app.kubernetes.io/part-of: aws-node-termination-handler
+    app.kubernetes.io/version: v1.18.3
+    k8s-addon: node-termination-handler.aws
+    k8s-app: aws-node-termination-handler
+  name: aws-node-termination-handler
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: aws-node-termination-handler
+      app.kubernetes.io/name: aws-node-termination-handler
+      kubernetes.io/os: linux
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app.kubernetes.io/component: deployment
+        app.kubernetes.io/instance: aws-node-termination-handler
+        app.kubernetes.io/name: aws-node-termination-handler
+        k8s-app: aws-node-termination-handler
+        kops.k8s.io/managed-by: kops
+        kops.k8s.io/nth-mode: sqs
+        kubernetes.io/os: linux
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
+      containers:
+      - env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: ENABLE_PROBES_SERVER
+          value: "true"
+        - name: PROBES_SERVER_PORT
+          value: "8080"
+        - name: PROBES_SERVER_ENDPOINT
+          value: /healthz
+        - name: LOG_LEVEL
+          value: info
+        - name: JSON_LOGGING
+          value: "true"
+        - name: LOG_FORMAT_VERSION
+          value: "2"
+        - name: ENABLE_PROMETHEUS_SERVER
+          value: "false"
+        - name: PROMETHEUS_SERVER_PORT
+          value: "9092"
+        - name: CHECK_TAG_BEFORE_DRAINING
+          value: "true"
+        - name: MANAGED_TAG
+          value: aws-node-termination-handler/managed
+        - name: USE_PROVIDER_ID
+          value: "true"
+        - name: DRY_RUN
+          value: "false"
+        - name: CORDON_ONLY
+          value: "false"
+        - name: TAINT_NODE
+          value: "false"
+        - name: EXCLUDE_FROM_LOAD_BALANCERS
+          value: "true"
+        - name: DELETE_LOCAL_DATA
+          value: "true"
+        - name: IGNORE_DAEMON_SETS
+          value: "true"
+        - name: POD_TERMINATION_GRACE_PERIOD
+          value: "-1"
+        - name: NODE_TERMINATION_GRACE_PERIOD
+          value: "120"
+        - name: EMIT_KUBERNETES_EVENTS
+          value: "true"
+        - name: COMPLETE_LIFECYCLE_ACTION_DELAY_SECONDS
+          value: "-1"
+        - name: ENABLE_SQS_TERMINATION_DRAINING
+          value: "true"
+        - name: QUEUE_URL
+          value: https://sqs.us-test-1.amazonaws.com/123456789012/containerd-example-com-nth
+        - name: WORKERS
+          value: "10"
+        image: public.ecr.aws/aws-ec2/aws-node-termination-handler:v1.18.3
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        name: aws-node-termination-handler
+        ports:
+        - containerPort: 8080
+          name: liveness-probe
+          protocol: TCP
+        - containerPort: 9092
+          name: metrics
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 50m
+            memory: 64Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsGroup: 1000
+          runAsNonRoot: true
+          runAsUser: 1000
+      hostNetwork: true
+      nodeSelector: null
+      priorityClassName: system-cluster-critical
+      securityContext:
+        fsGroup: 1000
+      serviceAccountName: aws-node-termination-handler
+      tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+      topologySpreadConstraints:
+      - labelSelector:
+          matchLabels:
+            app.kubernetes.io/instance: aws-node-termination-handler
+            app.kubernetes.io/name: aws-node-termination-handler
+            kops.k8s.io/nth-mode: sqs
+        maxSkew: 1
+        topologyKey: topology.kubernetes.io/zone
+        whenUnsatisfiable: ScheduleAnyway
+      - labelSelector:
+          matchLabels:
+            app.kubernetes.io/instance: aws-node-termination-handler
+            app.kubernetes.io/name: aws-node-termination-handler
+            kops.k8s.io/nth-mode: sqs
+        maxSkew: 1
+        topologyKey: kubernetes.io/hostname
+        whenUnsatisfiable: DoNotSchedule
+
+---
+
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: node-termination-handler.aws
+    app.kubernetes.io/instance: aws-node-termination-handler
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-node-termination-handler
+    k8s-addon: node-termination-handler.aws
+  name: aws-node-termination-handler
+  namespace: kube-system
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: aws-node-termination-handler
+      app.kubernetes.io/name: aws-node-termination-handler
+      kops.k8s.io/nth-mode: sqs

tests/integration/update_cluster/containerd-custom/data/aws_sqs_queue_containerd-example-com-nth_policy

@@ -0,0 +1,11 @@
+{
+		"Version": "2012-10-17",
+		"Statement": [{                     
+			"Effect": "Allow",
+			"Principal": {
+				"Service": ["events.amazonaws.com", "sqs.amazonaws.com"]
+			},
+			"Action": "sqs:SendMessage",
+			"Resource": "arn:aws-test:sqs:us-test-1:123456789012:containerd-example-com-nth"
+		}]
+	}

tests/integration/update_cluster/containerd-custom/kubernetes.tf

@@ -117,6 +117,11 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-containerd-example-c
     propagate_at_launch = true
     value               = "master-us-test-1a.masters.containerd.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"
     propagate_at_launch = true
@@ -177,6 +182,11 @@ resource "aws_autoscaling_group" "nodes-containerd-example-com" {
     propagate_at_launch = true
     value               = "nodes.containerd.example.com"
   }
+  tag {
+    key                 = "aws-node-termination-handler/managed"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
     propagate_at_launch = true
@@ -200,6 +210,82 @@ resource "aws_autoscaling_group" "nodes-containerd-example-com" {
   vpc_zone_identifier = [aws_subnet.us-test-1a-containerd-example-com.id]
 }
 
+resource "aws_autoscaling_lifecycle_hook" "master-us-test-1a-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-containerd-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "master-us-test-1a-NTHLifecycleHook"
+}
+
+resource "aws_autoscaling_lifecycle_hook" "nodes-NTHLifecycleHook" {
+  autoscaling_group_name = aws_autoscaling_group.nodes-containerd-example-com.id
+  default_result         = "CONTINUE"
+  heartbeat_timeout      = 300
+  lifecycle_transition   = "autoscaling:EC2_INSTANCE_TERMINATING"
+  name                   = "nodes-NTHLifecycleHook"
+}
+
+resource "aws_cloudwatch_event_rule" "containerd-example-com-ASGLifecycle" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_containerd.example.com-ASGLifecycle_event_pattern")
+  name          = "containerd.example.com-ASGLifecycle"
+  tags = {
+    "KubernetesCluster"                            = "containerd.example.com"
+    "Name"                                         = "containerd.example.com-ASGLifecycle"
+    "kubernetes.io/cluster/containerd.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "containerd-example-com-InstanceScheduledChange" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_containerd.example.com-InstanceScheduledChange_event_pattern")
+  name          = "containerd.example.com-InstanceScheduledChange"
+  tags = {
+    "KubernetesCluster"                            = "containerd.example.com"
+    "Name"                                         = "containerd.example.com-InstanceScheduledChange"
+    "kubernetes.io/cluster/containerd.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "containerd-example-com-InstanceStateChange" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_containerd.example.com-InstanceStateChange_event_pattern")
+  name          = "containerd.example.com-InstanceStateChange"
+  tags = {
+    "KubernetesCluster"                            = "containerd.example.com"
+    "Name"                                         = "containerd.example.com-InstanceStateChange"
+    "kubernetes.io/cluster/containerd.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_rule" "containerd-example-com-SpotInterruption" {
+  event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_containerd.example.com-SpotInterruption_event_pattern")
+  name          = "containerd.example.com-SpotInterruption"
+  tags = {
+    "KubernetesCluster"                            = "containerd.example.com"
+    "Name"                                         = "containerd.example.com-SpotInterruption"
+    "kubernetes.io/cluster/containerd.example.com" = "owned"
+  }
+}
+
+resource "aws_cloudwatch_event_target" "containerd-example-com-ASGLifecycle-Target" {
+  arn  = aws_sqs_queue.containerd-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.containerd-example-com-ASGLifecycle.id
+}
+
+resource "aws_cloudwatch_event_target" "containerd-example-com-InstanceScheduledChange-Target" {
+  arn  = aws_sqs_queue.containerd-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.containerd-example-com-InstanceScheduledChange.id
+}
+
+resource "aws_cloudwatch_event_target" "containerd-example-com-InstanceStateChange-Target" {
+  arn  = aws_sqs_queue.containerd-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.containerd-example-com-InstanceStateChange.id
+}
+
+resource "aws_cloudwatch_event_target" "containerd-example-com-SpotInterruption-Target" {
+  arn  = aws_sqs_queue.containerd-example-com-nth.arn
+  rule = aws_cloudwatch_event_rule.containerd-example-com-SpotInterruption.id
+}
+
 resource "aws_ebs_volume" "us-test-1a-etcd-events-containerd-example-com" {
   availability_zone = "us-test-1a"
   encrypted         = false
@@ -351,6 +437,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-containerd-example-com
     tags = {
       "KubernetesCluster"                                                                                     = "containerd.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.containerd.example.com"
+      "aws-node-termination-handler/managed"                                                                  = ""
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
       "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
@@ -365,6 +452,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-containerd-example-com
     tags = {
       "KubernetesCluster"                                                                                     = "containerd.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.containerd.example.com"
+      "aws-node-termination-handler/managed"                                                                  = ""
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
       "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
@@ -377,6 +465,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-containerd-example-com
   tags = {
     "KubernetesCluster"                                                                                     = "containerd.example.com"
     "Name"                                                                                                  = "master-us-test-1a.masters.containerd.example.com"
+    "aws-node-termination-handler/managed"                                                                  = ""
     "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
     "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
@@ -430,6 +519,7 @@ resource "aws_launch_template" "nodes-containerd-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "containerd.example.com"
       "Name"                                                                       = "nodes.containerd.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -441,6 +531,7 @@ resource "aws_launch_template" "nodes-containerd-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "containerd.example.com"
       "Name"                                                                       = "nodes.containerd.example.com"
+      "aws-node-termination-handler/managed"                                       = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -450,6 +541,7 @@ resource "aws_launch_template" "nodes-containerd-example-com" {
   tags = {
     "KubernetesCluster"                                                          = "containerd.example.com"
     "Name"                                                                       = "nodes.containerd.example.com"
+    "aws-node-termination-handler/managed"                                       = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/node"                                                           = "1"
     "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -557,6 +649,14 @@ resource "aws_s3_object" "containerd-example-com-addons-limit-range-addons-k8s-i
   server_side_encryption = "AES256"
 }
 
+resource "aws_s3_object" "containerd-example-com-addons-node-termination-handler-aws-k8s-1-11" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_containerd.example.com-addons-node-termination-handler.aws-k8s-1.11_content")
+  key                    = "clusters.example.com/containerd.example.com/addons/node-termination-handler.aws/k8s-1.11.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
 resource "aws_s3_object" "containerd-example-com-addons-storage-aws-addons-k8s-io-v1-15-0" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_object_containerd.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content")
@@ -777,6 +877,17 @@ resource "aws_security_group_rule" "from-nodes-containerd-example-com-ingress-ud
   type                     = "ingress"
 }
 
+resource "aws_sqs_queue" "containerd-example-com-nth" {
+  message_retention_seconds = 300
+  name                      = "containerd-example-com-nth"
+  policy                    = file("${path.module}/data/aws_sqs_queue_containerd-example-com-nth_policy")
+  tags = {
+    "KubernetesCluster"                            = "containerd.example.com"
+    "Name"                                         = "containerd-example-com-nth"
+    "kubernetes.io/cluster/containerd.example.com" = "owned"
+  }
+}
+
 resource "aws_subnet" "us-test-1a-containerd-example-com" {
   availability_zone                           = "us-test-1a"
   cidr_block                                  = "172.20.32.0/19"

tests/integration/update_cluster/containerd/data/aws_cloudwatch_event_rule_containerd.example.com-ASGLifecycle_event_pattern

@@ -0,0 +1 @@
+{"source":["aws.autoscaling"],"detail-type":["EC2 Instance-terminate Lifecycle Action"]}

tests/integration/update_cluster/containerd/data/aws_cloudwatch_event_rule_containerd.example.com-InstanceScheduledChange_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.health"],"detail-type": ["AWS Health Event"],"detail": {"service": ["EC2"],"eventTypeCategory": ["scheduledChange"]}}

tests/integration/update_cluster/containerd/data/aws_cloudwatch_event_rule_containerd.example.com-InstanceStateChange_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Instance State-change Notification"]}

tests/integration/update_cluster/containerd/data/aws_cloudwatch_event_rule_containerd.example.com-SpotInterruption_event_pattern

@@ -0,0 +1 @@
+{"source": ["aws.ec2"],"detail-type": ["EC2 Spot Instance Interruption Warning"]}

tests/integration/update_cluster/containerd/data/aws_iam_role_policy_masters.containerd.example.com_policy

@@ -192,13 +192,16 @@
         "iam:GetServerCertificate",
         "iam:ListServerCertificates",
         "kms:DescribeKey",
-        "kms:GenerateRandom"
+        "kms:GenerateRandom",
+        "sqs:DeleteMessage",
+        "sqs:ReceiveMessage"
       ],
       "Effect": "Allow",
       "Resource": "*"
     },
     {
       "Action": [
+        "autoscaling:CompleteLifecycleAction",
         "autoscaling:SetDesiredCapacity",
         "autoscaling:TerminateInstanceInAutoScalingGroup",
         "ec2:AttachVolume",

tests/integration/update_cluster/containerd/data/aws_s3_object_cluster-completed.spec_content

@@ -192,6 +192,18 @@ spec:
   networkCIDR: 172.20.0.0/16
   networking:
     cni: {}
+  nodeTerminationHandler:
+    cpuRequest: 50m
+    enableRebalanceDraining: false
+    enableRebalanceMonitoring: false
+    enableScheduledEventDraining: true
+    enableSpotInterruptionDraining: true
+    enabled: true
+    excludeFromLoadBalancers: true
+    managedASGTag: aws-node-termination-handler/managed
+    memoryRequest: 64Mi
+    prometheusEnable: false
+    version: v1.18.3
   nonMasqueradeCIDR: 100.64.0.0/10
   podCIDR: 100.96.0.0/11
   secretStore: memfs://clusters.example.com/containerd.example.com/secrets