Add missing cloudformation test invocations

2021-10-29 16:10:02 -05:00 · 2021-10-29 16:10:02 -05:00 · c860293783
parent 9c56f33d91
commit c860293783
5 changed files with 215 additions and 404 deletions
--- a/cmd/kops/integration_test.go
+++ b/cmd/kops/integration_test.go
@ -184,6 +184,7 @@ func TestNvidia(t *testing.T) {
 			"nvidia.addons.k8s.io-k8s-1.16",
 		).
 		runTestTerraformAWS(t)
+	newIntegrationTest("minimal.example.com", "nvidia").runTestCloudformation(t)
 }

 // TestMinimal runs the test on a minimum gossip configuration
@ -538,6 +539,7 @@ func TestExternalDNS(t *testing.T) {
 	newIntegrationTest("minimal.example.com", "external_dns").
 		withAddons("external-dns.addons.k8s.io-k8s-1.12").
 		runTestTerraformAWS(t)
+	newIntegrationTest("minimal.example.com", "external_dns").runTestCloudformation(t)
 }

 func TestExternalDNSIRSA(t *testing.T) {
--- a/tests/integration/update_cluster/external_dns/cloudformation.json
+++ b/tests/integration/update_cluster/external_dns/cloudformation.json
@ -243,6 +243,9 @@
            "HttpPutResponseHopLimit": 1,
            "HttpTokens": "optional"
          },
+          "Monitoring": {
+            "Enabled": false
+          },
          "NetworkInterfaces": [
            {
              "AssociatePublicIpAddress": true,
@ -382,6 +385,9 @@
            "HttpPutResponseHopLimit": 1,
            "HttpTokens": "optional"
          },
+          "Monitoring": {
+            "Enabled": false
+          },
          "NetworkInterfaces": [
            {
              "AssociatePublicIpAddress": true,
@ -947,97 +953,11 @@
        "PolicyDocument": {
          "Statement": [
            {
-              "Action": [
-                "ec2:DescribeAccountAttributes",
-                "ec2:DescribeInstances",
-                "ec2:DescribeInternetGateways",
-                "ec2:DescribeRegions",
-                "ec2:DescribeRouteTables",
-                "ec2:DescribeSecurityGroups",
-                "ec2:DescribeSubnets",
-                "ec2:DescribeVolumes"
-              ],
-              "Effect": "Allow",
-              "Resource": [
-                "*"
-              ]
-            },
-            {
-              "Action": [
-                "ec2:CreateSecurityGroup",
-                "ec2:CreateTags",
-                "ec2:ModifyInstanceAttribute"
-              ],
-              "Effect": "Allow",
-              "Resource": [
-                "*"
-              ]
-            },
-            {
-              "Action": [
-                "ec2:AttachVolume",
-                "ec2:AuthorizeSecurityGroupIngress",
-                "ec2:CreateRoute",
-                "ec2:DeleteRoute",
-                "ec2:DeleteSecurityGroup",
-                "ec2:RevokeSecurityGroupIngress"
-              ],
+              "Action": "ec2:AttachVolume",
              "Condition": {
                "StringEquals": {
-                  "ec2:ResourceTag/KubernetesCluster": "minimal.example.com"
-                }
-              },
-              "Effect": "Allow",
-              "Resource": [
-                "*"
-              ]
-            },
-            {
-              "Action": "autoscaling:CompleteLifecycleAction",
-              "Condition": {
-                "StringEquals": {
-                  "autoscaling:ResourceTag/KubernetesCluster": "minimal.example.com"
-                }
-              },
-              "Effect": "Allow",
-              "Resource": [
-                "*"
-              ]
-            },
-            {
-              "Action": "autoscaling:DescribeLifecycleHooks",
-              "Effect": "Allow",
-              "Resource": [
-                "*"
-              ]
-            },
-            {
-              "Action": "autoscaling:DescribeAutoScalingInstances",
-              "Effect": "Allow",
-              "Resource": [
-                "*"
-              ]
-            },
-            {
-              "Action": [
-                "autoscaling:DescribeAutoScalingGroups",
-                "autoscaling:DescribeLaunchConfigurations",
-                "autoscaling:DescribeTags",
-                "ec2:DescribeLaunchTemplateVersions"
-              ],
-              "Effect": "Allow",
-              "Resource": [
-                "*"
-              ]
-            },
-            {
-              "Action": [
-                "autoscaling:CompleteLifecycleAction",
-                "autoscaling:DescribeAutoScalingInstances"
-              ],
-              "Condition": {
-                "StringEquals": {
-                  "autoscaling:ResourceTag/KubernetesCluster": "minimal.example.com"
+                  "aws:ResourceTag/KubernetesCluster": "minimal.example.com",
+                  "aws:ResourceTag/k8s.io/role/master": "1"
                }
              },
              "Effect": "Allow",
@ -1047,57 +967,53 @@
            },
            {
              "Action": [
-                "ec2:DescribeVpcs",
-                "elasticloadbalancing:DescribeLoadBalancers",
-                "elasticloadbalancing:DescribeLoadBalancerAttributes",
-                "elasticloadbalancing:DescribeListeners",
-                "elasticloadbalancing:DescribeLoadBalancerPolicies",
-                "elasticloadbalancing:DescribeTargetGroups",
-                "elasticloadbalancing:DescribeTargetHealth",
-                "elasticloadbalancing:CreateListener",
-                "elasticloadbalancing:CreateTargetGroup",
-                "elasticloadbalancing:CreateLoadBalancer",
-                "elasticloadbalancing:CreateLoadBalancerPolicy",
-                "elasticloadbalancing:CreateLoadBalancerListeners",
-                "elasticloadbalancing:DeleteLoadBalancer",
-                "elasticloadbalancing:DeleteLoadBalancerListeners",
-                "elasticloadbalancing:DeleteListener",
-                "elasticloadbalancing:DeleteTargetGroup",
-                "elasticloadbalancing:AddTags",
-                "elasticloadbalancing:ModifyLoadBalancerAttributes",
-                "elasticloadbalancing:ModifyListener",
-                "elasticloadbalancing:ModifyTargetGroup",
-                "elasticloadbalancing:AttachLoadBalancerToSubnets",
-                "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer",
-                "elasticloadbalancing:ConfigureHealthCheck",
-                "elasticloadbalancing:DetachLoadBalancerFromSubnets",
-                "elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
-                "elasticloadbalancing:RegisterInstancesWithLoadBalancer",
-                "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer",
-                "elasticloadbalancing:DeregisterTargets",
-                "elasticloadbalancing:RegisterTargets",
-                "elasticloadbalancing:SetLoadBalancerPoliciesOfListener"
+                "s3:Get*"
+              ],
+              "Effect": "Allow",
+              "Resource": "arn:aws:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/*"
+            },
+            {
+              "Action": [
+                "s3:GetObject",
+                "s3:DeleteObject",
+                "s3:DeleteObjectVersion",
+                "s3:PutObject"
+              ],
+              "Effect": "Allow",
+              "Resource": "arn:aws:s3:::placeholder-write-bucket/clusters.example.com/minimal.example.com/backups/etcd/main/*"
+            },
+            {
+              "Action": [
+                "s3:GetObject",
+                "s3:DeleteObject",
+                "s3:DeleteObjectVersion",
+                "s3:PutObject"
+              ],
+              "Effect": "Allow",
+              "Resource": "arn:aws:s3:::placeholder-write-bucket/clusters.example.com/minimal.example.com/backups/etcd/events/*"
+            },
+            {
+              "Action": [
+                "s3:GetBucketLocation",
+                "s3:GetEncryptionConfiguration",
+                "s3:ListBucket",
+                "s3:ListBucketVersions"
              ],
              "Effect": "Allow",
              "Resource": [
-                "*"
+                "arn:aws:s3:::placeholder-read-bucket"
              ]
            },
            {
              "Action": [
-                "iam:ListServerCertificates",
-                "iam:GetServerCertificate"
+                "s3:GetBucketLocation",
+                "s3:GetEncryptionConfiguration",
+                "s3:ListBucket",
+                "s3:ListBucketVersions"
              ],
              "Effect": "Allow",
              "Resource": [
-                "*"
-              ]
-            },
-            {
-              "Action": "kms:GenerateRandom",
-              "Effect": "Allow",
-              "Resource": [
-                "*"
+                "arn:aws:s3:::placeholder-write-bucket"
              ]
            },
            {
@ -1131,43 +1047,20 @@
              ]
            },
            {
-              "Action": [
-                "ec2:DescribeAccountAttributes",
-                "ec2:DescribeInstances",
-                "ec2:DescribeVolumes",
-                "ec2:DescribeVolumesModifications",
-                "ec2:DescribeTags"
-              ],
-              "Effect": "Allow",
-              "Resource": "*"
-            },
-            {
-              "Action": [
-                "ec2:CreateVolume"
-              ],
+              "Action": "ec2:CreateTags",
              "Condition": {
                "StringEquals": {
-                  "aws:RequestTag/KubernetesCluster": "minimal.example.com"
+                  "ec2:CreateAction": [
+                    "CreateVolume",
+                    "CreateSnapshot"
+                  ]
                }
              },
              "Effect": "Allow",
-              "Resource": "*"
-            },
-            {
-              "Action": [
-                "ec2:ModifyVolume",
-                "ec2:ModifyInstanceAttribute",
-                "ec2:AttachVolume",
-                "ec2:DeleteVolume",
-                "ec2:DetachVolume"
-              ],
-              "Condition": {
-                "StringEquals": {
-                  "aws:ResourceTag/KubernetesCluster": "minimal.example.com"
-                }
-              },
-              "Effect": "Allow",
-              "Resource": "*"
+              "Resource": [
+                "arn:aws:ec2:*:*:volume/*",
+                "arn:aws:ec2:*:*:snapshot/*"
+              ]
            },
            {
              "Action": "ec2:CreateTags",
@ -1189,7 +1082,7 @@
              "Action": "ec2:DeleteTags",
              "Condition": {
                "StringEquals": {
-                  "ec2:ResourceTag/KubernetesCluster": "minimal.example.com"
+                  "aws:ResourceTag/KubernetesCluster": "minimal.example.com"
                }
              },
              "Effect": "Allow",
@ -1200,14 +1093,77 @@
            },
            {
              "Action": [
+                "autoscaling:DescribeAutoScalingGroups",
+                "autoscaling:DescribeAutoScalingInstances",
+                "autoscaling:DescribeLaunchConfigurations",
+                "autoscaling:DescribeTags",
+                "ec2:CreateSecurityGroup",
+                "ec2:CreateTags",
+                "ec2:DescribeAccountAttributes",
+                "ec2:DescribeInstanceTypes",
+                "ec2:DescribeInstances",
+                "ec2:DescribeLaunchTemplateVersions",
+                "ec2:DescribeRegions",
+                "ec2:DescribeRouteTables",
+                "ec2:DescribeSecurityGroups",
+                "ec2:DescribeSubnets",
+                "ec2:DescribeTags",
+                "ec2:DescribeVolumes",
+                "ec2:DescribeVolumesModifications",
+                "ec2:DescribeVpcs",
+                "elasticloadbalancing:AddTags",
+                "elasticloadbalancing:CreateListener",
+                "elasticloadbalancing:CreateTargetGroup",
+                "elasticloadbalancing:DescribeListeners",
+                "elasticloadbalancing:DescribeLoadBalancerAttributes",
+                "elasticloadbalancing:DescribeLoadBalancerPolicies",
+                "elasticloadbalancing:DescribeLoadBalancers",
+                "elasticloadbalancing:DescribeTargetGroups",
+                "elasticloadbalancing:DescribeTargetHealth",
+                "elasticloadbalancing:RegisterTargets",
+                "iam:GetServerCertificate",
+                "iam:ListServerCertificates",
+                "kms:DescribeKey",
+                "kms:GenerateRandom"
+              ],
+              "Effect": "Allow",
+              "Resource": "*"
+            },
+            {
+              "Action": [
+                "autoscaling:SetDesiredCapacity",
+                "autoscaling:TerminateInstanceInAutoScalingGroup",
                "ec2:AttachVolume",
+                "ec2:AuthorizeSecurityGroupIngress",
+                "ec2:DeleteRoute",
+                "ec2:DeleteSecurityGroup",
                "ec2:DeleteVolume",
                "ec2:DetachVolume",
-                "ec2:RevokeSecurityGroupIngress"
+                "ec2:ModifyInstanceAttribute",
+                "ec2:ModifyVolume",
+                "ec2:RevokeSecurityGroupIngress",
+                "elasticloadbalancing:AddTags",
+                "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer",
+                "elasticloadbalancing:AttachLoadBalancerToSubnets",
+                "elasticloadbalancing:ConfigureHealthCheck",
+                "elasticloadbalancing:DeleteListener",
+                "elasticloadbalancing:DeleteLoadBalancer",
+                "elasticloadbalancing:DeleteLoadBalancerListeners",
+                "elasticloadbalancing:DeleteTargetGroup",
+                "elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
+                "elasticloadbalancing:DeregisterTargets",
+                "elasticloadbalancing:DetachLoadBalancerFromSubnets",
+                "elasticloadbalancing:ModifyListener",
+                "elasticloadbalancing:ModifyLoadBalancerAttributes",
+                "elasticloadbalancing:ModifyTargetGroup",
+                "elasticloadbalancing:RegisterInstancesWithLoadBalancer",
+                "elasticloadbalancing:RegisterTargets",
+                "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer",
+                "elasticloadbalancing:SetLoadBalancerPoliciesOfListener"
              ],
              "Condition": {
                "StringEquals": {
-                  "ec2:ResourceTag/KubernetesCluster": "minimal.example.com"
+                  "aws:ResourceTag/KubernetesCluster": "minimal.example.com"
                }
              },
              "Effect": "Allow",
@ -1215,29 +1171,21 @@
            },
            {
              "Action": [
-                "autoscaling:SetDesiredCapacity",
-                "autoscaling:TerminateInstanceInAutoScalingGroup"
+                "ec2:CreateSecurityGroup",
+                "ec2:CreateVolume",
+                "elasticloadbalancing:CreateListener",
+                "elasticloadbalancing:CreateLoadBalancer",
+                "elasticloadbalancing:CreateLoadBalancerListeners",
+                "elasticloadbalancing:CreateLoadBalancerPolicy",
+                "elasticloadbalancing:CreateTargetGroup"
              ],
              "Condition": {
                "StringEquals": {
-                  "autoscaling:ResourceTag/KubernetesCluster": "minimal.example.com"
+                  "aws:RequestTag/KubernetesCluster": "minimal.example.com"
                }
              },
              "Effect": "Allow",
-              "Resource": [
-                "*"
-              ]
-            },
-            {
-              "Action": [
-                "autoscaling:DescribeAutoScalingGroups",
-                "autoscaling:DescribeAutoScalingInstances",
-                "autoscaling:DescribeLaunchConfigurations"
-              ],
-              "Effect": "Allow",
-              "Resource": [
-                "*"
-              ]
+              "Resource": "*"
            }
          ],
          "Version": "2012-10-17"
@ -1257,27 +1205,39 @@
          "Statement": [
            {
              "Action": [
-                "ec2:DescribeInstances",
-                "ec2:DescribeRegions"
+                "s3:Get*"
              ],
              "Effect": "Allow",
              "Resource": [
-                "*"
+                "arn:aws:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/addons/*",
+                "arn:aws:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/cluster-completed.spec",
+                "arn:aws:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/igconfig/node/*",
+                "arn:aws:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/secrets/dockerconfig"
              ]
            },
            {
-              "Action": "autoscaling:DescribeAutoScalingInstances",
+              "Action": [
+                "s3:GetBucketLocation",
+                "s3:GetEncryptionConfiguration",
+                "s3:ListBucket",
+                "s3:ListBucketVersions"
+              ],
              "Effect": "Allow",
              "Resource": [
-                "*"
+                "arn:aws:s3:::placeholder-read-bucket"
              ]
            },
            {
-              "Action": "kms:GenerateRandom",
+              "Action": [
+                "autoscaling:DescribeAutoScalingInstances",
+                "ec2:DescribeInstanceTypes",
+                "ec2:DescribeInstances",
+                "iam:GetServerCertificate",
+                "iam:ListServerCertificates",
+                "kms:GenerateRandom"
+              ],
              "Effect": "Allow",
-              "Resource": [
-                "*"
-              ]
+              "Resource": "*"
            }
          ],
          "Version": "2012-10-17"
--- a/tests/integration/update_cluster/external_dns/cloudformation.json.extracted.yaml
+++ b/tests/integration/update_cluster/external_dns/cloudformation.json.extracted.yaml
@ -131,7 +131,7 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
  containerRuntime: containerd
  containerd:
    logLevel: info
-    version: 1.4.6
+    version: 1.4.11
  docker:
    skipInstall: true
  encryptionConfig: null
@ -153,7 +153,6 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
    - NamespaceLifecycle
    - LimitRanger
    - ServiceAccount
-    - PersistentVolumeLabel
    - DefaultStorageClass
    - DefaultTolerationSeconds
    - MutatingAdmissionWebhook
@ -241,98 +240,11 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
  __EOF_CLUSTER_SPEC

  cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
-  Assets:
-    amd64:
-    - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet
-    - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl
-    - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
-    - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz
-    - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64
-    - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64
-    arm64:
-    - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet
-    - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl
-    - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
-    - be8c9a5a06ebec8fb1d36e867cd00fb5777746a9812a0cae2966778ff899c525@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.7.tgz
-    - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64
-    - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64
-  AuxConfigHash: /O5IS/dGo83lv2DbWn4k91OYfuOqtO79vjf5pD1DQlI=
-  CAs:
-    ca: |
-      -----BEGIN CERTIFICATE-----
-      MIIBaDCCARKgAwIBAgIMFoq6Pex4lTCM8fOIMA0GCSqGSIb3DQEBCwUAMBUxEzAR
-      BgNVBAMTCmt1YmVybmV0ZXMwHhcNMjEwNjE5MjI0MzEwWhcNMzEwNjE5MjI0MzEw
-      WjAVMRMwEQYDVQQDEwprdWJlcm5ldGVzMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJB
-      ANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m49pAEeIMsvwvD
-      MaLsb2v6JvXe0qvCmueU+/sCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud
-      EwEB/wQFMAMBAf8wHQYDVR0OBBYEFCOW3hR7ngBsk9aUOlEznWzH494EMA0GCSqG
-      SIb3DQEBCwUAA0EAVnZzkiku07kQFGAEXzWI6aZnAbzSoClYskEzCBMrOmdadjVp
-      VWcz76FwFlyd5jhzOJ49eMcVusSotKv2ZGimcA==
-      -----END CERTIFICATE-----
-      -----BEGIN CERTIFICATE-----
-      MIIBaDCCARKgAwIBAgIMFoq6PeyECsgUTfc2MA0GCSqGSIb3DQEBCwUAMBUxEzAR
-      BgNVBAMTCmt1YmVybmV0ZXMwHhcNMjEwNjE5MjI0MzEwWhcNMzEwNjE5MjI0MzEw
-      WjAVMRMwEQYDVQQDEwprdWJlcm5ldGVzMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJB
-      AKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCCrs6AdVRZNZ26
-      A6uHArhi33c2kHQkCjyLA7sCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud
-      EwEB/wQFMAMBAf8wHQYDVR0OBBYEFIT28RJlG8FTgmvn2YMa3hYX+u1BMA0GCSqG
-      SIb3DQEBCwUAA0EAKuaE5wKMP26AyfxkWu83iHoTPFtdjabXF0JcyPy0ijQZxfJq
-      9xc2CkttvgaDtT4H+E/ryQ3iq6kSfEYYPi8c0w==
-      -----END CERTIFICATE-----
  CloudProvider: aws
-  ClusterName: minimal.example.com
  ConfigBase: memfs://clusters.example.com/minimal.example.com
  InstanceGroupName: master-us-test-1a
  InstanceGroupRole: Master
-  KeypairIDs:
-    ca: "6976381481633145814258938760"
-  KubeletConfig:
-    anonymousAuth: false
-    cgroupDriver: systemd
-    cgroupRoot: /
-    cloudProvider: aws
-    clusterDNS: 100.64.0.10
-    clusterDomain: cluster.local
-    enableDebuggingHandlers: true
-    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
-    hostnameOverride: '@aws'
-    kubeconfigPath: /var/lib/kubelet/kubeconfig
-    logLevel: 2
-    networkPluginName: cni
-    nodeLabels:
-      kops.k8s.io/kops-controller-pki: ""
-      kubernetes.io/role: master
-      node-role.kubernetes.io/control-plane: ""
-      node-role.kubernetes.io/master: ""
-      node.kubernetes.io/exclude-from-external-load-balancers: ""
-    nonMasqueradeCIDR: 100.64.0.0/10
-    podManifestPath: /etc/kubernetes/manifests
-    registerSchedulable: false
-  UpdatePolicy: automatic
-  channels:
-  - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml
-  containerdConfig: |
-    version = 2
-
-    [plugins]
-
-      [plugins."io.containerd.grpc.v1.cri"]
-
-        [plugins."io.containerd.grpc.v1.cri".containerd]
-
-          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
-
-            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
-              runtime_type = "io.containerd.runc.v2"
-
-              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
-                SystemdCgroup = true
-  etcdManifests:
-  - memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml
-  - memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml
-  staticManifests:
-  - key: kube-apiserver-healthcheck
-    path: manifests/static/kube-apiserver-healthcheck.yaml
+  NodeupConfigHash: W80rsOsSg9anEDGL5fL40gNx08rzxsiXwJo/qjhA6jA=

  __EOF_KUBE_ENV

@ -471,7 +383,7 @@ Resources.AWSEC2LaunchTemplatenodesminimalexamplecom.Properties.LaunchTemplateDa
  containerRuntime: containerd
  containerd:
    logLevel: info
-    version: 1.4.6
+    version: 1.4.11
  docker:
    skipInstall: true
  kubeProxy:
@ -499,83 +411,11 @@ Resources.AWSEC2LaunchTemplatenodesminimalexamplecom.Properties.LaunchTemplateDa
  __EOF_CLUSTER_SPEC

  cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
-  Assets:
-    amd64:
-    - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet
-    - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl
-    - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
-    - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz
-    arm64:
-    - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet
-    - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl
-    - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
-    - be8c9a5a06ebec8fb1d36e867cd00fb5777746a9812a0cae2966778ff899c525@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.7.tgz
-  AuxConfigHash: /O5IS/dGo83lv2DbWn4k91OYfuOqtO79vjf5pD1DQlI=
-  CAs:
-    ca: |
-      -----BEGIN CERTIFICATE-----
-      MIIBaDCCARKgAwIBAgIMFoq6Pex4lTCM8fOIMA0GCSqGSIb3DQEBCwUAMBUxEzAR
-      BgNVBAMTCmt1YmVybmV0ZXMwHhcNMjEwNjE5MjI0MzEwWhcNMzEwNjE5MjI0MzEw
-      WjAVMRMwEQYDVQQDEwprdWJlcm5ldGVzMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJB
-      ANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m49pAEeIMsvwvD
-      MaLsb2v6JvXe0qvCmueU+/sCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud
-      EwEB/wQFMAMBAf8wHQYDVR0OBBYEFCOW3hR7ngBsk9aUOlEznWzH494EMA0GCSqG
-      SIb3DQEBCwUAA0EAVnZzkiku07kQFGAEXzWI6aZnAbzSoClYskEzCBMrOmdadjVp
-      VWcz76FwFlyd5jhzOJ49eMcVusSotKv2ZGimcA==
-      -----END CERTIFICATE-----
-      -----BEGIN CERTIFICATE-----
-      MIIBaDCCARKgAwIBAgIMFoq6PeyECsgUTfc2MA0GCSqGSIb3DQEBCwUAMBUxEzAR
-      BgNVBAMTCmt1YmVybmV0ZXMwHhcNMjEwNjE5MjI0MzEwWhcNMzEwNjE5MjI0MzEw
-      WjAVMRMwEQYDVQQDEwprdWJlcm5ldGVzMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJB
-      AKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCCrs6AdVRZNZ26
-      A6uHArhi33c2kHQkCjyLA7sCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud
-      EwEB/wQFMAMBAf8wHQYDVR0OBBYEFIT28RJlG8FTgmvn2YMa3hYX+u1BMA0GCSqG
-      SIb3DQEBCwUAA0EAKuaE5wKMP26AyfxkWu83iHoTPFtdjabXF0JcyPy0ijQZxfJq
-      9xc2CkttvgaDtT4H+E/ryQ3iq6kSfEYYPi8c0w==
-      -----END CERTIFICATE-----
  CloudProvider: aws
-  ClusterName: minimal.example.com
  ConfigBase: memfs://clusters.example.com/minimal.example.com
  InstanceGroupName: nodes
  InstanceGroupRole: Node
-  KeypairIDs: {}
-  KubeletConfig:
-    anonymousAuth: false
-    cgroupDriver: systemd
-    cgroupRoot: /
-    cloudProvider: aws
-    clusterDNS: 100.64.0.10
-    clusterDomain: cluster.local
-    enableDebuggingHandlers: true
-    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
-    hostnameOverride: '@aws'
-    kubeconfigPath: /var/lib/kubelet/kubeconfig
-    logLevel: 2
-    networkPluginName: cni
-    nodeLabels:
-      kubernetes.io/role: node
-      node-role.kubernetes.io/node: ""
-    nonMasqueradeCIDR: 100.64.0.0/10
-    podManifestPath: /etc/kubernetes/manifests
-  UpdatePolicy: automatic
-  channels:
-  - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml
-  containerdConfig: |
-    version = 2
-
-    [plugins]
-
-      [plugins."io.containerd.grpc.v1.cri"]
-
-        [plugins."io.containerd.grpc.v1.cri".containerd]
-
-          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
-
-            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
-              runtime_type = "io.containerd.runc.v2"
-
-              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
-                SystemdCgroup = true
+  NodeupConfigHash: /xc0hHryd87G4ro46rYwe9mCe7j9bWZRSru2pMpqqNs=

  __EOF_KUBE_ENV

--- a/tests/integration/update_cluster/nvidia/cloudformation.json
+++ b/tests/integration/update_cluster/nvidia/cloudformation.json
@ -124,6 +124,11 @@
            "Value": "nodes.minimal.example.com",
            "PropagateAtLaunch": true
          },
+          {
+            "Key": "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/gpu",
+            "Value": "1",
+            "PropagateAtLaunch": true
+          },
          {
            "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role",
            "Value": "node",
@ -379,7 +384,7 @@
            }
          },
          "ImageId": "ami-12345678",
-          "InstanceType": "t2.medium",
+          "InstanceType": "g4dn.xlarge",
          "KeyName": "kubernetes.minimal.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57",
          "MetadataOptions": {
            "HttpPutResponseHopLimit": 1,
@ -413,6 +418,10 @@
                  "Key": "Name",
                  "Value": "nodes.minimal.example.com"
                },
+                {
+                  "Key": "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/gpu",
+                  "Value": "1"
+                },
                {
                  "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role",
                  "Value": "node"
@ -446,6 +455,10 @@
                  "Key": "Name",
                  "Value": "nodes.minimal.example.com"
                },
+                {
+                  "Key": "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/gpu",
+                  "Value": "1"
+                },
                {
                  "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role",
                  "Value": "node"
@ -965,42 +978,6 @@
                "*"
              ]
            },
-            {
-              "Action": "ec2:CreateTags",
-              "Condition": {
-                "StringEquals": {
-                  "ec2:CreateAction": [
-                    "CreateVolume",
-                    "CreateSnapshot"
-                  ]
-                }
-              },
-              "Effect": "Allow",
-              "Resource": [
-                "arn:aws:ec2:*:*:volume/*",
-                "arn:aws:ec2:*:*:snapshot/*"
-              ]
-            },
-            {
-              "Action": [
-                "elasticloadbalancing:CreateLoadBalancer",
-                "elasticloadbalancing:CreateLoadBalancerPolicy",
-                "elasticloadbalancing:CreateLoadBalancerListeners",
-                "ec2:CreateSecurityGroup",
-                "ec2:CreateVolume",
-                "elasticloadbalancing:CreateListener",
-                "elasticloadbalancing:CreateTargetGroup"
-              ],
-              "Condition": {
-                "StringEquals": {
-                  "aws:RequestTag/KubernetesCluster": "minimal.example.com"
-                }
-              },
-              "Effect": "Allow",
-              "Resource": [
-                "*"
-              ]
-            },
            {
              "Action": [
                "s3:Get*"
@ -1074,7 +1051,8 @@
            },
            {
              "Action": [
-                "route53:ListHostedZones"
+                "route53:ListHostedZones",
+                "route53:ListTagsForResource"
              ],
              "Effect": "Allow",
              "Resource": [
@ -1082,16 +1060,20 @@
              ]
            },
            {
-              "Action": [
-                "ec2:CreateVolume"
-              ],
+              "Action": "ec2:CreateTags",
              "Condition": {
                "StringEquals": {
-                  "aws:RequestTag/KubernetesCluster": "minimal.example.com"
+                  "ec2:CreateAction": [
+                    "CreateVolume",
+                    "CreateSnapshot"
+                  ]
                }
              },
              "Effect": "Allow",
-              "Resource": "*"
+              "Resource": [
+                "arn:aws:ec2:*:*:volume/*",
+                "arn:aws:ec2:*:*:snapshot/*"
+              ]
            },
            {
              "Action": "ec2:CreateTags",
@ -1133,6 +1115,7 @@
                "ec2:DescribeAccountAttributes",
                "ec2:DescribeInstanceTypes",
                "ec2:DescribeInstances",
+                "ec2:DescribeLaunchTemplateVersions",
                "ec2:DescribeRegions",
                "ec2:DescribeRouteTables",
                "ec2:DescribeSecurityGroups",
@ -1141,12 +1124,16 @@
                "ec2:DescribeVolumes",
                "ec2:DescribeVolumesModifications",
                "ec2:DescribeVpcs",
+                "elasticloadbalancing:AddTags",
+                "elasticloadbalancing:CreateListener",
+                "elasticloadbalancing:CreateTargetGroup",
                "elasticloadbalancing:DescribeListeners",
                "elasticloadbalancing:DescribeLoadBalancerAttributes",
                "elasticloadbalancing:DescribeLoadBalancerPolicies",
                "elasticloadbalancing:DescribeLoadBalancers",
                "elasticloadbalancing:DescribeTargetGroups",
                "elasticloadbalancing:DescribeTargetHealth",
+                "elasticloadbalancing:RegisterTargets",
                "iam:GetServerCertificate",
                "iam:ListServerCertificates",
                "kms:DescribeKey",
@ -1194,6 +1181,24 @@
              },
              "Effect": "Allow",
              "Resource": "*"
+            },
+            {
+              "Action": [
+                "ec2:CreateSecurityGroup",
+                "ec2:CreateVolume",
+                "elasticloadbalancing:CreateListener",
+                "elasticloadbalancing:CreateLoadBalancer",
+                "elasticloadbalancing:CreateLoadBalancerListeners",
+                "elasticloadbalancing:CreateLoadBalancerPolicy",
+                "elasticloadbalancing:CreateTargetGroup"
+              ],
+              "Condition": {
+                "StringEquals": {
+                  "aws:RequestTag/KubernetesCluster": "minimal.example.com"
+                }
+              },
+              "Effect": "Allow",
+              "Resource": "*"
            }
          ],
          "Version": "2012-10-17"
@ -1220,7 +1225,6 @@
                "arn:aws:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/addons/*",
                "arn:aws:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/cluster-completed.spec",
                "arn:aws:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/igconfig/node/*",
-                "arn:aws:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/pki/ssh/*",
                "arn:aws:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/secrets/dockerconfig"
              ]
            },
--- a/tests/integration/update_cluster/nvidia/cloudformation.json.extracted.yaml
+++ b/tests/integration/update_cluster/nvidia/cloudformation.json.extracted.yaml
@ -131,7 +131,10 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
  containerRuntime: containerd
  containerd:
    logLevel: info
-    version: 1.4.6
+    nvidiaGPU:
+      enabled: true
+      package: nvidia-headless-460-server
+    version: 1.4.11
  docker:
    skipInstall: true
  encryptionConfig: null
@ -153,7 +156,6 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
    - NamespaceLifecycle
    - LimitRanger
    - ServiceAccount
-    - PersistentVolumeLabel
    - DefaultStorageClass
    - DefaultTolerationSeconds
    - MutatingAdmissionWebhook
@ -245,7 +247,7 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
  ConfigBase: memfs://clusters.example.com/minimal.example.com
  InstanceGroupName: master-us-test-1a
  InstanceGroupRole: Master
-  NodeupConfigHash: M484KjV19wtMDGWwO7cf5eb6cEAQW2nh2IqShjgpbHU=
+  NodeupConfigHash: N7p2va9tPSRTapQ/idYkE3zTSMGc04Er5uSryOA+EJY=

  __EOF_KUBE_ENV

@ -384,7 +386,10 @@ Resources.AWSEC2LaunchTemplatenodesminimalexamplecom.Properties.LaunchTemplateDa
  containerRuntime: containerd
  containerd:
    logLevel: info
-    version: 1.4.6
+    nvidiaGPU:
+      enabled: true
+      package: nvidia-headless-460-server
+    version: 1.4.11
  docker:
    skipInstall: true
  kubeProxy:
@ -416,7 +421,7 @@ Resources.AWSEC2LaunchTemplatenodesminimalexamplecom.Properties.LaunchTemplateDa
  ConfigBase: memfs://clusters.example.com/minimal.example.com
  InstanceGroupName: nodes
  InstanceGroupRole: Node
-  NodeupConfigHash: A0AyiJo03pbqluaXrVtbacjofP1NmBexAl0w2y4oS5o=
+  NodeupConfigHash: OyVtiYq+TGC8i2VY0pYPlgnx1r3FYU53FSn5Y0bLIQ4=

  __EOF_KUBE_ENV