Merge pull request #12637 from rifelpet/integration-test-cleanup

Integration test cleanup

cmd/kops/integration_test.go

@@ -184,6 +184,7 @@ func TestNvidia(t *testing.T) {
 			"nvidia.addons.k8s.io-k8s-1.16",
 		).
 		runTestTerraformAWS(t)
+	newIntegrationTest("minimal.example.com", "nvidia").runTestCloudformation(t)
 }
 
 // TestMinimal runs the test on a minimum gossip configuration
@@ -538,6 +539,7 @@ func TestExternalDNS(t *testing.T) {
 	newIntegrationTest("minimal.example.com", "external_dns").
 		withAddons("external-dns.addons.k8s.io-k8s-1.12").
 		runTestTerraformAWS(t)
+	newIntegrationTest("minimal.example.com", "external_dns").runTestCloudformation(t)
 }
 
 func TestExternalDNSIRSA(t *testing.T) {
@@ -800,6 +802,21 @@ func (i *integrationTest) runTest(t *testing.T, h *testutils.IntegrationTestHarn
 				golden.AssertMatchesFile(t, string(actualDataContent), path.Join(expectedDataPath, dataFileName))
 			}
 		}
+
+		existingExpectedFiles, err := ioutil.ReadDir(expectedDataPath)
+		if err != nil {
+			t.Fatalf("failed to read data dir: %v", err)
+		}
+		existingExpectedFilenames := make([]string, len(existingExpectedFiles))
+		for i, f := range existingExpectedFiles {
+			existingExpectedFilenames[i] = f.Name()
+		}
+		for j := 0; j < len(existingExpectedFilenames) && j < len(expectedDataFilenames); j++ {
+			if existingExpectedFilenames[j] != expectedDataFilenames[j] {
+				t.Errorf("diff with source directory @%d: %q vs %q", j, existingExpectedFilenames[j], expectedDataFilenames[j])
+				break
+			}
+		}
 	}
 }
 

tests/integration/update_cluster/aws-lb-controller/data/aws_s3_bucket_object_minimal.example.com-addons-anonymous-issuer-discovery.addons.k8s.io-k8s-1.16_content

@@ -1,19 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: anonymous-issuer-discovery.addons.k8s.io
-    addon.kops.k8s.io/version: 1.21.0-alpha.3
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: anonymous-issuer-discovery.addons.k8s.io
-  name: anonymous:service-account-issuer-discovery
-  namespace: kube-system
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:service-account-issuer-discovery
-subjects:
-- apiGroup: rbac.authorization.k8s.io
-  kind: User
-  name: system:anonymous

tests/integration/update_cluster/complex/data/aws_key_pair_kubernetes.complex.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key

@@ -1 +0,0 @@
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==

tests/integration/update_cluster/external_dns/cloudformation.json

@@ -243,6 +243,9 @@
             "HttpPutResponseHopLimit": 1,
             "HttpTokens": "optional"
           },
+          "Monitoring": {
+            "Enabled": false
+          },
           "NetworkInterfaces": [
             {
               "AssociatePublicIpAddress": true,
@@ -382,6 +385,9 @@
             "HttpPutResponseHopLimit": 1,
             "HttpTokens": "optional"
           },
+          "Monitoring": {
+            "Enabled": false
+          },
           "NetworkInterfaces": [
             {
               "AssociatePublicIpAddress": true,
@@ -947,97 +953,11 @@
         "PolicyDocument": {
           "Statement": [
             {
-              "Action": [
+              "Action": "ec2:AttachVolume",
-                "ec2:DescribeAccountAttributes",
-                "ec2:DescribeInstances",
-                "ec2:DescribeInternetGateways",
-                "ec2:DescribeRegions",
-                "ec2:DescribeRouteTables",
-                "ec2:DescribeSecurityGroups",
-                "ec2:DescribeSubnets",
-                "ec2:DescribeVolumes"
-              ],
-              "Effect": "Allow",
-              "Resource": [
-                "*"
-              ]
-            },
-            {
-              "Action": [
-                "ec2:CreateSecurityGroup",
-                "ec2:CreateTags",
-                "ec2:ModifyInstanceAttribute"
-              ],
-              "Effect": "Allow",
-              "Resource": [
-                "*"
-              ]
-            },
-            {
-              "Action": [
-                "ec2:AttachVolume",
-                "ec2:AuthorizeSecurityGroupIngress",
-                "ec2:CreateRoute",
-                "ec2:DeleteRoute",
-                "ec2:DeleteSecurityGroup",
-                "ec2:RevokeSecurityGroupIngress"
-              ],
               "Condition": {
                 "StringEquals": {
-                  "ec2:ResourceTag/KubernetesCluster": "minimal.example.com"
+                  "aws:ResourceTag/KubernetesCluster": "minimal.example.com",
-                }
+                  "aws:ResourceTag/k8s.io/role/master": "1"
-              },
-              "Effect": "Allow",
-              "Resource": [
-                "*"
-              ]
-            },
-            {
-              "Action": "autoscaling:CompleteLifecycleAction",
-              "Condition": {
-                "StringEquals": {
-                  "autoscaling:ResourceTag/KubernetesCluster": "minimal.example.com"
-                }
-              },
-              "Effect": "Allow",
-              "Resource": [
-                "*"
-              ]
-            },
-            {
-              "Action": "autoscaling:DescribeLifecycleHooks",
-              "Effect": "Allow",
-              "Resource": [
-                "*"
-              ]
-            },
-            {
-              "Action": "autoscaling:DescribeAutoScalingInstances",
-              "Effect": "Allow",
-              "Resource": [
-                "*"
-              ]
-            },
-            {
-              "Action": [
-                "autoscaling:DescribeAutoScalingGroups",
-                "autoscaling:DescribeLaunchConfigurations",
-                "autoscaling:DescribeTags",
-                "ec2:DescribeLaunchTemplateVersions"
-              ],
-              "Effect": "Allow",
-              "Resource": [
-                "*"
-              ]
-            },
-            {
-              "Action": [
-                "autoscaling:CompleteLifecycleAction",
-                "autoscaling:DescribeAutoScalingInstances"
-              ],
-              "Condition": {
-                "StringEquals": {
-                  "autoscaling:ResourceTag/KubernetesCluster": "minimal.example.com"
                 }
               },
               "Effect": "Allow",
@@ -1047,57 +967,53 @@
             },
             {
               "Action": [
-                "ec2:DescribeVpcs",
+                "s3:Get*"
-                "elasticloadbalancing:DescribeLoadBalancers",
+              ],
-                "elasticloadbalancing:DescribeLoadBalancerAttributes",
+              "Effect": "Allow",
-                "elasticloadbalancing:DescribeListeners",
+              "Resource": "arn:aws:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/*"
-                "elasticloadbalancing:DescribeLoadBalancerPolicies",
+            },
-                "elasticloadbalancing:DescribeTargetGroups",
+            {
-                "elasticloadbalancing:DescribeTargetHealth",
+              "Action": [
-                "elasticloadbalancing:CreateListener",
+                "s3:GetObject",
-                "elasticloadbalancing:CreateTargetGroup",
+                "s3:DeleteObject",
-                "elasticloadbalancing:CreateLoadBalancer",
+                "s3:DeleteObjectVersion",
-                "elasticloadbalancing:CreateLoadBalancerPolicy",
+                "s3:PutObject"
-                "elasticloadbalancing:CreateLoadBalancerListeners",
+              ],
-                "elasticloadbalancing:DeleteLoadBalancer",
+              "Effect": "Allow",
-                "elasticloadbalancing:DeleteLoadBalancerListeners",
+              "Resource": "arn:aws:s3:::placeholder-write-bucket/clusters.example.com/minimal.example.com/backups/etcd/main/*"
-                "elasticloadbalancing:DeleteListener",
+            },
-                "elasticloadbalancing:DeleteTargetGroup",
+            {
-                "elasticloadbalancing:AddTags",
+              "Action": [
-                "elasticloadbalancing:ModifyLoadBalancerAttributes",
+                "s3:GetObject",
-                "elasticloadbalancing:ModifyListener",
+                "s3:DeleteObject",
-                "elasticloadbalancing:ModifyTargetGroup",
+                "s3:DeleteObjectVersion",
-                "elasticloadbalancing:AttachLoadBalancerToSubnets",
+                "s3:PutObject"
-                "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer",
+              ],
-                "elasticloadbalancing:ConfigureHealthCheck",
+              "Effect": "Allow",
-                "elasticloadbalancing:DetachLoadBalancerFromSubnets",
+              "Resource": "arn:aws:s3:::placeholder-write-bucket/clusters.example.com/minimal.example.com/backups/etcd/events/*"
-                "elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
+            },
-                "elasticloadbalancing:RegisterInstancesWithLoadBalancer",
+            {
-                "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer",
+              "Action": [
-                "elasticloadbalancing:DeregisterTargets",
+                "s3:GetBucketLocation",
-                "elasticloadbalancing:RegisterTargets",
+                "s3:GetEncryptionConfiguration",
-                "elasticloadbalancing:SetLoadBalancerPoliciesOfListener"
+                "s3:ListBucket",
+                "s3:ListBucketVersions"
               ],
               "Effect": "Allow",
               "Resource": [
-                "*"
+                "arn:aws:s3:::placeholder-read-bucket"
               ]
             },
             {
               "Action": [
-                "iam:ListServerCertificates",
+                "s3:GetBucketLocation",
-                "iam:GetServerCertificate"
+                "s3:GetEncryptionConfiguration",
+                "s3:ListBucket",
+                "s3:ListBucketVersions"
               ],
               "Effect": "Allow",
               "Resource": [
-                "*"
+                "arn:aws:s3:::placeholder-write-bucket"
-              ]
-            },
-            {
-              "Action": "kms:GenerateRandom",
-              "Effect": "Allow",
-              "Resource": [
-                "*"
               ]
             },
             {
@@ -1131,43 +1047,20 @@
               ]
             },
             {
-              "Action": [
+              "Action": "ec2:CreateTags",
-                "ec2:DescribeAccountAttributes",
-                "ec2:DescribeInstances",
-                "ec2:DescribeVolumes",
-                "ec2:DescribeVolumesModifications",
-                "ec2:DescribeTags"
-              ],
-              "Effect": "Allow",
-              "Resource": "*"
-            },
-            {
-              "Action": [
-                "ec2:CreateVolume"
-              ],
               "Condition": {
                 "StringEquals": {
-                  "aws:RequestTag/KubernetesCluster": "minimal.example.com"
+                  "ec2:CreateAction": [
+                    "CreateVolume",
+                    "CreateSnapshot"
+                  ]
                 }
               },
               "Effect": "Allow",
-              "Resource": "*"
+              "Resource": [
-            },
+                "arn:aws:ec2:*:*:volume/*",
-            {
+                "arn:aws:ec2:*:*:snapshot/*"
-              "Action": [
+              ]
-                "ec2:ModifyVolume",
-                "ec2:ModifyInstanceAttribute",
-                "ec2:AttachVolume",
-                "ec2:DeleteVolume",
-                "ec2:DetachVolume"
-              ],
-              "Condition": {
-                "StringEquals": {
-                  "aws:ResourceTag/KubernetesCluster": "minimal.example.com"
-                }
-              },
-              "Effect": "Allow",
-              "Resource": "*"
             },
             {
               "Action": "ec2:CreateTags",
@@ -1189,7 +1082,7 @@
               "Action": "ec2:DeleteTags",
               "Condition": {
                 "StringEquals": {
-                  "ec2:ResourceTag/KubernetesCluster": "minimal.example.com"
+                  "aws:ResourceTag/KubernetesCluster": "minimal.example.com"
                 }
               },
               "Effect": "Allow",
@@ -1200,14 +1093,77 @@
             },
             {
               "Action": [
+                "autoscaling:DescribeAutoScalingGroups",
+                "autoscaling:DescribeAutoScalingInstances",
+                "autoscaling:DescribeLaunchConfigurations",
+                "autoscaling:DescribeTags",
+                "ec2:CreateSecurityGroup",
+                "ec2:CreateTags",
+                "ec2:DescribeAccountAttributes",
+                "ec2:DescribeInstanceTypes",
+                "ec2:DescribeInstances",
+                "ec2:DescribeLaunchTemplateVersions",
+                "ec2:DescribeRegions",
+                "ec2:DescribeRouteTables",
+                "ec2:DescribeSecurityGroups",
+                "ec2:DescribeSubnets",
+                "ec2:DescribeTags",
+                "ec2:DescribeVolumes",
+                "ec2:DescribeVolumesModifications",
+                "ec2:DescribeVpcs",
+                "elasticloadbalancing:AddTags",
+                "elasticloadbalancing:CreateListener",
+                "elasticloadbalancing:CreateTargetGroup",
+                "elasticloadbalancing:DescribeListeners",
+                "elasticloadbalancing:DescribeLoadBalancerAttributes",
+                "elasticloadbalancing:DescribeLoadBalancerPolicies",
+                "elasticloadbalancing:DescribeLoadBalancers",
+                "elasticloadbalancing:DescribeTargetGroups",
+                "elasticloadbalancing:DescribeTargetHealth",
+                "elasticloadbalancing:RegisterTargets",
+                "iam:GetServerCertificate",
+                "iam:ListServerCertificates",
+                "kms:DescribeKey",
+                "kms:GenerateRandom"
+              ],
+              "Effect": "Allow",
+              "Resource": "*"
+            },
+            {
+              "Action": [
+                "autoscaling:SetDesiredCapacity",
+                "autoscaling:TerminateInstanceInAutoScalingGroup",
                 "ec2:AttachVolume",
+                "ec2:AuthorizeSecurityGroupIngress",
+                "ec2:DeleteRoute",
+                "ec2:DeleteSecurityGroup",
                 "ec2:DeleteVolume",
                 "ec2:DetachVolume",
-                "ec2:RevokeSecurityGroupIngress"
+                "ec2:ModifyInstanceAttribute",
+                "ec2:ModifyVolume",
+                "ec2:RevokeSecurityGroupIngress",
+                "elasticloadbalancing:AddTags",
+                "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer",
+                "elasticloadbalancing:AttachLoadBalancerToSubnets",
+                "elasticloadbalancing:ConfigureHealthCheck",
+                "elasticloadbalancing:DeleteListener",
+                "elasticloadbalancing:DeleteLoadBalancer",
+                "elasticloadbalancing:DeleteLoadBalancerListeners",
+                "elasticloadbalancing:DeleteTargetGroup",
+                "elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
+                "elasticloadbalancing:DeregisterTargets",
+                "elasticloadbalancing:DetachLoadBalancerFromSubnets",
+                "elasticloadbalancing:ModifyListener",
+                "elasticloadbalancing:ModifyLoadBalancerAttributes",
+                "elasticloadbalancing:ModifyTargetGroup",
+                "elasticloadbalancing:RegisterInstancesWithLoadBalancer",
+                "elasticloadbalancing:RegisterTargets",
+                "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer",
+                "elasticloadbalancing:SetLoadBalancerPoliciesOfListener"
               ],
               "Condition": {
                 "StringEquals": {
-                  "ec2:ResourceTag/KubernetesCluster": "minimal.example.com"
+                  "aws:ResourceTag/KubernetesCluster": "minimal.example.com"
                 }
               },
               "Effect": "Allow",
@@ -1215,29 +1171,21 @@
             },
             {
               "Action": [
-                "autoscaling:SetDesiredCapacity",
+                "ec2:CreateSecurityGroup",
-                "autoscaling:TerminateInstanceInAutoScalingGroup"
+                "ec2:CreateVolume",
+                "elasticloadbalancing:CreateListener",
+                "elasticloadbalancing:CreateLoadBalancer",
+                "elasticloadbalancing:CreateLoadBalancerListeners",
+                "elasticloadbalancing:CreateLoadBalancerPolicy",
+                "elasticloadbalancing:CreateTargetGroup"
               ],
               "Condition": {
                 "StringEquals": {
-                  "autoscaling:ResourceTag/KubernetesCluster": "minimal.example.com"
+                  "aws:RequestTag/KubernetesCluster": "minimal.example.com"
                 }
               },
               "Effect": "Allow",
-              "Resource": [
+              "Resource": "*"
-                "*"
-              ]
-            },
-            {
-              "Action": [
-                "autoscaling:DescribeAutoScalingGroups",
-                "autoscaling:DescribeAutoScalingInstances",
-                "autoscaling:DescribeLaunchConfigurations"
-              ],
-              "Effect": "Allow",
-              "Resource": [
-                "*"
-              ]
             }
           ],
           "Version": "2012-10-17"
@@ -1257,27 +1205,39 @@
           "Statement": [
             {
               "Action": [
-                "ec2:DescribeInstances",
+                "s3:Get*"
-                "ec2:DescribeRegions"
               ],
               "Effect": "Allow",
               "Resource": [
-                "*"
+                "arn:aws:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/addons/*",
+                "arn:aws:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/cluster-completed.spec",
+                "arn:aws:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/igconfig/node/*",
+                "arn:aws:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/secrets/dockerconfig"
               ]
             },
             {
-              "Action": "autoscaling:DescribeAutoScalingInstances",
+              "Action": [
+                "s3:GetBucketLocation",
+                "s3:GetEncryptionConfiguration",
+                "s3:ListBucket",
+                "s3:ListBucketVersions"
+              ],
               "Effect": "Allow",
               "Resource": [
-                "*"
+                "arn:aws:s3:::placeholder-read-bucket"
               ]
             },
             {
-              "Action": "kms:GenerateRandom",
+              "Action": [
+                "autoscaling:DescribeAutoScalingInstances",
+                "ec2:DescribeInstanceTypes",
+                "ec2:DescribeInstances",
+                "iam:GetServerCertificate",
+                "iam:ListServerCertificates",
+                "kms:GenerateRandom"
+              ],
               "Effect": "Allow",
-              "Resource": [
+              "Resource": "*"
-                "*"
-              ]
             }
           ],
           "Version": "2012-10-17"

tests/integration/update_cluster/external_dns/cloudformation.json.extracted.yaml

@@ -131,7 +131,7 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
   containerRuntime: containerd
   containerd:
     logLevel: info
-    version: 1.4.6
+    version: 1.4.11
   docker:
     skipInstall: true
   encryptionConfig: null
@@ -153,7 +153,6 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
     - NamespaceLifecycle
     - LimitRanger
     - ServiceAccount
-    - PersistentVolumeLabel
     - DefaultStorageClass
     - DefaultTolerationSeconds
     - MutatingAdmissionWebhook
@@ -241,98 +240,11 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
   __EOF_CLUSTER_SPEC
 
   cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
-  Assets:
-    amd64:
-    - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet
-    - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl
-    - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
-    - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz
-    - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64
-    - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64
-    arm64:
-    - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet
-    - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl
-    - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
-    - be8c9a5a06ebec8fb1d36e867cd00fb5777746a9812a0cae2966778ff899c525@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.7.tgz
-    - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64
-    - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64
-  AuxConfigHash: /O5IS/dGo83lv2DbWn4k91OYfuOqtO79vjf5pD1DQlI=
-  CAs:
-    ca: |
-      -----BEGIN CERTIFICATE-----
-      MIIBaDCCARKgAwIBAgIMFoq6Pex4lTCM8fOIMA0GCSqGSIb3DQEBCwUAMBUxEzAR
-      BgNVBAMTCmt1YmVybmV0ZXMwHhcNMjEwNjE5MjI0MzEwWhcNMzEwNjE5MjI0MzEw
-      WjAVMRMwEQYDVQQDEwprdWJlcm5ldGVzMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJB
-      ANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m49pAEeIMsvwvD
-      MaLsb2v6JvXe0qvCmueU+/sCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud
-      EwEB/wQFMAMBAf8wHQYDVR0OBBYEFCOW3hR7ngBsk9aUOlEznWzH494EMA0GCSqG
-      SIb3DQEBCwUAA0EAVnZzkiku07kQFGAEXzWI6aZnAbzSoClYskEzCBMrOmdadjVp
-      VWcz76FwFlyd5jhzOJ49eMcVusSotKv2ZGimcA==
-      -----END CERTIFICATE-----
-      -----BEGIN CERTIFICATE-----
-      MIIBaDCCARKgAwIBAgIMFoq6PeyECsgUTfc2MA0GCSqGSIb3DQEBCwUAMBUxEzAR
-      BgNVBAMTCmt1YmVybmV0ZXMwHhcNMjEwNjE5MjI0MzEwWhcNMzEwNjE5MjI0MzEw
-      WjAVMRMwEQYDVQQDEwprdWJlcm5ldGVzMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJB
-      AKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCCrs6AdVRZNZ26
-      A6uHArhi33c2kHQkCjyLA7sCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud
-      EwEB/wQFMAMBAf8wHQYDVR0OBBYEFIT28RJlG8FTgmvn2YMa3hYX+u1BMA0GCSqG
-      SIb3DQEBCwUAA0EAKuaE5wKMP26AyfxkWu83iHoTPFtdjabXF0JcyPy0ijQZxfJq
-      9xc2CkttvgaDtT4H+E/ryQ3iq6kSfEYYPi8c0w==
-      -----END CERTIFICATE-----
   CloudProvider: aws
-  ClusterName: minimal.example.com
   ConfigBase: memfs://clusters.example.com/minimal.example.com
   InstanceGroupName: master-us-test-1a
   InstanceGroupRole: Master
-  KeypairIDs:
+  NodeupConfigHash: W80rsOsSg9anEDGL5fL40gNx08rzxsiXwJo/qjhA6jA=
-    ca: "6976381481633145814258938760"
-  KubeletConfig:
-    anonymousAuth: false
-    cgroupDriver: systemd
-    cgroupRoot: /
-    cloudProvider: aws
-    clusterDNS: 100.64.0.10
-    clusterDomain: cluster.local
-    enableDebuggingHandlers: true
-    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
-    hostnameOverride: '@aws'
-    kubeconfigPath: /var/lib/kubelet/kubeconfig
-    logLevel: 2
-    networkPluginName: cni
-    nodeLabels:
-      kops.k8s.io/kops-controller-pki: ""
-      kubernetes.io/role: master
-      node-role.kubernetes.io/control-plane: ""
-      node-role.kubernetes.io/master: ""
-      node.kubernetes.io/exclude-from-external-load-balancers: ""
-    nonMasqueradeCIDR: 100.64.0.0/10
-    podManifestPath: /etc/kubernetes/manifests
-    registerSchedulable: false
-  UpdatePolicy: automatic
-  channels:
-  - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml
-  containerdConfig: |
-    version = 2
-
-    [plugins]
-
-      [plugins."io.containerd.grpc.v1.cri"]
-
-        [plugins."io.containerd.grpc.v1.cri".containerd]
-
-          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
-
-            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
-              runtime_type = "io.containerd.runc.v2"
-
-              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
-                SystemdCgroup = true
-  etcdManifests:
-  - memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml
-  - memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml
-  staticManifests:
-  - key: kube-apiserver-healthcheck
-    path: manifests/static/kube-apiserver-healthcheck.yaml
 
   __EOF_KUBE_ENV
 
@@ -471,7 +383,7 @@ Resources.AWSEC2LaunchTemplatenodesminimalexamplecom.Properties.LaunchTemplateDa
   containerRuntime: containerd
   containerd:
     logLevel: info
-    version: 1.4.6
+    version: 1.4.11
   docker:
     skipInstall: true
   kubeProxy:
@@ -499,83 +411,11 @@ Resources.AWSEC2LaunchTemplatenodesminimalexamplecom.Properties.LaunchTemplateDa
   __EOF_CLUSTER_SPEC
 
   cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
-  Assets:
-    amd64:
-    - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet
-    - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl
-    - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
-    - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz
-    arm64:
-    - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet
-    - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl
-    - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
-    - be8c9a5a06ebec8fb1d36e867cd00fb5777746a9812a0cae2966778ff899c525@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.7.tgz
-  AuxConfigHash: /O5IS/dGo83lv2DbWn4k91OYfuOqtO79vjf5pD1DQlI=
-  CAs:
-    ca: |
-      -----BEGIN CERTIFICATE-----
-      MIIBaDCCARKgAwIBAgIMFoq6Pex4lTCM8fOIMA0GCSqGSIb3DQEBCwUAMBUxEzAR
-      BgNVBAMTCmt1YmVybmV0ZXMwHhcNMjEwNjE5MjI0MzEwWhcNMzEwNjE5MjI0MzEw
-      WjAVMRMwEQYDVQQDEwprdWJlcm5ldGVzMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJB
-      ANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m49pAEeIMsvwvD
-      MaLsb2v6JvXe0qvCmueU+/sCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud
-      EwEB/wQFMAMBAf8wHQYDVR0OBBYEFCOW3hR7ngBsk9aUOlEznWzH494EMA0GCSqG
-      SIb3DQEBCwUAA0EAVnZzkiku07kQFGAEXzWI6aZnAbzSoClYskEzCBMrOmdadjVp
-      VWcz76FwFlyd5jhzOJ49eMcVusSotKv2ZGimcA==
-      -----END CERTIFICATE-----
-      -----BEGIN CERTIFICATE-----
-      MIIBaDCCARKgAwIBAgIMFoq6PeyECsgUTfc2MA0GCSqGSIb3DQEBCwUAMBUxEzAR
-      BgNVBAMTCmt1YmVybmV0ZXMwHhcNMjEwNjE5MjI0MzEwWhcNMzEwNjE5MjI0MzEw
-      WjAVMRMwEQYDVQQDEwprdWJlcm5ldGVzMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJB
-      AKOE64nZbH+GM91AIrqf7HEk4hvzqsZFFtxc+8xir1XC3mI/RhCCrs6AdVRZNZ26
-      A6uHArhi33c2kHQkCjyLA7sCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud
-      EwEB/wQFMAMBAf8wHQYDVR0OBBYEFIT28RJlG8FTgmvn2YMa3hYX+u1BMA0GCSqG
-      SIb3DQEBCwUAA0EAKuaE5wKMP26AyfxkWu83iHoTPFtdjabXF0JcyPy0ijQZxfJq
-      9xc2CkttvgaDtT4H+E/ryQ3iq6kSfEYYPi8c0w==
-      -----END CERTIFICATE-----
   CloudProvider: aws
-  ClusterName: minimal.example.com
   ConfigBase: memfs://clusters.example.com/minimal.example.com
   InstanceGroupName: nodes
   InstanceGroupRole: Node
-  KeypairIDs: {}
+  NodeupConfigHash: /xc0hHryd87G4ro46rYwe9mCe7j9bWZRSru2pMpqqNs=
-  KubeletConfig:
-    anonymousAuth: false
-    cgroupDriver: systemd
-    cgroupRoot: /
-    cloudProvider: aws
-    clusterDNS: 100.64.0.10
-    clusterDomain: cluster.local
-    enableDebuggingHandlers: true
-    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
-    hostnameOverride: '@aws'
-    kubeconfigPath: /var/lib/kubelet/kubeconfig
-    logLevel: 2
-    networkPluginName: cni
-    nodeLabels:
-      kubernetes.io/role: node
-      node-role.kubernetes.io/node: ""
-    nonMasqueradeCIDR: 100.64.0.0/10
-    podManifestPath: /etc/kubernetes/manifests
-  UpdatePolicy: automatic
-  channels:
-  - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml
-  containerdConfig: |
-    version = 2
-
-    [plugins]
-
-      [plugins."io.containerd.grpc.v1.cri"]
-
-        [plugins."io.containerd.grpc.v1.cri".containerd]
-
-          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
-
-            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
-              runtime_type = "io.containerd.runc.v2"
-
-              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
-                SystemdCgroup = true
 
   __EOF_KUBE_ENV
 

tests/integration/update_cluster/external_dns/data/aws_s3_bucket_object_minimal.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content

@@ -1,121 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: dns-controller.addons.k8s.io
-    k8s-app: dns-controller
-    version: v1.22.0-alpha.2
-  name: dns-controller
-  namespace: kube-system
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      k8s-app: dns-controller
-  strategy:
-    type: Recreate
-  template:
-    metadata:
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ""
-      labels:
-        k8s-addon: dns-controller.addons.k8s.io
-        k8s-app: dns-controller
-        version: v1.22.0-alpha.2
-    spec:
-      containers:
-      - command:
-        - /dns-controller
-        - --watch-ingress=false
-        - --dns=aws-route53
-        - --zone=*/Z1AFAKE1ZON3YO
-        - --zone=*/*
-        - -v=2
-        env:
-        - name: KUBERNETES_SERVICE_HOST
-          value: 127.0.0.1
-        image: k8s.gcr.io/kops/dns-controller:1.23.0-alpha.1
-        name: dns-controller
-        resources:
-          requests:
-            cpu: 50m
-            memory: 50Mi
-        securityContext:
-          runAsNonRoot: true
-      dnsPolicy: Default
-      hostNetwork: true
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      priorityClassName: system-cluster-critical
-      serviceAccount: dns-controller
-      tolerations:
-      - operator: Exists
-
----
-
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: dns-controller.addons.k8s.io
-  name: dns-controller
-  namespace: kube-system
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: dns-controller.addons.k8s.io
-  name: kops:dns-controller
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - endpoints
-  - services
-  - pods
-  - ingress
-  - nodes
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - networking
-  resources:
-  - ingresses
-  verbs:
-  - get
-  - list
-  - watch
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: dns-controller.addons.k8s.io
-  name: kops:dns-controller
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: kops:dns-controller
-subjects:
-- apiGroup: rbac.authorization.k8s.io
-  kind: User
-  name: system:serviceaccount:kube-system:dns-controller

tests/integration/update_cluster/external_dns_irsa/data/aws_s3_bucket_object_minimal.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content

@@ -1,121 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: dns-controller.addons.k8s.io
-    k8s-app: dns-controller
-    version: v1.22.0-alpha.2
-  name: dns-controller
-  namespace: kube-system
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      k8s-app: dns-controller
-  strategy:
-    type: Recreate
-  template:
-    metadata:
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ""
-      labels:
-        k8s-addon: dns-controller.addons.k8s.io
-        k8s-app: dns-controller
-        version: v1.22.0-alpha.2
-    spec:
-      containers:
-      - command:
-        - /dns-controller
-        - --watch-ingress=false
-        - --dns=aws-route53
-        - --zone=*/Z1AFAKE1ZON3YO
-        - --zone=*/*
-        - -v=2
-        env:
-        - name: KUBERNETES_SERVICE_HOST
-          value: 127.0.0.1
-        image: k8s.gcr.io/kops/dns-controller:1.23.0-alpha.1
-        name: dns-controller
-        resources:
-          requests:
-            cpu: 50m
-            memory: 50Mi
-        securityContext:
-          runAsNonRoot: true
-      dnsPolicy: Default
-      hostNetwork: true
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      priorityClassName: system-cluster-critical
-      serviceAccount: dns-controller
-      tolerations:
-      - operator: Exists
-
----
-
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: dns-controller.addons.k8s.io
-  name: dns-controller
-  namespace: kube-system
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: dns-controller.addons.k8s.io
-  name: kops:dns-controller
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - endpoints
-  - services
-  - pods
-  - ingress
-  - nodes
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - networking
-  resources:
-  - ingresses
-  verbs:
-  - get
-  - list
-  - watch
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: dns-controller.addons.k8s.io
-  name: kops:dns-controller
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: kops:dns-controller
-subjects:
-- apiGroup: rbac.authorization.k8s.io
-  kind: User
-  name: system:serviceaccount:kube-system:dns-controller

tests/integration/update_cluster/irsa/data/aws_iam_role_dns-controller.kube-system.sa.minimal.example.com_policy

@@ -1,17 +0,0 @@
-{
-  "Statement": [
-    {
-      "Action": "sts:AssumeRoleWithWebIdentity",
-      "Condition": {
-        "StringEquals": {
-          "api.internal.minimal.example.com:sub": "system:serviceaccount:kube-system:dns-controller"
-        }
-      },
-      "Effect": "Allow",
-      "Principal": {
-        "Federated": "arn:aws:iam::123456789012:oidc-provider/api.internal.minimal.example.com"
-      }
-    }
-  ],
-  "Version": "2012-10-17"
-}

tests/integration/update_cluster/irsa/data/aws_iam_role_policy_dns-controller.kube-system.sa.minimal.example.com_policy

@@ -1,34 +0,0 @@
-{
-  "Statement": [
-    {
-      "Action": [
-        "route53:ChangeResourceRecordSets",
-        "route53:ListResourceRecordSets",
-        "route53:GetHostedZone"
-      ],
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO"
-      ]
-    },
-    {
-      "Action": [
-        "route53:GetChange"
-      ],
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws:route53:::change/*"
-      ]
-    },
-    {
-      "Action": [
-        "route53:ListHostedZones"
-      ],
-      "Effect": "Allow",
-      "Resource": [
-        "*"
-      ]
-    }
-  ],
-  "Version": "2012-10-17"
-}

tests/integration/update_cluster/nvidia/cloudformation.json

@@ -124,6 +124,11 @@
             "Value": "nodes.minimal.example.com",
             "PropagateAtLaunch": true
           },
+          {
+            "Key": "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/gpu",
+            "Value": "1",
+            "PropagateAtLaunch": true
+          },
           {
             "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role",
             "Value": "node",
@@ -379,7 +384,7 @@
             }
           },
           "ImageId": "ami-12345678",
-          "InstanceType": "t2.medium",
+          "InstanceType": "g4dn.xlarge",
           "KeyName": "kubernetes.minimal.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57",
           "MetadataOptions": {
             "HttpPutResponseHopLimit": 1,
@@ -413,6 +418,10 @@
                   "Key": "Name",
                   "Value": "nodes.minimal.example.com"
                 },
+                {
+                  "Key": "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/gpu",
+                  "Value": "1"
+                },
                 {
                   "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role",
                   "Value": "node"
@@ -446,6 +455,10 @@
                   "Key": "Name",
                   "Value": "nodes.minimal.example.com"
                 },
+                {
+                  "Key": "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/gpu",
+                  "Value": "1"
+                },
                 {
                   "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role",
                   "Value": "node"
@@ -965,42 +978,6 @@
                 "*"
               ]
             },
-            {
-              "Action": "ec2:CreateTags",
-              "Condition": {
-                "StringEquals": {
-                  "ec2:CreateAction": [
-                    "CreateVolume",
-                    "CreateSnapshot"
-                  ]
-                }
-              },
-              "Effect": "Allow",
-              "Resource": [
-                "arn:aws:ec2:*:*:volume/*",
-                "arn:aws:ec2:*:*:snapshot/*"
-              ]
-            },
-            {
-              "Action": [
-                "elasticloadbalancing:CreateLoadBalancer",
-                "elasticloadbalancing:CreateLoadBalancerPolicy",
-                "elasticloadbalancing:CreateLoadBalancerListeners",
-                "ec2:CreateSecurityGroup",
-                "ec2:CreateVolume",
-                "elasticloadbalancing:CreateListener",
-                "elasticloadbalancing:CreateTargetGroup"
-              ],
-              "Condition": {
-                "StringEquals": {
-                  "aws:RequestTag/KubernetesCluster": "minimal.example.com"
-                }
-              },
-              "Effect": "Allow",
-              "Resource": [
-                "*"
-              ]
-            },
             {
               "Action": [
                 "s3:Get*"
@@ -1074,7 +1051,8 @@
             },
             {
               "Action": [
-                "route53:ListHostedZones"
+                "route53:ListHostedZones",
+                "route53:ListTagsForResource"
               ],
               "Effect": "Allow",
               "Resource": [
@@ -1082,16 +1060,20 @@
               ]
             },
             {
-              "Action": [
+              "Action": "ec2:CreateTags",
-                "ec2:CreateVolume"
-              ],
               "Condition": {
                 "StringEquals": {
-                  "aws:RequestTag/KubernetesCluster": "minimal.example.com"
+                  "ec2:CreateAction": [
+                    "CreateVolume",
+                    "CreateSnapshot"
+                  ]
                 }
               },
               "Effect": "Allow",
-              "Resource": "*"
+              "Resource": [
+                "arn:aws:ec2:*:*:volume/*",
+                "arn:aws:ec2:*:*:snapshot/*"
+              ]
             },
             {
               "Action": "ec2:CreateTags",
@@ -1133,6 +1115,7 @@
                 "ec2:DescribeAccountAttributes",
                 "ec2:DescribeInstanceTypes",
                 "ec2:DescribeInstances",
+                "ec2:DescribeLaunchTemplateVersions",
                 "ec2:DescribeRegions",
                 "ec2:DescribeRouteTables",
                 "ec2:DescribeSecurityGroups",
@@ -1141,12 +1124,16 @@
                 "ec2:DescribeVolumes",
                 "ec2:DescribeVolumesModifications",
                 "ec2:DescribeVpcs",
+                "elasticloadbalancing:AddTags",
+                "elasticloadbalancing:CreateListener",
+                "elasticloadbalancing:CreateTargetGroup",
                 "elasticloadbalancing:DescribeListeners",
                 "elasticloadbalancing:DescribeLoadBalancerAttributes",
                 "elasticloadbalancing:DescribeLoadBalancerPolicies",
                 "elasticloadbalancing:DescribeLoadBalancers",
                 "elasticloadbalancing:DescribeTargetGroups",
                 "elasticloadbalancing:DescribeTargetHealth",
+                "elasticloadbalancing:RegisterTargets",
                 "iam:GetServerCertificate",
                 "iam:ListServerCertificates",
                 "kms:DescribeKey",
@@ -1194,6 +1181,24 @@
               },
               "Effect": "Allow",
               "Resource": "*"
+            },
+            {
+              "Action": [
+                "ec2:CreateSecurityGroup",
+                "ec2:CreateVolume",
+                "elasticloadbalancing:CreateListener",
+                "elasticloadbalancing:CreateLoadBalancer",
+                "elasticloadbalancing:CreateLoadBalancerListeners",
+                "elasticloadbalancing:CreateLoadBalancerPolicy",
+                "elasticloadbalancing:CreateTargetGroup"
+              ],
+              "Condition": {
+                "StringEquals": {
+                  "aws:RequestTag/KubernetesCluster": "minimal.example.com"
+                }
+              },
+              "Effect": "Allow",
+              "Resource": "*"
             }
           ],
           "Version": "2012-10-17"
@@ -1220,7 +1225,6 @@
                 "arn:aws:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/addons/*",
                 "arn:aws:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/cluster-completed.spec",
                 "arn:aws:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/igconfig/node/*",
-                "arn:aws:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/pki/ssh/*",
                 "arn:aws:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/secrets/dockerconfig"
               ]
             },

tests/integration/update_cluster/nvidia/cloudformation.json.extracted.yaml

@@ -131,7 +131,10 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
   containerRuntime: containerd
   containerd:
     logLevel: info
-    version: 1.4.6
+    nvidiaGPU:
+      enabled: true
+      package: nvidia-headless-460-server
+    version: 1.4.11
   docker:
     skipInstall: true
   encryptionConfig: null
@@ -153,7 +156,6 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
     - NamespaceLifecycle
     - LimitRanger
     - ServiceAccount
-    - PersistentVolumeLabel
     - DefaultStorageClass
     - DefaultTolerationSeconds
     - MutatingAdmissionWebhook
@@ -245,7 +247,7 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.
   ConfigBase: memfs://clusters.example.com/minimal.example.com
   InstanceGroupName: master-us-test-1a
   InstanceGroupRole: Master
-  NodeupConfigHash: M484KjV19wtMDGWwO7cf5eb6cEAQW2nh2IqShjgpbHU=
+  NodeupConfigHash: N7p2va9tPSRTapQ/idYkE3zTSMGc04Er5uSryOA+EJY=
 
   __EOF_KUBE_ENV
 
@@ -384,7 +386,10 @@ Resources.AWSEC2LaunchTemplatenodesminimalexamplecom.Properties.LaunchTemplateDa
   containerRuntime: containerd
   containerd:
     logLevel: info
-    version: 1.4.6
+    nvidiaGPU:
+      enabled: true
+      package: nvidia-headless-460-server
+    version: 1.4.11
   docker:
     skipInstall: true
   kubeProxy:
@@ -416,7 +421,7 @@ Resources.AWSEC2LaunchTemplatenodesminimalexamplecom.Properties.LaunchTemplateDa
   ConfigBase: memfs://clusters.example.com/minimal.example.com
   InstanceGroupName: nodes
   InstanceGroupRole: Node
-  NodeupConfigHash: A0AyiJo03pbqluaXrVtbacjofP1NmBexAl0w2y4oS5o=
+  NodeupConfigHash: OyVtiYq+TGC8i2VY0pYPlgnx1r3FYU53FSn5Y0bLIQ4=
 
   __EOF_KUBE_ENV
 

tests/integration/update_cluster/vfs-said/data/aws_iam_role_dns-controller.kube-system.sa.minimal.example.com_policy

@@ -1,17 +0,0 @@
-{
-  "Statement": [
-    {
-      "Action": "sts:AssumeRoleWithWebIdentity",
-      "Condition": {
-        "StringEquals": {
-          "discovery.example.com/minimal.example.com:sub": "system:serviceaccount:kube-system:dns-controller"
-        }
-      },
-      "Effect": "Allow",
-      "Principal": {
-        "Federated": "arn:aws:iam::123456789012:oidc-provider/discovery.example.com/minimal.example.com"
-      }
-    }
-  ],
-  "Version": "2012-10-17"
-}

tests/integration/update_cluster/vfs-said/data/aws_iam_role_policy_dns-controller.kube-system.sa.minimal.example.com_policy

@@ -1,34 +0,0 @@
-{
-  "Statement": [
-    {
-      "Action": [
-        "route53:ChangeResourceRecordSets",
-        "route53:ListResourceRecordSets",
-        "route53:GetHostedZone"
-      ],
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO"
-      ]
-    },
-    {
-      "Action": [
-        "route53:GetChange"
-      ],
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws:route53:::change/*"
-      ]
-    },
-    {
-      "Action": [
-        "route53:ListHostedZones"
-      ],
-      "Effect": "Allow",
-      "Resource": [
-        "*"
-      ]
-    }
-  ],
-  "Version": "2012-10-17"
-}