Bump k8s version in integration tests using 1.19

cmd/kops/integration_test.go

@@ -180,6 +180,22 @@ func (i *integrationTest) withAddons(addons ...string) *integrationTest {
 	return i
 }
 
+func (i integrationTest) withDefaultServiceAccountRoles24() *integrationTest {
+	return i.withServiceAccountRole("dns-controller.kube-system", true).
+		withServiceAccountRole("aws-cloud-controller-manager.kube-system", true).
+		withServiceAccountRole("ebs-csi-controller-sa.kube-system", true)
+}
+
+// withDefaultAddons24 adds the default addons for an AWS cluster running k8s 1.24
+func (i integrationTest) withDefaultAddons24() *integrationTest {
+	return i.withAddons(
+		awsCCMAddon,
+		awsEBSCSIAddon,
+		dnsControllerAddon,
+		leaderElectionAddon,
+	)
+}
+
 const (
 	awsCCMAddon         = "aws-cloud-controller.addons.k8s.io-k8s-1.18"
 	awsEBSCSIAddon      = "aws-ebs-csi-driver.addons.k8s.io-k8s-1.17"
@@ -431,8 +447,8 @@ func TestPrivateCilium(t *testing.T) {
 func TestPrivateCilium2(t *testing.T) {
 	newIntegrationTest("privatecilium.example.com", "privatecilium2").
 		withPrivate().
-		withAddons("networking.cilium.io-k8s-1.12", dnsControllerAddon).
+		withDefaultAddons24().
-		withKubeDNS().
+		withAddons("networking.cilium.io-k8s-1.12").
 		runTestTerraformAWS(t)
 	newIntegrationTest("privatecilium.example.com", "privatecilium2").
 		withPrivate().
@@ -516,10 +532,9 @@ func TestPrivateDns2(t *testing.T) {
 // TestDiscoveryFeatureGate runs a simple configuration, but with UseServiceAccountExternalPermissions and the ServiceAccountIssuerDiscovery feature gate enabled
 func TestDiscoveryFeatureGate(t *testing.T) {
 	newIntegrationTest("minimal.example.com", "public-jwks-apiserver").
-		withServiceAccountRole("dns-controller.kube-system", true).
+		withDefaultServiceAccountRoles24().
-		withAddons(dnsControllerAddon).
+		withDefaultAddons24().
 		withOIDCDiscovery().
-		withKubeDNS().
 		runTestTerraformAWS(t)
 }
 

tests/integration/update_cluster/privatecilium2/cloudformation.json

@@ -33,11 +33,6 @@
             "Value": "bastion.privatecilium.example.com",
             "PropagateAtLaunch": true
           },
-          {
-            "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role",
-            "Value": "node",
-            "PropagateAtLaunch": true
-          },
           {
             "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node",
             "Value": "",
@@ -119,21 +114,11 @@
             "Value": "",
             "PropagateAtLaunch": true
           },
-          {
-            "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role",
-            "Value": "master",
-            "PropagateAtLaunch": true
-          },
           {
             "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane",
             "Value": "",
             "PropagateAtLaunch": true
           },
-          {
-            "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master",
-            "Value": "",
-            "PropagateAtLaunch": true
-          },
           {
             "Key": "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers",
             "Value": "",
@@ -210,11 +195,6 @@
             "Value": "nodes.privatecilium.example.com",
             "PropagateAtLaunch": true
           },
-          {
-            "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role",
-            "Value": "node",
-            "PropagateAtLaunch": true
-          },
           {
             "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node",
             "Value": "",
@@ -373,10 +353,6 @@
                   "Key": "Name",
                   "Value": "bastion.privatecilium.example.com"
                 },
-                {
-                  "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role",
-                  "Value": "node"
-                },
                 {
                   "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node",
                   "Value": ""
@@ -406,10 +382,6 @@
                   "Key": "Name",
                   "Value": "bastion.privatecilium.example.com"
                 },
-                {
-                  "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role",
-                  "Value": "node"
-                },
                 {
                   "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node",
                   "Value": ""
@@ -499,18 +471,10 @@
                   "Key": "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki",
                   "Value": ""
                 },
-                {
-                  "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role",
-                  "Value": "master"
-                },
                 {
                   "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane",
                   "Value": ""
                 },
-                {
-                  "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master",
-                  "Value": ""
-                },
                 {
                   "Key": "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers",
                   "Value": ""
@@ -544,18 +508,10 @@
                   "Key": "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki",
                   "Value": ""
                 },
-                {
-                  "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role",
-                  "Value": "master"
-                },
                 {
                   "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane",
                   "Value": ""
                 },
-                {
-                  "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master",
-                  "Value": ""
-                },
                 {
                   "Key": "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers",
                   "Value": ""
@@ -637,10 +593,6 @@
                   "Key": "Name",
                   "Value": "nodes.privatecilium.example.com"
                 },
-                {
-                  "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role",
-                  "Value": "node"
-                },
                 {
                   "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node",
                   "Value": ""
@@ -670,10 +622,6 @@
                   "Key": "Name",
                   "Value": "nodes.privatecilium.example.com"
                 },
-                {
-                  "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role",
-                  "Value": "node"
-                },
                 {
                   "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node",
                   "Value": ""
@@ -1708,39 +1656,6 @@
                 "*"
               ]
             },
-            {
-              "Action": "ec2:CreateTags",
-              "Condition": {
-                "StringEquals": {
-                  "aws:RequestTag/KubernetesCluster": "privatecilium.example.com",
-                  "ec2:CreateAction": [
-                    "CreateSecurityGroup"
-                  ]
-                }
-              },
-              "Effect": "Allow",
-              "Resource": [
-                "arn:aws-test:ec2:*:*:security-group/*"
-              ]
-            },
-            {
-              "Action": [
-                "ec2:CreateTags",
-                "ec2:DeleteTags"
-              ],
-              "Condition": {
-                "Null": {
-                  "aws:RequestTag/KubernetesCluster": "true"
-                },
-                "StringEquals": {
-                  "aws:ResourceTag/KubernetesCluster": "privatecilium.example.com"
-                }
-              },
-              "Effect": "Allow",
-              "Resource": [
-                "arn:aws-test:ec2:*:*:security-group/*"
-              ]
-            },
             {
               "Action": "ec2:CreateTags",
               "Condition": {
@@ -1777,19 +1692,45 @@
                 "arn:aws-test:ec2:*:*:snapshot/*"
               ]
             },
+            {
+              "Action": "ec2:CreateTags",
+              "Condition": {
+                "StringEquals": {
+                  "aws:RequestTag/KubernetesCluster": "privatecilium.example.com",
+                  "ec2:CreateAction": [
+                    "CreateSecurityGroup"
+                  ]
+                }
+              },
+              "Effect": "Allow",
+              "Resource": [
+                "arn:aws-test:ec2:*:*:security-group/*"
+              ]
+            },
+            {
+              "Action": [
+                "ec2:CreateTags",
+                "ec2:DeleteTags"
+              ],
+              "Condition": {
+                "Null": {
+                  "aws:RequestTag/KubernetesCluster": "true"
+                },
+                "StringEquals": {
+                  "aws:ResourceTag/KubernetesCluster": "privatecilium.example.com"
+                }
+              },
+              "Effect": "Allow",
+              "Resource": [
+                "arn:aws-test:ec2:*:*:security-group/*"
+              ]
+            },
             {
               "Action": [
                 "autoscaling:DescribeAutoScalingGroups",
                 "autoscaling:DescribeAutoScalingInstances",
                 "autoscaling:DescribeLaunchConfigurations",
                 "autoscaling:DescribeTags",
-                "ec2:AttachVolume",
-                "ec2:AuthorizeSecurityGroupIngress",
-                "ec2:CreateSecurityGroup",
-                "ec2:CreateTags",
-                "ec2:DeleteRoute",
-                "ec2:DeleteSecurityGroup",
-                "ec2:DeleteVolume",
                 "ec2:DescribeAccountAttributes",
                 "ec2:DescribeInstanceTypes",
                 "ec2:DescribeInstances",
@@ -1802,19 +1743,12 @@
                 "ec2:DescribeVolumes",
                 "ec2:DescribeVolumesModifications",
                 "ec2:DescribeVpcs",
-                "ec2:DetachVolume",
-                "ec2:ModifyInstanceAttribute",
-                "ec2:ModifyVolume",
-                "elasticloadbalancing:AddTags",
-                "elasticloadbalancing:CreateListener",
-                "elasticloadbalancing:CreateTargetGroup",
                 "elasticloadbalancing:DescribeListeners",
                 "elasticloadbalancing:DescribeLoadBalancerAttributes",
                 "elasticloadbalancing:DescribeLoadBalancerPolicies",
                 "elasticloadbalancing:DescribeLoadBalancers",
                 "elasticloadbalancing:DescribeTargetGroups",
                 "elasticloadbalancing:DescribeTargetHealth",
-                "elasticloadbalancing:RegisterTargets",
                 "iam:GetServerCertificate",
                 "iam:ListServerCertificates",
                 "kms:DescribeKey",

tests/integration/update_cluster/privatecilium2/cloudformation.json.extracted.yaml

@@ -127,36 +127,30 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersprivateciliumexamplecom.Prope
   cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
   cloudConfig:
     awsEBSCSIDriver:
-      enabled: false
+      enabled: true
+      version: v1.6.2
     manageStorageClasses: true
-  containerRuntime: docker
+  containerRuntime: containerd
   containerd:
-    configOverride: |
-      disabled_plugins = ["cri"]
     logLevel: info
+    version: 1.6.6
   docker:
-    ipMasq: false
+    skipInstall: true
-    ipTables: false
-    logDriver: json-file
-    logLevel: info
-    logOpt:
-    - max-size=10m
-    - max-file=5
-    storage: overlay2,overlay,aufs
-    version: 19.03.15
   encryptionConfig: null
   etcdClusters:
     events:
-      version: 3.4.13
+      version: 3.5.4
     main:
-      version: 3.4.13
+      version: 3.5.4
   kubeAPIServer:
     allowPrivileged: true
     anonymousAuth: false
+    apiAudiences:
+    - kubernetes.svc.default
     apiServerCount: 1
     authorizationMode: AlwaysAllow
     bindAddress: 0.0.0.0
-    cloudProvider: aws
+    cloudProvider: external
     enableAdmissionPlugins:
     - NamespaceLifecycle
     - LimitRanger
@@ -171,8 +165,10 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersprivateciliumexamplecom.Prope
     - https://127.0.0.1:4001
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
-    image: registry.k8s.io/kube-apiserver:v1.19.0
+    featureGates:
-    insecurePort: 0
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
+    image: registry.k8s.io/kube-apiserver:v1.24.0
     kubeletPreferredAddressTypes:
     - InternalIP
     - Hostname
@@ -187,16 +183,21 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersprivateciliumexamplecom.Prope
     requestheaderUsernameHeaders:
     - X-Remote-User
     securePort: 443
+    serviceAccountIssuer: https://api.internal.privatecilium.example.com
+    serviceAccountJWKSURI: https://api.internal.privatecilium.example.com/openid/v1/jwks
     serviceClusterIPRange: 100.64.0.0/13
     storageBackend: etcd3
   kubeControllerManager:
     allocateNodeCIDRs: true
     attachDetachReconcileSyncPeriod: 1m0s
-    cloudProvider: aws
+    cloudProvider: external
     clusterCIDR: 100.96.0.0/11
     clusterName: privatecilium.example.com
     configureCloudRoutes: false
-    image: registry.k8s.io/kube-controller-manager:v1.19.0
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
+    image: registry.k8s.io/kube-controller-manager:v1.24.0
     leaderElection:
       leaderElect: true
     logLevel: 2
@@ -204,40 +205,55 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersprivateciliumexamplecom.Prope
   kubeProxy:
     clusterCIDR: 100.96.0.0/11
     cpuRequest: 100m
-    image: registry.k8s.io/kube-proxy:v1.19.0
+    image: registry.k8s.io/kube-proxy:v1.24.0
     logLevel: 2
   kubeScheduler:
-    image: registry.k8s.io/kube-scheduler:v1.19.0
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
+    image: registry.k8s.io/kube-scheduler:v1.24.0
     leaderElection:
       leaderElect: true
     logLevel: 2
   kubelet:
     anonymousAuth: false
+    cgroupDriver: systemd
     cgroupRoot: /
-    cloudProvider: aws
+    cloudProvider: external
     clusterDNS: 100.64.0.10
     clusterDomain: cluster.local
     enableDebuggingHandlers: true
     evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     kubeconfigPath: /var/lib/kubelet/kubeconfig
     logLevel: 2
-    networkPluginName: cni
     podInfraContainerImage: registry.k8s.io/pause:3.6
     podManifestPath: /etc/kubernetes/manifests
+    protectKernelDefaults: true
+    shutdownGracePeriod: 30s
+    shutdownGracePeriodCriticalPods: 10s
   masterKubelet:
     anonymousAuth: false
+    cgroupDriver: systemd
     cgroupRoot: /
-    cloudProvider: aws
+    cloudProvider: external
     clusterDNS: 100.64.0.10
     clusterDomain: cluster.local
     enableDebuggingHandlers: true
     evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     kubeconfigPath: /var/lib/kubelet/kubeconfig
     logLevel: 2
-    networkPluginName: cni
     podInfraContainerImage: registry.k8s.io/pause:3.6
     podManifestPath: /etc/kubernetes/manifests
+    protectKernelDefaults: true
     registerSchedulable: false
+    shutdownGracePeriod: 30s
+    shutdownGracePeriodCriticalPods: 10s
 
   __EOF_CLUSTER_SPEC
 
@@ -246,7 +262,7 @@ Resources.AWSEC2LaunchTemplatemasterustest1amastersprivateciliumexamplecom.Prope
   ConfigBase: memfs://clusters.example.com/privatecilium.example.com
   InstanceGroupName: master-us-test-1a
   InstanceGroupRole: Master
-  NodeupConfigHash: lQl5vPS8cn5z6ZqlLfEpYOp5FJd5Rvo3/U5dA3VWZ88=
+  NodeupConfigHash: RdSNFHb8HrZwhDUmxJnqm+aztTCrCDT836YrjbPrhZQ=
 
   __EOF_KUBE_ENV
 
@@ -380,41 +396,39 @@ Resources.AWSEC2LaunchTemplatenodesprivateciliumexamplecom.Properties.LaunchTemp
   cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
   cloudConfig:
     awsEBSCSIDriver:
-      enabled: false
+      enabled: true
+      version: v1.6.2
     manageStorageClasses: true
-  containerRuntime: docker
+  containerRuntime: containerd
   containerd:
-    configOverride: |
-      disabled_plugins = ["cri"]
     logLevel: info
+    version: 1.6.6
   docker:
-    ipMasq: false
+    skipInstall: true
-    ipTables: false
-    logDriver: json-file
-    logLevel: info
-    logOpt:
-    - max-size=10m
-    - max-file=5
-    storage: overlay2,overlay,aufs
-    version: 19.03.15
   kubeProxy:
     clusterCIDR: 100.96.0.0/11
     cpuRequest: 100m
-    image: registry.k8s.io/kube-proxy:v1.19.0
+    image: registry.k8s.io/kube-proxy:v1.24.0
     logLevel: 2
   kubelet:
     anonymousAuth: false
+    cgroupDriver: systemd
     cgroupRoot: /
-    cloudProvider: aws
+    cloudProvider: external
     clusterDNS: 100.64.0.10
     clusterDomain: cluster.local
     enableDebuggingHandlers: true
     evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     kubeconfigPath: /var/lib/kubelet/kubeconfig
     logLevel: 2
-    networkPluginName: cni
     podInfraContainerImage: registry.k8s.io/pause:3.6
     podManifestPath: /etc/kubernetes/manifests
+    protectKernelDefaults: true
+    shutdownGracePeriod: 30s
+    shutdownGracePeriodCriticalPods: 10s
 
   __EOF_CLUSTER_SPEC
 
@@ -423,7 +437,7 @@ Resources.AWSEC2LaunchTemplatenodesprivateciliumexamplecom.Properties.LaunchTemp
   ConfigBase: memfs://clusters.example.com/privatecilium.example.com
   InstanceGroupName: nodes
   InstanceGroupRole: Node
-  NodeupConfigHash: mONybsIQLRIuQ/k4BkPRuh6YET6I5XQVGqkv/UC114k=
+  NodeupConfigHash: qwUFYSeT5I8+V21GQ/JQY28wlCekz91WS9kK245Uuhg=
 
   __EOF_KUBE_ENV
 

tests/integration/update_cluster/privatecilium2/data/aws_iam_role_policy_masters.privatecilium.example.com_policy

@@ -94,39 +94,6 @@
         "*"
       ]
     },
-    {
-      "Action": "ec2:CreateTags",
-      "Condition": {
-        "StringEquals": {
-          "aws:RequestTag/KubernetesCluster": "privatecilium.example.com",
-          "ec2:CreateAction": [
-            "CreateSecurityGroup"
-          ]
-        }
-      },
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws-test:ec2:*:*:security-group/*"
-      ]
-    },
-    {
-      "Action": [
-        "ec2:CreateTags",
-        "ec2:DeleteTags"
-      ],
-      "Condition": {
-        "Null": {
-          "aws:RequestTag/KubernetesCluster": "true"
-        },
-        "StringEquals": {
-          "aws:ResourceTag/KubernetesCluster": "privatecilium.example.com"
-        }
-      },
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws-test:ec2:*:*:security-group/*"
-      ]
-    },
     {
       "Action": "ec2:CreateTags",
       "Condition": {
@@ -163,19 +130,45 @@
         "arn:aws-test:ec2:*:*:snapshot/*"
       ]
     },
+    {
+      "Action": "ec2:CreateTags",
+      "Condition": {
+        "StringEquals": {
+          "aws:RequestTag/KubernetesCluster": "privatecilium.example.com",
+          "ec2:CreateAction": [
+            "CreateSecurityGroup"
+          ]
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws-test:ec2:*:*:security-group/*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:CreateTags",
+        "ec2:DeleteTags"
+      ],
+      "Condition": {
+        "Null": {
+          "aws:RequestTag/KubernetesCluster": "true"
+        },
+        "StringEquals": {
+          "aws:ResourceTag/KubernetesCluster": "privatecilium.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws-test:ec2:*:*:security-group/*"
+      ]
+    },
     {
       "Action": [
         "autoscaling:DescribeAutoScalingGroups",
         "autoscaling:DescribeAutoScalingInstances",
         "autoscaling:DescribeLaunchConfigurations",
         "autoscaling:DescribeTags",
-        "ec2:AttachVolume",
-        "ec2:AuthorizeSecurityGroupIngress",
-        "ec2:CreateSecurityGroup",
-        "ec2:CreateTags",
-        "ec2:DeleteRoute",
-        "ec2:DeleteSecurityGroup",
-        "ec2:DeleteVolume",
         "ec2:DescribeAccountAttributes",
         "ec2:DescribeInstanceTypes",
         "ec2:DescribeInstances",
@@ -188,19 +181,12 @@
         "ec2:DescribeVolumes",
         "ec2:DescribeVolumesModifications",
         "ec2:DescribeVpcs",
-        "ec2:DetachVolume",
-        "ec2:ModifyInstanceAttribute",
-        "ec2:ModifyVolume",
-        "elasticloadbalancing:AddTags",
-        "elasticloadbalancing:CreateListener",
-        "elasticloadbalancing:CreateTargetGroup",
         "elasticloadbalancing:DescribeListeners",
         "elasticloadbalancing:DescribeLoadBalancerAttributes",
         "elasticloadbalancing:DescribeLoadBalancerPolicies",
         "elasticloadbalancing:DescribeLoadBalancers",
         "elasticloadbalancing:DescribeTargetGroups",
         "elasticloadbalancing:DescribeTargetHealth",
-        "elasticloadbalancing:RegisterTargets",
         "iam:GetServerCertificate",
         "iam:ListServerCertificates",
         "kms:DescribeKey",

tests/integration/update_cluster/privatecilium2/data/aws_launch_template_master-us-test-1a.masters.privatecilium.example.com_user_data

@@ -125,36 +125,30 @@ ensure-install-dir
 cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
 cloudConfig:
   awsEBSCSIDriver:
-    enabled: false
+    enabled: true
+    version: v1.6.2
   manageStorageClasses: true
-containerRuntime: docker
+containerRuntime: containerd
 containerd:
-  configOverride: |
-    disabled_plugins = ["cri"]
   logLevel: info
+  version: 1.6.6
 docker:
-  ipMasq: false
+  skipInstall: true
-  ipTables: false
-  logDriver: json-file
-  logLevel: info
-  logOpt:
-  - max-size=10m
-  - max-file=5
-  storage: overlay2,overlay,aufs
-  version: 19.03.15
 encryptionConfig: null
 etcdClusters:
   events:
-    version: 3.4.13
+    version: 3.5.4
   main:
-    version: 3.4.13
+    version: 3.5.4
 kubeAPIServer:
   allowPrivileged: true
   anonymousAuth: false
+  apiAudiences:
+  - kubernetes.svc.default
   apiServerCount: 1
   authorizationMode: AlwaysAllow
   bindAddress: 0.0.0.0
-  cloudProvider: aws
+  cloudProvider: external
   enableAdmissionPlugins:
   - NamespaceLifecycle
   - LimitRanger
@@ -169,8 +163,10 @@ kubeAPIServer:
   - https://127.0.0.1:4001
   etcdServersOverrides:
   - /events#https://127.0.0.1:4002
-  image: registry.k8s.io/kube-apiserver:v1.19.0
+  featureGates:
-  insecurePort: 0
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
+  image: registry.k8s.io/kube-apiserver:v1.24.0
   kubeletPreferredAddressTypes:
   - InternalIP
   - Hostname
@@ -185,16 +181,21 @@ kubeAPIServer:
   requestheaderUsernameHeaders:
   - X-Remote-User
   securePort: 443
+  serviceAccountIssuer: https://api.internal.privatecilium.example.com
+  serviceAccountJWKSURI: https://api.internal.privatecilium.example.com/openid/v1/jwks
   serviceClusterIPRange: 100.64.0.0/13
   storageBackend: etcd3
 kubeControllerManager:
   allocateNodeCIDRs: true
   attachDetachReconcileSyncPeriod: 1m0s
-  cloudProvider: aws
+  cloudProvider: external
   clusterCIDR: 100.96.0.0/11
   clusterName: privatecilium.example.com
   configureCloudRoutes: false
-  image: registry.k8s.io/kube-controller-manager:v1.19.0
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
+  image: registry.k8s.io/kube-controller-manager:v1.24.0
   leaderElection:
     leaderElect: true
   logLevel: 2
@@ -202,40 +203,55 @@ kubeControllerManager:
 kubeProxy:
   clusterCIDR: 100.96.0.0/11
   cpuRequest: 100m
-  image: registry.k8s.io/kube-proxy:v1.19.0
+  image: registry.k8s.io/kube-proxy:v1.24.0
   logLevel: 2
 kubeScheduler:
-  image: registry.k8s.io/kube-scheduler:v1.19.0
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
+  image: registry.k8s.io/kube-scheduler:v1.24.0
   leaderElection:
     leaderElect: true
   logLevel: 2
 kubelet:
   anonymousAuth: false
+  cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   podInfraContainerImage: registry.k8s.io/pause:3.6
   podManifestPath: /etc/kubernetes/manifests
+  protectKernelDefaults: true
+  shutdownGracePeriod: 30s
+  shutdownGracePeriodCriticalPods: 10s
 masterKubelet:
   anonymousAuth: false
+  cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   podInfraContainerImage: registry.k8s.io/pause:3.6
   podManifestPath: /etc/kubernetes/manifests
+  protectKernelDefaults: true
   registerSchedulable: false
+  shutdownGracePeriod: 30s
+  shutdownGracePeriodCriticalPods: 10s
 
 __EOF_CLUSTER_SPEC
 
@@ -244,7 +260,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/privatecilium.example.com
 InstanceGroupName: master-us-test-1a
 InstanceGroupRole: Master
-NodeupConfigHash: lQl5vPS8cn5z6ZqlLfEpYOp5FJd5Rvo3/U5dA3VWZ88=
+NodeupConfigHash: RdSNFHb8HrZwhDUmxJnqm+aztTCrCDT836YrjbPrhZQ=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/privatecilium2/data/aws_launch_template_nodes.privatecilium.example.com_user_data

@@ -125,41 +125,39 @@ ensure-install-dir
 cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
 cloudConfig:
   awsEBSCSIDriver:
-    enabled: false
+    enabled: true
+    version: v1.6.2
   manageStorageClasses: true
-containerRuntime: docker
+containerRuntime: containerd
 containerd:
-  configOverride: |
-    disabled_plugins = ["cri"]
   logLevel: info
+  version: 1.6.6
 docker:
-  ipMasq: false
+  skipInstall: true
-  ipTables: false
-  logDriver: json-file
-  logLevel: info
-  logOpt:
-  - max-size=10m
-  - max-file=5
-  storage: overlay2,overlay,aufs
-  version: 19.03.15
 kubeProxy:
   clusterCIDR: 100.96.0.0/11
   cpuRequest: 100m
-  image: registry.k8s.io/kube-proxy:v1.19.0
+  image: registry.k8s.io/kube-proxy:v1.24.0
   logLevel: 2
 kubelet:
   anonymousAuth: false
+  cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   podInfraContainerImage: registry.k8s.io/pause:3.6
   podManifestPath: /etc/kubernetes/manifests
+  protectKernelDefaults: true
+  shutdownGracePeriod: 30s
+  shutdownGracePeriodCriticalPods: 10s
 
 __EOF_CLUSTER_SPEC
 
@@ -168,7 +166,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/privatecilium.example.com
 InstanceGroupName: nodes
 InstanceGroupRole: Node
-NodeupConfigHash: mONybsIQLRIuQ/k4BkPRuh6YET6I5XQVGqkv/UC114k=
+NodeupConfigHash: qwUFYSeT5I8+V21GQ/JQY28wlCekz91WS9kK245Uuhg=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/privatecilium2/data/aws_s3_object_cluster-completed.spec_content

@@ -13,28 +13,29 @@ spec:
   channel: stable
   cloudConfig:
     awsEBSCSIDriver:
-      enabled: false
+      enabled: true
+      version: v1.6.2
     manageStorageClasses: true
+  cloudControllerManager:
+    allocateNodeCIDRs: true
+    clusterCIDR: 100.64.0.0/10
+    clusterName: privatecilium.example.com
+    configureCloudRoutes: false
+    enableLeaderMigration: true
+    image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.24.0
+    leaderElection:
+      leaderElect: true
   cloudProvider: aws
   clusterDNSDomain: cluster.local
   configBase: memfs://clusters.example.com/privatecilium.example.com
   configStore: memfs://clusters.example.com/privatecilium.example.com
-  containerRuntime: docker
+  containerRuntime: containerd
   containerd:
-    configOverride: |
-      disabled_plugins = ["cri"]
     logLevel: info
+    version: 1.6.6
   dnsZone: Z1AFAKE1ZON3YO
   docker:
-    ipMasq: false
+    skipInstall: true
-    ipTables: false
-    logDriver: json-file
-    logLevel: info
-    logOpt:
-    - max-size=10m
-    - max-file=5
-    storage: overlay2,overlay,aufs
-    version: 19.03.15
   etcdClusters:
   - backups:
       backupStore: memfs://clusters.example.com/privatecilium.example.com/backups/etcd/main
@@ -42,14 +43,14 @@ spec:
     - instanceGroup: master-us-test-1a
       name: us-test-1a
     name: main
-    version: 3.4.13
+    version: 3.5.4
   - backups:
       backupStore: memfs://clusters.example.com/privatecilium.example.com/backups/etcd/events
     etcdMembers:
     - instanceGroup: master-us-test-1a
       name: us-test-1a
     name: events
-    version: 3.4.13
+    version: 3.5.4
   externalDns:
     provider: dns-controller
   iam:
@@ -58,10 +59,12 @@ spec:
   kubeAPIServer:
     allowPrivileged: true
     anonymousAuth: false
+    apiAudiences:
+    - kubernetes.svc.default
     apiServerCount: 1
     authorizationMode: AlwaysAllow
     bindAddress: 0.0.0.0
-    cloudProvider: aws
+    cloudProvider: external
     enableAdmissionPlugins:
     - NamespaceLifecycle
     - LimitRanger
@@ -76,8 +79,10 @@ spec:
     - https://127.0.0.1:4001
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
-    image: registry.k8s.io/kube-apiserver:v1.19.0
+    featureGates:
-    insecurePort: 0
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
+    image: registry.k8s.io/kube-apiserver:v1.24.0
     kubeletPreferredAddressTypes:
     - InternalIP
     - Hostname
@@ -92,16 +97,21 @@ spec:
     requestheaderUsernameHeaders:
     - X-Remote-User
     securePort: 443
+    serviceAccountIssuer: https://api.internal.privatecilium.example.com
+    serviceAccountJWKSURI: https://api.internal.privatecilium.example.com/openid/v1/jwks
     serviceClusterIPRange: 100.64.0.0/13
     storageBackend: etcd3
   kubeControllerManager:
     allocateNodeCIDRs: true
     attachDetachReconcileSyncPeriod: 1m0s
-    cloudProvider: aws
+    cloudProvider: external
     clusterCIDR: 100.96.0.0/11
     clusterName: privatecilium.example.com
     configureCloudRoutes: false
-    image: registry.k8s.io/kube-controller-manager:v1.19.0
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
+    image: registry.k8s.io/kube-controller-manager:v1.24.0
     leaderElection:
       leaderElect: true
     logLevel: 2
@@ -118,49 +128,64 @@ spec:
       enabled: false
       image: registry.k8s.io/dns/k8s-dns-node-cache:1.21.3
       memoryRequest: 5Mi
-    provider: KubeDNS
+    provider: CoreDNS
     serverIP: 100.64.0.10
   kubeProxy:
     clusterCIDR: 100.96.0.0/11
     cpuRequest: 100m
-    image: registry.k8s.io/kube-proxy:v1.19.0
+    image: registry.k8s.io/kube-proxy:v1.24.0
     logLevel: 2
   kubeScheduler:
-    image: registry.k8s.io/kube-scheduler:v1.19.0
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
+    image: registry.k8s.io/kube-scheduler:v1.24.0
     leaderElection:
       leaderElect: true
     logLevel: 2
   kubelet:
     anonymousAuth: false
+    cgroupDriver: systemd
     cgroupRoot: /
-    cloudProvider: aws
+    cloudProvider: external
     clusterDNS: 100.64.0.10
     clusterDomain: cluster.local
     enableDebuggingHandlers: true
     evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     kubeconfigPath: /var/lib/kubelet/kubeconfig
     logLevel: 2
-    networkPluginName: cni
     podInfraContainerImage: registry.k8s.io/pause:3.6
     podManifestPath: /etc/kubernetes/manifests
+    protectKernelDefaults: true
+    shutdownGracePeriod: 30s
+    shutdownGracePeriodCriticalPods: 10s
   kubernetesApiAccess:
   - 0.0.0.0/0
-  kubernetesVersion: 1.19.0
+  kubernetesVersion: 1.24.0
   masterInternalName: api.internal.privatecilium.example.com
   masterKubelet:
     anonymousAuth: false
+    cgroupDriver: systemd
     cgroupRoot: /
-    cloudProvider: aws
+    cloudProvider: external
     clusterDNS: 100.64.0.10
     clusterDomain: cluster.local
     enableDebuggingHandlers: true
     evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     kubeconfigPath: /var/lib/kubelet/kubeconfig
     logLevel: 2
-    networkPluginName: cni
     podInfraContainerImage: registry.k8s.io/pause:3.6
     podManifestPath: /etc/kubernetes/manifests
+    protectKernelDefaults: true
     registerSchedulable: false
+    shutdownGracePeriod: 30s
+    shutdownGracePeriodCriticalPods: 10s
   masterPublicName: api.privatecilium.example.com
   networkCIDR: 172.20.0.0/16
   networking:

tests/integration/update_cluster/privatecilium2/data/aws_s3_object_etcd-cluster-spec-events_content

@@ -1,4 +1,4 @@
 {
   "memberCount": 1,
-  "etcdVersion": "3.4.13"
+  "etcdVersion": "3.5.4"
 }

tests/integration/update_cluster/privatecilium2/data/aws_s3_object_etcd-cluster-spec-main_content

@@ -1,4 +1,4 @@
 {
   "memberCount": 1,
-  "etcdVersion": "3.4.13"
+  "etcdVersion": "3.5.4"
 }

tests/integration/update_cluster/privatecilium2/data/aws_s3_object_nodeupconfig-master-us-test-1a_content

@@ -2,10 +2,12 @@ APIServerConfig:
   KubeAPIServer:
     allowPrivileged: true
     anonymousAuth: false
+    apiAudiences:
+    - kubernetes.svc.default
     apiServerCount: 1
     authorizationMode: AlwaysAllow
     bindAddress: 0.0.0.0
-    cloudProvider: aws
+    cloudProvider: external
     enableAdmissionPlugins:
     - NamespaceLifecycle
     - LimitRanger
@@ -20,8 +22,10 @@ APIServerConfig:
     - https://127.0.0.1:4001
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
-    image: registry.k8s.io/kube-apiserver:v1.19.0
+    featureGates:
-    insecurePort: 0
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
+    image: registry.k8s.io/kube-apiserver:v1.24.0
     kubeletPreferredAddressTypes:
     - InternalIP
     - Hostname
@@ -36,6 +40,8 @@ APIServerConfig:
     requestheaderUsernameHeaders:
     - X-Remote-User
     securePort: 443
+    serviceAccountIssuer: https://api.internal.privatecilium.example.com
+    serviceAccountJWKSURI: https://api.internal.privatecilium.example.com/openid/v1/jwks
     serviceClusterIPRange: 100.64.0.0/13
     storageBackend: etcd3
   ServiceAccountPublicKeys: |
@@ -49,17 +55,19 @@ APIServerConfig:
     -----END RSA PUBLIC KEY-----
 Assets:
   amd64:
-  - 3f03e5c160a8b658d30b34824a1c00abadbac96e62c4d01bf5c9271a2debc3ab@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubelet
+  - 3d98ac8b4fb8dc99f9952226f2565951cc366c442656a889facc5b1b2ec2ba52@https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubelet
-  - 79bb0d2f05487ff533999a639c075043c70a0a1ba25c1629eb1eef6ebe3ba70f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubectl
+  - 94d686bb6772f6fb59e3a32beff908ab406b79acdfb2427abdc4ac3ce1bb98d7@https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl
-  - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
+  - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
-  - 5504d190eef37355231325c176686d51ade6e0cabe2da526d561a38d8611506f@https://download.docker.com/linux/static/stable/x86_64/docker-19.03.15.tgz
+  - 0212869675742081d70600a1afc6cea4388435cc52bf5dc21f4efdcb9a92d2ef@https://github.com/containerd/containerd/releases/download/v1.6.6/containerd-1.6.6-linux-amd64.tar.gz
+  - e0436dfc5d26ca88f00e84cbdab5801dd9829b1e5ded05dcfc162ce5718c32ce@https://github.com/opencontainers/runc/releases/download/v1.1.2/runc.amd64
   - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64
   - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64
   arm64:
-  - d8fa5a9739ecc387dfcc55afa91ac6f4b0ccd01f1423c423dbd312d787bbb6bf@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubelet
+  - 8f066c9a048dd1704bf22ccf6e994e2fa2ea1175c9768a786f6cb6608765025e@https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/arm64/kubelet
-  - d4adf1b6b97252025cb2f7febf55daa3f42dc305822e3da133f77fd33071ec2f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubectl
+  - 449278789de283648e4076ade46816da249714f96e71567e035e9d17e1fff06d@https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/arm64/kubectl
-  - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
+  - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
-  - 264f3396630507606a8646fda6a28a98d3ced8927df84be8ee9a74ab73cc1566@https://download.docker.com/linux/static/stable/aarch64/docker-19.03.15.tgz
+  - 807bf333df331d713708ead66919189d7b142a0cc21ec32debbc988f9069d5eb@https://github.com/containerd/containerd/releases/download/v1.6.6/containerd-1.6.6-linux-arm64.tar.gz
+  - 6ebd968d46d00a3886e9a0cae2e0a7b399e110cf5d7b26e63ce23c1d81ea10ef@https://github.com/opencontainers/runc/releases/download/v1.1.2/runc.arm64
   - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64
   - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64
 CAs:
@@ -229,30 +237,38 @@ KeypairIDs:
   service-account: "2"
 KubeletConfig:
   anonymousAuth: false
+  cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   nodeLabels:
     kops.k8s.io/kops-controller-pki: ""
-    kubernetes.io/role: master
     node-role.kubernetes.io/control-plane: ""
-    node-role.kubernetes.io/master: ""
     node.kubernetes.io/exclude-from-external-load-balancers: ""
   podInfraContainerImage: registry.k8s.io/pause:3.6
   podManifestPath: /etc/kubernetes/manifests
+  protectKernelDefaults: true
   registerSchedulable: false
+  shutdownGracePeriod: 30s
+  shutdownGracePeriodCriticalPods: 10s
 UpdatePolicy: automatic
 channels:
 - memfs://clusters.example.com/privatecilium.example.com/addons/bootstrap-channel.yaml
+containerdConfig:
+  logLevel: info
+  version: 1.6.6
 etcdManifests:
 - memfs://clusters.example.com/privatecilium.example.com/manifests/etcd/main.yaml
 - memfs://clusters.example.com/privatecilium.example.com/manifests/etcd/events.yaml
 staticManifests:
 - key: kube-apiserver-healthcheck
   path: manifests/static/kube-apiserver-healthcheck.yaml
+useInstanceIDForNodeName: true

tests/integration/update_cluster/privatecilium2/data/aws_s3_object_nodeupconfig-nodes_content

@@ -1,14 +1,16 @@
 Assets:
   amd64:
-  - 3f03e5c160a8b658d30b34824a1c00abadbac96e62c4d01bf5c9271a2debc3ab@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubelet
+  - 3d98ac8b4fb8dc99f9952226f2565951cc366c442656a889facc5b1b2ec2ba52@https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubelet
-  - 79bb0d2f05487ff533999a639c075043c70a0a1ba25c1629eb1eef6ebe3ba70f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubectl
+  - 94d686bb6772f6fb59e3a32beff908ab406b79acdfb2427abdc4ac3ce1bb98d7@https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl
-  - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
+  - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
-  - 5504d190eef37355231325c176686d51ade6e0cabe2da526d561a38d8611506f@https://download.docker.com/linux/static/stable/x86_64/docker-19.03.15.tgz
+  - 0212869675742081d70600a1afc6cea4388435cc52bf5dc21f4efdcb9a92d2ef@https://github.com/containerd/containerd/releases/download/v1.6.6/containerd-1.6.6-linux-amd64.tar.gz
+  - e0436dfc5d26ca88f00e84cbdab5801dd9829b1e5ded05dcfc162ce5718c32ce@https://github.com/opencontainers/runc/releases/download/v1.1.2/runc.amd64
   arm64:
-  - d8fa5a9739ecc387dfcc55afa91ac6f4b0ccd01f1423c423dbd312d787bbb6bf@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubelet
+  - 8f066c9a048dd1704bf22ccf6e994e2fa2ea1175c9768a786f6cb6608765025e@https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/arm64/kubelet
-  - d4adf1b6b97252025cb2f7febf55daa3f42dc305822e3da133f77fd33071ec2f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubectl
+  - 449278789de283648e4076ade46816da249714f96e71567e035e9d17e1fff06d@https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/arm64/kubectl
-  - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
+  - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
-  - 264f3396630507606a8646fda6a28a98d3ced8927df84be8ee9a74ab73cc1566@https://download.docker.com/linux/static/stable/aarch64/docker-19.03.15.tgz
+  - 807bf333df331d713708ead66919189d7b142a0cc21ec32debbc988f9069d5eb@https://github.com/containerd/containerd/releases/download/v1.6.6/containerd-1.6.6-linux-arm64.tar.gz
+  - 6ebd968d46d00a3886e9a0cae2e0a7b399e110cf5d7b26e63ce23c1d81ea10ef@https://github.com/opencontainers/runc/releases/download/v1.1.2/runc.arm64
 CAs:
   kubernetes-ca: |
     -----BEGIN CERTIFICATE-----
@@ -39,20 +41,29 @@ KeypairIDs:
   kubernetes-ca: "6982820025135291416230495506"
 KubeletConfig:
   anonymousAuth: false
+  cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   nodeLabels:
-    kubernetes.io/role: node
     node-role.kubernetes.io/node: ""
   podInfraContainerImage: registry.k8s.io/pause:3.6
   podManifestPath: /etc/kubernetes/manifests
+  protectKernelDefaults: true
+  shutdownGracePeriod: 30s
+  shutdownGracePeriodCriticalPods: 10s
 UpdatePolicy: automatic
 channels:
 - memfs://clusters.example.com/privatecilium.example.com/addons/bootstrap-channel.yaml
+containerdConfig:
+  logLevel: info
+  version: 1.6.6
+useInstanceIDForNodeName: true

tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content

@@ -0,0 +1,238 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
+    k8s-app: aws-cloud-controller-manager
+  name: aws-cloud-controller-manager
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: aws-cloud-controller-manager
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        k8s-app: aws-cloud-controller-manager
+        kops.k8s.io/managed-by: kops
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
+      containers:
+      - args:
+        - --allocate-node-cidrs=true
+        - --cluster-cidr=100.64.0.0/10
+        - --cluster-name=privatecilium.example.com
+        - --configure-cloud-routes=false
+        - --enable-leader-migration=true
+        - --leader-elect=true
+        - --v=2
+        - --cloud-provider=aws
+        - --use-service-account-credentials=true
+        - --cloud-config=/etc/kubernetes/cloud.config
+        env:
+        - name: KUBERNETES_SERVICE_HOST
+          value: 127.0.0.1
+        image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.24.0
+        imagePullPolicy: IfNotPresent
+        name: aws-cloud-controller-manager
+        resources:
+          requests:
+            cpu: 200m
+        volumeMounts:
+        - mountPath: /etc/kubernetes/cloud.config
+          name: cloudconfig
+          readOnly: true
+      hostNetwork: true
+      nodeSelector: null
+      priorityClassName: system-cluster-critical
+      serviceAccountName: aws-cloud-controller-manager
+      tolerations:
+      - effect: NoSchedule
+        key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+      - effect: NoSchedule
+        key: node.kubernetes.io/not-ready
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+      volumes:
+      - hostPath:
+          path: /etc/kubernetes/cloud.config
+          type: ""
+        name: cloudconfig
+  updateStrategy:
+    type: RollingUpdate
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
+  name: aws-cloud-controller-manager
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
+  name: cloud-controller-manager:apiserver-authentication-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- apiGroup: ""
+  kind: ServiceAccount
+  name: aws-cloud-controller-manager
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
+  name: system:cloud-controller-manager
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - services/status
+  verbs:
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resourceNames:
+  - node-controller
+  - service-controller
+  - route-controller
+  resources:
+  - serviceaccounts/token
+  verbs:
+  - create
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
+  name: system:cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:cloud-controller-manager
+subjects:
+- apiGroup: ""
+  kind: ServiceAccount
+  name: aws-cloud-controller-manager
+  namespace: kube-system

tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content

@@ -0,0 +1,777 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-external-attacher-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - csi.storage.k8s.io
+  resources:
+  - csinodeinfos
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments/status
+  verbs:
+  - patch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-external-provisioner-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - storageclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - list
+  - watch
+  - create
+  - update
+  - patch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshots
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - csinodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - watch
+  - list
+  - delete
+  - update
+  - create
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments
+  verbs:
+  - get
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-external-resizer-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims/status
+  verbs:
+  - update
+  - patch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - storageclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - list
+  - watch
+  - create
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-external-snapshotter-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - list
+  - watch
+  - create
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+  - delete
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents/status
+  verbs:
+  - update
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-attacher-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ebs-external-attacher-role
+subjects:
+- kind: ServiceAccount
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-provisioner-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ebs-external-provisioner-role
+subjects:
+- kind: ServiceAccount
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-resizer-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ebs-external-resizer-role
+subjects:
+- kind: ServiceAccount
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-snapshotter-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ebs-external-snapshotter-role
+subjects:
+- kind: ServiceAccount
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-node-getter-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ebs-csi-node-role
+subjects:
+- kind: ServiceAccount
+  name: ebs-csi-node-sa
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-node-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-node-sa
+  namespace: kube-system
+
+---
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-node
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: ebs-csi-node
+      app.kubernetes.io/instance: aws-ebs-csi-driver
+      app.kubernetes.io/name: aws-ebs-csi-driver
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app: ebs-csi-node
+        app.kubernetes.io/instance: aws-ebs-csi-driver
+        app.kubernetes.io/name: aws-ebs-csi-driver
+        app.kubernetes.io/version: v1.6.2
+        kops.k8s.io/managed-by: kops
+    spec:
+      containers:
+      - args:
+        - node
+        - --endpoint=$(CSI_ENDPOINT)
+        - --logtostderr
+        - --v=2
+        env:
+        - name: CSI_ENDPOINT
+          value: unix:/csi/csi.sock
+        - name: CSI_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        image: registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.6.2
+        livenessProbe:
+          failureThreshold: 5
+          httpGet:
+            path: /healthz
+            port: healthz
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 3
+        name: ebs-plugin
+        ports:
+        - containerPort: 9808
+          name: healthz
+          protocol: TCP
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /var/lib/kubelet
+          mountPropagation: Bidirectional
+          name: kubelet-dir
+        - mountPath: /csi
+          name: plugin-dir
+        - mountPath: /dev
+          name: device-dir
+      - args:
+        - --csi-address=$(ADDRESS)
+        - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+        - --v=5
+        env:
+        - name: ADDRESS
+          value: /csi/csi.sock
+        - name: DRIVER_REG_SOCK_PATH
+          value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock
+        image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.1.0
+        lifecycle:
+          preStop:
+            exec:
+              command:
+              - /bin/sh
+              - -c
+              - rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock
+        name: node-driver-registrar
+        volumeMounts:
+        - mountPath: /csi
+          name: plugin-dir
+        - mountPath: /registration
+          name: registration-dir
+      - args:
+        - --csi-address=/csi/csi.sock
+        image: registry.k8s.io/sig-storage/livenessprobe:v2.2.0
+        name: liveness-probe
+        volumeMounts:
+        - mountPath: /csi
+          name: plugin-dir
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-node-critical
+      serviceAccountName: ebs-csi-node-sa
+      tolerations:
+      - operator: Exists
+      volumes:
+      - hostPath:
+          path: /var/lib/kubelet
+          type: Directory
+        name: kubelet-dir
+      - hostPath:
+          path: /var/lib/kubelet/plugins/ebs.csi.aws.com/
+          type: DirectoryOrCreate
+        name: plugin-dir
+      - hostPath:
+          path: /var/lib/kubelet/plugins_registry/
+          type: Directory
+        name: registration-dir
+      - hostPath:
+          path: /dev
+          type: Directory
+        name: device-dir
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-controller
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: ebs-csi-controller
+      app.kubernetes.io/instance: aws-ebs-csi-driver
+      app.kubernetes.io/name: aws-ebs-csi-driver
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app: ebs-csi-controller
+        app.kubernetes.io/instance: aws-ebs-csi-driver
+        app.kubernetes.io/name: aws-ebs-csi-driver
+        app.kubernetes.io/version: v1.6.2
+        kops.k8s.io/managed-by: kops
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+              - key: kubernetes.io/os
+                operator: In
+                values:
+                - linux
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
+              - key: kubernetes.io/os
+                operator: In
+                values:
+                - linux
+      containers:
+      - args:
+        - controller
+        - --endpoint=$(CSI_ENDPOINT)
+        - --logtostderr
+        - --k8s-tag-cluster-id=privatecilium.example.com
+        - --extra-tags=KubernetesCluster=privatecilium.example.com
+        - --v=5
+        env:
+        - name: CSI_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: CSI_ENDPOINT
+          value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+        - name: AWS_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              key: key_id
+              name: aws-secret
+              optional: true
+        - name: AWS_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              key: access_key
+              name: aws-secret
+              optional: true
+        image: registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.6.2
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          failureThreshold: 5
+          httpGet:
+            path: /healthz
+            port: healthz
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 3
+        name: ebs-plugin
+        ports:
+        - containerPort: 9808
+          name: healthz
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /var/lib/csi/sockets/pluginproxy/
+          name: socket-dir
+      - args:
+        - --csi-address=$(ADDRESS)
+        - --v=5
+        - --feature-gates=Topology=true
+        - --leader-election=true
+        - --extra-create-metadata=true
+        - --default-fstype=ext4
+        env:
+        - name: ADDRESS
+          value: /var/lib/csi/sockets/pluginproxy/csi.sock
+        image: registry.k8s.io/sig-storage/csi-provisioner:v2.2.0
+        name: csi-provisioner
+        volumeMounts:
+        - mountPath: /var/lib/csi/sockets/pluginproxy/
+          name: socket-dir
+      - args:
+        - --csi-address=$(ADDRESS)
+        - --v=5
+        - --leader-election=true
+        env:
+        - name: ADDRESS
+          value: /var/lib/csi/sockets/pluginproxy/csi.sock
+        image: registry.k8s.io/sig-storage/csi-attacher:v3.2.0
+        name: csi-attacher
+        volumeMounts:
+        - mountPath: /var/lib/csi/sockets/pluginproxy/
+          name: socket-dir
+      - args:
+        - --csi-address=$(ADDRESS)
+        - --v=5
+        env:
+        - name: ADDRESS
+          value: /var/lib/csi/sockets/pluginproxy/csi.sock
+        image: registry.k8s.io/sig-storage/csi-resizer:v1.1.0
+        imagePullPolicy: Always
+        name: csi-resizer
+        volumeMounts:
+        - mountPath: /var/lib/csi/sockets/pluginproxy/
+          name: socket-dir
+      - args:
+        - --csi-address=/csi/csi.sock
+        image: registry.k8s.io/sig-storage/livenessprobe:v2.4.0
+        name: liveness-probe
+        volumeMounts:
+        - mountPath: /csi
+          name: socket-dir
+      nodeSelector: null
+      priorityClassName: system-cluster-critical
+      serviceAccountName: ebs-csi-controller-sa
+      tolerations:
+      - operator: Exists
+      topologySpreadConstraints:
+      - labelSelector:
+          matchLabels:
+            app: ebs-csi-controller
+            app.kubernetes.io/instance: aws-ebs-csi-driver
+            app.kubernetes.io/name: aws-ebs-csi-driver
+        maxSkew: 1
+        topologyKey: topology.kubernetes.io/zone
+        whenUnsatisfiable: ScheduleAnyway
+      - labelSelector:
+          matchLabels:
+            app: ebs-csi-controller
+            app.kubernetes.io/instance: aws-ebs-csi-driver
+            app.kubernetes.io/name: aws-ebs-csi-driver
+        maxSkew: 1
+        topologyKey: kubernetes.io/hostname
+        whenUnsatisfiable: DoNotSchedule
+      volumes:
+      - emptyDir: {}
+        name: socket-dir
+
+---
+
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs.csi.aws.com
+spec:
+  attachRequired: true
+  podInfoOnMount: false
+
+---
+
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-controller
+  namespace: kube-system
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app: ebs-csi-controller
+      app.kubernetes.io/instance: aws-ebs-csi-driver

tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content

@@ -6,7 +6,7 @@ spec:
   addons:
   - id: k8s-1.16
     manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: aae486ec96f888c0b9440ec14edeae26a87e5c7fd94e8639d5991cb7c4ad2efa
+    manifestHash: 7ea6194d38b11cc8aa31b3593d765f6336c805e4a46297dff0efe46de35bec3a
     name: kops-controller.addons.k8s.io
     needsRollingUpdate: control-plane
     selector:
@@ -19,11 +19,11 @@ spec:
       k8s-addon: core.addons.k8s.io
     version: 9.99.0
   - id: k8s-1.12
-    manifest: kube-dns.addons.k8s.io/k8s-1.12.yaml
+    manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 6031d02efd43be93b5e17323df6b5eb80d4e3149675fa0c58e86f610d60b94e2
+    manifestHash: c930f84e40f98a66d2d21f5abb02300980216e73abcc78d4641c4d65335fc065
-    name: kube-dns.addons.k8s.io
+    name: coredns.addons.k8s.io
     selector:
-      k8s-addon: kube-dns.addons.k8s.io
+      k8s-addon: coredns.addons.k8s.io
     version: 9.99.0
   - id: k8s-1.9
     manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml
@@ -32,6 +32,13 @@ spec:
     selector:
       k8s-addon: kubelet-api.rbac.addons.k8s.io
     version: 9.99.0
+  - id: k8s-1.23
+    manifest: leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml
+    manifestHash: b9c91e09c0f28c9b74ff140b8395d611834c627d698846d625c10975a74a48c4
+    name: leader-migration.rbac.addons.k8s.io
+    selector:
+      k8s-addon: leader-migration.rbac.addons.k8s.io
+    version: 9.99.0
   - manifest: limit-range.addons.k8s.io/v1.5.0.yaml
     manifestHash: 2d55c3bc5e354e84a3730a65b42f39aba630a59dc8d32b30859fcce3d3178bc2
     name: limit-range.addons.k8s.io
@@ -47,7 +54,7 @@ spec:
     version: 9.99.0
   - id: v1.15.0
     manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
-    manifestHash: 065ae832ddac8d0931e9992d6a76f43a33a36975a38003b34f4c5d86a7d42780
+    manifestHash: 4e2cda50cd5048133aad1b5e28becb60f4629d3f9e09c514a2757c27998b4200
     name: storage-aws.addons.k8s.io
     selector:
       k8s-addon: storage-aws.addons.k8s.io
@@ -60,3 +67,17 @@ spec:
     selector:
       role.kubernetes.io/networking: "1"
     version: 9.99.0
+  - id: k8s-1.18
+    manifest: aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml
+    manifestHash: a955311c8f6122b21cdb8040d0ca562bec456e1a3d8ad79eebf3852854236aa3
+    name: aws-cloud-controller.addons.k8s.io
+    selector:
+      k8s-addon: aws-cloud-controller.addons.k8s.io
+    version: 9.99.0
+  - id: k8s-1.17
+    manifest: aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml
+    manifestHash: 86132447c612d4717a4ca5a33c152229de848a91e7f8eb79d292410f7d84f0ad
+    name: aws-ebs-csi-driver.addons.k8s.io
+    selector:
+      k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+    version: 9.99.0

tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -0,0 +1,385 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+    kubernetes.io/cluster-service: "true"
+  name: coredns
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+    kubernetes.io/bootstrapping: rbac-defaults
+  name: system:coredns
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  - services
+  - pods
+  - namespaces
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+    kubernetes.io/bootstrapping: rbac-defaults
+  name: system:coredns
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:coredns
+subjects:
+- kind: ServiceAccount
+  name: coredns
+  namespace: kube-system
+
+---
+
+apiVersion: v1
+data:
+  Corefile: |-
+    .:53 {
+        errors
+        health {
+          lameduck 5s
+        }
+        ready
+        kubernetes cluster.local. in-addr.arpa ip6.arpa {
+          pods insecure
+          fallthrough in-addr.arpa ip6.arpa
+          ttl 30
+        }
+        prometheus :9153
+        forward . /etc/resolv.conf {
+          max_concurrent 1000
+        }
+        cache 30
+        loop
+        reload
+        loadbalance
+    }
+kind: ConfigMap
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    addonmanager.kubernetes.io/mode: EnsureExists
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+  name: coredns
+  namespace: kube-system
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+    k8s-app: kube-dns
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: CoreDNS
+  name: coredns
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: kube-dns
+  strategy:
+    rollingUpdate:
+      maxSurge: 10%
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        k8s-app: kube-dns
+        kops.k8s.io/managed-by: kops
+    spec:
+      containers:
+      - args:
+        - -conf
+        - /etc/coredns/Corefile
+        image: registry.k8s.io/coredns/coredns:v1.8.6
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          failureThreshold: 5
+          httpGet:
+            path: /health
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 60
+          successThreshold: 1
+          timeoutSeconds: 5
+        name: coredns
+        ports:
+        - containerPort: 53
+          name: dns
+          protocol: UDP
+        - containerPort: 53
+          name: dns-tcp
+          protocol: TCP
+        - containerPort: 9153
+          name: metrics
+          protocol: TCP
+        readinessProbe:
+          httpGet:
+            path: /ready
+            port: 8181
+            scheme: HTTP
+        resources:
+          limits:
+            memory: 170Mi
+          requests:
+            cpu: 100m
+            memory: 70Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_BIND_SERVICE
+            drop:
+            - all
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - mountPath: /etc/coredns
+          name: config-volume
+          readOnly: true
+      dnsPolicy: Default
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-cluster-critical
+      serviceAccountName: coredns
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      topologySpreadConstraints:
+      - labelSelector:
+          matchLabels:
+            k8s-app: kube-dns
+        maxSkew: 1
+        topologyKey: topology.kubernetes.io/zone
+        whenUnsatisfiable: ScheduleAnyway
+      - labelSelector:
+          matchLabels:
+            k8s-app: kube-dns
+        maxSkew: 1
+        topologyKey: kubernetes.io/hostname
+        whenUnsatisfiable: ScheduleAnyway
+      volumes:
+      - configMap:
+          name: coredns
+        name: config-volume
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    prometheus.io/port: "9153"
+    prometheus.io/scrape: "true"
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+    k8s-app: kube-dns
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: CoreDNS
+  name: kube-dns
+  namespace: kube-system
+  resourceVersion: "0"
+spec:
+  clusterIP: 100.64.0.10
+  ports:
+  - name: dns
+    port: 53
+    protocol: UDP
+  - name: dns-tcp
+    port: 53
+    protocol: TCP
+  - name: metrics
+    port: 9153
+    protocol: TCP
+  selector:
+    k8s-app: kube-dns
+
+---
+
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+  name: kube-dns
+  namespace: kube-system
+spec:
+  maxUnavailable: 50%
+  selector:
+    matchLabels:
+      k8s-app: kube-dns
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+  name: coredns-autoscaler
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+  name: coredns-autoscaler
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - replicationcontrollers/scale
+  verbs:
+  - get
+  - update
+- apiGroups:
+  - extensions
+  - apps
+  resources:
+  - deployments/scale
+  - replicasets/scale
+  verbs:
+  - get
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - create
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+  name: coredns-autoscaler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: coredns-autoscaler
+subjects:
+- kind: ServiceAccount
+  name: coredns-autoscaler
+  namespace: kube-system
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+    k8s-app: coredns-autoscaler
+    kubernetes.io/cluster-service: "true"
+  name: coredns-autoscaler
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: coredns-autoscaler
+  template:
+    metadata:
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ""
+      creationTimestamp: null
+      labels:
+        k8s-app: coredns-autoscaler
+        kops.k8s.io/managed-by: kops
+    spec:
+      containers:
+      - command:
+        - /cluster-proportional-autoscaler
+        - --namespace=kube-system
+        - --configmap=coredns-autoscaler
+        - --target=Deployment/coredns
+        - --default-params={"linear":{"coresPerReplica":256,"nodesPerReplica":16,"preventSinglePointFailure":true}}
+        - --logtostderr=true
+        - --v=2
+        image: registry.k8s.io/cpa/cluster-proportional-autoscaler:1.8.4
+        name: autoscaler
+        resources:
+          requests:
+            cpu: 20m
+            memory: 10Mi
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-cluster-critical
+      serviceAccountName: coredns-autoscaler
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists

tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content

@@ -1,7 +1,7 @@
 apiVersion: v1
 data:
   config.yaml: |
-    {"cloud":"aws","configBase":"memfs://clusters.example.com/privatecilium.example.com","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["nodes.privatecilium.example.com"],"Region":"us-test-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"]}}
+    {"cloud":"aws","configBase":"memfs://clusters.example.com/privatecilium.example.com","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["nodes.privatecilium.example.com"],"Region":"us-test-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"],"useInstanceIDForNodeName":true}}
 kind: ConfigMap
 metadata:
   creationTimestamp: null

tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-kube-dns.addons.k8s.io-k8s-1.12_content

@@ -1,331 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kube-dns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kube-dns.addons.k8s.io
-    k8s-app: kube-dns-autoscaler
-    kubernetes.io/cluster-service: "true"
-  name: kube-dns-autoscaler
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      k8s-app: kube-dns-autoscaler
-  template:
-    metadata:
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ""
-      creationTimestamp: null
-      labels:
-        k8s-app: kube-dns-autoscaler
-        kops.k8s.io/managed-by: kops
-    spec:
-      containers:
-      - command:
-        - /cluster-proportional-autoscaler
-        - --namespace=kube-system
-        - --configmap=kube-dns-autoscaler
-        - --target=Deployment/kube-dns
-        - --default-params={"linear":{"coresPerReplica":256,"nodesPerReplica":16,"preventSinglePointFailure":true}}
-        - --logtostderr=true
-        - --v=2
-        image: registry.k8s.io/cpa/cluster-proportional-autoscaler:1.8.3
-        name: autoscaler
-        resources:
-          requests:
-            cpu: 20m
-            memory: 10Mi
-      priorityClassName: system-cluster-critical
-      serviceAccountName: kube-dns-autoscaler
-      tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
-
----
-
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kube-dns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kube-dns.addons.k8s.io
-    k8s-app: kube-dns
-    kubernetes.io/cluster-service: "true"
-  name: kube-dns
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      k8s-app: kube-dns
-  strategy:
-    rollingUpdate:
-      maxSurge: 10%
-      maxUnavailable: 0
-  template:
-    metadata:
-      annotations:
-        prometheus.io/port: "10055"
-        prometheus.io/scrape: "true"
-        scheduler.alpha.kubernetes.io/critical-pod: ""
-      creationTimestamp: null
-      labels:
-        k8s-app: kube-dns
-        kops.k8s.io/managed-by: kops
-    spec:
-      affinity:
-        podAntiAffinity:
-          preferredDuringSchedulingIgnoredDuringExecution:
-          - podAffinityTerm:
-              labelSelector:
-                matchExpressions:
-                - key: k8s-app
-                  operator: In
-                  values:
-                  - kube-dns
-              topologyKey: kubernetes.io/hostname
-            weight: 1
-      containers:
-      - args:
-        - --config-dir=/kube-dns-config
-        - --dns-port=10053
-        - --domain=cluster.local.
-        - --v=2
-        env:
-        - name: PROMETHEUS_PORT
-          value: "10055"
-        image: registry.k8s.io/k8s-dns-kube-dns:1.15.13
-        livenessProbe:
-          failureThreshold: 5
-          httpGet:
-            path: /healthcheck/kubedns
-            port: 10054
-            scheme: HTTP
-          initialDelaySeconds: 60
-          successThreshold: 1
-          timeoutSeconds: 5
-        name: kubedns
-        ports:
-        - containerPort: 10053
-          name: dns-local
-          protocol: UDP
-        - containerPort: 10053
-          name: dns-tcp-local
-          protocol: TCP
-        - containerPort: 10055
-          name: metrics
-          protocol: TCP
-        readinessProbe:
-          httpGet:
-            path: /readiness
-            port: 8081
-            scheme: HTTP
-          initialDelaySeconds: 3
-          timeoutSeconds: 5
-        resources:
-          limits:
-            memory: 170Mi
-          requests:
-            cpu: 100m
-            memory: 70Mi
-        volumeMounts:
-        - mountPath: /kube-dns-config
-          name: kube-dns-config
-      - args:
-        - -v=2
-        - -logtostderr
-        - -configDir=/etc/k8s/dns/dnsmasq-nanny
-        - -restartDnsmasq=true
-        - --
-        - -k
-        - --cache-size=1000
-        - --dns-forward-max=150
-        - --no-negcache
-        - --log-facility=-
-        - --server=/cluster.local/127.0.0.1#10053
-        - --server=/in-addr.arpa/127.0.0.1#10053
-        - --server=/in6.arpa/127.0.0.1#10053
-        - --min-port=1024
-        image: registry.k8s.io/k8s-dns-dnsmasq-nanny:1.15.13
-        livenessProbe:
-          failureThreshold: 5
-          httpGet:
-            path: /healthcheck/dnsmasq
-            port: 10054
-            scheme: HTTP
-          initialDelaySeconds: 60
-          successThreshold: 1
-          timeoutSeconds: 5
-        name: dnsmasq
-        ports:
-        - containerPort: 53
-          name: dns
-          protocol: UDP
-        - containerPort: 53
-          name: dns-tcp
-          protocol: TCP
-        resources:
-          requests:
-            cpu: 150m
-            memory: 20Mi
-        volumeMounts:
-        - mountPath: /etc/k8s/dns/dnsmasq-nanny
-          name: kube-dns-config
-      - args:
-        - --v=2
-        - --logtostderr
-        - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local,5,A
-        - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local,5,A
-        image: registry.k8s.io/k8s-dns-sidecar:1.15.13
-        livenessProbe:
-          failureThreshold: 5
-          httpGet:
-            path: /metrics
-            port: 10054
-            scheme: HTTP
-          initialDelaySeconds: 60
-          successThreshold: 1
-          timeoutSeconds: 5
-        name: sidecar
-        ports:
-        - containerPort: 10054
-          name: metrics
-          protocol: TCP
-        resources:
-          requests:
-            cpu: 10m
-            memory: 20Mi
-      dnsPolicy: Default
-      priorityClassName: system-cluster-critical
-      serviceAccountName: kube-dns
-      volumes:
-      - configMap:
-          name: kube-dns
-          optional: true
-        name: kube-dns-config
-
----
-
-apiVersion: v1
-kind: Service
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kube-dns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kube-dns.addons.k8s.io
-    k8s-app: kube-dns
-    kubernetes.io/cluster-service: "true"
-    kubernetes.io/name: KubeDNS
-  name: kube-dns
-  namespace: kube-system
-  resourceVersion: "0"
-spec:
-  clusterIP: 100.64.0.10
-  ports:
-  - name: dns
-    port: 53
-    protocol: UDP
-  - name: dns-tcp
-    port: 53
-    protocol: TCP
-  selector:
-    k8s-app: kube-dns
-
----
-
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kube-dns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kube-dns.addons.k8s.io
-  name: kube-dns-autoscaler
-  namespace: kube-system
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kube-dns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kube-dns.addons.k8s.io
-  name: kube-dns-autoscaler
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - list
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - replicationcontrollers/scale
-  verbs:
-  - get
-  - update
-- apiGroups:
-  - extensions
-  - apps
-  resources:
-  - deployments/scale
-  - replicasets/scale
-  verbs:
-  - get
-  - update
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - get
-  - create
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kube-dns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kube-dns.addons.k8s.io
-  name: kube-dns-autoscaler
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: kube-dns-autoscaler
-subjects:
-- kind: ServiceAccount
-  name: kube-dns-autoscaler
-  namespace: kube-system
-
----
-
-apiVersion: policy/v1beta1
-kind: PodDisruptionBudget
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kube-dns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kube-dns.addons.k8s.io
-  name: kube-dns
-  namespace: kube-system
-spec:
-  maxUnavailable: 50%
-  selector:
-    matchLabels:
-      k8s-app: kube-dns

tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content

@@ -0,0 +1,52 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: leader-migration.rbac.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: leader-migration.rbac.addons.k8s.io
+  name: system::leader-locking-migration
+  namespace: kube-system
+rules:
+- apiGroups:
+  - coordination.k8s.io
+  resourceNames:
+  - cloud-provider-extraction-migration
+  resources:
+  - leases
+  verbs:
+  - create
+  - list
+  - get
+  - update
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: leader-migration.rbac.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: leader-migration.rbac.addons.k8s.io
+  name: system::leader-locking-migration
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: system::leader-locking-migration
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:kube-controller-manager
+- kind: ServiceAccount
+  name: kube-controller-manager
+  namespace: kube-system
+- kind: ServiceAccount
+  name: aws-cloud-controller-manager
+  namespace: kube-system
+- kind: ServiceAccount
+  name: cloud-controller-manager
+  namespace: kube-system

tests/integration/update_cluster/privatecilium2/data/aws_s3_object_privatecilium.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content

@@ -35,7 +35,7 @@ apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
   annotations:
-    storageclass.kubernetes.io/is-default-class: "true"
+    storageclass.kubernetes.io/is-default-class: "false"
   creationTimestamp: null
   labels:
     addon.kops.k8s.io/name: storage-aws.addons.k8s.io
@@ -50,6 +50,26 @@ volumeBindingMode: WaitForFirstConsumer
 
 ---
 
+allowVolumeExpansion: true
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: storage-aws.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: storage-aws.addons.k8s.io
+  name: kops-csi-1-21
+parameters:
+  encrypted: "true"
+  type: gp3
+provisioner: ebs.csi.aws.com
+volumeBindingMode: WaitForFirstConsumer
+
+---
+
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:

tests/integration/update_cluster/privatecilium2/in-v1alpha2.yaml

@@ -21,7 +21,7 @@ spec:
   iam: {}
   kubelet:
     anonymousAuth: false
-  kubernetesVersion: v1.19.0
+  kubernetesVersion: v1.24.0
   masterInternalName: api.internal.privatecilium.example.com
   masterPublicName: api.privatecilium.example.com
   networkCIDR: 172.20.0.0/16

tests/integration/update_cluster/privatecilium2/kubernetes.tf

@@ -137,11 +137,6 @@ resource "aws_autoscaling_group" "bastion-privatecilium-example-com" {
     propagate_at_launch = true
     value               = "bastion.privatecilium.example.com"
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"
-    propagate_at_launch = true
-    value               = "node"
-  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
     propagate_at_launch = true
@@ -192,21 +187,11 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-privatecilium-exampl
     propagate_at_launch = true
     value               = ""
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"
-    propagate_at_launch = true
-    value               = "master"
-  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"
     propagate_at_launch = true
     value               = ""
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"
-    propagate_at_launch = true
-    value               = ""
-  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers"
     propagate_at_launch = true
@@ -251,11 +236,6 @@ resource "aws_autoscaling_group" "nodes-privatecilium-example-com" {
     propagate_at_launch = true
     value               = "nodes.privatecilium.example.com"
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"
-    propagate_at_launch = true
-    value               = "node"
-  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
     propagate_at_launch = true
@@ -512,7 +492,6 @@ resource "aws_launch_template" "bastion-privatecilium-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "privatecilium.example.com"
       "Name"                                                                       = "bastion.privatecilium.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                                                        = "1"
       "kops.k8s.io/instancegroup"                                                  = "bastion"
@@ -524,7 +503,6 @@ resource "aws_launch_template" "bastion-privatecilium-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "privatecilium.example.com"
       "Name"                                                                       = "bastion.privatecilium.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/bastion"                                                        = "1"
       "kops.k8s.io/instancegroup"                                                  = "bastion"
@@ -534,7 +512,6 @@ resource "aws_launch_template" "bastion-privatecilium-example-com" {
   tags = {
     "KubernetesCluster"                                                          = "privatecilium.example.com"
     "Name"                                                                       = "bastion.privatecilium.example.com"
-    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/bastion"                                                        = "1"
     "kops.k8s.io/instancegroup"                                                  = "bastion"
@@ -589,9 +566,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-privatecilium-example-
       "KubernetesCluster"                                                                                     = "privatecilium.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.privatecilium.example.com"
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
       "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
       "k8s.io/role/master"                                                                                    = "1"
       "kops.k8s.io/instancegroup"                                                                             = "master-us-test-1a"
@@ -604,9 +579,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-privatecilium-example-
       "KubernetesCluster"                                                                                     = "privatecilium.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.privatecilium.example.com"
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
       "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
       "k8s.io/role/master"                                                                                    = "1"
       "kops.k8s.io/instancegroup"                                                                             = "master-us-test-1a"
@@ -617,9 +590,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-privatecilium-example-
     "KubernetesCluster"                                                                                     = "privatecilium.example.com"
     "Name"                                                                                                  = "master-us-test-1a.masters.privatecilium.example.com"
     "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
-    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
     "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
     "k8s.io/role/master"                                                                                    = "1"
     "kops.k8s.io/instancegroup"                                                                             = "master-us-test-1a"
@@ -670,7 +641,6 @@ resource "aws_launch_template" "nodes-privatecilium-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "privatecilium.example.com"
       "Name"                                                                       = "nodes.privatecilium.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -682,7 +652,6 @@ resource "aws_launch_template" "nodes-privatecilium-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "privatecilium.example.com"
       "Name"                                                                       = "nodes.privatecilium.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -692,7 +661,6 @@ resource "aws_launch_template" "nodes-privatecilium-example-com" {
   tags = {
     "KubernetesCluster"                                                          = "privatecilium.example.com"
     "Name"                                                                       = "nodes.privatecilium.example.com"
-    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/node"                                                           = "1"
     "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -842,6 +810,22 @@ resource "aws_s3_object" "nodeupconfig-nodes" {
   server_side_encryption = "AES256"
 }
 
+resource "aws_s3_object" "privatecilium-example-com-addons-aws-cloud-controller-addons-k8s-io-k8s-1-18" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_privatecilium.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content")
+  key                    = "clusters.example.com/privatecilium.example.com/addons/aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "privatecilium-example-com-addons-aws-ebs-csi-driver-addons-k8s-io-k8s-1-17" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_privatecilium.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content")
+  key                    = "clusters.example.com/privatecilium.example.com/addons/aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
 resource "aws_s3_object" "privatecilium-example-com-addons-bootstrap" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_object_privatecilium.example.com-addons-bootstrap_content")
@@ -858,6 +842,14 @@ resource "aws_s3_object" "privatecilium-example-com-addons-core-addons-k8s-io" {
   server_side_encryption = "AES256"
 }
 
+resource "aws_s3_object" "privatecilium-example-com-addons-coredns-addons-k8s-io-k8s-1-12" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_privatecilium.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content")
+  key                    = "clusters.example.com/privatecilium.example.com/addons/coredns.addons.k8s.io/k8s-1.12.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
 resource "aws_s3_object" "privatecilium-example-com-addons-dns-controller-addons-k8s-io-k8s-1-12" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_object_privatecilium.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content")
@@ -874,14 +866,6 @@ resource "aws_s3_object" "privatecilium-example-com-addons-kops-controller-addon
   server_side_encryption = "AES256"
 }
 
-resource "aws_s3_object" "privatecilium-example-com-addons-kube-dns-addons-k8s-io-k8s-1-12" {
-  bucket                 = "testingBucket"
-  content                = file("${path.module}/data/aws_s3_object_privatecilium.example.com-addons-kube-dns.addons.k8s.io-k8s-1.12_content")
-  key                    = "clusters.example.com/privatecilium.example.com/addons/kube-dns.addons.k8s.io/k8s-1.12.yaml"
-  provider               = aws.files
-  server_side_encryption = "AES256"
-}
-
 resource "aws_s3_object" "privatecilium-example-com-addons-kubelet-api-rbac-addons-k8s-io-k8s-1-9" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_object_privatecilium.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content")
@@ -890,6 +874,14 @@ resource "aws_s3_object" "privatecilium-example-com-addons-kubelet-api-rbac-addo
   server_side_encryption = "AES256"
 }
 
+resource "aws_s3_object" "privatecilium-example-com-addons-leader-migration-rbac-addons-k8s-io-k8s-1-23" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_privatecilium.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content")
+  key                    = "clusters.example.com/privatecilium.example.com/addons/leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
 resource "aws_s3_object" "privatecilium-example-com-addons-limit-range-addons-k8s-io" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_object_privatecilium.example.com-addons-limit-range.addons.k8s.io_content")
@@ -1186,8 +1178,10 @@ resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" {
 }
 
 resource "aws_subnet" "us-test-1a-privatecilium-example-com" {
-  availability_zone = "us-test-1a"
+  availability_zone                           = "us-test-1a"
-  cidr_block        = "172.20.32.0/19"
+  cidr_block                                  = "172.20.32.0/19"
+  enable_resource_name_dns_a_record_on_launch = true
+  private_dns_hostname_type_on_launch         = "resource-name"
   tags = {
     "KubernetesCluster"                               = "privatecilium.example.com"
     "Name"                                            = "us-test-1a.privatecilium.example.com"
@@ -1199,8 +1193,10 @@ resource "aws_subnet" "us-test-1a-privatecilium-example-com" {
 }
 
 resource "aws_subnet" "utility-us-test-1a-privatecilium-example-com" {
-  availability_zone = "us-test-1a"
+  availability_zone                           = "us-test-1a"
-  cidr_block        = "172.20.4.0/22"
+  cidr_block                                  = "172.20.4.0/22"
+  enable_resource_name_dns_a_record_on_launch = true
+  private_dns_hostname_type_on_launch         = "resource-name"
   tags = {
     "KubernetesCluster"                               = "privatecilium.example.com"
     "Name"                                            = "utility-us-test-1a.privatecilium.example.com"

tests/integration/update_cluster/public-jwks-apiserver/data/aws_iam_role_aws-cloud-controller-manager.kube-system.sa.minimal.example.com_policy

@@ -0,0 +1,17 @@
+{
+  "Statement": [
+    {
+      "Action": "sts:AssumeRoleWithWebIdentity",
+      "Condition": {
+        "StringEquals": {
+          "discovery.example.com/minimal.example.com:sub": "system:serviceaccount:kube-system:aws-cloud-controller-manager"
+        }
+      },
+      "Effect": "Allow",
+      "Principal": {
+        "Federated": "arn:aws-test:iam::123456789012:oidc-provider/discovery.example.com/minimal.example.com"
+      }
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/public-jwks-apiserver/data/aws_iam_role_ebs-csi-controller-sa.kube-system.sa.minimal.example.com_policy

@@ -0,0 +1,17 @@
+{
+  "Statement": [
+    {
+      "Action": "sts:AssumeRoleWithWebIdentity",
+      "Condition": {
+        "StringEquals": {
+          "discovery.example.com/minimal.example.com:sub": "system:serviceaccount:kube-system:ebs-csi-controller-sa"
+        }
+      },
+      "Effect": "Allow",
+      "Principal": {
+        "Federated": "arn:aws-test:iam::123456789012:oidc-provider/discovery.example.com/minimal.example.com"
+      }
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/public-jwks-apiserver/data/aws_iam_role_policy_aws-cloud-controller-manager.kube-system.sa.minimal.example.com_policy

@@ -0,0 +1,114 @@
+{
+  "Statement": [
+    {
+      "Action": "ec2:CreateTags",
+      "Condition": {
+        "StringEquals": {
+          "aws:RequestTag/KubernetesCluster": "minimal.example.com",
+          "ec2:CreateAction": [
+            "CreateSecurityGroup"
+          ]
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws-test:ec2:*:*:security-group/*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:CreateTags",
+        "ec2:DeleteTags"
+      ],
+      "Condition": {
+        "Null": {
+          "aws:RequestTag/KubernetesCluster": "true"
+        },
+        "StringEquals": {
+          "aws:ResourceTag/KubernetesCluster": "minimal.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws-test:ec2:*:*:security-group/*"
+      ]
+    },
+    {
+      "Action": [
+        "autoscaling:DescribeAutoScalingGroups",
+        "autoscaling:DescribeTags",
+        "ec2:DescribeInstances",
+        "ec2:DescribeRegions",
+        "ec2:DescribeRouteTables",
+        "ec2:DescribeSecurityGroups",
+        "ec2:DescribeSubnets",
+        "ec2:DescribeVpcs",
+        "elasticloadbalancing:DescribeListeners",
+        "elasticloadbalancing:DescribeLoadBalancerAttributes",
+        "elasticloadbalancing:DescribeLoadBalancerPolicies",
+        "elasticloadbalancing:DescribeLoadBalancers",
+        "elasticloadbalancing:DescribeTargetGroups",
+        "elasticloadbalancing:DescribeTargetHealth",
+        "kms:DescribeKey"
+      ],
+      "Effect": "Allow",
+      "Resource": "*"
+    },
+    {
+      "Action": [
+        "ec2:AuthorizeSecurityGroupIngress",
+        "ec2:DeleteSecurityGroup",
+        "ec2:ModifyInstanceAttribute",
+        "ec2:RevokeSecurityGroupIngress",
+        "elasticloadbalancing:AddTags",
+        "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer",
+        "elasticloadbalancing:AttachLoadBalancerToSubnets",
+        "elasticloadbalancing:ConfigureHealthCheck",
+        "elasticloadbalancing:CreateLoadBalancerListeners",
+        "elasticloadbalancing:CreateLoadBalancerPolicy",
+        "elasticloadbalancing:DeleteListener",
+        "elasticloadbalancing:DeleteLoadBalancer",
+        "elasticloadbalancing:DeleteLoadBalancerListeners",
+        "elasticloadbalancing:DeleteTargetGroup",
+        "elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
+        "elasticloadbalancing:DeregisterTargets",
+        "elasticloadbalancing:DetachLoadBalancerFromSubnets",
+        "elasticloadbalancing:ModifyListener",
+        "elasticloadbalancing:ModifyLoadBalancerAttributes",
+        "elasticloadbalancing:ModifyTargetGroup",
+        "elasticloadbalancing:RegisterInstancesWithLoadBalancer",
+        "elasticloadbalancing:RegisterTargets",
+        "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer",
+        "elasticloadbalancing:SetLoadBalancerPoliciesOfListener"
+      ],
+      "Condition": {
+        "StringEquals": {
+          "aws:ResourceTag/KubernetesCluster": "minimal.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": "*"
+    },
+    {
+      "Action": [
+        "ec2:CreateSecurityGroup",
+        "elasticloadbalancing:CreateListener",
+        "elasticloadbalancing:CreateLoadBalancer",
+        "elasticloadbalancing:CreateTargetGroup"
+      ],
+      "Condition": {
+        "StringEquals": {
+          "aws:RequestTag/KubernetesCluster": "minimal.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": "*"
+    },
+    {
+      "Action": "ec2:CreateSecurityGroup",
+      "Effect": "Allow",
+      "Resource": "arn:aws-test:ec2:*:*:vpc/*"
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/public-jwks-apiserver/data/aws_iam_role_policy_ebs-csi-controller-sa.kube-system.sa.minimal.example.com_policy

@@ -0,0 +1,81 @@
+{
+  "Statement": [
+    {
+      "Action": "ec2:CreateTags",
+      "Condition": {
+        "StringEquals": {
+          "aws:RequestTag/KubernetesCluster": "minimal.example.com",
+          "ec2:CreateAction": [
+            "CreateVolume",
+            "CreateSnapshot"
+          ]
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws-test:ec2:*:*:volume/*",
+        "arn:aws-test:ec2:*:*:snapshot/*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:CreateTags",
+        "ec2:DeleteTags"
+      ],
+      "Condition": {
+        "Null": {
+          "aws:RequestTag/KubernetesCluster": "true"
+        },
+        "StringEquals": {
+          "aws:ResourceTag/KubernetesCluster": "minimal.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws-test:ec2:*:*:volume/*",
+        "arn:aws-test:ec2:*:*:snapshot/*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:DescribeAccountAttributes",
+        "ec2:DescribeInstances",
+        "ec2:DescribeTags",
+        "ec2:DescribeVolumes",
+        "ec2:DescribeVolumesModifications"
+      ],
+      "Effect": "Allow",
+      "Resource": "*"
+    },
+    {
+      "Action": [
+        "ec2:AttachVolume",
+        "ec2:DeleteVolume",
+        "ec2:DetachVolume",
+        "ec2:ModifyInstanceAttribute",
+        "ec2:ModifyVolume"
+      ],
+      "Condition": {
+        "StringEquals": {
+          "aws:ResourceTag/KubernetesCluster": "minimal.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": "*"
+    },
+    {
+      "Action": [
+        "ec2:CreateSnapshot",
+        "ec2:CreateVolume"
+      ],
+      "Condition": {
+        "StringEquals": {
+          "aws:RequestTag/KubernetesCluster": "minimal.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": "*"
+    }
+  ],
+  "Version": "2012-10-17"
+}

tests/integration/update_cluster/public-jwks-apiserver/data/aws_iam_role_policy_masters.minimal.example.com_policy

@@ -94,134 +94,19 @@
         "*"
       ]
     },
-    {
-      "Action": "ec2:CreateTags",
-      "Condition": {
-        "StringEquals": {
-          "aws:RequestTag/KubernetesCluster": "minimal.example.com",
-          "ec2:CreateAction": [
-            "CreateSecurityGroup"
-          ]
-        }
-      },
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws-test:ec2:*:*:security-group/*"
-      ]
-    },
     {
       "Action": [
-        "ec2:CreateTags",
-        "ec2:DeleteTags"
-      ],
-      "Condition": {
-        "Null": {
-          "aws:RequestTag/KubernetesCluster": "true"
-        },
-        "StringEquals": {
-          "aws:ResourceTag/KubernetesCluster": "minimal.example.com"
-        }
-      },
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws-test:ec2:*:*:security-group/*"
-      ]
-    },
-    {
-      "Action": [
-        "autoscaling:DescribeAutoScalingGroups",
         "autoscaling:DescribeAutoScalingInstances",
-        "autoscaling:DescribeTags",
-        "ec2:AttachVolume",
-        "ec2:AuthorizeSecurityGroupIngress",
-        "ec2:CreateSecurityGroup",
-        "ec2:CreateTags",
-        "ec2:DeleteRoute",
-        "ec2:DeleteSecurityGroup",
-        "ec2:DeleteVolume",
         "ec2:DescribeInstanceTypes",
         "ec2:DescribeInstances",
         "ec2:DescribeRegions",
-        "ec2:DescribeRouteTables",
-        "ec2:DescribeSecurityGroups",
-        "ec2:DescribeSubnets",
         "ec2:DescribeVolumes",
-        "ec2:DescribeVpcs",
-        "ec2:DetachVolume",
-        "ec2:ModifyInstanceAttribute",
-        "ec2:ModifyVolume",
-        "elasticloadbalancing:AddTags",
-        "elasticloadbalancing:CreateListener",
-        "elasticloadbalancing:CreateTargetGroup",
-        "elasticloadbalancing:DescribeListeners",
-        "elasticloadbalancing:DescribeLoadBalancerAttributes",
-        "elasticloadbalancing:DescribeLoadBalancerPolicies",
-        "elasticloadbalancing:DescribeLoadBalancers",
-        "elasticloadbalancing:DescribeTargetGroups",
-        "elasticloadbalancing:DescribeTargetHealth",
-        "elasticloadbalancing:RegisterTargets",
         "iam:GetServerCertificate",
         "iam:ListServerCertificates",
-        "kms:DescribeKey",
         "kms:GenerateRandom"
       ],
       "Effect": "Allow",
       "Resource": "*"
-    },
-    {
-      "Action": [
-        "ec2:AuthorizeSecurityGroupIngress",
-        "ec2:DeleteSecurityGroup",
-        "ec2:ModifyInstanceAttribute",
-        "ec2:RevokeSecurityGroupIngress",
-        "elasticloadbalancing:AddTags",
-        "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer",
-        "elasticloadbalancing:AttachLoadBalancerToSubnets",
-        "elasticloadbalancing:ConfigureHealthCheck",
-        "elasticloadbalancing:CreateLoadBalancerListeners",
-        "elasticloadbalancing:CreateLoadBalancerPolicy",
-        "elasticloadbalancing:DeleteListener",
-        "elasticloadbalancing:DeleteLoadBalancer",
-        "elasticloadbalancing:DeleteLoadBalancerListeners",
-        "elasticloadbalancing:DeleteTargetGroup",
-        "elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
-        "elasticloadbalancing:DeregisterTargets",
-        "elasticloadbalancing:DetachLoadBalancerFromSubnets",
-        "elasticloadbalancing:ModifyListener",
-        "elasticloadbalancing:ModifyLoadBalancerAttributes",
-        "elasticloadbalancing:ModifyTargetGroup",
-        "elasticloadbalancing:RegisterInstancesWithLoadBalancer",
-        "elasticloadbalancing:RegisterTargets",
-        "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer",
-        "elasticloadbalancing:SetLoadBalancerPoliciesOfListener"
-      ],
-      "Condition": {
-        "StringEquals": {
-          "aws:ResourceTag/KubernetesCluster": "minimal.example.com"
-        }
-      },
-      "Effect": "Allow",
-      "Resource": "*"
-    },
-    {
-      "Action": [
-        "ec2:CreateSecurityGroup",
-        "elasticloadbalancing:CreateListener",
-        "elasticloadbalancing:CreateLoadBalancer",
-        "elasticloadbalancing:CreateTargetGroup"
-      ],
-      "Condition": {
-        "StringEquals": {
-          "aws:RequestTag/KubernetesCluster": "minimal.example.com"
-        }
-      },
-      "Effect": "Allow",
-      "Resource": "*"
-    },
-    {
-      "Action": "ec2:CreateSecurityGroup",
-      "Effect": "Allow",
-      "Resource": "arn:aws-test:ec2:*:*:vpc/*"
     }
   ],
   "Version": "2012-10-17"

tests/integration/update_cluster/public-jwks-apiserver/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data

@@ -125,29 +125,21 @@ ensure-install-dir
 cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
 cloudConfig:
   awsEBSCSIDriver:
-    enabled: false
+    enabled: true
+    version: v1.6.2
   manageStorageClasses: true
-containerRuntime: docker
+containerRuntime: containerd
 containerd:
-  configOverride: |
-    disabled_plugins = ["cri"]
   logLevel: info
+  version: 1.6.6
 docker:
-  ipMasq: false
+  skipInstall: true
-  ipTables: false
-  logDriver: json-file
-  logLevel: info
-  logOpt:
-  - max-size=10m
-  - max-file=5
-  storage: overlay2,overlay,aufs
-  version: 19.03.15
 encryptionConfig: null
 etcdClusters:
   events:
-    version: 3.4.13
+    version: 3.5.4
   main:
-    version: 3.4.13
+    version: 3.5.4
 kubeAPIServer:
   allowPrivileged: true
   anonymousAuth: false
@@ -156,7 +148,7 @@ kubeAPIServer:
   apiServerCount: 1
   authorizationMode: AlwaysAllow
   bindAddress: 0.0.0.0
-  cloudProvider: aws
+  cloudProvider: external
   enableAdmissionPlugins:
   - NamespaceLifecycle
   - LimitRanger
@@ -172,9 +164,10 @@ kubeAPIServer:
   etcdServersOverrides:
   - /events#https://127.0.0.1:4002
   featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
     ServiceAccountIssuerDiscovery: "true"
-  image: registry.k8s.io/kube-apiserver:v1.19.0
+  image: registry.k8s.io/kube-apiserver:v1.24.0
-  insecurePort: 0
   kubeletPreferredAddressTypes:
   - InternalIP
   - Hostname
@@ -196,11 +189,14 @@ kubeAPIServer:
 kubeControllerManager:
   allocateNodeCIDRs: true
   attachDetachReconcileSyncPeriod: 1m0s
-  cloudProvider: aws
+  cloudProvider: external
   clusterCIDR: 100.96.0.0/11
   clusterName: minimal.example.com
   configureCloudRoutes: false
-  image: registry.k8s.io/kube-controller-manager:v1.19.0
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
+  image: registry.k8s.io/kube-controller-manager:v1.24.0
   leaderElection:
     leaderElect: true
   logLevel: 2
@@ -208,40 +204,55 @@ kubeControllerManager:
 kubeProxy:
   clusterCIDR: 100.96.0.0/11
   cpuRequest: 100m
-  image: registry.k8s.io/kube-proxy:v1.19.0
+  image: registry.k8s.io/kube-proxy:v1.24.0
   logLevel: 2
 kubeScheduler:
-  image: registry.k8s.io/kube-scheduler:v1.19.0
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
+  image: registry.k8s.io/kube-scheduler:v1.24.0
   leaderElection:
     leaderElect: true
   logLevel: 2
 kubelet:
   anonymousAuth: false
+  cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   podInfraContainerImage: registry.k8s.io/pause:3.6
   podManifestPath: /etc/kubernetes/manifests
+  protectKernelDefaults: true
+  shutdownGracePeriod: 30s
+  shutdownGracePeriodCriticalPods: 10s
 masterKubelet:
   anonymousAuth: false
+  cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   podInfraContainerImage: registry.k8s.io/pause:3.6
   podManifestPath: /etc/kubernetes/manifests
+  protectKernelDefaults: true
   registerSchedulable: false
+  shutdownGracePeriod: 30s
+  shutdownGracePeriodCriticalPods: 10s
 
 __EOF_CLUSTER_SPEC
 
@@ -250,7 +261,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/minimal.example.com
 InstanceGroupName: master-us-test-1a
 InstanceGroupRole: Master
-NodeupConfigHash: qJlHMqoSTZN0xRLilNEbnk3qYVnXLH3O1XsMf1PohHw=
+NodeupConfigHash: f2im/yoOf/CJCkwV7tCMXaUviDRLGMTziayJVoljRQs=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/public-jwks-apiserver/data/aws_launch_template_nodes.minimal.example.com_user_data

@@ -125,41 +125,39 @@ ensure-install-dir
 cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
 cloudConfig:
   awsEBSCSIDriver:
-    enabled: false
+    enabled: true
+    version: v1.6.2
   manageStorageClasses: true
-containerRuntime: docker
+containerRuntime: containerd
 containerd:
-  configOverride: |
-    disabled_plugins = ["cri"]
   logLevel: info
+  version: 1.6.6
 docker:
-  ipMasq: false
+  skipInstall: true
-  ipTables: false
-  logDriver: json-file
-  logLevel: info
-  logOpt:
-  - max-size=10m
-  - max-file=5
-  storage: overlay2,overlay,aufs
-  version: 19.03.15
 kubeProxy:
   clusterCIDR: 100.96.0.0/11
   cpuRequest: 100m
-  image: registry.k8s.io/kube-proxy:v1.19.0
+  image: registry.k8s.io/kube-proxy:v1.24.0
   logLevel: 2
 kubelet:
   anonymousAuth: false
+  cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   podInfraContainerImage: registry.k8s.io/pause:3.6
   podManifestPath: /etc/kubernetes/manifests
+  protectKernelDefaults: true
+  shutdownGracePeriod: 30s
+  shutdownGracePeriodCriticalPods: 10s
 
 __EOF_CLUSTER_SPEC
 
@@ -168,7 +166,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/minimal.example.com
 InstanceGroupName: nodes
 InstanceGroupRole: Node
-NodeupConfigHash: urnwCuCO8EPAvJFOTFu9P2/VrkW9b1gUzntYRX9HaNU=
+NodeupConfigHash: 7G+LRrw88cyCXykIPIY36rA7L1xKt4YQ9jT++plII6w=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/public-jwks-apiserver/data/aws_s3_object_cluster-completed.spec_content

@@ -11,28 +11,29 @@ spec:
   channel: stable
   cloudConfig:
     awsEBSCSIDriver:
-      enabled: false
+      enabled: true
+      version: v1.6.2
     manageStorageClasses: true
+  cloudControllerManager:
+    allocateNodeCIDRs: true
+    clusterCIDR: 100.64.0.0/10
+    clusterName: minimal.example.com
+    configureCloudRoutes: false
+    enableLeaderMigration: true
+    image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.24.0
+    leaderElection:
+      leaderElect: true
   cloudProvider: aws
   clusterDNSDomain: cluster.local
   configBase: memfs://clusters.example.com/minimal.example.com
   configStore: memfs://clusters.example.com/minimal.example.com
-  containerRuntime: docker
+  containerRuntime: containerd
   containerd:
-    configOverride: |
-      disabled_plugins = ["cri"]
     logLevel: info
+    version: 1.6.6
   dnsZone: Z1AFAKE1ZON3YO
   docker:
-    ipMasq: false
+    skipInstall: true
-    ipTables: false
-    logDriver: json-file
-    logLevel: info
-    logOpt:
-    - max-size=10m
-    - max-file=5
-    storage: overlay2,overlay,aufs
-    version: 19.03.15
   etcdClusters:
   - backups:
       backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd/main
@@ -40,14 +41,14 @@ spec:
     - instanceGroup: master-us-test-1a
       name: us-test-1a
     name: main
-    version: 3.4.13
+    version: 3.5.4
   - backups:
       backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd/events
     etcdMembers:
     - instanceGroup: master-us-test-1a
       name: us-test-1a
     name: events
-    version: 3.4.13
+    version: 3.5.4
   externalDns:
     provider: dns-controller
   iam:
@@ -62,7 +63,7 @@ spec:
     apiServerCount: 1
     authorizationMode: AlwaysAllow
     bindAddress: 0.0.0.0
-    cloudProvider: aws
+    cloudProvider: external
     enableAdmissionPlugins:
     - NamespaceLifecycle
     - LimitRanger
@@ -78,9 +79,10 @@ spec:
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
     featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
       ServiceAccountIssuerDiscovery: "true"
-    image: registry.k8s.io/kube-apiserver:v1.19.0
+    image: registry.k8s.io/kube-apiserver:v1.24.0
-    insecurePort: 0
     kubeletPreferredAddressTypes:
     - InternalIP
     - Hostname
@@ -102,11 +104,14 @@ spec:
   kubeControllerManager:
     allocateNodeCIDRs: true
     attachDetachReconcileSyncPeriod: 1m0s
-    cloudProvider: aws
+    cloudProvider: external
     clusterCIDR: 100.96.0.0/11
     clusterName: minimal.example.com
     configureCloudRoutes: false
-    image: registry.k8s.io/kube-controller-manager:v1.19.0
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
+    image: registry.k8s.io/kube-controller-manager:v1.24.0
     leaderElection:
       leaderElect: true
     logLevel: 2
@@ -123,49 +128,64 @@ spec:
       enabled: false
       image: registry.k8s.io/dns/k8s-dns-node-cache:1.21.3
       memoryRequest: 5Mi
-    provider: KubeDNS
+    provider: CoreDNS
     serverIP: 100.64.0.10
   kubeProxy:
     clusterCIDR: 100.96.0.0/11
     cpuRequest: 100m
-    image: registry.k8s.io/kube-proxy:v1.19.0
+    image: registry.k8s.io/kube-proxy:v1.24.0
     logLevel: 2
   kubeScheduler:
-    image: registry.k8s.io/kube-scheduler:v1.19.0
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
+    image: registry.k8s.io/kube-scheduler:v1.24.0
     leaderElection:
       leaderElect: true
     logLevel: 2
   kubelet:
     anonymousAuth: false
+    cgroupDriver: systemd
     cgroupRoot: /
-    cloudProvider: aws
+    cloudProvider: external
     clusterDNS: 100.64.0.10
     clusterDomain: cluster.local
     enableDebuggingHandlers: true
     evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     kubeconfigPath: /var/lib/kubelet/kubeconfig
     logLevel: 2
-    networkPluginName: cni
     podInfraContainerImage: registry.k8s.io/pause:3.6
     podManifestPath: /etc/kubernetes/manifests
+    protectKernelDefaults: true
+    shutdownGracePeriod: 30s
+    shutdownGracePeriodCriticalPods: 10s
   kubernetesApiAccess:
   - 0.0.0.0/0
-  kubernetesVersion: 1.19.0
+  kubernetesVersion: 1.24.0
   masterInternalName: api.internal.minimal.example.com
   masterKubelet:
     anonymousAuth: false
+    cgroupDriver: systemd
     cgroupRoot: /
-    cloudProvider: aws
+    cloudProvider: external
     clusterDNS: 100.64.0.10
     clusterDomain: cluster.local
     enableDebuggingHandlers: true
     evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     kubeconfigPath: /var/lib/kubelet/kubeconfig
     logLevel: 2
-    networkPluginName: cni
     podInfraContainerImage: registry.k8s.io/pause:3.6
     podManifestPath: /etc/kubernetes/manifests
+    protectKernelDefaults: true
     registerSchedulable: false
+    shutdownGracePeriod: 30s
+    shutdownGracePeriodCriticalPods: 10s
   masterPublicName: api.minimal.example.com
   networkCIDR: 172.20.0.0/16
   networking:

tests/integration/update_cluster/public-jwks-apiserver/data/aws_s3_object_etcd-cluster-spec-events_content

@@ -1,4 +1,4 @@
 {
   "memberCount": 1,
-  "etcdVersion": "3.4.13"
+  "etcdVersion": "3.5.4"
 }

tests/integration/update_cluster/public-jwks-apiserver/data/aws_s3_object_etcd-cluster-spec-main_content

@@ -1,4 +1,4 @@
 {
   "memberCount": 1,
-  "etcdVersion": "3.4.13"
+  "etcdVersion": "3.5.4"
 }

tests/integration/update_cluster/public-jwks-apiserver/data/aws_s3_object_minimal.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content

@@ -0,0 +1,254 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
+    k8s-app: aws-cloud-controller-manager
+  name: aws-cloud-controller-manager
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: aws-cloud-controller-manager
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        k8s-app: aws-cloud-controller-manager
+        kops.k8s.io/managed-by: kops
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
+      containers:
+      - args:
+        - --allocate-node-cidrs=true
+        - --cluster-cidr=100.64.0.0/10
+        - --cluster-name=minimal.example.com
+        - --configure-cloud-routes=false
+        - --enable-leader-migration=true
+        - --leader-elect=true
+        - --v=2
+        - --cloud-provider=aws
+        - --use-service-account-credentials=true
+        - --cloud-config=/etc/kubernetes/cloud.config
+        env:
+        - name: KUBERNETES_SERVICE_HOST
+          value: 127.0.0.1
+        - name: AWS_ROLE_ARN
+          value: arn:aws-test:iam::123456789012:role/aws-cloud-controller-manager.kube-system.sa.minimal.example.com
+        - name: AWS_WEB_IDENTITY_TOKEN_FILE
+          value: /var/run/secrets/amazonaws.com/token
+        image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.24.0
+        imagePullPolicy: IfNotPresent
+        name: aws-cloud-controller-manager
+        resources:
+          requests:
+            cpu: 200m
+        volumeMounts:
+        - mountPath: /etc/kubernetes/cloud.config
+          name: cloudconfig
+          readOnly: true
+        - mountPath: /var/run/secrets/amazonaws.com/
+          name: token-amazonaws-com
+          readOnly: true
+      hostNetwork: true
+      priorityClassName: system-cluster-critical
+      securityContext:
+        fsGroup: 10001
+      serviceAccountName: aws-cloud-controller-manager
+      tolerations:
+      - effect: NoSchedule
+        key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+      - effect: NoSchedule
+        key: node.kubernetes.io/not-ready
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+      volumes:
+      - hostPath:
+          path: /etc/kubernetes/cloud.config
+          type: ""
+        name: cloudconfig
+      - name: token-amazonaws-com
+        projected:
+          defaultMode: 420
+          sources:
+          - serviceAccountToken:
+              audience: amazonaws.com
+              expirationSeconds: 86400
+              path: token
+  updateStrategy:
+    type: RollingUpdate
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
+  name: aws-cloud-controller-manager
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
+  name: cloud-controller-manager:apiserver-authentication-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- apiGroup: ""
+  kind: ServiceAccount
+  name: aws-cloud-controller-manager
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
+  name: system:cloud-controller-manager
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - services/status
+  verbs:
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resourceNames:
+  - node-controller
+  - service-controller
+  - route-controller
+  resources:
+  - serviceaccounts/token
+  verbs:
+  - create
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
+  name: system:cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:cloud-controller-manager
+subjects:
+- apiGroup: ""
+  kind: ServiceAccount
+  name: aws-cloud-controller-manager
+  namespace: kube-system

tests/integration/update_cluster/public-jwks-apiserver/data/aws_s3_object_minimal.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content

@@ -0,0 +1,821 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-external-attacher-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - csi.storage.k8s.io
+  resources:
+  - csinodeinfos
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments/status
+  verbs:
+  - patch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-external-provisioner-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - storageclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - list
+  - watch
+  - create
+  - update
+  - patch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshots
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - csinodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - watch
+  - list
+  - delete
+  - update
+  - create
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments
+  verbs:
+  - get
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-external-resizer-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims/status
+  verbs:
+  - update
+  - patch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - storageclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - list
+  - watch
+  - create
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-external-snapshotter-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - list
+  - watch
+  - create
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+  - delete
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents/status
+  verbs:
+  - update
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-attacher-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ebs-external-attacher-role
+subjects:
+- kind: ServiceAccount
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-provisioner-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ebs-external-provisioner-role
+subjects:
+- kind: ServiceAccount
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-resizer-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ebs-external-resizer-role
+subjects:
+- kind: ServiceAccount
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-snapshotter-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ebs-external-snapshotter-role
+subjects:
+- kind: ServiceAccount
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-node-getter-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ebs-csi-node-role
+subjects:
+- kind: ServiceAccount
+  name: ebs-csi-node-sa
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-node-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-node-sa
+  namespace: kube-system
+
+---
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-node
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: ebs-csi-node
+      app.kubernetes.io/instance: aws-ebs-csi-driver
+      app.kubernetes.io/name: aws-ebs-csi-driver
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app: ebs-csi-node
+        app.kubernetes.io/instance: aws-ebs-csi-driver
+        app.kubernetes.io/name: aws-ebs-csi-driver
+        app.kubernetes.io/version: v1.6.2
+        kops.k8s.io/managed-by: kops
+    spec:
+      containers:
+      - args:
+        - node
+        - --endpoint=$(CSI_ENDPOINT)
+        - --logtostderr
+        - --v=2
+        env:
+        - name: CSI_ENDPOINT
+          value: unix:/csi/csi.sock
+        - name: CSI_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        image: registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.6.2
+        livenessProbe:
+          failureThreshold: 5
+          httpGet:
+            path: /healthz
+            port: healthz
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 3
+        name: ebs-plugin
+        ports:
+        - containerPort: 9808
+          name: healthz
+          protocol: TCP
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /var/lib/kubelet
+          mountPropagation: Bidirectional
+          name: kubelet-dir
+        - mountPath: /csi
+          name: plugin-dir
+        - mountPath: /dev
+          name: device-dir
+      - args:
+        - --csi-address=$(ADDRESS)
+        - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+        - --v=5
+        env:
+        - name: ADDRESS
+          value: /csi/csi.sock
+        - name: DRIVER_REG_SOCK_PATH
+          value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock
+        image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.1.0
+        lifecycle:
+          preStop:
+            exec:
+              command:
+              - /bin/sh
+              - -c
+              - rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock
+        name: node-driver-registrar
+        volumeMounts:
+        - mountPath: /csi
+          name: plugin-dir
+        - mountPath: /registration
+          name: registration-dir
+      - args:
+        - --csi-address=/csi/csi.sock
+        image: registry.k8s.io/sig-storage/livenessprobe:v2.2.0
+        name: liveness-probe
+        volumeMounts:
+        - mountPath: /csi
+          name: plugin-dir
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-node-critical
+      serviceAccountName: ebs-csi-node-sa
+      tolerations:
+      - operator: Exists
+      volumes:
+      - hostPath:
+          path: /var/lib/kubelet
+          type: Directory
+        name: kubelet-dir
+      - hostPath:
+          path: /var/lib/kubelet/plugins/ebs.csi.aws.com/
+          type: DirectoryOrCreate
+        name: plugin-dir
+      - hostPath:
+          path: /var/lib/kubelet/plugins_registry/
+          type: Directory
+        name: registration-dir
+      - hostPath:
+          path: /dev
+          type: Directory
+        name: device-dir
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-controller
+  namespace: kube-system
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: ebs-csi-controller
+      app.kubernetes.io/instance: aws-ebs-csi-driver
+      app.kubernetes.io/name: aws-ebs-csi-driver
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app: ebs-csi-controller
+        app.kubernetes.io/instance: aws-ebs-csi-driver
+        app.kubernetes.io/name: aws-ebs-csi-driver
+        app.kubernetes.io/version: v1.6.2
+        kops.k8s.io/managed-by: kops
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: kubernetes.io/os
+                operator: In
+                values:
+                - linux
+            - matchExpressions:
+              - key: kubernetes.io/os
+                operator: In
+                values:
+                - linux
+      containers:
+      - args:
+        - controller
+        - --endpoint=$(CSI_ENDPOINT)
+        - --logtostderr
+        - --k8s-tag-cluster-id=minimal.example.com
+        - --extra-tags=KubernetesCluster=minimal.example.com
+        - --v=5
+        env:
+        - name: CSI_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: CSI_ENDPOINT
+          value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+        - name: AWS_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              key: key_id
+              name: aws-secret
+              optional: true
+        - name: AWS_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              key: access_key
+              name: aws-secret
+              optional: true
+        - name: AWS_ROLE_ARN
+          value: arn:aws-test:iam::123456789012:role/ebs-csi-controller-sa.kube-system.sa.minimal.example.com
+        - name: AWS_WEB_IDENTITY_TOKEN_FILE
+          value: /var/run/secrets/amazonaws.com/token
+        image: registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.6.2
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          failureThreshold: 5
+          httpGet:
+            path: /healthz
+            port: healthz
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 3
+        name: ebs-plugin
+        ports:
+        - containerPort: 9808
+          name: healthz
+          protocol: TCP
+        resources: {}
+        volumeMounts:
+        - mountPath: /var/lib/csi/sockets/pluginproxy/
+          name: socket-dir
+        - mountPath: /var/run/secrets/amazonaws.com/
+          name: token-amazonaws-com
+          readOnly: true
+      - args:
+        - --csi-address=$(ADDRESS)
+        - --v=5
+        - --feature-gates=Topology=true
+        - --leader-election=true
+        - --extra-create-metadata=true
+        - --default-fstype=ext4
+        env:
+        - name: ADDRESS
+          value: /var/lib/csi/sockets/pluginproxy/csi.sock
+        - name: AWS_ROLE_ARN
+          value: arn:aws-test:iam::123456789012:role/ebs-csi-controller-sa.kube-system.sa.minimal.example.com
+        - name: AWS_WEB_IDENTITY_TOKEN_FILE
+          value: /var/run/secrets/amazonaws.com/token
+        image: registry.k8s.io/sig-storage/csi-provisioner:v2.2.0
+        name: csi-provisioner
+        resources: {}
+        volumeMounts:
+        - mountPath: /var/lib/csi/sockets/pluginproxy/
+          name: socket-dir
+        - mountPath: /var/run/secrets/amazonaws.com/
+          name: token-amazonaws-com
+          readOnly: true
+      - args:
+        - --csi-address=$(ADDRESS)
+        - --v=5
+        - --leader-election=true
+        env:
+        - name: ADDRESS
+          value: /var/lib/csi/sockets/pluginproxy/csi.sock
+        - name: AWS_ROLE_ARN
+          value: arn:aws-test:iam::123456789012:role/ebs-csi-controller-sa.kube-system.sa.minimal.example.com
+        - name: AWS_WEB_IDENTITY_TOKEN_FILE
+          value: /var/run/secrets/amazonaws.com/token
+        image: registry.k8s.io/sig-storage/csi-attacher:v3.2.0
+        name: csi-attacher
+        resources: {}
+        volumeMounts:
+        - mountPath: /var/lib/csi/sockets/pluginproxy/
+          name: socket-dir
+        - mountPath: /var/run/secrets/amazonaws.com/
+          name: token-amazonaws-com
+          readOnly: true
+      - args:
+        - --csi-address=$(ADDRESS)
+        - --v=5
+        env:
+        - name: ADDRESS
+          value: /var/lib/csi/sockets/pluginproxy/csi.sock
+        - name: AWS_ROLE_ARN
+          value: arn:aws-test:iam::123456789012:role/ebs-csi-controller-sa.kube-system.sa.minimal.example.com
+        - name: AWS_WEB_IDENTITY_TOKEN_FILE
+          value: /var/run/secrets/amazonaws.com/token
+        image: registry.k8s.io/sig-storage/csi-resizer:v1.1.0
+        imagePullPolicy: Always
+        name: csi-resizer
+        resources: {}
+        volumeMounts:
+        - mountPath: /var/lib/csi/sockets/pluginproxy/
+          name: socket-dir
+        - mountPath: /var/run/secrets/amazonaws.com/
+          name: token-amazonaws-com
+          readOnly: true
+      - args:
+        - --csi-address=/csi/csi.sock
+        env:
+        - name: AWS_ROLE_ARN
+          value: arn:aws-test:iam::123456789012:role/ebs-csi-controller-sa.kube-system.sa.minimal.example.com
+        - name: AWS_WEB_IDENTITY_TOKEN_FILE
+          value: /var/run/secrets/amazonaws.com/token
+        image: registry.k8s.io/sig-storage/livenessprobe:v2.4.0
+        name: liveness-probe
+        resources: {}
+        volumeMounts:
+        - mountPath: /csi
+          name: socket-dir
+        - mountPath: /var/run/secrets/amazonaws.com/
+          name: token-amazonaws-com
+          readOnly: true
+      priorityClassName: system-cluster-critical
+      securityContext:
+        fsGroup: 10001
+      serviceAccountName: ebs-csi-controller-sa
+      topologySpreadConstraints:
+      - labelSelector:
+          matchLabels:
+            app: ebs-csi-controller
+            app.kubernetes.io/instance: aws-ebs-csi-driver
+            app.kubernetes.io/name: aws-ebs-csi-driver
+        maxSkew: 1
+        topologyKey: topology.kubernetes.io/zone
+        whenUnsatisfiable: ScheduleAnyway
+      - labelSelector:
+          matchLabels:
+            app: ebs-csi-controller
+            app.kubernetes.io/instance: aws-ebs-csi-driver
+            app.kubernetes.io/name: aws-ebs-csi-driver
+        maxSkew: 1
+        topologyKey: kubernetes.io/hostname
+        whenUnsatisfiable: DoNotSchedule
+      volumes:
+      - emptyDir: {}
+        name: socket-dir
+      - name: token-amazonaws-com
+        projected:
+          defaultMode: 420
+          sources:
+          - serviceAccountToken:
+              audience: amazonaws.com
+              expirationSeconds: 86400
+              path: token
+
+---
+
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs.csi.aws.com
+spec:
+  attachRequired: true
+  podInfoOnMount: false
+
+---
+
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.6.2
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-controller
+  namespace: kube-system
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app: ebs-csi-controller
+      app.kubernetes.io/instance: aws-ebs-csi-driver

tests/integration/update_cluster/public-jwks-apiserver/data/aws_s3_object_minimal.example.com-addons-bootstrap_content

@@ -6,7 +6,7 @@ spec:
   addons:
   - id: k8s-1.16
     manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: 0ee9158a176ebece8f59c139e48dd9486a5bf914982ca285af7175b0fd9a7c7d
+    manifestHash: ed86d7bbf8c3453592465e82d42e8db4d0f3b4f750f5b30e20f1582c2d7d2265
     name: kops-controller.addons.k8s.io
     needsRollingUpdate: control-plane
     selector:
@@ -19,11 +19,11 @@ spec:
       k8s-addon: core.addons.k8s.io
     version: 9.99.0
   - id: k8s-1.12
-    manifest: kube-dns.addons.k8s.io/k8s-1.12.yaml
+    manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 6031d02efd43be93b5e17323df6b5eb80d4e3149675fa0c58e86f610d60b94e2
+    manifestHash: c930f84e40f98a66d2d21f5abb02300980216e73abcc78d4641c4d65335fc065
-    name: kube-dns.addons.k8s.io
+    name: coredns.addons.k8s.io
     selector:
-      k8s-addon: kube-dns.addons.k8s.io
+      k8s-addon: coredns.addons.k8s.io
     version: 9.99.0
   - id: k8s-1.9
     manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml
@@ -32,6 +32,13 @@ spec:
     selector:
       k8s-addon: kubelet-api.rbac.addons.k8s.io
     version: 9.99.0
+  - id: k8s-1.23
+    manifest: leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml
+    manifestHash: b9c91e09c0f28c9b74ff140b8395d611834c627d698846d625c10975a74a48c4
+    name: leader-migration.rbac.addons.k8s.io
+    selector:
+      k8s-addon: leader-migration.rbac.addons.k8s.io
+    version: 9.99.0
   - manifest: limit-range.addons.k8s.io/v1.5.0.yaml
     manifestHash: 2d55c3bc5e354e84a3730a65b42f39aba630a59dc8d32b30859fcce3d3178bc2
     name: limit-range.addons.k8s.io
@@ -47,8 +54,22 @@ spec:
     version: 9.99.0
   - id: v1.15.0
     manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
-    manifestHash: 065ae832ddac8d0931e9992d6a76f43a33a36975a38003b34f4c5d86a7d42780
+    manifestHash: 4e2cda50cd5048133aad1b5e28becb60f4629d3f9e09c514a2757c27998b4200
     name: storage-aws.addons.k8s.io
     selector:
       k8s-addon: storage-aws.addons.k8s.io
     version: 9.99.0
+  - id: k8s-1.18
+    manifest: aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml
+    manifestHash: e345ac4e8dc325d63dda12eb47411d5572e867db924c376274df24e5af0f7b2e
+    name: aws-cloud-controller.addons.k8s.io
+    selector:
+      k8s-addon: aws-cloud-controller.addons.k8s.io
+    version: 9.99.0
+  - id: k8s-1.17
+    manifest: aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml
+    manifestHash: 1c6f4843e13209c9be0c2f6873a2f1f68d3b2ee738c415a212ad044ea7ab3270
+    name: aws-ebs-csi-driver.addons.k8s.io
+    selector:
+      k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+    version: 9.99.0

tests/integration/update_cluster/public-jwks-apiserver/data/aws_s3_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -0,0 +1,385 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+    kubernetes.io/cluster-service: "true"
+  name: coredns
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+    kubernetes.io/bootstrapping: rbac-defaults
+  name: system:coredns
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  - services
+  - pods
+  - namespaces
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+    kubernetes.io/bootstrapping: rbac-defaults
+  name: system:coredns
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:coredns
+subjects:
+- kind: ServiceAccount
+  name: coredns
+  namespace: kube-system
+
+---
+
+apiVersion: v1
+data:
+  Corefile: |-
+    .:53 {
+        errors
+        health {
+          lameduck 5s
+        }
+        ready
+        kubernetes cluster.local. in-addr.arpa ip6.arpa {
+          pods insecure
+          fallthrough in-addr.arpa ip6.arpa
+          ttl 30
+        }
+        prometheus :9153
+        forward . /etc/resolv.conf {
+          max_concurrent 1000
+        }
+        cache 30
+        loop
+        reload
+        loadbalance
+    }
+kind: ConfigMap
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    addonmanager.kubernetes.io/mode: EnsureExists
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+  name: coredns
+  namespace: kube-system
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+    k8s-app: kube-dns
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: CoreDNS
+  name: coredns
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: kube-dns
+  strategy:
+    rollingUpdate:
+      maxSurge: 10%
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        k8s-app: kube-dns
+        kops.k8s.io/managed-by: kops
+    spec:
+      containers:
+      - args:
+        - -conf
+        - /etc/coredns/Corefile
+        image: registry.k8s.io/coredns/coredns:v1.8.6
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          failureThreshold: 5
+          httpGet:
+            path: /health
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 60
+          successThreshold: 1
+          timeoutSeconds: 5
+        name: coredns
+        ports:
+        - containerPort: 53
+          name: dns
+          protocol: UDP
+        - containerPort: 53
+          name: dns-tcp
+          protocol: TCP
+        - containerPort: 9153
+          name: metrics
+          protocol: TCP
+        readinessProbe:
+          httpGet:
+            path: /ready
+            port: 8181
+            scheme: HTTP
+        resources:
+          limits:
+            memory: 170Mi
+          requests:
+            cpu: 100m
+            memory: 70Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_BIND_SERVICE
+            drop:
+            - all
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - mountPath: /etc/coredns
+          name: config-volume
+          readOnly: true
+      dnsPolicy: Default
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-cluster-critical
+      serviceAccountName: coredns
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      topologySpreadConstraints:
+      - labelSelector:
+          matchLabels:
+            k8s-app: kube-dns
+        maxSkew: 1
+        topologyKey: topology.kubernetes.io/zone
+        whenUnsatisfiable: ScheduleAnyway
+      - labelSelector:
+          matchLabels:
+            k8s-app: kube-dns
+        maxSkew: 1
+        topologyKey: kubernetes.io/hostname
+        whenUnsatisfiable: ScheduleAnyway
+      volumes:
+      - configMap:
+          name: coredns
+        name: config-volume
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    prometheus.io/port: "9153"
+    prometheus.io/scrape: "true"
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+    k8s-app: kube-dns
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: CoreDNS
+  name: kube-dns
+  namespace: kube-system
+  resourceVersion: "0"
+spec:
+  clusterIP: 100.64.0.10
+  ports:
+  - name: dns
+    port: 53
+    protocol: UDP
+  - name: dns-tcp
+    port: 53
+    protocol: TCP
+  - name: metrics
+    port: 9153
+    protocol: TCP
+  selector:
+    k8s-app: kube-dns
+
+---
+
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+  name: kube-dns
+  namespace: kube-system
+spec:
+  maxUnavailable: 50%
+  selector:
+    matchLabels:
+      k8s-app: kube-dns
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+  name: coredns-autoscaler
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+  name: coredns-autoscaler
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - replicationcontrollers/scale
+  verbs:
+  - get
+  - update
+- apiGroups:
+  - extensions
+  - apps
+  resources:
+  - deployments/scale
+  - replicasets/scale
+  verbs:
+  - get
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - create
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+  name: coredns-autoscaler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: coredns-autoscaler
+subjects:
+- kind: ServiceAccount
+  name: coredns-autoscaler
+  namespace: kube-system
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: coredns.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: coredns.addons.k8s.io
+    k8s-app: coredns-autoscaler
+    kubernetes.io/cluster-service: "true"
+  name: coredns-autoscaler
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: coredns-autoscaler
+  template:
+    metadata:
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ""
+      creationTimestamp: null
+      labels:
+        k8s-app: coredns-autoscaler
+        kops.k8s.io/managed-by: kops
+    spec:
+      containers:
+      - command:
+        - /cluster-proportional-autoscaler
+        - --namespace=kube-system
+        - --configmap=coredns-autoscaler
+        - --target=Deployment/coredns
+        - --default-params={"linear":{"coresPerReplica":256,"nodesPerReplica":16,"preventSinglePointFailure":true}}
+        - --logtostderr=true
+        - --v=2
+        image: registry.k8s.io/cpa/cluster-proportional-autoscaler:1.8.4
+        name: autoscaler
+        resources:
+          requests:
+            cpu: 20m
+            memory: 10Mi
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-cluster-critical
+      serviceAccountName: coredns-autoscaler
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists

tests/integration/update_cluster/public-jwks-apiserver/data/aws_s3_object_minimal.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content

@@ -1,7 +1,7 @@
 apiVersion: v1
 data:
   config.yaml: |
-    {"cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["nodes.minimal.example.com"],"Region":"us-test-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"]}}
+    {"cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["nodes.minimal.example.com"],"Region":"us-test-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"],"useInstanceIDForNodeName":true}}
 kind: ConfigMap
 metadata:
   creationTimestamp: null

tests/integration/update_cluster/public-jwks-apiserver/data/aws_s3_object_minimal.example.com-addons-kube-dns.addons.k8s.io-k8s-1.12_content

@@ -1,331 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kube-dns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kube-dns.addons.k8s.io
-    k8s-app: kube-dns-autoscaler
-    kubernetes.io/cluster-service: "true"
-  name: kube-dns-autoscaler
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      k8s-app: kube-dns-autoscaler
-  template:
-    metadata:
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ""
-      creationTimestamp: null
-      labels:
-        k8s-app: kube-dns-autoscaler
-        kops.k8s.io/managed-by: kops
-    spec:
-      containers:
-      - command:
-        - /cluster-proportional-autoscaler
-        - --namespace=kube-system
-        - --configmap=kube-dns-autoscaler
-        - --target=Deployment/kube-dns
-        - --default-params={"linear":{"coresPerReplica":256,"nodesPerReplica":16,"preventSinglePointFailure":true}}
-        - --logtostderr=true
-        - --v=2
-        image: registry.k8s.io/cpa/cluster-proportional-autoscaler:1.8.3
-        name: autoscaler
-        resources:
-          requests:
-            cpu: 20m
-            memory: 10Mi
-      priorityClassName: system-cluster-critical
-      serviceAccountName: kube-dns-autoscaler
-      tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
-
----
-
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kube-dns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kube-dns.addons.k8s.io
-    k8s-app: kube-dns
-    kubernetes.io/cluster-service: "true"
-  name: kube-dns
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      k8s-app: kube-dns
-  strategy:
-    rollingUpdate:
-      maxSurge: 10%
-      maxUnavailable: 0
-  template:
-    metadata:
-      annotations:
-        prometheus.io/port: "10055"
-        prometheus.io/scrape: "true"
-        scheduler.alpha.kubernetes.io/critical-pod: ""
-      creationTimestamp: null
-      labels:
-        k8s-app: kube-dns
-        kops.k8s.io/managed-by: kops
-    spec:
-      affinity:
-        podAntiAffinity:
-          preferredDuringSchedulingIgnoredDuringExecution:
-          - podAffinityTerm:
-              labelSelector:
-                matchExpressions:
-                - key: k8s-app
-                  operator: In
-                  values:
-                  - kube-dns
-              topologyKey: kubernetes.io/hostname
-            weight: 1
-      containers:
-      - args:
-        - --config-dir=/kube-dns-config
-        - --dns-port=10053
-        - --domain=cluster.local.
-        - --v=2
-        env:
-        - name: PROMETHEUS_PORT
-          value: "10055"
-        image: registry.k8s.io/k8s-dns-kube-dns:1.15.13
-        livenessProbe:
-          failureThreshold: 5
-          httpGet:
-            path: /healthcheck/kubedns
-            port: 10054
-            scheme: HTTP
-          initialDelaySeconds: 60
-          successThreshold: 1
-          timeoutSeconds: 5
-        name: kubedns
-        ports:
-        - containerPort: 10053
-          name: dns-local
-          protocol: UDP
-        - containerPort: 10053
-          name: dns-tcp-local
-          protocol: TCP
-        - containerPort: 10055
-          name: metrics
-          protocol: TCP
-        readinessProbe:
-          httpGet:
-            path: /readiness
-            port: 8081
-            scheme: HTTP
-          initialDelaySeconds: 3
-          timeoutSeconds: 5
-        resources:
-          limits:
-            memory: 170Mi
-          requests:
-            cpu: 100m
-            memory: 70Mi
-        volumeMounts:
-        - mountPath: /kube-dns-config
-          name: kube-dns-config
-      - args:
-        - -v=2
-        - -logtostderr
-        - -configDir=/etc/k8s/dns/dnsmasq-nanny
-        - -restartDnsmasq=true
-        - --
-        - -k
-        - --cache-size=1000
-        - --dns-forward-max=150
-        - --no-negcache
-        - --log-facility=-
-        - --server=/cluster.local/127.0.0.1#10053
-        - --server=/in-addr.arpa/127.0.0.1#10053
-        - --server=/in6.arpa/127.0.0.1#10053
-        - --min-port=1024
-        image: registry.k8s.io/k8s-dns-dnsmasq-nanny:1.15.13
-        livenessProbe:
-          failureThreshold: 5
-          httpGet:
-            path: /healthcheck/dnsmasq
-            port: 10054
-            scheme: HTTP
-          initialDelaySeconds: 60
-          successThreshold: 1
-          timeoutSeconds: 5
-        name: dnsmasq
-        ports:
-        - containerPort: 53
-          name: dns
-          protocol: UDP
-        - containerPort: 53
-          name: dns-tcp
-          protocol: TCP
-        resources:
-          requests:
-            cpu: 150m
-            memory: 20Mi
-        volumeMounts:
-        - mountPath: /etc/k8s/dns/dnsmasq-nanny
-          name: kube-dns-config
-      - args:
-        - --v=2
-        - --logtostderr
-        - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local,5,A
-        - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local,5,A
-        image: registry.k8s.io/k8s-dns-sidecar:1.15.13
-        livenessProbe:
-          failureThreshold: 5
-          httpGet:
-            path: /metrics
-            port: 10054
-            scheme: HTTP
-          initialDelaySeconds: 60
-          successThreshold: 1
-          timeoutSeconds: 5
-        name: sidecar
-        ports:
-        - containerPort: 10054
-          name: metrics
-          protocol: TCP
-        resources:
-          requests:
-            cpu: 10m
-            memory: 20Mi
-      dnsPolicy: Default
-      priorityClassName: system-cluster-critical
-      serviceAccountName: kube-dns
-      volumes:
-      - configMap:
-          name: kube-dns
-          optional: true
-        name: kube-dns-config
-
----
-
-apiVersion: v1
-kind: Service
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kube-dns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kube-dns.addons.k8s.io
-    k8s-app: kube-dns
-    kubernetes.io/cluster-service: "true"
-    kubernetes.io/name: KubeDNS
-  name: kube-dns
-  namespace: kube-system
-  resourceVersion: "0"
-spec:
-  clusterIP: 100.64.0.10
-  ports:
-  - name: dns
-    port: 53
-    protocol: UDP
-  - name: dns-tcp
-    port: 53
-    protocol: TCP
-  selector:
-    k8s-app: kube-dns
-
----
-
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kube-dns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kube-dns.addons.k8s.io
-  name: kube-dns-autoscaler
-  namespace: kube-system
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kube-dns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kube-dns.addons.k8s.io
-  name: kube-dns-autoscaler
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - list
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - replicationcontrollers/scale
-  verbs:
-  - get
-  - update
-- apiGroups:
-  - extensions
-  - apps
-  resources:
-  - deployments/scale
-  - replicasets/scale
-  verbs:
-  - get
-  - update
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - get
-  - create
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kube-dns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kube-dns.addons.k8s.io
-  name: kube-dns-autoscaler
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: kube-dns-autoscaler
-subjects:
-- kind: ServiceAccount
-  name: kube-dns-autoscaler
-  namespace: kube-system
-
----
-
-apiVersion: policy/v1beta1
-kind: PodDisruptionBudget
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kube-dns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kube-dns.addons.k8s.io
-  name: kube-dns
-  namespace: kube-system
-spec:
-  maxUnavailable: 50%
-  selector:
-    matchLabels:
-      k8s-app: kube-dns

tests/integration/update_cluster/public-jwks-apiserver/data/aws_s3_object_minimal.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content

@@ -0,0 +1,52 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: leader-migration.rbac.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: leader-migration.rbac.addons.k8s.io
+  name: system::leader-locking-migration
+  namespace: kube-system
+rules:
+- apiGroups:
+  - coordination.k8s.io
+  resourceNames:
+  - cloud-provider-extraction-migration
+  resources:
+  - leases
+  verbs:
+  - create
+  - list
+  - get
+  - update
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: leader-migration.rbac.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: leader-migration.rbac.addons.k8s.io
+  name: system::leader-locking-migration
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: system::leader-locking-migration
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:kube-controller-manager
+- kind: ServiceAccount
+  name: kube-controller-manager
+  namespace: kube-system
+- kind: ServiceAccount
+  name: aws-cloud-controller-manager
+  namespace: kube-system
+- kind: ServiceAccount
+  name: cloud-controller-manager
+  namespace: kube-system

tests/integration/update_cluster/public-jwks-apiserver/data/aws_s3_object_minimal.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content

@@ -35,7 +35,7 @@ apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
   annotations:
-    storageclass.kubernetes.io/is-default-class: "true"
+    storageclass.kubernetes.io/is-default-class: "false"
   creationTimestamp: null
   labels:
     addon.kops.k8s.io/name: storage-aws.addons.k8s.io
@@ -50,6 +50,26 @@ volumeBindingMode: WaitForFirstConsumer
 
 ---
 
+allowVolumeExpansion: true
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: storage-aws.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: storage-aws.addons.k8s.io
+  name: kops-csi-1-21
+parameters:
+  encrypted: "true"
+  type: gp3
+provisioner: ebs.csi.aws.com
+volumeBindingMode: WaitForFirstConsumer
+
+---
+
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:

tests/integration/update_cluster/public-jwks-apiserver/data/aws_s3_object_nodeupconfig-master-us-test-1a_content

@@ -7,7 +7,7 @@ APIServerConfig:
     apiServerCount: 1
     authorizationMode: AlwaysAllow
     bindAddress: 0.0.0.0
-    cloudProvider: aws
+    cloudProvider: external
     enableAdmissionPlugins:
     - NamespaceLifecycle
     - LimitRanger
@@ -23,9 +23,10 @@ APIServerConfig:
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
     featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
       ServiceAccountIssuerDiscovery: "true"
-    image: registry.k8s.io/kube-apiserver:v1.19.0
+    image: registry.k8s.io/kube-apiserver:v1.24.0
-    insecurePort: 0
     kubeletPreferredAddressTypes:
     - InternalIP
     - Hostname
@@ -55,17 +56,19 @@ APIServerConfig:
     -----END RSA PUBLIC KEY-----
 Assets:
   amd64:
-  - 3f03e5c160a8b658d30b34824a1c00abadbac96e62c4d01bf5c9271a2debc3ab@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubelet
+  - 3d98ac8b4fb8dc99f9952226f2565951cc366c442656a889facc5b1b2ec2ba52@https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubelet
-  - 79bb0d2f05487ff533999a639c075043c70a0a1ba25c1629eb1eef6ebe3ba70f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubectl
+  - 94d686bb6772f6fb59e3a32beff908ab406b79acdfb2427abdc4ac3ce1bb98d7@https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl
-  - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
+  - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
-  - 5504d190eef37355231325c176686d51ade6e0cabe2da526d561a38d8611506f@https://download.docker.com/linux/static/stable/x86_64/docker-19.03.15.tgz
+  - 0212869675742081d70600a1afc6cea4388435cc52bf5dc21f4efdcb9a92d2ef@https://github.com/containerd/containerd/releases/download/v1.6.6/containerd-1.6.6-linux-amd64.tar.gz
+  - e0436dfc5d26ca88f00e84cbdab5801dd9829b1e5ded05dcfc162ce5718c32ce@https://github.com/opencontainers/runc/releases/download/v1.1.2/runc.amd64
   - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64
   - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64
   arm64:
-  - d8fa5a9739ecc387dfcc55afa91ac6f4b0ccd01f1423c423dbd312d787bbb6bf@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubelet
+  - 8f066c9a048dd1704bf22ccf6e994e2fa2ea1175c9768a786f6cb6608765025e@https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/arm64/kubelet
-  - d4adf1b6b97252025cb2f7febf55daa3f42dc305822e3da133f77fd33071ec2f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubectl
+  - 449278789de283648e4076ade46816da249714f96e71567e035e9d17e1fff06d@https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/arm64/kubectl
-  - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
+  - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
-  - 264f3396630507606a8646fda6a28a98d3ced8927df84be8ee9a74ab73cc1566@https://download.docker.com/linux/static/stable/aarch64/docker-19.03.15.tgz
+  - 807bf333df331d713708ead66919189d7b142a0cc21ec32debbc988f9069d5eb@https://github.com/containerd/containerd/releases/download/v1.6.6/containerd-1.6.6-linux-arm64.tar.gz
+  - 6ebd968d46d00a3886e9a0cae2e0a7b399e110cf5d7b26e63ce23c1d81ea10ef@https://github.com/opencontainers/runc/releases/download/v1.1.2/runc.arm64
   - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64
   - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64
 CAs:
@@ -235,30 +238,38 @@ KeypairIDs:
   service-account: "2"
 KubeletConfig:
   anonymousAuth: false
+  cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   nodeLabels:
     kops.k8s.io/kops-controller-pki: ""
-    kubernetes.io/role: master
     node-role.kubernetes.io/control-plane: ""
-    node-role.kubernetes.io/master: ""
     node.kubernetes.io/exclude-from-external-load-balancers: ""
   podInfraContainerImage: registry.k8s.io/pause:3.6
   podManifestPath: /etc/kubernetes/manifests
+  protectKernelDefaults: true
   registerSchedulable: false
+  shutdownGracePeriod: 30s
+  shutdownGracePeriodCriticalPods: 10s
 UpdatePolicy: automatic
 channels:
 - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml
+containerdConfig:
+  logLevel: info
+  version: 1.6.6
 etcdManifests:
 - memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml
 - memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml
 staticManifests:
 - key: kube-apiserver-healthcheck
   path: manifests/static/kube-apiserver-healthcheck.yaml
+useInstanceIDForNodeName: true

tests/integration/update_cluster/public-jwks-apiserver/data/aws_s3_object_nodeupconfig-nodes_content

@@ -1,14 +1,16 @@
 Assets:
   amd64:
-  - 3f03e5c160a8b658d30b34824a1c00abadbac96e62c4d01bf5c9271a2debc3ab@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubelet
+  - 3d98ac8b4fb8dc99f9952226f2565951cc366c442656a889facc5b1b2ec2ba52@https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubelet
-  - 79bb0d2f05487ff533999a639c075043c70a0a1ba25c1629eb1eef6ebe3ba70f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubectl
+  - 94d686bb6772f6fb59e3a32beff908ab406b79acdfb2427abdc4ac3ce1bb98d7@https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/amd64/kubectl
-  - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
+  - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
-  - 5504d190eef37355231325c176686d51ade6e0cabe2da526d561a38d8611506f@https://download.docker.com/linux/static/stable/x86_64/docker-19.03.15.tgz
+  - 0212869675742081d70600a1afc6cea4388435cc52bf5dc21f4efdcb9a92d2ef@https://github.com/containerd/containerd/releases/download/v1.6.6/containerd-1.6.6-linux-amd64.tar.gz
+  - e0436dfc5d26ca88f00e84cbdab5801dd9829b1e5ded05dcfc162ce5718c32ce@https://github.com/opencontainers/runc/releases/download/v1.1.2/runc.amd64
   arm64:
-  - d8fa5a9739ecc387dfcc55afa91ac6f4b0ccd01f1423c423dbd312d787bbb6bf@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubelet
+  - 8f066c9a048dd1704bf22ccf6e994e2fa2ea1175c9768a786f6cb6608765025e@https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/arm64/kubelet
-  - d4adf1b6b97252025cb2f7febf55daa3f42dc305822e3da133f77fd33071ec2f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubectl
+  - 449278789de283648e4076ade46816da249714f96e71567e035e9d17e1fff06d@https://storage.googleapis.com/kubernetes-release/release/v1.24.0/bin/linux/arm64/kubectl
-  - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
+  - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
-  - 264f3396630507606a8646fda6a28a98d3ced8927df84be8ee9a74ab73cc1566@https://download.docker.com/linux/static/stable/aarch64/docker-19.03.15.tgz
+  - 807bf333df331d713708ead66919189d7b142a0cc21ec32debbc988f9069d5eb@https://github.com/containerd/containerd/releases/download/v1.6.6/containerd-1.6.6-linux-arm64.tar.gz
+  - 6ebd968d46d00a3886e9a0cae2e0a7b399e110cf5d7b26e63ce23c1d81ea10ef@https://github.com/opencontainers/runc/releases/download/v1.1.2/runc.arm64
 CAs:
   kubernetes-ca: |
     -----BEGIN CERTIFICATE-----
@@ -39,20 +41,29 @@ KeypairIDs:
   kubernetes-ca: "6982820025135291416230495506"
 KubeletConfig:
   anonymousAuth: false
+  cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   nodeLabels:
-    kubernetes.io/role: node
     node-role.kubernetes.io/node: ""
   podInfraContainerImage: registry.k8s.io/pause:3.6
   podManifestPath: /etc/kubernetes/manifests
+  protectKernelDefaults: true
+  shutdownGracePeriod: 30s
+  shutdownGracePeriodCriticalPods: 10s
 UpdatePolicy: automatic
 channels:
 - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml
+containerdConfig:
+  logLevel: info
+  version: 1.6.6
+useInstanceIDForNodeName: true

tests/integration/update_cluster/public-jwks-apiserver/in-v1alpha2.yaml

@@ -25,7 +25,7 @@ spec:
       ServiceAccountIssuerDiscovery: "true"
   kubelet:
     anonymousAuth: false
-  kubernetesVersion: v1.19.0
+  kubernetesVersion: v1.24.0
   masterInternalName: api.internal.minimal.example.com
   masterPublicName: api.minimal.example.com
   networkCIDR: 172.20.0.0/16

tests/integration/update_cluster/public-jwks-apiserver/kubernetes.tf

@@ -1,23 +1,27 @@
 locals {
-  cluster_name                         = "minimal.example.com"
+  cluster_name                                       = "minimal.example.com"
-  iam_openid_connect_provider_arn      = aws_iam_openid_connect_provider.minimal-example-com.arn
+  iam_openid_connect_provider_arn                    = aws_iam_openid_connect_provider.minimal-example-com.arn
-  iam_openid_connect_provider_issuer   = "discovery.example.com/minimal.example.com"
+  iam_openid_connect_provider_issuer                 = "discovery.example.com/minimal.example.com"
-  kube-system-dns-controller_role_arn  = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.arn
+  kube-system-aws-cloud-controller-manager_role_arn  = aws_iam_role.aws-cloud-controller-manager-kube-system-sa-minimal-example-com.arn
-  kube-system-dns-controller_role_name = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.name
+  kube-system-aws-cloud-controller-manager_role_name = aws_iam_role.aws-cloud-controller-manager-kube-system-sa-minimal-example-com.name
-  master_autoscaling_group_ids         = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id]
+  kube-system-dns-controller_role_arn                = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.arn
-  master_security_group_ids            = [aws_security_group.masters-minimal-example-com.id]
+  kube-system-dns-controller_role_name               = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.name
-  masters_role_arn                     = aws_iam_role.masters-minimal-example-com.arn
+  kube-system-ebs-csi-controller-sa_role_arn         = aws_iam_role.ebs-csi-controller-sa-kube-system-sa-minimal-example-com.arn
-  masters_role_name                    = aws_iam_role.masters-minimal-example-com.name
+  kube-system-ebs-csi-controller-sa_role_name        = aws_iam_role.ebs-csi-controller-sa-kube-system-sa-minimal-example-com.name
-  node_autoscaling_group_ids           = [aws_autoscaling_group.nodes-minimal-example-com.id]
+  master_autoscaling_group_ids                       = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id]
-  node_security_group_ids              = [aws_security_group.nodes-minimal-example-com.id]
+  master_security_group_ids                          = [aws_security_group.masters-minimal-example-com.id]
-  node_subnet_ids                      = [aws_subnet.us-test-1a-minimal-example-com.id]
+  masters_role_arn                                   = aws_iam_role.masters-minimal-example-com.arn
-  nodes_role_arn                       = aws_iam_role.nodes-minimal-example-com.arn
+  masters_role_name                                  = aws_iam_role.masters-minimal-example-com.name
-  nodes_role_name                      = aws_iam_role.nodes-minimal-example-com.name
+  node_autoscaling_group_ids                         = [aws_autoscaling_group.nodes-minimal-example-com.id]
-  region                               = "us-test-1"
+  node_security_group_ids                            = [aws_security_group.nodes-minimal-example-com.id]
-  route_table_public_id                = aws_route_table.minimal-example-com.id
+  node_subnet_ids                                    = [aws_subnet.us-test-1a-minimal-example-com.id]
-  subnet_us-test-1a_id                 = aws_subnet.us-test-1a-minimal-example-com.id
+  nodes_role_arn                                     = aws_iam_role.nodes-minimal-example-com.arn
-  vpc_cidr_block                       = aws_vpc.minimal-example-com.cidr_block
+  nodes_role_name                                    = aws_iam_role.nodes-minimal-example-com.name
-  vpc_id                               = aws_vpc.minimal-example-com.id
+  region                                             = "us-test-1"
+  route_table_public_id                              = aws_route_table.minimal-example-com.id
+  subnet_us-test-1a_id                               = aws_subnet.us-test-1a-minimal-example-com.id
+  vpc_cidr_block                                     = aws_vpc.minimal-example-com.cidr_block
+  vpc_id                                             = aws_vpc.minimal-example-com.id
 }
 
 output "cluster_name" {
@@ -32,6 +36,14 @@ output "iam_openid_connect_provider_issuer" {
   value = "discovery.example.com/minimal.example.com"
 }
 
+output "kube-system-aws-cloud-controller-manager_role_arn" {
+  value = aws_iam_role.aws-cloud-controller-manager-kube-system-sa-minimal-example-com.arn
+}
+
+output "kube-system-aws-cloud-controller-manager_role_name" {
+  value = aws_iam_role.aws-cloud-controller-manager-kube-system-sa-minimal-example-com.name
+}
+
 output "kube-system-dns-controller_role_arn" {
   value = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.arn
 }
@@ -40,6 +52,14 @@ output "kube-system-dns-controller_role_name" {
   value = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.name
 }
 
+output "kube-system-ebs-csi-controller-sa_role_arn" {
+  value = aws_iam_role.ebs-csi-controller-sa-kube-system-sa-minimal-example-com.arn
+}
+
+output "kube-system-ebs-csi-controller-sa_role_name" {
+  value = aws_iam_role.ebs-csi-controller-sa-kube-system-sa-minimal-example-com.name
+}
+
 output "master_autoscaling_group_ids" {
   value = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id]
 }
@@ -131,21 +151,11 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com"
     propagate_at_launch = true
     value               = ""
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"
-    propagate_at_launch = true
-    value               = "master"
-  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"
     propagate_at_launch = true
     value               = ""
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"
-    propagate_at_launch = true
-    value               = ""
-  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers"
     propagate_at_launch = true
@@ -190,11 +200,6 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" {
     propagate_at_launch = true
     value               = "nodes.minimal.example.com"
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"
-    propagate_at_launch = true
-    value               = "node"
-  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
     propagate_at_launch = true
@@ -281,6 +286,16 @@ resource "aws_iam_openid_connect_provider" "minimal-example-com" {
   url             = "https://discovery.example.com/minimal.example.com"
 }
 
+resource "aws_iam_role" "aws-cloud-controller-manager-kube-system-sa-minimal-example-com" {
+  assume_role_policy = file("${path.module}/data/aws_iam_role_aws-cloud-controller-manager.kube-system.sa.minimal.example.com_policy")
+  name               = "aws-cloud-controller-manager.kube-system.sa.minimal.example.com"
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "aws-cloud-controller-manager.kube-system.sa.minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
 resource "aws_iam_role" "dns-controller-kube-system-sa-minimal-example-com" {
   assume_role_policy = file("${path.module}/data/aws_iam_role_dns-controller.kube-system.sa.minimal.example.com_policy")
   name               = "dns-controller.kube-system.sa.minimal.example.com"
@@ -291,6 +306,16 @@ resource "aws_iam_role" "dns-controller-kube-system-sa-minimal-example-com" {
   }
 }
 
+resource "aws_iam_role" "ebs-csi-controller-sa-kube-system-sa-minimal-example-com" {
+  assume_role_policy = file("${path.module}/data/aws_iam_role_ebs-csi-controller-sa.kube-system.sa.minimal.example.com_policy")
+  name               = "ebs-csi-controller-sa.kube-system.sa.minimal.example.com"
+  tags = {
+    "KubernetesCluster"                         = "minimal.example.com"
+    "Name"                                      = "ebs-csi-controller-sa.kube-system.sa.minimal.example.com"
+    "kubernetes.io/cluster/minimal.example.com" = "owned"
+  }
+}
+
 resource "aws_iam_role" "masters-minimal-example-com" {
   assume_role_policy = file("${path.module}/data/aws_iam_role_masters.minimal.example.com_policy")
   name               = "masters.minimal.example.com"
@@ -311,12 +336,24 @@ resource "aws_iam_role" "nodes-minimal-example-com" {
   }
 }
 
+resource "aws_iam_role_policy" "aws-cloud-controller-manager-kube-system-sa-minimal-example-com" {
+  name   = "aws-cloud-controller-manager.kube-system.sa.minimal.example.com"
+  policy = file("${path.module}/data/aws_iam_role_policy_aws-cloud-controller-manager.kube-system.sa.minimal.example.com_policy")
+  role   = aws_iam_role.aws-cloud-controller-manager-kube-system-sa-minimal-example-com.name
+}
+
 resource "aws_iam_role_policy" "dns-controller-kube-system-sa-minimal-example-com" {
   name   = "dns-controller.kube-system.sa.minimal.example.com"
   policy = file("${path.module}/data/aws_iam_role_policy_dns-controller.kube-system.sa.minimal.example.com_policy")
   role   = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.name
 }
 
+resource "aws_iam_role_policy" "ebs-csi-controller-sa-kube-system-sa-minimal-example-com" {
+  name   = "ebs-csi-controller-sa.kube-system.sa.minimal.example.com"
+  policy = file("${path.module}/data/aws_iam_role_policy_ebs-csi-controller-sa.kube-system.sa.minimal.example.com_policy")
+  role   = aws_iam_role.ebs-csi-controller-sa-kube-system-sa-minimal-example-com.name
+}
+
 resource "aws_iam_role_policy" "masters-minimal-example-com" {
   name   = "masters.minimal.example.com"
   policy = file("${path.module}/data/aws_iam_role_policy_masters.minimal.example.com_policy")
@@ -395,9 +432,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" {
       "KubernetesCluster"                                                                                     = "minimal.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.minimal.example.com"
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
       "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
       "k8s.io/role/master"                                                                                    = "1"
       "kops.k8s.io/instancegroup"                                                                             = "master-us-test-1a"
@@ -410,9 +445,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" {
       "KubernetesCluster"                                                                                     = "minimal.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.minimal.example.com"
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
       "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
       "k8s.io/role/master"                                                                                    = "1"
       "kops.k8s.io/instancegroup"                                                                             = "master-us-test-1a"
@@ -423,9 +456,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" {
     "KubernetesCluster"                                                                                     = "minimal.example.com"
     "Name"                                                                                                  = "master-us-test-1a.masters.minimal.example.com"
     "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
-    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
     "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
     "k8s.io/role/master"                                                                                    = "1"
     "kops.k8s.io/instancegroup"                                                                             = "master-us-test-1a"
@@ -476,7 +507,6 @@ resource "aws_launch_template" "nodes-minimal-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "minimal.example.com"
       "Name"                                                                       = "nodes.minimal.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -488,7 +518,6 @@ resource "aws_launch_template" "nodes-minimal-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "minimal.example.com"
       "Name"                                                                       = "nodes.minimal.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -498,7 +527,6 @@ resource "aws_launch_template" "nodes-minimal-example-com" {
   tags = {
     "KubernetesCluster"                                                          = "minimal.example.com"
     "Name"                                                                       = "nodes.minimal.example.com"
-    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/node"                                                           = "1"
     "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -608,6 +636,22 @@ resource "aws_s3_object" "manifests-static-kube-apiserver-healthcheck" {
   server_side_encryption = "AES256"
 }
 
+resource "aws_s3_object" "minimal-example-com-addons-aws-cloud-controller-addons-k8s-io-k8s-1-18" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content")
+  key                    = "clusters.example.com/minimal.example.com/addons/aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
+resource "aws_s3_object" "minimal-example-com-addons-aws-ebs-csi-driver-addons-k8s-io-k8s-1-17" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content")
+  key                    = "clusters.example.com/minimal.example.com/addons/aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
 resource "aws_s3_object" "minimal-example-com-addons-bootstrap" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-bootstrap_content")
@@ -624,6 +668,14 @@ resource "aws_s3_object" "minimal-example-com-addons-core-addons-k8s-io" {
   server_side_encryption = "AES256"
 }
 
+resource "aws_s3_object" "minimal-example-com-addons-coredns-addons-k8s-io-k8s-1-12" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content")
+  key                    = "clusters.example.com/minimal.example.com/addons/coredns.addons.k8s.io/k8s-1.12.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
 resource "aws_s3_object" "minimal-example-com-addons-dns-controller-addons-k8s-io-k8s-1-12" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content")
@@ -640,14 +692,6 @@ resource "aws_s3_object" "minimal-example-com-addons-kops-controller-addons-k8s-
   server_side_encryption = "AES256"
 }
 
-resource "aws_s3_object" "minimal-example-com-addons-kube-dns-addons-k8s-io-k8s-1-12" {
-  bucket                 = "testingBucket"
-  content                = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-kube-dns.addons.k8s.io-k8s-1.12_content")
-  key                    = "clusters.example.com/minimal.example.com/addons/kube-dns.addons.k8s.io/k8s-1.12.yaml"
-  provider               = aws.files
-  server_side_encryption = "AES256"
-}
-
 resource "aws_s3_object" "minimal-example-com-addons-kubelet-api-rbac-addons-k8s-io-k8s-1-9" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content")
@@ -656,6 +700,14 @@ resource "aws_s3_object" "minimal-example-com-addons-kubelet-api-rbac-addons-k8s
   server_side_encryption = "AES256"
 }
 
+resource "aws_s3_object" "minimal-example-com-addons-leader-migration-rbac-addons-k8s-io-k8s-1-23" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content")
+  key                    = "clusters.example.com/minimal.example.com/addons/leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
 resource "aws_s3_object" "minimal-example-com-addons-limit-range-addons-k8s-io" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-limit-range.addons.k8s.io_content")
@@ -837,8 +889,10 @@ resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-udp-1
 }
 
 resource "aws_subnet" "us-test-1a-minimal-example-com" {
-  availability_zone = "us-test-1a"
+  availability_zone                           = "us-test-1a"
-  cidr_block        = "172.20.32.0/19"
+  cidr_block                                  = "172.20.32.0/19"
+  enable_resource_name_dns_a_record_on_launch = true
+  private_dns_hostname_type_on_launch         = "resource-name"
   tags = {
     "KubernetesCluster"                         = "minimal.example.com"
     "Name"                                      = "us-test-1a.minimal.example.com"