Merge pull request #15531 from hakman/min_ver_k8s_1.28

Update min versions for kOps v1.28

channels/pkg/cmd/apply_channel.go

@@ -22,7 +22,6 @@ import (
 	"io"
 	"net/url"
 	"os"
-	"strings"
 
 	"github.com/blang/semver/v4"
 	"github.com/cert-manager/cert-manager/pkg/client/clientset/versioned"
@@ -32,9 +31,7 @@ import (
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/restmapper"
-	"k8s.io/klog/v2"
 	"k8s.io/kops/channels/pkg/channels"
-	"k8s.io/kops/pkg/apis/kops/util"
 	"k8s.io/kops/util/pkg/tables"
 )
 
@@ -227,23 +224,10 @@ func buildMenu(kubernetesVersion semver.Version, channelLocation string) (*chann
 		return nil, fmt.Errorf("unable to parse argument %q as url", channelLocation)
 	}
 	if !location.IsAbs() {
-		// We recognize the following "well-known" format:
-		// <name> with no slashes ->
-		if strings.Contains(channelLocation, "/") {
-			return nil, fmt.Errorf("channel format not recognized (did you mean to use `-f` to specify a local file?): %q", channelLocation)
-		}
 		expanded := "https://raw.githubusercontent.com/kubernetes/kops/master/addons/" + channelLocation + "/addon.yaml"
-		location, err = url.Parse(expanded)
-		if err != nil {
-			return nil, fmt.Errorf("unable to parse expanded argument %q as url", expanded)
-		}
 		// Disallow the use of legacy addons from the "well-known" location starting Kubernetes 1.23:
 		// https://raw.githubusercontent.com/kubernetes/kops/master/addons/<name>/addon.yaml
-		if util.IsKubernetesGTE("1.23", kubernetesVersion) {
 		return nil, fmt.Errorf("legacy addons are deprecated and unmaintained, use managed addons instead of %s", expanded)
-		} else {
-			klog.Warningf("Legacy addons are deprecated and unmaintained, use managed addons instead of %s", expanded)
-		}
 	}
 	o, err := channels.LoadAddons(channelLocation, location)
 	if err != nil {

cmd/kops/integration_test.go

@@ -600,16 +600,6 @@ func TestBastionAdditionalUserData(t *testing.T) {
 		runTestTerraformAWS(t)
 }
 
-const weaveAddon = "networking.weave-k8s-1.12"
-
-// TestPrivateWeave runs the test on a configuration with private topology, weave networking
-func TestPrivateWeave(t *testing.T) {
-	newIntegrationTest("privateweave.example.com", "privateweave").
-		withPrivate().
-		withAddons(awsEBSCSIAddon, weaveAddon, dnsControllerAddon).
-		runTestTerraformAWS(t)
-}
-
 // TestPrivateFlannel runs the test on a configuration with private topology, flannel networking
 func TestPrivateFlannel(t *testing.T) {
 	newIntegrationTest("privateflannel.example.com", "privateflannel").
@@ -739,7 +729,11 @@ func TestPrivateSharedIP(t *testing.T) {
 func TestPrivateDns1(t *testing.T) {
 	newIntegrationTest("privatedns1.example.com", "privatedns1").
 		withPrivate().
-		withAddons(awsEBSCSIAddon, weaveAddon, dnsControllerAddon).
+		withAddons(
+			awsCCMAddon,
+			awsEBSCSIAddon,
+			dnsControllerAddon,
+		).
 		runTestTerraformAWS(t)
 }
 
@@ -1159,7 +1153,11 @@ func TestAPIServerNodes(t *testing.T) {
 	defer unsetFeatureFlags()
 
 	newIntegrationTest("minimal.example.com", "apiservernodes").
-		withAddons(dnsControllerAddon, awsEBSCSIAddon).
+		withAddons(
+			awsCCMAddon,
+			awsEBSCSIAddon,
+			dnsControllerAddon,
+		).
 		withDedicatedAPIServer().
 		runTestTerraformAWS(t)
 }

nodeup/pkg/model/kube_apiserver.go

@@ -670,7 +670,7 @@ func (b *KubeAPIServerBuilder) buildPod(ctx context.Context, kubeAPIServer *kops
 	kubemanifest.AddHostPathMapping(pod, container, "logfile", "/var/log/kube-apiserver.log", kubemanifest.WithReadWrite())
 	// We use lighter containers that don't include shells
 	// But they have richer logging support via klog
-	if b.IsKubernetesGTE("1.23") {
+	{
 		container.Command = []string{"/go-runner"}
 		container.Args = []string{
 			"--log-file=/var/log/kube-apiserver.log",
@@ -678,19 +678,6 @@ func (b *KubeAPIServerBuilder) buildPod(ctx context.Context, kubeAPIServer *kops
 			"/usr/local/bin/kube-apiserver",
 		}
 		container.Args = append(container.Args, sortedStrings(flags)...)
-	} else {
-		container.Command = []string{"/usr/local/bin/kube-apiserver"}
-		if kubeAPIServer.LogFormat != "" && kubeAPIServer.LogFormat != "text" {
-			// When logging-format is not text, some flags are not accepted.
-			// https://github.com/kubernetes/kops/issues/13245
-			container.Args = sortedStrings(flags)
-		} else {
-			container.Args = append(
-				sortedStrings(flags),
-				"--logtostderr=false", // https://github.com/kubernetes/klog/issues/60
-				"--alsologtostderr",
-				"--log-file=/var/log/kube-apiserver.log")
-		}
 	}
 
 	for _, path := range b.SSLHostPaths() {

nodeup/pkg/model/kube_controller_manager.go

@@ -231,7 +231,7 @@ func (b *KubeControllerManagerBuilder) buildPod(kcm *kops.KubeControllerManagerC
 	kubemanifest.AddHostPathMapping(pod, container, "logfile", "/var/log/kube-controller-manager.log", kubemanifest.WithReadWrite())
 	// We use lighter containers that don't include shells
 	// But they have richer logging support via klog
-	if b.IsKubernetesGTE("1.23") {
+	{
 		container.Command = []string{"/go-runner"}
 		container.Args = []string{
 			"--log-file=/var/log/kube-controller-manager.log",
@@ -239,19 +239,6 @@ func (b *KubeControllerManagerBuilder) buildPod(kcm *kops.KubeControllerManagerC
 			"/usr/local/bin/kube-controller-manager",
 		}
 		container.Args = append(container.Args, sortedStrings(flags)...)
-	} else {
-		container.Command = []string{"/usr/local/bin/kube-controller-manager"}
-		if kcm.LogFormat != "" && kcm.LogFormat != "text" {
-			// When logging-format is not text, some flags are not accepted.
-			// https://github.com/kubernetes/kops/issues/14100
-			container.Args = sortedStrings(flags)
-		} else {
-			container.Args = append(
-				sortedStrings(flags),
-				"--logtostderr=false", // https://github.com/kubernetes/klog/issues/60
-				"--alsologtostderr",
-				"--log-file=/var/log/kube-controller-manager.log")
-		}
 	}
 	for _, path := range b.SSLHostPaths() {
 		name := strings.Replace(path, "/", "", -1)

nodeup/pkg/model/kube_proxy.go

@@ -183,7 +183,7 @@ func (b *KubeProxyBuilder) buildPod() (*v1.Pod, error) {
 	kubemanifest.AddHostPathMapping(pod, container, "logfile", "/var/log/kube-proxy.log", kubemanifest.WithReadWrite())
 	// We use lighter containers that don't include shells
 	// But they have richer logging support via klog
-	if b.IsKubernetesGTE("1.23") {
+	{
 		container.Command = []string{"/go-runner"}
 		container.Args = []string{
 			"--log-file=/var/log/kube-proxy.log",
@@ -191,13 +191,6 @@ func (b *KubeProxyBuilder) buildPod() (*v1.Pod, error) {
 			"/usr/local/bin/kube-proxy",
 		}
 		container.Args = append(container.Args, sortedStrings(flags)...)
-	} else {
-		container.Command = []string{"/usr/local/bin/kube-proxy"}
-		container.Args = append(
-			sortedStrings(flags),
-			"--logtostderr=false", // https://github.com/kubernetes/klog/issues/60
-			"--alsologtostderr",
-			"--log-file=/var/log/kube-proxy.log")
 	}
 	{
 		kubemanifest.AddHostPathMapping(pod, container, "kubeconfig", "/var/lib/kube-proxy/kubeconfig")

nodeup/pkg/model/kube_scheduler.go

@@ -222,11 +222,8 @@ func (b *KubeSchedulerBuilder) buildPod(kubeScheduler *kops.KubeSchedulerConfig)
 	healthAction := &v1.HTTPGetAction{
 		Host:   "127.0.0.1",
 		Path:   "/healthz",
-		Port: intstr.FromInt(10251),
-	}
-	if b.IsKubernetesGTE("1.23") {
-		healthAction.Port = intstr.FromInt(10259)
-		healthAction.Scheme = v1.URISchemeHTTPS
+		Port:   intstr.FromInt(10259),
+		Scheme: v1.URISchemeHTTPS,
 	}
 
 	container := &v1.Container{
@@ -251,7 +248,7 @@ func (b *KubeSchedulerBuilder) buildPod(kubeScheduler *kops.KubeSchedulerConfig)
 	kubemanifest.AddHostPathMapping(pod, container, "logfile", "/var/log/kube-scheduler.log", kubemanifest.WithReadWrite())
 	// We use lighter containers that don't include shells
 	// But they have richer logging support via klog
-	if b.IsKubernetesGTE("1.23") {
+	{
 		container.Command = []string{"/go-runner"}
 		container.Args = []string{
 			"--log-file=/var/log/kube-scheduler.log",
@@ -259,19 +256,6 @@ func (b *KubeSchedulerBuilder) buildPod(kubeScheduler *kops.KubeSchedulerConfig)
 			"/usr/local/bin/kube-scheduler",
 		}
 		container.Args = append(container.Args, sortedStrings(flags)...)
-	} else {
-		container.Command = []string{"/usr/local/bin/kube-scheduler"}
-		if kubeScheduler.LogFormat != "" && kubeScheduler.LogFormat != "text" {
-			// When logging-format is not text, some flags are not accepted.
-			// https://github.com/kubernetes/kops/issues/14100
-			container.Args = sortedStrings(flags)
-		} else {
-			container.Args = append(
-				sortedStrings(flags),
-				"--logtostderr=false", // https://github.com/kubernetes/klog/issues/60
-				"--alsologtostderr",
-				"--log-file=/var/log/kube-scheduler.log")
-		}
 	}
 
 	if kubeScheduler.MaxPersistentVolumes != nil {

nodeup/pkg/model/tests/golden/side-loading/cluster.yaml

@@ -30,7 +30,7 @@ spec:
   iam: {}
   kubelet:
     anonymousAuth: false
-  kubernetesVersion: https://dl.k8s.io/release/v1.22.0
+  kubernetesVersion: https://dl.k8s.io/release/v1.27.0
   masterPublicName: api.minimal.example.com
   networkCIDR: 172.20.0.0/16
   networking:

nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-amd64.yaml

@@ -14,6 +14,9 @@ contents: |
   spec:
     containers:
     - args:
+      - --log-file=/var/log/kube-apiserver.log
+      - --also-stdout
+      - /usr/local/bin/kube-apiserver
       - --allow-privileged=true
       - --anonymous-auth=false
       - --api-audiences=kubernetes.svc.default
@@ -22,14 +25,14 @@ contents: |
       - --bind-address=0.0.0.0
       - --client-ca-file=/srv/kubernetes/ca.crt
       - --cloud-config=/etc/kubernetes/in-tree-cloud.config
-      - --cloud-provider=aws
+      - --cloud-provider=external
       - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,NodeRestriction,ResourceQuota
       - --etcd-cafile=/srv/kubernetes/kube-apiserver/etcd-ca.crt
       - --etcd-certfile=/srv/kubernetes/kube-apiserver/etcd-client.crt
       - --etcd-keyfile=/srv/kubernetes/kube-apiserver/etcd-client.key
       - --etcd-servers-overrides=/events#https://127.0.0.1:4002
       - --etcd-servers=https://127.0.0.1:4001
-      - --feature-gates=CSIMigrationAWS=true,InTreePluginAWSUnregister=true
+      - --feature-gates=InTreePluginAWSUnregister=true
       - --kubelet-client-certificate=/srv/kubernetes/kube-apiserver/kubelet-api.crt
       - --kubelet-client-key=/srv/kubernetes/kube-apiserver/kubelet-api.key
       - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP
@@ -50,12 +53,9 @@ contents: |
       - --tls-cert-file=/srv/kubernetes/kube-apiserver/server.crt
       - --tls-private-key-file=/srv/kubernetes/kube-apiserver/server.key
       - --v=2
-      - --logtostderr=false
-      - --alsologtostderr
-      - --log-file=/var/log/kube-apiserver.log
       command:
-      - /usr/local/bin/kube-apiserver
-      image: registry.k8s.io/kube-apiserver-amd64:v1.22.0
+      - /go-runner
+      image: registry.k8s.io/kube-apiserver-amd64:v1.27.0
       livenessProbe:
         httpGet:
           host: 127.0.0.1

nodeup/pkg/model/tests/golden/side-loading/tasks-kube-apiserver-arm64.yaml

@@ -14,6 +14,9 @@ contents: |
   spec:
     containers:
     - args:
+      - --log-file=/var/log/kube-apiserver.log
+      - --also-stdout
+      - /usr/local/bin/kube-apiserver
       - --allow-privileged=true
       - --anonymous-auth=false
       - --api-audiences=kubernetes.svc.default
@@ -22,14 +25,14 @@ contents: |
       - --bind-address=0.0.0.0
       - --client-ca-file=/srv/kubernetes/ca.crt
       - --cloud-config=/etc/kubernetes/in-tree-cloud.config
-      - --cloud-provider=aws
+      - --cloud-provider=external
       - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,NodeRestriction,ResourceQuota
       - --etcd-cafile=/srv/kubernetes/kube-apiserver/etcd-ca.crt
       - --etcd-certfile=/srv/kubernetes/kube-apiserver/etcd-client.crt
       - --etcd-keyfile=/srv/kubernetes/kube-apiserver/etcd-client.key
       - --etcd-servers-overrides=/events#https://127.0.0.1:4002
       - --etcd-servers=https://127.0.0.1:4001
-      - --feature-gates=CSIMigrationAWS=true,InTreePluginAWSUnregister=true
+      - --feature-gates=InTreePluginAWSUnregister=true
       - --kubelet-client-certificate=/srv/kubernetes/kube-apiserver/kubelet-api.crt
       - --kubelet-client-key=/srv/kubernetes/kube-apiserver/kubelet-api.key
       - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP
@@ -50,12 +53,9 @@ contents: |
       - --tls-cert-file=/srv/kubernetes/kube-apiserver/server.crt
       - --tls-private-key-file=/srv/kubernetes/kube-apiserver/server.key
       - --v=2
-      - --logtostderr=false
-      - --alsologtostderr
-      - --log-file=/var/log/kube-apiserver.log
       command:
-      - /usr/local/bin/kube-apiserver
-      image: registry.k8s.io/kube-apiserver-arm64:v1.22.0
+      - /go-runner
+      image: registry.k8s.io/kube-apiserver-arm64:v1.27.0
       livenessProbe:
         httpGet:
           host: 127.0.0.1

nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-amd64.yaml

@@ -10,18 +10,21 @@ contents: |
   spec:
     containers:
     - args:
+      - --log-file=/var/log/kube-controller-manager.log
+      - --also-stdout
+      - /usr/local/bin/kube-controller-manager
       - --allocate-node-cidrs=true
       - --attach-detach-reconcile-sync-period=1m0s
       - --authentication-kubeconfig=/var/lib/kube-controller-manager/kubeconfig
       - --authorization-kubeconfig=/var/lib/kube-controller-manager/kubeconfig
       - --cloud-config=/etc/kubernetes/in-tree-cloud.config
-      - --cloud-provider=aws
+      - --cloud-provider=external
       - --cluster-cidr=100.96.0.0/11
       - --cluster-name=minimal.example.com
       - --cluster-signing-cert-file=/srv/kubernetes/kube-controller-manager/ca.crt
       - --cluster-signing-key-file=/srv/kubernetes/kube-controller-manager/ca.key
       - --configure-cloud-routes=true
-      - --feature-gates=CSIMigrationAWS=true,InTreePluginAWSUnregister=true
+      - --feature-gates=InTreePluginAWSUnregister=true
       - --flex-volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/
       - --kubeconfig=/var/lib/kube-controller-manager/kubeconfig
       - --leader-elect=true
@@ -31,12 +34,9 @@ contents: |
       - --tls-private-key-file=/srv/kubernetes/kube-controller-manager/server.key
       - --use-service-account-credentials=true
       - --v=2
-      - --logtostderr=false
-      - --alsologtostderr
-      - --log-file=/var/log/kube-controller-manager.log
       command:
-      - /usr/local/bin/kube-controller-manager
-      image: registry.k8s.io/kube-controller-manager-amd64:v1.22.0
+      - /go-runner
+      image: registry.k8s.io/kube-controller-manager-amd64:v1.27.0
       livenessProbe:
         httpGet:
           host: 127.0.0.1

nodeup/pkg/model/tests/golden/side-loading/tasks-kube-controller-manager-arm64.yaml

@@ -10,18 +10,21 @@ contents: |
   spec:
     containers:
     - args:
+      - --log-file=/var/log/kube-controller-manager.log
+      - --also-stdout
+      - /usr/local/bin/kube-controller-manager
       - --allocate-node-cidrs=true
       - --attach-detach-reconcile-sync-period=1m0s
       - --authentication-kubeconfig=/var/lib/kube-controller-manager/kubeconfig
       - --authorization-kubeconfig=/var/lib/kube-controller-manager/kubeconfig
       - --cloud-config=/etc/kubernetes/in-tree-cloud.config
-      - --cloud-provider=aws
+      - --cloud-provider=external
       - --cluster-cidr=100.96.0.0/11
       - --cluster-name=minimal.example.com
       - --cluster-signing-cert-file=/srv/kubernetes/kube-controller-manager/ca.crt
       - --cluster-signing-key-file=/srv/kubernetes/kube-controller-manager/ca.key
       - --configure-cloud-routes=true
-      - --feature-gates=CSIMigrationAWS=true,InTreePluginAWSUnregister=true
+      - --feature-gates=InTreePluginAWSUnregister=true
       - --flex-volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/
       - --kubeconfig=/var/lib/kube-controller-manager/kubeconfig
       - --leader-elect=true
@@ -31,12 +34,9 @@ contents: |
       - --tls-private-key-file=/srv/kubernetes/kube-controller-manager/server.key
       - --use-service-account-credentials=true
       - --v=2
-      - --logtostderr=false
-      - --alsologtostderr
-      - --log-file=/var/log/kube-controller-manager.log
       command:
-      - /usr/local/bin/kube-controller-manager
-      image: registry.k8s.io/kube-controller-manager-arm64:v1.22.0
+      - /go-runner
+      image: registry.k8s.io/kube-controller-manager-arm64:v1.27.0
       livenessProbe:
         httpGet:
           host: 127.0.0.1

nodeup/pkg/model/tests/golden/side-loading/tasks-kube-proxy-amd64.yaml

@@ -12,18 +12,18 @@ contents: |
   spec:
     containers:
     - args:
+      - --log-file=/var/log/kube-proxy.log
+      - --also-stdout
+      - /usr/local/bin/kube-proxy
       - --cluster-cidr=100.96.0.0/11
       - --conntrack-max-per-core=131072
       - --kubeconfig=/var/lib/kube-proxy/kubeconfig
       - --master=https://127.0.0.1
       - --oom-score-adj=-998
       - --v=2
-      - --logtostderr=false
-      - --alsologtostderr
-      - --log-file=/var/log/kube-proxy.log
       command:
-      - /usr/local/bin/kube-proxy
-      image: registry.k8s.io/kube-proxy-amd64:v1.22.0
+      - /go-runner
+      image: registry.k8s.io/kube-proxy-amd64:v1.27.0
       name: kube-proxy
       resources:
         requests:

nodeup/pkg/model/tests/golden/side-loading/tasks-kube-proxy-arm64.yaml

@@ -12,18 +12,18 @@ contents: |
   spec:
     containers:
     - args:
+      - --log-file=/var/log/kube-proxy.log
+      - --also-stdout
+      - /usr/local/bin/kube-proxy
       - --cluster-cidr=100.96.0.0/11
       - --conntrack-max-per-core=131072
       - --kubeconfig=/var/lib/kube-proxy/kubeconfig
       - --master=https://127.0.0.1
       - --oom-score-adj=-998
       - --v=2
-      - --logtostderr=false
-      - --alsologtostderr
-      - --log-file=/var/log/kube-proxy.log
       command:
-      - /usr/local/bin/kube-proxy
-      image: registry.k8s.io/kube-proxy-arm64:v1.22.0
+      - /go-runner
+      image: registry.k8s.io/kube-proxy-arm64:v1.27.0
       name: kube-proxy
       resources:
         requests:

nodeup/pkg/model/tests/golden/side-loading/tasks-kube-scheduler-amd64.yaml

@@ -10,25 +10,26 @@ contents: |
   spec:
     containers:
     - args:
+      - --log-file=/var/log/kube-scheduler.log
+      - --also-stdout
+      - /usr/local/bin/kube-scheduler
       - --authentication-kubeconfig=/var/lib/kube-scheduler/kubeconfig
       - --authorization-kubeconfig=/var/lib/kube-scheduler/kubeconfig
       - --config=/var/lib/kube-scheduler/config.yaml
-      - --feature-gates=CSIMigrationAWS=true,InTreePluginAWSUnregister=true
+      - --feature-gates=InTreePluginAWSUnregister=true
       - --leader-elect=true
       - --tls-cert-file=/srv/kubernetes/kube-scheduler/server.crt
       - --tls-private-key-file=/srv/kubernetes/kube-scheduler/server.key
       - --v=2
-      - --logtostderr=false
-      - --alsologtostderr
-      - --log-file=/var/log/kube-scheduler.log
       command:
-      - /usr/local/bin/kube-scheduler
-      image: registry.k8s.io/kube-scheduler-amd64:v1.22.0
+      - /go-runner
+      image: registry.k8s.io/kube-scheduler-amd64:v1.27.0
       livenessProbe:
         httpGet:
           host: 127.0.0.1
           path: /healthz
-          port: 10251
+          port: 10259
+          scheme: HTTPS
         initialDelaySeconds: 15
         timeoutSeconds: 15
       name: kube-scheduler
@@ -96,7 +97,7 @@ path: /srv/kubernetes/kube-scheduler/server.key
 type: file
 ---
 contents: |
-  apiVersion: kubescheduler.config.k8s.io/v1beta2
+  apiVersion: kubescheduler.config.k8s.io/v1
   clientConnection:
     kubeconfig: /var/lib/kube-scheduler/kubeconfig
   kind: KubeSchedulerConfiguration

nodeup/pkg/model/tests/golden/side-loading/tasks-kube-scheduler-arm64.yaml

@@ -10,25 +10,26 @@ contents: |
   spec:
     containers:
     - args:
+      - --log-file=/var/log/kube-scheduler.log
+      - --also-stdout
+      - /usr/local/bin/kube-scheduler
       - --authentication-kubeconfig=/var/lib/kube-scheduler/kubeconfig
       - --authorization-kubeconfig=/var/lib/kube-scheduler/kubeconfig
       - --config=/var/lib/kube-scheduler/config.yaml
-      - --feature-gates=CSIMigrationAWS=true,InTreePluginAWSUnregister=true
+      - --feature-gates=InTreePluginAWSUnregister=true
       - --leader-elect=true
       - --tls-cert-file=/srv/kubernetes/kube-scheduler/server.crt
       - --tls-private-key-file=/srv/kubernetes/kube-scheduler/server.key
       - --v=2
-      - --logtostderr=false
-      - --alsologtostderr
-      - --log-file=/var/log/kube-scheduler.log
       command:
-      - /usr/local/bin/kube-scheduler
-      image: registry.k8s.io/kube-scheduler-arm64:v1.22.0
+      - /go-runner
+      image: registry.k8s.io/kube-scheduler-arm64:v1.27.0
       livenessProbe:
         httpGet:
           host: 127.0.0.1
           path: /healthz
-          port: 10251
+          port: 10259
+          scheme: HTTPS
         initialDelaySeconds: 15
         timeoutSeconds: 15
       name: kube-scheduler
@@ -96,7 +97,7 @@ path: /srv/kubernetes/kube-scheduler/server.key
 type: file
 ---
 contents: |
-  apiVersion: kubescheduler.config.k8s.io/v1beta2
+  apiVersion: kubescheduler.config.k8s.io/v1
   clientConnection:
     kubeconfig: /var/lib/kube-scheduler/kubeconfig
   kind: KubeSchedulerConfiguration

pkg/apis/kops/validation/validation.go

@@ -1055,11 +1055,7 @@ func validateNetworking(cluster *kops.Cluster, v *kops.NetworkingSpec, fldPath *
 		}
 		optionTaken = true
 
-		if cluster.IsKubernetesGTE("1.23") {
-			allErrs = append(allErrs, field.Forbidden(fldPath.Child("weave"), "Weave is not supported for Kubernetes >= 1.23"))
-		} else if cluster.Spec.IsIPv6Only() {
-			allErrs = append(allErrs, field.Forbidden(fldPath.Child("weave"), "Weave does not support IPv6"))
-		}
+		allErrs = append(allErrs, field.Forbidden(fldPath.Child("weave"), "Weave is no longer supported"))
 	}
 
 	if v.Flannel != nil {

pkg/model/components/addonmanifests/awscloudcontrollermanager/iam.go

@@ -33,9 +33,6 @@ func (r *ServiceAccount) BuildAWSPolicy(b *iam.PolicyBuilder) (*iam.Policy, erro
 	clusterName := b.Cluster.ObjectMeta.Name
 	p := iam.NewPolicy(clusterName, b.Partition)
 	iam.AddCCMPermissions(p, b.Cluster.Spec.Networking.Kubenet != nil)
-	if b.Cluster.IsKubernetesLT("1.23") {
-		iam.AddLegacyCCMPermissions(p)
-	}
 	return p, nil
 }
 

pkg/model/components/containerd.go

@@ -47,9 +47,7 @@ func (b *ContainerdOptionsBuilder) BuildOptions(o interface{}) error {
 		// Set version based on Kubernetes version
 		if fi.ValueOf(containerd.Version) == "" {
 			switch {
-			case b.IsKubernetesLT("1.23"):
-				containerd.Version = fi.PtrTo("1.4.13")
-			case b.IsKubernetesGTE("1.23") && b.IsKubernetesLT("1.24.14"):
+			case b.IsKubernetesLT("1.24.14"):
 				fallthrough
 			case b.IsKubernetesGTE("1.25") && b.IsKubernetesLT("1.25.10"):
 				fallthrough

pkg/model/components/gcppdcsidriver.go

@@ -37,7 +37,7 @@ func (b *GCPPDCSIDriverOptionsBuilder) BuildOptions(o interface{}) error {
 
 	if gce.PDCSIDriver == nil {
 		gce.PDCSIDriver = &kops.PDCSIDriver{
-			Enabled: fi.PtrTo(b.IsKubernetesGTE("1.23")),
+			Enabled: fi.PtrTo(true),
 		}
 	}
 

pkg/model/components/kubecontrollermanager.go

@@ -108,7 +108,7 @@ func (b *KubeControllerManagerOptionsBuilder) BuildOptions(o interface{}) error
 	}
 
 	if clusterSpec.ExternalCloudControllerManager == nil {
-		if b.IsKubernetesGTE("1.23") && (kcm.CloudProvider == "aws" || kcm.CloudProvider == "gce") {
+		if kcm.CloudProvider == "aws" || kcm.CloudProvider == "gce" {
 			kcm.EnableLeaderMigration = fi.PtrTo(true)
 		}
 	} else {

pkg/model/iam/iam_builder.go

@@ -418,10 +418,6 @@ func (r *NodeRoleMaster) BuildAWSPolicy(b *PolicyBuilder) (*Policy, error) {
 
 		if b.Cluster.Spec.ExternalCloudControllerManager != nil {
 			AddCCMPermissions(p, b.Cluster.Spec.Networking.Kubenet != nil)
-
-			if b.Cluster.IsKubernetesLT("1.23") {
-				AddLegacyCCMPermissions(p)
-			}
 		}
 
 		if c := b.Cluster.Spec.CloudProvider.AWS.LoadBalancerController; c != nil && fi.ValueOf(b.Cluster.Spec.CloudProvider.AWS.LoadBalancerController.Enabled) {

tests/integration/update_cluster/apiservernodes/data/aws_iam_role_policy_masters.minimal.example.com_policy

@@ -94,39 +94,6 @@
         "*"
       ]
     },
-    {
-      "Action": "ec2:CreateTags",
-      "Condition": {
-        "StringEquals": {
-          "aws:RequestTag/KubernetesCluster": "minimal.example.com",
-          "ec2:CreateAction": [
-            "CreateSecurityGroup"
-          ]
-        }
-      },
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws-test:ec2:*:*:security-group/*"
-      ]
-    },
-    {
-      "Action": [
-        "ec2:CreateTags",
-        "ec2:DeleteTags"
-      ],
-      "Condition": {
-        "Null": {
-          "aws:RequestTag/KubernetesCluster": "true"
-        },
-        "StringEquals": {
-          "aws:ResourceTag/KubernetesCluster": "minimal.example.com"
-        }
-      },
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws-test:ec2:*:*:security-group/*"
-      ]
-    },
     {
       "Action": "ec2:CreateTags",
       "Condition": {
@@ -163,6 +130,39 @@
         "arn:aws-test:ec2:*:*:snapshot/*"
       ]
     },
+    {
+      "Action": "ec2:CreateTags",
+      "Condition": {
+        "StringEquals": {
+          "aws:RequestTag/KubernetesCluster": "minimal.example.com",
+          "ec2:CreateAction": [
+            "CreateSecurityGroup"
+          ]
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws-test:ec2:*:*:security-group/*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:CreateTags",
+        "ec2:DeleteTags"
+      ],
+      "Condition": {
+        "Null": {
+          "aws:RequestTag/KubernetesCluster": "true"
+        },
+        "StringEquals": {
+          "aws:ResourceTag/KubernetesCluster": "minimal.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws-test:ec2:*:*:security-group/*"
+      ]
+    },
     {
       "Action": [
         "autoscaling:DescribeAutoScalingGroups",
@@ -170,13 +170,6 @@
         "autoscaling:DescribeLaunchConfigurations",
         "autoscaling:DescribeScalingActivities",
         "autoscaling:DescribeTags",
-        "ec2:AttachVolume",
-        "ec2:AuthorizeSecurityGroupIngress",
-        "ec2:CreateSecurityGroup",
-        "ec2:CreateTags",
-        "ec2:DeleteRoute",
-        "ec2:DeleteSecurityGroup",
-        "ec2:DeleteVolume",
         "ec2:DescribeAccountAttributes",
         "ec2:DescribeAvailabilityZones",
         "ec2:DescribeInstanceTypes",
@@ -190,21 +183,12 @@
         "ec2:DescribeVolumes",
         "ec2:DescribeVolumesModifications",
         "ec2:DescribeVpcs",
-        "ec2:DetachVolume",
-        "ec2:ModifyInstanceAttribute",
-        "ec2:ModifyVolume",
-        "elasticloadbalancing:AddTags",
-        "elasticloadbalancing:CreateListener",
-        "elasticloadbalancing:CreateTargetGroup",
-        "elasticloadbalancing:DeleteListener",
         "elasticloadbalancing:DescribeListeners",
         "elasticloadbalancing:DescribeLoadBalancerAttributes",
         "elasticloadbalancing:DescribeLoadBalancerPolicies",
         "elasticloadbalancing:DescribeLoadBalancers",
         "elasticloadbalancing:DescribeTargetGroups",
         "elasticloadbalancing:DescribeTargetHealth",
-        "elasticloadbalancing:ModifyListener",
-        "elasticloadbalancing:RegisterTargets",
         "iam:GetServerCertificate",
         "iam:ListServerCertificates",
         "kms:DescribeKey",

tests/integration/update_cluster/apiservernodes/data/aws_launch_template_apiserver.apiservers.minimal.example.com_user_data

@@ -129,17 +129,15 @@ kubelet:
   anonymousAuth: false
   cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
   featureGates:
-    CSIMigrationAWS: "true"
     InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   podInfraContainerImage: registry.k8s.io/pause:3.9
   podManifestPath: /etc/kubernetes/manifests
   protectKernelDefaults: true
@@ -155,7 +153,7 @@ ClusterName: minimal.example.com
 ConfigBase: memfs://clusters.example.com/minimal.example.com
 InstanceGroupName: apiserver
 InstanceGroupRole: APIServer
-NodeupConfigHash: kSCdMp/gjLRgJzSElxpRrIDsHRkHYoR2yMeUTvyshEo=
+NodeupConfigHash: PWoLV0n5BXdHt+IXY3lx73jt/CbT6t4GFHAhW6XsJXw=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/apiservernodes/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data

@@ -149,7 +149,7 @@ kubeAPIServer:
   apiServerCount: 1
   authorizationMode: AlwaysAllow
   bindAddress: 0.0.0.0
-  cloudProvider: aws
+  cloudProvider: external
   enableAdmissionPlugins:
   - NamespaceLifecycle
   - LimitRanger
@@ -165,9 +165,8 @@ kubeAPIServer:
   etcdServersOverrides:
   - /events#https://127.0.0.1:4002
   featureGates:
-    CSIMigrationAWS: "true"
     InTreePluginAWSUnregister: "true"
-  image: registry.k8s.io/kube-apiserver:v1.22.0
+  image: registry.k8s.io/kube-apiserver:v1.27.0
   kubeletPreferredAddressTypes:
   - InternalIP
   - Hostname
@@ -189,23 +188,21 @@ kubeAPIServer:
 kubeControllerManager:
   allocateNodeCIDRs: true
   attachDetachReconcileSyncPeriod: 1m0s
-  cloudProvider: aws
+  cloudProvider: external
   clusterCIDR: 100.96.0.0/11
   clusterName: minimal.example.com
   configureCloudRoutes: false
   featureGates:
-    CSIMigrationAWS: "true"
     InTreePluginAWSUnregister: "true"
-  image: registry.k8s.io/kube-controller-manager:v1.22.0
+  image: registry.k8s.io/kube-controller-manager:v1.27.0
   leaderElection:
     leaderElect: true
   logLevel: 2
   useServiceAccountCredentials: true
 kubeScheduler:
   featureGates:
-    CSIMigrationAWS: "true"
     InTreePluginAWSUnregister: "true"
-  image: registry.k8s.io/kube-scheduler:v1.22.0
+  image: registry.k8s.io/kube-scheduler:v1.27.0
   leaderElection:
     leaderElect: true
   logLevel: 2
@@ -213,17 +210,15 @@ kubelet:
   anonymousAuth: false
   cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
   featureGates:
-    CSIMigrationAWS: "true"
     InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   podInfraContainerImage: registry.k8s.io/pause:3.9
   podManifestPath: /etc/kubernetes/manifests
   protectKernelDefaults: true
@@ -234,17 +229,15 @@ masterKubelet:
   anonymousAuth: false
   cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
   featureGates:
-    CSIMigrationAWS: "true"
     InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   podInfraContainerImage: registry.k8s.io/pause:3.9
   podManifestPath: /etc/kubernetes/manifests
   protectKernelDefaults: true
@@ -260,7 +253,7 @@ ClusterName: minimal.example.com
 ConfigBase: memfs://clusters.example.com/minimal.example.com
 InstanceGroupName: master-us-test-1a
 InstanceGroupRole: ControlPlane
-NodeupConfigHash: 2hk/XP+S4gkEXNYvU94yWg9JfIK7v2S1Uuc72c+KnlA=
+NodeupConfigHash: eqz+91ktDxdS8bg3Pu3LXKD0crf87N54bl8hPed6uEc=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/apiservernodes/data/aws_launch_template_nodes.minimal.example.com_user_data

@@ -129,17 +129,15 @@ kubelet:
   anonymousAuth: false
   cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
   featureGates:
-    CSIMigrationAWS: "true"
     InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   podInfraContainerImage: registry.k8s.io/pause:3.9
   podManifestPath: /etc/kubernetes/manifests
   protectKernelDefaults: true
@@ -178,7 +176,7 @@ ConfigServer:
   - https://kops-controller.internal.minimal.example.com:3988/
 InstanceGroupName: nodes
 InstanceGroupRole: Node
-NodeupConfigHash: qqMvOWLA7UFUqWONNc9G+iUfFB30mbyzWqZUwxw6Ts4=
+NodeupConfigHash: DFBI2DthES2C3bTVqrVylh4m1gsWDYpsoI0QgPxUlwE=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/apiservernodes/data/aws_s3_object_cluster-completed.spec_content

@@ -14,6 +14,14 @@ spec:
       enabled: true
       version: v1.14.1
     manageStorageClasses: true
+  cloudControllerManager:
+    allocateNodeCIDRs: true
+    clusterCIDR: 100.64.0.0/10
+    clusterName: minimal.example.com
+    configureCloudRoutes: false
+    image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.26.0
+    leaderElection:
+      leaderElect: true
   cloudProvider: aws
   clusterDNSDomain: cluster.local
   configBase: memfs://clusters.example.com/minimal.example.com
@@ -21,7 +29,9 @@ spec:
   containerRuntime: containerd
   containerd:
     logLevel: info
-    version: 1.4.13
+    runc:
+      version: 1.1.5
+    version: 1.6.20
   dnsZone: Z1AFAKE1ZON3YO
   docker:
     skipInstall: true
@@ -63,7 +73,7 @@ spec:
     apiServerCount: 1
     authorizationMode: AlwaysAllow
     bindAddress: 0.0.0.0
-    cloudProvider: aws
+    cloudProvider: external
     enableAdmissionPlugins:
     - NamespaceLifecycle
     - LimitRanger
@@ -79,9 +89,8 @@ spec:
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
     featureGates:
-      CSIMigrationAWS: "true"
       InTreePluginAWSUnregister: "true"
-    image: registry.k8s.io/kube-apiserver:v1.22.0
+    image: registry.k8s.io/kube-apiserver:v1.27.0
     kubeletPreferredAddressTypes:
     - InternalIP
     - Hostname
@@ -103,14 +112,13 @@ spec:
   kubeControllerManager:
     allocateNodeCIDRs: true
     attachDetachReconcileSyncPeriod: 1m0s
-    cloudProvider: aws
+    cloudProvider: external
     clusterCIDR: 100.96.0.0/11
     clusterName: minimal.example.com
     configureCloudRoutes: false
     featureGates:
-      CSIMigrationAWS: "true"
       InTreePluginAWSUnregister: "true"
-    image: registry.k8s.io/kube-controller-manager:v1.22.0
+    image: registry.k8s.io/kube-controller-manager:v1.27.0
     leaderElection:
       leaderElect: true
     logLevel: 2
@@ -132,13 +140,12 @@ spec:
   kubeProxy:
     clusterCIDR: 100.96.0.0/11
     cpuRequest: 100m
-    image: registry.k8s.io/kube-proxy:v1.22.0
+    image: registry.k8s.io/kube-proxy:v1.27.0
     logLevel: 2
   kubeScheduler:
     featureGates:
-      CSIMigrationAWS: "true"
       InTreePluginAWSUnregister: "true"
-    image: registry.k8s.io/kube-scheduler:v1.22.0
+    image: registry.k8s.io/kube-scheduler:v1.27.0
     leaderElection:
       leaderElect: true
     logLevel: 2
@@ -146,17 +153,15 @@ spec:
     anonymousAuth: false
     cgroupDriver: systemd
     cgroupRoot: /
-    cloudProvider: aws
+    cloudProvider: external
     clusterDNS: 100.64.0.10
     clusterDomain: cluster.local
     enableDebuggingHandlers: true
     evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
     featureGates:
-      CSIMigrationAWS: "true"
       InTreePluginAWSUnregister: "true"
     kubeconfigPath: /var/lib/kubelet/kubeconfig
     logLevel: 2
-    networkPluginName: cni
     podInfraContainerImage: registry.k8s.io/pause:3.9
     podManifestPath: /etc/kubernetes/manifests
     protectKernelDefaults: true
@@ -165,22 +170,20 @@ spec:
     shutdownGracePeriodCriticalPods: 10s
   kubernetesApiAccess:
   - 0.0.0.0/0
-  kubernetesVersion: 1.22.0
+  kubernetesVersion: 1.27.0
   masterKubelet:
     anonymousAuth: false
     cgroupDriver: systemd
     cgroupRoot: /
-    cloudProvider: aws
+    cloudProvider: external
     clusterDNS: 100.64.0.10
     clusterDomain: cluster.local
     enableDebuggingHandlers: true
     evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
     featureGates:
-      CSIMigrationAWS: "true"
       InTreePluginAWSUnregister: "true"
     kubeconfigPath: /var/lib/kubelet/kubeconfig
     logLevel: 2
-    networkPluginName: cni
     podInfraContainerImage: registry.k8s.io/pause:3.9
     podManifestPath: /etc/kubernetes/manifests
     protectKernelDefaults: true

tests/integration/update_cluster/apiservernodes/data/aws_s3_object_minimal.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content

@@ -0,0 +1,237 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
+    k8s-app: aws-cloud-controller-manager
+  name: aws-cloud-controller-manager
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: aws-cloud-controller-manager
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        k8s-app: aws-cloud-controller-manager
+        kops.k8s.io/managed-by: kops
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
+      containers:
+      - args:
+        - --allocate-node-cidrs=true
+        - --cluster-cidr=100.64.0.0/10
+        - --cluster-name=minimal.example.com
+        - --configure-cloud-routes=false
+        - --leader-elect=true
+        - --v=2
+        - --cloud-provider=aws
+        - --use-service-account-credentials=true
+        - --cloud-config=/etc/kubernetes/cloud.config
+        env:
+        - name: KUBERNETES_SERVICE_HOST
+          value: 127.0.0.1
+        image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.26.0
+        imagePullPolicy: IfNotPresent
+        name: aws-cloud-controller-manager
+        resources:
+          requests:
+            cpu: 200m
+        volumeMounts:
+        - mountPath: /etc/kubernetes/cloud.config
+          name: cloudconfig
+          readOnly: true
+      hostNetwork: true
+      nodeSelector: null
+      priorityClassName: system-cluster-critical
+      serviceAccountName: aws-cloud-controller-manager
+      tolerations:
+      - effect: NoSchedule
+        key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+      - effect: NoSchedule
+        key: node.kubernetes.io/not-ready
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+      volumes:
+      - hostPath:
+          path: /etc/kubernetes/cloud.config
+          type: ""
+        name: cloudconfig
+  updateStrategy:
+    type: RollingUpdate
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
+  name: aws-cloud-controller-manager
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
+  name: cloud-controller-manager:apiserver-authentication-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- apiGroup: ""
+  kind: ServiceAccount
+  name: aws-cloud-controller-manager
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
+  name: system:cloud-controller-manager
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - services/status
+  verbs:
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resourceNames:
+  - node-controller
+  - service-controller
+  - route-controller
+  resources:
+  - serviceaccounts/token
+  verbs:
+  - create
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
+  name: system:cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:cloud-controller-manager
+subjects:
+- apiGroup: ""
+  kind: ServiceAccount
+  name: aws-cloud-controller-manager
+  namespace: kube-system

tests/integration/update_cluster/apiservernodes/data/aws_s3_object_minimal.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content

@@ -798,7 +798,7 @@ spec:
 
 ---
 
-apiVersion: policy/v1beta1
+apiVersion: policy/v1
 kind: PodDisruptionBudget
 metadata:
   creationTimestamp: null

tests/integration/update_cluster/apiservernodes/data/aws_s3_object_minimal.example.com-addons-bootstrap_content

@@ -6,7 +6,7 @@ spec:
   addons:
   - id: k8s-1.16
     manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: 9dbd9a3614451ffda066ce8979fee1c9044ffd906d2c1ea97f2bbf1e81a52b5e
+    manifestHash: 87663bc0ea809ce11f6e766358b342a3b4a45e2ffb681b72f0560d1318254c9e
     name: kops-controller.addons.k8s.io
     needsRollingUpdate: control-plane
     selector:
@@ -14,7 +14,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 49273720e8c046ae8b72a3fe666d670f6efb5e26111dc521027218b1e8e1de51
+    manifestHash: d2bbb7cbee5835c3891fe80fbacf8963508359ef9159f8480325ce9a7174f14a
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io
@@ -46,9 +46,16 @@ spec:
     selector:
       k8s-addon: storage-aws.addons.k8s.io
     version: 9.99.0
+  - id: k8s-1.18
+    manifest: aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml
+    manifestHash: 557d71c430bb05a5b069fd8dc3a0a3247261795bfd0617b97cbf1f31fed3fc27
+    name: aws-cloud-controller.addons.k8s.io
+    selector:
+      k8s-addon: aws-cloud-controller.addons.k8s.io
+    version: 9.99.0
   - id: k8s-1.17
     manifest: aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml
-    manifestHash: 80a04c96830e1279702d4cdf8004416edc2020f7ada484e5213693962c0ade91
+    manifestHash: 9ebe176a18822b64f30849e1b29a147a73e49bb0c445c78cba85703ea3a3221f
     name: aws-ebs-csi-driver.addons.k8s.io
     selector:
       k8s-addon: aws-ebs-csi-driver.addons.k8s.io

tests/integration/update_cluster/apiservernodes/data/aws_s3_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -242,7 +242,7 @@ spec:
 
 ---
 
-apiVersion: policy/v1beta1
+apiVersion: policy/v1
 kind: PodDisruptionBudget
 metadata:
   creationTimestamp: null

tests/integration/update_cluster/apiservernodes/data/aws_s3_object_minimal.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content

@@ -1,7 +1,7 @@
 apiVersion: v1
 data:
   config.yaml: |
-    {"clusterName":"minimal.example.com","cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com","secretStore":"memfs://clusters.example.com/minimal.example.com/secrets","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["apiservers.minimal.example.com","nodes.minimal.example.com"],"Region":"us-test-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"]}}
+    {"clusterName":"minimal.example.com","cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com","secretStore":"memfs://clusters.example.com/minimal.example.com/secrets","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["apiservers.minimal.example.com","nodes.minimal.example.com"],"Region":"us-test-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"],"useInstanceIDForNodeName":true}}
 kind: ConfigMap
 metadata:
   creationTimestamp: null

tests/integration/update_cluster/apiservernodes/data/aws_s3_object_nodeupconfig-apiserver_content

@@ -7,7 +7,7 @@ APIServerConfig:
     apiServerCount: 1
     authorizationMode: AlwaysAllow
     bindAddress: 0.0.0.0
-    cloudProvider: aws
+    cloudProvider: external
     enableAdmissionPlugins:
     - NamespaceLifecycle
     - LimitRanger
@@ -23,9 +23,8 @@ APIServerConfig:
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
     featureGates:
-      CSIMigrationAWS: "true"
       InTreePluginAWSUnregister: "true"
-    image: registry.k8s.io/kube-apiserver:v1.22.0
+    image: registry.k8s.io/kube-apiserver:v1.27.0
     kubeletPreferredAddressTypes:
     - InternalIP
     - Hostname
@@ -55,15 +54,19 @@ APIServerConfig:
     -----END RSA PUBLIC KEY-----
 Assets:
   amd64:
-  - fec5c596f7f815f17f5d7d955e9707df1ef02a2ca5e788b223651f83376feb7f@https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubelet
-  - 703e70d49b82271535bc66bc7bd469a58c11d47f188889bd37101c9772f14fa1@https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubectl
-  - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
-  - 29ef1e8635795c2a49a20a56e778f45ff163c5400a5428ca33999ed53d44e3d8@https://github.com/containerd/containerd/releases/download/v1.4.13/cri-containerd-cni-1.4.13-linux-amd64.tar.gz
+  - 0b4ed4fcd75d33f5dff3ba17776e6089847fc83064d3f7a3ad59a34e94e60a29@https://dl.k8s.io/release/v1.27.0/bin/linux/amd64/kubelet
+  - 71a78259d70da9c5540c4cf4cff121f443e863376f68f89a759d90cef3f51e87@https://dl.k8s.io/release/v1.27.0/bin/linux/amd64/kubectl
+  - 5035d7814c95cd3cedbc5efb447ef25a4942ef05caab2159746d55ce1698c74a@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/amd64/ecr-credential-provider-linux-amd64
+  - f3a841324845ca6bf0d4091b4fc7f97e18a623172158b72fc3fdcdb9d42d2d37@https://storage.googleapis.com/k8s-artifacts-cni/release/v1.2.0/cni-plugins-linux-amd64-v1.2.0.tgz
+  - bb9a9ccd6517e2a54da748a9f60dc9aa9d79d19d4724663f2386812f083968e2@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-amd64.tar.gz
+  - f00b144e86f8c1db347a2e8f22caade07d55382c5f76dd5c0a5b1ab64eaec8bb@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.amd64
   arm64:
-  - cea637a7da4f1097b16b0195005351c07032a820a3d64c3ff326b9097cfac930@https://dl.k8s.io/release/v1.22.0/bin/linux/arm64/kubelet
-  - 8d9cc92dcc942f5ea2b2fc93c4934875d9e0e8ddecbde24c7d4c4e092cfc7afc@https://dl.k8s.io/release/v1.22.0/bin/linux/arm64/kubectl
-  - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
-  - debed306ed9a4e70dcbcb228a0b3898f9730099e324f34bb0e76abbaddf7a6a7@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.13.tgz
+  - 37aa2edc7c0c4b3e488518c6a4b44c8aade75a55010534ee2be291220c73d157@https://dl.k8s.io/release/v1.27.0/bin/linux/arm64/kubelet
+  - f8e09630211f2b7c6a8cc38835e7dea94708d401f5c84b23a37c70c604602ddc@https://dl.k8s.io/release/v1.27.0/bin/linux/arm64/kubectl
+  - b3d567bda9e2996fc1fbd9d13506bd16763d3865b5c7b0b3c4b48c6088c04481@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/arm64/ecr-credential-provider-linux-arm64
+  - 525e2b62ba92a1b6f3dc9612449a84aa61652e680f7ebf4eff579795fe464b57@https://storage.googleapis.com/k8s-artifacts-cni/release/v1.2.0/cni-plugins-linux-arm64-v1.2.0.tgz
+  - c3e6a054b18b20fce06c7c3ed53f0989bb4b255c849bede446ebca955f07a9ce@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-arm64.tar.gz
+  - 54e79e4d48b9e191767e4abc08be1a8476a1c757e9a9f8c45c6ded001226867f@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.arm64
 CAs:
   apiserver-aggregator-ca: |
     -----BEGIN CERTIFICATE-----
@@ -134,7 +137,7 @@ ClusterName: minimal.example.com
 ContainerRuntime: containerd
 FileAssets:
 - content: |
-    apiVersion: kubescheduler.config.k8s.io/v1beta2
+    apiVersion: kubescheduler.config.k8s.io/v1
     clientConnection:
       kubeconfig: /var/lib/kube-scheduler/kubeconfig
     kind: KubeSchedulerConfiguration
@@ -150,25 +153,22 @@ KeypairIDs:
 KubeProxy:
   clusterCIDR: 100.96.0.0/11
   cpuRequest: 100m
-  image: registry.k8s.io/kube-proxy:v1.22.0
+  image: registry.k8s.io/kube-proxy:v1.27.0
   logLevel: 2
 KubeletConfig:
   anonymousAuth: false
   cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
   featureGates:
-    CSIMigrationAWS: "true"
     InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   nodeLabels:
-    kubernetes.io/role: api-server
     node-role.kubernetes.io/api-server: ""
   podInfraContainerImage: registry.k8s.io/pause:3.9
   podManifestPath: /etc/kubernetes/manifests
@@ -178,7 +178,7 @@ KubeletConfig:
   shutdownGracePeriodCriticalPods: 10s
   taints:
   - node-role.kubernetes.io/api-server=:NoSchedule
-KubernetesVersion: 1.22.0
+KubernetesVersion: 1.27.0
 Networking:
   nonMasqueradeCIDR: 100.64.0.0/10
   serviceClusterIPRange: 100.64.0.0/13
@@ -187,11 +187,14 @@ channels:
 - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml
 containerdConfig:
   logLevel: info
-  version: 1.4.13
+  runc:
+    version: 1.1.5
+  version: 1.6.20
 docker:
   skipInstall: true
 staticManifests:
 - key: kube-apiserver-healthcheck
   path: manifests/static/kube-apiserver-healthcheck.yaml
+useInstanceIDForNodeName: true
 usesLegacyGossip: false
 usesNoneDNS: false

tests/integration/update_cluster/apiservernodes/data/aws_s3_object_nodeupconfig-master-us-test-1a_content

@@ -7,7 +7,7 @@ APIServerConfig:
     apiServerCount: 1
     authorizationMode: AlwaysAllow
     bindAddress: 0.0.0.0
-    cloudProvider: aws
+    cloudProvider: external
     enableAdmissionPlugins:
     - NamespaceLifecycle
     - LimitRanger
@@ -23,9 +23,8 @@ APIServerConfig:
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
     featureGates:
-      CSIMigrationAWS: "true"
       InTreePluginAWSUnregister: "true"
-    image: registry.k8s.io/kube-apiserver:v1.22.0
+    image: registry.k8s.io/kube-apiserver:v1.27.0
     kubeletPreferredAddressTypes:
     - InternalIP
     - Hostname
@@ -55,17 +54,21 @@ APIServerConfig:
     -----END RSA PUBLIC KEY-----
 Assets:
   amd64:
-  - fec5c596f7f815f17f5d7d955e9707df1ef02a2ca5e788b223651f83376feb7f@https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubelet
-  - 703e70d49b82271535bc66bc7bd469a58c11d47f188889bd37101c9772f14fa1@https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubectl
-  - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
-  - 29ef1e8635795c2a49a20a56e778f45ff163c5400a5428ca33999ed53d44e3d8@https://github.com/containerd/containerd/releases/download/v1.4.13/cri-containerd-cni-1.4.13-linux-amd64.tar.gz
+  - 0b4ed4fcd75d33f5dff3ba17776e6089847fc83064d3f7a3ad59a34e94e60a29@https://dl.k8s.io/release/v1.27.0/bin/linux/amd64/kubelet
+  - 71a78259d70da9c5540c4cf4cff121f443e863376f68f89a759d90cef3f51e87@https://dl.k8s.io/release/v1.27.0/bin/linux/amd64/kubectl
+  - 5035d7814c95cd3cedbc5efb447ef25a4942ef05caab2159746d55ce1698c74a@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/amd64/ecr-credential-provider-linux-amd64
+  - f3a841324845ca6bf0d4091b4fc7f97e18a623172158b72fc3fdcdb9d42d2d37@https://storage.googleapis.com/k8s-artifacts-cni/release/v1.2.0/cni-plugins-linux-amd64-v1.2.0.tgz
+  - bb9a9ccd6517e2a54da748a9f60dc9aa9d79d19d4724663f2386812f083968e2@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-amd64.tar.gz
+  - f00b144e86f8c1db347a2e8f22caade07d55382c5f76dd5c0a5b1ab64eaec8bb@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.amd64
   - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64
   - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64
   arm64:
-  - cea637a7da4f1097b16b0195005351c07032a820a3d64c3ff326b9097cfac930@https://dl.k8s.io/release/v1.22.0/bin/linux/arm64/kubelet
-  - 8d9cc92dcc942f5ea2b2fc93c4934875d9e0e8ddecbde24c7d4c4e092cfc7afc@https://dl.k8s.io/release/v1.22.0/bin/linux/arm64/kubectl
-  - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
-  - debed306ed9a4e70dcbcb228a0b3898f9730099e324f34bb0e76abbaddf7a6a7@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.13.tgz
+  - 37aa2edc7c0c4b3e488518c6a4b44c8aade75a55010534ee2be291220c73d157@https://dl.k8s.io/release/v1.27.0/bin/linux/arm64/kubelet
+  - f8e09630211f2b7c6a8cc38835e7dea94708d401f5c84b23a37c70c604602ddc@https://dl.k8s.io/release/v1.27.0/bin/linux/arm64/kubectl
+  - b3d567bda9e2996fc1fbd9d13506bd16763d3865b5c7b0b3c4b48c6088c04481@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/arm64/ecr-credential-provider-linux-arm64
+  - 525e2b62ba92a1b6f3dc9612449a84aa61652e680f7ebf4eff579795fe464b57@https://storage.googleapis.com/k8s-artifacts-cni/release/v1.2.0/cni-plugins-linux-arm64-v1.2.0.tgz
+  - c3e6a054b18b20fce06c7c3ed53f0989bb4b255c849bede446ebca955f07a9ce@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-arm64.tar.gz
+  - 54e79e4d48b9e191767e4abc08be1a8476a1c757e9a9f8c45c6ded001226867f@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.arm64
   - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64
   - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64
 CAs:
@@ -224,7 +227,7 @@ ClusterName: minimal.example.com
 ContainerRuntime: containerd
 FileAssets:
 - content: |
-    apiVersion: kubescheduler.config.k8s.io/v1beta2
+    apiVersion: kubescheduler.config.k8s.io/v1
     clientConnection:
       kubeconfig: /var/lib/kube-scheduler/kubeconfig
     kind: KubeSchedulerConfiguration
@@ -244,29 +247,25 @@ KeypairIDs:
 KubeProxy:
   clusterCIDR: 100.96.0.0/11
   cpuRequest: 100m
-  image: registry.k8s.io/kube-proxy:v1.22.0
+  image: registry.k8s.io/kube-proxy:v1.27.0
   logLevel: 2
 KubeletConfig:
   anonymousAuth: false
   cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
   featureGates:
-    CSIMigrationAWS: "true"
     InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   nodeLabels:
     kops.k8s.io/kops-controller-pki: ""
-    kubernetes.io/role: master
     node-role.kubernetes.io/api-server: ""
     node-role.kubernetes.io/control-plane: ""
-    node-role.kubernetes.io/master: ""
     node.kubernetes.io/exclude-from-external-load-balancers: ""
   podInfraContainerImage: registry.k8s.io/pause:3.9
   podManifestPath: /etc/kubernetes/manifests
@@ -275,8 +274,8 @@ KubeletConfig:
   shutdownGracePeriod: 30s
   shutdownGracePeriodCriticalPods: 10s
   taints:
-  - node-role.kubernetes.io/master=:NoSchedule
-KubernetesVersion: 1.22.0
+  - node-role.kubernetes.io/control-plane=:NoSchedule
+KubernetesVersion: 1.27.0
 Networking:
   nonMasqueradeCIDR: 100.64.0.0/10
   serviceClusterIPRange: 100.64.0.0/13
@@ -285,7 +284,9 @@ channels:
 - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml
 containerdConfig:
   logLevel: info
-  version: 1.4.13
+  runc:
+    version: 1.1.5
+  version: 1.6.20
 docker:
   skipInstall: true
 etcdManifests:
@@ -294,5 +295,6 @@ etcdManifests:
 staticManifests:
 - key: kube-apiserver-healthcheck
   path: manifests/static/kube-apiserver-healthcheck.yaml
+useInstanceIDForNodeName: true
 usesLegacyGossip: false
 usesNoneDNS: false

tests/integration/update_cluster/apiservernodes/data/aws_s3_object_nodeupconfig-nodes_content

@@ -1,14 +1,18 @@
 Assets:
   amd64:
-  - fec5c596f7f815f17f5d7d955e9707df1ef02a2ca5e788b223651f83376feb7f@https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubelet
-  - 703e70d49b82271535bc66bc7bd469a58c11d47f188889bd37101c9772f14fa1@https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubectl
-  - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
-  - 29ef1e8635795c2a49a20a56e778f45ff163c5400a5428ca33999ed53d44e3d8@https://github.com/containerd/containerd/releases/download/v1.4.13/cri-containerd-cni-1.4.13-linux-amd64.tar.gz
+  - 0b4ed4fcd75d33f5dff3ba17776e6089847fc83064d3f7a3ad59a34e94e60a29@https://dl.k8s.io/release/v1.27.0/bin/linux/amd64/kubelet
+  - 71a78259d70da9c5540c4cf4cff121f443e863376f68f89a759d90cef3f51e87@https://dl.k8s.io/release/v1.27.0/bin/linux/amd64/kubectl
+  - 5035d7814c95cd3cedbc5efb447ef25a4942ef05caab2159746d55ce1698c74a@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/amd64/ecr-credential-provider-linux-amd64
+  - f3a841324845ca6bf0d4091b4fc7f97e18a623172158b72fc3fdcdb9d42d2d37@https://storage.googleapis.com/k8s-artifacts-cni/release/v1.2.0/cni-plugins-linux-amd64-v1.2.0.tgz
+  - bb9a9ccd6517e2a54da748a9f60dc9aa9d79d19d4724663f2386812f083968e2@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-amd64.tar.gz
+  - f00b144e86f8c1db347a2e8f22caade07d55382c5f76dd5c0a5b1ab64eaec8bb@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.amd64
   arm64:
-  - cea637a7da4f1097b16b0195005351c07032a820a3d64c3ff326b9097cfac930@https://dl.k8s.io/release/v1.22.0/bin/linux/arm64/kubelet
-  - 8d9cc92dcc942f5ea2b2fc93c4934875d9e0e8ddecbde24c7d4c4e092cfc7afc@https://dl.k8s.io/release/v1.22.0/bin/linux/arm64/kubectl
-  - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
-  - debed306ed9a4e70dcbcb228a0b3898f9730099e324f34bb0e76abbaddf7a6a7@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.13.tgz
+  - 37aa2edc7c0c4b3e488518c6a4b44c8aade75a55010534ee2be291220c73d157@https://dl.k8s.io/release/v1.27.0/bin/linux/arm64/kubelet
+  - f8e09630211f2b7c6a8cc38835e7dea94708d401f5c84b23a37c70c604602ddc@https://dl.k8s.io/release/v1.27.0/bin/linux/arm64/kubectl
+  - b3d567bda9e2996fc1fbd9d13506bd16763d3865b5c7b0b3c4b48c6088c04481@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/arm64/ecr-credential-provider-linux-arm64
+  - 525e2b62ba92a1b6f3dc9612449a84aa61652e680f7ebf4eff579795fe464b57@https://storage.googleapis.com/k8s-artifacts-cni/release/v1.2.0/cni-plugins-linux-arm64-v1.2.0.tgz
+  - c3e6a054b18b20fce06c7c3ed53f0989bb4b255c849bede446ebca955f07a9ce@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-arm64.tar.gz
+  - 54e79e4d48b9e191767e4abc08be1a8476a1c757e9a9f8c45c6ded001226867f@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.arm64
 CAs: {}
 ClusterName: minimal.example.com
 ContainerRuntime: containerd
@@ -20,25 +24,22 @@ KeypairIDs:
 KubeProxy:
   clusterCIDR: 100.96.0.0/11
   cpuRequest: 100m
-  image: registry.k8s.io/kube-proxy:v1.22.0
+  image: registry.k8s.io/kube-proxy:v1.27.0
   logLevel: 2
 KubeletConfig:
   anonymousAuth: false
   cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
   featureGates:
-    CSIMigrationAWS: "true"
     InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   nodeLabels:
-    kubernetes.io/role: node
     node-role.kubernetes.io/node: ""
   podInfraContainerImage: registry.k8s.io/pause:3.9
   podManifestPath: /etc/kubernetes/manifests
@@ -46,7 +47,7 @@ KubeletConfig:
   registerSchedulable: true
   shutdownGracePeriod: 30s
   shutdownGracePeriodCriticalPods: 10s
-KubernetesVersion: 1.22.0
+KubernetesVersion: 1.27.0
 Networking:
   nonMasqueradeCIDR: 100.64.0.0/10
   serviceClusterIPRange: 100.64.0.0/13
@@ -55,8 +56,11 @@ channels:
 - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml
 containerdConfig:
   logLevel: info
-  version: 1.4.13
+  runc:
+    version: 1.1.5
+  version: 1.6.20
 docker:
   skipInstall: true
+useInstanceIDForNodeName: true
 usesLegacyGossip: false
 usesNoneDNS: false

tests/integration/update_cluster/apiservernodes/in-v1alpha2.yaml

@@ -21,7 +21,7 @@ spec:
   iam: {}
   kubelet:
     anonymousAuth: false
-  kubernetesVersion: v1.22.0
+  kubernetesVersion: v1.27.0
   masterInternalName: api.internal.minimal.example.com
   masterPublicName: api.minimal.example.com
   networkCIDR: 172.20.0.0/16

tests/integration/update_cluster/apiservernodes/kubernetes.tf

@@ -137,11 +137,6 @@ resource "aws_autoscaling_group" "apiserver-apiservers-minimal-example-com" {
     propagate_at_launch = true
     value               = "apiserver.apiservers.minimal.example.com"
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"
-    propagate_at_launch = true
-    value               = "api-server"
-  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/api-server"
     propagate_at_launch = true
@@ -192,11 +187,6 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com"
     propagate_at_launch = true
     value               = ""
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"
-    propagate_at_launch = true
-    value               = "master"
-  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/api-server"
     propagate_at_launch = true
@@ -207,11 +197,6 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com"
     propagate_at_launch = true
     value               = ""
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"
-    propagate_at_launch = true
-    value               = ""
-  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers"
     propagate_at_launch = true
@@ -262,11 +247,6 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" {
     propagate_at_launch = true
     value               = "nodes.minimal.example.com"
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"
-    propagate_at_launch = true
-    value               = "node"
-  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
     propagate_at_launch = true
@@ -446,7 +426,7 @@ resource "aws_launch_template" "apiserver-apiservers-minimal-example-com" {
     http_endpoint               = "enabled"
     http_protocol_ipv6          = "disabled"
     http_put_response_hop_limit = 1
-    http_tokens                 = "optional"
+    http_tokens                 = "required"
   }
   monitoring {
     enabled = false
@@ -463,7 +443,6 @@ resource "aws_launch_template" "apiserver-apiservers-minimal-example-com" {
     tags = {
       "KubernetesCluster"                                                                = "minimal.example.com"
       "Name"                                                                             = "apiserver.apiservers.minimal.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                 = "api-server"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/api-server" = ""
       "k8s.io/role/apiserver"                                                            = "1"
       "kops.k8s.io/instancegroup"                                                        = "apiserver"
@@ -475,7 +454,6 @@ resource "aws_launch_template" "apiserver-apiservers-minimal-example-com" {
     tags = {
       "KubernetesCluster"                                                                = "minimal.example.com"
       "Name"                                                                             = "apiserver.apiservers.minimal.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                 = "api-server"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/api-server" = ""
       "k8s.io/role/apiserver"                                                            = "1"
       "kops.k8s.io/instancegroup"                                                        = "apiserver"
@@ -485,7 +463,6 @@ resource "aws_launch_template" "apiserver-apiservers-minimal-example-com" {
   tags = {
     "KubernetesCluster"                                                                = "minimal.example.com"
     "Name"                                                                             = "apiserver.apiservers.minimal.example.com"
-    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                 = "api-server"
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/api-server" = ""
     "k8s.io/role/apiserver"                                                            = "1"
     "kops.k8s.io/instancegroup"                                                        = "apiserver"
@@ -523,7 +500,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" {
     http_endpoint               = "enabled"
     http_protocol_ipv6          = "disabled"
     http_put_response_hop_limit = 1
-    http_tokens                 = "optional"
+    http_tokens                 = "required"
   }
   monitoring {
     enabled = false
@@ -541,10 +518,8 @@ resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" {
       "KubernetesCluster"                                                                                     = "minimal.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.minimal.example.com"
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/api-server"                      = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
       "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
       "k8s.io/role/control-plane"                                                                             = "1"
       "k8s.io/role/master"                                                                                    = "1"
@@ -558,10 +533,8 @@ resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" {
       "KubernetesCluster"                                                                                     = "minimal.example.com"
       "Name"                                                                                                  = "master-us-test-1a.masters.minimal.example.com"
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/api-server"                      = ""
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
       "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
       "k8s.io/role/control-plane"                                                                             = "1"
       "k8s.io/role/master"                                                                                    = "1"
@@ -573,10 +546,8 @@ resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" {
     "KubernetesCluster"                                                                                     = "minimal.example.com"
     "Name"                                                                                                  = "master-us-test-1a.masters.minimal.example.com"
     "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
-    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/api-server"                      = ""
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
     "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
     "k8s.io/role/control-plane"                                                                             = "1"
     "k8s.io/role/master"                                                                                    = "1"
@@ -611,7 +582,7 @@ resource "aws_launch_template" "nodes-minimal-example-com" {
     http_endpoint               = "enabled"
     http_protocol_ipv6          = "disabled"
     http_put_response_hop_limit = 1
-    http_tokens                 = "optional"
+    http_tokens                 = "required"
   }
   monitoring {
     enabled = false
@@ -628,7 +599,6 @@ resource "aws_launch_template" "nodes-minimal-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "minimal.example.com"
       "Name"                                                                       = "nodes.minimal.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -640,7 +610,6 @@ resource "aws_launch_template" "nodes-minimal-example-com" {
     tags = {
       "KubernetesCluster"                                                          = "minimal.example.com"
       "Name"                                                                       = "nodes.minimal.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -650,7 +619,6 @@ resource "aws_launch_template" "nodes-minimal-example-com" {
   tags = {
     "KubernetesCluster"                                                          = "minimal.example.com"
     "Name"                                                                       = "nodes.minimal.example.com"
-    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/node"                                                           = "1"
     "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -742,6 +710,14 @@ resource "aws_s3_object" "manifests-static-kube-apiserver-healthcheck" {
   server_side_encryption = "AES256"
 }
 
+resource "aws_s3_object" "minimal-example-com-addons-aws-cloud-controller-addons-k8s-io-k8s-1-18" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content")
+  key                    = "clusters.example.com/minimal.example.com/addons/aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
 resource "aws_s3_object" "minimal-example-com-addons-aws-ebs-csi-driver-addons-k8s-io-k8s-1-17" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content")
@@ -981,6 +957,8 @@ resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-udp-1
 resource "aws_subnet" "us-test-1a-minimal-example-com" {
   availability_zone                           = "us-test-1a"
   cidr_block                                  = "172.20.32.0/19"
+  enable_resource_name_dns_a_record_on_launch = true
+  private_dns_hostname_type_on_launch         = "resource-name"
   tags = {
     "KubernetesCluster"                            = "minimal.example.com"
     "Name"                                         = "us-test-1a.minimal.example.com"

tests/integration/update_cluster/privatedns1/data/aws_iam_role_policy_masters.privatedns1.example.com_policy

@@ -94,39 +94,6 @@
         "*"
       ]
     },
-    {
-      "Action": "ec2:CreateTags",
-      "Condition": {
-        "StringEquals": {
-          "aws:RequestTag/KubernetesCluster": "privatedns1.example.com",
-          "ec2:CreateAction": [
-            "CreateSecurityGroup"
-          ]
-        }
-      },
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws-test:ec2:*:*:security-group/*"
-      ]
-    },
-    {
-      "Action": [
-        "ec2:CreateTags",
-        "ec2:DeleteTags"
-      ],
-      "Condition": {
-        "Null": {
-          "aws:RequestTag/KubernetesCluster": "true"
-        },
-        "StringEquals": {
-          "aws:ResourceTag/KubernetesCluster": "privatedns1.example.com"
-        }
-      },
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws-test:ec2:*:*:security-group/*"
-      ]
-    },
     {
       "Action": "ec2:CreateTags",
       "Condition": {
@@ -163,6 +130,39 @@
         "arn:aws-test:ec2:*:*:snapshot/*"
       ]
     },
+    {
+      "Action": "ec2:CreateTags",
+      "Condition": {
+        "StringEquals": {
+          "aws:RequestTag/KubernetesCluster": "privatedns1.example.com",
+          "ec2:CreateAction": [
+            "CreateSecurityGroup"
+          ]
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws-test:ec2:*:*:security-group/*"
+      ]
+    },
+    {
+      "Action": [
+        "ec2:CreateTags",
+        "ec2:DeleteTags"
+      ],
+      "Condition": {
+        "Null": {
+          "aws:RequestTag/KubernetesCluster": "true"
+        },
+        "StringEquals": {
+          "aws:ResourceTag/KubernetesCluster": "privatedns1.example.com"
+        }
+      },
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws-test:ec2:*:*:security-group/*"
+      ]
+    },
     {
       "Action": [
         "autoscaling:DescribeAutoScalingGroups",
@@ -170,13 +170,6 @@
         "autoscaling:DescribeLaunchConfigurations",
         "autoscaling:DescribeScalingActivities",
         "autoscaling:DescribeTags",
-        "ec2:AttachVolume",
-        "ec2:AuthorizeSecurityGroupIngress",
-        "ec2:CreateSecurityGroup",
-        "ec2:CreateTags",
-        "ec2:DeleteRoute",
-        "ec2:DeleteSecurityGroup",
-        "ec2:DeleteVolume",
         "ec2:DescribeAccountAttributes",
         "ec2:DescribeAvailabilityZones",
         "ec2:DescribeInstanceTypes",
@@ -190,21 +183,12 @@
         "ec2:DescribeVolumes",
         "ec2:DescribeVolumesModifications",
         "ec2:DescribeVpcs",
-        "ec2:DetachVolume",
-        "ec2:ModifyInstanceAttribute",
-        "ec2:ModifyVolume",
-        "elasticloadbalancing:AddTags",
-        "elasticloadbalancing:CreateListener",
-        "elasticloadbalancing:CreateTargetGroup",
-        "elasticloadbalancing:DeleteListener",
         "elasticloadbalancing:DescribeListeners",
         "elasticloadbalancing:DescribeLoadBalancerAttributes",
         "elasticloadbalancing:DescribeLoadBalancerPolicies",
         "elasticloadbalancing:DescribeLoadBalancers",
         "elasticloadbalancing:DescribeTargetGroups",
         "elasticloadbalancing:DescribeTargetHealth",
-        "elasticloadbalancing:ModifyListener",
-        "elasticloadbalancing:RegisterTargets",
         "iam:GetServerCertificate",
         "iam:ListServerCertificates",
         "kms:DescribeKey",

tests/integration/update_cluster/privatedns1/data/aws_launch_template_master-us-test-1a.masters.privatedns1.example.com_user_data

@@ -149,7 +149,7 @@ kubeAPIServer:
   apiServerCount: 1
   authorizationMode: AlwaysAllow
   bindAddress: 0.0.0.0
-  cloudProvider: aws
+  cloudProvider: external
   enableAdmissionPlugins:
   - NamespaceLifecycle
   - LimitRanger
@@ -165,9 +165,8 @@ kubeAPIServer:
   etcdServersOverrides:
   - /events#https://127.0.0.1:4002
   featureGates:
-    CSIMigrationAWS: "true"
     InTreePluginAWSUnregister: "true"
-  image: registry.k8s.io/kube-apiserver:v1.22.0
+  image: registry.k8s.io/kube-apiserver:v1.27.0
   kubeletPreferredAddressTypes:
   - InternalIP
   - Hostname
@@ -189,23 +188,21 @@ kubeAPIServer:
 kubeControllerManager:
   allocateNodeCIDRs: true
   attachDetachReconcileSyncPeriod: 1m0s
-  cloudProvider: aws
+  cloudProvider: external
   clusterCIDR: 100.96.0.0/11
   clusterName: privatedns1.example.com
   configureCloudRoutes: false
   featureGates:
-    CSIMigrationAWS: "true"
     InTreePluginAWSUnregister: "true"
-  image: registry.k8s.io/kube-controller-manager:v1.22.0
+  image: registry.k8s.io/kube-controller-manager:v1.27.0
   leaderElection:
     leaderElect: true
   logLevel: 2
   useServiceAccountCredentials: true
 kubeScheduler:
   featureGates:
-    CSIMigrationAWS: "true"
     InTreePluginAWSUnregister: "true"
-  image: registry.k8s.io/kube-scheduler:v1.22.0
+  image: registry.k8s.io/kube-scheduler:v1.27.0
   leaderElection:
     leaderElect: true
   logLevel: 2
@@ -213,17 +210,15 @@ kubelet:
   anonymousAuth: false
   cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
   featureGates:
-    CSIMigrationAWS: "true"
     InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   podInfraContainerImage: registry.k8s.io/pause:3.9
   podManifestPath: /etc/kubernetes/manifests
   protectKernelDefaults: true
@@ -234,17 +229,15 @@ masterKubelet:
   anonymousAuth: false
   cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
   featureGates:
-    CSIMigrationAWS: "true"
     InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   podInfraContainerImage: registry.k8s.io/pause:3.9
   podManifestPath: /etc/kubernetes/manifests
   protectKernelDefaults: true
@@ -260,7 +253,7 @@ ClusterName: privatedns1.example.com
 ConfigBase: memfs://clusters.example.com/privatedns1.example.com
 InstanceGroupName: master-us-test-1a
 InstanceGroupRole: ControlPlane
-NodeupConfigHash: o4y8EmBv12exLhOyO7r8fq3ZdF2AY3v3yy1QOGdT91A=
+NodeupConfigHash: 91rhU68bjHBvXgAdyTaKzfShk60Su3eS8U0ftbIISgg=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/privatedns1/data/aws_launch_template_nodes.privatedns1.example.com_user_data

@@ -129,17 +129,15 @@ kubelet:
   anonymousAuth: false
   cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
   featureGates:
-    CSIMigrationAWS: "true"
     InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   podInfraContainerImage: registry.k8s.io/pause:3.9
   podManifestPath: /etc/kubernetes/manifests
   protectKernelDefaults: true
@@ -178,7 +176,7 @@ ConfigServer:
   - https://kops-controller.internal.privatedns1.example.com:3988/
 InstanceGroupName: nodes
 InstanceGroupRole: Node
-NodeupConfigHash: OQVolkOKHllG4ToRe+nK5ATq+1QMsWml9ZItnjOTTNs=
+NodeupConfigHash: XQtsu7c/2iNacEPRguf+eh3CZIRALyy+Ir7Y1ajfkT4=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/privatedns1/data/aws_s3_object_cluster-completed.spec_content

@@ -16,6 +16,14 @@ spec:
       enabled: true
       version: v1.14.1
     manageStorageClasses: true
+  cloudControllerManager:
+    allocateNodeCIDRs: true
+    clusterCIDR: 100.64.0.0/10
+    clusterName: privatedns1.example.com
+    configureCloudRoutes: false
+    image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.26.0
+    leaderElection:
+      leaderElect: true
   cloudLabels:
     Owner: John Doe
     foo/bar: fib+baz
@@ -26,7 +34,9 @@ spec:
   containerRuntime: containerd
   containerd:
     logLevel: info
-    version: 1.4.13
+    runc:
+      version: 1.1.5
+    version: 1.6.20
   dnsZone: internal.example.com
   docker:
     skipInstall: true
@@ -68,7 +78,7 @@ spec:
     apiServerCount: 1
     authorizationMode: AlwaysAllow
     bindAddress: 0.0.0.0
-    cloudProvider: aws
+    cloudProvider: external
     enableAdmissionPlugins:
     - NamespaceLifecycle
     - LimitRanger
@@ -84,9 +94,8 @@ spec:
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
     featureGates:
-      CSIMigrationAWS: "true"
       InTreePluginAWSUnregister: "true"
-    image: registry.k8s.io/kube-apiserver:v1.22.0
+    image: registry.k8s.io/kube-apiserver:v1.27.0
     kubeletPreferredAddressTypes:
     - InternalIP
     - Hostname
@@ -108,14 +117,13 @@ spec:
   kubeControllerManager:
     allocateNodeCIDRs: true
     attachDetachReconcileSyncPeriod: 1m0s
-    cloudProvider: aws
+    cloudProvider: external
     clusterCIDR: 100.96.0.0/11
     clusterName: privatedns1.example.com
     configureCloudRoutes: false
     featureGates:
-      CSIMigrationAWS: "true"
       InTreePluginAWSUnregister: "true"
-    image: registry.k8s.io/kube-controller-manager:v1.22.0
+    image: registry.k8s.io/kube-controller-manager:v1.27.0
     leaderElection:
       leaderElect: true
     logLevel: 2
@@ -137,13 +145,12 @@ spec:
   kubeProxy:
     clusterCIDR: 100.96.0.0/11
     cpuRequest: 100m
-    image: registry.k8s.io/kube-proxy:v1.22.0
+    image: registry.k8s.io/kube-proxy:v1.27.0
     logLevel: 2
   kubeScheduler:
     featureGates:
-      CSIMigrationAWS: "true"
       InTreePluginAWSUnregister: "true"
-    image: registry.k8s.io/kube-scheduler:v1.22.0
+    image: registry.k8s.io/kube-scheduler:v1.27.0
     leaderElection:
       leaderElect: true
     logLevel: 2
@@ -151,17 +158,15 @@ spec:
     anonymousAuth: false
     cgroupDriver: systemd
     cgroupRoot: /
-    cloudProvider: aws
+    cloudProvider: external
     clusterDNS: 100.64.0.10
     clusterDomain: cluster.local
     enableDebuggingHandlers: true
     evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
     featureGates:
-      CSIMigrationAWS: "true"
       InTreePluginAWSUnregister: "true"
     kubeconfigPath: /var/lib/kubelet/kubeconfig
     logLevel: 2
-    networkPluginName: cni
     podInfraContainerImage: registry.k8s.io/pause:3.9
     podManifestPath: /etc/kubernetes/manifests
     protectKernelDefaults: true
@@ -170,22 +175,20 @@ spec:
     shutdownGracePeriodCriticalPods: 10s
   kubernetesApiAccess:
   - 0.0.0.0/0
-  kubernetesVersion: 1.22.0
+  kubernetesVersion: 1.27.0
   masterKubelet:
     anonymousAuth: false
     cgroupDriver: systemd
     cgroupRoot: /
-    cloudProvider: aws
+    cloudProvider: external
     clusterDNS: 100.64.0.10
     clusterDomain: cluster.local
     enableDebuggingHandlers: true
     evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
     featureGates:
-      CSIMigrationAWS: "true"
       InTreePluginAWSUnregister: "true"
     kubeconfigPath: /var/lib/kubelet/kubeconfig
     logLevel: 2
-    networkPluginName: cni
     podInfraContainerImage: registry.k8s.io/pause:3.9
     podManifestPath: /etc/kubernetes/manifests
     protectKernelDefaults: true
@@ -195,7 +198,7 @@ spec:
   masterPublicName: api.privatedns1.example.com
   networkCIDR: 172.20.0.0/16
   networking:
-    weave: {}
+    cni: {}
   nonMasqueradeCIDR: 100.64.0.0/10
   podCIDR: 100.96.0.0/11
   secretStore: memfs://clusters.example.com/privatedns1.example.com/secrets

tests/integration/update_cluster/privatedns1/data/aws_s3_object_nodeupconfig-master-us-test-1a_content

@@ -7,7 +7,7 @@ APIServerConfig:
     apiServerCount: 1
     authorizationMode: AlwaysAllow
     bindAddress: 0.0.0.0
-    cloudProvider: aws
+    cloudProvider: external
     enableAdmissionPlugins:
     - NamespaceLifecycle
     - LimitRanger
@@ -23,9 +23,8 @@ APIServerConfig:
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
     featureGates:
-      CSIMigrationAWS: "true"
       InTreePluginAWSUnregister: "true"
-    image: registry.k8s.io/kube-apiserver:v1.22.0
+    image: registry.k8s.io/kube-apiserver:v1.27.0
     kubeletPreferredAddressTypes:
     - InternalIP
     - Hostname
@@ -55,17 +54,21 @@ APIServerConfig:
     -----END RSA PUBLIC KEY-----
 Assets:
   amd64:
-  - fec5c596f7f815f17f5d7d955e9707df1ef02a2ca5e788b223651f83376feb7f@https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubelet
-  - 703e70d49b82271535bc66bc7bd469a58c11d47f188889bd37101c9772f14fa1@https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubectl
-  - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
-  - 29ef1e8635795c2a49a20a56e778f45ff163c5400a5428ca33999ed53d44e3d8@https://github.com/containerd/containerd/releases/download/v1.4.13/cri-containerd-cni-1.4.13-linux-amd64.tar.gz
+  - 0b4ed4fcd75d33f5dff3ba17776e6089847fc83064d3f7a3ad59a34e94e60a29@https://dl.k8s.io/release/v1.27.0/bin/linux/amd64/kubelet
+  - 71a78259d70da9c5540c4cf4cff121f443e863376f68f89a759d90cef3f51e87@https://dl.k8s.io/release/v1.27.0/bin/linux/amd64/kubectl
+  - 5035d7814c95cd3cedbc5efb447ef25a4942ef05caab2159746d55ce1698c74a@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/amd64/ecr-credential-provider-linux-amd64
+  - f3a841324845ca6bf0d4091b4fc7f97e18a623172158b72fc3fdcdb9d42d2d37@https://storage.googleapis.com/k8s-artifacts-cni/release/v1.2.0/cni-plugins-linux-amd64-v1.2.0.tgz
+  - bb9a9ccd6517e2a54da748a9f60dc9aa9d79d19d4724663f2386812f083968e2@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-amd64.tar.gz
+  - f00b144e86f8c1db347a2e8f22caade07d55382c5f76dd5c0a5b1ab64eaec8bb@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.amd64
   - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64
   - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64
   arm64:
-  - cea637a7da4f1097b16b0195005351c07032a820a3d64c3ff326b9097cfac930@https://dl.k8s.io/release/v1.22.0/bin/linux/arm64/kubelet
-  - 8d9cc92dcc942f5ea2b2fc93c4934875d9e0e8ddecbde24c7d4c4e092cfc7afc@https://dl.k8s.io/release/v1.22.0/bin/linux/arm64/kubectl
-  - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
-  - debed306ed9a4e70dcbcb228a0b3898f9730099e324f34bb0e76abbaddf7a6a7@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.13.tgz
+  - 37aa2edc7c0c4b3e488518c6a4b44c8aade75a55010534ee2be291220c73d157@https://dl.k8s.io/release/v1.27.0/bin/linux/arm64/kubelet
+  - f8e09630211f2b7c6a8cc38835e7dea94708d401f5c84b23a37c70c604602ddc@https://dl.k8s.io/release/v1.27.0/bin/linux/arm64/kubectl
+  - b3d567bda9e2996fc1fbd9d13506bd16763d3865b5c7b0b3c4b48c6088c04481@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/arm64/ecr-credential-provider-linux-arm64
+  - 525e2b62ba92a1b6f3dc9612449a84aa61652e680f7ebf4eff579795fe464b57@https://storage.googleapis.com/k8s-artifacts-cni/release/v1.2.0/cni-plugins-linux-arm64-v1.2.0.tgz
+  - c3e6a054b18b20fce06c7c3ed53f0989bb4b255c849bede446ebca955f07a9ce@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-arm64.tar.gz
+  - 54e79e4d48b9e191767e4abc08be1a8476a1c757e9a9f8c45c6ded001226867f@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.arm64
   - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64
   - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64
 CAs:
@@ -224,7 +227,7 @@ ClusterName: privatedns1.example.com
 ContainerRuntime: containerd
 FileAssets:
 - content: |
-    apiVersion: kubescheduler.config.k8s.io/v1beta2
+    apiVersion: kubescheduler.config.k8s.io/v1
     clientConnection:
       kubeconfig: /var/lib/kube-scheduler/kubeconfig
     kind: KubeSchedulerConfiguration
@@ -244,28 +247,24 @@ KeypairIDs:
 KubeProxy:
   clusterCIDR: 100.96.0.0/11
   cpuRequest: 100m
-  image: registry.k8s.io/kube-proxy:v1.22.0
+  image: registry.k8s.io/kube-proxy:v1.27.0
   logLevel: 2
 KubeletConfig:
   anonymousAuth: false
   cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
   featureGates:
-    CSIMigrationAWS: "true"
     InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   nodeLabels:
     kops.k8s.io/kops-controller-pki: ""
-    kubernetes.io/role: master
     node-role.kubernetes.io/control-plane: ""
-    node-role.kubernetes.io/master: ""
     node.kubernetes.io/exclude-from-external-load-balancers: ""
   podInfraContainerImage: registry.k8s.io/pause:3.9
   podManifestPath: /etc/kubernetes/manifests
@@ -274,8 +273,8 @@ KubeletConfig:
   shutdownGracePeriod: 30s
   shutdownGracePeriodCriticalPods: 10s
   taints:
-  - node-role.kubernetes.io/master=:NoSchedule
-KubernetesVersion: 1.22.0
+  - node-role.kubernetes.io/control-plane=:NoSchedule
+KubernetesVersion: 1.27.0
 Networking:
   nonMasqueradeCIDR: 100.64.0.0/10
   serviceClusterIPRange: 100.64.0.0/13
@@ -284,7 +283,9 @@ channels:
 - memfs://clusters.example.com/privatedns1.example.com/addons/bootstrap-channel.yaml
 containerdConfig:
   logLevel: info
-  version: 1.4.13
+  runc:
+    version: 1.1.5
+  version: 1.6.20
 docker:
   skipInstall: true
 etcdManifests:
@@ -293,5 +294,6 @@ etcdManifests:
 staticManifests:
 - key: kube-apiserver-healthcheck
   path: manifests/static/kube-apiserver-healthcheck.yaml
+useInstanceIDForNodeName: true
 usesLegacyGossip: false
 usesNoneDNS: false

tests/integration/update_cluster/privatedns1/data/aws_s3_object_nodeupconfig-nodes_content

@@ -1,14 +1,18 @@
 Assets:
   amd64:
-  - fec5c596f7f815f17f5d7d955e9707df1ef02a2ca5e788b223651f83376feb7f@https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubelet
-  - 703e70d49b82271535bc66bc7bd469a58c11d47f188889bd37101c9772f14fa1@https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubectl
-  - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
-  - 29ef1e8635795c2a49a20a56e778f45ff163c5400a5428ca33999ed53d44e3d8@https://github.com/containerd/containerd/releases/download/v1.4.13/cri-containerd-cni-1.4.13-linux-amd64.tar.gz
+  - 0b4ed4fcd75d33f5dff3ba17776e6089847fc83064d3f7a3ad59a34e94e60a29@https://dl.k8s.io/release/v1.27.0/bin/linux/amd64/kubelet
+  - 71a78259d70da9c5540c4cf4cff121f443e863376f68f89a759d90cef3f51e87@https://dl.k8s.io/release/v1.27.0/bin/linux/amd64/kubectl
+  - 5035d7814c95cd3cedbc5efb447ef25a4942ef05caab2159746d55ce1698c74a@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/amd64/ecr-credential-provider-linux-amd64
+  - f3a841324845ca6bf0d4091b4fc7f97e18a623172158b72fc3fdcdb9d42d2d37@https://storage.googleapis.com/k8s-artifacts-cni/release/v1.2.0/cni-plugins-linux-amd64-v1.2.0.tgz
+  - bb9a9ccd6517e2a54da748a9f60dc9aa9d79d19d4724663f2386812f083968e2@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-amd64.tar.gz
+  - f00b144e86f8c1db347a2e8f22caade07d55382c5f76dd5c0a5b1ab64eaec8bb@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.amd64
   arm64:
-  - cea637a7da4f1097b16b0195005351c07032a820a3d64c3ff326b9097cfac930@https://dl.k8s.io/release/v1.22.0/bin/linux/arm64/kubelet
-  - 8d9cc92dcc942f5ea2b2fc93c4934875d9e0e8ddecbde24c7d4c4e092cfc7afc@https://dl.k8s.io/release/v1.22.0/bin/linux/arm64/kubectl
-  - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
-  - debed306ed9a4e70dcbcb228a0b3898f9730099e324f34bb0e76abbaddf7a6a7@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.13.tgz
+  - 37aa2edc7c0c4b3e488518c6a4b44c8aade75a55010534ee2be291220c73d157@https://dl.k8s.io/release/v1.27.0/bin/linux/arm64/kubelet
+  - f8e09630211f2b7c6a8cc38835e7dea94708d401f5c84b23a37c70c604602ddc@https://dl.k8s.io/release/v1.27.0/bin/linux/arm64/kubectl
+  - b3d567bda9e2996fc1fbd9d13506bd16763d3865b5c7b0b3c4b48c6088c04481@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/arm64/ecr-credential-provider-linux-arm64
+  - 525e2b62ba92a1b6f3dc9612449a84aa61652e680f7ebf4eff579795fe464b57@https://storage.googleapis.com/k8s-artifacts-cni/release/v1.2.0/cni-plugins-linux-arm64-v1.2.0.tgz
+  - c3e6a054b18b20fce06c7c3ed53f0989bb4b255c849bede446ebca955f07a9ce@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-arm64.tar.gz
+  - 54e79e4d48b9e191767e4abc08be1a8476a1c757e9a9f8c45c6ded001226867f@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.arm64
 CAs: {}
 ClusterName: privatedns1.example.com
 ContainerRuntime: containerd
@@ -20,25 +24,22 @@ KeypairIDs:
 KubeProxy:
   clusterCIDR: 100.96.0.0/11
   cpuRequest: 100m
-  image: registry.k8s.io/kube-proxy:v1.22.0
+  image: registry.k8s.io/kube-proxy:v1.27.0
   logLevel: 2
 KubeletConfig:
   anonymousAuth: false
   cgroupDriver: systemd
   cgroupRoot: /
-  cloudProvider: aws
+  cloudProvider: external
   clusterDNS: 100.64.0.10
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
   featureGates:
-    CSIMigrationAWS: "true"
     InTreePluginAWSUnregister: "true"
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
-  networkPluginName: cni
   nodeLabels:
-    kubernetes.io/role: node
     node-role.kubernetes.io/node: ""
   podInfraContainerImage: registry.k8s.io/pause:3.9
   podManifestPath: /etc/kubernetes/manifests
@@ -46,7 +47,7 @@ KubeletConfig:
   registerSchedulable: true
   shutdownGracePeriod: 30s
   shutdownGracePeriodCriticalPods: 10s
-KubernetesVersion: 1.22.0
+KubernetesVersion: 1.27.0
 Networking:
   nonMasqueradeCIDR: 100.64.0.0/10
   serviceClusterIPRange: 100.64.0.0/13
@@ -55,8 +56,11 @@ channels:
 - memfs://clusters.example.com/privatedns1.example.com/addons/bootstrap-channel.yaml
 containerdConfig:
   logLevel: info
-  version: 1.4.13
+  runc:
+    version: 1.1.5
+  version: 1.6.20
 docker:
   skipInstall: true
+useInstanceIDForNodeName: true
 usesLegacyGossip: false
 usesNoneDNS: false

tests/integration/update_cluster/privatedns1/data/aws_s3_object_privatedns1.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content

@@ -0,0 +1,237 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
+    k8s-app: aws-cloud-controller-manager
+  name: aws-cloud-controller-manager
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: aws-cloud-controller-manager
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        k8s-app: aws-cloud-controller-manager
+        kops.k8s.io/managed-by: kops
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
+      containers:
+      - args:
+        - --allocate-node-cidrs=true
+        - --cluster-cidr=100.64.0.0/10
+        - --cluster-name=privatedns1.example.com
+        - --configure-cloud-routes=false
+        - --leader-elect=true
+        - --v=2
+        - --cloud-provider=aws
+        - --use-service-account-credentials=true
+        - --cloud-config=/etc/kubernetes/cloud.config
+        env:
+        - name: KUBERNETES_SERVICE_HOST
+          value: 127.0.0.1
+        image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.26.0
+        imagePullPolicy: IfNotPresent
+        name: aws-cloud-controller-manager
+        resources:
+          requests:
+            cpu: 200m
+        volumeMounts:
+        - mountPath: /etc/kubernetes/cloud.config
+          name: cloudconfig
+          readOnly: true
+      hostNetwork: true
+      nodeSelector: null
+      priorityClassName: system-cluster-critical
+      serviceAccountName: aws-cloud-controller-manager
+      tolerations:
+      - effect: NoSchedule
+        key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+      - effect: NoSchedule
+        key: node.kubernetes.io/not-ready
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+      volumes:
+      - hostPath:
+          path: /etc/kubernetes/cloud.config
+          type: ""
+        name: cloudconfig
+  updateStrategy:
+    type: RollingUpdate
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
+  name: aws-cloud-controller-manager
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
+  name: cloud-controller-manager:apiserver-authentication-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- apiGroup: ""
+  kind: ServiceAccount
+  name: aws-cloud-controller-manager
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
+  name: system:cloud-controller-manager
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - services/status
+  verbs:
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resourceNames:
+  - node-controller
+  - service-controller
+  - route-controller
+  resources:
+  - serviceaccounts/token
+  verbs:
+  - create
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
+  name: system:cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:cloud-controller-manager
+subjects:
+- apiGroup: ""
+  kind: ServiceAccount
+  name: aws-cloud-controller-manager
+  namespace: kube-system

tests/integration/update_cluster/privatedns1/data/aws_s3_object_privatedns1.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content

@@ -798,7 +798,7 @@ spec:
 
 ---
 
-apiVersion: policy/v1beta1
+apiVersion: policy/v1
 kind: PodDisruptionBudget
 metadata:
   creationTimestamp: null

tests/integration/update_cluster/privatedns1/data/aws_s3_object_privatedns1.example.com-addons-bootstrap_content

@@ -6,7 +6,7 @@ spec:
   addons:
   - id: k8s-1.16
     manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: 1777b09b9dbad5721d6273523d787efdc5ca50fdd9a4a1232d5c03ef3cb1b3c8
+    manifestHash: cf18fee164c0b94212c91ebf9c1691ddcf05afe7af61aed498680aa92ba2376b
     name: kops-controller.addons.k8s.io
     needsRollingUpdate: control-plane
     selector:
@@ -14,7 +14,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 49273720e8c046ae8b72a3fe666d670f6efb5e26111dc521027218b1e8e1de51
+    manifestHash: d2bbb7cbee5835c3891fe80fbacf8963508359ef9159f8480325ce9a7174f14a
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io
@@ -46,16 +46,16 @@ spec:
     selector:
       k8s-addon: storage-aws.addons.k8s.io
     version: 9.99.0
-  - id: k8s-1.12
-    manifest: networking.weave/k8s-1.12.yaml
-    manifestHash: c58a7acc6ed931d26b59892beb1f43e240fd51cbde223e3d95e15b3e04ced54d
-    name: networking.weave
+  - id: k8s-1.18
+    manifest: aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml
+    manifestHash: cff48979a983d2e54a115ca5176cfaf16b629b8c67cc429d0fff3951141f4193
+    name: aws-cloud-controller.addons.k8s.io
     selector:
-      role.kubernetes.io/networking: "1"
+      k8s-addon: aws-cloud-controller.addons.k8s.io
     version: 9.99.0
   - id: k8s-1.17
     manifest: aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml
-    manifestHash: 1b9a5e1ba9092ab22b11c9c8754747774873a7f0457509107707766c6ff44cc9
+    manifestHash: f0791dbf64945b9eb57ad23463b5e2716a83bb2b952f15ea5b766e3bd36f339f
     name: aws-ebs-csi-driver.addons.k8s.io
     selector:
       k8s-addon: aws-ebs-csi-driver.addons.k8s.io

tests/integration/update_cluster/privatedns1/data/aws_s3_object_privatedns1.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -242,7 +242,7 @@ spec:
 
 ---
 
-apiVersion: policy/v1beta1
+apiVersion: policy/v1
 kind: PodDisruptionBudget
 metadata:
   creationTimestamp: null

tests/integration/update_cluster/privatedns1/data/aws_s3_object_privatedns1.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content

@@ -1,7 +1,7 @@
 apiVersion: v1
 data:
   config.yaml: |
-    {"clusterName":"privatedns1.example.com","cloud":"aws","configBase":"memfs://clusters.example.com/privatedns1.example.com","secretStore":"memfs://clusters.example.com/privatedns1.example.com/secrets","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["nodes.privatedns1.example.com"],"Region":"us-test-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"]}}
+    {"clusterName":"privatedns1.example.com","cloud":"aws","configBase":"memfs://clusters.example.com/privatedns1.example.com","secretStore":"memfs://clusters.example.com/privatedns1.example.com/secrets","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["nodes.privatedns1.example.com"],"Region":"us-test-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"],"useInstanceIDForNodeName":true}}
 kind: ConfigMap
 metadata:
   creationTimestamp: null

tests/integration/update_cluster/privatedns1/data/aws_s3_object_privatedns1.example.com-addons-networking.weave-k8s-1.12_content

@@ -1,285 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.weave
-    app.kubernetes.io/managed-by: kops
-    name: weave-net
-    role.kubernetes.io/networking: "1"
-  name: weave-net
-  namespace: kube-system
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.weave
-    app.kubernetes.io/managed-by: kops
-    name: weave-net
-    role.kubernetes.io/networking: "1"
-  name: weave-net
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  - namespaces
-  - nodes
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - extensions
-  resources:
-  - networkpolicies
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - networking.k8s.io
-  resources:
-  - networkpolicies
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - nodes/status
-  verbs:
-  - patch
-  - update
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.weave
-    app.kubernetes.io/managed-by: kops
-    name: weave-net
-    role.kubernetes.io/networking: "1"
-  name: weave-net
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: weave-net
-subjects:
-- kind: ServiceAccount
-  name: weave-net
-  namespace: kube-system
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.weave
-    app.kubernetes.io/managed-by: kops
-    name: weave-net
-    role.kubernetes.io/networking: "1"
-  name: weave-net
-  namespace: kube-system
-rules:
-- apiGroups:
-  - ""
-  resourceNames:
-  - weave-net
-  resources:
-  - configmaps
-  verbs:
-  - get
-  - update
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - create
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.weave
-    app.kubernetes.io/managed-by: kops
-    name: weave-net
-    role.kubernetes.io/networking: "1"
-  name: weave-net
-  namespace: kube-system
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: weave-net
-subjects:
-- kind: ServiceAccount
-  name: weave-net
-  namespace: kube-system
-
----
-
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.weave
-    app.kubernetes.io/managed-by: kops
-    name: weave-net
-    role.kubernetes.io/networking: "1"
-  name: weave-net
-  namespace: kube-system
-spec:
-  minReadySeconds: 5
-  selector:
-    matchLabels:
-      name: weave-net
-  template:
-    metadata:
-      annotations:
-        prometheus.io/scrape: "true"
-      creationTimestamp: null
-      labels:
-        kops.k8s.io/managed-by: kops
-        name: weave-net
-    spec:
-      containers:
-      - command:
-        - /home/weave/launch.sh
-        env:
-        - name: INIT_CONTAINER
-          value: "true"
-        - name: HOSTNAME
-          valueFrom:
-            fieldRef:
-              apiVersion: v1
-              fieldPath: spec.nodeName
-        - name: IPALLOC_RANGE
-          value: 100.96.0.0/11
-        image: weaveworks/weave-kube:2.8.1
-        name: weave
-        ports:
-        - containerPort: 6782
-          name: metrics
-        readinessProbe:
-          httpGet:
-            host: 127.0.0.1
-            path: /status
-            port: 6784
-        resources:
-          limits:
-            memory: 200Mi
-          requests:
-            cpu: 50m
-            memory: 200Mi
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - mountPath: /weavedb
-          name: weavedb
-        - mountPath: /host/var/lib/dbus
-          name: dbus
-          readOnly: true
-        - mountPath: /host/etc/machine-id
-          name: cni-machine-id
-          readOnly: true
-        - mountPath: /run/xtables.lock
-          name: xtables-lock
-          readOnly: false
-      - env:
-        - name: HOSTNAME
-          valueFrom:
-            fieldRef:
-              apiVersion: v1
-              fieldPath: spec.nodeName
-        image: weaveworks/weave-npc:2.8.1
-        name: weave-npc
-        ports:
-        - containerPort: 6781
-          name: metrics
-        resources:
-          limits:
-            memory: 200Mi
-          requests:
-            cpu: 50m
-            memory: 200Mi
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - mountPath: /run/xtables.lock
-          name: xtables-lock
-          readOnly: false
-      dnsPolicy: ClusterFirstWithHostNet
-      hostNetwork: true
-      hostPID: false
-      initContainers:
-      - command:
-        - /home/weave/init.sh
-        image: weaveworks/weave-kube:2.8.1
-        name: weave-init
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - mountPath: /host/opt
-          name: cni-bin
-        - mountPath: /host/home
-          name: cni-bin2
-        - mountPath: /host/etc
-          name: cni-conf
-        - mountPath: /lib/modules
-          name: lib-modules
-        - mountPath: /run/xtables.lock
-          name: xtables-lock
-          readOnly: false
-      priorityClassName: system-node-critical
-      restartPolicy: Always
-      securityContext:
-        seLinuxOptions: {}
-      serviceAccountName: weave-net
-      tolerations:
-      - effect: NoSchedule
-        operator: Exists
-      - effect: NoExecute
-        operator: Exists
-      volumes:
-      - hostPath:
-          path: /var/lib/weave
-        name: weavedb
-      - hostPath:
-          path: /opt
-        name: cni-bin
-      - hostPath:
-          path: /home
-        name: cni-bin2
-      - hostPath:
-          path: /etc
-        name: cni-conf
-      - hostPath:
-          path: /etc/machine-id
-        name: cni-machine-id
-      - hostPath:
-          path: /var/lib/dbus
-        name: dbus
-      - hostPath:
-          path: /lib/modules
-        name: lib-modules
-      - hostPath:
-          path: /run/xtables.lock
-          type: FileOrCreate
-        name: xtables-lock
-  updateStrategy:
-    type: RollingUpdate

tests/integration/update_cluster/privatedns1/in-v1alpha2.yaml

@@ -25,11 +25,11 @@ spec:
   iam: {}
   kubelet:
     anonymousAuth: false
-  kubernetesVersion: v1.22.0
+  kubernetesVersion: v1.27.0
   masterPublicName: api.privatedns1.example.com
   networkCIDR: 172.20.0.0/16
   networking:
-    weave: {}
+    cni: {}
   nonMasqueradeCIDR: 100.64.0.0/10
   sshAccess:
   - 0.0.0.0/0

tests/integration/update_cluster/privatedns1/kubernetes.tf

@@ -214,21 +214,11 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-privatedns1-example-
     propagate_at_launch = true
     value               = ""
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"
-    propagate_at_launch = true
-    value               = "master"
-  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"
     propagate_at_launch = true
     value               = ""
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"
-    propagate_at_launch = true
-    value               = ""
-  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers"
     propagate_at_launch = true
@@ -289,11 +279,6 @@ resource "aws_autoscaling_group" "nodes-privatedns1-example-com" {
     propagate_at_launch = true
     value               = "fib+baz"
   }
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"
-    propagate_at_launch = true
-    value               = "node"
-  }
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node"
     propagate_at_launch = true
@@ -534,7 +519,7 @@ resource "aws_launch_template" "bastion-privatedns1-example-com" {
     http_endpoint               = "enabled"
     http_protocol_ipv6          = "disabled"
     http_put_response_hop_limit = 1
-    http_tokens                 = "optional"
+    http_tokens                 = "required"
   }
   monitoring {
     enabled = false
@@ -610,7 +595,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-privatedns1-example-co
     http_endpoint               = "enabled"
     http_protocol_ipv6          = "disabled"
     http_put_response_hop_limit = 1
-    http_tokens                 = "optional"
+    http_tokens                 = "required"
   }
   monitoring {
     enabled = false
@@ -630,9 +615,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-privatedns1-example-co
       "Owner"                                                                                                 = "John Doe"
       "foo/bar"                                                                                               = "fib+baz"
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
       "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
       "k8s.io/role/control-plane"                                                                             = "1"
       "k8s.io/role/master"                                                                                    = "1"
@@ -648,9 +631,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-privatedns1-example-co
       "Owner"                                                                                                 = "John Doe"
       "foo/bar"                                                                                               = "fib+baz"
       "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
       "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
       "k8s.io/role/control-plane"                                                                             = "1"
       "k8s.io/role/master"                                                                                    = "1"
@@ -664,9 +645,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-privatedns1-example-co
     "Owner"                                                                                                 = "John Doe"
     "foo/bar"                                                                                               = "fib+baz"
     "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
-    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
     "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
     "k8s.io/role/control-plane"                                                                             = "1"
     "k8s.io/role/master"                                                                                    = "1"
@@ -701,7 +680,7 @@ resource "aws_launch_template" "nodes-privatedns1-example-com" {
     http_endpoint               = "enabled"
     http_protocol_ipv6          = "disabled"
     http_put_response_hop_limit = 1
-    http_tokens                 = "optional"
+    http_tokens                 = "required"
   }
   monitoring {
     enabled = false
@@ -720,7 +699,6 @@ resource "aws_launch_template" "nodes-privatedns1-example-com" {
       "Name"                                                                       = "nodes.privatedns1.example.com"
       "Owner"                                                                      = "John Doe"
       "foo/bar"                                                                    = "fib+baz"
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -734,7 +712,6 @@ resource "aws_launch_template" "nodes-privatedns1-example-com" {
       "Name"                                                                       = "nodes.privatedns1.example.com"
       "Owner"                                                                      = "John Doe"
       "foo/bar"                                                                    = "fib+baz"
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
       "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
       "k8s.io/role/node"                                                           = "1"
       "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -746,7 +723,6 @@ resource "aws_launch_template" "nodes-privatedns1-example-com" {
     "Name"                                                                       = "nodes.privatedns1.example.com"
     "Owner"                                                                      = "John Doe"
     "foo/bar"                                                                    = "fib+baz"
-    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"           = "node"
     "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = ""
     "k8s.io/role/node"                                                           = "1"
     "kops.k8s.io/instancegroup"                                                  = "nodes"
@@ -967,6 +943,14 @@ resource "aws_s3_object" "nodeupconfig-nodes" {
   server_side_encryption = "AES256"
 }
 
+resource "aws_s3_object" "privatedns1-example-com-addons-aws-cloud-controller-addons-k8s-io-k8s-1-18" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_object_privatedns1.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content")
+  key                    = "clusters.example.com/privatedns1.example.com/addons/aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
 resource "aws_s3_object" "privatedns1-example-com-addons-aws-ebs-csi-driver-addons-k8s-io-k8s-1-17" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_object_privatedns1.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content")
@@ -1023,14 +1007,6 @@ resource "aws_s3_object" "privatedns1-example-com-addons-limit-range-addons-k8s-
   server_side_encryption = "AES256"
 }
 
-resource "aws_s3_object" "privatedns1-example-com-addons-networking-weave-k8s-1-12" {
-  bucket                 = "testingBucket"
-  content                = file("${path.module}/data/aws_s3_object_privatedns1.example.com-addons-networking.weave-k8s-1.12_content")
-  key                    = "clusters.example.com/privatedns1.example.com/addons/networking.weave/k8s-1.12.yaml"
-  provider               = aws.files
-  server_side_encryption = "AES256"
-}
-
 resource "aws_s3_object" "privatedns1-example-com-addons-storage-aws-addons-k8s-io-v1-15-0" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_object_privatedns1.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content")
@@ -1310,6 +1286,8 @@ resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-172-20-4-0--22" {
 resource "aws_subnet" "us-test-1a-privatedns1-example-com" {
   availability_zone                           = "us-test-1a"
   cidr_block                                  = "172.20.32.0/19"
+  enable_resource_name_dns_a_record_on_launch = true
+  private_dns_hostname_type_on_launch         = "resource-name"
   tags = {
     "KubernetesCluster"                             = "privatedns1.example.com"
     "Name"                                          = "us-test-1a.privatedns1.example.com"
@@ -1327,6 +1305,8 @@ resource "aws_subnet" "us-test-1a-privatedns1-example-com" {
 resource "aws_subnet" "utility-us-test-1a-privatedns1-example-com" {
   availability_zone                           = "us-test-1a"
   cidr_block                                  = "172.20.4.0/22"
+  enable_resource_name_dns_a_record_on_launch = true
+  private_dns_hostname_type_on_launch         = "resource-name"
   tags = {
     "KubernetesCluster"                             = "privatedns1.example.com"
     "Name"                                          = "utility-us-test-1a.privatedns1.example.com"

tests/integration/update_cluster/privateweave/data/aws_iam_role_bastions.privateweave.example.com_policy

@@ -1,10 +0,0 @@
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Principal": { "Service": "ec2.amazonaws.com"},
-      "Action": "sts:AssumeRole"
-    }
-  ]
-}

tests/integration/update_cluster/privateweave/data/aws_iam_role_masters.privateweave.example.com_policy

@@ -1,10 +0,0 @@
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Principal": { "Service": "ec2.amazonaws.com"},
-      "Action": "sts:AssumeRole"
-    }
-  ]
-}

tests/integration/update_cluster/privateweave/data/aws_iam_role_nodes.privateweave.example.com_policy

@@ -1,10 +0,0 @@
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Principal": { "Service": "ec2.amazonaws.com"},
-      "Action": "sts:AssumeRole"
-    }
-  ]
-}

tests/integration/update_cluster/privateweave/data/aws_iam_role_policy_bastions.privateweave.example.com_policy

@@ -1,10 +0,0 @@
-{
-  "Statement": [
-    {
-      "Action": "ec2:DescribeRegions",
-      "Effect": "Allow",
-      "Resource": "*"
-    }
-  ],
-  "Version": "2012-10-17"
-}

tests/integration/update_cluster/privateweave/data/aws_iam_role_policy_masters.privateweave.example.com_policy

@@ -1,281 +0,0 @@
-{
-  "Statement": [
-    {
-      "Action": "ec2:AttachVolume",
-      "Condition": {
-        "StringEquals": {
-          "aws:ResourceTag/KubernetesCluster": "privateweave.example.com",
-          "aws:ResourceTag/k8s.io/role/master": "1"
-        }
-      },
-      "Effect": "Allow",
-      "Resource": [
-        "*"
-      ]
-    },
-    {
-      "Action": [
-        "s3:Get*"
-      ],
-      "Effect": "Allow",
-      "Resource": "arn:aws-test:s3:::placeholder-read-bucket/clusters.example.com/privateweave.example.com/*"
-    },
-    {
-      "Action": [
-        "s3:GetObject",
-        "s3:DeleteObject",
-        "s3:DeleteObjectVersion",
-        "s3:PutObject"
-      ],
-      "Effect": "Allow",
-      "Resource": "arn:aws-test:s3:::placeholder-write-bucket/clusters.example.com/privateweave.example.com/backups/etcd/main/*"
-    },
-    {
-      "Action": [
-        "s3:GetObject",
-        "s3:DeleteObject",
-        "s3:DeleteObjectVersion",
-        "s3:PutObject"
-      ],
-      "Effect": "Allow",
-      "Resource": "arn:aws-test:s3:::placeholder-write-bucket/clusters.example.com/privateweave.example.com/backups/etcd/events/*"
-    },
-    {
-      "Action": [
-        "s3:GetBucketLocation",
-        "s3:GetEncryptionConfiguration",
-        "s3:ListBucket",
-        "s3:ListBucketVersions"
-      ],
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws-test:s3:::placeholder-read-bucket"
-      ]
-    },
-    {
-      "Action": [
-        "s3:GetBucketLocation",
-        "s3:GetEncryptionConfiguration",
-        "s3:ListBucket",
-        "s3:ListBucketVersions"
-      ],
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws-test:s3:::placeholder-write-bucket"
-      ]
-    },
-    {
-      "Action": [
-        "route53:ChangeResourceRecordSets",
-        "route53:ListResourceRecordSets",
-        "route53:GetHostedZone"
-      ],
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws-test:route53:::hostedzone/Z1AFAKE1ZON3YO"
-      ]
-    },
-    {
-      "Action": [
-        "route53:GetChange"
-      ],
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws-test:route53:::change/*"
-      ]
-    },
-    {
-      "Action": [
-        "route53:ListHostedZones",
-        "route53:ListTagsForResource"
-      ],
-      "Effect": "Allow",
-      "Resource": [
-        "*"
-      ]
-    },
-    {
-      "Action": "ec2:CreateTags",
-      "Condition": {
-        "StringEquals": {
-          "aws:RequestTag/KubernetesCluster": "privateweave.example.com",
-          "ec2:CreateAction": [
-            "CreateSecurityGroup"
-          ]
-        }
-      },
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws-test:ec2:*:*:security-group/*"
-      ]
-    },
-    {
-      "Action": [
-        "ec2:CreateTags",
-        "ec2:DeleteTags"
-      ],
-      "Condition": {
-        "Null": {
-          "aws:RequestTag/KubernetesCluster": "true"
-        },
-        "StringEquals": {
-          "aws:ResourceTag/KubernetesCluster": "privateweave.example.com"
-        }
-      },
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws-test:ec2:*:*:security-group/*"
-      ]
-    },
-    {
-      "Action": "ec2:CreateTags",
-      "Condition": {
-        "StringEquals": {
-          "aws:RequestTag/KubernetesCluster": "privateweave.example.com",
-          "ec2:CreateAction": [
-            "CreateVolume",
-            "CreateSnapshot"
-          ]
-        }
-      },
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws-test:ec2:*:*:volume/*",
-        "arn:aws-test:ec2:*:*:snapshot/*"
-      ]
-    },
-    {
-      "Action": [
-        "ec2:CreateTags",
-        "ec2:DeleteTags"
-      ],
-      "Condition": {
-        "Null": {
-          "aws:RequestTag/KubernetesCluster": "true"
-        },
-        "StringEquals": {
-          "aws:ResourceTag/KubernetesCluster": "privateweave.example.com"
-        }
-      },
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws-test:ec2:*:*:volume/*",
-        "arn:aws-test:ec2:*:*:snapshot/*"
-      ]
-    },
-    {
-      "Action": [
-        "autoscaling:DescribeAutoScalingGroups",
-        "autoscaling:DescribeAutoScalingInstances",
-        "autoscaling:DescribeLaunchConfigurations",
-        "autoscaling:DescribeScalingActivities",
-        "autoscaling:DescribeTags",
-        "ec2:AttachVolume",
-        "ec2:AuthorizeSecurityGroupIngress",
-        "ec2:CreateSecurityGroup",
-        "ec2:CreateTags",
-        "ec2:DeleteRoute",
-        "ec2:DeleteSecurityGroup",
-        "ec2:DeleteVolume",
-        "ec2:DescribeAccountAttributes",
-        "ec2:DescribeAvailabilityZones",
-        "ec2:DescribeInstanceTypes",
-        "ec2:DescribeInstances",
-        "ec2:DescribeLaunchTemplateVersions",
-        "ec2:DescribeRegions",
-        "ec2:DescribeRouteTables",
-        "ec2:DescribeSecurityGroups",
-        "ec2:DescribeSubnets",
-        "ec2:DescribeTags",
-        "ec2:DescribeVolumes",
-        "ec2:DescribeVolumesModifications",
-        "ec2:DescribeVpcs",
-        "ec2:DetachVolume",
-        "ec2:ModifyInstanceAttribute",
-        "ec2:ModifyVolume",
-        "elasticloadbalancing:AddTags",
-        "elasticloadbalancing:CreateListener",
-        "elasticloadbalancing:CreateTargetGroup",
-        "elasticloadbalancing:DeleteListener",
-        "elasticloadbalancing:DescribeListeners",
-        "elasticloadbalancing:DescribeLoadBalancerAttributes",
-        "elasticloadbalancing:DescribeLoadBalancerPolicies",
-        "elasticloadbalancing:DescribeLoadBalancers",
-        "elasticloadbalancing:DescribeTargetGroups",
-        "elasticloadbalancing:DescribeTargetHealth",
-        "elasticloadbalancing:ModifyListener",
-        "elasticloadbalancing:RegisterTargets",
-        "iam:GetServerCertificate",
-        "iam:ListServerCertificates",
-        "kms:DescribeKey",
-        "kms:GenerateRandom"
-      ],
-      "Effect": "Allow",
-      "Resource": "*"
-    },
-    {
-      "Action": [
-        "autoscaling:SetDesiredCapacity",
-        "autoscaling:TerminateInstanceInAutoScalingGroup",
-        "ec2:AttachVolume",
-        "ec2:AuthorizeSecurityGroupIngress",
-        "ec2:DeleteSecurityGroup",
-        "ec2:DeleteVolume",
-        "ec2:DetachVolume",
-        "ec2:ModifyInstanceAttribute",
-        "ec2:ModifyVolume",
-        "ec2:RevokeSecurityGroupIngress",
-        "elasticloadbalancing:AddTags",
-        "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer",
-        "elasticloadbalancing:AttachLoadBalancerToSubnets",
-        "elasticloadbalancing:ConfigureHealthCheck",
-        "elasticloadbalancing:CreateLoadBalancerListeners",
-        "elasticloadbalancing:CreateLoadBalancerPolicy",
-        "elasticloadbalancing:DeleteListener",
-        "elasticloadbalancing:DeleteLoadBalancer",
-        "elasticloadbalancing:DeleteLoadBalancerListeners",
-        "elasticloadbalancing:DeleteTargetGroup",
-        "elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
-        "elasticloadbalancing:DeregisterTargets",
-        "elasticloadbalancing:DetachLoadBalancerFromSubnets",
-        "elasticloadbalancing:ModifyListener",
-        "elasticloadbalancing:ModifyLoadBalancerAttributes",
-        "elasticloadbalancing:ModifyTargetGroup",
-        "elasticloadbalancing:RegisterInstancesWithLoadBalancer",
-        "elasticloadbalancing:RegisterTargets",
-        "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer",
-        "elasticloadbalancing:SetLoadBalancerPoliciesOfListener"
-      ],
-      "Condition": {
-        "StringEquals": {
-          "aws:ResourceTag/KubernetesCluster": "privateweave.example.com"
-        }
-      },
-      "Effect": "Allow",
-      "Resource": "*"
-    },
-    {
-      "Action": [
-        "ec2:CreateSecurityGroup",
-        "ec2:CreateSnapshot",
-        "ec2:CreateVolume",
-        "elasticloadbalancing:CreateListener",
-        "elasticloadbalancing:CreateLoadBalancer",
-        "elasticloadbalancing:CreateTargetGroup"
-      ],
-      "Condition": {
-        "StringEquals": {
-          "aws:RequestTag/KubernetesCluster": "privateweave.example.com"
-        }
-      },
-      "Effect": "Allow",
-      "Resource": "*"
-    },
-    {
-      "Action": "ec2:CreateSecurityGroup",
-      "Effect": "Allow",
-      "Resource": "arn:aws-test:ec2:*:*:vpc/*"
-    }
-  ],
-  "Version": "2012-10-17"
-}

tests/integration/update_cluster/privateweave/data/aws_iam_role_policy_nodes.privateweave.example.com_policy

@@ -1,30 +0,0 @@
-{
-  "Statement": [
-    {
-      "Action": [
-        "s3:GetBucketLocation",
-        "s3:GetEncryptionConfiguration",
-        "s3:ListBucket",
-        "s3:ListBucketVersions"
-      ],
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws-test:s3:::placeholder-read-bucket"
-      ]
-    },
-    {
-      "Action": [
-        "autoscaling:DescribeAutoScalingInstances",
-        "ec2:DescribeInstanceTypes",
-        "ec2:DescribeInstances",
-        "ec2:DescribeRegions",
-        "iam:GetServerCertificate",
-        "iam:ListServerCertificates",
-        "kms:GenerateRandom"
-      ],
-      "Effect": "Allow",
-      "Resource": "*"
-    }
-  ],
-  "Version": "2012-10-17"
-}

tests/integration/update_cluster/privateweave/data/aws_key_pair_kubernetes.privateweave.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key

@@ -1 +0,0 @@
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==

tests/integration/update_cluster/privateweave/data/aws_launch_template_master-us-test-1a.masters.privateweave.example.com_user_data

@@ -1,268 +0,0 @@
-#!/bin/bash
-set -o errexit
-set -o nounset
-set -o pipefail
-
-NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64
-NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924
-NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64
-NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865
-
-export AWS_REGION=us-test-1
-
-
-
-
-sysctl -w net.core.rmem_max=16777216 || true
-sysctl -w net.core.wmem_max=16777216 || true
-sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true
-sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true
-
-
-function ensure-install-dir() {
-  INSTALL_DIR="/opt/kops"
-  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
-  if [[ -d /var/lib/toolbox ]]; then
-    INSTALL_DIR="/var/lib/toolbox/kops"
-  fi
-  mkdir -p ${INSTALL_DIR}/bin
-  mkdir -p ${INSTALL_DIR}/conf
-  cd ${INSTALL_DIR}
-}
-
-# Retry a download until we get it. args: name, sha, urls
-download-or-bust() {
-  local -r file="$1"
-  local -r hash="$2"
-  local -r urls=( $(split-commas "$3") )
-
-  if [[ -f "${file}" ]]; then
-    if ! validate-hash "${file}" "${hash}"; then
-      rm -f "${file}"
-    else
-      return 0
-    fi
-  fi
-
-  while true; do
-    for url in "${urls[@]}"; do
-      commands=(
-        "curl -f --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
-        "wget --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
-        "curl -f -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
-        "wget -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
-      )
-      for cmd in "${commands[@]}"; do
-        echo "Attempting download with: ${cmd} {url}"
-        if ! (${cmd} "${url}"); then
-          echo "== Download failed with ${cmd} =="
-          continue
-        fi
-        if ! validate-hash "${file}" "${hash}"; then
-          echo "== Hash validation of ${url} failed. Retrying. =="
-          rm -f "${file}"
-        else
-          echo "== Downloaded ${url} (SHA256 = ${hash}) =="
-          return 0
-        fi
-      done
-    done
-
-    echo "All downloads failed; sleeping before retrying"
-    sleep 60
-  done
-}
-
-validate-hash() {
-  local -r file="$1"
-  local -r expected="$2"
-  local actual
-
-  actual=$(sha256sum ${file} | awk '{ print $1 }') || true
-  if [[ "${actual}" != "${expected}" ]]; then
-    echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} =="
-    return 1
-  fi
-}
-
-function split-commas() {
-  echo $1 | tr "," "\n"
-}
-
-function download-release() {
-  case "$(uname -m)" in
-  x86_64*|i?86_64*|amd64*)
-    NODEUP_URL="${NODEUP_URL_AMD64}"
-    NODEUP_HASH="${NODEUP_HASH_AMD64}"
-    ;;
-  aarch64*|arm64*)
-    NODEUP_URL="${NODEUP_URL_ARM64}"
-    NODEUP_HASH="${NODEUP_HASH_ARM64}"
-    ;;
-  *)
-    echo "Unsupported host arch: $(uname -m)" >&2
-    exit 1
-    ;;
-  esac
-
-  cd ${INSTALL_DIR}/bin
-  download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}"
-
-  chmod +x nodeup
-
-  echo "Running nodeup"
-  # We can't run in the foreground because of https://github.com/docker/docker/issues/23793
-  ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8  )
-}
-
-####################################################################################
-
-/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
-
-echo "== nodeup node config starting =="
-ensure-install-dir
-
-cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
-cloudConfig:
-  manageStorageClasses: true
-encryptionConfig: null
-etcdClusters:
-  events:
-    manager:
-      backupRetentionDays: 90
-      env:
-      - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION
-        value: 90d
-    version: 3.5.9
-  main:
-    manager:
-      backupRetentionDays: 90
-      env:
-      - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION
-        value: 90d
-    version: 3.5.9
-kubeAPIServer:
-  allowPrivileged: true
-  anonymousAuth: false
-  apiAudiences:
-  - kubernetes.svc.default
-  apiServerCount: 1
-  authorizationMode: AlwaysAllow
-  bindAddress: 0.0.0.0
-  cloudProvider: aws
-  enableAdmissionPlugins:
-  - NamespaceLifecycle
-  - LimitRanger
-  - ServiceAccount
-  - DefaultStorageClass
-  - DefaultTolerationSeconds
-  - MutatingAdmissionWebhook
-  - ValidatingAdmissionWebhook
-  - NodeRestriction
-  - ResourceQuota
-  etcdServers:
-  - https://127.0.0.1:4001
-  etcdServersOverrides:
-  - /events#https://127.0.0.1:4002
-  featureGates:
-    CSIMigrationAWS: "true"
-    InTreePluginAWSUnregister: "true"
-  image: registry.k8s.io/kube-apiserver:v1.22.0
-  kubeletPreferredAddressTypes:
-  - InternalIP
-  - Hostname
-  - ExternalIP
-  logLevel: 2
-  requestheaderAllowedNames:
-  - aggregator
-  requestheaderExtraHeaderPrefixes:
-  - X-Remote-Extra-
-  requestheaderGroupHeaders:
-  - X-Remote-Group
-  requestheaderUsernameHeaders:
-  - X-Remote-User
-  securePort: 443
-  serviceAccountIssuer: https://api.internal.privateweave.example.com
-  serviceAccountJWKSURI: https://api.internal.privateweave.example.com/openid/v1/jwks
-  serviceClusterIPRange: 100.64.0.0/13
-  storageBackend: etcd3
-kubeControllerManager:
-  allocateNodeCIDRs: true
-  attachDetachReconcileSyncPeriod: 1m0s
-  cloudProvider: aws
-  clusterCIDR: 100.96.0.0/11
-  clusterName: privateweave.example.com
-  configureCloudRoutes: false
-  featureGates:
-    CSIMigrationAWS: "true"
-    InTreePluginAWSUnregister: "true"
-  image: registry.k8s.io/kube-controller-manager:v1.22.0
-  leaderElection:
-    leaderElect: true
-  logLevel: 2
-  useServiceAccountCredentials: true
-kubeScheduler:
-  featureGates:
-    CSIMigrationAWS: "true"
-    InTreePluginAWSUnregister: "true"
-  image: registry.k8s.io/kube-scheduler:v1.22.0
-  leaderElection:
-    leaderElect: true
-  logLevel: 2
-kubelet:
-  anonymousAuth: false
-  cgroupDriver: systemd
-  cgroupRoot: /
-  cloudProvider: aws
-  clusterDNS: 100.64.0.10
-  clusterDomain: cluster.local
-  enableDebuggingHandlers: true
-  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
-  featureGates:
-    CSIMigrationAWS: "true"
-    InTreePluginAWSUnregister: "true"
-  kubeconfigPath: /var/lib/kubelet/kubeconfig
-  logLevel: 2
-  networkPluginName: cni
-  podInfraContainerImage: registry.k8s.io/pause:3.9
-  podManifestPath: /etc/kubernetes/manifests
-  protectKernelDefaults: true
-  registerSchedulable: true
-  shutdownGracePeriod: 30s
-  shutdownGracePeriodCriticalPods: 10s
-masterKubelet:
-  anonymousAuth: false
-  cgroupDriver: systemd
-  cgroupRoot: /
-  cloudProvider: aws
-  clusterDNS: 100.64.0.10
-  clusterDomain: cluster.local
-  enableDebuggingHandlers: true
-  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
-  featureGates:
-    CSIMigrationAWS: "true"
-    InTreePluginAWSUnregister: "true"
-  kubeconfigPath: /var/lib/kubelet/kubeconfig
-  logLevel: 2
-  networkPluginName: cni
-  podInfraContainerImage: registry.k8s.io/pause:3.9
-  podManifestPath: /etc/kubernetes/manifests
-  protectKernelDefaults: true
-  registerSchedulable: true
-  shutdownGracePeriod: 30s
-  shutdownGracePeriodCriticalPods: 10s
-
-__EOF_CLUSTER_SPEC
-
-cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
-CloudProvider: aws
-ClusterName: privateweave.example.com
-ConfigBase: memfs://clusters.example.com/privateweave.example.com
-InstanceGroupName: master-us-test-1a
-InstanceGroupRole: ControlPlane
-NodeupConfigHash: rNA78SiuIXSsxqg8VR3v7nVCAvxFd4tXGvmJT5ZFR7s=
-
-__EOF_KUBE_ENV
-
-download-release
-echo "== nodeup node config done =="

tests/integration/update_cluster/privateweave/data/aws_launch_template_nodes.privateweave.example.com_user_data

@@ -1,186 +0,0 @@
-#!/bin/bash
-set -o errexit
-set -o nounset
-set -o pipefail
-
-NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64
-NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924
-NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64
-NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865
-
-export AWS_REGION=us-test-1
-
-
-
-
-sysctl -w net.core.rmem_max=16777216 || true
-sysctl -w net.core.wmem_max=16777216 || true
-sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true
-sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true
-
-
-function ensure-install-dir() {
-  INSTALL_DIR="/opt/kops"
-  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
-  if [[ -d /var/lib/toolbox ]]; then
-    INSTALL_DIR="/var/lib/toolbox/kops"
-  fi
-  mkdir -p ${INSTALL_DIR}/bin
-  mkdir -p ${INSTALL_DIR}/conf
-  cd ${INSTALL_DIR}
-}
-
-# Retry a download until we get it. args: name, sha, urls
-download-or-bust() {
-  local -r file="$1"
-  local -r hash="$2"
-  local -r urls=( $(split-commas "$3") )
-
-  if [[ -f "${file}" ]]; then
-    if ! validate-hash "${file}" "${hash}"; then
-      rm -f "${file}"
-    else
-      return 0
-    fi
-  fi
-
-  while true; do
-    for url in "${urls[@]}"; do
-      commands=(
-        "curl -f --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
-        "wget --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
-        "curl -f -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
-        "wget -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
-      )
-      for cmd in "${commands[@]}"; do
-        echo "Attempting download with: ${cmd} {url}"
-        if ! (${cmd} "${url}"); then
-          echo "== Download failed with ${cmd} =="
-          continue
-        fi
-        if ! validate-hash "${file}" "${hash}"; then
-          echo "== Hash validation of ${url} failed. Retrying. =="
-          rm -f "${file}"
-        else
-          echo "== Downloaded ${url} (SHA256 = ${hash}) =="
-          return 0
-        fi
-      done
-    done
-
-    echo "All downloads failed; sleeping before retrying"
-    sleep 60
-  done
-}
-
-validate-hash() {
-  local -r file="$1"
-  local -r expected="$2"
-  local actual
-
-  actual=$(sha256sum ${file} | awk '{ print $1 }') || true
-  if [[ "${actual}" != "${expected}" ]]; then
-    echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} =="
-    return 1
-  fi
-}
-
-function split-commas() {
-  echo $1 | tr "," "\n"
-}
-
-function download-release() {
-  case "$(uname -m)" in
-  x86_64*|i?86_64*|amd64*)
-    NODEUP_URL="${NODEUP_URL_AMD64}"
-    NODEUP_HASH="${NODEUP_HASH_AMD64}"
-    ;;
-  aarch64*|arm64*)
-    NODEUP_URL="${NODEUP_URL_ARM64}"
-    NODEUP_HASH="${NODEUP_HASH_ARM64}"
-    ;;
-  *)
-    echo "Unsupported host arch: $(uname -m)" >&2
-    exit 1
-    ;;
-  esac
-
-  cd ${INSTALL_DIR}/bin
-  download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}"
-
-  chmod +x nodeup
-
-  echo "Running nodeup"
-  # We can't run in the foreground because of https://github.com/docker/docker/issues/23793
-  ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8  )
-}
-
-####################################################################################
-
-/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
-
-echo "== nodeup node config starting =="
-ensure-install-dir
-
-cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
-cloudConfig:
-  manageStorageClasses: true
-kubelet:
-  anonymousAuth: false
-  cgroupDriver: systemd
-  cgroupRoot: /
-  cloudProvider: aws
-  clusterDNS: 100.64.0.10
-  clusterDomain: cluster.local
-  enableDebuggingHandlers: true
-  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
-  featureGates:
-    CSIMigrationAWS: "true"
-    InTreePluginAWSUnregister: "true"
-  kubeconfigPath: /var/lib/kubelet/kubeconfig
-  logLevel: 2
-  networkPluginName: cni
-  podInfraContainerImage: registry.k8s.io/pause:3.9
-  podManifestPath: /etc/kubernetes/manifests
-  protectKernelDefaults: true
-  registerSchedulable: true
-  shutdownGracePeriod: 30s
-  shutdownGracePeriodCriticalPods: 10s
-
-__EOF_CLUSTER_SPEC
-
-cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
-CloudProvider: aws
-ClusterName: privateweave.example.com
-ConfigServer:
-  CACertificates: |
-    -----BEGIN CERTIFICATE-----
-    MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU
-    BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw
-    ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
-    SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1
-    jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
-    MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA
-    MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8
-    tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w==
-    -----END CERTIFICATE-----
-    -----BEGIN CERTIFICATE-----
-    MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU
-    BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw
-    OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
-    SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7
-    WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
-    MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn
-    MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA
-    9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw==
-    -----END CERTIFICATE-----
-  servers:
-  - https://kops-controller.internal.privateweave.example.com:3988/
-InstanceGroupName: nodes
-InstanceGroupRole: Node
-NodeupConfigHash: JWsz1eFTXS3WjLjgzoAxb/ne6UMcpoXyi7sCh/C0SzA=
-
-__EOF_KUBE_ENV
-
-download-release
-echo "== nodeup node config done =="

tests/integration/update_cluster/privateweave/data/aws_s3_object_cluster-completed.spec_content

@@ -1,215 +0,0 @@
-apiVersion: kops.k8s.io/v1alpha2
-kind: Cluster
-metadata:
-  creationTimestamp: "2016-12-12T04:13:14Z"
-  name: privateweave.example.com
-spec:
-  api:
-    loadBalancer:
-      class: Classic
-      type: Public
-  authorization:
-    alwaysAllow: {}
-  channel: stable
-  cloudConfig:
-    awsEBSCSIDriver:
-      enabled: true
-      version: v1.14.1
-    manageStorageClasses: true
-  cloudProvider: aws
-  clusterDNSDomain: cluster.local
-  configBase: memfs://clusters.example.com/privateweave.example.com
-  configStore: memfs://clusters.example.com/privateweave.example.com
-  containerRuntime: containerd
-  containerd:
-    logLevel: info
-    version: 1.4.13
-  dnsZone: Z1AFAKE1ZON3YO
-  docker:
-    skipInstall: true
-  etcdClusters:
-  - backups:
-      backupStore: memfs://clusters.example.com/privateweave.example.com/backups/etcd/main
-    etcdMembers:
-    - instanceGroup: master-us-test-1a
-      name: us-test-1a
-    manager:
-      backupRetentionDays: 90
-      env:
-      - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION
-        value: 90d
-    name: main
-    version: 3.5.9
-  - backups:
-      backupStore: memfs://clusters.example.com/privateweave.example.com/backups/etcd/events
-    etcdMembers:
-    - instanceGroup: master-us-test-1a
-      name: us-test-1a
-    manager:
-      backupRetentionDays: 90
-      env:
-      - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION
-        value: 90d
-    name: events
-    version: 3.5.9
-  externalDns:
-    provider: dns-controller
-  iam:
-    legacy: false
-  keyStore: memfs://clusters.example.com/privateweave.example.com/pki
-  kubeAPIServer:
-    allowPrivileged: true
-    anonymousAuth: false
-    apiAudiences:
-    - kubernetes.svc.default
-    apiServerCount: 1
-    authorizationMode: AlwaysAllow
-    bindAddress: 0.0.0.0
-    cloudProvider: aws
-    enableAdmissionPlugins:
-    - NamespaceLifecycle
-    - LimitRanger
-    - ServiceAccount
-    - DefaultStorageClass
-    - DefaultTolerationSeconds
-    - MutatingAdmissionWebhook
-    - ValidatingAdmissionWebhook
-    - NodeRestriction
-    - ResourceQuota
-    etcdServers:
-    - https://127.0.0.1:4001
-    etcdServersOverrides:
-    - /events#https://127.0.0.1:4002
-    featureGates:
-      CSIMigrationAWS: "true"
-      InTreePluginAWSUnregister: "true"
-    image: registry.k8s.io/kube-apiserver:v1.22.0
-    kubeletPreferredAddressTypes:
-    - InternalIP
-    - Hostname
-    - ExternalIP
-    logLevel: 2
-    requestheaderAllowedNames:
-    - aggregator
-    requestheaderExtraHeaderPrefixes:
-    - X-Remote-Extra-
-    requestheaderGroupHeaders:
-    - X-Remote-Group
-    requestheaderUsernameHeaders:
-    - X-Remote-User
-    securePort: 443
-    serviceAccountIssuer: https://api.internal.privateweave.example.com
-    serviceAccountJWKSURI: https://api.internal.privateweave.example.com/openid/v1/jwks
-    serviceClusterIPRange: 100.64.0.0/13
-    storageBackend: etcd3
-  kubeControllerManager:
-    allocateNodeCIDRs: true
-    attachDetachReconcileSyncPeriod: 1m0s
-    cloudProvider: aws
-    clusterCIDR: 100.96.0.0/11
-    clusterName: privateweave.example.com
-    configureCloudRoutes: false
-    featureGates:
-      CSIMigrationAWS: "true"
-      InTreePluginAWSUnregister: "true"
-    image: registry.k8s.io/kube-controller-manager:v1.22.0
-    leaderElection:
-      leaderElect: true
-    logLevel: 2
-    useServiceAccountCredentials: true
-  kubeDNS:
-    cacheMaxConcurrent: 150
-    cacheMaxSize: 1000
-    cpuRequest: 100m
-    domain: cluster.local
-    memoryLimit: 170Mi
-    memoryRequest: 70Mi
-    nodeLocalDNS:
-      cpuRequest: 25m
-      enabled: false
-      image: registry.k8s.io/dns/k8s-dns-node-cache:1.22.20
-      memoryRequest: 5Mi
-    provider: CoreDNS
-    serverIP: 100.64.0.10
-  kubeProxy:
-    clusterCIDR: 100.96.0.0/11
-    cpuRequest: 100m
-    image: registry.k8s.io/kube-proxy:v1.22.0
-    logLevel: 2
-  kubeScheduler:
-    featureGates:
-      CSIMigrationAWS: "true"
-      InTreePluginAWSUnregister: "true"
-    image: registry.k8s.io/kube-scheduler:v1.22.0
-    leaderElection:
-      leaderElect: true
-    logLevel: 2
-  kubelet:
-    anonymousAuth: false
-    cgroupDriver: systemd
-    cgroupRoot: /
-    cloudProvider: aws
-    clusterDNS: 100.64.0.10
-    clusterDomain: cluster.local
-    enableDebuggingHandlers: true
-    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
-    featureGates:
-      CSIMigrationAWS: "true"
-      InTreePluginAWSUnregister: "true"
-    kubeconfigPath: /var/lib/kubelet/kubeconfig
-    logLevel: 2
-    networkPluginName: cni
-    podInfraContainerImage: registry.k8s.io/pause:3.9
-    podManifestPath: /etc/kubernetes/manifests
-    protectKernelDefaults: true
-    registerSchedulable: true
-    shutdownGracePeriod: 30s
-    shutdownGracePeriodCriticalPods: 10s
-  kubernetesApiAccess:
-  - 0.0.0.0/0
-  kubernetesVersion: 1.22.0
-  masterKubelet:
-    anonymousAuth: false
-    cgroupDriver: systemd
-    cgroupRoot: /
-    cloudProvider: aws
-    clusterDNS: 100.64.0.10
-    clusterDomain: cluster.local
-    enableDebuggingHandlers: true
-    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
-    featureGates:
-      CSIMigrationAWS: "true"
-      InTreePluginAWSUnregister: "true"
-    kubeconfigPath: /var/lib/kubelet/kubeconfig
-    logLevel: 2
-    networkPluginName: cni
-    podInfraContainerImage: registry.k8s.io/pause:3.9
-    podManifestPath: /etc/kubernetes/manifests
-    protectKernelDefaults: true
-    registerSchedulable: true
-    shutdownGracePeriod: 30s
-    shutdownGracePeriodCriticalPods: 10s
-  masterPublicName: api.privateweave.example.com
-  networkCIDR: 172.20.0.0/16
-  networking:
-    weave: {}
-  nonMasqueradeCIDR: 100.64.0.0/10
-  podCIDR: 100.96.0.0/11
-  secretStore: memfs://clusters.example.com/privateweave.example.com/secrets
-  serviceClusterIPRange: 100.64.0.0/13
-  sshAccess:
-  - 0.0.0.0/0
-  subnets:
-  - cidr: 172.20.32.0/19
-    name: us-test-1a
-    type: Private
-    zone: us-test-1a
-  - cidr: 172.20.4.0/22
-    name: utility-us-test-1a
-    type: Utility
-    zone: us-test-1a
-  topology:
-    dns:
-      type: Public
-    masters: private
-    nodes: private

tests/integration/update_cluster/privateweave/data/aws_s3_object_etcd-cluster-spec-events_content

@@ -1,4 +0,0 @@
-{
-  "memberCount": 1,
-  "etcdVersion": "3.5.9"
-}

tests/integration/update_cluster/privateweave/data/aws_s3_object_etcd-cluster-spec-main_content

@@ -1,4 +0,0 @@
-{
-  "memberCount": 1,
-  "etcdVersion": "3.5.9"
-}

tests/integration/update_cluster/privateweave/data/aws_s3_object_kops-version.txt_content

@@ -1 +0,0 @@
-1.21.0-alpha.1

tests/integration/update_cluster/privateweave/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content

@@ -1,141 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  labels:
-    k8s-app: etcd-manager-events
-  name: etcd-manager-events
-  namespace: kube-system
-spec:
-  containers:
-  - command:
-    - /bin/sh
-    - -c
-    - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /etcd-manager
-      --backup-store=memfs://clusters.example.com/privateweave.example.com/backups/etcd/events
-      --client-urls=https://__name__:4002 --cluster-name=etcd-events --containerized=true
-      --dns-suffix=.internal.privateweave.example.com --grpc-port=3997 --peer-urls=https://__name__:2381
-      --quarantine-client-urls=https://__name__:3995 --v=6 --volume-name-tag=k8s.io/etcd/events
-      --volume-provider=aws --volume-tag=k8s.io/etcd/events --volume-tag=k8s.io/role/control-plane=1
-      --volume-tag=kubernetes.io/cluster/privateweave.example.com=owned > /tmp/pipe
-      2>&1
-    env:
-    - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION
-      value: 90d
-    image: registry.k8s.io/etcdadm/etcd-manager-slim:v3.0.20230516
-    name: etcd-manager
-    resources:
-      requests:
-        cpu: 200m
-        memory: 100Mi
-    securityContext:
-      privileged: true
-    volumeMounts:
-    - mountPath: /rootfs
-      name: rootfs
-    - mountPath: /run
-      name: run
-    - mountPath: /etc/kubernetes/pki/etcd-manager
-      name: pki
-    - mountPath: /opt
-      name: opt
-    - mountPath: /var/log/etcd.log
-      name: varlogetcd
-  hostNetwork: true
-  hostPID: true
-  initContainers:
-  - args:
-    - /ko-app/kops-utils-cp
-    - /opt/bin
-    command:
-    - /ko-app/kops-utils-cp
-    image: registry.k8s.io/kops/kops-utils-cp:1.27.0-beta.1
-    name: kops-utils-cp
-    resources: {}
-    volumeMounts:
-    - mountPath: /opt
-      name: opt
-  - args:
-    - /usr/local/bin/etcd
-    - /usr/local/bin/etcdctl
-    - /opt/etcd-v3.2.24
-    command:
-    - /opt/bin/kops-utils-cp
-    image: registry.k8s.io/etcd:3.2.24-1
-    name: init-etcd-3-2-24
-    resources: {}
-    volumeMounts:
-    - mountPath: /opt
-      name: opt
-  - args:
-    - /usr/local/bin/etcd
-    - /usr/local/bin/etcdctl
-    - /opt/etcd-v3.3.17
-    command:
-    - /opt/bin/kops-utils-cp
-    image: registry.k8s.io/etcd:3.3.17-0
-    name: init-etcd-3-3-17
-    resources: {}
-    volumeMounts:
-    - mountPath: /opt
-      name: opt
-  - args:
-    - /usr/local/bin/etcd
-    - /usr/local/bin/etcdctl
-    - /opt/etcd-v3.4.13
-    command:
-    - /opt/bin/kops-utils-cp
-    image: registry.k8s.io/etcd:3.4.13-0
-    name: init-etcd-3-4-13
-    resources: {}
-    volumeMounts:
-    - mountPath: /opt
-      name: opt
-  - args:
-    - /usr/local/bin/etcd
-    - /usr/local/bin/etcdctl
-    - /opt/etcd-v3.5.7
-    command:
-    - /opt/bin/kops-utils-cp
-    image: registry.k8s.io/etcd:3.5.7-0
-    name: init-etcd-3-5-7
-    resources: {}
-    volumeMounts:
-    - mountPath: /opt
-      name: opt
-  - args:
-    - /usr/local/bin/etcd
-    - /usr/local/bin/etcdctl
-    - /opt/etcd-v3.5.9
-    command:
-    - /opt/bin/kops-utils-cp
-    image: registry.k8s.io/etcd:3.5.9-0
-    name: init-etcd-3-5-9
-    resources: {}
-    volumeMounts:
-    - mountPath: /opt
-      name: opt
-  priorityClassName: system-cluster-critical
-  tolerations:
-  - key: CriticalAddonsOnly
-    operator: Exists
-  volumes:
-  - hostPath:
-      path: /
-      type: Directory
-    name: rootfs
-  - hostPath:
-      path: /run
-      type: DirectoryOrCreate
-    name: run
-  - hostPath:
-      path: /etc/kubernetes/pki/etcd-manager-events
-      type: DirectoryOrCreate
-    name: pki
-  - emptyDir: {}
-    name: opt
-  - hostPath:
-      path: /var/log/etcd-events.log
-      type: FileOrCreate
-    name: varlogetcd
-status: {}

tests/integration/update_cluster/privateweave/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content

@@ -1,141 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  labels:
-    k8s-app: etcd-manager-main
-  name: etcd-manager-main
-  namespace: kube-system
-spec:
-  containers:
-  - command:
-    - /bin/sh
-    - -c
-    - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /etcd-manager
-      --backup-store=memfs://clusters.example.com/privateweave.example.com/backups/etcd/main
-      --client-urls=https://__name__:4001 --cluster-name=etcd --containerized=true
-      --dns-suffix=.internal.privateweave.example.com --grpc-port=3996 --peer-urls=https://__name__:2380
-      --quarantine-client-urls=https://__name__:3994 --v=6 --volume-name-tag=k8s.io/etcd/main
-      --volume-provider=aws --volume-tag=k8s.io/etcd/main --volume-tag=k8s.io/role/control-plane=1
-      --volume-tag=kubernetes.io/cluster/privateweave.example.com=owned > /tmp/pipe
-      2>&1
-    env:
-    - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION
-      value: 90d
-    image: registry.k8s.io/etcdadm/etcd-manager-slim:v3.0.20230516
-    name: etcd-manager
-    resources:
-      requests:
-        cpu: 200m
-        memory: 100Mi
-    securityContext:
-      privileged: true
-    volumeMounts:
-    - mountPath: /rootfs
-      name: rootfs
-    - mountPath: /run
-      name: run
-    - mountPath: /etc/kubernetes/pki/etcd-manager
-      name: pki
-    - mountPath: /opt
-      name: opt
-    - mountPath: /var/log/etcd.log
-      name: varlogetcd
-  hostNetwork: true
-  hostPID: true
-  initContainers:
-  - args:
-    - /ko-app/kops-utils-cp
-    - /opt/bin
-    command:
-    - /ko-app/kops-utils-cp
-    image: registry.k8s.io/kops/kops-utils-cp:1.27.0-beta.1
-    name: kops-utils-cp
-    resources: {}
-    volumeMounts:
-    - mountPath: /opt
-      name: opt
-  - args:
-    - /usr/local/bin/etcd
-    - /usr/local/bin/etcdctl
-    - /opt/etcd-v3.2.24
-    command:
-    - /opt/bin/kops-utils-cp
-    image: registry.k8s.io/etcd:3.2.24-1
-    name: init-etcd-3-2-24
-    resources: {}
-    volumeMounts:
-    - mountPath: /opt
-      name: opt
-  - args:
-    - /usr/local/bin/etcd
-    - /usr/local/bin/etcdctl
-    - /opt/etcd-v3.3.17
-    command:
-    - /opt/bin/kops-utils-cp
-    image: registry.k8s.io/etcd:3.3.17-0
-    name: init-etcd-3-3-17
-    resources: {}
-    volumeMounts:
-    - mountPath: /opt
-      name: opt
-  - args:
-    - /usr/local/bin/etcd
-    - /usr/local/bin/etcdctl
-    - /opt/etcd-v3.4.13
-    command:
-    - /opt/bin/kops-utils-cp
-    image: registry.k8s.io/etcd:3.4.13-0
-    name: init-etcd-3-4-13
-    resources: {}
-    volumeMounts:
-    - mountPath: /opt
-      name: opt
-  - args:
-    - /usr/local/bin/etcd
-    - /usr/local/bin/etcdctl
-    - /opt/etcd-v3.5.7
-    command:
-    - /opt/bin/kops-utils-cp
-    image: registry.k8s.io/etcd:3.5.7-0
-    name: init-etcd-3-5-7
-    resources: {}
-    volumeMounts:
-    - mountPath: /opt
-      name: opt
-  - args:
-    - /usr/local/bin/etcd
-    - /usr/local/bin/etcdctl
-    - /opt/etcd-v3.5.9
-    command:
-    - /opt/bin/kops-utils-cp
-    image: registry.k8s.io/etcd:3.5.9-0
-    name: init-etcd-3-5-9
-    resources: {}
-    volumeMounts:
-    - mountPath: /opt
-      name: opt
-  priorityClassName: system-cluster-critical
-  tolerations:
-  - key: CriticalAddonsOnly
-    operator: Exists
-  volumes:
-  - hostPath:
-      path: /
-      type: Directory
-    name: rootfs
-  - hostPath:
-      path: /run
-      type: DirectoryOrCreate
-    name: run
-  - hostPath:
-      path: /etc/kubernetes/pki/etcd-manager-main
-      type: DirectoryOrCreate
-    name: pki
-  - emptyDir: {}
-    name: opt
-  - hostPath:
-      path: /var/log/etcd.log
-      type: FileOrCreate
-    name: varlogetcd
-status: {}

tests/integration/update_cluster/privateweave/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content

@@ -1,33 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-spec:
-  containers:
-  - args:
-    - --ca-cert=/secrets/ca.crt
-    - --client-cert=/secrets/client.crt
-    - --client-key=/secrets/client.key
-    image: registry.k8s.io/kops/kube-apiserver-healthcheck:1.27.0-beta.1
-    livenessProbe:
-      httpGet:
-        host: 127.0.0.1
-        path: /.kube-apiserver-healthcheck/healthz
-        port: 3990
-      initialDelaySeconds: 5
-      timeoutSeconds: 5
-    name: healthcheck
-    resources: {}
-    securityContext:
-      runAsNonRoot: true
-      runAsUser: 10012
-    volumeMounts:
-    - mountPath: /secrets
-      name: healthcheck-secrets
-      readOnly: true
-  volumes:
-  - hostPath:
-      path: /etc/kubernetes/kube-apiserver-healthcheck/secrets
-      type: Directory
-    name: healthcheck-secrets
-status: {}

tests/integration/update_cluster/privateweave/data/aws_s3_object_nodeupconfig-master-us-test-1a_content

@@ -1,297 +0,0 @@
-APIServerConfig:
-  KubeAPIServer:
-    allowPrivileged: true
-    anonymousAuth: false
-    apiAudiences:
-    - kubernetes.svc.default
-    apiServerCount: 1
-    authorizationMode: AlwaysAllow
-    bindAddress: 0.0.0.0
-    cloudProvider: aws
-    enableAdmissionPlugins:
-    - NamespaceLifecycle
-    - LimitRanger
-    - ServiceAccount
-    - DefaultStorageClass
-    - DefaultTolerationSeconds
-    - MutatingAdmissionWebhook
-    - ValidatingAdmissionWebhook
-    - NodeRestriction
-    - ResourceQuota
-    etcdServers:
-    - https://127.0.0.1:4001
-    etcdServersOverrides:
-    - /events#https://127.0.0.1:4002
-    featureGates:
-      CSIMigrationAWS: "true"
-      InTreePluginAWSUnregister: "true"
-    image: registry.k8s.io/kube-apiserver:v1.22.0
-    kubeletPreferredAddressTypes:
-    - InternalIP
-    - Hostname
-    - ExternalIP
-    logLevel: 2
-    requestheaderAllowedNames:
-    - aggregator
-    requestheaderExtraHeaderPrefixes:
-    - X-Remote-Extra-
-    requestheaderGroupHeaders:
-    - X-Remote-Group
-    requestheaderUsernameHeaders:
-    - X-Remote-User
-    securePort: 443
-    serviceAccountIssuer: https://api.internal.privateweave.example.com
-    serviceAccountJWKSURI: https://api.internal.privateweave.example.com/openid/v1/jwks
-    serviceClusterIPRange: 100.64.0.0/13
-    storageBackend: etcd3
-  ServiceAccountPublicKeys: |
-    -----BEGIN RSA PUBLIC KEY-----
-    MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm
-    XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ==
-    -----END RSA PUBLIC KEY-----
-    -----BEGIN RSA PUBLIC KEY-----
-    MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF
-    Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ==
-    -----END RSA PUBLIC KEY-----
-Assets:
-  amd64:
-  - fec5c596f7f815f17f5d7d955e9707df1ef02a2ca5e788b223651f83376feb7f@https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubelet
-  - 703e70d49b82271535bc66bc7bd469a58c11d47f188889bd37101c9772f14fa1@https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubectl
-  - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
-  - 29ef1e8635795c2a49a20a56e778f45ff163c5400a5428ca33999ed53d44e3d8@https://github.com/containerd/containerd/releases/download/v1.4.13/cri-containerd-cni-1.4.13-linux-amd64.tar.gz
-  - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64
-  - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64
-  arm64:
-  - cea637a7da4f1097b16b0195005351c07032a820a3d64c3ff326b9097cfac930@https://dl.k8s.io/release/v1.22.0/bin/linux/arm64/kubelet
-  - 8d9cc92dcc942f5ea2b2fc93c4934875d9e0e8ddecbde24c7d4c4e092cfc7afc@https://dl.k8s.io/release/v1.22.0/bin/linux/arm64/kubectl
-  - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
-  - debed306ed9a4e70dcbcb228a0b3898f9730099e324f34bb0e76abbaddf7a6a7@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.13.tgz
-  - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64
-  - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64
-CAs:
-  apiserver-aggregator-ca: |
-    -----BEGIN CERTIFICATE-----
-    MIIBgjCCASygAwIBAgIMFo3gINaZLHjisEcbMA0GCSqGSIb3DQEBCwUAMCIxIDAe
-    BgNVBAMTF2FwaXNlcnZlci1hZ2dyZWdhdG9yLWNhMB4XDTIxMDYzMDA0NTExMloX
-    DTMxMDYzMDA0NTExMlowIjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3It
-    Y2EwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAyyE71AOU3go5XFegLQ6fidI0LhhM
-    x7CzpTzh2xWKcHUfbNI7itgJvC/+GlyG5W+DF5V7ba0IJiQLsFve0oLdewIDAQAB
-    o0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU
-    ALfqF5ZmfqvqORuJIFilZYKF3d0wDQYJKoZIhvcNAQELBQADQQAHAomFKsF4jvYX
-    WM/UzQXDj9nSAFTf8dBPCXyZZNotsOH7+P6W4mMiuVs8bAuGiXGUdbsQ2lpiT/Rk
-    CzMeMdr4
-    -----END CERTIFICATE-----
-    -----BEGIN CERTIFICATE-----
-    MIIBgjCCASygAwIBAgIMFo3gM0nxQpiX/agfMA0GCSqGSIb3DQEBCwUAMCIxIDAe
-    BgNVBAMTF2FwaXNlcnZlci1hZ2dyZWdhdG9yLWNhMB4XDTIxMDYzMDA0NTIzMVoX
-    DTMxMDYzMDA0NTIzMVowIjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3It
-    Y2EwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAyyE71AOU3go5XFegLQ6fidI0LhhM
-    x7CzpTzh2xWKcHUfbNI7itgJvC/+GlyG5W+DF5V7ba0IJiQLsFve0oLdewIDAQAB
-    o0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU
-    ALfqF5ZmfqvqORuJIFilZYKF3d0wDQYJKoZIhvcNAQELBQADQQCXsoezoxXu2CEN
-    QdlXZOfmBT6cqxIX/RMHXhpHwRiqPsTO8IO2bVA8CSzxNwMuSv/ZtrMHoh8+PcVW
-    HLtkTXH8
-    -----END CERTIFICATE-----
-  etcd-clients-ca: |
-    -----BEGIN CERTIFICATE-----
-    MIIBcjCCARygAwIBAgIMFo1ogHnr26DL9YkqMA0GCSqGSIb3DQEBCwUAMBoxGDAW
-    BgNVBAMTD2V0Y2QtY2xpZW50cy1jYTAeFw0yMTA2MjgxNjE5MDFaFw0zMTA2Mjgx
-    NjE5MDFaMBoxGDAWBgNVBAMTD2V0Y2QtY2xpZW50cy1jYTBcMA0GCSqGSIb3DQEB
-    AQUAA0sAMEgCQQDYlt4Xx03Cp8QooPrloaVWznx9aQDSpl1UsrDyoBPNEElOLWep
-    uPaQBHiDLL8LwzGi7G9r+ib13tKrwprnlPv7AgMBAAGjQjBAMA4GA1UdDwEB/wQE
-    AwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQjlt4Ue54AbJPWlDpRM51s
-    x+PeBDANBgkqhkiG9w0BAQsFAANBAAZAdf8ROEVkr3Rf7I+s+CQOil2toadlKWOY
-    qCeJ2XaEROfp9aUTEIU1MGM3g57MPyAPPU7mURskuOQz6B1UFaY=
-    -----END CERTIFICATE-----
-    -----BEGIN CERTIFICATE-----
-    MIIBcjCCARygAwIBAgIMFo1olfBnC/CsT+dqMA0GCSqGSIb3DQEBCwUAMBoxGDAW
-    BgNVBAMTD2V0Y2QtY2xpZW50cy1jYTAeFw0yMTA2MjgxNjIwMzNaFw0zMTA2Mjgx
-    NjIwMzNaMBoxGDAWBgNVBAMTD2V0Y2QtY2xpZW50cy1jYTBcMA0GCSqGSIb3DQEB
-    AQUAA0sAMEgCQQDYlt4Xx03Cp8QooPrloaVWznx9aQDSpl1UsrDyoBPNEElOLWep
-    uPaQBHiDLL8LwzGi7G9r+ib13tKrwprnlPv7AgMBAAGjQjBAMA4GA1UdDwEB/wQE
-    AwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQjlt4Ue54AbJPWlDpRM51s
-    x+PeBDANBgkqhkiG9w0BAQsFAANBAF1xUz77PlUVUnd9duF8F7plou0TONC9R6/E
-    YQ8C6vM1b+9NSDGjCW8YmwEU2fBgskb/BBX2lwVZ32/RUEju4Co=
-    -----END CERTIFICATE-----
-  etcd-manager-ca-events: |
-    -----BEGIN CERTIFICATE-----
-    MIIBgDCCASqgAwIBAgIMFo+bKjm04vB4rNtaMA0GCSqGSIb3DQEBCwUAMCExHzAd
-    BgNVBAMTFmV0Y2QtbWFuYWdlci1jYS1ldmVudHMwHhcNMjEwNzA1MjAwOTU2WhcN
-    MzEwNzA1MjAwOTU2WjAhMR8wHQYDVQQDExZldGNkLW1hbmFnZXItY2EtZXZlbnRz
-    MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKiC8tndMlEFZ7qzeKxeKqFVjaYpsh/H
-    g7RxWo15+1kgH3suO0lxp9+RxSVv97hnsfbySTPZVhy2cIQj7eZtZt8CAwEAAaNC
-    MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFBg6
-    CEZkQNnRkARBwFce03AEWa+sMA0GCSqGSIb3DQEBCwUAA0EAJMnBThok/uUe8q8O
-    sS5q19KUuE8YCTUzMDj36EBKf6NX4NoakCa1h6kfQVtlMtEIMWQZCjbm8xGK5ffs
-    GS/VUw==
-    -----END CERTIFICATE-----
-    -----BEGIN CERTIFICATE-----
-    MIIBgDCCASqgAwIBAgIMFo+bQ+EgIiBmGghjMA0GCSqGSIb3DQEBCwUAMCExHzAd
-    BgNVBAMTFmV0Y2QtbWFuYWdlci1jYS1ldmVudHMwHhcNMjEwNzA1MjAxMTQ2WhcN
-    MzEwNzA1MjAxMTQ2WjAhMR8wHQYDVQQDExZldGNkLW1hbmFnZXItY2EtZXZlbnRz
-    MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKFhHVVxxDGv8d1jBvtdSxz7KIVoBOjL
-    DMxsmTsINiQkTQaFlb+XPlnY1ar4+RhE519AFUkqfhypk4Zxqf1YFXUCAwEAAaNC
-    MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNuW
-    LLH5c8kDubDbr6BHgedW0iJ9MA0GCSqGSIb3DQEBCwUAA0EAiKUoBoaGu7XzboFE
-    hjfKlX0TujqWuW3qMxDEJwj4dVzlSLrAoB/G01MJ+xxYKh456n48aG6N827UPXhV
-    cPfVNg==
-    -----END CERTIFICATE-----
-  etcd-manager-ca-main: |
-    -----BEGIN CERTIFICATE-----
-    MIIBfDCCASagAwIBAgIMFo+bKjm1c3jfv6hIMA0GCSqGSIb3DQEBCwUAMB8xHTAb
-    BgNVBAMTFGV0Y2QtbWFuYWdlci1jYS1tYWluMB4XDTIxMDcwNTIwMDk1NloXDTMx
-    MDcwNTIwMDk1NlowHzEdMBsGA1UEAxMUZXRjZC1tYW5hZ2VyLWNhLW1haW4wXDAN
-    BgkqhkiG9w0BAQEFAANLADBIAkEAxbkDbGYmCSShpRG3r+lzTOFujyuruRfjOhYm
-    ZRX4w1Utd5y63dUc98sjc9GGUYMHd+0k1ql/a48tGhnK6N6jJwIDAQABo0IwQDAO
-    BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUWZLkbBFx
-    GAgPU4i62c52unSo7RswDQYJKoZIhvcNAQELBQADQQAj6Pgd0va/8FtkyMlnohLu
-    Gf4v8RJO6zk3Y6jJ4+cwWziipFM1ielMzSOZfFcCZgH3m5Io40is4hPSqyq2TOA6
-    -----END CERTIFICATE-----
-    -----BEGIN CERTIFICATE-----
-    MIIBfDCCASagAwIBAgIMFo+bQ+Eg8Si30gr4MA0GCSqGSIb3DQEBCwUAMB8xHTAb
-    BgNVBAMTFGV0Y2QtbWFuYWdlci1jYS1tYWluMB4XDTIxMDcwNTIwMTE0NloXDTMx
-    MDcwNTIwMTE0NlowHzEdMBsGA1UEAxMUZXRjZC1tYW5hZ2VyLWNhLW1haW4wXDAN
-    BgkqhkiG9w0BAQEFAANLADBIAkEAw33jzcd/iosN04b0WXbDt7B0c3sJ3aafcGLP
-    vG3xRB9N5bYr9+qZAq3mzAFkxscn4j1ce5b1/GKTDEAClmZgdQIDAQABo0IwQDAO
-    BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUE/h+3gDP
-    DvKwHRyiYlXM8voZ1wowDQYJKoZIhvcNAQELBQADQQBXuimeEoAOu5HN4hG7NqL9
-    t40K3ZRhRZv3JQWnRVJCBDjg1rD0GQJR/n+DoWvbeijI5C9pNjr2pWSIYR1eYCvd
-    -----END CERTIFICATE-----
-  etcd-peers-ca-events: |
-    -----BEGIN CERTIFICATE-----
-    MIIBfDCCASagAwIBAgIMFo+bKjmxTPh3/lYJMA0GCSqGSIb3DQEBCwUAMB8xHTAb
-    BgNVBAMTFGV0Y2QtcGVlcnMtY2EtZXZlbnRzMB4XDTIxMDcwNTIwMDk1NloXDTMx
-    MDcwNTIwMDk1NlowHzEdMBsGA1UEAxMUZXRjZC1wZWVycy1jYS1ldmVudHMwXDAN
-    BgkqhkiG9w0BAQEFAANLADBIAkEAv5g4HF2xmrYyouJfY9jXx1M3gPLD/pupvxPY
-    xyjJw5pNCy5M5XGS3iTqRD5RDE0fWudVHFZKLIe8WPc06NApXwIDAQABo0IwQDAO
-    BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUf6xiDI+O
-    Yph1ziCGr2hZaQYt+fUwDQYJKoZIhvcNAQELBQADQQBBxj5hqEQstonTb8lnqeGB
-    DEYtUeAk4eR/HzvUMjF52LVGuvN3XVt+JTrFeKNvb6/RDUbBNRj3azalcUkpPh6V
-    -----END CERTIFICATE-----
-    -----BEGIN CERTIFICATE-----
-    MIIBfDCCASagAwIBAgIMFo+bQ+Eq69jgzpKwMA0GCSqGSIb3DQEBCwUAMB8xHTAb
-    BgNVBAMTFGV0Y2QtcGVlcnMtY2EtZXZlbnRzMB4XDTIxMDcwNTIwMTE0NloXDTMx
-    MDcwNTIwMTE0NlowHzEdMBsGA1UEAxMUZXRjZC1wZWVycy1jYS1ldmVudHMwXDAN
-    BgkqhkiG9w0BAQEFAANLADBIAkEAo5Nj2CjX1qp3mEPw1H5nHAFWLoGNSLSlRFJW
-    03NxaNPMFzL5PrCoyOXrX8/MWczuZYw0Crf8EPOOQWi2+W0XLwIDAQABo0IwQDAO
-    BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUxauhhKQh
-    cvdZND78rHe0RQVTTiswDQYJKoZIhvcNAQELBQADQQB+cq4jIS9q0zXslaRa+ViI
-    J+dviA3sMygbmSJO0s4DxYmoazKJblux5q0ASSvS9iL1l9ShuZ1dWyp2tpZawHyb
-    -----END CERTIFICATE-----
-  etcd-peers-ca-main: |
-    -----BEGIN CERTIFICATE-----
-    MIIBeDCCASKgAwIBAgIMFo+bKjmuLDDLcDHsMA0GCSqGSIb3DQEBCwUAMB0xGzAZ
-    BgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjAeFw0yMTA3MDUyMDA5NTZaFw0zMTA3
-    MDUyMDA5NTZaMB0xGzAZBgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjBcMA0GCSqG
-    SIb3DQEBAQUAA0sAMEgCQQCyRaXWpwgN6INQqws9p/BvPElJv2Rno9dVTFhlQqDA
-    aUJXe7MBmiO4NJcW76EozeBh5ztR3/4NE1FM2x8TisS3AgMBAAGjQjBAMA4GA1Ud
-    DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQtE1d49uSvpURf
-    OQ25Vlu6liY20DANBgkqhkiG9w0BAQsFAANBAAgLVaetJZcfOA3OIMMvQbz2Ydrt
-    uWF9BKkIad8jrcIrm3IkOtR8bKGmDIIaRKuG/ZUOL6NMe2fky3AAfKwleL4=
-    -----END CERTIFICATE-----
-    -----BEGIN CERTIFICATE-----
-    MIIBeDCCASKgAwIBAgIMFo+bQ+EuVthBfuZvMA0GCSqGSIb3DQEBCwUAMB0xGzAZ
-    BgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjAeFw0yMTA3MDUyMDExNDZaFw0zMTA3
-    MDUyMDExNDZaMB0xGzAZBgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjBcMA0GCSqG
-    SIb3DQEBAQUAA0sAMEgCQQCxNbycDZNx5V1ZOiXxZSvaFpHRwKeHDfcuMUitdoPt
-    naVMlMTGDWAMuCVmFHFAWohIYynemEegmZkZ15S7AErfAgMBAAGjQjBAMA4GA1Ud
-    DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTAjQ8T4HclPIsC
-    qipEfUIcLP6jqTANBgkqhkiG9w0BAQsFAANBAJdZ17TN3HlWrH7HQgfR12UBwz8K
-    G9DurDznVaBVUYaHY8Sg5AvAXeb+yIF2JMmRR+bK+/G1QYY2D3/P31Ic2Oo=
-    -----END CERTIFICATE-----
-  kubernetes-ca: |
-    -----BEGIN CERTIFICATE-----
-    MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU
-    BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw
-    ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
-    SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1
-    jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
-    MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA
-    MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8
-    tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w==
-    -----END CERTIFICATE-----
-    -----BEGIN CERTIFICATE-----
-    MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU
-    BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw
-    OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
-    SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7
-    WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
-    MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn
-    MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA
-    9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw==
-    -----END CERTIFICATE-----
-ClusterName: privateweave.example.com
-ContainerRuntime: containerd
-FileAssets:
-- content: |
-    apiVersion: kubescheduler.config.k8s.io/v1beta2
-    clientConnection:
-      kubeconfig: /var/lib/kube-scheduler/kubeconfig
-    kind: KubeSchedulerConfiguration
-  path: /var/lib/kube-scheduler/config.yaml
-Hooks:
-- null
-- null
-KeypairIDs:
-  apiserver-aggregator-ca: "6980187172486667078076483355"
-  etcd-clients-ca: "6979622252718071085282986282"
-  etcd-manager-ca-events: "6982279354000777253151890266"
-  etcd-manager-ca-main: "6982279354000936168671127624"
-  etcd-peers-ca-events: "6982279353999767935825892873"
-  etcd-peers-ca-main: "6982279353998887468930183660"
-  kubernetes-ca: "6982820025135291416230495506"
-  service-account: "2"
-KubeProxy:
-  clusterCIDR: 100.96.0.0/11
-  cpuRequest: 100m
-  image: registry.k8s.io/kube-proxy:v1.22.0
-  logLevel: 2
-KubeletConfig:
-  anonymousAuth: false
-  cgroupDriver: systemd
-  cgroupRoot: /
-  cloudProvider: aws
-  clusterDNS: 100.64.0.10
-  clusterDomain: cluster.local
-  enableDebuggingHandlers: true
-  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
-  featureGates:
-    CSIMigrationAWS: "true"
-    InTreePluginAWSUnregister: "true"
-  kubeconfigPath: /var/lib/kubelet/kubeconfig
-  logLevel: 2
-  networkPluginName: cni
-  nodeLabels:
-    kops.k8s.io/kops-controller-pki: ""
-    kubernetes.io/role: master
-    node-role.kubernetes.io/control-plane: ""
-    node-role.kubernetes.io/master: ""
-    node.kubernetes.io/exclude-from-external-load-balancers: ""
-  podInfraContainerImage: registry.k8s.io/pause:3.9
-  podManifestPath: /etc/kubernetes/manifests
-  protectKernelDefaults: true
-  registerSchedulable: true
-  shutdownGracePeriod: 30s
-  shutdownGracePeriodCriticalPods: 10s
-  taints:
-  - node-role.kubernetes.io/master=:NoSchedule
-KubernetesVersion: 1.22.0
-Networking:
-  nonMasqueradeCIDR: 100.64.0.0/10
-  serviceClusterIPRange: 100.64.0.0/13
-UpdatePolicy: automatic
-channels:
-- memfs://clusters.example.com/privateweave.example.com/addons/bootstrap-channel.yaml
-containerdConfig:
-  logLevel: info
-  version: 1.4.13
-docker:
-  skipInstall: true
-etcdManifests:
-- memfs://clusters.example.com/privateweave.example.com/manifests/etcd/main-master-us-test-1a.yaml
-- memfs://clusters.example.com/privateweave.example.com/manifests/etcd/events-master-us-test-1a.yaml
-staticManifests:
-- key: kube-apiserver-healthcheck
-  path: manifests/static/kube-apiserver-healthcheck.yaml
-usesLegacyGossip: false
-usesNoneDNS: false

tests/integration/update_cluster/privateweave/data/aws_s3_object_nodeupconfig-nodes_content

@@ -1,62 +0,0 @@
-Assets:
-  amd64:
-  - fec5c596f7f815f17f5d7d955e9707df1ef02a2ca5e788b223651f83376feb7f@https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubelet
-  - 703e70d49b82271535bc66bc7bd469a58c11d47f188889bd37101c9772f14fa1@https://dl.k8s.io/release/v1.22.0/bin/linux/amd64/kubectl
-  - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
-  - 29ef1e8635795c2a49a20a56e778f45ff163c5400a5428ca33999ed53d44e3d8@https://github.com/containerd/containerd/releases/download/v1.4.13/cri-containerd-cni-1.4.13-linux-amd64.tar.gz
-  arm64:
-  - cea637a7da4f1097b16b0195005351c07032a820a3d64c3ff326b9097cfac930@https://dl.k8s.io/release/v1.22.0/bin/linux/arm64/kubelet
-  - 8d9cc92dcc942f5ea2b2fc93c4934875d9e0e8ddecbde24c7d4c4e092cfc7afc@https://dl.k8s.io/release/v1.22.0/bin/linux/arm64/kubectl
-  - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
-  - debed306ed9a4e70dcbcb228a0b3898f9730099e324f34bb0e76abbaddf7a6a7@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.13.tgz
-CAs: {}
-ClusterName: privateweave.example.com
-ContainerRuntime: containerd
-Hooks:
-- null
-- null
-KeypairIDs:
-  kubernetes-ca: "6982820025135291416230495506"
-KubeProxy:
-  clusterCIDR: 100.96.0.0/11
-  cpuRequest: 100m
-  image: registry.k8s.io/kube-proxy:v1.22.0
-  logLevel: 2
-KubeletConfig:
-  anonymousAuth: false
-  cgroupDriver: systemd
-  cgroupRoot: /
-  cloudProvider: aws
-  clusterDNS: 100.64.0.10
-  clusterDomain: cluster.local
-  enableDebuggingHandlers: true
-  evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
-  featureGates:
-    CSIMigrationAWS: "true"
-    InTreePluginAWSUnregister: "true"
-  kubeconfigPath: /var/lib/kubelet/kubeconfig
-  logLevel: 2
-  networkPluginName: cni
-  nodeLabels:
-    kubernetes.io/role: node
-    node-role.kubernetes.io/node: ""
-  podInfraContainerImage: registry.k8s.io/pause:3.9
-  podManifestPath: /etc/kubernetes/manifests
-  protectKernelDefaults: true
-  registerSchedulable: true
-  shutdownGracePeriod: 30s
-  shutdownGracePeriodCriticalPods: 10s
-KubernetesVersion: 1.22.0
-Networking:
-  nonMasqueradeCIDR: 100.64.0.0/10
-  serviceClusterIPRange: 100.64.0.0/13
-UpdatePolicy: automatic
-channels:
-- memfs://clusters.example.com/privateweave.example.com/addons/bootstrap-channel.yaml
-containerdConfig:
-  logLevel: info
-  version: 1.4.13
-docker:
-  skipInstall: true
-usesLegacyGossip: false
-usesNoneDNS: false

tests/integration/update_cluster/privateweave/data/aws_s3_object_privateweave.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content

@@ -1,819 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
-    app.kubernetes.io/instance: aws-ebs-csi-driver
-    app.kubernetes.io/managed-by: kops
-    app.kubernetes.io/name: aws-ebs-csi-driver
-    app.kubernetes.io/version: v1.14.1
-    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
-  name: ebs-csi-controller-sa
-  namespace: kube-system
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
-    app.kubernetes.io/instance: aws-ebs-csi-driver
-    app.kubernetes.io/managed-by: kops
-    app.kubernetes.io/name: aws-ebs-csi-driver
-    app.kubernetes.io/version: v1.14.1
-    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
-  name: ebs-external-attacher-role
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - persistentvolumes
-  verbs:
-  - get
-  - list
-  - watch
-  - update
-  - patch
-- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - csi.storage.k8s.io
-  resources:
-  - csinodeinfos
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - storage.k8s.io
-  resources:
-  - volumeattachments
-  verbs:
-  - get
-  - list
-  - watch
-  - update
-  - patch
-- apiGroups:
-  - storage.k8s.io
-  resources:
-  - volumeattachments/status
-  verbs:
-  - patch
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
-    app.kubernetes.io/instance: aws-ebs-csi-driver
-    app.kubernetes.io/managed-by: kops
-    app.kubernetes.io/name: aws-ebs-csi-driver
-    app.kubernetes.io/version: v1.14.1
-    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
-  name: ebs-external-provisioner-role
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - persistentvolumes
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - persistentvolumeclaims
-  verbs:
-  - get
-  - list
-  - watch
-  - update
-- apiGroups:
-  - storage.k8s.io
-  resources:
-  - storageclasses
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - events
-  verbs:
-  - list
-  - watch
-  - create
-  - update
-  - patch
-- apiGroups:
-  - snapshot.storage.k8s.io
-  resources:
-  - volumesnapshots
-  verbs:
-  - get
-  - list
-- apiGroups:
-  - snapshot.storage.k8s.io
-  resources:
-  - volumesnapshotcontents
-  verbs:
-  - get
-  - list
-- apiGroups:
-  - storage.k8s.io
-  resources:
-  - csinodes
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - coordination.k8s.io
-  resources:
-  - leases
-  verbs:
-  - get
-  - watch
-  - list
-  - delete
-  - update
-  - create
-- apiGroups:
-  - storage.k8s.io
-  resources:
-  - volumeattachments
-  verbs:
-  - get
-  - list
-  - watch
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
-    app.kubernetes.io/instance: aws-ebs-csi-driver
-    app.kubernetes.io/managed-by: kops
-    app.kubernetes.io/name: aws-ebs-csi-driver
-    app.kubernetes.io/version: v1.14.1
-    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
-  name: ebs-external-resizer-role
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - persistentvolumes
-  verbs:
-  - get
-  - list
-  - watch
-  - update
-  - patch
-- apiGroups:
-  - ""
-  resources:
-  - persistentvolumeclaims
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - persistentvolumeclaims/status
-  verbs:
-  - update
-  - patch
-- apiGroups:
-  - storage.k8s.io
-  resources:
-  - storageclasses
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - events
-  verbs:
-  - list
-  - watch
-  - create
-  - update
-  - patch
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
-    app.kubernetes.io/instance: aws-ebs-csi-driver
-    app.kubernetes.io/managed-by: kops
-    app.kubernetes.io/name: aws-ebs-csi-driver
-    app.kubernetes.io/version: v1.14.1
-    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
-  name: ebs-external-snapshotter-role
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - events
-  verbs:
-  - list
-  - watch
-  - create
-  - update
-  - patch
-- apiGroups:
-  - ""
-  resources:
-  - secrets
-  verbs:
-  - get
-  - list
-- apiGroups:
-  - snapshot.storage.k8s.io
-  resources:
-  - volumesnapshotclasses
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - snapshot.storage.k8s.io
-  resources:
-  - volumesnapshotcontents
-  verbs:
-  - create
-  - get
-  - list
-  - watch
-  - update
-  - delete
-  - patch
-- apiGroups:
-  - snapshot.storage.k8s.io
-  resources:
-  - volumesnapshotcontents/status
-  verbs:
-  - update
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
-    app.kubernetes.io/instance: aws-ebs-csi-driver
-    app.kubernetes.io/managed-by: kops
-    app.kubernetes.io/name: aws-ebs-csi-driver
-    app.kubernetes.io/version: v1.14.1
-    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
-  name: ebs-csi-attacher-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: ebs-external-attacher-role
-subjects:
-- kind: ServiceAccount
-  name: ebs-csi-controller-sa
-  namespace: kube-system
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
-    app.kubernetes.io/instance: aws-ebs-csi-driver
-    app.kubernetes.io/managed-by: kops
-    app.kubernetes.io/name: aws-ebs-csi-driver
-    app.kubernetes.io/version: v1.14.1
-    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
-  name: ebs-csi-provisioner-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: ebs-external-provisioner-role
-subjects:
-- kind: ServiceAccount
-  name: ebs-csi-controller-sa
-  namespace: kube-system
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
-    app.kubernetes.io/instance: aws-ebs-csi-driver
-    app.kubernetes.io/managed-by: kops
-    app.kubernetes.io/name: aws-ebs-csi-driver
-    app.kubernetes.io/version: v1.14.1
-    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
-  name: ebs-csi-resizer-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: ebs-external-resizer-role
-subjects:
-- kind: ServiceAccount
-  name: ebs-csi-controller-sa
-  namespace: kube-system
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
-    app.kubernetes.io/instance: aws-ebs-csi-driver
-    app.kubernetes.io/managed-by: kops
-    app.kubernetes.io/name: aws-ebs-csi-driver
-    app.kubernetes.io/version: v1.14.1
-    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
-  name: ebs-csi-snapshotter-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: ebs-external-snapshotter-role
-subjects:
-- kind: ServiceAccount
-  name: ebs-csi-controller-sa
-  namespace: kube-system
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    app.kubernetes.io/name: aws-ebs-csi-driver
-    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
-  name: ebs-csi-node-getter-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: ebs-csi-node-role
-subjects:
-- kind: ServiceAccount
-  name: ebs-csi-node-sa
-  namespace: kube-system
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    app.kubernetes.io/name: aws-ebs-csi-driver
-    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
-  name: ebs-csi-node-role
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - get
-
----
-
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
-    app.kubernetes.io/instance: aws-ebs-csi-driver
-    app.kubernetes.io/managed-by: kops
-    app.kubernetes.io/name: aws-ebs-csi-driver
-    app.kubernetes.io/version: v1.14.1
-    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
-  name: ebs-csi-node-sa
-  namespace: kube-system
-
----
-
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
-    app.kubernetes.io/instance: aws-ebs-csi-driver
-    app.kubernetes.io/managed-by: kops
-    app.kubernetes.io/name: aws-ebs-csi-driver
-    app.kubernetes.io/version: v1.14.1
-    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
-  name: ebs-csi-node
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      app: ebs-csi-node
-      app.kubernetes.io/instance: aws-ebs-csi-driver
-      app.kubernetes.io/name: aws-ebs-csi-driver
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        app: ebs-csi-node
-        app.kubernetes.io/instance: aws-ebs-csi-driver
-        app.kubernetes.io/name: aws-ebs-csi-driver
-        app.kubernetes.io/version: v1.14.1
-        kops.k8s.io/managed-by: kops
-    spec:
-      containers:
-      - args:
-        - node
-        - --endpoint=$(CSI_ENDPOINT)
-        - --logtostderr
-        - --v=2
-        env:
-        - name: CSI_ENDPOINT
-          value: unix:/csi/csi.sock
-        - name: CSI_NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: spec.nodeName
-        image: registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1
-        imagePullPolicy: IfNotPresent
-        livenessProbe:
-          failureThreshold: 5
-          httpGet:
-            path: /healthz
-            port: healthz
-          initialDelaySeconds: 10
-          periodSeconds: 10
-          timeoutSeconds: 3
-        name: ebs-plugin
-        ports:
-        - containerPort: 9808
-          name: healthz
-          protocol: TCP
-        securityContext:
-          privileged: true
-          readOnlyRootFilesystem: true
-        volumeMounts:
-        - mountPath: /var/lib/kubelet
-          mountPropagation: Bidirectional
-          name: kubelet-dir
-        - mountPath: /csi
-          name: plugin-dir
-        - mountPath: /dev
-          name: device-dir
-      - args:
-        - --csi-address=$(ADDRESS)
-        - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
-        - --v=5
-        env:
-        - name: ADDRESS
-          value: /csi/csi.sock
-        - name: DRIVER_REG_SOCK_PATH
-          value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock
-        image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
-        imagePullPolicy: IfNotPresent
-        name: node-driver-registrar
-        securityContext:
-          allowPrivilegeEscalation: false
-          readOnlyRootFilesystem: true
-        volumeMounts:
-        - mountPath: /csi
-          name: plugin-dir
-        - mountPath: /registration
-          name: registration-dir
-      - args:
-        - --csi-address=/csi/csi.sock
-        image: registry.k8s.io/sig-storage/livenessprobe:v2.6.0
-        imagePullPolicy: IfNotPresent
-        name: liveness-probe
-        securityContext:
-          allowPrivilegeEscalation: false
-          readOnlyRootFilesystem: true
-        volumeMounts:
-        - mountPath: /csi
-          name: plugin-dir
-      nodeSelector:
-        kubernetes.io/os: linux
-      priorityClassName: system-node-critical
-      securityContext:
-        fsGroup: 0
-        runAsGroup: 0
-        runAsNonRoot: false
-        runAsUser: 0
-      serviceAccountName: ebs-csi-node-sa
-      tolerations:
-      - operator: Exists
-      volumes:
-      - hostPath:
-          path: /var/lib/kubelet
-          type: Directory
-        name: kubelet-dir
-      - hostPath:
-          path: /var/lib/kubelet/plugins/ebs.csi.aws.com/
-          type: DirectoryOrCreate
-        name: plugin-dir
-      - hostPath:
-          path: /var/lib/kubelet/plugins_registry/
-          type: Directory
-        name: registration-dir
-      - hostPath:
-          path: /dev
-          type: Directory
-        name: device-dir
-
----
-
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
-    app.kubernetes.io/instance: aws-ebs-csi-driver
-    app.kubernetes.io/managed-by: kops
-    app.kubernetes.io/name: aws-ebs-csi-driver
-    app.kubernetes.io/version: v1.14.1
-    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
-  name: ebs-csi-controller
-  namespace: kube-system
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: ebs-csi-controller
-      app.kubernetes.io/instance: aws-ebs-csi-driver
-      app.kubernetes.io/name: aws-ebs-csi-driver
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        app: ebs-csi-controller
-        app.kubernetes.io/instance: aws-ebs-csi-driver
-        app.kubernetes.io/name: aws-ebs-csi-driver
-        app.kubernetes.io/version: v1.14.1
-        kops.k8s.io/managed-by: kops
-    spec:
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-            - matchExpressions:
-              - key: node-role.kubernetes.io/control-plane
-                operator: Exists
-              - key: kubernetes.io/os
-                operator: In
-                values:
-                - linux
-            - matchExpressions:
-              - key: node-role.kubernetes.io/master
-                operator: Exists
-              - key: kubernetes.io/os
-                operator: In
-                values:
-                - linux
-      containers:
-      - args:
-        - controller
-        - --endpoint=$(CSI_ENDPOINT)
-        - --logtostderr
-        - --k8s-tag-cluster-id=privateweave.example.com
-        - --extra-tags=KubernetesCluster=privateweave.example.com
-        - --http-endpoint=0.0.0.0:3301
-        - --v=5
-        env:
-        - name: CSI_ENDPOINT
-          value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
-        - name: CSI_NODE_NAME
-          valueFrom:
-            fieldRef:
-              apiVersion: v1
-              fieldPath: spec.nodeName
-        - name: AWS_ACCESS_KEY_ID
-          valueFrom:
-            secretKeyRef:
-              key: key_id
-              name: aws-secret
-              optional: true
-        - name: AWS_SECRET_ACCESS_KEY
-          valueFrom:
-            secretKeyRef:
-              key: access_key
-              name: aws-secret
-              optional: true
-        image: registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1
-        imagePullPolicy: IfNotPresent
-        livenessProbe:
-          failureThreshold: 5
-          httpGet:
-            path: /healthz
-            port: healthz
-          initialDelaySeconds: 10
-          periodSeconds: 10
-          timeoutSeconds: 3
-        name: ebs-plugin
-        ports:
-        - containerPort: 9808
-          name: healthz
-          protocol: TCP
-        - containerPort: 3301
-          name: metrics
-          protocol: TCP
-        readinessProbe:
-          failureThreshold: 5
-          httpGet:
-            path: /healthz
-            port: healthz
-          initialDelaySeconds: 10
-          periodSeconds: 10
-          timeoutSeconds: 3
-        securityContext:
-          allowPrivilegeEscalation: false
-          readOnlyRootFilesystem: true
-        volumeMounts:
-        - mountPath: /var/lib/csi/sockets/pluginproxy/
-          name: socket-dir
-      - args:
-        - --csi-address=$(ADDRESS)
-        - --v=5
-        - --feature-gates=Topology=true
-        - --extra-create-metadata
-        - --leader-election=true
-        - --default-fstype=ext4
-        env:
-        - name: ADDRESS
-          value: /var/lib/csi/sockets/pluginproxy/csi.sock
-        image: registry.k8s.io/sig-storage/csi-provisioner:v3.1.0
-        imagePullPolicy: IfNotPresent
-        name: csi-provisioner
-        securityContext:
-          allowPrivilegeEscalation: false
-          readOnlyRootFilesystem: true
-        volumeMounts:
-        - mountPath: /var/lib/csi/sockets/pluginproxy/
-          name: socket-dir
-      - args:
-        - --csi-address=$(ADDRESS)
-        - --v=5
-        - --leader-election=true
-        env:
-        - name: ADDRESS
-          value: /var/lib/csi/sockets/pluginproxy/csi.sock
-        image: registry.k8s.io/sig-storage/csi-attacher:v3.4.0
-        imagePullPolicy: IfNotPresent
-        name: csi-attacher
-        securityContext:
-          allowPrivilegeEscalation: false
-          readOnlyRootFilesystem: true
-        volumeMounts:
-        - mountPath: /var/lib/csi/sockets/pluginproxy/
-          name: socket-dir
-      - args:
-        - --csi-address=$(ADDRESS)
-        - --v=5
-        env:
-        - name: ADDRESS
-          value: /var/lib/csi/sockets/pluginproxy/csi.sock
-        image: registry.k8s.io/sig-storage/csi-resizer:v1.4.0
-        imagePullPolicy: IfNotPresent
-        name: csi-resizer
-        securityContext:
-          allowPrivilegeEscalation: false
-          readOnlyRootFilesystem: true
-        volumeMounts:
-        - mountPath: /var/lib/csi/sockets/pluginproxy/
-          name: socket-dir
-      - args:
-        - --csi-address=/csi/csi.sock
-        image: registry.k8s.io/sig-storage/livenessprobe:v2.6.0
-        imagePullPolicy: IfNotPresent
-        name: liveness-probe
-        volumeMounts:
-        - mountPath: /csi
-          name: socket-dir
-      hostNetwork: true
-      nodeSelector: null
-      priorityClassName: system-cluster-critical
-      securityContext:
-        fsGroup: 1000
-        runAsGroup: 1000
-        runAsNonRoot: true
-        runAsUser: 1000
-      serviceAccountName: ebs-csi-controller-sa
-      tolerations:
-      - operator: Exists
-      topologySpreadConstraints:
-      - labelSelector:
-          matchLabels:
-            app: ebs-csi-controller
-            app.kubernetes.io/instance: aws-ebs-csi-driver
-            app.kubernetes.io/name: aws-ebs-csi-driver
-        maxSkew: 1
-        topologyKey: topology.kubernetes.io/zone
-        whenUnsatisfiable: ScheduleAnyway
-      - labelSelector:
-          matchLabels:
-            app: ebs-csi-controller
-            app.kubernetes.io/instance: aws-ebs-csi-driver
-            app.kubernetes.io/name: aws-ebs-csi-driver
-        maxSkew: 1
-        topologyKey: kubernetes.io/hostname
-        whenUnsatisfiable: DoNotSchedule
-      volumes:
-      - emptyDir: {}
-        name: socket-dir
-
----
-
-apiVersion: storage.k8s.io/v1
-kind: CSIDriver
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
-    app.kubernetes.io/instance: aws-ebs-csi-driver
-    app.kubernetes.io/managed-by: kops
-    app.kubernetes.io/name: aws-ebs-csi-driver
-    app.kubernetes.io/version: v1.14.1
-    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
-  name: ebs.csi.aws.com
-spec:
-  attachRequired: true
-  podInfoOnMount: false
-
----
-
-apiVersion: policy/v1beta1
-kind: PodDisruptionBudget
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
-    app.kubernetes.io/instance: aws-ebs-csi-driver
-    app.kubernetes.io/managed-by: kops
-    app.kubernetes.io/name: aws-ebs-csi-driver
-    app.kubernetes.io/version: v1.14.1
-    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
-  name: ebs-csi-controller
-  namespace: kube-system
-spec:
-  maxUnavailable: 1
-  selector:
-    matchLabels:
-      app: ebs-csi-controller
-      app.kubernetes.io/instance: aws-ebs-csi-driver

tests/integration/update_cluster/privateweave/data/aws_s3_object_privateweave.example.com-addons-bootstrap_content

@@ -1,62 +0,0 @@
-kind: Addons
-metadata:
-  creationTimestamp: null
-  name: bootstrap
-spec:
-  addons:
-  - id: k8s-1.16
-    manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: f74dee34ac8258319f7e310b635f98c21759e0d3e05574f2fe6640430be2cd2f
-    name: kops-controller.addons.k8s.io
-    needsRollingUpdate: control-plane
-    selector:
-      k8s-addon: kops-controller.addons.k8s.io
-    version: 9.99.0
-  - id: k8s-1.12
-    manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 49273720e8c046ae8b72a3fe666d670f6efb5e26111dc521027218b1e8e1de51
-    name: coredns.addons.k8s.io
-    selector:
-      k8s-addon: coredns.addons.k8s.io
-    version: 9.99.0
-  - id: k8s-1.9
-    manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml
-    manifestHash: 01c120e887bd98d82ef57983ad58a0b22bc85efb48108092a24c4b82e4c9ea81
-    name: kubelet-api.rbac.addons.k8s.io
-    selector:
-      k8s-addon: kubelet-api.rbac.addons.k8s.io
-    version: 9.99.0
-  - manifest: limit-range.addons.k8s.io/v1.5.0.yaml
-    manifestHash: 2d55c3bc5e354e84a3730a65b42f39aba630a59dc8d32b30859fcce3d3178bc2
-    name: limit-range.addons.k8s.io
-    selector:
-      k8s-addon: limit-range.addons.k8s.io
-    version: 9.99.0
-  - id: k8s-1.12
-    manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: c648d0743139214b71a6454f4d0712d5d3006039522661560d16e839d70820a6
-    name: dns-controller.addons.k8s.io
-    selector:
-      k8s-addon: dns-controller.addons.k8s.io
-    version: 9.99.0
-  - id: v1.15.0
-    manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
-    manifestHash: 4e2cda50cd5048133aad1b5e28becb60f4629d3f9e09c514a2757c27998b4200
-    name: storage-aws.addons.k8s.io
-    selector:
-      k8s-addon: storage-aws.addons.k8s.io
-    version: 9.99.0
-  - id: k8s-1.12
-    manifest: networking.weave/k8s-1.12.yaml
-    manifestHash: c58a7acc6ed931d26b59892beb1f43e240fd51cbde223e3d95e15b3e04ced54d
-    name: networking.weave
-    selector:
-      role.kubernetes.io/networking: "1"
-    version: 9.99.0
-  - id: k8s-1.17
-    manifest: aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml
-    manifestHash: 78af8219079e3a720207de5c69498484b83c058f244cc59392f06f1d9d341d7b
-    name: aws-ebs-csi-driver.addons.k8s.io
-    selector:
-      k8s-addon: aws-ebs-csi-driver.addons.k8s.io
-    version: 9.99.0

tests/integration/update_cluster/privateweave/data/aws_s3_object_privateweave.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -1,383 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: coredns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: coredns.addons.k8s.io
-    kubernetes.io/cluster-service: "true"
-  name: coredns
-  namespace: kube-system
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: coredns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: coredns.addons.k8s.io
-    kubernetes.io/bootstrapping: rbac-defaults
-  name: system:coredns
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - endpoints
-  - services
-  - pods
-  - namespaces
-  verbs:
-  - list
-  - watch
-- apiGroups:
-  - discovery.k8s.io
-  resources:
-  - endpointslices
-  verbs:
-  - list
-  - watch
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: coredns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: coredns.addons.k8s.io
-    kubernetes.io/bootstrapping: rbac-defaults
-  name: system:coredns
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:coredns
-subjects:
-- kind: ServiceAccount
-  name: coredns
-  namespace: kube-system
-
----
-
-apiVersion: v1
-data:
-  Corefile: |-
-    .:53 {
-        errors
-        health {
-          lameduck 5s
-        }
-        ready
-        kubernetes cluster.local. in-addr.arpa ip6.arpa {
-          pods insecure
-          fallthrough in-addr.arpa ip6.arpa
-          ttl 30
-        }
-        prometheus :9153
-        forward . /etc/resolv.conf {
-          max_concurrent 1000
-        }
-        cache 30
-        loop
-        reload
-        loadbalance
-    }
-kind: ConfigMap
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: coredns.addons.k8s.io
-    addonmanager.kubernetes.io/mode: EnsureExists
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: coredns.addons.k8s.io
-  name: coredns
-  namespace: kube-system
-
----
-
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: coredns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: coredns.addons.k8s.io
-    k8s-app: kube-dns
-    kubernetes.io/cluster-service: "true"
-    kubernetes.io/name: CoreDNS
-  name: coredns
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      k8s-app: kube-dns
-  strategy:
-    rollingUpdate:
-      maxSurge: 10%
-      maxUnavailable: 1
-    type: RollingUpdate
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        k8s-app: kube-dns
-        kops.k8s.io/managed-by: kops
-    spec:
-      containers:
-      - args:
-        - -conf
-        - /etc/coredns/Corefile
-        image: registry.k8s.io/coredns/coredns:v1.10.1
-        imagePullPolicy: IfNotPresent
-        livenessProbe:
-          failureThreshold: 5
-          httpGet:
-            path: /health
-            port: 8080
-            scheme: HTTP
-          initialDelaySeconds: 60
-          successThreshold: 1
-          timeoutSeconds: 5
-        name: coredns
-        ports:
-        - containerPort: 53
-          name: dns
-          protocol: UDP
-        - containerPort: 53
-          name: dns-tcp
-          protocol: TCP
-        - containerPort: 9153
-          name: metrics
-          protocol: TCP
-        readinessProbe:
-          httpGet:
-            path: /ready
-            port: 8181
-            scheme: HTTP
-        resources:
-          limits:
-            memory: 170Mi
-          requests:
-            cpu: 100m
-            memory: 70Mi
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            add:
-            - NET_BIND_SERVICE
-            drop:
-            - all
-          readOnlyRootFilesystem: true
-        volumeMounts:
-        - mountPath: /etc/coredns
-          name: config-volume
-          readOnly: true
-      dnsPolicy: Default
-      nodeSelector:
-        kubernetes.io/os: linux
-      priorityClassName: system-cluster-critical
-      serviceAccountName: coredns
-      tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
-      topologySpreadConstraints:
-      - labelSelector:
-          matchLabels:
-            k8s-app: kube-dns
-        maxSkew: 1
-        topologyKey: topology.kubernetes.io/zone
-        whenUnsatisfiable: ScheduleAnyway
-      - labelSelector:
-          matchLabels:
-            k8s-app: kube-dns
-        maxSkew: 1
-        topologyKey: kubernetes.io/hostname
-        whenUnsatisfiable: ScheduleAnyway
-      volumes:
-      - configMap:
-          name: coredns
-        name: config-volume
-
----
-
-apiVersion: v1
-kind: Service
-metadata:
-  annotations:
-    prometheus.io/port: "9153"
-    prometheus.io/scrape: "true"
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: coredns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: coredns.addons.k8s.io
-    k8s-app: kube-dns
-    kubernetes.io/cluster-service: "true"
-    kubernetes.io/name: CoreDNS
-  name: kube-dns
-  namespace: kube-system
-  resourceVersion: "0"
-spec:
-  clusterIP: 100.64.0.10
-  ports:
-  - name: dns
-    port: 53
-    protocol: UDP
-  - name: dns-tcp
-    port: 53
-    protocol: TCP
-  - name: metrics
-    port: 9153
-    protocol: TCP
-  selector:
-    k8s-app: kube-dns
-
----
-
-apiVersion: policy/v1beta1
-kind: PodDisruptionBudget
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: coredns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: coredns.addons.k8s.io
-  name: kube-dns
-  namespace: kube-system
-spec:
-  maxUnavailable: 50%
-  selector:
-    matchLabels:
-      k8s-app: kube-dns
-
----
-
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: coredns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: coredns.addons.k8s.io
-  name: coredns-autoscaler
-  namespace: kube-system
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: coredns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: coredns.addons.k8s.io
-  name: coredns-autoscaler
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - list
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - replicationcontrollers/scale
-  verbs:
-  - get
-  - update
-- apiGroups:
-  - extensions
-  - apps
-  resources:
-  - deployments/scale
-  - replicasets/scale
-  verbs:
-  - get
-  - update
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - get
-  - create
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: coredns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: coredns.addons.k8s.io
-  name: coredns-autoscaler
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: coredns-autoscaler
-subjects:
-- kind: ServiceAccount
-  name: coredns-autoscaler
-  namespace: kube-system
-
----
-
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: coredns.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: coredns.addons.k8s.io
-    k8s-app: coredns-autoscaler
-    kubernetes.io/cluster-service: "true"
-  name: coredns-autoscaler
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      k8s-app: coredns-autoscaler
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        k8s-app: coredns-autoscaler
-        kops.k8s.io/managed-by: kops
-    spec:
-      containers:
-      - command:
-        - /cluster-proportional-autoscaler
-        - --namespace=kube-system
-        - --configmap=coredns-autoscaler
-        - --target=Deployment/coredns
-        - --default-params={"linear":{"coresPerReplica":256,"nodesPerReplica":16,"preventSinglePointFailure":true}}
-        - --logtostderr=true
-        - --v=2
-        image: registry.k8s.io/cpa/cluster-proportional-autoscaler:v1.8.8
-        name: autoscaler
-        resources:
-          requests:
-            cpu: 20m
-            memory: 10Mi
-      nodeSelector:
-        kubernetes.io/os: linux
-      priorityClassName: system-cluster-critical
-      serviceAccountName: coredns-autoscaler
-      tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists

tests/integration/update_cluster/privateweave/data/aws_s3_object_privateweave.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content

@@ -1,138 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: dns-controller.addons.k8s.io
-    k8s-app: dns-controller
-    version: v1.27.0-beta.1
-  name: dns-controller
-  namespace: kube-system
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      k8s-app: dns-controller
-  strategy:
-    type: Recreate
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        k8s-addon: dns-controller.addons.k8s.io
-        k8s-app: dns-controller
-        kops.k8s.io/managed-by: kops
-        version: v1.27.0-beta.1
-    spec:
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-            - matchExpressions:
-              - key: node-role.kubernetes.io/control-plane
-                operator: Exists
-            - matchExpressions:
-              - key: node-role.kubernetes.io/master
-                operator: Exists
-      containers:
-      - args:
-        - --watch-ingress=false
-        - --dns=aws-route53
-        - --zone=*/Z1AFAKE1ZON3YO
-        - --internal-ipv4
-        - --zone=*/*
-        - -v=2
-        command: null
-        env:
-        - name: KUBERNETES_SERVICE_HOST
-          value: 127.0.0.1
-        image: registry.k8s.io/kops/dns-controller:1.27.0-beta.1
-        name: dns-controller
-        resources:
-          requests:
-            cpu: 50m
-            memory: 50Mi
-        securityContext:
-          runAsNonRoot: true
-      dnsPolicy: Default
-      hostNetwork: true
-      nodeSelector: null
-      priorityClassName: system-cluster-critical
-      serviceAccount: dns-controller
-      tolerations:
-      - key: node.cloudprovider.kubernetes.io/uninitialized
-        operator: Exists
-      - key: node.kubernetes.io/not-ready
-        operator: Exists
-      - key: node-role.kubernetes.io/control-plane
-        operator: Exists
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-
----
-
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: dns-controller.addons.k8s.io
-  name: dns-controller
-  namespace: kube-system
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: dns-controller.addons.k8s.io
-  name: kops:dns-controller
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - endpoints
-  - services
-  - pods
-  - ingress
-  - nodes
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - networking.k8s.io
-  resources:
-  - ingresses
-  verbs:
-  - get
-  - list
-  - watch
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: dns-controller.addons.k8s.io
-  name: kops:dns-controller
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: kops:dns-controller
-subjects:
-- apiGroup: rbac.authorization.k8s.io
-  kind: User
-  name: system:serviceaccount:kube-system:dns-controller

tests/integration/update_cluster/privateweave/data/aws_s3_object_privateweave.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content

@@ -1,225 +0,0 @@
-apiVersion: v1
-data:
-  config.yaml: |
-    {"clusterName":"privateweave.example.com","cloud":"aws","configBase":"memfs://clusters.example.com/privateweave.example.com","secretStore":"memfs://clusters.example.com/privateweave.example.com/secrets","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["nodes.privateweave.example.com"],"Region":"us-test-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"]}}
-kind: ConfigMap
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kops-controller.addons.k8s.io
-  name: kops-controller
-  namespace: kube-system
-
----
-
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kops-controller.addons.k8s.io
-    k8s-app: kops-controller
-    version: v1.27.0-beta.1
-  name: kops-controller
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      k8s-app: kops-controller
-  template:
-    metadata:
-      annotations:
-        dns.alpha.kubernetes.io/internal: kops-controller.internal.privateweave.example.com
-      creationTimestamp: null
-      labels:
-        k8s-addon: kops-controller.addons.k8s.io
-        k8s-app: kops-controller
-        kops.k8s.io/managed-by: kops
-        version: v1.27.0-beta.1
-    spec:
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-            - matchExpressions:
-              - key: node-role.kubernetes.io/control-plane
-                operator: Exists
-              - key: kops.k8s.io/kops-controller-pki
-                operator: Exists
-            - matchExpressions:
-              - key: node-role.kubernetes.io/master
-                operator: Exists
-              - key: kops.k8s.io/kops-controller-pki
-                operator: Exists
-      containers:
-      - args:
-        - --v=2
-        - --conf=/etc/kubernetes/kops-controller/config/config.yaml
-        command: null
-        env:
-        - name: KUBERNETES_SERVICE_HOST
-          value: 127.0.0.1
-        image: registry.k8s.io/kops/kops-controller:1.27.0-beta.1
-        name: kops-controller
-        resources:
-          requests:
-            cpu: 50m
-            memory: 50Mi
-        securityContext:
-          runAsNonRoot: true
-          runAsUser: 10011
-        volumeMounts:
-        - mountPath: /etc/kubernetes/kops-controller/config/
-          name: kops-controller-config
-        - mountPath: /etc/kubernetes/kops-controller/pki/
-          name: kops-controller-pki
-      dnsPolicy: Default
-      hostNetwork: true
-      nodeSelector: null
-      priorityClassName: system-cluster-critical
-      serviceAccount: kops-controller
-      tolerations:
-      - key: node.cloudprovider.kubernetes.io/uninitialized
-        operator: Exists
-      - key: node.kubernetes.io/not-ready
-        operator: Exists
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-      - key: node-role.kubernetes.io/control-plane
-        operator: Exists
-      volumes:
-      - configMap:
-          name: kops-controller
-        name: kops-controller-config
-      - hostPath:
-          path: /etc/kubernetes/kops-controller/
-          type: Directory
-        name: kops-controller-pki
-  updateStrategy:
-    type: OnDelete
-
----
-
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kops-controller.addons.k8s.io
-  name: kops-controller
-  namespace: kube-system
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kops-controller.addons.k8s.io
-  name: kops-controller
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - get
-  - list
-  - watch
-  - patch
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kops-controller.addons.k8s.io
-  name: kops-controller
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: kops-controller
-subjects:
-- apiGroup: rbac.authorization.k8s.io
-  kind: User
-  name: system:serviceaccount:kube-system:kops-controller
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kops-controller.addons.k8s.io
-  name: kops-controller
-  namespace: kube-system
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - events
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-- apiGroups:
-  - ""
-  - coordination.k8s.io
-  resourceNames:
-  - kops-controller-leader
-  resources:
-  - configmaps
-  - leases
-  verbs:
-  - get
-  - list
-  - watch
-  - patch
-  - update
-  - delete
-- apiGroups:
-  - ""
-  - coordination.k8s.io
-  resources:
-  - configmaps
-  - leases
-  verbs:
-  - create
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kops-controller.addons.k8s.io
-  name: kops-controller
-  namespace: kube-system
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: kops-controller
-subjects:
-- apiGroup: rbac.authorization.k8s.io
-  kind: User
-  name: system:serviceaccount:kube-system:kops-controller

tests/integration/update_cluster/privateweave/data/aws_s3_object_privateweave.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content

@@ -1,17 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: kubelet-api.rbac.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: kubelet-api.rbac.addons.k8s.io
-  name: kops:system:kubelet-api-admin
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kubelet-api-admin
-subjects:
-- apiGroup: rbac.authorization.k8s.io
-  kind: User
-  name: kubelet-api

tests/integration/update_cluster/privateweave/data/aws_s3_object_privateweave.example.com-addons-limit-range.addons.k8s.io_content

@@ -1,15 +0,0 @@
-apiVersion: v1
-kind: LimitRange
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: limit-range.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: limit-range.addons.k8s.io
-  name: limits
-  namespace: default
-spec:
-  limits:
-  - defaultRequest:
-      cpu: 100m
-    type: Container

tests/integration/update_cluster/privateweave/data/aws_s3_object_privateweave.example.com-addons-networking.weave-k8s-1.12_content

@@ -1,285 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.weave
-    app.kubernetes.io/managed-by: kops
-    name: weave-net
-    role.kubernetes.io/networking: "1"
-  name: weave-net
-  namespace: kube-system
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.weave
-    app.kubernetes.io/managed-by: kops
-    name: weave-net
-    role.kubernetes.io/networking: "1"
-  name: weave-net
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  - namespaces
-  - nodes
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - extensions
-  resources:
-  - networkpolicies
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - networking.k8s.io
-  resources:
-  - networkpolicies
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - nodes/status
-  verbs:
-  - patch
-  - update
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.weave
-    app.kubernetes.io/managed-by: kops
-    name: weave-net
-    role.kubernetes.io/networking: "1"
-  name: weave-net
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: weave-net
-subjects:
-- kind: ServiceAccount
-  name: weave-net
-  namespace: kube-system
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.weave
-    app.kubernetes.io/managed-by: kops
-    name: weave-net
-    role.kubernetes.io/networking: "1"
-  name: weave-net
-  namespace: kube-system
-rules:
-- apiGroups:
-  - ""
-  resourceNames:
-  - weave-net
-  resources:
-  - configmaps
-  verbs:
-  - get
-  - update
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - create
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.weave
-    app.kubernetes.io/managed-by: kops
-    name: weave-net
-    role.kubernetes.io/networking: "1"
-  name: weave-net
-  namespace: kube-system
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: weave-net
-subjects:
-- kind: ServiceAccount
-  name: weave-net
-  namespace: kube-system
-
----
-
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.weave
-    app.kubernetes.io/managed-by: kops
-    name: weave-net
-    role.kubernetes.io/networking: "1"
-  name: weave-net
-  namespace: kube-system
-spec:
-  minReadySeconds: 5
-  selector:
-    matchLabels:
-      name: weave-net
-  template:
-    metadata:
-      annotations:
-        prometheus.io/scrape: "true"
-      creationTimestamp: null
-      labels:
-        kops.k8s.io/managed-by: kops
-        name: weave-net
-    spec:
-      containers:
-      - command:
-        - /home/weave/launch.sh
-        env:
-        - name: INIT_CONTAINER
-          value: "true"
-        - name: HOSTNAME
-          valueFrom:
-            fieldRef:
-              apiVersion: v1
-              fieldPath: spec.nodeName
-        - name: IPALLOC_RANGE
-          value: 100.96.0.0/11
-        image: weaveworks/weave-kube:2.8.1
-        name: weave
-        ports:
-        - containerPort: 6782
-          name: metrics
-        readinessProbe:
-          httpGet:
-            host: 127.0.0.1
-            path: /status
-            port: 6784
-        resources:
-          limits:
-            memory: 200Mi
-          requests:
-            cpu: 50m
-            memory: 200Mi
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - mountPath: /weavedb
-          name: weavedb
-        - mountPath: /host/var/lib/dbus
-          name: dbus
-          readOnly: true
-        - mountPath: /host/etc/machine-id
-          name: cni-machine-id
-          readOnly: true
-        - mountPath: /run/xtables.lock
-          name: xtables-lock
-          readOnly: false
-      - env:
-        - name: HOSTNAME
-          valueFrom:
-            fieldRef:
-              apiVersion: v1
-              fieldPath: spec.nodeName
-        image: weaveworks/weave-npc:2.8.1
-        name: weave-npc
-        ports:
-        - containerPort: 6781
-          name: metrics
-        resources:
-          limits:
-            memory: 200Mi
-          requests:
-            cpu: 50m
-            memory: 200Mi
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - mountPath: /run/xtables.lock
-          name: xtables-lock
-          readOnly: false
-      dnsPolicy: ClusterFirstWithHostNet
-      hostNetwork: true
-      hostPID: false
-      initContainers:
-      - command:
-        - /home/weave/init.sh
-        image: weaveworks/weave-kube:2.8.1
-        name: weave-init
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - mountPath: /host/opt
-          name: cni-bin
-        - mountPath: /host/home
-          name: cni-bin2
-        - mountPath: /host/etc
-          name: cni-conf
-        - mountPath: /lib/modules
-          name: lib-modules
-        - mountPath: /run/xtables.lock
-          name: xtables-lock
-          readOnly: false
-      priorityClassName: system-node-critical
-      restartPolicy: Always
-      securityContext:
-        seLinuxOptions: {}
-      serviceAccountName: weave-net
-      tolerations:
-      - effect: NoSchedule
-        operator: Exists
-      - effect: NoExecute
-        operator: Exists
-      volumes:
-      - hostPath:
-          path: /var/lib/weave
-        name: weavedb
-      - hostPath:
-          path: /opt
-        name: cni-bin
-      - hostPath:
-          path: /home
-        name: cni-bin2
-      - hostPath:
-          path: /etc
-        name: cni-conf
-      - hostPath:
-          path: /etc/machine-id
-        name: cni-machine-id
-      - hostPath:
-          path: /var/lib/dbus
-        name: dbus
-      - hostPath:
-          path: /lib/modules
-        name: lib-modules
-      - hostPath:
-          path: /run/xtables.lock
-          type: FileOrCreate
-        name: xtables-lock
-  updateStrategy:
-    type: RollingUpdate

tests/integration/update_cluster/privateweave/data/aws_s3_object_privateweave.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content

@@ -1,118 +0,0 @@
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: storage-aws.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: storage-aws.addons.k8s.io
-  name: default
-parameters:
-  type: gp2
-provisioner: kubernetes.io/aws-ebs
-
----
-
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  annotations:
-    storageclass.kubernetes.io/is-default-class: "false"
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: storage-aws.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: storage-aws.addons.k8s.io
-  name: gp2
-parameters:
-  type: gp2
-provisioner: kubernetes.io/aws-ebs
-
----
-
-allowVolumeExpansion: true
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  annotations:
-    storageclass.kubernetes.io/is-default-class: "false"
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: storage-aws.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: storage-aws.addons.k8s.io
-  name: kops-ssd-1-17
-parameters:
-  encrypted: "true"
-  type: gp2
-provisioner: kubernetes.io/aws-ebs
-volumeBindingMode: WaitForFirstConsumer
-
----
-
-allowVolumeExpansion: true
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  annotations:
-    storageclass.kubernetes.io/is-default-class: "true"
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: storage-aws.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: storage-aws.addons.k8s.io
-  name: kops-csi-1-21
-parameters:
-  encrypted: "true"
-  type: gp3
-provisioner: ebs.csi.aws.com
-volumeBindingMode: WaitForFirstConsumer
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: storage-aws.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: storage-aws.addons.k8s.io
-  name: system:aws-cloud-provider
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - get
-  - list
-  - patch
-- apiGroups:
-  - ""
-  resources:
-  - events
-  verbs:
-  - create
-  - patch
-  - update
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: storage-aws.addons.k8s.io
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: storage-aws.addons.k8s.io
-  name: system:aws-cloud-provider
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:aws-cloud-provider
-subjects:
-- kind: ServiceAccount
-  name: aws-cloud-provider
-  namespace: kube-system

tests/integration/update_cluster/privateweave/id_rsa.pub

@@ -1 +0,0 @@
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==

tests/integration/update_cluster/privateweave/in-v1alpha2.yaml

@@ -1,101 +0,0 @@
-apiVersion: kops.k8s.io/v1alpha2
-kind: Cluster
-metadata:
-  creationTimestamp: "2016-12-12T04:13:14Z"
-  name: privateweave.example.com
-spec:
-  kubernetesApiAccess:
-  - 0.0.0.0/0
-  channel: stable
-  cloudProvider: aws
-  configBase: memfs://clusters.example.com/privateweave.example.com
-  etcdClusters:
-  - etcdMembers:
-    - instanceGroup: master-us-test-1a
-      name: us-test-1a
-    name: main
-  - etcdMembers:
-    - instanceGroup: master-us-test-1a
-      name: us-test-1a
-    name: events
-  iam: {}
-  kubelet:
-    anonymousAuth: false
-  kubernetesVersion: v1.22.0
-  masterPublicName: api.privateweave.example.com
-  networkCIDR: 172.20.0.0/16
-  networking:
-    weave: {}
-  nonMasqueradeCIDR: 100.64.0.0/10
-  sshAccess:
-  - 0.0.0.0/0
-  topology:
-    masters: private
-    nodes: private
-  subnets:
-  - cidr: 172.20.32.0/19
-    name: us-test-1a
-    type: Private
-    zone: us-test-1a
-  - cidr: 172.20.4.0/22
-    name: utility-us-test-1a
-    type: Utility
-    zone: us-test-1a
-
----
-
-apiVersion: kops.k8s.io/v1alpha2
-kind: InstanceGroup
-metadata:
-  creationTimestamp: "2016-12-12T04:13:15Z"
-  name: master-us-test-1a
-  labels:
-    kops.k8s.io/cluster: privateweave.example.com
-spec:
-  associatePublicIp: true
-  image: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220404
-  machineType: m3.medium
-  maxSize: 1
-  minSize: 1
-  role: Master
-  subnets:
-  - us-test-1a
-
----
-
-apiVersion: kops.k8s.io/v1alpha2
-kind: InstanceGroup
-metadata:
-  creationTimestamp: "2016-12-12T04:13:15Z"
-  name: nodes
-  labels:
-    kops.k8s.io/cluster: privateweave.example.com
-spec:
-  associatePublicIp: true
-  image: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220404
-  machineType: t2.medium
-  maxSize: 2
-  minSize: 2
-  role: Node
-  subnets:
-  - us-test-1a
-
-
----
-
-apiVersion: kops.k8s.io/v1alpha2
-kind: InstanceGroup
-metadata:
-  creationTimestamp: "2016-12-14T15:32:41Z"
-  name: bastion
-  labels:
-    kops.k8s.io/cluster: privateweave.example.com
-spec:
-  associatePublicIp: true
-  image: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220404
-  machineType: t2.micro
-  maxSize: 1
-  minSize: 1
-  role: Bastion
-  subnets:
-  - utility-us-test-1a

upup/pkg/fi/cloudup/apply_cluster.go

@@ -81,9 +81,9 @@ const (
 	starline = "*********************************************************************************"
 
 	// OldestSupportedKubernetesVersion is the oldest kubernetes version that is supported in kOps.
-	OldestSupportedKubernetesVersion = "1.22.0"
+	OldestSupportedKubernetesVersion = "1.23.0"
 	// OldestRecommendedKubernetesVersion is the oldest kubernetes version that is not deprecated in kOps.
-	OldestRecommendedKubernetesVersion = "1.24.0"
+	OldestRecommendedKubernetesVersion = "1.27.0"
 )
 
 // TerraformCloudProviders is the list of cloud providers with terraform target support

upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go

@@ -455,7 +455,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext)
 		}
 	}
 
-	if b.IsKubernetesGTE("1.23") && b.IsKubernetesLT("1.26") &&
+	if b.IsKubernetesLT("1.26") &&
 		(b.Cluster.Spec.GetCloudProvider() == kops.CloudProviderAWS ||
 			b.Cluster.Spec.GetCloudProvider() == kops.CloudProviderGCE) {
 		// AWS and GCE KCM-to-CCM leader migration

upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go

@@ -48,7 +48,6 @@ func TestBootstrapChannelBuilder_BuildTasks(t *testing.T) {
 	runChannelBuilderTest(t, "simple", []string{"kops-controller.addons.k8s.io-k8s-1.16"})
 	// Use cilium networking, proxy
 	runChannelBuilderTest(t, "cilium", []string{"kops-controller.addons.k8s.io-k8s-1.16"})
-	runChannelBuilderTest(t, "weave", []string{})
 	runChannelBuilderTest(t, "amazonvpc", []string{"networking.amazon-vpc-routed-eni-k8s-1.16"})
 	runChannelBuilderTest(t, "amazonvpc-containerd", []string{"networking.amazon-vpc-routed-eni-k8s-1.16"})
 	runChannelBuilderTest(t, "awsiamauthenticator/crd", []string{"authentication.aws-k8s-1.12"})

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/cluster.yaml

@@ -21,7 +21,7 @@ spec:
       name: master-us-test-1a
     name: events
   iam: {}
-  kubernetesVersion: 1.22.0
+  kubernetesVersion: 1.27.0
   masterPublicName: api.minimal.example.com
   additionalSans:
   - proxy.api.minimal.example.com

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 data:
   config.yaml: |
-    {"clusterName":"minimal.example.com","cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com","secretStore":"memfs://clusters.example.com/minimal.example.com/secrets","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["kops-custom-node-role","nodes.minimal.example.com"],"Region":"us-east-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"]}}
+    {"clusterName":"minimal.example.com","cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com","secretStore":"memfs://clusters.example.com/minimal.example.com/secrets","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["kops-custom-node-role","nodes.minimal.example.com"],"Region":"us-east-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"],"useInstanceIDForNodeName":true}}
 kind: ConfigMap
 metadata:
   creationTimestamp: null

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml

@@ -6,7 +6,7 @@ spec:
   addons:
   - id: k8s-1.16
     manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: 2c44b103e72bfe14fdcaac5ef4f98895a32509a34e0b54dd5b9e941761978ee4
+    manifestHash: e0b7bb1c7846d06cd678aa05bbb3f135ca4c1f1409c60be31bd59e85396a1ed0
     name: kops-controller.addons.k8s.io
     needsRollingUpdate: control-plane
     selector:
@@ -14,7 +14,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 49273720e8c046ae8b72a3fe666d670f6efb5e26111dc521027218b1e8e1de51
+    manifestHash: d2bbb7cbee5835c3891fe80fbacf8963508359ef9159f8480325ce9a7174f14a
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io
@@ -48,15 +48,22 @@ spec:
     version: 9.99.0
   - id: k8s-1.16
     manifest: networking.cilium.io/k8s-1.16-v1.12.yaml
-    manifestHash: 2045965a451579b2a01239022b29fe8e47c01659a11e2e1ebb951e6c0fd7ccbc
+    manifestHash: 4e82169ed7f2247b5347427539cba5ea4140120b716e4c28cbe59dc28fd20d16
     name: networking.cilium.io
     needsRollingUpdate: all
     selector:
       role.kubernetes.io/networking: "1"
     version: 9.99.0
+  - id: k8s-1.18
+    manifest: aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml
+    manifestHash: 557d71c430bb05a5b069fd8dc3a0a3247261795bfd0617b97cbf1f31fed3fc27
+    name: aws-cloud-controller.addons.k8s.io
+    selector:
+      k8s-addon: aws-cloud-controller.addons.k8s.io
+    version: 9.99.0
   - id: k8s-1.17
     manifest: aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml
-    manifestHash: 80a04c96830e1279702d4cdf8004416edc2020f7ada484e5213693962c0ade91
+    manifestHash: 9ebe176a18822b64f30849e1b29a147a73e49bb0c445c78cba85703ea3a3221f
     name: aws-ebs-csi-driver.addons.k8s.io
     selector:
       k8s-addon: aws-ebs-csi-driver.addons.k8s.io

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/insecure-1.19/manifest.yaml

@@ -26,6 +26,13 @@ spec:
     selector:
       k8s-addon: kubelet-api.rbac.addons.k8s.io
     version: 9.99.0
+  - id: k8s-1.23
+    manifest: leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml
+    manifestHash: b9c91e09c0f28c9b74ff140b8395d611834c627d698846d625c10975a74a48c4
+    name: leader-migration.rbac.addons.k8s.io
+    selector:
+      k8s-addon: leader-migration.rbac.addons.k8s.io
+    version: 9.99.0
   - manifest: limit-range.addons.k8s.io/v1.5.0.yaml
     manifestHash: 2d55c3bc5e354e84a3730a65b42f39aba630a59dc8d32b30859fcce3d3178bc2
     name: limit-range.addons.k8s.io

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/metrics-server/secure-1.19/manifest.yaml

@@ -26,6 +26,13 @@ spec:
     selector:
       k8s-addon: kubelet-api.rbac.addons.k8s.io
     version: 9.99.0
+  - id: k8s-1.23
+    manifest: leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml
+    manifestHash: b9c91e09c0f28c9b74ff140b8395d611834c627d698846d625c10975a74a48c4
+    name: leader-migration.rbac.addons.k8s.io
+    selector:
+      k8s-addon: leader-migration.rbac.addons.k8s.io
+    version: 9.99.0
   - manifest: limit-range.addons.k8s.io/v1.5.0.yaml
     manifestHash: 2d55c3bc5e354e84a3730a65b42f39aba630a59dc8d32b30859fcce3d3178bc2
     name: limit-range.addons.k8s.io

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/service-account-iam/cluster.yaml

@@ -22,7 +22,7 @@ spec:
     name: events
   iam:
     useServiceAccountExternalPermissions: true
-  kubernetesVersion: v1.22.6
+  kubernetesVersion: v1.27.0
   masterPublicName: api.minimal.example.com
   additionalSans:
   - proxy.api.minimal.example.com

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/service-account-iam/kops-controller.addons.k8s.io-k8s-1.16.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 data:
   config.yaml: |
-    {"clusterName":"minimal.example.com","cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com","secretStore":"memfs://clusters.example.com/minimal.example.com/secrets","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["kops-custom-node-role","nodes.minimal.example.com"],"Region":"us-east-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"]}}
+    {"clusterName":"minimal.example.com","cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com","secretStore":"memfs://clusters.example.com/minimal.example.com/secrets","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["kops-custom-node-role","nodes.minimal.example.com"],"Region":"us-east-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"],"useInstanceIDForNodeName":true}}
 kind: ConfigMap
 metadata:
   creationTimestamp: null

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/service-account-iam/manifest.yaml

@@ -6,7 +6,7 @@ spec:
   addons:
   - id: k8s-1.16
     manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: 2c44b103e72bfe14fdcaac5ef4f98895a32509a34e0b54dd5b9e941761978ee4
+    manifestHash: e0b7bb1c7846d06cd678aa05bbb3f135ca4c1f1409c60be31bd59e85396a1ed0
     name: kops-controller.addons.k8s.io
     needsRollingUpdate: control-plane
     selector:
@@ -14,7 +14,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 49273720e8c046ae8b72a3fe666d670f6efb5e26111dc521027218b1e8e1de51
+    manifestHash: d2bbb7cbee5835c3891fe80fbacf8963508359ef9159f8480325ce9a7174f14a
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io
@@ -46,9 +46,16 @@ spec:
     selector:
       k8s-addon: storage-aws.addons.k8s.io
     version: 9.99.0
+  - id: k8s-1.18
+    manifest: aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml
+    manifestHash: 304c8eb52f04d8a87d9001a8cc7759bf4a15e27b9687d4163285c5467b759ee5
+    name: aws-cloud-controller.addons.k8s.io
+    selector:
+      k8s-addon: aws-cloud-controller.addons.k8s.io
+    version: 9.99.0
   - id: k8s-1.17
     manifest: aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml
-    manifestHash: 9d4fdbc3b961de3787b596fd84f57b257013aa2ddd7d83020e26b6c390f99708
+    manifestHash: f4378373dd1f6893c91fda5196e03f8b95b610ea68a896afa0caa522b1f96a1e
     name: aws-ebs-csi-driver.addons.k8s.io
     selector:
       k8s-addon: aws-ebs-csi-driver.addons.k8s.io

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/cluster.yaml

@@ -1,47 +0,0 @@
-apiVersion: kops.k8s.io/v1alpha2
-kind: Cluster
-metadata:
-  creationTimestamp: "2016-12-10T22:42:27Z"
-  name: minimal.example.com
-spec:
-  kubernetesApiAccess:
-  - 0.0.0.0/0
-  channel: stable
-  cloudProvider: aws
-  configBase: memfs://clusters.example.com/minimal.example.com
-  etcdClusters:
-  - etcdMembers:
-    - instanceGroup: master-us-test-1a
-      name: master-us-test-1a
-    name: main
-  - etcdMembers:
-    - instanceGroup: master-us-test-1a
-      name: master-us-test-1a
-    name: events
-  iam: {}
-  kubernetesVersion: v1.22.0
-  masterPublicName: api.minimal.example.com
-  networkCIDR: 172.20.0.0/16
-  networking:
-    weave:
-      memoryRequest: 300Mi
-      cpuRequest: 100m
-      memoryLimit: 300Mi
-      cpuLimit: 200m
-      netExtraArgs: --log-level=info
-      npcMemoryRequest: 300Mi
-      npcCPURequest: 100m
-      npcMemoryLimit: 300Mi
-      npcCPULimit: 200m
-      npcExtraArgs: --log-level=info
-  nonMasqueradeCIDR: 100.64.0.0/10
-  sshAccess:
-    - 0.0.0.0/0
-  topology:
-    masters: public
-    nodes: public
-  subnets:
-  - cidr: 172.20.32.0/19
-    name: us-test-1a
-    type: Public
-    zone: us-test-1a

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml

@@ -1,62 +0,0 @@
-kind: Addons
-metadata:
-  creationTimestamp: null
-  name: bootstrap
-spec:
-  addons:
-  - id: k8s-1.16
-    manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: 2c44b103e72bfe14fdcaac5ef4f98895a32509a34e0b54dd5b9e941761978ee4
-    name: kops-controller.addons.k8s.io
-    needsRollingUpdate: control-plane
-    selector:
-      k8s-addon: kops-controller.addons.k8s.io
-    version: 9.99.0
-  - id: k8s-1.12
-    manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 49273720e8c046ae8b72a3fe666d670f6efb5e26111dc521027218b1e8e1de51
-    name: coredns.addons.k8s.io
-    selector:
-      k8s-addon: coredns.addons.k8s.io
-    version: 9.99.0
-  - id: k8s-1.9
-    manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml
-    manifestHash: 01c120e887bd98d82ef57983ad58a0b22bc85efb48108092a24c4b82e4c9ea81
-    name: kubelet-api.rbac.addons.k8s.io
-    selector:
-      k8s-addon: kubelet-api.rbac.addons.k8s.io
-    version: 9.99.0
-  - manifest: limit-range.addons.k8s.io/v1.5.0.yaml
-    manifestHash: 2d55c3bc5e354e84a3730a65b42f39aba630a59dc8d32b30859fcce3d3178bc2
-    name: limit-range.addons.k8s.io
-    selector:
-      k8s-addon: limit-range.addons.k8s.io
-    version: 9.99.0
-  - id: k8s-1.12
-    manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: c648d0743139214b71a6454f4d0712d5d3006039522661560d16e839d70820a6
-    name: dns-controller.addons.k8s.io
-    selector:
-      k8s-addon: dns-controller.addons.k8s.io
-    version: 9.99.0
-  - id: v1.15.0
-    manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
-    manifestHash: 4e2cda50cd5048133aad1b5e28becb60f4629d3f9e09c514a2757c27998b4200
-    name: storage-aws.addons.k8s.io
-    selector:
-      k8s-addon: storage-aws.addons.k8s.io
-    version: 9.99.0
-  - id: k8s-1.12
-    manifest: networking.weave/k8s-1.12.yaml
-    manifestHash: 88a53d6a9d91f7515d7d369200e3773db2244222bbb964a5119611b45a6db1d6
-    name: networking.weave
-    selector:
-      role.kubernetes.io/networking: "1"
-    version: 9.99.0
-  - id: k8s-1.17
-    manifest: aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml
-    manifestHash: 80a04c96830e1279702d4cdf8004416edc2020f7ada484e5213693962c0ade91
-    name: aws-ebs-csi-driver.addons.k8s.io
-    selector:
-      k8s-addon: aws-ebs-csi-driver.addons.k8s.io
-    version: 9.99.0