Merge pull request #12584 from rifelpet/canal320

Upgrade Canal to v3.20 with k8s 1.22 support

cmd/kops/integration_test.go

@@ -377,7 +377,7 @@ func TestPrivateCiliumAdvanced(t *testing.T) {
 func TestPrivateCanal(t *testing.T) {
 	newIntegrationTest("privatecanal.example.com", "privatecanal").
 		withPrivate().
-		withAddons("networking.projectcalico.org.canal-k8s-1.16", dnsControllerAddon).
+		withAddons("networking.projectcalico.org.canal-k8s-1.22", "aws-ebs-csi-driver.addons.k8s.io-k8s-1.17", dnsControllerAddon).
 		runTestTerraformAWS(t)
 }
 

pkg/apis/kops/validation/validation.go

@@ -815,10 +815,6 @@ func validateNetworkingCanal(c *kops.Cluster, v *kops.CanalNetworkingSpec, fldPa
 		allErrs = append(allErrs, IsValidValue(fldPath.Child("iptablesBackend"), &v.IptablesBackend, valid)...)
 	}
 
-	if c.IsKubernetesGTE("1.22") {
-		allErrs = append(allErrs, field.Forbidden(fldPath, "Canal is supported only for Kubernetes 1.21 and lower"))
-	}
-
 	return allErrs
 }
 

tests/integration/update_cluster/privatecanal/data/aws_launch_template_master-us-test-1a.masters.privatecanal.example.com_user_data

@@ -125,7 +125,8 @@ ensure-install-dir
 cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
 cloudConfig:
   awsEBSCSIDriver:
-    enabled: false
+    enabled: true
+    version: v1.4.0
   manageStorageClasses: true
 containerRuntime: containerd
 containerd:
@@ -136,9 +137,9 @@ docker:
 encryptionConfig: null
 etcdClusters:
   events:
-    version: 3.4.13
+    version: 3.5.0
   main:
-    version: 3.4.13
+    version: 3.5.0
 kubeAPIServer:
   allowPrivileged: true
   anonymousAuth: false
@@ -162,7 +163,7 @@ kubeAPIServer:
   - https://127.0.0.1:4001
   etcdServersOverrides:
   - /events#https://127.0.0.1:4002
-  image: k8s.gcr.io/kube-apiserver:v1.21.0
+  image: k8s.gcr.io/kube-apiserver:v1.22.0
   kubeletPreferredAddressTypes:
   - InternalIP
   - Hostname
@@ -188,7 +189,9 @@ kubeControllerManager:
   clusterCIDR: 100.96.0.0/11
   clusterName: privatecanal.example.com
   configureCloudRoutes: false
-  image: k8s.gcr.io/kube-controller-manager:v1.21.0
+  featureGates:
+    CSIMigrationAWS: "true"
+  image: k8s.gcr.io/kube-controller-manager:v1.22.0
   leaderElection:
     leaderElect: true
   logLevel: 2
@@ -197,10 +200,10 @@ kubeProxy:
   clusterCIDR: 100.96.0.0/11
   cpuRequest: 100m
   hostnameOverride: '@aws'
-  image: k8s.gcr.io/kube-proxy:v1.21.0
+  image: k8s.gcr.io/kube-proxy:v1.22.0
   logLevel: 2
 kubeScheduler:
-  image: k8s.gcr.io/kube-scheduler:v1.21.0
+  image: k8s.gcr.io/kube-scheduler:v1.22.0
   leaderElection:
     leaderElect: true
   logLevel: 2
@@ -213,12 +216,16 @@ kubelet:
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   hostnameOverride: '@aws'
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
   networkPluginName: cni
   nonMasqueradeCIDR: 100.64.0.0/10
   podManifestPath: /etc/kubernetes/manifests
+  protectKernelDefaults: true
 masterKubelet:
   anonymousAuth: false
   cgroupDriver: systemd
@@ -228,12 +235,16 @@ masterKubelet:
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   hostnameOverride: '@aws'
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
   networkPluginName: cni
   nonMasqueradeCIDR: 100.64.0.0/10
   podManifestPath: /etc/kubernetes/manifests
+  protectKernelDefaults: true
   registerSchedulable: false
 
 __EOF_CLUSTER_SPEC
@@ -243,7 +254,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/privatecanal.example.com
 InstanceGroupName: master-us-test-1a
 InstanceGroupRole: Master
-NodeupConfigHash: KYvsy5BReL0Xr64Y49pvZchubc2wvbhQhVQHvRI5Exw=
+NodeupConfigHash: cDWy7b28g2HeI4L2fYq3N4v96GeAjsauylqNFQvNTEI=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/privatecanal/data/aws_launch_template_nodes.privatecanal.example.com_user_data

@@ -125,7 +125,8 @@ ensure-install-dir
 cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
 cloudConfig:
   awsEBSCSIDriver:
-    enabled: false
+    enabled: true
+    version: v1.4.0
   manageStorageClasses: true
 containerRuntime: containerd
 containerd:
@@ -137,7 +138,7 @@ kubeProxy:
   clusterCIDR: 100.96.0.0/11
   cpuRequest: 100m
   hostnameOverride: '@aws'
-  image: k8s.gcr.io/kube-proxy:v1.21.0
+  image: k8s.gcr.io/kube-proxy:v1.22.0
   logLevel: 2
 kubelet:
   anonymousAuth: false
@@ -148,12 +149,16 @@ kubelet:
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   hostnameOverride: '@aws'
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
   networkPluginName: cni
   nonMasqueradeCIDR: 100.64.0.0/10
   podManifestPath: /etc/kubernetes/manifests
+  protectKernelDefaults: true
 
 __EOF_CLUSTER_SPEC
 
@@ -162,7 +167,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/privatecanal.example.com
 InstanceGroupName: nodes
 InstanceGroupRole: Node
-NodeupConfigHash: GbTQ9y60njLR8mZO898FmHzRH7QpWPe3zvOE4j5T0UE=
+NodeupConfigHash: yaHIJa6pnpGnjNVhmc6Q+KUAQEfb2aZQnSNqwggjxms=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/privatecanal/data/aws_s3_bucket_object_cluster-completed.spec_content

@@ -13,7 +13,8 @@ spec:
   channel: stable
   cloudConfig:
     awsEBSCSIDriver:
-      enabled: false
+      enabled: true
+      version: v1.4.0
     manageStorageClasses: true
   cloudProvider: aws
   clusterDNSDomain: cluster.local
@@ -33,14 +34,14 @@ spec:
     - instanceGroup: master-us-test-1a
       name: us-test-1a
     name: main
-    version: 3.4.13
+    version: 3.5.0
   - backups:
       backupStore: memfs://clusters.example.com/privatecanal.example.com/backups/etcd/events
     etcdMembers:
     - instanceGroup: master-us-test-1a
       name: us-test-1a
     name: events
-    version: 3.4.13
+    version: 3.5.0
   externalDns:
     provider: dns-controller
   iam:
@@ -69,7 +70,7 @@ spec:
     - https://127.0.0.1:4001
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
-    image: k8s.gcr.io/kube-apiserver:v1.21.0
+    image: k8s.gcr.io/kube-apiserver:v1.22.0
     kubeletPreferredAddressTypes:
     - InternalIP
     - Hostname
@@ -95,7 +96,9 @@ spec:
     clusterCIDR: 100.96.0.0/11
     clusterName: privatecanal.example.com
     configureCloudRoutes: false
-    image: k8s.gcr.io/kube-controller-manager:v1.21.0
+    featureGates:
+      CSIMigrationAWS: "true"
+    image: k8s.gcr.io/kube-controller-manager:v1.22.0
     leaderElection:
       leaderElect: true
     logLevel: 2
@@ -119,10 +122,10 @@ spec:
     clusterCIDR: 100.96.0.0/11
     cpuRequest: 100m
     hostnameOverride: '@aws'
-    image: k8s.gcr.io/kube-proxy:v1.21.0
+    image: k8s.gcr.io/kube-proxy:v1.22.0
     logLevel: 2
   kubeScheduler:
-    image: k8s.gcr.io/kube-scheduler:v1.21.0
+    image: k8s.gcr.io/kube-scheduler:v1.22.0
     leaderElection:
       leaderElect: true
     logLevel: 2
@@ -135,15 +138,19 @@ spec:
     clusterDomain: cluster.local
     enableDebuggingHandlers: true
     evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     hostnameOverride: '@aws'
     kubeconfigPath: /var/lib/kubelet/kubeconfig
     logLevel: 2
     networkPluginName: cni
     nonMasqueradeCIDR: 100.64.0.0/10
     podManifestPath: /etc/kubernetes/manifests
+    protectKernelDefaults: true
   kubernetesApiAccess:
   - 0.0.0.0/0
-  kubernetesVersion: 1.21.0
+  kubernetesVersion: 1.22.0
   masterInternalName: api.internal.privatecanal.example.com
   masterKubelet:
     anonymousAuth: false
@@ -154,12 +161,16 @@ spec:
     clusterDomain: cluster.local
     enableDebuggingHandlers: true
     evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    featureGates:
+      CSIMigrationAWS: "true"
+      InTreePluginAWSUnregister: "true"
     hostnameOverride: '@aws'
     kubeconfigPath: /var/lib/kubelet/kubeconfig
     logLevel: 2
     networkPluginName: cni
     nonMasqueradeCIDR: 100.64.0.0/10
     podManifestPath: /etc/kubernetes/manifests
+    protectKernelDefaults: true
     registerSchedulable: false
   masterPublicName: api.privatecanal.example.com
   networkCIDR: 172.20.0.0/16

tests/integration/update_cluster/privatecanal/data/aws_s3_bucket_object_etcd-cluster-spec-events_content

@@ -1,4 +1,4 @@
 {
   "memberCount": 1,
-  "etcdVersion": "3.4.13"
+  "etcdVersion": "3.5.0"
 }

tests/integration/update_cluster/privatecanal/data/aws_s3_bucket_object_etcd-cluster-spec-main_content

@@ -1,4 +1,4 @@
 {
   "memberCount": 1,
-  "etcdVersion": "3.4.13"
+  "etcdVersion": "3.5.0"
 }

tests/integration/update_cluster/privatecanal/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content

@@ -22,7 +22,7 @@ APIServerConfig:
     - https://127.0.0.1:4001
     etcdServersOverrides:
     - /events#https://127.0.0.1:4002
-    image: k8s.gcr.io/kube-apiserver:v1.21.0
+    image: k8s.gcr.io/kube-apiserver:v1.22.0
     kubeletPreferredAddressTypes:
     - InternalIP
     - Hostname
@@ -52,16 +52,16 @@ APIServerConfig:
     -----END RSA PUBLIC KEY-----
 Assets:
   amd64:
-  - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet
+  - fec5c596f7f815f17f5d7d955e9707df1ef02a2ca5e788b223651f83376feb7f@https://storage.googleapis.com/kubernetes-release/release/v1.22.0/bin/linux/amd64/kubelet
-  - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl
+  - 703e70d49b82271535bc66bc7bd469a58c11d47f188889bd37101c9772f14fa1@https://storage.googleapis.com/kubernetes-release/release/v1.22.0/bin/linux/amd64/kubectl
-  - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
+  - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
   - a4a4af4776316833cad5996c66d59f8b4a2af4da716b7902b7a2d5f5ac362dcc@https://github.com/containerd/containerd/releases/download/v1.4.11/cri-containerd-cni-1.4.11-linux-amd64.tar.gz
   - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64
   - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64
   arm64:
-  - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet
+  - cea637a7da4f1097b16b0195005351c07032a820a3d64c3ff326b9097cfac930@https://storage.googleapis.com/kubernetes-release/release/v1.22.0/bin/linux/arm64/kubelet
-  - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl
+  - 8d9cc92dcc942f5ea2b2fc93c4934875d9e0e8ddecbde24c7d4c4e092cfc7afc@https://storage.googleapis.com/kubernetes-release/release/v1.22.0/bin/linux/arm64/kubectl
-  - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
+  - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
   - 0259f8b6572f02cf0dafd7388ca0e4adfdbbfaba81cfb1b7443e89fccbed22c7@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.9.tgz
   - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64
   - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64
@@ -239,6 +239,9 @@ KubeletConfig:
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   hostnameOverride: '@aws'
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
@@ -251,6 +254,7 @@ KubeletConfig:
     node.kubernetes.io/exclude-from-external-load-balancers: ""
   nonMasqueradeCIDR: 100.64.0.0/10
   podManifestPath: /etc/kubernetes/manifests
+  protectKernelDefaults: true
   registerSchedulable: false
 UpdatePolicy: automatic
 channels:

tests/integration/update_cluster/privatecanal/data/aws_s3_bucket_object_nodeupconfig-nodes_content

@@ -1,13 +1,13 @@
 Assets:
   amd64:
-  - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet
+  - fec5c596f7f815f17f5d7d955e9707df1ef02a2ca5e788b223651f83376feb7f@https://storage.googleapis.com/kubernetes-release/release/v1.22.0/bin/linux/amd64/kubelet
-  - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl
+  - 703e70d49b82271535bc66bc7bd469a58c11d47f188889bd37101c9772f14fa1@https://storage.googleapis.com/kubernetes-release/release/v1.22.0/bin/linux/amd64/kubectl
-  - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
+  - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
   - a4a4af4776316833cad5996c66d59f8b4a2af4da716b7902b7a2d5f5ac362dcc@https://github.com/containerd/containerd/releases/download/v1.4.11/cri-containerd-cni-1.4.11-linux-amd64.tar.gz
   arm64:
-  - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet
+  - cea637a7da4f1097b16b0195005351c07032a820a3d64c3ff326b9097cfac930@https://storage.googleapis.com/kubernetes-release/release/v1.22.0/bin/linux/arm64/kubelet
-  - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl
+  - 8d9cc92dcc942f5ea2b2fc93c4934875d9e0e8ddecbde24c7d4c4e092cfc7afc@https://storage.googleapis.com/kubernetes-release/release/v1.22.0/bin/linux/arm64/kubectl
-  - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
+  - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
   - 0259f8b6572f02cf0dafd7388ca0e4adfdbbfaba81cfb1b7443e89fccbed22c7@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.9.tgz
 CAs:
   kubernetes-ca: |
@@ -46,6 +46,9 @@ KubeletConfig:
   clusterDomain: cluster.local
   enableDebuggingHandlers: true
   evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+  featureGates:
+    CSIMigrationAWS: "true"
+    InTreePluginAWSUnregister: "true"
   hostnameOverride: '@aws'
   kubeconfigPath: /var/lib/kubelet/kubeconfig
   logLevel: 2
@@ -55,6 +58,7 @@ KubeletConfig:
     node-role.kubernetes.io/node: ""
   nonMasqueradeCIDR: 100.64.0.0/10
   podManifestPath: /etc/kubernetes/manifests
+  protectKernelDefaults: true
 UpdatePolicy: automatic
 channels:
 - memfs://clusters.example.com/privatecanal.example.com/addons/bootstrap-channel.yaml

tests/integration/update_cluster/privatecanal/data/aws_s3_bucket_object_privatecanal.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content

@@ -0,0 +1,770 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.4.0
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.4.0
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-external-attacher-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - csi.storage.k8s.io
+  resources:
+  - csinodeinfos
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments/status
+  verbs:
+  - patch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.4.0
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-external-provisioner-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - storageclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - list
+  - watch
+  - create
+  - update
+  - patch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshots
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - csinodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - watch
+  - list
+  - delete
+  - update
+  - create
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - volumeattachments
+  verbs:
+  - get
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.4.0
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-external-resizer-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims/status
+  verbs:
+  - update
+  - patch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - storageclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - list
+  - watch
+  - create
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.4.0
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-external-snapshotter-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - list
+  - watch
+  - create
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+  - delete
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents/status
+  verbs:
+  - update
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.4.0
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-attacher-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ebs-external-attacher-role
+subjects:
+- kind: ServiceAccount
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.4.0
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-provisioner-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ebs-external-provisioner-role
+subjects:
+- kind: ServiceAccount
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.4.0
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-resizer-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ebs-external-resizer-role
+subjects:
+- kind: ServiceAccount
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.4.0
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-snapshotter-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ebs-external-snapshotter-role
+subjects:
+- kind: ServiceAccount
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-node-getter-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ebs-csi-node-role
+subjects:
+- kind: ServiceAccount
+  name: ebs-csi-node-sa
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-node-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.4.0
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-node-sa
+  namespace: kube-system
+
+---
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.4.0
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-node
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: ebs-csi-node
+      app.kubernetes.io/instance: aws-ebs-csi-driver
+      app.kubernetes.io/name: aws-ebs-csi-driver
+  template:
+    metadata:
+      labels:
+        app: ebs-csi-node
+        app.kubernetes.io/instance: aws-ebs-csi-driver
+        app.kubernetes.io/name: aws-ebs-csi-driver
+        app.kubernetes.io/version: v1.4.0
+    spec:
+      containers:
+      - args:
+        - node
+        - --endpoint=$(CSI_ENDPOINT)
+        - --logtostderr
+        - --v=2
+        env:
+        - name: CSI_ENDPOINT
+          value: unix:/csi/csi.sock
+        - name: CSI_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        image: k8s.gcr.io/provider-aws/aws-ebs-csi-driver:v1.4.0
+        livenessProbe:
+          failureThreshold: 5
+          httpGet:
+            path: /healthz
+            port: healthz
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 3
+        name: ebs-plugin
+        ports:
+        - containerPort: 9808
+          name: healthz
+          protocol: TCP
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /var/lib/kubelet
+          mountPropagation: Bidirectional
+          name: kubelet-dir
+        - mountPath: /csi
+          name: plugin-dir
+        - mountPath: /dev
+          name: device-dir
+      - args:
+        - --csi-address=$(ADDRESS)
+        - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+        - --v=5
+        env:
+        - name: ADDRESS
+          value: /csi/csi.sock
+        - name: DRIVER_REG_SOCK_PATH
+          value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock
+        image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0
+        lifecycle:
+          preStop:
+            exec:
+              command:
+              - /bin/sh
+              - -c
+              - rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock
+        name: node-driver-registrar
+        volumeMounts:
+        - mountPath: /csi
+          name: plugin-dir
+        - mountPath: /registration
+          name: registration-dir
+      - args:
+        - --csi-address=/csi/csi.sock
+        image: k8s.gcr.io/sig-storage/livenessprobe:v2.2.0
+        name: liveness-probe
+        volumeMounts:
+        - mountPath: /csi
+          name: plugin-dir
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-node-critical
+      serviceAccountName: ebs-csi-node-sa
+      tolerations:
+      - operator: Exists
+      volumes:
+      - hostPath:
+          path: /var/lib/kubelet
+          type: Directory
+        name: kubelet-dir
+      - hostPath:
+          path: /var/lib/kubelet/plugins/ebs.csi.aws.com/
+          type: DirectoryOrCreate
+        name: plugin-dir
+      - hostPath:
+          path: /var/lib/kubelet/plugins_registry/
+          type: Directory
+        name: registration-dir
+      - hostPath:
+          path: /dev
+          type: Directory
+        name: device-dir
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.4.0
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-controller
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: ebs-csi-controller
+      app.kubernetes.io/instance: aws-ebs-csi-driver
+      app.kubernetes.io/name: aws-ebs-csi-driver
+  template:
+    metadata:
+      labels:
+        app: ebs-csi-controller
+        app.kubernetes.io/instance: aws-ebs-csi-driver
+        app.kubernetes.io/name: aws-ebs-csi-driver
+        app.kubernetes.io/version: v1.4.0
+    spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app
+                  operator: In
+                  values:
+                  - ebs-csi-controller
+              topologyKey: topology.kubernetes.io/zone
+            weight: 100
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: app
+                operator: In
+                values:
+                - ebs-csi-controller
+            topologyKey: kubernetes.com/hostname
+      containers:
+      - args:
+        - controller
+        - --endpoint=$(CSI_ENDPOINT)
+        - --logtostderr
+        - --k8s-tag-cluster-id=privatecanal.example.com
+        - --extra-tags=KubernetesCluster=privatecanal.example.com
+        - --v=5
+        env:
+        - name: CSI_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: CSI_ENDPOINT
+          value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+        - name: AWS_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              key: key_id
+              name: aws-secret
+              optional: true
+        - name: AWS_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              key: access_key
+              name: aws-secret
+              optional: true
+        image: k8s.gcr.io/provider-aws/aws-ebs-csi-driver:v1.4.0
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          failureThreshold: 5
+          httpGet:
+            path: /healthz
+            port: healthz
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          timeoutSeconds: 3
+        name: ebs-plugin
+        ports:
+        - containerPort: 9808
+          name: healthz
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /var/lib/csi/sockets/pluginproxy/
+          name: socket-dir
+      - args:
+        - --csi-address=$(ADDRESS)
+        - --v=5
+        - --feature-gates=Topology=true
+        - --leader-election=true
+        - --extra-create-metadata=true
+        - --default-fstype=ext4
+        env:
+        - name: ADDRESS
+          value: /var/lib/csi/sockets/pluginproxy/csi.sock
+        image: k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0
+        name: csi-provisioner
+        volumeMounts:
+        - mountPath: /var/lib/csi/sockets/pluginproxy/
+          name: socket-dir
+      - args:
+        - --csi-address=$(ADDRESS)
+        - --v=5
+        - --leader-election=true
+        env:
+        - name: ADDRESS
+          value: /var/lib/csi/sockets/pluginproxy/csi.sock
+        image: k8s.gcr.io/sig-storage/csi-attacher:v3.2.0
+        name: csi-attacher
+        volumeMounts:
+        - mountPath: /var/lib/csi/sockets/pluginproxy/
+          name: socket-dir
+      - args:
+        - --csi-address=$(ADDRESS)
+        - --leader-election=true
+        env:
+        - name: ADDRESS
+          value: /var/lib/csi/sockets/pluginproxy/csi.sock
+        image: k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0
+        name: csi-snapshotter
+        volumeMounts:
+        - mountPath: /var/lib/csi/sockets/pluginproxy/
+          name: socket-dir
+      - args:
+        - --csi-address=$(ADDRESS)
+        - --v=5
+        env:
+        - name: ADDRESS
+          value: /var/lib/csi/sockets/pluginproxy/csi.sock
+        image: k8s.gcr.io/sig-storage/csi-resizer:v1.1.0
+        imagePullPolicy: Always
+        name: csi-resizer
+        volumeMounts:
+        - mountPath: /var/lib/csi/sockets/pluginproxy/
+          name: socket-dir
+      - args:
+        - --csi-address=/csi/csi.sock
+        image: k8s.gcr.io/sig-storage/livenessprobe:v2.2.0
+        name: liveness-probe
+        volumeMounts:
+        - mountPath: /csi
+          name: socket-dir
+      nodeSelector:
+        kubernetes.io/os: linux
+        node-role.kubernetes.io/master: ""
+      priorityClassName: system-cluster-critical
+      serviceAccountName: ebs-csi-controller-sa
+      tolerations:
+      - operator: Exists
+      volumes:
+      - emptyDir: {}
+        name: socket-dir
+
+---
+
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.4.0
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs.csi.aws.com
+spec:
+  attachRequired: true
+  podInfoOnMount: false
+
+---
+
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io
+    app.kubernetes.io/instance: aws-ebs-csi-driver
+    app.kubernetes.io/managed-by: kops
+    app.kubernetes.io/name: aws-ebs-csi-driver
+    app.kubernetes.io/version: v1.4.0
+    k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+  name: ebs-csi-controller
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app: ebs-csi-controller
+      app.kubernetes.io/instance: aws-ebs-csi-driver

tests/integration/update_cluster/privatecanal/data/aws_s3_bucket_object_privatecanal.example.com-addons-bootstrap_content

@@ -47,15 +47,22 @@ spec:
     version: 9.99.0
   - id: v1.15.0
     manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
-    manifestHash: 065ae832ddac8d0931e9992d6a76f43a33a36975a38003b34f4c5d86a7d42780
+    manifestHash: 4e2cda50cd5048133aad1b5e28becb60f4629d3f9e09c514a2757c27998b4200
     name: storage-aws.addons.k8s.io
     selector:
       k8s-addon: storage-aws.addons.k8s.io
     version: 9.99.0
-  - id: k8s-1.16
+  - id: k8s-1.22
-    manifest: networking.projectcalico.org.canal/k8s-1.16.yaml
+    manifest: networking.projectcalico.org.canal/k8s-1.22.yaml
-    manifestHash: 95eac57e6511368d555dde5c7e50ed95f5b6e9bec9c65c4e846fe8b0437edeef
+    manifestHash: b026de0876284d3167c957c3eec392866900c5b0c81ce92ef458e7bac343644a
     name: networking.projectcalico.org.canal
     selector:
       role.kubernetes.io/networking: "1"
     version: 9.99.0
+  - id: k8s-1.17
+    manifest: aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml
+    manifestHash: dd8a6632d53821b6fc49ad83d524cff280e3be3205f2d09aa6905af7cebe4af6
+    name: aws-ebs-csi-driver.addons.k8s.io
+    selector:
+      k8s-addon: aws-ebs-csi-driver.addons.k8s.io
+    version: 9.99.0

tests/integration/update_cluster/privatecanal/data/aws_s3_bucket_object_privatecanal.example.com-addons-networking.projectcalico.org.canal-k8s-1.16_content

@@ -1,756 +0,0 @@
-apiVersion: v1
-data:
-  canal_iface: ""
-  cni_network_config: |-
-    {
-      "name": "k8s-pod-network",
-      "cniVersion": "0.3.1",
-      "plugins": [
-        {
-          "type": "calico",
-          "log_level": "info",
-          "datastore_type": "kubernetes",
-          "nodename": "__KUBERNETES_NODE_NAME__",
-          "mtu": __CNI_MTU__,
-          "ipam": {
-              "type": "host-local",
-              "subnet": "usePodCidr"
-          },
-          "policy": {
-              "type": "k8s"
-          },
-          "kubernetes": {
-              "kubeconfig": "__KUBECONFIG_FILEPATH__"
-          }
-        },
-        {
-          "type": "portmap",
-          "snat": true,
-          "capabilities": {"portMappings": true}
-        },
-        {
-          "type": "bandwidth",
-          "capabilities": {"bandwidth": true}
-        }
-      ]
-    }
-  masquerade: "true"
-  net-conf.json: |
-    {
-      "Network": "100.64.0.0/10",
-      "Backend": {
-        "Type": "vxlan"
-      }
-    }
-  typha_service_name: none
-  veth_mtu: "1440"
-kind: ConfigMap
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.projectcalico.org.canal
-    app.kubernetes.io/managed-by: kops
-    role.kubernetes.io/networking: "1"
-  name: canal-config
-  namespace: kube-system
-
----
-
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.projectcalico.org.canal
-    app.kubernetes.io/managed-by: kops
-    role.kubernetes.io/networking: "1"
-  name: bgpconfigurations.crd.projectcalico.org
-spec:
-  group: crd.projectcalico.org
-  names:
-    kind: BGPConfiguration
-    plural: bgpconfigurations
-    singular: bgpconfiguration
-  scope: Cluster
-  version: v1
-
----
-
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.projectcalico.org.canal
-    app.kubernetes.io/managed-by: kops
-    role.kubernetes.io/networking: "1"
-  name: bgppeers.crd.projectcalico.org
-spec:
-  group: crd.projectcalico.org
-  names:
-    kind: BGPPeer
-    plural: bgppeers
-    singular: bgppeer
-  scope: Cluster
-  version: v1
-
----
-
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.projectcalico.org.canal
-    app.kubernetes.io/managed-by: kops
-    role.kubernetes.io/networking: "1"
-  name: blockaffinities.crd.projectcalico.org
-spec:
-  group: crd.projectcalico.org
-  names:
-    kind: BlockAffinity
-    plural: blockaffinities
-    singular: blockaffinity
-  scope: Cluster
-  version: v1
-
----
-
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.projectcalico.org.canal
-    app.kubernetes.io/managed-by: kops
-    role.kubernetes.io/networking: "1"
-  name: clusterinformations.crd.projectcalico.org
-spec:
-  group: crd.projectcalico.org
-  names:
-    kind: ClusterInformation
-    plural: clusterinformations
-    singular: clusterinformation
-  scope: Cluster
-  version: v1
-
----
-
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.projectcalico.org.canal
-    app.kubernetes.io/managed-by: kops
-    role.kubernetes.io/networking: "1"
-  name: felixconfigurations.crd.projectcalico.org
-spec:
-  group: crd.projectcalico.org
-  names:
-    kind: FelixConfiguration
-    plural: felixconfigurations
-    singular: felixconfiguration
-  scope: Cluster
-  version: v1
-
----
-
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.projectcalico.org.canal
-    app.kubernetes.io/managed-by: kops
-    role.kubernetes.io/networking: "1"
-  name: globalnetworkpolicies.crd.projectcalico.org
-spec:
-  group: crd.projectcalico.org
-  names:
-    kind: GlobalNetworkPolicy
-    plural: globalnetworkpolicies
-    singular: globalnetworkpolicy
-  scope: Cluster
-  version: v1
-
----
-
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.projectcalico.org.canal
-    app.kubernetes.io/managed-by: kops
-    role.kubernetes.io/networking: "1"
-  name: globalnetworksets.crd.projectcalico.org
-spec:
-  group: crd.projectcalico.org
-  names:
-    kind: GlobalNetworkSet
-    plural: globalnetworksets
-    singular: globalnetworkset
-  scope: Cluster
-  version: v1
-
----
-
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.projectcalico.org.canal
-    app.kubernetes.io/managed-by: kops
-    role.kubernetes.io/networking: "1"
-  name: hostendpoints.crd.projectcalico.org
-spec:
-  group: crd.projectcalico.org
-  names:
-    kind: HostEndpoint
-    plural: hostendpoints
-    singular: hostendpoint
-  scope: Cluster
-  version: v1
-
----
-
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.projectcalico.org.canal
-    app.kubernetes.io/managed-by: kops
-    role.kubernetes.io/networking: "1"
-  name: ipamblocks.crd.projectcalico.org
-spec:
-  group: crd.projectcalico.org
-  names:
-    kind: IPAMBlock
-    plural: ipamblocks
-    singular: ipamblock
-  scope: Cluster
-  version: v1
-
----
-
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.projectcalico.org.canal
-    app.kubernetes.io/managed-by: kops
-    role.kubernetes.io/networking: "1"
-  name: ipamconfigs.crd.projectcalico.org
-spec:
-  group: crd.projectcalico.org
-  names:
-    kind: IPAMConfig
-    plural: ipamconfigs
-    singular: ipamconfig
-  scope: Cluster
-  version: v1
-
----
-
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.projectcalico.org.canal
-    app.kubernetes.io/managed-by: kops
-    role.kubernetes.io/networking: "1"
-  name: ipamhandles.crd.projectcalico.org
-spec:
-  group: crd.projectcalico.org
-  names:
-    kind: IPAMHandle
-    plural: ipamhandles
-    singular: ipamhandle
-  scope: Cluster
-  version: v1
-
----
-
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.projectcalico.org.canal
-    app.kubernetes.io/managed-by: kops
-    role.kubernetes.io/networking: "1"
-  name: ippools.crd.projectcalico.org
-spec:
-  group: crd.projectcalico.org
-  names:
-    kind: IPPool
-    plural: ippools
-    singular: ippool
-  scope: Cluster
-  version: v1
-
----
-
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.projectcalico.org.canal
-    app.kubernetes.io/managed-by: kops
-    role.kubernetes.io/networking: "1"
-  name: networkpolicies.crd.projectcalico.org
-spec:
-  group: crd.projectcalico.org
-  names:
-    kind: NetworkPolicy
-    plural: networkpolicies
-    singular: networkpolicy
-  scope: Namespaced
-  version: v1
-
----
-
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.projectcalico.org.canal
-    app.kubernetes.io/managed-by: kops
-    role.kubernetes.io/networking: "1"
-  name: networksets.crd.projectcalico.org
-spec:
-  group: crd.projectcalico.org
-  names:
-    kind: NetworkSet
-    plural: networksets
-    singular: networkset
-  scope: Namespaced
-  version: v1
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.projectcalico.org.canal
-    app.kubernetes.io/managed-by: kops
-    role.kubernetes.io/networking: "1"
-  name: calico
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  - nodes
-  - namespaces
-  verbs:
-  - get
-- apiGroups:
-  - ""
-  resources:
-  - endpoints
-  - services
-  verbs:
-  - watch
-  - list
-  - get
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - get
-- apiGroups:
-  - ""
-  resources:
-  - nodes/status
-  verbs:
-  - patch
-  - update
-- apiGroups:
-  - networking.k8s.io
-  resources:
-  - networkpolicies
-  verbs:
-  - watch
-  - list
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  - namespaces
-  - serviceaccounts
-  verbs:
-  - list
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - pods/status
-  verbs:
-  - patch
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - globalfelixconfigs
-  - felixconfigurations
-  - bgppeers
-  - globalbgpconfigs
-  - bgpconfigurations
-  - ippools
-  - ipamblocks
-  - globalnetworkpolicies
-  - globalnetworksets
-  - networkpolicies
-  - networksets
-  - clusterinformations
-  - hostendpoints
-  - blockaffinities
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - ippools
-  - felixconfigurations
-  - clusterinformations
-  verbs:
-  - create
-  - update
-- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - crd.projectcalico.org
-  resources:
-  - bgpconfigurations
-  - bgppeers
-  verbs:
-  - create
-  - update
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.projectcalico.org.canal
-    app.kubernetes.io/managed-by: kops
-    role.kubernetes.io/networking: "1"
-  name: flannel
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
-- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - list
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - nodes/status
-  verbs:
-  - patch
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.projectcalico.org.canal
-    app.kubernetes.io/managed-by: kops
-    role.kubernetes.io/networking: "1"
-  name: canal-flannel
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: flannel
-subjects:
-- kind: ServiceAccount
-  name: canal
-  namespace: kube-system
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.projectcalico.org.canal
-    app.kubernetes.io/managed-by: kops
-    role.kubernetes.io/networking: "1"
-  name: canal-calico
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: calico
-subjects:
-- kind: ServiceAccount
-  name: canal
-  namespace: kube-system
-
----
-
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.projectcalico.org.canal
-    app.kubernetes.io/managed-by: kops
-    k8s-app: canal
-    role.kubernetes.io/networking: "1"
-  name: canal
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      k8s-app: canal
-  template:
-    metadata:
-      labels:
-        k8s-app: canal
-    spec:
-      containers:
-      - env:
-        - name: DATASTORE_TYPE
-          value: kubernetes
-        - name: USE_POD_CIDR
-          value: "true"
-        - name: WAIT_FOR_DATASTORE
-          value: "true"
-        - name: NODENAME
-          valueFrom:
-            fieldRef:
-              fieldPath: spec.nodeName
-        - name: CALICO_NETWORKING_BACKEND
-          value: none
-        - name: CLUSTER_TYPE
-          value: k8s,canal
-        - name: FELIX_IPTABLESREFRESHINTERVAL
-          value: "60"
-        - name: IP
-          value: ""
-        - name: FELIX_IPINIPMTU
-          valueFrom:
-            configMapKeyRef:
-              key: veth_mtu
-              name: canal-config
-        - name: CALICO_DISABLE_FILE_LOGGING
-          value: "true"
-        - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
-          value: ACCEPT
-        - name: FELIX_IPV6SUPPORT
-          value: "false"
-        - name: FELIX_LOGSEVERITYSCREEN
-          value: info
-        - name: FELIX_HEALTHENABLED
-          value: "true"
-        - name: FELIX_CHAININSERTMODE
-          value: insert
-        - name: FELIX_IPTABLESBACKEND
-          value: Auto
-        - name: FELIX_PROMETHEUSMETRICSENABLED
-          value: "false"
-        - name: FELIX_PROMETHEUSMETRICSPORT
-          value: "9091"
-        - name: FELIX_PROMETHEUSGOMETRICSENABLED
-          value: "true"
-        - name: FELIX_PROMETHEUSPROCESSMETRICSENABLED
-          value: "true"
-        image: calico/node:v3.13.4
-        livenessProbe:
-          exec:
-            command:
-            - /bin/calico-node
-            - -felix-live
-          failureThreshold: 6
-          initialDelaySeconds: 10
-          periodSeconds: 10
-        name: calico-node
-        readinessProbe:
-          httpGet:
-            host: localhost
-            path: /readiness
-            port: 9099
-          periodSeconds: 10
-        resources:
-          requests:
-            cpu: 100m
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - mountPath: /lib/modules
-          name: lib-modules
-          readOnly: true
-        - mountPath: /run/xtables.lock
-          name: xtables-lock
-          readOnly: false
-        - mountPath: /var/run/calico
-          name: var-run-calico
-          readOnly: false
-        - mountPath: /var/lib/calico
-          name: var-lib-calico
-          readOnly: false
-        - mountPath: /var/run/nodeagent
-          name: policysync
-      - command:
-        - /opt/bin/flanneld
-        - --ip-masq
-        - --kube-subnet-mgr
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        - name: FLANNELD_IFACE
-          valueFrom:
-            configMapKeyRef:
-              key: canal_iface
-              name: canal-config
-        - name: FLANNELD_IP_MASQ
-          valueFrom:
-            configMapKeyRef:
-              key: masquerade
-              name: canal-config
-        image: quay.io/coreos/flannel:v0.11.0
-        name: kube-flannel
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - mountPath: /run/xtables.lock
-          name: xtables-lock
-          readOnly: false
-        - mountPath: /etc/kube-flannel/
-          name: flannel-cfg
-      hostNetwork: true
-      initContainers:
-      - command:
-        - /install-cni.sh
-        env:
-        - name: CNI_CONF_NAME
-          value: 10-canal.conflist
-        - name: CNI_NETWORK_CONFIG
-          valueFrom:
-            configMapKeyRef:
-              key: cni_network_config
-              name: canal-config
-        - name: KUBERNETES_NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: spec.nodeName
-        - name: CNI_MTU
-          valueFrom:
-            configMapKeyRef:
-              key: veth_mtu
-              name: canal-config
-        - name: SLEEP
-          value: "false"
-        image: calico/cni:v3.13.4
-        name: install-cni
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - mountPath: /host/opt/cni/bin
-          name: cni-bin-dir
-        - mountPath: /host/etc/cni/net.d
-          name: cni-net-dir
-      - image: calico/pod2daemon-flexvol:v3.13.4
-        name: flexvol-driver
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - mountPath: /host/driver
-          name: flexvol-driver-host
-      nodeSelector:
-        kubernetes.io/os: linux
-      priorityClassName: system-node-critical
-      serviceAccountName: canal
-      terminationGracePeriodSeconds: 0
-      tolerations:
-      - effect: NoSchedule
-        operator: Exists
-      - key: CriticalAddonsOnly
-        operator: Exists
-      - effect: NoExecute
-        operator: Exists
-      volumes:
-      - hostPath:
-          path: /lib/modules
-        name: lib-modules
-      - hostPath:
-          path: /var/run/calico
-        name: var-run-calico
-      - hostPath:
-          path: /var/lib/calico
-        name: var-lib-calico
-      - hostPath:
-          path: /run/xtables.lock
-          type: FileOrCreate
-        name: xtables-lock
-      - configMap:
-          name: canal-config
-        name: flannel-cfg
-      - hostPath:
-          path: /opt/cni/bin
-        name: cni-bin-dir
-      - hostPath:
-          path: /etc/cni/net.d
-        name: cni-net-dir
-      - hostPath:
-          path: /var/run/nodeagent
-          type: DirectoryOrCreate
-        name: policysync
-      - hostPath:
-          path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds
-          type: DirectoryOrCreate
-        name: flexvol-driver-host
-  updateStrategy:
-    rollingUpdate:
-      maxUnavailable: 1
-    type: RollingUpdate
-
----
-
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: networking.projectcalico.org.canal
-    app.kubernetes.io/managed-by: kops
-    role.kubernetes.io/networking: "1"
-  name: canal
-  namespace: kube-system

tests/integration/update_cluster/privatecanal/data/aws_s3_bucket_object_privatecanal.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content

@@ -35,7 +35,7 @@ apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
   annotations:
-    storageclass.kubernetes.io/is-default-class: "true"
+    storageclass.kubernetes.io/is-default-class: "false"
   creationTimestamp: null
   labels:
     addon.kops.k8s.io/name: storage-aws.addons.k8s.io
@@ -50,6 +50,26 @@ volumeBindingMode: WaitForFirstConsumer
 
 ---
 
+allowVolumeExpansion: true
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: storage-aws.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: storage-aws.addons.k8s.io
+  name: kops-csi-1-21
+parameters:
+  encrypted: "true"
+  type: gp3
+provisioner: ebs.csi.aws.com
+volumeBindingMode: WaitForFirstConsumer
+
+---
+
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:

tests/integration/update_cluster/privatecanal/in-v1alpha2.yaml

@@ -21,7 +21,7 @@ spec:
   iam: {}
   kubelet:
     anonymousAuth: false
-  kubernetesVersion: v1.21.0
+  kubernetesVersion: v1.22.0
   masterInternalName: api.internal.privatecanal.example.com
   masterPublicName: api.privatecanal.example.com
   networkCIDR: 172.20.0.0/16

tests/integration/update_cluster/privatecanal/kubernetes.tf

@@ -840,6 +840,14 @@ resource "aws_s3_bucket_object" "nodeupconfig-nodes" {
   server_side_encryption = "AES256"
 }
 
+resource "aws_s3_bucket_object" "privatecanal-example-com-addons-aws-ebs-csi-driver-addons-k8s-io-k8s-1-17" {
+  bucket                 = "testingBucket"
+  content                = file("${path.module}/data/aws_s3_bucket_object_privatecanal.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content")
+  key                    = "clusters.example.com/privatecanal.example.com/addons/aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml"
+  provider               = aws.files
+  server_side_encryption = "AES256"
+}
+
 resource "aws_s3_bucket_object" "privatecanal-example-com-addons-bootstrap" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_bucket_object_privatecanal.example.com-addons-bootstrap_content")
@@ -896,10 +904,10 @@ resource "aws_s3_bucket_object" "privatecanal-example-com-addons-limit-range-add
   server_side_encryption = "AES256"
 }
 
-resource "aws_s3_bucket_object" "privatecanal-example-com-addons-networking-projectcalico-org-canal-k8s-1-16" {
+resource "aws_s3_bucket_object" "privatecanal-example-com-addons-networking-projectcalico-org-canal-k8s-1-22" {
   bucket                 = "testingBucket"
-  content                = file("${path.module}/data/aws_s3_bucket_object_privatecanal.example.com-addons-networking.projectcalico.org.canal-k8s-1.16_content")
+  content                = file("${path.module}/data/aws_s3_bucket_object_privatecanal.example.com-addons-networking.projectcalico.org.canal-k8s-1.22_content")
-  key                    = "clusters.example.com/privatecanal.example.com/addons/networking.projectcalico.org.canal/k8s-1.16.yaml"
+  key                    = "clusters.example.com/privatecanal.example.com/addons/networking.projectcalico.org.canal/k8s-1.22.yaml"
   provider               = aws.files
   server_side_encryption = "AES256"
 }

upup/models/BUILD.bazel

@@ -51,6 +51,7 @@ go_library(
         "cloudup/resources/addons/snapshot-controller.addons.k8s.io/k8s-1.20.yaml.template",
         "cloudup/resources/addons/node-problem-detector.addons.k8s.io/k8s-1.17.yaml.template",
         "cloudup/resources/addons/nvidia.addons.k8s.io/k8s-1.16.yaml",
+        "cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.22.yaml.template",
     ],
     importpath = "k8s.io/kops/upup/models",
     visibility = ["//visibility:public"],

upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go

@@ -822,7 +822,17 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*Addon
 	if b.Cluster.Spec.Networking.Canal != nil {
 		key := "networking.projectcalico.org.canal"
 
-		{
+		if b.IsKubernetesGTE("v1.22.0") {
+			id := "k8s-1.22"
+			location := key + "/" + id + ".yaml"
+
+			addons.Add(&channelsapi.AddonSpec{
+				Name:     fi.String(key),
+				Selector: networkingSelector(),
+				Manifest: fi.String(location),
+				Id:       id,
+			})
+		} else {
 			id := "k8s-1.16"
 			location := key + "/" + id + ".yaml"