diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index d9e9ce9cff..51ea930e2e 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -209,6 +209,7 @@ const ( certManagerAddon = "certmanager.io-k8s-1.16" ciliumAddon = "networking.cilium.io-k8s-1.16" dnsControllerAddon = "dns-controller.addons.k8s.io-k8s-1.12" + flannelAddon = "networking.flannel-k8s-1.25" leaderElectionAddon = "leader-migration.rbac.addons.k8s.io-k8s-1.23" ) @@ -491,7 +492,8 @@ func TestPrivateWeave(t *testing.T) { func TestPrivateFlannel(t *testing.T) { newIntegrationTest("privateflannel.example.com", "privateflannel"). withPrivate(). - withAddons("networking.flannel-k8s-1.12", dnsControllerAddon). + withDefaultAddons24(). + withAddons(flannelAddon). runTestTerraformAWS(t) } diff --git a/tests/integration/update_cluster/privateflannel/data/aws_iam_role_policy_masters.privateflannel.example.com_policy b/tests/integration/update_cluster/privateflannel/data/aws_iam_role_policy_masters.privateflannel.example.com_policy index fc6fd9acb4..29ea93df1d 100644 --- a/tests/integration/update_cluster/privateflannel/data/aws_iam_role_policy_masters.privateflannel.example.com_policy +++ b/tests/integration/update_cluster/privateflannel/data/aws_iam_role_policy_masters.privateflannel.example.com_policy @@ -94,39 +94,6 @@ "*" ] }, - { - "Action": "ec2:CreateTags", - "Condition": { - "StringEquals": { - "aws:RequestTag/KubernetesCluster": "privateflannel.example.com", - "ec2:CreateAction": [ - "CreateSecurityGroup" - ] - } - }, - "Effect": "Allow", - "Resource": [ - "arn:aws-test:ec2:*:*:security-group/*" - ] - }, - { - "Action": [ - "ec2:CreateTags", - "ec2:DeleteTags" - ], - "Condition": { - "Null": { - "aws:RequestTag/KubernetesCluster": "true" - }, - "StringEquals": { - "aws:ResourceTag/KubernetesCluster": "privateflannel.example.com" - } - }, - "Effect": "Allow", - "Resource": [ - "arn:aws-test:ec2:*:*:security-group/*" - ] - }, { "Action": "ec2:CreateTags", "Condition": { @@ -163,19 +130,45 @@ "arn:aws-test:ec2:*:*:snapshot/*" ] }, + { + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "aws:RequestTag/KubernetesCluster": "privateflannel.example.com", + "ec2:CreateAction": [ + "CreateSecurityGroup" + ] + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:security-group/*" + ] + }, + { + "Action": [ + "ec2:CreateTags", + "ec2:DeleteTags" + ], + "Condition": { + "Null": { + "aws:RequestTag/KubernetesCluster": "true" + }, + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "privateflannel.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:security-group/*" + ] + }, { "Action": [ "autoscaling:DescribeAutoScalingGroups", "autoscaling:DescribeAutoScalingInstances", "autoscaling:DescribeLaunchConfigurations", "autoscaling:DescribeTags", - "ec2:AttachVolume", - "ec2:AuthorizeSecurityGroupIngress", - "ec2:CreateSecurityGroup", - "ec2:CreateTags", - "ec2:DeleteRoute", - "ec2:DeleteSecurityGroup", - "ec2:DeleteVolume", "ec2:DescribeAccountAttributes", "ec2:DescribeInstanceTypes", "ec2:DescribeInstances", @@ -188,19 +181,12 @@ "ec2:DescribeVolumes", "ec2:DescribeVolumesModifications", "ec2:DescribeVpcs", - "ec2:DetachVolume", - "ec2:ModifyInstanceAttribute", - "ec2:ModifyVolume", - "elasticloadbalancing:AddTags", - "elasticloadbalancing:CreateListener", - "elasticloadbalancing:CreateTargetGroup", "elasticloadbalancing:DescribeListeners", "elasticloadbalancing:DescribeLoadBalancerAttributes", "elasticloadbalancing:DescribeLoadBalancerPolicies", "elasticloadbalancing:DescribeLoadBalancers", "elasticloadbalancing:DescribeTargetGroups", "elasticloadbalancing:DescribeTargetHealth", - "elasticloadbalancing:RegisterTargets", "iam:GetServerCertificate", "iam:ListServerCertificates", "kms:DescribeKey", diff --git a/tests/integration/update_cluster/privateflannel/data/aws_launch_template_master-us-test-1a.masters.privateflannel.example.com_user_data b/tests/integration/update_cluster/privateflannel/data/aws_launch_template_master-us-test-1a.masters.privateflannel.example.com_user_data index cf0603adcd..a31eb8e868 100644 --- a/tests/integration/update_cluster/privateflannel/data/aws_launch_template_master-us-test-1a.masters.privateflannel.example.com_user_data +++ b/tests/integration/update_cluster/privateflannel/data/aws_launch_template_master-us-test-1a.masters.privateflannel.example.com_user_data @@ -125,20 +125,23 @@ ensure-install-dir cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' cloudConfig: awsEBSCSIDriver: - enabled: false + enabled: true + version: v1.8.0 manageStorageClasses: true containerRuntime: containerd containerd: logLevel: info - version: 1.4.13 + runc: + version: 1.1.4 + version: 1.6.8 docker: skipInstall: true encryptionConfig: null etcdClusters: events: - version: 3.4.13 + version: 3.5.4 main: - version: 3.4.13 + version: 3.5.4 kubeAPIServer: allowPrivileged: true anonymousAuth: false @@ -147,7 +150,7 @@ kubeAPIServer: apiServerCount: 1 authorizationMode: AlwaysAllow bindAddress: 0.0.0.0 - cloudProvider: aws + cloudProvider: external enableAdmissionPlugins: - NamespaceLifecycle - LimitRanger @@ -162,7 +165,10 @@ kubeAPIServer: - https://127.0.0.1:4001 etcdServersOverrides: - /events#https://127.0.0.1:4002 - image: registry.k8s.io/kube-apiserver:v1.21.0 + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-apiserver:v1.25.0 kubeletPreferredAddressTypes: - InternalIP - Hostname @@ -184,11 +190,14 @@ kubeAPIServer: kubeControllerManager: allocateNodeCIDRs: true attachDetachReconcileSyncPeriod: 1m0s - cloudProvider: aws + cloudProvider: external clusterCIDR: 100.96.0.0/11 clusterName: privateflannel.example.com configureCloudRoutes: false - image: registry.k8s.io/kube-controller-manager:v1.21.0 + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-controller-manager:v1.25.0 leaderElection: leaderElect: true logLevel: 2 @@ -196,10 +205,13 @@ kubeControllerManager: kubeProxy: clusterCIDR: 100.96.0.0/11 cpuRequest: 100m - image: registry.k8s.io/kube-proxy:v1.21.0 + image: registry.k8s.io/kube-proxy:v1.25.0 logLevel: 2 kubeScheduler: - image: registry.k8s.io/kube-scheduler:v1.21.0 + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-scheduler:v1.25.0 leaderElection: leaderElect: true logLevel: 2 @@ -207,32 +219,38 @@ kubelet: anonymousAuth: false cgroupDriver: systemd cgroupRoot: / - cloudProvider: aws + cloudProvider: external clusterDNS: 100.64.0.10 clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 - networkPluginName: cni podInfraContainerImage: registry.k8s.io/pause:3.6 podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true shutdownGracePeriod: 30s shutdownGracePeriodCriticalPods: 10s masterKubelet: anonymousAuth: false cgroupDriver: systemd cgroupRoot: / - cloudProvider: aws + cloudProvider: external clusterDNS: 100.64.0.10 clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 - networkPluginName: cni podInfraContainerImage: registry.k8s.io/pause:3.6 podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true registerSchedulable: false shutdownGracePeriod: 30s shutdownGracePeriodCriticalPods: 10s @@ -244,7 +262,7 @@ CloudProvider: aws ConfigBase: memfs://clusters.example.com/privateflannel.example.com InstanceGroupName: master-us-test-1a InstanceGroupRole: Master -NodeupConfigHash: 9bl7nc3fmJzQ97SKZjYIIXw8RtxfvsO5G7JlEGSfwGo= +NodeupConfigHash: /XEEEkIpyD9DFl1M2yERdnj0V8fAcWHjXqyvF2Ou0sY= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/privateflannel/data/aws_launch_template_nodes.privateflannel.example.com_user_data b/tests/integration/update_cluster/privateflannel/data/aws_launch_template_nodes.privateflannel.example.com_user_data index 68bfe4215e..b4b0eb9610 100644 --- a/tests/integration/update_cluster/privateflannel/data/aws_launch_template_nodes.privateflannel.example.com_user_data +++ b/tests/integration/update_cluster/privateflannel/data/aws_launch_template_nodes.privateflannel.example.com_user_data @@ -125,33 +125,39 @@ ensure-install-dir cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' cloudConfig: awsEBSCSIDriver: - enabled: false + enabled: true + version: v1.8.0 manageStorageClasses: true containerRuntime: containerd containerd: logLevel: info - version: 1.4.13 + runc: + version: 1.1.4 + version: 1.6.8 docker: skipInstall: true kubeProxy: clusterCIDR: 100.96.0.0/11 cpuRequest: 100m - image: registry.k8s.io/kube-proxy:v1.21.0 + image: registry.k8s.io/kube-proxy:v1.25.0 logLevel: 2 kubelet: anonymousAuth: false cgroupDriver: systemd cgroupRoot: / - cloudProvider: aws + cloudProvider: external clusterDNS: 100.64.0.10 clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 - networkPluginName: cni podInfraContainerImage: registry.k8s.io/pause:3.6 podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true shutdownGracePeriod: 30s shutdownGracePeriodCriticalPods: 10s @@ -162,7 +168,7 @@ CloudProvider: aws ConfigBase: memfs://clusters.example.com/privateflannel.example.com InstanceGroupName: nodes InstanceGroupRole: Node -NodeupConfigHash: GY/MFKBVJLXIsiWxM43gVcm4X8DArd0PWbyo8MoJ1oA= +NodeupConfigHash: SD5kNk5k3nRVYsHkhayCQtFZXAjYZdRPDKJCxaF6Cro= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_cluster-completed.spec_content index 08e4b9b4ec..805b617d06 100644 --- a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_cluster-completed.spec_content @@ -13,8 +13,17 @@ spec: channel: stable cloudConfig: awsEBSCSIDriver: - enabled: false + enabled: true + version: v1.8.0 manageStorageClasses: true + cloudControllerManager: + allocateNodeCIDRs: true + clusterCIDR: 100.64.0.0/10 + clusterName: privateflannel.example.com + configureCloudRoutes: false + image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.25.0 + leaderElection: + leaderElect: true cloudProvider: aws clusterDNSDomain: cluster.local configBase: memfs://clusters.example.com/privateflannel.example.com @@ -22,7 +31,9 @@ spec: containerRuntime: containerd containerd: logLevel: info - version: 1.4.13 + runc: + version: 1.1.4 + version: 1.6.8 dnsZone: Z1AFAKE1ZON3YO docker: skipInstall: true @@ -33,14 +44,14 @@ spec: - instanceGroup: master-us-test-1a name: us-test-1a name: main - version: 3.4.13 + version: 3.5.4 - backups: backupStore: memfs://clusters.example.com/privateflannel.example.com/backups/etcd/events etcdMembers: - instanceGroup: master-us-test-1a name: us-test-1a name: events - version: 3.4.13 + version: 3.5.4 externalDns: provider: dns-controller iam: @@ -54,7 +65,7 @@ spec: apiServerCount: 1 authorizationMode: AlwaysAllow bindAddress: 0.0.0.0 - cloudProvider: aws + cloudProvider: external enableAdmissionPlugins: - NamespaceLifecycle - LimitRanger @@ -69,7 +80,10 @@ spec: - https://127.0.0.1:4001 etcdServersOverrides: - /events#https://127.0.0.1:4002 - image: registry.k8s.io/kube-apiserver:v1.21.0 + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-apiserver:v1.25.0 kubeletPreferredAddressTypes: - InternalIP - Hostname @@ -91,11 +105,14 @@ spec: kubeControllerManager: allocateNodeCIDRs: true attachDetachReconcileSyncPeriod: 1m0s - cloudProvider: aws + cloudProvider: external clusterCIDR: 100.96.0.0/11 clusterName: privateflannel.example.com configureCloudRoutes: false - image: registry.k8s.io/kube-controller-manager:v1.21.0 + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-controller-manager:v1.25.0 leaderElection: leaderElect: true logLevel: 2 @@ -117,10 +134,13 @@ spec: kubeProxy: clusterCIDR: 100.96.0.0/11 cpuRequest: 100m - image: registry.k8s.io/kube-proxy:v1.21.0 + image: registry.k8s.io/kube-proxy:v1.25.0 logLevel: 2 kubeScheduler: - image: registry.k8s.io/kube-scheduler:v1.21.0 + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-scheduler:v1.25.0 leaderElection: leaderElect: true logLevel: 2 @@ -128,36 +148,42 @@ spec: anonymousAuth: false cgroupDriver: systemd cgroupRoot: / - cloudProvider: aws + cloudProvider: external clusterDNS: 100.64.0.10 clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 - networkPluginName: cni podInfraContainerImage: registry.k8s.io/pause:3.6 podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true shutdownGracePeriod: 30s shutdownGracePeriodCriticalPods: 10s kubernetesApiAccess: - 0.0.0.0/0 - kubernetesVersion: 1.21.0 + kubernetesVersion: 1.25.0 masterInternalName: api.internal.privateflannel.example.com masterKubelet: anonymousAuth: false cgroupDriver: systemd cgroupRoot: / - cloudProvider: aws + cloudProvider: external clusterDNS: 100.64.0.10 clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 - networkPluginName: cni podInfraContainerImage: registry.k8s.io/pause:3.6 podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true registerSchedulable: false shutdownGracePeriod: 30s shutdownGracePeriodCriticalPods: 10s diff --git a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_etcd-cluster-spec-events_content b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_etcd-cluster-spec-events_content index bb8ddb0e2e..06759fa2b2 100644 --- a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_etcd-cluster-spec-events_content +++ b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_etcd-cluster-spec-events_content @@ -1,4 +1,4 @@ { "memberCount": 1, - "etcdVersion": "3.4.13" + "etcdVersion": "3.5.4" } diff --git a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_etcd-cluster-spec-main_content b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_etcd-cluster-spec-main_content index bb8ddb0e2e..06759fa2b2 100644 --- a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_etcd-cluster-spec-main_content +++ b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_etcd-cluster-spec-main_content @@ -1,4 +1,4 @@ { "memberCount": 1, - "etcdVersion": "3.4.13" + "etcdVersion": "3.5.4" } diff --git a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_nodeupconfig-master-us-test-1a_content b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_nodeupconfig-master-us-test-1a_content index a67bd1e311..368191e6f9 100644 --- a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_nodeupconfig-master-us-test-1a_content +++ b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_nodeupconfig-master-us-test-1a_content @@ -7,7 +7,7 @@ APIServerConfig: apiServerCount: 1 authorizationMode: AlwaysAllow bindAddress: 0.0.0.0 - cloudProvider: aws + cloudProvider: external enableAdmissionPlugins: - NamespaceLifecycle - LimitRanger @@ -22,7 +22,10 @@ APIServerConfig: - https://127.0.0.1:4001 etcdServersOverrides: - /events#https://127.0.0.1:4002 - image: registry.k8s.io/kube-apiserver:v1.21.0 + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-apiserver:v1.25.0 kubeletPreferredAddressTypes: - InternalIP - Hostname @@ -52,17 +55,19 @@ APIServerConfig: -----END RSA PUBLIC KEY----- Assets: amd64: - - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet - - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl - - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz - - 29ef1e8635795c2a49a20a56e778f45ff163c5400a5428ca33999ed53d44e3d8@https://github.com/containerd/containerd/releases/download/v1.4.13/cri-containerd-cni-1.4.13-linux-amd64.tar.gz + - 7f9183fce12606818612ce80b6c09757452c4fb50aefea5fc5843951c5020e24@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/amd64/kubelet + - e23cc7092218c95c22d8ee36fb9499194a36ac5b5349ca476886b7edc0203885@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/amd64/kubectl + - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz + - 3a1322c18ee5ff4b9bd5af6b7b30c923a3eab8af1df05554f530ef8e2b24ac5e@https://github.com/containerd/containerd/releases/download/v1.6.8/containerd-1.6.8-linux-amd64.tar.gz + - db772be63147a4e747b4fe286c7c16a2edc4a8458bd3092ea46aaee77750e8ce@https://github.com/opencontainers/runc/releases/download/v1.1.4/runc.amd64 - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 arm64: - - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet - - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl - - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz - - debed306ed9a4e70dcbcb228a0b3898f9730099e324f34bb0e76abbaddf7a6a7@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.13.tgz + - 69572a7b3d179d4a479aa2e0f90e2f091d8d84ef33a35422fc89975dc137a590@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/arm64/kubelet + - 24db547bbae294c5c44f2b4a777e45f0e2f3d6295eace0d0c4be2b2dfa45330d@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/arm64/kubectl + - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz + - b114e36ecce78cef9d611416c01b784a420928c82766d6df7dc02b10d9da94cd@https://github.com/containerd/containerd/releases/download/v1.6.8/containerd-1.6.8-linux-arm64.tar.gz + - dbb71e737eaef454a406ce21fd021bd8f1b35afb7635016745992bbd7c17a223@https://github.com/opencontainers/runc/releases/download/v1.1.4/runc.arm64 - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 CAs: @@ -220,7 +225,7 @@ CAs: ClusterName: privateflannel.example.com FileAssets: - content: | - apiVersion: kubescheduler.config.k8s.io/v1beta1 + apiVersion: kubescheduler.config.k8s.io/v1beta2 clientConnection: kubeconfig: /var/lib/kube-scheduler/kubeconfig kind: KubeSchedulerConfiguration @@ -241,22 +246,23 @@ KubeletConfig: anonymousAuth: false cgroupDriver: systemd cgroupRoot: / - cloudProvider: aws + cloudProvider: external clusterDNS: 100.64.0.10 clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 - networkPluginName: cni nodeLabels: kops.k8s.io/kops-controller-pki: "" - kubernetes.io/role: master node-role.kubernetes.io/control-plane: "" - node-role.kubernetes.io/master: "" node.kubernetes.io/exclude-from-external-load-balancers: "" podInfraContainerImage: registry.k8s.io/pause:3.6 podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true registerSchedulable: false shutdownGracePeriod: 30s shutdownGracePeriodCriticalPods: 10s @@ -265,10 +271,13 @@ channels: - memfs://clusters.example.com/privateflannel.example.com/addons/bootstrap-channel.yaml containerdConfig: logLevel: info - version: 1.4.13 + runc: + version: 1.1.4 + version: 1.6.8 etcdManifests: - memfs://clusters.example.com/privateflannel.example.com/manifests/etcd/main-master-us-test-1a.yaml - memfs://clusters.example.com/privateflannel.example.com/manifests/etcd/events-master-us-test-1a.yaml staticManifests: - key: kube-apiserver-healthcheck path: manifests/static/kube-apiserver-healthcheck.yaml +useInstanceIDForNodeName: true diff --git a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_nodeupconfig-nodes_content b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_nodeupconfig-nodes_content index c239bd32a7..797832d98f 100644 --- a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_nodeupconfig-nodes_content +++ b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_nodeupconfig-nodes_content @@ -1,14 +1,16 @@ Assets: amd64: - - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet - - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl - - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz - - 29ef1e8635795c2a49a20a56e778f45ff163c5400a5428ca33999ed53d44e3d8@https://github.com/containerd/containerd/releases/download/v1.4.13/cri-containerd-cni-1.4.13-linux-amd64.tar.gz + - 7f9183fce12606818612ce80b6c09757452c4fb50aefea5fc5843951c5020e24@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/amd64/kubelet + - e23cc7092218c95c22d8ee36fb9499194a36ac5b5349ca476886b7edc0203885@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/amd64/kubectl + - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz + - 3a1322c18ee5ff4b9bd5af6b7b30c923a3eab8af1df05554f530ef8e2b24ac5e@https://github.com/containerd/containerd/releases/download/v1.6.8/containerd-1.6.8-linux-amd64.tar.gz + - db772be63147a4e747b4fe286c7c16a2edc4a8458bd3092ea46aaee77750e8ce@https://github.com/opencontainers/runc/releases/download/v1.1.4/runc.amd64 arm64: - - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet - - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl - - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz - - debed306ed9a4e70dcbcb228a0b3898f9730099e324f34bb0e76abbaddf7a6a7@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.13.tgz + - 69572a7b3d179d4a479aa2e0f90e2f091d8d84ef33a35422fc89975dc137a590@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/arm64/kubelet + - 24db547bbae294c5c44f2b4a777e45f0e2f3d6295eace0d0c4be2b2dfa45330d@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/arm64/kubectl + - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz + - b114e36ecce78cef9d611416c01b784a420928c82766d6df7dc02b10d9da94cd@https://github.com/containerd/containerd/releases/download/v1.6.8/containerd-1.6.8-linux-arm64.tar.gz + - dbb71e737eaef454a406ce21fd021bd8f1b35afb7635016745992bbd7c17a223@https://github.com/opencontainers/runc/releases/download/v1.1.4/runc.arm64 CAs: kubernetes-ca: | -----BEGIN CERTIFICATE----- @@ -41,19 +43,21 @@ KubeletConfig: anonymousAuth: false cgroupDriver: systemd cgroupRoot: / - cloudProvider: aws + cloudProvider: external clusterDNS: 100.64.0.10 clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 - networkPluginName: cni nodeLabels: - kubernetes.io/role: node node-role.kubernetes.io/node: "" podInfraContainerImage: registry.k8s.io/pause:3.6 podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true shutdownGracePeriod: 30s shutdownGracePeriodCriticalPods: 10s UpdatePolicy: automatic @@ -61,4 +65,7 @@ channels: - memfs://clusters.example.com/privateflannel.example.com/addons/bootstrap-channel.yaml containerdConfig: logLevel: info - version: 1.4.13 + runc: + version: 1.1.4 + version: 1.6.8 +useInstanceIDForNodeName: true diff --git a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content new file mode 100644 index 0000000000..581e2fb8ad --- /dev/null +++ b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content @@ -0,0 +1,237 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + k8s-app: aws-cloud-controller-manager + name: aws-cloud-controller-manager + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + template: + metadata: + creationTimestamp: null + labels: + k8s-app: aws-cloud-controller-manager + kops.k8s.io/managed-by: kops + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + containers: + - args: + - --allocate-node-cidrs=true + - --cluster-cidr=100.64.0.0/10 + - --cluster-name=privateflannel.example.com + - --configure-cloud-routes=false + - --leader-elect=true + - --v=2 + - --cloud-provider=aws + - --use-service-account-credentials=true + - --cloud-config=/etc/kubernetes/cloud.config + env: + - name: KUBERNETES_SERVICE_HOST + value: 127.0.0.1 + image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.25.0 + imagePullPolicy: IfNotPresent + name: aws-cloud-controller-manager + resources: + requests: + cpu: 200m + volumeMounts: + - mountPath: /etc/kubernetes/cloud.config + name: cloudconfig + readOnly: true + hostNetwork: true + nodeSelector: null + priorityClassName: system-cluster-critical + serviceAccountName: aws-cloud-controller-manager + tolerations: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + - effect: NoSchedule + key: node.kubernetes.io/not-ready + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + - effect: NoSchedule + key: node-role.kubernetes.io/master + volumes: + - hostPath: + path: /etc/kubernetes/cloud.config + type: "" + name: cloudconfig + updateStrategy: + type: RollingUpdate + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + name: aws-cloud-controller-manager + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- apiGroup: "" + kind: ServiceAccount + name: aws-cloud-controller-manager + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + name: system:cloud-controller-manager +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - '*' +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch +- apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update +- apiGroups: + - "" + resources: + - secrets + verbs: + - list + - watch +- apiGroups: + - "" + resourceNames: + - node-controller + - service-controller + - route-controller + resources: + - serviceaccounts/token + verbs: + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + name: system:cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager +subjects: +- apiGroup: "" + kind: ServiceAccount + name: aws-cloud-controller-manager + namespace: kube-system diff --git a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content new file mode 100644 index 0000000000..314275a864 --- /dev/null +++ b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content @@ -0,0 +1,785 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-external-attacher-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-external-provisioner-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list +- apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-external-resizer-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-external-snapshotter-role +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-attacher-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-provisioner-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-resizer-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-snapshotter-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-node-getter-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role +subjects: +- kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-node-role +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-node-sa + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-node + namespace: kube-system +spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + creationTimestamp: null + labels: + app: ebs-csi-node + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + kops.k8s.io/managed-by: kops + spec: + containers: + - args: + - node + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --v=2 + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.8.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=5 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1 + imagePullPolicy: IfNotPresent + name: node-driver-registrar + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.5.0 + imagePullPolicy: IfNotPresent + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + creationTimestamp: null + labels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + kops.k8s.io/managed-by: kops + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: kubernetes.io/os + operator: In + values: + - linux + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - args: + - controller + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --k8s-tag-cluster-id=privateflannel.example.com + - --extra-tags=KubernetesCluster=privateflannel.example.com + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + image: registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.8.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --v=5 + - --feature-gates=Topology=true + - --extra-create-metadata + - --leader-election=true + - --default-fstype=ext4 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-provisioner:v3.1.0 + imagePullPolicy: IfNotPresent + name: csi-provisioner + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --v=5 + - --leader-election=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-attacher:v3.4.0 + imagePullPolicy: IfNotPresent + name: csi-attacher + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --v=5 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-resizer:v1.4.0 + imagePullPolicy: IfNotPresent + name: csi-resizer + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.5.0 + imagePullPolicy: IfNotPresent + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: null + priorityClassName: system-cluster-critical + serviceAccountName: ebs-csi-controller-sa + tolerations: + - operator: Exists + topologySpreadConstraints: + - labelSelector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + - labelSelector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + volumes: + - emptyDir: {} + name: socket-dir + +--- + +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs.csi.aws.com +spec: + attachRequired: true + podInfoOnMount: false + +--- + +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-controller + namespace: kube-system +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver diff --git a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-bootstrap_content b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-bootstrap_content index e379ee0dae..ad590eb3fe 100644 --- a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-bootstrap_content @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 3de75ed2bea6a0076e6e7a310cb74fdd696dac51fb0fd0b899732835fdb4ab58 + manifestHash: 15f1202c0fbd8b3065c87b0409464ebbbe7185b38983e0baf1ce272054ff9fb4 name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: @@ -14,7 +14,7 @@ spec: version: 9.99.0 - id: k8s-1.12 manifest: coredns.addons.k8s.io/k8s-1.12.yaml - manifestHash: cd1e8f47fe52b13fee5536b0d4b4429ef256829d87a51cbc189fa0f21ff3503b + manifestHash: 6a1db11adb764a3138401cf615c57780df760e7688d4d0d94bd434d6a6b9d370 name: coredns.addons.k8s.io selector: k8s-addon: coredns.addons.k8s.io @@ -26,6 +26,13 @@ spec: selector: k8s-addon: kubelet-api.rbac.addons.k8s.io version: 9.99.0 + - id: k8s-1.23 + manifest: leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml + manifestHash: b9c91e09c0f28c9b74ff140b8395d611834c627d698846d625c10975a74a48c4 + name: leader-migration.rbac.addons.k8s.io + selector: + k8s-addon: leader-migration.rbac.addons.k8s.io + version: 9.99.0 - manifest: limit-range.addons.k8s.io/v1.5.0.yaml manifestHash: 2d55c3bc5e354e84a3730a65b42f39aba630a59dc8d32b30859fcce3d3178bc2 name: limit-range.addons.k8s.io @@ -41,15 +48,67 @@ spec: version: 9.99.0 - id: v1.15.0 manifest: storage-aws.addons.k8s.io/v1.15.0.yaml - manifestHash: 065ae832ddac8d0931e9992d6a76f43a33a36975a38003b34f4c5d86a7d42780 + manifestHash: 4e2cda50cd5048133aad1b5e28becb60f4629d3f9e09c514a2757c27998b4200 name: storage-aws.addons.k8s.io selector: k8s-addon: storage-aws.addons.k8s.io version: 9.99.0 - - id: k8s-1.12 - manifest: networking.flannel/k8s-1.12.yaml - manifestHash: 9a1dce09e6c31f0da54c1a7f593f3bb111cd9b50453f6f31a382215290794571 + - id: k8s-1.25 + manifest: networking.flannel/k8s-1.25.yaml + manifestHash: 0b0d13083ea6ee5196f49234a338d9d96e21684a622d2320bd4d9241f7b2e2d7 name: networking.flannel + prune: + kinds: + - kind: ConfigMap + labelSelector: addon.kops.k8s.io/name=networking.flannel,app.kubernetes.io/managed-by=kops + namespaces: + - kube-flannel + - kind: Service + labelSelector: addon.kops.k8s.io/name=networking.flannel,app.kubernetes.io/managed-by=kops + - kind: ServiceAccount + labelSelector: addon.kops.k8s.io/name=networking.flannel,app.kubernetes.io/managed-by=kops + namespaces: + - kube-flannel + - group: apps + kind: DaemonSet + labelSelector: addon.kops.k8s.io/name=networking.flannel,app.kubernetes.io/managed-by=kops + namespaces: + - kube-flannel + - group: apps + kind: Deployment + labelSelector: addon.kops.k8s.io/name=networking.flannel,app.kubernetes.io/managed-by=kops + - group: apps + kind: StatefulSet + labelSelector: addon.kops.k8s.io/name=networking.flannel,app.kubernetes.io/managed-by=kops + - group: policy + kind: PodDisruptionBudget + labelSelector: addon.kops.k8s.io/name=networking.flannel,app.kubernetes.io/managed-by=kops + - group: rbac.authorization.k8s.io + kind: ClusterRole + labelSelector: addon.kops.k8s.io/name=networking.flannel,app.kubernetes.io/managed-by=kops + - group: rbac.authorization.k8s.io + kind: ClusterRoleBinding + labelSelector: addon.kops.k8s.io/name=networking.flannel,app.kubernetes.io/managed-by=kops + - group: rbac.authorization.k8s.io + kind: Role + labelSelector: addon.kops.k8s.io/name=networking.flannel,app.kubernetes.io/managed-by=kops + - group: rbac.authorization.k8s.io + kind: RoleBinding + labelSelector: addon.kops.k8s.io/name=networking.flannel,app.kubernetes.io/managed-by=kops selector: role.kubernetes.io/networking: "1" version: 9.99.0 + - id: k8s-1.18 + manifest: aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml + manifestHash: 2991b9b59bbca24f083a9e6106120a0954d25d47b747c4b785a4f4c341440203 + name: aws-cloud-controller.addons.k8s.io + selector: + k8s-addon: aws-cloud-controller.addons.k8s.io + version: 9.99.0 + - id: k8s-1.17 + manifest: aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml + manifestHash: 41a7b8bd924e91ac42d61a71908578205d6292c973f53062f055ffbc3cbf8ba9 + name: aws-ebs-csi-driver.addons.k8s.io + selector: + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + version: 9.99.0 diff --git a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content index fd5b8a7c05..ad2b9fcd21 100644 --- a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content +++ b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content @@ -242,7 +242,7 @@ spec: --- -apiVersion: policy/v1beta1 +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: creationTimestamp: null diff --git a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content index 302ac399f1..a0660f1c8e 100644 --- a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content +++ b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content @@ -1,7 +1,7 @@ apiVersion: v1 data: config.yaml: | - {"cloud":"aws","configBase":"memfs://clusters.example.com/privateflannel.example.com","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["nodes.privateflannel.example.com"],"Region":"us-test-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"]}} + {"cloud":"aws","configBase":"memfs://clusters.example.com/privateflannel.example.com","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["nodes.privateflannel.example.com"],"Region":"us-test-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"],"useInstanceIDForNodeName":true}} kind: ConfigMap metadata: creationTimestamp: null diff --git a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content new file mode 100644 index 0000000000..11ed6d46fd --- /dev/null +++ b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content @@ -0,0 +1,52 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: leader-migration.rbac.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: leader-migration.rbac.addons.k8s.io + name: system::leader-locking-migration + namespace: kube-system +rules: +- apiGroups: + - coordination.k8s.io + resourceNames: + - cloud-provider-extraction-migration + resources: + - leases + verbs: + - create + - list + - get + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: leader-migration.rbac.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: leader-migration.rbac.addons.k8s.io + name: system::leader-locking-migration + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: system::leader-locking-migration +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:kube-controller-manager +- kind: ServiceAccount + name: kube-controller-manager + namespace: kube-system +- kind: ServiceAccount + name: aws-cloud-controller-manager + namespace: kube-system +- kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system diff --git a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-networking.flannel-k8s-1.12_content b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-networking.flannel-k8s-1.25_content similarity index 73% rename from tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-networking.flannel-k8s-1.12_content rename to tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-networking.flannel-k8s-1.25_content index 2e7a431e09..95206046dd 100644 --- a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-networking.flannel-k8s-1.12_content +++ b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-networking.flannel-k8s-1.25_content @@ -1,51 +1,13 @@ -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy +apiVersion: v1 +kind: Namespace metadata: - annotations: - apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default - apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default - seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default - seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default creationTimestamp: null labels: addon.kops.k8s.io/name: networking.flannel app.kubernetes.io/managed-by: kops + pod-security.kubernetes.io/enforce: privileged role.kubernetes.io/networking: "1" - name: psp.flannel.unprivileged -spec: - allowPrivilegeEscalation: false - allowedCapabilities: - - NET_ADMIN - - NET_RAW - allowedHostPaths: - - pathPrefix: /dev/net - - pathPrefix: /etc/cni/net.d - - pathPrefix: /etc/kube-flannel - - pathPrefix: /run/flannel - defaultAddCapabilities: [] - defaultAllowPrivilegeEscalation: false - fsGroup: - rule: RunAsAny - hostIPC: false - hostNetwork: true - hostPID: false - hostPorts: - - max: 65535 - min: 0 - privileged: false - readOnlyRootFilesystem: false - requiredDropCapabilities: [] - runAsUser: - rule: RunAsAny - seLinux: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - volumes: - - configMap - - secret - - emptyDir - - hostPath + name: kube-flannel --- @@ -59,14 +21,6 @@ metadata: role.kubernetes.io/networking: "1" name: flannel rules: -- apiGroups: - - extensions - resourceNames: - - psp.flannel.unprivileged - resources: - - podsecuritypolicies - verbs: - - use - apiGroups: - "" resources: @@ -105,7 +59,7 @@ roleRef: subjects: - kind: ServiceAccount name: flannel - namespace: kube-system + namespace: kube-flannel --- @@ -118,7 +72,7 @@ metadata: app.kubernetes.io/managed-by: kops role.kubernetes.io/networking: "1" name: flannel - namespace: kube-system + namespace: kube-flannel --- @@ -158,11 +112,10 @@ metadata: addon.kops.k8s.io/name: networking.flannel app: flannel app.kubernetes.io/managed-by: kops - k8s-app: flannel role.kubernetes.io/networking: "1" tier: node name: kube-flannel-cfg - namespace: kube-system + namespace: kube-flannel --- @@ -174,11 +127,10 @@ metadata: addon.kops.k8s.io/name: networking.flannel app: flannel app.kubernetes.io/managed-by: kops - k8s-app: flannel role.kubernetes.io/networking: "1" tier: node name: kube-flannel-ds - namespace: kube-system + namespace: kube-flannel spec: selector: matchLabels: @@ -217,14 +169,17 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: rancher/mirrored-flannelcni-flannel:v0.17.0 + - name: EVENT_QUEUE_DEPTH + value: "5000" + image: docker.io/rancher/mirrored-flannelcni-flannel:v0.19.2 name: kube-flannel resources: limits: - memory: 100Mi + cpu: 100m + memory: 50Mi requests: cpu: 100m - memory: 100Mi + memory: 50Mi securityContext: capabilities: add: @@ -234,8 +189,6 @@ spec: volumeMounts: - mountPath: /run/flannel name: run - - mountPath: /dev/net - name: dev-net - mountPath: /etc/kube-flannel/ name: flannel-cfg - mountPath: /run/xtables.lock @@ -248,7 +201,7 @@ spec: - /opt/cni/bin/flannel command: - cp - image: rancher/mirrored-flannelcni-flannel-cni-plugin:v1.0.1 + image: docker.io/rancher/mirrored-flannelcni-flannel-cni-plugin:v1.1.0 name: install-cni-plugin volumeMounts: - mountPath: /opt/cni/bin @@ -259,7 +212,7 @@ spec: - /etc/cni/net.d/10-flannel.conflist command: - cp - image: rancher/mirrored-flannelcni-flannel:v0.17.0 + image: docker.io/rancher/mirrored-flannelcni-flannel:v0.19.2 name: install-cni volumeMounts: - mountPath: /etc/cni/net.d @@ -269,14 +222,12 @@ spec: priorityClassName: system-node-critical serviceAccountName: flannel tolerations: - - operator: Exists + - effect: NoSchedule + operator: Exists volumes: - hostPath: path: /run/flannel name: run - - hostPath: - path: /dev/net - name: dev-net - hostPath: path: /opt/cni/bin name: cni-plugin diff --git a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content index 21efd54326..bea3e88be3 100644 --- a/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content +++ b/tests/integration/update_cluster/privateflannel/data/aws_s3_object_privateflannel.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content @@ -35,7 +35,7 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: annotations: - storageclass.kubernetes.io/is-default-class: "true" + storageclass.kubernetes.io/is-default-class: "false" creationTimestamp: null labels: addon.kops.k8s.io/name: storage-aws.addons.k8s.io @@ -50,6 +50,26 @@ volumeBindingMode: WaitForFirstConsumer --- +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: + storageclass.kubernetes.io/is-default-class: "true" + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: kops-csi-1-21 +parameters: + encrypted: "true" + type: gp3 +provisioner: ebs.csi.aws.com +volumeBindingMode: WaitForFirstConsumer + +--- + apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/tests/integration/update_cluster/privateflannel/in-v1alpha2.yaml b/tests/integration/update_cluster/privateflannel/in-v1alpha2.yaml index 4fc009bada..bb2e632582 100644 --- a/tests/integration/update_cluster/privateflannel/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/privateflannel/in-v1alpha2.yaml @@ -21,7 +21,7 @@ spec: iam: {} kubelet: anonymousAuth: false - kubernetesVersion: v1.21.0 + kubernetesVersion: v1.25.0 masterInternalName: api.internal.privateflannel.example.com masterPublicName: api.privateflannel.example.com networkCIDR: 172.20.0.0/16 diff --git a/tests/integration/update_cluster/privateflannel/kubernetes.tf b/tests/integration/update_cluster/privateflannel/kubernetes.tf index 43c054f433..2f91bbfb3d 100644 --- a/tests/integration/update_cluster/privateflannel/kubernetes.tf +++ b/tests/integration/update_cluster/privateflannel/kubernetes.tf @@ -138,11 +138,6 @@ resource "aws_autoscaling_group" "bastion-privateflannel-example-com" { propagate_at_launch = true value = "bastion.privateflannel.example.com" } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" - propagate_at_launch = true - value = "node" - } tag { key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" propagate_at_launch = true @@ -194,21 +189,11 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-privateflannel-examp propagate_at_launch = true value = "" } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" - propagate_at_launch = true - value = "master" - } tag { key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" propagate_at_launch = true value = "" } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" - propagate_at_launch = true - value = "" - } tag { key = "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" propagate_at_launch = true @@ -254,11 +239,6 @@ resource "aws_autoscaling_group" "nodes-privateflannel-example-com" { propagate_at_launch = true value = "nodes.privateflannel.example.com" } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" - propagate_at_launch = true - value = "node" - } tag { key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" propagate_at_launch = true @@ -515,7 +495,6 @@ resource "aws_launch_template" "bastion-privateflannel-example-com" { tags = { "KubernetesCluster" = "privateflannel.example.com" "Name" = "bastion.privateflannel.example.com" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/bastion" = "1" "kops.k8s.io/instancegroup" = "bastion" @@ -527,7 +506,6 @@ resource "aws_launch_template" "bastion-privateflannel-example-com" { tags = { "KubernetesCluster" = "privateflannel.example.com" "Name" = "bastion.privateflannel.example.com" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/bastion" = "1" "kops.k8s.io/instancegroup" = "bastion" @@ -537,7 +515,6 @@ resource "aws_launch_template" "bastion-privateflannel-example-com" { tags = { "KubernetesCluster" = "privateflannel.example.com" "Name" = "bastion.privateflannel.example.com" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/bastion" = "1" "kops.k8s.io/instancegroup" = "bastion" @@ -592,9 +569,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-privateflannel-example "KubernetesCluster" = "privateflannel.example.com" "Name" = "master-us-test-1a.masters.privateflannel.example.com" "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "master" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" = "" "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" "k8s.io/role/master" = "1" "kops.k8s.io/instancegroup" = "master-us-test-1a" @@ -607,9 +582,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-privateflannel-example "KubernetesCluster" = "privateflannel.example.com" "Name" = "master-us-test-1a.masters.privateflannel.example.com" "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "master" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" = "" "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" "k8s.io/role/master" = "1" "kops.k8s.io/instancegroup" = "master-us-test-1a" @@ -620,9 +593,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-privateflannel-example "KubernetesCluster" = "privateflannel.example.com" "Name" = "master-us-test-1a.masters.privateflannel.example.com" "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "master" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" = "" "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" "k8s.io/role/master" = "1" "kops.k8s.io/instancegroup" = "master-us-test-1a" @@ -673,7 +644,6 @@ resource "aws_launch_template" "nodes-privateflannel-example-com" { tags = { "KubernetesCluster" = "privateflannel.example.com" "Name" = "nodes.privateflannel.example.com" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/node" = "1" "kops.k8s.io/instancegroup" = "nodes" @@ -685,7 +655,6 @@ resource "aws_launch_template" "nodes-privateflannel-example-com" { tags = { "KubernetesCluster" = "privateflannel.example.com" "Name" = "nodes.privateflannel.example.com" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/node" = "1" "kops.k8s.io/instancegroup" = "nodes" @@ -695,7 +664,6 @@ resource "aws_launch_template" "nodes-privateflannel-example-com" { tags = { "KubernetesCluster" = "privateflannel.example.com" "Name" = "nodes.privateflannel.example.com" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/node" = "1" "kops.k8s.io/instancegroup" = "nodes" @@ -845,6 +813,22 @@ resource "aws_s3_object" "nodeupconfig-nodes" { server_side_encryption = "AES256" } +resource "aws_s3_object" "privateflannel-example-com-addons-aws-cloud-controller-addons-k8s-io-k8s-1-18" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privateflannel.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content") + key = "clusters.example.com/privateflannel.example.com/addons/aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privateflannel-example-com-addons-aws-ebs-csi-driver-addons-k8s-io-k8s-1-17" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privateflannel.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content") + key = "clusters.example.com/privateflannel.example.com/addons/aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + resource "aws_s3_object" "privateflannel-example-com-addons-bootstrap" { bucket = "testingBucket" content = file("${path.module}/data/aws_s3_object_privateflannel.example.com-addons-bootstrap_content") @@ -885,6 +869,14 @@ resource "aws_s3_object" "privateflannel-example-com-addons-kubelet-api-rbac-add server_side_encryption = "AES256" } +resource "aws_s3_object" "privateflannel-example-com-addons-leader-migration-rbac-addons-k8s-io-k8s-1-23" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privateflannel.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content") + key = "clusters.example.com/privateflannel.example.com/addons/leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + resource "aws_s3_object" "privateflannel-example-com-addons-limit-range-addons-k8s-io" { bucket = "testingBucket" content = file("${path.module}/data/aws_s3_object_privateflannel.example.com-addons-limit-range.addons.k8s.io_content") @@ -893,10 +885,10 @@ resource "aws_s3_object" "privateflannel-example-com-addons-limit-range-addons-k server_side_encryption = "AES256" } -resource "aws_s3_object" "privateflannel-example-com-addons-networking-flannel-k8s-1-12" { +resource "aws_s3_object" "privateflannel-example-com-addons-networking-flannel-k8s-1-25" { bucket = "testingBucket" - content = file("${path.module}/data/aws_s3_object_privateflannel.example.com-addons-networking.flannel-k8s-1.12_content") - key = "clusters.example.com/privateflannel.example.com/addons/networking.flannel/k8s-1.12.yaml" + content = file("${path.module}/data/aws_s3_object_privateflannel.example.com-addons-networking.flannel-k8s-1.25_content") + key = "clusters.example.com/privateflannel.example.com/addons/networking.flannel/k8s-1.25.yaml" provider = aws.files server_side_encryption = "AES256" } @@ -1181,8 +1173,10 @@ resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" { } resource "aws_subnet" "us-test-1a-privateflannel-example-com" { - availability_zone = "us-test-1a" - cidr_block = "172.20.32.0/19" + availability_zone = "us-test-1a" + cidr_block = "172.20.32.0/19" + enable_resource_name_dns_a_record_on_launch = true + private_dns_hostname_type_on_launch = "resource-name" tags = { "KubernetesCluster" = "privateflannel.example.com" "Name" = "us-test-1a.privateflannel.example.com" @@ -1196,8 +1190,10 @@ resource "aws_subnet" "us-test-1a-privateflannel-example-com" { } resource "aws_subnet" "utility-us-test-1a-privateflannel-example-com" { - availability_zone = "us-test-1a" - cidr_block = "172.20.4.0/22" + availability_zone = "us-test-1a" + cidr_block = "172.20.4.0/22" + enable_resource_name_dns_a_record_on_launch = true + private_dns_hostname_type_on_launch = "resource-name" tags = { "KubernetesCluster" = "privateflannel.example.com" "Name" = "utility-us-test-1a.privateflannel.example.com" diff --git a/upup/models/cloudup/resources/addons/networking.flannel/k8s-1.25.yaml.template b/upup/models/cloudup/resources/addons/networking.flannel/k8s-1.25.yaml.template new file mode 100644 index 0000000000..8e4ee6c73a --- /dev/null +++ b/upup/models/cloudup/resources/addons/networking.flannel/k8s-1.25.yaml.template @@ -0,0 +1,207 @@ +# Pulled and modified from: https://raw.githubusercontent.com/coreos/flannel/v0.19.2/Documentation/kube-flannel.yml +--- +kind: Namespace +apiVersion: v1 +metadata: + name: kube-flannel + labels: + pod-security.kubernetes.io/enforce: privileged +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: flannel +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: flannel +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flannel +subjects: +- kind: ServiceAccount + name: flannel + namespace: kube-flannel +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flannel + namespace: kube-flannel +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-flannel-cfg + namespace: kube-flannel + labels: + tier: node + app: flannel +data: + cni-conf.json: | + { + "name": "cbr0", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "flannel", + "delegate": { + "hairpinMode": true, + "isDefaultGateway": true + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + } + ] + } + net-conf.json: | + { + "Network": "{{ .NonMasqueradeCIDR }}", + "Backend": { + "Type": "{{ FlannelBackendType }}" + } + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-flannel-ds + namespace: kube-flannel + labels: + tier: node + app: flannel +spec: + selector: + matchLabels: + tier: node + app: flannel + template: + metadata: + labels: + tier: node + app: flannel + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + hostNetwork: true + priorityClassName: system-node-critical + tolerations: + - operator: Exists + effect: NoSchedule + serviceAccountName: flannel + initContainers: + - name: install-cni-plugin + #image: flannelcni/flannel-cni-plugin:v1.1.0 for ppc64le and mips64le (dockerhub limitations may apply) + image: docker.io/rancher/mirrored-flannelcni-flannel-cni-plugin:v1.1.0 + command: + - cp + args: + - -f + - /flannel + - /opt/cni/bin/flannel + volumeMounts: + - name: cni-plugin + mountPath: /opt/cni/bin + - name: install-cni + #image: flannelcni/flannel:v0.19.2 for ppc64le and mips64le (dockerhub limitations may apply) + image: docker.io/rancher/mirrored-flannelcni-flannel:v0.19.2 + command: + - cp + args: + - -f + - /etc/kube-flannel/cni-conf.json + - /etc/cni/net.d/10-flannel.conflist + volumeMounts: + - name: cni + mountPath: /etc/cni/net.d + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + containers: + - name: kube-flannel + #image: flannelcni/flannel:v0.19.2 for ppc64le and mips64le (dockerhub limitations may apply) + image: docker.io/rancher/mirrored-flannelcni-flannel:v0.19.2 + command: + - /opt/bin/flanneld + args: + - --ip-masq + - --kube-subnet-mgr + - --iptables-resync={{- or .Networking.Flannel.IptablesResyncSeconds "5" }} + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: false + capabilities: + add: ["NET_ADMIN", "NET_RAW"] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: EVENT_QUEUE_DEPTH + value: "5000" + volumeMounts: + - name: run + mountPath: /run/flannel + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + - name: xtables-lock + mountPath: /run/xtables.lock + volumes: + - name: run + hostPath: + path: /run/flannel + - name: cni-plugin + hostPath: + path: /opt/cni/bin + - name: cni + hostPath: + path: /etc/cni/net.d + - name: flannel-cfg + configMap: + name: kube-flannel-cfg + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go index 653ef09af4..6546cefee1 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go @@ -906,16 +906,28 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*Addon if b.Cluster.Spec.Networking.Flannel != nil { key := "networking.flannel" - { - location := key + "/k8s-1.12.yaml" - id := "k8s-1.12" + if b.IsKubernetesGTE("v1.25.0") { + id := "k8s-1.25" + location := key + "/" + id + ".yaml" - addons.Add(&channelsapi.AddonSpec{ + addon := addons.Add(&channelsapi.AddonSpec{ Name: fi.String(key), Selector: networkingSelector(), Manifest: fi.String(location), Id: id, }) + addon.BuildPrune = true + } else { + id := "k8s-1.12" + location := key + "/" + id + ".yaml" + + addon := addons.Add(&channelsapi.AddonSpec{ + Name: fi.String(key), + Selector: networkingSelector(), + Manifest: fi.String(location), + Id: id, + }) + addon.BuildPrune = true } }