From 8a40c544d2283eb747cbf19ef1e9a4f2db759151 Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Thu, 25 Aug 2022 11:19:46 +0200 Subject: [PATCH 1/2] Bump kubernetes version in karpenter e2e test --- cmd/kops/integration_test.go | 7 +- ....kube-system.sa.minimal.example.com_policy | 17 + ....kube-system.sa.minimal.example.com_policy | 17 + ....kube-system.sa.minimal.example.com_policy | 114 +++ ....kube-system.sa.minimal.example.com_policy | 81 ++ ..._policy_masters.minimal.example.com_policy | 115 --- ...odes-default.minimal.example.com_user_data | 18 +- ...-machinetype.minimal.example.com_user_data | 18 +- ...t-1a.masters.minimal.example.com_user_data | 48 +- ...mplate_nodes.minimal.example.com_user_data | 18 +- ...s_s3_object_cluster-completed.spec_content | 56 +- ...s3_object_etcd-cluster-spec-events_content | 2 +- ...s_s3_object_etcd-cluster-spec-main_content | 2 +- ...-controller.addons.k8s.io-k8s-1.18_content | 253 ++++++ ...-csi-driver.addons.k8s.io-k8s-1.17_content | 829 ++++++++++++++++++ ...nimal.example.com-addons-bootstrap_content | 29 +- ...ons-coredns.addons.k8s.io-k8s-1.12_content | 2 +- ...e.com-addons-karpenter.sh-k8s-1.19_content | 4 +- ...-controller.addons.k8s.io-k8s-1.16_content | 2 +- ...ration.rbac.addons.k8s.io-k8s-1.23_content | 52 ++ ...-storage-aws.addons.k8s.io-v1.15.0_content | 22 +- ...deupconfig-karpenter-nodes-default_content | 31 +- ...karpenter-nodes-single-machinetype_content | 31 +- ...ect_nodeupconfig-master-us-test-1a_content | 41 +- .../aws_s3_object_nodeupconfig-nodes_content | 31 +- .../update_cluster/karpenter/in-v1alpha2.yaml | 2 +- .../update_cluster/karpenter/kubernetes.tf | 158 ++-- 27 files changed, 1719 insertions(+), 281 deletions(-) create mode 100644 tests/integration/update_cluster/karpenter/data/aws_iam_role_aws-cloud-controller-manager.kube-system.sa.minimal.example.com_policy create mode 100644 tests/integration/update_cluster/karpenter/data/aws_iam_role_ebs-csi-controller-sa.kube-system.sa.minimal.example.com_policy create mode 100644 tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_aws-cloud-controller-manager.kube-system.sa.minimal.example.com_policy create mode 100644 tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_ebs-csi-controller-sa.kube-system.sa.minimal.example.com_policy create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index 0aec390332..2df0fda8c7 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -196,6 +196,10 @@ func (i integrationTest) withDefaultAddons24() *integrationTest { ) } +func (i integrationTest) withDefaults24() *integrationTest { + return i.withDefaultAddons24().withDefaultServiceAccountRoles24() +} + const ( awsAuthenticatorAddon = "authentication.aws-k8s-1.12" awsCCMAddon = "aws-cloud-controller.addons.k8s.io-k8s-1.18" @@ -797,8 +801,7 @@ func TestKarpenter(t *testing.T) { test := newIntegrationTest("minimal.example.com", "karpenter"). withOIDCDiscovery(). - withAddons(dnsControllerAddon). - withServiceAccountRole("dns-controller.kube-system", true). + withDefaults24(). withAddons("karpenter.sh-k8s-1.19"). withServiceAccountRole("karpenter.kube-system", true) test.expectTerraformFilenames = append(test.expectTerraformFilenames, diff --git a/tests/integration/update_cluster/karpenter/data/aws_iam_role_aws-cloud-controller-manager.kube-system.sa.minimal.example.com_policy b/tests/integration/update_cluster/karpenter/data/aws_iam_role_aws-cloud-controller-manager.kube-system.sa.minimal.example.com_policy new file mode 100644 index 0000000000..6c18833de4 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_iam_role_aws-cloud-controller-manager.kube-system.sa.minimal.example.com_policy @@ -0,0 +1,17 @@ +{ + "Statement": [ + { + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "discovery.example.com/minimal.example.com:sub": "system:serviceaccount:kube-system:aws-cloud-controller-manager" + } + }, + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws-test:iam::123456789012:oidc-provider/discovery.example.com/minimal.example.com" + } + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/karpenter/data/aws_iam_role_ebs-csi-controller-sa.kube-system.sa.minimal.example.com_policy b/tests/integration/update_cluster/karpenter/data/aws_iam_role_ebs-csi-controller-sa.kube-system.sa.minimal.example.com_policy new file mode 100644 index 0000000000..6bd6048b16 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_iam_role_ebs-csi-controller-sa.kube-system.sa.minimal.example.com_policy @@ -0,0 +1,17 @@ +{ + "Statement": [ + { + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "discovery.example.com/minimal.example.com:sub": "system:serviceaccount:kube-system:ebs-csi-controller-sa" + } + }, + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws-test:iam::123456789012:oidc-provider/discovery.example.com/minimal.example.com" + } + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_aws-cloud-controller-manager.kube-system.sa.minimal.example.com_policy b/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_aws-cloud-controller-manager.kube-system.sa.minimal.example.com_policy new file mode 100644 index 0000000000..829a0ea20a --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_aws-cloud-controller-manager.kube-system.sa.minimal.example.com_policy @@ -0,0 +1,114 @@ +{ + "Statement": [ + { + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "aws:RequestTag/KubernetesCluster": "minimal.example.com", + "ec2:CreateAction": [ + "CreateSecurityGroup" + ] + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:security-group/*" + ] + }, + { + "Action": [ + "ec2:CreateTags", + "ec2:DeleteTags" + ], + "Condition": { + "Null": { + "aws:RequestTag/KubernetesCluster": "true" + }, + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:security-group/*" + ] + }, + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeTags", + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "kms:DescribeKey" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:DeleteSecurityGroup", + "ec2:ModifyInstanceAttribute", + "ec2:RevokeSecurityGroupIngress", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:DeregisterTargets", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ec2:CreateSecurityGroup", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateTargetGroup" + ], + "Condition": { + "StringEquals": { + "aws:RequestTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": "ec2:CreateSecurityGroup", + "Effect": "Allow", + "Resource": "arn:aws-test:ec2:*:*:vpc/*" + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_ebs-csi-controller-sa.kube-system.sa.minimal.example.com_policy b/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_ebs-csi-controller-sa.kube-system.sa.minimal.example.com_policy new file mode 100644 index 0000000000..bf357c57d8 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_ebs-csi-controller-sa.kube-system.sa.minimal.example.com_policy @@ -0,0 +1,81 @@ +{ + "Statement": [ + { + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "aws:RequestTag/KubernetesCluster": "minimal.example.com", + "ec2:CreateAction": [ + "CreateVolume", + "CreateSnapshot" + ] + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:volume/*", + "arn:aws-test:ec2:*:*:snapshot/*" + ] + }, + { + "Action": [ + "ec2:CreateTags", + "ec2:DeleteTags" + ], + "Condition": { + "Null": { + "aws:RequestTag/KubernetesCluster": "true" + }, + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:volume/*", + "arn:aws-test:ec2:*:*:snapshot/*" + ] + }, + { + "Action": [ + "ec2:DescribeAccountAttributes", + "ec2:DescribeInstances", + "ec2:DescribeTags", + "ec2:DescribeVolumes", + "ec2:DescribeVolumesModifications" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ec2:AttachVolume", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ec2:CreateSnapshot", + "ec2:CreateVolume" + ], + "Condition": { + "StringEquals": { + "aws:RequestTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_masters.minimal.example.com_policy b/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_masters.minimal.example.com_policy index b6ffff68df..1c48ca7b58 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_masters.minimal.example.com_policy +++ b/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_masters.minimal.example.com_policy @@ -94,134 +94,19 @@ "*" ] }, - { - "Action": "ec2:CreateTags", - "Condition": { - "StringEquals": { - "aws:RequestTag/KubernetesCluster": "minimal.example.com", - "ec2:CreateAction": [ - "CreateSecurityGroup" - ] - } - }, - "Effect": "Allow", - "Resource": [ - "arn:aws-test:ec2:*:*:security-group/*" - ] - }, { "Action": [ - "ec2:CreateTags", - "ec2:DeleteTags" - ], - "Condition": { - "Null": { - "aws:RequestTag/KubernetesCluster": "true" - }, - "StringEquals": { - "aws:ResourceTag/KubernetesCluster": "minimal.example.com" - } - }, - "Effect": "Allow", - "Resource": [ - "arn:aws-test:ec2:*:*:security-group/*" - ] - }, - { - "Action": [ - "autoscaling:DescribeAutoScalingGroups", "autoscaling:DescribeAutoScalingInstances", - "autoscaling:DescribeTags", - "ec2:AttachVolume", - "ec2:AuthorizeSecurityGroupIngress", - "ec2:CreateSecurityGroup", - "ec2:CreateTags", - "ec2:DeleteRoute", - "ec2:DeleteSecurityGroup", - "ec2:DeleteVolume", "ec2:DescribeInstanceTypes", "ec2:DescribeInstances", "ec2:DescribeRegions", - "ec2:DescribeRouteTables", - "ec2:DescribeSecurityGroups", - "ec2:DescribeSubnets", "ec2:DescribeVolumes", - "ec2:DescribeVpcs", - "ec2:DetachVolume", - "ec2:ModifyInstanceAttribute", - "ec2:ModifyVolume", - "elasticloadbalancing:AddTags", - "elasticloadbalancing:CreateListener", - "elasticloadbalancing:CreateTargetGroup", - "elasticloadbalancing:DescribeListeners", - "elasticloadbalancing:DescribeLoadBalancerAttributes", - "elasticloadbalancing:DescribeLoadBalancerPolicies", - "elasticloadbalancing:DescribeLoadBalancers", - "elasticloadbalancing:DescribeTargetGroups", - "elasticloadbalancing:DescribeTargetHealth", - "elasticloadbalancing:RegisterTargets", "iam:GetServerCertificate", "iam:ListServerCertificates", - "kms:DescribeKey", "kms:GenerateRandom" ], "Effect": "Allow", "Resource": "*" - }, - { - "Action": [ - "ec2:AuthorizeSecurityGroupIngress", - "ec2:DeleteSecurityGroup", - "ec2:ModifyInstanceAttribute", - "ec2:RevokeSecurityGroupIngress", - "elasticloadbalancing:AddTags", - "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", - "elasticloadbalancing:AttachLoadBalancerToSubnets", - "elasticloadbalancing:ConfigureHealthCheck", - "elasticloadbalancing:CreateLoadBalancerListeners", - "elasticloadbalancing:CreateLoadBalancerPolicy", - "elasticloadbalancing:DeleteListener", - "elasticloadbalancing:DeleteLoadBalancer", - "elasticloadbalancing:DeleteLoadBalancerListeners", - "elasticloadbalancing:DeleteTargetGroup", - "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", - "elasticloadbalancing:DeregisterTargets", - "elasticloadbalancing:DetachLoadBalancerFromSubnets", - "elasticloadbalancing:ModifyListener", - "elasticloadbalancing:ModifyLoadBalancerAttributes", - "elasticloadbalancing:ModifyTargetGroup", - "elasticloadbalancing:RegisterInstancesWithLoadBalancer", - "elasticloadbalancing:RegisterTargets", - "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", - "elasticloadbalancing:SetLoadBalancerPoliciesOfListener" - ], - "Condition": { - "StringEquals": { - "aws:ResourceTag/KubernetesCluster": "minimal.example.com" - } - }, - "Effect": "Allow", - "Resource": "*" - }, - { - "Action": [ - "ec2:CreateSecurityGroup", - "elasticloadbalancing:CreateListener", - "elasticloadbalancing:CreateLoadBalancer", - "elasticloadbalancing:CreateTargetGroup" - ], - "Condition": { - "StringEquals": { - "aws:RequestTag/KubernetesCluster": "minimal.example.com" - } - }, - "Effect": "Allow", - "Resource": "*" - }, - { - "Action": "ec2:CreateSecurityGroup", - "Effect": "Allow", - "Resource": "arn:aws-test:ec2:*:*:vpc/*" } ], "Version": "2012-10-17" diff --git a/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-default.minimal.example.com_user_data b/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-default.minimal.example.com_user_data index 461a387218..3ed0f2dc6c 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-default.minimal.example.com_user_data +++ b/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-default.minimal.example.com_user_data @@ -125,33 +125,39 @@ ensure-install-dir cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' cloudConfig: awsEBSCSIDriver: - enabled: false + enabled: true + version: v1.8.0 manageStorageClasses: true containerRuntime: containerd containerd: logLevel: info - version: 1.4.13 + runc: + version: 1.1.3 + version: 1.6.8 docker: skipInstall: true kubeProxy: clusterCIDR: 100.96.0.0/11 cpuRequest: 100m - image: registry.k8s.io/kube-proxy:v1.21.0 + image: registry.k8s.io/kube-proxy:v1.25.0 logLevel: 2 kubelet: anonymousAuth: false cgroupDriver: systemd cgroupRoot: / - cloudProvider: aws + cloudProvider: external clusterDNS: 100.64.0.10 clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 - networkPluginName: cni podInfraContainerImage: registry.k8s.io/pause:3.6 podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true shutdownGracePeriod: 30s shutdownGracePeriodCriticalPods: 10s @@ -162,7 +168,7 @@ CloudProvider: aws ConfigBase: memfs://clusters.example.com/minimal.example.com InstanceGroupName: karpenter-nodes-default InstanceGroupRole: Node -NodeupConfigHash: QN8rK2AIAaBVnHT2avJE/9kH2/CprJgqN38gV0ME1cQ= +NodeupConfigHash: 1FfgFY6VB5afH7KFhehAAy3hUBSDD/zXVUA23C/ojJY= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-single-machinetype.minimal.example.com_user_data b/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-single-machinetype.minimal.example.com_user_data index 7ac9749a5e..be3c367f65 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-single-machinetype.minimal.example.com_user_data +++ b/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-single-machinetype.minimal.example.com_user_data @@ -125,33 +125,39 @@ ensure-install-dir cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' cloudConfig: awsEBSCSIDriver: - enabled: false + enabled: true + version: v1.8.0 manageStorageClasses: true containerRuntime: containerd containerd: logLevel: info - version: 1.4.13 + runc: + version: 1.1.3 + version: 1.6.8 docker: skipInstall: true kubeProxy: clusterCIDR: 100.96.0.0/11 cpuRequest: 100m - image: registry.k8s.io/kube-proxy:v1.21.0 + image: registry.k8s.io/kube-proxy:v1.25.0 logLevel: 2 kubelet: anonymousAuth: false cgroupDriver: systemd cgroupRoot: / - cloudProvider: aws + cloudProvider: external clusterDNS: 100.64.0.10 clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 - networkPluginName: cni podInfraContainerImage: registry.k8s.io/pause:3.6 podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true shutdownGracePeriod: 30s shutdownGracePeriodCriticalPods: 10s @@ -162,7 +168,7 @@ CloudProvider: aws ConfigBase: memfs://clusters.example.com/minimal.example.com InstanceGroupName: karpenter-nodes-single-machinetype InstanceGroupRole: Node -NodeupConfigHash: jFa5ZXlMB3owMtyf+ycLQPUeytsaEbampn2XMdSkiNM= +NodeupConfigHash: C0E1AeWAAdfmmgPAT+IqvpmDy30Yv9Gn5ncFwOk8IVE= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/karpenter/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data b/tests/integration/update_cluster/karpenter/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data index 5f5f6da22f..175b53b139 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data +++ b/tests/integration/update_cluster/karpenter/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data @@ -125,20 +125,23 @@ ensure-install-dir cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' cloudConfig: awsEBSCSIDriver: - enabled: false + enabled: true + version: v1.8.0 manageStorageClasses: true containerRuntime: containerd containerd: logLevel: info - version: 1.4.13 + runc: + version: 1.1.3 + version: 1.6.8 docker: skipInstall: true encryptionConfig: null etcdClusters: events: - version: 3.4.13 + version: 3.5.4 main: - version: 3.4.13 + version: 3.5.4 kubeAPIServer: allowPrivileged: true anonymousAuth: false @@ -147,7 +150,7 @@ kubeAPIServer: apiServerCount: 1 authorizationMode: AlwaysAllow bindAddress: 0.0.0.0 - cloudProvider: aws + cloudProvider: external enableAdmissionPlugins: - NamespaceLifecycle - LimitRanger @@ -162,7 +165,10 @@ kubeAPIServer: - https://127.0.0.1:4001 etcdServersOverrides: - /events#https://127.0.0.1:4002 - image: registry.k8s.io/kube-apiserver:v1.21.0 + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-apiserver:v1.25.0 kubeletPreferredAddressTypes: - InternalIP - Hostname @@ -184,11 +190,14 @@ kubeAPIServer: kubeControllerManager: allocateNodeCIDRs: true attachDetachReconcileSyncPeriod: 1m0s - cloudProvider: aws + cloudProvider: external clusterCIDR: 100.96.0.0/11 clusterName: minimal.example.com configureCloudRoutes: false - image: registry.k8s.io/kube-controller-manager:v1.21.0 + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-controller-manager:v1.25.0 leaderElection: leaderElect: true logLevel: 2 @@ -196,10 +205,13 @@ kubeControllerManager: kubeProxy: clusterCIDR: 100.96.0.0/11 cpuRequest: 100m - image: registry.k8s.io/kube-proxy:v1.21.0 + image: registry.k8s.io/kube-proxy:v1.25.0 logLevel: 2 kubeScheduler: - image: registry.k8s.io/kube-scheduler:v1.21.0 + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-scheduler:v1.25.0 leaderElection: leaderElect: true logLevel: 2 @@ -207,32 +219,38 @@ kubelet: anonymousAuth: false cgroupDriver: systemd cgroupRoot: / - cloudProvider: aws + cloudProvider: external clusterDNS: 100.64.0.10 clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 - networkPluginName: cni podInfraContainerImage: registry.k8s.io/pause:3.6 podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true shutdownGracePeriod: 30s shutdownGracePeriodCriticalPods: 10s masterKubelet: anonymousAuth: false cgroupDriver: systemd cgroupRoot: / - cloudProvider: aws + cloudProvider: external clusterDNS: 100.64.0.10 clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 - networkPluginName: cni podInfraContainerImage: registry.k8s.io/pause:3.6 podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true registerSchedulable: false shutdownGracePeriod: 30s shutdownGracePeriodCriticalPods: 10s @@ -244,7 +262,7 @@ CloudProvider: aws ConfigBase: memfs://clusters.example.com/minimal.example.com InstanceGroupName: master-us-test-1a InstanceGroupRole: Master -NodeupConfigHash: F56oipBerHI/IM58aPmR1lXYLb5nkwRq2LaknRGeeNQ= +NodeupConfigHash: druZrgEqRg3bQtuYmxpUASgMeDHZb+5b/fhw9+2O9a0= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/karpenter/data/aws_launch_template_nodes.minimal.example.com_user_data b/tests/integration/update_cluster/karpenter/data/aws_launch_template_nodes.minimal.example.com_user_data index a237d188a8..6e48286940 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_launch_template_nodes.minimal.example.com_user_data +++ b/tests/integration/update_cluster/karpenter/data/aws_launch_template_nodes.minimal.example.com_user_data @@ -125,33 +125,39 @@ ensure-install-dir cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' cloudConfig: awsEBSCSIDriver: - enabled: false + enabled: true + version: v1.8.0 manageStorageClasses: true containerRuntime: containerd containerd: logLevel: info - version: 1.4.13 + runc: + version: 1.1.3 + version: 1.6.8 docker: skipInstall: true kubeProxy: clusterCIDR: 100.96.0.0/11 cpuRequest: 100m - image: registry.k8s.io/kube-proxy:v1.21.0 + image: registry.k8s.io/kube-proxy:v1.25.0 logLevel: 2 kubelet: anonymousAuth: false cgroupDriver: systemd cgroupRoot: / - cloudProvider: aws + cloudProvider: external clusterDNS: 100.64.0.10 clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 - networkPluginName: cni podInfraContainerImage: registry.k8s.io/pause:3.6 podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true shutdownGracePeriod: 30s shutdownGracePeriodCriticalPods: 10s @@ -162,7 +168,7 @@ CloudProvider: aws ConfigBase: memfs://clusters.example.com/minimal.example.com InstanceGroupName: nodes InstanceGroupRole: Node -NodeupConfigHash: vbE8fon7Gz+yUJcoRvevJp6c22GnUv6MsvhgR0aUJfs= +NodeupConfigHash: T25MM8XyNbt7mlbFdFtrq5ywmHdGcC80E95ANq/+gOY= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_cluster-completed.spec_content index 92b4d90f06..ad7a10f987 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_cluster-completed.spec_content @@ -11,8 +11,17 @@ spec: channel: stable cloudConfig: awsEBSCSIDriver: - enabled: false + enabled: true + version: v1.8.0 manageStorageClasses: true + cloudControllerManager: + allocateNodeCIDRs: true + clusterCIDR: 100.64.0.0/10 + clusterName: minimal.example.com + configureCloudRoutes: false + image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.24.1 + leaderElection: + leaderElect: true cloudProvider: aws clusterDNSDomain: cluster.local configBase: memfs://clusters.example.com/minimal.example.com @@ -20,7 +29,9 @@ spec: containerRuntime: containerd containerd: logLevel: info - version: 1.4.13 + runc: + version: 1.1.3 + version: 1.6.8 dnsZone: Z1AFAKE1ZON3YO docker: skipInstall: true @@ -31,14 +42,14 @@ spec: - instanceGroup: master-us-test-1a name: us-test-1a name: main - version: 3.4.13 + version: 3.5.4 - backups: backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd/events etcdMembers: - instanceGroup: master-us-test-1a name: us-test-1a name: events - version: 3.4.13 + version: 3.5.4 externalDns: provider: dns-controller iam: @@ -55,7 +66,7 @@ spec: apiServerCount: 1 authorizationMode: AlwaysAllow bindAddress: 0.0.0.0 - cloudProvider: aws + cloudProvider: external enableAdmissionPlugins: - NamespaceLifecycle - LimitRanger @@ -70,7 +81,10 @@ spec: - https://127.0.0.1:4001 etcdServersOverrides: - /events#https://127.0.0.1:4002 - image: registry.k8s.io/kube-apiserver:v1.21.0 + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-apiserver:v1.25.0 kubeletPreferredAddressTypes: - InternalIP - Hostname @@ -92,11 +106,14 @@ spec: kubeControllerManager: allocateNodeCIDRs: true attachDetachReconcileSyncPeriod: 1m0s - cloudProvider: aws + cloudProvider: external clusterCIDR: 100.96.0.0/11 clusterName: minimal.example.com configureCloudRoutes: false - image: registry.k8s.io/kube-controller-manager:v1.21.0 + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-controller-manager:v1.25.0 leaderElection: leaderElect: true logLevel: 2 @@ -118,10 +135,13 @@ spec: kubeProxy: clusterCIDR: 100.96.0.0/11 cpuRequest: 100m - image: registry.k8s.io/kube-proxy:v1.21.0 + image: registry.k8s.io/kube-proxy:v1.25.0 logLevel: 2 kubeScheduler: - image: registry.k8s.io/kube-scheduler:v1.21.0 + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-scheduler:v1.25.0 leaderElection: leaderElect: true logLevel: 2 @@ -129,36 +149,42 @@ spec: anonymousAuth: false cgroupDriver: systemd cgroupRoot: / - cloudProvider: aws + cloudProvider: external clusterDNS: 100.64.0.10 clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 - networkPluginName: cni podInfraContainerImage: registry.k8s.io/pause:3.6 podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true shutdownGracePeriod: 30s shutdownGracePeriodCriticalPods: 10s kubernetesApiAccess: - 0.0.0.0/0 - kubernetesVersion: 1.21.0 + kubernetesVersion: 1.25.0 masterInternalName: api.internal.minimal.example.com masterKubelet: anonymousAuth: false cgroupDriver: systemd cgroupRoot: / - cloudProvider: aws + cloudProvider: external clusterDNS: 100.64.0.10 clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 - networkPluginName: cni podInfraContainerImage: registry.k8s.io/pause:3.6 podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true registerSchedulable: false shutdownGracePeriod: 30s shutdownGracePeriodCriticalPods: 10s diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_etcd-cluster-spec-events_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_etcd-cluster-spec-events_content index bb8ddb0e2e..06759fa2b2 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_etcd-cluster-spec-events_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_etcd-cluster-spec-events_content @@ -1,4 +1,4 @@ { "memberCount": 1, - "etcdVersion": "3.4.13" + "etcdVersion": "3.5.4" } diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_etcd-cluster-spec-main_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_etcd-cluster-spec-main_content index bb8ddb0e2e..06759fa2b2 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_etcd-cluster-spec-main_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_etcd-cluster-spec-main_content @@ -1,4 +1,4 @@ { "memberCount": 1, - "etcdVersion": "3.4.13" + "etcdVersion": "3.5.4" } diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content new file mode 100644 index 0000000000..eca7aec39d --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content @@ -0,0 +1,253 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + k8s-app: aws-cloud-controller-manager + name: aws-cloud-controller-manager + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + template: + metadata: + creationTimestamp: null + labels: + k8s-app: aws-cloud-controller-manager + kops.k8s.io/managed-by: kops + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + containers: + - args: + - --allocate-node-cidrs=true + - --cluster-cidr=100.64.0.0/10 + - --cluster-name=minimal.example.com + - --configure-cloud-routes=false + - --leader-elect=true + - --v=2 + - --cloud-provider=aws + - --use-service-account-credentials=true + - --cloud-config=/etc/kubernetes/cloud.config + env: + - name: KUBERNETES_SERVICE_HOST + value: 127.0.0.1 + - name: AWS_ROLE_ARN + value: arn:aws-test:iam::123456789012:role/aws-cloud-controller-manager.kube-system.sa.minimal.example.com + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: /var/run/secrets/amazonaws.com/token + image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.24.1 + imagePullPolicy: IfNotPresent + name: aws-cloud-controller-manager + resources: + requests: + cpu: 200m + volumeMounts: + - mountPath: /etc/kubernetes/cloud.config + name: cloudconfig + readOnly: true + - mountPath: /var/run/secrets/amazonaws.com/ + name: token-amazonaws-com + readOnly: true + hostNetwork: true + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 10001 + serviceAccountName: aws-cloud-controller-manager + tolerations: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + - effect: NoSchedule + key: node.kubernetes.io/not-ready + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + - effect: NoSchedule + key: node-role.kubernetes.io/master + volumes: + - hostPath: + path: /etc/kubernetes/cloud.config + type: "" + name: cloudconfig + - name: token-amazonaws-com + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: amazonaws.com + expirationSeconds: 86400 + path: token + updateStrategy: + type: RollingUpdate + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + name: aws-cloud-controller-manager + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- apiGroup: "" + kind: ServiceAccount + name: aws-cloud-controller-manager + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + name: system:cloud-controller-manager +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - '*' +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch +- apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update +- apiGroups: + - "" + resources: + - secrets + verbs: + - list + - watch +- apiGroups: + - "" + resourceNames: + - node-controller + - service-controller + - route-controller + resources: + - serviceaccounts/token + verbs: + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + name: system:cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager +subjects: +- apiGroup: "" + kind: ServiceAccount + name: aws-cloud-controller-manager + namespace: kube-system diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content new file mode 100644 index 0000000000..fc71f6c722 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content @@ -0,0 +1,829 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-external-attacher-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-external-provisioner-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list +- apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-external-resizer-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-external-snapshotter-role +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-attacher-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-provisioner-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-resizer-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-snapshotter-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-node-getter-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role +subjects: +- kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-node-role +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-node-sa + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-node + namespace: kube-system +spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + creationTimestamp: null + labels: + app: ebs-csi-node + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + kops.k8s.io/managed-by: kops + spec: + containers: + - args: + - node + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --v=2 + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.8.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=5 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1 + imagePullPolicy: IfNotPresent + name: node-driver-registrar + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.5.0 + imagePullPolicy: IfNotPresent + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-controller + namespace: kube-system +spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + creationTimestamp: null + labels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + kops.k8s.io/managed-by: kops + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - args: + - controller + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --k8s-tag-cluster-id=minimal.example.com + - --extra-tags=KubernetesCluster=minimal.example.com + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_ROLE_ARN + value: arn:aws-test:iam::123456789012:role/ebs-csi-controller-sa.kube-system.sa.minimal.example.com + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: /var/run/secrets/amazonaws.com/token + image: registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.8.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: {} + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - mountPath: /var/run/secrets/amazonaws.com/ + name: token-amazonaws-com + readOnly: true + - args: + - --csi-address=$(ADDRESS) + - --v=5 + - --feature-gates=Topology=true + - --extra-create-metadata + - --leader-election=true + - --default-fstype=ext4 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + - name: AWS_ROLE_ARN + value: arn:aws-test:iam::123456789012:role/ebs-csi-controller-sa.kube-system.sa.minimal.example.com + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: /var/run/secrets/amazonaws.com/token + image: registry.k8s.io/sig-storage/csi-provisioner:v3.1.0 + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: {} + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - mountPath: /var/run/secrets/amazonaws.com/ + name: token-amazonaws-com + readOnly: true + - args: + - --csi-address=$(ADDRESS) + - --v=5 + - --leader-election=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + - name: AWS_ROLE_ARN + value: arn:aws-test:iam::123456789012:role/ebs-csi-controller-sa.kube-system.sa.minimal.example.com + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: /var/run/secrets/amazonaws.com/token + image: registry.k8s.io/sig-storage/csi-attacher:v3.4.0 + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: {} + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - mountPath: /var/run/secrets/amazonaws.com/ + name: token-amazonaws-com + readOnly: true + - args: + - --csi-address=$(ADDRESS) + - --v=5 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + - name: AWS_ROLE_ARN + value: arn:aws-test:iam::123456789012:role/ebs-csi-controller-sa.kube-system.sa.minimal.example.com + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: /var/run/secrets/amazonaws.com/token + image: registry.k8s.io/sig-storage/csi-resizer:v1.4.0 + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: {} + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - mountPath: /var/run/secrets/amazonaws.com/ + name: token-amazonaws-com + readOnly: true + - args: + - --csi-address=/csi/csi.sock + env: + - name: AWS_ROLE_ARN + value: arn:aws-test:iam::123456789012:role/ebs-csi-controller-sa.kube-system.sa.minimal.example.com + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: /var/run/secrets/amazonaws.com/token + image: registry.k8s.io/sig-storage/livenessprobe:v2.5.0 + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: {} + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /var/run/secrets/amazonaws.com/ + name: token-amazonaws-com + readOnly: true + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 10001 + serviceAccountName: ebs-csi-controller-sa + topologySpreadConstraints: + - labelSelector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + - labelSelector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + volumes: + - emptyDir: {} + name: socket-dir + - name: token-amazonaws-com + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: amazonaws.com + expirationSeconds: 86400 + path: token + +--- + +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs.csi.aws.com +spec: + attachRequired: true + podInfoOnMount: false + +--- + +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.8.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-controller + namespace: kube-system +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content index 73ae70a153..4e4caa94ca 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content @@ -6,7 +6,7 @@ spec: addons: - id: k8s-1.16 manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: ed4392aad695145055951a9b4988e5e4cd377faad5ab42a34a01af27141524aa + manifestHash: 29870be2a6c0f1962f8a2ea34b30899636b2f47dfeb7b13f1ae46cc262dff3a1 name: kops-controller.addons.k8s.io needsRollingUpdate: control-plane selector: @@ -14,7 +14,7 @@ spec: version: 9.99.0 - id: k8s-1.12 manifest: coredns.addons.k8s.io/k8s-1.12.yaml - manifestHash: cd1e8f47fe52b13fee5536b0d4b4429ef256829d87a51cbc189fa0f21ff3503b + manifestHash: 6a1db11adb764a3138401cf615c57780df760e7688d4d0d94bd434d6a6b9d370 name: coredns.addons.k8s.io selector: k8s-addon: coredns.addons.k8s.io @@ -26,6 +26,13 @@ spec: selector: k8s-addon: kubelet-api.rbac.addons.k8s.io version: 9.99.0 + - id: k8s-1.23 + manifest: leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml + manifestHash: b9c91e09c0f28c9b74ff140b8395d611834c627d698846d625c10975a74a48c4 + name: leader-migration.rbac.addons.k8s.io + selector: + k8s-addon: leader-migration.rbac.addons.k8s.io + version: 9.99.0 - manifest: limit-range.addons.k8s.io/v1.5.0.yaml manifestHash: 2d55c3bc5e354e84a3730a65b42f39aba630a59dc8d32b30859fcce3d3178bc2 name: limit-range.addons.k8s.io @@ -41,14 +48,28 @@ spec: version: 9.99.0 - id: v1.15.0 manifest: storage-aws.addons.k8s.io/v1.15.0.yaml - manifestHash: 065ae832ddac8d0931e9992d6a76f43a33a36975a38003b34f4c5d86a7d42780 + manifestHash: 4e2cda50cd5048133aad1b5e28becb60f4629d3f9e09c514a2757c27998b4200 name: storage-aws.addons.k8s.io selector: k8s-addon: storage-aws.addons.k8s.io version: 9.99.0 + - id: k8s-1.18 + manifest: aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml + manifestHash: 95b07c1842cf1050aa3ba57d864dbd4bc15f2bdd58a120955e34dfc7f8e3f03a + name: aws-cloud-controller.addons.k8s.io + selector: + k8s-addon: aws-cloud-controller.addons.k8s.io + version: 9.99.0 + - id: k8s-1.17 + manifest: aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml + manifestHash: 44d1bdde6aead8f31385caa491f59079d6953b4cfcdb05781ce5f19e9652f0aa + name: aws-ebs-csi-driver.addons.k8s.io + selector: + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + version: 9.99.0 - id: k8s-1.19 manifest: karpenter.sh/k8s-1.19.yaml - manifestHash: 456d5115709499873dbe4fbee5f558795a51575a1bab9eff17f2242d8f24d578 + manifestHash: 41eb3f8deddc9ffa10651af1269a7d5048f8b2c5bb79460f235a24ad0290343c name: karpenter.sh selector: k8s-addon: karpenter.sh diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content index fd5b8a7c05..ad2b9fcd21 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content @@ -242,7 +242,7 @@ spec: --- -apiVersion: policy/v1beta1 +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: creationTimestamp: null diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content index f5218a2563..f394b4d1b4 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content @@ -1038,6 +1038,8 @@ spec: - env: - name: AWS_ENI_LIMITED_POD_DENSITY value: "true" + - name: AWS_NODE_NAME_CONVENTION + value: resource-name - name: AWS_REGION value: us-test-1 - name: CLUSTER_NAME @@ -1373,7 +1375,7 @@ webhooks: --- -apiVersion: policy/v1beta1 +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: creationTimestamp: null diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content index fa0174988e..d4d68a8f82 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content @@ -1,7 +1,7 @@ apiVersion: v1 data: config.yaml: | - {"cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["nodes.minimal.example.com"],"Region":"us-test-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"]}} + {"cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["nodes.minimal.example.com"],"Region":"us-test-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"],"useInstanceIDForNodeName":true}} kind: ConfigMap metadata: creationTimestamp: null diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content new file mode 100644 index 0000000000..11ed6d46fd --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content @@ -0,0 +1,52 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: leader-migration.rbac.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: leader-migration.rbac.addons.k8s.io + name: system::leader-locking-migration + namespace: kube-system +rules: +- apiGroups: + - coordination.k8s.io + resourceNames: + - cloud-provider-extraction-migration + resources: + - leases + verbs: + - create + - list + - get + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: leader-migration.rbac.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: leader-migration.rbac.addons.k8s.io + name: system::leader-locking-migration + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: system::leader-locking-migration +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:kube-controller-manager +- kind: ServiceAccount + name: kube-controller-manager + namespace: kube-system +- kind: ServiceAccount + name: aws-cloud-controller-manager + namespace: kube-system +- kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content index 21efd54326..bea3e88be3 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content @@ -35,7 +35,7 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: annotations: - storageclass.kubernetes.io/is-default-class: "true" + storageclass.kubernetes.io/is-default-class: "false" creationTimestamp: null labels: addon.kops.k8s.io/name: storage-aws.addons.k8s.io @@ -50,6 +50,26 @@ volumeBindingMode: WaitForFirstConsumer --- +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: + storageclass.kubernetes.io/is-default-class: "true" + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: kops-csi-1-21 +parameters: + encrypted: "true" + type: gp3 +provisioner: ebs.csi.aws.com +volumeBindingMode: WaitForFirstConsumer + +--- + apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-karpenter-nodes-default_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-karpenter-nodes-default_content index 0185d46037..faabaa3652 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-karpenter-nodes-default_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-karpenter-nodes-default_content @@ -1,14 +1,16 @@ Assets: amd64: - - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet - - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl - - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz - - 29ef1e8635795c2a49a20a56e778f45ff163c5400a5428ca33999ed53d44e3d8@https://github.com/containerd/containerd/releases/download/v1.4.13/cri-containerd-cni-1.4.13-linux-amd64.tar.gz + - 7f9183fce12606818612ce80b6c09757452c4fb50aefea5fc5843951c5020e24@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/amd64/kubelet + - e23cc7092218c95c22d8ee36fb9499194a36ac5b5349ca476886b7edc0203885@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/amd64/kubectl + - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz + - 3a1322c18ee5ff4b9bd5af6b7b30c923a3eab8af1df05554f530ef8e2b24ac5e@https://github.com/containerd/containerd/releases/download/v1.6.8/containerd-1.6.8-linux-amd64.tar.gz + - 6e8b24be90fffce6b025d254846da9d2ca6d65125f9139b6354bab0272253d01@https://github.com/opencontainers/runc/releases/download/v1.1.3/runc.amd64 arm64: - - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet - - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl - - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz - - debed306ed9a4e70dcbcb228a0b3898f9730099e324f34bb0e76abbaddf7a6a7@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.13.tgz + - 69572a7b3d179d4a479aa2e0f90e2f091d8d84ef33a35422fc89975dc137a590@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/arm64/kubelet + - 24db547bbae294c5c44f2b4a777e45f0e2f3d6295eace0d0c4be2b2dfa45330d@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/arm64/kubectl + - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz + - b114e36ecce78cef9d611416c01b784a420928c82766d6df7dc02b10d9da94cd@https://github.com/containerd/containerd/releases/download/v1.6.8/containerd-1.6.8-linux-arm64.tar.gz + - 00c9ad161a77a01d9dcbd25b1d76fa9822e57d8e4abf26ba8907c98f6bcfcd0f@https://github.com/opencontainers/runc/releases/download/v1.1.3/runc.arm64 CAs: kubernetes-ca: | -----BEGIN CERTIFICATE----- @@ -41,20 +43,22 @@ KubeletConfig: anonymousAuth: false cgroupDriver: systemd cgroupRoot: / - cloudProvider: aws + cloudProvider: external clusterDNS: 100.64.0.10 clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 - networkPluginName: cni nodeLabels: karpenter.sh/provisioner-name: karpenter-nodes-default - kubernetes.io/role: node node-role.kubernetes.io/node: "" podInfraContainerImage: registry.k8s.io/pause:3.6 podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true shutdownGracePeriod: 30s shutdownGracePeriodCriticalPods: 10s UpdatePolicy: automatic @@ -62,4 +66,7 @@ channels: - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml containerdConfig: logLevel: info - version: 1.4.13 + runc: + version: 1.1.3 + version: 1.6.8 +useInstanceIDForNodeName: true diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-karpenter-nodes-single-machinetype_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-karpenter-nodes-single-machinetype_content index daa46b6c9b..b0ec1a35d3 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-karpenter-nodes-single-machinetype_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-karpenter-nodes-single-machinetype_content @@ -1,14 +1,16 @@ Assets: amd64: - - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet - - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl - - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz - - 29ef1e8635795c2a49a20a56e778f45ff163c5400a5428ca33999ed53d44e3d8@https://github.com/containerd/containerd/releases/download/v1.4.13/cri-containerd-cni-1.4.13-linux-amd64.tar.gz + - 7f9183fce12606818612ce80b6c09757452c4fb50aefea5fc5843951c5020e24@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/amd64/kubelet + - e23cc7092218c95c22d8ee36fb9499194a36ac5b5349ca476886b7edc0203885@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/amd64/kubectl + - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz + - 3a1322c18ee5ff4b9bd5af6b7b30c923a3eab8af1df05554f530ef8e2b24ac5e@https://github.com/containerd/containerd/releases/download/v1.6.8/containerd-1.6.8-linux-amd64.tar.gz + - 6e8b24be90fffce6b025d254846da9d2ca6d65125f9139b6354bab0272253d01@https://github.com/opencontainers/runc/releases/download/v1.1.3/runc.amd64 arm64: - - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet - - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl - - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz - - debed306ed9a4e70dcbcb228a0b3898f9730099e324f34bb0e76abbaddf7a6a7@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.13.tgz + - 69572a7b3d179d4a479aa2e0f90e2f091d8d84ef33a35422fc89975dc137a590@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/arm64/kubelet + - 24db547bbae294c5c44f2b4a777e45f0e2f3d6295eace0d0c4be2b2dfa45330d@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/arm64/kubectl + - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz + - b114e36ecce78cef9d611416c01b784a420928c82766d6df7dc02b10d9da94cd@https://github.com/containerd/containerd/releases/download/v1.6.8/containerd-1.6.8-linux-arm64.tar.gz + - 00c9ad161a77a01d9dcbd25b1d76fa9822e57d8e4abf26ba8907c98f6bcfcd0f@https://github.com/opencontainers/runc/releases/download/v1.1.3/runc.arm64 CAs: kubernetes-ca: | -----BEGIN CERTIFICATE----- @@ -41,20 +43,22 @@ KubeletConfig: anonymousAuth: false cgroupDriver: systemd cgroupRoot: / - cloudProvider: aws + cloudProvider: external clusterDNS: 100.64.0.10 clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 - networkPluginName: cni nodeLabels: karpenter.sh/provisioner-name: karpenter-nodes-single-machinetype - kubernetes.io/role: node node-role.kubernetes.io/node: "" podInfraContainerImage: registry.k8s.io/pause:3.6 podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true shutdownGracePeriod: 30s shutdownGracePeriodCriticalPods: 10s UpdatePolicy: automatic @@ -62,4 +66,7 @@ channels: - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml containerdConfig: logLevel: info - version: 1.4.13 + runc: + version: 1.1.3 + version: 1.6.8 +useInstanceIDForNodeName: true diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-master-us-test-1a_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-master-us-test-1a_content index f5aa57071a..0c50d652ac 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-master-us-test-1a_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-master-us-test-1a_content @@ -7,7 +7,7 @@ APIServerConfig: apiServerCount: 1 authorizationMode: AlwaysAllow bindAddress: 0.0.0.0 - cloudProvider: aws + cloudProvider: external enableAdmissionPlugins: - NamespaceLifecycle - LimitRanger @@ -22,7 +22,10 @@ APIServerConfig: - https://127.0.0.1:4001 etcdServersOverrides: - /events#https://127.0.0.1:4002 - image: registry.k8s.io/kube-apiserver:v1.21.0 + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-apiserver:v1.25.0 kubeletPreferredAddressTypes: - InternalIP - Hostname @@ -52,17 +55,19 @@ APIServerConfig: -----END RSA PUBLIC KEY----- Assets: amd64: - - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet - - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl - - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz - - 29ef1e8635795c2a49a20a56e778f45ff163c5400a5428ca33999ed53d44e3d8@https://github.com/containerd/containerd/releases/download/v1.4.13/cri-containerd-cni-1.4.13-linux-amd64.tar.gz + - 7f9183fce12606818612ce80b6c09757452c4fb50aefea5fc5843951c5020e24@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/amd64/kubelet + - e23cc7092218c95c22d8ee36fb9499194a36ac5b5349ca476886b7edc0203885@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/amd64/kubectl + - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz + - 3a1322c18ee5ff4b9bd5af6b7b30c923a3eab8af1df05554f530ef8e2b24ac5e@https://github.com/containerd/containerd/releases/download/v1.6.8/containerd-1.6.8-linux-amd64.tar.gz + - 6e8b24be90fffce6b025d254846da9d2ca6d65125f9139b6354bab0272253d01@https://github.com/opencontainers/runc/releases/download/v1.1.3/runc.amd64 - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 arm64: - - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet - - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl - - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz - - debed306ed9a4e70dcbcb228a0b3898f9730099e324f34bb0e76abbaddf7a6a7@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.13.tgz + - 69572a7b3d179d4a479aa2e0f90e2f091d8d84ef33a35422fc89975dc137a590@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/arm64/kubelet + - 24db547bbae294c5c44f2b4a777e45f0e2f3d6295eace0d0c4be2b2dfa45330d@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/arm64/kubectl + - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz + - b114e36ecce78cef9d611416c01b784a420928c82766d6df7dc02b10d9da94cd@https://github.com/containerd/containerd/releases/download/v1.6.8/containerd-1.6.8-linux-arm64.tar.gz + - 00c9ad161a77a01d9dcbd25b1d76fa9822e57d8e4abf26ba8907c98f6bcfcd0f@https://github.com/opencontainers/runc/releases/download/v1.1.3/runc.arm64 - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 CAs: @@ -220,7 +225,7 @@ CAs: ClusterName: minimal.example.com FileAssets: - content: | - apiVersion: kubescheduler.config.k8s.io/v1beta1 + apiVersion: kubescheduler.config.k8s.io/v1beta2 clientConnection: kubeconfig: /var/lib/kube-scheduler/kubeconfig kind: KubeSchedulerConfiguration @@ -241,22 +246,23 @@ KubeletConfig: anonymousAuth: false cgroupDriver: systemd cgroupRoot: / - cloudProvider: aws + cloudProvider: external clusterDNS: 100.64.0.10 clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 - networkPluginName: cni nodeLabels: kops.k8s.io/kops-controller-pki: "" - kubernetes.io/role: master node-role.kubernetes.io/control-plane: "" - node-role.kubernetes.io/master: "" node.kubernetes.io/exclude-from-external-load-balancers: "" podInfraContainerImage: registry.k8s.io/pause:3.6 podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true registerSchedulable: false shutdownGracePeriod: 30s shutdownGracePeriodCriticalPods: 10s @@ -265,10 +271,13 @@ channels: - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml containerdConfig: logLevel: info - version: 1.4.13 + runc: + version: 1.1.3 + version: 1.6.8 etcdManifests: - memfs://clusters.example.com/minimal.example.com/manifests/etcd/main-master-us-test-1a.yaml - memfs://clusters.example.com/minimal.example.com/manifests/etcd/events-master-us-test-1a.yaml staticManifests: - key: kube-apiserver-healthcheck path: manifests/static/kube-apiserver-healthcheck.yaml +useInstanceIDForNodeName: true diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-nodes_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-nodes_content index 2985309644..342d43f9b5 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-nodes_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-nodes_content @@ -1,14 +1,16 @@ Assets: amd64: - - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet - - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl - - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz - - 29ef1e8635795c2a49a20a56e778f45ff163c5400a5428ca33999ed53d44e3d8@https://github.com/containerd/containerd/releases/download/v1.4.13/cri-containerd-cni-1.4.13-linux-amd64.tar.gz + - 7f9183fce12606818612ce80b6c09757452c4fb50aefea5fc5843951c5020e24@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/amd64/kubelet + - e23cc7092218c95c22d8ee36fb9499194a36ac5b5349ca476886b7edc0203885@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/amd64/kubectl + - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz + - 3a1322c18ee5ff4b9bd5af6b7b30c923a3eab8af1df05554f530ef8e2b24ac5e@https://github.com/containerd/containerd/releases/download/v1.6.8/containerd-1.6.8-linux-amd64.tar.gz + - 6e8b24be90fffce6b025d254846da9d2ca6d65125f9139b6354bab0272253d01@https://github.com/opencontainers/runc/releases/download/v1.1.3/runc.amd64 arm64: - - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet - - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl - - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz - - debed306ed9a4e70dcbcb228a0b3898f9730099e324f34bb0e76abbaddf7a6a7@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.13.tgz + - 69572a7b3d179d4a479aa2e0f90e2f091d8d84ef33a35422fc89975dc137a590@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/arm64/kubelet + - 24db547bbae294c5c44f2b4a777e45f0e2f3d6295eace0d0c4be2b2dfa45330d@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/arm64/kubectl + - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz + - b114e36ecce78cef9d611416c01b784a420928c82766d6df7dc02b10d9da94cd@https://github.com/containerd/containerd/releases/download/v1.6.8/containerd-1.6.8-linux-arm64.tar.gz + - 00c9ad161a77a01d9dcbd25b1d76fa9822e57d8e4abf26ba8907c98f6bcfcd0f@https://github.com/opencontainers/runc/releases/download/v1.1.3/runc.arm64 CAs: kubernetes-ca: | -----BEGIN CERTIFICATE----- @@ -41,19 +43,21 @@ KubeletConfig: anonymousAuth: false cgroupDriver: systemd cgroupRoot: / - cloudProvider: aws + cloudProvider: external clusterDNS: 100.64.0.10 clusterDomain: cluster.local enableDebuggingHandlers: true evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 - networkPluginName: cni nodeLabels: - kubernetes.io/role: node node-role.kubernetes.io/node: "" podInfraContainerImage: registry.k8s.io/pause:3.6 podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true shutdownGracePeriod: 30s shutdownGracePeriodCriticalPods: 10s UpdatePolicy: automatic @@ -61,4 +65,7 @@ channels: - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml containerdConfig: logLevel: info - version: 1.4.13 + runc: + version: 1.1.3 + version: 1.6.8 +useInstanceIDForNodeName: true diff --git a/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml b/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml index 1dcf35ca7e..142d89191c 100644 --- a/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml @@ -24,7 +24,7 @@ spec: enabled: true kubelet: anonymousAuth: false - kubernetesVersion: v1.21.0 + kubernetesVersion: v1.25.0 masterInternalName: api.internal.minimal.example.com masterPublicName: api.minimal.example.com networkCIDR: 172.20.0.0/16 diff --git a/tests/integration/update_cluster/karpenter/kubernetes.tf b/tests/integration/update_cluster/karpenter/kubernetes.tf index b748cb8414..b0216fed06 100644 --- a/tests/integration/update_cluster/karpenter/kubernetes.tf +++ b/tests/integration/update_cluster/karpenter/kubernetes.tf @@ -1,25 +1,29 @@ locals { - cluster_name = "minimal.example.com" - iam_openid_connect_provider_arn = aws_iam_openid_connect_provider.minimal-example-com.arn - iam_openid_connect_provider_issuer = "discovery.example.com/minimal.example.com" - kube-system-dns-controller_role_arn = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.arn - kube-system-dns-controller_role_name = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.name - kube-system-karpenter_role_arn = aws_iam_role.karpenter-kube-system-sa-minimal-example-com.arn - kube-system-karpenter_role_name = aws_iam_role.karpenter-kube-system-sa-minimal-example-com.name - master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id] - master_security_group_ids = [aws_security_group.masters-minimal-example-com.id] - masters_role_arn = aws_iam_role.masters-minimal-example-com.arn - masters_role_name = aws_iam_role.masters-minimal-example-com.name - node_autoscaling_group_ids = [aws_autoscaling_group.nodes-minimal-example-com.id] - node_security_group_ids = [aws_security_group.nodes-minimal-example-com.id] - node_subnet_ids = [aws_subnet.us-test-1a-minimal-example-com.id] - nodes_role_arn = aws_iam_role.nodes-minimal-example-com.arn - nodes_role_name = aws_iam_role.nodes-minimal-example-com.name - region = "us-test-1" - route_table_public_id = aws_route_table.minimal-example-com.id - subnet_us-test-1a_id = aws_subnet.us-test-1a-minimal-example-com.id - vpc_cidr_block = aws_vpc.minimal-example-com.cidr_block - vpc_id = aws_vpc.minimal-example-com.id + cluster_name = "minimal.example.com" + iam_openid_connect_provider_arn = aws_iam_openid_connect_provider.minimal-example-com.arn + iam_openid_connect_provider_issuer = "discovery.example.com/minimal.example.com" + kube-system-aws-cloud-controller-manager_role_arn = aws_iam_role.aws-cloud-controller-manager-kube-system-sa-minimal-example-com.arn + kube-system-aws-cloud-controller-manager_role_name = aws_iam_role.aws-cloud-controller-manager-kube-system-sa-minimal-example-com.name + kube-system-dns-controller_role_arn = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.arn + kube-system-dns-controller_role_name = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.name + kube-system-ebs-csi-controller-sa_role_arn = aws_iam_role.ebs-csi-controller-sa-kube-system-sa-minimal-example-com.arn + kube-system-ebs-csi-controller-sa_role_name = aws_iam_role.ebs-csi-controller-sa-kube-system-sa-minimal-example-com.name + kube-system-karpenter_role_arn = aws_iam_role.karpenter-kube-system-sa-minimal-example-com.arn + kube-system-karpenter_role_name = aws_iam_role.karpenter-kube-system-sa-minimal-example-com.name + master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id] + master_security_group_ids = [aws_security_group.masters-minimal-example-com.id] + masters_role_arn = aws_iam_role.masters-minimal-example-com.arn + masters_role_name = aws_iam_role.masters-minimal-example-com.name + node_autoscaling_group_ids = [aws_autoscaling_group.nodes-minimal-example-com.id] + node_security_group_ids = [aws_security_group.nodes-minimal-example-com.id] + node_subnet_ids = [aws_subnet.us-test-1a-minimal-example-com.id] + nodes_role_arn = aws_iam_role.nodes-minimal-example-com.arn + nodes_role_name = aws_iam_role.nodes-minimal-example-com.name + region = "us-test-1" + route_table_public_id = aws_route_table.minimal-example-com.id + subnet_us-test-1a_id = aws_subnet.us-test-1a-minimal-example-com.id + vpc_cidr_block = aws_vpc.minimal-example-com.cidr_block + vpc_id = aws_vpc.minimal-example-com.id } output "cluster_name" { @@ -34,6 +38,14 @@ output "iam_openid_connect_provider_issuer" { value = "discovery.example.com/minimal.example.com" } +output "kube-system-aws-cloud-controller-manager_role_arn" { + value = aws_iam_role.aws-cloud-controller-manager-kube-system-sa-minimal-example-com.arn +} + +output "kube-system-aws-cloud-controller-manager_role_name" { + value = aws_iam_role.aws-cloud-controller-manager-kube-system-sa-minimal-example-com.name +} + output "kube-system-dns-controller_role_arn" { value = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.arn } @@ -42,6 +54,14 @@ output "kube-system-dns-controller_role_name" { value = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.name } +output "kube-system-ebs-csi-controller-sa_role_arn" { + value = aws_iam_role.ebs-csi-controller-sa-kube-system-sa-minimal-example-com.arn +} + +output "kube-system-ebs-csi-controller-sa_role_name" { + value = aws_iam_role.ebs-csi-controller-sa-kube-system-sa-minimal-example-com.name +} + output "kube-system-karpenter_role_arn" { value = aws_iam_role.karpenter-kube-system-sa-minimal-example-com.arn } @@ -142,21 +162,11 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" propagate_at_launch = true value = "" } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" - propagate_at_launch = true - value = "master" - } tag { key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" propagate_at_launch = true value = "" } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" - propagate_at_launch = true - value = "" - } tag { key = "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" propagate_at_launch = true @@ -202,11 +212,6 @@ resource "aws_autoscaling_group" "nodes-minimal-example-com" { propagate_at_launch = true value = "nodes.minimal.example.com" } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" - propagate_at_launch = true - value = "node" - } tag { key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" propagate_at_launch = true @@ -293,6 +298,18 @@ resource "aws_iam_openid_connect_provider" "minimal-example-com" { url = "https://discovery.example.com/minimal.example.com" } +resource "aws_iam_role" "aws-cloud-controller-manager-kube-system-sa-minimal-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_aws-cloud-controller-manager.kube-system.sa.minimal.example.com_policy") + name = "aws-cloud-controller-manager.kube-system.sa.minimal.example.com" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "aws-cloud-controller-manager.kube-system.sa.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "service-account.kops.k8s.io/name" = "aws-cloud-controller-manager" + "service-account.kops.k8s.io/namespace" = "kube-system" + } +} + resource "aws_iam_role" "dns-controller-kube-system-sa-minimal-example-com" { assume_role_policy = file("${path.module}/data/aws_iam_role_dns-controller.kube-system.sa.minimal.example.com_policy") name = "dns-controller.kube-system.sa.minimal.example.com" @@ -305,6 +322,18 @@ resource "aws_iam_role" "dns-controller-kube-system-sa-minimal-example-com" { } } +resource "aws_iam_role" "ebs-csi-controller-sa-kube-system-sa-minimal-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_ebs-csi-controller-sa.kube-system.sa.minimal.example.com_policy") + name = "ebs-csi-controller-sa.kube-system.sa.minimal.example.com" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "ebs-csi-controller-sa.kube-system.sa.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "service-account.kops.k8s.io/name" = "ebs-csi-controller-sa" + "service-account.kops.k8s.io/namespace" = "kube-system" + } +} + resource "aws_iam_role" "karpenter-kube-system-sa-minimal-example-com" { assume_role_policy = file("${path.module}/data/aws_iam_role_karpenter.kube-system.sa.minimal.example.com_policy") name = "karpenter.kube-system.sa.minimal.example.com" @@ -337,12 +366,24 @@ resource "aws_iam_role" "nodes-minimal-example-com" { } } +resource "aws_iam_role_policy" "aws-cloud-controller-manager-kube-system-sa-minimal-example-com" { + name = "aws-cloud-controller-manager.kube-system.sa.minimal.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_aws-cloud-controller-manager.kube-system.sa.minimal.example.com_policy") + role = aws_iam_role.aws-cloud-controller-manager-kube-system-sa-minimal-example-com.name +} + resource "aws_iam_role_policy" "dns-controller-kube-system-sa-minimal-example-com" { name = "dns-controller.kube-system.sa.minimal.example.com" policy = file("${path.module}/data/aws_iam_role_policy_dns-controller.kube-system.sa.minimal.example.com_policy") role = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.name } +resource "aws_iam_role_policy" "ebs-csi-controller-sa-kube-system-sa-minimal-example-com" { + name = "ebs-csi-controller-sa.kube-system.sa.minimal.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_ebs-csi-controller-sa.kube-system.sa.minimal.example.com_policy") + role = aws_iam_role.ebs-csi-controller-sa-kube-system-sa-minimal-example-com.name +} + resource "aws_iam_role_policy" "karpenter-kube-system-sa-minimal-example-com" { name = "karpenter.kube-system.sa.minimal.example.com" policy = file("${path.module}/data/aws_iam_role_policy_karpenter.kube-system.sa.minimal.example.com_policy") @@ -422,7 +463,6 @@ resource "aws_launch_template" "karpenter-nodes-default-minimal-example-com" { "KubernetesCluster" = "minimal.example.com" "Name" = "karpenter-nodes-default.minimal.example.com" "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes-default" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/node" = "1" "kops.k8s.io/instancegroup" = "karpenter-nodes-default" @@ -435,7 +475,6 @@ resource "aws_launch_template" "karpenter-nodes-default-minimal-example-com" { "KubernetesCluster" = "minimal.example.com" "Name" = "karpenter-nodes-default.minimal.example.com" "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes-default" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/node" = "1" "kops.k8s.io/instancegroup" = "karpenter-nodes-default" @@ -446,7 +485,6 @@ resource "aws_launch_template" "karpenter-nodes-default-minimal-example-com" { "KubernetesCluster" = "minimal.example.com" "Name" = "karpenter-nodes-default.minimal.example.com" "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes-default" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/node" = "1" "kops.k8s.io/instancegroup" = "karpenter-nodes-default" @@ -497,7 +535,6 @@ resource "aws_launch_template" "karpenter-nodes-single-machinetype-minimal-examp "KubernetesCluster" = "minimal.example.com" "Name" = "karpenter-nodes-single-machinetype.minimal.example.com" "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes-single-machinetype" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/node" = "1" "kops.k8s.io/instancegroup" = "karpenter-nodes-single-machinetype" @@ -510,7 +547,6 @@ resource "aws_launch_template" "karpenter-nodes-single-machinetype-minimal-examp "KubernetesCluster" = "minimal.example.com" "Name" = "karpenter-nodes-single-machinetype.minimal.example.com" "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes-single-machinetype" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/node" = "1" "kops.k8s.io/instancegroup" = "karpenter-nodes-single-machinetype" @@ -521,7 +557,6 @@ resource "aws_launch_template" "karpenter-nodes-single-machinetype-minimal-examp "KubernetesCluster" = "minimal.example.com" "Name" = "karpenter-nodes-single-machinetype.minimal.example.com" "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes-single-machinetype" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/node" = "1" "kops.k8s.io/instancegroup" = "karpenter-nodes-single-machinetype" @@ -577,9 +612,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" { "KubernetesCluster" = "minimal.example.com" "Name" = "master-us-test-1a.masters.minimal.example.com" "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "master" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" = "" "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" "k8s.io/role/master" = "1" "kops.k8s.io/instancegroup" = "master-us-test-1a" @@ -592,9 +625,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" { "KubernetesCluster" = "minimal.example.com" "Name" = "master-us-test-1a.masters.minimal.example.com" "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "master" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" = "" "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" "k8s.io/role/master" = "1" "kops.k8s.io/instancegroup" = "master-us-test-1a" @@ -605,9 +636,7 @@ resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" { "KubernetesCluster" = "minimal.example.com" "Name" = "master-us-test-1a.masters.minimal.example.com" "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "master" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" = "" "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" "k8s.io/role/master" = "1" "kops.k8s.io/instancegroup" = "master-us-test-1a" @@ -658,7 +687,6 @@ resource "aws_launch_template" "nodes-minimal-example-com" { tags = { "KubernetesCluster" = "minimal.example.com" "Name" = "nodes.minimal.example.com" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/node" = "1" "kops.k8s.io/instancegroup" = "nodes" @@ -670,7 +698,6 @@ resource "aws_launch_template" "nodes-minimal-example-com" { tags = { "KubernetesCluster" = "minimal.example.com" "Name" = "nodes.minimal.example.com" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/node" = "1" "kops.k8s.io/instancegroup" = "nodes" @@ -680,7 +707,6 @@ resource "aws_launch_template" "nodes-minimal-example-com" { tags = { "KubernetesCluster" = "minimal.example.com" "Name" = "nodes.minimal.example.com" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" "k8s.io/role/node" = "1" "kops.k8s.io/instancegroup" = "nodes" @@ -790,6 +816,22 @@ resource "aws_s3_object" "manifests-static-kube-apiserver-healthcheck" { server_side_encryption = "AES256" } +resource "aws_s3_object" "minimal-example-com-addons-aws-cloud-controller-addons-k8s-io-k8s-1-18" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content") + key = "clusters.example.com/minimal.example.com/addons/aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "minimal-example-com-addons-aws-ebs-csi-driver-addons-k8s-io-k8s-1-17" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content") + key = "clusters.example.com/minimal.example.com/addons/aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + resource "aws_s3_object" "minimal-example-com-addons-bootstrap" { bucket = "testingBucket" content = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-bootstrap_content") @@ -838,6 +880,14 @@ resource "aws_s3_object" "minimal-example-com-addons-kubelet-api-rbac-addons-k8s server_side_encryption = "AES256" } +resource "aws_s3_object" "minimal-example-com-addons-leader-migration-rbac-addons-k8s-io-k8s-1-23" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content") + key = "clusters.example.com/minimal.example.com/addons/leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + resource "aws_s3_object" "minimal-example-com-addons-limit-range-addons-k8s-io" { bucket = "testingBucket" content = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-limit-range.addons.k8s.io_content") @@ -1035,8 +1085,10 @@ resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-udp-1 } resource "aws_subnet" "us-test-1a-minimal-example-com" { - availability_zone = "us-test-1a" - cidr_block = "172.20.32.0/19" + availability_zone = "us-test-1a" + cidr_block = "172.20.32.0/19" + enable_resource_name_dns_a_record_on_launch = true + private_dns_hostname_type_on_launch = "resource-name" tags = { "KubernetesCluster" = "minimal.example.com" "Name" = "us-test-1a.minimal.example.com" From 679135462daaf505c41d6c56485204c43c729e93 Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Thu, 25 Aug 2022 11:16:40 +0200 Subject: [PATCH 2/2] Map up kubelet config and add startup taints to karpeneter provisioner --- ...odes-default.minimal.example.com_user_data | 2 +- ...nimal.example.com-addons-bootstrap_content | 2 +- ...e.com-addons-karpenter.sh-k8s-1.19_content | 11 +++++++++++ ...deupconfig-karpenter-nodes-default_content | 4 ++++ .../update_cluster/karpenter/in-v1alpha2.yaml | 5 +++++ .../karpenter.sh/k8s-1.19.yaml.template | 19 +++++++++++++++++++ 6 files changed, 41 insertions(+), 2 deletions(-) diff --git a/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-default.minimal.example.com_user_data b/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-default.minimal.example.com_user_data index 3ed0f2dc6c..84a65d038a 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-default.minimal.example.com_user_data +++ b/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes-default.minimal.example.com_user_data @@ -168,7 +168,7 @@ CloudProvider: aws ConfigBase: memfs://clusters.example.com/minimal.example.com InstanceGroupName: karpenter-nodes-default InstanceGroupRole: Node -NodeupConfigHash: 1FfgFY6VB5afH7KFhehAAy3hUBSDD/zXVUA23C/ojJY= +NodeupConfigHash: IsrI1UToKBMNZ5BZejHeUUKJ2lszE+TvvFeYc9UOlwA= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content index 4e4caa94ca..a1802bc9ae 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content @@ -69,7 +69,7 @@ spec: version: 9.99.0 - id: k8s-1.19 manifest: karpenter.sh/k8s-1.19.yaml - manifestHash: 41eb3f8deddc9ffa10651af1269a7d5048f8b2c5bb79460f235a24ad0290343c + manifestHash: cdcc6eea95393014d4b1d416a4a8e1b0ca997f2336e7c21a5488ad809b4463c9 name: karpenter.sh selector: k8s-addon: karpenter.sh diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content index f394b4d1b4..2ff46a01b3 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content @@ -1404,6 +1404,11 @@ metadata: spec: consolidation: enabled: true + kubeletConfiguration: + maxPods: 50 + systemReserved: + cpu: 500m + memory: 1G provider: launchTemplate: karpenter-nodes-default.minimal.example.com subnetSelector: @@ -1422,6 +1427,9 @@ spec: operator: In values: - c5.large + startupTaints: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized --- @@ -1455,3 +1463,6 @@ spec: operator: In values: - t2.medium + startupTaints: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-karpenter-nodes-default_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-karpenter-nodes-default_content index faabaa3652..33c0fd74c1 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-karpenter-nodes-default_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_nodeupconfig-karpenter-nodes-default_content @@ -53,6 +53,7 @@ KubeletConfig: InTreePluginAWSUnregister: "true" kubeconfigPath: /var/lib/kubelet/kubeconfig logLevel: 2 + maxPods: 50 nodeLabels: karpenter.sh/provisioner-name: karpenter-nodes-default node-role.kubernetes.io/node: "" @@ -61,6 +62,9 @@ KubeletConfig: protectKernelDefaults: true shutdownGracePeriod: 30s shutdownGracePeriodCriticalPods: 10s + systemReserved: + cpu: 500m + memory: 1G UpdatePolicy: automatic channels: - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml diff --git a/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml b/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml index 142d89191c..24647b3efa 100644 --- a/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml @@ -111,6 +111,11 @@ metadata: kops.k8s.io/cluster: minimal.example.com spec: manager: Karpenter + kubelet: + maxPods: 50 + systemReserved: + cpu: 500m + memory: 1G associatePublicIp: true image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 role: Node diff --git a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template index 6e08e223d4..04c11a5851 100644 --- a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template +++ b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template @@ -1142,6 +1142,20 @@ metadata: spec: consolidation: enabled: true +{{ with $spec.Kubelet }} +{{ if or .MaxPods .SystemReserved }} + kubeletConfiguration: + {{ if .MaxPods }} + maxPods: {{ .MaxPods }} + {{ end }} + {{ if .SystemReserved }} + systemReserved: + {{ range $key, $val := .SystemReserved}} + {{ $key }}: "{{ $val }}" + {{ end }} + {{ end }} +{{ end }} +{{ end }} requirements: - key: karpenter.sh/capacity-type operator: In @@ -1164,7 +1178,12 @@ spec: {{ if $taint.value }} value: "{{ $taint.value }}" {{ end }} + {{ end }} {{ end }} +{{ if $.ExternalCloudControllerManager }} + startupTaints: + - key: node.cloudprovider.kubernetes.io/uninitialized + effect: NoSchedule {{ end }} {{ with $spec.NodeLabels }} labels: