diff --git a/channels/alpha b/channels/alpha index aaf5c6b844..1dc6948e90 100644 --- a/channels/alpha +++ b/channels/alpha @@ -61,11 +61,19 @@ spec: - name: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20230502 providerID: aws architectureID: amd64 - kubernetesVersion: ">=1.18.0" + kubernetesVersion: ">=1.18.0 <1.27.0" - name: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-arm64-server-20230502 providerID: aws architectureID: arm64 - kubernetesVersion: ">=1.20.0" + kubernetesVersion: ">=1.20.0 <1.27.0" + - name: 099720109477/ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-20230601 + providerID: aws + architectureID: amd64 + kubernetesVersion: ">=1.27.0" + - name: 099720109477/ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-arm64-server-20230601 + providerID: aws + architectureID: arm64 + kubernetesVersion: ">=1.27.0" - name: cos-cloud/cos-stable-65-10323-99-0 providerID: gce architectureID: amd64 @@ -77,11 +85,19 @@ spec: - name: ubuntu-os-cloud/ubuntu-2004-focal-v20230302 providerID: gce architectureID: amd64 - kubernetesVersion: ">=1.18.0" + kubernetesVersion: ">=1.18.0 <1.27.0" + - name: ubuntu-os-cloud/ubuntu-2204-jammy-v20230606 + providerID: gce + architectureID: amd64 + kubernetesVersion: ">=1.27.0" - name: Canonical:0001-com-ubuntu-server-focal:20_04-lts-gen2:20.04.202303020 providerID: azure architectureID: amd64 - kubernetesVersion: ">=1.20.0" + kubernetesVersion: ">=1.20.0 <1.27.0" + - name: Canonical:0001-com-ubuntu-server-jammy:22_04-lts-gen2:22.04.202306010 + providerID: azure + architectureID: amd64 + kubernetesVersion: ">=1.27.0" cluster: kubernetesVersion: v1.5.8 networking: diff --git a/channels/stable b/channels/stable index 138720feb8..7f67de5570 100644 --- a/channels/stable +++ b/channels/stable @@ -61,11 +61,19 @@ spec: - name: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20230502 providerID: aws architectureID: amd64 - kubernetesVersion: ">=1.18.0" + kubernetesVersion: ">=1.18.0 <1.27.0" - name: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-arm64-server-20230502 providerID: aws architectureID: arm64 - kubernetesVersion: ">=1.20.0" + kubernetesVersion: ">=1.20.0 <1.27.0" + - name: 099720109477/ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-20230601 + providerID: aws + architectureID: amd64 + kubernetesVersion: ">=1.27.0" + - name: 099720109477/ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-arm64-server-20230601 + providerID: aws + architectureID: arm64 + kubernetesVersion: ">=1.27.0" - name: cos-cloud/cos-stable-65-10323-99-0 providerID: gce architectureID: amd64 @@ -77,11 +85,19 @@ spec: - name: ubuntu-os-cloud/ubuntu-2004-focal-v20230302 providerID: gce architectureID: amd64 - kubernetesVersion: ">=1.18.0" + kubernetesVersion: ">=1.18.0 <1.27.0" + - name: ubuntu-os-cloud/ubuntu-2204-jammy-v20230606 + providerID: gce + architectureID: amd64 + kubernetesVersion: ">=1.27.0" - name: Canonical:0001-com-ubuntu-server-focal:20_04-lts-gen2:20.04.202303020 providerID: azure architectureID: amd64 - kubernetesVersion: ">=1.20.0" + kubernetesVersion: ">=1.20.0 <1.27.0" + - name: Canonical:0001-com-ubuntu-server-jammy:22_04-lts-gen2:22.04.202306010 + providerID: azure + architectureID: amd64 + kubernetesVersion: ">=1.27.0" cluster: kubernetesVersion: v1.5.8 networking: diff --git a/cmd/kops/create_cluster_integration_test.go b/cmd/kops/create_cluster_integration_test.go index 13c2647349..447645f296 100644 --- a/cmd/kops/create_cluster_integration_test.go +++ b/cmd/kops/create_cluster_integration_test.go @@ -51,6 +51,7 @@ func TestCreateClusterMinimal(t *testing.T) { runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.24", "v1alpha2") runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.25", "v1alpha2") runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.26", "v1alpha2") + runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.27", "v1alpha2") runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.26-arm64", "v1alpha2") runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.26-irsa", "v1alpha2") } diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index 48b640a547..75aa39a69e 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -279,6 +279,19 @@ func TestMinimal_v1_26(t *testing.T) { runTestTerraformAWS(t) } +// TestMinimal runs the test on a minimum configuration +func TestMinimal_v1_27(t *testing.T) { + t.Setenv("KOPS_RUN_TOO_NEW_VERSION", "1") + + newIntegrationTest("minimal.example.com", "minimal-1.27"). + withAddons( + awsEBSCSIAddon, + dnsControllerAddon, + awsCCMAddon, + ). + runTestTerraformAWS(t) +} + // TestMinimal_NoneDNS runs the test on a minimum configuration with --dns=none func TestMinimal_NoneDNS(t *testing.T) { t.Setenv("KOPS_RUN_TOO_NEW_VERSION", "1") diff --git a/docs/releases/1.27-NOTES.md b/docs/releases/1.27-NOTES.md index d47f3a25eb..afad70f7a0 100644 --- a/docs/releases/1.27-NOTES.md +++ b/docs/releases/1.27-NOTES.md @@ -6,6 +6,8 @@ This is a document to gather the release notes prior to the release. # Significant changes +* **The default image has been updated to [Ubuntu 22.04 (Jammy)](https://kops.sigs.k8s.io/operations/images/#ubuntu-2204-jammy)**. + * The default retention duration for the etcd backups is now set to 90 days. This behaviour can be overridden by setting `spec.etcdClusters[*].manager.backupRetentionDays` in the cluster spec. diff --git a/pkg/apis/kops/channel.go b/pkg/apis/kops/channel.go index d2e16c5b0d..1817bb4a4c 100644 --- a/pkg/apis/kops/channel.go +++ b/pkg/apis/kops/channel.go @@ -358,10 +358,13 @@ func RecommendedKubernetesVersion(c *Channel, kopsVersionString string) *semver. // Returns true if the given image name has the stable or alpha channel images prefix. Otherwise false. func (c *Channel) HasUpstreamImagePrefix(image string) bool { return strings.HasPrefix(image, "kope.io/k8s-") || + strings.HasPrefix(image, "099720109477/ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-") || strings.HasPrefix(image, "099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-") || strings.HasPrefix(image, "cos-cloud/cos-stable-") || strings.HasPrefix(image, "ubuntu-os-cloud/ubuntu-2004-focal-") || - strings.HasPrefix(image, "Canonical:0001-com-ubuntu-server-focal:20_04-lts-gen2:") + strings.HasPrefix(image, "ubuntu-os-cloud/ubuntu-2204-jammy-") || + strings.HasPrefix(image, "Canonical:0001-com-ubuntu-server-focal:20_04-lts-gen2:") || + strings.HasPrefix(image, "Canonical:0001-com-ubuntu-server-jammy:22_04-lts-gen2:") } // GetPackageVersion returns the version for the package, or an error if could not be found. diff --git a/pkg/util/templater/templater_test.go b/pkg/util/templater/templater_test.go index 81fb2128a6..d926d8ed54 100644 --- a/pkg/util/templater/templater_test.go +++ b/pkg/util/templater/templater_test.go @@ -101,7 +101,7 @@ func TestRenderChannelFunctions(t *testing.T) { }, { Context: map[string]interface{}{}, - Template: `{{ ChannelRecommendedImage "aws" "1.19.2" "arm64" }}`, + Template: `{{ ChannelRecommendedImage "aws" "1.20.2" "arm64" }}`, Expected: "099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-arm64-server-20210315", }, } diff --git a/tests/integration/channel/integration_test.go b/tests/integration/channel/integration_test.go index 0156b20bc6..ea418f5ef0 100644 --- a/tests/integration/channel/integration_test.go +++ b/tests/integration/channel/integration_test.go @@ -241,7 +241,7 @@ func TestFindImage(t *testing.T) { ExpectedImage: "099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20210315", }, { - KubernetesVersion: "1.19.1", + KubernetesVersion: "1.20.1", Architecture: "arm64", ExpectedImage: "099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-arm64-server-20210315", }, diff --git a/tests/integration/channel/simple/channel.yaml b/tests/integration/channel/simple/channel.yaml index 14cbae8487..b60649c856 100644 --- a/tests/integration/channel/simple/channel.yaml +++ b/tests/integration/channel/simple/channel.yaml @@ -31,11 +31,19 @@ spec: - name: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20210315 providerID: aws architectureID: amd64 - kubernetesVersion: ">=1.18.0" + kubernetesVersion: ">=1.18.0 <1.27.0" - name: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-arm64-server-20210315 providerID: aws architectureID: arm64 - kubernetesVersion: ">=1.18.0" + kubernetesVersion: ">=1.20.0 <1.27.0" + - name: 099720109477/ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-20230601 + providerID: aws + architectureID: amd64 + kubernetesVersion: ">=1.27.0" + - name: 099720109477/ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-arm64-server-20230601 + providerID: aws + architectureID: arm64 + kubernetesVersion: ">=1.27.0" cluster: kubernetesVersion: v1.4.7 networking: diff --git a/tests/integration/create_cluster/minimal-1.27/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.27/expected-v1alpha2.yaml new file mode 100644 index 0000000000..853bbb8a53 --- /dev/null +++ b/tests/integration/create_cluster/minimal-1.27/expected-v1alpha2.yaml @@ -0,0 +1,95 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + name: minimal.example.com +spec: + api: + dns: {} + authorization: + rbac: {} + channel: stable + cloudProvider: aws + configBase: memfs://tests/minimal.example.com + etcdClusters: + - cpuRequest: 200m + etcdMembers: + - encryptedVolume: true + instanceGroup: control-plane-us-test-1a + name: a + manager: + backupRetentionDays: 90 + memoryRequest: 100Mi + name: main + - cpuRequest: 100m + etcdMembers: + - encryptedVolume: true + instanceGroup: control-plane-us-test-1a + name: a + manager: + backupRetentionDays: 90 + memoryRequest: 100Mi + name: events + iam: + allowContainerRegistry: true + legacy: false + kubelet: + anonymousAuth: false + kubernetesApiAccess: + - 0.0.0.0/0 + - ::/0 + kubernetesVersion: v1.27.0 + masterPublicName: api.minimal.example.com + networkCIDR: 172.20.0.0/16 + networking: + cni: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + - ::/0 + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Public + zone: us-test-1a + topology: + dns: + type: Public + masters: public + nodes: public + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: minimal.example.com + name: control-plane-us-test-1a +spec: + image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-20230601 + machineType: m3.medium + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: minimal.example.com + name: nodes-us-test-1a +spec: + image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-20230601 + machineType: t2.medium + maxSize: 1 + minSize: 1 + role: Node + subnets: + - us-test-1a diff --git a/tests/integration/create_cluster/minimal-1.27/options.yaml b/tests/integration/create_cluster/minimal-1.27/options.yaml new file mode 100644 index 0000000000..21fd13e746 --- /dev/null +++ b/tests/integration/create_cluster/minimal-1.27/options.yaml @@ -0,0 +1,6 @@ +ClusterName: minimal.example.com +Zones: +- us-test-1a +CloudProvider: aws +Networking: cni +KubernetesVersion: v1.27.0 \ No newline at end of file diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_iam_role_masters.minimal.example.com_policy b/tests/integration/update_cluster/minimal-1.27/data/aws_iam_role_masters.minimal.example.com_policy new file mode 100644 index 0000000000..66d5de1d5a --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_iam_role_masters.minimal.example.com_policy @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { "Service": "ec2.amazonaws.com"}, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_iam_role_nodes.minimal.example.com_policy b/tests/integration/update_cluster/minimal-1.27/data/aws_iam_role_nodes.minimal.example.com_policy new file mode 100644 index 0000000000..66d5de1d5a --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_iam_role_nodes.minimal.example.com_policy @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { "Service": "ec2.amazonaws.com"}, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_iam_role_policy_masters.minimal.example.com_policy b/tests/integration/update_cluster/minimal-1.27/data/aws_iam_role_policy_masters.minimal.example.com_policy new file mode 100644 index 0000000000..a203beedf9 --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_iam_role_policy_masters.minimal.example.com_policy @@ -0,0 +1,272 @@ +{ + "Statement": [ + { + "Action": "ec2:AttachVolume", + "Condition": { + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "minimal.example.com", + "aws:ResourceTag/k8s.io/role/master": "1" + } + }, + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "s3:Get*" + ], + "Effect": "Allow", + "Resource": "arn:aws-test:s3:::placeholder-read-bucket/tests/minimal.example.com/*" + }, + { + "Action": [ + "s3:GetObject", + "s3:DeleteObject", + "s3:DeleteObjectVersion", + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": "arn:aws-test:s3:::placeholder-write-bucket/tests/minimal.example.com/backups/etcd/main/*" + }, + { + "Action": [ + "s3:GetObject", + "s3:DeleteObject", + "s3:DeleteObjectVersion", + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": "arn:aws-test:s3:::placeholder-write-bucket/tests/minimal.example.com/backups/etcd/events/*" + }, + { + "Action": [ + "s3:GetBucketLocation", + "s3:GetEncryptionConfiguration", + "s3:ListBucket", + "s3:ListBucketVersions" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:s3:::placeholder-read-bucket" + ] + }, + { + "Action": [ + "s3:GetBucketLocation", + "s3:GetEncryptionConfiguration", + "s3:ListBucket", + "s3:ListBucketVersions" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:s3:::placeholder-write-bucket" + ] + }, + { + "Action": [ + "route53:ChangeResourceRecordSets", + "route53:ListResourceRecordSets", + "route53:GetHostedZone" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:route53:::hostedzone/Z1AFAKE1ZON3YO" + ] + }, + { + "Action": [ + "route53:GetChange" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:route53:::change/*" + ] + }, + { + "Action": [ + "route53:ListHostedZones", + "route53:ListTagsForResource" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "aws:RequestTag/KubernetesCluster": "minimal.example.com", + "ec2:CreateAction": [ + "CreateVolume", + "CreateSnapshot" + ] + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:volume/*", + "arn:aws-test:ec2:*:*:snapshot/*" + ] + }, + { + "Action": [ + "ec2:CreateTags", + "ec2:DeleteTags" + ], + "Condition": { + "Null": { + "aws:RequestTag/KubernetesCluster": "true" + }, + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:volume/*", + "arn:aws-test:ec2:*:*:snapshot/*" + ] + }, + { + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "aws:RequestTag/KubernetesCluster": "minimal.example.com", + "ec2:CreateAction": [ + "CreateSecurityGroup" + ] + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:security-group/*" + ] + }, + { + "Action": [ + "ec2:CreateTags", + "ec2:DeleteTags" + ], + "Condition": { + "Null": { + "aws:RequestTag/KubernetesCluster": "true" + }, + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:security-group/*" + ] + }, + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeScalingActivities", + "autoscaling:DescribeTags", + "ec2:DescribeAccountAttributes", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeTags", + "ec2:DescribeVolumes", + "ec2:DescribeVolumesModifications", + "ec2:DescribeVpcs", + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:DescribeRepositories", + "ecr:GetAuthorizationToken", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:ListImages", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "iam:GetServerCertificate", + "iam:ListServerCertificates", + "kms:DescribeKey", + "kms:GenerateRandom" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume", + "ec2:RevokeSecurityGroupIngress", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:DeregisterTargets", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ec2:CreateSecurityGroup", + "ec2:CreateSnapshot", + "ec2:CreateVolume", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateTargetGroup" + ], + "Condition": { + "StringEquals": { + "aws:RequestTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": "ec2:CreateSecurityGroup", + "Effect": "Allow", + "Resource": "arn:aws-test:ec2:*:*:vpc/*" + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_iam_role_policy_nodes.minimal.example.com_policy b/tests/integration/update_cluster/minimal-1.27/data/aws_iam_role_policy_nodes.minimal.example.com_policy new file mode 100644 index 0000000000..b6eaf07f36 --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_iam_role_policy_nodes.minimal.example.com_policy @@ -0,0 +1,37 @@ +{ + "Statement": [ + { + "Action": [ + "s3:GetBucketLocation", + "s3:GetEncryptionConfiguration", + "s3:ListBucket", + "s3:ListBucketVersions" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:s3:::placeholder-read-bucket" + ] + }, + { + "Action": [ + "autoscaling:DescribeAutoScalingInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:DescribeRepositories", + "ecr:GetAuthorizationToken", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:ListImages", + "iam:GetServerCertificate", + "iam:ListServerCertificates", + "kms:GenerateRandom" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key b/tests/integration/update_cluster/minimal-1.27/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key new file mode 100644 index 0000000000..81cb012783 --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data b/tests/integration/update_cluster/minimal-1.27/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data new file mode 100644 index 0000000000..03281c0c0a --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data @@ -0,0 +1,265 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 +NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 +NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 +NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + +export AWS_REGION=us-test-1 + + + + +sysctl -w net.core.rmem_max=16777216 || true +sysctl -w net.core.wmem_max=16777216 || true +sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true +sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, urls +download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return 0 + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return 0 + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: + manageStorageClasses: true +encryptionConfig: null +etcdClusters: + events: + cpuRequest: 100m + manager: + backupRetentionDays: 90 + env: + - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION + value: 90d + memoryRequest: 100Mi + version: 3.5.9 + main: + cpuRequest: 200m + manager: + backupRetentionDays: 90 + env: + - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION + value: 90d + memoryRequest: 100Mi + version: 3.5.9 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: Node,RBAC + bindAddress: 0.0.0.0 + cloudProvider: external + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + featureGates: + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-apiserver:v1.27.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.minimal.example.com + serviceAccountJWKSURI: https://api.internal.minimal.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: external + clusterCIDR: 100.96.0.0/11 + clusterName: minimal.example.com + configureCloudRoutes: false + featureGates: + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-controller-manager:v1.27.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeScheduler: + featureGates: + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-scheduler:v1.27.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + InTreePluginAWSUnregister: "true" + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + podInfraContainerImage: registry.k8s.io/pause:3.6 + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + registerSchedulable: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s +masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + InTreePluginAWSUnregister: "true" + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + podInfraContainerImage: registry.k8s.io/pause:3.6 + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + registerSchedulable: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s + +__EOF_CLUSTER_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +CloudProvider: aws +ClusterName: minimal.example.com +ConfigBase: memfs://tests/minimal.example.com +InstanceGroupName: master-us-test-1a +InstanceGroupRole: ControlPlane +NodeupConfigHash: wSQyjFNHgxGXvEYgZ68/2kU0AwR+RvoUacjV2f/2KBA= + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_launch_template_nodes.minimal.example.com_user_data b/tests/integration/update_cluster/minimal-1.27/data/aws_launch_template_nodes.minimal.example.com_user_data new file mode 100644 index 0000000000..d2daa0f5a2 --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_launch_template_nodes.minimal.example.com_user_data @@ -0,0 +1,184 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 +NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 +NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 +NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + +export AWS_REGION=us-test-1 + + + + +sysctl -w net.core.rmem_max=16777216 || true +sysctl -w net.core.wmem_max=16777216 || true +sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true +sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, urls +download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return 0 + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return 0 + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: + manageStorageClasses: true +kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + InTreePluginAWSUnregister: "true" + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + podInfraContainerImage: registry.k8s.io/pause:3.6 + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + registerSchedulable: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s + +__EOF_CLUSTER_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +CloudProvider: aws +ClusterName: minimal.example.com +ConfigServer: + CACertificates: | + -----BEGIN CERTIFICATE----- + MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU + BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw + ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD + SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1 + jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG + MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA + MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8 + tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w== + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU + BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw + OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD + SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7 + WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG + MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn + MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA + 9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw== + -----END CERTIFICATE----- + servers: + - https://kops-controller.internal.minimal.example.com:3988/ +InstanceGroupName: nodes +InstanceGroupRole: Node +NodeupConfigHash: u0pUte4rAuplx3LL4SC2zlCV9ppPlsYWZCR/CoGQXBw= + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_cluster-completed.spec_content new file mode 100644 index 0000000000..39f7de0703 --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_cluster-completed.spec_content @@ -0,0 +1,221 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + name: minimal.example.com +spec: + api: + dns: {} + authorization: + rbac: {} + channel: stable + cloudConfig: + awsEBSCSIDriver: + enabled: true + version: v1.14.1 + manageStorageClasses: true + cloudControllerManager: + allocateNodeCIDRs: true + clusterCIDR: 100.64.0.0/10 + clusterName: minimal.example.com + configureCloudRoutes: false + image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.26.0 + leaderElection: + leaderElect: true + cloudProvider: aws + clusterDNSDomain: cluster.local + configBase: memfs://tests/minimal.example.com + configStore: memfs://tests/minimal.example.com + containerRuntime: containerd + containerd: + logLevel: info + runc: + version: 1.1.5 + version: 1.6.20 + dnsZone: Z1AFAKE1ZON3YO + docker: + skipInstall: true + etcdClusters: + - backups: + backupStore: memfs://tests/minimal.example.com/backups/etcd/main + cpuRequest: 200m + etcdMembers: + - encryptedVolume: true + instanceGroup: master-us-test-1a + name: a + manager: + backupRetentionDays: 90 + env: + - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION + value: 90d + memoryRequest: 100Mi + name: main + version: 3.5.9 + - backups: + backupStore: memfs://tests/minimal.example.com/backups/etcd/events + cpuRequest: 100m + etcdMembers: + - encryptedVolume: true + instanceGroup: master-us-test-1a + name: a + manager: + backupRetentionDays: 90 + env: + - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION + value: 90d + memoryRequest: 100Mi + name: events + version: 3.5.9 + externalDns: + provider: dns-controller + iam: + allowContainerRegistry: true + legacy: false + keyStore: memfs://tests/minimal.example.com/pki + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: Node,RBAC + bindAddress: 0.0.0.0 + cloudProvider: external + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + featureGates: + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-apiserver:v1.27.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.minimal.example.com + serviceAccountJWKSURI: https://api.internal.minimal.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: external + clusterCIDR: 100.96.0.0/11 + clusterName: minimal.example.com + configureCloudRoutes: false + featureGates: + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-controller-manager:v1.27.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeDNS: + cacheMaxConcurrent: 150 + cacheMaxSize: 1000 + cpuRequest: 100m + domain: cluster.local + memoryLimit: 170Mi + memoryRequest: 70Mi + nodeLocalDNS: + cpuRequest: 25m + enabled: false + image: registry.k8s.io/dns/k8s-dns-node-cache:1.22.20 + memoryRequest: 5Mi + provider: CoreDNS + serverIP: 100.64.0.10 + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + image: registry.k8s.io/kube-proxy:v1.27.0 + logLevel: 2 + kubeScheduler: + featureGates: + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-scheduler:v1.27.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + InTreePluginAWSUnregister: "true" + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + podInfraContainerImage: registry.k8s.io/pause:3.6 + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + registerSchedulable: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s + kubernetesApiAccess: + - 0.0.0.0/0 + - ::/0 + kubernetesVersion: 1.27.0 + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + InTreePluginAWSUnregister: "true" + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + podInfraContainerImage: registry.k8s.io/pause:3.6 + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + registerSchedulable: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s + masterPublicName: api.minimal.example.com + networkCIDR: 172.20.0.0/16 + networking: + cni: {} + nonMasqueradeCIDR: 100.64.0.0/10 + podCIDR: 100.96.0.0/11 + secretStore: memfs://tests/minimal.example.com/secrets + serviceClusterIPRange: 100.64.0.0/13 + sshAccess: + - 0.0.0.0/0 + - ::/0 + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Public + zone: us-test-1a + topology: + dns: + type: Public + masters: public + nodes: public diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_etcd-cluster-spec-events_content b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_etcd-cluster-spec-events_content new file mode 100644 index 0000000000..3bde2f95b6 --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_etcd-cluster-spec-events_content @@ -0,0 +1,4 @@ +{ + "memberCount": 1, + "etcdVersion": "3.5.9" +} diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_etcd-cluster-spec-main_content b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_etcd-cluster-spec-main_content new file mode 100644 index 0000000000..3bde2f95b6 --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_etcd-cluster-spec-main_content @@ -0,0 +1,4 @@ +{ + "memberCount": 1, + "etcdVersion": "3.5.9" +} diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_kops-version.txt_content b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_kops-version.txt_content new file mode 100644 index 0000000000..b7340298dc --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_kops-version.txt_content @@ -0,0 +1 @@ +1.21.0-alpha.1 diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content new file mode 100644 index 0000000000..fb92c18102 --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content @@ -0,0 +1,140 @@ +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + labels: + k8s-app: etcd-manager-events + name: etcd-manager-events + namespace: kube-system +spec: + containers: + - command: + - /bin/sh + - -c + - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /etcd-manager + --backup-store=memfs://tests/minimal.example.com/backups/etcd/events --client-urls=https://__name__:4002 + --cluster-name=etcd-events --containerized=true --dns-suffix=.internal.minimal.example.com + --grpc-port=3997 --peer-urls=https://__name__:2381 --quarantine-client-urls=https://__name__:3995 + --v=6 --volume-name-tag=k8s.io/etcd/events --volume-provider=aws --volume-tag=k8s.io/etcd/events + --volume-tag=k8s.io/role/control-plane=1 --volume-tag=kubernetes.io/cluster/minimal.example.com=owned + > /tmp/pipe 2>&1 + env: + - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION + value: 90d + image: registry.k8s.io/etcdadm/etcd-manager-slim:v3.0.20230516 + name: etcd-manager + resources: + requests: + cpu: 100m + memory: 100Mi + securityContext: + privileged: true + volumeMounts: + - mountPath: /rootfs + name: rootfs + - mountPath: /run + name: run + - mountPath: /etc/kubernetes/pki/etcd-manager + name: pki + - mountPath: /opt + name: opt + - mountPath: /var/log/etcd.log + name: varlogetcd + hostNetwork: true + hostPID: true + initContainers: + - args: + - /ko-app/kops-copy + - /opt/bin + command: + - /ko-app/kops-copy + image: registry.k8s.io/kops/kops-copy:1.27.0-alpha.2 + name: kops-copy + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - /usr/local/bin/etcd + - /usr/local/bin/etcdctl + - /opt/etcd-v3.2.24 + command: + - /opt/bin/kops-copy + image: registry.k8s.io/etcd:3.2.24-1 + name: init-etcd-3-2-24 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - /usr/local/bin/etcd + - /usr/local/bin/etcdctl + - /opt/etcd-v3.3.17 + command: + - /opt/bin/kops-copy + image: registry.k8s.io/etcd:3.3.17-0 + name: init-etcd-3-3-17 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - /usr/local/bin/etcd + - /usr/local/bin/etcdctl + - /opt/etcd-v3.4.13 + command: + - /opt/bin/kops-copy + image: registry.k8s.io/etcd:3.4.13-0 + name: init-etcd-3-4-13 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - /usr/local/bin/etcd + - /usr/local/bin/etcdctl + - /opt/etcd-v3.5.7 + command: + - /opt/bin/kops-copy + image: registry.k8s.io/etcd:3.5.7-0 + name: init-etcd-3-5-7 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - /usr/local/bin/etcd + - /usr/local/bin/etcdctl + - /opt/etcd-v3.5.9 + command: + - /opt/bin/kops-copy + image: registry.k8s.io/etcd:3.5.9-0 + name: init-etcd-3-5-9 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: / + type: Directory + name: rootfs + - hostPath: + path: /run + type: DirectoryOrCreate + name: run + - hostPath: + path: /etc/kubernetes/pki/etcd-manager-events + type: DirectoryOrCreate + name: pki + - emptyDir: {} + name: opt + - hostPath: + path: /var/log/etcd-events.log + type: FileOrCreate + name: varlogetcd +status: {} diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content new file mode 100644 index 0000000000..7dfb6d2217 --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content @@ -0,0 +1,140 @@ +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + labels: + k8s-app: etcd-manager-main + name: etcd-manager-main + namespace: kube-system +spec: + containers: + - command: + - /bin/sh + - -c + - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /etcd-manager + --backup-store=memfs://tests/minimal.example.com/backups/etcd/main --client-urls=https://__name__:4001 + --cluster-name=etcd --containerized=true --dns-suffix=.internal.minimal.example.com + --grpc-port=3996 --peer-urls=https://__name__:2380 --quarantine-client-urls=https://__name__:3994 + --v=6 --volume-name-tag=k8s.io/etcd/main --volume-provider=aws --volume-tag=k8s.io/etcd/main + --volume-tag=k8s.io/role/control-plane=1 --volume-tag=kubernetes.io/cluster/minimal.example.com=owned + > /tmp/pipe 2>&1 + env: + - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION + value: 90d + image: registry.k8s.io/etcdadm/etcd-manager-slim:v3.0.20230516 + name: etcd-manager + resources: + requests: + cpu: 200m + memory: 100Mi + securityContext: + privileged: true + volumeMounts: + - mountPath: /rootfs + name: rootfs + - mountPath: /run + name: run + - mountPath: /etc/kubernetes/pki/etcd-manager + name: pki + - mountPath: /opt + name: opt + - mountPath: /var/log/etcd.log + name: varlogetcd + hostNetwork: true + hostPID: true + initContainers: + - args: + - /ko-app/kops-copy + - /opt/bin + command: + - /ko-app/kops-copy + image: registry.k8s.io/kops/kops-copy:1.27.0-alpha.2 + name: kops-copy + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - /usr/local/bin/etcd + - /usr/local/bin/etcdctl + - /opt/etcd-v3.2.24 + command: + - /opt/bin/kops-copy + image: registry.k8s.io/etcd:3.2.24-1 + name: init-etcd-3-2-24 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - /usr/local/bin/etcd + - /usr/local/bin/etcdctl + - /opt/etcd-v3.3.17 + command: + - /opt/bin/kops-copy + image: registry.k8s.io/etcd:3.3.17-0 + name: init-etcd-3-3-17 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - /usr/local/bin/etcd + - /usr/local/bin/etcdctl + - /opt/etcd-v3.4.13 + command: + - /opt/bin/kops-copy + image: registry.k8s.io/etcd:3.4.13-0 + name: init-etcd-3-4-13 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - /usr/local/bin/etcd + - /usr/local/bin/etcdctl + - /opt/etcd-v3.5.7 + command: + - /opt/bin/kops-copy + image: registry.k8s.io/etcd:3.5.7-0 + name: init-etcd-3-5-7 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - /usr/local/bin/etcd + - /usr/local/bin/etcdctl + - /opt/etcd-v3.5.9 + command: + - /opt/bin/kops-copy + image: registry.k8s.io/etcd:3.5.9-0 + name: init-etcd-3-5-9 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: / + type: Directory + name: rootfs + - hostPath: + path: /run + type: DirectoryOrCreate + name: run + - hostPath: + path: /etc/kubernetes/pki/etcd-manager-main + type: DirectoryOrCreate + name: pki + - emptyDir: {} + name: opt + - hostPath: + path: /var/log/etcd.log + type: FileOrCreate + name: varlogetcd +status: {} diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content new file mode 100644 index 0000000000..4bc42f9fd6 --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null +spec: + containers: + - args: + - --ca-cert=/secrets/ca.crt + - --client-cert=/secrets/client.crt + - --client-key=/secrets/client.key + image: registry.k8s.io/kops/kube-apiserver-healthcheck:1.27.0-alpha.2 + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /.kube-apiserver-healthcheck/healthz + port: 3990 + initialDelaySeconds: 5 + timeoutSeconds: 5 + name: healthcheck + resources: {} + securityContext: + runAsNonRoot: true + runAsUser: 10012 + volumeMounts: + - mountPath: /secrets + name: healthcheck-secrets + readOnly: true + volumes: + - hostPath: + path: /etc/kubernetes/kube-apiserver-healthcheck/secrets + type: Directory + name: healthcheck-secrets +status: {} diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content new file mode 100644 index 0000000000..4b10587240 --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content @@ -0,0 +1,237 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + k8s-app: aws-cloud-controller-manager + name: aws-cloud-controller-manager + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + template: + metadata: + creationTimestamp: null + labels: + k8s-app: aws-cloud-controller-manager + kops.k8s.io/managed-by: kops + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + containers: + - args: + - --allocate-node-cidrs=true + - --cluster-cidr=100.64.0.0/10 + - --cluster-name=minimal.example.com + - --configure-cloud-routes=false + - --leader-elect=true + - --v=2 + - --cloud-provider=aws + - --use-service-account-credentials=true + - --cloud-config=/etc/kubernetes/cloud.config + env: + - name: KUBERNETES_SERVICE_HOST + value: 127.0.0.1 + image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.26.0 + imagePullPolicy: IfNotPresent + name: aws-cloud-controller-manager + resources: + requests: + cpu: 200m + volumeMounts: + - mountPath: /etc/kubernetes/cloud.config + name: cloudconfig + readOnly: true + hostNetwork: true + nodeSelector: null + priorityClassName: system-cluster-critical + serviceAccountName: aws-cloud-controller-manager + tolerations: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + - effect: NoSchedule + key: node.kubernetes.io/not-ready + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + - effect: NoSchedule + key: node-role.kubernetes.io/master + volumes: + - hostPath: + path: /etc/kubernetes/cloud.config + type: "" + name: cloudconfig + updateStrategy: + type: RollingUpdate + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + name: aws-cloud-controller-manager + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- apiGroup: "" + kind: ServiceAccount + name: aws-cloud-controller-manager + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + name: system:cloud-controller-manager +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - '*' +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch +- apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update +- apiGroups: + - "" + resources: + - secrets + verbs: + - list + - watch +- apiGroups: + - "" + resourceNames: + - node-controller + - service-controller + - route-controller + resources: + - serviceaccounts/token + verbs: + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + name: system:cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager +subjects: +- apiGroup: "" + kind: ServiceAccount + name: aws-cloud-controller-manager + namespace: kube-system diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content new file mode 100644 index 0000000000..12cb865696 --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content @@ -0,0 +1,819 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.14.1 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.14.1 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-external-attacher-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.14.1 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-external-provisioner-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list +- apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.14.1 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-external-resizer-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.14.1 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-external-snapshotter-role +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.14.1 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-attacher-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.14.1 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-provisioner-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.14.1 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-resizer-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.14.1 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-snapshotter-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-node-getter-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role +subjects: +- kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-node-role +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.14.1 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-node-sa + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.14.1 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-node + namespace: kube-system +spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + creationTimestamp: null + labels: + app: ebs-csi-node + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.14.1 + kops.k8s.io/managed-by: kops + spec: + containers: + - args: + - node + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --v=2 + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=5 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1 + imagePullPolicy: IfNotPresent + name: node-driver-registrar + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.6.0 + imagePullPolicy: IfNotPresent + name: liveness-probe + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.14.1 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + creationTimestamp: null + labels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.14.1 + kops.k8s.io/managed-by: kops + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: kubernetes.io/os + operator: In + values: + - linux + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - args: + - controller + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --k8s-tag-cluster-id=minimal.example.com + - --extra-tags=KubernetesCluster=minimal.example.com + - --http-endpoint=0.0.0.0:3301 + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + image: registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.14.1 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + - containerPort: 3301 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --v=5 + - --feature-gates=Topology=true + - --extra-create-metadata + - --leader-election=true + - --default-fstype=ext4 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-provisioner:v3.1.0 + imagePullPolicy: IfNotPresent + name: csi-provisioner + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --v=5 + - --leader-election=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-attacher:v3.4.0 + imagePullPolicy: IfNotPresent + name: csi-attacher + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --v=5 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-resizer:v1.4.0 + imagePullPolicy: IfNotPresent + name: csi-resizer + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.6.0 + imagePullPolicy: IfNotPresent + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + hostNetwork: true + nodeSelector: null + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - operator: Exists + topologySpreadConstraints: + - labelSelector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + - labelSelector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + volumes: + - emptyDir: {} + name: socket-dir + +--- + +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.14.1 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs.csi.aws.com +spec: + attachRequired: true + podInfoOnMount: false + +--- + +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.14.1 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-controller + namespace: kube-system +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-bootstrap_content new file mode 100644 index 0000000000..255a49b65f --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-bootstrap_content @@ -0,0 +1,62 @@ +kind: Addons +metadata: + creationTimestamp: null + name: bootstrap +spec: + addons: + - id: k8s-1.16 + manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml + manifestHash: 7aa6902c8c96c5778e3c73bd357055f35e8b5f2e0d299be22c1dec5c9fa8749e + name: kops-controller.addons.k8s.io + needsRollingUpdate: control-plane + selector: + k8s-addon: kops-controller.addons.k8s.io + version: 9.99.0 + - id: k8s-1.12 + manifest: coredns.addons.k8s.io/k8s-1.12.yaml + manifestHash: d2bbb7cbee5835c3891fe80fbacf8963508359ef9159f8480325ce9a7174f14a + name: coredns.addons.k8s.io + selector: + k8s-addon: coredns.addons.k8s.io + version: 9.99.0 + - id: k8s-1.9 + manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml + manifestHash: 01c120e887bd98d82ef57983ad58a0b22bc85efb48108092a24c4b82e4c9ea81 + name: kubelet-api.rbac.addons.k8s.io + selector: + k8s-addon: kubelet-api.rbac.addons.k8s.io + version: 9.99.0 + - manifest: limit-range.addons.k8s.io/v1.5.0.yaml + manifestHash: 2d55c3bc5e354e84a3730a65b42f39aba630a59dc8d32b30859fcce3d3178bc2 + name: limit-range.addons.k8s.io + selector: + k8s-addon: limit-range.addons.k8s.io + version: 9.99.0 + - id: k8s-1.12 + manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml + manifestHash: 89b181fbeb7149df1ea98c61f478d26dc1cef6c3a9512049bb64bd1ae7e7096a + name: dns-controller.addons.k8s.io + selector: + k8s-addon: dns-controller.addons.k8s.io + version: 9.99.0 + - id: v1.15.0 + manifest: storage-aws.addons.k8s.io/v1.15.0.yaml + manifestHash: 4e2cda50cd5048133aad1b5e28becb60f4629d3f9e09c514a2757c27998b4200 + name: storage-aws.addons.k8s.io + selector: + k8s-addon: storage-aws.addons.k8s.io + version: 9.99.0 + - id: k8s-1.18 + manifest: aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml + manifestHash: 557d71c430bb05a5b069fd8dc3a0a3247261795bfd0617b97cbf1f31fed3fc27 + name: aws-cloud-controller.addons.k8s.io + selector: + k8s-addon: aws-cloud-controller.addons.k8s.io + version: 9.99.0 + - id: k8s-1.17 + manifest: aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml + manifestHash: 9ebe176a18822b64f30849e1b29a147a73e49bb0c445c78cba85703ea3a3221f + name: aws-ebs-csi-driver.addons.k8s.io + selector: + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + version: 9.99.0 diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content new file mode 100644 index 0000000000..477b37ebbe --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content @@ -0,0 +1,383 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + kubernetes.io/cluster-service: "true" + name: coredns + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns +subjects: +- kind: ServiceAccount + name: coredns + namespace: kube-system + +--- + +apiVersion: v1 +data: + Corefile: |- + .:53 { + errors + health { + lameduck 5s + } + ready + kubernetes cluster.local. in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + } + prometheus :9153 + forward . /etc/resolv.conf { + max_concurrent 1000 + } + cache 30 + loop + reload + loadbalance + } +kind: ConfigMap +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + addonmanager.kubernetes.io/mode: EnsureExists + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: coredns + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: CoreDNS + name: coredns + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-dns + strategy: + rollingUpdate: + maxSurge: 10% + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + k8s-app: kube-dns + kops.k8s.io/managed-by: kops + spec: + containers: + - args: + - -conf + - /etc/coredns/Corefile + image: registry.k8s.io/coredns/coredns:v1.10.1 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + successThreshold: 1 + timeoutSeconds: 5 + name: coredns + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - all + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /etc/coredns + name: config-volume + readOnly: true + dnsPolicy: Default + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: coredns + tolerations: + - key: CriticalAddonsOnly + operator: Exists + topologySpreadConstraints: + - labelSelector: + matchLabels: + k8s-app: kube-dns + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + - labelSelector: + matchLabels: + k8s-app: kube-dns + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + volumes: + - configMap: + name: coredns + name: config-volume + +--- + +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: CoreDNS + name: kube-dns + namespace: kube-system + resourceVersion: "0" +spec: + clusterIP: 100.64.0.10 + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP + selector: + k8s-app: kube-dns + +--- + +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: kube-dns + namespace: kube-system +spec: + maxUnavailable: 50% + selector: + matchLabels: + k8s-app: kube-dns + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: coredns-autoscaler + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: coredns-autoscaler +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - replicationcontrollers/scale + verbs: + - get + - update +- apiGroups: + - extensions + - apps + resources: + - deployments/scale + - replicasets/scale + verbs: + - get + - update +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: coredns-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: coredns-autoscaler +subjects: +- kind: ServiceAccount + name: coredns-autoscaler + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + k8s-app: coredns-autoscaler + kubernetes.io/cluster-service: "true" + name: coredns-autoscaler + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: coredns-autoscaler + template: + metadata: + creationTimestamp: null + labels: + k8s-app: coredns-autoscaler + kops.k8s.io/managed-by: kops + spec: + containers: + - command: + - /cluster-proportional-autoscaler + - --namespace=kube-system + - --configmap=coredns-autoscaler + - --target=Deployment/coredns + - --default-params={"linear":{"coresPerReplica":256,"nodesPerReplica":16,"preventSinglePointFailure":true}} + - --logtostderr=true + - --v=2 + image: registry.k8s.io/cpa/cluster-proportional-autoscaler:v1.8.8 + name: autoscaler + resources: + requests: + cpu: 20m + memory: 10Mi + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: coredns-autoscaler + tolerations: + - key: CriticalAddonsOnly + operator: Exists diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content new file mode 100644 index 0000000000..8cc21282dc --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content @@ -0,0 +1,138 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: dns-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: dns-controller.addons.k8s.io + k8s-app: dns-controller + version: v1.27.0-alpha.2 + name: dns-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: dns-controller + strategy: + type: Recreate + template: + metadata: + creationTimestamp: null + labels: + k8s-addon: dns-controller.addons.k8s.io + k8s-app: dns-controller + kops.k8s.io/managed-by: kops + version: v1.27.0-alpha.2 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + containers: + - args: + - --watch-ingress=false + - --dns=aws-route53 + - --zone=*/Z1AFAKE1ZON3YO + - --internal-ipv4 + - --zone=*/* + - -v=2 + command: null + env: + - name: KUBERNETES_SERVICE_HOST + value: 127.0.0.1 + image: registry.k8s.io/kops/dns-controller:1.27.0-alpha.2 + name: dns-controller + resources: + requests: + cpu: 50m + memory: 50Mi + securityContext: + runAsNonRoot: true + dnsPolicy: Default + hostNetwork: true + nodeSelector: null + priorityClassName: system-cluster-critical + serviceAccount: dns-controller + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + - key: node.kubernetes.io/not-ready + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: dns-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: dns-controller.addons.k8s.io + name: dns-controller + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: dns-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: dns-controller.addons.k8s.io + name: kops:dns-controller +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - ingress + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: dns-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: dns-controller.addons.k8s.io + name: kops:dns-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kops:dns-controller +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:kube-system:dns-controller diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content new file mode 100644 index 0000000000..2a093e5c85 --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content @@ -0,0 +1,225 @@ +apiVersion: v1 +data: + config.yaml: | + {"clusterName":"minimal.example.com","cloud":"aws","configBase":"memfs://tests/minimal.example.com","secretStore":"memfs://tests/minimal.example.com/secrets","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["nodes.minimal.example.com"],"Region":"us-test-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"],"useInstanceIDForNodeName":true}} +kind: ConfigMap +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + k8s-app: kops-controller + version: v1.27.0-alpha.2 + name: kops-controller + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kops-controller + template: + metadata: + annotations: + dns.alpha.kubernetes.io/internal: kops-controller.internal.minimal.example.com + creationTimestamp: null + labels: + k8s-addon: kops-controller.addons.k8s.io + k8s-app: kops-controller + kops.k8s.io/managed-by: kops + version: v1.27.0-alpha.2 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: kops.k8s.io/kops-controller-pki + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - key: kops.k8s.io/kops-controller-pki + operator: Exists + containers: + - args: + - --v=2 + - --conf=/etc/kubernetes/kops-controller/config/config.yaml + command: null + env: + - name: KUBERNETES_SERVICE_HOST + value: 127.0.0.1 + image: registry.k8s.io/kops/kops-controller:1.27.0-alpha.2 + name: kops-controller + resources: + requests: + cpu: 50m + memory: 50Mi + securityContext: + runAsNonRoot: true + runAsUser: 10011 + volumeMounts: + - mountPath: /etc/kubernetes/kops-controller/config/ + name: kops-controller-config + - mountPath: /etc/kubernetes/kops-controller/pki/ + name: kops-controller-pki + dnsPolicy: Default + hostNetwork: true + nodeSelector: null + priorityClassName: system-cluster-critical + serviceAccount: kops-controller + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + - key: node.kubernetes.io/not-ready + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + volumes: + - configMap: + name: kops-controller + name: kops-controller-config + - hostPath: + path: /etc/kubernetes/kops-controller/ + type: Directory + name: kops-controller-pki + updateStrategy: + type: OnDelete + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - patch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kops-controller +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:kube-system:kops-controller + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - create +- apiGroups: + - "" + - coordination.k8s.io + resourceNames: + - kops-controller-leader + resources: + - configmaps + - leases + verbs: + - get + - list + - watch + - patch + - update + - delete +- apiGroups: + - "" + - coordination.k8s.io + resources: + - configmaps + - leases + verbs: + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kops-controller +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:kube-system:kops-controller diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content new file mode 100644 index 0000000000..36761e1c56 --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kubelet-api.rbac.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kubelet-api.rbac.addons.k8s.io + name: kops:system:kubelet-api-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kubelet-api-admin +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: kubelet-api diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-limit-range.addons.k8s.io_content b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-limit-range.addons.k8s.io_content new file mode 100644 index 0000000000..4dcdce48b9 --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-limit-range.addons.k8s.io_content @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: LimitRange +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: limit-range.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: limit-range.addons.k8s.io + name: limits + namespace: default +spec: + limits: + - defaultRequest: + cpu: 100m + type: Container diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content new file mode 100644 index 0000000000..bea3e88be3 --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_minimal.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content @@ -0,0 +1,118 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: default +parameters: + type: gp2 +provisioner: kubernetes.io/aws-ebs + +--- + +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: + storageclass.kubernetes.io/is-default-class: "false" + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: gp2 +parameters: + type: gp2 +provisioner: kubernetes.io/aws-ebs + +--- + +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: + storageclass.kubernetes.io/is-default-class: "false" + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: kops-ssd-1-17 +parameters: + encrypted: "true" + type: gp2 +provisioner: kubernetes.io/aws-ebs +volumeBindingMode: WaitForFirstConsumer + +--- + +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: + storageclass.kubernetes.io/is-default-class: "true" + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: kops-csi-1-21 +parameters: + encrypted: "true" + type: gp3 +provisioner: ebs.csi.aws.com +volumeBindingMode: WaitForFirstConsumer + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: system:aws-cloud-provider +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - patch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: system:aws-cloud-provider +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:aws-cloud-provider +subjects: +- kind: ServiceAccount + name: aws-cloud-provider + namespace: kube-system diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_nodeupconfig-master-us-test-1a_content b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_nodeupconfig-master-us-test-1a_content new file mode 100644 index 0000000000..df89c687ff --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_nodeupconfig-master-us-test-1a_content @@ -0,0 +1,300 @@ +APIServerConfig: + KubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: Node,RBAC + bindAddress: 0.0.0.0 + cloudProvider: external + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + featureGates: + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-apiserver:v1.27.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.minimal.example.com + serviceAccountJWKSURI: https://api.internal.minimal.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + ServiceAccountPublicKeys: | + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm + XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ== + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF + Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ== + -----END RSA PUBLIC KEY----- +Assets: + amd64: + - 0b4ed4fcd75d33f5dff3ba17776e6089847fc83064d3f7a3ad59a34e94e60a29@https://dl.k8s.io/release/v1.27.0/bin/linux/amd64/kubelet + - 71a78259d70da9c5540c4cf4cff121f443e863376f68f89a759d90cef3f51e87@https://dl.k8s.io/release/v1.27.0/bin/linux/amd64/kubectl + - 5035d7814c95cd3cedbc5efb447ef25a4942ef05caab2159746d55ce1698c74a@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/amd64/ecr-credential-provider-linux-amd64 + - f3a841324845ca6bf0d4091b4fc7f97e18a623172158b72fc3fdcdb9d42d2d37@https://storage.googleapis.com/k8s-artifacts-cni/release/v1.2.0/cni-plugins-linux-amd64-v1.2.0.tgz + - bb9a9ccd6517e2a54da748a9f60dc9aa9d79d19d4724663f2386812f083968e2@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-amd64.tar.gz + - f00b144e86f8c1db347a2e8f22caade07d55382c5f76dd5c0a5b1ab64eaec8bb@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.amd64 + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 37aa2edc7c0c4b3e488518c6a4b44c8aade75a55010534ee2be291220c73d157@https://dl.k8s.io/release/v1.27.0/bin/linux/arm64/kubelet + - f8e09630211f2b7c6a8cc38835e7dea94708d401f5c84b23a37c70c604602ddc@https://dl.k8s.io/release/v1.27.0/bin/linux/arm64/kubectl + - b3d567bda9e2996fc1fbd9d13506bd16763d3865b5c7b0b3c4b48c6088c04481@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/arm64/ecr-credential-provider-linux-arm64 + - 525e2b62ba92a1b6f3dc9612449a84aa61652e680f7ebf4eff579795fe464b57@https://storage.googleapis.com/k8s-artifacts-cni/release/v1.2.0/cni-plugins-linux-arm64-v1.2.0.tgz + - c3e6a054b18b20fce06c7c3ed53f0989bb4b255c849bede446ebca955f07a9ce@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-arm64.tar.gz + - 54e79e4d48b9e191767e4abc08be1a8476a1c757e9a9f8c45c6ded001226867f@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.arm64 + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 +CAs: + apiserver-aggregator-ca: | + -----BEGIN CERTIFICATE----- + MIIBgjCCASygAwIBAgIMFo3gINaZLHjisEcbMA0GCSqGSIb3DQEBCwUAMCIxIDAe + BgNVBAMTF2FwaXNlcnZlci1hZ2dyZWdhdG9yLWNhMB4XDTIxMDYzMDA0NTExMloX + DTMxMDYzMDA0NTExMlowIjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3It + Y2EwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAyyE71AOU3go5XFegLQ6fidI0LhhM + x7CzpTzh2xWKcHUfbNI7itgJvC/+GlyG5W+DF5V7ba0IJiQLsFve0oLdewIDAQAB + o0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU + ALfqF5ZmfqvqORuJIFilZYKF3d0wDQYJKoZIhvcNAQELBQADQQAHAomFKsF4jvYX + WM/UzQXDj9nSAFTf8dBPCXyZZNotsOH7+P6W4mMiuVs8bAuGiXGUdbsQ2lpiT/Rk + CzMeMdr4 + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBgjCCASygAwIBAgIMFo3gM0nxQpiX/agfMA0GCSqGSIb3DQEBCwUAMCIxIDAe + BgNVBAMTF2FwaXNlcnZlci1hZ2dyZWdhdG9yLWNhMB4XDTIxMDYzMDA0NTIzMVoX + DTMxMDYzMDA0NTIzMVowIjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3It + Y2EwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAyyE71AOU3go5XFegLQ6fidI0LhhM + x7CzpTzh2xWKcHUfbNI7itgJvC/+GlyG5W+DF5V7ba0IJiQLsFve0oLdewIDAQAB + o0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU + ALfqF5ZmfqvqORuJIFilZYKF3d0wDQYJKoZIhvcNAQELBQADQQCXsoezoxXu2CEN + QdlXZOfmBT6cqxIX/RMHXhpHwRiqPsTO8IO2bVA8CSzxNwMuSv/ZtrMHoh8+PcVW + HLtkTXH8 + -----END CERTIFICATE----- + etcd-clients-ca: | + -----BEGIN CERTIFICATE----- + MIIBcjCCARygAwIBAgIMFo1ogHnr26DL9YkqMA0GCSqGSIb3DQEBCwUAMBoxGDAW + BgNVBAMTD2V0Y2QtY2xpZW50cy1jYTAeFw0yMTA2MjgxNjE5MDFaFw0zMTA2Mjgx + NjE5MDFaMBoxGDAWBgNVBAMTD2V0Y2QtY2xpZW50cy1jYTBcMA0GCSqGSIb3DQEB + AQUAA0sAMEgCQQDYlt4Xx03Cp8QooPrloaVWznx9aQDSpl1UsrDyoBPNEElOLWep + uPaQBHiDLL8LwzGi7G9r+ib13tKrwprnlPv7AgMBAAGjQjBAMA4GA1UdDwEB/wQE + AwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQjlt4Ue54AbJPWlDpRM51s + x+PeBDANBgkqhkiG9w0BAQsFAANBAAZAdf8ROEVkr3Rf7I+s+CQOil2toadlKWOY + qCeJ2XaEROfp9aUTEIU1MGM3g57MPyAPPU7mURskuOQz6B1UFaY= + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBcjCCARygAwIBAgIMFo1olfBnC/CsT+dqMA0GCSqGSIb3DQEBCwUAMBoxGDAW + BgNVBAMTD2V0Y2QtY2xpZW50cy1jYTAeFw0yMTA2MjgxNjIwMzNaFw0zMTA2Mjgx + NjIwMzNaMBoxGDAWBgNVBAMTD2V0Y2QtY2xpZW50cy1jYTBcMA0GCSqGSIb3DQEB + AQUAA0sAMEgCQQDYlt4Xx03Cp8QooPrloaVWznx9aQDSpl1UsrDyoBPNEElOLWep + uPaQBHiDLL8LwzGi7G9r+ib13tKrwprnlPv7AgMBAAGjQjBAMA4GA1UdDwEB/wQE + AwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQjlt4Ue54AbJPWlDpRM51s + x+PeBDANBgkqhkiG9w0BAQsFAANBAF1xUz77PlUVUnd9duF8F7plou0TONC9R6/E + YQ8C6vM1b+9NSDGjCW8YmwEU2fBgskb/BBX2lwVZ32/RUEju4Co= + -----END CERTIFICATE----- + etcd-manager-ca-events: | + -----BEGIN CERTIFICATE----- + MIIBgDCCASqgAwIBAgIMFo+bKjm04vB4rNtaMA0GCSqGSIb3DQEBCwUAMCExHzAd + BgNVBAMTFmV0Y2QtbWFuYWdlci1jYS1ldmVudHMwHhcNMjEwNzA1MjAwOTU2WhcN + MzEwNzA1MjAwOTU2WjAhMR8wHQYDVQQDExZldGNkLW1hbmFnZXItY2EtZXZlbnRz + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKiC8tndMlEFZ7qzeKxeKqFVjaYpsh/H + g7RxWo15+1kgH3suO0lxp9+RxSVv97hnsfbySTPZVhy2cIQj7eZtZt8CAwEAAaNC + MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFBg6 + CEZkQNnRkARBwFce03AEWa+sMA0GCSqGSIb3DQEBCwUAA0EAJMnBThok/uUe8q8O + sS5q19KUuE8YCTUzMDj36EBKf6NX4NoakCa1h6kfQVtlMtEIMWQZCjbm8xGK5ffs + GS/VUw== + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBgDCCASqgAwIBAgIMFo+bQ+EgIiBmGghjMA0GCSqGSIb3DQEBCwUAMCExHzAd + BgNVBAMTFmV0Y2QtbWFuYWdlci1jYS1ldmVudHMwHhcNMjEwNzA1MjAxMTQ2WhcN + MzEwNzA1MjAxMTQ2WjAhMR8wHQYDVQQDExZldGNkLW1hbmFnZXItY2EtZXZlbnRz + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKFhHVVxxDGv8d1jBvtdSxz7KIVoBOjL + DMxsmTsINiQkTQaFlb+XPlnY1ar4+RhE519AFUkqfhypk4Zxqf1YFXUCAwEAAaNC + MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNuW + LLH5c8kDubDbr6BHgedW0iJ9MA0GCSqGSIb3DQEBCwUAA0EAiKUoBoaGu7XzboFE + hjfKlX0TujqWuW3qMxDEJwj4dVzlSLrAoB/G01MJ+xxYKh456n48aG6N827UPXhV + cPfVNg== + -----END CERTIFICATE----- + etcd-manager-ca-main: | + -----BEGIN CERTIFICATE----- + MIIBfDCCASagAwIBAgIMFo+bKjm1c3jfv6hIMA0GCSqGSIb3DQEBCwUAMB8xHTAb + BgNVBAMTFGV0Y2QtbWFuYWdlci1jYS1tYWluMB4XDTIxMDcwNTIwMDk1NloXDTMx + MDcwNTIwMDk1NlowHzEdMBsGA1UEAxMUZXRjZC1tYW5hZ2VyLWNhLW1haW4wXDAN + BgkqhkiG9w0BAQEFAANLADBIAkEAxbkDbGYmCSShpRG3r+lzTOFujyuruRfjOhYm + ZRX4w1Utd5y63dUc98sjc9GGUYMHd+0k1ql/a48tGhnK6N6jJwIDAQABo0IwQDAO + BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUWZLkbBFx + GAgPU4i62c52unSo7RswDQYJKoZIhvcNAQELBQADQQAj6Pgd0va/8FtkyMlnohLu + Gf4v8RJO6zk3Y6jJ4+cwWziipFM1ielMzSOZfFcCZgH3m5Io40is4hPSqyq2TOA6 + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBfDCCASagAwIBAgIMFo+bQ+Eg8Si30gr4MA0GCSqGSIb3DQEBCwUAMB8xHTAb + BgNVBAMTFGV0Y2QtbWFuYWdlci1jYS1tYWluMB4XDTIxMDcwNTIwMTE0NloXDTMx + MDcwNTIwMTE0NlowHzEdMBsGA1UEAxMUZXRjZC1tYW5hZ2VyLWNhLW1haW4wXDAN + BgkqhkiG9w0BAQEFAANLADBIAkEAw33jzcd/iosN04b0WXbDt7B0c3sJ3aafcGLP + vG3xRB9N5bYr9+qZAq3mzAFkxscn4j1ce5b1/GKTDEAClmZgdQIDAQABo0IwQDAO + BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUE/h+3gDP + DvKwHRyiYlXM8voZ1wowDQYJKoZIhvcNAQELBQADQQBXuimeEoAOu5HN4hG7NqL9 + t40K3ZRhRZv3JQWnRVJCBDjg1rD0GQJR/n+DoWvbeijI5C9pNjr2pWSIYR1eYCvd + -----END CERTIFICATE----- + etcd-peers-ca-events: | + -----BEGIN CERTIFICATE----- + MIIBfDCCASagAwIBAgIMFo+bKjmxTPh3/lYJMA0GCSqGSIb3DQEBCwUAMB8xHTAb + BgNVBAMTFGV0Y2QtcGVlcnMtY2EtZXZlbnRzMB4XDTIxMDcwNTIwMDk1NloXDTMx + MDcwNTIwMDk1NlowHzEdMBsGA1UEAxMUZXRjZC1wZWVycy1jYS1ldmVudHMwXDAN + BgkqhkiG9w0BAQEFAANLADBIAkEAv5g4HF2xmrYyouJfY9jXx1M3gPLD/pupvxPY + xyjJw5pNCy5M5XGS3iTqRD5RDE0fWudVHFZKLIe8WPc06NApXwIDAQABo0IwQDAO + BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUf6xiDI+O + Yph1ziCGr2hZaQYt+fUwDQYJKoZIhvcNAQELBQADQQBBxj5hqEQstonTb8lnqeGB + DEYtUeAk4eR/HzvUMjF52LVGuvN3XVt+JTrFeKNvb6/RDUbBNRj3azalcUkpPh6V + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBfDCCASagAwIBAgIMFo+bQ+Eq69jgzpKwMA0GCSqGSIb3DQEBCwUAMB8xHTAb + BgNVBAMTFGV0Y2QtcGVlcnMtY2EtZXZlbnRzMB4XDTIxMDcwNTIwMTE0NloXDTMx + MDcwNTIwMTE0NlowHzEdMBsGA1UEAxMUZXRjZC1wZWVycy1jYS1ldmVudHMwXDAN + BgkqhkiG9w0BAQEFAANLADBIAkEAo5Nj2CjX1qp3mEPw1H5nHAFWLoGNSLSlRFJW + 03NxaNPMFzL5PrCoyOXrX8/MWczuZYw0Crf8EPOOQWi2+W0XLwIDAQABo0IwQDAO + BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUxauhhKQh + cvdZND78rHe0RQVTTiswDQYJKoZIhvcNAQELBQADQQB+cq4jIS9q0zXslaRa+ViI + J+dviA3sMygbmSJO0s4DxYmoazKJblux5q0ASSvS9iL1l9ShuZ1dWyp2tpZawHyb + -----END CERTIFICATE----- + etcd-peers-ca-main: | + -----BEGIN CERTIFICATE----- + MIIBeDCCASKgAwIBAgIMFo+bKjmuLDDLcDHsMA0GCSqGSIb3DQEBCwUAMB0xGzAZ + BgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjAeFw0yMTA3MDUyMDA5NTZaFw0zMTA3 + MDUyMDA5NTZaMB0xGzAZBgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjBcMA0GCSqG + SIb3DQEBAQUAA0sAMEgCQQCyRaXWpwgN6INQqws9p/BvPElJv2Rno9dVTFhlQqDA + aUJXe7MBmiO4NJcW76EozeBh5ztR3/4NE1FM2x8TisS3AgMBAAGjQjBAMA4GA1Ud + DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQtE1d49uSvpURf + OQ25Vlu6liY20DANBgkqhkiG9w0BAQsFAANBAAgLVaetJZcfOA3OIMMvQbz2Ydrt + uWF9BKkIad8jrcIrm3IkOtR8bKGmDIIaRKuG/ZUOL6NMe2fky3AAfKwleL4= + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBeDCCASKgAwIBAgIMFo+bQ+EuVthBfuZvMA0GCSqGSIb3DQEBCwUAMB0xGzAZ + BgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjAeFw0yMTA3MDUyMDExNDZaFw0zMTA3 + MDUyMDExNDZaMB0xGzAZBgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjBcMA0GCSqG + SIb3DQEBAQUAA0sAMEgCQQCxNbycDZNx5V1ZOiXxZSvaFpHRwKeHDfcuMUitdoPt + naVMlMTGDWAMuCVmFHFAWohIYynemEegmZkZ15S7AErfAgMBAAGjQjBAMA4GA1Ud + DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTAjQ8T4HclPIsC + qipEfUIcLP6jqTANBgkqhkiG9w0BAQsFAANBAJdZ17TN3HlWrH7HQgfR12UBwz8K + G9DurDznVaBVUYaHY8Sg5AvAXeb+yIF2JMmRR+bK+/G1QYY2D3/P31Ic2Oo= + -----END CERTIFICATE----- + kubernetes-ca: | + -----BEGIN CERTIFICATE----- + MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU + BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw + ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD + SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1 + jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG + MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA + MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8 + tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w== + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU + BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw + OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD + SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7 + WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG + MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn + MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA + 9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw== + -----END CERTIFICATE----- +ClusterName: minimal.example.com +ContainerRuntime: containerd +FileAssets: +- content: | + apiVersion: kubescheduler.config.k8s.io/v1 + clientConnection: + kubeconfig: /var/lib/kube-scheduler/kubeconfig + kind: KubeSchedulerConfiguration + path: /var/lib/kube-scheduler/config.yaml +Hooks: +- null +- null +KeypairIDs: + apiserver-aggregator-ca: "6980187172486667078076483355" + etcd-clients-ca: "6979622252718071085282986282" + etcd-manager-ca-events: "6982279354000777253151890266" + etcd-manager-ca-main: "6982279354000936168671127624" + etcd-peers-ca-events: "6982279353999767935825892873" + etcd-peers-ca-main: "6982279353998887468930183660" + kubernetes-ca: "6982820025135291416230495506" + service-account: "2" +KubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + image: registry.k8s.io/kube-proxy:v1.27.0 + logLevel: 2 +KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + InTreePluginAWSUnregister: "true" + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + nodeLabels: + kops.k8s.io/instancegroup: master-us-test-1a + kops.k8s.io/kops-controller-pki: "" + node-role.kubernetes.io/control-plane: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + podInfraContainerImage: registry.k8s.io/pause:3.6 + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + registerSchedulable: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s + taints: + - node-role.kubernetes.io/control-plane=:NoSchedule +KubernetesVersion: 1.27.0 +Networking: + nonMasqueradeCIDR: 100.64.0.0/10 + serviceClusterIPRange: 100.64.0.0/13 +UpdatePolicy: automatic +channels: +- memfs://tests/minimal.example.com/addons/bootstrap-channel.yaml +containerdConfig: + logLevel: info + runc: + version: 1.1.5 + version: 1.6.20 +docker: + skipInstall: true +etcdManifests: +- memfs://tests/minimal.example.com/manifests/etcd/main-master-us-test-1a.yaml +- memfs://tests/minimal.example.com/manifests/etcd/events-master-us-test-1a.yaml +staticManifests: +- key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml +useInstanceIDForNodeName: true +usesLegacyGossip: false +usesNoneDNS: false diff --git a/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_nodeupconfig-nodes_content b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_nodeupconfig-nodes_content new file mode 100644 index 0000000000..737ef2f9fb --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/data/aws_s3_object_nodeupconfig-nodes_content @@ -0,0 +1,67 @@ +Assets: + amd64: + - 0b4ed4fcd75d33f5dff3ba17776e6089847fc83064d3f7a3ad59a34e94e60a29@https://dl.k8s.io/release/v1.27.0/bin/linux/amd64/kubelet + - 71a78259d70da9c5540c4cf4cff121f443e863376f68f89a759d90cef3f51e87@https://dl.k8s.io/release/v1.27.0/bin/linux/amd64/kubectl + - 5035d7814c95cd3cedbc5efb447ef25a4942ef05caab2159746d55ce1698c74a@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/amd64/ecr-credential-provider-linux-amd64 + - f3a841324845ca6bf0d4091b4fc7f97e18a623172158b72fc3fdcdb9d42d2d37@https://storage.googleapis.com/k8s-artifacts-cni/release/v1.2.0/cni-plugins-linux-amd64-v1.2.0.tgz + - bb9a9ccd6517e2a54da748a9f60dc9aa9d79d19d4724663f2386812f083968e2@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-amd64.tar.gz + - f00b144e86f8c1db347a2e8f22caade07d55382c5f76dd5c0a5b1ab64eaec8bb@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.amd64 + arm64: + - 37aa2edc7c0c4b3e488518c6a4b44c8aade75a55010534ee2be291220c73d157@https://dl.k8s.io/release/v1.27.0/bin/linux/arm64/kubelet + - f8e09630211f2b7c6a8cc38835e7dea94708d401f5c84b23a37c70c604602ddc@https://dl.k8s.io/release/v1.27.0/bin/linux/arm64/kubectl + - b3d567bda9e2996fc1fbd9d13506bd16763d3865b5c7b0b3c4b48c6088c04481@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/arm64/ecr-credential-provider-linux-arm64 + - 525e2b62ba92a1b6f3dc9612449a84aa61652e680f7ebf4eff579795fe464b57@https://storage.googleapis.com/k8s-artifacts-cni/release/v1.2.0/cni-plugins-linux-arm64-v1.2.0.tgz + - c3e6a054b18b20fce06c7c3ed53f0989bb4b255c849bede446ebca955f07a9ce@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-arm64.tar.gz + - 54e79e4d48b9e191767e4abc08be1a8476a1c757e9a9f8c45c6ded001226867f@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.arm64 +CAs: {} +ClusterName: minimal.example.com +ContainerRuntime: containerd +Hooks: +- null +- null +KeypairIDs: + kubernetes-ca: "6982820025135291416230495506" +KubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + image: registry.k8s.io/kube-proxy:v1.27.0 + logLevel: 2 +KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + InTreePluginAWSUnregister: "true" + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + nodeLabels: + kops.k8s.io/instancegroup: nodes-us-test-1a + node-role.kubernetes.io/node: "" + podInfraContainerImage: registry.k8s.io/pause:3.6 + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + registerSchedulable: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s +KubernetesVersion: 1.27.0 +Networking: + nonMasqueradeCIDR: 100.64.0.0/10 + serviceClusterIPRange: 100.64.0.0/13 +UpdatePolicy: automatic +channels: +- memfs://tests/minimal.example.com/addons/bootstrap-channel.yaml +containerdConfig: + logLevel: info + runc: + version: 1.1.5 + version: 1.6.20 +docker: + skipInstall: true +useInstanceIDForNodeName: true +usesLegacyGossip: false +usesNoneDNS: false diff --git a/tests/integration/update_cluster/minimal-1.27/id_rsa.pub b/tests/integration/update_cluster/minimal-1.27/id_rsa.pub new file mode 100755 index 0000000000..81cb012783 --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/minimal-1.27/in-v1alpha2.yaml b/tests/integration/update_cluster/minimal-1.27/in-v1alpha2.yaml new file mode 100644 index 0000000000..b62b1eb61a --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/in-v1alpha2.yaml @@ -0,0 +1,101 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + name: minimal.example.com +spec: + api: + dns: {} + authorization: + rbac: {} + channel: stable + cloudProvider: aws + configBase: memfs://tests/minimal.example.com + etcdClusters: + - cpuRequest: 200m + etcdMembers: + - encryptedVolume: true + instanceGroup: master-us-test-1a + name: a + memoryRequest: 100Mi + name: main + - cpuRequest: 100m + etcdMembers: + - encryptedVolume: true + instanceGroup: master-us-test-1a + name: a + memoryRequest: 100Mi + name: events + iam: + allowContainerRegistry: true + legacy: false + kubelet: + anonymousAuth: false + kubernetesApiAccess: + - 0.0.0.0/0 + - ::/0 + kubernetesVersion: v1.27.0 + masterPublicName: api.minimal.example.com + networkCIDR: 172.20.0.0/16 + networking: + cni: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + - ::/0 + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Public + zone: us-test-1a + topology: + dns: + type: Public + masters: public + nodes: public + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: minimal.example.com + name: master-us-test-1a +spec: + image: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220404 + instanceMetadata: + httpPutResponseHopLimit: 3 + httpTokens: required + machineType: m3.medium + maxSize: 1 + minSize: 1 + nodeLabels: + kops.k8s.io/instancegroup: master-us-test-1a + role: Master + subnets: + - us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: minimal.example.com + name: nodes +spec: + image: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220404 + instanceMetadata: + httpPutResponseHopLimit: 1 + httpTokens: required + machineType: t2.medium + maxSize: 1 + minSize: 1 + nodeLabels: + kops.k8s.io/instancegroup: nodes-us-test-1a + role: Node + subnets: + - us-test-1a diff --git a/tests/integration/update_cluster/minimal-1.27/kubernetes.tf b/tests/integration/update_cluster/minimal-1.27/kubernetes.tf new file mode 100644 index 0000000000..15b317b6ee --- /dev/null +++ b/tests/integration/update_cluster/minimal-1.27/kubernetes.tf @@ -0,0 +1,877 @@ +locals { + cluster_name = "minimal.example.com" + master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id] + master_security_group_ids = [aws_security_group.masters-minimal-example-com.id] + masters_role_arn = aws_iam_role.masters-minimal-example-com.arn + masters_role_name = aws_iam_role.masters-minimal-example-com.name + node_autoscaling_group_ids = [aws_autoscaling_group.nodes-minimal-example-com.id] + node_security_group_ids = [aws_security_group.nodes-minimal-example-com.id] + node_subnet_ids = [aws_subnet.us-test-1a-minimal-example-com.id] + nodes_role_arn = aws_iam_role.nodes-minimal-example-com.arn + nodes_role_name = aws_iam_role.nodes-minimal-example-com.name + region = "us-test-1" + route_table_public_id = aws_route_table.minimal-example-com.id + subnet_us-test-1a_id = aws_subnet.us-test-1a-minimal-example-com.id + vpc_cidr_block = aws_vpc.minimal-example-com.cidr_block + vpc_id = aws_vpc.minimal-example-com.id + vpc_ipv6_cidr_block = aws_vpc.minimal-example-com.ipv6_cidr_block + vpc_ipv6_cidr_length = local.vpc_ipv6_cidr_block == "" ? null : tonumber(regex(".*/(\\d+)", local.vpc_ipv6_cidr_block)[0]) +} + +output "cluster_name" { + value = "minimal.example.com" +} + +output "master_autoscaling_group_ids" { + value = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id] +} + +output "master_security_group_ids" { + value = [aws_security_group.masters-minimal-example-com.id] +} + +output "masters_role_arn" { + value = aws_iam_role.masters-minimal-example-com.arn +} + +output "masters_role_name" { + value = aws_iam_role.masters-minimal-example-com.name +} + +output "node_autoscaling_group_ids" { + value = [aws_autoscaling_group.nodes-minimal-example-com.id] +} + +output "node_security_group_ids" { + value = [aws_security_group.nodes-minimal-example-com.id] +} + +output "node_subnet_ids" { + value = [aws_subnet.us-test-1a-minimal-example-com.id] +} + +output "nodes_role_arn" { + value = aws_iam_role.nodes-minimal-example-com.arn +} + +output "nodes_role_name" { + value = aws_iam_role.nodes-minimal-example-com.name +} + +output "region" { + value = "us-test-1" +} + +output "route_table_public_id" { + value = aws_route_table.minimal-example-com.id +} + +output "subnet_us-test-1a_id" { + value = aws_subnet.us-test-1a-minimal-example-com.id +} + +output "vpc_cidr_block" { + value = aws_vpc.minimal-example-com.cidr_block +} + +output "vpc_id" { + value = aws_vpc.minimal-example-com.id +} + +output "vpc_ipv6_cidr_block" { + value = aws_vpc.minimal-example-com.ipv6_cidr_block +} + +output "vpc_ipv6_cidr_length" { + value = local.vpc_ipv6_cidr_block == "" ? null : tonumber(regex(".*/(\\d+)", local.vpc_ipv6_cidr_block)[0]) +} + +provider "aws" { + region = "us-test-1" +} + +provider "aws" { + alias = "files" + region = "us-test-1" +} + +resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" { + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + launch_template { + id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id + version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version + } + max_instance_lifetime = 0 + max_size = 1 + metrics_granularity = "1Minute" + min_size = 1 + name = "master-us-test-1a.masters.minimal.example.com" + protect_from_scale_in = false + tag { + key = "KubernetesCluster" + propagate_at_launch = true + value = "minimal.example.com" + } + tag { + key = "Name" + propagate_at_launch = true + value = "master-us-test-1a.masters.minimal.example.com" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/instancegroup" + propagate_at_launch = true + value = "master-us-test-1a" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/role/control-plane" + propagate_at_launch = true + value = "1" + } + tag { + key = "k8s.io/role/master" + propagate_at_launch = true + value = "1" + } + tag { + key = "kops.k8s.io/instancegroup" + propagate_at_launch = true + value = "master-us-test-1a" + } + tag { + key = "kubernetes.io/cluster/minimal.example.com" + propagate_at_launch = true + value = "owned" + } + vpc_zone_identifier = [aws_subnet.us-test-1a-minimal-example-com.id] +} + +resource "aws_autoscaling_group" "nodes-minimal-example-com" { + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + launch_template { + id = aws_launch_template.nodes-minimal-example-com.id + version = aws_launch_template.nodes-minimal-example-com.latest_version + } + max_instance_lifetime = 0 + max_size = 1 + metrics_granularity = "1Minute" + min_size = 1 + name = "nodes.minimal.example.com" + protect_from_scale_in = false + tag { + key = "KubernetesCluster" + propagate_at_launch = true + value = "minimal.example.com" + } + tag { + key = "Name" + propagate_at_launch = true + value = "nodes.minimal.example.com" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/instancegroup" + propagate_at_launch = true + value = "nodes-us-test-1a" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/role/node" + propagate_at_launch = true + value = "1" + } + tag { + key = "kops.k8s.io/instancegroup" + propagate_at_launch = true + value = "nodes" + } + tag { + key = "kubernetes.io/cluster/minimal.example.com" + propagate_at_launch = true + value = "owned" + } + vpc_zone_identifier = [aws_subnet.us-test-1a-minimal-example-com.id] +} + +resource "aws_ebs_volume" "a-etcd-events-minimal-example-com" { + availability_zone = "us-test-1a" + encrypted = true + iops = 3000 + size = 20 + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "a.etcd-events.minimal.example.com" + "k8s.io/etcd/events" = "a/a" + "k8s.io/role/control-plane" = "1" + "k8s.io/role/master" = "1" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + throughput = 125 + type = "gp3" +} + +resource "aws_ebs_volume" "a-etcd-main-minimal-example-com" { + availability_zone = "us-test-1a" + encrypted = true + iops = 3000 + size = 20 + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "a.etcd-main.minimal.example.com" + "k8s.io/etcd/main" = "a/a" + "k8s.io/role/control-plane" = "1" + "k8s.io/role/master" = "1" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + throughput = 125 + type = "gp3" +} + +resource "aws_iam_instance_profile" "masters-minimal-example-com" { + name = "masters.minimal.example.com" + role = aws_iam_role.masters-minimal-example-com.name + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "masters.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_iam_instance_profile" "nodes-minimal-example-com" { + name = "nodes.minimal.example.com" + role = aws_iam_role.nodes-minimal-example-com.name + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_iam_role" "masters-minimal-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_masters.minimal.example.com_policy") + name = "masters.minimal.example.com" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "masters.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_iam_role" "nodes-minimal-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_nodes.minimal.example.com_policy") + name = "nodes.minimal.example.com" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_iam_role_policy" "masters-minimal-example-com" { + name = "masters.minimal.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_masters.minimal.example.com_policy") + role = aws_iam_role.masters-minimal-example-com.name +} + +resource "aws_iam_role_policy" "nodes-minimal-example-com" { + name = "nodes.minimal.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_nodes.minimal.example.com_policy") + role = aws_iam_role.nodes-minimal-example-com.name +} + +resource "aws_internet_gateway" "minimal-example-com" { + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_key_pair" "kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" { + key_name = "kubernetes.minimal.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57" + public_key = file("${path.module}/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key") + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" { + block_device_mappings { + device_name = "/dev/xvda" + ebs { + delete_on_termination = true + encrypted = true + iops = 3000 + throughput = 125 + volume_size = 64 + volume_type = "gp3" + } + } + block_device_mappings { + device_name = "/dev/sdc" + virtual_name = "ephemeral0" + } + iam_instance_profile { + name = aws_iam_instance_profile.masters-minimal-example-com.id + } + image_id = "ami-12345678" + instance_type = "m3.medium" + key_name = aws_key_pair.kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id + lifecycle { + create_before_destroy = true + } + metadata_options { + http_endpoint = "enabled" + http_protocol_ipv6 = "disabled" + http_put_response_hop_limit = 3 + http_tokens = "required" + } + monitoring { + enabled = false + } + name = "master-us-test-1a.masters.minimal.example.com" + network_interfaces { + associate_public_ip_address = true + delete_on_termination = true + ipv6_address_count = 0 + security_groups = [aws_security_group.masters-minimal-example-com.id] + } + tag_specifications { + resource_type = "instance" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "master-us-test-1a.masters.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/instancegroup" = "master-us-test-1a" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" + "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" + "k8s.io/role/control-plane" = "1" + "k8s.io/role/master" = "1" + "kops.k8s.io/instancegroup" = "master-us-test-1a" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + } + tag_specifications { + resource_type = "volume" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "master-us-test-1a.masters.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/instancegroup" = "master-us-test-1a" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" + "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" + "k8s.io/role/control-plane" = "1" + "k8s.io/role/master" = "1" + "kops.k8s.io/instancegroup" = "master-us-test-1a" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + } + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "master-us-test-1a.masters.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/instancegroup" = "master-us-test-1a" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" + "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" + "k8s.io/role/control-plane" = "1" + "k8s.io/role/master" = "1" + "kops.k8s.io/instancegroup" = "master-us-test-1a" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + user_data = filebase64("${path.module}/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data") +} + +resource "aws_launch_template" "nodes-minimal-example-com" { + block_device_mappings { + device_name = "/dev/xvda" + ebs { + delete_on_termination = true + encrypted = true + iops = 3000 + throughput = 125 + volume_size = 128 + volume_type = "gp3" + } + } + iam_instance_profile { + name = aws_iam_instance_profile.nodes-minimal-example-com.id + } + image_id = "ami-12345678" + instance_type = "t2.medium" + key_name = aws_key_pair.kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id + lifecycle { + create_before_destroy = true + } + metadata_options { + http_endpoint = "enabled" + http_protocol_ipv6 = "disabled" + http_put_response_hop_limit = 1 + http_tokens = "required" + } + monitoring { + enabled = false + } + name = "nodes.minimal.example.com" + network_interfaces { + associate_public_ip_address = true + delete_on_termination = true + ipv6_address_count = 0 + security_groups = [aws_security_group.nodes-minimal-example-com.id] + } + tag_specifications { + resource_type = "instance" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/instancegroup" = "nodes-us-test-1a" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "nodes" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + } + tag_specifications { + resource_type = "volume" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/instancegroup" = "nodes-us-test-1a" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "nodes" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + } + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/instancegroup" = "nodes-us-test-1a" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "nodes" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + user_data = filebase64("${path.module}/data/aws_launch_template_nodes.minimal.example.com_user_data") +} + +resource "aws_route" "route-0-0-0-0--0" { + destination_cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.minimal-example-com.id + route_table_id = aws_route_table.minimal-example-com.id +} + +resource "aws_route" "route-__--0" { + destination_ipv6_cidr_block = "::/0" + gateway_id = aws_internet_gateway.minimal-example-com.id + route_table_id = aws_route_table.minimal-example-com.id +} + +resource "aws_route_table" "minimal-example-com" { + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/kops/role" = "public" + } + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_route_table_association" "us-test-1a-minimal-example-com" { + route_table_id = aws_route_table.minimal-example-com.id + subnet_id = aws_subnet.us-test-1a-minimal-example-com.id +} + +resource "aws_s3_object" "cluster-completed-spec" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_cluster-completed.spec_content") + key = "tests/minimal.example.com/cluster-completed.spec" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "etcd-cluster-spec-events" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_etcd-cluster-spec-events_content") + key = "tests/minimal.example.com/backups/etcd/events/control/etcd-cluster-spec" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "etcd-cluster-spec-main" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_etcd-cluster-spec-main_content") + key = "tests/minimal.example.com/backups/etcd/main/control/etcd-cluster-spec" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "kops-version-txt" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_kops-version.txt_content") + key = "tests/minimal.example.com/kops-version.txt" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "manifests-etcdmanager-events-master-us-test-1a" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content") + key = "tests/minimal.example.com/manifests/etcd/events-master-us-test-1a.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "manifests-etcdmanager-main-master-us-test-1a" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content") + key = "tests/minimal.example.com/manifests/etcd/main-master-us-test-1a.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "manifests-static-kube-apiserver-healthcheck" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content") + key = "tests/minimal.example.com/manifests/static/kube-apiserver-healthcheck.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "minimal-example-com-addons-aws-cloud-controller-addons-k8s-io-k8s-1-18" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content") + key = "tests/minimal.example.com/addons/aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "minimal-example-com-addons-aws-ebs-csi-driver-addons-k8s-io-k8s-1-17" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content") + key = "tests/minimal.example.com/addons/aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "minimal-example-com-addons-bootstrap" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-bootstrap_content") + key = "tests/minimal.example.com/addons/bootstrap-channel.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "minimal-example-com-addons-coredns-addons-k8s-io-k8s-1-12" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content") + key = "tests/minimal.example.com/addons/coredns.addons.k8s.io/k8s-1.12.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "minimal-example-com-addons-dns-controller-addons-k8s-io-k8s-1-12" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content") + key = "tests/minimal.example.com/addons/dns-controller.addons.k8s.io/k8s-1.12.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "minimal-example-com-addons-kops-controller-addons-k8s-io-k8s-1-16" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content") + key = "tests/minimal.example.com/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "minimal-example-com-addons-kubelet-api-rbac-addons-k8s-io-k8s-1-9" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content") + key = "tests/minimal.example.com/addons/kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "minimal-example-com-addons-limit-range-addons-k8s-io" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-limit-range.addons.k8s.io_content") + key = "tests/minimal.example.com/addons/limit-range.addons.k8s.io/v1.5.0.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "minimal-example-com-addons-storage-aws-addons-k8s-io-v1-15-0" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content") + key = "tests/minimal.example.com/addons/storage-aws.addons.k8s.io/v1.15.0.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "nodeupconfig-master-us-test-1a" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_nodeupconfig-master-us-test-1a_content") + key = "tests/minimal.example.com/igconfig/control-plane/master-us-test-1a/nodeupconfig.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "nodeupconfig-nodes" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_nodeupconfig-nodes_content") + key = "tests/minimal.example.com/igconfig/node/nodes/nodeupconfig.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_security_group" "masters-minimal-example-com" { + description = "Security group for masters" + name = "masters.minimal.example.com" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "masters.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_security_group" "nodes-minimal-example-com" { + description = "Security group for nodes" + name = "nodes.minimal.example.com" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-masters-minimal-example-com" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-nodes-minimal-example-com" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-masters-minimal-example-com" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 443 + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 443 + type = "ingress" +} + +resource "aws_security_group_rule" "from-__--0-ingress-tcp-22to22-masters-minimal-example-com" { + from_port = 22 + ipv6_cidr_blocks = ["::/0"] + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "from-__--0-ingress-tcp-22to22-nodes-minimal-example-com" { + from_port = 22 + ipv6_cidr_blocks = ["::/0"] + protocol = "tcp" + security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "from-__--0-ingress-tcp-443to443-masters-minimal-example-com" { + from_port = 443 + ipv6_cidr_blocks = ["::/0"] + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 443 + type = "ingress" +} + +resource "aws_security_group_rule" "from-masters-minimal-example-com-egress-all-0to0-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-masters-minimal-example-com-egress-all-0to0-__--0" { + from_port = 0 + ipv6_cidr_blocks = ["::/0"] + protocol = "-1" + security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-masters-minimal-example-com-ingress-all-0to0-masters-minimal-example-com" { + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.masters-minimal-example-com.id + source_security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 0 + type = "ingress" +} + +resource "aws_security_group_rule" "from-masters-minimal-example-com-ingress-all-0to0-nodes-minimal-example-com" { + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes-minimal-example-com.id + source_security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 0 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-minimal-example-com-egress-all-0to0-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-nodes-minimal-example-com-egress-all-0to0-__--0" { + from_port = 0 + ipv6_cidr_blocks = ["::/0"] + protocol = "-1" + security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-all-0to0-nodes-minimal-example-com" { + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes-minimal-example-com.id + source_security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 0 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-tcp-1to2379-masters-minimal-example-com" { + from_port = 1 + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + source_security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 2379 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-tcp-2382to4000-masters-minimal-example-com" { + from_port = 2382 + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + source_security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 4000 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-tcp-4003to65535-masters-minimal-example-com" { + from_port = 4003 + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + source_security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 65535 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-udp-1to65535-masters-minimal-example-com" { + from_port = 1 + protocol = "udp" + security_group_id = aws_security_group.masters-minimal-example-com.id + source_security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 65535 + type = "ingress" +} + +resource "aws_subnet" "us-test-1a-minimal-example-com" { + availability_zone = "us-test-1a" + cidr_block = "172.20.32.0/19" + enable_resource_name_dns_a_record_on_launch = true + private_dns_hostname_type_on_launch = "resource-name" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kops.k8s.io/instance-group/master-us-test-1a" = "true" + "kops.k8s.io/instance-group/nodes" = "true" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" + } + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_vpc" "minimal-example-com" { + assign_generated_ipv6_cidr_block = true + cidr_block = "172.20.0.0/16" + enable_dns_hostnames = true + enable_dns_support = true + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_vpc_dhcp_options" "minimal-example-com" { + domain_name = "us-test-1.compute.internal" + domain_name_servers = ["AmazonProvidedDNS"] + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_vpc_dhcp_options_association" "minimal-example-com" { + dhcp_options_id = aws_vpc_dhcp_options.minimal-example-com.id + vpc_id = aws_vpc.minimal-example-com.id +} + +terraform { + required_version = ">= 0.15.0" + required_providers { + aws = { + "configuration_aliases" = [aws.files] + "source" = "hashicorp/aws" + "version" = ">= 4.0.0" + } + } +} diff --git a/upup/pkg/fi/cloudup/new_cluster.go b/upup/pkg/fi/cloudup/new_cluster.go index e5eff8cfc8..87ba6db39f 100644 --- a/upup/pkg/fi/cloudup/new_cluster.go +++ b/upup/pkg/fi/cloudup/new_cluster.go @@ -1526,30 +1526,36 @@ func addCiliumNetwork(cluster *api.Cluster) { // defaultImage returns the default Image, based on the cloudprovider func defaultImage(cluster *api.Cluster, channel *api.Channel, architecture architectures.Architecture) (string, error) { + kubernetesVersion, err := util.ParseKubernetesVersion(cluster.Spec.KubernetesVersion) + if err != nil { + return "", fmt.Errorf("unable to parse kubernetes version %q", cluster.Spec.KubernetesVersion) + } + if channel != nil { - var kubernetesVersion *semver.Version - if cluster.Spec.KubernetesVersion != "" { - var err error - kubernetesVersion, err = util.ParseKubernetesVersion(cluster.Spec.KubernetesVersion) - if err != nil { - return "", fmt.Errorf("unable to parse kubernetes version %q", cluster.Spec.KubernetesVersion) - } - } - if channel != nil && kubernetesVersion != nil { - image := channel.FindImage(cluster.Spec.GetCloudProvider(), *kubernetesVersion, architecture) - if image != nil { - return image.Name, nil - } + image := channel.FindImage(cluster.Spec.GetCloudProvider(), *kubernetesVersion, architecture) + if image != nil { + return image.Name, nil } } - switch cluster.Spec.GetCloudProvider() { - case api.CloudProviderDO: - return defaultDOImage, nil - case api.CloudProviderHetzner: - return defaultHetznerImage, nil - case api.CloudProviderScaleway: - return defaultScalewayImage, nil + if kubernetesVersion.LT(semver.MustParse("1.27.0")) { + switch cluster.Spec.GetCloudProvider() { + case api.CloudProviderDO: + return defaultDOImageFocal, nil + case api.CloudProviderHetzner: + return defaultHetznerImageFocal, nil + case api.CloudProviderScaleway: + return defaultScalewayImageFocal, nil + } + } else { + switch cluster.Spec.GetCloudProvider() { + case api.CloudProviderDO: + return defaultDOImageJammy, nil + case api.CloudProviderHetzner: + return defaultHetznerImageJammy, nil + case api.CloudProviderScaleway: + return defaultScalewayImageJammy, nil + } } return "", fmt.Errorf("unable to determine default image for cloud provider %q and architecture %q", cluster.Spec.GetCloudProvider(), architecture) diff --git a/upup/pkg/fi/cloudup/new_cluster_test.go b/upup/pkg/fi/cloudup/new_cluster_test.go index e1a03f371b..b6c9048b1a 100644 --- a/upup/pkg/fi/cloudup/new_cluster_test.go +++ b/upup/pkg/fi/cloudup/new_cluster_test.go @@ -400,7 +400,19 @@ func TestDefaultImage(t *testing.T) { }, }, architecture: architectures.ArchitectureAmd64, - expected: defaultDOImage, + expected: defaultDOImageFocal, + }, + { + cluster: &api.Cluster{ + Spec: api.ClusterSpec{ + KubernetesVersion: "v1.27.0", + CloudProvider: api.CloudProviderSpec{ + DO: &api.DOSpec{}, + }, + }, + }, + architecture: architectures.ArchitectureAmd64, + expected: defaultDOImageJammy, }, { cluster: &api.Cluster{ @@ -412,7 +424,19 @@ func TestDefaultImage(t *testing.T) { }, }, architecture: architectures.ArchitectureAmd64, - expected: defaultHetznerImage, + expected: defaultHetznerImageFocal, + }, + { + cluster: &api.Cluster{ + Spec: api.ClusterSpec{ + KubernetesVersion: "v1.27.0", + CloudProvider: api.CloudProviderSpec{ + Hetzner: &api.HetznerSpec{}, + }, + }, + }, + architecture: architectures.ArchitectureAmd64, + expected: defaultHetznerImageJammy, }, { cluster: &api.Cluster{ @@ -424,7 +448,19 @@ func TestDefaultImage(t *testing.T) { }, }, architecture: architectures.ArchitectureAmd64, - expected: defaultScalewayImage, + expected: defaultScalewayImageFocal, + }, + { + cluster: &api.Cluster{ + Spec: api.ClusterSpec{ + KubernetesVersion: "v1.27.0", + CloudProvider: api.CloudProviderSpec{ + Scaleway: &api.ScalewaySpec{}, + }, + }, + }, + architecture: architectures.ArchitectureAmd64, + expected: defaultScalewayImageJammy, }, } diff --git a/upup/pkg/fi/cloudup/populate_instancegroup_spec.go b/upup/pkg/fi/cloudup/populate_instancegroup_spec.go index 2028c2861d..51a4c2e621 100644 --- a/upup/pkg/fi/cloudup/populate_instancegroup_spec.go +++ b/upup/pkg/fi/cloudup/populate_instancegroup_spec.go @@ -52,9 +52,12 @@ const ( defaultMasterMachineTypeHetzner = "cx21" defaultMasterMachineTypeScaleway = "DEV1-M" - defaultDOImage = "ubuntu-20-04-x64" - defaultHetznerImage = "ubuntu-20.04" - defaultScalewayImage = "ubuntu_focal" + defaultDOImageFocal = "ubuntu-20-04-x64" + defaultHetznerImageFocal = "ubuntu-20.04" + defaultScalewayImageFocal = "ubuntu_focal" + defaultDOImageJammy = "ubuntu-22-04-x64" + defaultHetznerImageJammy = "ubuntu-22.04" + defaultScalewayImageJammy = "ubuntu_jammy" ) // TODO: this hardcoded list can be replaced with DescribeInstanceTypes' DedicatedHostsSupported field