kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove support for K8s 1.26 in kOps 1.32

This commit is contained in:
Ciprian Hacman 2025-01-04 14:38:29 +02:00
parent 6b0d029366
commit eaf796c3c0
19 changed files with 74 additions and 10483 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
cmd/kops/create_cluster_integration_test.go
View File

@ -46,14 +46,14 @@ var MagicTimestamp = metav1.Time{Time: time.Date(2017, 1, 1, 0, 0, 0, 0, time.UT
// TestCreateClusterMinimal runs kops create cluster minimal.example.com --zones us-test-1a
func TestCreateClusterMinimal(t *testing.T) {
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.25", "v1alpha2")
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.26", "v1alpha2")
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.27", "v1alpha2")
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.28", "v1alpha2")
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.29", "v1alpha2")
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.30", "v1alpha2")
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.26-arm64", "v1alpha2")
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.26-irsa", "v1alpha2")
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.31", "v1alpha2")
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.32", "v1alpha2")
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-arm64", "v1alpha2")
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-irsa", "v1alpha2")
}
// TestCreateClusterHetzner runs kops create cluster minimal.k8s.local --zones fsn1
@ -111,9 +111,8 @@ func TestCreateClusterHA(t *testing.T) {
// TestCreateClusterMinimalGCE runs kops create cluster minimal.example.com --cloud gce --zones us-test1-a
func TestCreateClusterMinimalGCE(t *testing.T) {
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.26-gce", "v1alpha2")
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.26-gce-dns-none", "v1alpha2")
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.29-gce", "v1alpha2")
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-gce", "v1alpha2")
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-gce-dns-none", "v1alpha2")
}
// TestCreateClusterHAGCE runs kops create cluster ha-gce.example.com --cloud gce --zones us-test1-a,us-test1-b,us-test1-c --master-zones us-test1-a,us-test1-b,us-test1-c

150
cmd/kops/integration_test.go
View File

@ -190,16 +190,6 @@ func (i integrationTest) withDefaultServiceAccountRoles24() *integrationTest {
withServiceAccountRole("ebs-csi-controller-sa.kube-system", true)
}
// withDefaultAddons24 adds the default addons for an AWS cluster running k8s 1.24
func (i integrationTest) withDefaultAddons24() *integrationTest {
return i.withAddons(
awsCCMAddon,
awsEBSCSIAddon,
dnsControllerAddon,
leaderElectionAddon,
)
}
// withDefaultAddons30 adds the default addons for an AWS cluster running k8s 1.30
func (i integrationTest) withDefaultAddons30() *integrationTest {
return i.withAddons(
@ -210,7 +200,7 @@ func (i integrationTest) withDefaultAddons30() *integrationTest {
}
func (i integrationTest) withDefaults24() *integrationTest {
return i.withDefaultAddons24().withDefaultServiceAccountRoles24()
return i.withDefaultAddons30().withDefaultServiceAccountRoles24()
}
const (
@ -232,7 +222,6 @@ const (
certManagerAddon = "certmanager.io-k8s-1.16"
clusterAutoscalerAddon = "cluster-autoscaler.addons.k8s.io-k8s-1.15"
dnsControllerAddon = "dns-controller.addons.k8s.io-k8s-1.12"
leaderElectionAddon = "leader-migration.rbac.addons.k8s.io-k8s-1.23"
metricsServerAddon = "metrics-server.addons.k8s.io-k8s-1.11"
nodeProblemDetectorAddon = "node-problem-detector.addons.k8s.io-k8s-1.17"
)
@ -248,35 +237,8 @@ func TestMinimalAWS(t *testing.T) {
runTestTerraformAWS(t)
}
// TestMinimal runs the test on a minimum configuration
func TestMinimal_v1_25(t *testing.T) {
newIntegrationTest("minimal.example.com", "minimal-1.25").
withAddons(
awsEBSCSIAddon,
dnsControllerAddon,
awsCCMAddon,
leaderElectionAddon,
).
runTestTerraformAWS(t)
}
// TestMinimal runs the test on a minimum configuration
func TestMinimal_v1_26(t *testing.T) {
t.Setenv("KOPS_RUN_TOO_NEW_VERSION", "1")
newIntegrationTest("minimal.example.com", "minimal-1.26").
withAddons(
awsEBSCSIAddon,
dnsControllerAddon,
awsCCMAddon,
).
runTestTerraformAWS(t)
}
// TestMinimal runs the test on a minimum configuration
func TestMinimal_v1_27(t *testing.T) {
t.Setenv("KOPS_RUN_TOO_NEW_VERSION", "1")
newIntegrationTest("minimal.example.com", "minimal-1.27").
withAddons(
awsEBSCSIAddon,
@ -288,8 +250,6 @@ func TestMinimal_v1_27(t *testing.T) {
// TestMinimal runs the test on a minimum configuration
func TestMinimal_v1_28(t *testing.T) {
t.Setenv("KOPS_RUN_TOO_NEW_VERSION", "1")
newIntegrationTest("minimal.example.com", "minimal-1.28").
withAddons(
awsEBSCSIAddon,
@ -301,8 +261,6 @@ func TestMinimal_v1_28(t *testing.T) {
// TestMinimal runs the test on a minimum configuration
func TestMinimal_v1_29(t *testing.T) {
t.Setenv("KOPS_RUN_TOO_NEW_VERSION", "1")
newIntegrationTest("minimal.example.com", "minimal-1.29").
withAddons(
awsEBSCSIAddon,
@ -314,8 +272,6 @@ func TestMinimal_v1_29(t *testing.T) {
// TestMinimal runs the test on a minimum configuration
func TestMinimal_v1_30(t *testing.T) {
t.Setenv("KOPS_RUN_TOO_NEW_VERSION", "1")
newIntegrationTest("minimal.example.com", "minimal-1.30").
withAddons(
awsEBSCSIAddon,
@ -325,10 +281,30 @@ func TestMinimal_v1_30(t *testing.T) {
runTestTerraformAWS(t)
}
// TestMinimal runs the test on a minimum configuration
func TestMinimal_v1_31(t *testing.T) {
newIntegrationTest("minimal.example.com", "minimal-1.31").
withAddons(
awsEBSCSIAddon,
dnsControllerAddon,
awsCCMAddon,
).
runTestTerraformAWS(t)
}
// TestMinimal runs the test on a minimum configuration
func TestMinimal_v1_32(t *testing.T) {
newIntegrationTest("minimal.example.com", "minimal-1.32").
withAddons(
awsEBSCSIAddon,
dnsControllerAddon,
awsCCMAddon,
).
runTestTerraformAWS(t)
}
// TestMinimal_NoneDNS runs the test on a minimum configuration with --dns=none
func TestMinimal_NoneDNS(t *testing.T) {
t.Setenv("KOPS_RUN_TOO_NEW_VERSION", "1")
newIntegrationTest("minimal.example.com", "minimal-dns-none").
withAddons(
awsEBSCSIAddon,
@ -538,14 +514,14 @@ func TestExternalPolicies(t *testing.T) {
// TestMinimalIPv6 runs the test on a minimum IPv6 configuration
func TestMinimalIPv6(t *testing.T) {
newIntegrationTest("minimal-ipv6.example.com", "minimal-ipv6").
withDefaultAddons24().
withDefaultAddons30().
runTestTerraformAWS(t)
}
// TestMinimalIPv6Calico runs the test on a minimum IPv6 configuration with Calico
func TestMinimalIPv6Calico(t *testing.T) {
newIntegrationTest("minimal-ipv6.example.com", "minimal-ipv6-calico").
withDefaultAddons24().
withDefaultAddons30().
withAddons(calicoAddon).
runTestTerraformAWS(t)
}
@ -553,7 +529,7 @@ func TestMinimalIPv6Calico(t *testing.T) {
// TestMinimalIPv6Cilium runs the test on a minimum IPv6 configuration with Cilium
func TestMinimalIPv6Cilium(t *testing.T) {
newIntegrationTest("minimal-ipv6.example.com", "minimal-ipv6-cilium").
withDefaultAddons24().
withDefaultAddons30().
withAddons(ciliumAddon).
runTestTerraformAWS(t)
}
@ -561,7 +537,7 @@ func TestMinimalIPv6Cilium(t *testing.T) {
// TestMinimalIPv6NoSubnetPrefix runs the test with "/64#N" subnet notation
func TestMinimalIPv6NoSubnetPrefix(t *testing.T) {
newIntegrationTest("minimal-ipv6.example.com", "minimal-ipv6-no-subnet-prefix").
withDefaultAddons24().
withDefaultAddons30().
runTestTerraformAWS(t)
}
@ -637,7 +613,7 @@ func TestBastionAdditionalUserData(t *testing.T) {
func TestPrivateFlannel(t *testing.T) {
newIntegrationTest("privateflannel.example.com", "privateflannel").
withPrivate().
withDefaultAddons24().
withDefaultAddons30().
withAddons(flannelAddon).
runTestTerraformAWS(t)
}
@ -646,7 +622,7 @@ func TestPrivateFlannel(t *testing.T) {
func TestPrivateCalico(t *testing.T) {
newIntegrationTest("privatecalico.example.com", "privatecalico").
withPrivate().
withDefaultAddons24().
withDefaultAddons30().
withAddons(calicoAddon).
runTestTerraformAWS(t)
}
@ -702,7 +678,7 @@ func TestPrivateCiliumENI(t *testing.T) {
func TestPrivateCanal(t *testing.T) {
newIntegrationTest("privatecanal.example.com", "privatecanal").
withPrivate().
withDefaultAddons24().
withDefaultAddons30().
withAddons(canalAddon).
runTestTerraformAWS(t)
}
@ -861,57 +837,7 @@ func TestManyAddonsCCMIRSA(t *testing.T) {
runTestTerraformAWS(t)
}
func TestManyAddonsCCMIRSA25(t *testing.T) {
newIntegrationTest("minimal.example.com", "many-addons-ccm-irsa25").
withOIDCDiscovery().
withServiceAccountRole("aws-load-balancer-controller.kube-system", true).
withServiceAccountRole("dns-controller.kube-system", true).
withServiceAccountRole("aws-cloud-controller-manager.kube-system", true).
withServiceAccountRole("cluster-autoscaler.kube-system", true).
withServiceAccountRole("ebs-csi-controller-sa.kube-system", true).
withServiceAccountRole("aws-node-termination-handler.kube-system", true).
withAddons(
"aws-load-balancer-controller.addons.k8s.io-k8s-1.19",
"aws-ebs-csi-driver.addons.k8s.io-k8s-1.17",
"certmanager.io-k8s-1.16",
"cluster-autoscaler.addons.k8s.io-k8s-1.15",
"networking.amazon-vpc-routed-eni-k8s-1.16",
"snapshot-controller.addons.k8s.io-k8s-1.20",
"aws-cloud-controller.addons.k8s.io-k8s-1.18",
leaderElectionAddon,
metricsServerAddon,
dnsControllerAddon,
).
runTestTerraformAWS(t)
}
func TestManyAddonsCCMIRSA26(t *testing.T) {
t.Setenv("KOPS_RUN_TOO_NEW_VERSION", "1")
newIntegrationTest("minimal.example.com", "many-addons-ccm-irsa26").
withOIDCDiscovery().
withServiceAccountRole("aws-load-balancer-controller.kube-system", true).
withServiceAccountRole("dns-controller.kube-system", true).
withServiceAccountRole("aws-cloud-controller-manager.kube-system", true).
withServiceAccountRole("cluster-autoscaler.kube-system", true).
withServiceAccountRole("ebs-csi-controller-sa.kube-system", true).
withServiceAccountRole("aws-node-termination-handler.kube-system", true).
withAddons(
"aws-load-balancer-controller.addons.k8s.io-k8s-1.19",
"aws-ebs-csi-driver.addons.k8s.io-k8s-1.17",
"certmanager.io-k8s-1.16",
"cluster-autoscaler.addons.k8s.io-k8s-1.15",
"networking.amazon-vpc-routed-eni-k8s-1.16",
"snapshot-controller.addons.k8s.io-k8s-1.20",
"aws-cloud-controller.addons.k8s.io-k8s-1.18",
"nodelocaldns.addons.k8s.io-k8s-1.12",
metricsServerAddon,
dnsControllerAddon,
).
runTestTerraformAWS(t)
}
func TestManyAddonsGCE(t *testing.T) {
t.Setenv("KOPS_RUN_TOO_NEW_VERSION", "1")
newIntegrationTest("minimal.example.com", "many-addons-gce").
withAddons(
certManagerAddon,
@ -1007,7 +933,7 @@ func TestSharedVPC(t *testing.T) {
// TestSharedVPCIPv6 runs the test on a configuration with a shared VPC using IPv6
func TestSharedVPCIPv6(t *testing.T) {
newIntegrationTest("minimal-ipv6.example.com", "shared_vpc_ipv6").
withDefaultAddons24().
withDefaultAddons30().
runTestTerraformAWS(t)
}
@ -1527,6 +1453,8 @@ func storeKeyset(t *testing.T, ctx context.Context, keyStore fi.Keystore, name s
}
func (i *integrationTest) runTestTerraformAWS(t *testing.T) {
t.Setenv("KOPS_RUN_TOO_NEW_VERSION", "1")
ctx := testcontext.ForTest(t)
h := testutils.NewIntegrationTestHarness(t)
defer h.Close()
@ -1615,6 +1543,8 @@ func (i *integrationTest) runTestTerraformAWS(t *testing.T) {
}
func (i *integrationTest) runTestPhase(t *testing.T, phase cloudup.Phase) {
t.Setenv("KOPS_RUN_TOO_NEW_VERSION", "1")
ctx := testcontext.ForTest(t)
h := testutils.NewIntegrationTestHarness(t)
defer h.Close()
@ -1660,6 +1590,8 @@ func (i *integrationTest) runTestPhase(t *testing.T, phase cloudup.Phase) {
}
func (i *integrationTest) runTestTerraformGCE(t *testing.T) {
t.Setenv("KOPS_RUN_TOO_NEW_VERSION", "1")
ctx := testcontext.ForTest(t)
h := testutils.NewIntegrationTestHarness(t)
defer h.Close()
@ -1688,8 +1620,8 @@ func (i *integrationTest) runTestTerraformGCE(t *testing.T) {
"aws_s3_object_"+i.clusterName+"-addons-kops-controller.addons.k8s.io-k8s-1.16_content",
"aws_s3_object_"+i.clusterName+"-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content",
"aws_s3_object_"+i.clusterName+"-addons-limit-range.addons.k8s.io_content",
"aws_s3_object_"+i.clusterName+"-addons-metadata-proxy.addons.k8s.io-v0.1.12_content",
"aws_s3_object_"+i.clusterName+"-addons-storage-gce.addons.k8s.io-v1.7.0_content")
"aws_s3_object_"+i.clusterName+"-addons-storage-gce.addons.k8s.io-v1.7.0_content",
)
for j := 0; j < i.zones; j++ {
zone := "us-test1-" + string([]byte{byte('a') + byte(j)})
@ -1710,6 +1642,8 @@ func (i *integrationTest) runTestTerraformGCE(t *testing.T) {
}
func (i *integrationTest) runTestTerraformHetzner(t *testing.T) {
t.Setenv("KOPS_RUN_TOO_NEW_VERSION", "1")
ctx := testcontext.ForTest(t)
h := testutils.NewIntegrationTestHarness(t)
defer h.Close()
@ -1743,6 +1677,8 @@ func (i *integrationTest) runTestTerraformHetzner(t *testing.T) {
}
func (i *integrationTest) runTestTerraformScaleway(t *testing.T) {
t.Setenv("KOPS_RUN_TOO_NEW_VERSION", "1")
featureflag.ParseFlags("+Scaleway")
unsetFeatureFlags := func() {
featureflag.ParseFlags("-Scaleway")

10
cmd/kops/lifecycle_integration_test.go
View File

@ -371,6 +371,7 @@ func runLifecycleTestAWS(o *LifecycleTestOptions) {
o.AddDefaults()
t := o.t
t.Setenv("KOPS_RUN_TOO_NEW_VERSION", "1")
h := testutils.NewIntegrationTestHarness(o.t)
defer h.Close()
@ -401,16 +402,12 @@ func runLifecycleTestOpenstack(o *LifecycleTestOptions) {
o.AddDefaults()
t := o.t
t.Setenv("KOPS_RUN_TOO_NEW_VERSION", "1")
t.Setenv("OS_REGION_NAME", "us-test1")
h := testutils.NewIntegrationTestHarness(o.t)
defer h.Close()
origRegion := os.Getenv("OS_REGION_NAME")
os.Setenv("OS_REGION_NAME", "us-test1")
defer func() {
os.Setenv("OS_REGION_NAME", origRegion)
}()
h.MockKopsVersion("1.21.0-alpha.1")
cloud := testutils.SetupMockOpenstack()
@ -459,6 +456,7 @@ func runLifecycleTestGCE(o *LifecycleTestOptions) {
o.AddDefaults()
t := o.t
t.Setenv("KOPS_RUN_TOO_NEW_VERSION", "1")
h := testutils.NewIntegrationTestHarness(o.t)
defer h.Close()

16
pkg/apis/kops/validation/validation.go
View File

@ -757,20 +757,18 @@ func validateKubeAPIServer(v *kops.KubeAPIServerConfig, c *kops.Cluster, fldPath
"admissionControl is mutually exclusive with disableAdmissionPlugins˚"))
}
if c.IsKubernetesGTE("1.26") {
allErrs = append(allErrs, field.Forbidden(fldPath.Child("admissionControl"), "admissionControl has been replaced with enableAdmissionPlugins"))
}
allErrs = append(allErrs, field.Forbidden(fldPath.Child("admissionControl"), "admissionControl has been replaced with enableAdmissionPlugins"))
}
for _, plugin := range v.EnableAdmissionPlugins {
if plugin == "PodSecurityPolicy" && c.IsKubernetesGTE("1.25") {
if plugin == "PodSecurityPolicy" {
allErrs = append(allErrs, field.Forbidden(fldPath.Child("enableAdmissionPlugins"),
"PodSecurityPolicy has been removed from Kubernetes 1.25"))
}
}
for _, plugin := range v.AdmissionControl {
if plugin == "PodSecurityPolicy" && c.IsKubernetesGTE("1.25") {
if plugin == "PodSecurityPolicy" {
allErrs = append(allErrs, field.Forbidden(fldPath.Child("admissionControl"),
"PodSecurityPolicy has been removed from Kubernetes 1.25"))
}
@ -853,9 +851,7 @@ func validateKubeControllerManager(v *kops.KubeControllerManagerConfig, c *kops.
// We aren't aiming to do comprehensive validation, but we can add some best-effort validation where it helps guide users
// Users reported encountered this in #15909
if v.ExperimentalClusterSigningDuration != nil {
if c.IsKubernetesGTE("1.25") {
allErrs = append(allErrs, field.Forbidden(fldPath.Child("experimentalClusterSigningDuration"), "experimentalClusterSigningDuration has been replaced with clusterSigningDuration as of kubernetes 1.25"))
}
allErrs = append(allErrs, field.Forbidden(fldPath.Child("experimentalClusterSigningDuration"), "experimentalClusterSigningDuration has been replaced with clusterSigningDuration as of kubernetes 1.25"))
}
return allErrs
@ -1111,9 +1107,7 @@ func validateNetworking(cluster *kops.Cluster, v *kops.NetworkingSpec, fldPath *
allErrs = append(allErrs, field.Forbidden(fldPath.Child("external"), "only one networking option permitted"))
}
if cluster.IsKubernetesGTE("1.26") {
allErrs = append(allErrs, field.Forbidden(fldPath.Child("external"), "external is not supported for Kubernetes >= 1.26"))
}
allErrs = append(allErrs, field.Forbidden(fldPath.Child("external"), "external is not supported for Kubernetes >= 1.26"))
optionTaken = true
}

8
pkg/model/components/awscloudcontrollermanager.go
View File

@ -75,10 +75,6 @@ func (b *AWSCloudControllerManagerOptionsBuilder) BuildOptions(cluster *kops.Clu
if eccm.Image == "" {
// See https://us.gcr.io/k8s-artifacts-prod/provider-aws/cloud-controller-manager
switch b.ControlPlaneKubernetesVersion().Minor() {
case 25:
eccm.Image = "registry.k8s.io/provider-aws/cloud-controller-manager:v1.25.15"
case 26:
eccm.Image = "registry.k8s.io/provider-aws/cloud-controller-manager:v1.26.12"
case 27:
eccm.Image = "registry.k8s.io/provider-aws/cloud-controller-manager:v1.27.9"
case 28:
@ -94,9 +90,5 @@ func (b *AWSCloudControllerManagerOptionsBuilder) BuildOptions(cluster *kops.Clu
}
}
if b.ControlPlaneKubernetesVersion().IsLT("1.25") {
eccm.EnableLeaderMigration = fi.PtrTo(true)
}
return nil
}

4
pkg/model/components/clusterautoscaler.go
View File

@ -43,10 +43,6 @@ func (b *ClusterAutoscalerOptionsBuilder) BuildOptions(o *kops.Cluster) error {
v, err := util.ParseKubernetesVersion(clusterSpec.KubernetesVersion)
if err == nil {
switch v.Minor {
case 25:
image = "registry.k8s.io/autoscaling/cluster-autoscaler:v1.25.3"
case 26:
image = "registry.k8s.io/autoscaling/cluster-autoscaler:v1.26.8"
case 27:
image = "registry.k8s.io/autoscaling/cluster-autoscaler:v1.27.7"
case 28:

13
pkg/model/components/containerd.go
View File

@ -42,22 +42,13 @@ func (b *ContainerdOptionsBuilder) BuildOptions(o *kops.Cluster) error {
// Set version based on Kubernetes version
if fi.ValueOf(containerd.Version) == "" {
switch {
case b.IsKubernetesLT("1.25.10"):
fallthrough
case b.IsKubernetesGTE("1.26") && b.IsKubernetesLT("1.26.5"):
fallthrough
case b.IsKubernetesGTE("1.27") && b.IsKubernetesLT("1.27.2"):
case b.IsKubernetesLT("1.27.2"):
containerd.Version = fi.PtrTo("1.6.20")
containerd.Runc = &kops.Runc{
Version: fi.PtrTo("1.1.5"),
}
case b.IsKubernetesGTE("1.27.2"):
containerd.Version = fi.PtrTo("1.7.22")
containerd.Runc = &kops.Runc{
Version: fi.PtrTo("1.1.14"),
}
default:
containerd.Version = fi.PtrTo("1.6.36")
containerd.Version = fi.PtrTo("1.7.22")
containerd.Runc = &kops.Runc{
Version: fi.PtrTo("1.1.14"),
}

4
pkg/model/components/gcpcloudcontrollermanager.go
View File

@ -79,9 +79,5 @@ func (b *GCPCloudControllerManagerOptionsBuilder) BuildOptions(cluster *kops.Clu
}
}
if b.ControlPlaneKubernetesVersion().IsLT("1.25") {
ccmConfig.EnableLeaderMigration = fi.PtrTo(true)
}
return nil
}

3
pkg/model/components/kubescheduler/model.go
View File

@ -88,9 +88,6 @@ func (b *KubeSchedulerBuilder) buildSchedulerConfig() ([]byte, error) {
config = &unstructured.Unstructured{}
config.SetKind("KubeSchedulerConfiguration")
config.SetAPIVersion("kubescheduler.config.k8s.io/v1")
if b.IsKubernetesLT("1.25") {
config.SetAPIVersion("kubescheduler.config.k8s.io/v1beta2")
}
// We need to store the object, because we are often called repeatedly (until we converge)
b.AdditionalObjects = append(b.AdditionalObjects, kubemanifest.NewObject(config.Object))
}

22
tests/e2e/pkg/tester/skip_regex.go
View File

@ -50,14 +50,6 @@ func (t *Tester) setSkipRegexFlag() error {
skipRegex := skipRegexBase
if k8sVersion.Minor == 26 && cluster.Spec.LegacyCloudProvider == "aws" {
// This test was introduced in k8s 1.26
// and skipped automatically for AWS clusters as of k8s 1.27
// because Classic Load Balancers dont support UDP
// https://github.com/kubernetes/kubernetes/pull/113650
// https://github.com/kubernetes/kubernetes/pull/115977
skipRegex += "|LoadBalancers.should.be.able.to.preserve.UDP.traffic"
}
if !isPre28 {
// K8s 1.28 promoted ProxyTerminatingEndpoints to GA, but it has limited CNI support
// https://github.com/kubernetes/kubernetes/pull/117718
@ -167,20 +159,6 @@ func (t *Tester) setSkipRegexFlag() error {
skipRegex += "|should.verify.that.all.nodes.have.volume.limits"
}
if cluster.Spec.LegacyCloudProvider == "aws" {
if k8sVersion.Minor <= 26 {
// Prow jobs are being migrated to community-owned EKS clusters.
// The e2e.test binaries from older k/k builds dont have new enough aws-sdk-go versions to authenticate from EKS pods.
// This disables tests that depend on e2e.test's aws-sdk-go.
//
// > Couldn't create a new PD in zone "ap-northeast-1c", sleeping 5 seconds: NoCredentialProviders: no valid providers in chain. Deprecated.
//
// We can remove this once we remove the old upgrade jobs.
// Example: https://prow.k8s.io/view/gs/kubernetes-jenkins/logs/e2e-kops-aws-upgrade-k125-ko128-to-k126-kolatest/1808210907088556032
skipRegex += "|\\[Driver:.aws\\].\\[Testpattern:.Pre-provisioned.PV|\\[Driver:.aws\\].\\[Testpattern:.Inline-volume"
}
}
// This test fails on RHEL-based distros because they return fully qualified hostnames yet the k8s node names are not fully qualified.
// Dedicated job testing this: https://testgrid.k8s.io/kops-misc#kops-aws-k28-hostname-bug123255
// ref: https://github.com/kubernetes/kops/issues/16349

44
upup/models/cloudup/resources/addons/leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml
View File

@ -1,44 +0,0 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
labels:
k8s-addon: leader-migration.rbac.addons.k8s.io
name: system::leader-locking-migration
namespace: kube-system
rules:
- apiGroups:
- coordination.k8s.io
resources:
- leases
resourceNames:
- cloud-provider-extraction-migration
verbs:
- create
- list
- get
- update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
k8s-addon: leader-migration.rbac.addons.k8s.io
name: system::leader-locking-migration
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: system::leader-locking-migration
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: system:kube-controller-manager
- kind: ServiceAccount
name: kube-controller-manager
namespace: kube-system
- kind: ServiceAccount
name: aws-cloud-controller-manager
namespace: kube-system
- kind: ServiceAccount
name: cloud-controller-manager
namespace: kube-system

258
upup/models/cloudup/resources/addons/networking.flannel/k8s-1.12.yaml.template
View File

@ -1,258 +0,0 @@
# Pulled and modified from: https://raw.githubusercontent.com/coreos/flannel/v0.17.0/Documentation/kube-flannel.yml
---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
name: psp.flannel.unprivileged
annotations:
seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
spec:
privileged: false
volumes:
- configMap
- secret
- emptyDir
- hostPath
allowedHostPaths:
- pathPrefix: "/dev/net"
- pathPrefix: "/etc/cni/net.d"
- pathPrefix: "/etc/kube-flannel"
- pathPrefix: "/run/flannel"
readOnlyRootFilesystem: false
# Users and groups
runAsUser:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
fsGroup:
rule: RunAsAny
# Privilege Escalation
allowPrivilegeEscalation: false
defaultAllowPrivilegeEscalation: false
# Capabilities
allowedCapabilities: ['NET_ADMIN', 'NET_RAW']
defaultAddCapabilities: []
requiredDropCapabilities: []
# Host namespaces
hostPID: false
hostIPC: false
hostNetwork: true
hostPorts:
- min: 0
max: 65535
# SELinux
seLinux:
# SELinux is unused in CaaSP
rule: 'RunAsAny'
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: flannel
rules:
- apiGroups: ['extensions']
resources: ['podsecuritypolicies']
verbs: ['use']
resourceNames: ['psp.flannel.unprivileged']
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- apiGroups:
- ""
resources:
- nodes
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes/status
verbs:
- patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: flannel
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: flannel
subjects:
- kind: ServiceAccount
name: flannel
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: flannel
namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
name: kube-flannel-cfg
namespace: kube-system
labels:
k8s-app: flannel
tier: node
app: flannel
data:
cni-conf.json: |
{
"name": "cbr0",
"cniVersion": "0.3.1",
"plugins": [
{
"type": "flannel",
"delegate": {
"hairpinMode": true,
"isDefaultGateway": true
}
},
{
"type": "portmap",
"capabilities": {
"portMappings": true
}
}
]
}
net-conf.json: |
{
"Network": "{{ .Networking.NonMasqueradeCIDR }}",
"Backend": {
"Type": "{{ FlannelBackendType }}"
}
}
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kube-flannel-ds
namespace: kube-system
labels:
k8s-app: flannel
tier: node
app: flannel
spec:
selector:
matchLabels:
tier: node
app: flannel
template:
metadata:
labels:
tier: node
app: flannel
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/os
operator: In
values:
- linux
hostNetwork: true
priorityClassName: system-node-critical
tolerations:
- operator: Exists
serviceAccountName: flannel
initContainers:
- name: install-cni-plugin
image: rancher/mirrored-flannelcni-flannel-cni-plugin:v1.0.1
command:
- cp
args:
- -f
- /flannel
- /opt/cni/bin/flannel
volumeMounts:
- name: cni-plugin
mountPath: /opt/cni/bin
- name: install-cni
image: rancher/mirrored-flannelcni-flannel:v0.17.0
command:
- cp
args:
- -f
- /etc/kube-flannel/cni-conf.json
- /etc/cni/net.d/10-flannel.conflist
volumeMounts:
- name: cni
mountPath: /etc/cni/net.d
- name: flannel-cfg
mountPath: /etc/kube-flannel/
containers:
- name: kube-flannel
image: rancher/mirrored-flannelcni-flannel:v0.17.0
command:
- /opt/bin/flanneld
args:
- --ip-masq
- --kube-subnet-mgr
- --iptables-resync={{- or .Networking.Flannel.IptablesResyncSeconds "5" }}
resources:
limits:
memory: 100Mi
requests:
cpu: 100m
memory: 100Mi
securityContext:
privileged: false
capabilities:
add: ["NET_ADMIN", "NET_RAW"]
{{ if ContainerdSELinuxEnabled }}
seLinuxOptions:
type: spc_t
level: s0
{{ end }}
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
volumeMounts:
- name: run
mountPath: /run/flannel
- name: dev-net
mountPath: /dev/net
- name: flannel-cfg
mountPath: /etc/kube-flannel/
- name: xtables-lock
mountPath: /run/xtables.lock
volumes:
- name: run
hostPath:
path: /run/flannel
- name: dev-net
hostPath:
path: /dev/net
- name: cni-plugin
hostPath:
path: /opt/cni/bin
- name: cni
hostPath:
path: /etc/cni/net.d
- name: flannel-cfg
configMap:
name: kube-flannel-cfg
- name: xtables-lock
hostPath:
path: /run/xtables.lock
type: FileOrCreate

4904
upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.22.yaml.template
View File

File diff suppressed because it is too large Load Diff

4964
upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.22.yaml.template
View File

File diff suppressed because it is too large Load Diff

4
upup/pkg/fi/cloudup/apply_cluster.go
View File

@ -72,9 +72,9 @@ const (
starline = "*********************************************************************************"
// OldestSupportedKubernetesVersion is the oldest kubernetes version that is supported in kOps.
OldestSupportedKubernetesVersion = "1.25.0"
OldestSupportedKubernetesVersion = "1.27.0"
// OldestRecommendedKubernetesVersion is the oldest kubernetes version that is not deprecated in kOps.
OldestRecommendedKubernetesVersion = "1.27.0"
OldestRecommendedKubernetesVersion = "1.29.0"
)
// TerraformCloudProviders is the list of cloud providers with terraform target support

58
upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go
View File

@ -426,25 +426,6 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext)
}
}
if b.IsKubernetesLT("1.26") &&
(b.Cluster.GetCloudProvider() == kops.CloudProviderAWS ||
b.Cluster.GetCloudProvider() == kops.CloudProviderGCE) {
// AWS and GCE KCM-to-CCM leader migration
key := "leader-migration.rbac.addons.k8s.io"
{
location := key + "/k8s-1.23.yaml"
id := "k8s-1.23"
addons.Add(&channelsapi.AddonSpec{
Name: fi.PtrTo(key),
Selector: map[string]string{"k8s-addon": key},
Manifest: fi.PtrTo(location),
Id: id,
})
}
}
{
key := "limit-range.addons.k8s.io"
version := "1.5.0"
@ -951,21 +932,10 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext)
if b.Cluster.Spec.Networking.Flannel != nil {
key := "networking.flannel"
if b.IsKubernetesGTE("v1.25.0") {
{
id := "k8s-1.25"
location := key + "/" + id + ".yaml"
addon := addons.Add(&channelsapi.AddonSpec{
Name: fi.PtrTo(key),
Selector: networkingSelector(),
Manifest: fi.PtrTo(location),
Id: id,
})
addon.BuildPrune = true
} else {
id := "k8s-1.12"
location := key + "/" + id + ".yaml"
addon := addons.Add(&channelsapi.AddonSpec{
Name: fi.PtrTo(key),
Selector: networkingSelector(),
@ -979,21 +949,10 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext)
if b.Cluster.Spec.Networking.Calico != nil {
key := "networking.projectcalico.org"
if b.IsKubernetesGTE("v1.25.0") {
{
id := "k8s-1.25"
location := key + "/" + id + ".yaml"
addon := addons.Add(&channelsapi.AddonSpec{
Name: fi.PtrTo(key),
Selector: networkingSelector(),
Manifest: fi.PtrTo(location),
Id: id,
})
addon.BuildPrune = true
} else {
id := "k8s-1.22"
location := key + "/" + id + ".yaml"
addon := addons.Add(&channelsapi.AddonSpec{
Name: fi.PtrTo(key),
Selector: networkingSelector(),
@ -1007,21 +966,10 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext)
if b.Cluster.Spec.Networking.Canal != nil {
key := "networking.projectcalico.org.canal"
if b.IsKubernetesGTE("v1.25.0") {
{
id := "k8s-1.25"
location := key + "/" + id + ".yaml"
addon := addons.Add(&channelsapi.AddonSpec{
Name: fi.PtrTo(key),
Selector: networkingSelector(),
Manifest: fi.PtrTo(location),
Id: id,
})
addon.BuildPrune = true
} else {
id := "k8s-1.22"
location := key + "/" + id + ".yaml"
addon := addons.Add(&channelsapi.AddonSpec{
Name: fi.PtrTo(key),
Selector: networkingSelector(),

50
upup/pkg/fi/cloudup/new_cluster_test.go
View File

@ -441,7 +441,7 @@ func TestDefaultImage(t *testing.T) {
{
cluster: &api.Cluster{
Spec: api.ClusterSpec{
KubernetesVersion: "v1.25.0",
KubernetesVersion: "v1.32.0",
CloudProvider: api.CloudProviderSpec{
AWS: &api.AWSSpec{},
},
@ -453,7 +453,7 @@ func TestDefaultImage(t *testing.T) {
{
cluster: &api.Cluster{
Spec: api.ClusterSpec{
KubernetesVersion: "v1.25.0",
KubernetesVersion: "v1.32.0",
CloudProvider: api.CloudProviderSpec{
AWS: &api.AWSSpec{},
},
@ -465,7 +465,7 @@ func TestDefaultImage(t *testing.T) {
{
cluster: &api.Cluster{
Spec: api.ClusterSpec{
KubernetesVersion: "v1.25.0",
KubernetesVersion: "v1.32.0",
CloudProvider: api.CloudProviderSpec{
Azure: &api.AzureSpec{},
},
@ -477,7 +477,7 @@ func TestDefaultImage(t *testing.T) {
{
cluster: &api.Cluster{
Spec: api.ClusterSpec{
KubernetesVersion: "v1.25.0",
KubernetesVersion: "v1.32.0",
CloudProvider: api.CloudProviderSpec{
GCE: &api.GCESpec{},
},
@ -489,19 +489,7 @@ func TestDefaultImage(t *testing.T) {
{
cluster: &api.Cluster{
Spec: api.ClusterSpec{
KubernetesVersion: "v1.25.0",
CloudProvider: api.CloudProviderSpec{
DO: &api.DOSpec{},
},
},
},
architecture: architectures.ArchitectureAmd64,
expected: defaultDOImageFocal,
},
{
cluster: &api.Cluster{
Spec: api.ClusterSpec{
KubernetesVersion: "v1.27.0",
KubernetesVersion: "v1.32.0",
CloudProvider: api.CloudProviderSpec{
DO: &api.DOSpec{},
},
@ -513,19 +501,7 @@ func TestDefaultImage(t *testing.T) {
{
cluster: &api.Cluster{
Spec: api.ClusterSpec{
KubernetesVersion: "v1.25.0",
CloudProvider: api.CloudProviderSpec{
Hetzner: &api.HetznerSpec{},
},
},
},
architecture: architectures.ArchitectureAmd64,
expected: defaultHetznerImageFocal,
},
{
cluster: &api.Cluster{
Spec: api.ClusterSpec{
KubernetesVersion: "v1.27.0",
KubernetesVersion: "v1.32.0",
CloudProvider: api.CloudProviderSpec{
Hetzner: &api.HetznerSpec{},
},
@ -537,19 +513,7 @@ func TestDefaultImage(t *testing.T) {
{
cluster: &api.Cluster{
Spec: api.ClusterSpec{
KubernetesVersion: "v1.25.0",
CloudProvider: api.CloudProviderSpec{
Scaleway: &api.ScalewaySpec{},
},
},
},
architecture: architectures.ArchitectureAmd64,
expected: defaultScalewayImageFocal,
},
{
cluster: &api.Cluster{
Spec: api.ClusterSpec{
KubernetesVersion: "v1.27.0",
KubernetesVersion: "v1.32.0",
CloudProvider: api.CloudProviderSpec{
Scaleway: &api.ScalewaySpec{},
},

12
upup/pkg/fi/cloudup/template_functions.go
View File

@ -913,11 +913,7 @@ func (tf *TemplateFunctions) OpenStackCCMTag() string {
if err != nil {
tag = "latest"
} else {
if parsed.Minor == 25 {
tag = "v1.25.5"
} else if parsed.Minor == 26 {
tag = "v1.26.2"
} else if parsed.Minor == 27 {
if parsed.Minor == 27 {
tag = "v1.27.1"
} else {
// otherwise we use always .0 ccm image, if needed that can be overrided using clusterspec
@ -935,11 +931,7 @@ func (tf *TemplateFunctions) OpenStackCSITag() string {
if err != nil {
tag = "latest"
} else {
if parsed.Minor == 25 {
tag = "v1.25.5"
} else if parsed.Minor == 26 {
tag = "v1.26.2"
} else if parsed.Minor == 27 {
if parsed.Minor == 27 {
tag = "v1.27.1"
} else {
// otherwise we use always .0 csi image, if needed that can be overrided using cloud config spec

20
upup/pkg/fi/cloudup/template_functions_test.go
View File

@ -242,26 +242,6 @@ func Test_TemplateFunctions_CloudControllerConfigArgv(t *testing.T) {
"--cloud-config=/etc/kubernetes/cloud.config",
},
},
{
desc: "Leader Migration",
cluster: &kops.Cluster{Spec: kops.ClusterSpec{
CloudProvider: kops.CloudProviderSpec{
Openstack: &kops.OpenstackSpec{},
},
ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
LeaderElection: &kops.LeaderElectionConfiguration{LeaderElect: fi.PtrTo(true)},
EnableLeaderMigration: fi.PtrTo(true),
},
}},
expectedArgv: []string{
"--enable-leader-migration=true",
"--leader-elect=true",
"--v=2",
"--cloud-provider=openstack",
"--use-service-account-credentials=true",
"--cloud-config=/etc/kubernetes/cloud.config",
},
},
{
desc: "Disable Controller",
cluster: &kops.Cluster{Spec: kops.ClusterSpec{