diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index 9208a67615..ee57268355 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -237,7 +237,7 @@ func TestMinimalGossip(t *testing.T) { // TestMinimalGCE runs tests on a minimal GCE configuration func TestMinimalGCE(t *testing.T) { newIntegrationTest("minimal-gce.example.com", "minimal_gce"). - withAddons(dnsControllerAddon, "gcp-pd-csi-driver.addons.k8s.io-k8s-1.23"). + withAddons(dnsControllerAddon, leaderElectionAddon, "gcp-pd-csi-driver.addons.k8s.io-k8s-1.23"). runTestTerraformGCE(t) } diff --git a/pkg/model/components/gcpcloudcontrollermanager.go b/pkg/model/components/gcpcloudcontrollermanager.go index de4c710b50..344a86e4e8 100644 --- a/pkg/model/components/gcpcloudcontrollermanager.go +++ b/pkg/model/components/gcpcloudcontrollermanager.go @@ -36,7 +36,12 @@ func (b *GCPCloudControllerManagerOptionsBuilder) BuildOptions(options interface return nil } + if clusterSpec.ExternalCloudControllerManager == nil && b.IsKubernetesGTE("1.24") { + clusterSpec.ExternalCloudControllerManager = &kops.CloudControllerManagerConfig{} + } + ccmConfig := clusterSpec.ExternalCloudControllerManager + if ccmConfig == nil { return nil } @@ -51,5 +56,10 @@ func (b *GCPCloudControllerManagerOptionsBuilder) BuildOptions(options interface // TODO: Implement CCM image publishing ccmConfig.Image = "k8scloudprovidergcp/cloud-controller-manager:v1.23.0" } + + if b.IsKubernetesGTE("1.24") && b.IsKubernetesLT("1.25") { + ccmConfig.EnableLeaderMigration = fi.Bool(true) + } + return nil } diff --git a/pkg/model/components/kubecontrollermanager.go b/pkg/model/components/kubecontrollermanager.go index d6b5036a96..0bcd211786 100644 --- a/pkg/model/components/kubecontrollermanager.go +++ b/pkg/model/components/kubecontrollermanager.go @@ -98,7 +98,7 @@ func (b *KubeControllerManagerOptionsBuilder) BuildOptions(o interface{}) error } if clusterSpec.ExternalCloudControllerManager == nil { - if kcm.CloudProvider == "aws" && b.IsKubernetesGTE("1.23") { + if b.IsKubernetesGTE("1.23") && (kcm.CloudProvider == "aws" || kcm.CloudProvider == "gce") { kcm.EnableLeaderMigration = fi.Bool(true) } } else { diff --git a/tests/integration/update_cluster/minimal-1.23/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/minimal-1.23/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content index c66326199d..17337b67b3 100644 --- a/tests/integration/update_cluster/minimal-1.23/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/minimal-1.23/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content @@ -34,7 +34,7 @@ spec: version: 9.99.0 - id: k8s-1.23 manifest: leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml - manifestHash: 6211e71f8175cebcba7812f74c41d175604cbff7bab9ac788f80bac290a7b981 + manifestHash: b9c91e09c0f28c9b74ff140b8395d611834c627d698846d625c10975a74a48c4 name: leader-migration.rbac.addons.k8s.io selector: k8s-addon: leader-migration.rbac.addons.k8s.io diff --git a/tests/integration/update_cluster/minimal-1.23/data/aws_s3_bucket_object_minimal.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content b/tests/integration/update_cluster/minimal-1.23/data/aws_s3_bucket_object_minimal.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content index 0005b049b5..11ed6d46fd 100644 --- a/tests/integration/update_cluster/minimal-1.23/data/aws_s3_bucket_object_minimal.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content +++ b/tests/integration/update_cluster/minimal-1.23/data/aws_s3_bucket_object_minimal.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content @@ -47,3 +47,6 @@ subjects: - kind: ServiceAccount name: aws-cloud-controller-manager namespace: kube-system +- kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system diff --git a/tests/integration/update_cluster/minimal-1.24/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/minimal-1.24/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content index 714ceb9d2c..3b9fe886eb 100644 --- a/tests/integration/update_cluster/minimal-1.24/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/minimal-1.24/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content @@ -34,7 +34,7 @@ spec: version: 9.99.0 - id: k8s-1.23 manifest: leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml - manifestHash: 6211e71f8175cebcba7812f74c41d175604cbff7bab9ac788f80bac290a7b981 + manifestHash: b9c91e09c0f28c9b74ff140b8395d611834c627d698846d625c10975a74a48c4 name: leader-migration.rbac.addons.k8s.io selector: k8s-addon: leader-migration.rbac.addons.k8s.io diff --git a/tests/integration/update_cluster/minimal-1.24/data/aws_s3_bucket_object_minimal.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content b/tests/integration/update_cluster/minimal-1.24/data/aws_s3_bucket_object_minimal.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content index 0005b049b5..11ed6d46fd 100644 --- a/tests/integration/update_cluster/minimal-1.24/data/aws_s3_bucket_object_minimal.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content +++ b/tests/integration/update_cluster/minimal-1.24/data/aws_s3_bucket_object_minimal.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content @@ -47,3 +47,6 @@ subjects: - kind: ServiceAccount name: aws-cloud-controller-manager namespace: kube-system +- kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system diff --git a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-bootstrap_content b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-bootstrap_content index b7e3c3dfb0..6bc513a80b 100644 --- a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-bootstrap_content @@ -34,7 +34,7 @@ spec: version: 9.99.0 - id: k8s-1.23 manifest: leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml - manifestHash: 6211e71f8175cebcba7812f74c41d175604cbff7bab9ac788f80bac290a7b981 + manifestHash: b9c91e09c0f28c9b74ff140b8395d611834c627d698846d625c10975a74a48c4 name: leader-migration.rbac.addons.k8s.io selector: k8s-addon: leader-migration.rbac.addons.k8s.io diff --git a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content index 0005b049b5..11ed6d46fd 100644 --- a/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content +++ b/tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content @@ -47,3 +47,6 @@ subjects: - kind: ServiceAccount name: aws-cloud-controller-manager namespace: kube-system +- kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system diff --git a/tests/integration/update_cluster/minimal_gce/data/aws_s3_bucket_object_cluster-completed.spec_content b/tests/integration/update_cluster/minimal_gce/data/aws_s3_bucket_object_cluster-completed.spec_content index 6097a36f92..72453fd8e6 100644 --- a/tests/integration/update_cluster/minimal_gce/data/aws_s3_bucket_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/minimal_gce/data/aws_s3_bucket_object_cluster-completed.spec_content @@ -95,6 +95,7 @@ spec: clusterCIDR: 100.96.0.0/11 clusterName: minimal-gce-example-com configureCloudRoutes: false + enableLeaderMigration: true image: k8s.gcr.io/kube-controller-manager:v1.23.0 leaderElection: leaderElect: true diff --git a/tests/integration/update_cluster/minimal_gce/data/aws_s3_bucket_object_minimal-gce.example.com-addons-bootstrap_content b/tests/integration/update_cluster/minimal_gce/data/aws_s3_bucket_object_minimal-gce.example.com-addons-bootstrap_content index 23768b29f2..658692c2f7 100644 --- a/tests/integration/update_cluster/minimal_gce/data/aws_s3_bucket_object_minimal-gce.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/minimal_gce/data/aws_s3_bucket_object_minimal-gce.example.com-addons-bootstrap_content @@ -32,6 +32,13 @@ spec: selector: k8s-addon: kubelet-api.rbac.addons.k8s.io version: 9.99.0 + - id: k8s-1.23 + manifest: leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml + manifestHash: b9c91e09c0f28c9b74ff140b8395d611834c627d698846d625c10975a74a48c4 + name: leader-migration.rbac.addons.k8s.io + selector: + k8s-addon: leader-migration.rbac.addons.k8s.io + version: 9.99.0 - manifest: limit-range.addons.k8s.io/v1.5.0.yaml manifestHash: 2d55c3bc5e354e84a3730a65b42f39aba630a59dc8d32b30859fcce3d3178bc2 name: limit-range.addons.k8s.io diff --git a/tests/integration/update_cluster/minimal_gce/data/aws_s3_bucket_object_minimal-gce.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content b/tests/integration/update_cluster/minimal_gce/data/aws_s3_bucket_object_minimal-gce.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content new file mode 100644 index 0000000000..11ed6d46fd --- /dev/null +++ b/tests/integration/update_cluster/minimal_gce/data/aws_s3_bucket_object_minimal-gce.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content @@ -0,0 +1,52 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: leader-migration.rbac.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: leader-migration.rbac.addons.k8s.io + name: system::leader-locking-migration + namespace: kube-system +rules: +- apiGroups: + - coordination.k8s.io + resourceNames: + - cloud-provider-extraction-migration + resources: + - leases + verbs: + - create + - list + - get + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: leader-migration.rbac.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: leader-migration.rbac.addons.k8s.io + name: system::leader-locking-migration + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: system::leader-locking-migration +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:kube-controller-manager +- kind: ServiceAccount + name: kube-controller-manager + namespace: kube-system +- kind: ServiceAccount + name: aws-cloud-controller-manager + namespace: kube-system +- kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system diff --git a/tests/integration/update_cluster/minimal_gce/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_startup-script b/tests/integration/update_cluster/minimal_gce/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_startup-script index 7096015e73..a7bc868e32 100644 --- a/tests/integration/update_cluster/minimal_gce/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_startup-script +++ b/tests/integration/update_cluster/minimal_gce/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_startup-script @@ -189,6 +189,7 @@ kubeControllerManager: clusterCIDR: 100.96.0.0/11 clusterName: minimal-gce-example-com configureCloudRoutes: false + enableLeaderMigration: true image: k8s.gcr.io/kube-controller-manager:v1.23.0 leaderElection: leaderElect: true diff --git a/tests/integration/update_cluster/minimal_gce/kubernetes.tf b/tests/integration/update_cluster/minimal_gce/kubernetes.tf index 0700274363..7abe520e17 100644 --- a/tests/integration/update_cluster/minimal_gce/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_gce/kubernetes.tf @@ -138,6 +138,14 @@ resource "aws_s3_bucket_object" "minimal-gce-example-com-addons-kubelet-api-rbac server_side_encryption = "AES256" } +resource "aws_s3_bucket_object" "minimal-gce-example-com-addons-leader-migration-rbac-addons-k8s-io-k8s-1-23" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_minimal-gce.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content") + key = "tests/minimal-gce.example.com/addons/leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + resource "aws_s3_bucket_object" "minimal-gce-example-com-addons-limit-range-addons-k8s-io" { bucket = "testingBucket" content = file("${path.module}/data/aws_s3_bucket_object_minimal-gce.example.com-addons-limit-range.addons.k8s.io_content") diff --git a/upup/models/cloudup/resources/addons/leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml b/upup/models/cloudup/resources/addons/leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml index 80d3d94176..f56e8d7153 100644 --- a/upup/models/cloudup/resources/addons/leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml +++ b/upup/models/cloudup/resources/addons/leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml @@ -39,3 +39,6 @@ subjects: - kind: ServiceAccount name: aws-cloud-controller-manager namespace: kube-system +- kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go index a444b86574..762d37046a 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go @@ -452,10 +452,10 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*Addon } } - if kops.CloudProviderID(b.Cluster.Spec.CloudProvider) == kops.CloudProviderAWS && - b.IsKubernetesGTE("1.23") && - b.IsKubernetesLT("1.26") { - // AWS KCM-to-CCM leader migration + if b.IsKubernetesGTE("1.23") && b.IsKubernetesLT("1.26") && + (kops.CloudProviderID(b.Cluster.Spec.CloudProvider) == kops.CloudProviderAWS || + kops.CloudProviderID(b.Cluster.Spec.CloudProvider) == kops.CloudProviderGCE) { + // AWS and GCE KCM-to-CCM leader migration key := "leader-migration.rbac.addons.k8s.io" if b.IsKubernetesLT("1.25") {