kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Migrate to GCE CCM in k8s 1.24

This commit is contained in:
John Gardiner Myers 2021-12-21 15:02:24 -08:00
parent e578d6b42b
commit dae281d30e
16 changed files with 100 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/kops/integration_test.go
View File

@ -237,7 +237,7 @@ func TestMinimalGossip(t *testing.T) {
// TestMinimalGCE runs tests on a minimal GCE configuration
func TestMinimalGCE(t *testing.T) {
newIntegrationTest("minimal-gce.example.com", "minimal_gce").
withAddons(dnsControllerAddon, "gcp-pd-csi-driver.addons.k8s.io-k8s-1.23").
withAddons(dnsControllerAddon, leaderElectionAddon, "gcp-pd-csi-driver.addons.k8s.io-k8s-1.23").
runTestTerraformGCE(t)
}

10
pkg/model/components/gcpcloudcontrollermanager.go
View File

@ -36,7 +36,12 @@ func (b *GCPCloudControllerManagerOptionsBuilder) BuildOptions(options interface
return nil
}
if clusterSpec.ExternalCloudControllerManager == nil && b.IsKubernetesGTE("1.24") {
clusterSpec.ExternalCloudControllerManager = &kops.CloudControllerManagerConfig{}
}
ccmConfig := clusterSpec.ExternalCloudControllerManager
if ccmConfig == nil {
return nil
}
@ -51,5 +56,10 @@ func (b *GCPCloudControllerManagerOptionsBuilder) BuildOptions(options interface
// TODO: Implement CCM image publishing
ccmConfig.Image = "k8scloudprovidergcp/cloud-controller-manager:v1.23.0"
}
if b.IsKubernetesGTE("1.24") && b.IsKubernetesLT("1.25") {
ccmConfig.EnableLeaderMigration = fi.Bool(true)
}
return nil
}

2
pkg/model/components/kubecontrollermanager.go
View File

@ -98,7 +98,7 @@ func (b *KubeControllerManagerOptionsBuilder) BuildOptions(o interface{}) error
}
if clusterSpec.ExternalCloudControllerManager == nil {
if kcm.CloudProvider == "aws" && b.IsKubernetesGTE("1.23") {
if b.IsKubernetesGTE("1.23") && (kcm.CloudProvider == "aws" || kcm.CloudProvider == "gce") {
kcm.EnableLeaderMigration = fi.Bool(true)
}
} else {

2
tests/integration/update_cluster/minimal-1.23/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content
View File

@ -34,7 +34,7 @@ spec:
version: 9.99.0
- id: k8s-1.23
manifest: leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml
manifestHash: 6211e71f8175cebcba7812f74c41d175604cbff7bab9ac788f80bac290a7b981
manifestHash: b9c91e09c0f28c9b74ff140b8395d611834c627d698846d625c10975a74a48c4
name: leader-migration.rbac.addons.k8s.io
selector:
k8s-addon: leader-migration.rbac.addons.k8s.io

3
tests/integration/update_cluster/minimal-1.23/data/aws_s3_bucket_object_minimal.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content
View File

@ -47,3 +47,6 @@ subjects:
- kind: ServiceAccount
name: aws-cloud-controller-manager
namespace: kube-system
- kind: ServiceAccount
name: cloud-controller-manager
namespace: kube-system

2
tests/integration/update_cluster/minimal-1.24/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content
View File

@ -34,7 +34,7 @@ spec:
version: 9.99.0
- id: k8s-1.23
manifest: leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml
manifestHash: 6211e71f8175cebcba7812f74c41d175604cbff7bab9ac788f80bac290a7b981
manifestHash: b9c91e09c0f28c9b74ff140b8395d611834c627d698846d625c10975a74a48c4
name: leader-migration.rbac.addons.k8s.io
selector:
k8s-addon: leader-migration.rbac.addons.k8s.io

3
tests/integration/update_cluster/minimal-1.24/data/aws_s3_bucket_object_minimal.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content
View File

@ -47,3 +47,6 @@ subjects:
- kind: ServiceAccount
name: aws-cloud-controller-manager
namespace: kube-system
- kind: ServiceAccount
name: cloud-controller-manager
namespace: kube-system

2
tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-bootstrap_content
View File

@ -34,7 +34,7 @@ spec:
version: 9.99.0
- id: k8s-1.23
manifest: leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml
manifestHash: 6211e71f8175cebcba7812f74c41d175604cbff7bab9ac788f80bac290a7b981
manifestHash: b9c91e09c0f28c9b74ff140b8395d611834c627d698846d625c10975a74a48c4
name: leader-migration.rbac.addons.k8s.io
selector:
k8s-addon: leader-migration.rbac.addons.k8s.io

3
tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content
View File

@ -47,3 +47,6 @@ subjects:
- kind: ServiceAccount
name: aws-cloud-controller-manager
namespace: kube-system
- kind: ServiceAccount
name: cloud-controller-manager
namespace: kube-system

1
tests/integration/update_cluster/minimal_gce/data/aws_s3_bucket_object_cluster-completed.spec_content
View File

@ -95,6 +95,7 @@ spec:
clusterCIDR: 100.96.0.0/11
clusterName: minimal-gce-example-com
configureCloudRoutes: false
enableLeaderMigration: true
image: k8s.gcr.io/kube-controller-manager:v1.23.0
leaderElection:
leaderElect: true

7
tests/integration/update_cluster/minimal_gce/data/aws_s3_bucket_object_minimal-gce.example.com-addons-bootstrap_content
View File

@ -32,6 +32,13 @@ spec:
selector:
k8s-addon: kubelet-api.rbac.addons.k8s.io
version: 9.99.0
- id: k8s-1.23
manifest: leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml
manifestHash: b9c91e09c0f28c9b74ff140b8395d611834c627d698846d625c10975a74a48c4
name: leader-migration.rbac.addons.k8s.io
selector:
k8s-addon: leader-migration.rbac.addons.k8s.io
version: 9.99.0
- manifest: limit-range.addons.k8s.io/v1.5.0.yaml
manifestHash: 2d55c3bc5e354e84a3730a65b42f39aba630a59dc8d32b30859fcce3d3178bc2
name: limit-range.addons.k8s.io

52
tests/integration/update_cluster/minimal_gce/data/aws_s3_bucket_object_minimal-gce.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content Normal file
View File

@ -0,0 +1,52 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: leader-migration.rbac.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: leader-migration.rbac.addons.k8s.io
name: system::leader-locking-migration
namespace: kube-system
rules:
- apiGroups:
- coordination.k8s.io
resourceNames:
- cloud-provider-extraction-migration
resources:
- leases
verbs:
- create
- list
- get
- update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: leader-migration.rbac.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: leader-migration.rbac.addons.k8s.io
name: system::leader-locking-migration
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: system::leader-locking-migration
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: system:kube-controller-manager
- kind: ServiceAccount
name: kube-controller-manager
namespace: kube-system
- kind: ServiceAccount
name: aws-cloud-controller-manager
namespace: kube-system
- kind: ServiceAccount
name: cloud-controller-manager
namespace: kube-system

1
tests/integration/update_cluster/minimal_gce/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_startup-script
View File

@ -189,6 +189,7 @@ kubeControllerManager:
clusterCIDR: 100.96.0.0/11
clusterName: minimal-gce-example-com
configureCloudRoutes: false
enableLeaderMigration: true
image: k8s.gcr.io/kube-controller-manager:v1.23.0
leaderElection:
leaderElect: true

8
tests/integration/update_cluster/minimal_gce/kubernetes.tf
View File

@ -138,6 +138,14 @@ resource "aws_s3_bucket_object" "minimal-gce-example-com-addons-kubelet-api-rbac
server_side_encryption = "AES256"
}
resource "aws_s3_bucket_object" "minimal-gce-example-com-addons-leader-migration-rbac-addons-k8s-io-k8s-1-23" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_bucket_object_minimal-gce.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content")
key = "tests/minimal-gce.example.com/addons/leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml"
provider = aws.files
server_side_encryption = "AES256"
}
resource "aws_s3_bucket_object" "minimal-gce-example-com-addons-limit-range-addons-k8s-io" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_bucket_object_minimal-gce.example.com-addons-limit-range.addons.k8s.io_content")

3
upup/models/cloudup/resources/addons/leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml
View File

@ -39,3 +39,6 @@ subjects:
- kind: ServiceAccount
name: aws-cloud-controller-manager
namespace: kube-system
- kind: ServiceAccount
name: cloud-controller-manager
namespace: kube-system

8
upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go
View File

@ -452,10 +452,10 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*Addon
}
}
if kops.CloudProviderID(b.Cluster.Spec.CloudProvider) == kops.CloudProviderAWS &&
b.IsKubernetesGTE("1.23") &&
b.IsKubernetesLT("1.26") {
// AWS KCM-to-CCM leader migration
if b.IsKubernetesGTE("1.23") && b.IsKubernetesLT("1.26") &&
(kops.CloudProviderID(b.Cluster.Spec.CloudProvider) == kops.CloudProviderAWS ||
kops.CloudProviderID(b.Cluster.Spec.CloudProvider) == kops.CloudProviderGCE) {
// AWS and GCE KCM-to-CCM leader migration
key := "leader-migration.rbac.addons.k8s.io"
if b.IsKubernetesLT("1.25") {