Migrate to AWS CCM in k8s 1.24

cmd/kops/integration_test.go

@@ -184,6 +184,7 @@ func TestMinimal_v1_23(t *testing.T) {
 		withAddons(
 			awsEBSCSIAddon,
 			dnsControllerAddon,
+			"leader-migration.rbac.addons.k8s.io-k8s-1.23",
 		).
 		runTestTerraformAWS(t)
 	newIntegrationTest("minimal.example.com", "minimal").runTestCloudformation(t)

pkg/apis/kops/componentconfig.go

@@ -650,6 +650,8 @@ type KubeControllerManagerConfig struct {
 
 	// EnableProfiling enables profiling via web interface host:port/debug/pprof/
 	EnableProfiling *bool `json:"enableProfiling,omitempty" flag:"profiling"`
+	// EnableLeaderMigration enables controller leader migration.
+	EnableLeaderMigration *bool `json:"enableLeaderMigration,omitempty" flag:"enable-leader-migration"`
 }
 
 // CloudControllerManagerConfig is the configuration of the cloud controller
@@ -677,6 +679,8 @@ type CloudControllerManagerConfig struct {
 	LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"`
 	// UseServiceAccountCredentials controls whether we use individual service account credentials for each controller.
 	UseServiceAccountCredentials *bool `json:"useServiceAccountCredentials,omitempty" flag:"use-service-account-credentials"`
+	// EnableLeaderMigration enables controller leader migration.
+	EnableLeaderMigration *bool `json:"enableLeaderMigration,omitempty" flag:"enable-leader-migration"`
 }
 
 // KubeSchedulerConfig is the configuration for the kube-scheduler

pkg/apis/kops/v1alpha2/componentconfig.go

@@ -649,6 +649,8 @@ type KubeControllerManagerConfig struct {
 
 	// EnableProfiling enables profiling via web interface host:port/debug/pprof/
 	EnableProfiling *bool `json:"enableProfiling,omitempty" flag:"profiling"`
+	// EnableLeaderMigration enables controller leader migration.
+	EnableLeaderMigration *bool `json:"enableLeaderMigration,omitempty" flag:"enable-leader-migration"`
 }
 
 // CloudControllerManagerConfig is the configuration of the cloud controller
@@ -676,6 +678,8 @@ type CloudControllerManagerConfig struct {
 	LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"`
 	// UseServiceAccountCredentials controls whether we use individual service account credentials for each controller.
 	UseServiceAccountCredentials *bool `json:"useServiceAccountCredentials,omitempty" flag:"use-service-account-credentials"`
+	// EnableLeaderMigration enables controller leader migration.
+	EnableLeaderMigration *bool `json:"enableLeaderMigration,omitempty" flag:"enable-leader-migration"`
 }
 
 // KubeSchedulerConfig is the configuration for the kube-scheduler

pkg/apis/kops/v1alpha3/componentconfig.go

@@ -647,6 +647,8 @@ type KubeControllerManagerConfig struct {
 
 	// EnableProfiling enables profiling via web interface host:port/debug/pprof/
 	EnableProfiling *bool `json:"enableProfiling,omitempty" flag:"profiling"`
+	// EnableLeaderMigration enables controller leader migration.
+	EnableLeaderMigration *bool `json:"enableLeaderMigration,omitempty" flag:"enable-leader-migration"`
 }
 
 // CloudControllerManagerConfig is the configuration of the cloud controller
@@ -674,6 +676,8 @@ type CloudControllerManagerConfig struct {
 	LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"`
 	// UseServiceAccountCredentials controls whether we use individual service account credentials for each controller.
 	UseServiceAccountCredentials *bool `json:"useServiceAccountCredentials,omitempty" flag:"use-service-account-credentials"`
+	// EnableLeaderMigration enables controller leader migration.
+	EnableLeaderMigration *bool `json:"enableLeaderMigration,omitempty" flag:"enable-leader-migration"`
 }
 
 // KubeSchedulerConfig is the configuration for the kube-scheduler

pkg/model/components/awscloudcontrollermanager.go

@@ -24,7 +24,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi/loader"
 )
 
-// KubeControllerManagerOptionsBuilder adds options for the kubernetes controller manager to the model.
+// AWSCloudControllerManagerOptionsBuilder adds options for the kubernetes controller manager to the model.
 type AWSCloudControllerManagerOptionsBuilder struct {
 	*OptionsContext
 }
@@ -38,7 +38,15 @@ func (b *AWSCloudControllerManagerOptionsBuilder) BuildOptions(o interface{}) er
 
 	eccm := clusterSpec.ExternalCloudControllerManager
 
-	if eccm == nil || kops.CloudProviderID(clusterSpec.CloudProvider) != kops.CloudProviderAWS {
+	if kops.CloudProviderID(clusterSpec.CloudProvider) != kops.CloudProviderAWS {
+		return nil
+	}
+
+	if eccm == nil && b.IsKubernetesGTE("1.24") {
+		eccm = &kops.CloudControllerManagerConfig{}
+	}
+
+	if eccm == nil {
 		return nil
 	}
 
@@ -91,5 +99,9 @@ func (b *AWSCloudControllerManagerOptionsBuilder) BuildOptions(o interface{}) er
 		}
 	}
 
+	if b.IsKubernetesGTE("1.24") && b.IsKubernetesLT("1.25") {
+		eccm.EnableLeaderMigration = fi.Bool(true)
+	}
+
 	return nil
 }

pkg/model/components/kubecontrollermanager.go

@@ -79,7 +79,11 @@ func (b *KubeControllerManagerOptionsBuilder) BuildOptions(o interface{}) error
 	kcm.ClusterName = b.ClusterName
 	switch kops.CloudProviderID(clusterSpec.CloudProvider) {
 	case kops.CloudProviderAWS:
-		kcm.CloudProvider = "aws"
+		if b.IsKubernetesGTE("1.24") {
+			kcm.CloudProvider = "external"
+		} else {
+			kcm.CloudProvider = "aws"
+		}
 
 	case kops.CloudProviderGCE:
 		kcm.CloudProvider = "gce"
@@ -101,7 +105,11 @@ func (b *KubeControllerManagerOptionsBuilder) BuildOptions(o interface{}) error
 		return fmt.Errorf("unknown cloudprovider %q", clusterSpec.CloudProvider)
 	}
 
-	if clusterSpec.ExternalCloudControllerManager != nil {
+	if clusterSpec.ExternalCloudControllerManager == nil {
+		if kcm.CloudProvider == "aws" && b.IsKubernetesGTE("1.23") && b.IsKubernetesLT("1.24") {
+			kcm.EnableLeaderMigration = fi.Bool(true)
+		}
+	} else {
 		kcm.CloudProvider = "external"
 	}
 

upup/models/cloudup/resources/addons/leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml

@@ -0,0 +1,41 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    k8s-addon: leader-migration.rbac.addons.k8s.io
+  name: system::leader-locking-migration
+  namespace: kube-system
+rules:
+  apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  resourceNames:
+  - cloud-provider-extraction-migration
+  verbs:
+  - create
+  - list
+  - get
+  - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    k8s-addon: leader-migration.rbac.addons.k8s.io
+  name: system::leader-locking-migration
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: system::leader-locking-migration
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:kube-controller-manager
+- kind: ServiceAccount
+  name: kube-controller-manager
+  namespace: kube-system
+- kind: ServiceAccount
+  name: aws-cloud-controller-manager
+  namespace: kube-system

upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go

@@ -451,6 +451,37 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*Addon
 		}
 	}
 
+	if kops.CloudProviderID(b.Cluster.Spec.CloudProvider) == kops.CloudProviderAWS &&
+		b.IsKubernetesGTE("1.23") &&
+		b.IsKubernetesLT("1.26") {
+		// AWS KCM-to-CCM leader migration
+		key := "leader-migration.rbac.addons.k8s.io"
+
+		if b.IsKubernetesLT("1.25") {
+			location := key + "/k8s-1.23.yaml"
+			id := "k8s-1.23"
+
+			addons.Add(&channelsapi.AddonSpec{
+				Name:     fi.String(key),
+				Selector: map[string]string{"k8s-addon": key},
+				Manifest: fi.String(location),
+				Id:       id,
+			})
+		}
+
+		if b.IsKubernetesGTE("1.25") {
+			location := key + "/k8s-1.25.yaml"
+			id := "k8s-1.25"
+
+			addons.Add(&channelsapi.AddonSpec{
+				Name:     fi.String(key),
+				Selector: map[string]string{"k8s-addon": key},
+				Manifest: fi.String(location),
+				Id:       id,
+			})
+		}
+	}
+
 	{
 		key := "limit-range.addons.k8s.io"
 		version := "1.5.0"