Add validation to help users move from usePolicyConfigMap

k8s/crds/kops.k8s.io_clusters.yaml

@@ -3872,7 +3872,8 @@ spec:
                     type: string
                   usePolicyConfigMap:
                     description: UsePolicyConfigMap enable setting the scheduler policy
-                      from a configmap
+                      from a configmap Deprecated - use KubeSchedulerConfiguration
+                      instead
                     type: boolean
                 type: object
               kubelet:

nodeup/pkg/model/kube_scheduler.go

@@ -196,10 +196,6 @@ func (b *KubeSchedulerBuilder) buildPod(kubeScheduler *kops.KubeSchedulerConfig)
 		flags = append(flags, "--"+flag+"kubeconfig="+kubescheduler.KubeConfigPath)
 	}
 
-	if fi.ValueOf(kubeScheduler.UsePolicyConfigMap) {
-		flags = append(flags, "--policy-configmap=scheduler-policy", "--policy-configmap-namespace=kube-system")
-	}
-
 	pod := &v1.Pod{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "v1",

pkg/apis/kops/componentconfig.go

@@ -745,6 +745,7 @@ type KubeSchedulerConfig struct {
 	// LeaderElection defines the configuration of leader election client.
 	LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"`
 	// UsePolicyConfigMap enable setting the scheduler policy from a configmap
+	// Deprecated - use KubeSchedulerConfiguration instead
 	UsePolicyConfigMap *bool `json:"usePolicyConfigMap,omitempty"`
 	// FeatureGates is set of key=value pairs that describe feature gates for alpha/experimental features.
 	FeatureGates map[string]string `json:"featureGates,omitempty" flag:"feature-gates"`

pkg/apis/kops/v1alpha2/componentconfig.go

@@ -751,6 +751,7 @@ type KubeSchedulerConfig struct {
 	// LeaderElection defines the configuration of leader election client.
 	LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"`
 	// UsePolicyConfigMap enable setting the scheduler policy from a configmap
+	// Deprecated - use KubeSchedulerConfiguration instead
 	UsePolicyConfigMap *bool `json:"usePolicyConfigMap,omitempty"`
 	// FeatureGates is set of key=value pairs that describe feature gates for alpha/experimental features.
 	FeatureGates map[string]string `json:"featureGates,omitempty" flag:"feature-gates"`

pkg/apis/kops/v1alpha3/componentconfig.go

@@ -742,6 +742,7 @@ type KubeSchedulerConfig struct {
 	// LeaderElection defines the configuration of leader election client.
 	LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"`
 	// UsePolicyConfigMap enable setting the scheduler policy from a configmap
+	// Deprecated - use KubeSchedulerConfiguration instead
 	UsePolicyConfigMap *bool `json:"usePolicyConfigMap,omitempty"`
 	// FeatureGates is set of key=value pairs that describe feature gates for alpha/experimental features.
 	FeatureGates map[string]string `json:"featureGates,omitempty" flag:"feature-gates"`

pkg/apis/kops/validation/validation.go

@@ -134,6 +134,10 @@ func validateClusterSpec(spec *kops.ClusterSpec, c *kops.Cluster, fieldPath *fie
 		allErrs = append(allErrs, validateKubeControllerManager(spec.KubeControllerManager, c, fieldPath.Child("kubeControllerManager"), strict)...)
 	}
 
+	if spec.KubeScheduler != nil {
+		allErrs = append(allErrs, validateKubeScheduler(spec.KubeScheduler, c, fieldPath.Child("kubeScheduler"), strict)...)
+	}
+
 	if spec.KubeProxy != nil {
 		allErrs = append(allErrs, validateKubeProxy(spec.KubeProxy, fieldPath.Child("kubeProxy"))...)
 	}
@@ -842,6 +846,18 @@ func validateKubeControllerManager(v *kops.KubeControllerManagerConfig, c *kops.
 	return allErrs
 }
 
+func validateKubeScheduler(v *kops.KubeSchedulerConfig, c *kops.Cluster, fldPath *field.Path, strict bool) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	// We aren't aiming to do comprehensive validation, but we can add some best-effort validation where it helps guide users.
+	// Users reported encountered this in #16388
+	if v.UsePolicyConfigMap != nil {
+		allErrs = append(allErrs, field.Forbidden(fldPath.Child("usePolicyConfigMap"), "usePolicyConfigMap is deprecated, use KubeSchedulerConfiguration"))
+	}
+
+	return allErrs
+}
+
 func validateKubeProxy(k *kops.KubeProxyConfig, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
 

pkg/model/components/kubescheduler_test.go

@@ -19,29 +19,12 @@ package components
 import (
 	"testing"
 
-	api "k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/apis/kops/util"
 	"k8s.io/kops/pkg/assets"
 	"k8s.io/kops/util/pkg/vfs"
 )
 
-func buildSchedulerConfigMapCluster(version string) *api.Cluster {
+func Test_Build_Scheduler(t *testing.T) {
-	usePolicyConfigMap := true
-
-	return &api.Cluster{
-		Spec: api.ClusterSpec{
-			CloudProvider: api.CloudProviderSpec{
-				AWS: &api.AWSSpec{},
-			},
-			KubernetesVersion: version,
-			KubeScheduler: &api.KubeSchedulerConfig{
-				UsePolicyConfigMap: &usePolicyConfigMap,
-			},
-		},
-	}
-}
-
-func Test_Build_Scheduler_Without_PolicyConfigMap(t *testing.T) {
 	versions := []string{"v1.6.0", "v1.6.4", "v1.7.0", "v1.7.4"}
 
 	for _, v := range versions {
@@ -69,30 +52,3 @@ func Test_Build_Scheduler_Without_PolicyConfigMap(t *testing.T) {
 		}
 	}
 }
-
-func Test_Build_Scheduler_PolicyConfigMap_Supported_Version(t *testing.T) {
-	versions := []string{"v1.9.0", "v1.10.5", "v1.18.0"}
-
-	for _, v := range versions {
-
-		c := buildSchedulerConfigMapCluster(v)
-		b := assets.NewAssetBuilder(vfs.Context, c.Spec.Assets, c.Spec.KubernetesVersion, false)
-
-		version, err := util.ParseKubernetesVersion(v)
-		if err != nil {
-			t.Fatalf("unexpected error from ParseKubernetesVersion %s: %v", v, err)
-		}
-
-		ks := &KubeSchedulerOptionsBuilder{
-			&OptionsContext{
-				AssetBuilder:      b,
-				KubernetesVersion: *version,
-			},
-		}
-
-		err = ks.BuildOptions(&c.Spec)
-		if err != nil {
-			t.Fatalf("unexpected error from BuildOptions %s: %v", v, err)
-		}
-	}
-}

upup/models/cloudup/resources/addons/scheduler.addons.k8s.io/v1.7.0.yaml

@@ -1,38 +0,0 @@
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: scheduler-policy
-  namespace: kube-system
-  labels:
-    k8s-addon: scheduler.addons.k8s.io
-data:
-  policy.cfg: |
-    {
-      "kind" : "Policy",
-      "apiVersion" : "v1",
-      "predicates" : [
-        {"name": "NoDiskConflict"},
-        {"name": "NoVolumeZoneConflict"},
-        {"name": "MaxEBSVolumeCount"},
-        {"name": "MaxGCEPDVolumeCount"},
-        {"name": "MaxAzureDiskVolumeCount"},
-        {"name": "MatchInterPodAffinity"},
-        {"name": "NoDiskConflict"},
-        {"name": "GeneralPredicates"},
-        {"name": "CheckNodeMemoryPressure"},
-        {"name": "CheckNodeDiskPressure"},
-        {"name": "CheckNodeCondition"},
-        {"name": "PodToleratesNodeTaints"},
-        {"name": "NoVolumeNodeConflict"}
-      ],
-      "priorities" : [
-        {"name": "SelectorSpreadPriority", "weight" : 1},
-        {"name": "LeastRequestedPriority", "weight" : 1},
-        {"name": "BalancedResourceAllocation", "weight" : 1},
-        {"name": "NodePreferAvoidPodsPriority", "weight" : 1},
-        {"name": "NodeAffinityPriority", "weight" : 1},
-        {"name": "TaintTolerationPriority", "weight" : 1},
-        {"name": "InterPodAffinityPriority", "weight" : 1}
-      ],
-      "hardPodAffinitySymmetricWeight" : 1
-    }

upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go

@@ -1212,18 +1212,6 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext)
 		}
 	}
 
-	if b.Cluster.Spec.KubeScheduler.UsePolicyConfigMap != nil {
-		key := "scheduler.addons.k8s.io"
-		version := "1.7.0"
-		location := key + "/v" + version + ".yaml"
-
-		addons.Add(&channelsapi.AddonSpec{
-			Name:     fi.PtrTo(key),
-			Selector: map[string]string{"k8s-addon": key},
-			Manifest: fi.PtrTo(location),
-		})
-	}
-
 	serviceAccounts := make(map[string]iam.Subject)
 
 	if b.Cluster.Spec.GetCloudProvider() == kops.CloudProviderAWS && b.Cluster.Spec.KubeAPIServer.ServiceAccountIssuer != nil {