diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index baea5fdad6..e9fa7b9d11 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -3872,7 +3872,8 @@ spec: type: string usePolicyConfigMap: description: UsePolicyConfigMap enable setting the scheduler policy - from a configmap + from a configmap Deprecated - use KubeSchedulerConfiguration + instead type: boolean type: object kubelet: diff --git a/nodeup/pkg/model/kube_scheduler.go b/nodeup/pkg/model/kube_scheduler.go index 355bd31ff1..36e2203ea8 100644 --- a/nodeup/pkg/model/kube_scheduler.go +++ b/nodeup/pkg/model/kube_scheduler.go @@ -196,10 +196,6 @@ func (b *KubeSchedulerBuilder) buildPod(kubeScheduler *kops.KubeSchedulerConfig) flags = append(flags, "--"+flag+"kubeconfig="+kubescheduler.KubeConfigPath) } - if fi.ValueOf(kubeScheduler.UsePolicyConfigMap) { - flags = append(flags, "--policy-configmap=scheduler-policy", "--policy-configmap-namespace=kube-system") - } - pod := &v1.Pod{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index 2f334ba72c..f8dc1c8b1d 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -745,6 +745,7 @@ type KubeSchedulerConfig struct { // LeaderElection defines the configuration of leader election client. LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"` // UsePolicyConfigMap enable setting the scheduler policy from a configmap + // Deprecated - use KubeSchedulerConfiguration instead UsePolicyConfigMap *bool `json:"usePolicyConfigMap,omitempty"` // FeatureGates is set of key=value pairs that describe feature gates for alpha/experimental features. FeatureGates map[string]string `json:"featureGates,omitempty" flag:"feature-gates"` diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go index 204231119c..85baf1622e 100644 --- a/pkg/apis/kops/v1alpha2/componentconfig.go +++ b/pkg/apis/kops/v1alpha2/componentconfig.go @@ -751,6 +751,7 @@ type KubeSchedulerConfig struct { // LeaderElection defines the configuration of leader election client. LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"` // UsePolicyConfigMap enable setting the scheduler policy from a configmap + // Deprecated - use KubeSchedulerConfiguration instead UsePolicyConfigMap *bool `json:"usePolicyConfigMap,omitempty"` // FeatureGates is set of key=value pairs that describe feature gates for alpha/experimental features. FeatureGates map[string]string `json:"featureGates,omitempty" flag:"feature-gates"` diff --git a/pkg/apis/kops/v1alpha3/componentconfig.go b/pkg/apis/kops/v1alpha3/componentconfig.go index ca0730f53b..ca8767318f 100644 --- a/pkg/apis/kops/v1alpha3/componentconfig.go +++ b/pkg/apis/kops/v1alpha3/componentconfig.go @@ -742,6 +742,7 @@ type KubeSchedulerConfig struct { // LeaderElection defines the configuration of leader election client. LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"` // UsePolicyConfigMap enable setting the scheduler policy from a configmap + // Deprecated - use KubeSchedulerConfiguration instead UsePolicyConfigMap *bool `json:"usePolicyConfigMap,omitempty"` // FeatureGates is set of key=value pairs that describe feature gates for alpha/experimental features. FeatureGates map[string]string `json:"featureGates,omitempty" flag:"feature-gates"` diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index 5e9dd02fab..99c7188985 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -134,6 +134,10 @@ func validateClusterSpec(spec *kops.ClusterSpec, c *kops.Cluster, fieldPath *fie allErrs = append(allErrs, validateKubeControllerManager(spec.KubeControllerManager, c, fieldPath.Child("kubeControllerManager"), strict)...) } + if spec.KubeScheduler != nil { + allErrs = append(allErrs, validateKubeScheduler(spec.KubeScheduler, c, fieldPath.Child("kubeScheduler"), strict)...) + } + if spec.KubeProxy != nil { allErrs = append(allErrs, validateKubeProxy(spec.KubeProxy, fieldPath.Child("kubeProxy"))...) } @@ -842,6 +846,18 @@ func validateKubeControllerManager(v *kops.KubeControllerManagerConfig, c *kops. return allErrs } +func validateKubeScheduler(v *kops.KubeSchedulerConfig, c *kops.Cluster, fldPath *field.Path, strict bool) field.ErrorList { + allErrs := field.ErrorList{} + + // We aren't aiming to do comprehensive validation, but we can add some best-effort validation where it helps guide users. + // Users reported encountered this in #16388 + if v.UsePolicyConfigMap != nil { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("usePolicyConfigMap"), "usePolicyConfigMap is deprecated, use KubeSchedulerConfiguration")) + } + + return allErrs +} + func validateKubeProxy(k *kops.KubeProxyConfig, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} diff --git a/pkg/model/components/kubescheduler_test.go b/pkg/model/components/kubescheduler_test.go index fd93470d7f..a73c7a6163 100644 --- a/pkg/model/components/kubescheduler_test.go +++ b/pkg/model/components/kubescheduler_test.go @@ -19,29 +19,12 @@ package components import ( "testing" - api "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/kops/util" "k8s.io/kops/pkg/assets" "k8s.io/kops/util/pkg/vfs" ) -func buildSchedulerConfigMapCluster(version string) *api.Cluster { - usePolicyConfigMap := true - - return &api.Cluster{ - Spec: api.ClusterSpec{ - CloudProvider: api.CloudProviderSpec{ - AWS: &api.AWSSpec{}, - }, - KubernetesVersion: version, - KubeScheduler: &api.KubeSchedulerConfig{ - UsePolicyConfigMap: &usePolicyConfigMap, - }, - }, - } -} - -func Test_Build_Scheduler_Without_PolicyConfigMap(t *testing.T) { +func Test_Build_Scheduler(t *testing.T) { versions := []string{"v1.6.0", "v1.6.4", "v1.7.0", "v1.7.4"} for _, v := range versions { @@ -69,30 +52,3 @@ func Test_Build_Scheduler_Without_PolicyConfigMap(t *testing.T) { } } } - -func Test_Build_Scheduler_PolicyConfigMap_Supported_Version(t *testing.T) { - versions := []string{"v1.9.0", "v1.10.5", "v1.18.0"} - - for _, v := range versions { - - c := buildSchedulerConfigMapCluster(v) - b := assets.NewAssetBuilder(vfs.Context, c.Spec.Assets, c.Spec.KubernetesVersion, false) - - version, err := util.ParseKubernetesVersion(v) - if err != nil { - t.Fatalf("unexpected error from ParseKubernetesVersion %s: %v", v, err) - } - - ks := &KubeSchedulerOptionsBuilder{ - &OptionsContext{ - AssetBuilder: b, - KubernetesVersion: *version, - }, - } - - err = ks.BuildOptions(&c.Spec) - if err != nil { - t.Fatalf("unexpected error from BuildOptions %s: %v", v, err) - } - } -} diff --git a/upup/models/cloudup/resources/addons/scheduler.addons.k8s.io/v1.7.0.yaml b/upup/models/cloudup/resources/addons/scheduler.addons.k8s.io/v1.7.0.yaml deleted file mode 100644 index e5af28ea76..0000000000 --- a/upup/models/cloudup/resources/addons/scheduler.addons.k8s.io/v1.7.0.yaml +++ /dev/null @@ -1,38 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: scheduler-policy - namespace: kube-system - labels: - k8s-addon: scheduler.addons.k8s.io -data: - policy.cfg: | - { - "kind" : "Policy", - "apiVersion" : "v1", - "predicates" : [ - {"name": "NoDiskConflict"}, - {"name": "NoVolumeZoneConflict"}, - {"name": "MaxEBSVolumeCount"}, - {"name": "MaxGCEPDVolumeCount"}, - {"name": "MaxAzureDiskVolumeCount"}, - {"name": "MatchInterPodAffinity"}, - {"name": "NoDiskConflict"}, - {"name": "GeneralPredicates"}, - {"name": "CheckNodeMemoryPressure"}, - {"name": "CheckNodeDiskPressure"}, - {"name": "CheckNodeCondition"}, - {"name": "PodToleratesNodeTaints"}, - {"name": "NoVolumeNodeConflict"} - ], - "priorities" : [ - {"name": "SelectorSpreadPriority", "weight" : 1}, - {"name": "LeastRequestedPriority", "weight" : 1}, - {"name": "BalancedResourceAllocation", "weight" : 1}, - {"name": "NodePreferAvoidPodsPriority", "weight" : 1}, - {"name": "NodeAffinityPriority", "weight" : 1}, - {"name": "TaintTolerationPriority", "weight" : 1}, - {"name": "InterPodAffinityPriority", "weight" : 1} - ], - "hardPodAffinitySymmetricWeight" : 1 - } \ No newline at end of file diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go index 27b99a4f52..7c1e8c8896 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go @@ -1212,18 +1212,6 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext) } } - if b.Cluster.Spec.KubeScheduler.UsePolicyConfigMap != nil { - key := "scheduler.addons.k8s.io" - version := "1.7.0" - location := key + "/v" + version + ".yaml" - - addons.Add(&channelsapi.AddonSpec{ - Name: fi.PtrTo(key), - Selector: map[string]string{"k8s-addon": key}, - Manifest: fi.PtrTo(location), - }) - } - serviceAccounts := make(map[string]iam.Subject) if b.Cluster.Spec.GetCloudProvider() == kops.CloudProviderAWS && b.Cluster.Spec.KubeAPIServer.ServiceAccountIssuer != nil {