From a708919bf4facfca1564b7016b821fa5df8eb194 Mon Sep 17 00:00:00 2001 From: Manatsawin Hanmongkolchai Date: Tue, 10 Oct 2017 10:51:30 +0700 Subject: [PATCH 1/2] Generate scheduler policy by dynamic cluster addons --- nodeup/pkg/model/kube_scheduler.go | 4 ++ pkg/apis/kops/componentconfig.go | 6 +-- pkg/apis/kops/v1alpha1/componentconfig.go | 6 +-- .../kops/v1alpha1/zz_generated.conversion.go | 6 +-- .../kops/v1alpha1/zz_generated.deepcopy.go | 9 +++++ pkg/apis/kops/v1alpha2/componentconfig.go | 6 +-- .../kops/v1alpha2/zz_generated.conversion.go | 6 +-- .../kops/v1alpha2/zz_generated.deepcopy.go | 9 +++++ pkg/apis/kops/zz_generated.deepcopy.go | 9 +++++ pkg/model/components/kubescheduler.go | 4 +- pkg/model/components/kubescheduler_test.go | 4 +- .../scheduler.addons.k8s.io/v1.7.0.yaml | 38 +++++++++++++++++++ .../pkg/fi/cloudup/bootstrapchannelbuilder.go | 15 ++++++++ 13 files changed, 99 insertions(+), 23 deletions(-) create mode 100644 upup/models/cloudup/resources/addons/scheduler.addons.k8s.io/v1.7.0.yaml diff --git a/nodeup/pkg/model/kube_scheduler.go b/nodeup/pkg/model/kube_scheduler.go index e4ed249fca..359cb23dd0 100644 --- a/nodeup/pkg/model/kube_scheduler.go +++ b/nodeup/pkg/model/kube_scheduler.go @@ -101,6 +101,10 @@ func (b *KubeSchedulerBuilder) buildPod() (*v1.Pod, error) { // Add kubeconfig flag flags = append(flags, "--kubeconfig="+"/var/lib/kube-scheduler/kubeconfig") + if c.UsePolicyConfigMap != nil { + flags = append(flags, "--policy-configmap=scheduler-policy") + } + pod := &v1.Pod{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index fa4fb2df56..7574bc42f6 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -345,10 +345,8 @@ type KubeSchedulerConfig struct { Image string `json:"image,omitempty"` // LeaderElection defines the configuration of leader election client. LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"` - // PolicyConfigMap is the name of configmap to use for scheduler policy - PolicyConfigMap string `json:"policyConfigMap,omitempty" flag:"policy-configmap"` - // PolicyConfigMapNamespace is the namespace containing the configmap - PolicyConfigMapNamespace string `json:"policyConfigMapNamespace,omitempty" flag:"policy-configmap-namespace"` + // UsePolicyConfigMap enable setting the scheduler policy from a configmap + UsePolicyConfigMap *bool `json:"usePolicyConfigMap,omitempty"` } // LeaderElectionConfiguration defines the configuration of leader election diff --git a/pkg/apis/kops/v1alpha1/componentconfig.go b/pkg/apis/kops/v1alpha1/componentconfig.go index e082edbb2e..6d3f9638f3 100644 --- a/pkg/apis/kops/v1alpha1/componentconfig.go +++ b/pkg/apis/kops/v1alpha1/componentconfig.go @@ -345,10 +345,8 @@ type KubeSchedulerConfig struct { Image string `json:"image,omitempty"` // LeaderElection defines the configuration of leader election client. LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"` - // PolicyConfigMap is the name of configmap to use for scheduler policy - PolicyConfigMap string `json:"policyConfigMap,omitempty" flag:"policy-configmap"` - // PolicyConfigMapNamespace is the namespace containing the configmap - PolicyConfigMapNamespace string `json:"policyConfigMapNamespace,omitempty" flag:"policy-configmap-namespace"` + // UsePolicyConfigMap enable setting the scheduler policy from a configmap + UsePolicyConfigMap *bool `json:"usePolicyConfigMap,omitempty"` } // LeaderElectionConfiguration defines the configuration of leader election diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index b60912f6d6..6c44a43dc2 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -2021,8 +2021,7 @@ func autoConvert_v1alpha1_KubeSchedulerConfig_To_kops_KubeSchedulerConfig(in *Ku } else { out.LeaderElection = nil } - out.PolicyConfigMap = in.PolicyConfigMap - out.PolicyConfigMapNamespace = in.PolicyConfigMapNamespace + out.UsePolicyConfigMap = in.UsePolicyConfigMap return nil } @@ -2044,8 +2043,7 @@ func autoConvert_kops_KubeSchedulerConfig_To_v1alpha1_KubeSchedulerConfig(in *ko } else { out.LeaderElection = nil } - out.PolicyConfigMap = in.PolicyConfigMap - out.PolicyConfigMapNamespace = in.PolicyConfigMapNamespace + out.UsePolicyConfigMap = in.UsePolicyConfigMap return nil } diff --git a/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go index e53176233d..b7a7c85606 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go @@ -2198,6 +2198,15 @@ func (in *KubeSchedulerConfig) DeepCopyInto(out *KubeSchedulerConfig) { (*in).DeepCopyInto(*out) } } + if in.UsePolicyConfigMap != nil { + in, out := &in.UsePolicyConfigMap, &out.UsePolicyConfigMap + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } return } diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go index c197254367..1ea0558103 100644 --- a/pkg/apis/kops/v1alpha2/componentconfig.go +++ b/pkg/apis/kops/v1alpha2/componentconfig.go @@ -345,10 +345,8 @@ type KubeSchedulerConfig struct { Image string `json:"image,omitempty"` // LeaderElection defines the configuration of leader election client. LeaderElection *LeaderElectionConfiguration `json:"leaderElection,omitempty"` - // PolicyConfigMap is the name of configmap to use for scheduler policy - PolicyConfigMap string `json:"policyConfigMap,omitempty" flag:"policy-configmap"` - // PolicyConfigMapNamespace is the namespace containing the configmap - PolicyConfigMapNamespace string `json:"policyConfigMapNamespace,omitempty" flag:"policy-configmap-namespace"` + // UsePolicyConfigMap enable setting the scheduler policy from a configmap + UsePolicyConfigMap *bool `json:"usePolicyConfigMap,omitempty"` } // LeaderElectionConfiguration defines the configuration of leader election diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 0cb00dc8e5..57f356351c 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -2283,8 +2283,7 @@ func autoConvert_v1alpha2_KubeSchedulerConfig_To_kops_KubeSchedulerConfig(in *Ku } else { out.LeaderElection = nil } - out.PolicyConfigMap = in.PolicyConfigMap - out.PolicyConfigMapNamespace = in.PolicyConfigMapNamespace + out.UsePolicyConfigMap = in.UsePolicyConfigMap return nil } @@ -2306,8 +2305,7 @@ func autoConvert_kops_KubeSchedulerConfig_To_v1alpha2_KubeSchedulerConfig(in *ko } else { out.LeaderElection = nil } - out.PolicyConfigMap = in.PolicyConfigMap - out.PolicyConfigMapNamespace = in.PolicyConfigMapNamespace + out.UsePolicyConfigMap = in.UsePolicyConfigMap return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index 2560b2b235..3d5e8ce21c 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -2324,6 +2324,15 @@ func (in *KubeSchedulerConfig) DeepCopyInto(out *KubeSchedulerConfig) { (*in).DeepCopyInto(*out) } } + if in.UsePolicyConfigMap != nil { + in, out := &in.UsePolicyConfigMap, &out.UsePolicyConfigMap + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } return } diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index dde59839df..1aa1913562 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -2543,6 +2543,15 @@ func (in *KubeSchedulerConfig) DeepCopyInto(out *KubeSchedulerConfig) { (*in).DeepCopyInto(*out) } } + if in.UsePolicyConfigMap != nil { + in, out := &in.UsePolicyConfigMap, &out.UsePolicyConfigMap + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } return } diff --git a/pkg/model/components/kubescheduler.go b/pkg/model/components/kubescheduler.go index 85de3667cb..ac4ab4ad1f 100644 --- a/pkg/model/components/kubescheduler.go +++ b/pkg/model/components/kubescheduler.go @@ -39,8 +39,8 @@ func (b *KubeSchedulerOptionsBuilder) BuildOptions(o interface{}) error { config := clusterSpec.KubeScheduler - if config.PolicyConfigMap != "" && b.IsKubernetesLT("v1.7.0") { - return fmt.Errorf("policyConfigMap is only supported in Kubernetes 1.7.0 or later") + if config.UsePolicyConfigMap != nil && b.IsKubernetesLT("v1.7.0") { + return fmt.Errorf("usePolicyConfigMap is only supported in Kubernetes 1.7.0 or later") } if config.LogLevel == 0 { diff --git a/pkg/model/components/kubescheduler_test.go b/pkg/model/components/kubescheduler_test.go index be950af64c..f02b266838 100644 --- a/pkg/model/components/kubescheduler_test.go +++ b/pkg/model/components/kubescheduler_test.go @@ -25,12 +25,14 @@ import ( ) func buildSchedulerConfigMapCluster() *api.Cluster { + usePolicyConfigMap := true + return &api.Cluster{ Spec: api.ClusterSpec{ CloudProvider: "aws", KubernetesVersion: "v1.4.0", KubeScheduler: &api.KubeSchedulerConfig{ - PolicyConfigMap: "scheduler-config", + UsePolicyConfigMap: &usePolicyConfigMap, }, }, } diff --git a/upup/models/cloudup/resources/addons/scheduler.addons.k8s.io/v1.7.0.yaml b/upup/models/cloudup/resources/addons/scheduler.addons.k8s.io/v1.7.0.yaml new file mode 100644 index 0000000000..e5af28ea76 --- /dev/null +++ b/upup/models/cloudup/resources/addons/scheduler.addons.k8s.io/v1.7.0.yaml @@ -0,0 +1,38 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: scheduler-policy + namespace: kube-system + labels: + k8s-addon: scheduler.addons.k8s.io +data: + policy.cfg: | + { + "kind" : "Policy", + "apiVersion" : "v1", + "predicates" : [ + {"name": "NoDiskConflict"}, + {"name": "NoVolumeZoneConflict"}, + {"name": "MaxEBSVolumeCount"}, + {"name": "MaxGCEPDVolumeCount"}, + {"name": "MaxAzureDiskVolumeCount"}, + {"name": "MatchInterPodAffinity"}, + {"name": "NoDiskConflict"}, + {"name": "GeneralPredicates"}, + {"name": "CheckNodeMemoryPressure"}, + {"name": "CheckNodeDiskPressure"}, + {"name": "CheckNodeCondition"}, + {"name": "PodToleratesNodeTaints"}, + {"name": "NoVolumeNodeConflict"} + ], + "priorities" : [ + {"name": "SelectorSpreadPriority", "weight" : 1}, + {"name": "LeastRequestedPriority", "weight" : 1}, + {"name": "BalancedResourceAllocation", "weight" : 1}, + {"name": "NodePreferAvoidPodsPriority", "weight" : 1}, + {"name": "NodeAffinityPriority", "weight" : 1}, + {"name": "TaintTolerationPriority", "weight" : 1}, + {"name": "InterPodAffinityPriority", "weight" : 1} + ], + "hardPodAffinitySymmetricWeight" : 1 + } \ No newline at end of file diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index b4d888809c..34f5ee9eec 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -639,5 +639,20 @@ func (b *BootstrapChannelBuilder) buildManifest() (*channelsapi.Addons, map[stri } } + if b.cluster.Spec.KubeScheduler.UsePolicyConfigMap != nil { + key := "scheduler.addons.k8s.io" + version := "1.7.0" + + location := key + "/v" + version + ".yaml" + + addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ + Name: fi.String(key), + Version: fi.String(version), + Selector: map[string]string{"k8s-addon": key}, + Manifest: fi.String(location), + }) + manifests[key] = "addons/" + location + } + return addons, manifests, nil } From 170bf5200eaa10f1245e4b90ce2c186c68f134d0 Mon Sep 17 00:00:00 2001 From: Manatsawin Hanmongkolchai Date: Tue, 10 Oct 2017 11:02:02 +0700 Subject: [PATCH 2/2] Updated docs for UsePolicyConfigMap --- docs/cluster_spec.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index 03f201ed23..cdee5fa345 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -211,13 +211,12 @@ This block contains configurations for `kube-scheduler`. See https://kubernetes ```yaml spec: kubeScheduler: - policyConfigMap: scheduler-policy - policyConfigMapNamespace: default + usePolicyConfigMap: true ``` -Will resulting to running kube-scheduler with the arguments `--policy-configmap=scheduler-policy --policy-configmap-namespace=default`. +Will make kube-scheduler use the scheduler policy from configmap "scheduler-policy" in namespace kube-system. -Note that as of Kubernetes 1.8.0 kube-scheduler does not reload its configuration from configmap automatically. You will need to ssh into the master instance and restart the Docker container manually. Also, this option is not supported during cluster creation, only during updates. +Note that as of Kubernetes 1.8.0 kube-scheduler does not reload its configuration from configmap automatically. You will need to ssh into the master instance and restart the Docker container manually. ### kubeControllerManager This block contains configurations for the `controller-manager`.