From a9f3db63fc83a517cfb5baf80c50f5896bb22edd Mon Sep 17 00:00:00 2001 From: Roberto Rodriguez Alcala Date: Thu, 23 Jan 2020 15:32:28 -0800 Subject: [PATCH] Support additional kube-scheduler config parameters via config file Mentioned in #6942 This change allows using the --config flag and a generated configfile to set options that were not previously supported and the use via flags is deprecated. (https://kubernetes.io/docs/reference/command-line-tools-reference/kube-scheduler/) I thought that it might be better to have them in a config file to ensure support in newer kubernetes versions. It also makes it easy to add more. --- nodeup/pkg/model/kube_scheduler.go | 22 +++- pkg/apis/kops/componentconfig.go | 6 ++ pkg/configbuilder/BUILD.bazel | 26 +++++ pkg/configbuilder/buildconfigfile.go | 119 ++++++++++++++++++++++ pkg/configbuilder/buildconfigfile_test.go | 85 ++++++++++++++++ 5 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 pkg/configbuilder/BUILD.bazel create mode 100644 pkg/configbuilder/buildconfigfile.go create mode 100644 pkg/configbuilder/buildconfigfile_test.go diff --git a/nodeup/pkg/model/kube_scheduler.go b/nodeup/pkg/model/kube_scheduler.go index cd9dd4a69f..7b7ed63bec 100644 --- a/nodeup/pkg/model/kube_scheduler.go +++ b/nodeup/pkg/model/kube_scheduler.go @@ -20,6 +20,7 @@ import ( "fmt" "strconv" + "k8s.io/kops/pkg/configbuilder" "k8s.io/kops/pkg/flagbuilder" "k8s.io/kops/pkg/k8scodecs" "k8s.io/kops/pkg/kubemanifest" @@ -41,6 +42,8 @@ type KubeSchedulerBuilder struct { var _ fi.ModelBuilder = &KubeSchedulerBuilder{} +var defaultKubeConfig = "/var/lib/kube-scheduler/kubeconfig" + // Build is responsible for building the manifest for the kube-scheduler func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error { if !b.IsMaster { @@ -79,6 +82,23 @@ func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error { }) } + if b.Cluster.Spec.KubeScheduler.KubeConfig == nil { + b.Cluster.Spec.KubeScheduler.KubeConfig = &defaultKubeConfig + } + { + config, err := configbuilder.BuildConfigYaml(b.Cluster.Spec.KubeScheduler) + if err != nil { + return err + } + + c.AddTask(&nodetasks.File{ + Path: "/var/lib/kube-scheduler/config", + Contents: fi.NewBytesResource(config), + Type: nodetasks.FileType_File, + Mode: s("0400"), + }) + } + { c.AddTask(&nodetasks.File{ Path: "/var/log/kube-scheduler.log", @@ -101,7 +121,7 @@ func (b *KubeSchedulerBuilder) buildPod() (*v1.Pod, error) { return nil, fmt.Errorf("error building kube-scheduler flags: %v", err) } // Add kubeconfig flag - flags = append(flags, "--kubeconfig="+"/var/lib/kube-scheduler/kubeconfig") + flags = append(flags, "--config="+"/var/lib/kube-scheduler/config") if c.UsePolicyConfigMap != nil { flags = append(flags, "--policy-configmap=scheduler-policy", "--policy-configmap-namespace=kube-system") diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index 0d50c02aa1..5469a8bf91 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -609,6 +609,12 @@ type KubeSchedulerConfig struct { // which has been supported as far back as Kubernetes 1.7. The default depends on the version and the cloud provider // as outlined: https://kubernetes.io/docs/concepts/storage/storage-limits/ MaxPersistentVolumes *int32 `json:"maxPersistentVolumes,omitempty"` + // Qps sets the maximum qps to send to apiserver after the burst quota is exhausted + Qps *float32 `json:"qps,omitempty" configfile:"ClientConnection.QPS"` + // Burst sets the maximum qps to send to apiserver after the burst quota is exhausted + Burst *float32 `json:"qps,omitempty" configfile:"ClientConnection.Burst"` + // Overrides the default kubeconfig path. + KubeConfig *string `json:"kubeConfig,omitempty" configfile:"ClientConnection.Kubeconfig"` } // LeaderElectionConfiguration defines the configuration of leader election diff --git a/pkg/configbuilder/BUILD.bazel b/pkg/configbuilder/BUILD.bazel new file mode 100644 index 0000000000..c377d6dd2c --- /dev/null +++ b/pkg/configbuilder/BUILD.bazel @@ -0,0 +1,26 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["buildconfigfile.go"], + importpath = "k8s.io/kops/pkg/configbuilder", + visibility = ["//visibility:public"], + deps = [ + "//util/pkg/reflectutils:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["buildconfigfile_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/apis/kops:go_default_library", + "//upup/pkg/fi:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + ], +) diff --git a/pkg/configbuilder/buildconfigfile.go b/pkg/configbuilder/buildconfigfile.go new file mode 100644 index 0000000000..fcb56d4a9f --- /dev/null +++ b/pkg/configbuilder/buildconfigfile.go @@ -0,0 +1,119 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package configbuilder + +import ( + "fmt" + "reflect" + "strings" + + "gopkg.in/yaml.v2" + + "k8s.io/klog" + "k8s.io/kops/util/pkg/reflectutils" +) + +// ClientConnectionConfig for kube-scheduler +type ClientConnectionConfig struct { + Burst *int32 `yaml:"burst,omitempty"` + Kubeconfig *string `yaml:"kubeconfig"` + QPS *float32 `yaml:"qps,omitempty"` +} + +// SchedulerConfig used to generate the config file +type SchedulerConfig struct { + APIVersion string `yaml:"apiVersion"` + Kind string `yaml:"Kind"` + BindTimeoutSeconds *int64 `yaml:"bindTimeoutSeconds,omitempty"` + ClientConnection *ClientConnectionConfig `yaml:"clientConnection,omitempty"` +} + +// BuildConfigYaml reflects the options interface and extracts the parameters for the config file +func BuildConfigYaml(options interface{}) ([]byte, error) { + + schedConfig := new(SchedulerConfig) + schedConfig.APIVersion = "kubescheduler.config.k8s.io/v1alpha1" + schedConfig.Kind = "KubeSchedulerConfiguration" + schedConfig.ClientConnection = new(ClientConnectionConfig) + + walker := func(path string, field *reflect.StructField, val reflect.Value) error { + if field == nil { + klog.V(8).Infof("ignoring non-field: %s", path) + return nil + } + tag := field.Tag.Get("configfile") + if tag == "" { + klog.V(4).Infof("not writing field with no flag tag: %s", path) + // We want to descend - it could be a structure containing flags + return nil + } + if tag == "-" { + klog.V(4).Infof("skipping field with %q flag tag: %s", tag, path) + return reflectutils.SkipReflection + } + + tokens := strings.Split(tag, ",") + + flagName := tokens[0] + + targetValue, error := getValueFromStruct(flagName, schedConfig) + if error != nil { + return fmt.Errorf("conversion error for field %s: %s", flagName, error) + } + // We do have to do this, even though the recursive walk will do it for us + // because when we descend we won't have `field` set + if val.Kind() == reflect.Ptr { + if val.IsNil() { + return nil + } + } + targetValue.Set(val) + + return reflectutils.SkipReflection + } + + err := reflectutils.ReflectRecursive(reflect.ValueOf(options), walker) + if err != nil { + return nil, fmt.Errorf("BuildFlagsList to reflect value: %s", err) + } + + configFile, err := yaml.Marshal(schedConfig) + if err != nil { + return nil, err + } + + return configFile, nil +} + +func getValueFromStruct(keyWithDots string, object *SchedulerConfig) (*reflect.Value, error) { + keySlice := strings.Split(keyWithDots, ".") + v := reflect.ValueOf(object) + // iterate through field names ,ignore the first name as it might be the current instance name + // you can make it recursive also if want to support types like slice,map etc along with struct + for _, key := range keySlice { + for v.Kind() == reflect.Ptr { + v = v.Elem() + } + // we only accept structs + if v.Kind() != reflect.Struct { + return nil, fmt.Errorf("only accepts structs; got %T", v) + } + v = v.FieldByName(key) + } + + return &v, nil +} diff --git a/pkg/configbuilder/buildconfigfile_test.go b/pkg/configbuilder/buildconfigfile_test.go new file mode 100644 index 0000000000..7caf9c109e --- /dev/null +++ b/pkg/configbuilder/buildconfigfile_test.go @@ -0,0 +1,85 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package configbuilder + +import ( + "bytes" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/kops/pkg/apis/kops" + "testing" +) + +func resourceValue(s string) *resource.Quantity { + q := resource.MustParse(s) + return &q +} + +func TestParseBasic(t *testing.T) { + expect := []byte( + `apiVersion: kubescheduler.config.k8s.io/v1alpha1 +Kind: KubeSchedulerConfiguration +clientConnection: + kubeconfig: null + qps: 3 +`) + qps := float32(3.0) + s := &kops.KubeSchedulerConfig{Qps: &qps} + + yaml, err := BuildConfigYaml(s) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + if !bytes.Equal(yaml, expect) { + t.Errorf("unexpected result: %v, expected: %v", expect, yaml) + } +} + +func TestGetStructVal(t *testing.T) { + str := "test" + s := &SchedulerConfig{ + ClientConnection: &ClientConnectionConfig{ + Kubeconfig: &str, + }, + } + v, err := getValueFromStruct("ClientConnection.Kubeconfig", s) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + inStruct := v.Elem().String() + if inStruct != str { + t.Errorf("unexpected value: %s, %s, expected: %s", inStruct, err, str) + } + +} + +func TestWrongStructField(t *testing.T) { + str := "test" + s := &SchedulerConfig{ + ClientConnection: &ClientConnectionConfig{ + Kubeconfig: &str, + }, + } + v, err := getValueFromStruct("ClientConnection.NotExistent", s) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + if v.IsValid() { + t.Errorf("unexpected Valid value from non-existent field lookup") + } + +}