Support additional kube-scheduler config parameters via config file

Mentioned in #6942

This change allows using the --config flag and a generated configfile to set
options that were not previously supported and the use via flags is deprecated.
(https://kubernetes.io/docs/reference/command-line-tools-reference/kube-scheduler/)

I thought that it might be better to have them in a config file to ensure
support in newer kubernetes versions.

It also makes it easy to add more.

nodeup/pkg/model/kube_scheduler.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"strconv"
 
+	"k8s.io/kops/pkg/configbuilder"
 	"k8s.io/kops/pkg/flagbuilder"
 	"k8s.io/kops/pkg/k8scodecs"
 	"k8s.io/kops/pkg/kubemanifest"
@@ -41,6 +42,8 @@ type KubeSchedulerBuilder struct {
 
 var _ fi.ModelBuilder = &KubeSchedulerBuilder{}
 
+var defaultKubeConfig = "/var/lib/kube-scheduler/kubeconfig"
+
 // Build is responsible for building the manifest for the kube-scheduler
 func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error {
 	if !b.IsMaster {
@@ -79,6 +82,23 @@ func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error {
 		})
 	}
 
+	if b.Cluster.Spec.KubeScheduler.KubeConfig == nil {
+		b.Cluster.Spec.KubeScheduler.KubeConfig = &defaultKubeConfig
+	}
+	{
+		config, err := configbuilder.BuildConfigYaml(b.Cluster.Spec.KubeScheduler)
+		if err != nil {
+			return err
+		}
+
+		c.AddTask(&nodetasks.File{
+			Path:     "/var/lib/kube-scheduler/config",
+			Contents: fi.NewBytesResource(config),
+			Type:     nodetasks.FileType_File,
+			Mode:     s("0400"),
+		})
+	}
+
 	{
 		c.AddTask(&nodetasks.File{
 			Path:        "/var/log/kube-scheduler.log",
@@ -101,7 +121,7 @@ func (b *KubeSchedulerBuilder) buildPod() (*v1.Pod, error) {
 		return nil, fmt.Errorf("error building kube-scheduler flags: %v", err)
 	}
 	// Add kubeconfig flag
-	flags = append(flags, "--kubeconfig="+"/var/lib/kube-scheduler/kubeconfig")
+	flags = append(flags, "--config="+"/var/lib/kube-scheduler/config")
 
 	if c.UsePolicyConfigMap != nil {
 		flags = append(flags, "--policy-configmap=scheduler-policy", "--policy-configmap-namespace=kube-system")

pkg/apis/kops/componentconfig.go

@@ -609,6 +609,12 @@ type KubeSchedulerConfig struct {
 	// which has been supported as far back as Kubernetes 1.7. The default depends on the version and the cloud provider
 	// as outlined: https://kubernetes.io/docs/concepts/storage/storage-limits/
 	MaxPersistentVolumes *int32 `json:"maxPersistentVolumes,omitempty"`
+	// Qps sets the maximum qps to send to apiserver after the burst quota is exhausted
+	Qps *float32 `json:"qps,omitempty" configfile:"ClientConnection.QPS"`
+	// Burst sets the maximum qps to send to apiserver after the burst quota is exhausted
+	Burst *float32 `json:"qps,omitempty" configfile:"ClientConnection.Burst"`
+	// Overrides the default kubeconfig path.
+	KubeConfig *string `json:"kubeConfig,omitempty" configfile:"ClientConnection.Kubeconfig"`
 }
 
 // LeaderElectionConfiguration defines the configuration of leader election

pkg/configbuilder/BUILD.bazel

@@ -0,0 +1,26 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["buildconfigfile.go"],
+    importpath = "k8s.io/kops/pkg/configbuilder",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//util/pkg/reflectutils:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/klog:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["buildconfigfile_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//pkg/apis/kops:go_default_library",
+        "//upup/pkg/fi:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+    ],
+)

pkg/configbuilder/buildconfigfile.go

@@ -0,0 +1,119 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package configbuilder
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+
+	"gopkg.in/yaml.v2"
+
+	"k8s.io/klog"
+	"k8s.io/kops/util/pkg/reflectutils"
+)
+
+// ClientConnectionConfig for kube-scheduler
+type ClientConnectionConfig struct {
+	Burst      *int32   `yaml:"burst,omitempty"`
+	Kubeconfig *string  `yaml:"kubeconfig"`
+	QPS        *float32 `yaml:"qps,omitempty"`
+}
+
+// SchedulerConfig used to generate the config file
+type SchedulerConfig struct {
+	APIVersion         string                  `yaml:"apiVersion"`
+	Kind               string                  `yaml:"Kind"`
+	BindTimeoutSeconds *int64                  `yaml:"bindTimeoutSeconds,omitempty"`
+	ClientConnection   *ClientConnectionConfig `yaml:"clientConnection,omitempty"`
+}
+
+// BuildConfigYaml reflects the options interface and extracts the parameters for the config file
+func BuildConfigYaml(options interface{}) ([]byte, error) {
+
+	schedConfig := new(SchedulerConfig)
+	schedConfig.APIVersion = "kubescheduler.config.k8s.io/v1alpha1"
+	schedConfig.Kind = "KubeSchedulerConfiguration"
+	schedConfig.ClientConnection = new(ClientConnectionConfig)
+
+	walker := func(path string, field *reflect.StructField, val reflect.Value) error {
+		if field == nil {
+			klog.V(8).Infof("ignoring non-field: %s", path)
+			return nil
+		}
+		tag := field.Tag.Get("configfile")
+		if tag == "" {
+			klog.V(4).Infof("not writing field with no flag tag: %s", path)
+			// We want to descend - it could be a structure containing flags
+			return nil
+		}
+		if tag == "-" {
+			klog.V(4).Infof("skipping field with %q flag tag: %s", tag, path)
+			return reflectutils.SkipReflection
+		}
+
+		tokens := strings.Split(tag, ",")
+
+		flagName := tokens[0]
+
+		targetValue, error := getValueFromStruct(flagName, schedConfig)
+		if error != nil {
+			return fmt.Errorf("conversion error for field %s: %s", flagName, error)
+		}
+		// We do have to do this, even though the recursive walk will do it for us
+		// because when we descend we won't have `field` set
+		if val.Kind() == reflect.Ptr {
+			if val.IsNil() {
+				return nil
+			}
+		}
+		targetValue.Set(val)
+
+		return reflectutils.SkipReflection
+	}
+
+	err := reflectutils.ReflectRecursive(reflect.ValueOf(options), walker)
+	if err != nil {
+		return nil, fmt.Errorf("BuildFlagsList to reflect value: %s", err)
+	}
+
+	configFile, err := yaml.Marshal(schedConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return configFile, nil
+}
+
+func getValueFromStruct(keyWithDots string, object *SchedulerConfig) (*reflect.Value, error) {
+	keySlice := strings.Split(keyWithDots, ".")
+	v := reflect.ValueOf(object)
+	// iterate through field names ,ignore the first name as it might be the current instance name
+	// you can make it recursive also if want to support types like slice,map etc along with struct
+	for _, key := range keySlice {
+		for v.Kind() == reflect.Ptr {
+			v = v.Elem()
+		}
+		// we only accept structs
+		if v.Kind() != reflect.Struct {
+			return nil, fmt.Errorf("only accepts structs; got %T", v)
+		}
+		v = v.FieldByName(key)
+	}
+
+	return &v, nil
+}

pkg/configbuilder/buildconfigfile_test.go

@@ -0,0 +1,85 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package configbuilder
+
+import (
+	"bytes"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/kops/pkg/apis/kops"
+	"testing"
+)
+
+func resourceValue(s string) *resource.Quantity {
+	q := resource.MustParse(s)
+	return &q
+}
+
+func TestParseBasic(t *testing.T) {
+	expect := []byte(
+		`apiVersion: kubescheduler.config.k8s.io/v1alpha1
+Kind: KubeSchedulerConfiguration
+clientConnection:
+  kubeconfig: null
+  qps: 3
+`)
+	qps := float32(3.0)
+	s := &kops.KubeSchedulerConfig{Qps: &qps}
+
+	yaml, err := BuildConfigYaml(s)
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
+
+	if !bytes.Equal(yaml, expect) {
+		t.Errorf("unexpected result: %v, expected: %v", expect, yaml)
+	}
+}
+
+func TestGetStructVal(t *testing.T) {
+	str := "test"
+	s := &SchedulerConfig{
+		ClientConnection: &ClientConnectionConfig{
+			Kubeconfig: &str,
+		},
+	}
+	v, err := getValueFromStruct("ClientConnection.Kubeconfig", s)
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
+	inStruct := v.Elem().String()
+	if inStruct != str {
+		t.Errorf("unexpected value: %s, %s, expected: %s", inStruct, err, str)
+	}
+
+}
+
+func TestWrongStructField(t *testing.T) {
+	str := "test"
+	s := &SchedulerConfig{
+		ClientConnection: &ClientConnectionConfig{
+			Kubeconfig: &str,
+		},
+	}
+	v, err := getValueFromStruct("ClientConnection.NotExistent", s)
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
+	if v.IsValid() {
+		t.Errorf("unexpected Valid value from non-existent field lookup")
+	}
+
+}