Merge pull request #8407 from rralcala/master

Support additional kube-scheduler config parameters via config file

hack/.packages

@@ -82,6 +82,7 @@ k8s.io/kops/pkg/client/simple/api
 k8s.io/kops/pkg/client/simple/vfsclientset
 k8s.io/kops/pkg/cloudinstances
 k8s.io/kops/pkg/commands
+k8s.io/kops/pkg/configbuilder
 k8s.io/kops/pkg/diff
 k8s.io/kops/pkg/dns
 k8s.io/kops/pkg/drain

k8s/crds/kops.k8s.io_clusters.yaml

@@ -1609,6 +1609,11 @@ spec:
             kubeScheduler:
               description: KubeSchedulerConfig is the configuration for the kube-scheduler
               properties:
+                burst:
+                  description: Burst sets the maximum qps to send to apiserver after
+                    the burst quota is exhausted
+                  format: int32
+                  type: integer
                 featureGates:
                   additionalProperties:
                     type: string
@@ -1677,6 +1682,10 @@ spec:
                     and the cloud provider as outlined: https://kubernetes.io/docs/concepts/storage/storage-limits/'
                   format: int32
                   type: integer
+                qps:
+                  description: Qps sets the maximum qps to send to apiserver after
+                    the burst quota is exhausted
+                  type: string
                 usePolicyConfigMap:
                   description: UsePolicyConfigMap enable setting the scheduler policy
                     from a configmap

nodeup/pkg/model/BUILD.bazel

@@ -46,6 +46,7 @@ go_library(
         "//pkg/apis/kops/util:go_default_library",
         "//pkg/apis/nodeup:go_default_library",
         "//pkg/assets:go_default_library",
+        "//pkg/configbuilder:go_default_library",
         "//pkg/dns:go_default_library",
         "//pkg/flagbuilder:go_default_library",
         "//pkg/k8scodecs:go_default_library",
@@ -88,6 +89,7 @@ go_test(
         "docker_test.go",
         "kube_apiserver_test.go",
         "kube_proxy_test.go",
+        "kube_scheduler_test.go",
         "kubelet_test.go",
         "protokube_test.go",
     ],
@@ -97,6 +99,7 @@ go_test(
         "//nodeup/pkg/distros:go_default_library",
         "//pkg/apis/kops:go_default_library",
         "//pkg/apis/nodeup:go_default_library",
+        "//pkg/configbuilder:go_default_library",
         "//pkg/flagbuilder:go_default_library",
         "//pkg/testutils:go_default_library",
         "//upup/pkg/fi:go_default_library",

nodeup/pkg/model/kube_scheduler.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"strconv"
 
+	"k8s.io/kops/pkg/configbuilder"
 	"k8s.io/kops/pkg/flagbuilder"
 	"k8s.io/kops/pkg/k8scodecs"
 	"k8s.io/kops/pkg/kubemanifest"
@@ -34,6 +35,20 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
+// ClientConnectionConfig is used by kube-scheduler to talk to the api server
+type ClientConnectionConfig struct {
+	Burst      int32    `yaml:"burst,omitempty"`
+	Kubeconfig string   `yaml:"kubeconfig"`
+	QPS        *float64 `yaml:"qps,omitempty"`
+}
+
+// SchedulerConfig is used to generate the config file
+type SchedulerConfig struct {
+	APIVersion       string                 `yaml:"apiVersion"`
+	Kind             string                 `yaml:"kind"`
+	ClientConnection ClientConnectionConfig `yaml:"clientConnection,omitempty"`
+}
+
 // KubeSchedulerBuilder install kube-scheduler
 type KubeSchedulerBuilder struct {
 	*NodeupModelContext
@@ -41,14 +56,16 @@ type KubeSchedulerBuilder struct {
 
 var _ fi.ModelBuilder = &KubeSchedulerBuilder{}
 
+const defaultKubeConfig = "/var/lib/kube-scheduler/kubeconfig"
+
 // Build is responsible for building the manifest for the kube-scheduler
 func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error {
 	if !b.IsMaster {
 		return nil
 	}
-
+	useConfigFile := b.IsKubernetesGTE("1.11")
 	{
-		pod, err := b.buildPod()
+		pod, err := b.buildPod(useConfigFile)
 		if err != nil {
 			return fmt.Errorf("error building kube-scheduler pod: %v", err)
 		}
@@ -78,6 +95,19 @@ func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error {
 			Mode:     s("0400"),
 		})
 	}
+	if useConfigFile {
+		config, err := configbuilder.BuildConfigYaml(b.Cluster.Spec.KubeScheduler, NewSchedulerConfig())
+		if err != nil {
+			return err
+		}
+
+		c.AddTask(&nodetasks.File{
+			Path:     "/var/lib/kube-scheduler/config.yaml",
+			Contents: fi.NewBytesResource(config),
+			Type:     nodetasks.FileType_File,
+			Mode:     s("0400"),
+		})
+	}
 
 	{
 		c.AddTask(&nodetasks.File{
@@ -92,16 +122,30 @@ func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error {
 	return nil
 }
 
+// NewSchedulerConfig initializes a new kube-scheduler config file
+func NewSchedulerConfig() *SchedulerConfig {
+	schedConfig := new(SchedulerConfig)
+	schedConfig.APIVersion = "kubescheduler.config.k8s.io/v1alpha1"
+	schedConfig.Kind = "KubeSchedulerConfiguration"
+	schedConfig.ClientConnection = ClientConnectionConfig{}
+	schedConfig.ClientConnection.Kubeconfig = defaultKubeConfig
+	return schedConfig
+}
+
 // buildPod is responsible for constructing the pod specification
-func (b *KubeSchedulerBuilder) buildPod() (*v1.Pod, error) {
+func (b *KubeSchedulerBuilder) buildPod(useConfigFile bool) (*v1.Pod, error) {
 	c := b.Cluster.Spec.KubeScheduler
 
 	flags, err := flagbuilder.BuildFlagsList(c)
 	if err != nil {
 		return nil, fmt.Errorf("error building kube-scheduler flags: %v", err)
 	}
-	// Add kubeconfig flag
+	if useConfigFile {
-	flags = append(flags, "--kubeconfig="+"/var/lib/kube-scheduler/kubeconfig")
+		flags = append(flags, "--config="+"/var/lib/kube-scheduler/config.yaml")
+	} else {
+		// Add kubeconfig flag
+		flags = append(flags, "--kubeconfig="+defaultKubeConfig)
+	}
 
 	if c.UsePolicyConfigMap != nil {
 		flags = append(flags, "--policy-configmap=scheduler-policy", "--policy-configmap-namespace=kube-system")

nodeup/pkg/model/kube_scheduler_test.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package model
+
+import (
+	"bytes"
+	"testing"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/configbuilder"
+)
+
+func TestParseDefault(t *testing.T) {
+	expect := []byte(
+		`apiVersion: kubescheduler.config.k8s.io/v1alpha1
+kind: KubeSchedulerConfiguration
+clientConnection:
+  kubeconfig: /var/lib/kube-scheduler/kubeconfig
+`)
+
+	s := &kops.KubeSchedulerConfig{}
+
+	yaml, err := configbuilder.BuildConfigYaml(s, NewSchedulerConfig())
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
+
+	if !bytes.Equal(yaml, expect) {
+		t.Errorf("unexpected result: \n%s, expected: \n%s", yaml, expect)
+	}
+}
+
+func TestParse(t *testing.T) {
+	expect := []byte(
+		`apiVersion: kubescheduler.config.k8s.io/v1alpha1
+kind: KubeSchedulerConfiguration
+clientConnection:
+  burst: 100
+  kubeconfig: /var/lib/kube-scheduler/kubeconfig
+  qps: 3.1
+`)
+	qps, _ := resource.ParseQuantity("3.1")
+
+	s := &kops.KubeSchedulerConfig{Qps: &qps, Burst: 100}
+
+	yaml, err := configbuilder.BuildConfigYaml(s, NewSchedulerConfig())
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
+
+	if !bytes.Equal(yaml, expect) {
+		t.Errorf("unexpected result: \n%s, expected: \n%s", yaml, expect)
+	}
+}

pkg/apis/kops/componentconfig.go

@@ -609,6 +609,10 @@ type KubeSchedulerConfig struct {
 	// which has been supported as far back as Kubernetes 1.7. The default depends on the version and the cloud provider
 	// as outlined: https://kubernetes.io/docs/concepts/storage/storage-limits/
 	MaxPersistentVolumes *int32 `json:"maxPersistentVolumes,omitempty"`
+	// Qps sets the maximum qps to send to apiserver after the burst quota is exhausted
+	Qps *resource.Quantity `json:"qps,omitempty" configfile:"ClientConnection.QPS"`
+	// Burst sets the maximum qps to send to apiserver after the burst quota is exhausted
+	Burst int32 `json:"burst,omitempty" configfile:"ClientConnection.Burst"`
 }
 
 // LeaderElectionConfiguration defines the configuration of leader election

pkg/apis/kops/v1alpha1/componentconfig.go

@@ -609,6 +609,10 @@ type KubeSchedulerConfig struct {
 	// which has been supported as far back as Kubernetes 1.7. The default depends on the version and the cloud provider
 	// as outlined: https://kubernetes.io/docs/concepts/storage/storage-limits/
 	MaxPersistentVolumes *int32 `json:"maxPersistentVolumes,omitempty"`
+	// Qps sets the maximum qps to send to apiserver after the burst quota is exhausted
+	Qps *resource.Quantity `json:"qps,omitempty" configfile:"ClientConnection.QPS"`
+	// Burst sets the maximum qps to send to apiserver after the burst quota is exhausted
+	Burst int32 `json:"burst,omitempty" configfile:"ClientConnection.Burst"`
 }
 
 // LeaderElectionConfiguration defines the configuration of leader election

pkg/apis/kops/v1alpha1/zz_generated.conversion.go

@@ -3656,6 +3656,8 @@ func autoConvert_v1alpha1_KubeSchedulerConfig_To_kops_KubeSchedulerConfig(in *Ku
 	out.UsePolicyConfigMap = in.UsePolicyConfigMap
 	out.FeatureGates = in.FeatureGates
 	out.MaxPersistentVolumes = in.MaxPersistentVolumes
+	out.Qps = in.Qps
+	out.Burst = in.Burst
 	return nil
 }
 
@@ -3680,6 +3682,8 @@ func autoConvert_kops_KubeSchedulerConfig_To_v1alpha1_KubeSchedulerConfig(in *ko
 	out.UsePolicyConfigMap = in.UsePolicyConfigMap
 	out.FeatureGates = in.FeatureGates
 	out.MaxPersistentVolumes = in.MaxPersistentVolumes
+	out.Qps = in.Qps
+	out.Burst = in.Burst
 	return nil
 }
 

pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go

@@ -2415,6 +2415,11 @@ func (in *KubeSchedulerConfig) DeepCopyInto(out *KubeSchedulerConfig) {
 		*out = new(int32)
 		**out = **in
 	}
+	if in.Qps != nil {
+		in, out := &in.Qps, &out.Qps
+		x := (*in).DeepCopy()
+		*out = &x
+	}
 	return
 }
 

pkg/apis/kops/v1alpha2/componentconfig.go

@@ -610,6 +610,10 @@ type KubeSchedulerConfig struct {
 	// which has been supported as far back as Kubernetes 1.7. The default depends on the version and the cloud provider
 	// as outlined: https://kubernetes.io/docs/concepts/storage/storage-limits/
 	MaxPersistentVolumes *int32 `json:"maxPersistentVolumes,omitempty"`
+	// Qps sets the maximum qps to send to apiserver after the burst quota is exhausted
+	Qps *resource.Quantity `json:"qps,omitempty"`
+	// Burst sets the maximum qps to send to apiserver after the burst quota is exhausted
+	Burst int32 `json:"burst,omitempty"`
 }
 
 // LeaderElectionConfiguration defines the configuration of leader election

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -3926,6 +3926,8 @@ func autoConvert_v1alpha2_KubeSchedulerConfig_To_kops_KubeSchedulerConfig(in *Ku
 	out.UsePolicyConfigMap = in.UsePolicyConfigMap
 	out.FeatureGates = in.FeatureGates
 	out.MaxPersistentVolumes = in.MaxPersistentVolumes
+	out.Qps = in.Qps
+	out.Burst = in.Burst
 	return nil
 }
 
@@ -3950,6 +3952,8 @@ func autoConvert_kops_KubeSchedulerConfig_To_v1alpha2_KubeSchedulerConfig(in *ko
 	out.UsePolicyConfigMap = in.UsePolicyConfigMap
 	out.FeatureGates = in.FeatureGates
 	out.MaxPersistentVolumes = in.MaxPersistentVolumes
+	out.Qps = in.Qps
+	out.Burst = in.Burst
 	return nil
 }
 

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -2486,6 +2486,11 @@ func (in *KubeSchedulerConfig) DeepCopyInto(out *KubeSchedulerConfig) {
 		*out = new(int32)
 		**out = **in
 	}
+	if in.Qps != nil {
+		in, out := &in.Qps, &out.Qps
+		x := (*in).DeepCopy()
+		*out = &x
+	}
 	return
 }
 

pkg/apis/kops/zz_generated.deepcopy.go

@@ -2668,6 +2668,11 @@ func (in *KubeSchedulerConfig) DeepCopyInto(out *KubeSchedulerConfig) {
 		*out = new(int32)
 		**out = **in
 	}
+	if in.Qps != nil {
+		in, out := &in.Qps, &out.Qps
+		x := (*in).DeepCopy()
+		*out = &x
+	}
 	return
 }
 

pkg/configbuilder/BUILD.bazel

@@ -0,0 +1,21 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["buildconfigfile.go"],
+    importpath = "k8s.io/kops/pkg/configbuilder",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/apis/kops:go_default_library",
+        "//util/pkg/reflectutils:go_default_library",
+        "//vendor/gopkg.in/yaml.v2:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//vendor/k8s.io/klog:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["buildconfigfile_test.go"],
+    embed = [":go_default_library"],
+)

pkg/configbuilder/buildconfigfile.go

@@ -0,0 +1,111 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package configbuilder
+
+import (
+	"fmt"
+
+	"reflect"
+	"strconv"
+	"strings"
+
+	"gopkg.in/yaml.v2"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/klog"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/util/pkg/reflectutils"
+)
+
+// BuildConfigYaml reflects the options interface and extracts the parameters for the config file
+func BuildConfigYaml(options *kops.KubeSchedulerConfig, target interface{}) ([]byte, error) {
+	walker := func(path string, field *reflect.StructField, val reflect.Value) error {
+		if field == nil {
+			klog.V(8).Infof("ignoring non-field: %s", path)
+			return nil
+		}
+		tag := field.Tag.Get("configfile")
+		if tag == "" {
+			klog.V(4).Infof("not writing field with no configfile tag: %s", path)
+			// We want to descend - it could be a structure containing flags
+			return nil
+		}
+		if tag == "-" {
+			klog.V(4).Infof("skipping field with %q configfile tag: %s", tag, path)
+			return reflectutils.SkipReflection
+		}
+
+		tokens := strings.Split(tag, ",")
+
+		flagName := tokens[0]
+
+		targetValue, error := getValueFromStruct(flagName, target)
+		if error != nil {
+			return fmt.Errorf("conversion error for field %s: %s", flagName, error)
+		}
+		// We do have to do this, even though the recursive walk will do it for us
+		// because when we descend we won't have `field` set
+		if val.Kind() == reflect.Ptr {
+			if val.IsNil() {
+				return nil
+			}
+		}
+		switch v := val.Interface().(type) {
+		case *resource.Quantity:
+			floatVal, err := strconv.ParseFloat(v.AsDec().String(), 64)
+			if err != nil {
+				return fmt.Errorf("unable to convert from Quantity %v to float", v)
+			}
+			targetValue.Set(reflect.ValueOf(&floatVal))
+		default:
+			targetValue.Set(val)
+		}
+
+		return reflectutils.SkipReflection
+
+	}
+
+	err := reflectutils.ReflectRecursive(reflect.ValueOf(options), walker)
+	if err != nil {
+		return nil, fmt.Errorf("BuildFlagsList to reflect value: %s", err)
+	}
+
+	configFile, err := yaml.Marshal(target)
+	if err != nil {
+		return nil, err
+	}
+
+	return configFile, nil
+}
+
+func getValueFromStruct(keyWithDots string, object interface{}) (*reflect.Value, error) {
+	keySlice := strings.Split(keyWithDots, ".")
+	v := reflect.ValueOf(object)
+	// iterate through field names, ignoring the first name as it might be the current instance name
+	// you can make it recursive also if want to support types like slice,map etc along with struct
+	for _, key := range keySlice {
+		for v.Kind() == reflect.Ptr {
+			v = v.Elem()
+		}
+		// we only accept structs
+		if v.Kind() != reflect.Struct {
+			return nil, fmt.Errorf("only accepts structs; got %T", v)
+		}
+		v = v.FieldByName(key)
+	}
+
+	return &v, nil
+}

pkg/configbuilder/buildconfigfile_test.go

@@ -0,0 +1,67 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package configbuilder
+
+import (
+	"testing"
+)
+
+// ClientConnectionConfig is used by kube-scheduler to talk to the api server
+type DummyNestedStruct struct {
+	Name *string  `yaml:"name,omitempty"`
+	QPS  *float64 `yaml:"qps,omitempty"`
+}
+
+// SchedulerConfig is used to generate the config file
+type DummyStruct struct {
+	ClientConnection *DummyNestedStruct `yaml:"clientConnection,omitempty"`
+}
+
+func TestGetStructVal(t *testing.T) {
+	str := "test"
+	s := &DummyStruct{
+		ClientConnection: &DummyNestedStruct{
+			Name: &str,
+		},
+	}
+	v, err := getValueFromStruct("ClientConnection.Name", s)
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
+	inStruct := v.Elem().String()
+	if inStruct != str {
+		t.Errorf("unexpected value: %s, %s, expected: %s", inStruct, err, str)
+	}
+
+}
+
+func TestWrongStructField(t *testing.T) {
+	str := "test"
+	s := &DummyStruct{
+		ClientConnection: &DummyNestedStruct{
+			Name: &str,
+		},
+	}
+	v, err := getValueFromStruct("ClientConnection.NotExistent", s)
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
+	if v.IsValid() {
+		t.Errorf("unexpected Valid value from non-existent field lookup")
+	}
+
+}