diff --git a/hack/.packages b/hack/.packages index 479e397c9a..0819aa77fa 100644 --- a/hack/.packages +++ b/hack/.packages @@ -82,6 +82,7 @@ k8s.io/kops/pkg/client/simple/api k8s.io/kops/pkg/client/simple/vfsclientset k8s.io/kops/pkg/cloudinstances k8s.io/kops/pkg/commands +k8s.io/kops/pkg/configbuilder k8s.io/kops/pkg/diff k8s.io/kops/pkg/dns k8s.io/kops/pkg/drain diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 71f4093b3a..49bea005dc 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -1609,6 +1609,11 @@ spec: kubeScheduler: description: KubeSchedulerConfig is the configuration for the kube-scheduler properties: + burst: + description: Burst sets the maximum qps to send to apiserver after + the burst quota is exhausted + format: int32 + type: integer featureGates: additionalProperties: type: string @@ -1677,6 +1682,10 @@ spec: and the cloud provider as outlined: https://kubernetes.io/docs/concepts/storage/storage-limits/' format: int32 type: integer + qps: + description: Qps sets the maximum qps to send to apiserver after + the burst quota is exhausted + type: string usePolicyConfigMap: description: UsePolicyConfigMap enable setting the scheduler policy from a configmap diff --git a/nodeup/pkg/model/BUILD.bazel b/nodeup/pkg/model/BUILD.bazel index acb16f8fdf..9f786cf1c0 100644 --- a/nodeup/pkg/model/BUILD.bazel +++ b/nodeup/pkg/model/BUILD.bazel @@ -46,6 +46,7 @@ go_library( "//pkg/apis/kops/util:go_default_library", "//pkg/apis/nodeup:go_default_library", "//pkg/assets:go_default_library", + "//pkg/configbuilder:go_default_library", "//pkg/dns:go_default_library", "//pkg/flagbuilder:go_default_library", "//pkg/k8scodecs:go_default_library", @@ -88,6 +89,7 @@ go_test( "docker_test.go", "kube_apiserver_test.go", "kube_proxy_test.go", + "kube_scheduler_test.go", "kubelet_test.go", "protokube_test.go", ], @@ -97,6 +99,7 @@ go_test( "//nodeup/pkg/distros:go_default_library", "//pkg/apis/kops:go_default_library", "//pkg/apis/nodeup:go_default_library", + "//pkg/configbuilder:go_default_library", "//pkg/flagbuilder:go_default_library", "//pkg/testutils:go_default_library", "//upup/pkg/fi:go_default_library", diff --git a/nodeup/pkg/model/kube_scheduler.go b/nodeup/pkg/model/kube_scheduler.go index cd9dd4a69f..2c7c454bfa 100644 --- a/nodeup/pkg/model/kube_scheduler.go +++ b/nodeup/pkg/model/kube_scheduler.go @@ -20,6 +20,7 @@ import ( "fmt" "strconv" + "k8s.io/kops/pkg/configbuilder" "k8s.io/kops/pkg/flagbuilder" "k8s.io/kops/pkg/k8scodecs" "k8s.io/kops/pkg/kubemanifest" @@ -34,6 +35,20 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) +// ClientConnectionConfig is used by kube-scheduler to talk to the api server +type ClientConnectionConfig struct { + Burst int32 `yaml:"burst,omitempty"` + Kubeconfig string `yaml:"kubeconfig"` + QPS *float64 `yaml:"qps,omitempty"` +} + +// SchedulerConfig is used to generate the config file +type SchedulerConfig struct { + APIVersion string `yaml:"apiVersion"` + Kind string `yaml:"kind"` + ClientConnection ClientConnectionConfig `yaml:"clientConnection,omitempty"` +} + // KubeSchedulerBuilder install kube-scheduler type KubeSchedulerBuilder struct { *NodeupModelContext @@ -41,14 +56,16 @@ type KubeSchedulerBuilder struct { var _ fi.ModelBuilder = &KubeSchedulerBuilder{} +const defaultKubeConfig = "/var/lib/kube-scheduler/kubeconfig" + // Build is responsible for building the manifest for the kube-scheduler func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error { if !b.IsMaster { return nil } - + useConfigFile := b.IsKubernetesGTE("1.11") { - pod, err := b.buildPod() + pod, err := b.buildPod(useConfigFile) if err != nil { return fmt.Errorf("error building kube-scheduler pod: %v", err) } @@ -78,6 +95,19 @@ func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error { Mode: s("0400"), }) } + if useConfigFile { + config, err := configbuilder.BuildConfigYaml(b.Cluster.Spec.KubeScheduler, NewSchedulerConfig()) + if err != nil { + return err + } + + c.AddTask(&nodetasks.File{ + Path: "/var/lib/kube-scheduler/config.yaml", + Contents: fi.NewBytesResource(config), + Type: nodetasks.FileType_File, + Mode: s("0400"), + }) + } { c.AddTask(&nodetasks.File{ @@ -92,16 +122,30 @@ func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error { return nil } +// NewSchedulerConfig initializes a new kube-scheduler config file +func NewSchedulerConfig() *SchedulerConfig { + schedConfig := new(SchedulerConfig) + schedConfig.APIVersion = "kubescheduler.config.k8s.io/v1alpha1" + schedConfig.Kind = "KubeSchedulerConfiguration" + schedConfig.ClientConnection = ClientConnectionConfig{} + schedConfig.ClientConnection.Kubeconfig = defaultKubeConfig + return schedConfig +} + // buildPod is responsible for constructing the pod specification -func (b *KubeSchedulerBuilder) buildPod() (*v1.Pod, error) { +func (b *KubeSchedulerBuilder) buildPod(useConfigFile bool) (*v1.Pod, error) { c := b.Cluster.Spec.KubeScheduler flags, err := flagbuilder.BuildFlagsList(c) if err != nil { return nil, fmt.Errorf("error building kube-scheduler flags: %v", err) } - // Add kubeconfig flag - flags = append(flags, "--kubeconfig="+"/var/lib/kube-scheduler/kubeconfig") + if useConfigFile { + flags = append(flags, "--config="+"/var/lib/kube-scheduler/config.yaml") + } else { + // Add kubeconfig flag + flags = append(flags, "--kubeconfig="+defaultKubeConfig) + } if c.UsePolicyConfigMap != nil { flags = append(flags, "--policy-configmap=scheduler-policy", "--policy-configmap-namespace=kube-system") diff --git a/nodeup/pkg/model/kube_scheduler_test.go b/nodeup/pkg/model/kube_scheduler_test.go new file mode 100644 index 0000000000..d52936462d --- /dev/null +++ b/nodeup/pkg/model/kube_scheduler_test.go @@ -0,0 +1,69 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "bytes" + "testing" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/configbuilder" +) + +func TestParseDefault(t *testing.T) { + expect := []byte( + `apiVersion: kubescheduler.config.k8s.io/v1alpha1 +kind: KubeSchedulerConfiguration +clientConnection: + kubeconfig: /var/lib/kube-scheduler/kubeconfig +`) + + s := &kops.KubeSchedulerConfig{} + + yaml, err := configbuilder.BuildConfigYaml(s, NewSchedulerConfig()) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + if !bytes.Equal(yaml, expect) { + t.Errorf("unexpected result: \n%s, expected: \n%s", yaml, expect) + } +} + +func TestParse(t *testing.T) { + expect := []byte( + `apiVersion: kubescheduler.config.k8s.io/v1alpha1 +kind: KubeSchedulerConfiguration +clientConnection: + burst: 100 + kubeconfig: /var/lib/kube-scheduler/kubeconfig + qps: 3.1 +`) + qps, _ := resource.ParseQuantity("3.1") + + s := &kops.KubeSchedulerConfig{Qps: &qps, Burst: 100} + + yaml, err := configbuilder.BuildConfigYaml(s, NewSchedulerConfig()) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + if !bytes.Equal(yaml, expect) { + t.Errorf("unexpected result: \n%s, expected: \n%s", yaml, expect) + } +} diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index 0d50c02aa1..9abd3d4451 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -609,6 +609,10 @@ type KubeSchedulerConfig struct { // which has been supported as far back as Kubernetes 1.7. The default depends on the version and the cloud provider // as outlined: https://kubernetes.io/docs/concepts/storage/storage-limits/ MaxPersistentVolumes *int32 `json:"maxPersistentVolumes,omitempty"` + // Qps sets the maximum qps to send to apiserver after the burst quota is exhausted + Qps *resource.Quantity `json:"qps,omitempty" configfile:"ClientConnection.QPS"` + // Burst sets the maximum qps to send to apiserver after the burst quota is exhausted + Burst int32 `json:"burst,omitempty" configfile:"ClientConnection.Burst"` } // LeaderElectionConfiguration defines the configuration of leader election diff --git a/pkg/apis/kops/v1alpha1/componentconfig.go b/pkg/apis/kops/v1alpha1/componentconfig.go index 2b7fa7c587..dfcb28fe21 100644 --- a/pkg/apis/kops/v1alpha1/componentconfig.go +++ b/pkg/apis/kops/v1alpha1/componentconfig.go @@ -609,6 +609,10 @@ type KubeSchedulerConfig struct { // which has been supported as far back as Kubernetes 1.7. The default depends on the version and the cloud provider // as outlined: https://kubernetes.io/docs/concepts/storage/storage-limits/ MaxPersistentVolumes *int32 `json:"maxPersistentVolumes,omitempty"` + // Qps sets the maximum qps to send to apiserver after the burst quota is exhausted + Qps *resource.Quantity `json:"qps,omitempty" configfile:"ClientConnection.QPS"` + // Burst sets the maximum qps to send to apiserver after the burst quota is exhausted + Burst int32 `json:"burst,omitempty" configfile:"ClientConnection.Burst"` } // LeaderElectionConfiguration defines the configuration of leader election diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index 3bf36384dd..d1b21da2e5 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -3656,6 +3656,8 @@ func autoConvert_v1alpha1_KubeSchedulerConfig_To_kops_KubeSchedulerConfig(in *Ku out.UsePolicyConfigMap = in.UsePolicyConfigMap out.FeatureGates = in.FeatureGates out.MaxPersistentVolumes = in.MaxPersistentVolumes + out.Qps = in.Qps + out.Burst = in.Burst return nil } @@ -3680,6 +3682,8 @@ func autoConvert_kops_KubeSchedulerConfig_To_v1alpha1_KubeSchedulerConfig(in *ko out.UsePolicyConfigMap = in.UsePolicyConfigMap out.FeatureGates = in.FeatureGates out.MaxPersistentVolumes = in.MaxPersistentVolumes + out.Qps = in.Qps + out.Burst = in.Burst return nil } diff --git a/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go index c27185539a..ba8f86806e 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go @@ -2415,6 +2415,11 @@ func (in *KubeSchedulerConfig) DeepCopyInto(out *KubeSchedulerConfig) { *out = new(int32) **out = **in } + if in.Qps != nil { + in, out := &in.Qps, &out.Qps + x := (*in).DeepCopy() + *out = &x + } return } diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go index 25ef37748a..091058a4a2 100644 --- a/pkg/apis/kops/v1alpha2/componentconfig.go +++ b/pkg/apis/kops/v1alpha2/componentconfig.go @@ -610,6 +610,10 @@ type KubeSchedulerConfig struct { // which has been supported as far back as Kubernetes 1.7. The default depends on the version and the cloud provider // as outlined: https://kubernetes.io/docs/concepts/storage/storage-limits/ MaxPersistentVolumes *int32 `json:"maxPersistentVolumes,omitempty"` + // Qps sets the maximum qps to send to apiserver after the burst quota is exhausted + Qps *resource.Quantity `json:"qps,omitempty"` + // Burst sets the maximum qps to send to apiserver after the burst quota is exhausted + Burst int32 `json:"burst,omitempty"` } // LeaderElectionConfiguration defines the configuration of leader election diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 6668f16534..3f41ed8b0a 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -3926,6 +3926,8 @@ func autoConvert_v1alpha2_KubeSchedulerConfig_To_kops_KubeSchedulerConfig(in *Ku out.UsePolicyConfigMap = in.UsePolicyConfigMap out.FeatureGates = in.FeatureGates out.MaxPersistentVolumes = in.MaxPersistentVolumes + out.Qps = in.Qps + out.Burst = in.Burst return nil } @@ -3950,6 +3952,8 @@ func autoConvert_kops_KubeSchedulerConfig_To_v1alpha2_KubeSchedulerConfig(in *ko out.UsePolicyConfigMap = in.UsePolicyConfigMap out.FeatureGates = in.FeatureGates out.MaxPersistentVolumes = in.MaxPersistentVolumes + out.Qps = in.Qps + out.Burst = in.Burst return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index 2994fda089..839dc262f9 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -2486,6 +2486,11 @@ func (in *KubeSchedulerConfig) DeepCopyInto(out *KubeSchedulerConfig) { *out = new(int32) **out = **in } + if in.Qps != nil { + in, out := &in.Qps, &out.Qps + x := (*in).DeepCopy() + *out = &x + } return } diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index f54d81b551..916c6c4c76 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -2668,6 +2668,11 @@ func (in *KubeSchedulerConfig) DeepCopyInto(out *KubeSchedulerConfig) { *out = new(int32) **out = **in } + if in.Qps != nil { + in, out := &in.Qps, &out.Qps + x := (*in).DeepCopy() + *out = &x + } return } diff --git a/pkg/configbuilder/BUILD.bazel b/pkg/configbuilder/BUILD.bazel new file mode 100644 index 0000000000..9c18771803 --- /dev/null +++ b/pkg/configbuilder/BUILD.bazel @@ -0,0 +1,21 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["buildconfigfile.go"], + importpath = "k8s.io/kops/pkg/configbuilder", + visibility = ["//visibility:public"], + deps = [ + "//pkg/apis/kops:go_default_library", + "//util/pkg/reflectutils:go_default_library", + "//vendor/gopkg.in/yaml.v2:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["buildconfigfile_test.go"], + embed = [":go_default_library"], +) diff --git a/pkg/configbuilder/buildconfigfile.go b/pkg/configbuilder/buildconfigfile.go new file mode 100644 index 0000000000..f65d47bb85 --- /dev/null +++ b/pkg/configbuilder/buildconfigfile.go @@ -0,0 +1,111 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package configbuilder + +import ( + "fmt" + + "reflect" + "strconv" + "strings" + + "gopkg.in/yaml.v2" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/klog" + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/util/pkg/reflectutils" +) + +// BuildConfigYaml reflects the options interface and extracts the parameters for the config file +func BuildConfigYaml(options *kops.KubeSchedulerConfig, target interface{}) ([]byte, error) { + walker := func(path string, field *reflect.StructField, val reflect.Value) error { + if field == nil { + klog.V(8).Infof("ignoring non-field: %s", path) + return nil + } + tag := field.Tag.Get("configfile") + if tag == "" { + klog.V(4).Infof("not writing field with no configfile tag: %s", path) + // We want to descend - it could be a structure containing flags + return nil + } + if tag == "-" { + klog.V(4).Infof("skipping field with %q configfile tag: %s", tag, path) + return reflectutils.SkipReflection + } + + tokens := strings.Split(tag, ",") + + flagName := tokens[0] + + targetValue, error := getValueFromStruct(flagName, target) + if error != nil { + return fmt.Errorf("conversion error for field %s: %s", flagName, error) + } + // We do have to do this, even though the recursive walk will do it for us + // because when we descend we won't have `field` set + if val.Kind() == reflect.Ptr { + if val.IsNil() { + return nil + } + } + switch v := val.Interface().(type) { + case *resource.Quantity: + floatVal, err := strconv.ParseFloat(v.AsDec().String(), 64) + if err != nil { + return fmt.Errorf("unable to convert from Quantity %v to float", v) + } + targetValue.Set(reflect.ValueOf(&floatVal)) + default: + targetValue.Set(val) + } + + return reflectutils.SkipReflection + + } + + err := reflectutils.ReflectRecursive(reflect.ValueOf(options), walker) + if err != nil { + return nil, fmt.Errorf("BuildFlagsList to reflect value: %s", err) + } + + configFile, err := yaml.Marshal(target) + if err != nil { + return nil, err + } + + return configFile, nil +} + +func getValueFromStruct(keyWithDots string, object interface{}) (*reflect.Value, error) { + keySlice := strings.Split(keyWithDots, ".") + v := reflect.ValueOf(object) + // iterate through field names, ignoring the first name as it might be the current instance name + // you can make it recursive also if want to support types like slice,map etc along with struct + for _, key := range keySlice { + for v.Kind() == reflect.Ptr { + v = v.Elem() + } + // we only accept structs + if v.Kind() != reflect.Struct { + return nil, fmt.Errorf("only accepts structs; got %T", v) + } + v = v.FieldByName(key) + } + + return &v, nil +} diff --git a/pkg/configbuilder/buildconfigfile_test.go b/pkg/configbuilder/buildconfigfile_test.go new file mode 100644 index 0000000000..ff23af100f --- /dev/null +++ b/pkg/configbuilder/buildconfigfile_test.go @@ -0,0 +1,67 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package configbuilder + +import ( + "testing" +) + +// ClientConnectionConfig is used by kube-scheduler to talk to the api server +type DummyNestedStruct struct { + Name *string `yaml:"name,omitempty"` + QPS *float64 `yaml:"qps,omitempty"` +} + +// SchedulerConfig is used to generate the config file +type DummyStruct struct { + ClientConnection *DummyNestedStruct `yaml:"clientConnection,omitempty"` +} + +func TestGetStructVal(t *testing.T) { + str := "test" + s := &DummyStruct{ + ClientConnection: &DummyNestedStruct{ + Name: &str, + }, + } + v, err := getValueFromStruct("ClientConnection.Name", s) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + inStruct := v.Elem().String() + if inStruct != str { + t.Errorf("unexpected value: %s, %s, expected: %s", inStruct, err, str) + } + +} + +func TestWrongStructField(t *testing.T) { + str := "test" + s := &DummyStruct{ + ClientConnection: &DummyNestedStruct{ + Name: &str, + }, + } + v, err := getValueFromStruct("ClientConnection.NotExistent", s) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + if v.IsValid() { + t.Errorf("unexpected Valid value from non-existent field lookup") + } + +}