Updates based on feedback

This commit is contained in:
Roberto Rodriguez Alcala 2020-01-25 12:05:35 -08:00
parent f5c8e46ae2
commit 1298d541cc
18 changed files with 155 additions and 160 deletions

View File

@ -1609,11 +1609,6 @@ spec:
kubeScheduler:
description: KubeSchedulerConfig is the configuration for the kube-scheduler
properties:
burst:
description: Burst sets the maximum qps to send to apiserver after
the burst quota is exhausted
format: int32
type: integer
featureGates:
additionalProperties:
type: string
@ -1623,9 +1618,6 @@ spec:
image:
description: Image is the docker image to use
type: string
kubeConfig:
description: KubeConfig overrides the default kubeconfig path.
type: string
leaderElection:
description: LeaderElection defines the configuration of leader
election client.
@ -1685,10 +1677,6 @@ spec:
and the cloud provider as outlined: https://kubernetes.io/docs/concepts/storage/storage-limits/'
format: int32
type: integer
qps:
description: Qps sets the maximum qps to send to apiserver after
the burst quota is exhausted
type: string
usePolicyConfigMap:
description: UsePolicyConfigMap enable setting the scheduler policy
from a configmap

View File

@ -89,6 +89,7 @@ go_test(
"docker_test.go",
"kube_apiserver_test.go",
"kube_proxy_test.go",
"kube_scheduler_test.go",
"kubelet_test.go",
"protokube_test.go",
],
@ -98,6 +99,7 @@ go_test(
"//nodeup/pkg/distros:go_default_library",
"//pkg/apis/kops:go_default_library",
"//pkg/apis/nodeup:go_default_library",
"//pkg/configbuilder:go_default_library",
"//pkg/flagbuilder:go_default_library",
"//pkg/testutils:go_default_library",
"//upup/pkg/fi:go_default_library",

View File

@ -35,6 +35,21 @@ import (
"k8s.io/apimachinery/pkg/util/intstr"
)
// ClientConnectionConfig is used by kube-scheduler to talk to the api server
type ClientConnectionConfig struct {
Burst int32 `yaml:"burst,omitempty"`
Kubeconfig *string `yaml:"kubeconfig"`
QPS *float64 `yaml:"qps,omitempty"`
}
// SchedulerConfig is used to generate the config file
type SchedulerConfig struct {
APIVersion string `yaml:"apiVersion"`
Kind string `yaml:"kind"`
BindTimeoutSeconds *int64 `yaml:"bindTimeoutSeconds,omitempty"`
ClientConnection *ClientConnectionConfig `yaml:"clientConnection,omitempty"`
}
// KubeSchedulerBuilder install kube-scheduler
type KubeSchedulerBuilder struct {
*NodeupModelContext
@ -49,9 +64,9 @@ func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error {
if !b.IsMaster {
return nil
}
useConfigFile := b.IsKubernetesGTE("1.11")
{
pod, err := b.buildPod()
pod, err := b.buildPod(useConfigFile)
if err != nil {
return fmt.Errorf("error building kube-scheduler pod: %v", err)
}
@ -81,18 +96,14 @@ func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error {
Mode: s("0400"),
})
}
if b.Cluster.Spec.KubeScheduler.KubeConfig == nil {
b.Cluster.Spec.KubeScheduler.KubeConfig = &defaultKubeConfig
}
{
config, err := configbuilder.BuildConfigYaml(b.Cluster.Spec.KubeScheduler)
if useConfigFile {
config, err := configbuilder.BuildConfigYaml(b.Cluster.Spec.KubeScheduler, NewSchedulerConfig())
if err != nil {
return err
}
c.AddTask(&nodetasks.File{
Path: "/var/lib/kube-scheduler/config",
Path: "/var/lib/kube-scheduler/config.yaml",
Contents: fi.NewBytesResource(config),
Type: nodetasks.FileType_File,
Mode: s("0400"),
@ -112,16 +123,30 @@ func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error {
return nil
}
// NewSchedulerConfig initializes a new kube-scheduler config file
func NewSchedulerConfig() *SchedulerConfig {
schedConfig := new(SchedulerConfig)
schedConfig.APIVersion = "kubescheduler.config.k8s.io/v1alpha1"
schedConfig.Kind = "KubeSchedulerConfiguration"
schedConfig.ClientConnection = new(ClientConnectionConfig)
schedConfig.ClientConnection.Kubeconfig = &defaultKubeConfig
return schedConfig
}
// buildPod is responsible for constructing the pod specification
func (b *KubeSchedulerBuilder) buildPod() (*v1.Pod, error) {
func (b *KubeSchedulerBuilder) buildPod(useConfigFile bool) (*v1.Pod, error) {
c := b.Cluster.Spec.KubeScheduler
flags, err := flagbuilder.BuildFlagsList(c)
if err != nil {
return nil, fmt.Errorf("error building kube-scheduler flags: %v", err)
}
if useConfigFile {
flags = append(flags, "--config="+"/var/lib/kube-scheduler/config.yaml")
} else {
// Add kubeconfig flag
flags = append(flags, "--config="+"/var/lib/kube-scheduler/config")
flags = append(flags, "--config="+defaultKubeConfig)
}
if c.UsePolicyConfigMap != nil {
flags = append(flags, "--policy-configmap=scheduler-policy", "--policy-configmap-namespace=kube-system")

View File

@ -0,0 +1,48 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package model
import (
"bytes"
"testing"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/configbuilder"
)
func TestParseBasic(t *testing.T) {
expect := []byte(
`apiVersion: kubescheduler.config.k8s.io/v1alpha1
kind: KubeSchedulerConfiguration
clientConnection:
kubeconfig: /var/lib/kube-scheduler/kubeconfig
qps: 3.1
`)
qps, _ := resource.ParseQuantity("3.1")
s := &kops.KubeSchedulerConfig{Qps: &qps}
yaml, err := configbuilder.BuildConfigYaml(s, NewSchedulerConfig())
if err != nil {
t.Errorf("unexpected error: %s", err)
}
if !bytes.Equal(yaml, expect) {
t.Errorf("unexpected result: \n%s, expected: \n%s", yaml, expect)
}
}

View File

@ -612,9 +612,7 @@ type KubeSchedulerConfig struct {
// Qps sets the maximum qps to send to apiserver after the burst quota is exhausted
Qps *resource.Quantity `json:"qps,omitempty" configfile:"ClientConnection.QPS"`
// Burst sets the maximum qps to send to apiserver after the burst quota is exhausted
Burst *int32 `json:"burst,omitempty" configfile:"ClientConnection.Burst"`
// KubeConfig overrides the default kubeconfig path.
KubeConfig *string `json:"kubeConfig,omitempty" configfile:"ClientConnection.Kubeconfig"`
Burst int32 `json:"burst,omitempty" configfile:"ClientConnection.Burst"`
}
// LeaderElectionConfiguration defines the configuration of leader election

View File

@ -609,12 +609,6 @@ type KubeSchedulerConfig struct {
// which has been supported as far back as Kubernetes 1.7. The default depends on the version and the cloud provider
// as outlined: https://kubernetes.io/docs/concepts/storage/storage-limits/
MaxPersistentVolumes *int32 `json:"maxPersistentVolumes,omitempty"`
// Qps sets the maximum qps to send to apiserver after the burst quota is exhausted
Qps *resource.Quantity `json:"qps,omitempty" configfile:"ClientConnection.QPS"`
// Burst sets the maximum qps to send to apiserver after the burst quota is exhausted
Burst *int32 `json:"burst,omitempty" configfile:"ClientConnection.Burst"`
// KubeConfig overrides the default kubeconfig path.
KubeConfig *string `json:"kubeConfig,omitempty" configfile:"ClientConnection.Kubeconfig"`
}
// LeaderElectionConfiguration defines the configuration of leader election

View File

@ -38,6 +38,10 @@ func Convert_v1alpha1_BastionSpec_To_kops_BastionSpec(in *BastionSpec, out *kops
return nil
}
func Convert_kops_KubeSchedulerConfig_To_v1alpha1_KubeSchedulerConfig(in *kops.KubeSchedulerConfig, out *KubeSchedulerConfig, s conversion.Scope) error {
return autoConvert_kops_KubeSchedulerConfig_To_v1alpha1_KubeSchedulerConfig(in, out, s)
}
func Convert_kops_BastionSpec_To_v1alpha1_BastionSpec(in *kops.BastionSpec, out *BastionSpec, s conversion.Scope) error {
out.PublicName = in.BastionPublicName
out.IdleTimeout = in.IdleTimeoutSeconds

View File

@ -813,6 +813,11 @@ func RegisterConversions(s *runtime.Scheme) error {
}); err != nil {
return err
}
if err := s.AddConversionFunc((*kops.KubeSchedulerConfig)(nil), (*KubeSchedulerConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_kops_KubeSchedulerConfig_To_v1alpha1_KubeSchedulerConfig(a.(*kops.KubeSchedulerConfig), b.(*KubeSchedulerConfig), scope)
}); err != nil {
return err
}
if err := s.AddConversionFunc((*kops.TopologySpec)(nil), (*TopologySpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_kops_TopologySpec_To_v1alpha1_TopologySpec(a.(*kops.TopologySpec), b.(*TopologySpec), scope)
}); err != nil {
@ -3656,9 +3661,6 @@ func autoConvert_v1alpha1_KubeSchedulerConfig_To_kops_KubeSchedulerConfig(in *Ku
out.UsePolicyConfigMap = in.UsePolicyConfigMap
out.FeatureGates = in.FeatureGates
out.MaxPersistentVolumes = in.MaxPersistentVolumes
out.Qps = in.Qps
out.Burst = in.Burst
out.KubeConfig = in.KubeConfig
return nil
}
@ -3683,17 +3685,11 @@ func autoConvert_kops_KubeSchedulerConfig_To_v1alpha1_KubeSchedulerConfig(in *ko
out.UsePolicyConfigMap = in.UsePolicyConfigMap
out.FeatureGates = in.FeatureGates
out.MaxPersistentVolumes = in.MaxPersistentVolumes
out.Qps = in.Qps
out.Burst = in.Burst
out.KubeConfig = in.KubeConfig
// WARNING: in.Qps requires manual conversion: does not exist in peer-type
// WARNING: in.Burst requires manual conversion: does not exist in peer-type
return nil
}
// Convert_kops_KubeSchedulerConfig_To_v1alpha1_KubeSchedulerConfig is an autogenerated conversion function.
func Convert_kops_KubeSchedulerConfig_To_v1alpha1_KubeSchedulerConfig(in *kops.KubeSchedulerConfig, out *KubeSchedulerConfig, s conversion.Scope) error {
return autoConvert_kops_KubeSchedulerConfig_To_v1alpha1_KubeSchedulerConfig(in, out, s)
}
func autoConvert_v1alpha1_KubeletConfigSpec_To_kops_KubeletConfigSpec(in *KubeletConfigSpec, out *kops.KubeletConfigSpec, s conversion.Scope) error {
out.APIServers = in.APIServers
out.AnonymousAuth = in.AnonymousAuth

View File

@ -2415,21 +2415,6 @@ func (in *KubeSchedulerConfig) DeepCopyInto(out *KubeSchedulerConfig) {
*out = new(int32)
**out = **in
}
if in.Qps != nil {
in, out := &in.Qps, &out.Qps
x := (*in).DeepCopy()
*out = &x
}
if in.Burst != nil {
in, out := &in.Burst, &out.Burst
*out = new(int32)
**out = **in
}
if in.KubeConfig != nil {
in, out := &in.KubeConfig, &out.KubeConfig
*out = new(string)
**out = **in
}
return
}

View File

@ -7,6 +7,7 @@ go_library(
"cluster.go",
"componentconfig.go",
"containerdconfig.go",
"conversion.go",
"defaults.go",
"doc.go",
"dockerconfig.go",

View File

@ -610,12 +610,6 @@ type KubeSchedulerConfig struct {
// which has been supported as far back as Kubernetes 1.7. The default depends on the version and the cloud provider
// as outlined: https://kubernetes.io/docs/concepts/storage/storage-limits/
MaxPersistentVolumes *int32 `json:"maxPersistentVolumes,omitempty"`
// Qps sets the maximum qps to send to apiserver after the burst quota is exhausted
Qps *resource.Quantity `json:"qps,omitempty" configfile:"ClientConnection.QPS"`
// Burst sets the maximum qps to send to apiserver after the burst quota is exhausted
Burst *int32 `json:"burst,omitempty" configfile:"ClientConnection.Burst"`
// KubeConfig overrides the default kubeconfig path.
KubeConfig *string `json:"kubeConfig,omitempty" configfile:"ClientConnection.Kubeconfig"`
}
// LeaderElectionConfiguration defines the configuration of leader election

View File

@ -0,0 +1,26 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1alpha2
import (
"k8s.io/apimachinery/pkg/conversion"
"k8s.io/kops/pkg/apis/kops"
)
func Convert_kops_KubeSchedulerConfig_To_v1alpha2_KubeSchedulerConfig(in *kops.KubeSchedulerConfig, out *KubeSchedulerConfig, s conversion.Scope) error {
return autoConvert_kops_KubeSchedulerConfig_To_v1alpha2_KubeSchedulerConfig(in, out, s)
}

View File

@ -863,6 +863,11 @@ func RegisterConversions(s *runtime.Scheme) error {
}); err != nil {
return err
}
if err := s.AddConversionFunc((*kops.KubeSchedulerConfig)(nil), (*KubeSchedulerConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_kops_KubeSchedulerConfig_To_v1alpha2_KubeSchedulerConfig(a.(*kops.KubeSchedulerConfig), b.(*KubeSchedulerConfig), scope)
}); err != nil {
return err
}
return nil
}
@ -3926,9 +3931,6 @@ func autoConvert_v1alpha2_KubeSchedulerConfig_To_kops_KubeSchedulerConfig(in *Ku
out.UsePolicyConfigMap = in.UsePolicyConfigMap
out.FeatureGates = in.FeatureGates
out.MaxPersistentVolumes = in.MaxPersistentVolumes
out.Qps = in.Qps
out.Burst = in.Burst
out.KubeConfig = in.KubeConfig
return nil
}
@ -3953,17 +3955,11 @@ func autoConvert_kops_KubeSchedulerConfig_To_v1alpha2_KubeSchedulerConfig(in *ko
out.UsePolicyConfigMap = in.UsePolicyConfigMap
out.FeatureGates = in.FeatureGates
out.MaxPersistentVolumes = in.MaxPersistentVolumes
out.Qps = in.Qps
out.Burst = in.Burst
out.KubeConfig = in.KubeConfig
// WARNING: in.Qps requires manual conversion: does not exist in peer-type
// WARNING: in.Burst requires manual conversion: does not exist in peer-type
return nil
}
// Convert_kops_KubeSchedulerConfig_To_v1alpha2_KubeSchedulerConfig is an autogenerated conversion function.
func Convert_kops_KubeSchedulerConfig_To_v1alpha2_KubeSchedulerConfig(in *kops.KubeSchedulerConfig, out *KubeSchedulerConfig, s conversion.Scope) error {
return autoConvert_kops_KubeSchedulerConfig_To_v1alpha2_KubeSchedulerConfig(in, out, s)
}
func autoConvert_v1alpha2_KubeletConfigSpec_To_kops_KubeletConfigSpec(in *KubeletConfigSpec, out *kops.KubeletConfigSpec, s conversion.Scope) error {
out.APIServers = in.APIServers
out.AnonymousAuth = in.AnonymousAuth

View File

@ -2486,21 +2486,6 @@ func (in *KubeSchedulerConfig) DeepCopyInto(out *KubeSchedulerConfig) {
*out = new(int32)
**out = **in
}
if in.Qps != nil {
in, out := &in.Qps, &out.Qps
x := (*in).DeepCopy()
*out = &x
}
if in.Burst != nil {
in, out := &in.Burst, &out.Burst
*out = new(int32)
**out = **in
}
if in.KubeConfig != nil {
in, out := &in.KubeConfig, &out.KubeConfig
*out = new(string)
**out = **in
}
return
}

View File

@ -2673,16 +2673,6 @@ func (in *KubeSchedulerConfig) DeepCopyInto(out *KubeSchedulerConfig) {
x := (*in).DeepCopy()
*out = &x
}
if in.Burst != nil {
in, out := &in.Burst, &out.Burst
*out = new(int32)
**out = **in
}
if in.KubeConfig != nil {
in, out := &in.KubeConfig, &out.KubeConfig
*out = new(string)
**out = **in
}
return
}

View File

@ -6,6 +6,7 @@ go_library(
importpath = "k8s.io/kops/pkg/configbuilder",
visibility = ["//visibility:public"],
deps = [
"//pkg/apis/kops:go_default_library",
"//util/pkg/reflectutils:go_default_library",
"//vendor/gopkg.in/yaml.v2:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
@ -17,8 +18,4 @@ go_test(
name = "go_default_test",
srcs = ["buildconfigfile_test.go"],
embed = [":go_default_library"],
deps = [
"//pkg/apis/kops:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
],
)

View File

@ -18,6 +18,7 @@ package configbuilder
import (
"fmt"
"reflect"
"strconv"
"strings"
@ -25,32 +26,12 @@ import (
"gopkg.in/yaml.v2"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/klog"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/util/pkg/reflectutils"
)
// ClientConnectionConfig is used by kube-scheduler to talk to the api server
type ClientConnectionConfig struct {
Burst *int32 `yaml:"burst,omitempty"`
Kubeconfig *string `yaml:"kubeconfig"`
QPS *float64 `yaml:"qps,omitempty"`
}
// SchedulerConfig is used to generate the config file
type SchedulerConfig struct {
APIVersion string `yaml:"apiVersion"`
Kind string `yaml:"kind"`
BindTimeoutSeconds *int64 `yaml:"bindTimeoutSeconds,omitempty"`
ClientConnection *ClientConnectionConfig `yaml:"clientConnection,omitempty"`
}
// BuildConfigYaml reflects the options interface and extracts the parameters for the config file
func BuildConfigYaml(options interface{}) ([]byte, error) {
schedConfig := new(SchedulerConfig)
schedConfig.APIVersion = "kubescheduler.config.k8s.io/v1alpha1"
schedConfig.Kind = "KubeSchedulerConfiguration"
schedConfig.ClientConnection = new(ClientConnectionConfig)
func BuildConfigYaml(options *kops.KubeSchedulerConfig, target interface{}) ([]byte, error) {
walker := func(path string, field *reflect.StructField, val reflect.Value) error {
if field == nil {
klog.V(8).Infof("ignoring non-field: %s", path)
@ -71,7 +52,7 @@ func BuildConfigYaml(options interface{}) ([]byte, error) {
flagName := tokens[0]
targetValue, error := getValueFromStruct(flagName, schedConfig)
targetValue, error := getValueFromStruct(flagName, target)
if error != nil {
return fmt.Errorf("conversion error for field %s: %s", flagName, error)
}
@ -102,7 +83,7 @@ func BuildConfigYaml(options interface{}) ([]byte, error) {
return nil, fmt.Errorf("BuildFlagsList to reflect value: %s", err)
}
configFile, err := yaml.Marshal(schedConfig)
configFile, err := yaml.Marshal(target)
if err != nil {
return nil, err
}
@ -110,7 +91,7 @@ func BuildConfigYaml(options interface{}) ([]byte, error) {
return configFile, nil
}
func getValueFromStruct(keyWithDots string, object *SchedulerConfig) (*reflect.Value, error) {
func getValueFromStruct(keyWithDots string, object interface{}) (*reflect.Value, error) {
keySlice := strings.Split(keyWithDots, ".")
v := reflect.ValueOf(object)
// iterate through field names, ignoring the first name as it might be the current instance name

View File

@ -17,43 +17,28 @@ limitations under the License.
package configbuilder
import (
"bytes"
"testing"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/kops/pkg/apis/kops"
)
func TestParseBasic(t *testing.T) {
expect := []byte(
`apiVersion: kubescheduler.config.k8s.io/v1alpha1
kind: KubeSchedulerConfiguration
clientConnection:
kubeconfig: null
qps: 3.1
`)
qps, _ := resource.ParseQuantity("3.1")
// ClientConnectionConfig is used by kube-scheduler to talk to the api server
type DummyNestedStruct struct {
Name *string `yaml:"name,omitempty"`
QPS *float64 `yaml:"qps,omitempty"`
}
s := &kops.KubeSchedulerConfig{Qps: &qps}
yaml, err := BuildConfigYaml(s)
if err != nil {
t.Errorf("unexpected error: %s", err)
}
if !bytes.Equal(yaml, expect) {
t.Errorf("unexpected result: \n%s, expected: \n%s", expect, yaml)
}
// SchedulerConfig is used to generate the config file
type DummyStruct struct {
ClientConnection *DummyNestedStruct `yaml:"clientConnection,omitempty"`
}
func TestGetStructVal(t *testing.T) {
str := "test"
s := &SchedulerConfig{
ClientConnection: &ClientConnectionConfig{
Kubeconfig: &str,
s := &DummyStruct{
ClientConnection: &DummyNestedStruct{
Name: &str,
},
}
v, err := getValueFromStruct("ClientConnection.Kubeconfig", s)
v, err := getValueFromStruct("ClientConnection.Name", s)
if err != nil {
t.Errorf("unexpected error: %s", err)
}
@ -66,9 +51,9 @@ func TestGetStructVal(t *testing.T) {
func TestWrongStructField(t *testing.T) {
str := "test"
s := &SchedulerConfig{
ClientConnection: &ClientConnectionConfig{
Kubeconfig: &str,
s := &DummyStruct{
ClientConnection: &DummyNestedStruct{
Name: &str,
},
}
v, err := getValueFromStruct("ClientConnection.NotExistent", s)