kops/pkg/model/components/kubescheduler/model.go

/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubescheduler

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"

	"k8s.io/apimachinery/pkg/api/resource"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/klog/v2"
	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/assets"
	"k8s.io/kops/pkg/kubemanifest"
	"k8s.io/kops/pkg/model"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/util/pkg/reflectutils"
	"sigs.k8s.io/yaml"
)

// KubeSchedulerConfigPath is the path where we write the kube-scheduler config file (on the control-plane nodes)
const KubeSchedulerConfigPath = "/var/lib/kube-scheduler/config.yaml"

// Kubeconfig is the path where we write the kube-scheduler kubeconfig file (on the control-plane nodes)
const KubeConfigPath = "/var/lib/kube-scheduler/kubeconfig"

// KubeSchedulerBuilder builds the configuration file for kube-scheduler
type KubeSchedulerBuilder struct {
	*model.KopsModelContext
	Lifecycle    fi.Lifecycle
	AssetBuilder *assets.AssetBuilder
}

var _ fi.CloudupModelBuilder = &KubeSchedulerBuilder{}

// Build creates the tasks relating to kube-scheduler
func (b *KubeSchedulerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
	configYAML, err := b.buildSchedulerConfig()
	if err != nil {
		return err
	}

	b.AssetBuilder.StaticFiles = append(b.AssetBuilder.StaticFiles, &assets.StaticFile{
		Path:    KubeSchedulerConfigPath,
		Content: string(configYAML),
		Roles:   []kops.InstanceGroupRole{kops.InstanceGroupRoleControlPlane, kops.InstanceGroupRoleAPIServer},
	})
	return nil
}

func (b *KubeSchedulerBuilder) buildSchedulerConfig() ([]byte, error) {
	var matches []*kubemanifest.Object
	for _, additionalObject := range b.AdditionalObjects {
		gvk := additionalObject.GroupVersionKind()
		if gvk.Group != "kubescheduler.config.k8s.io" {
			continue
		}
		if gvk.Kind != "KubeSchedulerConfiguration" {
			continue
		}
		matches = append(matches, additionalObject)
	}

	if len(matches) > 1 {
		return nil, fmt.Errorf("found multiple KubeSchedulerConfiguration objects in cluster configuration; expected at most one")
	}

	var config *unstructured.Unstructured
	if len(matches) == 1 {
		config = matches[0].ToUnstructured()
	} else {
		config = &unstructured.Unstructured{}
		config.SetKind("KubeSchedulerConfiguration")
		config.SetAPIVersion("kubescheduler.config.k8s.io/v1")
		if b.IsKubernetesLT("1.25") {
			config.SetAPIVersion("kubescheduler.config.k8s.io/v1beta2")
		}
		// We need to store the object, because we are often called repeatedly (until we converge)
		b.AdditionalObjects = append(b.AdditionalObjects, kubemanifest.NewObject(config.Object))
	}

	// TODO: Handle different versions? e.g. gvk := config.GroupVersionKind()

	if err := unstructured.SetNestedField(config.Object, KubeConfigPath, "clientConnection", "kubeconfig"); err != nil {
		return nil, fmt.Errorf("error setting clientConnection.kubeconfig in kube-scheduler configuration: %w", err)
	}

	kubeScheduler := b.Cluster.Spec.KubeScheduler
	if kubeScheduler != nil {
		if err := MapToUnstructured(kubeScheduler, config); err != nil {
			return nil, err
		}
	}

	configYAML, err := yaml.Marshal(config)
	if err != nil {
		return nil, err
	}
	return configYAML, nil
}

// MapToUnstructured reflects the options interface and extracts the parameters for the config file
func MapToUnstructured(options interface{}, target *unstructured.Unstructured) error {
	setValue := func(targetPath string, val interface{}) error {
		fields := strings.Split(targetPath, ".")
		// Cannot use unstructured.SetNestedField, because it fails with e.g. "cannot deep copy int32"
		parent := target.Object
		for i := 0; i < len(fields)-1; i++ {
			v := parent[fields[i]]
			if v == nil {
				v = make(map[string]interface{})
				parent[fields[i]] = v
			}
			m, ok := v.(map[string]interface{})
			if !ok {
				return fmt.Errorf("value was not a map at position %d in %s", i, targetPath)
			}
			parent = m
		}
		parent[fields[len(fields)-1]] = val
		return nil
	}

	walker := func(path *reflectutils.FieldPath, field *reflect.StructField, val reflect.Value) error {
		if field == nil {
			klog.V(8).Infof("ignoring non-field: %s", path)
			return nil
		}

		tag := field.Tag.Get("config")
		if tag == "" {
			klog.V(4).Infof("not writing field with no config tag: %s", path)
			// We want to descend - it could be a structure containing flags
			return nil
		}
		if tag == "-" {
			klog.V(4).Infof("skipping field with %q config tag: %s", tag, path)
			return reflectutils.SkipReflection
		}

		tagTokens := strings.Split(tag, ",")
		omitEmpty := false
		for _, token := range tagTokens {
			if token == "omitempty" {
				omitEmpty = true
			}
		}
		targetPath := tagTokens[0]

		// We do have to do this, even though the recursive walk will do it for us
		// because when we descend we won't have `field` set
		if val.Kind() == reflect.Ptr {
			if val.IsNil() {
				return nil
			}
		}

		isEmpty := val.IsZero()

		if !isEmpty || !omitEmpty {
			switch v := val.Interface().(type) {
			case *resource.Quantity:
				floatVal, err := strconv.ParseFloat(v.AsDec().String(), 64)
				if err != nil {
					return fmt.Errorf("unable to convert from Quantity %v to float", v)
				}
				if err := setValue(targetPath, floatVal); err != nil {
					return err
				}
				// Clear the field, so we don't set the flag
				val.Set(reflect.ValueOf(nil))
			default:
				if err := setValue(targetPath, val.Interface()); err != nil {
					return err
				}
				// Clear the field, so we don't set the flag
				empty := reflect.New(val.Type()).Elem()
				val.Set(empty)
			}
		}

		return reflectutils.SkipReflection
	}

	err := reflectutils.ReflectRecursive(reflect.ValueOf(options), walker, &reflectutils.ReflectOptions{DeprecatedDoubleVisit: true, JSONNames: true})
	if err != nil {
		return fmt.Errorf("error walking over %T: %w", options, err)
	}

	return nil
}