Configuration file for kube-scheduler

We generate a kube-scheduler configuration file in the kops CLI, and
nodeup will use it if provided (instead of generating one).

We put the configuration file into the fileAssets.

Users can provide a kube-scheduler configuration in additional
objects, and this will be used as the base configuration (we add the
kubeconfig path).

Issue #13352

Co-authored-by: Ciprian Hacman <ciprian@hakman.dev>

nodeup/pkg/model/context.go

@@ -670,3 +670,28 @@ func (c *NodeupModelContext) GetMetadataLocalIP() (string, error) {
 
 	return internalIP, nil
 }
+
+func (c *NodeupModelContext) findStaticManifest(key string) *nodeup.StaticManifest {
+	if c == nil || c.NodeupConfig == nil {
+		return nil
+	}
+	for _, manifest := range c.NodeupConfig.StaticManifests {
+		if manifest.Key == key {
+			return manifest
+		}
+	}
+	return nil
+}
+
+func (c *NodeupModelContext) findFileAsset(path string) *kops.FileAssetSpec {
+	if c == nil || c.NodeupConfig == nil {
+		return nil
+	}
+	for i := range c.NodeupConfig.FileAssets {
+		f := &c.NodeupConfig.FileAssets[i]
+		if f.Path == path {
+			return f
+		}
+	}
+	return nil
+}

nodeup/pkg/model/kube_apiserver_healthcheck.go

@@ -28,15 +28,7 @@ import (
 )
 
 func (b *KubeAPIServerBuilder) findHealthcheckManifest() *nodeup.StaticManifest {
-	if b.NodeupConfig == nil {
-		return nil
-	}
-	for _, manifest := range b.NodeupConfig.StaticManifests {
-		if manifest.Key == "kube-apiserver-healthcheck" {
-			return manifest
-		}
-	}
-	return nil
+	return b.findStaticManifest("kube-apiserver-healthcheck")
 }
 
 func (b *KubeAPIServerBuilder) addHealthcheckSidecar(pod *corev1.Pod) error {

nodeup/pkg/model/kube_scheduler.go

@@ -22,12 +22,14 @@ import (
 	"strconv"
 	"strings"
 
+	"k8s.io/klog/v2"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/configbuilder"
 	"k8s.io/kops/pkg/flagbuilder"
 	"k8s.io/kops/pkg/k8scodecs"
 	"k8s.io/kops/pkg/kubemanifest"
 	"k8s.io/kops/pkg/model/components"
+	"k8s.io/kops/pkg/model/components/kubescheduler"
 	"k8s.io/kops/pkg/rbac"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
@@ -61,8 +63,6 @@ type KubeSchedulerBuilder struct {
 
 var _ fi.ModelBuilder = &KubeSchedulerBuilder{}
 
-const defaultKubeConfig = "/var/lib/kube-scheduler/kubeconfig"
-
 // Build is responsible for building the manifest for the kube-scheduler
 func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error {
 	if !b.IsMaster {
@@ -97,13 +97,22 @@ func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error {
 		kubeconfig := b.BuildIssuedKubeconfig("kube-scheduler", nodetasks.PKIXName{CommonName: rbac.KubeScheduler}, c)
 
 		c.AddTask(&nodetasks.File{
-			Path:     "/var/lib/kube-scheduler/kubeconfig",
+			Path:     kubescheduler.KubeConfigPath,
 			Contents: kubeconfig,
 			Type:     nodetasks.FileType_File,
 			Mode:     s("0400"),
 		})
 	}
-	{
+
+	// Load the kube-scheduler config object if one has been provided.
+	kubeSchedulerConfigAsset := b.findFileAsset(kubescheduler.KubeSchedulerConfigPath)
+
+	if kubeSchedulerConfigAsset != nil {
+		klog.Infof("using kubescheduler configuration from file assets")
+		// FileAssets are written automatically, we don't need to write it.
+	} else {
+		// We didn't get a kubescheduler configuration; warn as we're aiming to move this to generation in the kops CLI
+		klog.Warningf("using embedded kubescheduler configuration")
 		var config *SchedulerConfig
 		if b.IsKubernetesGTE("1.22") {
 			config = NewSchedulerConfig("kubescheduler.config.k8s.io/v1beta2")
@@ -111,14 +120,13 @@ func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error {
 			config = NewSchedulerConfig("kubescheduler.config.k8s.io/v1beta1")
 		}
 
-		manifest, err := configbuilder.BuildConfigYaml(&kubeScheduler, config)
+		kubeSchedulerConfig, err := configbuilder.BuildConfigYaml(&kubeScheduler, config)
 		if err != nil {
 			return err
 		}
-
 		c.AddTask(&nodetasks.File{
-			Path:     "/var/lib/kube-scheduler/config.yaml",
-			Contents: fi.NewBytesResource(manifest),
+			Path:     kubescheduler.KubeSchedulerConfigPath,
+			Contents: fi.NewBytesResource(kubeSchedulerConfig),
 			Type:     nodetasks.FileType_File,
 			Mode:     s("0400"),
 		})
@@ -143,7 +151,7 @@ func NewSchedulerConfig(apiVersion string) *SchedulerConfig {
 	schedConfig.APIVersion = apiVersion
 	schedConfig.Kind = "KubeSchedulerConfiguration"
 	schedConfig.ClientConnection = ClientConnectionConfig{}
-	schedConfig.ClientConnection.Kubeconfig = defaultKubeConfig
+	schedConfig.ClientConnection.Kubeconfig = kubescheduler.KubeConfigPath
 	return schedConfig
 }
 
@@ -190,7 +198,7 @@ func (b *KubeSchedulerBuilder) buildPod(kubeScheduler *kops.KubeSchedulerConfig)
 
 	// Add kubeconfig flags
 	for _, flag := range []string{"authentication-", "authorization-"} {
-		flags = append(flags, "--"+flag+"kubeconfig="+defaultKubeConfig)
+		flags = append(flags, "--"+flag+"kubeconfig="+kubescheduler.KubeConfigPath)
 	}
 
 	if fi.BoolValue(kubeScheduler.UsePolicyConfigMap) {

pkg/apis/kops/componentconfig.go

@@ -720,9 +720,9 @@ type KubeSchedulerConfig struct {
 	// as outlined: https://kubernetes.io/docs/concepts/storage/storage-limits/
 	MaxPersistentVolumes *int32 `json:"maxPersistentVolumes,omitempty"`
 	// Qps sets the maximum qps to send to apiserver after the burst quota is exhausted
-	Qps *resource.Quantity `json:"qps,omitempty" configfile:"ClientConnection.QPS"`
+	Qps *resource.Quantity `json:"qps,omitempty" configfile:"ClientConnection.QPS" config:"clientConnection.qps,omitempty"`
 	// Burst sets the maximum qps to send to apiserver after the burst quota is exhausted
-	Burst int32 `json:"burst,omitempty" configfile:"ClientConnection.Burst"`
+	Burst int32 `json:"burst,omitempty" configfile:"ClientConnection.Burst" config:"clientConnection.burst,omitempty"`
 	// AuthenticationKubeconfig is the path to an Authentication Kubeconfig
 	AuthenticationKubeconfig string `json:"authenticationKubeconfig,omitempty" flag:"authentication-kubeconfig"`
 	// AuthorizationKubeconfig is the path to an Authorization Kubeconfig

pkg/assets/builder.go

@@ -50,15 +50,31 @@ type AssetBuilder struct {
 	// KubernetesVersion is the version of kubernetes we are installing
 	KubernetesVersion semver.Version
 
-	// StaticManifests records static manifests
+	// StaticManifests records manifests used by nodeup:
+	// * e.g. sidecar manifests for static pods run by kubelet
 	StaticManifests []*StaticManifest
+
+	// StaticFiles records static files:
+	// * Configuration files supporting static pods
+	StaticFiles []*StaticFile
+}
+
+type StaticFile struct {
+	// Path is the path to the manifest.
+	Path string
+
+	// Content holds the desired file contents.
+	Content string
+
+	// The static manifest will only be applied to instances matching the specified role
+	Roles []kops.InstanceGroupRole
 }
 
 type StaticManifest struct {
 	// Key is the unique identifier of the manifest
 	Key string
 
-	// Path is the path to the manifest
+	// Path is the path to the manifest.
 	Path string
 
 	// The static manifest will only be applied to instances matching the specified role

pkg/kubemanifest/manifest.go

@@ -23,6 +23,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/klog/v2"
 	"k8s.io/kops/util/pkg/text"
 	"sigs.k8s.io/yaml"
@@ -43,6 +44,10 @@ func (o *Object) ToUnstructured() *unstructured.Unstructured {
 	return &unstructured.Unstructured{Object: o.data}
 }
 
+func (o *Object) GroupVersionKind() schema.GroupVersionKind {
+	return o.ToUnstructured().GroupVersionKind()
+}
+
 // FromRuntimeObject converts from a runtime.Object.
 func FromRuntimeObject(obj runtime.Object) (*Object, error) {
 	data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)

pkg/model/components/kubescheduler/model.go

@@ -0,0 +1,208 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubescheduler
+
+import (
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/klog/v2"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/assets"
+	"k8s.io/kops/pkg/kubemanifest"
+	"k8s.io/kops/pkg/model"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/util/pkg/reflectutils"
+	"sigs.k8s.io/yaml"
+)
+
+// KubeSchedulerConfigPath is the path where we write the kube-scheduler config file (on the control-plane nodes)
+const KubeSchedulerConfigPath = "/var/lib/kube-scheduler/config.yaml"
+
+// Kubeconfig is the path where we write the kube-scheduler kubeconfig file (on the control-plane nodes)
+const KubeConfigPath = "/var/lib/kube-scheduler/kubeconfig"
+
+// KubeSchedulerBuilder builds the configuration file for kube-scheduler
+type KubeSchedulerBuilder struct {
+	*model.KopsModelContext
+	Lifecycle    fi.Lifecycle
+	AssetBuilder *assets.AssetBuilder
+}
+
+var _ fi.ModelBuilder = &KubeSchedulerBuilder{}
+
+// Build creates the tasks relating to kube-scheduler
+func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error {
+	configYAML, err := b.buildSchedulerConfig()
+	if err != nil {
+		return err
+	}
+
+	b.AssetBuilder.StaticFiles = append(b.AssetBuilder.StaticFiles, &assets.StaticFile{
+		Path:    KubeSchedulerConfigPath,
+		Content: string(configYAML),
+		Roles:   []kops.InstanceGroupRole{kops.InstanceGroupRoleMaster, kops.InstanceGroupRoleAPIServer},
+	})
+	return nil
+}
+
+func (b *KubeSchedulerBuilder) buildSchedulerConfig() ([]byte, error) {
+	var matches []*kubemanifest.Object
+	for _, additionalObject := range b.AdditionalObjects {
+		gvk := additionalObject.GroupVersionKind()
+		if gvk.Group != "kubescheduler.config.k8s.io" {
+			continue
+		}
+		if gvk.Kind != "KubeSchedulerConfiguration" {
+			continue
+		}
+		matches = append(matches, additionalObject)
+	}
+
+	if len(matches) > 1 {
+		return nil, fmt.Errorf("found multiple KubeSchedulerConfiguration objects in cluster configuration; expected at most one")
+	}
+
+	var config *unstructured.Unstructured
+	if len(matches) == 1 {
+		config = matches[0].ToUnstructured()
+	} else {
+		config = &unstructured.Unstructured{}
+		config.SetKind("KubeSchedulerConfiguration")
+		if b.IsKubernetesGTE("1.22") {
+			config.SetAPIVersion("kubescheduler.config.k8s.io/v1beta2")
+		} else {
+			config.SetAPIVersion("kubescheduler.config.k8s.io/v1beta1")
+		}
+		// We need to store the object, because we are often called repeatedly (until we converge)
+		b.AdditionalObjects = append(b.AdditionalObjects, kubemanifest.NewObject(config.Object))
+	}
+
+	// TODO: Handle different versions? e.g. gvk := config.GroupVersionKind()
+
+	if err := unstructured.SetNestedField(config.Object, KubeConfigPath, "clientConnection", "kubeconfig"); err != nil {
+		return nil, fmt.Errorf("error setting clientConnection.kubeconfig in kube-scheduler configuration: %w", err)
+	}
+
+	kubeScheduler := b.Cluster.Spec.KubeScheduler
+	if kubeScheduler != nil {
+		if err := MapToUnstructured(kubeScheduler, config); err != nil {
+			return nil, err
+		}
+	}
+
+	configYAML, err := yaml.Marshal(config)
+	if err != nil {
+		return nil, err
+	}
+	return configYAML, nil
+}
+
+// MapToUnstructured reflects the options interface and extracts the parameters for the config file
+func MapToUnstructured(options interface{}, target *unstructured.Unstructured) error {
+	setValue := func(targetPath string, val interface{}) error {
+		fields := strings.Split(targetPath, ".")
+		// Cannot use unstructured.SetNestedField, because it fails with e.g. "cannot deep copy int32"
+		parent := target.Object
+		for i := 0; i < len(fields)-1; i++ {
+			v := parent[fields[i]]
+			if v == nil {
+				v = make(map[string]interface{})
+				parent[fields[i]] = v
+			}
+			m, ok := v.(map[string]interface{})
+			if !ok {
+				return fmt.Errorf("value was not a map at position %d in %s", i, targetPath)
+			}
+			parent = m
+		}
+		parent[fields[len(fields)-1]] = val
+		return nil
+	}
+
+	walker := func(path *reflectutils.FieldPath, field *reflect.StructField, val reflect.Value) error {
+		if field == nil {
+			klog.V(8).Infof("ignoring non-field: %s", path)
+			return nil
+		}
+
+		tag := field.Tag.Get("config")
+		if tag == "" {
+			klog.V(4).Infof("not writing field with no config tag: %s", path)
+			// We want to descend - it could be a structure containing flags
+			return nil
+		}
+		if tag == "-" {
+			klog.V(4).Infof("skipping field with %q config tag: %s", tag, path)
+			return reflectutils.SkipReflection
+		}
+
+		tagTokens := strings.Split(tag, ",")
+		omitEmpty := false
+		for _, token := range tagTokens {
+			if token == "omitempty" {
+				omitEmpty = true
+			}
+		}
+		targetPath := tagTokens[0]
+
+		// We do have to do this, even though the recursive walk will do it for us
+		// because when we descend we won't have `field` set
+		if val.Kind() == reflect.Ptr {
+			if val.IsNil() {
+				return nil
+			}
+		}
+
+		isEmpty := val.IsZero()
+
+		if !isEmpty || !omitEmpty {
+			switch v := val.Interface().(type) {
+			case *resource.Quantity:
+				floatVal, err := strconv.ParseFloat(v.AsDec().String(), 64)
+				if err != nil {
+					return fmt.Errorf("unable to convert from Quantity %v to float", v)
+				}
+				if err := setValue(targetPath, floatVal); err != nil {
+					return err
+				}
+				// Clear the field, so we don't set the flag
+				val.Set(reflect.ValueOf(nil))
+			default:
+				if err := setValue(targetPath, val.Interface()); err != nil {
+					return err
+				}
+				// Clear the field, so we don't set the flag
+				empty := reflect.New(val.Type()).Elem()
+				val.Set(empty)
+			}
+		}
+
+		return reflectutils.SkipReflection
+	}
+
+	err := reflectutils.ReflectRecursive(reflect.ValueOf(options), walker, &reflectutils.ReflectOptions{DeprecatedDoubleVisit: true, JSONNames: true})
+	if err != nil {
+		return fmt.Errorf("error walking over %T: %w", options, err)
+	}
+
+	return nil
+}

pkg/model/components/kubescheduler/model_test.go

@@ -0,0 +1,86 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubescheduler
+
+import (
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	"k8s.io/kops/pkg/assets"
+	"k8s.io/kops/pkg/kubemanifest"
+	"k8s.io/kops/pkg/model"
+	"k8s.io/kops/pkg/model/iam"
+	"k8s.io/kops/pkg/testutils"
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+func Test_RunKubeSchedulerBuilder(t *testing.T) {
+	tests := []string{
+		"tests/minimal",
+		"tests/kubeschedulerconfig",
+		"tests/mixing",
+	}
+	for _, basedir := range tests {
+		basedir := basedir
+
+		t.Run(fmt.Sprintf("basedir=%s", basedir), func(t *testing.T) {
+			context := &fi.ModelBuilderContext{
+				Tasks: make(map[string]fi.Task),
+			}
+			kopsModelContext, err := LoadKopsModelContext(basedir)
+			if err != nil {
+				t.Fatalf("error loading model %q: %v", basedir, err)
+				return
+			}
+
+			builder := KubeSchedulerBuilder{
+				KopsModelContext: kopsModelContext,
+				AssetBuilder:     assets.NewAssetBuilder(kopsModelContext.Cluster, false),
+			}
+
+			if err := builder.Build(context); err != nil {
+				t.Fatalf("error from Build: %v", err)
+				return
+			}
+
+			testutils.ValidateTasks(t, filepath.Join(basedir, "tasks.yaml"), context)
+		})
+	}
+}
+
+func LoadKopsModelContext(basedir string) (*model.KopsModelContext, error) {
+	spec, err := testutils.LoadModel(basedir)
+	if err != nil {
+		return nil, err
+	}
+
+	if spec.Cluster == nil {
+		return nil, fmt.Errorf("no cluster found in %s", basedir)
+	}
+
+	kopsContext := &model.KopsModelContext{
+		IAMModelContext: iam.IAMModelContext{Cluster: spec.Cluster},
+		InstanceGroups:  spec.InstanceGroups,
+	}
+
+	for _, u := range spec.AdditionalObjects {
+		kopsContext.AdditionalObjects = append(kopsContext.AdditionalObjects, kubemanifest.NewObject(u.Object))
+	}
+
+	return kopsContext, nil
+}

pkg/model/components/kubescheduler/tests/kubeschedulerconfig/cluster.yaml

@@ -0,0 +1,21 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  name: minimal.example.com
+spec:
+  kubernetesVersion: v1.24.0
+
+---
+
+apiVersion: kubescheduler.config.k8s.io/v1beta2
+kind: KubeSchedulerConfiguration
+profiles:
+- plugins:
+    score:
+      disabled:
+      - name: PodTopologySpread
+      enabled:
+      - name: MyCustomPluginA
+        weight: 2
+      - name: MyCustomPluginB
+        weight: 1

pkg/model/components/kubescheduler/tests/kubeschedulerconfig/tasks.yaml

@@ -0,0 +1 @@
+

pkg/model/components/kubescheduler/tests/minimal/cluster.yaml

@@ -0,0 +1,6 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  name: minimal.example.com
+spec:
+  kubernetesVersion: v1.21.0

pkg/model/components/kubescheduler/tests/minimal/tasks.yaml

@@ -0,0 +1 @@
+

pkg/model/components/kubescheduler/tests/mixing/cluster.yaml

@@ -0,0 +1,23 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  name: minimal.example.com
+spec:
+  kubernetesVersion: v1.24.0
+  kubeScheduler:
+    burst: 123
+
+---
+
+apiVersion: kubescheduler.config.k8s.io/v1beta2
+kind: KubeSchedulerConfiguration
+profiles:
+- plugins:
+    score:
+      disabled:
+      - name: PodTopologySpread
+      enabled:
+      - name: MyCustomPluginA
+        weight: 2
+      - name: MyCustomPluginB
+        weight: 1

pkg/model/components/kubescheduler/tests/mixing/static-kube-scheduler-config

@@ -0,0 +1,15 @@
+apiVersion: kubescheduler.config.k8s.io/v1beta2
+clientConnection:
+  burst: 123
+  kubeconfig: /var/lib/kube-scheduler/kubeconfig
+kind: KubeSchedulerConfiguration
+profiles:
+- plugins:
+    score:
+      disabled:
+      - name: PodTopologySpread
+      enabled:
+      - name: MyCustomPluginA
+        weight: 2
+      - name: MyCustomPluginB
+        weight: 1

pkg/model/components/kubescheduler/tests/mixing/tasks.yaml

@@ -0,0 +1 @@
+

pkg/model/context.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/kops/pkg/apis/kops/model"
 	"k8s.io/kops/pkg/apis/kops/util"
 	"k8s.io/kops/pkg/dns"
+	"k8s.io/kops/pkg/kubemanifest"
 	"k8s.io/kops/pkg/model/components"
 	"k8s.io/kops/pkg/model/iam"
 	nodeidentityaws "k8s.io/kops/pkg/nodeidentity/aws"
@@ -48,6 +49,9 @@ type KopsModelContext struct {
 	InstanceGroups []*kops.InstanceGroup
 	Region         string
 	SSHPublicKeys  [][]byte
+
+	// AdditionalObjects holds cluster-asssociated configuration objects, other than the Cluster and InstanceGroups.
+	AdditionalObjects kubemanifest.ObjectList
 }
 
 // GatherSubnets maps the subnet names in an InstanceGroup to the ClusterSubnetSpec objects (which are stored on the Cluster)

pkg/testutils/modelharness.go

@@ -24,6 +24,7 @@ import (
 	"strings"
 	"testing"
 
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/apis/kops/v1alpha2"
@@ -36,6 +37,9 @@ import (
 type Model struct {
 	Cluster        *kops.Cluster
 	InstanceGroups []*kops.InstanceGroup
+
+	// AdditionalObjects holds cluster-asssociated configuration objects, other than the Cluster and InstanceGroups.
+	AdditionalObjects []*unstructured.Unstructured
 }
 
 // LoadModel loads a cluster and instancegroups from a cluster.yaml file found in basedir
@@ -68,8 +72,11 @@ func LoadModel(basedir string) (*Model, error) {
 		case *kops.InstanceGroup:
 			spec.InstanceGroups = append(spec.InstanceGroups, v)
 
+		case *unstructured.Unstructured:
+			spec.AdditionalObjects = append(spec.AdditionalObjects, v)
+
 		default:
-			return nil, fmt.Errorf("unhandled kind %q", gvk)
+			return nil, fmt.Errorf("unhandled kind %T %q", o, gvk)
 		}
 	}
 

upup/pkg/fi/cloudup/apply_cluster.go

@@ -44,12 +44,14 @@ import (
 	"k8s.io/kops/pkg/client/simple"
 	"k8s.io/kops/pkg/dns"
 	"k8s.io/kops/pkg/featureflag"
+	"k8s.io/kops/pkg/kubemanifest"
 	"k8s.io/kops/pkg/model"
 	"k8s.io/kops/pkg/model/awsmodel"
 	"k8s.io/kops/pkg/model/azuremodel"
 	"k8s.io/kops/pkg/model/components"
 	"k8s.io/kops/pkg/model/components/etcdmanager"
 	"k8s.io/kops/pkg/model/components/kubeapiserver"
+	"k8s.io/kops/pkg/model/components/kubescheduler"
 	"k8s.io/kops/pkg/model/domodel"
 	"k8s.io/kops/pkg/model/gcemodel"
 	"k8s.io/kops/pkg/model/hetznermodel"
@@ -144,6 +146,9 @@ type ApplyClusterCmd struct {
 	ImageAssets []*assets.ImageAsset
 	// FileAssets are the file assets we use (output).
 	FileAssets []*assets.FileAsset
+
+	// AdditionalObjects holds cluster-asssociated configuration objects, other than the Cluster and InstanceGroups.
+	AdditionalObjects kubemanifest.ObjectList
 }
 
 func (c *ApplyClusterCmd) Run(ctx context.Context) error {
@@ -171,6 +176,18 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error {
 		c.InstanceGroups = instanceGroups
 	}
 
+	if c.AdditionalObjects == nil {
+		additionalObjects, err := c.Clientset.AddonsFor(c.Cluster).List()
+		if err != nil {
+			return err
+		}
+		// We use the nil object to mean "uninitialized"
+		if additionalObjects == nil {
+			additionalObjects = []*kubemanifest.Object{}
+		}
+		c.AdditionalObjects = additionalObjects
+	}
+
 	for _, ig := range c.InstanceGroups {
 		// Try to guess the path for additional third party volume plugins in Flatcar
 		image := strings.ToLower(ig.Spec.Image)
@@ -392,8 +409,9 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error {
 	}
 
 	modelContext := &model.KopsModelContext{
-		IAMModelContext: iam.IAMModelContext{Cluster: cluster},
-		InstanceGroups:  c.InstanceGroups,
+		IAMModelContext:   iam.IAMModelContext{Cluster: cluster},
+		InstanceGroups:    c.InstanceGroups,
+		AdditionalObjects: c.AdditionalObjects,
 	}
 
 	switch cluster.Spec.GetCloudProvider() {
@@ -525,6 +543,11 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error {
 				KopsModelContext: modelContext,
 				Lifecycle:        clusterLifecycle,
 			},
+			&kubescheduler.KubeSchedulerBuilder{
+				AssetBuilder:     assetBuilder,
+				KopsModelContext: modelContext,
+				Lifecycle:        clusterLifecycle,
+			},
 			&etcdmanager.EtcdManagerBuilder{
 				AssetBuilder:     assetBuilder,
 				KopsModelContext: modelContext,
@@ -1408,6 +1431,24 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAddit
 		})
 	}
 
+	for _, staticFile := range n.assetBuilder.StaticFiles {
+		match := false
+		for _, r := range staticFile.Roles {
+			if r == role {
+				match = true
+			}
+		}
+
+		if !match {
+			continue
+		}
+
+		config.FileAssets = append(config.FileAssets, kops.FileAssetSpec{
+			Content: staticFile.Content,
+			Path:    staticFile.Path,
+		})
+	}
+
 	config.Images = n.images[role]
 	config.Channels = n.channels
 	config.EtcdManifests = n.etcdManifests[role]