diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go index 257dbe218e..ca740f882b 100644 --- a/nodeup/pkg/model/context.go +++ b/nodeup/pkg/model/context.go @@ -670,3 +670,28 @@ func (c *NodeupModelContext) GetMetadataLocalIP() (string, error) { return internalIP, nil } + +func (c *NodeupModelContext) findStaticManifest(key string) *nodeup.StaticManifest { + if c == nil || c.NodeupConfig == nil { + return nil + } + for _, manifest := range c.NodeupConfig.StaticManifests { + if manifest.Key == key { + return manifest + } + } + return nil +} + +func (c *NodeupModelContext) findFileAsset(path string) *kops.FileAssetSpec { + if c == nil || c.NodeupConfig == nil { + return nil + } + for i := range c.NodeupConfig.FileAssets { + f := &c.NodeupConfig.FileAssets[i] + if f.Path == path { + return f + } + } + return nil +} diff --git a/nodeup/pkg/model/kube_apiserver_healthcheck.go b/nodeup/pkg/model/kube_apiserver_healthcheck.go index 417d3fb13b..47fc754fdb 100644 --- a/nodeup/pkg/model/kube_apiserver_healthcheck.go +++ b/nodeup/pkg/model/kube_apiserver_healthcheck.go @@ -28,15 +28,7 @@ import ( ) func (b *KubeAPIServerBuilder) findHealthcheckManifest() *nodeup.StaticManifest { - if b.NodeupConfig == nil { - return nil - } - for _, manifest := range b.NodeupConfig.StaticManifests { - if manifest.Key == "kube-apiserver-healthcheck" { - return manifest - } - } - return nil + return b.findStaticManifest("kube-apiserver-healthcheck") } func (b *KubeAPIServerBuilder) addHealthcheckSidecar(pod *corev1.Pod) error { diff --git a/nodeup/pkg/model/kube_scheduler.go b/nodeup/pkg/model/kube_scheduler.go index f7c381a8c8..c09255cf29 100644 --- a/nodeup/pkg/model/kube_scheduler.go +++ b/nodeup/pkg/model/kube_scheduler.go @@ -22,12 +22,14 @@ import ( "strconv" "strings" + "k8s.io/klog/v2" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/configbuilder" "k8s.io/kops/pkg/flagbuilder" "k8s.io/kops/pkg/k8scodecs" "k8s.io/kops/pkg/kubemanifest" "k8s.io/kops/pkg/model/components" + "k8s.io/kops/pkg/model/components/kubescheduler" "k8s.io/kops/pkg/rbac" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" @@ -61,8 +63,6 @@ type KubeSchedulerBuilder struct { var _ fi.ModelBuilder = &KubeSchedulerBuilder{} -const defaultKubeConfig = "/var/lib/kube-scheduler/kubeconfig" - // Build is responsible for building the manifest for the kube-scheduler func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error { if !b.IsMaster { @@ -97,13 +97,22 @@ func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error { kubeconfig := b.BuildIssuedKubeconfig("kube-scheduler", nodetasks.PKIXName{CommonName: rbac.KubeScheduler}, c) c.AddTask(&nodetasks.File{ - Path: "/var/lib/kube-scheduler/kubeconfig", + Path: kubescheduler.KubeConfigPath, Contents: kubeconfig, Type: nodetasks.FileType_File, Mode: s("0400"), }) } - { + + // Load the kube-scheduler config object if one has been provided. + kubeSchedulerConfigAsset := b.findFileAsset(kubescheduler.KubeSchedulerConfigPath) + + if kubeSchedulerConfigAsset != nil { + klog.Infof("using kubescheduler configuration from file assets") + // FileAssets are written automatically, we don't need to write it. + } else { + // We didn't get a kubescheduler configuration; warn as we're aiming to move this to generation in the kops CLI + klog.Warningf("using embedded kubescheduler configuration") var config *SchedulerConfig if b.IsKubernetesGTE("1.22") { config = NewSchedulerConfig("kubescheduler.config.k8s.io/v1beta2") @@ -111,14 +120,13 @@ func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error { config = NewSchedulerConfig("kubescheduler.config.k8s.io/v1beta1") } - manifest, err := configbuilder.BuildConfigYaml(&kubeScheduler, config) + kubeSchedulerConfig, err := configbuilder.BuildConfigYaml(&kubeScheduler, config) if err != nil { return err } - c.AddTask(&nodetasks.File{ - Path: "/var/lib/kube-scheduler/config.yaml", - Contents: fi.NewBytesResource(manifest), + Path: kubescheduler.KubeSchedulerConfigPath, + Contents: fi.NewBytesResource(kubeSchedulerConfig), Type: nodetasks.FileType_File, Mode: s("0400"), }) @@ -143,7 +151,7 @@ func NewSchedulerConfig(apiVersion string) *SchedulerConfig { schedConfig.APIVersion = apiVersion schedConfig.Kind = "KubeSchedulerConfiguration" schedConfig.ClientConnection = ClientConnectionConfig{} - schedConfig.ClientConnection.Kubeconfig = defaultKubeConfig + schedConfig.ClientConnection.Kubeconfig = kubescheduler.KubeConfigPath return schedConfig } @@ -190,7 +198,7 @@ func (b *KubeSchedulerBuilder) buildPod(kubeScheduler *kops.KubeSchedulerConfig) // Add kubeconfig flags for _, flag := range []string{"authentication-", "authorization-"} { - flags = append(flags, "--"+flag+"kubeconfig="+defaultKubeConfig) + flags = append(flags, "--"+flag+"kubeconfig="+kubescheduler.KubeConfigPath) } if fi.BoolValue(kubeScheduler.UsePolicyConfigMap) { diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index 2e42e5e62e..98d754a7ee 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -720,9 +720,9 @@ type KubeSchedulerConfig struct { // as outlined: https://kubernetes.io/docs/concepts/storage/storage-limits/ MaxPersistentVolumes *int32 `json:"maxPersistentVolumes,omitempty"` // Qps sets the maximum qps to send to apiserver after the burst quota is exhausted - Qps *resource.Quantity `json:"qps,omitempty" configfile:"ClientConnection.QPS"` + Qps *resource.Quantity `json:"qps,omitempty" configfile:"ClientConnection.QPS" config:"clientConnection.qps,omitempty"` // Burst sets the maximum qps to send to apiserver after the burst quota is exhausted - Burst int32 `json:"burst,omitempty" configfile:"ClientConnection.Burst"` + Burst int32 `json:"burst,omitempty" configfile:"ClientConnection.Burst" config:"clientConnection.burst,omitempty"` // AuthenticationKubeconfig is the path to an Authentication Kubeconfig AuthenticationKubeconfig string `json:"authenticationKubeconfig,omitempty" flag:"authentication-kubeconfig"` // AuthorizationKubeconfig is the path to an Authorization Kubeconfig diff --git a/pkg/assets/builder.go b/pkg/assets/builder.go index d59e1a55de..b67a6408a0 100644 --- a/pkg/assets/builder.go +++ b/pkg/assets/builder.go @@ -50,15 +50,31 @@ type AssetBuilder struct { // KubernetesVersion is the version of kubernetes we are installing KubernetesVersion semver.Version - // StaticManifests records static manifests + // StaticManifests records manifests used by nodeup: + // * e.g. sidecar manifests for static pods run by kubelet StaticManifests []*StaticManifest + + // StaticFiles records static files: + // * Configuration files supporting static pods + StaticFiles []*StaticFile +} + +type StaticFile struct { + // Path is the path to the manifest. + Path string + + // Content holds the desired file contents. + Content string + + // The static manifest will only be applied to instances matching the specified role + Roles []kops.InstanceGroupRole } type StaticManifest struct { // Key is the unique identifier of the manifest Key string - // Path is the path to the manifest + // Path is the path to the manifest. Path string // The static manifest will only be applied to instances matching the specified role diff --git a/pkg/kubemanifest/manifest.go b/pkg/kubemanifest/manifest.go index 0545fbc703..a490fc1cd8 100644 --- a/pkg/kubemanifest/manifest.go +++ b/pkg/kubemanifest/manifest.go @@ -23,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/klog/v2" "k8s.io/kops/util/pkg/text" "sigs.k8s.io/yaml" @@ -43,6 +44,10 @@ func (o *Object) ToUnstructured() *unstructured.Unstructured { return &unstructured.Unstructured{Object: o.data} } +func (o *Object) GroupVersionKind() schema.GroupVersionKind { + return o.ToUnstructured().GroupVersionKind() +} + // FromRuntimeObject converts from a runtime.Object. func FromRuntimeObject(obj runtime.Object) (*Object, error) { data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) diff --git a/pkg/model/components/kubescheduler/model.go b/pkg/model/components/kubescheduler/model.go new file mode 100644 index 0000000000..f4df8e65eb --- /dev/null +++ b/pkg/model/components/kubescheduler/model.go @@ -0,0 +1,208 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubescheduler + +import ( + "fmt" + "reflect" + "strconv" + "strings" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/klog/v2" + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/assets" + "k8s.io/kops/pkg/kubemanifest" + "k8s.io/kops/pkg/model" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/util/pkg/reflectutils" + "sigs.k8s.io/yaml" +) + +// KubeSchedulerConfigPath is the path where we write the kube-scheduler config file (on the control-plane nodes) +const KubeSchedulerConfigPath = "/var/lib/kube-scheduler/config.yaml" + +// Kubeconfig is the path where we write the kube-scheduler kubeconfig file (on the control-plane nodes) +const KubeConfigPath = "/var/lib/kube-scheduler/kubeconfig" + +// KubeSchedulerBuilder builds the configuration file for kube-scheduler +type KubeSchedulerBuilder struct { + *model.KopsModelContext + Lifecycle fi.Lifecycle + AssetBuilder *assets.AssetBuilder +} + +var _ fi.ModelBuilder = &KubeSchedulerBuilder{} + +// Build creates the tasks relating to kube-scheduler +func (b *KubeSchedulerBuilder) Build(c *fi.ModelBuilderContext) error { + configYAML, err := b.buildSchedulerConfig() + if err != nil { + return err + } + + b.AssetBuilder.StaticFiles = append(b.AssetBuilder.StaticFiles, &assets.StaticFile{ + Path: KubeSchedulerConfigPath, + Content: string(configYAML), + Roles: []kops.InstanceGroupRole{kops.InstanceGroupRoleMaster, kops.InstanceGroupRoleAPIServer}, + }) + return nil +} + +func (b *KubeSchedulerBuilder) buildSchedulerConfig() ([]byte, error) { + var matches []*kubemanifest.Object + for _, additionalObject := range b.AdditionalObjects { + gvk := additionalObject.GroupVersionKind() + if gvk.Group != "kubescheduler.config.k8s.io" { + continue + } + if gvk.Kind != "KubeSchedulerConfiguration" { + continue + } + matches = append(matches, additionalObject) + } + + if len(matches) > 1 { + return nil, fmt.Errorf("found multiple KubeSchedulerConfiguration objects in cluster configuration; expected at most one") + } + + var config *unstructured.Unstructured + if len(matches) == 1 { + config = matches[0].ToUnstructured() + } else { + config = &unstructured.Unstructured{} + config.SetKind("KubeSchedulerConfiguration") + if b.IsKubernetesGTE("1.22") { + config.SetAPIVersion("kubescheduler.config.k8s.io/v1beta2") + } else { + config.SetAPIVersion("kubescheduler.config.k8s.io/v1beta1") + } + // We need to store the object, because we are often called repeatedly (until we converge) + b.AdditionalObjects = append(b.AdditionalObjects, kubemanifest.NewObject(config.Object)) + } + + // TODO: Handle different versions? e.g. gvk := config.GroupVersionKind() + + if err := unstructured.SetNestedField(config.Object, KubeConfigPath, "clientConnection", "kubeconfig"); err != nil { + return nil, fmt.Errorf("error setting clientConnection.kubeconfig in kube-scheduler configuration: %w", err) + } + + kubeScheduler := b.Cluster.Spec.KubeScheduler + if kubeScheduler != nil { + if err := MapToUnstructured(kubeScheduler, config); err != nil { + return nil, err + } + } + + configYAML, err := yaml.Marshal(config) + if err != nil { + return nil, err + } + return configYAML, nil +} + +// MapToUnstructured reflects the options interface and extracts the parameters for the config file +func MapToUnstructured(options interface{}, target *unstructured.Unstructured) error { + setValue := func(targetPath string, val interface{}) error { + fields := strings.Split(targetPath, ".") + // Cannot use unstructured.SetNestedField, because it fails with e.g. "cannot deep copy int32" + parent := target.Object + for i := 0; i < len(fields)-1; i++ { + v := parent[fields[i]] + if v == nil { + v = make(map[string]interface{}) + parent[fields[i]] = v + } + m, ok := v.(map[string]interface{}) + if !ok { + return fmt.Errorf("value was not a map at position %d in %s", i, targetPath) + } + parent = m + } + parent[fields[len(fields)-1]] = val + return nil + } + + walker := func(path *reflectutils.FieldPath, field *reflect.StructField, val reflect.Value) error { + if field == nil { + klog.V(8).Infof("ignoring non-field: %s", path) + return nil + } + + tag := field.Tag.Get("config") + if tag == "" { + klog.V(4).Infof("not writing field with no config tag: %s", path) + // We want to descend - it could be a structure containing flags + return nil + } + if tag == "-" { + klog.V(4).Infof("skipping field with %q config tag: %s", tag, path) + return reflectutils.SkipReflection + } + + tagTokens := strings.Split(tag, ",") + omitEmpty := false + for _, token := range tagTokens { + if token == "omitempty" { + omitEmpty = true + } + } + targetPath := tagTokens[0] + + // We do have to do this, even though the recursive walk will do it for us + // because when we descend we won't have `field` set + if val.Kind() == reflect.Ptr { + if val.IsNil() { + return nil + } + } + + isEmpty := val.IsZero() + + if !isEmpty || !omitEmpty { + switch v := val.Interface().(type) { + case *resource.Quantity: + floatVal, err := strconv.ParseFloat(v.AsDec().String(), 64) + if err != nil { + return fmt.Errorf("unable to convert from Quantity %v to float", v) + } + if err := setValue(targetPath, floatVal); err != nil { + return err + } + // Clear the field, so we don't set the flag + val.Set(reflect.ValueOf(nil)) + default: + if err := setValue(targetPath, val.Interface()); err != nil { + return err + } + // Clear the field, so we don't set the flag + empty := reflect.New(val.Type()).Elem() + val.Set(empty) + } + } + + return reflectutils.SkipReflection + } + + err := reflectutils.ReflectRecursive(reflect.ValueOf(options), walker, &reflectutils.ReflectOptions{DeprecatedDoubleVisit: true, JSONNames: true}) + if err != nil { + return fmt.Errorf("error walking over %T: %w", options, err) + } + + return nil +} diff --git a/pkg/model/components/kubescheduler/model_test.go b/pkg/model/components/kubescheduler/model_test.go new file mode 100644 index 0000000000..e5cb3bb4ad --- /dev/null +++ b/pkg/model/components/kubescheduler/model_test.go @@ -0,0 +1,86 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubescheduler + +import ( + "fmt" + "path/filepath" + "testing" + + "k8s.io/kops/pkg/assets" + "k8s.io/kops/pkg/kubemanifest" + "k8s.io/kops/pkg/model" + "k8s.io/kops/pkg/model/iam" + "k8s.io/kops/pkg/testutils" + "k8s.io/kops/upup/pkg/fi" +) + +func Test_RunKubeSchedulerBuilder(t *testing.T) { + tests := []string{ + "tests/minimal", + "tests/kubeschedulerconfig", + "tests/mixing", + } + for _, basedir := range tests { + basedir := basedir + + t.Run(fmt.Sprintf("basedir=%s", basedir), func(t *testing.T) { + context := &fi.ModelBuilderContext{ + Tasks: make(map[string]fi.Task), + } + kopsModelContext, err := LoadKopsModelContext(basedir) + if err != nil { + t.Fatalf("error loading model %q: %v", basedir, err) + return + } + + builder := KubeSchedulerBuilder{ + KopsModelContext: kopsModelContext, + AssetBuilder: assets.NewAssetBuilder(kopsModelContext.Cluster, false), + } + + if err := builder.Build(context); err != nil { + t.Fatalf("error from Build: %v", err) + return + } + + testutils.ValidateTasks(t, filepath.Join(basedir, "tasks.yaml"), context) + }) + } +} + +func LoadKopsModelContext(basedir string) (*model.KopsModelContext, error) { + spec, err := testutils.LoadModel(basedir) + if err != nil { + return nil, err + } + + if spec.Cluster == nil { + return nil, fmt.Errorf("no cluster found in %s", basedir) + } + + kopsContext := &model.KopsModelContext{ + IAMModelContext: iam.IAMModelContext{Cluster: spec.Cluster}, + InstanceGroups: spec.InstanceGroups, + } + + for _, u := range spec.AdditionalObjects { + kopsContext.AdditionalObjects = append(kopsContext.AdditionalObjects, kubemanifest.NewObject(u.Object)) + } + + return kopsContext, nil +} diff --git a/pkg/model/components/kubescheduler/tests/kubeschedulerconfig/cluster.yaml b/pkg/model/components/kubescheduler/tests/kubeschedulerconfig/cluster.yaml new file mode 100644 index 0000000000..0eb1693368 --- /dev/null +++ b/pkg/model/components/kubescheduler/tests/kubeschedulerconfig/cluster.yaml @@ -0,0 +1,21 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + name: minimal.example.com +spec: + kubernetesVersion: v1.24.0 + +--- + +apiVersion: kubescheduler.config.k8s.io/v1beta2 +kind: KubeSchedulerConfiguration +profiles: +- plugins: + score: + disabled: + - name: PodTopologySpread + enabled: + - name: MyCustomPluginA + weight: 2 + - name: MyCustomPluginB + weight: 1 \ No newline at end of file diff --git a/pkg/model/components/kubescheduler/tests/kubeschedulerconfig/tasks.yaml b/pkg/model/components/kubescheduler/tests/kubeschedulerconfig/tasks.yaml new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/pkg/model/components/kubescheduler/tests/kubeschedulerconfig/tasks.yaml @@ -0,0 +1 @@ + diff --git a/pkg/model/components/kubescheduler/tests/minimal/cluster.yaml b/pkg/model/components/kubescheduler/tests/minimal/cluster.yaml new file mode 100644 index 0000000000..2978cc6b5e --- /dev/null +++ b/pkg/model/components/kubescheduler/tests/minimal/cluster.yaml @@ -0,0 +1,6 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + name: minimal.example.com +spec: + kubernetesVersion: v1.21.0 diff --git a/pkg/model/components/kubescheduler/tests/minimal/tasks.yaml b/pkg/model/components/kubescheduler/tests/minimal/tasks.yaml new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/pkg/model/components/kubescheduler/tests/minimal/tasks.yaml @@ -0,0 +1 @@ + diff --git a/pkg/model/components/kubescheduler/tests/mixing/cluster.yaml b/pkg/model/components/kubescheduler/tests/mixing/cluster.yaml new file mode 100644 index 0000000000..b4b6040b00 --- /dev/null +++ b/pkg/model/components/kubescheduler/tests/mixing/cluster.yaml @@ -0,0 +1,23 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + name: minimal.example.com +spec: + kubernetesVersion: v1.24.0 + kubeScheduler: + burst: 123 + +--- + +apiVersion: kubescheduler.config.k8s.io/v1beta2 +kind: KubeSchedulerConfiguration +profiles: +- plugins: + score: + disabled: + - name: PodTopologySpread + enabled: + - name: MyCustomPluginA + weight: 2 + - name: MyCustomPluginB + weight: 1 \ No newline at end of file diff --git a/pkg/model/components/kubescheduler/tests/mixing/static-kube-scheduler-config b/pkg/model/components/kubescheduler/tests/mixing/static-kube-scheduler-config new file mode 100644 index 0000000000..e912238b88 --- /dev/null +++ b/pkg/model/components/kubescheduler/tests/mixing/static-kube-scheduler-config @@ -0,0 +1,15 @@ +apiVersion: kubescheduler.config.k8s.io/v1beta2 +clientConnection: + burst: 123 + kubeconfig: /var/lib/kube-scheduler/kubeconfig +kind: KubeSchedulerConfiguration +profiles: +- plugins: + score: + disabled: + - name: PodTopologySpread + enabled: + - name: MyCustomPluginA + weight: 2 + - name: MyCustomPluginB + weight: 1 diff --git a/pkg/model/components/kubescheduler/tests/mixing/tasks.yaml b/pkg/model/components/kubescheduler/tests/mixing/tasks.yaml new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/pkg/model/components/kubescheduler/tests/mixing/tasks.yaml @@ -0,0 +1 @@ + diff --git a/pkg/model/context.go b/pkg/model/context.go index 12e1563272..0bb2609571 100644 --- a/pkg/model/context.go +++ b/pkg/model/context.go @@ -25,6 +25,7 @@ import ( "k8s.io/kops/pkg/apis/kops/model" "k8s.io/kops/pkg/apis/kops/util" "k8s.io/kops/pkg/dns" + "k8s.io/kops/pkg/kubemanifest" "k8s.io/kops/pkg/model/components" "k8s.io/kops/pkg/model/iam" nodeidentityaws "k8s.io/kops/pkg/nodeidentity/aws" @@ -48,6 +49,9 @@ type KopsModelContext struct { InstanceGroups []*kops.InstanceGroup Region string SSHPublicKeys [][]byte + + // AdditionalObjects holds cluster-asssociated configuration objects, other than the Cluster and InstanceGroups. + AdditionalObjects kubemanifest.ObjectList } // GatherSubnets maps the subnet names in an InstanceGroup to the ClusterSubnetSpec objects (which are stored on the Cluster) diff --git a/pkg/testutils/modelharness.go b/pkg/testutils/modelharness.go index b33294f404..2839c63d26 100644 --- a/pkg/testutils/modelharness.go +++ b/pkg/testutils/modelharness.go @@ -24,6 +24,7 @@ import ( "strings" "testing" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/kops/v1alpha2" @@ -36,6 +37,9 @@ import ( type Model struct { Cluster *kops.Cluster InstanceGroups []*kops.InstanceGroup + + // AdditionalObjects holds cluster-asssociated configuration objects, other than the Cluster and InstanceGroups. + AdditionalObjects []*unstructured.Unstructured } // LoadModel loads a cluster and instancegroups from a cluster.yaml file found in basedir @@ -68,8 +72,11 @@ func LoadModel(basedir string) (*Model, error) { case *kops.InstanceGroup: spec.InstanceGroups = append(spec.InstanceGroups, v) + case *unstructured.Unstructured: + spec.AdditionalObjects = append(spec.AdditionalObjects, v) + default: - return nil, fmt.Errorf("unhandled kind %q", gvk) + return nil, fmt.Errorf("unhandled kind %T %q", o, gvk) } } diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index 1863e70097..5371a1183d 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -44,12 +44,14 @@ import ( "k8s.io/kops/pkg/client/simple" "k8s.io/kops/pkg/dns" "k8s.io/kops/pkg/featureflag" + "k8s.io/kops/pkg/kubemanifest" "k8s.io/kops/pkg/model" "k8s.io/kops/pkg/model/awsmodel" "k8s.io/kops/pkg/model/azuremodel" "k8s.io/kops/pkg/model/components" "k8s.io/kops/pkg/model/components/etcdmanager" "k8s.io/kops/pkg/model/components/kubeapiserver" + "k8s.io/kops/pkg/model/components/kubescheduler" "k8s.io/kops/pkg/model/domodel" "k8s.io/kops/pkg/model/gcemodel" "k8s.io/kops/pkg/model/hetznermodel" @@ -144,6 +146,9 @@ type ApplyClusterCmd struct { ImageAssets []*assets.ImageAsset // FileAssets are the file assets we use (output). FileAssets []*assets.FileAsset + + // AdditionalObjects holds cluster-asssociated configuration objects, other than the Cluster and InstanceGroups. + AdditionalObjects kubemanifest.ObjectList } func (c *ApplyClusterCmd) Run(ctx context.Context) error { @@ -171,6 +176,18 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error { c.InstanceGroups = instanceGroups } + if c.AdditionalObjects == nil { + additionalObjects, err := c.Clientset.AddonsFor(c.Cluster).List() + if err != nil { + return err + } + // We use the nil object to mean "uninitialized" + if additionalObjects == nil { + additionalObjects = []*kubemanifest.Object{} + } + c.AdditionalObjects = additionalObjects + } + for _, ig := range c.InstanceGroups { // Try to guess the path for additional third party volume plugins in Flatcar image := strings.ToLower(ig.Spec.Image) @@ -392,8 +409,9 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error { } modelContext := &model.KopsModelContext{ - IAMModelContext: iam.IAMModelContext{Cluster: cluster}, - InstanceGroups: c.InstanceGroups, + IAMModelContext: iam.IAMModelContext{Cluster: cluster}, + InstanceGroups: c.InstanceGroups, + AdditionalObjects: c.AdditionalObjects, } switch cluster.Spec.GetCloudProvider() { @@ -525,6 +543,11 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error { KopsModelContext: modelContext, Lifecycle: clusterLifecycle, }, + &kubescheduler.KubeSchedulerBuilder{ + AssetBuilder: assetBuilder, + KopsModelContext: modelContext, + Lifecycle: clusterLifecycle, + }, &etcdmanager.EtcdManagerBuilder{ AssetBuilder: assetBuilder, KopsModelContext: modelContext, @@ -1408,6 +1431,24 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAddit }) } + for _, staticFile := range n.assetBuilder.StaticFiles { + match := false + for _, r := range staticFile.Roles { + if r == role { + match = true + } + } + + if !match { + continue + } + + config.FileAssets = append(config.FileAssets, kops.FileAssetSpec{ + Content: staticFile.Content, + Path: staticFile.Path, + }) + } + config.Images = n.images[role] config.Channels = n.channels config.EtcdManifests = n.etcdManifests[role]