From f32fcc35fa23d5f08ca26b365a9d7807ffe972be Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sun, 15 Dec 2019 23:32:03 -0500 Subject: [PATCH] Addons: Support arbitrary additional objects We will be managing cluster addons using CRDs, and so we want to be able to apply arbitrary objects as part of cluster bringup. Start by allowing (behind a feature-flag) for arbitrary objects to be specified. Co-authored-by: John Gardiner Myers --- cmd/kops/BUILD.bazel | 2 + cmd/kops/create_cluster.go | 20 +++- hack/.packages | 1 + pkg/apis/kops/registry/BUILD.bazel | 1 + pkg/apis/kops/registry/helpers.go | 11 ++- pkg/assets/builder.go | 2 +- pkg/client/simple/BUILD.bazel | 1 + pkg/client/simple/api/BUILD.bazel | 1 + pkg/client/simple/api/clientset.go | 8 ++ pkg/client/simple/clientset.go | 16 +++- pkg/client/simple/vfsclientset/BUILD.bazel | 2 + pkg/client/simple/vfsclientset/addons.go | 96 +++++++++++++++++++ pkg/client/simple/vfsclientset/clientset.go | 7 ++ pkg/clusteraddons/BUILD.bazel | 13 +++ pkg/clusteraddons/load.go | 65 +++++++++++++ pkg/featureflag/featureflag.go | 2 + pkg/kubemanifest/manifest.go | 28 +++++- upup/pkg/fi/cloudup/BUILD.bazel | 1 + upup/pkg/fi/cloudup/apply_cluster.go | 7 ++ .../pkg/fi/cloudup/bootstrapchannelbuilder.go | 56 +++++++++-- .../cloudup/bootstrapchannelbuilder_test.go | 4 +- upup/pkg/kutil/convert_kubeup_cluster.go | 2 +- upup/pkg/kutil/import_cluster.go | 2 +- 23 files changed, 328 insertions(+), 20 deletions(-) create mode 100644 pkg/client/simple/vfsclientset/addons.go create mode 100644 pkg/clusteraddons/BUILD.bazel create mode 100644 pkg/clusteraddons/load.go diff --git a/cmd/kops/BUILD.bazel b/cmd/kops/BUILD.bazel index bf658890cd..d8ed70e52f 100644 --- a/cmd/kops/BUILD.bazel +++ b/cmd/kops/BUILD.bazel @@ -67,6 +67,7 @@ go_library( "//pkg/assets:go_default_library", "//pkg/client/simple:go_default_library", "//pkg/cloudinstances:go_default_library", + "//pkg/clusteraddons:go_default_library", "//pkg/commands:go_default_library", "//pkg/dump:go_default_library", "//pkg/edit:go_default_library", @@ -75,6 +76,7 @@ go_library( "//pkg/instancegroups:go_default_library", "//pkg/kopscodecs:go_default_library", "//pkg/kubeconfig:go_default_library", + "//pkg/kubemanifest:go_default_library", "//pkg/pki:go_default_library", "//pkg/pretty:go_default_library", "//pkg/resources:go_default_library", diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index e345e4724d..f23d952fd8 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -35,9 +35,11 @@ import ( "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/pkg/apis/kops/validation" "k8s.io/kops/pkg/assets" + "k8s.io/kops/pkg/clusteraddons" "k8s.io/kops/pkg/commands" "k8s.io/kops/pkg/featureflag" "k8s.io/kops/pkg/kubeconfig" + "k8s.io/kops/pkg/kubemanifest" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup" "k8s.io/kops/upup/pkg/fi/utils" @@ -87,6 +89,9 @@ type CreateClusterOptions struct { DryRun bool // Output type during a DryRun Output string + + // AddonPaths specify paths to additional components that we can add to a cluster + AddonPaths []string } func (o *CreateClusterOptions) InitDefaults() { @@ -209,6 +214,10 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command { cmd.Flags().StringSliceVar(&options.Zones, "zones", options.Zones, "Zones in which to run the cluster") cmd.Flags().StringSliceVar(&options.MasterZones, "master-zones", options.MasterZones, "Zones in which to run masters (must be an odd number)") + if featureflag.ClusterAddons.Enabled() { + cmd.Flags().StringSliceVar(&options.AddonPaths, "add", options.AddonPaths, "Paths to addons we should add to the cluster") + } + cmd.Flags().StringVar(&options.KubernetesVersion, "kubernetes-version", options.KubernetesVersion, "Version of kubernetes to run (defaults to version in channel)") cmd.Flags().StringVar(&options.ContainerRuntime, "container-runtime", options.ContainerRuntime, "Container runtime to use: containerd, docker") @@ -545,8 +554,17 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr } } + var addons kubemanifest.ObjectList + for _, p := range c.AddonPaths { + addon, err := clusteraddons.LoadClusterAddon(p) + if err != nil { + return fmt.Errorf("error loading cluster addon %s: %v", p, err) + } + addons = append(addons, addon.Objects...) + } + // Note we perform as much validation as we can, before writing a bad config - err = registry.CreateClusterConfig(ctx, clientset, cluster, fullInstanceGroups) + err = registry.CreateClusterConfig(ctx, clientset, cluster, fullInstanceGroups, addons) if err != nil { return fmt.Errorf("error writing updated configuration: %v", err) } diff --git a/hack/.packages b/hack/.packages index 6b368f3db9..a5d598f443 100644 --- a/hack/.packages +++ b/hack/.packages @@ -82,6 +82,7 @@ k8s.io/kops/pkg/client/simple k8s.io/kops/pkg/client/simple/api k8s.io/kops/pkg/client/simple/vfsclientset k8s.io/kops/pkg/cloudinstances +k8s.io/kops/pkg/clusteraddons k8s.io/kops/pkg/commands k8s.io/kops/pkg/configbuilder k8s.io/kops/pkg/diff diff --git a/pkg/apis/kops/registry/BUILD.bazel b/pkg/apis/kops/registry/BUILD.bazel index 274de0622b..1ff321c416 100644 --- a/pkg/apis/kops/registry/BUILD.bazel +++ b/pkg/apis/kops/registry/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "//pkg/acls:go_default_library", "//pkg/apis/kops:go_default_library", "//pkg/client/simple:go_default_library", + "//pkg/kubemanifest:go_default_library", "//upup/pkg/fi/utils:go_default_library", "//util/pkg/vfs:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/pkg/apis/kops/registry/helpers.go b/pkg/apis/kops/registry/helpers.go index 701b4dddee..fce4025b1a 100644 --- a/pkg/apis/kops/registry/helpers.go +++ b/pkg/apis/kops/registry/helpers.go @@ -23,9 +23,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" api "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/client/simple" + "k8s.io/kops/pkg/kubemanifest" ) -func CreateClusterConfig(ctx context.Context, clientset simple.Clientset, cluster *api.Cluster, groups []*api.InstanceGroup) error { +func CreateClusterConfig(ctx context.Context, clientset simple.Clientset, cluster *api.Cluster, groups []*api.InstanceGroup, addons kubemanifest.ObjectList) error { // Check for instancegroup Name duplicates before writing { names := map[string]bool{} @@ -52,5 +53,13 @@ func CreateClusterConfig(ctx context.Context, clientset simple.Clientset, cluste } } + { + addonsClient := clientset.AddonsFor(cluster) + + if err := addonsClient.Replace(addons); err != nil { + return fmt.Errorf("error writing updated addon configuration: %v", err) + } + } + return nil } diff --git a/pkg/assets/builder.go b/pkg/assets/builder.go index 74257e1375..82cecb487d 100644 --- a/pkg/assets/builder.go +++ b/pkg/assets/builder.go @@ -123,7 +123,7 @@ func (a *AssetBuilder) RemapManifest(data []byte) ([]byte, error) { } } - return kubemanifest.ToYAML(objects) + return objects.ToYAML() } // RemapImage normalizes a containers location if a user sets the AssetsLocation ContainerRegistry location. diff --git a/pkg/client/simple/BUILD.bazel b/pkg/client/simple/BUILD.bazel index bf1b613cba..f95d6c93f2 100644 --- a/pkg/client/simple/BUILD.bazel +++ b/pkg/client/simple/BUILD.bazel @@ -8,6 +8,7 @@ go_library( deps = [ "//pkg/apis/kops:go_default_library", "//pkg/client/clientset_generated/clientset/typed/kops/internalversion:go_default_library", + "//pkg/kubemanifest:go_default_library", "//upup/pkg/fi:go_default_library", "//util/pkg/vfs:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/pkg/client/simple/api/BUILD.bazel b/pkg/client/simple/api/BUILD.bazel index d6bbfcf15a..f457454fe5 100644 --- a/pkg/client/simple/api/BUILD.bazel +++ b/pkg/client/simple/api/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "//pkg/apis/kops/registry:go_default_library", "//pkg/apis/kops/validation:go_default_library", "//pkg/client/clientset_generated/clientset/typed/kops/internalversion:go_default_library", + "//pkg/client/simple:go_default_library", "//pkg/client/simple/vfsclientset:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/secrets:go_default_library", diff --git a/pkg/client/simple/api/clientset.go b/pkg/client/simple/api/clientset.go index 44cda7d152..cf7c0ff63b 100644 --- a/pkg/client/simple/api/clientset.go +++ b/pkg/client/simple/api/clientset.go @@ -29,6 +29,7 @@ import ( "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/pkg/apis/kops/validation" kopsinternalversion "k8s.io/kops/pkg/client/clientset_generated/clientset/typed/kops/internalversion" + "k8s.io/kops/pkg/client/simple" "k8s.io/kops/pkg/client/simple/vfsclientset" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/secrets" @@ -47,6 +48,13 @@ func (c *RESTClientset) GetCluster(ctx context.Context, name string) (*kops.Clus return c.KopsClient.Clusters(namespace).Get(ctx, name, metav1.GetOptions{}) } +// AddonsFor fetches the AddonsClient for the cluster +func (c *RESTClientset) AddonsFor(cluster *kops.Cluster) simple.AddonsClient { + // We should manage these directly in the cluster + klog.Fatalf("AddonsFor not implemented for RESTClientset") + return nil +} + // CreateCluster implements the CreateCluster method of Clientset for a kubernetes-API state store func (c *RESTClientset) CreateCluster(ctx context.Context, cluster *kops.Cluster) (*kops.Cluster, error) { namespace := restNamespaceForClusterName(cluster.Name) diff --git a/pkg/client/simple/clientset.go b/pkg/client/simple/clientset.go index 66f7b5b76a..86e9b0637d 100644 --- a/pkg/client/simple/clientset.go +++ b/pkg/client/simple/clientset.go @@ -22,6 +22,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kops/pkg/apis/kops" kopsinternalversion "k8s.io/kops/pkg/client/clientset_generated/clientset/typed/kops/internalversion" + "k8s.io/kops/pkg/kubemanifest" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/util/pkg/vfs" ) @@ -42,9 +43,12 @@ type Clientset interface { // ConfigBaseFor returns the vfs path where we will read configuration information from ConfigBaseFor(cluster *kops.Cluster) (vfs.Path, error) - // InstanceGroupsFor returns the InstanceGroupInterface bounds to the namespace for a particular Cluster + // InstanceGroupsFor returns the InstanceGroupInterface bound to the namespace for a particular Cluster InstanceGroupsFor(cluster *kops.Cluster) kopsinternalversion.InstanceGroupInterface + // AddonsFor returns the client for addon objects for a particular Cluster + AddonsFor(cluster *kops.Cluster) AddonsClient + // SecretStore builds the secret store for the specified cluster SecretStore(cluster *kops.Cluster) (fi.SecretStore, error) @@ -57,3 +61,13 @@ type Clientset interface { // DeleteCluster deletes all the state for the specified cluster DeleteCluster(ctx context.Context, cluster *kops.Cluster) error } + +// AddonsClient is a client for manipulating cluster addons +// Because we want to support storing these directly in a cluster, we don't group them +type AddonsClient interface { + // Replace replaces all the addon objects with the provided list + Replace(objects kubemanifest.ObjectList) error + + // List returns all the addon objects + List() (kubemanifest.ObjectList, error) +} diff --git a/pkg/client/simple/vfsclientset/BUILD.bazel b/pkg/client/simple/vfsclientset/BUILD.bazel index 0ad4c48f2d..cf66dd88be 100644 --- a/pkg/client/simple/vfsclientset/BUILD.bazel +++ b/pkg/client/simple/vfsclientset/BUILD.bazel @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", srcs = [ + "addons.go", "clientset.go", "cluster.go", "commonvfs.go", @@ -20,6 +21,7 @@ go_library( "//pkg/client/clientset_generated/clientset/typed/kops/internalversion:go_default_library", "//pkg/client/simple:go_default_library", "//pkg/kopscodecs:go_default_library", + "//pkg/kubemanifest:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/secrets:go_default_library", "//util/pkg/vfs:go_default_library", diff --git a/pkg/client/simple/vfsclientset/addons.go b/pkg/client/simple/vfsclientset/addons.go new file mode 100644 index 0000000000..c9e6e19d0f --- /dev/null +++ b/pkg/client/simple/vfsclientset/addons.go @@ -0,0 +1,96 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vfsclientset + +import ( + "bytes" + "fmt" + "os" + + "k8s.io/klog/v2" + "k8s.io/kops/pkg/acls" + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/client/simple" + "k8s.io/kops/pkg/kubemanifest" + "k8s.io/kops/util/pkg/vfs" +) + +type vfsAddonsClient struct { + basePath vfs.Path + + clusterName string + cluster *kops.Cluster +} + +var _ simple.AddonsClient = &vfsAddonsClient{} + +func newAddonsVFS(c *VFSClientset, cluster *kops.Cluster) *vfsAddonsClient { + if cluster == nil || cluster.Name == "" { + klog.Fatalf("cluster / cluster.Name is required") + } + + clusterName := cluster.Name + + r := &vfsAddonsClient{ + cluster: cluster, + clusterName: clusterName, + } + r.basePath = c.basePath.Join(clusterName, "clusteraddons") + + return r +} + +// TODO: Offer partial replacement? +func (c *vfsAddonsClient) Replace(addons kubemanifest.ObjectList) error { + b, err := addons.ToYAML() + if err != nil { + return err + } + + configPath := c.basePath.Join("default") + + acl, err := acls.GetACL(configPath, c.cluster) + if err != nil { + return err + } + + rs := bytes.NewReader(b) + if err := configPath.WriteFile(rs, acl); err != nil { + return fmt.Errorf("error writing addons file %s: %v", configPath, err) + } + + return nil +} + +func (c *vfsAddonsClient) List() (kubemanifest.ObjectList, error) { + configPath := c.basePath.Join("default") + + b, err := configPath.ReadFile() + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("error reading addons file %s: %v", configPath, err) + } + + objects, err := kubemanifest.LoadObjectsFrom(b) + if err != nil { + return nil, err + } + + return objects, nil +} diff --git a/pkg/client/simple/vfsclientset/clientset.go b/pkg/client/simple/vfsclientset/clientset.go index 40e2233949..e15a94aa6a 100644 --- a/pkg/client/simple/vfsclientset/clientset.go +++ b/pkg/client/simple/vfsclientset/clientset.go @@ -76,6 +76,10 @@ func (c *VFSClientset) InstanceGroupsFor(cluster *kops.Cluster) kopsinternalvers return newInstanceGroupVFS(c, cluster) } +func (c *VFSClientset) AddonsFor(cluster *kops.Cluster) simple.AddonsClient { + return newAddonsVFS(c, cluster) +} + func (c *VFSClientset) SecretStore(cluster *kops.Cluster) (fi.SecretStore, error) { if cluster.Spec.SecretStore == "" { configBase, err := registry.ConfigBase(cluster) @@ -145,6 +149,9 @@ func DeleteAllClusterState(basePath vfs.Path) error { if strings.HasPrefix(relativePath, "addons/") { continue } + if strings.HasPrefix(relativePath, "clusteraddons/") { + continue + } if strings.HasPrefix(relativePath, "pki/") { continue } diff --git a/pkg/clusteraddons/BUILD.bazel b/pkg/clusteraddons/BUILD.bazel new file mode 100644 index 0000000000..ecfcd84d3e --- /dev/null +++ b/pkg/clusteraddons/BUILD.bazel @@ -0,0 +1,13 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["load.go"], + importpath = "k8s.io/kops/pkg/clusteraddons", + visibility = ["//visibility:public"], + deps = [ + "//pkg/kubemanifest:go_default_library", + "//util/pkg/vfs:go_default_library", + "//vendor/k8s.io/klog/v2:go_default_library", + ], +) diff --git a/pkg/clusteraddons/load.go b/pkg/clusteraddons/load.go new file mode 100644 index 0000000000..139526409a --- /dev/null +++ b/pkg/clusteraddons/load.go @@ -0,0 +1,65 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clusteraddons + +import ( + "fmt" + "net/url" + + "k8s.io/klog/v2" + "k8s.io/kops/pkg/kubemanifest" + "k8s.io/kops/util/pkg/vfs" +) + +type ClusterAddon struct { + Raw string + Objects kubemanifest.ObjectList +} + +// LoadClusterAddon loads a set of objects from the specified VFS location +func LoadClusterAddon(location string) (*ClusterAddon, error) { + u, err := url.Parse(location) + if err != nil { + return nil, fmt.Errorf("invalid addon location: %q", location) + } + + // TODO: Should we support relative paths for "standard" addons? See equivalent code in LoadChannel + + resolved := u.String() + klog.V(2).Infof("Loading addon from %q", resolved) + addonBytes, err := vfs.Context.ReadFile(resolved) + if err != nil { + return nil, fmt.Errorf("error reading addon %q: %v", resolved, err) + } + addon, err := ParseClusterAddon(addonBytes) + if err != nil { + return nil, fmt.Errorf("error parsing addon %q: %v", resolved, err) + } + klog.V(4).Infof("Addon contents: %s", string(addonBytes)) + + return addon, nil +} + +// ParseClusterAddon parses a ClusterAddon object +func ParseClusterAddon(raw []byte) (*ClusterAddon, error) { + objects, err := kubemanifest.LoadObjectsFrom(raw) + if err != nil { + return nil, fmt.Errorf("error parsing addon %v", err) + } + + return &ClusterAddon{Raw: string(raw), Objects: objects}, nil +} diff --git a/pkg/featureflag/featureflag.go b/pkg/featureflag/featureflag.go index 3ae14150ab..715a135066 100644 --- a/pkg/featureflag/featureflag.go +++ b/pkg/featureflag/featureflag.go @@ -92,6 +92,8 @@ var ( Terraform012 = New("Terraform-0.12", Bool(true)) // LegacyIAM will permit use of legacy IAM permissions. LegacyIAM = New("LegacyIAM", Bool(false)) + // ClusterAddons activates experimental cluster-addons support + ClusterAddons = New("ClusterAddons", Bool(false)) ) // FeatureFlag defines a feature flag diff --git a/pkg/kubemanifest/manifest.go b/pkg/kubemanifest/manifest.go index 8c689a6aac..91f19aa06b 100644 --- a/pkg/kubemanifest/manifest.go +++ b/pkg/kubemanifest/manifest.go @@ -30,14 +30,21 @@ type Object struct { data map[string]interface{} } +// ObjectList describes a list of objects, allowing us to add bulk-methods +type ObjectList []*Object + // LoadObjectsFrom parses multiple objects from a yaml file -func LoadObjectsFrom(contents []byte) ([]*Object, error) { +func LoadObjectsFrom(contents []byte) (ObjectList, error) { var objects []*Object - // TODO: Support more separators? sections := text.SplitContentToSections(contents) for _, section := range sections { + // We need this so we don't error on a section that is empty / commented out + if !hasYAMLContent(section) { + continue + } + data := make(map[string]interface{}) err := yaml.Unmarshal(section, &data) if err != nil { @@ -54,11 +61,24 @@ func LoadObjectsFrom(contents []byte) ([]*Object, error) { return objects, nil } +// hasYAMLContent determines if the byte slice has any content, +// because yaml parsing gives an error if called with no content. +// TODO: How does apimachinery avoid this problem? +func hasYAMLContent(yamlData []byte) bool { + for _, line := range bytes.Split(yamlData, []byte("\n")) { + l := bytes.TrimSpace(line) + if len(l) != 0 && !bytes.HasPrefix(l, []byte("#")) { + return true + } + } + return false +} + // ToYAML serializes a list of objects back to bytes; it is the opposite of LoadObjectsFrom -func ToYAML(objects []*Object) ([]byte, error) { +func (l ObjectList) ToYAML() ([]byte, error) { var yamlSeparator = []byte("\n---\n\n") var yamls [][]byte - for _, object := range objects { + for _, object := range l { // Don't serialize empty objects - they confuse yaml parsers if object.IsEmptyObject() { continue diff --git a/upup/pkg/fi/cloudup/BUILD.bazel b/upup/pkg/fi/cloudup/BUILD.bazel index 5be5cd8632..30671269ea 100644 --- a/upup/pkg/fi/cloudup/BUILD.bazel +++ b/upup/pkg/fi/cloudup/BUILD.bazel @@ -42,6 +42,7 @@ go_library( "//pkg/client/simple/vfsclientset:go_default_library", "//pkg/dns:go_default_library", "//pkg/featureflag:go_default_library", + "//pkg/kubemanifest:go_default_library", "//pkg/model:go_default_library", "//pkg/model/alimodel:go_default_library", "//pkg/model/awsmodel:go_default_library", diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index f732a55ac0..ab4f0891cb 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -299,6 +299,12 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error { return err } + addonsClient := c.Clientset.AddonsFor(cluster) + addons, err := addonsClient.List() + if err != nil { + return fmt.Errorf("error fetching addons: %v", err) + } + // Normalize k8s version versionWithoutV := strings.TrimSpace(cluster.Spec.KubernetesVersion) versionWithoutV = strings.TrimPrefix(versionWithoutV, "v") @@ -478,6 +484,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error { Lifecycle: &clusterLifecycle, assetBuilder: assetBuilder, templates: templates, + ClusterAddons: addons, }, &model.PKIModelBuilder{ KopsModelContext: modelContext, diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index f43840e164..8f78975a65 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -27,6 +27,7 @@ import ( "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/assets" "k8s.io/kops/pkg/featureflag" + "k8s.io/kops/pkg/kubemanifest" "k8s.io/kops/pkg/model" "k8s.io/kops/pkg/templates" "k8s.io/kops/upup/pkg/fi" @@ -37,9 +38,10 @@ import ( // BootstrapChannelBuilder is responsible for handling the addons in channels type BootstrapChannelBuilder struct { *model.KopsModelContext - Lifecycle *fi.Lifecycle - templates *templates.Templates - assetBuilder *assets.AssetBuilder + ClusterAddons kubemanifest.ObjectList + Lifecycle *fi.Lifecycle + templates *templates.Templates + assetBuilder *assets.AssetBuilder } var _ fi.ModelBuilder = &BootstrapChannelBuilder{} @@ -51,8 +53,6 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error { return err } - tasks := c.Tasks - for _, a := range addons.Spec.Addons { key := *a.Name if a.Id != "" { @@ -91,13 +91,53 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error { } a.ManifestHash = manifestHash - tasks[name] = &fitasks.ManagedFile{ + c.AddTask(&fitasks.ManagedFile{ Contents: fi.WrapResource(fi.NewBytesResource(manifestBytes)), Lifecycle: b.Lifecycle, Location: fi.String(manifestPath), Name: fi.String(name), + }) + } + + if b.ClusterAddons != nil { + key := "cluster-addons.kops.k8s.io" + version := "0.0.0" + location := key + "/default.yaml" + + a := &channelsapi.AddonSpec{ + Name: fi.String(key), + Version: fi.String(version), + Selector: map[string]string{"k8s-addon": key}, + Manifest: fi.String(location), } + name := b.Cluster.ObjectMeta.Name + "-addons-" + key + manifestPath := "addons/" + *a.Manifest + + manifestBytes, err := b.ClusterAddons.ToYAML() + if err != nil { + return fmt.Errorf("error serializing addons: %v", err) + } + + // Trim whitespace + manifestBytes = []byte(strings.TrimSpace(string(manifestBytes))) + + rawManifest := string(manifestBytes) + + manifestHash, err := utils.HashString(rawManifest) + if err != nil { + return fmt.Errorf("error hashing manifest: %v", err) + } + a.ManifestHash = manifestHash + + c.AddTask(&fitasks.ManagedFile{ + Contents: fi.WrapResource(fi.NewBytesResource(manifestBytes)), + Lifecycle: b.Lifecycle, + Location: fi.String(manifestPath), + Name: fi.String(name), + }) + + addons.Spec.Addons = append(addons.Spec.Addons, a) } addonsYAML, err := utils.YamlMarshal(addons) @@ -107,12 +147,12 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error { name := b.Cluster.ObjectMeta.Name + "-addons-bootstrap" - tasks[name] = &fitasks.ManagedFile{ + c.AddTask(&fitasks.ManagedFile{ Contents: fi.WrapResource(fi.NewBytesResource(addonsYAML)), Lifecycle: b.Lifecycle, Location: fi.String("addons/bootstrap-channel.yaml"), Name: fi.String(name), - } + }) return nil } diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go index d9fbde4d3c..964473ba04 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go @@ -118,7 +118,7 @@ func runChannelBuilderTest(t *testing.T, key string, addonManifests []string) { { name := cluster.ObjectMeta.Name + "-addons-bootstrap" - manifestTask := context.Tasks[name] + manifestTask := context.Tasks["ManagedFile/"+name] if manifestTask == nil { t.Fatalf("manifest task not found (%q)", name) } @@ -135,7 +135,7 @@ func runChannelBuilderTest(t *testing.T, key string, addonManifests []string) { for _, k := range addonManifests { name := cluster.ObjectMeta.Name + "-addons-" + k - manifestTask := context.Tasks[name] + manifestTask := context.Tasks["ManagedFile/"+name] if manifestTask == nil { for k := range context.Tasks { t.Logf("found task %s", k) diff --git a/upup/pkg/kutil/convert_kubeup_cluster.go b/upup/pkg/kutil/convert_kubeup_cluster.go index 1b34cb735d..8386c9b6c0 100644 --- a/upup/pkg/kutil/convert_kubeup_cluster.go +++ b/upup/pkg/kutil/convert_kubeup_cluster.go @@ -465,7 +465,7 @@ func (x *ConvertKubeupCluster) Upgrade(ctx context.Context) error { } } - err = registry.CreateClusterConfig(ctx, x.Clientset, cluster, x.InstanceGroups) + err = registry.CreateClusterConfig(ctx, x.Clientset, cluster, x.InstanceGroups, nil) if err != nil { return fmt.Errorf("error writing updated configuration: %v", err) } diff --git a/upup/pkg/kutil/import_cluster.go b/upup/pkg/kutil/import_cluster.go index 3b11808a4f..7e5a181ac0 100644 --- a/upup/pkg/kutil/import_cluster.go +++ b/upup/pkg/kutil/import_cluster.go @@ -488,7 +488,7 @@ func (x *ImportCluster) ImportAWSCluster(ctx context.Context) error { fullInstanceGroups = append(fullInstanceGroups, full) } - err = registry.CreateClusterConfig(ctx, x.Clientset, cluster, fullInstanceGroups) + err = registry.CreateClusterConfig(ctx, x.Clientset, cluster, fullInstanceGroups, nil) if err != nil { return err }