diff --git a/cmd/kops/BUILD.bazel b/cmd/kops/BUILD.bazel index bf658890cd..d8ed70e52f 100644 --- a/cmd/kops/BUILD.bazel +++ b/cmd/kops/BUILD.bazel @@ -67,6 +67,7 @@ go_library( "//pkg/assets:go_default_library", "//pkg/client/simple:go_default_library", "//pkg/cloudinstances:go_default_library", + "//pkg/clusteraddons:go_default_library", "//pkg/commands:go_default_library", "//pkg/dump:go_default_library", "//pkg/edit:go_default_library", @@ -75,6 +76,7 @@ go_library( "//pkg/instancegroups:go_default_library", "//pkg/kopscodecs:go_default_library", "//pkg/kubeconfig:go_default_library", + "//pkg/kubemanifest:go_default_library", "//pkg/pki:go_default_library", "//pkg/pretty:go_default_library", "//pkg/resources:go_default_library", diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index e345e4724d..f23d952fd8 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -35,9 +35,11 @@ import ( "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/pkg/apis/kops/validation" "k8s.io/kops/pkg/assets" + "k8s.io/kops/pkg/clusteraddons" "k8s.io/kops/pkg/commands" "k8s.io/kops/pkg/featureflag" "k8s.io/kops/pkg/kubeconfig" + "k8s.io/kops/pkg/kubemanifest" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup" "k8s.io/kops/upup/pkg/fi/utils" @@ -87,6 +89,9 @@ type CreateClusterOptions struct { DryRun bool // Output type during a DryRun Output string + + // AddonPaths specify paths to additional components that we can add to a cluster + AddonPaths []string } func (o *CreateClusterOptions) InitDefaults() { @@ -209,6 +214,10 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command { cmd.Flags().StringSliceVar(&options.Zones, "zones", options.Zones, "Zones in which to run the cluster") cmd.Flags().StringSliceVar(&options.MasterZones, "master-zones", options.MasterZones, "Zones in which to run masters (must be an odd number)") + if featureflag.ClusterAddons.Enabled() { + cmd.Flags().StringSliceVar(&options.AddonPaths, "add", options.AddonPaths, "Paths to addons we should add to the cluster") + } + cmd.Flags().StringVar(&options.KubernetesVersion, "kubernetes-version", options.KubernetesVersion, "Version of kubernetes to run (defaults to version in channel)") cmd.Flags().StringVar(&options.ContainerRuntime, "container-runtime", options.ContainerRuntime, "Container runtime to use: containerd, docker") @@ -545,8 +554,17 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr } } + var addons kubemanifest.ObjectList + for _, p := range c.AddonPaths { + addon, err := clusteraddons.LoadClusterAddon(p) + if err != nil { + return fmt.Errorf("error loading cluster addon %s: %v", p, err) + } + addons = append(addons, addon.Objects...) + } + // Note we perform as much validation as we can, before writing a bad config - err = registry.CreateClusterConfig(ctx, clientset, cluster, fullInstanceGroups) + err = registry.CreateClusterConfig(ctx, clientset, cluster, fullInstanceGroups, addons) if err != nil { return fmt.Errorf("error writing updated configuration: %v", err) } diff --git a/hack/.packages b/hack/.packages index 6b368f3db9..a5d598f443 100644 --- a/hack/.packages +++ b/hack/.packages @@ -82,6 +82,7 @@ k8s.io/kops/pkg/client/simple k8s.io/kops/pkg/client/simple/api k8s.io/kops/pkg/client/simple/vfsclientset k8s.io/kops/pkg/cloudinstances +k8s.io/kops/pkg/clusteraddons k8s.io/kops/pkg/commands k8s.io/kops/pkg/configbuilder k8s.io/kops/pkg/diff diff --git a/pkg/apis/kops/registry/BUILD.bazel b/pkg/apis/kops/registry/BUILD.bazel index 274de0622b..1ff321c416 100644 --- a/pkg/apis/kops/registry/BUILD.bazel +++ b/pkg/apis/kops/registry/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "//pkg/acls:go_default_library", "//pkg/apis/kops:go_default_library", "//pkg/client/simple:go_default_library", + "//pkg/kubemanifest:go_default_library", "//upup/pkg/fi/utils:go_default_library", "//util/pkg/vfs:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/pkg/apis/kops/registry/helpers.go b/pkg/apis/kops/registry/helpers.go index 701b4dddee..fce4025b1a 100644 --- a/pkg/apis/kops/registry/helpers.go +++ b/pkg/apis/kops/registry/helpers.go @@ -23,9 +23,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" api "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/client/simple" + "k8s.io/kops/pkg/kubemanifest" ) -func CreateClusterConfig(ctx context.Context, clientset simple.Clientset, cluster *api.Cluster, groups []*api.InstanceGroup) error { +func CreateClusterConfig(ctx context.Context, clientset simple.Clientset, cluster *api.Cluster, groups []*api.InstanceGroup, addons kubemanifest.ObjectList) error { // Check for instancegroup Name duplicates before writing { names := map[string]bool{} @@ -52,5 +53,13 @@ func CreateClusterConfig(ctx context.Context, clientset simple.Clientset, cluste } } + { + addonsClient := clientset.AddonsFor(cluster) + + if err := addonsClient.Replace(addons); err != nil { + return fmt.Errorf("error writing updated addon configuration: %v", err) + } + } + return nil } diff --git a/pkg/assets/builder.go b/pkg/assets/builder.go index 74257e1375..82cecb487d 100644 --- a/pkg/assets/builder.go +++ b/pkg/assets/builder.go @@ -123,7 +123,7 @@ func (a *AssetBuilder) RemapManifest(data []byte) ([]byte, error) { } } - return kubemanifest.ToYAML(objects) + return objects.ToYAML() } // RemapImage normalizes a containers location if a user sets the AssetsLocation ContainerRegistry location. diff --git a/pkg/client/simple/BUILD.bazel b/pkg/client/simple/BUILD.bazel index bf1b613cba..f95d6c93f2 100644 --- a/pkg/client/simple/BUILD.bazel +++ b/pkg/client/simple/BUILD.bazel @@ -8,6 +8,7 @@ go_library( deps = [ "//pkg/apis/kops:go_default_library", "//pkg/client/clientset_generated/clientset/typed/kops/internalversion:go_default_library", + "//pkg/kubemanifest:go_default_library", "//upup/pkg/fi:go_default_library", "//util/pkg/vfs:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/pkg/client/simple/api/BUILD.bazel b/pkg/client/simple/api/BUILD.bazel index d6bbfcf15a..f457454fe5 100644 --- a/pkg/client/simple/api/BUILD.bazel +++ b/pkg/client/simple/api/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "//pkg/apis/kops/registry:go_default_library", "//pkg/apis/kops/validation:go_default_library", "//pkg/client/clientset_generated/clientset/typed/kops/internalversion:go_default_library", + "//pkg/client/simple:go_default_library", "//pkg/client/simple/vfsclientset:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/secrets:go_default_library", diff --git a/pkg/client/simple/api/clientset.go b/pkg/client/simple/api/clientset.go index 44cda7d152..cf7c0ff63b 100644 --- a/pkg/client/simple/api/clientset.go +++ b/pkg/client/simple/api/clientset.go @@ -29,6 +29,7 @@ import ( "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/pkg/apis/kops/validation" kopsinternalversion "k8s.io/kops/pkg/client/clientset_generated/clientset/typed/kops/internalversion" + "k8s.io/kops/pkg/client/simple" "k8s.io/kops/pkg/client/simple/vfsclientset" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/secrets" @@ -47,6 +48,13 @@ func (c *RESTClientset) GetCluster(ctx context.Context, name string) (*kops.Clus return c.KopsClient.Clusters(namespace).Get(ctx, name, metav1.GetOptions{}) } +// AddonsFor fetches the AddonsClient for the cluster +func (c *RESTClientset) AddonsFor(cluster *kops.Cluster) simple.AddonsClient { + // We should manage these directly in the cluster + klog.Fatalf("AddonsFor not implemented for RESTClientset") + return nil +} + // CreateCluster implements the CreateCluster method of Clientset for a kubernetes-API state store func (c *RESTClientset) CreateCluster(ctx context.Context, cluster *kops.Cluster) (*kops.Cluster, error) { namespace := restNamespaceForClusterName(cluster.Name) diff --git a/pkg/client/simple/clientset.go b/pkg/client/simple/clientset.go index 66f7b5b76a..86e9b0637d 100644 --- a/pkg/client/simple/clientset.go +++ b/pkg/client/simple/clientset.go @@ -22,6 +22,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kops/pkg/apis/kops" kopsinternalversion "k8s.io/kops/pkg/client/clientset_generated/clientset/typed/kops/internalversion" + "k8s.io/kops/pkg/kubemanifest" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/util/pkg/vfs" ) @@ -42,9 +43,12 @@ type Clientset interface { // ConfigBaseFor returns the vfs path where we will read configuration information from ConfigBaseFor(cluster *kops.Cluster) (vfs.Path, error) - // InstanceGroupsFor returns the InstanceGroupInterface bounds to the namespace for a particular Cluster + // InstanceGroupsFor returns the InstanceGroupInterface bound to the namespace for a particular Cluster InstanceGroupsFor(cluster *kops.Cluster) kopsinternalversion.InstanceGroupInterface + // AddonsFor returns the client for addon objects for a particular Cluster + AddonsFor(cluster *kops.Cluster) AddonsClient + // SecretStore builds the secret store for the specified cluster SecretStore(cluster *kops.Cluster) (fi.SecretStore, error) @@ -57,3 +61,13 @@ type Clientset interface { // DeleteCluster deletes all the state for the specified cluster DeleteCluster(ctx context.Context, cluster *kops.Cluster) error } + +// AddonsClient is a client for manipulating cluster addons +// Because we want to support storing these directly in a cluster, we don't group them +type AddonsClient interface { + // Replace replaces all the addon objects with the provided list + Replace(objects kubemanifest.ObjectList) error + + // List returns all the addon objects + List() (kubemanifest.ObjectList, error) +} diff --git a/pkg/client/simple/vfsclientset/BUILD.bazel b/pkg/client/simple/vfsclientset/BUILD.bazel index 0ad4c48f2d..cf66dd88be 100644 --- a/pkg/client/simple/vfsclientset/BUILD.bazel +++ b/pkg/client/simple/vfsclientset/BUILD.bazel @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", srcs = [ + "addons.go", "clientset.go", "cluster.go", "commonvfs.go", @@ -20,6 +21,7 @@ go_library( "//pkg/client/clientset_generated/clientset/typed/kops/internalversion:go_default_library", "//pkg/client/simple:go_default_library", "//pkg/kopscodecs:go_default_library", + "//pkg/kubemanifest:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/secrets:go_default_library", "//util/pkg/vfs:go_default_library", diff --git a/pkg/client/simple/vfsclientset/addons.go b/pkg/client/simple/vfsclientset/addons.go new file mode 100644 index 0000000000..c9e6e19d0f --- /dev/null +++ b/pkg/client/simple/vfsclientset/addons.go @@ -0,0 +1,96 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vfsclientset + +import ( + "bytes" + "fmt" + "os" + + "k8s.io/klog/v2" + "k8s.io/kops/pkg/acls" + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/client/simple" + "k8s.io/kops/pkg/kubemanifest" + "k8s.io/kops/util/pkg/vfs" +) + +type vfsAddonsClient struct { + basePath vfs.Path + + clusterName string + cluster *kops.Cluster +} + +var _ simple.AddonsClient = &vfsAddonsClient{} + +func newAddonsVFS(c *VFSClientset, cluster *kops.Cluster) *vfsAddonsClient { + if cluster == nil || cluster.Name == "" { + klog.Fatalf("cluster / cluster.Name is required") + } + + clusterName := cluster.Name + + r := &vfsAddonsClient{ + cluster: cluster, + clusterName: clusterName, + } + r.basePath = c.basePath.Join(clusterName, "clusteraddons") + + return r +} + +// TODO: Offer partial replacement? +func (c *vfsAddonsClient) Replace(addons kubemanifest.ObjectList) error { + b, err := addons.ToYAML() + if err != nil { + return err + } + + configPath := c.basePath.Join("default") + + acl, err := acls.GetACL(configPath, c.cluster) + if err != nil { + return err + } + + rs := bytes.NewReader(b) + if err := configPath.WriteFile(rs, acl); err != nil { + return fmt.Errorf("error writing addons file %s: %v", configPath, err) + } + + return nil +} + +func (c *vfsAddonsClient) List() (kubemanifest.ObjectList, error) { + configPath := c.basePath.Join("default") + + b, err := configPath.ReadFile() + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("error reading addons file %s: %v", configPath, err) + } + + objects, err := kubemanifest.LoadObjectsFrom(b) + if err != nil { + return nil, err + } + + return objects, nil +} diff --git a/pkg/client/simple/vfsclientset/clientset.go b/pkg/client/simple/vfsclientset/clientset.go index 40e2233949..e15a94aa6a 100644 --- a/pkg/client/simple/vfsclientset/clientset.go +++ b/pkg/client/simple/vfsclientset/clientset.go @@ -76,6 +76,10 @@ func (c *VFSClientset) InstanceGroupsFor(cluster *kops.Cluster) kopsinternalvers return newInstanceGroupVFS(c, cluster) } +func (c *VFSClientset) AddonsFor(cluster *kops.Cluster) simple.AddonsClient { + return newAddonsVFS(c, cluster) +} + func (c *VFSClientset) SecretStore(cluster *kops.Cluster) (fi.SecretStore, error) { if cluster.Spec.SecretStore == "" { configBase, err := registry.ConfigBase(cluster) @@ -145,6 +149,9 @@ func DeleteAllClusterState(basePath vfs.Path) error { if strings.HasPrefix(relativePath, "addons/") { continue } + if strings.HasPrefix(relativePath, "clusteraddons/") { + continue + } if strings.HasPrefix(relativePath, "pki/") { continue } diff --git a/pkg/clusteraddons/BUILD.bazel b/pkg/clusteraddons/BUILD.bazel new file mode 100644 index 0000000000..ecfcd84d3e --- /dev/null +++ b/pkg/clusteraddons/BUILD.bazel @@ -0,0 +1,13 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["load.go"], + importpath = "k8s.io/kops/pkg/clusteraddons", + visibility = ["//visibility:public"], + deps = [ + "//pkg/kubemanifest:go_default_library", + "//util/pkg/vfs:go_default_library", + "//vendor/k8s.io/klog/v2:go_default_library", + ], +) diff --git a/pkg/clusteraddons/load.go b/pkg/clusteraddons/load.go new file mode 100644 index 0000000000..139526409a --- /dev/null +++ b/pkg/clusteraddons/load.go @@ -0,0 +1,65 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clusteraddons + +import ( + "fmt" + "net/url" + + "k8s.io/klog/v2" + "k8s.io/kops/pkg/kubemanifest" + "k8s.io/kops/util/pkg/vfs" +) + +type ClusterAddon struct { + Raw string + Objects kubemanifest.ObjectList +} + +// LoadClusterAddon loads a set of objects from the specified VFS location +func LoadClusterAddon(location string) (*ClusterAddon, error) { + u, err := url.Parse(location) + if err != nil { + return nil, fmt.Errorf("invalid addon location: %q", location) + } + + // TODO: Should we support relative paths for "standard" addons? See equivalent code in LoadChannel + + resolved := u.String() + klog.V(2).Infof("Loading addon from %q", resolved) + addonBytes, err := vfs.Context.ReadFile(resolved) + if err != nil { + return nil, fmt.Errorf("error reading addon %q: %v", resolved, err) + } + addon, err := ParseClusterAddon(addonBytes) + if err != nil { + return nil, fmt.Errorf("error parsing addon %q: %v", resolved, err) + } + klog.V(4).Infof("Addon contents: %s", string(addonBytes)) + + return addon, nil +} + +// ParseClusterAddon parses a ClusterAddon object +func ParseClusterAddon(raw []byte) (*ClusterAddon, error) { + objects, err := kubemanifest.LoadObjectsFrom(raw) + if err != nil { + return nil, fmt.Errorf("error parsing addon %v", err) + } + + return &ClusterAddon{Raw: string(raw), Objects: objects}, nil +} diff --git a/pkg/featureflag/featureflag.go b/pkg/featureflag/featureflag.go index 3ae14150ab..715a135066 100644 --- a/pkg/featureflag/featureflag.go +++ b/pkg/featureflag/featureflag.go @@ -92,6 +92,8 @@ var ( Terraform012 = New("Terraform-0.12", Bool(true)) // LegacyIAM will permit use of legacy IAM permissions. LegacyIAM = New("LegacyIAM", Bool(false)) + // ClusterAddons activates experimental cluster-addons support + ClusterAddons = New("ClusterAddons", Bool(false)) ) // FeatureFlag defines a feature flag diff --git a/pkg/kubemanifest/manifest.go b/pkg/kubemanifest/manifest.go index 8c689a6aac..91f19aa06b 100644 --- a/pkg/kubemanifest/manifest.go +++ b/pkg/kubemanifest/manifest.go @@ -30,14 +30,21 @@ type Object struct { data map[string]interface{} } +// ObjectList describes a list of objects, allowing us to add bulk-methods +type ObjectList []*Object + // LoadObjectsFrom parses multiple objects from a yaml file -func LoadObjectsFrom(contents []byte) ([]*Object, error) { +func LoadObjectsFrom(contents []byte) (ObjectList, error) { var objects []*Object - // TODO: Support more separators? sections := text.SplitContentToSections(contents) for _, section := range sections { + // We need this so we don't error on a section that is empty / commented out + if !hasYAMLContent(section) { + continue + } + data := make(map[string]interface{}) err := yaml.Unmarshal(section, &data) if err != nil { @@ -54,11 +61,24 @@ func LoadObjectsFrom(contents []byte) ([]*Object, error) { return objects, nil } +// hasYAMLContent determines if the byte slice has any content, +// because yaml parsing gives an error if called with no content. +// TODO: How does apimachinery avoid this problem? +func hasYAMLContent(yamlData []byte) bool { + for _, line := range bytes.Split(yamlData, []byte("\n")) { + l := bytes.TrimSpace(line) + if len(l) != 0 && !bytes.HasPrefix(l, []byte("#")) { + return true + } + } + return false +} + // ToYAML serializes a list of objects back to bytes; it is the opposite of LoadObjectsFrom -func ToYAML(objects []*Object) ([]byte, error) { +func (l ObjectList) ToYAML() ([]byte, error) { var yamlSeparator = []byte("\n---\n\n") var yamls [][]byte - for _, object := range objects { + for _, object := range l { // Don't serialize empty objects - they confuse yaml parsers if object.IsEmptyObject() { continue diff --git a/upup/pkg/fi/cloudup/BUILD.bazel b/upup/pkg/fi/cloudup/BUILD.bazel index 5be5cd8632..30671269ea 100644 --- a/upup/pkg/fi/cloudup/BUILD.bazel +++ b/upup/pkg/fi/cloudup/BUILD.bazel @@ -42,6 +42,7 @@ go_library( "//pkg/client/simple/vfsclientset:go_default_library", "//pkg/dns:go_default_library", "//pkg/featureflag:go_default_library", + "//pkg/kubemanifest:go_default_library", "//pkg/model:go_default_library", "//pkg/model/alimodel:go_default_library", "//pkg/model/awsmodel:go_default_library", diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index f732a55ac0..ab4f0891cb 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -299,6 +299,12 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error { return err } + addonsClient := c.Clientset.AddonsFor(cluster) + addons, err := addonsClient.List() + if err != nil { + return fmt.Errorf("error fetching addons: %v", err) + } + // Normalize k8s version versionWithoutV := strings.TrimSpace(cluster.Spec.KubernetesVersion) versionWithoutV = strings.TrimPrefix(versionWithoutV, "v") @@ -478,6 +484,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error { Lifecycle: &clusterLifecycle, assetBuilder: assetBuilder, templates: templates, + ClusterAddons: addons, }, &model.PKIModelBuilder{ KopsModelContext: modelContext, diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index f43840e164..8f78975a65 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -27,6 +27,7 @@ import ( "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/assets" "k8s.io/kops/pkg/featureflag" + "k8s.io/kops/pkg/kubemanifest" "k8s.io/kops/pkg/model" "k8s.io/kops/pkg/templates" "k8s.io/kops/upup/pkg/fi" @@ -37,9 +38,10 @@ import ( // BootstrapChannelBuilder is responsible for handling the addons in channels type BootstrapChannelBuilder struct { *model.KopsModelContext - Lifecycle *fi.Lifecycle - templates *templates.Templates - assetBuilder *assets.AssetBuilder + ClusterAddons kubemanifest.ObjectList + Lifecycle *fi.Lifecycle + templates *templates.Templates + assetBuilder *assets.AssetBuilder } var _ fi.ModelBuilder = &BootstrapChannelBuilder{} @@ -51,8 +53,6 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error { return err } - tasks := c.Tasks - for _, a := range addons.Spec.Addons { key := *a.Name if a.Id != "" { @@ -91,13 +91,53 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error { } a.ManifestHash = manifestHash - tasks[name] = &fitasks.ManagedFile{ + c.AddTask(&fitasks.ManagedFile{ Contents: fi.WrapResource(fi.NewBytesResource(manifestBytes)), Lifecycle: b.Lifecycle, Location: fi.String(manifestPath), Name: fi.String(name), + }) + } + + if b.ClusterAddons != nil { + key := "cluster-addons.kops.k8s.io" + version := "0.0.0" + location := key + "/default.yaml" + + a := &channelsapi.AddonSpec{ + Name: fi.String(key), + Version: fi.String(version), + Selector: map[string]string{"k8s-addon": key}, + Manifest: fi.String(location), } + name := b.Cluster.ObjectMeta.Name + "-addons-" + key + manifestPath := "addons/" + *a.Manifest + + manifestBytes, err := b.ClusterAddons.ToYAML() + if err != nil { + return fmt.Errorf("error serializing addons: %v", err) + } + + // Trim whitespace + manifestBytes = []byte(strings.TrimSpace(string(manifestBytes))) + + rawManifest := string(manifestBytes) + + manifestHash, err := utils.HashString(rawManifest) + if err != nil { + return fmt.Errorf("error hashing manifest: %v", err) + } + a.ManifestHash = manifestHash + + c.AddTask(&fitasks.ManagedFile{ + Contents: fi.WrapResource(fi.NewBytesResource(manifestBytes)), + Lifecycle: b.Lifecycle, + Location: fi.String(manifestPath), + Name: fi.String(name), + }) + + addons.Spec.Addons = append(addons.Spec.Addons, a) } addonsYAML, err := utils.YamlMarshal(addons) @@ -107,12 +147,12 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error { name := b.Cluster.ObjectMeta.Name + "-addons-bootstrap" - tasks[name] = &fitasks.ManagedFile{ + c.AddTask(&fitasks.ManagedFile{ Contents: fi.WrapResource(fi.NewBytesResource(addonsYAML)), Lifecycle: b.Lifecycle, Location: fi.String("addons/bootstrap-channel.yaml"), Name: fi.String(name), - } + }) return nil } diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go index d9fbde4d3c..964473ba04 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go @@ -118,7 +118,7 @@ func runChannelBuilderTest(t *testing.T, key string, addonManifests []string) { { name := cluster.ObjectMeta.Name + "-addons-bootstrap" - manifestTask := context.Tasks[name] + manifestTask := context.Tasks["ManagedFile/"+name] if manifestTask == nil { t.Fatalf("manifest task not found (%q)", name) } @@ -135,7 +135,7 @@ func runChannelBuilderTest(t *testing.T, key string, addonManifests []string) { for _, k := range addonManifests { name := cluster.ObjectMeta.Name + "-addons-" + k - manifestTask := context.Tasks[name] + manifestTask := context.Tasks["ManagedFile/"+name] if manifestTask == nil { for k := range context.Tasks { t.Logf("found task %s", k) diff --git a/upup/pkg/kutil/convert_kubeup_cluster.go b/upup/pkg/kutil/convert_kubeup_cluster.go index 1b34cb735d..8386c9b6c0 100644 --- a/upup/pkg/kutil/convert_kubeup_cluster.go +++ b/upup/pkg/kutil/convert_kubeup_cluster.go @@ -465,7 +465,7 @@ func (x *ConvertKubeupCluster) Upgrade(ctx context.Context) error { } } - err = registry.CreateClusterConfig(ctx, x.Clientset, cluster, x.InstanceGroups) + err = registry.CreateClusterConfig(ctx, x.Clientset, cluster, x.InstanceGroups, nil) if err != nil { return fmt.Errorf("error writing updated configuration: %v", err) } diff --git a/upup/pkg/kutil/import_cluster.go b/upup/pkg/kutil/import_cluster.go index 3b11808a4f..7e5a181ac0 100644 --- a/upup/pkg/kutil/import_cluster.go +++ b/upup/pkg/kutil/import_cluster.go @@ -488,7 +488,7 @@ func (x *ImportCluster) ImportAWSCluster(ctx context.Context) error { fullInstanceGroups = append(fullInstanceGroups, full) } - err = registry.CreateClusterConfig(ctx, x.Clientset, cluster, fullInstanceGroups) + err = registry.CreateClusterConfig(ctx, x.Clientset, cluster, fullInstanceGroups, nil) if err != nil { return err }