Addons: Support arbitrary additional objects

We will be managing cluster addons using CRDs, and so we want to be
able to apply arbitrary objects as part of cluster bringup.

Start by allowing (behind a feature-flag) for arbitrary objects to be
specified.

Co-authored-by: John Gardiner Myers <jgmyers@proofpoint.com>

cmd/kops/BUILD.bazel

@@ -67,6 +67,7 @@ go_library(
         "//pkg/assets:go_default_library",
         "//pkg/client/simple:go_default_library",
         "//pkg/cloudinstances:go_default_library",
+        "//pkg/clusteraddons:go_default_library",
         "//pkg/commands:go_default_library",
         "//pkg/dump:go_default_library",
         "//pkg/edit:go_default_library",
@@ -75,6 +76,7 @@ go_library(
         "//pkg/instancegroups:go_default_library",
         "//pkg/kopscodecs:go_default_library",
         "//pkg/kubeconfig:go_default_library",
+        "//pkg/kubemanifest:go_default_library",
         "//pkg/pki:go_default_library",
         "//pkg/pretty:go_default_library",
         "//pkg/resources:go_default_library",

cmd/kops/create_cluster.go

@@ -35,9 +35,11 @@ import (
 	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/pkg/apis/kops/validation"
 	"k8s.io/kops/pkg/assets"
+	"k8s.io/kops/pkg/clusteraddons"
 	"k8s.io/kops/pkg/commands"
 	"k8s.io/kops/pkg/featureflag"
 	"k8s.io/kops/pkg/kubeconfig"
+	"k8s.io/kops/pkg/kubemanifest"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup"
 	"k8s.io/kops/upup/pkg/fi/utils"
@@ -87,6 +89,9 @@ type CreateClusterOptions struct {
 	DryRun bool
 	// Output type during a DryRun
 	Output string
+
+	// AddonPaths specify paths to additional components that we can add to a cluster
+	AddonPaths []string
 }
 
 func (o *CreateClusterOptions) InitDefaults() {
@@ -209,6 +214,10 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 	cmd.Flags().StringSliceVar(&options.Zones, "zones", options.Zones, "Zones in which to run the cluster")
 	cmd.Flags().StringSliceVar(&options.MasterZones, "master-zones", options.MasterZones, "Zones in which to run masters (must be an odd number)")
 
+	if featureflag.ClusterAddons.Enabled() {
+		cmd.Flags().StringSliceVar(&options.AddonPaths, "add", options.AddonPaths, "Paths to addons we should add to the cluster")
+	}
+
 	cmd.Flags().StringVar(&options.KubernetesVersion, "kubernetes-version", options.KubernetesVersion, "Version of kubernetes to run (defaults to version in channel)")
 
 	cmd.Flags().StringVar(&options.ContainerRuntime, "container-runtime", options.ContainerRuntime, "Container runtime to use: containerd, docker")
@@ -545,8 +554,17 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr
 		}
 	}
 
+	var addons kubemanifest.ObjectList
+	for _, p := range c.AddonPaths {
+		addon, err := clusteraddons.LoadClusterAddon(p)
+		if err != nil {
+			return fmt.Errorf("error loading cluster addon %s: %v", p, err)
+		}
+		addons = append(addons, addon.Objects...)
+	}
+
 	// Note we perform as much validation as we can, before writing a bad config
-	err = registry.CreateClusterConfig(ctx, clientset, cluster, fullInstanceGroups)
+	err = registry.CreateClusterConfig(ctx, clientset, cluster, fullInstanceGroups, addons)
 	if err != nil {
 		return fmt.Errorf("error writing updated configuration: %v", err)
 	}

hack/.packages

@@ -82,6 +82,7 @@ k8s.io/kops/pkg/client/simple
 k8s.io/kops/pkg/client/simple/api
 k8s.io/kops/pkg/client/simple/vfsclientset
 k8s.io/kops/pkg/cloudinstances
+k8s.io/kops/pkg/clusteraddons
 k8s.io/kops/pkg/commands
 k8s.io/kops/pkg/configbuilder
 k8s.io/kops/pkg/diff

pkg/apis/kops/registry/BUILD.bazel

@@ -13,6 +13,7 @@ go_library(
         "//pkg/acls:go_default_library",
         "//pkg/apis/kops:go_default_library",
         "//pkg/client/simple:go_default_library",
+        "//pkg/kubemanifest:go_default_library",
         "//upup/pkg/fi/utils:go_default_library",
         "//util/pkg/vfs:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

pkg/apis/kops/registry/helpers.go

@@ -23,9 +23,10 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	api "k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/client/simple"
+	"k8s.io/kops/pkg/kubemanifest"
 )
 
-func CreateClusterConfig(ctx context.Context, clientset simple.Clientset, cluster *api.Cluster, groups []*api.InstanceGroup) error {
+func CreateClusterConfig(ctx context.Context, clientset simple.Clientset, cluster *api.Cluster, groups []*api.InstanceGroup, addons kubemanifest.ObjectList) error {
 	// Check for instancegroup Name duplicates before writing
 	{
 		names := map[string]bool{}
@@ -52,5 +53,13 @@ func CreateClusterConfig(ctx context.Context, clientset simple.Clientset, cluste
 		}
 	}
 
+	{
+		addonsClient := clientset.AddonsFor(cluster)
+
+		if err := addonsClient.Replace(addons); err != nil {
+			return fmt.Errorf("error writing updated addon configuration: %v", err)
+		}
+	}
+
 	return nil
 }

pkg/assets/builder.go

@@ -123,7 +123,7 @@ func (a *AssetBuilder) RemapManifest(data []byte) ([]byte, error) {
 		}
 	}
 
-	return kubemanifest.ToYAML(objects)
+	return objects.ToYAML()
 }
 
 // RemapImage normalizes a containers location if a user sets the AssetsLocation ContainerRegistry location.

pkg/client/simple/BUILD.bazel

@@ -8,6 +8,7 @@ go_library(
     deps = [
         "//pkg/apis/kops:go_default_library",
         "//pkg/client/clientset_generated/clientset/typed/kops/internalversion:go_default_library",
+        "//pkg/kubemanifest:go_default_library",
         "//upup/pkg/fi:go_default_library",
         "//util/pkg/vfs:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

pkg/client/simple/api/BUILD.bazel

@@ -10,6 +10,7 @@ go_library(
         "//pkg/apis/kops/registry:go_default_library",
         "//pkg/apis/kops/validation:go_default_library",
         "//pkg/client/clientset_generated/clientset/typed/kops/internalversion:go_default_library",
+        "//pkg/client/simple:go_default_library",
         "//pkg/client/simple/vfsclientset:go_default_library",
         "//upup/pkg/fi:go_default_library",
         "//upup/pkg/fi/secrets:go_default_library",

pkg/client/simple/api/clientset.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/pkg/apis/kops/validation"
 	kopsinternalversion "k8s.io/kops/pkg/client/clientset_generated/clientset/typed/kops/internalversion"
+	"k8s.io/kops/pkg/client/simple"
 	"k8s.io/kops/pkg/client/simple/vfsclientset"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/secrets"
@@ -47,6 +48,13 @@ func (c *RESTClientset) GetCluster(ctx context.Context, name string) (*kops.Clus
 	return c.KopsClient.Clusters(namespace).Get(ctx, name, metav1.GetOptions{})
 }
 
+// AddonsFor fetches the AddonsClient for the cluster
+func (c *RESTClientset) AddonsFor(cluster *kops.Cluster) simple.AddonsClient {
+	// We should manage these directly in the cluster
+	klog.Fatalf("AddonsFor not implemented for RESTClientset")
+	return nil
+}
+
 // CreateCluster implements the CreateCluster method of Clientset for a kubernetes-API state store
 func (c *RESTClientset) CreateCluster(ctx context.Context, cluster *kops.Cluster) (*kops.Cluster, error) {
 	namespace := restNamespaceForClusterName(cluster.Name)

pkg/client/simple/clientset.go

@@ -22,6 +22,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kops/pkg/apis/kops"
 	kopsinternalversion "k8s.io/kops/pkg/client/clientset_generated/clientset/typed/kops/internalversion"
+	"k8s.io/kops/pkg/kubemanifest"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/util/pkg/vfs"
 )
@@ -42,9 +43,12 @@ type Clientset interface {
 	// ConfigBaseFor returns the vfs path where we will read configuration information from
 	ConfigBaseFor(cluster *kops.Cluster) (vfs.Path, error)
 
-	// InstanceGroupsFor returns the InstanceGroupInterface bounds to the namespace for a particular Cluster
+	// InstanceGroupsFor returns the InstanceGroupInterface bound to the namespace for a particular Cluster
 	InstanceGroupsFor(cluster *kops.Cluster) kopsinternalversion.InstanceGroupInterface
 
+	// AddonsFor returns the client for addon objects for a particular Cluster
+	AddonsFor(cluster *kops.Cluster) AddonsClient
+
 	// SecretStore builds the secret store for the specified cluster
 	SecretStore(cluster *kops.Cluster) (fi.SecretStore, error)
 
@@ -57,3 +61,13 @@ type Clientset interface {
 	// DeleteCluster deletes all the state for the specified cluster
 	DeleteCluster(ctx context.Context, cluster *kops.Cluster) error
 }
+
+// AddonsClient is a client for manipulating cluster addons
+// Because we want to support storing these directly in a cluster, we don't group them
+type AddonsClient interface {
+	// Replace replaces all the addon objects with the provided list
+	Replace(objects kubemanifest.ObjectList) error
+
+	// List returns all the addon objects
+	List() (kubemanifest.ObjectList, error)
+}

pkg/client/simple/vfsclientset/BUILD.bazel

@@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "go_default_library",
     srcs = [
+        "addons.go",
         "clientset.go",
         "cluster.go",
         "commonvfs.go",
@@ -20,6 +21,7 @@ go_library(
         "//pkg/client/clientset_generated/clientset/typed/kops/internalversion:go_default_library",
         "//pkg/client/simple:go_default_library",
         "//pkg/kopscodecs:go_default_library",
+        "//pkg/kubemanifest:go_default_library",
         "//upup/pkg/fi:go_default_library",
         "//upup/pkg/fi/secrets:go_default_library",
         "//util/pkg/vfs:go_default_library",

pkg/client/simple/vfsclientset/addons.go

@@ -0,0 +1,96 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vfsclientset
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+
+	"k8s.io/klog/v2"
+	"k8s.io/kops/pkg/acls"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/client/simple"
+	"k8s.io/kops/pkg/kubemanifest"
+	"k8s.io/kops/util/pkg/vfs"
+)
+
+type vfsAddonsClient struct {
+	basePath vfs.Path
+
+	clusterName string
+	cluster     *kops.Cluster
+}
+
+var _ simple.AddonsClient = &vfsAddonsClient{}
+
+func newAddonsVFS(c *VFSClientset, cluster *kops.Cluster) *vfsAddonsClient {
+	if cluster == nil || cluster.Name == "" {
+		klog.Fatalf("cluster / cluster.Name is required")
+	}
+
+	clusterName := cluster.Name
+
+	r := &vfsAddonsClient{
+		cluster:     cluster,
+		clusterName: clusterName,
+	}
+	r.basePath = c.basePath.Join(clusterName, "clusteraddons")
+
+	return r
+}
+
+// TODO: Offer partial replacement?
+func (c *vfsAddonsClient) Replace(addons kubemanifest.ObjectList) error {
+	b, err := addons.ToYAML()
+	if err != nil {
+		return err
+	}
+
+	configPath := c.basePath.Join("default")
+
+	acl, err := acls.GetACL(configPath, c.cluster)
+	if err != nil {
+		return err
+	}
+
+	rs := bytes.NewReader(b)
+	if err := configPath.WriteFile(rs, acl); err != nil {
+		return fmt.Errorf("error writing addons file %s: %v", configPath, err)
+	}
+
+	return nil
+}
+
+func (c *vfsAddonsClient) List() (kubemanifest.ObjectList, error) {
+	configPath := c.basePath.Join("default")
+
+	b, err := configPath.ReadFile()
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("error reading addons file %s: %v", configPath, err)
+	}
+
+	objects, err := kubemanifest.LoadObjectsFrom(b)
+	if err != nil {
+		return nil, err
+	}
+
+	return objects, nil
+}

pkg/client/simple/vfsclientset/clientset.go

@@ -76,6 +76,10 @@ func (c *VFSClientset) InstanceGroupsFor(cluster *kops.Cluster) kopsinternalvers
 	return newInstanceGroupVFS(c, cluster)
 }
 
+func (c *VFSClientset) AddonsFor(cluster *kops.Cluster) simple.AddonsClient {
+	return newAddonsVFS(c, cluster)
+}
+
 func (c *VFSClientset) SecretStore(cluster *kops.Cluster) (fi.SecretStore, error) {
 	if cluster.Spec.SecretStore == "" {
 		configBase, err := registry.ConfigBase(cluster)
@@ -145,6 +149,9 @@ func DeleteAllClusterState(basePath vfs.Path) error {
 		if strings.HasPrefix(relativePath, "addons/") {
 			continue
 		}
+		if strings.HasPrefix(relativePath, "clusteraddons/") {
+			continue
+		}
 		if strings.HasPrefix(relativePath, "pki/") {
 			continue
 		}

pkg/clusteraddons/BUILD.bazel

@@ -0,0 +1,13 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["load.go"],
+    importpath = "k8s.io/kops/pkg/clusteraddons",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/kubemanifest:go_default_library",
+        "//util/pkg/vfs:go_default_library",
+        "//vendor/k8s.io/klog/v2:go_default_library",
+    ],
+)

pkg/clusteraddons/load.go

@@ -0,0 +1,65 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package clusteraddons
+
+import (
+	"fmt"
+	"net/url"
+
+	"k8s.io/klog/v2"
+	"k8s.io/kops/pkg/kubemanifest"
+	"k8s.io/kops/util/pkg/vfs"
+)
+
+type ClusterAddon struct {
+	Raw     string
+	Objects kubemanifest.ObjectList
+}
+
+// LoadClusterAddon loads a set of objects from the specified VFS location
+func LoadClusterAddon(location string) (*ClusterAddon, error) {
+	u, err := url.Parse(location)
+	if err != nil {
+		return nil, fmt.Errorf("invalid addon location: %q", location)
+	}
+
+	// TODO: Should we support relative paths for "standard" addons?  See equivalent code in LoadChannel
+
+	resolved := u.String()
+	klog.V(2).Infof("Loading addon from %q", resolved)
+	addonBytes, err := vfs.Context.ReadFile(resolved)
+	if err != nil {
+		return nil, fmt.Errorf("error reading addon %q: %v", resolved, err)
+	}
+	addon, err := ParseClusterAddon(addonBytes)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing addon %q: %v", resolved, err)
+	}
+	klog.V(4).Infof("Addon contents: %s", string(addonBytes))
+
+	return addon, nil
+}
+
+// ParseClusterAddon parses a ClusterAddon object
+func ParseClusterAddon(raw []byte) (*ClusterAddon, error) {
+	objects, err := kubemanifest.LoadObjectsFrom(raw)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing addon %v", err)
+	}
+
+	return &ClusterAddon{Raw: string(raw), Objects: objects}, nil
+}

pkg/featureflag/featureflag.go

@@ -92,6 +92,8 @@ var (
 	Terraform012 = New("Terraform-0.12", Bool(true))
 	// LegacyIAM will permit use of legacy IAM permissions.
 	LegacyIAM = New("LegacyIAM", Bool(false))
+	// ClusterAddons activates experimental cluster-addons support
+	ClusterAddons = New("ClusterAddons", Bool(false))
 )
 
 // FeatureFlag defines a feature flag

pkg/kubemanifest/manifest.go

@@ -30,14 +30,21 @@ type Object struct {
 	data map[string]interface{}
 }
 
+// ObjectList describes a list of objects, allowing us to add bulk-methods
+type ObjectList []*Object
+
 // LoadObjectsFrom parses multiple objects from a yaml file
-func LoadObjectsFrom(contents []byte) ([]*Object, error) {
+func LoadObjectsFrom(contents []byte) (ObjectList, error) {
 	var objects []*Object
 
-	// TODO: Support more separators?
 	sections := text.SplitContentToSections(contents)
 
 	for _, section := range sections {
+		// We need this so we don't error on a section that is empty / commented out
+		if !hasYAMLContent(section) {
+			continue
+		}
+
 		data := make(map[string]interface{})
 		err := yaml.Unmarshal(section, &data)
 		if err != nil {
@@ -54,11 +61,24 @@ func LoadObjectsFrom(contents []byte) ([]*Object, error) {
 	return objects, nil
 }
 
+// hasYAMLContent determines if the byte slice has any content,
+// because yaml parsing gives an error if called with no content.
+// TODO: How does apimachinery avoid this problem?
+func hasYAMLContent(yamlData []byte) bool {
+	for _, line := range bytes.Split(yamlData, []byte("\n")) {
+		l := bytes.TrimSpace(line)
+		if len(l) != 0 && !bytes.HasPrefix(l, []byte("#")) {
+			return true
+		}
+	}
+	return false
+}
+
 // ToYAML serializes a list of objects back to bytes; it is the opposite of LoadObjectsFrom
-func ToYAML(objects []*Object) ([]byte, error) {
+func (l ObjectList) ToYAML() ([]byte, error) {
 	var yamlSeparator = []byte("\n---\n\n")
 	var yamls [][]byte
-	for _, object := range objects {
+	for _, object := range l {
 		// Don't serialize empty objects - they confuse yaml parsers
 		if object.IsEmptyObject() {
 			continue

upup/pkg/fi/cloudup/BUILD.bazel

@@ -42,6 +42,7 @@ go_library(
         "//pkg/client/simple/vfsclientset:go_default_library",
         "//pkg/dns:go_default_library",
         "//pkg/featureflag:go_default_library",
+        "//pkg/kubemanifest:go_default_library",
         "//pkg/model:go_default_library",
         "//pkg/model/alimodel:go_default_library",
         "//pkg/model/awsmodel:go_default_library",

upup/pkg/fi/cloudup/apply_cluster.go

@@ -299,6 +299,12 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error {
 		return err
 	}
 
+	addonsClient := c.Clientset.AddonsFor(cluster)
+	addons, err := addonsClient.List()
+	if err != nil {
+		return fmt.Errorf("error fetching addons: %v", err)
+	}
+
 	// Normalize k8s version
 	versionWithoutV := strings.TrimSpace(cluster.Spec.KubernetesVersion)
 	versionWithoutV = strings.TrimPrefix(versionWithoutV, "v")
@@ -478,6 +484,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error {
 				Lifecycle:        &clusterLifecycle,
 				assetBuilder:     assetBuilder,
 				templates:        templates,
+				ClusterAddons:    addons,
 			},
 			&model.PKIModelBuilder{
 				KopsModelContext: modelContext,

upup/pkg/fi/cloudup/bootstrapchannelbuilder.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/assets"
 	"k8s.io/kops/pkg/featureflag"
+	"k8s.io/kops/pkg/kubemanifest"
 	"k8s.io/kops/pkg/model"
 	"k8s.io/kops/pkg/templates"
 	"k8s.io/kops/upup/pkg/fi"
@@ -37,9 +38,10 @@ import (
 // BootstrapChannelBuilder is responsible for handling the addons in channels
 type BootstrapChannelBuilder struct {
 	*model.KopsModelContext
-	Lifecycle    *fi.Lifecycle
-	templates    *templates.Templates
-	assetBuilder *assets.AssetBuilder
+	ClusterAddons kubemanifest.ObjectList
+	Lifecycle     *fi.Lifecycle
+	templates     *templates.Templates
+	assetBuilder  *assets.AssetBuilder
 }
 
 var _ fi.ModelBuilder = &BootstrapChannelBuilder{}
@@ -51,8 +53,6 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error {
 		return err
 	}
 
-	tasks := c.Tasks
-
 	for _, a := range addons.Spec.Addons {
 		key := *a.Name
 		if a.Id != "" {
@@ -91,13 +91,53 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error {
 		}
 		a.ManifestHash = manifestHash
 
-		tasks[name] = &fitasks.ManagedFile{
+		c.AddTask(&fitasks.ManagedFile{
 			Contents:  fi.WrapResource(fi.NewBytesResource(manifestBytes)),
 			Lifecycle: b.Lifecycle,
 			Location:  fi.String(manifestPath),
 			Name:      fi.String(name),
+		})
+	}
+
+	if b.ClusterAddons != nil {
+		key := "cluster-addons.kops.k8s.io"
+		version := "0.0.0"
+		location := key + "/default.yaml"
+
+		a := &channelsapi.AddonSpec{
+			Name:     fi.String(key),
+			Version:  fi.String(version),
+			Selector: map[string]string{"k8s-addon": key},
+			Manifest: fi.String(location),
 		}
 
+		name := b.Cluster.ObjectMeta.Name + "-addons-" + key
+		manifestPath := "addons/" + *a.Manifest
+
+		manifestBytes, err := b.ClusterAddons.ToYAML()
+		if err != nil {
+			return fmt.Errorf("error serializing addons: %v", err)
+		}
+
+		// Trim whitespace
+		manifestBytes = []byte(strings.TrimSpace(string(manifestBytes)))
+
+		rawManifest := string(manifestBytes)
+
+		manifestHash, err := utils.HashString(rawManifest)
+		if err != nil {
+			return fmt.Errorf("error hashing manifest: %v", err)
+		}
+		a.ManifestHash = manifestHash
+
+		c.AddTask(&fitasks.ManagedFile{
+			Contents:  fi.WrapResource(fi.NewBytesResource(manifestBytes)),
+			Lifecycle: b.Lifecycle,
+			Location:  fi.String(manifestPath),
+			Name:      fi.String(name),
+		})
+
+		addons.Spec.Addons = append(addons.Spec.Addons, a)
 	}
 
 	addonsYAML, err := utils.YamlMarshal(addons)
@@ -107,12 +147,12 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error {
 
 	name := b.Cluster.ObjectMeta.Name + "-addons-bootstrap"
 
-	tasks[name] = &fitasks.ManagedFile{
+	c.AddTask(&fitasks.ManagedFile{
 		Contents:  fi.WrapResource(fi.NewBytesResource(addonsYAML)),
 		Lifecycle: b.Lifecycle,
 		Location:  fi.String("addons/bootstrap-channel.yaml"),
 		Name:      fi.String(name),
-	}
+	})
 
 	return nil
 }

upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go

@@ -118,7 +118,7 @@ func runChannelBuilderTest(t *testing.T, key string, addonManifests []string) {
 
 	{
 		name := cluster.ObjectMeta.Name + "-addons-bootstrap"
-		manifestTask := context.Tasks[name]
+		manifestTask := context.Tasks["ManagedFile/"+name]
 		if manifestTask == nil {
 			t.Fatalf("manifest task not found (%q)", name)
 		}
@@ -135,7 +135,7 @@ func runChannelBuilderTest(t *testing.T, key string, addonManifests []string) {
 
 	for _, k := range addonManifests {
 		name := cluster.ObjectMeta.Name + "-addons-" + k
-		manifestTask := context.Tasks[name]
+		manifestTask := context.Tasks["ManagedFile/"+name]
 		if manifestTask == nil {
 			for k := range context.Tasks {
 				t.Logf("found task %s", k)

upup/pkg/kutil/convert_kubeup_cluster.go

@@ -465,7 +465,7 @@ func (x *ConvertKubeupCluster) Upgrade(ctx context.Context) error {
 		}
 	}
 
-	err = registry.CreateClusterConfig(ctx, x.Clientset, cluster, x.InstanceGroups)
+	err = registry.CreateClusterConfig(ctx, x.Clientset, cluster, x.InstanceGroups, nil)
 	if err != nil {
 		return fmt.Errorf("error writing updated configuration: %v", err)
 	}

upup/pkg/kutil/import_cluster.go

@@ -488,7 +488,7 @@ func (x *ImportCluster) ImportAWSCluster(ctx context.Context) error {
 		fullInstanceGroups = append(fullInstanceGroups, full)
 	}
 
-	err = registry.CreateClusterConfig(ctx, x.Clientset, cluster, fullInstanceGroups)
+	err = registry.CreateClusterConfig(ctx, x.Clientset, cluster, fullInstanceGroups, nil)
 	if err != nil {
 		return err
 	}