Merge pull request #12156 from justinsb/prune_via_labels

Prune addons via labels

channels/pkg/api/channel.go

@@ -46,7 +46,7 @@ type AddonSpec struct {
 	// Manifest is the URL to the manifest that should be applied
 	Manifest *string `json:"manifest,omitempty"`
 
-	// Manifesthash is the sha256 hash of our manifest
+	// ManifestHash is the sha256 hash of our manifest
 	ManifestHash string `json:"manifestHash,omitempty"`
 
 	// KubernetesVersion is a semver version range on which this version of the addon can be applied
@@ -69,6 +69,33 @@ type AddonSpec struct {
 	NeedsPKI bool `json:"needsPKI,omitempty"`
 
 	Version string `json:"version,omitempty"`
+
+	// PruneSpec specifies how old objects should be removed (pruned).
+	Prune *PruneSpec `json:"prune,omitempty"`
+}
+
+// PruneSpec specifies how old objects should be removed (pruned).
+type PruneSpec struct {
+	// Kinds specifies the objects to be pruned, by Kind.
+	Kinds []PruneKindSpec `json:"kinds,omitempty"`
+}
+
+// PruneKindSpec specifies pruning for a particular Kind of object.
+type PruneKindSpec struct {
+	// Group specifies the object Group to be pruned (required).
+	Group string `json:"group,omitempty"`
+	// Kind specifies the object Kind to be pruned (required).
+	Kind string `json:"kind,omitempty"`
+
+	// Namespaces limits pruning only to objects in certain namespaces.
+	Namespaces []string `json:"namespaces,omitempty"`
+
+	// LabelSelector limits pruning only to objects matching the specified labels.
+	LabelSelector string `json:"labelSelector,omitempty"`
+
+	// FieldSelector allows pruning only of objects matching the field selector.
+	// (This isn't currently used, but adding it now lets us start without worrying about version skew)
+	FieldSelector string `json:"fieldSelector,omitempty"`
 }
 
 func (a *Addons) Verify() error {

channels/pkg/channels/BUILD.bazel

@@ -7,11 +7,13 @@ go_library(
         "addons.go",
         "apply.go",
         "channel_version.go",
+        "prune.go",
     ],
     importpath = "k8s.io/kops/channels/pkg/channels",
     visibility = ["//visibility:public"],
     deps = [
         "//channels/pkg/api:go_default_library",
+        "//pkg/kubemanifest:go_default_library",
         "//pkg/pki:go_default_library",
         "//upup/pkg/fi/utils:go_default_library",
         "//util/pkg/vfs:go_default_library",
@@ -21,9 +23,13 @@ go_library(
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
+        "//vendor/k8s.io/client-go/dynamic:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/restmapper:go_default_library",
         "//vendor/k8s.io/klog/v2:go_default_library",
     ],
 )

channels/pkg/channels/addon.go

@@ -24,6 +24,7 @@ import (
 	"net/url"
 
 	"k8s.io/kops/pkg/pki"
+	"k8s.io/kops/util/pkg/vfs"
 
 	certmanager "github.com/jetstack/cert-manager/pkg/client/clientset/versioned"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -72,7 +73,7 @@ func (m *AddonMenu) MergeAddons(o *AddonMenu) {
 		if existing == nil {
 			m.Addons[k] = v
 		} else {
-			if v.ChannelVersion().replaces(existing.ChannelVersion()) {
+			if v.ChannelVersion().replaces(k, existing.ChannelVersion()) {
 				m.Addons[k] = v
 			}
 		}
@@ -84,6 +85,7 @@ func (a *Addon) ChannelVersion() *ChannelVersion {
 		Channel:          &a.ChannelName,
 		Id:               a.Spec.Id,
 		ManifestHash:     a.Spec.ManifestHash,
+		SystemGeneration: CurrentSystemGeneration,
 	}
 }
 
@@ -119,7 +121,7 @@ func (a *Addon) GetRequiredUpdates(ctx context.Context, k8sClient kubernetes.Int
 		}
 	}
 
-	if existingVersion != nil && !newVersion.replaces(existingVersion) {
+	if existingVersion != nil && !newVersion.replaces(a.Name, existingVersion) {
 		newVersion = nil
 	}
 
@@ -151,7 +153,7 @@ func (a *Addon) GetManifestFullUrl() (*url.URL, error) {
 	return manifestURL, nil
 }
 
-func (a *Addon) EnsureUpdated(ctx context.Context, k8sClient kubernetes.Interface, cmClient certmanager.Interface) (*AddonUpdate, error) {
+func (a *Addon) EnsureUpdated(ctx context.Context, k8sClient kubernetes.Interface, cmClient certmanager.Interface, pruner *Pruner) (*AddonUpdate, error) {
 	required, err := a.GetRequiredUpdates(ctx, k8sClient, cmClient)
 	if err != nil {
 		return nil, err
@@ -167,9 +169,18 @@ func (a *Addon) EnsureUpdated(ctx context.Context, k8sClient kubernetes.Interfac
 		}
 		klog.Infof("Applying update from %q", manifestURL)
 
-		err = Apply(manifestURL.String())
+		// We copy the manifest to a temp file because it is likely e.g. an s3 URL, which kubectl can't read
+		data, err := vfs.Context.ReadFile(manifestURL.String())
 		if err != nil {
-			return nil, fmt.Errorf("error applying update from %q: %v", manifestURL, err)
+			return nil, fmt.Errorf("error reading manifest: %w", err)
+		}
+
+		if err := Apply(data); err != nil {
+			return nil, fmt.Errorf("error applying update from %q: %w", manifestURL, err)
+		}
+
+		if err := pruner.Prune(ctx, data, a.Spec.Prune); err != nil {
+			return nil, fmt.Errorf("error pruning manifest from %q: %w", manifestURL, err)
 		}
 
 		if err := a.AddNeedsUpdateLabel(ctx, k8sClient, required); err != nil {

channels/pkg/channels/addons.go

@@ -74,7 +74,7 @@ func (a *Addons) GetCurrent(kubernetesVersion semver.Version) (*AddonMenu, error
 		name := addon.Name
 
 		existing := menu.Addons[name]
-		if existing == nil || addon.ChannelVersion().replaces(existing.ChannelVersion()) {
+		if existing == nil || addon.ChannelVersion().replaces(name, existing.ChannelVersion()) {
 			menu.Addons[name] = addon
 		}
 	}

channels/pkg/channels/addons_test.go

@@ -85,6 +85,9 @@ func Test_Filtering(t *testing.T) {
 }
 
 func Test_Replacement(t *testing.T) {
+	hash1 := "3544de6578b2b582c0323b15b7b05a28c60b9430"
+	hash2 := "ea9e79bf29adda450446487d65a8fc6b3fdf8c2b"
+
 	grid := []struct {
 		Old      *ChannelVersion
 		New      *ChannelVersion
@@ -92,23 +95,38 @@ func Test_Replacement(t *testing.T) {
 	}{
 		//Test ManifestHash Changes
 		{
-			Old:      &ChannelVersion{Id: "a", ManifestHash: "3544de6578b2b582c0323b15b7b05a28c60b9430"},
+			Old:      &ChannelVersion{Id: "a", ManifestHash: hash1},
-			New:      &ChannelVersion{Id: "a", ManifestHash: "3544de6578b2b582c0323b15b7b05a28c60b9430"},
+			New:      &ChannelVersion{Id: "a", ManifestHash: hash1},
 			Replaces: false,
 		},
 		{
 			Old:      &ChannelVersion{Id: "a", ManifestHash: ""},
-			New:      &ChannelVersion{Id: "a", ManifestHash: "3544de6578b2b582c0323b15b7b05a28c60b9430"},
+			New:      &ChannelVersion{Id: "a", ManifestHash: hash1},
 			Replaces: true,
 		},
 		{
-			Old:      &ChannelVersion{Id: "a", ManifestHash: "3544de6578b2b582c0323b15b7b05a28c60b9430"},
+			Old:      &ChannelVersion{Id: "a", ManifestHash: hash1},
-			New:      &ChannelVersion{Id: "a", ManifestHash: "ea9e79bf29adda450446487d65a8fc6b3fdf8c2b"},
+			New:      &ChannelVersion{Id: "a", ManifestHash: hash2},
 			Replaces: true,
 		},
+		{
+			Old:      &ChannelVersion{Id: "a", ManifestHash: hash1},
+			New:      &ChannelVersion{Id: "a", ManifestHash: hash1, SystemGeneration: 1},
+			Replaces: true,
+		},
+		{
+			Old:      &ChannelVersion{Id: "a", ManifestHash: hash1, SystemGeneration: 1},
+			New:      &ChannelVersion{Id: "a", ManifestHash: hash1},
+			Replaces: false,
+		},
+		{
+			Old:      &ChannelVersion{Id: "a", ManifestHash: hash1, SystemGeneration: 1},
+			New:      &ChannelVersion{Id: "a", ManifestHash: hash1, SystemGeneration: 1},
+			Replaces: false,
+		},
 	}
 	for _, g := range grid {
-		actual := g.New.replaces(g.Old)
+		actual := g.New.replaces(t.Name(), g.Old)
 		if actual != g.Replaces {
 			t.Errorf("unexpected result from %v -> %v, expect %t.  actual %v", g.Old, g.New, g.Replaces, actual)
 		}
@@ -218,7 +236,7 @@ func Test_NeedsRollingUpdate(t *testing.T) {
 		ctx := context.Background()
 
 		annotations := map[string]string{
-			"addons.k8s.io/test": "{\"manifestHash\":\"originalHash\"}",
+			"addons.k8s.io/test": "{\"manifestHash\":\"originalHash\",\"systemGeneration\": 1}",
 		}
 		if len(g.originalAnnotations) > 0 {
 			annotations = g.originalAnnotations

channels/pkg/channels/apply.go

@@ -25,18 +25,12 @@ import (
 	"strings"
 
 	"k8s.io/klog/v2"
-	"k8s.io/kops/util/pkg/vfs"
 )
 
 // Apply calls kubectl apply to apply the manifest.
 // We will likely in future change this to create things directly (or more likely embed this logic into kubectl itself)
-func Apply(manifest string) error {
+func Apply(data []byte) error {
 	// We copy the manifest to a temp file because it is likely e.g. an s3 URL, which kubectl can't read
-	data, err := vfs.Context.ReadFile(manifest)
-	if err != nil {
-		return fmt.Errorf("error reading manifest: %v", err)
-	}
-
 	tmpDir, err := ioutil.TempDir("", "channel")
 	if err != nil {
 		return fmt.Errorf("error creating temp dir: %v", err)

channels/pkg/channels/channel_version.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"strconv"
 	"strings"
 
 	certmanager "github.com/jetstack/cert-manager/pkg/client/clientset/versioned"
@@ -38,10 +39,20 @@ type Channel struct {
 	Name      string
 }
 
+// CurrentSystemGeneration holds our current SystemGeneration value.
+// Version history:
+//   0  Pre-history (and the default value); versions prior to prune.
+//   1  Prune functionality introduced.
+const CurrentSystemGeneration = 1
+
 type ChannelVersion struct {
 	Channel      *string `json:"channel,omitempty"`
 	Id           string  `json:"id,omitempty"`
 	ManifestHash string  `json:"manifestHash,omitempty"`
+
+	// SystemGeneration holds the generation of the channels functionality.
+	// It is used so that we reapply when we introduce new features, such as prune.
+	SystemGeneration int `json:"systemGeneration,omitempty"`
 }
 
 func stringValue(s *string) string {
@@ -59,6 +70,7 @@ func (c *ChannelVersion) String() string {
 	if c.ManifestHash != "" {
 		s += " ManifestHash=" + c.ManifestHash
 	}
+	s += " SystemGeneration=" + strconv.Itoa(c.SystemGeneration)
 	return s
 }
 
@@ -102,23 +114,33 @@ func (c *Channel) AnnotationName() string {
 	return AnnotationPrefix + c.Name
 }
 
-func (c *ChannelVersion) replaces(existing *ChannelVersion) bool {
+func (c *ChannelVersion) replaces(name string, existing *ChannelVersion) bool {
-	klog.V(4).Infof("Checking existing channel: %v compared to new channel: %v", existing, c)
+	klog.V(6).Infof("Checking existing config for %q: %v compared to new channel: %v", name, existing, c)
-
-	if c.Id == existing.Id {
-		// Same id; check manifests
-		if c.ManifestHash == existing.ManifestHash {
-			klog.V(4).Infof("Manifest Match")
-			return false
-		}
-		klog.V(4).Infof("Channels had same ids %q, %q but different ManifestHash (%q vs %q); will replace", c.Id, c.ManifestHash, existing.ManifestHash)
-	} else {
-		klog.V(4).Infof("Channels had different ids (%q vs %q); will replace", c.Id, existing.Id)
-	}
 
+	if c.Id != existing.Id {
+		klog.V(4).Infof("cluster has different ids for %q (%q vs %q); will replace", name, c.Id, existing.Id)
 		return true
 	}
 
+	if c.ManifestHash != existing.ManifestHash {
+		klog.V(4).Infof("cluster has different ManifestHash for %q (%q vs %q); will replace", name, c.ManifestHash, existing.ManifestHash)
+		return true
+	}
+
+	if existing.SystemGeneration != c.SystemGeneration {
+		if existing.SystemGeneration > c.SystemGeneration {
+			klog.V(4).Infof("cluster has newer SystemGeneration for %q (%v vs %v), will not replace", name, existing.SystemGeneration, c.SystemGeneration)
+			return false
+		} else {
+			klog.V(4).Infof("cluster has different SystemGeneration for %q (%v vs %v); will replace", name, existing.SystemGeneration, c.SystemGeneration)
+			return true
+		}
+	}
+
+	klog.V(4).Infof("manifest Match for %q: %v", name, existing)
+	return false
+}
+
 func (c *Channel) GetInstalledVersion(ctx context.Context, k8sClient kubernetes.Interface) (*ChannelVersion, error) {
 	ns, err := k8sClient.CoreV1().Namespaces().Get(ctx, c.Namespace, metav1.GetOptions{})
 	if err != nil {

channels/pkg/channels/prune.go

@@ -0,0 +1,149 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package channels
+
+import (
+	"context"
+	"fmt"
+
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/restmapper"
+	"k8s.io/klog/v2"
+	"k8s.io/kops/channels/pkg/api"
+	"k8s.io/kops/pkg/kubemanifest"
+)
+
+type Pruner struct {
+	Client     dynamic.Interface
+	RESTMapper *restmapper.DeferredDiscoveryRESTMapper
+}
+
+// Prune prunes objects not in the manifest, according to PruneSpec.
+func (p *Pruner) Prune(ctx context.Context, manifest []byte, spec *api.PruneSpec) error {
+	klog.Infof("Prune spec: %v", spec)
+
+	if spec == nil {
+		return nil
+	}
+
+	objects, err := kubemanifest.LoadObjectsFrom(manifest)
+	if err != nil {
+		return fmt.Errorf("failed to parse objects: %w", err)
+	}
+
+	objectsByKind := make(map[schema.GroupKind][]*kubemanifest.Object)
+	for _, object := range objects {
+		gv, err := schema.ParseGroupVersion(object.APIVersion())
+		if err != nil || gv.Version == "" {
+			return fmt.Errorf("failed to parse apiVersion %q", object.APIVersion())
+		}
+		kind := object.Kind()
+		if kind == "" {
+			return fmt.Errorf("failed to find kind in object")
+		}
+
+		gvk := gv.WithKind(kind)
+		gk := gvk.GroupKind()
+		objectsByKind[gk] = append(objectsByKind[gk], object)
+	}
+
+	for i := range spec.Kinds {
+		pruneKind := &spec.Kinds[i]
+		gk := schema.GroupKind{Group: pruneKind.Group, Kind: pruneKind.Kind}
+		if err := p.pruneObjectsOfKind(ctx, gk, pruneKind, objectsByKind[gk]); err != nil {
+			return fmt.Errorf("failed to prune objects of kind %s: %w", gk, err)
+		}
+	}
+
+	return nil
+}
+
+func (p *Pruner) pruneObjectsOfKind(ctx context.Context, gk schema.GroupKind, spec *api.PruneKindSpec, keepObjects []*kubemanifest.Object) error {
+	klog.Infof("pruning objects of kind: %v", gk)
+
+	restMapping, err := p.RESTMapper.RESTMapping(gk)
+	if err != nil {
+		return fmt.Errorf("unable to find resource for %s: %w", gk, err)
+	}
+
+	gvr := restMapping.Resource
+
+	var listOptions v1.ListOptions
+	listOptions.LabelSelector = spec.LabelSelector
+	listOptions.FieldSelector = spec.FieldSelector
+
+	baseResource := p.Client.Resource(gvr)
+	if len(spec.Namespaces) == 0 {
+		objects, err := baseResource.List(ctx, listOptions)
+		if err != nil {
+			return fmt.Errorf("error listing objects: %w", err)
+		}
+		if err := p.pruneObjects(ctx, gvr, objects, keepObjects); err != nil {
+			return err
+		}
+	} else {
+		for _, namespace := range spec.Namespaces {
+			resource := baseResource.Namespace(namespace)
+			actualObjects, err := resource.List(ctx, listOptions)
+			if err != nil {
+				return fmt.Errorf("error listing objects in namespace %s: %w", namespace, err)
+			}
+			if err := p.pruneObjects(ctx, gvr, actualObjects, keepObjects); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (p *Pruner) pruneObjects(ctx context.Context, gvr schema.GroupVersionResource, actualObjects *unstructured.UnstructuredList, keepObjects []*kubemanifest.Object) error {
+	keepMap := make(map[string]*kubemanifest.Object)
+	for _, keepObject := range keepObjects {
+		key := keepObject.GetNamespace() + "/" + keepObject.GetName()
+		keepMap[key] = keepObject
+	}
+
+	for _, actualObject := range actualObjects.Items {
+		name := actualObject.GetName()
+		namespace := actualObject.GetNamespace()
+		key := namespace + "/" + name
+		if _, found := keepMap[key]; found {
+			// Object is in manifest, don't delete
+			continue
+		}
+
+		klog.Infof("pruning %s %s", gvr, key)
+
+		var resource dynamic.ResourceInterface
+		if namespace != "" {
+			resource = p.Client.Resource(gvr).Namespace(namespace)
+		} else {
+			resource = p.Client.Resource(gvr)
+		}
+
+		var opts v1.DeleteOptions
+		if err := resource.Delete(ctx, name, opts); err != nil {
+			return fmt.Errorf("failed to delete %s: %w", key, err)
+		}
+	}
+
+	return nil
+}

channels/pkg/cmd/BUILD.bazel

@@ -22,10 +22,12 @@ go_library(
         "//vendor/github.com/spf13/viper:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library",
+        "//vendor/k8s.io/client-go/dynamic:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/k8s.io/client-go/plugin/pkg/client/auth:go_default_library",
         "//vendor/k8s.io/client-go/rest:go_default_library",
-        "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
+        "//vendor/k8s.io/client-go/restmapper:go_default_library",
         "//vendor/k8s.io/klog/v2:go_default_library",
     ],
 )

channels/pkg/cmd/apply_channel.go

@@ -66,6 +66,16 @@ func RunApplyChannel(ctx context.Context, f Factory, out io.Writer, options *App
 		return err
 	}
 
+	dynamicClient, err := f.DynamicClient()
+	if err != nil {
+		return err
+	}
+
+	restMapper, err := f.RESTMapper()
+	if err != nil {
+		return err
+	}
+
 	kubernetesVersionInfo, err := k8sClient.Discovery().ServerVersion()
 	if err != nil {
 		return fmt.Errorf("error querying kubernetes version: %v", err)
@@ -200,8 +210,13 @@ func RunApplyChannel(ctx context.Context, f Factory, out io.Writer, options *App
 		return nil
 	}
 
+	pruner := &channels.Pruner{
+		Client:     dynamicClient,
+		RESTMapper: restMapper,
+	}
+
 	for _, needUpdate := range needUpdates {
-		update, err := needUpdate.EnsureUpdated(ctx, k8sClient, cmClient)
+		update, err := needUpdate.EnsureUpdated(ctx, k8sClient, cmClient, pruner)
 		if err != nil {
 			fmt.Printf("error updating %q: %v", needUpdate.Name, err)
 		} else if update != nil {

channels/pkg/cmd/factory.go

@@ -19,9 +19,11 @@ package cmd
 import (
 	"fmt"
 
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/restmapper"
 
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
@@ -31,37 +33,43 @@ import (
 type Factory interface {
 	KubernetesClient() (kubernetes.Interface, error)
 	CertManagerClient() (certmanager.Interface, error)
+	RESTMapper() (*restmapper.DeferredDiscoveryRESTMapper, error)
+	DynamicClient() (dynamic.Interface, error)
 }
 
 type DefaultFactory struct {
+	ConfigFlags genericclioptions.ConfigFlags
+
 	kubernetesClient  kubernetes.Interface
 	certManagerClient certmanager.Interface
+
+	cachedRESTConfig *rest.Config
+	dynamicClient    dynamic.Interface
+	restMapper       *restmapper.DeferredDiscoveryRESTMapper
 }
 
 var _ Factory = &DefaultFactory{}
 
-func loadConfig() (*rest.Config, error) {
+func (f *DefaultFactory) restConfig() (*rest.Config, error) {
-	loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
+	if f.cachedRESTConfig == nil {
-	loadingRules.DefaultClientConfig = &clientcmd.DefaultClientConfig
+		restConfig, err := f.ConfigFlags.ToRESTConfig()
-
+		if err != nil {
-	configOverrides := &clientcmd.ConfigOverrides{
+			return nil, fmt.Errorf("cannot load kubecfg settings: %w", err)
-		ClusterDefaults: clientcmd.ClusterDefaults,
 		}
-
+		f.cachedRESTConfig = restConfig
-	kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides)
+	}
-	return kubeConfig.ClientConfig()
+	return f.cachedRESTConfig, nil
-
 }
 
 func (f *DefaultFactory) KubernetesClient() (kubernetes.Interface, error) {
 	if f.kubernetesClient == nil {
-		config, err := loadConfig()
+		restConfig, err := f.restConfig()
 		if err != nil {
-			return nil, fmt.Errorf("cannot load kubecfg settings: %v", err)
+			return nil, err
 		}
-		k8sClient, err := kubernetes.NewForConfig(config)
+		k8sClient, err := kubernetes.NewForConfig(restConfig)
 		if err != nil {
-			return nil, fmt.Errorf("cannot build kube client: %v", err)
+			return nil, fmt.Errorf("cannot build kube client: %w", err)
 		}
 		f.kubernetesClient = k8sClient
 	}
@@ -69,13 +77,29 @@ func (f *DefaultFactory) KubernetesClient() (kubernetes.Interface, error) {
 	return f.kubernetesClient, nil
 }
 
+func (f *DefaultFactory) DynamicClient() (dynamic.Interface, error) {
+	if f.dynamicClient == nil {
+		restConfig, err := f.restConfig()
+		if err != nil {
+			return nil, fmt.Errorf("cannot load kubecfg settings: %w", err)
+		}
+		dynamicClient, err := dynamic.NewForConfig(restConfig)
+		if err != nil {
+			return nil, fmt.Errorf("cannot build dynamicClient client: %v", err)
+		}
+		f.dynamicClient = dynamicClient
+	}
+
+	return f.dynamicClient, nil
+}
+
 func (f *DefaultFactory) CertManagerClient() (certmanager.Interface, error) {
 	if f.certManagerClient == nil {
-		config, err := loadConfig()
+		restConfig, err := f.restConfig()
 		if err != nil {
-			return nil, fmt.Errorf("cannot load kubecfg settings: %v", err)
+			return nil, err
 		}
-		certManagerClient, err := certmanager.NewForConfig(config)
+		certManagerClient, err := certmanager.NewForConfig(restConfig)
 		if err != nil {
 			return nil, fmt.Errorf("cannot build kube client: %v", err)
 		}
@@ -84,3 +108,18 @@ func (f *DefaultFactory) CertManagerClient() (certmanager.Interface, error) {
 
 	return f.certManagerClient, nil
 }
+
+func (f *DefaultFactory) RESTMapper() (*restmapper.DeferredDiscoveryRESTMapper, error) {
+	if f.restMapper == nil {
+		discoveryClient, err := f.ConfigFlags.ToDiscoveryClient()
+		if err != nil {
+			return nil, err
+		}
+
+		restMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient)
+
+		f.restMapper = restMapper
+	}
+
+	return f.restMapper, nil
+}

nodeup/pkg/model/update_service.go

@@ -92,7 +92,7 @@ func (b *UpdateServiceBuilder) buildDebianPackage(c *fi.ModelBuilderContext) {
 `
 	} else {
 
-		klog.Infof("Detected OS %s; installing %s package", b.Distribution, debianPackageName)
+		klog.Infof("Detected OS %v; installing %s package", b.Distribution, debianPackageName)
 		c.AddTask(&nodetasks.Package{Name: debianPackageName})
 
 		contents = `APT::Periodic::Update-Package-Lists "1";

pkg/kubemanifest/manifest.go

@@ -140,6 +140,44 @@ func (m *Object) Kind() string {
 	return s
 }
 
+// GetNamespace returns the namespace field of the object, or "" if it cannot be found or is invalid
+func (m *Object) GetNamespace() string {
+	metadata := m.metadata()
+	return getStringValue(metadata, "namespace")
+}
+
+// GetName returns the namespace field of the object, or "" if it cannot be found or is invalid
+func (m *Object) GetName() string {
+	metadata := m.metadata()
+	return getStringValue(metadata, "name")
+}
+
+// getStringValue returns the specified field of the object, or "" if it cannot be found or is invalid
+func getStringValue(m map[string]interface{}, key string) string {
+	v, found := m[key]
+	if !found {
+		return ""
+	}
+	s, ok := v.(string)
+	if !ok {
+		return ""
+	}
+	return s
+}
+
+// metadata returns the metadata map of the object, or nil if it cannot be found or is invalid
+func (m *Object) metadata() map[string]interface{} {
+	v, found := m.data["metadata"]
+	if !found {
+		return nil
+	}
+	metadata, ok := v.(map[string]interface{})
+	if !ok {
+		return nil
+	}
+	return metadata
+}
+
 // APIVersion returns the apiVersion field of the object, or "" if it cannot be found or is invalid
 func (m *Object) APIVersion() string {
 	v, found := m.data["apiVersion"]

tests/e2e/kubetest2-kops/builder/build.go

@@ -18,7 +18,10 @@ package builder
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
+	"strings"
 
 	"sigs.k8s.io/kubetest2/pkg/exec"
 )
@@ -39,5 +42,21 @@ func (b *BuildOptions) Build() error {
 	)
 	cmd.SetDir(b.KopsRoot)
 	exec.InheritOutput(cmd)
-	return cmd.Run()
+	if err := cmd.Run(); err != nil {
+		return err
+	}
+
+	// Write some meta files so that other tooling can know e.g. KOPS_BASE_URL
+	metaDir := filepath.Join(b.KopsRoot, ".kubetest2")
+
+	if err := os.MkdirAll(metaDir, 0755); err != nil {
+		return fmt.Errorf("failed to Mkdir(%q): %w", metaDir, err)
+	}
+	p := filepath.Join(metaDir, "kops-base-url")
+	kopsBaseURL := strings.Replace(b.StageLocation, "gs://", "https://storage.googleapis.com/", 1)
+	if err := ioutil.WriteFile(p, []byte(kopsBaseURL), 0644); err != nil {
+		return fmt.Errorf("failed to WriteFile(%q): %w", p, err)
+	}
+
+	return nil
 }

tests/e2e/scenarios/addon-resource-tracking/run-test.sh

@@ -19,7 +19,7 @@ source "${REPO_ROOT}"/tests/e2e/scenarios/lib/common.sh
 
 function haveds() {
 	local ds=0
-	kubectl get ds -n kube-system aws-node-termination-handler || ds=$?
+	kubectl get ds -n kube-system aws-node-termination-handler --show-labels || ds=$?
 	return $ds
 }
 
@@ -39,7 +39,10 @@ ${KUBETEST2} \
     --create-args="$ARGS"
 
 
-haveds
+if ! haveds; then
+  echo "Expected aws-node-termination-handler to exist"
+  exit 1
+fi
 
 # Upgrade to a version that should adopt existing resources and apply the change below
 kops-acquire-latest
@@ -53,11 +56,14 @@ kops edit cluster "${CLUSTER_NAME}" "--set=cluster.spec.nodeTerminationHandler.e
 kops update cluster --allow-kops-downgrade
 kops update cluster --yes --allow-kops-downgrade
 
-# wait for channels to deploy
+# Rolling-upgrade is needed so we get the new channels binary that supports prune
-sleep 90s
+kops rolling-update cluster --instance-group-roles=master --yes
 
 # just make sure pods are ready
 kops validate cluster --wait=5m
 
 # We should no longer have a daemonset called aws-node-termination-handler
-haveds && exit 1
+if haveds; then
+  echo "Expected aws-node-termination-handler to have been pruned"
+  exit 1
+fi

tests/e2e/scenarios/lib/common.sh

@@ -108,6 +108,9 @@ function kops-acquire-latest() {
          fi
          $KUBETEST2 --build
          KOPS="${REPO_ROOT}/.bazelbuild/dist/linux/amd64/kops"
+         KOPS_BASE_URL=$(cat "${REPO_ROOT}/.kubetest2/kops-base-url")
+         export KOPS_BASE_URL
+         echo "KOPS_BASE_URL=$KOPS_BASE_URL"
     fi
 }
 

tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content

@@ -62,6 +62,42 @@ spec:
     manifest: node-termination-handler.aws/k8s-1.11.yaml
     manifestHash: 4197a26e91677e28e68617b12e8b2e9f2825d397578dcd7d8b6500e47a05c4c2
     name: node-termination-handler.aws
+    prune:
+      kinds:
+      - kind: ConfigMap
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: Service
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: ServiceAccount
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: apps
+        kind: DaemonSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: apps
+        kind: Deployment
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: StatefulSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: policy
+        kind: PodDisruptionBudget
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRole
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: Role
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: RoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
     selector:
       k8s-addon: node-termination-handler.aws
     version: 9.99.0

tests/integration/update_cluster/many-addons-ccm/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content

@@ -62,6 +62,42 @@ spec:
     manifest: node-termination-handler.aws/k8s-1.11.yaml
     manifestHash: dd42ae2f7510700d37bf0214e0afdd87c12968c5c67ec88791f20f06fef90caf
     name: node-termination-handler.aws
+    prune:
+      kinds:
+      - kind: ConfigMap
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: Service
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: ServiceAccount
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: apps
+        kind: DaemonSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: apps
+        kind: Deployment
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: StatefulSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: policy
+        kind: PodDisruptionBudget
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRole
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: Role
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: RoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
     selector:
       k8s-addon: node-termination-handler.aws
     version: 9.99.0

tests/integration/update_cluster/many-addons/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content

@@ -62,6 +62,42 @@ spec:
     manifest: node-termination-handler.aws/k8s-1.11.yaml
     manifestHash: dd42ae2f7510700d37bf0214e0afdd87c12968c5c67ec88791f20f06fef90caf
     name: node-termination-handler.aws
+    prune:
+      kinds:
+      - kind: ConfigMap
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: Service
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: ServiceAccount
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: apps
+        kind: DaemonSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: apps
+        kind: Deployment
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: StatefulSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: policy
+        kind: PodDisruptionBudget
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRole
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: Role
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: RoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
     selector:
       k8s-addon: node-termination-handler.aws
     version: 9.99.0

tests/integration/update_cluster/nth_sqs_resources/data/aws_s3_bucket_object_nthsqsresources.longclustername.example.com-addons-bootstrap_content

@@ -49,6 +49,42 @@ spec:
     manifest: node-termination-handler.aws/k8s-1.11.yaml
     manifestHash: 424354959edcf24bcc3e1a3099b5b0a4525d59e2336a36940995ae51ead4ab08
     name: node-termination-handler.aws
+    prune:
+      kinds:
+      - kind: ConfigMap
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: Service
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - kind: ServiceAccount
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: apps
+        kind: DaemonSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: Deployment
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: apps
+        kind: StatefulSet
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: policy
+        kind: PodDisruptionBudget
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRole
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: Role
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: RoleBinding
+        labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops
     selector:
       k8s-addon: node-termination-handler.aws
     version: 9.99.0

tests/integration/update_cluster/privatekopeio/data/aws_s3_bucket_object_privatekopeio.example.com-addons-bootstrap_content

@@ -56,6 +56,42 @@ spec:
     manifest: networking.kope.io/k8s-1.12.yaml
     manifestHash: 294272eb01da2938395ff6425ac74690788b6f7ebe80327a83a77b2951b63968
     name: networking.kope.io
+    prune:
+      kinds:
+      - kind: ConfigMap
+        labelSelector: addon.kops.k8s.io/name=networking.kope.io,app.kubernetes.io/managed-by=kops
+      - kind: Service
+        labelSelector: addon.kops.k8s.io/name=networking.kope.io,app.kubernetes.io/managed-by=kops
+      - kind: ServiceAccount
+        labelSelector: addon.kops.k8s.io/name=networking.kope.io,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: apps
+        kind: DaemonSet
+        labelSelector: addon.kops.k8s.io/name=networking.kope.io,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: apps
+        kind: Deployment
+        labelSelector: addon.kops.k8s.io/name=networking.kope.io,app.kubernetes.io/managed-by=kops
+      - group: apps
+        kind: StatefulSet
+        labelSelector: addon.kops.k8s.io/name=networking.kope.io,app.kubernetes.io/managed-by=kops
+      - group: policy
+        kind: PodDisruptionBudget
+        labelSelector: addon.kops.k8s.io/name=networking.kope.io,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRole
+        labelSelector: addon.kops.k8s.io/name=networking.kope.io,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRoleBinding
+        labelSelector: addon.kops.k8s.io/name=networking.kope.io,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: Role
+        labelSelector: addon.kops.k8s.io/name=networking.kope.io,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: RoleBinding
+        labelSelector: addon.kops.k8s.io/name=networking.kope.io,app.kubernetes.io/managed-by=kops
     selector:
       role.kubernetes.io/networking: "1"
     version: 9.99.0

upup/pkg/fi/cloudup/bootstrapchannelbuilder/BUILD.bazel

@@ -5,6 +5,7 @@ go_library(
     srcs = [
         "bootstrapchannelbuilder.go",
         "cilium.go",
+        "pruning.go",
     ],
     importpath = "k8s.io/kops/upup/pkg/fi/cloudup/bootstrapchannelbuilder",
     visibility = ["//visibility:public"],
@@ -31,6 +32,9 @@ go_library(
         "//upup/pkg/fi/fitasks:go_default_library",
         "//upup/pkg/fi/utils:go_default_library",
         "//vendor/github.com/blang/semver/v4:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/klog/v2:go_default_library",
     ],
 )

upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go

@@ -100,22 +100,18 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error {
 		return err
 	}
 
-	if err := addons.Verify(); err != nil {
+	for _, a := range addons.Items {
-		return err
-	}
-
-	for _, a := range addons.Spec.Addons {
 		// Older versions of channels that may be running on the upgrading cluster requires Version to be set
 		// We hardcode version to a high version to ensure an update is triggered on first run, and from then on
 		// only a hash change will trigger an addon update.
-		a.Version = "9.99.0"
+		a.Spec.Version = "9.99.0"
 
-		key := *a.Name
+		key := *a.Spec.Name
-		if a.Id != "" {
+		if a.Spec.Id != "" {
-			key = key + "-" + a.Id
+			key = key + "-" + a.Spec.Id
 		}
 		name := b.Cluster.ObjectMeta.Name + "-addons-" + key
-		manifestPath := "addons/" + *a.Manifest
+		manifestPath := "addons/" + *a.Spec.Manifest
 		klog.V(4).Infof("Addon %q", name)
 
 		manifestResource := b.templates.Find(manifestPath)
@@ -129,7 +125,7 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error {
 		}
 
 		// Go through any transforms that are best expressed as code
-		remapped, err := addonmanifests.RemapAddonManifest(a, b.KopsModelContext, b.assetBuilder, manifestBytes)
+		remapped, err := addonmanifests.RemapAddonManifest(a.Spec, b.KopsModelContext, b.assetBuilder, manifestBytes)
 		if err != nil {
 			klog.Infof("invalid manifest: %s", string(manifestBytes))
 			return fmt.Errorf("error remapping manifest %s: %v", manifestPath, err)
@@ -139,6 +135,8 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error {
 		// Trim whitespace
 		manifestBytes = []byte(strings.TrimSpace(string(manifestBytes)))
 
+		a.ManifestData = manifestBytes
+
 		rawManifest := string(manifestBytes)
 		klog.V(4).Infof("Manifest %v", rawManifest)
 
@@ -147,7 +145,7 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error {
 		if err != nil {
 			return fmt.Errorf("error hashing manifest: %v", err)
 		}
-		a.ManifestHash = manifestHash
+		a.Spec.ManifestHash = manifestHash
 
 		c.AddTask(&fitasks.ManagedFile{
 			Contents:  fi.NewBytesResource(manifestBytes),
@@ -202,7 +200,8 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error {
 				Name:      fi.String(name),
 			})
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &a.Spec)
+			addon := addons.Add(&a.Spec)
+			addon.ManifestData = manifestBytes
 		}
 
 		b.ClusterAddons = append(b.ClusterAddons, crds...)
@@ -244,10 +243,25 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error {
 			Name:      fi.String(name),
 		})
 
-		addons.Spec.Addons = append(addons.Spec.Addons, a)
+		addons.Add(a)
 	}
 
-	addonsYAML, err := utils.YamlMarshal(addons)
+	if err := b.addPruneDirectives(addons); err != nil {
+		return err
+	}
+
+	addonsObject := &channelsapi.Addons{}
+	addonsObject.Kind = "Addons"
+	addonsObject.ObjectMeta.Name = "bootstrap"
+	for _, addon := range addons.Items {
+		addonsObject.Spec.Addons = append(addonsObject.Spec.Addons, addon.Spec)
+	}
+
+	if err := addonsObject.Verify(); err != nil {
+		return err
+	}
+
+	addonsYAML, err := utils.YamlMarshal(addonsObject)
 	if err != nil {
 		return fmt.Errorf("error serializing addons yaml: %v", err)
 	}
@@ -264,12 +278,33 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error {
 	return nil
 }
 
-func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*channelsapi.Addons, error) {
+type AddonList struct {
-	serviceAccountRoles := []iam.Subject{}
+	Items []*Addon
+}
 
-	addons := &channelsapi.Addons{}
+func (a *AddonList) Add(spec *channelsapi.AddonSpec) *Addon {
-	addons.Kind = "Addons"
+	addon := &Addon{
-	addons.ObjectMeta.Name = "bootstrap"
+		Spec: spec,
+	}
+	a.Items = append(a.Items, addon)
+	return addon
+}
+
+type Addon struct {
+	// Spec is the spec that will (eventually) be passed to the channels binary.
+	Spec *channelsapi.AddonSpec
+
+	// ManifestData is the object data loaded from the manifest.
+	ManifestData []byte
+
+	// BuildPrune is set if we should automatically build prune specifiers, based on the manifest.
+	BuildPrune bool
+}
+
+func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*AddonList, error) {
+	addons := &AddonList{}
+
+	serviceAccountRoles := []iam.Subject{}
 
 	{
 		key := "kops-controller.addons.k8s.io"
@@ -278,7 +313,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.16.yaml"
 			id := "k8s-1.16"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:               fi.String(key),
 				Selector:           map[string]string{"k8s-addon": key},
 				Manifest:           fi.String(location),
@@ -293,7 +328,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 		version := "1.4.0"
 		location := key + "/v" + version + ".yaml"
 
-		addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+		addons.Add(&channelsapi.AddonSpec{
 			Name:     fi.String(key),
 			Selector: map[string]string{"k8s-addon": key},
 			Manifest: fi.String(location),
@@ -308,7 +343,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.12.yaml"
 			id := "k8s-1.12"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
@@ -338,7 +373,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.12.yaml"
 				id := "k8s-1.12"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: map[string]string{"k8s-addon": key},
 					Manifest: fi.String(location),
@@ -356,7 +391,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.12.yaml"
 				id := "k8s-1.12"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: map[string]string{"k8s-addon": key},
 					Manifest: fi.String(location),
@@ -387,7 +422,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.8.yaml"
 				id := "k8s-1.8"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: map[string]string{"k8s-addon": key},
 					Manifest: fi.String(location),
@@ -407,7 +442,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.9.yaml"
 			id := "k8s-1.9"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
@@ -421,7 +456,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 		version := "1.5.0"
 		location := key + "/v" + version + ".yaml"
 
-		addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+		addons.Add(&channelsapi.AddonSpec{
 			Name:     fi.String(key),
 			Selector: map[string]string{"k8s-addon": key},
 			Manifest: fi.String(location),
@@ -434,19 +469,16 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 		if externalDNS == nil || !externalDNS.Disable {
 			{
 				key := "dns-controller.addons.k8s.io"
-
-				{
 				location := key + "/k8s-1.12.yaml"
 				id := "k8s-1.12"
 
-					addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: map[string]string{"k8s-addon": key},
 					Manifest: fi.String(location),
 					Id:       id,
 				})
 			}
-			}
 
 			// Generate dns-controller ServiceAccount IAM permissions
 			if b.UseServiceAccountExternalPermissions() {
@@ -461,7 +493,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.12.yaml"
 				id := "k8s-1.12"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: map[string]string{"k8s-addon": key},
 					Manifest: fi.String(location),
@@ -485,7 +517,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.12.yaml"
 				id := "k8s-1.12"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: map[string]string{"k8s-addon": key},
 					Manifest: fi.String(location),
@@ -503,7 +535,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.15.yaml"
 				id := "k8s-1.15"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: map[string]string{"k8s-addon": key},
 					Manifest: fi.String(location),
@@ -526,7 +558,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.11.yaml"
 				id := "k8s-1.11"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: map[string]string{"k8s-app": "metrics-server"},
 					Manifest: fi.String(location),
@@ -545,7 +577,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.16.yaml"
 				id := "k8s-1.16"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Manifest: fi.String(location),
 					Id:       id,
@@ -564,12 +596,13 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.11.yaml"
 			id := "k8s-1.11"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addon := addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
 				Id:       id,
 			})
+			addon.BuildPrune = true
 		}
 
 		if b.UseServiceAccountExternalPermissions() {
@@ -587,7 +620,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.17.yaml"
 			id := "k8s-1.17"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
@@ -606,7 +639,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.16.yaml"
 			id := "k8s-1.16"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
@@ -623,7 +656,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.9.yaml"
 			id := "k8s-1.9"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
@@ -645,7 +678,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			id := "v1.15.0"
 			location := key + "/" + id + ".yaml"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
@@ -661,7 +694,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			id := "k8s-1.8"
 			location := key + "/" + id + ".yaml"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
@@ -677,7 +710,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			id := "v1.7.0"
 			location := key + "/" + id + ".yaml"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
@@ -693,7 +726,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			id := "v1.14.0"
 			location := key + "/" + id + ".yaml"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
@@ -711,7 +744,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			id := "v0.1.12"
 			location := key + "/" + id + ".yaml"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
@@ -727,12 +760,14 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.12.yaml"
 			id := "k8s-1.12"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addon := addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: networkingSelector(),
 				Manifest: fi.String(location),
 				Id:       id,
 			})
+
+			addon.BuildPrune = true
 		}
 	}
 
@@ -743,7 +778,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.12.yaml"
 			id := "k8s-1.12"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: networkingSelector(),
 				Manifest: fi.String(location),
@@ -759,7 +794,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.12.yaml"
 			id := "k8s-1.12"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: networkingSelector(),
 				Manifest: fi.String(location),
@@ -775,7 +810,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			id := "k8s-1.16"
 			location := key + "/" + id + ".yaml"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: networkingSelector(),
 				Manifest: fi.String(location),
@@ -791,7 +826,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			id := "k8s-1.16"
 			location := key + "/" + id + ".yaml"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: networkingSelector(),
 				Manifest: fi.String(location),
@@ -807,7 +842,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.12.yaml"
 			id := "k8s-1.12"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: networkingSelector(),
 				Manifest: fi.String(location),
@@ -823,7 +858,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			id := "k8s-1.16"
 			location := key + "/" + id + ".yaml"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:               fi.String(key),
 				Selector:           networkingSelector(),
 				Manifest:           fi.String(location),
@@ -848,7 +883,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.12.yaml"
 				id := "k8s-1.12"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: authenticationSelector,
 					Manifest: fi.String(location),
@@ -863,7 +898,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.12.yaml"
 				id := "k8s-1.12"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: authenticationSelector,
 					Manifest: fi.String(location),
@@ -880,7 +915,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			id := "k8s-1.16"
 			location := key + "/" + id + ".yaml"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Manifest: fi.String(location),
 				Selector: map[string]string{"k8s-addon": key},
@@ -896,7 +931,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.13.yaml"
 				id := "k8s-1.13-ccm"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Manifest: fi.String(location),
 					Selector: map[string]string{"k8s-addon": key},
@@ -910,7 +945,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.12.yaml"
 				id := "k8s-1.12-ccm"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: map[string]string{"k8s-addon": key},
 					Manifest: fi.String(location),
@@ -928,7 +963,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			{
 				id := "k8s-1.18"
 				location := key + "/" + id + ".yaml"
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Manifest: fi.String(location),
 					Selector: map[string]string{"k8s-addon": key},
@@ -945,7 +980,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			{
 				id := "k8s-1.17"
 				location := key + "/" + id + ".yaml"
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Manifest: fi.String(location),
 					Selector: map[string]string{"k8s-addon": key},
@@ -964,7 +999,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 		{
 			id := "k8s-1.20"
 			location := key + "/" + id + ".yaml"
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Manifest: fi.String(location),
 				Selector: map[string]string{"k8s-addon": key},
@@ -979,7 +1014,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 		version := "1.7.0"
 		location := key + "/v" + version + ".yaml"
 
-		addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+		addons.Add(&channelsapi.AddonSpec{
 			Name:     fi.String(key),
 			Selector: map[string]string{"k8s-addon": key},
 			Manifest: fi.String(location),

upup/pkg/fi/cloudup/bootstrapchannelbuilder/cilium.go

@@ -24,7 +24,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi"
 )
 
-func addCiliumAddon(b *BootstrapChannelBuilder, addons *api.Addons) error {
+func addCiliumAddon(b *BootstrapChannelBuilder, addons *AddonList) error {
 
 	cilium := b.Cluster.Spec.Networking.Cilium
 	if cilium != nil {
@@ -39,7 +39,7 @@ func addCiliumAddon(b *BootstrapChannelBuilder, addons *api.Addons) error {
 				id := "k8s-1.12"
 				location := key + "/" + id + "-v1.8.yaml"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &api.AddonSpec{
+				addons.Add(&api.AddonSpec{
 					Name:               fi.String(key),
 					Selector:           networkingSelector(),
 					Manifest:           fi.String(location),
@@ -62,7 +62,7 @@ func addCiliumAddon(b *BootstrapChannelBuilder, addons *api.Addons) error {
 				if cilium.Hubble != nil && fi.BoolValue(cilium.Hubble.Enabled) {
 					addon.NeedsPKI = true
 				}
-				addons.Spec.Addons = append(addons.Spec.Addons, addon)
+				addons.Add(addon)
 			}
 		} else if ver.Minor == 10 {
 			{
@@ -79,7 +79,7 @@ func addCiliumAddon(b *BootstrapChannelBuilder, addons *api.Addons) error {
 				if cilium.Hubble != nil && fi.BoolValue(cilium.Hubble.Enabled) {
 					addon.NeedsPKI = true
 				}
-				addons.Spec.Addons = append(addons.Spec.Addons, addon)
+				addons.Add(addon)
 			}
 		} else {
 			return fmt.Errorf("unknown cilium version: %q", cilium.Version)

upup/pkg/fi/cloudup/bootstrapchannelbuilder/pruning.go

@@ -0,0 +1,151 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bootstrapchannelbuilder
+
+import (
+	"fmt"
+	"sort"
+
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/klog/v2"
+
+	channelsapi "k8s.io/kops/channels/pkg/api"
+	"k8s.io/kops/pkg/kubemanifest"
+)
+
+func (b *BootstrapChannelBuilder) addPruneDirectives(addons *AddonList) error {
+	for _, addon := range addons.Items {
+		if !addon.BuildPrune {
+			continue
+		}
+
+		id := *addon.Spec.Name
+
+		if err := b.addPruneDirectivesForAddon(addon); err != nil {
+			return fmt.Errorf("failed to configure pruning for %s: %w", id, err)
+		}
+	}
+	return nil
+}
+
+func (b *BootstrapChannelBuilder) addPruneDirectivesForAddon(addon *Addon) error {
+	addon.Spec.Prune = &channelsapi.PruneSpec{}
+
+	// We add these labels to all objects we manage, so we reuse them for pruning.
+	selectorMap := map[string]string{
+		"app.kubernetes.io/managed-by": "kops",
+		"addon.kops.k8s.io/name":       *addon.Spec.Name,
+	}
+	selector, err := labels.ValidatedSelectorFromSet(selectorMap)
+	if err != nil {
+		return fmt.Errorf("error parsing selector %v: %w", selectorMap, err)
+	}
+
+	// We always include a set of well-known group kinds,
+	// so that we prune even if we end up removing something from the manifest.
+	alwaysPruneGroupKinds := []schema.GroupKind{
+		{Group: "", Kind: "ConfigMap"},
+		{Group: "", Kind: "Service"},
+		{Group: "", Kind: "ServiceAccount"},
+		{Group: "apps", Kind: "Deployment"},
+		{Group: "apps", Kind: "DaemonSet"},
+		{Group: "apps", Kind: "StatefulSet"},
+		{Group: "rbac.authorization.k8s.io", Kind: "ClusterRole"},
+		{Group: "rbac.authorization.k8s.io", Kind: "ClusterRoleBinding"},
+		{Group: "rbac.authorization.k8s.io", Kind: "Role"},
+		{Group: "rbac.authorization.k8s.io", Kind: "RoleBinding"},
+		{Group: "policy", Kind: "PodDisruptionBudget"},
+	}
+	pruneGroupKind := make(map[schema.GroupKind]bool)
+	for _, gk := range alwaysPruneGroupKinds {
+		pruneGroupKind[gk] = true
+	}
+
+	// In addition, we deliberately exclude a few types that are riskier to delete:
+	//
+	//  * Namespace: because it deletes anything else that happens to be in the namespace
+	//
+	//  * CustomResourceDefinition: because it deletes all instances of the CRD
+	neverPruneGroupKinds := map[schema.GroupKind]bool{
+		{Group: "", Kind: "Namespace"}:                                    true,
+		{Group: "apiextensions.k8s.io", Kind: "CustomResourceDefinition"}: true,
+	}
+
+	// Parse the manifest; we use this to scope pruning to namespaces
+	objects, err := kubemanifest.LoadObjectsFrom(addon.ManifestData)
+	if err != nil {
+		return fmt.Errorf("failed to parse manifest: %w", err)
+	}
+	objectsByGK := make(map[schema.GroupKind][]*kubemanifest.Object)
+	for _, object := range objects {
+		gv, err := schema.ParseGroupVersion(object.APIVersion())
+		if err != nil || gv.Version == "" {
+			return fmt.Errorf("failed to parse apiVersion %q", object.APIVersion())
+		}
+		gvk := gv.WithKind(object.Kind())
+		if gvk.Kind == "" {
+			return fmt.Errorf("failed to get kind for object")
+		}
+
+		gk := gvk.GroupKind()
+		objectsByGK[gk] = append(objectsByGK[gk], object)
+
+		// Warn if there are objects in the manifest that we haven't considered
+		if !pruneGroupKind[gk] {
+			if !neverPruneGroupKinds[gk] {
+				klog.Warningf("manifest includes an object of GroupKind %v, which will not be pruned", gk)
+			}
+		}
+	}
+
+	var groupKinds []schema.GroupKind
+	for gk := range pruneGroupKind {
+		groupKinds = append(groupKinds, gk)
+	}
+
+	sort.Slice(groupKinds, func(i, j int) bool {
+		if groupKinds[i].Group != groupKinds[j].Group {
+			return groupKinds[i].Group < groupKinds[j].Group
+		}
+		return groupKinds[i].Kind < groupKinds[j].Kind
+	})
+
+	for _, gk := range groupKinds {
+		pruneSpec := channelsapi.PruneKindSpec{}
+		pruneSpec.Group = gk.Group
+		pruneSpec.Kind = gk.Kind
+
+		namespaces := sets.NewString()
+		for _, object := range objectsByGK[gk] {
+			namespace := object.GetNamespace()
+			if namespace != "" {
+				namespaces.Insert(namespace)
+			}
+		}
+		if namespaces.Len() != 0 {
+			pruneSpec.Namespaces = namespaces.List()
+		}
+
+		pruneSpec.LabelSelector = selector.String()
+
+		addon.Spec.Prune.Kinds = append(addon.Spec.Prune.Kinds, pruneSpec)
+	}
+
+	return nil
+}