Automatically generate prune specifiers

We use the "app.kubernetes.io/managed-by" and "addon.kops.k8s.io/name"
labels, which are automatically added.

tests/integration/update_cluster/privatekopeio/data/aws_s3_bucket_object_privatekopeio.example.com-addons-bootstrap_content

@@ -56,6 +56,23 @@ spec:
     manifest: networking.kope.io/k8s-1.12.yaml
     manifestHash: 294272eb01da2938395ff6425ac74690788b6f7ebe80327a83a77b2951b63968
     name: networking.kope.io
+    prune:
+      kinds:
+      - kind: ServiceAccount
+        labelSelector: addon.kops.k8s.io/name=networking.kope.io,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: apps
+        kind: DaemonSet
+        labelSelector: addon.kops.k8s.io/name=networking.kope.io,app.kubernetes.io/managed-by=kops
+        namespaces:
+        - kube-system
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRole
+        labelSelector: addon.kops.k8s.io/name=networking.kope.io,app.kubernetes.io/managed-by=kops
+      - group: rbac.authorization.k8s.io
+        kind: ClusterRoleBinding
+        labelSelector: addon.kops.k8s.io/name=networking.kope.io,app.kubernetes.io/managed-by=kops
     selector:
       role.kubernetes.io/networking: "1"
     version: 9.99.0

upup/pkg/fi/cloudup/bootstrapchannelbuilder/BUILD.bazel

@@ -5,6 +5,7 @@ go_library(
     srcs = [
         "bootstrapchannelbuilder.go",
         "cilium.go",
+        "pruning.go",
     ],
     importpath = "k8s.io/kops/upup/pkg/fi/cloudup/bootstrapchannelbuilder",
     visibility = ["//visibility:public"],
@@ -31,6 +32,9 @@ go_library(
         "//upup/pkg/fi/fitasks:go_default_library",
         "//upup/pkg/fi/utils:go_default_library",
         "//vendor/github.com/blang/semver/v4:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/klog/v2:go_default_library",
     ],
 )

upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go

@@ -100,22 +100,18 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error {
 		return err
 	}
 
-	if err := addons.Verify(); err != nil {
+	for _, a := range addons.Items {
-		return err
-	}
-
-	for _, a := range addons.Spec.Addons {
 		// Older versions of channels that may be running on the upgrading cluster requires Version to be set
 		// We hardcode version to a high version to ensure an update is triggered on first run, and from then on
 		// only a hash change will trigger an addon update.
-		a.Version = "9.99.0"
+		a.Spec.Version = "9.99.0"
 
-		key := *a.Name
+		key := *a.Spec.Name
-		if a.Id != "" {
+		if a.Spec.Id != "" {
-			key = key + "-" + a.Id
+			key = key + "-" + a.Spec.Id
 		}
 		name := b.Cluster.ObjectMeta.Name + "-addons-" + key
-		manifestPath := "addons/" + *a.Manifest
+		manifestPath := "addons/" + *a.Spec.Manifest
 		klog.V(4).Infof("Addon %q", name)
 
 		manifestResource := b.templates.Find(manifestPath)
@@ -129,7 +125,7 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error {
 		}
 
 		// Go through any transforms that are best expressed as code
-		remapped, err := addonmanifests.RemapAddonManifest(a, b.KopsModelContext, b.assetBuilder, manifestBytes)
+		remapped, err := addonmanifests.RemapAddonManifest(a.Spec, b.KopsModelContext, b.assetBuilder, manifestBytes)
 		if err != nil {
 			klog.Infof("invalid manifest: %s", string(manifestBytes))
 			return fmt.Errorf("error remapping manifest %s: %v", manifestPath, err)
@@ -139,6 +135,8 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error {
 		// Trim whitespace
 		manifestBytes = []byte(strings.TrimSpace(string(manifestBytes)))
 
+		a.ManifestData = manifestBytes
+
 		rawManifest := string(manifestBytes)
 		klog.V(4).Infof("Manifest %v", rawManifest)
 
@@ -147,7 +145,7 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error {
 		if err != nil {
 			return fmt.Errorf("error hashing manifest: %v", err)
 		}
-		a.ManifestHash = manifestHash
+		a.Spec.ManifestHash = manifestHash
 
 		c.AddTask(&fitasks.ManagedFile{
 			Contents:  fi.NewBytesResource(manifestBytes),
@@ -202,7 +200,8 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error {
 				Name:      fi.String(name),
 			})
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &a.Spec)
+			addon := addons.Add(&a.Spec)
+			addon.ManifestData = manifestBytes
 		}
 
 		b.ClusterAddons = append(b.ClusterAddons, crds...)
@@ -244,10 +243,25 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error {
 			Name:      fi.String(name),
 		})
 
-		addons.Spec.Addons = append(addons.Spec.Addons, a)
+		addons.Add(a)
 	}
 
-	addonsYAML, err := utils.YamlMarshal(addons)
+	if err := b.addPruneDirectives(addons); err != nil {
+		return err
+	}
+
+	addonsObject := &channelsapi.Addons{}
+	addonsObject.Kind = "Addons"
+	addonsObject.ObjectMeta.Name = "bootstrap"
+	for _, addon := range addons.Items {
+		addonsObject.Spec.Addons = append(addonsObject.Spec.Addons, addon.Spec)
+	}
+
+	if err := addonsObject.Verify(); err != nil {
+		return err
+	}
+
+	addonsYAML, err := utils.YamlMarshal(addonsObject)
 	if err != nil {
 		return fmt.Errorf("error serializing addons yaml: %v", err)
 	}
@@ -264,12 +278,33 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error {
 	return nil
 }
 
-func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*channelsapi.Addons, error) {
+type AddonList struct {
-	serviceAccountRoles := []iam.Subject{}
+	Items []*Addon
+}
 
-	addons := &channelsapi.Addons{}
+func (a *AddonList) Add(spec *channelsapi.AddonSpec) *Addon {
-	addons.Kind = "Addons"
+	addon := &Addon{
-	addons.ObjectMeta.Name = "bootstrap"
+		Spec: spec,
+	}
+	a.Items = append(a.Items, addon)
+	return addon
+}
+
+type Addon struct {
+	// Spec is the spec that will (eventually) be passed to the channels binary.
+	Spec *channelsapi.AddonSpec
+
+	// ManifestData is the object data loaded from the manifest.
+	ManifestData []byte
+
+	// BuildPrune is set if we should automatically build prune specifiers, based on the manifest.
+	BuildPrune bool
+}
+
+func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*AddonList, error) {
+	addons := &AddonList{}
+
+	serviceAccountRoles := []iam.Subject{}
 
 	{
 		key := "kops-controller.addons.k8s.io"
@@ -278,7 +313,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.16.yaml"
 			id := "k8s-1.16"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:               fi.String(key),
 				Selector:           map[string]string{"k8s-addon": key},
 				Manifest:           fi.String(location),
@@ -293,7 +328,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 		version := "1.4.0"
 		location := key + "/v" + version + ".yaml"
 
-		addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+		addons.Add(&channelsapi.AddonSpec{
 			Name:     fi.String(key),
 			Selector: map[string]string{"k8s-addon": key},
 			Manifest: fi.String(location),
@@ -308,7 +343,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.12.yaml"
 			id := "k8s-1.12"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
@@ -338,7 +373,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.12.yaml"
 				id := "k8s-1.12"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: map[string]string{"k8s-addon": key},
 					Manifest: fi.String(location),
@@ -356,7 +391,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.12.yaml"
 				id := "k8s-1.12"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: map[string]string{"k8s-addon": key},
 					Manifest: fi.String(location),
@@ -387,7 +422,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.8.yaml"
 				id := "k8s-1.8"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: map[string]string{"k8s-addon": key},
 					Manifest: fi.String(location),
@@ -407,7 +442,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.9.yaml"
 			id := "k8s-1.9"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
@@ -421,7 +456,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 		version := "1.5.0"
 		location := key + "/v" + version + ".yaml"
 
-		addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+		addons.Add(&channelsapi.AddonSpec{
 			Name:     fi.String(key),
 			Selector: map[string]string{"k8s-addon": key},
 			Manifest: fi.String(location),
@@ -434,19 +469,16 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 		if externalDNS == nil || !externalDNS.Disable {
 			{
 				key := "dns-controller.addons.k8s.io"
-
-				{
 				location := key + "/k8s-1.12.yaml"
 				id := "k8s-1.12"
 
-					addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: map[string]string{"k8s-addon": key},
 					Manifest: fi.String(location),
 					Id:       id,
 				})
 			}
-			}
 
 			// Generate dns-controller ServiceAccount IAM permissions
 			if b.UseServiceAccountExternalPermissions() {
@@ -461,7 +493,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.12.yaml"
 				id := "k8s-1.12"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: map[string]string{"k8s-addon": key},
 					Manifest: fi.String(location),
@@ -485,7 +517,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.12.yaml"
 				id := "k8s-1.12"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: map[string]string{"k8s-addon": key},
 					Manifest: fi.String(location),
@@ -503,7 +535,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.15.yaml"
 				id := "k8s-1.15"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: map[string]string{"k8s-addon": key},
 					Manifest: fi.String(location),
@@ -526,7 +558,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.11.yaml"
 				id := "k8s-1.11"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: map[string]string{"k8s-app": "metrics-server"},
 					Manifest: fi.String(location),
@@ -545,7 +577,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.16.yaml"
 				id := "k8s-1.16"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Manifest: fi.String(location),
 					Id:       id,
@@ -564,7 +596,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.11.yaml"
 			id := "k8s-1.11"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
@@ -587,7 +619,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.17.yaml"
 			id := "k8s-1.17"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
@@ -606,7 +638,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.16.yaml"
 			id := "k8s-1.16"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
@@ -623,7 +655,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.9.yaml"
 			id := "k8s-1.9"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
@@ -645,7 +677,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			id := "v1.15.0"
 			location := key + "/" + id + ".yaml"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
@@ -661,7 +693,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			id := "k8s-1.8"
 			location := key + "/" + id + ".yaml"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
@@ -677,7 +709,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			id := "v1.7.0"
 			location := key + "/" + id + ".yaml"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
@@ -693,7 +725,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			id := "v1.14.0"
 			location := key + "/" + id + ".yaml"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
@@ -711,7 +743,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			id := "v0.1.12"
 			location := key + "/" + id + ".yaml"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: map[string]string{"k8s-addon": key},
 				Manifest: fi.String(location),
@@ -727,12 +759,14 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.12.yaml"
 			id := "k8s-1.12"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addon := addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: networkingSelector(),
 				Manifest: fi.String(location),
 				Id:       id,
 			})
+
+			addon.BuildPrune = true
 		}
 	}
 
@@ -743,7 +777,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.12.yaml"
 			id := "k8s-1.12"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: networkingSelector(),
 				Manifest: fi.String(location),
@@ -759,7 +793,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.12.yaml"
 			id := "k8s-1.12"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: networkingSelector(),
 				Manifest: fi.String(location),
@@ -775,7 +809,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			id := "k8s-1.16"
 			location := key + "/" + id + ".yaml"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: networkingSelector(),
 				Manifest: fi.String(location),
@@ -791,7 +825,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			id := "k8s-1.16"
 			location := key + "/" + id + ".yaml"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: networkingSelector(),
 				Manifest: fi.String(location),
@@ -807,7 +841,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			location := key + "/k8s-1.12.yaml"
 			id := "k8s-1.12"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Selector: networkingSelector(),
 				Manifest: fi.String(location),
@@ -823,7 +857,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			id := "k8s-1.16"
 			location := key + "/" + id + ".yaml"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:               fi.String(key),
 				Selector:           networkingSelector(),
 				Manifest:           fi.String(location),
@@ -848,7 +882,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.12.yaml"
 				id := "k8s-1.12"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: authenticationSelector,
 					Manifest: fi.String(location),
@@ -863,7 +897,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.12.yaml"
 				id := "k8s-1.12"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: authenticationSelector,
 					Manifest: fi.String(location),
@@ -880,7 +914,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			id := "k8s-1.16"
 			location := key + "/" + id + ".yaml"
 
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Manifest: fi.String(location),
 				Selector: map[string]string{"k8s-addon": key},
@@ -896,7 +930,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.13.yaml"
 				id := "k8s-1.13-ccm"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Manifest: fi.String(location),
 					Selector: map[string]string{"k8s-addon": key},
@@ -910,7 +944,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				location := key + "/k8s-1.12.yaml"
 				id := "k8s-1.12-ccm"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Selector: map[string]string{"k8s-addon": key},
 					Manifest: fi.String(location),
@@ -928,7 +962,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			{
 				id := "k8s-1.18"
 				location := key + "/" + id + ".yaml"
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Manifest: fi.String(location),
 					Selector: map[string]string{"k8s-addon": key},
@@ -945,7 +979,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 			{
 				id := "k8s-1.17"
 				location := key + "/" + id + ".yaml"
-				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+				addons.Add(&channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Manifest: fi.String(location),
 					Selector: map[string]string{"k8s-addon": key},
@@ -964,7 +998,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 		{
 			id := "k8s-1.20"
 			location := key + "/" + id + ".yaml"
-			addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+			addons.Add(&channelsapi.AddonSpec{
 				Name:     fi.String(key),
 				Manifest: fi.String(location),
 				Selector: map[string]string{"k8s-addon": key},
@@ -979,7 +1013,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 		version := "1.7.0"
 		location := key + "/v" + version + ".yaml"
 
-		addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+		addons.Add(&channelsapi.AddonSpec{
 			Name:     fi.String(key),
 			Selector: map[string]string{"k8s-addon": key},
 			Manifest: fi.String(location),

upup/pkg/fi/cloudup/bootstrapchannelbuilder/cilium.go

@@ -24,7 +24,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi"
 )
 
-func addCiliumAddon(b *BootstrapChannelBuilder, addons *api.Addons) error {
+func addCiliumAddon(b *BootstrapChannelBuilder, addons *AddonList) error {
 
 	cilium := b.Cluster.Spec.Networking.Cilium
 	if cilium != nil {
@@ -39,7 +39,7 @@ func addCiliumAddon(b *BootstrapChannelBuilder, addons *api.Addons) error {
 				id := "k8s-1.12"
 				location := key + "/" + id + "-v1.8.yaml"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &api.AddonSpec{
+				addons.Add(&api.AddonSpec{
 					Name:               fi.String(key),
 					Selector:           networkingSelector(),
 					Manifest:           fi.String(location),
@@ -62,7 +62,7 @@ func addCiliumAddon(b *BootstrapChannelBuilder, addons *api.Addons) error {
 				if cilium.Hubble != nil && fi.BoolValue(cilium.Hubble.Enabled) {
 					addon.NeedsPKI = true
 				}
-				addons.Spec.Addons = append(addons.Spec.Addons, addon)
+				addons.Add(addon)
 			}
 		} else if ver.Minor == 10 {
 			{
@@ -79,7 +79,7 @@ func addCiliumAddon(b *BootstrapChannelBuilder, addons *api.Addons) error {
 				if cilium.Hubble != nil && fi.BoolValue(cilium.Hubble.Enabled) {
 					addon.NeedsPKI = true
 				}
-				addons.Spec.Addons = append(addons.Spec.Addons, addon)
+				addons.Add(addon)
 			}
 		} else {
 			return fmt.Errorf("unknown cilium version: %q", cilium.Version)

upup/pkg/fi/cloudup/bootstrapchannelbuilder/pruning.go

@@ -0,0 +1,111 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bootstrapchannelbuilder
+
+import (
+	"fmt"
+	"sort"
+
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/sets"
+	channelsapi "k8s.io/kops/channels/pkg/api"
+	"k8s.io/kops/pkg/kubemanifest"
+)
+
+func (b *BootstrapChannelBuilder) addPruneDirectives(addons *AddonList) error {
+	for _, addon := range addons.Items {
+		if !addon.BuildPrune {
+			continue
+		}
+
+		id := *addon.Spec.Name
+
+		if err := b.addPruneDirectivesForAddon(addon); err != nil {
+			return fmt.Errorf("failed to configure pruning for %s: %w", id, err)
+		}
+	}
+	return nil
+}
+
+func (b *BootstrapChannelBuilder) addPruneDirectivesForAddon(addon *Addon) error {
+	addon.Spec.Prune = &channelsapi.PruneSpec{}
+
+	// We add these labels to all objects we manage, so we reuse them for pruning.
+	selectorMap := map[string]string{
+		"app.kubernetes.io/managed-by": "kops",
+		"addon.kops.k8s.io/name":       *addon.Spec.Name,
+	}
+	selector, err := labels.ValidatedSelectorFromSet(selectorMap)
+	if err != nil {
+		return fmt.Errorf("error parsing selector %v: %w", selectorMap, err)
+	}
+
+	objects, err := kubemanifest.LoadObjectsFrom(addon.ManifestData)
+	if err != nil {
+		return fmt.Errorf("failed to parse manifest: %w", err)
+	}
+
+	byGroupKind := make(map[schema.GroupKind][]*kubemanifest.Object)
+	for _, object := range objects {
+		gv, err := schema.ParseGroupVersion(object.APIVersion())
+		if err != nil || gv.Version == "" {
+			return fmt.Errorf("failed to parse apiVersion %q", object.APIVersion())
+		}
+		gvk := gv.WithKind(object.Kind())
+		if gvk.Kind == "" {
+			return fmt.Errorf("failed to get kind for object")
+		}
+
+		gk := gvk.GroupKind()
+		byGroupKind[gk] = append(byGroupKind[gk], object)
+	}
+
+	var groupKinds []schema.GroupKind
+	for gk := range byGroupKind {
+		groupKinds = append(groupKinds, gk)
+	}
+	sort.Slice(groupKinds, func(i, j int) bool {
+		if groupKinds[i].Group != groupKinds[j].Group {
+			return groupKinds[i].Group < groupKinds[j].Group
+		}
+		return groupKinds[i].Kind < groupKinds[j].Kind
+	})
+
+	for _, gk := range groupKinds {
+		pruneSpec := channelsapi.PruneKindSpec{}
+		pruneSpec.Group = gk.Group
+		pruneSpec.Kind = gk.Kind
+
+		namespaces := sets.NewString()
+		for _, object := range byGroupKind[gk] {
+			namespace := object.GetNamespace()
+			if namespace != "" {
+				namespaces.Insert(namespace)
+			}
+		}
+		if namespaces.Len() != 0 {
+			pruneSpec.Namespaces = namespaces.List()
+		}
+
+		pruneSpec.LabelSelector = selector.String()
+
+		addon.Spec.Prune.Kinds = append(addon.Spec.Prune.Kinds, pruneSpec)
+	}
+
+	return nil
+}