Allow built-in manifests to be replaced by external addons

We identify the external manifests by checking for our labels.
Currently that label is kOps specific, and we'll likely have to evolve
that to something ecosystem-netural.

We only support the GCE CCM addon and the kopeio-networking addon at
first.

For the GCE CCM addon, we need to replace the arguments, in particular
we likely need the Pod CIDR.  Here we need to work with the GCE CCM to
find a mechanism that can allow some of these flags to be communicated
via a more extensible mechanism (env vars or config maps, likely,
though possibly CRDs).

This is all behind the ClusterAddons feature flag at the moment, so we
can figure this out with other projects safely.

Co-authored-by: John Gardiner Myers <jgmyers@proofpoint.com>

pkg/kubemanifest/containerargs.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubemanifest
+
+import (
+	"fmt"
+	"strings"
+)
+
+type ContainerVisitorFunction func(container map[string]interface{}) error
+
+func (m *Object) VisitContainers(visitorFn ContainerVisitorFunction) error {
+	visitorObj := &containerVisitor{
+		visitor: visitorFn,
+	}
+	err := m.accept(visitorObj)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+type containerVisitor struct {
+	visitorBase
+	visitor ContainerVisitorFunction
+}
+
+func (m *containerVisitor) VisitMap(path []string, v map[string]interface{}) error {
+	n := len(path)
+	if n < 2 || path[n-2] != "containers" || !strings.HasPrefix(path[n-1], "[") {
+		return nil
+	}
+
+	if err := m.visitor(v); err != nil {
+		return fmt.Errorf("error visiting container %v: %w", v, err)
+	}
+
+	return nil
+}

pkg/kubemanifest/visitor.go

@@ -40,10 +40,16 @@ func (m *visitorBase) VisitFloat64(path []string, v float64, mutator func(float6
 	return nil
 }
 
+func (m *visitorBase) VisitMap(path []string, v map[string]interface{}) error {
+	klog.V(10).Infof("object value at %s: %f", strings.Join(path, "."), v)
+	return nil
+}
+
 type Visitor interface {
 	VisitBool(path []string, v bool, mutator func(bool)) error
 	VisitString(path []string, v string, mutator func(string)) error
 	VisitFloat64(path []string, v float64, mutator func(float64)) error
+	VisitMap(path []string, v map[string]interface{}) error
 }
 
 func visit(visitor Visitor, data interface{}, path []string, mutator func(interface{})) error {
@@ -78,6 +84,11 @@ func visit(visitor Visitor, data interface{}, path []string, mutator func(interf
 	case nil:
 	case map[string]interface{}:
 		m := data
+
+		if err := visitor.VisitMap(path, m); err != nil {
+			return err
+		}
+
 		for k, v := range m {
 			path = append(path, k)
 
@@ -104,6 +115,9 @@ func visit(visitor Visitor, data interface{}, path []string, mutator func(interf
 			path = path[:len(path)-1]
 		}
 
+	case []string:
+		// ignore - we don't have any visitors for this currently
+
 	default:
 		return fmt.Errorf("unhandled type in manifest: %T", data)
 	}

pkg/model/components/addonmanifests/remap.go

@@ -31,6 +31,10 @@ import (
 	"k8s.io/kops/upup/pkg/fi"
 )
 
+// KopsAddonLabelKey is the label we use for objects in kOps manifests.
+// The value corresponds to the name of the addon.
+const KopsAddonLabelKey = "addon.kops.k8s.io/name"
+
 func RemapAddonManifest(addon *addonsapi.AddonSpec, context *model.KopsModelContext, assetBuilder *assets.AssetBuilder, manifest []byte, serviceAccounts map[string]iam.Subject) ([]byte, error) {
 	name := fi.ValueOf(addon.Name)
 
@@ -119,7 +123,7 @@ func addLabels(addon *addonsapi.AddonSpec, objects kubemanifest.ObjectList) erro
 		}
 
 		meta.Labels["app.kubernetes.io/managed-by"] = "kops"
-		meta.Labels["addon.kops.k8s.io/name"] = *addon.Name
+		meta.Labels[KopsAddonLabelKey] = *addon.Name
 
 		// ensure selector is set where applicable
 		for key, val := range addon.Selector {

upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go

@@ -870,7 +870,47 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext)
 		if b.Cluster.Spec.GetCloudProvider() == kops.CloudProviderGCE {
 			if b.Cluster.Spec.ExternalCloudControllerManager != nil {
 				key := "gcp-cloud-controller.addons.k8s.io"
-				{
+				useBuiltin := !b.hasExternalAddon(key)
+
+				if !useBuiltin {
+					klog.Infof("Found cloud-controller-manager in addons; won't use builtin")
+
+					// Until we make the manifest extensible, we still need to inject our arguments.
+					// TODO(justinsb): we don't really want to do this, it limits the ability for users to override things.
+					// However, this is behind a feature flag at the moment, and this way we can work towards something better.
+					gkDaemonset := schema.GroupKind{Group: "apps", Kind: "DaemonSet"}
+					for _, addon := range b.ClusterAddons {
+						if addon.GroupVersionKind().GroupKind() == gkDaemonset &&
+							addon.GetName() == "cloud-controller-manager" &&
+							addon.GetNamespace() == "kube-system" {
+
+							klog.Infof("replacing arguments in externally provided cloud-controller-manager")
+
+							fnAny, ok := b.templates.TemplateFunctions["CloudControllerConfigArgv"]
+							if !ok {
+								return nil, nil, fmt.Errorf("unable to find TemplateFunction CloudControllerConfigArgv")
+							}
+							fn, ok := fnAny.(func() ([]string, error))
+							if !ok {
+								return nil, nil, fmt.Errorf("unexpected type for TemplateFunction CloudControllerConfigArgv: %T", fnAny)
+							}
+							args, err := fn()
+							if err != nil {
+								return nil, nil, fmt.Errorf("in TemplateFunction CloudControllerConfigArgv: %w", err)
+							}
+
+							if err := addon.VisitContainers(func(container map[string]interface{}) error {
+								// TODO: Check name?
+								container["args"] = args
+								return nil
+							}); err != nil {
+								return nil, nil, fmt.Errorf("error visiting containers: %w", err)
+							}
+						}
+					}
+				}
+
+				if useBuiltin {
 					id := "k8s-1.23"
 					location := key + "/" + id + ".yaml"
 					addon := addons.Add(&channelsapi.AddonSpec{
@@ -887,8 +927,13 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext)
 
 	if b.Cluster.Spec.Networking.Kopeio != nil && !featureflag.UseAddonOperators.Enabled() {
 		key := "networking.kope.io"
+		useBuiltin := !b.hasExternalAddon(key)
 
-		{
+		if !useBuiltin {
+			klog.Infof("Found kopeio-networking-agent in addons; won't use builtin")
+		}
+
+		if useBuiltin {
 			location := key + "/k8s-1.12.yaml"
 			id := "k8s-1.12"
 
@@ -1220,3 +1265,15 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext)
 	}
 	return addons, serviceAccounts, nil
 }
+
+// hasExternalAddon checks if the user has overridden a built-in manifest via additional objects.
+// We identify this by looking for objects with the matching label.
+func (b *BootstrapChannelBuilder) hasExternalAddon(key string) bool {
+	for _, o := range b.ClusterAddons {
+		labels := o.ToUnstructured().GetLabels()
+		if labels[addonmanifests.KopsAddonLabelKey] == key {
+			return true
+		}
+	}
+	return false
+}

upup/pkg/fi/cloudup/bootstrapchannelbuilder/pruning.go

@@ -27,6 +27,7 @@ import (
 
 	channelsapi "k8s.io/kops/channels/pkg/api"
 	"k8s.io/kops/pkg/kubemanifest"
+	"k8s.io/kops/pkg/model/components/addonmanifests"
 )
 
 func (b *BootstrapChannelBuilder) addPruneDirectives(addons *AddonList) error {
@@ -49,8 +50,8 @@ func (b *BootstrapChannelBuilder) addPruneDirectivesForAddon(addon *Addon) error
 
 	// We add these labels to all objects we manage, so we reuse them for pruning.
 	selectorMap := map[string]string{
-		"app.kubernetes.io/managed-by": "kops",
-		"addon.kops.k8s.io/name":       *addon.Spec.Name,
+		"app.kubernetes.io/managed-by":   "kops",
+		addonmanifests.KopsAddonLabelKey: *addon.Spec.Name,
 	}
 	selector, err := labels.ValidatedSelectorFromSet(selectorMap)
 	if err != nil {