kops-controller: create IPAM controller for GCE

We observe the IPv6 CIDRs assigned to nodes, and reflect them into the node.

Co-authored-by: John Gardiner Myers <jgmyers@proofpoint.com>

cmd/kops-controller/controllers/awsipam.go

@@ -146,8 +146,10 @@ func (r *AWSIPAMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 			return ctrl.Result{}, fmt.Errorf("unexpected amount of ipv6 prefixes on interface %q: %v", *eni.NetworkInterfaces[0].NetworkInterfaceId, len(eni.NetworkInterfaces[0].Ipv6Prefixes))
 		}
 
-		patchNodePodCIDRs(r.coreV1Client, ctx, node, *eni.NetworkInterfaces[0].Ipv6Prefixes[0].Ipv6Prefix)
-
+		ipv6Address := aws.StringValue(eni.NetworkInterfaces[0].Ipv6Prefixes[0].Ipv6Prefix)
+		if err := patchNodePodCIDRs(r.coreV1Client, ctx, node, ipv6Address); err != nil {
+			return ctrl.Result{}, err
+		}
 	}
 
 	return ctrl.Result{}, nil

cmd/kops-controller/controllers/gceipam.go

@@ -0,0 +1,149 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/go-logr/logr"
+	"google.golang.org/api/compute/v1"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/klog/v2"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+)
+
+// NewGCEIPAMReconciler is the constructor for a GCEIPAMReconciler
+func NewGCEIPAMReconciler(mgr manager.Manager) (*GCEIPAMReconciler, error) {
+	klog.Info("starting gce ipam controller")
+	r := &GCEIPAMReconciler{
+		client: mgr.GetClient(),
+		log:    ctrl.Log.WithName("controllers").WithName("gce-ipam"),
+	}
+
+	coreClient, err := corev1client.NewForConfig(mgr.GetConfig())
+	if err != nil {
+		return nil, fmt.Errorf("building corev1 client: %w", err)
+	}
+	r.coreV1Client = coreClient
+
+	gceClient, err := compute.NewService(context.Background())
+	if err != nil {
+		return nil, fmt.Errorf("building compute API client: %w", err)
+	}
+	r.gceClient = gceClient
+
+	return r, nil
+}
+
+// GCEIPAMReconciler observes Node objects, assigning their`PodCIDRs` from the instance's `ExternalIpv6`.
+type GCEIPAMReconciler struct {
+	// client is the controller-runtime client
+	client client.Client
+
+	// log is a logr
+	log logr.Logger
+
+	// coreV1Client is a client-go client for patching nodes
+	coreV1Client *corev1client.CoreV1Client
+
+	// gceClient is a client for GCE
+	gceClient *compute.Service
+}
+
+// +kubebuilder:rbac:groups=,resources=nodes,verbs=get;list;watch;patch
+// Reconcile is the main reconciler function that observes node changes.
+func (r *GCEIPAMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	_ = r.log.WithValues("node", req.NamespacedName)
+
+	node := &corev1.Node{}
+	if err := r.client.Get(ctx, req.NamespacedName, node); err != nil {
+		klog.Warningf("unable to fetch node %s: %w", node.Name, err)
+		if apierrors.IsNotFound(err) {
+			// we'll ignore not-found errors, since they can't be fixed by an immediate
+			// requeue (we'll need to wait for a new notification), and we can get them
+			// on deleted requests.
+			return ctrl.Result{}, nil
+		}
+		return ctrl.Result{}, err
+	}
+
+	if len(node.Spec.PodCIDRs) == 0 {
+		// CCM Node Controller has not done its thing yet
+		if node.Spec.ProviderID == "" {
+			klog.Infof("node %q has empty provider ID", node.Name)
+			return ctrl.Result{}, nil
+		}
+
+		// e.g. providerID: gce://example-project-id/us-west2-a/instance-id
+		providerURL, err := url.Parse(node.Spec.ProviderID)
+		if err != nil {
+			return ctrl.Result{}, fmt.Errorf("parsing providerID %q: %w", node.Spec.ProviderID, err)
+		}
+		tokens := strings.Split(strings.Trim(providerURL.Path, "/"), "/")
+		if len(tokens) != 2 {
+			return ctrl.Result{}, fmt.Errorf("unexpected format for providerID %q", node.Spec.ProviderID)
+		}
+		project := providerURL.Host
+		zone := tokens[0]
+		instanceID := tokens[1]
+		if project == "" || zone == "" || instanceID == "" {
+			return ctrl.Result{}, fmt.Errorf("unexpected format for providerID %q", node.Spec.ProviderID)
+		}
+
+		instance, err := r.gceClient.Instances.Get(project, zone, instanceID).Context(ctx).Do()
+		if err != nil {
+			return ctrl.Result{}, fmt.Errorf("getting instance %s/%s/%s: %w", project, zone, instanceID, err)
+		}
+
+		var ipv6Addresses []string
+		for _, nic := range instance.NetworkInterfaces {
+			for _, ipv6AccessConfig := range nic.Ipv6AccessConfigs {
+				if ipv6AccessConfig.ExternalIpv6 != "" {
+					ipv6Address := fmt.Sprintf("%s/%d", ipv6AccessConfig.ExternalIpv6, ipv6AccessConfig.ExternalIpv6PrefixLength)
+					ipv6Addresses = append(ipv6Addresses, ipv6Address)
+				}
+			}
+		}
+
+		if len(ipv6Addresses) == 0 {
+			return ctrl.Result{}, fmt.Errorf("no ipv6 address found on interface %q", instance.NetworkInterfaces[0].Name)
+		}
+		if len(ipv6Addresses) != 1 {
+			return ctrl.Result{}, fmt.Errorf("multiple ipv6 addresses found on interface %q: %v", instance.NetworkInterfaces[0].Name, ipv6Addresses)
+		}
+
+		ipv6Address := ipv6Addresses[0]
+		if err := patchNodePodCIDRs(r.coreV1Client, ctx, node, ipv6Address); err != nil {
+			return ctrl.Result{}, err
+		}
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func (r *GCEIPAMReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.Node{}).
+		Complete(r)
+}

cmd/kops-controller/main.go

@@ -175,19 +175,10 @@ func main() {
 	}
 
 	if opt.EnableCloudIPAM {
-		setupLog.Info("enabling IPAM controller")
-		if opt.Cloud != "aws" {
-			klog.Error("IPAM controller only supported by aws")
-			os.Exit(1)
-		}
-		ipamController, err := controllers.NewAWSIPAMReconciler(mgr)
-		if err != nil {
-			setupLog.Error(err, "unable to create controller", "controller", "IPAMController")
-			os.Exit(1)
-		}
-		if err := ipamController.SetupWithManager(mgr); err != nil {
-			setupLog.Error(err, "unable to create controller", "controller", "IPAMController")
+		if err := setupCloudIPAM(mgr, &opt); err != nil {
+			setupLog.Error(err, "unable to setup cloud IPAM")
 			os.Exit(1)
+
 		}
 	}
 
@@ -324,3 +315,35 @@ func addGossipController(mgr manager.Manager, opt *config.Options) error {
 
 	return nil
 }
+
+// Reconciler is the interface for a standard Reconciler.
+type Reconciler interface {
+	SetupWithManager(mgr manager.Manager) error
+}
+
+func setupCloudIPAM(mgr manager.Manager, opt *config.Options) error {
+	setupLog.Info("enabling IPAM controller")
+	var controller Reconciler
+	switch opt.Cloud {
+	case "aws":
+		ipamController, err := controllers.NewAWSIPAMReconciler(mgr)
+		if err != nil {
+			return fmt.Errorf("creating aws IPAM controller: %w", err)
+		}
+		controller = ipamController
+	case "gce":
+		ipamController, err := controllers.NewGCEIPAMReconciler(mgr)
+		if err != nil {
+			return fmt.Errorf("creating gce IPAM controller: %w", err)
+		}
+		controller = ipamController
+	default:
+		return fmt.Errorf("kOps IPAM controller is not supported on cloud %q", opt.Cloud)
+	}
+
+	if err := controller.SetupWithManager(mgr); err != nil {
+		return fmt.Errorf("registering IPAM controller: %w", err)
+	}
+
+	return nil
+}

pkg/model/components/gcpcloudcontrollermanager.go

@@ -65,13 +65,20 @@ func (b *GCPCloudControllerManagerOptionsBuilder) BuildOptions(options interface
 	}
 
 	if ccmConfig.Controllers == nil {
-		ccmConfig.Controllers = []string{
-			"*",
+		var changes []string
 
 		// Don't run gkenetworkparamset controller, looks for some CRDs (GKENetworkParamSet and Network) which are only installed on GKE
 		// However, the version we're current running doesn't support this controller anyway, so we need to introduce this later,
 		// possibly based on the image version.
-			// "-gkenetworkparams",
+		// changes = append(ccmConfig.Controllers, "-gkenetworkparams")
+
+		// Turn off some controllers if kops-controller is running them
+		if clusterSpec.IsKopsControllerIPAM() {
+			changes = append(ccmConfig.Controllers, "-nodeipam", "-route")
+		}
+
+		if len(changes) != 0 {
+			ccmConfig.Controllers = append([]string{"*"}, changes...)
 		}
 	}