diff --git a/cmd/kops-controller/controllers/awsipam.go b/cmd/kops-controller/controllers/awsipam.go index 2f364933c4..f9b93377a7 100644 --- a/cmd/kops-controller/controllers/awsipam.go +++ b/cmd/kops-controller/controllers/awsipam.go @@ -146,8 +146,10 @@ func (r *AWSIPAMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, fmt.Errorf("unexpected amount of ipv6 prefixes on interface %q: %v", *eni.NetworkInterfaces[0].NetworkInterfaceId, len(eni.NetworkInterfaces[0].Ipv6Prefixes)) } - patchNodePodCIDRs(r.coreV1Client, ctx, node, *eni.NetworkInterfaces[0].Ipv6Prefixes[0].Ipv6Prefix) - + ipv6Address := aws.StringValue(eni.NetworkInterfaces[0].Ipv6Prefixes[0].Ipv6Prefix) + if err := patchNodePodCIDRs(r.coreV1Client, ctx, node, ipv6Address); err != nil { + return ctrl.Result{}, err + } } return ctrl.Result{}, nil diff --git a/cmd/kops-controller/controllers/gceipam.go b/cmd/kops-controller/controllers/gceipam.go new file mode 100644 index 0000000000..04b18c9ed8 --- /dev/null +++ b/cmd/kops-controller/controllers/gceipam.go @@ -0,0 +1,149 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "net/url" + "strings" + + "github.com/go-logr/logr" + "google.golang.org/api/compute/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +// NewGCEIPAMReconciler is the constructor for a GCEIPAMReconciler +func NewGCEIPAMReconciler(mgr manager.Manager) (*GCEIPAMReconciler, error) { + klog.Info("starting gce ipam controller") + r := &GCEIPAMReconciler{ + client: mgr.GetClient(), + log: ctrl.Log.WithName("controllers").WithName("gce-ipam"), + } + + coreClient, err := corev1client.NewForConfig(mgr.GetConfig()) + if err != nil { + return nil, fmt.Errorf("building corev1 client: %w", err) + } + r.coreV1Client = coreClient + + gceClient, err := compute.NewService(context.Background()) + if err != nil { + return nil, fmt.Errorf("building compute API client: %w", err) + } + r.gceClient = gceClient + + return r, nil +} + +// GCEIPAMReconciler observes Node objects, assigning their`PodCIDRs` from the instance's `ExternalIpv6`. +type GCEIPAMReconciler struct { + // client is the controller-runtime client + client client.Client + + // log is a logr + log logr.Logger + + // coreV1Client is a client-go client for patching nodes + coreV1Client *corev1client.CoreV1Client + + // gceClient is a client for GCE + gceClient *compute.Service +} + +// +kubebuilder:rbac:groups=,resources=nodes,verbs=get;list;watch;patch +// Reconcile is the main reconciler function that observes node changes. +func (r *GCEIPAMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + _ = r.log.WithValues("node", req.NamespacedName) + + node := &corev1.Node{} + if err := r.client.Get(ctx, req.NamespacedName, node); err != nil { + klog.Warningf("unable to fetch node %s: %w", node.Name, err) + if apierrors.IsNotFound(err) { + // we'll ignore not-found errors, since they can't be fixed by an immediate + // requeue (we'll need to wait for a new notification), and we can get them + // on deleted requests. + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + if len(node.Spec.PodCIDRs) == 0 { + // CCM Node Controller has not done its thing yet + if node.Spec.ProviderID == "" { + klog.Infof("node %q has empty provider ID", node.Name) + return ctrl.Result{}, nil + } + + // e.g. providerID: gce://example-project-id/us-west2-a/instance-id + providerURL, err := url.Parse(node.Spec.ProviderID) + if err != nil { + return ctrl.Result{}, fmt.Errorf("parsing providerID %q: %w", node.Spec.ProviderID, err) + } + tokens := strings.Split(strings.Trim(providerURL.Path, "/"), "/") + if len(tokens) != 2 { + return ctrl.Result{}, fmt.Errorf("unexpected format for providerID %q", node.Spec.ProviderID) + } + project := providerURL.Host + zone := tokens[0] + instanceID := tokens[1] + if project == "" || zone == "" || instanceID == "" { + return ctrl.Result{}, fmt.Errorf("unexpected format for providerID %q", node.Spec.ProviderID) + } + + instance, err := r.gceClient.Instances.Get(project, zone, instanceID).Context(ctx).Do() + if err != nil { + return ctrl.Result{}, fmt.Errorf("getting instance %s/%s/%s: %w", project, zone, instanceID, err) + } + + var ipv6Addresses []string + for _, nic := range instance.NetworkInterfaces { + for _, ipv6AccessConfig := range nic.Ipv6AccessConfigs { + if ipv6AccessConfig.ExternalIpv6 != "" { + ipv6Address := fmt.Sprintf("%s/%d", ipv6AccessConfig.ExternalIpv6, ipv6AccessConfig.ExternalIpv6PrefixLength) + ipv6Addresses = append(ipv6Addresses, ipv6Address) + } + } + } + + if len(ipv6Addresses) == 0 { + return ctrl.Result{}, fmt.Errorf("no ipv6 address found on interface %q", instance.NetworkInterfaces[0].Name) + } + if len(ipv6Addresses) != 1 { + return ctrl.Result{}, fmt.Errorf("multiple ipv6 addresses found on interface %q: %v", instance.NetworkInterfaces[0].Name, ipv6Addresses) + } + + ipv6Address := ipv6Addresses[0] + if err := patchNodePodCIDRs(r.coreV1Client, ctx, node, ipv6Address); err != nil { + return ctrl.Result{}, err + } + } + + return ctrl.Result{}, nil +} + +func (r *GCEIPAMReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.Node{}). + Complete(r) +} diff --git a/cmd/kops-controller/main.go b/cmd/kops-controller/main.go index 278676639b..94b00e3342 100644 --- a/cmd/kops-controller/main.go +++ b/cmd/kops-controller/main.go @@ -175,19 +175,10 @@ func main() { } if opt.EnableCloudIPAM { - setupLog.Info("enabling IPAM controller") - if opt.Cloud != "aws" { - klog.Error("IPAM controller only supported by aws") - os.Exit(1) - } - ipamController, err := controllers.NewAWSIPAMReconciler(mgr) - if err != nil { - setupLog.Error(err, "unable to create controller", "controller", "IPAMController") - os.Exit(1) - } - if err := ipamController.SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "IPAMController") + if err := setupCloudIPAM(mgr, &opt); err != nil { + setupLog.Error(err, "unable to setup cloud IPAM") os.Exit(1) + } } @@ -324,3 +315,35 @@ func addGossipController(mgr manager.Manager, opt *config.Options) error { return nil } + +// Reconciler is the interface for a standard Reconciler. +type Reconciler interface { + SetupWithManager(mgr manager.Manager) error +} + +func setupCloudIPAM(mgr manager.Manager, opt *config.Options) error { + setupLog.Info("enabling IPAM controller") + var controller Reconciler + switch opt.Cloud { + case "aws": + ipamController, err := controllers.NewAWSIPAMReconciler(mgr) + if err != nil { + return fmt.Errorf("creating aws IPAM controller: %w", err) + } + controller = ipamController + case "gce": + ipamController, err := controllers.NewGCEIPAMReconciler(mgr) + if err != nil { + return fmt.Errorf("creating gce IPAM controller: %w", err) + } + controller = ipamController + default: + return fmt.Errorf("kOps IPAM controller is not supported on cloud %q", opt.Cloud) + } + + if err := controller.SetupWithManager(mgr); err != nil { + return fmt.Errorf("registering IPAM controller: %w", err) + } + + return nil +} diff --git a/pkg/model/components/gcpcloudcontrollermanager.go b/pkg/model/components/gcpcloudcontrollermanager.go index c934eb35b9..2ca83d9aa0 100644 --- a/pkg/model/components/gcpcloudcontrollermanager.go +++ b/pkg/model/components/gcpcloudcontrollermanager.go @@ -65,13 +65,20 @@ func (b *GCPCloudControllerManagerOptionsBuilder) BuildOptions(options interface } if ccmConfig.Controllers == nil { - ccmConfig.Controllers = []string{ - "*", + var changes []string - // Don't run gkenetworkparamset controller, looks for some CRDs (GKENetworkParamSet and Network) which are only installed on GKE - // However, the version we're current running doesn't support this controller anyway, so we need to introduce this later, - // possibly based on the image version. - // "-gkenetworkparams", + // Don't run gkenetworkparamset controller, looks for some CRDs (GKENetworkParamSet and Network) which are only installed on GKE + // However, the version we're current running doesn't support this controller anyway, so we need to introduce this later, + // possibly based on the image version. + // changes = append(ccmConfig.Controllers, "-gkenetworkparams") + + // Turn off some controllers if kops-controller is running them + if clusterSpec.IsKopsControllerIPAM() { + changes = append(ccmConfig.Controllers, "-nodeipam", "-route") + } + + if len(changes) != 0 { + ccmConfig.Controllers = append([]string{"*"}, changes...) } }