Merge pull request #15591 from justinsb/gce_ipam_1

kops-controller: create IPAM controller for GCE
This commit is contained in:
Kubernetes Prow Robot 2023-07-05 10:58:57 -07:00 committed by GitHub
commit ac1c1bd2ec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 210 additions and 56 deletions

View File

@ -146,8 +146,10 @@ func (r *AWSIPAMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
return ctrl.Result{}, fmt.Errorf("unexpected amount of ipv6 prefixes on interface %q: %v", *eni.NetworkInterfaces[0].NetworkInterfaceId, len(eni.NetworkInterfaces[0].Ipv6Prefixes))
}
patchNodePodCIDRs(r.coreV1Client, ctx, node, *eni.NetworkInterfaces[0].Ipv6Prefixes[0].Ipv6Prefix)
ipv6Address := aws.StringValue(eni.NetworkInterfaces[0].Ipv6Prefixes[0].Ipv6Prefix)
if err := patchNodePodCIDRs(r.coreV1Client, ctx, node, ipv6Address); err != nil {
return ctrl.Result{}, err
}
}
return ctrl.Result{}, nil

View File

@ -0,0 +1,149 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controllers
import (
"context"
"fmt"
"net/url"
"strings"
"github.com/go-logr/logr"
"google.golang.org/api/compute/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/klog/v2"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/manager"
)
// NewGCEIPAMReconciler is the constructor for a GCEIPAMReconciler
func NewGCEIPAMReconciler(mgr manager.Manager) (*GCEIPAMReconciler, error) {
klog.Info("starting gce ipam controller")
r := &GCEIPAMReconciler{
client: mgr.GetClient(),
log: ctrl.Log.WithName("controllers").WithName("gce-ipam"),
}
coreClient, err := corev1client.NewForConfig(mgr.GetConfig())
if err != nil {
return nil, fmt.Errorf("building corev1 client: %w", err)
}
r.coreV1Client = coreClient
gceClient, err := compute.NewService(context.Background())
if err != nil {
return nil, fmt.Errorf("building compute API client: %w", err)
}
r.gceClient = gceClient
return r, nil
}
// GCEIPAMReconciler observes Node objects, assigning their`PodCIDRs` from the instance's `ExternalIpv6`.
type GCEIPAMReconciler struct {
// client is the controller-runtime client
client client.Client
// log is a logr
log logr.Logger
// coreV1Client is a client-go client for patching nodes
coreV1Client *corev1client.CoreV1Client
// gceClient is a client for GCE
gceClient *compute.Service
}
// +kubebuilder:rbac:groups=,resources=nodes,verbs=get;list;watch;patch
// Reconcile is the main reconciler function that observes node changes.
func (r *GCEIPAMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
_ = r.log.WithValues("node", req.NamespacedName)
node := &corev1.Node{}
if err := r.client.Get(ctx, req.NamespacedName, node); err != nil {
klog.Warningf("unable to fetch node %s: %w", node.Name, err)
if apierrors.IsNotFound(err) {
// we'll ignore not-found errors, since they can't be fixed by an immediate
// requeue (we'll need to wait for a new notification), and we can get them
// on deleted requests.
return ctrl.Result{}, nil
}
return ctrl.Result{}, err
}
if len(node.Spec.PodCIDRs) == 0 {
// CCM Node Controller has not done its thing yet
if node.Spec.ProviderID == "" {
klog.Infof("node %q has empty provider ID", node.Name)
return ctrl.Result{}, nil
}
// e.g. providerID: gce://example-project-id/us-west2-a/instance-id
providerURL, err := url.Parse(node.Spec.ProviderID)
if err != nil {
return ctrl.Result{}, fmt.Errorf("parsing providerID %q: %w", node.Spec.ProviderID, err)
}
tokens := strings.Split(strings.Trim(providerURL.Path, "/"), "/")
if len(tokens) != 2 {
return ctrl.Result{}, fmt.Errorf("unexpected format for providerID %q", node.Spec.ProviderID)
}
project := providerURL.Host
zone := tokens[0]
instanceID := tokens[1]
if project == "" || zone == "" || instanceID == "" {
return ctrl.Result{}, fmt.Errorf("unexpected format for providerID %q", node.Spec.ProviderID)
}
instance, err := r.gceClient.Instances.Get(project, zone, instanceID).Context(ctx).Do()
if err != nil {
return ctrl.Result{}, fmt.Errorf("getting instance %s/%s/%s: %w", project, zone, instanceID, err)
}
var ipv6Addresses []string
for _, nic := range instance.NetworkInterfaces {
for _, ipv6AccessConfig := range nic.Ipv6AccessConfigs {
if ipv6AccessConfig.ExternalIpv6 != "" {
ipv6Address := fmt.Sprintf("%s/%d", ipv6AccessConfig.ExternalIpv6, ipv6AccessConfig.ExternalIpv6PrefixLength)
ipv6Addresses = append(ipv6Addresses, ipv6Address)
}
}
}
if len(ipv6Addresses) == 0 {
return ctrl.Result{}, fmt.Errorf("no ipv6 address found on interface %q", instance.NetworkInterfaces[0].Name)
}
if len(ipv6Addresses) != 1 {
return ctrl.Result{}, fmt.Errorf("multiple ipv6 addresses found on interface %q: %v", instance.NetworkInterfaces[0].Name, ipv6Addresses)
}
ipv6Address := ipv6Addresses[0]
if err := patchNodePodCIDRs(r.coreV1Client, ctx, node, ipv6Address); err != nil {
return ctrl.Result{}, err
}
}
return ctrl.Result{}, nil
}
func (r *GCEIPAMReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&corev1.Node{}).
Complete(r)
}

View File

@ -175,19 +175,10 @@ func main() {
}
if opt.EnableCloudIPAM {
setupLog.Info("enabling IPAM controller")
if opt.Cloud != "aws" {
klog.Error("IPAM controller only supported by aws")
os.Exit(1)
}
ipamController, err := controllers.NewAWSIPAMReconciler(mgr)
if err != nil {
setupLog.Error(err, "unable to create controller", "controller", "IPAMController")
os.Exit(1)
}
if err := ipamController.SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "IPAMController")
if err := setupCloudIPAM(mgr, &opt); err != nil {
setupLog.Error(err, "unable to setup cloud IPAM")
os.Exit(1)
}
}
@ -324,3 +315,35 @@ func addGossipController(mgr manager.Manager, opt *config.Options) error {
return nil
}
// Reconciler is the interface for a standard Reconciler.
type Reconciler interface {
SetupWithManager(mgr manager.Manager) error
}
func setupCloudIPAM(mgr manager.Manager, opt *config.Options) error {
setupLog.Info("enabling IPAM controller")
var controller Reconciler
switch opt.Cloud {
case "aws":
ipamController, err := controllers.NewAWSIPAMReconciler(mgr)
if err != nil {
return fmt.Errorf("creating aws IPAM controller: %w", err)
}
controller = ipamController
case "gce":
ipamController, err := controllers.NewGCEIPAMReconciler(mgr)
if err != nil {
return fmt.Errorf("creating gce IPAM controller: %w", err)
}
controller = ipamController
default:
return fmt.Errorf("kOps IPAM controller is not supported on cloud %q", opt.Cloud)
}
if err := controller.SetupWithManager(mgr); err != nil {
return fmt.Errorf("registering IPAM controller: %w", err)
}
return nil
}

View File

@ -65,13 +65,20 @@ func (b *GCPCloudControllerManagerOptionsBuilder) BuildOptions(options interface
}
if ccmConfig.Controllers == nil {
ccmConfig.Controllers = []string{
"*",
var changes []string
// Don't run gkenetworkparamset controller, looks for some CRDs (GKENetworkParamSet and Network) which are only installed on GKE
// However, the version we're current running doesn't support this controller anyway, so we need to introduce this later,
// possibly based on the image version.
// "-gkenetworkparams",
// Don't run gkenetworkparamset controller, looks for some CRDs (GKENetworkParamSet and Network) which are only installed on GKE
// However, the version we're current running doesn't support this controller anyway, so we need to introduce this later,
// possibly based on the image version.
// changes = append(ccmConfig.Controllers, "-gkenetworkparams")
// Turn off some controllers if kops-controller is running them
if clusterSpec.IsKopsControllerIPAM() {
changes = append(ccmConfig.Controllers, "-nodeipam", "-route")
}
if len(changes) != 0 {
ccmConfig.Controllers = append([]string{"*"}, changes...)
}
}

View File

@ -20,8 +20,6 @@ spec:
cidrAllocatorType: CloudAllocator
clusterCIDR: 100.96.0.0/11
clusterName: ha-gce-example-com
controllers:
- '*'
image: k8scloudprovidergcp/cloud-controller-manager:latest
leaderElection:
leaderElect: true

View File

@ -62,7 +62,7 @@ spec:
version: 9.99.0
- id: k8s-1.23
manifest: gcp-cloud-controller.addons.k8s.io/k8s-1.23.yaml
manifestHash: 865e79337a96353842b6f5c359701cdfe3d1da805d235ea49c91d4ab79a35050
manifestHash: 39ecc974e248a7e0f0c6c0813e8e27be9713af775939bd09c83dfc2c1946bf2f
name: gcp-cloud-controller.addons.k8s.io
prune:
kinds:

View File

@ -37,7 +37,6 @@ spec:
- --cidr-allocator-type=CloudAllocator
- --cluster-cidr=100.96.0.0/11
- --cluster-name=ha-gce-example-com
- --controllers=*
- --leader-elect=true
- --v=2
- --cloud-provider=gce

View File

@ -22,8 +22,6 @@ spec:
cidrAllocatorType: CloudAllocator
clusterCIDR: 100.96.0.0/11
clusterName: minimal-example-com
controllers:
- '*'
image: k8scloudprovidergcp/cloud-controller-manager:latest
leaderElection:
leaderElect: true

View File

@ -133,7 +133,7 @@ spec:
version: 9.99.0
- id: k8s-1.23
manifest: gcp-cloud-controller.addons.k8s.io/k8s-1.23.yaml
manifestHash: 5b2863c906b3bd49e625b3ef83f27f3bd5840c52118ed57b38bd20e12330d37f
manifestHash: 49730987c3f5dba8914954740cc96dbb08c090c0ac9c943a160a32cb94b53764
name: gcp-cloud-controller.addons.k8s.io
prune:
kinds:

View File

@ -37,7 +37,6 @@ spec:
- --cidr-allocator-type=CloudAllocator
- --cluster-cidr=100.96.0.0/11
- --cluster-name=minimal-example-com
- --controllers=*
- --leader-elect=true
- --v=2
- --cloud-provider=gce

View File

@ -20,8 +20,6 @@ spec:
cidrAllocatorType: CloudAllocator
clusterCIDR: 100.96.0.0/11
clusterName: minimal-gce-example-com
controllers:
- '*'
image: k8scloudprovidergcp/cloud-controller-manager:latest
leaderElection:
leaderElect: true

View File

@ -62,7 +62,7 @@ spec:
version: 9.99.0
- id: k8s-1.23
manifest: gcp-cloud-controller.addons.k8s.io/k8s-1.23.yaml
manifestHash: 79edeb95bce8a2771bd84b7ecd55639b2fc895e3cdd533015727368996cab6ce
manifestHash: 1e7b574d8a629441fef3f17dec13a0c73ab48b2c40160c30a8c78a99652db702
name: gcp-cloud-controller.addons.k8s.io
prune:
kinds:

View File

@ -37,7 +37,6 @@ spec:
- --cidr-allocator-type=CloudAllocator
- --cluster-cidr=100.96.0.0/11
- --cluster-name=minimal-gce-example-com
- --controllers=*
- --leader-elect=true
- --v=2
- --cloud-provider=gce

View File

@ -24,8 +24,6 @@ spec:
cidrAllocatorType: CloudAllocator
clusterCIDR: 100.96.0.0/11
clusterName: minimal-gce-example-com
controllers:
- '*'
image: k8scloudprovidergcp/cloud-controller-manager:latest
leaderElection:
leaderElect: true

View File

@ -55,7 +55,7 @@ spec:
version: 9.99.0
- id: k8s-1.23
manifest: gcp-cloud-controller.addons.k8s.io/k8s-1.23.yaml
manifestHash: 79edeb95bce8a2771bd84b7ecd55639b2fc895e3cdd533015727368996cab6ce
manifestHash: 1e7b574d8a629441fef3f17dec13a0c73ab48b2c40160c30a8c78a99652db702
name: gcp-cloud-controller.addons.k8s.io
prune:
kinds:

View File

@ -37,7 +37,6 @@ spec:
- --cidr-allocator-type=CloudAllocator
- --cluster-cidr=100.96.0.0/11
- --cluster-name=minimal-gce-example-com
- --controllers=*
- --leader-elect=true
- --v=2
- --cloud-provider=gce

View File

@ -24,8 +24,6 @@ spec:
cidrAllocatorType: CloudAllocator
clusterCIDR: 100.96.0.0/11
clusterName: minimal-gce-ilb-example-com
controllers:
- '*'
image: k8scloudprovidergcp/cloud-controller-manager:latest
leaderElection:
leaderElect: true

View File

@ -62,7 +62,7 @@ spec:
version: 9.99.0
- id: k8s-1.23
manifest: gcp-cloud-controller.addons.k8s.io/k8s-1.23.yaml
manifestHash: 9311c506f8a1674f99ad4959e584fe982a33a2a36692477ca5d9d8e5318fb135
manifestHash: 8e9e414b952c3c6298c27cdc50866ab48e18ac2689c70ce926a8639fc54ff786
name: gcp-cloud-controller.addons.k8s.io
prune:
kinds:

View File

@ -37,7 +37,6 @@ spec:
- --cidr-allocator-type=CloudAllocator
- --cluster-cidr=100.96.0.0/11
- --cluster-name=minimal-gce-ilb-example-com
- --controllers=*
- --leader-elect=true
- --v=2
- --cloud-provider=gce

View File

@ -24,8 +24,6 @@ spec:
cidrAllocatorType: CloudAllocator
clusterCIDR: 100.96.0.0/11
clusterName: minimal-gce-with-a-very-very-very-very-very-long-name-example-com
controllers:
- '*'
image: k8scloudprovidergcp/cloud-controller-manager:latest
leaderElection:
leaderElect: true

View File

@ -62,7 +62,7 @@ spec:
version: 9.99.0
- id: k8s-1.23
manifest: gcp-cloud-controller.addons.k8s.io/k8s-1.23.yaml
manifestHash: 2b00060fec2abeb81de79e436f13dfe1f9a88c5f3b1e449043a24d85de3016a1
manifestHash: 24c22638647064142f86c461088862d4de22ca298d5f018bf2801c20f5df21d4
name: gcp-cloud-controller.addons.k8s.io
prune:
kinds:

View File

@ -37,7 +37,6 @@ spec:
- --cidr-allocator-type=CloudAllocator
- --cluster-cidr=100.96.0.0/11
- --cluster-name=minimal-gce-with-a-very-very-very-very-very-long-name-example-com
- --controllers=*
- --leader-elect=true
- --v=2
- --cloud-provider=gce

View File

@ -20,8 +20,6 @@ spec:
cidrAllocatorType: CloudAllocator
clusterCIDR: 100.96.0.0/11
clusterName: minimal-gce-with-a-very-very-very-very-very-long-name-example-com
controllers:
- '*'
image: k8scloudprovidergcp/cloud-controller-manager:latest
leaderElection:
leaderElect: true

View File

@ -62,7 +62,7 @@ spec:
version: 9.99.0
- id: k8s-1.23
manifest: gcp-cloud-controller.addons.k8s.io/k8s-1.23.yaml
manifestHash: 2b00060fec2abeb81de79e436f13dfe1f9a88c5f3b1e449043a24d85de3016a1
manifestHash: 24c22638647064142f86c461088862d4de22ca298d5f018bf2801c20f5df21d4
name: gcp-cloud-controller.addons.k8s.io
prune:
kinds:

View File

@ -37,7 +37,6 @@ spec:
- --cidr-allocator-type=CloudAllocator
- --cluster-cidr=100.96.0.0/11
- --cluster-name=minimal-gce-with-a-very-very-very-very-very-long-name-example-com
- --controllers=*
- --leader-elect=true
- --v=2
- --cloud-provider=gce

View File

@ -24,8 +24,6 @@ spec:
cidrAllocatorType: CloudAllocator
clusterCIDR: 100.96.0.0/11
clusterName: minimal-gce-plb-example-com
controllers:
- '*'
image: k8scloudprovidergcp/cloud-controller-manager:latest
leaderElection:
leaderElect: true

View File

@ -62,7 +62,7 @@ spec:
version: 9.99.0
- id: k8s-1.23
manifest: gcp-cloud-controller.addons.k8s.io/k8s-1.23.yaml
manifestHash: ab96089a9249097410dfe8884fc1ef7ea2808a7c267dd6f224e156550d4666ae
manifestHash: 2132eed9950043d4883ddc77f664aedaa403207b2f8efdeba0b3481afc397f5a
name: gcp-cloud-controller.addons.k8s.io
prune:
kinds:

View File

@ -37,7 +37,6 @@ spec:
- --cidr-allocator-type=CloudAllocator
- --cluster-cidr=100.96.0.0/11
- --cluster-name=minimal-gce-plb-example-com
- --controllers=*
- --leader-elect=true
- --v=2
- --cloud-provider=gce

View File

@ -20,8 +20,6 @@ spec:
cidrAllocatorType: CloudAllocator
clusterCIDR: 100.96.0.0/11
clusterName: minimal-gce-private-example-com
controllers:
- '*'
image: k8scloudprovidergcp/cloud-controller-manager:latest
leaderElection:
leaderElect: true

View File

@ -62,7 +62,7 @@ spec:
version: 9.99.0
- id: k8s-1.23
manifest: gcp-cloud-controller.addons.k8s.io/k8s-1.23.yaml
manifestHash: 8f75ce02c86162faf35e02cb10835ae33b86abd8bdb72a8e4b489a9bd559fdbc
manifestHash: bf9a99e3927ad9ce2106ef090effc743f380dbde6437e58caebf08223d70f4b4
name: gcp-cloud-controller.addons.k8s.io
prune:
kinds:

View File

@ -37,7 +37,6 @@ spec:
- --cidr-allocator-type=CloudAllocator
- --cluster-cidr=100.96.0.0/11
- --cluster-name=minimal-gce-private-example-com
- --controllers=*
- --leader-elect=true
- --v=2
- --cloud-provider=gce