Merge pull request #16944 from justinsb/bare-metal-ipv6

(Experimental) bare-metal with IPv6

.github/workflows/e2e.yml

@@ -46,3 +46,29 @@ jobs:
         with:
           name: tests-e2e-scenarios-bare-metal
           path: /tmp/artifacts/
+
+  tests-e2e-scenarios-bare-metal-ipv6:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 70
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          path: ${{ env.GOPATH }}/src/k8s.io/kops
+
+      - name: Set up go
+        uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed
+        with:
+          go-version-file: '${{ env.GOPATH }}/src/k8s.io/kops/go.mod'
+
+      - name: tests/e2e/scenarios/bare-metal/run-test
+        working-directory: ${{ env.GOPATH }}/src/k8s.io/kops
+        run: |
+          timeout 60m tests/e2e/scenarios/bare-metal/scenario-ipv6
+        env:
+          ARTIFACTS: /tmp/artifacts
+      - name: Archive production artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: tests-e2e-scenarios-bare-metal-ipv6
+          path: /tmp/artifacts/

cmd/kops-controller/controllers/gceipam.go

@@ -56,7 +56,7 @@ func NewGCEIPAMReconciler(mgr manager.Manager) (*GCEIPAMReconciler, error) {
 	return r, nil
 }
 
-// GCEIPAMReconciler observes Node objects, assigning their`PodCIDRs` from the instance's `ExternalIpv6`.
+// GCEIPAMReconciler observes Node objects, assigning their `PodCIDRs` from the instance's `ExternalIpv6`.
 type GCEIPAMReconciler struct {
 	// client is the controller-runtime client
 	client client.Client

cmd/kops-controller/controllers/metalipam.go

@@ -0,0 +1,115 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/klog/v2"
+	kopsapi "k8s.io/kops/pkg/apis/kops/v1alpha2"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+)
+
+// NewMetalIPAMReconciler is the constructor for a MetalIPAMReconciler
+func NewMetalIPAMReconciler(ctx context.Context, mgr manager.Manager) (*MetalIPAMReconciler, error) {
+	klog.Info("starting metal ipam controller")
+	r := &MetalIPAMReconciler{
+		client: mgr.GetClient(),
+		log:    ctrl.Log.WithName("controllers").WithName("metal_ipam"),
+	}
+
+	coreClient, err := corev1client.NewForConfig(mgr.GetConfig())
+	if err != nil {
+		return nil, fmt.Errorf("building corev1 client: %w", err)
+	}
+	r.coreV1Client = coreClient
+
+	return r, nil
+}
+
+// MetalIPAMReconciler observes Node objects, assigning their `PodCIDRs` from the instance's `ExternalIpv6`.
+type MetalIPAMReconciler struct {
+	// client is the controller-runtime client
+	client client.Client
+
+	// log is a logr
+	log logr.Logger
+
+	// coreV1Client is a client-go client for patching nodes
+	coreV1Client *corev1client.CoreV1Client
+}
+
+// +kubebuilder:rbac:groups=,resources=nodes,verbs=get;list;watch;patch
+// Reconcile is the main reconciler function that observes node changes.
+func (r *MetalIPAMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	log := klog.FromContext(ctx)
+
+	node := &corev1.Node{}
+	if err := r.client.Get(ctx, req.NamespacedName, node); err != nil {
+		if apierrors.IsNotFound(err) {
+			// we'll ignore not-found errors, since they can't be fixed by an immediate
+			// requeue (we'll need to wait for a new notification), and we can get them
+			// on deleted requests.
+			return ctrl.Result{}, nil
+		}
+		log.Error(err, "unable to fetch node", "node.name", node.Name)
+		return ctrl.Result{}, err
+	}
+
+	host := &kopsapi.Host{}
+	id := types.NamespacedName{
+		Namespace: "kops-system",
+		Name:      node.Name,
+	}
+	if err := r.client.Get(ctx, id, host); err != nil {
+		log.Error(err, "unable to fetch host", "id", id)
+		return ctrl.Result{}, err
+	}
+
+	if len(node.Spec.PodCIDRs) != 0 && len(host.Spec.PodCIDRs) > 0 {
+		log.V(2).Info("node has pod cidrs, skipping", "node.name", node.Name)
+		return ctrl.Result{}, nil
+	}
+
+	if len(host.Spec.PodCIDRs) == 0 {
+		log.Info("host record has no pod cidrs, cannot assign pod cidrs to node", "node.name", node.Name)
+		return ctrl.Result{}, nil
+	}
+
+	log.Info("assigning pod cidrs to node", "node.name", node.Name, "pod.cidrs", host.Spec.PodCIDRs)
+	if err := patchNodePodCIDRs(r.coreV1Client, ctx, node, host.Spec.PodCIDRs); err != nil {
+		log.Error(err, "unable to patch node", "node.name", node.Name)
+		return ctrl.Result{}, err
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func (r *MetalIPAMReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		Named("metal_ipam").
+		For(&corev1.Node{}).
+		Complete(r)
+}

cmd/kops-controller/main.go

@@ -392,6 +392,12 @@ func setupCloudIPAM(ctx context.Context, mgr manager.Manager, opt *config.Option
 			return fmt.Errorf("creating gce IPAM controller: %w", err)
 		}
 		controller = ipamController
+	case "metal":
+		ipamController, err := controllers.NewMetalIPAMReconciler(ctx, mgr)
+		if err != nil {
+			return fmt.Errorf("creating metal IPAM controller: %w", err)
+		}
+		controller = ipamController
 	default:
 		return fmt.Errorf("kOps IPAM controller is not supported on cloud %q", opt.Cloud)
 	}

cmd/kops/toolbox_enroll.go

@@ -45,11 +45,14 @@ func NewCmdToolboxEnroll(f commandutils.Factory, out io.Writer) *cobra.Command {
 
 	cmd.Flags().StringVar(&options.ClusterName, "cluster", options.ClusterName, "Name of cluster to join")
 	cmd.Flags().StringVar(&options.InstanceGroup, "instance-group", options.InstanceGroup, "Name of instance-group to join")
+	cmd.Flags().StringSliceVar(&options.PodCIDRs, "pod-cidr", options.PodCIDRs, "IP Address range to use for pods that run on this node")
 
 	cmd.Flags().StringVar(&options.Host, "host", options.Host, "IP/hostname for machine to add")
 	cmd.Flags().StringVar(&options.SSHUser, "ssh-user", options.SSHUser, "user for ssh")
 	cmd.Flags().IntVar(&options.SSHPort, "ssh-port", options.SSHPort, "port for ssh")
 
+	cmd.Flags().BoolVar(&options.BuildHost, "build-host", options.BuildHost, "only build the host resource, don't apply it or enroll the node")
+
 	options.CreateKubecfgOptions.AddCommonFlags(cmd.Flags())
 
 	return cmd

docs/cli/kops_toolbox_enroll.md

@@ -23,10 +23,12 @@ kops toolbox enroll [CLUSTER] [flags]
 
 ```
       --api-server string       Override the API server used when communicating with the cluster kube-apiserver
+      --build-host              only build the host resource, don't apply it or enroll the node
       --cluster string          Name of cluster to join
   -h, --help                    help for enroll
       --host string             IP/hostname for machine to add
       --instance-group string   Name of instance-group to join
+      --pod-cidr strings        IP Address range to use for pods that run on this node
       --ssh-port int            port for ssh (default 22)
       --ssh-user string         user for ssh (default "root")
       --use-kubeconfig          Use the server endpoint from the local kubeconfig instead of inferring from cluster name

k8s/crds/kops.k8s.io_hosts.yaml

@@ -42,6 +42,12 @@ spec:
             properties:
               instanceGroup:
                 type: string
+              podCIDRs:
+                description: PodCIDRs configures the IP ranges to be used for pods
+                  on this node/host.
+                items:
+                  type: string
+                type: array
               publicKey:
                 type: string
             type: object

nodeup/pkg/model/kube_apiserver.go

@@ -19,10 +19,12 @@ package model
 import (
 	"context"
 	"fmt"
+	"net"
 	"path/filepath"
 	"sort"
 	"strings"
 
+	"k8s.io/klog/v2"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/flagbuilder"
 	"k8s.io/kops/pkg/k8scodecs"
@@ -77,6 +79,55 @@ func (b *KubeAPIServerBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 		}
 	}
 
+	if b.CloudProvider() == kops.CloudProviderMetal {
+		// Workaround for https://github.com/kubernetes/kubernetes/issues/111671
+		if b.IsIPv6Only() {
+			interfaces, err := net.Interfaces()
+			if err != nil {
+				return fmt.Errorf("getting local network interfaces: %w", err)
+			}
+			var ipv6s []net.IP
+			for _, intf := range interfaces {
+				addresses, err := intf.Addrs()
+				if err != nil {
+					return fmt.Errorf("getting addresses for network interface %q: %w", intf.Name, err)
+				}
+				for _, addr := range addresses {
+					ip, _, err := net.ParseCIDR(addr.String())
+					if ip == nil {
+						return fmt.Errorf("parsing ip address %q (bound to network %q): %w", addr.String(), intf.Name, err)
+					}
+					if ip.To4() != nil {
+						// We're only looking for ipv6
+						continue
+					}
+					if ip.IsLinkLocalUnicast() {
+						klog.V(4).Infof("ignoring link-local unicast addr %v", addr)
+						continue
+					}
+					if ip.IsLinkLocalMulticast() {
+						klog.V(4).Infof("ignoring link-local multicast addr %v", addr)
+						continue
+					}
+					if ip.IsLoopback() {
+						klog.V(4).Infof("ignoring loopback addr %v", addr)
+						continue
+					}
+					ipv6s = append(ipv6s, ip)
+				}
+			}
+			if len(ipv6s) > 1 {
+				klog.Warningf("found multiple ipv6s, choosing first: %v", ipv6s)
+			}
+			if len(ipv6s) == 0 {
+				klog.Warningf("did not find ipv6 address for kube-apiserver --advertise-address")
+			}
+			if len(ipv6s) > 0 {
+				kubeAPIServer.AdvertiseAddress = ipv6s[0].String()
+			}
+		}
+	}
+
 	b.configureOIDC(&kubeAPIServer)
 	if err := b.writeAuthenticationConfig(c, &kubeAPIServer); err != nil {
 		return err

nodeup/pkg/model/prefix.go

@@ -41,6 +41,8 @@ func (b *PrefixBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 		})
 	case kops.CloudProviderGCE:
 		// Prefix is assigned by GCE
+	case kops.CloudProviderMetal:
+		// IPv6 must be configured externally (not by nodeup)
 	default:
 		return fmt.Errorf("kOps IPAM controller not supported on cloud %q", b.CloudProvider())
 	}

pkg/apis/kops/v1alpha2/host.go

@@ -36,6 +36,9 @@ type Host struct {
 type HostSpec struct {
 	PublicKey     string `json:"publicKey,omitempty"`
 	InstanceGroup string `json:"instanceGroup,omitempty"`
+
+	// PodCIDRs configures the IP ranges to be used for pods on this node/host.
+	PodCIDRs []string `json:"podCIDRs,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -2370,7 +2370,7 @@ func (in *Host) DeepCopyInto(out *Host) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
 	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
-	out.Spec = in.Spec
+	in.Spec.DeepCopyInto(&out.Spec)
 	return
 }
 
@@ -2428,6 +2428,11 @@ func (in *HostList) DeepCopyObject() runtime.Object {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *HostSpec) DeepCopyInto(out *HostSpec) {
 	*out = *in
+	if in.PodCIDRs != nil {
+		in, out := &in.PodCIDRs, &out.PodCIDRs
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 

pkg/commands/toolbox_enroll.go

@@ -67,6 +67,12 @@ type ToolboxEnrollOptions struct {
 	SSHUser string
 	SSHPort int
 
+	// BuildHost is a flag to only build the host resource, don't apply it or enroll the node
+	BuildHost bool
+
+	// PodCIDRs is the list of IP Address ranges to use for pods that run on this node
+	PodCIDRs []string
+
 	kubeconfig.CreateKubecfgOptions
 }
 
@@ -85,6 +91,11 @@ func RunToolboxEnroll(ctx context.Context, f commandutils.Factory, out io.Writer
 	if options.InstanceGroup == "" {
 		return fmt.Errorf("instance-group is required")
 	}
+	if options.Host == "" {
+		// Technically we could build the host resource without the PKI, but this isn't the case we are targeting right now.
+		return fmt.Errorf("host is required")
+	}
+
 	clientset, err := f.KopsClient()
 	if err != nil {
 		return err
@@ -100,40 +111,11 @@ func RunToolboxEnroll(ctx context.Context, f commandutils.Factory, out io.Writer
 	if err != nil {
 		return err
 	}
-	fullInstanceGroup, err := configBuilder.GetFullInstanceGroup(ctx)
-	if err != nil {
-		return err
-	}
-	bootstrapData, err := configBuilder.GetBootstrapData(ctx)
-	if err != nil {
-		return err
-	}
 
 	// Enroll the node over SSH.
-	if options.Host != "" {
+	restConfig, err := f.RESTConfig(ctx, fullCluster, options.CreateKubecfgOptions)
-		restConfig, err := f.RESTConfig(ctx, fullCluster, options.CreateKubecfgOptions)
-		if err != nil {
-			return err
-		}
-
-		if err := enrollHost(ctx, fullInstanceGroup, options, bootstrapData, restConfig); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func enrollHost(ctx context.Context, ig *kops.InstanceGroup, options *ToolboxEnrollOptions, bootstrapData *BootstrapData, restConfig *rest.Config) error {
-	scheme := runtime.NewScheme()
-	if err := v1alpha2.AddToScheme(scheme); err != nil {
-		return fmt.Errorf("building kubernetes scheme: %w", err)
-	}
-	kubeClient, err := client.New(restConfig, client.Options{
-		Scheme: scheme,
-	})
 	if err != nil {
-		return fmt.Errorf("building kubernetes client: %w", err)
+		return err
 	}
 
 	sudo := true
@@ -147,6 +129,39 @@ func enrollHost(ctx context.Context, ig *kops.InstanceGroup, options *ToolboxEnr
 	}
 	defer sshTarget.Close()
 
+	hostData, err := buildHostData(ctx, sshTarget, options)
+	if err != nil {
+		return err
+	}
+
+	if options.BuildHost {
+		klog.Infof("building host data for %+v", hostData)
+		b, err := yaml.Marshal(hostData)
+		if err != nil {
+			return fmt.Errorf("error marshalling host data: %w", err)
+		}
+		fmt.Fprintf(out, "%s\n", string(b))
+		return nil
+	}
+
+	fullInstanceGroup, err := configBuilder.GetFullInstanceGroup(ctx)
+	if err != nil {
+		return err
+	}
+	bootstrapData, err := configBuilder.GetBootstrapData(ctx)
+	if err != nil {
+		return err
+	}
+
+	if err := enrollHost(ctx, fullInstanceGroup, bootstrapData, restConfig, hostData, sshTarget); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// buildHostData builds an instance of the Host CRD, based on information in the options and by SSHing to the target host.
+func buildHostData(ctx context.Context, sshTarget *SSHHost, options *ToolboxEnrollOptions) (*v1alpha2.Host, error) {
 	publicKeyPath := "/etc/kubernetes/kops/pki/machine/public.pem"
 
 	publicKeyBytes, err := sshTarget.readFile(ctx, publicKeyPath)
@@ -154,19 +169,20 @@ func enrollHost(ctx context.Context, ig *kops.InstanceGroup, options *ToolboxEnr
 		if errors.Is(err, fs.ErrNotExist) {
 			publicKeyBytes = nil
 		} else {
-			return fmt.Errorf("error reading public key %q: %w", publicKeyPath, err)
+			return nil, fmt.Errorf("error reading public key %q: %w", publicKeyPath, err)
 		}
 	}
 
+	// Create the key if it doesn't exist
 	publicKeyBytes = bytes.TrimSpace(publicKeyBytes)
 	if len(publicKeyBytes) == 0 {
-		if _, err := sshTarget.runScript(ctx, scriptCreateKey, ExecOptions{Sudo: sudo, Echo: true}); err != nil {
+		if _, err := sshTarget.runScript(ctx, scriptCreateKey, ExecOptions{Echo: true}); err != nil {
-			return err
+			return nil, err
 		}
 
 		b, err := sshTarget.readFile(ctx, publicKeyPath)
 		if err != nil {
-			return fmt.Errorf("error reading public key %q (after creation): %w", publicKeyPath, err)
+			return nil, fmt.Errorf("error reading public key %q (after creation): %w", publicKeyPath, err)
 		}
 		publicKeyBytes = b
 	}
@@ -174,14 +190,37 @@ func enrollHost(ctx context.Context, ig *kops.InstanceGroup, options *ToolboxEnr
 
 	hostname, err := sshTarget.getHostname(ctx)
 	if err != nil {
-		return err
+		return nil, err
+	}
+
+	host := &v1alpha2.Host{}
+	host.SetGroupVersionKind(v1alpha2.SchemeGroupVersion.WithKind("Host"))
+	host.Namespace = "kops-system"
+	host.Name = hostname
+	host.Spec.InstanceGroup = options.InstanceGroup
+	host.Spec.PublicKey = string(publicKeyBytes)
+	host.Spec.PodCIDRs = options.PodCIDRs
+
+	return host, nil
+}
+
+func enrollHost(ctx context.Context, ig *kops.InstanceGroup, bootstrapData *BootstrapData, restConfig *rest.Config, hostData *v1alpha2.Host, sshTarget *SSHHost) error {
+	scheme := runtime.NewScheme()
+	if err := v1alpha2.AddToScheme(scheme); err != nil {
+		return fmt.Errorf("building kubernetes scheme: %w", err)
+	}
+	kubeClient, err := client.New(restConfig, client.Options{
+		Scheme: scheme,
+	})
+	if err != nil {
+		return fmt.Errorf("building kubernetes client: %w", err)
 	}
 
 	// We can't create the host resource in the API server for control-plane nodes,
 	// because the API server (likely) isn't running yet.
 	if !ig.IsControlPlane() {
-		if err := createHostResourceInAPIServer(ctx, options, hostname, publicKeyBytes, kubeClient); err != nil {
+		if err := kubeClient.Create(ctx, hostData); err != nil {
-			return err
+			return fmt.Errorf("failed to create host %s/%s: %w", hostData.Namespace, hostData.Name, err)
 		}
 	}
 
@@ -192,27 +231,13 @@ func enrollHost(ctx context.Context, ig *kops.InstanceGroup, options *ToolboxEnr
 	}
 
 	if len(bootstrapData.NodeupScript) != 0 {
-		if _, err := sshTarget.runScript(ctx, string(bootstrapData.NodeupScript), ExecOptions{Sudo: sudo, Echo: true}); err != nil {
+		if _, err := sshTarget.runScript(ctx, string(bootstrapData.NodeupScript), ExecOptions{Echo: true}); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func createHostResourceInAPIServer(ctx context.Context, options *ToolboxEnrollOptions, nodeName string, publicKey []byte, client client.Client) error {
-	host := &v1alpha2.Host{}
-	host.Namespace = "kops-system"
-	host.Name = nodeName
-	host.Spec.InstanceGroup = options.InstanceGroup
-	host.Spec.PublicKey = string(publicKey)
-
-	if err := client.Create(ctx, host); err != nil {
-		return fmt.Errorf("failed to create host %s/%s: %w", host.Namespace, host.Name, err)
-	}
-
-	return nil
-}
-
 const scriptCreateKey = `
 #!/bin/bash
 set -o errexit
@@ -323,7 +348,7 @@ func (s *SSHHost) runScript(ctx context.Context, script string, options ExecOpti
 	p := vfs.NewSSHPath(s.sshClient, s.hostname, scriptPath, s.sudo)
 
 	defer func() {
-		if _, err := s.runCommand(ctx, "rm -rf "+tempDir, ExecOptions{Sudo: s.sudo, Echo: false}); err != nil {
+		if _, err := s.runCommand(ctx, "rm -rf "+tempDir, ExecOptions{Echo: false}); err != nil {
 			klog.Warningf("error cleaning up temp directory %q: %v", tempDir, err)
 		}
 	}()
@@ -344,7 +369,6 @@ type CommandOutput struct {
 
 // ExecOptions holds options for running a command remotely.
 type ExecOptions struct {
-	Sudo bool
 	Echo bool
 }
 
@@ -361,10 +385,11 @@ func (s *SSHHost) runCommand(ctx context.Context, command string, options ExecOp
 	session.Stderr = &output.Stderr
 
 	if options.Echo {
-		session.Stdout = io.MultiWriter(os.Stdout, session.Stdout)
+		// We send both to stderr, so we don't "corrupt" stdout
+		session.Stdout = io.MultiWriter(os.Stderr, session.Stdout)
 		session.Stderr = io.MultiWriter(os.Stderr, session.Stderr)
 	}
-	if options.Sudo {
+	if s.sudo {
 		command = "sudo " + command
 	}
 	if err := session.Run(command); err != nil {
@@ -376,7 +401,7 @@ func (s *SSHHost) runCommand(ctx context.Context, command string, options ExecOp
 // getHostname gets the hostname of the SSH target.
 // This is used as the node name when registering the node.
 func (s *SSHHost) getHostname(ctx context.Context) (string, error) {
-	output, err := s.runCommand(ctx, "hostname", ExecOptions{Sudo: false, Echo: true})
+	output, err := s.runCommand(ctx, "hostname", ExecOptions{Echo: true})
 	if err != nil {
 		return "", fmt.Errorf("failed to get hostname: %w", err)
 	}

pkg/kubeconfig/create_kubecfg.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"crypto/x509/pkix"
 	"fmt"
+	"net"
 	"os/user"
 	"sort"
 	"time"
@@ -91,7 +92,7 @@ func BuildKubecfg(ctx context.Context, cluster *kops.Cluster, keyStore fi.Keysto
 		server = "https://" + cluster.APIInternalName()
 	} else {
 		if cluster.Spec.API.PublicName != "" {
-			server = "https://" + cluster.Spec.API.PublicName
+			server = "https://" + wrapIPv6Address(cluster.Spec.API.PublicName)
 		} else {
 			server = "https://api." + clusterName
 		}
@@ -132,7 +133,7 @@ func BuildKubecfg(ctx context.Context, cluster *kops.Cluster, keyStore fi.Keysto
 				if len(targets) != 1 {
 					klog.Warningf("Found multiple API endpoints (%v), choosing arbitrarily", targets)
 				}
-				server = "https://" + targets[0]
+				server = "https://" + wrapIPv6Address(targets[0])
 			}
 		}
 	}
@@ -221,3 +222,14 @@ func BuildKubecfg(ctx context.Context, cluster *kops.Cluster, keyStore fi.Keysto
 
 	return b, nil
 }
+
+// wrapIPv6Address will wrap IPv6 addresses in square brackets,
+// for use in URLs; other endpoints are unchanged.
+func wrapIPv6Address(endpoint string) string {
+	ip := net.ParseIP(endpoint)
+	// IPv6 addresses are wrapped in square brackets in URLs
+	if ip != nil && ip.To4() == nil {
+		return "[" + endpoint + "]"
+	}
+	return endpoint
+}

pkg/model/components/kubelet.go

@@ -124,30 +124,37 @@ func (b *KubeletOptionsBuilder) configureKubelet(cluster *kops.Cluster, kubelet
 
 	cloudProvider := cluster.GetCloudProvider()
 	klog.V(1).Infof("Cloud Provider: %s", cloudProvider)
-	if cloudProvider != kops.CloudProviderMetal {
+	if b.controlPlaneKubernetesVersion.IsLT("1.31") {
-		if b.controlPlaneKubernetesVersion.IsLT("1.31") {
+		switch cloudProvider {
-			switch cloudProvider {
+		case kops.CloudProviderAWS:
-			case kops.CloudProviderAWS:
+			kubelet.CloudProvider = "aws"
-				kubelet.CloudProvider = "aws"
+		case kops.CloudProviderGCE:
-			case kops.CloudProviderGCE:
+			kubelet.CloudProvider = "gce"
-				kubelet.CloudProvider = "gce"
+		case kops.CloudProviderDO:
-			case kops.CloudProviderDO:
+			kubelet.CloudProvider = "external"
-				kubelet.CloudProvider = "external"
+		case kops.CloudProviderHetzner:
-			case kops.CloudProviderHetzner:
+			kubelet.CloudProvider = "external"
-				kubelet.CloudProvider = "external"
+		case kops.CloudProviderOpenstack:
-			case kops.CloudProviderOpenstack:
+			kubelet.CloudProvider = "openstack"
-				kubelet.CloudProvider = "openstack"
+		case kops.CloudProviderAzure:
-			case kops.CloudProviderAzure:
+			kubelet.CloudProvider = "azure"
-				kubelet.CloudProvider = "azure"
+		case kops.CloudProviderScaleway:
-			case kops.CloudProviderScaleway:
+			kubelet.CloudProvider = "external"
-				kubelet.CloudProvider = "external"
+		case kops.CloudProviderMetal:
-			default:
+			kubelet.CloudProvider = ""
-				kubelet.CloudProvider = "external"
+		default:
-			}
+			kubelet.CloudProvider = "external"
+		}
 
-			if cluster.Spec.ExternalCloudControllerManager != nil {
+		if cluster.Spec.ExternalCloudControllerManager != nil {
-				kubelet.CloudProvider = "external"
+			kubelet.CloudProvider = "external"
-			}
+		}
+	} else {
+		if cloudProvider == kops.CloudProviderMetal {
+			// metal does not (yet) have a cloud-controller-manager, so we don't need to set the cloud-provider flag
+			// If we do set it to external, kubelet will taint the node with the node.kops.k8s.io/uninitialized taint
+			// and there is no cloud-controller-manager to remove it
+			kubelet.CloudProvider = ""
 		} else {
 			kubelet.CloudProvider = "external"
 		}

tests/e2e/scenarios/bare-metal/cleanup

@@ -38,6 +38,8 @@ sudo ip link del dev tap-vm0 || true
 sudo ip link del dev tap-vm1 || true
 sudo ip link del dev tap-vm2 || true
 
+sudo ip link del dev br0 || true
+
 rm -rf .build/vm0
 rm -rf .build/vm1
 rm -rf .build/vm2

tests/e2e/scenarios/bare-metal/e2e_test.go

@@ -54,6 +54,35 @@ func TestNodeAddresses(t *testing.T) {
 		}
 	}
 }
+func TestNodesNotTainted(t *testing.T) {
+	h := NewHarness(context.Background(), t)
+
+	nodes := h.Nodes()
+
+	// Quick check that we have some nodes
+	if len(nodes) == 0 {
+		t.Errorf("expected some nodes, got 0 nodes")
+	}
+
+	// Verify that the nodes aren't tainted
+	// In particular, we are checking for the node.cloudprovider.kubernetes.io/uninitialized taint
+	for _, node := range nodes {
+		t.Logf("node %s has taints: %v", node.Name, node.Spec.Taints)
+		for _, taint := range node.Spec.Taints {
+			switch taint.Key {
+			case "node.kops.k8s.io/uninitialized":
+				t.Errorf("unexpected taint for node %s: %s", node.Name, taint.Key)
+				t.Errorf("if we pass the --cloud-provider=external flag to kubelet, the node will be tainted with the node.kops.k8s.io/uninitialize taint")
+				t.Errorf("the taint is expected to be removed by the cloud-contoller-manager")
+				t.Errorf("(likely should be running a cloud-controller-manager in the cluster, or we should not pass the --cloud-provider=external flag to kubelet)")
+			case "node-role.kubernetes.io/control-plane":
+				// expected for control-plane nodes
+			default:
+				t.Errorf("unexpected taint for node %s: %s", node.Name, taint.Key)
+			}
+		}
+	}
+}
 
 // Harness is a test harness for our bare-metal e2e tests
 type Harness struct {

tests/e2e/scenarios/bare-metal/run-test

@@ -40,30 +40,39 @@ function cleanup() {
   fi
 }
 
-if [[ -z "${SKIP_CLEANUP:-}" ]]; then
+trap cleanup EXIT
-  trap cleanup EXIT
-fi
 
 # Create the directory that will back our mock s3 storage
 rm -rf ${WORKDIR}/s3
 mkdir -p ${WORKDIR}/s3/
 
+IPV4_PREFIX=10.123.45.
+
+VM0_IP=${IPV4_PREFIX}10
+VM1_IP=${IPV4_PREFIX}11
+VM2_IP=${IPV4_PREFIX}12
+
 # Start our VMs
 ${REPO_ROOT}/tests/e2e/scenarios/bare-metal/start-vms
 
+# Start an SSH agent; enroll assumes SSH connectivity to the VMs with the key in the agent
+eval $(ssh-agent)
+ssh-add ${REPO_ROOT}/.build/.ssh/id_ed25519
+
 . hack/dev-build-metal.sh
 
 echo "Waiting 10 seconds for VMs to start"
 sleep 10
 
 # Remove from known-hosts in case of reuse
-ssh-keygen -f ~/.ssh/known_hosts -R 10.123.45.10 || true
+ssh-keygen -f ~/.ssh/known_hosts -R ${VM0_IP} || true
-ssh-keygen -f ~/.ssh/known_hosts -R 10.123.45.11 || true
+ssh-keygen -f ~/.ssh/known_hosts -R ${VM1_IP} || true
-ssh-keygen -f ~/.ssh/known_hosts -R 10.123.45.12 || true
+ssh-keygen -f ~/.ssh/known_hosts -R ${VM2_IP} || true
 
-ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@10.123.45.10 uptime
+# Check SSH is working and accept the host keys
-ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@10.123.45.11 uptime
+ssh -o StrictHostKeyChecking=accept-new root@${VM0_IP} uptime
-ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@10.123.45.12 uptime
+ssh -o StrictHostKeyChecking=accept-new root@${VM1_IP} uptime
+ssh -o StrictHostKeyChecking=accept-new root@${VM2_IP} uptime
 
 cd ${REPO_ROOT}
 
@@ -93,7 +102,7 @@ ${KOPS} create cluster --cloud=metal metal.k8s.local --zones main --networking c
 
 # Set the IP ingress, required for metal cloud
 # TODO: is this the best option?
-${KOPS} edit cluster metal.k8s.local  --set spec.api.publicName=10.123.45.10
+${KOPS} edit cluster metal.k8s.local  --set spec.api.publicName=${VM0_IP}
 
 # Use latest etcd-manager image (while we're adding features)
 #${KOPS} edit cluster metal.k8s.local --set 'spec.etcdClusters[*].manager.image=us-central1-docker.pkg.dev/k8s-staging-images/etcd-manager/etcd-manager-slim:v3.0.20250628-7-ga7be11fb'
@@ -114,28 +123,24 @@ ${KOPS} get ig --name metal.k8s.local -oyaml
 ${KOPS} update cluster metal.k8s.local
 ${KOPS} update cluster metal.k8s.local --yes --admin
 
-# Start an SSH agent; enroll assumes SSH connectivity to the VMs with the key in the agent
-eval $(ssh-agent)
-ssh-add ${REPO_ROOT}/.build/.ssh/id_ed25519
-
 # Enroll the control-plane VM
-${KOPS} toolbox enroll --cluster metal.k8s.local --instance-group control-plane-main --host 10.123.45.10 --v=2
+${KOPS} toolbox enroll --cluster metal.k8s.local --instance-group control-plane-main --host ${VM0_IP} --v=2
 
 # Manual creation of "volumes" for etcd, and setting up peer nodes
-cat <<EOF | ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@10.123.45.10 tee -a /etc/hosts
+cat <<EOF | ssh root@${VM0_IP} tee -a /etc/hosts
 
 # Hosts added for etcd discovery
-10.123.45.10 node0.main.metal.k8s.local
+${VM0_IP} node0.main.metal.k8s.local
-10.123.45.10 node0.events.metal.k8s.local
+${VM0_IP} node0.events.metal.k8s.local
 EOF
 
-ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@10.123.45.10 cat /etc/hosts
+ssh root@${VM0_IP} cat /etc/hosts
 
-ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@10.123.45.10 mkdir -p /mnt/disks/metal.k8s.local--main--0/mnt
+ssh root@${VM0_IP} mkdir -p /mnt/disks/metal.k8s.local--main--0/mnt
-ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@10.123.45.10 touch    /mnt/disks/metal.k8s.local--main--0/mnt/please-create-new-cluster
+ssh root@${VM0_IP} touch    /mnt/disks/metal.k8s.local--main--0/mnt/please-create-new-cluster
 
-ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@10.123.45.10 mkdir -p /mnt/disks/metal.k8s.local--events--0/mnt
+ssh root@${VM0_IP} mkdir -p /mnt/disks/metal.k8s.local--events--0/mnt
-ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@10.123.45.10 touch    /mnt/disks/metal.k8s.local--events--0/mnt/please-create-new-cluster
+ssh root@${VM0_IP} touch    /mnt/disks/metal.k8s.local--events--0/mnt/please-create-new-cluster
 
 
 echo "Waiting for kube to start"
@@ -204,18 +209,18 @@ function enroll_node() {
 
 # Manual "discovery" for control-plane endpoints
 # TODO: Replace with well-known IP
-cat <<EOF | ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@${node_ip} tee -a /etc/hosts
+cat <<EOF | ssh root@${node_ip} tee -a /etc/hosts
 
 # Hosts added for leader discovery
-10.123.45.10 kops-controller.internal.metal.k8s.local
+${VM0_IP} kops-controller.internal.metal.k8s.local
-10.123.45.10 api.internal.metal.k8s.local
+${VM0_IP} api.internal.metal.k8s.local
 EOF
 
 timeout 10m ${KOPS} toolbox enroll --cluster metal.k8s.local --instance-group nodes-main --host ${node_ip} --v=2
 }
 
-enroll_node 10.123.45.11
+enroll_node ${VM1_IP}
-enroll_node 10.123.45.12
+enroll_node ${VM2_IP}
 
 echo "Waiting 30 seconds for nodes to be ready"
 sleep 30

tests/e2e/scenarios/bare-metal/scenario-ipv6

@@ -0,0 +1,322 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o xtrace
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+cd ${REPO_ROOT}
+
+WORKDIR=${REPO_ROOT}/.build/
+
+BINDIR=${WORKDIR}/bin
+mkdir -p "${BINDIR}"
+go build -o ${BINDIR}/kops ./cmd/kops
+
+export KOPS=${BINDIR}/kops
+
+function cleanup() {
+  echo "running dump-artifacts"
+  ${REPO_ROOT}/tests/e2e/scenarios/bare-metal/dump-artifacts || true
+
+  if [[ -z "${SKIP_CLEANUP:-}" ]]; then
+    echo "running cleanup"
+    ${REPO_ROOT}/tests/e2e/scenarios/bare-metal/cleanup || true
+  fi
+}
+
+trap cleanup EXIT
+
+# Create the directory that will back our mock s3 storage
+rm -rf ${WORKDIR}/s3
+mkdir -p ${WORKDIR}/s3/
+
+IPV6_PREFIX=fd00:10:123:45:
+IPV4_PREFIX=10.123.45.
+
+VM0_IP=${IPV4_PREFIX}10
+VM1_IP=${IPV4_PREFIX}11
+VM2_IP=${IPV4_PREFIX}12
+
+VM0_IPV6=${IPV6_PREFIX}a::
+VM1_IPV6=${IPV6_PREFIX}b::
+VM2_IPV6=${IPV6_PREFIX}c::
+
+VM0_POD_CIDR=${IPV6_PREFIX}a::/96
+VM1_POD_CIDR=${IPV6_PREFIX}b::/96
+VM2_POD_CIDR=${IPV6_PREFIX}c::/96
+
+# Start our VMs
+${REPO_ROOT}/tests/e2e/scenarios/bare-metal/start-vms
+
+# Start an SSH agent; enroll assumes SSH connectivity to the VMs with the key in the agent
+eval $(ssh-agent)
+ssh-add ${REPO_ROOT}/.build/.ssh/id_ed25519
+
+. hack/dev-build-metal.sh
+
+echo "Waiting 10 seconds for VMs to start"
+sleep 10
+
+# Remove from known-hosts in case of reuse
+ssh-keygen -f ~/.ssh/known_hosts -R ${VM0_IP} || true
+ssh-keygen -f ~/.ssh/known_hosts -R ${VM1_IP} || true
+ssh-keygen -f ~/.ssh/known_hosts -R ${VM2_IP} || true
+
+# Check SSH is working and accept the host keys
+ssh -o StrictHostKeyChecking=accept-new root@${VM0_IP} uptime
+ssh -o StrictHostKeyChecking=accept-new root@${VM1_IP} uptime
+ssh -o StrictHostKeyChecking=accept-new root@${VM2_IP} uptime
+
+cd ${REPO_ROOT}
+
+# Configure IPv6 networking
+function configure_ipv6() {
+  local hostname=$1
+  local node_ip=$2
+  local ipv6_ip=$3
+  local ipv6_range=$4
+
+  ssh root@${node_ip} ip link
+  ssh root@${node_ip} ip -6 addr add ${ipv6_range} dev enp0s3
+
+  # Set our node as the default route
+  # (otherwise the kubelet will not discover the IPv6 addresses in ResolveBindAddress)
+  # node-to-node routes will be discovered by radvd
+  ssh root@${node_ip} ip -6 route add ${IPV6_PREFIX}0::/96 dev enp0s3
+  ssh root@${node_ip} ip -6 route add default via ${IPV6_PREFIX}0::
+
+  cat <<EOF | ssh root@${node_ip} tee /etc/resolv.conf
+nameserver 8.8.8.8
+nameserver 8.8.4.4
+nameserver 2001:4860:4860::8888
+nameserver 2001:4860:4860::8844
+EOF
+
+  # Ensure /etc/hosts has an entry for the host
+  cat <<EOF | ssh root@${node_ip} tee -a /etc/hosts
+::1 ${hostname} localhost
+EOF
+
+  cat << EOF | ssh root@${node_ip} tee /etc/radvd.conf
+interface enp0s3
+{
+  AdvSendAdvert on;
+  AdvDefaultLifetime 0; # Not a default router
+  route ${ipv6_range}
+  {
+  };
+};
+EOF
+
+  ssh root@${node_ip} apt-get update
+  ssh root@${node_ip} apt-get install -y radvd
+  ssh root@${node_ip} systemctl restart radvd
+
+  ssh root@${node_ip} sysctl net.ipv6.conf.enp0s3.accept_ra=2
+  ssh root@${node_ip} sysctl net.ipv6.conf.enp0s3.accept_ra_rt_info_max_plen=96
+
+  
+  ssh root@${node_ip} ip -6 addr
+  ssh root@${node_ip} ip -6 route
+}
+
+# Configure our IPv6 addresses on the bridge
+sudo ip address add ${IPV6_PREFIX}0::/96 dev br0 || true
+sudo sysctl net.ipv6.conf.br0.accept_ra=2
+sudo sysctl net.ipv6.conf.br0.accept_ra_rt_info_max_plen=96
+
+# Configure the VMs on the bridge
+configure_ipv6 vm0 ${VM0_IP} ${VM0_IPV6} ${VM0_POD_CIDR}
+configure_ipv6 vm1 ${VM1_IP} ${VM1_IPV6} ${VM1_POD_CIDR}
+configure_ipv6 vm2 ${VM2_IP} ${VM2_IPV6} ${VM2_POD_CIDR}
+
+ip -6 route
+
+# Check the VMs are OK
+ping6 -c 1 ${VM0_IPV6}
+ping6 -c 1 ${VM1_IPV6}
+ping6 -c 1 ${VM2_IPV6}
+
+# Enable feature flag for bare metal
+export KOPS_FEATURE_FLAGS=Metal
+
+# Set up the AWS credentials
+export AWS_SECRET_ACCESS_KEY=secret
+export AWS_ACCESS_KEY_ID=accesskey
+export AWS_ENDPOINT_URL=http://10.123.45.1:8443
+export AWS_REGION=us-east-1
+
+export S3_ENDPOINT=${AWS_ENDPOINT_URL}
+export S3_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+export S3_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+
+# Create the state-store bucket in our mock s3 server
+export KOPS_STATE_STORE=s3://kops-state-store/
+aws --version
+aws s3 ls s3://kops-state-store || aws s3 mb s3://kops-state-store
+
+export CLUSTER_NAME=metalipv6.k8s.local
+
+# List clusters (there should not be any yet)
+${KOPS} get cluster || true
+
+# Create a cluster
+${KOPS} create cluster --cloud=metal ${CLUSTER_NAME} --zones main --networking cni --ipv6
+
+# Set the IP ingress, required for metal cloud
+# TODO: is this the best option?
+${KOPS} edit cluster ${CLUSTER_NAME} --set spec.api.publicName=${VM0_IPV6}
+
+# Use 1.32 kubernetes so we get https://github.com/kubernetes/kubernetes/pull/125337
+export KOPS_RUN_TOO_NEW_VERSION=1
+"${KOPS}" edit cluster ${CLUSTER_NAME} "--set=cluster.spec.kubernetesVersion=1.32.0"
+
+# List clusters
+${KOPS} get cluster
+${KOPS} get cluster -oyaml
+
+# List instance groups
+${KOPS} get ig --name ${CLUSTER_NAME}
+${KOPS} get ig --name ${CLUSTER_NAME} -oyaml
+
+# Apply basic configuration
+${KOPS} update cluster ${CLUSTER_NAME}
+${KOPS} update cluster ${CLUSTER_NAME} --yes --admin
+
+
+# Enroll the control-plane VM
+${KOPS} toolbox enroll --cluster ${CLUSTER_NAME} --instance-group control-plane-main --host ${VM0_IP} --pod-cidr ${VM0_POD_CIDR} --v=2
+
+# Manual creation of "volumes" for etcd, and setting up peer nodes
+cat <<EOF | ssh root@${VM0_IP} tee -a /etc/hosts
+
+# Hosts added for etcd discovery
+${VM0_IP} node0.main.${CLUSTER_NAME}
+${VM0_IP} node0.events.${CLUSTER_NAME}
+EOF
+
+ssh root@${VM0_IP} cat /etc/hosts
+
+ssh root@${VM0_IP} mkdir -p /mnt/disks/${CLUSTER_NAME}--main--0/mnt
+ssh root@${VM0_IP} touch    /mnt/disks/${CLUSTER_NAME}--main--0/mnt/please-create-new-cluster
+
+ssh root@${VM0_IP} mkdir -p /mnt/disks/${CLUSTER_NAME}--events--0/mnt
+ssh root@${VM0_IP} touch    /mnt/disks/${CLUSTER_NAME}--events--0/mnt/please-create-new-cluster
+
+echo "Waiting for kube to start"
+# Wait for kube-apiserver to be ready, timeout after 10 minutes
+for i in {1..60}; do
+  if kubectl get nodes; then
+    break
+  fi
+  sleep 10
+done
+
+# Create CRD and namespace for host records
+kubectl create ns kops-system
+kubectl apply -f ${REPO_ROOT}/k8s/crds/kops.k8s.io_hosts.yaml
+
+# Create the host record (we can't auto create for control plane nodes)
+${KOPS} toolbox enroll --cluster ${CLUSTER_NAME} --instance-group control-plane-main --host ${VM0_IP} --pod-cidr ${VM0_POD_CIDR} --v=2 --build-host | kubectl apply -f -
+
+kubectl get nodes
+kubectl get pods -A
+
+# Install kindnet
+kubectl create -f https://raw.githubusercontent.com/aojea/kindnet/main/install-kindnet.yaml
+echo "Waiting 10 seconds for kindnet to start"
+sleep 10
+kubectl get nodes
+kubectl get pods -A
+
+
+# kops-controller extra permissions
+kubectl apply --server-side -f - <<EOF
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kops-controller:pki-verifier
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kops-controller:pki-verifier
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:serviceaccount:kube-system:kops-controller
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kops-controller:pki-verifier
+rules:
+- apiGroups:
+  - "kops.k8s.io"
+  resources:
+  - hosts
+  verbs:
+  - get
+  - list
+  - watch
+# Must be able to set node addresses
+# TODO: Move out?
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+EOF
+
+function enroll_node() {
+  local node_ip=$1
+  local pod_cidr=$2
+
+# Manual "discovery" for control-plane endpoints
+# TODO: Replace with well-known IP
+cat <<EOF | ssh root@${node_ip} tee -a /etc/hosts
+
+# Hosts added for leader discovery
+${VM0_IP} kops-controller.internal.${CLUSTER_NAME}
+${VM0_IP} api.internal.${CLUSTER_NAME}
+EOF
+
+timeout 10m ${KOPS} toolbox enroll --cluster ${CLUSTER_NAME} --instance-group nodes-main --host ${node_ip} --pod-cidr ${pod_cidr} --v=2
+}
+
+enroll_node ${VM1_IP} ${VM1_POD_CIDR}
+enroll_node ${VM2_IP} ${VM2_POD_CIDR}
+
+echo "Waiting 30 seconds for nodes to be ready"
+sleep 30
+
+kubectl get nodes
+kubectl get nodes -o yaml
+
+kubectl get pods -A
+
+# Ensure the cluster passes validation
+${KOPS} validate cluster ${CLUSTER_NAME} --wait=10m
+
+# Run a few bare-metal e2e tests
+echo "running e2e tests"
+cd ${REPO_ROOT}/tests/e2e/scenarios/bare-metal
+go test -v .
+
+echo "Test successful"

upup/pkg/fi/loader/options_loader.go

@@ -55,7 +55,7 @@ func (l *OptionsLoader[T]) iterate(userConfig T, current T) (T, error) {
 	reflectutils.JSONMergeStruct(next, current)
 
 	for _, t := range l.Builders {
-		klog.V(2).Infof("executing builder %T", t)
+		klog.V(4).Infof("executing builder %T", t)
 
 		err := t.BuildOptions(next)
 		if err != nil {