Merge pull request #16844 from justinsb/add_node

metal: initial support for adding hosts

.github/workflows/e2e.yml

@@ -15,6 +15,7 @@ permissions:
 jobs:
   tests-e2e-scenarios-bare-metal:
     runs-on: ubuntu-24.04
+    timeout-minutes: 70
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
         with:
@@ -28,7 +29,7 @@ jobs:
       - name: tests/e2e/scenarios/bare-metal/run-test
         working-directory: ${{ env.GOPATH }}/src/k8s.io/kops
         run: |
-          tests/e2e/scenarios/bare-metal/run-test
+          timeout 60m tests/e2e/scenarios/bare-metal/run-test
         env:
           ARTIFACTS: /tmp/artifacts
       - name: Archive production artifacts

cmd/kops-controller/pkg/controllerclientset/addons.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllerclientset
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"k8s.io/klog/v2"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/client/simple"
+	"k8s.io/kops/pkg/kubemanifest"
+	"k8s.io/kops/util/pkg/vfs"
+)
+
+type addonsClient struct {
+	basePath vfs.Path
+
+	cluster *kops.Cluster
+}
+
+var _ simple.AddonsClient = &addonsClient{}
+
+func newAddonsClient(basePath vfs.Path, cluster *kops.Cluster) *addonsClient {
+	if cluster == nil || cluster.Name == "" {
+		klog.Fatalf("cluster / cluster.Name is required")
+	}
+
+	r := &addonsClient{
+		basePath: basePath,
+		cluster:  cluster,
+	}
+
+	return r
+}
+
+func (c *addonsClient) Replace(addons kubemanifest.ObjectList) error {
+	return fmt.Errorf("server-side addons client does not support Addons::Replace")
+}
+
+func (c *addonsClient) List(ctx context.Context) (kubemanifest.ObjectList, error) {
+	configPath := c.basePath.Join("default")
+
+	b, err := configPath.ReadFile(ctx)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("error reading addons file %s: %v", configPath, err)
+	}
+
+	objects, err := kubemanifest.LoadObjectsFrom(b)
+	if err != nil {
+		return nil, err
+	}
+
+	return objects, nil
+}

cmd/kops-controller/pkg/controllerclientset/clientset.go

@@ -0,0 +1,170 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllerclientset
+
+import (
+	"context"
+	"fmt"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/klog/v2"
+	"k8s.io/kops/pkg/apis/kops"
+	kopsinternalversion "k8s.io/kops/pkg/client/clientset_generated/clientset/typed/kops/internalversion"
+	"k8s.io/kops/pkg/client/simple"
+	"k8s.io/kops/pkg/kopscodecs"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/util/pkg/vfs"
+)
+
+// New constructs a new client for querying cluster and instancegroup information
+// We split it out because we expect it to support CRDs etc in future.
+func New(vfsContext *vfs.VFSContext, clusterBasePath vfs.Path, clusterName string, clusterKeystore fi.CAStore, clusterSecretStore fi.SecretStore) (simple.Clientset, error) {
+	return &client{
+		vfsContext: vfsContext,
+
+		clusterBasePath: clusterBasePath,
+		clusterName:     clusterName,
+
+		clusterKeystore:    clusterKeystore,
+		clusterSecretStore: clusterSecretStore,
+	}, nil
+}
+
+type client struct {
+	vfsContext         *vfs.VFSContext
+	clusterBasePath    vfs.Path
+	clusterName        string
+	clusterKeystore    fi.CAStore
+	clusterSecretStore fi.SecretStore
+}
+
+// GetCluster reads a cluster by name
+func (c *client) GetCluster(ctx context.Context, name string) (*kops.Cluster, error) {
+	if name != c.clusterName {
+		return nil, fmt.Errorf("clientset bound to cluster %q, got cluster %q", c.clusterName, name)
+	}
+
+	p := c.clusterBasePath.Join("config")
+	b, err := p.ReadFile(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("reading file %v: %w", p, err)
+	}
+
+	gvk := kops.SchemeGroupVersion.WithKind("Cluster")
+	object, _, err := kopscodecs.Decode(b, &gvk)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing %v: %w", p, err)
+	}
+
+	cluster, ok := object.(*kops.Cluster)
+	if !ok {
+		return nil, fmt.Errorf("unexpected kind for cluster, got %T, want kops.Cluster", object)
+	}
+	return cluster, nil
+}
+
+// VFSContext returns a VFSContext.
+func (c *client) VFSContext() *vfs.VFSContext {
+	return c.vfsContext
+}
+
+// CreateCluster creates a cluster
+func (c *client) CreateCluster(ctx context.Context, cluster *kops.Cluster) (*kops.Cluster, error) {
+	return nil, fmt.Errorf("method CreateCluster not supported in server-side client")
+}
+
+// UpdateCluster updates a cluster
+func (c *client) UpdateCluster(ctx context.Context, cluster *kops.Cluster, status *kops.ClusterStatus) (*kops.Cluster, error) {
+	return nil, fmt.Errorf("method UpdateCluster not supported in server-side client")
+}
+
+// ListClusters returns all clusters
+func (c *client) ListClusters(ctx context.Context, options metav1.ListOptions) (*kops.ClusterList, error) {
+	return nil, fmt.Errorf("method ListClusters not supported in server-side client")
+}
+
+// ConfigBaseFor returns the vfs path where we will read configuration information from
+func (c *client) ConfigBaseFor(cluster *kops.Cluster) (vfs.Path, error) {
+	return nil, fmt.Errorf("method ConfigBaseFor not supported in server-side client")
+}
+
+// InstanceGroupsFor returns the InstanceGroupInterface bound to the namespace for a particular Cluster
+func (c *client) InstanceGroupsFor(cluster *kops.Cluster) kopsinternalversion.InstanceGroupInterface {
+	clusterName := cluster.Name
+	if clusterName != c.clusterName {
+		klog.Fatalf("clientset bound to cluster %q, got cluster %q", c.clusterName, clusterName)
+	}
+
+	return newInstanceGroups(c.vfsContext, cluster, c.clusterBasePath)
+}
+
+// AddonsFor returns the client for addon objects for a particular Cluster
+func (c *client) AddonsFor(cluster *kops.Cluster) simple.AddonsClient {
+	clusterName := cluster.Name
+	if clusterName != c.clusterName {
+		klog.Fatalf("clientset bound to cluster %q, got cluster %q", c.clusterName, clusterName)
+	}
+
+	basePath := c.clusterBasePath.Join("clusteraddons")
+
+	return newAddonsClient(basePath, cluster)
+}
+
+// SecretStore builds the secret store for the specified cluster
+func (c *client) SecretStore(cluster *kops.Cluster) (fi.SecretStore, error) {
+	clusterName := cluster.Name
+	if clusterName != c.clusterName {
+		klog.Fatalf("clientset bound to cluster %q, got cluster %q", c.clusterName, clusterName)
+	}
+
+	return c.clusterSecretStore, nil
+}
+
+// KeystoreReader builds the read-only key store for the specified cluster
+func (c *client) KeystoreReader(cluster *kops.Cluster) (fi.KeystoreReader, error) {
+	clusterName := cluster.Name
+	if clusterName != c.clusterName {
+		klog.Fatalf("clientset bound to cluster %q, got cluster %q", c.clusterName, clusterName)
+	}
+
+	return c.clusterKeystore, nil
+}
+
+// KeyStore builds the Keystore Writer for the specified cluster
+func (c *client) KeyStore(cluster *kops.Cluster) (fi.CAStore, error) {
+	clusterName := cluster.Name
+	if clusterName != c.clusterName {
+		klog.Fatalf("clientset bound to cluster %q, got cluster %q", c.clusterName, clusterName)
+	}
+
+	return c.clusterKeystore, nil
+}
+
+// SSHCredentialStore builds the SSHCredential store for the specified cluster
+func (c *client) SSHCredentialStore(cluster *kops.Cluster) (fi.SSHCredentialStore, error) {
+	clusterName := cluster.Name
+	if clusterName != c.clusterName {
+		klog.Fatalf("clientset bound to cluster %q, got cluster %q", c.clusterName, clusterName)
+	}
+
+	return newSSHCredentialStore(c.clusterBasePath, cluster), nil
+}
+
+// DeleteCluster deletes all the state for the specified cluster
+func (c *client) DeleteCluster(ctx context.Context, cluster *kops.Cluster) error {
+	return fmt.Errorf("method DeleteCluster not supported in server-side client")
+}

cmd/kops-controller/pkg/controllerclientset/instancegroups.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllerclientset
+
+import (
+	"context"
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/klog/v2"
+	kopsapi "k8s.io/kops/pkg/apis/kops"
+	kopsinternalversion "k8s.io/kops/pkg/client/clientset_generated/clientset/typed/kops/internalversion"
+	"k8s.io/kops/pkg/client/simple/vfsclientset"
+	"k8s.io/kops/util/pkg/vfs"
+)
+
+type instanceGroups struct {
+	base        vfsclientset.VFSClientBase
+	clusterName string
+}
+
+var _ kopsinternalversion.InstanceGroupInterface = &instanceGroups{}
+
+func newInstanceGroups(vfsContext *vfs.VFSContext, cluster *kopsapi.Cluster, clusterBasePath vfs.Path) *instanceGroups {
+	if cluster == nil || cluster.Name == "" {
+		klog.Fatalf("cluster / cluster.Name is required")
+	}
+
+	clusterName := cluster.Name
+	kind := "InstanceGroup"
+
+	r := &instanceGroups{
+		// cluster:     cluster,
+		clusterName: clusterName,
+	}
+	// We don't expect to need encoding
+	var storeVersion runtime.GroupVersioner
+	r.base.Init(kind, vfsContext, clusterBasePath.Join("instancegroup"), storeVersion)
+	return r
+}
+
+func (c *instanceGroups) Get(ctx context.Context, name string, options metav1.GetOptions) (*kopsapi.InstanceGroup, error) {
+	if options.ResourceVersion != "" {
+		return nil, fmt.Errorf("ResourceVersion not supported in InstanceGroupVFS::Get")
+	}
+
+	o, err := c.base.Find(ctx, name)
+	if err != nil {
+		return nil, err
+	}
+	if o == nil {
+		return nil, errors.NewNotFound(schema.GroupResource{Group: kopsapi.GroupName, Resource: "InstanceGroup"}, name)
+	}
+	ig := o.(*kopsapi.InstanceGroup)
+	c.addLabels(ig)
+
+	return ig, nil
+}
+
+func (c *instanceGroups) addLabels(ig *kopsapi.InstanceGroup) {
+	if ig.ObjectMeta.Labels == nil {
+		ig.ObjectMeta.Labels = make(map[string]string)
+	}
+	ig.ObjectMeta.Labels[kopsapi.LabelClusterName] = c.clusterName
+}
+
+func (c *instanceGroups) List(ctx context.Context, options metav1.ListOptions) (*kopsapi.InstanceGroupList, error) {
+	list := &kopsapi.InstanceGroupList{}
+	items, err := c.base.List(ctx, list.Items, options)
+	if err != nil {
+		return nil, err
+	}
+	list.Items = items.([]kopsapi.InstanceGroup)
+	for i := range list.Items {
+		c.addLabels(&list.Items[i])
+	}
+	return list, nil
+}
+
+func (c *instanceGroups) Create(ctx context.Context, g *kopsapi.InstanceGroup, opts metav1.CreateOptions) (*kopsapi.InstanceGroup, error) {
+	return nil, fmt.Errorf("InstanceGroups::Create not supported for server-side client")
+}
+
+func (c *instanceGroups) Update(ctx context.Context, g *kopsapi.InstanceGroup, opts metav1.UpdateOptions) (*kopsapi.InstanceGroup, error) {
+	return nil, fmt.Errorf("InstanceGroups::Update not supported for server-side client")
+}
+
+func (c *instanceGroups) Delete(ctx context.Context, name string, options metav1.DeleteOptions) error {
+	return fmt.Errorf("InstanceGroups::Delete not supported for server-side client")
+}
+
+func (r *instanceGroups) DeleteCollection(ctx context.Context, options metav1.DeleteOptions, listOptions metav1.ListOptions) error {
+	return fmt.Errorf("InstanceGroups::DeleteCollection not supported for server-side client")
+}
+
+func (r *instanceGroups) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
+	return nil, fmt.Errorf("InstanceGroups::Watch not supported for server-side client")
+}
+
+func (r *instanceGroups) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *kopsapi.InstanceGroup, err error) {
+	return nil, fmt.Errorf("InstanceGroups::Patch not supported for server-side client")
+}

cmd/kops-controller/pkg/controllerclientset/sshcredentialstore.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllerclientset
+
+import (
+	"context"
+	"fmt"
+
+	"k8s.io/klog/v2"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/util/pkg/vfs"
+)
+
+type sshCredentialStore struct {
+	clusterBasePath vfs.Path
+
+	cluster *kops.Cluster
+}
+
+var _ fi.SSHCredentialStore = &sshCredentialStore{}
+
+func newSSHCredentialStore(clusterBasePath vfs.Path, cluster *kops.Cluster) *sshCredentialStore {
+	if cluster == nil || cluster.Name == "" {
+		klog.Fatalf("cluster / cluster.Name is required")
+	}
+
+	s := &sshCredentialStore{
+		clusterBasePath: clusterBasePath,
+		cluster:         cluster,
+	}
+
+	return s
+}
+
+// DeleteSSHCredential deletes the specified SSH credential.
+func (s *sshCredentialStore) DeleteSSHCredential() error {
+	return fmt.Errorf("method DeleteSSHCredential not supported in server-side client")
+}
+
+// AddSSHPublicKey adds an SSH public key.
+func (s *sshCredentialStore) AddSSHPublicKey(ctx context.Context, data []byte) error {
+	return fmt.Errorf("method AddSSHPublicKey not supported in server-side client")
+}
+
+// FindSSHPublicKeys retrieves the SSH public keys.
+func (s *sshCredentialStore) FindSSHPublicKeys() ([]*kops.SSHCredential, error) {
+	klog.Warningf("method FindSSHPublicKeys is stub-implemented supported in server-side client")
+	return nil, nil
+}

cmd/kops-controller/pkg/server/keystore.go

@@ -22,12 +22,16 @@ import (
 	"os"
 	"path"
 
+	"k8s.io/klog/v2"
 	"k8s.io/kops/pkg/pki"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/util/pkg/vfs"
 	"sigs.k8s.io/yaml"
 )
 
 type keystore struct {
-	keys map[string]keystoreEntry
+	keys    map[string]keystoreEntry
+	keySets map[string]*fi.Keyset
 }
 
 type keystoreEntry struct {
@@ -35,10 +39,11 @@ type keystoreEntry struct {
 	key         *pki.PrivateKey
 }
 
-var _ pki.Keystore = keystore{}
+var _ pki.Keystore = &keystore{}
+var _ fi.CAStore = &keystore{}
 
 // FindPrimaryKeypair implements pki.Keystore
-func (k keystore) FindPrimaryKeypair(ctx context.Context, name string) (*pki.Certificate, *pki.PrivateKey, error) {
+func (k *keystore) FindPrimaryKeypair(ctx context.Context, name string) (*pki.Certificate, *pki.PrivateKey, error) {
 	entry, ok := k.keys[name]
 	if !ok {
 		return nil, nil, fmt.Errorf("unknown CA %q", name)
@@ -46,15 +51,41 @@ func (k keystore) FindPrimaryKeypair(ctx context.Context, name string) (*pki.Cer
 	return entry.certificate, entry.key, nil
 }
 
-func newKeystore(basePath string, cas []string) (pki.Keystore, map[string]string, error) {
+// FindKeyset finds a Keyset.  If the keyset is not found, it returns (nil, nil).
+func (k *keystore) FindKeyset(ctx context.Context, name string) (*fi.Keyset, error) {
+	keySet, ok := k.keySets[name]
+	if !ok {
+		return nil, nil
+	}
+	return keySet, nil
+}
+
+// StoreKeyset writes a Keyset to the store.
+func (k *keystore) StoreKeyset(ctx context.Context, name string, keyset *fi.Keyset) error {
+	return fmt.Errorf("server-side client does not support StoreKeyset")
+}
+
+// MirrorTo will copy secrets to a vfs.Path, which is often easier for a machine to read
+func (k *keystore) MirrorTo(ctx context.Context, basedir vfs.Path) error {
+	return fmt.Errorf("server-side client does not support MirrorTo")
+}
+
+// ListKeysets will return all the KeySets.
+func (k *keystore) ListKeysets() (map[string]*fi.Keyset, error) {
+	return nil, fmt.Errorf("server-side client does not support ListKeysets")
+}
+
+func newKeystore(basePath string, cas []string) (*keystore, map[string]string, error) {
 	keystore := &keystore{
-		keys: map[string]keystoreEntry{},
+		keys:    map[string]keystoreEntry{},
+		keySets: map[string]*fi.Keyset{},
 	}
 	for _, name := range cas {
 		certBytes, err := os.ReadFile(path.Join(basePath, name+".crt"))
 		if err != nil {
 			return nil, nil, fmt.Errorf("reading %q certificate: %v", name, err)
 		}
+		// TODO: Support multiple certificates?
 		certificate, err := pki.ParsePEMCertificate(certBytes)
 		if err != nil {
 			return nil, nil, fmt.Errorf("parsing %q certificate: %v", name, err)
@@ -80,10 +111,29 @@ func newKeystore(basePath string, cas []string) (pki.Keystore, map[string]string
 	if err != nil {
 		return nil, nil, fmt.Errorf("reading keypair-ids.yaml")
 	}
-	err = yaml.Unmarshal(keypairIDsBytes, &keypairIDs)
-	if err != nil {
+	if err := yaml.Unmarshal(keypairIDsBytes, &keypairIDs); err != nil {
 		return nil, nil, fmt.Errorf("parsing keypair-ids.yaml")
 	}
 
+	// Build keysets
+	for name, keypairID := range keypairIDs {
+		entry, found := keystore.keys[name]
+		if !found {
+			klog.Warningf("keypair %q found in keypair IDs, not found as keypair", name)
+			continue
+		}
+		primary := &fi.KeysetItem{}
+		primary.Id = keypairID
+		primary.Certificate = entry.certificate
+		primary.PrivateKey = entry.key
+
+		keyset := &fi.Keyset{}
+		keyset.Primary = primary
+		keyset.Items = make(map[string]*fi.KeysetItem)
+		keyset.Items[primary.Id] = primary
+
+		keystore.keySets[name] = keyset
+	}
+
 	return keystore, keypairIDs, nil
 }

cmd/kops-controller/pkg/server/node_config.go

@@ -18,11 +18,13 @@ package server
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 
 	"k8s.io/klog/v2"
 	"k8s.io/kops/pkg/apis/nodeup"
 	"k8s.io/kops/pkg/bootstrap"
+	"k8s.io/kops/pkg/commands"
 )
 
 func (s *Server) getNodeConfig(ctx context.Context, req *nodeup.BootstrapRequest, identity *bootstrap.VerifyResult) (*nodeup.NodeConfig, error) {
@@ -35,10 +37,21 @@ func (s *Server) getNodeConfig(ctx context.Context, req *nodeup.BootstrapRequest
 
 	nodeConfig := &nodeup.NodeConfig{}
 
-	// Note: For now, we're assuming there is only a single cluster, and it is ours.
-	// We therefore use the configured base path
+	if s.opt.Cloud == "metal" {
+		bootstrapData, err := s.buildNodeupConfig(ctx, s.opt.ClusterName, identity.InstanceGroupName)
+		if err != nil {
+			return nil, fmt.Errorf("building nodeConfig for instanceGroup: %w", err)
+		}
+		nodeupConfig, err := json.Marshal(bootstrapData.NodeupConfig)
+		if err != nil {
+			return nil, fmt.Errorf("marshalling nodeupConfig: %w", err)
+		}
+		nodeConfig.NodeupConfig = string(nodeupConfig)
+	} else {
+
+		// Note: For now, we're assuming there is only a single cluster, and it is ours.
+		// We therefore use the configured base path
 
-	{
 		p := s.configBase.Join("igconfig", "node", instanceGroupName, "nodeupconfig.yaml")
 
 		b, err := p.ReadFile(ctx)
@@ -66,3 +79,18 @@ func (s *Server) getNodeConfig(ctx context.Context, req *nodeup.BootstrapRequest
 
 	return nodeConfig, nil
 }
+
+func (s *Server) buildNodeupConfig(ctx context.Context, clusterName string, instanceGroupName string) (*commands.BootstrapData, error) {
+	configBuilder := &commands.ConfigBuilder{
+		Clientset:         s.clientset,
+		ClusterName:       clusterName,
+		InstanceGroupName: instanceGroupName,
+	}
+
+	bootstrapData, err := configBuilder.GetBootstrapData(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return bootstrapData, nil
+}

cmd/kops-controller/pkg/server/server.go

@@ -36,10 +36,12 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog/v2"
 	"k8s.io/kops/cmd/kops-controller/pkg/config"
+	"k8s.io/kops/cmd/kops-controller/pkg/controllerclientset"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/apis/kops/model"
 	"k8s.io/kops/pkg/apis/nodeup"
 	"k8s.io/kops/pkg/bootstrap"
+	"k8s.io/kops/pkg/client/simple"
 	"k8s.io/kops/pkg/pki"
 	"k8s.io/kops/pkg/rbac"
 	"k8s.io/kops/upup/pkg/fi"
@@ -55,9 +57,11 @@ type Server struct {
 	keypairIDs  map[string]string
 	server      *http.Server
 	verifier    bootstrap.Verifier
-	keystore    pki.Keystore
+	keystore    *keystore
 	secretStore fi.SecretStore
 
+	clientset simple.Clientset
+
 	// configBase is the base of the configuration storage.
 	configBase vfs.Path
 
@@ -92,16 +96,22 @@ func NewServer(vfsContext *vfs.VFSContext, opt *config.Options, verifier bootstr
 	}
 	s.configBase = configBase
 
+	s.keystore, s.keypairIDs, err = newKeystore(opt.Server.CABasePath, opt.Server.SigningCAs)
+	if err != nil {
+		return nil, err
+	}
+
 	p, err := vfsContext.BuildVfsPath(opt.SecretStore)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse SecretStore %q: %w", opt.SecretStore, err)
 	}
 	s.secretStore = secrets.NewVFSSecretStore(nil, p)
 
-	s.keystore, s.keypairIDs, err = newKeystore(opt.Server.CABasePath, opt.Server.SigningCAs)
+	clientset, err := controllerclientset.New(vfsContext, configBase, opt.ClusterName, s.keystore, s.secretStore)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("building controller clientset: %w", err)
 	}
+	s.clientset = clientset
 
 	challengeClient, err := bootstrap.NewChallengeClient(s.keystore)
 	if err != nil {

k8s/crds/kops.k8s.io_hosts.yaml

@@ -3,6 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
+    api-approved.kubernetes.io: https://github.com/kubernetes/enhancements/pull/1111
     controller-gen.kubebuilder.io/version: v0.16.0
   name: hosts.kops.k8s.io
 spec:
@@ -47,3 +48,5 @@ spec:
         type: object
     served: true
     storage: true
+    subresources:
+      status: {}

pkg/apis/kops/parse.go

@@ -54,7 +54,7 @@ func ParseInstanceGroupRole(input string, lenient bool) (InstanceGroupRole, bool
 }
 
 // ParseRawYaml parses an object just using yaml, without the full api machinery
-// Deprecated: prefer using the API machinery
+// Deprecated: prefer using the API machinery (package kopscodecs)
 func ParseRawYaml(data []byte, dest interface{}) error {
 	// Yaml can't parse empty strings
 	configString := string(data)
@@ -71,7 +71,7 @@ func ParseRawYaml(data []byte, dest interface{}) error {
 }
 
 // ToRawYaml marshals an object to yaml, without the full api machinery
-// Deprecated: prefer using the API machinery
+// Deprecated: prefer using the API machinery (package kopscodecs)
 func ToRawYaml(obj interface{}) ([]byte, error) {
 	data, err := utils.YamlMarshal(obj)
 	if err != nil {

pkg/apis/kops/v1alpha2/host.go

@@ -22,6 +22,8 @@ import (
 
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+// +kubebuilder:metadata:annotations="api-approved.kubernetes.io=https://github.com/kubernetes/enhancements/pull/1111"
+// +kubebuilder:subresource:status
 
 // Host represents a bare-metal machine that could be registered as a Node.
 type Host struct {

pkg/apis/kops/v1alpha3/host.go

@@ -22,6 +22,8 @@ import (
 
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+// +kubebuilder:metadata:annotations="api-approved.kubernetes.io=https://github.com/kubernetes/enhancements/pull/1111"
+// +kubebuilder:subresource:status
 
 // Host represents a bare-metal machine that could be registered as a Node.
 type Host struct {

pkg/client/simple/clientset.go

@@ -55,7 +55,7 @@ type Clientset interface {
 	// SecretStore builds the secret store for the specified cluster
 	SecretStore(cluster *kops.Cluster) (fi.SecretStore, error)
 
-	// KeyStore builds the key store for the specified cluster
+	// KeyStore gets the read-write keystore store for the specified cluster
 	KeyStore(cluster *kops.Cluster) (fi.CAStore, error)
 
 	// SSHCredentialStore builds the SSHCredential store for the specified cluster

pkg/client/simple/vfsclientset/cluster.go

@@ -38,12 +38,12 @@ import (
 )
 
 type ClusterVFS struct {
-	commonVFS
+	VFSClientBase
 }
 
 func newClusterVFS(vfsContext *vfs.VFSContext, basePath vfs.Path) *ClusterVFS {
 	c := &ClusterVFS{}
-	c.init("Cluster", vfsContext, basePath, StoreVersion)
+	c.Init("Cluster", vfsContext, basePath, StoreVersion)
 	return c
 }
 

pkg/client/simple/vfsclientset/commonvfs.go

@@ -40,7 +40,7 @@ var StoreVersion = v1alpha2.SchemeGroupVersion
 
 type ValidationFunction func(o runtime.Object) error
 
-type commonVFS struct {
+type VFSClientBase struct {
 	kind       string
 	vfsContext *vfs.VFSContext
 	basePath   vfs.Path
@@ -48,7 +48,7 @@ type commonVFS struct {
 	validate   ValidationFunction
 }
 
-func (c *commonVFS) init(kind string, vfsContext *vfs.VFSContext, basePath vfs.Path, storeVersion runtime.GroupVersioner) {
+func (c *VFSClientBase) Init(kind string, vfsContext *vfs.VFSContext, basePath vfs.Path, storeVersion runtime.GroupVersioner) {
 	codecs := kopscodecs.Codecs
 	yaml, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), "application/yaml")
 	if !ok {
@@ -61,7 +61,7 @@ func (c *commonVFS) init(kind string, vfsContext *vfs.VFSContext, basePath vfs.P
 	c.basePath = basePath
 }
 
-func (c *commonVFS) find(ctx context.Context, name string) (runtime.Object, error) {
+func (c *VFSClientBase) Find(ctx context.Context, name string) (runtime.Object, error) {
 	o, err := c.readConfig(ctx, c.basePath.Join(name))
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -72,11 +72,11 @@ func (c *commonVFS) find(ctx context.Context, name string) (runtime.Object, erro
 	return o, nil
 }
 
-func (c *commonVFS) list(ctx context.Context, items interface{}, options metav1.ListOptions) (interface{}, error) {
+func (c *VFSClientBase) List(ctx context.Context, items interface{}, options metav1.ListOptions) (interface{}, error) {
 	return c.readAll(ctx, items)
 }
 
-func (c *commonVFS) create(ctx context.Context, cluster *kops.Cluster, i runtime.Object) error {
+func (c *VFSClientBase) create(ctx context.Context, cluster *kops.Cluster, i runtime.Object) error {
 	objectMeta, err := meta.Accessor(i)
 	if err != nil {
 		return err
@@ -105,7 +105,7 @@ func (c *commonVFS) create(ctx context.Context, cluster *kops.Cluster, i runtime
 	return nil
 }
 
-func (c *commonVFS) serialize(o runtime.Object) ([]byte, error) {
+func (c *VFSClientBase) serialize(o runtime.Object) ([]byte, error) {
 	var b bytes.Buffer
 	err := c.encoder.Encode(o, &b)
 	if err != nil {
@@ -115,7 +115,7 @@ func (c *commonVFS) serialize(o runtime.Object) ([]byte, error) {
 	return b.Bytes(), nil
 }
 
-func (c *commonVFS) readConfig(ctx context.Context, configPath vfs.Path) (runtime.Object, error) {
+func (c *VFSClientBase) readConfig(ctx context.Context, configPath vfs.Path) (runtime.Object, error) {
 	data, err := configPath.ReadFile(ctx)
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -131,7 +131,7 @@ func (c *commonVFS) readConfig(ctx context.Context, configPath vfs.Path) (runtim
 	return object, nil
 }
 
-func (c *commonVFS) writeConfig(ctx context.Context, cluster *kops.Cluster, configPath vfs.Path, o runtime.Object, writeOptions ...vfs.WriteOption) error {
+func (c *VFSClientBase) writeConfig(ctx context.Context, cluster *kops.Cluster, configPath vfs.Path, o runtime.Object, writeOptions ...vfs.WriteOption) error {
 	data, err := c.serialize(o)
 	if err != nil {
 		return fmt.Errorf("error marshaling object: %v", err)
@@ -176,7 +176,7 @@ func (c *commonVFS) writeConfig(ctx context.Context, cluster *kops.Cluster, conf
 	return nil
 }
 
-func (c *commonVFS) update(ctx context.Context, cluster *kops.Cluster, i runtime.Object) error {
+func (c *VFSClientBase) update(ctx context.Context, cluster *kops.Cluster, i runtime.Object) error {
 	objectMeta, err := meta.Accessor(i)
 	if err != nil {
 		return err
@@ -202,7 +202,7 @@ func (c *commonVFS) update(ctx context.Context, cluster *kops.Cluster, i runtime
 	return nil
 }
 
-func (c *commonVFS) delete(ctx context.Context, name string, options metav1.DeleteOptions) error {
+func (c *VFSClientBase) delete(ctx context.Context, name string, options metav1.DeleteOptions) error {
 	p := c.basePath.Join(name)
 	err := p.Remove(ctx)
 	if err != nil {
@@ -214,7 +214,7 @@ func (c *commonVFS) delete(ctx context.Context, name string, options metav1.Dele
 	return nil
 }
 
-func (c *commonVFS) listNames(ctx context.Context) ([]string, error) {
+func (c *VFSClientBase) listNames(ctx context.Context) ([]string, error) {
 	keys, err := listChildNames(ctx, c.basePath)
 	if err != nil {
 		return nil, fmt.Errorf("error listing %s in state store: %v", c.kind, err)
@@ -226,7 +226,7 @@ func (c *commonVFS) listNames(ctx context.Context) ([]string, error) {
 	return keys, nil
 }
 
-func (c *commonVFS) readAll(ctx context.Context, items interface{}) (interface{}, error) {
+func (c *VFSClientBase) readAll(ctx context.Context, items interface{}) (interface{}, error) {
 	sliceValue := reflect.ValueOf(items)
 	sliceType := reflect.TypeOf(items)
 	if sliceType.Kind() != reflect.Slice {
@@ -239,7 +239,7 @@ func (c *commonVFS) readAll(ctx context.Context, items interface{}) (interface{}
 	}
 
 	for _, name := range names {
-		o, err := c.find(ctx, name)
+		o, err := c.Find(ctx, name)
 		if err != nil {
 			return nil, err
 		}

pkg/client/simple/vfsclientset/instancegroup.go

@@ -34,7 +34,7 @@ import (
 )
 
 type InstanceGroupVFS struct {
-	commonVFS
+	VFSClientBase
 
 	clusterName string
 	cluster     *kopsapi.Cluster
@@ -52,7 +52,7 @@ func newInstanceGroupVFS(c *VFSClientset, cluster *kopsapi.Cluster) *InstanceGro
 		cluster:     cluster,
 		clusterName: clusterName,
 	}
-	r.init(kind, c.VFSContext(), c.basePath.Join(clusterName, "instancegroup"), StoreVersion)
+	r.Init(kind, c.VFSContext(), c.basePath.Join(clusterName, "instancegroup"), StoreVersion)
 	r.validate = func(o runtime.Object) error {
 		return validation.ValidateInstanceGroup(o.(*kopsapi.InstanceGroup), nil, false).ToAggregate()
 	}
@@ -66,7 +66,7 @@ func (c *InstanceGroupVFS) Get(ctx context.Context, name string, options metav1.
 		return nil, fmt.Errorf("ResourceVersion not supported in InstanceGroupVFS::Get")
 	}
 
-	o, err := c.find(ctx, name)
+	o, err := c.Find(ctx, name)
 	if err != nil {
 		return nil, err
 	}
@@ -88,7 +88,7 @@ func (c *InstanceGroupVFS) addLabels(ig *kopsapi.InstanceGroup) {
 
 func (c *InstanceGroupVFS) List(ctx context.Context, options metav1.ListOptions) (*kopsapi.InstanceGroupList, error) {
 	list := &kopsapi.InstanceGroupList{}
-	items, err := c.list(ctx, list.Items, options)
+	items, err := c.VFSClientBase.List(ctx, list.Items, options)
 	if err != nil {
 		return nil, err
 	}

pkg/commands/toolbox_enroll.go

@@ -45,6 +45,7 @@ import (
 
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/apis/kops/v1alpha2"
+	"k8s.io/kops/pkg/apis/nodeup"
 	"k8s.io/kops/pkg/assets"
 	"k8s.io/kops/pkg/client/simple"
 	"k8s.io/kops/pkg/commands/commandutils"
@@ -188,7 +189,7 @@ func enrollHost(ctx context.Context, ig *kops.InstanceGroup, options *ToolboxEnr
 		}
 	}
 
-	for k, v := range bootstrapData.ConfigFiles {
+	for k, v := range bootstrapData.NodeupScriptAdditionalFiles {
 		if err := host.writeFile(ctx, k, bytes.NewReader(v)); err != nil {
 			return fmt.Errorf("writing file %q over SSH: %w", k, err)
 		}
@@ -383,8 +384,12 @@ func (s *SSHHost) getHostname(ctx context.Context) (string, error) {
 }
 
 type BootstrapData struct {
+	// NodeupScript is a script that can be used to bootstrap the node.
 	NodeupScript []byte
-	ConfigFiles  map[string][]byte
+	// NodeupConfig is structured configuration, provided by kops-controller (for example).
+	NodeupConfig *nodeup.Config
+	// NodeupScriptAdditionalFiles are additional files that are needed by the nodeup script.
+	NodeupScriptAdditionalFiles map[string][]byte
 }
 
 // ConfigBuilder builds bootstrap configuration for a node.
@@ -730,7 +735,7 @@ func (b *ConfigBuilder) GetBootstrapData(ctx context.Context) (*BootstrapData, e
 	}
 
 	bootstrapData := &BootstrapData{}
-	bootstrapData.ConfigFiles = make(map[string][]byte)
+	bootstrapData.NodeupScriptAdditionalFiles = make(map[string][]byte)
 
 	encryptionConfigSecretHash := ""
 	// TODO: Support encryption config?
@@ -812,7 +817,7 @@ func (b *ConfigBuilder) GetBootstrapData(ctx context.Context) (*BootstrapData, e
 		// bootConfig.NodeupConfigHash = base64.StdEncoding.EncodeToString(sum256[:])
 
 		p := filepath.Join("/etc/kubernetes/kops/config", "igconfig", bootConfig.InstanceGroupRole.ToLowerString(), ig.Name, "nodeupconfig.yaml")
-		bootstrapData.ConfigFiles[p] = nodeupConfigBytes
+		bootstrapData.NodeupScriptAdditionalFiles[p] = nodeupConfigBytes
 
 		// Copy any static manifests we need on the control plane
 		for _, staticManifest := range assetBuilder.StaticManifests {
@@ -820,7 +825,7 @@ func (b *ConfigBuilder) GetBootstrapData(ctx context.Context) (*BootstrapData, e
 				continue
 			}
 			p := filepath.Join("/etc/kubernetes/kops/config", staticManifest.Path)
-			bootstrapData.ConfigFiles[p] = staticManifest.Contents
+			bootstrapData.NodeupScriptAdditionalFiles[p] = staticManifest.Contents
 		}
 	}
 
@@ -829,6 +834,7 @@ func (b *ConfigBuilder) GetBootstrapData(ctx context.Context) (*BootstrapData, e
 		return nil, err
 	}
 	bootstrapData.NodeupScript = nodeupScriptBytes
+	bootstrapData.NodeupConfig = nodeupConfig
 
 	b.bootstrapData = bootstrapData
 

pkg/model/bootstrapscript.go

@@ -125,12 +125,19 @@ func (b *BootstrapScript) kubeEnv(ig *kops.InstanceGroup, c *fi.CloudupContext)
 }
 
 func KeypairNamesForInstanceGroup(cluster *kops.Cluster, ig *kops.InstanceGroup) []string {
-	keypairs := []string{"kubernetes-ca", "etcd-clients-ca"}
-	for _, etcdCluster := range cluster.Spec.EtcdClusters {
-		k := etcdCluster.Name
-		keypairs = append(keypairs, "etcd-manager-ca-"+k, "etcd-peers-ca-"+k)
-		if k != "events" && k != "main" {
-			keypairs = append(keypairs, "etcd-clients-ca-"+k)
+	keypairs := []string{"kubernetes-ca"}
+
+	// Add keypairs for default etcd clusters (main and events, not cilium)
+	if ig.IsControlPlane() {
+		for _, etcdCluster := range cluster.Spec.EtcdClusters {
+			k := etcdCluster.Name
+			if k != "events" && k != "main" {
+				// Likely cilium
+				continue
+			}
+			keypairs = append(keypairs, "etcd-manager-ca-"+k, "etcd-peers-ca-"+k)
+			// The client ca certificate is shared between events and main etcd clusters
+			keypairs = append(keypairs, "etcd-clients-ca")
 		}
 	}
 
@@ -138,6 +145,17 @@ func KeypairNamesForInstanceGroup(cluster *kops.Cluster, ig *kops.InstanceGroup)
 		keypairs = append(keypairs, "apiserver-aggregator-ca", "service-account", "etcd-clients-ca")
 	}
 
+	// Add keypairs for cilium etcd clusters (not the default etcd clusters)
+	for _, etcdCluster := range cluster.Spec.EtcdClusters {
+		k := etcdCluster.Name
+		if k == "events" || k == "main" {
+			// Not cilium
+			continue
+		}
+
+		keypairs = append(keypairs, "etcd-manager-ca-"+k, "etcd-peers-ca-"+k, "etcd-clients-ca-"+k)
+	}
+
 	if ig.IsBastion() {
 		keypairs = nil
 	}

pkg/nodemodel/nodeupconfigbuilder.go

@@ -459,8 +459,8 @@ func loadCertificates(keysets map[string]*fi.Keyset, name string, config *nodeup
 	}
 	config.CAs[name] = string(certificates)
 	if includeKeypairID {
-		if keyset.Primary == nil {
-			return fmt.Errorf("key %q did not have primary set", name)
+		if keyset.Primary == nil || keyset.Primary.Id == "" {
+			return fmt.Errorf("key %q did not have primary id set", name)
 		}
 		config.KeypairIDs[name] = keyset.Primary.Id
 	}

pkg/pki/issue_test.go

@@ -39,7 +39,7 @@ type mockKeystore struct {
 	invoked bool
 }
 
-// FindPrimaryKeypair implements pki.Keystore
+// FindPrimaryKeypair implements Keystore
 func (m *mockKeystore) FindPrimaryKeypair(ctx context.Context, name string) (*Certificate, *PrivateKey, error) {
 	assert.False(m.t, m.invoked, "invoked already")
 	m.invoked = true

tests/e2e/scenarios/bare-metal/run-test

@@ -38,7 +38,9 @@ function cleanup() {
   ${REPO_ROOT}/tests/e2e/scenarios/bare-metal/cleanup || true
 }
 
-trap cleanup EXIT
+if [[ -z "${SKIP_CLEANUP:-}" ]]; then
+  trap cleanup EXIT
+fi
 
 # Create the directory that will back our mock s3 storage
 rm -rf ${WORKDIR}/s3
@@ -115,7 +117,7 @@ eval $(ssh-agent)
 ssh-add ${REPO_ROOT}/.build/.ssh/id_ed25519
 
 # Enroll the control-plane VM
-${KOPS} toolbox enroll --cluster metal.k8s.local --instance-group control-plane-main --host 10.123.45.10 --v=8
+${KOPS} toolbox enroll --cluster metal.k8s.local --instance-group control-plane-main --host 10.123.45.10 --v=2
 
 # Manual creation of "volumes" for etcd, and setting up peer nodes
 cat <<EOF | ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@10.123.45.10 tee -a /etc/hosts
@@ -142,9 +144,76 @@ kubectl get pods -A
 
 # Install kindnet
 kubectl create -f https://raw.githubusercontent.com/aojea/kindnet/main/install-kindnet.yaml
-echo "Waiting 10 seconds for kube to start"
+echo "Waiting 10 seconds for kindnet to start"
 sleep 10
 kubectl get nodes
 kubectl get pods -A
 
+# For host records
+kubectl create ns kops-system
+kubectl apply -f ${REPO_ROOT}/k8s/crds/kops.k8s.io_hosts.yaml
+
+# kops-controller extra permissions
+kubectl apply --server-side -f - <<EOF
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kops-controller:pki-verifier
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kops-controller:pki-verifier
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:serviceaccount:kube-system:kops-controller
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kops-controller:pki-verifier
+rules:
+- apiGroups:
+  - "kops.k8s.io"
+  resources:
+  - hosts
+  verbs:
+  - get
+  - list
+  - watch
+# Must be able to set node addresses
+# TODO: Move out?
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+EOF
+
+function enroll_node() {
+  local node_ip=$1
+
+# Manual "discovery" for control-plane endpoints
+# TODO: Replace with well-known IP
+cat <<EOF | ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@${node_ip} tee -a /etc/hosts
+
+# Hosts added for leader discovery
+10.123.45.10 kops-controller.internal.metal.k8s.local
+10.123.45.10 api.internal.metal.k8s.local
+EOF
+
+timeout 10m ${KOPS} toolbox enroll --cluster metal.k8s.local --instance-group nodes-main --host ${node_ip} --v=2
+}
+
+enroll_node 10.123.45.11
+enroll_node 10.123.45.12
+
+echo "Waiting 30 seconds for nodes to be ready"
+sleep 30
+
+kubectl get nodes
+kubectl get pods -A
+
+
 echo "Test successful"

upup/pkg/fi/clientset_castore.go

@@ -45,7 +45,7 @@ var (
 )
 
 // NewClientsetCAStore is the constructor for ClientsetCAStore
-func NewClientsetCAStore(cluster *kops.Cluster, clientset kopsinternalversion.KopsInterface, namespace string) CAStore {
+func NewClientsetCAStore(cluster *kops.Cluster, clientset kopsinternalversion.KopsInterface, namespace string) *ClientsetCAStore {
 	c := &ClientsetCAStore{
 		cluster:   cluster,
 		clientset: clientset,
@@ -160,6 +160,25 @@ func (c *ClientsetCAStore) FindKeyset(ctx context.Context, name string) (*Keyset
 	return c.loadKeyset(ctx, name)
 }
 
+// FindPrimaryKeypair implements pki.Keystore
+func (c *ClientsetCAStore) FindPrimaryKeypair(ctx context.Context, name string) (*pki.Certificate, *pki.PrivateKey, error) {
+	keyset, err := c.FindKeyset(ctx, name)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if keyset == nil {
+		return nil, nil, nil
+	}
+	if keyset.Primary == nil {
+		return nil, nil, nil
+	}
+	if keyset.Primary.Certificate == nil {
+		return nil, nil, nil
+	}
+	return keyset.Primary.Certificate, keyset.Primary.PrivateKey, nil
+}
+
 // ListKeysets implements CAStore::ListKeysets
 func (c *ClientsetCAStore) ListKeysets() (map[string]*Keyset, error) {
 	ctx := context.TODO()

upup/pkg/fi/vfs_keystorereader.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/apis/kops/v1alpha2"
 	"k8s.io/kops/pkg/kopscodecs"
+	"k8s.io/kops/pkg/pki"
 	"k8s.io/kops/util/pkg/vfs"
 )
 
@@ -108,6 +109,25 @@ var legacyKeysetMappings = map[string]string{
 	"kubernetes-ca": "ca",
 }
 
+// FindPrimaryKeypair implements pki.Keystore
+func (c *VFSKeystoreReader) FindPrimaryKeypair(ctx context.Context, name string) (*pki.Certificate, *pki.PrivateKey, error) {
+	keyset, err := c.FindKeyset(ctx, name)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if keyset == nil {
+		return nil, nil, nil
+	}
+	if keyset.Primary == nil {
+		return nil, nil, nil
+	}
+	if keyset.Primary.Certificate == nil {
+		return nil, nil, nil
+	}
+	return keyset.Primary.Certificate, keyset.Primary.PrivateKey, nil
+}
+
 func (c *VFSKeystoreReader) FindKeyset(ctx context.Context, id string) (*Keyset, error) {
 	keys, err := c.findPrivateKeyset(ctx, id)
 	if keys == nil || os.IsNotExist(err) {