Perform challenge callbacks into a node

In order to verify that the caller is running on the specified node,
we source the expected IP address from the cloud, and require that the
node set up a simple challenge/response server to answer requests.

Because the challenge server runs on a port outside of the nodePort
range, this also makes it harder for pods to impersonate their host
nodes - though we do combine this with TPM and similar functionality
where it is available.

cmd/kops-controller/pkg/server/server.go

@@ -61,6 +61,9 @@ type Server struct {
 
 	// uncachedClient is an uncached client for the kube apiserver
 	uncachedClient client.Client
+
+	// challengeClient performs our callback-challenge into the node
+	challengeClient *bootstrap.ChallengeClient
 }
 
 var _ manager.LeaderElectionRunnable = &Server{}
@@ -94,6 +97,17 @@ func NewServer(opt *config.Options, verifier bootstrap.Verifier, uncachedClient
 	}
 	s.secretStore = secrets.NewVFSSecretStore(nil, p)
 
+	s.keystore, s.keypairIDs, err = newKeystore(opt.Server.CABasePath, opt.Server.SigningCAs)
+	if err != nil {
+		return nil, err
+	}
+
+	challengeClient, err := bootstrap.NewChallengeClient(s.keystore)
+	if err != nil {
+		return nil, err
+	}
+	s.challengeClient = challengeClient
+
 	r := http.NewServeMux()
 	r.Handle("/bootstrap", http.HandlerFunc(s.bootstrap))
 	server.Handler = recovery(r)
@@ -106,12 +120,6 @@ func (s *Server) NeedLeaderElection() bool {
 }
 
 func (s *Server) Start(ctx context.Context) error {
-	var err error
-	s.keystore, s.keypairIDs, err = newKeystore(s.opt.Server.CABasePath, s.opt.Server.SigningCAs)
-	if err != nil {
-		return err
-	}
-
 	go func() {
 		<-ctx.Done()
 
@@ -198,6 +206,15 @@ func (s *Server) bootstrap(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if err := s.challengeClient.DoCallbackChallenge(ctx, s.opt.ClusterName, id.ChallengeEndpoint, req); err != nil {
+		klog.Infof("bootstrap %s callback challenge failed: %v", r.RemoteAddr, err)
+		w.WriteHeader(http.StatusBadRequest)
+		_, _ = w.Write([]byte("callback failed"))
+		return
+	}
+
+	klog.Infof("performed successful callback challenge with %s; identified as %s", id.ChallengeEndpoint, id.NodeName)
+
 	resp := &nodeup.BootstrapResponse{
 		Certs: map[string]string{},
 	}

nodeup/pkg/model/bootstrap_client.go

@@ -102,6 +102,8 @@ func (b BootstrapClientBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 		Certs:      b.bootstrapCerts,
 		KeypairIDs: b.bootstrapKeypairIDs,
 	}
+	bootstrapClientTask.UseChallengeCallback = b.UseChallengeCallback()
+	bootstrapClientTask.ClusterName = b.NodeupConfig.ClusterName
 
 	for _, cert := range b.bootstrapCerts {
 		cert.Cert.Task = bootstrapClientTask

nodeup/pkg/model/context.go

@@ -396,6 +396,11 @@ func (c *NodeupModelContext) UseKopsControllerForNodeBootstrap() bool {
 	return model.UseKopsControllerForNodeBootstrap(c.Cluster)
 }
 
+// UseChallengeCallback is true if we should use a callback challenge during node provisioning with kops-controller.
+func (c *NodeupModelContext) UseChallengeCallback() bool {
+	return model.UseChallengeCallback()
+}
+
 // UsesSecondaryIP checks if the CNI in use attaches secondary interfaces to the host.
 func (c *NodeupModelContext) UsesSecondaryIP() bool {
 	return (c.NodeupConfig.Networking.CNI != nil && c.NodeupConfig.Networking.CNI.UsesSecondaryIP) ||

pkg/apis/kops/model/features.go

@@ -36,6 +36,11 @@ func UseKopsControllerForNodeBootstrap(cluster *kops.Cluster) bool {
 	}
 }
 
+// UseChallengeCallback is true if we should use a callback challenge during node provisioning with kops-controller.
+func UseChallengeCallback() bool {
+	return true
+}
+
 // UseKopsControllerForNodeConfig checks if nodeup should use kops-controller to get nodeup.Config.
 func UseKopsControllerForNodeConfig(cluster *kops.Cluster) bool {
 	switch cluster.Spec.GetCloudProvider() {

pkg/apis/nodeup/bootstrap.go

@@ -30,6 +30,17 @@ type BootstrapRequest struct {
 	// IncludeNodeConfig controls whether the cluster & instance group configuration should be returned.
 	// This allows for nodes without access to the kops state store.
 	IncludeNodeConfig bool `json:"includeNodeConfig"`
+
+	// Challenge is for a callback challenge.
+	Challenge *ChallengeRequest `json:"challenge,omitempty"`
+}
+
+// ChallengeRequest describes the callback challenge.
+type ChallengeRequest struct {
+	Endpoint        string `json:"endpoint,omitempty"`
+	ServerCA        []byte `json:"ca,omitempty"`
+	ChallengeID     string `json:"challengeID,omitempty"`
+	ChallengeSecret []byte `json:"challengeSecret,omitempty"`
 }
 
 // BootstrapResponse is a response to a BootstrapRequest.

pkg/apis/nodeup/config.go

@@ -127,6 +127,8 @@ type BootConfig struct {
 	// APIServerIPs is the API server IP addresses.
 	// This field is used for adding an alias for api.internal. in /etc/hosts, when Topology.DNS.Type == DNSTypeNone.
 	APIServerIPs []string `json:",omitempty"`
+	// ClusterName is the name of the cluster.
+	ClusterName string `json:",omitempty"`
 	// InstanceGroupName is the name of the instance group.
 	InstanceGroupName string `json:",omitempty"`
 	// InstanceGroupRole is the instance group role.
@@ -200,6 +202,7 @@ func NewConfig(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) (*Confi
 
 	bootConfig := BootConfig{
 		CloudProvider:     cluster.Spec.GetCloudProvider(),
+		ClusterName:       cluster.ObjectMeta.Name,
 		InstanceGroupName: instanceGroup.ObjectMeta.Name,
 		InstanceGroupRole: role,
 	}

pkg/bootstrap/authenticate.go

@@ -39,6 +39,12 @@ type VerifyResult struct {
 
 	// CertificateNames is the alternate names the node is authorized to use for certificates.
 	CertificateNames []string
+
+	// ChallengeEndpoint is a valid endpoints to which we should issue a challenge request,
+	// corresponding to the node the request identified as.
+	// This should be sourced from e.g. the cloud, and acts as a cross-check
+	// that this is the correct instance.
+	ChallengeEndpoint string
 }
 
 // Verifier verifies authentication credentials for requests.

pkg/bootstrap/challenge.go

@@ -0,0 +1,101 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bootstrap
+
+import (
+	cryptorand "crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"fmt"
+	"math/big"
+	"time"
+
+	"k8s.io/klog/v2"
+	"k8s.io/kops/pkg/pki"
+)
+
+func randomBytes(length int) []byte {
+	b := make([]byte, length)
+	if _, err := cryptorand.Read(b); err != nil {
+		klog.Fatalf("failed to read from crypto/rand: %v", err)
+	}
+	return b
+}
+
+func challengeKopsControllerSubject(clusterName string) pkix.Name {
+	// Note: keep in sync with subjectsMatch if you add (additional) fields here
+	return pkix.Name{
+		CommonName: "kops-controller." + clusterName,
+	}
+}
+
+func subjectsMatch(l, r pkix.Name) bool {
+	// We need to check all the fields in challengeKopsControllerSubject
+	return l.CommonName == r.CommonName
+}
+
+func challengeServerHostName(clusterName string) string {
+	return "challenge-server." + clusterName
+}
+
+func BuildChallengeServerCertificate(clusterName string) (*tls.Certificate, error) {
+	serverName := challengeServerHostName(clusterName)
+
+	privateKey, err := pki.GeneratePrivateKey()
+	if err != nil {
+		return nil, fmt.Errorf("generating ecdsa key: %w", err)
+	}
+
+	keyUsage := x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment
+
+	now := time.Now()
+	notBefore := now.Add(-15 * time.Minute)
+	notAfter := notBefore.Add(time.Hour)
+
+	template := x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		Subject: pkix.Name{
+			CommonName: serverName,
+		},
+		NotBefore: notBefore,
+		NotAfter:  notAfter,
+
+		KeyUsage:              keyUsage,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+	}
+
+	template.DNSNames = append(template.DNSNames, serverName)
+
+	der, err := x509.CreateCertificate(cryptorand.Reader, &template, &template, privateKey.Key.Public(), privateKey.Key)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create certificate: %w", err)
+	}
+
+	parsed, err := x509.ParseCertificate(der)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse certificate: %w", err)
+	}
+	tlsCertificate := &tls.Certificate{
+		PrivateKey:  privateKey.Key,
+		Certificate: [][]byte{parsed.Raw},
+		Leaf:        parsed,
+	}
+
+	return tlsCertificate, nil
+}

pkg/bootstrap/challenge_client.go

@@ -0,0 +1,128 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bootstrap
+
+import (
+	"context"
+	"crypto/subtle"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"k8s.io/kops/pkg/apis/nodeup"
+	"k8s.io/kops/pkg/pki"
+	pb "k8s.io/kops/proto/kops/bootstrap/v1"
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+type ChallengeClient struct {
+	keystore pki.Keystore
+}
+
+func NewChallengeClient(keystore pki.Keystore) (*ChallengeClient, error) {
+	return &ChallengeClient{
+		keystore: keystore,
+	}, nil
+}
+
+func (c *ChallengeClient) getClientCertificate(ctx context.Context, clusterName string) (*tls.Certificate, error) {
+	subject := challengeKopsControllerSubject(clusterName)
+
+	certificate, privateKey, _, err := pki.IssueCert(ctx, &pki.IssueCertRequest{
+		Validity: 1 * time.Hour,
+		Signer:   fi.CertificateIDCA,
+		Type:     "client",
+		Subject:  subject,
+	}, c.keystore)
+	if err != nil {
+		return nil, fmt.Errorf("error creating certificate: %w", err)
+	}
+
+	// TODO: Caching and rotation
+	clientCertificate := &tls.Certificate{
+		PrivateKey:  privateKey.Key,
+		Certificate: [][]byte{certificate.Certificate.Raw},
+		Leaf:        certificate.Certificate,
+	}
+	return clientCertificate, nil
+}
+
+func (c *ChallengeClient) DoCallbackChallenge(ctx context.Context, clusterName string, targetEndpoint string, bootstrapRequest *nodeup.BootstrapRequest) error {
+	challenge := bootstrapRequest.Challenge
+
+	if challenge == nil {
+		return fmt.Errorf("challenge not set")
+	}
+	if challenge.ChallengeID == "" {
+		return fmt.Errorf("challenge.id not set")
+	}
+	if len(challenge.ChallengeSecret) == 0 {
+		return fmt.Errorf("challenge.secret not set")
+	}
+	if challenge.Endpoint == "" {
+		return fmt.Errorf("challenge.endpoint not set")
+	}
+	if len(challenge.ServerCA) == 0 {
+		return fmt.Errorf("challenge.ca not set")
+	}
+
+	clientCertificate, err := c.getClientCertificate(ctx, clusterName)
+	if err != nil {
+		return err
+	}
+
+	serverCAs := x509.NewCertPool()
+	if !serverCAs.AppendCertsFromPEM(challenge.ServerCA) {
+		return fmt.Errorf("error loading certificate pool")
+	}
+
+	serverName := challengeServerHostName(clusterName)
+	tlsConfig := &tls.Config{
+		RootCAs:      serverCAs,
+		Certificates: []tls.Certificate{*clientCertificate},
+		ServerName:   serverName,
+	}
+
+	kospControllerNonce := randomBytes(16)
+	req := &pb.ChallengeRequest{
+		ChallengeId:     challenge.ChallengeID,
+		ChallengeRandom: kospControllerNonce,
+	}
+
+	expectedChallengeResponse := buildChallengeResponse(challenge.ChallengeSecret, kospControllerNonce)
+
+	var opts []grpc.DialOption
+	opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)))
+	conn, err := grpc.DialContext(ctx, targetEndpoint, opts...)
+	if err != nil {
+		return fmt.Errorf("error dialing target %q: %w", targetEndpoint, err)
+	}
+	client := pb.NewCallbackServiceClient(conn)
+
+	response, err := client.Challenge(ctx, req)
+	if err != nil {
+		return fmt.Errorf("error from callback challenge: %w", err)
+	}
+
+	if subtle.ConstantTimeCompare(response.GetChallengeResponse(), expectedChallengeResponse) != 1 {
+		return fmt.Errorf("callback challenge returned wrong result")
+	}
+	return nil
+}

pkg/bootstrap/challenge_server.go

@@ -0,0 +1,217 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bootstrap
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/hex"
+	"encoding/pem"
+	"fmt"
+	"net"
+	"sync"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/status"
+
+	"k8s.io/klog/v2"
+	"k8s.io/kops/pkg/apis/nodeup"
+	pb "k8s.io/kops/proto/kops/bootstrap/v1"
+)
+
+type ChallengeServer struct {
+	tlsConfig *tls.Config
+	servingCA []byte
+
+	mutex      sync.Mutex
+	challenges map[string]*Challenge
+
+	RequiredSubject pkix.Name
+
+	pb.UnimplementedCallbackServiceServer
+}
+
+func NewChallengeServer(clusterName string, caBundle []byte) (*ChallengeServer, error) {
+	serverCertificate, err := BuildChallengeServerCertificate(clusterName)
+	if err != nil {
+		return nil, err
+	}
+	tlsConfig := &tls.Config{
+		Certificates: []tls.Certificate{*serverCertificate},
+	}
+
+	var servingCA bytes.Buffer
+	for _, cert := range serverCertificate.Certificate {
+		if err := pem.Encode(&servingCA, &pem.Block{Type: "CERTIFICATE", Bytes: cert}); err != nil {
+			return nil, err
+		}
+	}
+
+	clientCAs := x509.NewCertPool()
+	if !clientCAs.AppendCertsFromPEM(caBundle) {
+		return nil, fmt.Errorf("unable to build client-cert CA pools")
+	}
+	tlsConfig.ClientCAs = clientCAs
+	tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
+
+	return &ChallengeServer{
+		RequiredSubject: challengeKopsControllerSubject(clusterName),
+		tlsConfig:       tlsConfig,
+		servingCA:       servingCA.Bytes(),
+	}, nil
+}
+
+type Challenge struct {
+	ChallengeID     string
+	ChallengeSecret []byte
+}
+
+func (s *ChallengeServer) createChallenge() *Challenge {
+	c := &Challenge{}
+	c.ChallengeID = hex.EncodeToString(randomBytes(16))
+	c.ChallengeSecret = randomBytes(16)
+
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	if s.challenges == nil {
+		s.challenges = make(map[string]*Challenge)
+	}
+	s.challenges[c.ChallengeID] = c
+	return c
+}
+
+type ChallengeListener struct {
+	endpoint   string
+	server     *ChallengeServer
+	grpcServer *grpc.Server
+}
+
+func (s *ChallengeListener) CreateChallenge() *nodeup.ChallengeRequest {
+	challenge := s.server.createChallenge()
+
+	return &nodeup.ChallengeRequest{
+		Endpoint:        s.Endpoint(),
+		ChallengeID:     challenge.ChallengeID,
+		ChallengeSecret: challenge.ChallengeSecret,
+		ServerCA:        s.server.servingCA,
+	}
+}
+
+func (s *ChallengeListener) Stop() {
+	s.grpcServer.Stop()
+}
+
+func (s *ChallengeListener) Endpoint() string {
+	return s.endpoint
+}
+
+func (s *ChallengeServer) NewListener(ctx context.Context, listen string) (*ChallengeListener, error) {
+	var opts []grpc.ServerOption
+
+	opts = append(opts, grpc.Creds(credentials.NewTLS(s.tlsConfig)))
+	grpcServer := grpc.NewServer(opts...)
+	pb.RegisterCallbackServiceServer(grpcServer, s)
+
+	lis, err := net.Listen("tcp", listen)
+	if err != nil {
+		return nil, fmt.Errorf("error listening on %q: %w", listen, err)
+	}
+
+	grpcListener := &ChallengeListener{
+		server:     s,
+		grpcServer: grpcServer,
+		endpoint:   lis.Addr().String(),
+	}
+
+	go func() {
+		klog.Infof("starting node-challenge listener on %v", lis.Addr())
+		if err := grpcServer.Serve(lis); err != nil {
+			lis.Close()
+
+			klog.Warningf("error serving GRPC: %v", err)
+		}
+	}()
+
+	return grpcListener, nil
+}
+
+// Answers challenges to cross-check bootstrap requests.
+func (s *ChallengeServer) Challenge(ctx context.Context, req *pb.ChallengeRequest) (*pb.ChallengeResponse, error) {
+	klog.Infof("got node-challenge request")
+	// Explicitly authenticate the username for safety
+	peerInfo, ok := peer.FromContext(ctx)
+	if !ok {
+		klog.Warningf("no peer in context")
+		return nil, status.Error(codes.Unauthenticated, "peer was nil")
+	}
+
+	tlsInfo, ok := peerInfo.AuthInfo.(credentials.TLSInfo)
+	if !ok {
+		klog.Warningf("peer.AuthInfo was of unexpected type %T", peerInfo.AuthInfo)
+		return nil, status.Error(codes.Unauthenticated, "unexpected peer transport credentials")
+	}
+
+	if len(tlsInfo.State.VerifiedChains) == 0 || len(tlsInfo.State.VerifiedChains[0]) == 0 {
+		klog.Warningf("no VerifiedChains in TLSInfo")
+		return nil, status.Error(codes.Unauthenticated, "verified chains were empty")
+	}
+
+	if got, want := tlsInfo.State.VerifiedChains[0][0].Subject, s.RequiredSubject; !subjectsMatch(got, want) {
+		klog.Warningf("certificate subjects did not match expected; got %q, want %q", got, want)
+		return nil, status.Error(codes.Unauthenticated, "certificate subjects did not match")
+	}
+
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+
+	key := req.ChallengeId
+	if key == "" {
+		return nil, status.Errorf(codes.InvalidArgument, "challenge_id is required")
+	}
+
+	challenge := s.challenges[key]
+	if challenge == nil {
+		return nil, status.Errorf(codes.NotFound, "challenge was not found")
+	}
+	// Prevent replay attacks
+	delete(s.challenges, key)
+
+	hash := buildChallengeResponse(challenge.ChallengeSecret, req.GetChallengeRandom())
+	response := &pb.ChallengeResponse{
+		ChallengeResponse: hash,
+	}
+
+	return response, nil
+}
+
+func buildChallengeResponse(nodeNonce []byte, kopsControllerNonde []byte) []byte {
+	// Arguably this is overkill because the TLS handshake is stronger and everything is encrypted.
+	hasher := sha256.New()
+	hasher.Sum(nodeNonce)
+	hasher.Sum(kopsControllerNonde)
+
+	hash := hasher.Sum(nil)
+
+	return hash
+}

pkg/wellknownports/wellknownports.go

@@ -20,6 +20,9 @@ const (
 	// KubeAPIServer is the port where kube-apiserver listens.
 	KubeAPIServer = 443
 
+	// NodeupChallenge is the port where nodeup listens for challenges.
+	NodeupChallenge = 3987
+
 	// KopsControllerPort is the port where kops-controller listens.
 	KopsControllerPort = 3988
 

proto/buf.gen.yaml

@@ -0,0 +1,8 @@
+version: v1
+plugins:
+  - plugin: go
+    out: .
+    opt: paths=source_relative
+  - plugin: go-grpc
+    out: .
+    opt: paths=source_relative

proto/buf.yaml

@@ -0,0 +1,7 @@
+version: v1
+breaking:
+  use:
+    - FILE
+lint:
+  use:
+    - DEFAULT

proto/kops/bootstrap/v1/bootstrap.proto

@@ -0,0 +1,28 @@
+syntax = "proto3";
+
+package kops.bootstrap.v1;
+
+option go_package = "k8s.io/kops/proto/kops/bootstrap/v1";
+
+// CallbackService is the service that the node must run during bootstrapping,
+// we perform a simple callback from the controller.
+service CallbackService {
+  // Answers challenges to cross-check bootstrap requests.
+  rpc Challenge(ChallengeRequest) returns (ChallengeResponse) {}
+} 
+
+message ChallengeRequest {
+  // challenge_id is a random value (nonce) that the node generated and passed in its bootstrap request.
+  string challenge_id = 1;
+
+  // challenge_random is a random value (nonce) that kops-controller generated, it is hashed into the response.
+  bytes challenge_random = 2;
+}
+
+message ChallengeResponse {
+  // challenge_response combines the node nonce with the kops-controller nonce.
+  // The node nonce is passed in the bootstrap request.
+  // The kops-controller nonce is challenge_random in the ChallengeRequest.
+  // challenge_response is expected to be sha256(node-node + kops-controller-nonce)
+  bytes challenge_response = 1;
+}

upup/pkg/fi/cloudup/awsup/aws_authenticator.go

@@ -71,7 +71,7 @@ func NewAWSAuthenticator(region string) (bootstrap.Authenticator, error) {
 	}, nil
 }
 
-func (a awsAuthenticator) CreateToken(body []byte) (string, error) {
+func (a *awsAuthenticator) CreateToken(body []byte) (string, error) {
 	sha := sha256.Sum256(body)
 
 	stsRequest, _ := a.sts.GetCallerIdentityRequest(nil)

upup/pkg/fi/cloudup/awsup/aws_verifier.go

@@ -38,6 +38,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/sts"
 	"k8s.io/kops/pkg/bootstrap"
 	nodeidentityaws "k8s.io/kops/pkg/nodeidentity/aws"
+	"k8s.io/kops/pkg/wellknownports"
 )
 
 type AWSVerifierOptions struct {
@@ -237,9 +238,32 @@ func (a awsVerifier) VerifyToken(ctx context.Context, rawRequest *http.Request,
 		return nil, err
 	}
 
+	var challengeEndpoints []string
+	for _, nic := range instance.NetworkInterfaces {
+		if ip := aws.StringValue(nic.PrivateIpAddress); ip != "" {
+			challengeEndpoints = append(challengeEndpoints, net.JoinHostPort(ip, strconv.Itoa(wellknownports.NodeupChallenge)))
+		}
+		for _, a := range nic.PrivateIpAddresses {
+			if ip := aws.StringValue(a.PrivateIpAddress); ip != "" {
+				challengeEndpoints = append(challengeEndpoints, net.JoinHostPort(ip, strconv.Itoa(wellknownports.NodeupChallenge)))
+			}
+		}
+
+		for _, a := range nic.Ipv6Addresses {
+			if ip := aws.StringValue(a.Ipv6Address); ip != "" {
+				challengeEndpoints = append(challengeEndpoints, net.JoinHostPort(ip, strconv.Itoa(wellknownports.NodeupChallenge)))
+			}
+		}
+	}
+
+	if len(challengeEndpoints) == 0 {
+		return nil, fmt.Errorf("cannot determine challenge endpoint for instance id: %s", instanceID)
+	}
+
 	result := &bootstrap.VerifyResult{
-		NodeName:         addrs[0],
-		CertificateNames: addrs,
+		NodeName:          addrs[0],
+		CertificateNames:  addrs,
+		ChallengeEndpoint: challengeEndpoints[0],
 	}
 
 	for _, tag := range instance.Tags {

upup/pkg/fi/cloudup/gce/tpm/gcetpmverifier/tpmverifier.go

@@ -29,6 +29,7 @@ import (
 	"fmt"
 	"math"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
@@ -36,6 +37,7 @@ import (
 	"google.golang.org/api/googleapi"
 	"k8s.io/kops/pkg/bootstrap"
 	"k8s.io/kops/pkg/nodeidentity/gce"
+	"k8s.io/kops/pkg/wellknownports"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/gce/gcemetadata"
 	gcetpm "k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm"
@@ -171,10 +173,13 @@ func (v *tpmVerifier) VerifyToken(ctx context.Context, rawRequest *http.Request,
 		return nil, err
 	}
 
+	challengeEndpoint := instance.NetworkInterfaces[0].NetworkIP + ":" + strconv.Itoa(wellknownports.NodeupChallenge)
+
 	result := &bootstrap.VerifyResult{
 		NodeName:          instance.Name,
 		InstanceGroupName: instanceGroupName,
 		CertificateNames:  sans,
+		ChallengeEndpoint: challengeEndpoint,
 	}
 
 	return result, nil

upup/pkg/fi/cloudup/hetzner/authenticator.go

@@ -35,11 +35,10 @@ func NewHetznerAuthenticator() (bootstrap.Authenticator, error) {
 	return &hetznerAuthenticator{}, nil
 }
 
-func (h hetznerAuthenticator) CreateToken(body []byte) (string, error) {
+func (h *hetznerAuthenticator) CreateToken(body []byte) (string, error) {
 	serverID, err := metadata.NewClient().InstanceID()
 	if err != nil {
 		return "", fmt.Errorf("failed to retrieve server ID: %w", err)
 	}
-
 	return HetznerAuthenticationTokenPrefix + strconv.Itoa(serverID), nil
 }

upup/pkg/fi/cloudup/hetzner/verifier.go

@@ -19,6 +19,7 @@ package hetzner
 import (
 	"context"
 	"fmt"
+	"net"
 	"net/http"
 	"os"
 	"strconv"
@@ -26,6 +27,7 @@ import (
 
 	"github.com/hetznercloud/hcloud-go/hcloud"
 	"k8s.io/kops/pkg/bootstrap"
+	"k8s.io/kops/pkg/wellknownports"
 )
 
 type HetznerVerifierOptions struct {
@@ -71,18 +73,26 @@ func (h hetznerVerifier) VerifyToken(ctx context.Context, rawRequest *http.Reque
 	}
 
 	var addrs []string
+	var challengeEndpoints []string
 	if server.PublicNet.IPv4.IP != nil {
+		// Don't challenge over the public network
 		addrs = append(addrs, server.PublicNet.IPv4.IP.String())
 	}
 	for _, network := range server.PrivateNet {
 		if network.IP != nil {
 			addrs = append(addrs, network.IP.String())
+			challengeEndpoints = append(challengeEndpoints, net.JoinHostPort(network.IP.String(), strconv.Itoa(wellknownports.NodeupChallenge)))
 		}
 	}
 
+	if len(challengeEndpoints) == 0 {
+		return nil, fmt.Errorf("cannot determine challenge endpoint for server %q", serverID)
+	}
+
 	result := &bootstrap.VerifyResult{
-		NodeName:         server.Name,
-		CertificateNames: addrs,
+		NodeName:          server.Name,
+		CertificateNames:  addrs,
+		ChallengeEndpoint: challengeEndpoints[0],
 	}
 
 	for key, value := range server.Labels {

upup/pkg/fi/cloudup/openstack/authenticator.go

@@ -33,7 +33,7 @@ func NewOpenstackAuthenticator() (bootstrap.Authenticator, error) {
 	return &openstackAuthenticator{}, nil
 }
 
-func (o openstackAuthenticator) CreateToken(body []byte) (string, error) {
+func (o *openstackAuthenticator) CreateToken(body []byte) (string, error) {
 	metadata, err := GetLocalMetadata()
 	if err != nil {
 		return "", fmt.Errorf("unable to fetch metadata: %w", err)

upup/pkg/fi/cloudup/openstack/verifier.go

@@ -22,6 +22,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
 
 	"github.com/gophercloud/gophercloud"
@@ -35,6 +36,7 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
 	"k8s.io/kops/pkg/bootstrap"
+	"k8s.io/kops/pkg/wellknownports"
 )
 
 type OpenStackVerifierOptions struct {
@@ -151,6 +153,9 @@ func (o openstackVerifier) VerifyToken(ctx context.Context, rawRequest *http.Req
 		return nil, fmt.Errorf("authentication request address %q does not match server addresses %v", requestAddr, addrs)
 	}
 
+	// We will call back onto this address, now that we have verified it is an instance IP
+	challengeEndpoint := net.JoinHostPort(requestAddr, strconv.Itoa(wellknownports.NodeupChallenge))
+
 	// check from kubernetes API does the instance already exist
 	_, err = o.kubeClient.CoreV1().Nodes().Get(ctx, instance.Name, v1.GetOptions{})
 	if err == nil {
@@ -161,8 +166,9 @@ func (o openstackVerifier) VerifyToken(ctx context.Context, rawRequest *http.Req
 	}
 
 	result := &bootstrap.VerifyResult{
-		NodeName:         instance.Name,
-		CertificateNames: addrs,
+		NodeName:          instance.Name,
+		CertificateNames:  addrs,
+		ChallengeEndpoint: challengeEndpoint,
 	}
 	value, ok := instance.Metadata[TagKopsInstanceGroup]
 	if ok {

upup/pkg/fi/nodeup/command.go

@@ -42,6 +42,7 @@ import (
 	"k8s.io/kops/nodeup/pkg/model"
 	"k8s.io/kops/nodeup/pkg/model/networking"
 	api "k8s.io/kops/pkg/apis/kops"
+	kopsmodel "k8s.io/kops/pkg/apis/kops/model"
 	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/pkg/apis/nodeup"
 	"k8s.io/kops/pkg/assets"
@@ -50,6 +51,7 @@ import (
 	"k8s.io/kops/pkg/kopscodecs"
 	"k8s.io/kops/pkg/kopscontrollerclient"
 	"k8s.io/kops/pkg/resolver"
+	"k8s.io/kops/pkg/wellknownports"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	"k8s.io/kops/upup/pkg/fi/cloudup/gce/gcediscovery"
@@ -763,6 +765,23 @@ func getNodeConfigFromServers(ctx context.Context, bootConfig *nodeup.BootConfig
 		return nil, fmt.Errorf("unsupported cloud provider for node configuration %s", bootConfig.CloudProvider)
 	}
 
+	var challengeListener *bootstrap.ChallengeListener
+
+	if kopsmodel.UseChallengeCallback() {
+		challengeServer, err := bootstrap.NewChallengeServer(bootConfig.ClusterName, []byte(bootConfig.ConfigServer.CACertificates))
+		if err != nil {
+			return nil, err
+		}
+		listen := ":" + strconv.Itoa(wellknownports.NodeupChallenge)
+
+		l, err := challengeServer.NewListener(ctx, listen)
+		if err != nil {
+			return nil, fmt.Errorf("error starting challenge listener: %w", err)
+		}
+		challengeListener = l
+		defer challengeListener.Stop()
+	}
+
 	client := &kopscontrollerclient.Client{
 		Authenticator: authenticator,
 		Resolver:      resolver,
@@ -782,6 +801,11 @@ func getNodeConfigFromServers(ctx context.Context, bootConfig *nodeup.BootConfig
 			APIVersion:        nodeup.BootstrapAPIVersion,
 			IncludeNodeConfig: true,
 		}
+
+		if challengeListener != nil {
+			request.Challenge = challengeListener.CreateChallenge()
+		}
+
 		var resp nodeup.BootstrapResponse
 		err = client.Query(ctx, &request, &resp)
 		if err != nil {

upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go

@@ -17,14 +17,16 @@ limitations under the License.
 package nodetasks
 
 import (
-	"context"
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
+	"strconv"
 
 	"k8s.io/kops/pkg/apis/nodeup"
+	"k8s.io/kops/pkg/bootstrap"
 	"k8s.io/kops/pkg/kopscontrollerclient"
 	"k8s.io/kops/pkg/pki"
+	"k8s.io/kops/pkg/wellknownports"
 	"k8s.io/kops/upup/pkg/fi"
 )
 
@@ -37,6 +39,12 @@ type BootstrapClientTask struct {
 	// Client holds the client wrapper for the kops-bootstrap protocol
 	Client *kopscontrollerclient.Client
 
+	// UseChallengeCallback is true if we should run a challenge responder during the request.
+	UseChallengeCallback bool
+
+	// ClusterName is the name of the cluster
+	ClusterName string
+
 	keys map[string]*pki.PrivateKey
 }
 
@@ -76,7 +84,7 @@ func (b *BootstrapClientTask) String() string {
 }
 
 func (b *BootstrapClientTask) Run(c *fi.NodeupContext) error {
-	ctx := context.TODO()
+	ctx := c.Context()
 
 	req := nodeup.BootstrapRequest{
 		APIVersion: nodeup.BootstrapAPIVersion,
@@ -84,6 +92,24 @@ func (b *BootstrapClientTask) Run(c *fi.NodeupContext) error {
 		KeypairIDs: b.KeypairIDs,
 	}
 
+	var challengeServer *bootstrap.ChallengeServer
+	if b.UseChallengeCallback {
+		s, err := bootstrap.NewChallengeServer(b.ClusterName, b.Client.CAs)
+		if err != nil {
+			return err
+		}
+		challengeServer = s
+		listen := ":" + strconv.Itoa(wellknownports.NodeupChallenge)
+
+		listener, err := challengeServer.NewListener(ctx, listen)
+		if err != nil {
+			return fmt.Errorf("error starting challenge listener: %w", err)
+		}
+		defer listener.Stop()
+
+		req.Challenge = listener.CreateChallenge()
+	}
+
 	if b.keys == nil {
 		b.keys = map[string]*pki.PrivateKey{}
 	}