feat: Support PKI bootstrap

Similar to the TPM bootstrapping on GCE (indeed, a lot of the code is
modified from there), but we verify the PKI signature against a public
key in a Host CRD object.

cmd/kops-controller/main.go

@@ -32,7 +32,9 @@ import (
 	"k8s.io/kops/cmd/kops-controller/controllers"
 	"k8s.io/kops/cmd/kops-controller/pkg/config"
 	"k8s.io/kops/cmd/kops-controller/pkg/server"
+	"k8s.io/kops/pkg/apis/kops/v1alpha2"
 	"k8s.io/kops/pkg/bootstrap"
+	"k8s.io/kops/pkg/bootstrap/pkibootstrap"
 	"k8s.io/kops/pkg/nodeidentity"
 	nodeidentityaws "k8s.io/kops/pkg/nodeidentity/aws"
 	nodeidentityazure "k8s.io/kops/pkg/nodeidentity/azure"
@@ -58,7 +60,6 @@ import (
 )
 
 var (
-	scheme   = runtime.NewScheme()
 	setupLog = ctrl.Log.WithName("setup")
 )
 
@@ -100,7 +101,8 @@ func main() {
 
 	ctrl.SetLogger(klogr.New())
 
-	if err := buildScheme(); err != nil {
+	scheme, err := buildScheme()
+	if err != nil {
 		setupLog.Error(err, "error building scheme")
 		os.Exit(1)
 	}
@@ -108,6 +110,7 @@ func main() {
 	kubeConfig := ctrl.GetConfigOrDie()
 	kubeConfig.Burst = 200
 	kubeConfig.QPS = 100
+
 	mgr, err := ctrl.NewManager(kubeConfig, ctrl.Options{
 		Scheme: scheme,
 		Metrics: metricsserver.Options{
@@ -183,6 +186,15 @@ func main() {
 			verifiers = append(verifiers, verifier)
 		}
 
+		if opt.Server.PKI != nil {
+			verifier, err := pkibootstrap.NewVerifier(opt.Server.PKI, mgr.GetClient())
+			if err != nil {
+				setupLog.Error(err, "unable to create verifier")
+				os.Exit(1)
+			}
+			verifiers = append(verifiers, verifier)
+		}
+
 		if len(verifiers) == 0 {
 			klog.Fatalf("server verifiers not provided")
 		}
@@ -233,15 +245,19 @@ func main() {
 	}
 }
 
-func buildScheme() error {
+func buildScheme() (*runtime.Scheme, error) {
+	scheme := runtime.NewScheme()
 	if err := corev1.AddToScheme(scheme); err != nil {
-		return fmt.Errorf("error registering corev1: %v", err)
+		return nil, fmt.Errorf("error registering corev1: %v", err)
+	}
+	if err := v1alpha2.AddToScheme(scheme); err != nil {
+		return nil, fmt.Errorf("error registering kops/v1alpha2 API: %v", err)
 	}
 	// Needed so that the leader-election system can post events
 	if err := coordinationv1.AddToScheme(scheme); err != nil {
-		return fmt.Errorf("error registering coordinationv1: %v", err)
+		return nil, fmt.Errorf("error registering coordinationv1: %v", err)
 	}
-	return nil
+	return scheme, nil
 }
 
 func addNodeController(mgr manager.Manager, vfsContext *vfs.VFSContext, opt *config.Options) error {

cmd/kops-controller/pkg/config/options.go

@@ -17,6 +17,7 @@ limitations under the License.
 package config
 
 import (
+	"k8s.io/kops/pkg/bootstrap/pkibootstrap"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	"k8s.io/kops/upup/pkg/fi/cloudup/azure"
 	"k8s.io/kops/upup/pkg/fi/cloudup/do"
@@ -51,6 +52,9 @@ type ServerOptions struct {
 	// Provider is the cloud provider.
 	Provider ServerProviderOptions `json:"provider"`
 
+	// PKI configures private/public key node authentication.
+	PKI *pkibootstrap.Options `json:"pki,omitempty"`
+
 	// ServerKeyPath is the path to our TLS serving private key.
 	ServerKeyPath string `json:"serverKeyPath,omitempty"`
 	// ServerCertificatePath is the path to our TLS serving certificate.

nodeup/pkg/model/bootstrap_client.go

@@ -24,6 +24,7 @@ import (
 
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/bootstrap"
+	"k8s.io/kops/pkg/bootstrap/pkibootstrap"
 	"k8s.io/kops/pkg/kopscontrollerclient"
 	"k8s.io/kops/pkg/resolver"
 	"k8s.io/kops/pkg/wellknownports"
@@ -101,6 +102,13 @@ func (b BootstrapClientBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 		}
 		authenticator = a
 
+	case "metal":
+		a, err := pkibootstrap.NewAuthenticatorFromFile("/etc/kubernetes/kops/pki/machine/private.pem")
+		if err != nil {
+			return err
+		}
+		authenticator = a
+
 	default:
 		return fmt.Errorf("unsupported cloud provider for authenticator %q", b.CloudProvider())
 	}

pkg/bootstrap/pkibootstrap/options.go

@@ -0,0 +1,26 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pkibootstrap
+
+// Options describes how we authenticate instances with known-public-key authentication.
+type Options struct {
+	// MaxTimeSkew is the maximum time skew to allow (in seconds)
+	MaxTimeSkew int64 `json:"MaxTimeSkew,omitempty"`
+}
+
+// AuthenticationTokenPrefix is the prefix used for authentication using PKI
+const AuthenticationTokenPrefix = "x-pki-tpm "

pkg/bootstrap/pkibootstrap/pkisigner.go

@@ -0,0 +1,151 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pkibootstrap
+
+import (
+	"bytes"
+	"crypto"
+	cryptorand "crypto/rand"
+	"crypto/sha256"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"os"
+	"time"
+
+	"k8s.io/klog/v2"
+	"k8s.io/kops/pkg/bootstrap"
+	"k8s.io/kops/pkg/pki"
+)
+
+type pkiAuthenticator struct {
+	signer   crypto.Signer
+	keyID    string
+	hostname string
+}
+
+// AuthTokenData is the code data that is signed as part of the header.
+type AuthTokenData struct {
+	// Instance is the name/id of the instance we are claiming
+	Instance string `json:"instance,omitempty"`
+
+	// KeyID is the identifier of the public key we are signing with, if we're using a fixed key.
+	KeyID string `json:"keyID,omitempty"`
+
+	// RequestHash is the hash of the request
+	RequestHash []byte `json:"requestHash,omitempty"`
+
+	// Timestamp is the time of this request (to help prevent replay attacks)
+	Timestamp int64 `json:"timestamp,omitempty"`
+
+	// Audience is the audience for this request (to help prevent replay attacks)
+	Audience string `json:"audience,omitempty"`
+}
+
+var _ bootstrap.Authenticator = &pkiAuthenticator{}
+
+func NewAuthenticator(hostname string, signer crypto.Signer) (bootstrap.Authenticator, error) {
+	keyID, err := computeKeyID(signer)
+	if err != nil {
+		return nil, err
+	}
+
+	return &pkiAuthenticator{hostname: hostname, signer: signer, keyID: keyID}, nil
+}
+
+func computeKeyID(signer crypto.Signer) (string, error) {
+	publicKey := signer.Public()
+	pkData, err := x509.MarshalPKIXPublicKey(publicKey)
+	if err != nil {
+		return "", fmt.Errorf("error converting public key to x509: %w", err)
+	}
+
+	var b bytes.Buffer
+	if err := pem.Encode(&b, &pem.Block{Type: "PUBLIC KEY", Bytes: pkData}); err != nil {
+		return "", fmt.Errorf("error encoding public key: %w", err)
+	}
+	return b.String(), nil
+}
+
+func NewAuthenticatorFromFile(p string) (bootstrap.Authenticator, error) {
+	hostname, err := os.Hostname()
+	if err != nil {
+		return nil, fmt.Errorf("couldn't determine hostname: %w", err)
+	}
+
+	keyBytes, err := os.ReadFile(p)
+	if err != nil {
+		return nil, fmt.Errorf("error reading %q: %w", p, err)
+	}
+	key, err := pki.ParsePEMPrivateKey(keyBytes)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing key from %q: %w", p, err)
+	}
+
+	return NewAuthenticator(hostname, key.Key)
+}
+
+func (a *pkiAuthenticator) CreateToken(body []byte) (string, error) {
+	requestHash := sha256.Sum256(body)
+
+	data := AuthTokenData{
+		Timestamp:   time.Now().Unix(),
+		Audience:    AudienceNodeAuthentication,
+		RequestHash: requestHash[:],
+
+		KeyID:    a.keyID,
+		Instance: a.hostname,
+	}
+
+	payload, err := json.Marshal(&data)
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal token data: %w", err)
+	}
+
+	signature, err := a.sign(payload)
+	if err != nil {
+		return "", fmt.Errorf("failed to sign token data: %w", err)
+	}
+	token := &AuthToken{
+		Data:      payload,
+		Signature: signature,
+	}
+
+	b, err := json.Marshal(token)
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal token: %w", err)
+	}
+	return AuthenticationTokenPrefix + base64.StdEncoding.EncodeToString(b), nil
+}
+
+// sign performs a TPM signature with the tpmKey, and sanity checks the result.
+func (a *pkiAuthenticator) sign(payload []byte) ([]byte, error) {
+	beforeSign := time.Now()
+
+	digest := sha256.Sum256(payload)
+
+	signature, err := a.signer.Sign(cryptorand.Reader, digest[:], crypto.SHA256)
+	if err != nil {
+		return nil, fmt.Errorf("failed to sign data: %w", err)
+	}
+
+	klog.Infof("signing took %v", time.Since(beforeSign))
+
+	return signature, nil
+}

pkg/bootstrap/pkibootstrap/pkiverifier.go

@@ -0,0 +1,179 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pkibootstrap
+
+import (
+	"bytes"
+	"context"
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"math"
+	"net/http"
+	"strings"
+	"time"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/klog/v2"
+	kops "k8s.io/kops/pkg/apis/kops/v1alpha2"
+	"k8s.io/kops/pkg/bootstrap"
+	"k8s.io/kops/pkg/pki"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type verifier struct {
+	opt    Options
+	client client.Client
+}
+
+// NewVerifier constructs a new verifier.
+func NewVerifier(options *Options, client client.Client) (bootstrap.Verifier, error) {
+	opt := *options
+	if opt.MaxTimeSkew == 0 {
+		opt.MaxTimeSkew = 300
+	}
+	return &verifier{
+		opt:    opt,
+		client: client,
+	}, nil
+}
+
+var _ bootstrap.Verifier = &verifier{}
+
+// TODO: Dedup with gce
+func (v *verifier) parseTokenData(tokenPrefix string, authToken string, body []byte) (*AuthToken, *AuthTokenData, error) {
+	if !strings.HasPrefix(authToken, tokenPrefix) {
+		return nil, nil, fmt.Errorf("incorrect authorization type")
+	}
+	authToken = strings.TrimPrefix(authToken, tokenPrefix)
+
+	tokenBytes, err := base64.StdEncoding.DecodeString(authToken)
+	if err != nil {
+		return nil, nil, fmt.Errorf("decoding authorization token: %w", err)
+	}
+
+	token := &AuthToken{}
+	if err = json.Unmarshal(tokenBytes, token); err != nil {
+		return nil, nil, fmt.Errorf("unmarshalling authorization token: %w", err)
+	}
+
+	tokenData := &AuthTokenData{}
+	if err := json.Unmarshal(token.Data, tokenData); err != nil {
+		return nil, nil, fmt.Errorf("unmarshalling authorization token data: %w", err)
+	}
+
+	// Guard against replay attacks
+	if tokenData.Audience != AudienceNodeAuthentication {
+		return nil, nil, fmt.Errorf("incorrect Audience")
+	}
+	timeSkew := math.Abs(time.Since(time.Unix(tokenData.Timestamp, 0)).Seconds())
+	if timeSkew > float64(v.opt.MaxTimeSkew) {
+		return nil, nil, fmt.Errorf("incorrect Timestamp %v", tokenData.Timestamp)
+	}
+
+	// Verify the token has signed the body content.
+	requestHash := sha256.Sum256(body)
+	if !bytes.Equal(requestHash[:], tokenData.RequestHash) {
+		return nil, nil, fmt.Errorf("incorrect RequestHash")
+	}
+
+	return token, tokenData, nil
+}
+
+// Can generate keys with
+// openssl ecparam -name prime256v1 -genkey -noout -out ec-priv-key.pem
+// openssl ec -in ec-priv-key.pem -pubout > ec-pub-key.pem
+// Note that golang doesn't support secp256k1: https://groups.google.com/g/golang-nuts/c/Mbkug5t3ZYA
+
+func (v *verifier) VerifyToken(ctx context.Context, rawRequest *http.Request, authToken string, body []byte) (*bootstrap.VerifyResult, error) {
+	// Reminder: we shouldn't trust any data we get from the client until we've checked the signature (and even then...)
+	// Thankfully the GCE SDK does seem to escape the parameters correctly, for example.
+
+	token, tokenData, err := v.parseTokenData(AuthenticationTokenPrefix, authToken, body)
+	if err != nil {
+		return nil, err
+	}
+
+	// Verify the token has a valid signature.
+	result, signingKey, err := v.getSigningKey(ctx, tokenData)
+	if err != nil {
+		return nil, err
+	}
+
+	if !verifySignature(signingKey, token.Data, token.Signature) {
+		return nil, fmt.Errorf("failed to verify claim signature for node")
+	}
+
+	return result, nil
+}
+
+func (v *verifier) getSigningKey(ctx context.Context, tokenData *AuthTokenData) (*bootstrap.VerifyResult, crypto.PublicKey, error) {
+	nodeName := tokenData.Instance
+	id := types.NamespacedName{
+		Namespace: "kops-system",
+		Name:      nodeName,
+	}
+	var host kops.Host
+	if err := v.client.Get(ctx, id, &host); err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil, nil, fmt.Errorf("host not found for %v", id)
+		}
+		return nil, nil, fmt.Errorf("error getting host %v: %w", id, err)
+	}
+
+	// TODO: Check instance-group matches request (does it matter?)
+
+	if host.Spec.PublicKey == "" {
+		return nil, nil, fmt.Errorf("host %v did not have public-key", id)
+	}
+	instanceGroup := host.Spec.InstanceGroup
+	if instanceGroup == "" {
+		return nil, nil, fmt.Errorf("host %v did not have spec.instanceGroup", id)
+	}
+	pubKey, err := pki.ParsePEMPublicKey([]byte(host.Spec.PublicKey))
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to parse public key: %w", err)
+	}
+
+	var sans []string
+
+	result := &bootstrap.VerifyResult{
+		NodeName:          nodeName,
+		InstanceGroupName: instanceGroup,
+		CertificateNames:  sans,
+	}
+
+	return result, pubKey.Key, nil
+}
+
+func verifySignature(signingKey crypto.PublicKey, payload []byte, signature []byte) bool {
+	attestHash := sha256.Sum256(payload)
+	switch signingKey := signingKey.(type) {
+	case *ecdsa.PublicKey:
+		klog.Infof("attestHash %x", attestHash)
+		klog.Infof("sig %x", signature)
+		return ecdsa.VerifyASN1(signingKey, attestHash[:], signature)
+
+	default:
+		klog.Warningf("key type %T not supported", signingKey)
+		return false
+	}
+}

pkg/bootstrap/pkibootstrap/token.go

@@ -0,0 +1,30 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pkibootstrap
+
+// AuthToken describes the authentication header data when using GCE TPM authentication.
+type AuthToken struct {
+	// Signature is the TPM or PKI signature for data
+	Signature []byte `json:"signature,omitempty"`
+
+	// Data is the data we are signing.
+	// It is a JSON encoded form of AuthTokenData.
+	Data []byte `json:"data,omitempty"`
+}
+
+// AudienceNodeAuthentication is used in case we have multiple audiences using the TPM in future
+const AudienceNodeAuthentication = "kops.k8s.io/node-bootstrap"

upup/pkg/fi/nodeup/command.go

@@ -45,6 +45,7 @@ import (
 	"k8s.io/kops/pkg/apis/nodeup"
 	"k8s.io/kops/pkg/assets"
 	"k8s.io/kops/pkg/bootstrap"
+	"k8s.io/kops/pkg/bootstrap/pkibootstrap"
 	"k8s.io/kops/pkg/configserver"
 	"k8s.io/kops/pkg/kopscontrollerclient"
 	"k8s.io/kops/pkg/resolver"
@@ -654,6 +655,14 @@ func getNodeConfigFromServers(ctx context.Context, bootConfig *nodeup.BootConfig
 			return nil, err
 		}
 		authenticator = a
+
+	case "metal":
+		a, err := pkibootstrap.NewAuthenticatorFromFile("/etc/kubernetes/kops/pki/machine/private.pem")
+		if err != nil {
+			return nil, err
+		}
+		authenticator = a
+
 	default:
 		return nil, fmt.Errorf("unsupported cloud provider for node configuration %s", bootConfig.CloudProvider)
 	}