diff --git a/cmd/kops-controller/main.go b/cmd/kops-controller/main.go index d1ba60b9e8..846a8d8d3a 100644 --- a/cmd/kops-controller/main.go +++ b/cmd/kops-controller/main.go @@ -32,7 +32,9 @@ import ( "k8s.io/kops/cmd/kops-controller/controllers" "k8s.io/kops/cmd/kops-controller/pkg/config" "k8s.io/kops/cmd/kops-controller/pkg/server" + "k8s.io/kops/pkg/apis/kops/v1alpha2" "k8s.io/kops/pkg/bootstrap" + "k8s.io/kops/pkg/bootstrap/pkibootstrap" "k8s.io/kops/pkg/nodeidentity" nodeidentityaws "k8s.io/kops/pkg/nodeidentity/aws" nodeidentityazure "k8s.io/kops/pkg/nodeidentity/azure" @@ -58,7 +60,6 @@ import ( ) var ( - scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("setup") ) @@ -100,7 +101,8 @@ func main() { ctrl.SetLogger(klogr.New()) - if err := buildScheme(); err != nil { + scheme, err := buildScheme() + if err != nil { setupLog.Error(err, "error building scheme") os.Exit(1) } @@ -108,6 +110,7 @@ func main() { kubeConfig := ctrl.GetConfigOrDie() kubeConfig.Burst = 200 kubeConfig.QPS = 100 + mgr, err := ctrl.NewManager(kubeConfig, ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{ @@ -183,6 +186,15 @@ func main() { verifiers = append(verifiers, verifier) } + if opt.Server.PKI != nil { + verifier, err := pkibootstrap.NewVerifier(opt.Server.PKI, mgr.GetClient()) + if err != nil { + setupLog.Error(err, "unable to create verifier") + os.Exit(1) + } + verifiers = append(verifiers, verifier) + } + if len(verifiers) == 0 { klog.Fatalf("server verifiers not provided") } @@ -233,15 +245,19 @@ func main() { } } -func buildScheme() error { +func buildScheme() (*runtime.Scheme, error) { + scheme := runtime.NewScheme() if err := corev1.AddToScheme(scheme); err != nil { - return fmt.Errorf("error registering corev1: %v", err) + return nil, fmt.Errorf("error registering corev1: %v", err) + } + if err := v1alpha2.AddToScheme(scheme); err != nil { + return nil, fmt.Errorf("error registering kops/v1alpha2 API: %v", err) } // Needed so that the leader-election system can post events if err := coordinationv1.AddToScheme(scheme); err != nil { - return fmt.Errorf("error registering coordinationv1: %v", err) + return nil, fmt.Errorf("error registering coordinationv1: %v", err) } - return nil + return scheme, nil } func addNodeController(mgr manager.Manager, vfsContext *vfs.VFSContext, opt *config.Options) error { diff --git a/cmd/kops-controller/pkg/config/options.go b/cmd/kops-controller/pkg/config/options.go index 39770c1231..9c6fb65d20 100644 --- a/cmd/kops-controller/pkg/config/options.go +++ b/cmd/kops-controller/pkg/config/options.go @@ -17,6 +17,7 @@ limitations under the License. package config import ( + "k8s.io/kops/pkg/bootstrap/pkibootstrap" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/upup/pkg/fi/cloudup/azure" "k8s.io/kops/upup/pkg/fi/cloudup/do" @@ -51,6 +52,9 @@ type ServerOptions struct { // Provider is the cloud provider. Provider ServerProviderOptions `json:"provider"` + // PKI configures private/public key node authentication. + PKI *pkibootstrap.Options `json:"pki,omitempty"` + // ServerKeyPath is the path to our TLS serving private key. ServerKeyPath string `json:"serverKeyPath,omitempty"` // ServerCertificatePath is the path to our TLS serving certificate. diff --git a/nodeup/pkg/model/bootstrap_client.go b/nodeup/pkg/model/bootstrap_client.go index 68ff62448c..031e55a966 100644 --- a/nodeup/pkg/model/bootstrap_client.go +++ b/nodeup/pkg/model/bootstrap_client.go @@ -24,6 +24,7 @@ import ( "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/bootstrap" + "k8s.io/kops/pkg/bootstrap/pkibootstrap" "k8s.io/kops/pkg/kopscontrollerclient" "k8s.io/kops/pkg/resolver" "k8s.io/kops/pkg/wellknownports" @@ -101,6 +102,13 @@ func (b BootstrapClientBuilder) Build(c *fi.NodeupModelBuilderContext) error { } authenticator = a + case "metal": + a, err := pkibootstrap.NewAuthenticatorFromFile("/etc/kubernetes/kops/pki/machine/private.pem") + if err != nil { + return err + } + authenticator = a + default: return fmt.Errorf("unsupported cloud provider for authenticator %q", b.CloudProvider()) } diff --git a/pkg/bootstrap/pkibootstrap/options.go b/pkg/bootstrap/pkibootstrap/options.go new file mode 100644 index 0000000000..718c8a9493 --- /dev/null +++ b/pkg/bootstrap/pkibootstrap/options.go @@ -0,0 +1,26 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pkibootstrap + +// Options describes how we authenticate instances with known-public-key authentication. +type Options struct { + // MaxTimeSkew is the maximum time skew to allow (in seconds) + MaxTimeSkew int64 `json:"MaxTimeSkew,omitempty"` +} + +// AuthenticationTokenPrefix is the prefix used for authentication using PKI +const AuthenticationTokenPrefix = "x-pki-tpm " diff --git a/pkg/bootstrap/pkibootstrap/pkisigner.go b/pkg/bootstrap/pkibootstrap/pkisigner.go new file mode 100644 index 0000000000..25d59d8191 --- /dev/null +++ b/pkg/bootstrap/pkibootstrap/pkisigner.go @@ -0,0 +1,151 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pkibootstrap + +import ( + "bytes" + "crypto" + cryptorand "crypto/rand" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "fmt" + "os" + "time" + + "k8s.io/klog/v2" + "k8s.io/kops/pkg/bootstrap" + "k8s.io/kops/pkg/pki" +) + +type pkiAuthenticator struct { + signer crypto.Signer + keyID string + hostname string +} + +// AuthTokenData is the code data that is signed as part of the header. +type AuthTokenData struct { + // Instance is the name/id of the instance we are claiming + Instance string `json:"instance,omitempty"` + + // KeyID is the identifier of the public key we are signing with, if we're using a fixed key. + KeyID string `json:"keyID,omitempty"` + + // RequestHash is the hash of the request + RequestHash []byte `json:"requestHash,omitempty"` + + // Timestamp is the time of this request (to help prevent replay attacks) + Timestamp int64 `json:"timestamp,omitempty"` + + // Audience is the audience for this request (to help prevent replay attacks) + Audience string `json:"audience,omitempty"` +} + +var _ bootstrap.Authenticator = &pkiAuthenticator{} + +func NewAuthenticator(hostname string, signer crypto.Signer) (bootstrap.Authenticator, error) { + keyID, err := computeKeyID(signer) + if err != nil { + return nil, err + } + + return &pkiAuthenticator{hostname: hostname, signer: signer, keyID: keyID}, nil +} + +func computeKeyID(signer crypto.Signer) (string, error) { + publicKey := signer.Public() + pkData, err := x509.MarshalPKIXPublicKey(publicKey) + if err != nil { + return "", fmt.Errorf("error converting public key to x509: %w", err) + } + + var b bytes.Buffer + if err := pem.Encode(&b, &pem.Block{Type: "PUBLIC KEY", Bytes: pkData}); err != nil { + return "", fmt.Errorf("error encoding public key: %w", err) + } + return b.String(), nil +} + +func NewAuthenticatorFromFile(p string) (bootstrap.Authenticator, error) { + hostname, err := os.Hostname() + if err != nil { + return nil, fmt.Errorf("couldn't determine hostname: %w", err) + } + + keyBytes, err := os.ReadFile(p) + if err != nil { + return nil, fmt.Errorf("error reading %q: %w", p, err) + } + key, err := pki.ParsePEMPrivateKey(keyBytes) + if err != nil { + return nil, fmt.Errorf("error parsing key from %q: %w", p, err) + } + + return NewAuthenticator(hostname, key.Key) +} + +func (a *pkiAuthenticator) CreateToken(body []byte) (string, error) { + requestHash := sha256.Sum256(body) + + data := AuthTokenData{ + Timestamp: time.Now().Unix(), + Audience: AudienceNodeAuthentication, + RequestHash: requestHash[:], + + KeyID: a.keyID, + Instance: a.hostname, + } + + payload, err := json.Marshal(&data) + if err != nil { + return "", fmt.Errorf("failed to marshal token data: %w", err) + } + + signature, err := a.sign(payload) + if err != nil { + return "", fmt.Errorf("failed to sign token data: %w", err) + } + token := &AuthToken{ + Data: payload, + Signature: signature, + } + + b, err := json.Marshal(token) + if err != nil { + return "", fmt.Errorf("failed to marshal token: %w", err) + } + return AuthenticationTokenPrefix + base64.StdEncoding.EncodeToString(b), nil +} + +// sign performs a TPM signature with the tpmKey, and sanity checks the result. +func (a *pkiAuthenticator) sign(payload []byte) ([]byte, error) { + beforeSign := time.Now() + + digest := sha256.Sum256(payload) + + signature, err := a.signer.Sign(cryptorand.Reader, digest[:], crypto.SHA256) + if err != nil { + return nil, fmt.Errorf("failed to sign data: %w", err) + } + + klog.Infof("signing took %v", time.Since(beforeSign)) + + return signature, nil +} diff --git a/pkg/bootstrap/pkibootstrap/pkiverifier.go b/pkg/bootstrap/pkibootstrap/pkiverifier.go new file mode 100644 index 0000000000..c12068e4b7 --- /dev/null +++ b/pkg/bootstrap/pkibootstrap/pkiverifier.go @@ -0,0 +1,179 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pkibootstrap + +import ( + "bytes" + "context" + "crypto" + "crypto/ecdsa" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "math" + "net/http" + "strings" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + kops "k8s.io/kops/pkg/apis/kops/v1alpha2" + "k8s.io/kops/pkg/bootstrap" + "k8s.io/kops/pkg/pki" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type verifier struct { + opt Options + client client.Client +} + +// NewVerifier constructs a new verifier. +func NewVerifier(options *Options, client client.Client) (bootstrap.Verifier, error) { + opt := *options + if opt.MaxTimeSkew == 0 { + opt.MaxTimeSkew = 300 + } + return &verifier{ + opt: opt, + client: client, + }, nil +} + +var _ bootstrap.Verifier = &verifier{} + +// TODO: Dedup with gce +func (v *verifier) parseTokenData(tokenPrefix string, authToken string, body []byte) (*AuthToken, *AuthTokenData, error) { + if !strings.HasPrefix(authToken, tokenPrefix) { + return nil, nil, fmt.Errorf("incorrect authorization type") + } + authToken = strings.TrimPrefix(authToken, tokenPrefix) + + tokenBytes, err := base64.StdEncoding.DecodeString(authToken) + if err != nil { + return nil, nil, fmt.Errorf("decoding authorization token: %w", err) + } + + token := &AuthToken{} + if err = json.Unmarshal(tokenBytes, token); err != nil { + return nil, nil, fmt.Errorf("unmarshalling authorization token: %w", err) + } + + tokenData := &AuthTokenData{} + if err := json.Unmarshal(token.Data, tokenData); err != nil { + return nil, nil, fmt.Errorf("unmarshalling authorization token data: %w", err) + } + + // Guard against replay attacks + if tokenData.Audience != AudienceNodeAuthentication { + return nil, nil, fmt.Errorf("incorrect Audience") + } + timeSkew := math.Abs(time.Since(time.Unix(tokenData.Timestamp, 0)).Seconds()) + if timeSkew > float64(v.opt.MaxTimeSkew) { + return nil, nil, fmt.Errorf("incorrect Timestamp %v", tokenData.Timestamp) + } + + // Verify the token has signed the body content. + requestHash := sha256.Sum256(body) + if !bytes.Equal(requestHash[:], tokenData.RequestHash) { + return nil, nil, fmt.Errorf("incorrect RequestHash") + } + + return token, tokenData, nil +} + +// Can generate keys with +// openssl ecparam -name prime256v1 -genkey -noout -out ec-priv-key.pem +// openssl ec -in ec-priv-key.pem -pubout > ec-pub-key.pem +// Note that golang doesn't support secp256k1: https://groups.google.com/g/golang-nuts/c/Mbkug5t3ZYA + +func (v *verifier) VerifyToken(ctx context.Context, rawRequest *http.Request, authToken string, body []byte) (*bootstrap.VerifyResult, error) { + // Reminder: we shouldn't trust any data we get from the client until we've checked the signature (and even then...) + // Thankfully the GCE SDK does seem to escape the parameters correctly, for example. + + token, tokenData, err := v.parseTokenData(AuthenticationTokenPrefix, authToken, body) + if err != nil { + return nil, err + } + + // Verify the token has a valid signature. + result, signingKey, err := v.getSigningKey(ctx, tokenData) + if err != nil { + return nil, err + } + + if !verifySignature(signingKey, token.Data, token.Signature) { + return nil, fmt.Errorf("failed to verify claim signature for node") + } + + return result, nil +} + +func (v *verifier) getSigningKey(ctx context.Context, tokenData *AuthTokenData) (*bootstrap.VerifyResult, crypto.PublicKey, error) { + nodeName := tokenData.Instance + id := types.NamespacedName{ + Namespace: "kops-system", + Name: nodeName, + } + var host kops.Host + if err := v.client.Get(ctx, id, &host); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil, fmt.Errorf("host not found for %v", id) + } + return nil, nil, fmt.Errorf("error getting host %v: %w", id, err) + } + + // TODO: Check instance-group matches request (does it matter?) + + if host.Spec.PublicKey == "" { + return nil, nil, fmt.Errorf("host %v did not have public-key", id) + } + instanceGroup := host.Spec.InstanceGroup + if instanceGroup == "" { + return nil, nil, fmt.Errorf("host %v did not have spec.instanceGroup", id) + } + pubKey, err := pki.ParsePEMPublicKey([]byte(host.Spec.PublicKey)) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse public key: %w", err) + } + + var sans []string + + result := &bootstrap.VerifyResult{ + NodeName: nodeName, + InstanceGroupName: instanceGroup, + CertificateNames: sans, + } + + return result, pubKey.Key, nil +} + +func verifySignature(signingKey crypto.PublicKey, payload []byte, signature []byte) bool { + attestHash := sha256.Sum256(payload) + switch signingKey := signingKey.(type) { + case *ecdsa.PublicKey: + klog.Infof("attestHash %x", attestHash) + klog.Infof("sig %x", signature) + return ecdsa.VerifyASN1(signingKey, attestHash[:], signature) + + default: + klog.Warningf("key type %T not supported", signingKey) + return false + } +} diff --git a/pkg/bootstrap/pkibootstrap/token.go b/pkg/bootstrap/pkibootstrap/token.go new file mode 100644 index 0000000000..277644b10b --- /dev/null +++ b/pkg/bootstrap/pkibootstrap/token.go @@ -0,0 +1,30 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pkibootstrap + +// AuthToken describes the authentication header data when using GCE TPM authentication. +type AuthToken struct { + // Signature is the TPM or PKI signature for data + Signature []byte `json:"signature,omitempty"` + + // Data is the data we are signing. + // It is a JSON encoded form of AuthTokenData. + Data []byte `json:"data,omitempty"` +} + +// AudienceNodeAuthentication is used in case we have multiple audiences using the TPM in future +const AudienceNodeAuthentication = "kops.k8s.io/node-bootstrap" diff --git a/upup/pkg/fi/nodeup/command.go b/upup/pkg/fi/nodeup/command.go index 1388cc33a7..c511f643c9 100644 --- a/upup/pkg/fi/nodeup/command.go +++ b/upup/pkg/fi/nodeup/command.go @@ -45,6 +45,7 @@ import ( "k8s.io/kops/pkg/apis/nodeup" "k8s.io/kops/pkg/assets" "k8s.io/kops/pkg/bootstrap" + "k8s.io/kops/pkg/bootstrap/pkibootstrap" "k8s.io/kops/pkg/configserver" "k8s.io/kops/pkg/kopscontrollerclient" "k8s.io/kops/pkg/resolver" @@ -654,6 +655,14 @@ func getNodeConfigFromServers(ctx context.Context, bootConfig *nodeup.BootConfig return nil, err } authenticator = a + + case "metal": + a, err := pkibootstrap.NewAuthenticatorFromFile("/etc/kubernetes/kops/pki/machine/private.pem") + if err != nil { + return nil, err + } + authenticator = a + default: return nil, fmt.Errorf("unsupported cloud provider for node configuration %s", bootConfig.CloudProvider) }