From bec273ebf1dec1f8db4eaabf30051a099a160c9a Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Sun, 2 Aug 2020 14:36:55 -0700 Subject: [PATCH] Implement signing of kubelet cert in kops-controller --- cmd/kops-controller/pkg/config/options.go | 5 ++ cmd/kops-controller/pkg/server/BUILD.bazel | 7 +- cmd/kops-controller/pkg/server/keystore.go | 76 ++++++++++++++++++++++ cmd/kops-controller/pkg/server/server.go | 76 ++++++++++++++++++++-- nodeup/pkg/model/kops_controller.go | 8 +++ pkg/apis/nodeup/bootstrap.go | 4 ++ pkg/pki/issue.go | 9 ++- upup/pkg/fi/cloudup/template_functions.go | 2 + 8 files changed, 180 insertions(+), 7 deletions(-) create mode 100644 cmd/kops-controller/pkg/server/keystore.go diff --git a/cmd/kops-controller/pkg/config/options.go b/cmd/kops-controller/pkg/config/options.go index cdb2b4d0e3..5a82a3bfbe 100644 --- a/cmd/kops-controller/pkg/config/options.go +++ b/cmd/kops-controller/pkg/config/options.go @@ -38,6 +38,11 @@ type ServerOptions struct { ServerKeyPath string `json:"serverKeyPath,omitempty"` // ServerCertificatePath is the path to our TLS serving certificate. ServerCertificatePath string `json:"serverCertificatePath,omitempty"` + + // CABasePath is a base of the path to CA certificate and key files. + CABasePath string `json:"caBasePath"` + // SigningCAs is the list of active signing CAs. + SigningCAs []string `json:"signingCAs"` } type ServerProviderOptions struct { diff --git a/cmd/kops-controller/pkg/server/BUILD.bazel b/cmd/kops-controller/pkg/server/BUILD.bazel index 8ba8961085..522bbaba93 100644 --- a/cmd/kops-controller/pkg/server/BUILD.bazel +++ b/cmd/kops-controller/pkg/server/BUILD.bazel @@ -2,12 +2,17 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", - srcs = ["server.go"], + srcs = [ + "keystore.go", + "server.go", + ], importpath = "k8s.io/kops/cmd/kops-controller/pkg/server", visibility = ["//visibility:public"], deps = [ "//cmd/kops-controller/pkg/config:go_default_library", "//pkg/apis/nodeup:go_default_library", + "//pkg/pki:go_default_library", + "//pkg/rbac:go_default_library", "//upup/pkg/fi:go_default_library", "//vendor/github.com/gorilla/mux:go_default_library", "//vendor/k8s.io/klog:go_default_library", diff --git a/cmd/kops-controller/pkg/server/keystore.go b/cmd/kops-controller/pkg/server/keystore.go new file mode 100644 index 0000000000..52dfb8a52b --- /dev/null +++ b/cmd/kops-controller/pkg/server/keystore.go @@ -0,0 +1,76 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "fmt" + "io/ioutil" + "path" + + "k8s.io/kops/pkg/pki" +) + +type keystore struct { + keys map[string]keystoreEntry +} + +type keystoreEntry struct { + certificate *pki.Certificate + key *pki.PrivateKey +} + +var _ pki.Keystore = keystore{} + +func (k keystore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) { + entry, ok := k.keys[name] + if !ok { + return nil, nil, false, fmt.Errorf("unknown CA %q", name) + } + return entry.certificate, entry.key, false, nil +} + +func newKeystore(basePath string, cas []string) (pki.Keystore, error) { + keystore := &keystore{ + keys: map[string]keystoreEntry{}, + } + for _, name := range cas { + certBytes, err := ioutil.ReadFile(path.Join(basePath, name+".pem")) + if err != nil { + return nil, fmt.Errorf("reading %q certificate: %v", name, err) + } + certificate, err := pki.ParsePEMCertificate(certBytes) + if err != nil { + return nil, fmt.Errorf("parsing %q certificate: %v", name, err) + } + + keyBytes, err := ioutil.ReadFile(path.Join(basePath, name+"-key.pem")) + if err != nil { + return nil, fmt.Errorf("reading %q key: %v", name, err) + } + key, err := pki.ParsePEMPrivateKey(keyBytes) + if err != nil { + return nil, fmt.Errorf("parsing %q key: %v", name, err) + } + + keystore.keys[name] = keystoreEntry{ + certificate: certificate, + key: key, + } + } + + return keystore, nil +} diff --git a/cmd/kops-controller/pkg/server/server.go b/cmd/kops-controller/pkg/server/server.go index 0210d5b96e..7a28de8ac3 100644 --- a/cmd/kops-controller/pkg/server/server.go +++ b/cmd/kops-controller/pkg/server/server.go @@ -18,16 +18,23 @@ package server import ( "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" "encoding/json" + "encoding/pem" "fmt" + "hash/fnv" "io/ioutil" "net/http" "runtime/debug" + "time" "github.com/gorilla/mux" "k8s.io/klog" "k8s.io/kops/cmd/kops-controller/pkg/config" "k8s.io/kops/pkg/apis/nodeup" + "k8s.io/kops/pkg/pki" + "k8s.io/kops/pkg/rbac" "k8s.io/kops/upup/pkg/fi" ) @@ -35,6 +42,7 @@ type Server struct { opt *config.Options server *http.Server verifier fi.Verifier + keystore pki.Keystore } func NewServer(opt *config.Options, verifier fi.Verifier) (*Server, error) { @@ -59,6 +67,12 @@ func NewServer(opt *config.Options, verifier fi.Verifier) (*Server, error) { } func (s *Server) Start() error { + var err error + s.keystore, err = newKeystore(s.opt.Server.CABasePath, s.opt.Server.SigningCAs) + if err != nil { + return err + } + return s.server.ListenAndServeTLS(s.opt.Server.ServerCertificatePath, s.opt.Server.ServerKeyPath) } @@ -85,8 +99,6 @@ func (s *Server) bootstrap(w http.ResponseWriter, r *http.Request) { return } - klog.Infof("id is %s", id.Instance) // todo do something with id - req := &nodeup.BootstrapRequest{} err = json.Unmarshal(body, req) if err != nil { @@ -103,10 +115,66 @@ func (s *Server) bootstrap(w http.ResponseWriter, r *http.Request) { return } + resp := &nodeup.BootstrapResponse{ + Certs: map[string]string{}, + } + + // Skew the certificate lifetime by up to 30 days based on information about the requesting node. + // This is so that different nodes created at the same time have the certificates they generated + // expire at different times, but all certificates on a given node expire around the same time. + hash := fnv.New32() + _, _ = hash.Write([]byte(r.RemoteAddr)) + validHours := (455 * 24) + (hash.Sum32() % (30 * 24)) + + for name, pubKey := range req.Certs { + cert, err := s.issueCert(name, pubKey, id, validHours) + if err != nil { + klog.Infof("bootstrap %s cert %q issue err: %v", r.RemoteAddr, name, err) + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(fmt.Sprintf("failed to issue %q: %v", name, err))) + return + } + resp.Certs[name] = cert + } + w.Header().Set("Content-Type", "application/json") - resp := &nodeup.BootstrapResponse{} _ = json.NewEncoder(w).Encode(resp) - klog.Infof("bootstrap %s success", r.RemoteAddr) + klog.Infof("bootstrap %s %s success", r.RemoteAddr, id.Instance) +} + +func (s *Server) issueCert(name string, pubKey string, id *fi.VerifyResult, validHours uint32) (string, error) { + block, _ := pem.Decode([]byte(pubKey)) + if block.Type != "RSA PUBLIC KEY" { + return "", fmt.Errorf("unexpected key type %q", block.Type) + } + key, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return "", fmt.Errorf("parsing key: %v", err) + } + + issueReq := &pki.IssueCertRequest{ + Signer: fi.CertificateIDCA, + Type: "client", + PublicKey: key, + Validity: time.Hour * time.Duration(validHours), + } + + switch name { + case "kubelet": + issueReq.Subject = pkix.Name{ + CommonName: fmt.Sprintf("system:node:%s", id.Instance), + Organization: []string{rbac.NodesGroup}, + } + default: + return "", fmt.Errorf("unexpected key name") + } + + cert, _, _, err := pki.IssueCert(issueReq, s.keystore) + if err != nil { + return "", fmt.Errorf("issuing certificate: %v", err) + } + + return cert.AsString() } // recovery is responsible for ensuring we don't exit on a panic. diff --git a/nodeup/pkg/model/kops_controller.go b/nodeup/pkg/model/kops_controller.go index 1a27af5e85..d8c6f4789e 100644 --- a/nodeup/pkg/model/kops_controller.go +++ b/nodeup/pkg/model/kops_controller.go @@ -81,5 +81,13 @@ func (b *KopsControllerBuilder) Build(c *fi.ModelBuilderContext) error { Owner: s(wellknownusers.KopsControllerName), }) + for _, cert := range []string{fi.CertificateIDCA} { + owner := wellknownusers.KopsControllerName + err := b.BuildCertificatePairTask(c, cert, pkiDir, cert, &owner) + if err != nil { + return err + } + } + return nil } diff --git a/pkg/apis/nodeup/bootstrap.go b/pkg/apis/nodeup/bootstrap.go index e2b8119b7b..095718233d 100644 --- a/pkg/apis/nodeup/bootstrap.go +++ b/pkg/apis/nodeup/bootstrap.go @@ -22,8 +22,12 @@ const BootstrapAPIVersion = "bootstrap.kops.k8s.io/v1alpha1" type BootstrapRequest struct { // APIVersion defines the versioned schema of this representation of a request. APIVersion string `json:"apiVersion"` + // Certs are the requested certificates and their respective public keys. + Certs map[string]string `json:"certs"` } // BootstrapRespose is a response to a BootstrapRequest. type BootstrapResponse struct { + // Certs are the issued certificates. + Certs map[string]string } diff --git a/pkg/pki/issue.go b/pkg/pki/issue.go index ccf4238973..a35e00b317 100644 --- a/pkg/pki/issue.go +++ b/pkg/pki/issue.go @@ -17,6 +17,7 @@ limitations under the License. package pki import ( + "crypto" "crypto/x509" "crypto/x509/pkix" "fmt" @@ -43,7 +44,9 @@ type IssueCertRequest struct { // AlternateNames is a list of alternative names for this certificate. AlternateNames []string - // PrivateKey is the private key for this certificate. If nil, a new private key will be generated. + // PublicKey is the public key for this certificate. If nil, it will be calculated from PrivateKey. + PublicKey crypto.PublicKey + // PrivateKey is the private key for this certificate. If both this and PublicKey are nil, a new private key will be generated. PrivateKey *PrivateKey // Validity is the certificate validity. The default is 10 years. Validity time.Duration @@ -130,7 +133,9 @@ func IssueCert(request *IssueCertRequest, keystore Keystore) (issuedCertificate } privateKey := request.PrivateKey - if privateKey == nil { + if request.PublicKey != nil { + template.PublicKey = request.PublicKey + } else if privateKey == nil { var err error privateKey, err = GeneratePrivateKey() if err != nil { diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index bb598db88b..67066f11d1 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -388,6 +388,8 @@ func (tf *TemplateFunctions) KopsControllerConfig() (string, error) { Listen: fmt.Sprintf(":%d", wellknownports.KopsControllerPort), ServerCertificatePath: path.Join(pkiDir, "kops-controller.crt"), ServerKeyPath: path.Join(pkiDir, "kops-controller.key"), + CABasePath: pkiDir, + SigningCAs: []string{fi.CertificateIDCA}, } switch kops.CloudProviderID(cluster.Spec.CloudProvider) {