diff --git a/cmd/kops-controller/pkg/server/BUILD.bazel b/cmd/kops-controller/pkg/server/BUILD.bazel index 45e5bc9f9a..2b4a39e9f9 100644 --- a/cmd/kops-controller/pkg/server/BUILD.bazel +++ b/cmd/kops-controller/pkg/server/BUILD.bazel @@ -19,5 +19,6 @@ go_library( "//util/pkg/vfs:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/cmd/kops-controller/pkg/server/keystore.go b/cmd/kops-controller/pkg/server/keystore.go index 5dd6d288b8..db2b00a802 100644 --- a/cmd/kops-controller/pkg/server/keystore.go +++ b/cmd/kops-controller/pkg/server/keystore.go @@ -22,6 +22,7 @@ import ( "path" "k8s.io/kops/pkg/pki" + "sigs.k8s.io/yaml" ) type keystore struct { @@ -43,27 +44,27 @@ func (k keystore) FindPrimaryKeypair(name string) (*pki.Certificate, *pki.Privat return entry.certificate, entry.key, nil } -func newKeystore(basePath string, cas []string) (pki.Keystore, error) { +func newKeystore(basePath string, cas []string) (pki.Keystore, map[string]string, error) { keystore := &keystore{ keys: map[string]keystoreEntry{}, } for _, name := range cas { certBytes, err := ioutil.ReadFile(path.Join(basePath, name+".crt")) if err != nil { - return nil, fmt.Errorf("reading %q certificate: %v", name, err) + return nil, nil, fmt.Errorf("reading %q certificate: %v", name, err) } certificate, err := pki.ParsePEMCertificate(certBytes) if err != nil { - return nil, fmt.Errorf("parsing %q certificate: %v", name, err) + return nil, nil, fmt.Errorf("parsing %q certificate: %v", name, err) } keyBytes, err := ioutil.ReadFile(path.Join(basePath, name+".key")) if err != nil { - return nil, fmt.Errorf("reading %q key: %v", name, err) + return nil, nil, fmt.Errorf("reading %q key: %v", name, err) } key, err := pki.ParsePEMPrivateKey(keyBytes) if err != nil { - return nil, fmt.Errorf("parsing %q key: %v", name, err) + return nil, nil, fmt.Errorf("parsing %q key: %v", name, err) } keystore.keys[name] = keystoreEntry{ @@ -72,5 +73,15 @@ func newKeystore(basePath string, cas []string) (pki.Keystore, error) { } } - return keystore, nil + var keypairIDs map[string]string + keypairIDsBytes, err := ioutil.ReadFile(path.Join(basePath, "keypair-ids.yaml")) + if err != nil { + return nil, nil, fmt.Errorf("reading keypair-ids.yaml") + } + err = yaml.Unmarshal(keypairIDsBytes, &keypairIDs) + if err != nil { + return nil, nil, fmt.Errorf("parsing keypair-ids.yaml") + } + + return keystore, keypairIDs, nil } diff --git a/cmd/kops-controller/pkg/server/server.go b/cmd/kops-controller/pkg/server/server.go index b2a2ca9dae..2feb3c97ac 100644 --- a/cmd/kops-controller/pkg/server/server.go +++ b/cmd/kops-controller/pkg/server/server.go @@ -40,11 +40,12 @@ import ( ) type Server struct { - opt *config.Options - certNames sets.String - server *http.Server - verifier fi.Verifier - keystore pki.Keystore + opt *config.Options + certNames sets.String + keypairIDs map[string]string + server *http.Server + verifier fi.Verifier + keystore pki.Keystore // configBase is the base of the configuration storage. configBase vfs.Path @@ -81,7 +82,7 @@ func NewServer(opt *config.Options, verifier fi.Verifier) (*Server, error) { func (s *Server) Start() error { var err error - s.keystore, err = newKeystore(s.opt.Server.CABasePath, s.opt.Server.SigningCAs) + s.keystore, s.keypairIDs, err = newKeystore(s.opt.Server.CABasePath, s.opt.Server.SigningCAs) if err != nil { return err } @@ -152,7 +153,7 @@ func (s *Server) bootstrap(w http.ResponseWriter, r *http.Request) { validHours := (455 * 24) + (hash.Sum32() % (30 * 24)) for name, pubKey := range req.Certs { - cert, err := s.issueCert(name, pubKey, id, validHours) + cert, err := s.issueCert(name, pubKey, id, validHours, req.KeypairIDs) if err != nil { klog.Infof("bootstrap %s cert %q issue err: %v", r.RemoteAddr, name, err) w.WriteHeader(http.StatusBadRequest) @@ -167,7 +168,7 @@ func (s *Server) bootstrap(w http.ResponseWriter, r *http.Request) { klog.Infof("bootstrap %s %s success", r.RemoteAddr, id.NodeName) } -func (s *Server) issueCert(name string, pubKey string, id *fi.VerifyResult, validHours uint32) (string, error) { +func (s *Server) issueCert(name string, pubKey string, id *fi.VerifyResult, validHours uint32, keypairIDs map[string]string) (string, error) { block, _ := pem.Decode([]byte(pubKey)) if block.Type != "RSA PUBLIC KEY" { return "", fmt.Errorf("unexpected key type %q", block.Type) @@ -216,6 +217,13 @@ func (s *Server) issueCert(name string, pubKey string, id *fi.VerifyResult, vali return "", fmt.Errorf("unexpected key name") } + // This field was added to the protocol in kOps 1.22. + if len(keypairIDs) > 0 { + if keypairIDs[issueReq.Signer] != s.keypairIDs[issueReq.Signer] { + return "", fmt.Errorf("request's keypair ID %q for %s didn't match server's %q", keypairIDs[issueReq.Signer], issueReq.Signer, s.keypairIDs[issueReq.Signer]) + } + } + cert, _, _, err := pki.IssueCert(issueReq, s.keystore) if err != nil { return "", fmt.Errorf("issuing certificate: %v", err) diff --git a/nodeup/pkg/model/bootstrap_client.go b/nodeup/pkg/model/bootstrap_client.go index 834a0c791b..2cb874a6c2 100644 --- a/nodeup/pkg/model/bootstrap_client.go +++ b/nodeup/pkg/model/bootstrap_client.go @@ -64,8 +64,9 @@ func (b BootstrapClientBuilder) Build(c *fi.ModelBuilderContext) error { } bootstrapClientTask := &nodetasks.BootstrapClientTask{ - Client: bootstrapClient, - Certs: b.bootstrapCerts, + Client: bootstrapClient, + Certs: b.bootstrapCerts, + KeypairIDs: b.bootstrapKeypairIDs, } for _, cert := range b.bootstrapCerts { diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go index 6f74ab5aaf..6f3c84f64c 100644 --- a/nodeup/pkg/model/context.go +++ b/nodeup/pkg/model/context.go @@ -65,8 +65,9 @@ type NodeupModelContext struct { // HasAPIServer is true if the InstanceGroup has a role of master or apiserver (pupulated by Init) HasAPIServer bool - kubernetesVersion semver.Version - bootstrapCerts map[string]*nodetasks.BootstrapCert + kubernetesVersion semver.Version + bootstrapCerts map[string]*nodetasks.BootstrapCert + bootstrapKeypairIDs map[string]string // ConfigurationMode determines if we are prewarming an instance or running it live ConfigurationMode string @@ -81,6 +82,7 @@ func (c *NodeupModelContext) Init() error { } c.kubernetesVersion = *k8sVersion c.bootstrapCerts = map[string]*nodetasks.BootstrapCert{} + c.bootstrapKeypairIDs = map[string]string{} role := c.BootConfig.InstanceGroupRole @@ -240,7 +242,7 @@ func (c *NodeupModelContext) BuildIssuedKubeconfig(name string, subject nodetask } // GetBootstrapCert requests a certificate keypair from kops-controller. -func (c *NodeupModelContext) GetBootstrapCert(name string) (cert, key fi.Resource) { +func (c *NodeupModelContext) GetBootstrapCert(name string, signer string) (cert, key fi.Resource, err error) { if c.IsMaster { panic("control plane nodes can't get certs from kops-controller") } @@ -252,13 +254,20 @@ func (c *NodeupModelContext) GetBootstrapCert(name string) (cert, key fi.Resourc } c.bootstrapCerts[name] = b } - return b.Cert, b.Key + c.bootstrapKeypairIDs[signer] = c.NodeupConfig.KeypairIDs[signer] + if c.bootstrapKeypairIDs[signer] == "" { + return nil, nil, fmt.Errorf("no keypairID for %q", signer) + } + return b.Cert, b.Key, nil } // BuildBootstrapKubeconfig generates a kubeconfig with a client certificate from either kops-controller or the state store. func (c *NodeupModelContext) BuildBootstrapKubeconfig(name string, ctx *fi.ModelBuilderContext) (fi.Resource, error) { if c.UseKopsControllerForNodeBootstrap() { - cert, key := c.GetBootstrapCert(name) + cert, key, err := c.GetBootstrapCert(name, fi.CertificateIDCA) + if err != nil { + return nil, err + } kubeConfig := &nodetasks.KubeConfig{ Name: name, @@ -273,7 +282,7 @@ func (c *NodeupModelContext) BuildBootstrapKubeconfig(name string, ctx *fi.Model kubeConfig.ServerURL = "https://" + c.Cluster.Spec.MasterInternalName } - err := ctx.EnsureTask(kubeConfig) + err = ctx.EnsureTask(kubeConfig) if err != nil { return nil, err } diff --git a/nodeup/pkg/model/kops_controller.go b/nodeup/pkg/model/kops_controller.go index 618bcd01fa..ec72f66f66 100644 --- a/nodeup/pkg/model/kops_controller.go +++ b/nodeup/pkg/model/kops_controller.go @@ -23,6 +23,7 @@ import ( "k8s.io/kops/pkg/wellknownusers" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" + "sigs.k8s.io/yaml" ) // KopsControllerBuilder installs the keys for a kops-controller. @@ -91,5 +92,17 @@ func (b *KopsControllerBuilder) Build(c *fi.ModelBuilderContext) error { } } + keypairIDs, err := yaml.Marshal(b.NodeupConfig.KeypairIDs) + if err != nil { + return err + } + c.AddTask(&nodetasks.File{ + Path: filepath.Join(pkiDir, "keypair-ids.yaml"), + Contents: fi.NewBytesResource(keypairIDs), + Type: nodetasks.FileType_File, + Mode: s("0600"), + Owner: s(wellknownusers.KopsControllerName), + }) + return nil } diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go index 6b1d37675a..9d715d5036 100644 --- a/nodeup/pkg/model/kubelet.go +++ b/nodeup/pkg/model/kubelet.go @@ -559,7 +559,10 @@ func (b *KubeletBuilder) buildKubeletServingCertificate(c *fi.ModelBuilderContex } if !b.HasAPIServer { - cert, key := b.GetBootstrapCert(name) + cert, key, err := b.GetBootstrapCert(name, fi.CertificateIDCA) + if err != nil { + return err + } c.AddTask(&nodetasks.File{ Path: filepath.Join(dir, name+".crt"), diff --git a/nodeup/pkg/model/networking/cilium.go b/nodeup/pkg/model/networking/cilium.go index 47ec1e3e02..ca8eec4c5a 100644 --- a/nodeup/pkg/model/networking/cilium.go +++ b/nodeup/pkg/model/networking/cilium.go @@ -123,7 +123,10 @@ func (b *CiliumBuilder) buildCiliumEtcdSecrets(c *fi.ModelBuilderContext) error return issueCert.AddFileTasks(c, dir, name, "", nil) } else { if b.UseKopsControllerForNodeBootstrap() { - cert, key := b.GetBootstrapCert(name) + cert, key, err := b.GetBootstrapCert(name, signer) + if err != nil { + return err + } c.AddTask(&nodetasks.File{ Path: filepath.Join(dir, name+".crt"), diff --git a/pkg/apis/nodeup/bootstrap.go b/pkg/apis/nodeup/bootstrap.go index 5b84a7cfec..b77730149b 100644 --- a/pkg/apis/nodeup/bootstrap.go +++ b/pkg/apis/nodeup/bootstrap.go @@ -24,6 +24,8 @@ type BootstrapRequest struct { APIVersion string `json:"apiVersion"` // Certs are the requested certificates and their respective public keys. Certs map[string]string `json:"certs"` + // KeypairIDs are the keypair IDs of the CAs to use for issuing certificates. + KeypairIDs map[string]string `json:"keypairIDs"` // IncludeNodeConfig controls whether the cluster & instance group configuration should be returned. // This allows for nodes without access to the kops state store. diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index 791298b924..417e08f840 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -35,6 +35,7 @@ import ( "k8s.io/klog/v2" kopsbase "k8s.io/kops" "k8s.io/kops/pkg/apis/kops" + apiModel "k8s.io/kops/pkg/apis/kops/model" "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/pkg/apis/kops/util" "k8s.io/kops/pkg/apis/kops/validation" @@ -1295,17 +1296,16 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAddit } } - if err := getTasksCertificate(caTasks, fi.CertificateIDCA, config, false); err != nil { + if err := getTasksCertificate(caTasks, fi.CertificateIDCA, config, true); err != nil { return nil, nil, err } if caTasks["etcd-clients-ca-cilium"] != nil { - if err := getTasksCertificate(caTasks, "etcd-clients-ca-cilium", config, hasAPIServer); err != nil { + if err := getTasksCertificate(caTasks, "etcd-clients-ca-cilium", config, hasAPIServer || apiModel.UseKopsControllerForNodeBootstrap(n.cluster)); err != nil { return nil, nil, err } } if isMaster { - config.KeypairIDs[fi.CertificateIDCA] = caTasks[fi.CertificateIDCA].Keyset().Primary.Id if err := getTasksCertificate(caTasks, "etcd-clients-ca", config, true); err != nil { return nil, nil, err } diff --git a/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go b/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go index cfaacfea7e..882554001d 100644 --- a/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go +++ b/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go @@ -41,6 +41,8 @@ import ( type BootstrapClientTask struct { // Certs are the requested certificates. Certs map[string]*BootstrapCert + // KeypairIDs are the keypair IDs of the CAs to use for issuing certificates. + KeypairIDs map[string]string // Client holds the client wrapper for the kops-bootstrap protocol Client *KopsBootstrapClient @@ -83,6 +85,7 @@ func (b *BootstrapClientTask) Run(c *fi.Context) error { req := nodeup.BootstrapRequest{ APIVersion: nodeup.BootstrapAPIVersion, Certs: map[string]string{}, + KeypairIDs: b.KeypairIDs, } if b.keys == nil {