diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go index fd749af760..e4d7e721b2 100644 --- a/nodeup/pkg/model/context.go +++ b/nodeup/pkg/model/context.go @@ -98,16 +98,16 @@ func (c *NodeupModelContext) CNIConfDir() string { // buildPKIKubeconfig generates a kubeconfig func (c *NodeupModelContext) buildPKIKubeconfig(id string) (string, error) { - caCertificate, err := c.KeyStore.Cert(fi.CertificateId_CA) + caCertificate, err := c.KeyStore.Cert(fi.CertificateId_CA, false) if err != nil { return "", fmt.Errorf("error fetching CA certificate from keystore: %v", err) } - certificate, err := c.KeyStore.Cert(id) + certificate, err := c.KeyStore.Cert(id, false) if err != nil { return "", fmt.Errorf("error fetching %q certificate from keystore: %v", id, err) } - privateKey, err := c.KeyStore.PrivateKey(id) + privateKey, err := c.KeyStore.PrivateKey(id, false) if err != nil { return "", fmt.Errorf("error fetching %q private key from keystore: %v", id, err) } diff --git a/nodeup/pkg/model/convenience.go b/nodeup/pkg/model/convenience.go index 5f98ee4513..6e4503a17c 100644 --- a/nodeup/pkg/model/convenience.go +++ b/nodeup/pkg/model/convenience.go @@ -94,7 +94,7 @@ func getProxyEnvVars(proxies *kops.EgressProxySpec) []v1.EnvVar { // buildCertificateRequest retrieves the certificate from a keystore func buildCertificateRequest(c *fi.ModelBuilderContext, b *NodeupModelContext, name, path string) error { - cert, err := b.KeyStore.Cert(name) + cert, err := b.KeyStore.Cert(name, false) if err != nil { return err } @@ -120,7 +120,7 @@ func buildCertificateRequest(c *fi.ModelBuilderContext, b *NodeupModelContext, n // buildPrivateKeyRequest retrieves a private key from the store func buildPrivateKeyRequest(c *fi.ModelBuilderContext, b *NodeupModelContext, name, path string) error { - k, err := b.KeyStore.PrivateKey(name) + k, err := b.KeyStore.PrivateKey(name, false) if err != nil { return err } diff --git a/nodeup/pkg/model/kubecontrollermanager.go b/nodeup/pkg/model/kubecontrollermanager.go index e433575f0e..75518c30c8 100644 --- a/nodeup/pkg/model/kubecontrollermanager.go +++ b/nodeup/pkg/model/kubecontrollermanager.go @@ -45,7 +45,7 @@ func (b *KubeControllerManagerBuilder) Build(c *fi.ModelBuilderContext) error { // If we're using the CertificateSigner, include the CA Key // TODO: use a per-machine key? use KMS? if b.useCertificateSigner() { - ca, err := b.KeyStore.PrivateKey(fi.CertificateId_CA) + ca, err := b.KeyStore.PrivateKey(fi.CertificateId_CA, false) if err != nil { return err } diff --git a/nodeup/pkg/model/protokube.go b/nodeup/pkg/model/protokube.go index 8f0eaa1569..2b84166a43 100644 --- a/nodeup/pkg/model/protokube.go +++ b/nodeup/pkg/model/protokube.go @@ -331,7 +331,7 @@ func (t *ProtokubeBuilder) writeProxyEnvVars(buffer *bytes.Buffer) { // buildCertificateTask is responsible for build a certificate request task func (t *ProtokubeBuilder) buildCeritificateTask(c *fi.ModelBuilderContext, name, filename string) error { - cert, err := t.KeyStore.Cert(name) + cert, err := t.KeyStore.Cert(name, false) if err != nil { return err } @@ -353,7 +353,7 @@ func (t *ProtokubeBuilder) buildCeritificateTask(c *fi.ModelBuilderContext, name // buildPrivateKeyTask is responsible for build a certificate request task func (t *ProtokubeBuilder) buildPrivateTask(c *fi.ModelBuilderContext, name, filename string) error { - cert, err := t.KeyStore.PrivateKey(name) + cert, err := t.KeyStore.PrivateKey(name, false) if err != nil { return err } diff --git a/nodeup/pkg/model/secrets.go b/nodeup/pkg/model/secrets.go index 846e09d6aa..0fbc1b3492 100644 --- a/nodeup/pkg/model/secrets.go +++ b/nodeup/pkg/model/secrets.go @@ -40,7 +40,7 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error { // retrieve the platform ca { - ca, err := b.KeyStore.CertificatePool(fi.CertificateId_CA) + ca, err := b.KeyStore.CertificatePool(fi.CertificateId_CA, false) if err != nil { return err } @@ -79,7 +79,7 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error { } { - cert, err := b.KeyStore.Cert("master") + cert, err := b.KeyStore.Cert("master", false) if err != nil { return err } @@ -98,7 +98,7 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error { } { - k, err := b.KeyStore.PrivateKey("master") + k, err := b.KeyStore.PrivateKey("master", false) if err != nil { return err } @@ -118,7 +118,7 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error { if b.IsKubernetesGTE("1.7") { - cert, err := b.KeyStore.Cert("apiserver-proxy-client") + cert, err := b.KeyStore.Cert("apiserver-proxy-client", false) if err != nil { return fmt.Errorf("apiserver proxy client cert lookup failed: %v", err.Error()) } @@ -135,7 +135,7 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error { } c.AddTask(t) - key, err := b.KeyStore.PrivateKey("apiserver-proxy-client") + key, err := b.KeyStore.PrivateKey("apiserver-proxy-client", false) if err != nil { return fmt.Errorf("apiserver proxy client private key lookup failed: %v", err.Error()) } diff --git a/upup/pkg/fi/ca.go b/upup/pkg/fi/ca.go index bacb0c35c4..dc20767544 100644 --- a/upup/pkg/fi/ca.go +++ b/upup/pkg/fi/ca.go @@ -59,10 +59,10 @@ type CAStore interface { Keystore // Cert returns the primary specified certificate - Cert(name string) (*pki.Certificate, error) + Cert(name string, createIfMissing bool) (*pki.Certificate, error) // CertificatePool returns all active certificates with the specified id - CertificatePool(name string) (*CertificatePool, error) - PrivateKey(name string) (*pki.PrivateKey, error) + CertificatePool(name string, createIfMissing bool) (*CertificatePool, error) + PrivateKey(name string, createIfMissing bool) (*pki.PrivateKey, error) FindCert(name string) (*pki.Certificate, error) FindPrivateKey(name string) (*pki.PrivateKey, error) diff --git a/upup/pkg/fi/clientset_castore.go b/upup/pkg/fi/clientset_castore.go new file mode 100644 index 0000000000..b17e3b9a63 --- /dev/null +++ b/upup/pkg/fi/clientset_castore.go @@ -0,0 +1,624 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fi + +import ( + "bytes" + crypto_rand "crypto/rand" + "crypto/rsa" + "crypto/x509" + "fmt" + "math/big" + "sync" + "time" + + "github.com/golang/glog" + "golang.org/x/crypto/ssh" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kops/pkg/apis/kops" + kopsinternalversion "k8s.io/kops/pkg/client/clientset_generated/clientset/typed/kops/internalversion" + "k8s.io/kops/pkg/pki" + "k8s.io/kops/util/pkg/vfs" +) + +// ClientsetCAStore is a CAStore implementation that stores keypairs in Keyset on a API server +type ClientsetCAStore struct { + namespace string + clientset kopsinternalversion.KopsInterface + + mutex sync.Mutex + cacheCaKeyset *keyset +} + +var _ CAStore = &ClientsetCAStore{} + +// NewClientsetCAStore is the constructor for ClientsetCAStore +func NewClientsetCAStore(clientset kopsinternalversion.KopsInterface, namespace string) CAStore { + c := &ClientsetCAStore{ + clientset: clientset, + namespace: namespace, + } + + return c +} + +// readCAKeypairs retrieves the CA keypair, generating a new keypair if not found +func (c *ClientsetCAStore) readCAKeypairs() (*keyset, error) { + c.mutex.Lock() + defer c.mutex.Unlock() + + if c.cacheCaKeyset != nil { + return c.cacheCaKeyset, nil + } + + keyset, err := c.loadKeyset(CertificateId_CA) + if err != nil { + return nil, err + } + + if keyset == nil { + keyset, err = c.generateCACertificate() + if err != nil { + return nil, err + } + + } + c.cacheCaKeyset = keyset + + return keyset, nil +} + +// generateCACertificate creates and stores a CA keypair +// Should be called with the mutex held, to prevent concurrent creation of different keys +func (c *ClientsetCAStore) generateCACertificate() (*keyset, error) { + template := BuildCAX509Template() + + caRsaKey, err := rsa.GenerateKey(crypto_rand.Reader, 2048) + if err != nil { + return nil, fmt.Errorf("error generating RSA private key: %v", err) + } + + caPrivateKey := &pki.PrivateKey{Key: caRsaKey} + + t := time.Now().UnixNano() + template.SerialNumber = pki.BuildPKISerial(t) + + caCertificate, err := pki.SignNewCertificate(caPrivateKey, template, nil, nil) + if err != nil { + return nil, err + } + + return c.storeAndVerifyKeypair(CertificateId_CA, caCertificate, caPrivateKey) +} + +// keyset is a parsed Keyset +type keyset struct { + items map[string]*keysetItem + primary *keysetItem +} + +// keysetItem is a parsed KeysetItem +type keysetItem struct { + id string + certificate *pki.Certificate + privateKey *pki.PrivateKey +} + +// loadKeyset gets the named keyset +func (c *ClientsetCAStore) loadKeyset(name string) (*keyset, error) { + o, err := c.clientset.Keysets(c.namespace).Get(name, v1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return nil, nil + } + return nil, fmt.Errorf("error reading keyset %q: %v", name, err) + } + + keyset := &keyset{ + items: make(map[string]*keysetItem), + } + + for _, key := range o.Spec.Keys { + cert, err := pki.LoadPEMCertificate(key.PublicMaterial) + if err != nil { + glog.Warningf("key public material was %s", key.PublicMaterial) + return nil, fmt.Errorf("error loading certificate %s/%s: %v", name, key.Id, err) + } + privateKey, err := pki.ParsePEMPrivateKey(key.PrivateMaterial) + if err != nil { + return nil, fmt.Errorf("error loading private key %s/%s: %v", name, key.Id, err) + } + keyset.items[key.Id] = &keysetItem{ + id: key.Id, + certificate: cert, + privateKey: privateKey, + } + } + + primary := FindPrimary(o) + if primary != nil { + keyset.primary = keyset.items[primary.Id] + } + + return keyset, nil +} + +// FindPrimary returns the primary KeysetItem in the Keyset +func FindPrimary(keyset *kops.Keyset) *kops.KeysetItem { + var primary *kops.KeysetItem + var primaryVersion *big.Int + for i := range keyset.Spec.Keys { + item := &keyset.Spec.Keys[i] + version, ok := big.NewInt(0).SetString(item.Id, 10) + if !ok { + glog.Warningf("Ignoring key item with non-integer version: %q", item.Id) + continue + } + + if primaryVersion == nil || version.Cmp(primaryVersion) > 0 { + primary = item + primaryVersion = version + } + } + return primary +} + +// Cert implements CAStore::Cert +func (c *ClientsetCAStore) Cert(name string, createIfMissing bool) (*pki.Certificate, error) { + cert, err := c.FindCert(name) + if err == nil && cert == nil { + if !createIfMissing { + glog.Warningf("using empty certificate, because running with DryRun") + return &pki.Certificate{}, err + } + return nil, fmt.Errorf("cannot find certificate %q", name) + } + return cert, err + +} + +// CertificatePool implements CAStore::CertificatePool +func (c *ClientsetCAStore) CertificatePool(id string, createIfMissing bool) (*CertificatePool, error) { + cert, err := c.FindCertificatePool(id) + if err == nil && cert == nil { + if !createIfMissing { + glog.Warningf("using empty certificate, because running with DryRun") + return &CertificatePool{}, err + } + return nil, fmt.Errorf("cannot find certificate pool %q", id) + } + return cert, err + +} + +// FindKeypair implements CAStore::FindKeypair +func (c *ClientsetCAStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) { + keyset, err := c.loadKeyset(name) + if err != nil { + return nil, nil, err + } + + if keyset != nil && keyset.primary != nil { + return keyset.primary.certificate, keyset.primary.privateKey, nil + } + + return nil, nil, nil +} + +// FindCert implements CAStore::FindCert +func (c *ClientsetCAStore) FindCert(name string) (*pki.Certificate, error) { + keyset, err := c.loadKeyset(name) + if err != nil { + return nil, err + } + + var cert *pki.Certificate + if keyset != nil && keyset.primary != nil { + cert = keyset.primary.certificate + } + + return cert, nil +} + +// FindCertificatePool implements CAStore::FindCertificatePool +func (c *ClientsetCAStore) FindCertificatePool(name string) (*CertificatePool, error) { + keyset, err := c.loadKeyset(name) + if err != nil { + return nil, err + } + + pool := &CertificatePool{} + + if keyset != nil { + if keyset.primary != nil { + pool.Primary = keyset.primary.certificate + } + + for id, item := range keyset.items { + if id == keyset.primary.id { + continue + } + pool.Secondary = append(pool.Secondary, item.certificate) + } + } + return pool, nil +} + +// List implements CAStore::List +func (c *ClientsetCAStore) List() ([]*KeystoreItem, error) { + var items []*KeystoreItem + + { + list, err := c.clientset.Keysets(c.namespace).List(v1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("error listing Keysets: %v", err) + } + + for _, keyset := range list.Items { + for _, item := range keyset.Spec.Keys { + ki := &KeystoreItem{ + Name: keyset.Name, + Id: item.Id, + } + + switch keyset.Spec.Type { + case kops.SecretTypeKeypair: + ki.Type = SecretTypeKeypair + case kops.SecretTypeSecret: + //ki.Type = SecretTypeSecret + continue // Ignore - this is handled by ClientsetSecretStore + default: + return nil, fmt.Errorf("unhandled secret type %q: %v", ki.Type, err) + } + items = append(items, ki) + } + } + } + + { + list, err := c.clientset.SSHCredentials(c.namespace).List(v1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("error listing SSHCredentials: %v", err) + } + + for _, sshCredential := range list.Items { + ki := &KeystoreItem{ + Name: sshCredential.Name, + Type: SecretTypeSSHPublicKey, + } + items = append(items, ki) + } + } + + return items, nil +} + +// IssueCert implements CAStore::IssueCert +func (c *ClientsetCAStore) IssueCert(name string, serial *big.Int, privateKey *pki.PrivateKey, template *x509.Certificate) (*pki.Certificate, error) { + glog.Infof("Issuing new certificate: %q", name) + + template.SerialNumber = serial + + caKeyset, err := c.readCAKeypairs() + if err != nil { + return nil, err + } + + if caKeyset == nil { + return nil, fmt.Errorf("ca keyset was not found; cannot issue certificates") + } + if caKeyset.primary == nil { + return nil, fmt.Errorf("ca keyset did not have any key data; cannot issue certificates") + } + if caKeyset.primary.certificate == nil { + return nil, fmt.Errorf("ca certificate was not found; cannot issue certificates") + } + if caKeyset.primary.privateKey == nil { + return nil, fmt.Errorf("ca privateKey was not found; cannot issue certificates") + } + cert, err := pki.SignNewCertificate(privateKey, template, caKeyset.primary.certificate.Certificate, caKeyset.primary.privateKey) + if err != nil { + return nil, err + } + + if _, err := c.storeAndVerifyKeypair(name, cert, privateKey); err != nil { + return nil, err + } + + return cert, nil +} + +// storeAndVerifyKeypair writes the keypair, also re-reading it to double-check it +func (c *ClientsetCAStore) storeAndVerifyKeypair(name string, cert *pki.Certificate, privateKey *pki.PrivateKey) (*keyset, error) { + id := cert.Certificate.SerialNumber.String() + if err := c.storeKeypair(name, id, cert, privateKey); err != nil { + return nil, err + } + + // Make double-sure it round-trips + keyset, err := c.loadKeyset(name) + if err != nil { + return nil, fmt.Errorf("error fetching stored certificate: %v", err) + } + + if keyset == nil { + return nil, fmt.Errorf("stored certificate not found: %v", err) + } + if keyset.primary == nil { + return nil, fmt.Errorf("stored certificate did not have data: %v", err) + } + if keyset.primary.id != id { + return nil, fmt.Errorf("stored certificate changed concurrently (id mismatch)") + } + return keyset, nil +} + +// StoreKeypair implements CAStore::StoreKeypair +func (c *ClientsetCAStore) StoreKeypair(name string, cert *pki.Certificate, privateKey *pki.PrivateKey) error { + return c.storeKeypair(name, cert.Certificate.SerialNumber.String(), cert, privateKey) +} + +// AddCert implements CAStore::AddCert +func (c *ClientsetCAStore) AddCert(name string, cert *pki.Certificate) error { + glog.Infof("Adding TLS certificate: %q", name) + + // We add with a timestamp of zero so this will never be the newest cert + serial := pki.BuildPKISerial(0) + + err := c.storeKeypair(name, serial.String(), cert, nil) + if err != nil { + return err + } + + return nil +} + +// FindPrivateKey implements CAStore::FindPrivateKey +func (c *ClientsetCAStore) FindPrivateKey(name string) (*pki.PrivateKey, error) { + keyset, err := c.loadKeyset(name) + if err != nil { + return nil, err + } + + if keyset != nil && keyset.primary != nil { + return keyset.primary.privateKey, nil + } + return nil, nil +} + +// PrivateKey implements CAStore::PrivateKey +func (c *ClientsetCAStore) PrivateKey(name string, createIfMissing bool) (*pki.PrivateKey, error) { + key, err := c.FindPrivateKey(name) + if err == nil && key == nil { + if !createIfMissing { + glog.Warningf("using empty certificate, because running with DryRun") + return &pki.PrivateKey{}, err + } + return nil, fmt.Errorf("cannot find SSL key %q", name) + } + return key, err +} + +// CreateKeypair implements CAStore::CreateKeypair +func (c *ClientsetCAStore) CreateKeypair(id string, template *x509.Certificate, privateKey *pki.PrivateKey) (*pki.Certificate, error) { + serial := c.buildSerial() + + cert, err := c.IssueCert(id, serial, privateKey, template) + if err != nil { + return nil, err + } + + return cert, nil +} + +// addKey saves the specified key to the registry +func (c *ClientsetCAStore) addKey(name string, keysetType kops.KeysetType, item *kops.KeysetItem) error { + create := false + client := c.clientset.Keysets(c.namespace) + keyset, err := client.Get(name, v1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + keyset = nil + } else { + return fmt.Errorf("error reading keyset %q: %v", name, err) + } + } + if keyset == nil { + keyset = &kops.Keyset{} + keyset.Name = name + keyset.Spec.Type = keysetType + create = true + } + keyset.Spec.Keys = append(keyset.Spec.Keys, *item) + if create { + if _, err := client.Create(keyset); err != nil { + return fmt.Errorf("error creating keyset %q: %v", name, err) + } + } else { + if _, err := client.Update(keyset); err != nil { + return fmt.Errorf("error updating keyset %q: %v", name, err) + } + } + return nil +} + +// DeleteKeysetItem deletes the specified key from the registry; deleting the whole keyset if it was the last one +func DeleteKeysetItem(client kopsinternalversion.KeysetInterface, name string, keysetType kops.KeysetType, id string) error { + keyset, err := client.Get(name, v1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return nil + } else { + return fmt.Errorf("error reading Keyset %q: %v", name, err) + } + } + + if keyset.Spec.Type != keysetType { + return fmt.Errorf("mismatch on Keyset type on %q", name) + } + + var newKeys []kops.KeysetItem + found := false + for _, ki := range keyset.Spec.Keys { + if ki.Id == id { + found = true + } else { + newKeys = append(newKeys, ki) + } + } + if !found { + return fmt.Errorf("KeysetItem %q not found in Keyset %q", id, name) + } + if len(newKeys) == 0 { + if err := client.Delete(name, &v1.DeleteOptions{}); err != nil { + return fmt.Errorf("error deleting Keyset %q: %v", name, err) + } + } else { + keyset.Spec.Keys = newKeys + if _, err := client.Update(keyset); err != nil { + return fmt.Errorf("error updating Keyset %q: %v", name, err) + } + } + return nil +} + +// addSshCredential saves the specified SSH Credential to the registry, doing an update or insert +func (c *ClientsetCAStore) addSshCredential(name string, publicKey string) error { + create := false + client := c.clientset.SSHCredentials(c.namespace) + sshCredential, err := client.Get(name, v1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + sshCredential = nil + } else { + return fmt.Errorf("error reading SSHCredential %q: %v", name, err) + } + } + if sshCredential == nil { + sshCredential = &kops.SSHCredential{} + sshCredential.Name = name + create = true + } + sshCredential.Spec.PublicKey = publicKey + if create { + if _, err := client.Create(sshCredential); err != nil { + return fmt.Errorf("error creating SSHCredential %q: %v", name, err) + } + } else { + if _, err := client.Update(sshCredential); err != nil { + return fmt.Errorf("error updating SSHCredential %q: %v", name, err) + } + } + return nil +} + +// deleteSSHCredential deletes the specified SSHCredential from the registry +func (c *ClientsetCAStore) deleteSSHCredential(name string) error { + client := c.clientset.SSHCredentials(c.namespace) + err := client.Delete(name, &v1.DeleteOptions{}) + if err != nil { + return fmt.Errorf("error deleting SSHCredential %q: %v", name, err) + } + return nil +} + +// addKey saves the specified keypair to the registry +func (c *ClientsetCAStore) storeKeypair(name string, id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error { + var publicMaterial bytes.Buffer + if _, err := cert.WriteTo(&publicMaterial); err != nil { + return err + } + + var privateMaterial bytes.Buffer + if _, err := privateKey.WriteTo(&privateMaterial); err != nil { + return err + } + + item := &kops.KeysetItem{ + Id: id, + PublicMaterial: publicMaterial.Bytes(), + PrivateMaterial: privateMaterial.Bytes(), + } + return c.addKey(name, kops.SecretTypeKeypair, item) +} + +// buildSerial returns a serial for use when issuing certificates +func (c *ClientsetCAStore) buildSerial() *big.Int { + t := time.Now().UnixNano() + return pki.BuildPKISerial(t) +} + +// AddSSHPublicKey implements CAStore::AddSSHPublicKey +func (c *ClientsetCAStore) AddSSHPublicKey(name string, pubkey []byte) error { + _, _, _, _, err := ssh.ParseAuthorizedKey(pubkey) + if err != nil { + return fmt.Errorf("error parsing SSH public key: %v", err) + } + + // TODO: Reintroduce or remove + //// compute fingerprint to serve as id + //h := md5.New() + //_, err = h.Write(sshPublicKey.Marshal()) + //if err != nil { + // return err + //} + //id = formatFingerprint(h.Sum(nil)) + + return c.addSshCredential(name, string(pubkey)) +} + +// FindSSHPublicKeys implements CAStore::FindSSHPublicKeys +func (c *ClientsetCAStore) FindSSHPublicKeys(name string) ([]*KeystoreItem, error) { + o, err := c.clientset.SSHCredentials(c.namespace).Get(name, v1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("error reading SSHCredential %q: %v", name, err) + } + + var items []*KeystoreItem + item := &KeystoreItem{ + Type: SecretTypeSSHPublicKey, + Name: name, + //Id: insertFingerprintColons(k.Id), + Data: []byte(o.Spec.PublicKey), + } + items = append(items, item) + + return items, nil +} + +// DeleteSecret implements CAStore::DeleteSecret +func (c *ClientsetCAStore) DeleteSecret(item *KeystoreItem) error { + switch item.Type { + case SecretTypeSSHPublicKey: + return c.deleteSSHCredential(item.Name) + + case SecretTypeKeypair: + client := c.clientset.Keysets(c.namespace) + return DeleteKeysetItem(client, item.Name, kops.SecretTypeKeypair, item.Id) + + default: + // Primarily because we need to make sure users can recreate them! + return fmt.Errorf("deletion of keystore items of type %v not (yet) supported", item.Type) + } +} + +// VFSPath implements CAStore::VFSPath +func (c *ClientsetCAStore) VFSPath() vfs.Path { + // We will implement mirroring instead + panic("ClientsetCAStore::VFSPath not implemented") +} diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index f1ce1f007a..52bd6ae8a8 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -192,7 +192,6 @@ func (c *ApplyClusterCmd) Run() error { if err != nil { return err } - keyStore.(*fi.VFSCAStore).DryRun = c.DryRun secretStore, err := registry.SecretStore(cluster) if err != nil { diff --git a/upup/pkg/fi/cloudup/populate_cluster_spec.go b/upup/pkg/fi/cloudup/populate_cluster_spec.go index 04c7a1db4a..89d6eaf68f 100644 --- a/upup/pkg/fi/cloudup/populate_cluster_spec.go +++ b/upup/pkg/fi/cloudup/populate_cluster_spec.go @@ -181,8 +181,6 @@ func (c *populateClusterSpec) run() error { if err != nil { return err } - // Always assume a dry run during this phase - keyStore.(*fi.VFSCAStore).DryRun = true secretStore, err := registry.SecretStore(cluster) if err != nil { diff --git a/upup/pkg/fi/nodeup/template_functions.go b/upup/pkg/fi/nodeup/template_functions.go index 425d53c821..df77a0f497 100644 --- a/upup/pkg/fi/nodeup/template_functions.go +++ b/upup/pkg/fi/nodeup/template_functions.go @@ -27,7 +27,6 @@ import ( api "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/nodeup" "k8s.io/kops/pkg/flagbuilder" - "k8s.io/kops/pkg/pki" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/secrets" "k8s.io/kops/util/pkg/vfs" @@ -93,9 +92,6 @@ func (t *templateFunctions) populate(dest template.FuncMap) { return runtime.GOARCH } - dest["CACertificate"] = t.CACertificate - dest["PrivateKey"] = t.PrivateKey - dest["Certificate"] = t.Certificate dest["GetToken"] = t.GetToken dest["BuildFlags"] = flagbuilder.BuildFlags @@ -122,21 +118,6 @@ func (t *templateFunctions) populate(dest template.FuncMap) { } } -// CACertificate returns the primary CA certificate for the cluster -func (t *templateFunctions) CACertificate() (*pki.Certificate, error) { - return t.keyStore.Cert(fi.CertificateId_CA) -} - -// PrivateKey returns the specified private key -func (t *templateFunctions) PrivateKey(id string) (*pki.PrivateKey, error) { - return t.keyStore.PrivateKey(id) -} - -// Certificate returns the specified private key -func (t *templateFunctions) Certificate(id string) (*pki.Certificate, error) { - return t.keyStore.Cert(id) -} - // GetToken returns the specified token func (t *templateFunctions) GetToken(key string) (string, error) { token, err := t.secretStore.FindSecret(key) diff --git a/upup/pkg/fi/secrets.go b/upup/pkg/fi/secrets.go index d2dc66cb7e..9b45f087db 100644 --- a/upup/pkg/fi/secrets.go +++ b/upup/pkg/fi/secrets.go @@ -25,15 +25,15 @@ import ( ) type SecretStore interface { - // Get a secret. Returns an error if not found + // Secret returns a secret. Returns an error if not found Secret(id string) (*Secret, error) // DeleteSecret deletes the specified secret DeleteSecret(item *KeystoreItem) error - // Find a secret, if exists. Returns nil,nil if not found + // FindSecret finds a secret, if exists. Returns nil,nil if not found FindSecret(id string) (*Secret, error) - // Create or replace a secret + // GetOrCreateSecret creates or replace a secret GetOrCreateSecret(id string, secret *Secret) (current *Secret, created bool, err error) - // Lists the ids of all known secrets + // ListSecrets lists the ids of all known secrets ListSecrets() ([]string, error) // VFSPath returns the path where the SecretStore is stored diff --git a/upup/pkg/fi/secrets/clientset_secretstore.go b/upup/pkg/fi/secrets/clientset_secretstore.go new file mode 100644 index 0000000000..a34c9ed93d --- /dev/null +++ b/upup/pkg/fi/secrets/clientset_secretstore.go @@ -0,0 +1,186 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package secrets + +import ( + "fmt" + "strings" + "time" + + "github.com/golang/glog" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kops/pkg/apis/kops" + kopsinternalversion "k8s.io/kops/pkg/client/clientset_generated/clientset/typed/kops/internalversion" + "k8s.io/kops/pkg/pki" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/util/pkg/vfs" +) + +// NamePrefix is a prefix we use to avoid collisions with other keysets +const NamePrefix = "token-" + +// ClientsetSecretStore is a SecretStore backed by Keyset objects in an API server +type ClientsetSecretStore struct { + namespace string + clientset kopsinternalversion.KopsInterface +} + +var _ fi.SecretStore = &ClientsetSecretStore{} + +// NewClientsetSecretStore is the constructor for ClientsetSecretStore +func NewClientsetSecretStore(clientset kopsinternalversion.KopsInterface, namespace string) fi.SecretStore { + c := &ClientsetSecretStore{ + clientset: clientset, + namespace: namespace, + } + return c +} + +// FindSecret implements fi.SecretStore::FindSecret +func (c *ClientsetSecretStore) FindSecret(name string) (*fi.Secret, error) { + s, err := c.loadSecret(name) + if err != nil { + return nil, err + } + return s, nil +} + +// ListSecrets implements fi.SecretStore::ListSecrets +func (c *ClientsetSecretStore) ListSecrets() ([]string, error) { + list, err := c.clientset.Keysets(c.namespace).List(v1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("error listing keysets: %v", err) + } + + var names []string + for i := range list.Items { + keyset := &list.Items[i] + + switch keyset.Spec.Type { + case kops.SecretTypeSecret: + name := strings.TrimPrefix(keyset.Name, NamePrefix) + names = append(names, name) + } + } + + return names, nil +} + +// Secret implements fi.SecretStore::Secret +func (c *ClientsetSecretStore) Secret(name string) (*fi.Secret, error) { + s, err := c.FindSecret(name) + if err != nil { + return nil, err + } + if s == nil { + return nil, fmt.Errorf("Secret not found: %q", name) + } + return s, nil +} + +// DeleteSecret implements fi.SecretStore::DeleteSecret +func (c *ClientsetSecretStore) DeleteSecret(item *fi.KeystoreItem) error { + client := c.clientset.Keysets(c.namespace) + return fi.DeleteKeysetItem(client, item.Name, kops.SecretTypeKeypair, item.Id) +} + +// GetOrCreateSecret implements fi.SecretStore::GetOrCreateSecret +func (c *ClientsetSecretStore) GetOrCreateSecret(name string, secret *fi.Secret) (*fi.Secret, bool, error) { + for i := 0; i < 2; i++ { + s, err := c.FindSecret(name) + if err != nil { + return nil, false, err + } + + if s != nil { + return s, false, nil + } + + _, err = c.createSecret(secret, name) + if err != nil { + if errors.IsAlreadyExists(err) && i == 0 { + glog.Infof("Got already-exists error when writing secret; likely due to concurrent creation. Will retry") + continue + } else { + return nil, false, err + } + } + + if err == nil { + break + } + } + + // Make double-sure it round-trips + s, err := c.loadSecret(name) + if err != nil { + glog.Fatalf("unable to load secret immmediately after creation %v: %v", name, err) + return nil, false, err + } + return s, true, nil +} + +// loadSecret returns the named secret, if it exists, otherwise returns nil +func (c *ClientsetSecretStore) loadSecret(name string) (*fi.Secret, error) { + name = NamePrefix + name + keyset, err := c.clientset.Keysets(c.namespace).Get(name, v1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return nil, nil + } + return nil, fmt.Errorf("error reading keyset %q: %v", name, err) + } + + return parseSecret(keyset) +} + +// parseSecret attempts to parse the primary secret, otherwise returns nil +func parseSecret(keyset *kops.Keyset) (*fi.Secret, error) { + primary := fi.FindPrimary(keyset) + if primary == nil { + return nil, nil + } + + s := &fi.Secret{} + s.Data = primary.PrivateMaterial + return s, nil +} + +// createSecret writes the secret, but only if it does not exist +func (c *ClientsetSecretStore) createSecret(s *fi.Secret, name string) (*kops.Keyset, error) { + keyset := &kops.Keyset{} + keyset.Name = NamePrefix + name + keyset.Spec.Type = kops.SecretTypeSecret + + t := time.Now().UnixNano() + id := pki.BuildPKISerial(t) + + keyset.Spec.Keys = append(keyset.Spec.Keys, kops.KeysetItem{ + Id: id.String(), + PrivateMaterial: s.Data, + }) + + return c.clientset.Keysets(c.namespace).Create(keyset) +} + +// VFSPath implements fi.SecretStore::VFSPath +func (c *ClientsetSecretStore) VFSPath() vfs.Path { + // We will implement mirroring instead + glog.Fatalf("ClientsetSecretStore::VFSPath not implemented") + return nil +} diff --git a/upup/pkg/fi/secrets/vfs_secretstore.go b/upup/pkg/fi/secrets/vfs_secretstore.go index 8613ba59a6..dc07158dc1 100644 --- a/upup/pkg/fi/secrets/vfs_secretstore.go +++ b/upup/pkg/fi/secrets/vfs_secretstore.go @@ -42,8 +42,8 @@ func (c *VFSSecretStore) VFSPath() vfs.Path { return c.basedir } -func (c *VFSSecretStore) buildSecretPath(id string) vfs.Path { - return c.basedir.Join(id) +func (c *VFSSecretStore) buildSecretPath(name string) vfs.Path { + return c.basedir.Join(name) } func (c *VFSSecretStore) FindSecret(id string) (*fi.Secret, error) { diff --git a/upup/pkg/fi/vfs_castore.go b/upup/pkg/fi/vfs_castore.go index 82c8ffe20b..4b4ee35b9f 100644 --- a/upup/pkg/fi/vfs_castore.go +++ b/upup/pkg/fi/vfs_castore.go @@ -38,7 +38,6 @@ import ( ) type VFSCAStore struct { - DryRun bool basedir vfs.Path mutex sync.Mutex @@ -260,10 +259,10 @@ func (c *VFSCAStore) loadOneCertificate(p vfs.Path) (*pki.Certificate, error) { return cert, nil } -func (c *VFSCAStore) Cert(id string) (*pki.Certificate, error) { +func (c *VFSCAStore) Cert(id string, createIfMissing bool) (*pki.Certificate, error) { cert, err := c.FindCert(id) if err == nil && cert == nil { - if c.DryRun { + if !createIfMissing { glog.Warningf("using empty certificate, because running with DryRun") return &pki.Certificate{}, err } @@ -273,10 +272,10 @@ func (c *VFSCAStore) Cert(id string) (*pki.Certificate, error) { } -func (c *VFSCAStore) CertificatePool(id string) (*CertificatePool, error) { +func (c *VFSCAStore) CertificatePool(id string, createIfMissing bool) (*CertificatePool, error) { cert, err := c.FindCertificatePool(id) if err == nil && cert == nil { - if c.DryRun { + if !createIfMissing { glog.Warningf("using empty certificate, because running with DryRun") return &CertificatePool{}, err } @@ -570,10 +569,10 @@ func (c *VFSCAStore) FindPrivateKey(id string) (*pki.PrivateKey, error) { return key, nil } -func (c *VFSCAStore) PrivateKey(id string) (*pki.PrivateKey, error) { +func (c *VFSCAStore) PrivateKey(id string, createIfMissing bool) (*pki.PrivateKey, error) { key, err := c.FindPrivateKey(id) if err == nil && key == nil { - if c.DryRun { + if !createIfMissing { glog.Warningf("using empty certificate, because running with DryRun") return &pki.PrivateKey{}, err } @@ -647,6 +646,7 @@ func insertFingerprintColons(id string) string { if len(remaining) < 2 { glog.Warningf("unexpected format for SSH public key id: %q", id) buf.WriteString(remaining) + break } else { buf.WriteString(remaining[0:2]) remaining = remaining[2:] diff --git a/upup/pkg/kutil/convert_kubeup_cluster.go b/upup/pkg/kutil/convert_kubeup_cluster.go index 396f93d330..4be0b1e5f8 100644 --- a/upup/pkg/kutil/convert_kubeup_cluster.go +++ b/upup/pkg/kutil/convert_kubeup_cluster.go @@ -480,7 +480,7 @@ func (x *ConvertKubeupCluster) Upgrade() error { return fmt.Errorf("error writing completed cluster spec: %v", err) } - oldCACertPool, err := oldKeyStore.CertificatePool(fi.CertificateId_CA) + oldCACertPool, err := oldKeyStore.CertificatePool(fi.CertificateId_CA, true) if err != nil { return fmt.Errorf("error reading old CA certs: %v", err) }