Merge pull request #3409 from justinsb/clientset_secret_stores

Automatic merge from submit-queue. .

SecretStore and CAStore implementations backed by API

nodeup/pkg/model/context.go

@@ -98,16 +98,16 @@ func (c *NodeupModelContext) CNIConfDir() string {
 
 // buildPKIKubeconfig generates a kubeconfig
 func (c *NodeupModelContext) buildPKIKubeconfig(id string) (string, error) {
-	caCertificate, err := c.KeyStore.Cert(fi.CertificateId_CA)
+	caCertificate, err := c.KeyStore.Cert(fi.CertificateId_CA, false)
 	if err != nil {
 		return "", fmt.Errorf("error fetching CA certificate from keystore: %v", err)
 	}
 
-	certificate, err := c.KeyStore.Cert(id)
+	certificate, err := c.KeyStore.Cert(id, false)
 	if err != nil {
 		return "", fmt.Errorf("error fetching %q certificate from keystore: %v", id, err)
 	}
-	privateKey, err := c.KeyStore.PrivateKey(id)
+	privateKey, err := c.KeyStore.PrivateKey(id, false)
 	if err != nil {
 		return "", fmt.Errorf("error fetching %q private key from keystore: %v", id, err)
 	}

nodeup/pkg/model/convenience.go

@@ -94,7 +94,7 @@ func getProxyEnvVars(proxies *kops.EgressProxySpec) []v1.EnvVar {
 
 // buildCertificateRequest retrieves the certificate from a keystore
 func buildCertificateRequest(c *fi.ModelBuilderContext, b *NodeupModelContext, name, path string) error {
-	cert, err := b.KeyStore.Cert(name)
+	cert, err := b.KeyStore.Cert(name, false)
 	if err != nil {
 		return err
 	}
@@ -120,7 +120,7 @@ func buildCertificateRequest(c *fi.ModelBuilderContext, b *NodeupModelContext, n
 
 // buildPrivateKeyRequest retrieves a private key from the store
 func buildPrivateKeyRequest(c *fi.ModelBuilderContext, b *NodeupModelContext, name, path string) error {
-	k, err := b.KeyStore.PrivateKey(name)
+	k, err := b.KeyStore.PrivateKey(name, false)
 	if err != nil {
 		return err
 	}

nodeup/pkg/model/kubecontrollermanager.go

@@ -45,7 +45,7 @@ func (b *KubeControllerManagerBuilder) Build(c *fi.ModelBuilderContext) error {
 	// If we're using the CertificateSigner, include the CA Key
 	// TODO: use a per-machine key?  use KMS?
 	if b.useCertificateSigner() {
-		ca, err := b.KeyStore.PrivateKey(fi.CertificateId_CA)
+		ca, err := b.KeyStore.PrivateKey(fi.CertificateId_CA, false)
 		if err != nil {
 			return err
 		}

nodeup/pkg/model/protokube.go

@@ -331,7 +331,7 @@ func (t *ProtokubeBuilder) writeProxyEnvVars(buffer *bytes.Buffer) {
 
 // buildCertificateTask is responsible for build a certificate request task
 func (t *ProtokubeBuilder) buildCeritificateTask(c *fi.ModelBuilderContext, name, filename string) error {
-	cert, err := t.KeyStore.Cert(name)
+	cert, err := t.KeyStore.Cert(name, false)
 	if err != nil {
 		return err
 	}
@@ -353,7 +353,7 @@ func (t *ProtokubeBuilder) buildCeritificateTask(c *fi.ModelBuilderContext, name
 
 // buildPrivateKeyTask is responsible for build a certificate request task
 func (t *ProtokubeBuilder) buildPrivateTask(c *fi.ModelBuilderContext, name, filename string) error {
-	cert, err := t.KeyStore.PrivateKey(name)
+	cert, err := t.KeyStore.PrivateKey(name, false)
 	if err != nil {
 		return err
 	}

nodeup/pkg/model/secrets.go

@@ -40,7 +40,7 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error {
 
 	// retrieve the platform ca
 	{
-		ca, err := b.KeyStore.CertificatePool(fi.CertificateId_CA)
+		ca, err := b.KeyStore.CertificatePool(fi.CertificateId_CA, false)
 		if err != nil {
 			return err
 		}
@@ -79,7 +79,7 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error {
 	}
 
 	{
-		cert, err := b.KeyStore.Cert("master")
+		cert, err := b.KeyStore.Cert("master", false)
 		if err != nil {
 			return err
 		}
@@ -98,7 +98,7 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error {
 	}
 
 	{
-		k, err := b.KeyStore.PrivateKey("master")
+		k, err := b.KeyStore.PrivateKey("master", false)
 		if err != nil {
 			return err
 		}
@@ -118,7 +118,7 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error {
 
 	if b.IsKubernetesGTE("1.7") {
 
-		cert, err := b.KeyStore.Cert("apiserver-proxy-client")
+		cert, err := b.KeyStore.Cert("apiserver-proxy-client", false)
 		if err != nil {
 			return fmt.Errorf("apiserver proxy client cert lookup failed: %v", err.Error())
 		}
@@ -135,7 +135,7 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error {
 		}
 		c.AddTask(t)
 
-		key, err := b.KeyStore.PrivateKey("apiserver-proxy-client")
+		key, err := b.KeyStore.PrivateKey("apiserver-proxy-client", false)
 		if err != nil {
 			return fmt.Errorf("apiserver proxy client private key lookup failed: %v", err.Error())
 		}

upup/pkg/fi/ca.go

@@ -59,10 +59,10 @@ type CAStore interface {
 	Keystore
 
 	// Cert returns the primary specified certificate
-	Cert(name string) (*pki.Certificate, error)
+	Cert(name string, createIfMissing bool) (*pki.Certificate, error)
 	// CertificatePool returns all active certificates with the specified id
-	CertificatePool(name string) (*CertificatePool, error)
+	CertificatePool(name string, createIfMissing bool) (*CertificatePool, error)
-	PrivateKey(name string) (*pki.PrivateKey, error)
+	PrivateKey(name string, createIfMissing bool) (*pki.PrivateKey, error)
 
 	FindCert(name string) (*pki.Certificate, error)
 	FindPrivateKey(name string) (*pki.PrivateKey, error)

upup/pkg/fi/clientset_castore.go

@@ -0,0 +1,624 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fi
+
+import (
+	"bytes"
+	crypto_rand "crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"fmt"
+	"math/big"
+	"sync"
+	"time"
+
+	"github.com/golang/glog"
+	"golang.org/x/crypto/ssh"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kops/pkg/apis/kops"
+	kopsinternalversion "k8s.io/kops/pkg/client/clientset_generated/clientset/typed/kops/internalversion"
+	"k8s.io/kops/pkg/pki"
+	"k8s.io/kops/util/pkg/vfs"
+)
+
+// ClientsetCAStore is a CAStore implementation that stores keypairs in Keyset on a API server
+type ClientsetCAStore struct {
+	namespace string
+	clientset kopsinternalversion.KopsInterface
+
+	mutex         sync.Mutex
+	cacheCaKeyset *keyset
+}
+
+var _ CAStore = &ClientsetCAStore{}
+
+// NewClientsetCAStore is the constructor for ClientsetCAStore
+func NewClientsetCAStore(clientset kopsinternalversion.KopsInterface, namespace string) CAStore {
+	c := &ClientsetCAStore{
+		clientset: clientset,
+		namespace: namespace,
+	}
+
+	return c
+}
+
+// readCAKeypairs retrieves the CA keypair, generating a new keypair if not found
+func (c *ClientsetCAStore) readCAKeypairs() (*keyset, error) {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+
+	if c.cacheCaKeyset != nil {
+		return c.cacheCaKeyset, nil
+	}
+
+	keyset, err := c.loadKeyset(CertificateId_CA)
+	if err != nil {
+		return nil, err
+	}
+
+	if keyset == nil {
+		keyset, err = c.generateCACertificate()
+		if err != nil {
+			return nil, err
+		}
+
+	}
+	c.cacheCaKeyset = keyset
+
+	return keyset, nil
+}
+
+// generateCACertificate creates and stores a CA keypair
+// Should be called with the mutex held, to prevent concurrent creation of different keys
+func (c *ClientsetCAStore) generateCACertificate() (*keyset, error) {
+	template := BuildCAX509Template()
+
+	caRsaKey, err := rsa.GenerateKey(crypto_rand.Reader, 2048)
+	if err != nil {
+		return nil, fmt.Errorf("error generating RSA private key: %v", err)
+	}
+
+	caPrivateKey := &pki.PrivateKey{Key: caRsaKey}
+
+	t := time.Now().UnixNano()
+	template.SerialNumber = pki.BuildPKISerial(t)
+
+	caCertificate, err := pki.SignNewCertificate(caPrivateKey, template, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return c.storeAndVerifyKeypair(CertificateId_CA, caCertificate, caPrivateKey)
+}
+
+// keyset is a parsed Keyset
+type keyset struct {
+	items   map[string]*keysetItem
+	primary *keysetItem
+}
+
+// keysetItem is a parsed KeysetItem
+type keysetItem struct {
+	id          string
+	certificate *pki.Certificate
+	privateKey  *pki.PrivateKey
+}
+
+// loadKeyset gets the named keyset
+func (c *ClientsetCAStore) loadKeyset(name string) (*keyset, error) {
+	o, err := c.clientset.Keysets(c.namespace).Get(name, v1.GetOptions{})
+	if err != nil {
+		if errors.IsNotFound(err) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("error reading keyset %q: %v", name, err)
+	}
+
+	keyset := &keyset{
+		items: make(map[string]*keysetItem),
+	}
+
+	for _, key := range o.Spec.Keys {
+		cert, err := pki.LoadPEMCertificate(key.PublicMaterial)
+		if err != nil {
+			glog.Warningf("key public material was %s", key.PublicMaterial)
+			return nil, fmt.Errorf("error loading certificate %s/%s: %v", name, key.Id, err)
+		}
+		privateKey, err := pki.ParsePEMPrivateKey(key.PrivateMaterial)
+		if err != nil {
+			return nil, fmt.Errorf("error loading private key %s/%s: %v", name, key.Id, err)
+		}
+		keyset.items[key.Id] = &keysetItem{
+			id:          key.Id,
+			certificate: cert,
+			privateKey:  privateKey,
+		}
+	}
+
+	primary := FindPrimary(o)
+	if primary != nil {
+		keyset.primary = keyset.items[primary.Id]
+	}
+
+	return keyset, nil
+}
+
+// FindPrimary returns the primary KeysetItem in the Keyset
+func FindPrimary(keyset *kops.Keyset) *kops.KeysetItem {
+	var primary *kops.KeysetItem
+	var primaryVersion *big.Int
+	for i := range keyset.Spec.Keys {
+		item := &keyset.Spec.Keys[i]
+		version, ok := big.NewInt(0).SetString(item.Id, 10)
+		if !ok {
+			glog.Warningf("Ignoring key item with non-integer version: %q", item.Id)
+			continue
+		}
+
+		if primaryVersion == nil || version.Cmp(primaryVersion) > 0 {
+			primary = item
+			primaryVersion = version
+		}
+	}
+	return primary
+}
+
+// Cert implements CAStore::Cert
+func (c *ClientsetCAStore) Cert(name string, createIfMissing bool) (*pki.Certificate, error) {
+	cert, err := c.FindCert(name)
+	if err == nil && cert == nil {
+		if !createIfMissing {
+			glog.Warningf("using empty certificate, because running with DryRun")
+			return &pki.Certificate{}, err
+		}
+		return nil, fmt.Errorf("cannot find certificate %q", name)
+	}
+	return cert, err
+
+}
+
+// CertificatePool implements CAStore::CertificatePool
+func (c *ClientsetCAStore) CertificatePool(id string, createIfMissing bool) (*CertificatePool, error) {
+	cert, err := c.FindCertificatePool(id)
+	if err == nil && cert == nil {
+		if !createIfMissing {
+			glog.Warningf("using empty certificate, because running with DryRun")
+			return &CertificatePool{}, err
+		}
+		return nil, fmt.Errorf("cannot find certificate pool %q", id)
+	}
+	return cert, err
+
+}
+
+// FindKeypair implements CAStore::FindKeypair
+func (c *ClientsetCAStore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, error) {
+	keyset, err := c.loadKeyset(name)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if keyset != nil && keyset.primary != nil {
+		return keyset.primary.certificate, keyset.primary.privateKey, nil
+	}
+
+	return nil, nil, nil
+}
+
+// FindCert implements CAStore::FindCert
+func (c *ClientsetCAStore) FindCert(name string) (*pki.Certificate, error) {
+	keyset, err := c.loadKeyset(name)
+	if err != nil {
+		return nil, err
+	}
+
+	var cert *pki.Certificate
+	if keyset != nil && keyset.primary != nil {
+		cert = keyset.primary.certificate
+	}
+
+	return cert, nil
+}
+
+// FindCertificatePool implements CAStore::FindCertificatePool
+func (c *ClientsetCAStore) FindCertificatePool(name string) (*CertificatePool, error) {
+	keyset, err := c.loadKeyset(name)
+	if err != nil {
+		return nil, err
+	}
+
+	pool := &CertificatePool{}
+
+	if keyset != nil {
+		if keyset.primary != nil {
+			pool.Primary = keyset.primary.certificate
+		}
+
+		for id, item := range keyset.items {
+			if id == keyset.primary.id {
+				continue
+			}
+			pool.Secondary = append(pool.Secondary, item.certificate)
+		}
+	}
+	return pool, nil
+}
+
+// List implements CAStore::List
+func (c *ClientsetCAStore) List() ([]*KeystoreItem, error) {
+	var items []*KeystoreItem
+
+	{
+		list, err := c.clientset.Keysets(c.namespace).List(v1.ListOptions{})
+		if err != nil {
+			return nil, fmt.Errorf("error listing Keysets: %v", err)
+		}
+
+		for _, keyset := range list.Items {
+			for _, item := range keyset.Spec.Keys {
+				ki := &KeystoreItem{
+					Name: keyset.Name,
+					Id:   item.Id,
+				}
+
+				switch keyset.Spec.Type {
+				case kops.SecretTypeKeypair:
+					ki.Type = SecretTypeKeypair
+				case kops.SecretTypeSecret:
+					//ki.Type = SecretTypeSecret
+					continue // Ignore - this is handled by ClientsetSecretStore
+				default:
+					return nil, fmt.Errorf("unhandled secret type %q: %v", ki.Type, err)
+				}
+				items = append(items, ki)
+			}
+		}
+	}
+
+	{
+		list, err := c.clientset.SSHCredentials(c.namespace).List(v1.ListOptions{})
+		if err != nil {
+			return nil, fmt.Errorf("error listing SSHCredentials: %v", err)
+		}
+
+		for _, sshCredential := range list.Items {
+			ki := &KeystoreItem{
+				Name: sshCredential.Name,
+				Type: SecretTypeSSHPublicKey,
+			}
+			items = append(items, ki)
+		}
+	}
+
+	return items, nil
+}
+
+// IssueCert implements CAStore::IssueCert
+func (c *ClientsetCAStore) IssueCert(name string, serial *big.Int, privateKey *pki.PrivateKey, template *x509.Certificate) (*pki.Certificate, error) {
+	glog.Infof("Issuing new certificate: %q", name)
+
+	template.SerialNumber = serial
+
+	caKeyset, err := c.readCAKeypairs()
+	if err != nil {
+		return nil, err
+	}
+
+	if caKeyset == nil {
+		return nil, fmt.Errorf("ca keyset was not found; cannot issue certificates")
+	}
+	if caKeyset.primary == nil {
+		return nil, fmt.Errorf("ca keyset did not have any key data; cannot issue certificates")
+	}
+	if caKeyset.primary.certificate == nil {
+		return nil, fmt.Errorf("ca certificate was not found; cannot issue certificates")
+	}
+	if caKeyset.primary.privateKey == nil {
+		return nil, fmt.Errorf("ca privateKey was not found; cannot issue certificates")
+	}
+	cert, err := pki.SignNewCertificate(privateKey, template, caKeyset.primary.certificate.Certificate, caKeyset.primary.privateKey)
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err := c.storeAndVerifyKeypair(name, cert, privateKey); err != nil {
+		return nil, err
+	}
+
+	return cert, nil
+}
+
+// storeAndVerifyKeypair writes the keypair, also re-reading it to double-check it
+func (c *ClientsetCAStore) storeAndVerifyKeypair(name string, cert *pki.Certificate, privateKey *pki.PrivateKey) (*keyset, error) {
+	id := cert.Certificate.SerialNumber.String()
+	if err := c.storeKeypair(name, id, cert, privateKey); err != nil {
+		return nil, err
+	}
+
+	// Make double-sure it round-trips
+	keyset, err := c.loadKeyset(name)
+	if err != nil {
+		return nil, fmt.Errorf("error fetching stored certificate: %v", err)
+	}
+
+	if keyset == nil {
+		return nil, fmt.Errorf("stored certificate not found: %v", err)
+	}
+	if keyset.primary == nil {
+		return nil, fmt.Errorf("stored certificate did not have data: %v", err)
+	}
+	if keyset.primary.id != id {
+		return nil, fmt.Errorf("stored certificate changed concurrently (id mismatch)")
+	}
+	return keyset, nil
+}
+
+// StoreKeypair implements CAStore::StoreKeypair
+func (c *ClientsetCAStore) StoreKeypair(name string, cert *pki.Certificate, privateKey *pki.PrivateKey) error {
+	return c.storeKeypair(name, cert.Certificate.SerialNumber.String(), cert, privateKey)
+}
+
+// AddCert implements CAStore::AddCert
+func (c *ClientsetCAStore) AddCert(name string, cert *pki.Certificate) error {
+	glog.Infof("Adding TLS certificate: %q", name)
+
+	// We add with a timestamp of zero so this will never be the newest cert
+	serial := pki.BuildPKISerial(0)
+
+	err := c.storeKeypair(name, serial.String(), cert, nil)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// FindPrivateKey implements CAStore::FindPrivateKey
+func (c *ClientsetCAStore) FindPrivateKey(name string) (*pki.PrivateKey, error) {
+	keyset, err := c.loadKeyset(name)
+	if err != nil {
+		return nil, err
+	}
+
+	if keyset != nil && keyset.primary != nil {
+		return keyset.primary.privateKey, nil
+	}
+	return nil, nil
+}
+
+// PrivateKey implements CAStore::PrivateKey
+func (c *ClientsetCAStore) PrivateKey(name string, createIfMissing bool) (*pki.PrivateKey, error) {
+	key, err := c.FindPrivateKey(name)
+	if err == nil && key == nil {
+		if !createIfMissing {
+			glog.Warningf("using empty certificate, because running with DryRun")
+			return &pki.PrivateKey{}, err
+		}
+		return nil, fmt.Errorf("cannot find SSL key %q", name)
+	}
+	return key, err
+}
+
+// CreateKeypair implements CAStore::CreateKeypair
+func (c *ClientsetCAStore) CreateKeypair(id string, template *x509.Certificate, privateKey *pki.PrivateKey) (*pki.Certificate, error) {
+	serial := c.buildSerial()
+
+	cert, err := c.IssueCert(id, serial, privateKey, template)
+	if err != nil {
+		return nil, err
+	}
+
+	return cert, nil
+}
+
+// addKey saves the specified key to the registry
+func (c *ClientsetCAStore) addKey(name string, keysetType kops.KeysetType, item *kops.KeysetItem) error {
+	create := false
+	client := c.clientset.Keysets(c.namespace)
+	keyset, err := client.Get(name, v1.GetOptions{})
+	if err != nil {
+		if errors.IsNotFound(err) {
+			keyset = nil
+		} else {
+			return fmt.Errorf("error reading keyset %q: %v", name, err)
+		}
+	}
+	if keyset == nil {
+		keyset = &kops.Keyset{}
+		keyset.Name = name
+		keyset.Spec.Type = keysetType
+		create = true
+	}
+	keyset.Spec.Keys = append(keyset.Spec.Keys, *item)
+	if create {
+		if _, err := client.Create(keyset); err != nil {
+			return fmt.Errorf("error creating keyset %q: %v", name, err)
+		}
+	} else {
+		if _, err := client.Update(keyset); err != nil {
+			return fmt.Errorf("error updating keyset %q: %v", name, err)
+		}
+	}
+	return nil
+}
+
+// DeleteKeysetItem deletes the specified key from the registry; deleting the whole keyset if it was the last one
+func DeleteKeysetItem(client kopsinternalversion.KeysetInterface, name string, keysetType kops.KeysetType, id string) error {
+	keyset, err := client.Get(name, v1.GetOptions{})
+	if err != nil {
+		if errors.IsNotFound(err) {
+			return nil
+		} else {
+			return fmt.Errorf("error reading Keyset %q: %v", name, err)
+		}
+	}
+
+	if keyset.Spec.Type != keysetType {
+		return fmt.Errorf("mismatch on Keyset type on %q", name)
+	}
+
+	var newKeys []kops.KeysetItem
+	found := false
+	for _, ki := range keyset.Spec.Keys {
+		if ki.Id == id {
+			found = true
+		} else {
+			newKeys = append(newKeys, ki)
+		}
+	}
+	if !found {
+		return fmt.Errorf("KeysetItem %q not found in Keyset %q", id, name)
+	}
+	if len(newKeys) == 0 {
+		if err := client.Delete(name, &v1.DeleteOptions{}); err != nil {
+			return fmt.Errorf("error deleting Keyset %q: %v", name, err)
+		}
+	} else {
+		keyset.Spec.Keys = newKeys
+		if _, err := client.Update(keyset); err != nil {
+			return fmt.Errorf("error updating Keyset %q: %v", name, err)
+		}
+	}
+	return nil
+}
+
+// addSshCredential saves the specified SSH Credential to the registry, doing an update or insert
+func (c *ClientsetCAStore) addSshCredential(name string, publicKey string) error {
+	create := false
+	client := c.clientset.SSHCredentials(c.namespace)
+	sshCredential, err := client.Get(name, v1.GetOptions{})
+	if err != nil {
+		if errors.IsNotFound(err) {
+			sshCredential = nil
+		} else {
+			return fmt.Errorf("error reading SSHCredential %q: %v", name, err)
+		}
+	}
+	if sshCredential == nil {
+		sshCredential = &kops.SSHCredential{}
+		sshCredential.Name = name
+		create = true
+	}
+	sshCredential.Spec.PublicKey = publicKey
+	if create {
+		if _, err := client.Create(sshCredential); err != nil {
+			return fmt.Errorf("error creating SSHCredential %q: %v", name, err)
+		}
+	} else {
+		if _, err := client.Update(sshCredential); err != nil {
+			return fmt.Errorf("error updating SSHCredential %q: %v", name, err)
+		}
+	}
+	return nil
+}
+
+// deleteSSHCredential deletes the specified SSHCredential from the registry
+func (c *ClientsetCAStore) deleteSSHCredential(name string) error {
+	client := c.clientset.SSHCredentials(c.namespace)
+	err := client.Delete(name, &v1.DeleteOptions{})
+	if err != nil {
+		return fmt.Errorf("error deleting SSHCredential %q: %v", name, err)
+	}
+	return nil
+}
+
+// addKey saves the specified keypair to the registry
+func (c *ClientsetCAStore) storeKeypair(name string, id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error {
+	var publicMaterial bytes.Buffer
+	if _, err := cert.WriteTo(&publicMaterial); err != nil {
+		return err
+	}
+
+	var privateMaterial bytes.Buffer
+	if _, err := privateKey.WriteTo(&privateMaterial); err != nil {
+		return err
+	}
+
+	item := &kops.KeysetItem{
+		Id:              id,
+		PublicMaterial:  publicMaterial.Bytes(),
+		PrivateMaterial: privateMaterial.Bytes(),
+	}
+	return c.addKey(name, kops.SecretTypeKeypair, item)
+}
+
+// buildSerial returns a serial for use when issuing certificates
+func (c *ClientsetCAStore) buildSerial() *big.Int {
+	t := time.Now().UnixNano()
+	return pki.BuildPKISerial(t)
+}
+
+// AddSSHPublicKey implements CAStore::AddSSHPublicKey
+func (c *ClientsetCAStore) AddSSHPublicKey(name string, pubkey []byte) error {
+	_, _, _, _, err := ssh.ParseAuthorizedKey(pubkey)
+	if err != nil {
+		return fmt.Errorf("error parsing SSH public key: %v", err)
+	}
+
+	// TODO: Reintroduce or remove
+	//// compute fingerprint to serve as id
+	//h := md5.New()
+	//_, err = h.Write(sshPublicKey.Marshal())
+	//if err != nil {
+	//	return err
+	//}
+	//id = formatFingerprint(h.Sum(nil))
+
+	return c.addSshCredential(name, string(pubkey))
+}
+
+// FindSSHPublicKeys implements CAStore::FindSSHPublicKeys
+func (c *ClientsetCAStore) FindSSHPublicKeys(name string) ([]*KeystoreItem, error) {
+	o, err := c.clientset.SSHCredentials(c.namespace).Get(name, v1.GetOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("error reading SSHCredential %q: %v", name, err)
+	}
+
+	var items []*KeystoreItem
+	item := &KeystoreItem{
+		Type: SecretTypeSSHPublicKey,
+		Name: name,
+		//Id:   insertFingerprintColons(k.Id),
+		Data: []byte(o.Spec.PublicKey),
+	}
+	items = append(items, item)
+
+	return items, nil
+}
+
+// DeleteSecret implements CAStore::DeleteSecret
+func (c *ClientsetCAStore) DeleteSecret(item *KeystoreItem) error {
+	switch item.Type {
+	case SecretTypeSSHPublicKey:
+		return c.deleteSSHCredential(item.Name)
+
+	case SecretTypeKeypair:
+		client := c.clientset.Keysets(c.namespace)
+		return DeleteKeysetItem(client, item.Name, kops.SecretTypeKeypair, item.Id)
+
+	default:
+		// Primarily because we need to make sure users can recreate them!
+		return fmt.Errorf("deletion of keystore items of type %v not (yet) supported", item.Type)
+	}
+}
+
+// VFSPath implements CAStore::VFSPath
+func (c *ClientsetCAStore) VFSPath() vfs.Path {
+	// We will implement mirroring instead
+	panic("ClientsetCAStore::VFSPath not implemented")
+}

upup/pkg/fi/cloudup/apply_cluster.go

@@ -192,7 +192,6 @@ func (c *ApplyClusterCmd) Run() error {
 	if err != nil {
 		return err
 	}
-	keyStore.(*fi.VFSCAStore).DryRun = c.DryRun
 
 	secretStore, err := registry.SecretStore(cluster)
 	if err != nil {

upup/pkg/fi/cloudup/populate_cluster_spec.go

@@ -181,8 +181,6 @@ func (c *populateClusterSpec) run() error {
 	if err != nil {
 		return err
 	}
-	// Always assume a dry run during this phase
-	keyStore.(*fi.VFSCAStore).DryRun = true
 
 	secretStore, err := registry.SecretStore(cluster)
 	if err != nil {

upup/pkg/fi/nodeup/template_functions.go

@@ -27,7 +27,6 @@ import (
 	api "k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/apis/nodeup"
 	"k8s.io/kops/pkg/flagbuilder"
-	"k8s.io/kops/pkg/pki"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/secrets"
 	"k8s.io/kops/util/pkg/vfs"
@@ -93,9 +92,6 @@ func (t *templateFunctions) populate(dest template.FuncMap) {
 		return runtime.GOARCH
 	}
 
-	dest["CACertificate"] = t.CACertificate
-	dest["PrivateKey"] = t.PrivateKey
-	dest["Certificate"] = t.Certificate
 	dest["GetToken"] = t.GetToken
 
 	dest["BuildFlags"] = flagbuilder.BuildFlags
@@ -122,21 +118,6 @@ func (t *templateFunctions) populate(dest template.FuncMap) {
 	}
 }
 
-// CACertificate returns the primary CA certificate for the cluster
-func (t *templateFunctions) CACertificate() (*pki.Certificate, error) {
-	return t.keyStore.Cert(fi.CertificateId_CA)
-}
-
-// PrivateKey returns the specified private key
-func (t *templateFunctions) PrivateKey(id string) (*pki.PrivateKey, error) {
-	return t.keyStore.PrivateKey(id)
-}
-
-// Certificate returns the specified private key
-func (t *templateFunctions) Certificate(id string) (*pki.Certificate, error) {
-	return t.keyStore.Cert(id)
-}
-
 // GetToken returns the specified token
 func (t *templateFunctions) GetToken(key string) (string, error) {
 	token, err := t.secretStore.FindSecret(key)

upup/pkg/fi/secrets.go

@@ -25,15 +25,15 @@ import (
 )
 
 type SecretStore interface {
-	// Get a secret.  Returns an error if not found
+	// Secret returns a secret.  Returns an error if not found
 	Secret(id string) (*Secret, error)
 	// DeleteSecret deletes the specified secret
 	DeleteSecret(item *KeystoreItem) error
-	// Find a secret, if exists.  Returns nil,nil if not found
+	// FindSecret finds a secret, if exists.  Returns nil,nil if not found
 	FindSecret(id string) (*Secret, error)
-	// Create or replace a secret
+	// GetOrCreateSecret creates or replace a secret
 	GetOrCreateSecret(id string, secret *Secret) (current *Secret, created bool, err error)
-	// Lists the ids of all known secrets
+	// ListSecrets lists the ids of all known secrets
 	ListSecrets() ([]string, error)
 
 	// VFSPath returns the path where the SecretStore is stored

upup/pkg/fi/secrets/clientset_secretstore.go

@@ -0,0 +1,186 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package secrets
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kops/pkg/apis/kops"
+	kopsinternalversion "k8s.io/kops/pkg/client/clientset_generated/clientset/typed/kops/internalversion"
+	"k8s.io/kops/pkg/pki"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/util/pkg/vfs"
+)
+
+// NamePrefix is a prefix we use to avoid collisions with other keysets
+const NamePrefix = "token-"
+
+// ClientsetSecretStore is a SecretStore backed by Keyset objects in an API server
+type ClientsetSecretStore struct {
+	namespace string
+	clientset kopsinternalversion.KopsInterface
+}
+
+var _ fi.SecretStore = &ClientsetSecretStore{}
+
+// NewClientsetSecretStore is the constructor for ClientsetSecretStore
+func NewClientsetSecretStore(clientset kopsinternalversion.KopsInterface, namespace string) fi.SecretStore {
+	c := &ClientsetSecretStore{
+		clientset: clientset,
+		namespace: namespace,
+	}
+	return c
+}
+
+// FindSecret implements fi.SecretStore::FindSecret
+func (c *ClientsetSecretStore) FindSecret(name string) (*fi.Secret, error) {
+	s, err := c.loadSecret(name)
+	if err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
+// ListSecrets implements fi.SecretStore::ListSecrets
+func (c *ClientsetSecretStore) ListSecrets() ([]string, error) {
+	list, err := c.clientset.Keysets(c.namespace).List(v1.ListOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("error listing keysets: %v", err)
+	}
+
+	var names []string
+	for i := range list.Items {
+		keyset := &list.Items[i]
+
+		switch keyset.Spec.Type {
+		case kops.SecretTypeSecret:
+			name := strings.TrimPrefix(keyset.Name, NamePrefix)
+			names = append(names, name)
+		}
+	}
+
+	return names, nil
+}
+
+// Secret implements fi.SecretStore::Secret
+func (c *ClientsetSecretStore) Secret(name string) (*fi.Secret, error) {
+	s, err := c.FindSecret(name)
+	if err != nil {
+		return nil, err
+	}
+	if s == nil {
+		return nil, fmt.Errorf("Secret not found: %q", name)
+	}
+	return s, nil
+}
+
+// DeleteSecret implements fi.SecretStore::DeleteSecret
+func (c *ClientsetSecretStore) DeleteSecret(item *fi.KeystoreItem) error {
+	client := c.clientset.Keysets(c.namespace)
+	return fi.DeleteKeysetItem(client, item.Name, kops.SecretTypeKeypair, item.Id)
+}
+
+// GetOrCreateSecret implements fi.SecretStore::GetOrCreateSecret
+func (c *ClientsetSecretStore) GetOrCreateSecret(name string, secret *fi.Secret) (*fi.Secret, bool, error) {
+	for i := 0; i < 2; i++ {
+		s, err := c.FindSecret(name)
+		if err != nil {
+			return nil, false, err
+		}
+
+		if s != nil {
+			return s, false, nil
+		}
+
+		_, err = c.createSecret(secret, name)
+		if err != nil {
+			if errors.IsAlreadyExists(err) && i == 0 {
+				glog.Infof("Got already-exists error when writing secret; likely due to concurrent creation.  Will retry")
+				continue
+			} else {
+				return nil, false, err
+			}
+		}
+
+		if err == nil {
+			break
+		}
+	}
+
+	// Make double-sure it round-trips
+	s, err := c.loadSecret(name)
+	if err != nil {
+		glog.Fatalf("unable to load secret immmediately after creation %v: %v", name, err)
+		return nil, false, err
+	}
+	return s, true, nil
+}
+
+// loadSecret returns the named secret, if it exists, otherwise returns nil
+func (c *ClientsetSecretStore) loadSecret(name string) (*fi.Secret, error) {
+	name = NamePrefix + name
+	keyset, err := c.clientset.Keysets(c.namespace).Get(name, v1.GetOptions{})
+	if err != nil {
+		if errors.IsNotFound(err) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("error reading keyset %q: %v", name, err)
+	}
+
+	return parseSecret(keyset)
+}
+
+// parseSecret attempts to parse the primary secret, otherwise returns nil
+func parseSecret(keyset *kops.Keyset) (*fi.Secret, error) {
+	primary := fi.FindPrimary(keyset)
+	if primary == nil {
+		return nil, nil
+	}
+
+	s := &fi.Secret{}
+	s.Data = primary.PrivateMaterial
+	return s, nil
+}
+
+// createSecret writes the secret, but only if it does not exist
+func (c *ClientsetSecretStore) createSecret(s *fi.Secret, name string) (*kops.Keyset, error) {
+	keyset := &kops.Keyset{}
+	keyset.Name = NamePrefix + name
+	keyset.Spec.Type = kops.SecretTypeSecret
+
+	t := time.Now().UnixNano()
+	id := pki.BuildPKISerial(t)
+
+	keyset.Spec.Keys = append(keyset.Spec.Keys, kops.KeysetItem{
+		Id:              id.String(),
+		PrivateMaterial: s.Data,
+	})
+
+	return c.clientset.Keysets(c.namespace).Create(keyset)
+}
+
+// VFSPath implements fi.SecretStore::VFSPath
+func (c *ClientsetSecretStore) VFSPath() vfs.Path {
+	// We will implement mirroring instead
+	glog.Fatalf("ClientsetSecretStore::VFSPath not implemented")
+	return nil
+}

upup/pkg/fi/secrets/vfs_secretstore.go

@@ -42,8 +42,8 @@ func (c *VFSSecretStore) VFSPath() vfs.Path {
 	return c.basedir
 }
 
-func (c *VFSSecretStore) buildSecretPath(id string) vfs.Path {
+func (c *VFSSecretStore) buildSecretPath(name string) vfs.Path {
-	return c.basedir.Join(id)
+	return c.basedir.Join(name)
 }
 
 func (c *VFSSecretStore) FindSecret(id string) (*fi.Secret, error) {

upup/pkg/fi/vfs_castore.go

@@ -38,7 +38,6 @@ import (
 )
 
 type VFSCAStore struct {
-	DryRun  bool
 	basedir vfs.Path
 
 	mutex               sync.Mutex
@@ -260,10 +259,10 @@ func (c *VFSCAStore) loadOneCertificate(p vfs.Path) (*pki.Certificate, error) {
 	return cert, nil
 }
 
-func (c *VFSCAStore) Cert(id string) (*pki.Certificate, error) {
+func (c *VFSCAStore) Cert(id string, createIfMissing bool) (*pki.Certificate, error) {
 	cert, err := c.FindCert(id)
 	if err == nil && cert == nil {
-		if c.DryRun {
+		if !createIfMissing {
 			glog.Warningf("using empty certificate, because running with DryRun")
 			return &pki.Certificate{}, err
 		}
@@ -273,10 +272,10 @@ func (c *VFSCAStore) Cert(id string) (*pki.Certificate, error) {
 
 }
 
-func (c *VFSCAStore) CertificatePool(id string) (*CertificatePool, error) {
+func (c *VFSCAStore) CertificatePool(id string, createIfMissing bool) (*CertificatePool, error) {
 	cert, err := c.FindCertificatePool(id)
 	if err == nil && cert == nil {
-		if c.DryRun {
+		if !createIfMissing {
 			glog.Warningf("using empty certificate, because running with DryRun")
 			return &CertificatePool{}, err
 		}
@@ -570,10 +569,10 @@ func (c *VFSCAStore) FindPrivateKey(id string) (*pki.PrivateKey, error) {
 	return key, nil
 }
 
-func (c *VFSCAStore) PrivateKey(id string) (*pki.PrivateKey, error) {
+func (c *VFSCAStore) PrivateKey(id string, createIfMissing bool) (*pki.PrivateKey, error) {
 	key, err := c.FindPrivateKey(id)
 	if err == nil && key == nil {
-		if c.DryRun {
+		if !createIfMissing {
 			glog.Warningf("using empty certificate, because running with DryRun")
 			return &pki.PrivateKey{}, err
 		}
@@ -647,6 +646,7 @@ func insertFingerprintColons(id string) string {
 		if len(remaining) < 2 {
 			glog.Warningf("unexpected format for SSH public key id: %q", id)
 			buf.WriteString(remaining)
+			break
 		} else {
 			buf.WriteString(remaining[0:2])
 			remaining = remaining[2:]

upup/pkg/kutil/convert_kubeup_cluster.go

@@ -480,7 +480,7 @@ func (x *ConvertKubeupCluster) Upgrade() error {
 		return fmt.Errorf("error writing completed cluster spec: %v", err)
 	}
 
-	oldCACertPool, err := oldKeyStore.CertificatePool(fi.CertificateId_CA)
+	oldCACertPool, err := oldKeyStore.CertificatePool(fi.CertificateId_CA, true)
 	if err != nil {
 		return fmt.Errorf("error reading old CA certs: %v", err)
 	}