Merge pull request #3411 from justinsb/mirror_stores

Automatic merge from submit-queue.

Mirror keystore & secretstore

cmd/kops/create.go

@@ -27,7 +27,6 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/kops/cmd/kops/util"
 	kopsapi "k8s.io/kops/pkg/apis/kops"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/pkg/apis/kops/v1alpha1"
 	"k8s.io/kops/upup/pkg/fi/cloudup"
 	"k8s.io/kops/util/pkg/vfs"
@@ -208,7 +207,7 @@ func RunCreate(f *util.Factory, out io.Writer, c *CreateOptions) error {
 					return err
 				}
 
-				keyStore, err := registry.KeyStore(cluster)
+				keyStore, err := clientset.KeyStore(cluster)
 				if err != nil {
 					return err
 				}

cmd/kops/create_cluster.go

@@ -121,6 +121,9 @@ type CreateClusterOptions struct {
 	// We need VSphereDatastore to support Kubernetes vSphere Cloud Provider (v1.5.3)
 	// We can remove this once we support higher versions.
 	VSphereDatastore string
+
+	// ConfigBase is the location where we will store the configuration, it defaults to the state store
+	ConfigBase string
 }
 
 func (o *CreateClusterOptions) InitDefaults() {
@@ -227,6 +230,11 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 	cmd.Flags().StringVar(&options.Target, "target", options.Target, "Target - direct, terraform, cloudformation")
 	cmd.Flags().StringVar(&options.Models, "model", options.Models, "Models to apply (separate multiple models with commas)")
 
+	// Configuration / state location
+	if featureflag.EnableSeparateConfigBase.Enabled() {
+		cmd.Flags().StringVar(&options.ConfigBase, "config-base", options.ConfigBase, "A cluster-readable location where we mirror configuration information, separate from the state store.  Allows for a state store that is not accessible from the cluster.")
+	}
+
 	cmd.Flags().StringVar(&options.Cloud, "cloud", options.Cloud, "Cloud provider to use - gce, aws, vsphere")
 
 	cmd.Flags().StringSliceVar(&options.Zones, "zones", options.Zones, "Zones in which to run the cluster")
@@ -371,6 +379,7 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
 	}
 	cluster.Spec.Channel = c.Channel
 
+	cluster.Spec.ConfigBase = c.ConfigBase
 	configBase, err := clientset.ConfigBaseFor(cluster)
 	if err != nil {
 		return fmt.Errorf("error building ConfigBase for cluster: %v", err)
@@ -933,7 +942,7 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
 	}
 
 	assetBuilder := assets.NewAssetBuilder(cluster.Spec.Assets)
-	fullCluster, err := cloudup.PopulateClusterSpec(cluster, assetBuilder)
+	fullCluster, err := cloudup.PopulateClusterSpec(clientset, cluster, assetBuilder)
 	if err != nil {
 		return err
 	}
@@ -958,7 +967,7 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
 		return fmt.Errorf("error writing updated configuration: %v", err)
 	}
 
-	keyStore, err := registry.KeyStore(cluster)
+	keyStore, err := clientset.KeyStore(cluster)
 	if err != nil {
 		return err
 	}

cmd/kops/create_secret_dockerconfig.go

@@ -25,7 +25,6 @@ import (
 
 	"github.com/spf13/cobra"
 	"k8s.io/kops/cmd/kops/util"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
 	"k8s.io/kubernetes/pkg/util/i18n"
@@ -97,7 +96,12 @@ func RunCreateSecretDockerConfig(f *util.Factory, out io.Writer, options *Create
 		return err
 	}
 
-	secretStore, err := registry.SecretStore(cluster)
+	clientset, err := f.Clientset()
+	if err != nil {
+		return err
+	}
+
+	secretStore, err := clientset.SecretStore(cluster)
 	if err != nil {
 		return err
 	}

cmd/kops/create_secret_encryptionconfig.go

@@ -25,7 +25,6 @@ import (
 	"github.com/spf13/cobra"
 	"k8s.io/kops/cmd/kops/util"
 	"k8s.io/kops/pkg/apis/kops"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
 	"k8s.io/kubernetes/pkg/util/i18n"
@@ -98,7 +97,12 @@ func RunCreateSecretEncryptionConfig(f *util.Factory, out io.Writer, options *Cr
 		return err
 	}
 
-	secretStore, err := registry.SecretStore(cluster)
+	clientset, err := f.Clientset()
+	if err != nil {
+		return err
+	}
+
+	secretStore, err := clientset.SecretStore(cluster)
 	if err != nil {
 		return err
 	}

cmd/kops/create_secret_sshpublickey.go

@@ -24,7 +24,6 @@ import (
 
 	"github.com/spf13/cobra"
 	"k8s.io/kops/cmd/kops/util"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
 	"k8s.io/kubernetes/pkg/util/i18n"
 )
@@ -99,7 +98,12 @@ func RunCreateSecretPublicKey(f *util.Factory, out io.Writer, options *CreateSec
 		return err
 	}
 
-	keyStore, err := registry.KeyStore(cluster)
+	clientset, err := f.Clientset()
+	if err != nil {
+		return err
+	}
+
+	keyStore, err := clientset.KeyStore(cluster)
 	if err != nil {
 		return err
 	}

cmd/kops/delete_cluster.go

@@ -24,7 +24,6 @@ import (
 	"github.com/spf13/cobra"
 	"k8s.io/kops/cmd/kops/util"
 	api "k8s.io/kops/pkg/apis/kops"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/pkg/kubeconfig"
 	"k8s.io/kops/pkg/resources"
 	"k8s.io/kops/pkg/resources/tracker"
@@ -32,7 +31,6 @@ import (
 	"k8s.io/kops/upup/pkg/fi/cloudup"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	"k8s.io/kops/util/pkg/tables"
-	"k8s.io/kops/util/pkg/vfs"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
 	"k8s.io/kubernetes/pkg/util/i18n"
 )
@@ -96,8 +94,6 @@ func NewCmdDeleteCluster(f *util.Factory, out io.Writer) *cobra.Command {
 type getter func(o interface{}) interface{}
 
 func RunDeleteCluster(f *util.Factory, out io.Writer, options *DeleteClusterOptions) error {
-	var configBase vfs.Path
-
 	clusterName := options.ClusterName
 	if clusterName == "" {
 		return fmt.Errorf("--name is required (for safety)")
@@ -123,11 +119,6 @@ func RunDeleteCluster(f *util.Factory, out io.Writer, options *DeleteClusterOpti
 		if err != nil {
 			return err
 		}
-
-		configBase, err = registry.ConfigBase(cluster)
-		if err != nil {
-			return err
-		}
 	}
 
 	wouldDeleteCloudResources := false
@@ -205,7 +196,11 @@ func RunDeleteCluster(f *util.Factory, out io.Writer, options *DeleteClusterOpti
 			}
 			return nil
 		}
-		err := registry.DeleteAllClusterState(configBase)
+		clientset, err := f.Clientset()
+		if err != nil {
+			return err
+		}
+		err = clientset.DeleteCluster(cluster)
 		if err != nil {
 			return fmt.Errorf("error removing cluster from state store: %v", err)
 		}

cmd/kops/delete_secret.go

@@ -22,7 +22,6 @@ import (
 
 	"github.com/spf13/cobra"
 	"k8s.io/kops/cmd/kops/util"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
 	"k8s.io/kubernetes/pkg/util/i18n"
@@ -88,17 +87,22 @@ func RunDeleteSecret(f *util.Factory, out io.Writer, options *DeleteSecretOption
 		return fmt.Errorf("SecretName is required")
 	}
 
+	clientset, err := f.Clientset()
+	if err != nil {
+		return err
+	}
+
 	cluster, err := GetCluster(f, options.ClusterName)
 	if err != nil {
 		return err
 	}
 
-	keyStore, err := registry.KeyStore(cluster)
+	keyStore, err := clientset.KeyStore(cluster)
 	if err != nil {
 		return err
 	}
 
-	secretStore, err := registry.SecretStore(cluster)
+	secretStore, err := clientset.SecretStore(cluster)
 	if err != nil {
 		return err
 	}

cmd/kops/describe_secrets.go

@@ -17,17 +17,15 @@ limitations under the License.
 package main
 
 import (
-	"fmt"
-
 	"bytes"
 	"crypto/rsa"
+	"fmt"
 	"os"
 	"sort"
 	"strings"
 	"text/tabwriter"
 
 	"github.com/spf13/cobra"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
 	"k8s.io/kubernetes/pkg/util/i18n"
@@ -77,12 +75,17 @@ func (c *DescribeSecretsCommand) Run(args []string) error {
 		return err
 	}
 
-	keyStore, err := registry.KeyStore(cluster)
+	clientset, err := rootCommand.Clientset()
 	if err != nil {
 		return err
 	}
 
-	secretStore, err := registry.SecretStore(cluster)
+	keyStore, err := clientset.KeyStore(cluster)
+	if err != nil {
+		return err
+	}
+
+	secretStore, err := clientset.SecretStore(cluster)
 	if err != nil {
 		return err
 	}

cmd/kops/edit_cluster.go

@@ -211,7 +211,7 @@ func RunEditCluster(f *util.Factory, cmd *cobra.Command, args []string, out io.W
 		}
 
 		assetBuilder := assets.NewAssetBuilder(newCluster.Spec.Assets)
-		fullCluster, err := cloudup.PopulateClusterSpec(newCluster, assetBuilder)
+		fullCluster, err := cloudup.PopulateClusterSpec(clientset, newCluster, assetBuilder)
 		if err != nil {
 			results = editResults{
 				file: file,

cmd/kops/edit_instancegroup.go

@@ -168,7 +168,7 @@ func RunEditInstanceGroup(f *util.Factory, cmd *cobra.Command, args []string, ou
 	}
 
 	assetBuilder := assets.NewAssetBuilder(cluster.Spec.Assets)
-	fullCluster, err := cloudup.PopulateClusterSpec(cluster, assetBuilder)
+	fullCluster, err := cloudup.PopulateClusterSpec(clientset, cluster, assetBuilder)
 	if err != nil {
 		return err
 	}

cmd/kops/export_kubecfg.go

@@ -21,7 +21,6 @@ import (
 
 	"github.com/spf13/cobra"
 	"k8s.io/kops/cmd/kops/util"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/pkg/kubeconfig"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
@@ -73,17 +72,22 @@ func RunExportKubecfg(f *util.Factory, out io.Writer, options *ExportKubecfgOpti
 		return err
 	}
 
+	clientset, err := rootCommand.Clientset()
+	if err != nil {
+		return err
+	}
+
 	cluster, err := rootCommand.Cluster()
 	if err != nil {
 		return err
 	}
 
-	keyStore, err := registry.KeyStore(cluster)
+	keyStore, err := clientset.KeyStore(cluster)
 	if err != nil {
 		return err
 	}
 
-	secretStore, err := registry.SecretStore(cluster)
+	secretStore, err := clientset.SecretStore(cluster)
 	if err != nil {
 		return err
 	}

cmd/kops/get_secrets.go

@@ -18,14 +18,12 @@ package main
 
 import (
 	"fmt"
+	"io"
 	"os"
-
 	"strings"
 
 	"github.com/spf13/cobra"
-	"io"
 	"k8s.io/kops/cmd/kops/util"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/util/pkg/tables"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
@@ -148,12 +146,17 @@ func RunGetSecrets(options *GetSecretsOptions, args []string) error {
 		return err
 	}
 
-	keyStore, err := registry.KeyStore(cluster)
+	clientset, err := rootCommand.Clientset()
 	if err != nil {
 		return err
 	}
 
-	secretStore, err := registry.SecretStore(cluster)
+	keyStore, err := clientset.KeyStore(cluster)
+	if err != nil {
+		return err
+	}
+
+	secretStore, err := clientset.SecretStore(cluster)
 	if err != nil {
 		return err
 	}

cmd/kops/update_cluster.go

@@ -30,7 +30,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kops/cmd/kops/util"
 	"k8s.io/kops/pkg/apis/kops"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/pkg/kubeconfig"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup"
@@ -144,21 +143,21 @@ func RunUpdateCluster(f *util.Factory, clusterName string, out io.Writer, c *Upd
 		return err
 	}
 
-	keyStore, err := registry.KeyStore(cluster)
-	if err != nil {
-		return err
-	}
-
-	secretStore, err := registry.SecretStore(cluster)
-	if err != nil {
-		return err
-	}
-
 	clientset, err := f.Clientset()
 	if err != nil {
 		return err
 	}
 
+	keyStore, err := clientset.KeyStore(cluster)
+	if err != nil {
+		return err
+	}
+
+	secretStore, err := clientset.SecretStore(cluster)
+	if err != nil {
+		return err
+	}
+
 	if c.SSHPublicKey != "" {
 		fmt.Fprintf(out, "--ssh-public-key on update is deprecated - please use `kops create secret --name %s sshpublickey admin -i ~/.ssh/id_rsa.pub` instead\n", cluster.ObjectMeta.Name)
 

cmd/kops/upgrade_cluster.go

@@ -290,7 +290,7 @@ func (c *UpgradeClusterCmd) Run(args []string) error {
 		}
 
 		assetBuilder := assets.NewAssetBuilder(cluster.Spec.Assets)
-		fullCluster, err := cloudup.PopulateClusterSpec(cluster, assetBuilder)
+		fullCluster, err := cloudup.PopulateClusterSpec(clientset, cluster, assetBuilder)
 		if err != nil {
 			return err
 		}

cmd/kops/util/factory.go

@@ -29,6 +29,7 @@ import (
 	// Register our APIs
 	"github.com/golang/glog"
 	_ "k8s.io/kops/pkg/apis/kops/install"
+	"k8s.io/kops/pkg/client/simple/api"
 	"net/url"
 	"strings"
 )
@@ -74,6 +75,8 @@ func (f *Factory) Clientset() (simple.Clientset, error) {
 				return nil, fmt.Errorf("Invalid kops server url: %q", registryPath)
 			}
 
+			u.Scheme = "https"
+
 			config := &rest.Config{
 				Host: u.Scheme + "://" + u.Host,
 			}
@@ -86,7 +89,7 @@ func (f *Factory) Clientset() (simple.Clientset, error) {
 				return nil, fmt.Errorf("error building kops API client: %v", err)
 			}
 
-			f.clientset = &simple.RESTClientset{
+			f.clientset = &api.RESTClientset{
 				BaseURL: &url.URL{
 					Scheme: "k8s",
 					Host:   u.Host,

examples/kops-api-example/up.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	api "k8s.io/kops/pkg/apis/kops"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/pkg/client/simple/vfsclientset"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup"
@@ -101,7 +100,7 @@ func up() error {
 		}
 	}
 
-	keyStore, err := registry.KeyStore(cluster)
+	keyStore, err := clientset.KeyStore(cluster)
 	if err != nil {
 		return err
 	}

federation/apply_federation.go

@@ -33,7 +33,6 @@ import (
 	"k8s.io/kops/federation/targets/kubernetestarget"
 	"k8s.io/kops/federation/tasks"
 	kopsapi "k8s.io/kops/pkg/apis/kops"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/pkg/client/simple"
 	"k8s.io/kops/pkg/kubeconfig"
 	"k8s.io/kops/pkg/pki"
@@ -177,7 +176,7 @@ func (o *ApplyFederationOperation) Run() error {
 			ClusterName:       clusterName,
 			ApiserverHostname: cluster.Spec.MasterPublicName,
 		}
-		err = a.Run(cluster)
+		err = a.Run(o.KopsClient, cluster)
 		if err != nil {
 			return err
 		}
@@ -195,7 +194,7 @@ func (o *ApplyFederationOperation) Run() error {
 // Builds a fi.Context applying to the federation namespace in the specified cluster
 // Note that this operates inside the cluster, for example the KeyStore is backed by secrets in the namespace
 func (o *ApplyFederationOperation) federationContextForCluster(cluster *kopsapi.Cluster) (*fi.Context, error) {
-	clusterKeystore, err := registry.KeyStore(cluster)
+	clusterKeystore, err := o.KopsClient.KeyStore(cluster)
 	if err != nil {
 		return nil, err
 	}

federation/federation_cluster.go

@@ -24,7 +24,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/pkg/api/v1"
 	kopsapi "k8s.io/kops/pkg/apis/kops"
-	"k8s.io/kops/pkg/apis/kops/registry"
+	"k8s.io/kops/pkg/client/simple"
 	"k8s.io/kops/pkg/kubeconfig"
 	"k8s.io/kubernetes/federation/apis/federation/v1beta1"
 	"k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
@@ -43,12 +43,12 @@ type FederationCluster struct {
 	ApiserverHostname string
 }
 
-func (o *FederationCluster) Run(cluster *kopsapi.Cluster) error {
+func (o *FederationCluster) Run(clientset simple.Clientset, cluster *kopsapi.Cluster) error {
-	keyStore, err := registry.KeyStore(cluster)
+	keyStore, err := clientset.KeyStore(cluster)
 	if err != nil {
 		return err
 	}
-	secretStore, err := registry.SecretStore(cluster)
+	secretStore, err := clientset.SecretStore(cluster)
 	if err != nil {
 		return err
 	}

hack/.packages

@@ -56,6 +56,7 @@ k8s.io/kops/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1
 k8s.io/kops/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2
 k8s.io/kops/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake
 k8s.io/kops/pkg/client/simple
+k8s.io/kops/pkg/client/simple/api
 k8s.io/kops/pkg/client/simple/vfsclientset
 k8s.io/kops/pkg/cloudinstances
 k8s.io/kops/pkg/diff

pkg/apis/kops/registry/registry.go

@@ -20,10 +20,7 @@ import (
 	"fmt"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	api "k8s.io/kops/pkg/apis/kops"
-	"k8s.io/kops/upup/pkg/fi"
-	"k8s.io/kops/upup/pkg/fi/secrets"
 	"k8s.io/kops/util/pkg/vfs"
-	"strings"
 )
 
 // Path for the user-specified cluster spec
@@ -32,46 +29,6 @@ const PathCluster = "config"
 // Path for completed cluster spec in the state store
 const PathClusterCompleted = "cluster.spec"
 
-func DeleteAllClusterState(basePath vfs.Path) error {
-	paths, err := basePath.ReadTree()
-	if err != nil {
-		return fmt.Errorf("error listing files in state store: %v", err)
-	}
-
-	for _, path := range paths {
-		relativePath, err := vfs.RelativePath(basePath, path)
-		if err != nil {
-			return err
-		}
-		if relativePath == "config" || relativePath == "cluster.spec" {
-			continue
-		}
-		if strings.HasPrefix(relativePath, "addons/") {
-			continue
-		}
-		if strings.HasPrefix(relativePath, "pki/") {
-			continue
-		}
-		if strings.HasPrefix(relativePath, "secrets/") {
-			continue
-		}
-		if strings.HasPrefix(relativePath, "instancegroup/") {
-			continue
-		}
-
-		return fmt.Errorf("refusing to delete: unknown file found: %s", path)
-	}
-
-	for _, path := range paths {
-		err = path.Remove()
-		if err != nil {
-			return fmt.Errorf("error deleting cluster file %s: %v", path, err)
-		}
-	}
-
-	return nil
-}
-
 func ConfigBase(c *api.Cluster) (vfs.Path, error) {
 	if c.Spec.ConfigBase == "" {
 		return nil, field.Required(field.NewPath("Spec", "ConfigBase"), "")
@@ -82,21 +39,3 @@ func ConfigBase(c *api.Cluster) (vfs.Path, error) {
 	}
 	return configBase, nil
 }
-
-func SecretStore(c *api.Cluster) (fi.SecretStore, error) {
-	configBase, err := ConfigBase(c)
-	if err != nil {
-		return nil, err
-	}
-	basedir := configBase.Join("secrets")
-	return secrets.NewVFSSecretStore(basedir), nil
-}
-
-func KeyStore(c *api.Cluster) (fi.CAStore, error) {
-	configBase, err := ConfigBase(c)
-	if err != nil {
-		return nil, err
-	}
-	basedir := configBase.Join("pki")
-	return fi.NewVFSCAStore(basedir), nil
-}

pkg/client/simple/api/clientset.go

@@ -0,0 +1,198 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/golang/glog"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/apis/kops/registry"
+	"k8s.io/kops/pkg/apis/kops/validation"
+	kopsinternalversion "k8s.io/kops/pkg/client/clientset_generated/clientset/typed/kops/internalversion"
+	"k8s.io/kops/pkg/client/simple/vfsclientset"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/secrets"
+	"k8s.io/kops/util/pkg/vfs"
+	"k8s.io/kubernetes/staging/src/k8s.io/apimachinery/pkg/api/errors"
+)
+
+// RESTClientset is an implementation of clientset that uses a "real" generated REST client
+type RESTClientset struct {
+	BaseURL    *url.URL
+	KopsClient kopsinternalversion.KopsInterface
+}
+
+// GetCluster implements the GetCluster method of Clientset for a kubernetes-API state store
+func (c *RESTClientset) GetCluster(name string) (*kops.Cluster, error) {
+	namespace := restNamespaceForClusterName(name)
+	return c.KopsClient.Clusters(namespace).Get(name, metav1.GetOptions{})
+}
+
+// CreateCluster implements the CreateCluster method of Clientset for a kubernetes-API state store
+func (c *RESTClientset) CreateCluster(cluster *kops.Cluster) (*kops.Cluster, error) {
+	namespace := restNamespaceForClusterName(cluster.Name)
+	return c.KopsClient.Clusters(namespace).Create(cluster)
+}
+
+// UpdateCluster implements the UpdateCluster method of Clientset for a kubernetes-API state store
+func (c *RESTClientset) UpdateCluster(cluster *kops.Cluster, status *kops.ClusterStatus) (*kops.Cluster, error) {
+	glog.Warningf("validating cluster update client side; needs to move to server")
+	old, err := c.GetCluster(cluster.Name)
+	if err != nil {
+		return nil, err
+	}
+	if err := validation.ValidateClusterUpdate(cluster, status, old).ToAggregate(); err != nil {
+		return nil, err
+	}
+
+	namespace := restNamespaceForClusterName(cluster.Name)
+	return c.KopsClient.Clusters(namespace).Update(cluster)
+}
+
+// ConfigBaseFor implements the ConfigBaseFor method of Clientset for a kubernetes-API state store
+func (c *RESTClientset) ConfigBaseFor(cluster *kops.Cluster) (vfs.Path, error) {
+	if cluster.Spec.ConfigBase != "" {
+		return vfs.Context.BuildVfsPath(cluster.Spec.ConfigBase)
+	}
+	// URL for clusters looks like  https://<server>/apis/kops/v1alpha2/namespaces/<cluster>/clusters/<cluster>
+	// We probably want to add a subresource for full resources
+	return vfs.Context.BuildVfsPath(c.BaseURL.String())
+}
+
+// ListClusters implements the ListClusters method of Clientset for a kubernetes-API state store
+func (c *RESTClientset) ListClusters(options metav1.ListOptions) (*kops.ClusterList, error) {
+	return c.KopsClient.Clusters(metav1.NamespaceAll).List(options)
+}
+
+// InstanceGroupsFor implements the InstanceGroupsFor method of Clientset for a kubernetes-API state store
+func (c *RESTClientset) InstanceGroupsFor(cluster *kops.Cluster) kopsinternalversion.InstanceGroupInterface {
+	namespace := restNamespaceForClusterName(cluster.Name)
+	return c.KopsClient.InstanceGroups(namespace)
+}
+
+// FederationsFor implements the FederationsFor method of Clientset for a kubernetes-API state store
+func (c *RESTClientset) FederationsFor(federation *kops.Federation) kopsinternalversion.FederationInterface {
+	// Unsure if this should be namespaced or not - probably, so that we can RBAC it...
+	panic("Federations are currently not supported by the server API")
+	//namespace := restNamespaceForFederationName(federation.Name)
+	//return c.KopsClient.Federations(namespace)
+}
+
+// ListFederations implements the ListFederations method of Clientset for a kubernetes-API state store
+func (c *RESTClientset) ListFederations(options metav1.ListOptions) (*kops.FederationList, error) {
+	return c.KopsClient.Federations(metav1.NamespaceAll).List(options)
+}
+
+// GetFederation implements the GetFederation method of Clientset for a kubernetes-API state store
+func (c *RESTClientset) GetFederation(name string) (*kops.Federation, error) {
+	namespace := restNamespaceForFederationName(name)
+	return c.KopsClient.Federations(namespace).Get(name, metav1.GetOptions{})
+}
+
+func (c *RESTClientset) SecretStore(cluster *kops.Cluster) (fi.SecretStore, error) {
+	namespace := restNamespaceForClusterName(cluster.Name)
+	return secrets.NewClientsetSecretStore(c.KopsClient, namespace), nil
+}
+
+func (c *RESTClientset) KeyStore(cluster *kops.Cluster) (fi.CAStore, error) {
+	namespace := restNamespaceForClusterName(cluster.Name)
+	return fi.NewClientsetCAStore(c.KopsClient, namespace), nil
+}
+
+func (c *RESTClientset) DeleteCluster(cluster *kops.Cluster) error {
+	configBase, err := registry.ConfigBase(cluster)
+	if err != nil {
+		return err
+	}
+
+	err = vfsclientset.DeleteAllClusterState(configBase)
+	if err != nil {
+		return err
+	}
+
+	name := cluster.Name
+	namespace := restNamespaceForClusterName(name)
+
+	{
+		keysets, err := c.KopsClient.Keysets(namespace).List(metav1.ListOptions{})
+		if err != nil {
+			return fmt.Errorf("error listing Keysets: %v", err)
+		}
+
+		for i := range keysets.Items {
+			keyset := &keysets.Items[i]
+			err = c.KopsClient.Keysets(namespace).Delete(keyset.Name, &metav1.DeleteOptions{})
+			if err != nil {
+				if errors.IsNotFound(err) {
+					// Unlikely...
+					glog.Warningf("Keyset was concurrently deleted")
+				} else {
+					return fmt.Errorf("error deleting Keyset %q: %v", keyset.Name, err)
+				}
+			}
+		}
+	}
+
+	{
+		igs, err := c.KopsClient.InstanceGroups(namespace).List(metav1.ListOptions{})
+		if err != nil {
+			return fmt.Errorf("error listing instance groups: %v", err)
+		}
+
+		for i := range igs.Items {
+			ig := &igs.Items[i]
+			err = c.KopsClient.InstanceGroups(namespace).Delete(ig.Name, &metav1.DeleteOptions{})
+			if err != nil {
+				if errors.IsNotFound(err) {
+					// Unlikely...
+					glog.Warningf("instance group was concurrently deleted")
+				} else {
+					return fmt.Errorf("error deleting instance group %q: %v", ig.Name, err)
+				}
+			}
+		}
+	}
+
+	err = c.KopsClient.Clusters(namespace).Delete(name, &metav1.DeleteOptions{})
+	if err != nil {
+		if errors.IsNotFound(err) {
+			// Unlikely...
+			glog.Warningf("cluster %q was concurrently deleted", name)
+		} else {
+			return fmt.Errorf("error deleting cluster%q: %v", name, err)
+		}
+	}
+
+	return nil
+}
+
+func restNamespaceForClusterName(clusterName string) string {
+	// We are not allowed dots, so we map them to dashes
+	// This can conflict, but this will simply be a limitation that we pass on to the user
+	// i.e. it will not be possible to create a.b.example.com and a-b.example.com
+	namespace := strings.Replace(clusterName, ".", "-", -1)
+	return namespace
+}
+
+func restNamespaceForFederationName(clusterName string) string {
+	namespace := strings.Replace(clusterName, ".", "-", -1)
+	return namespace
+}

pkg/client/simple/clientset.go

@@ -17,14 +17,11 @@ limitations under the License.
 package simple
 
 import (
-	"github.com/golang/glog"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kops/pkg/apis/kops"
-	"k8s.io/kops/pkg/apis/kops/validation"
 	kopsinternalversion "k8s.io/kops/pkg/client/clientset_generated/clientset/typed/kops/internalversion"
+	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/util/pkg/vfs"
-	"net/url"
-	"strings"
 )
 
 type Clientset interface {
@@ -54,87 +51,13 @@ type Clientset interface {
 
 	// ListFederations returns all federations
 	ListFederations(options metav1.ListOptions) (*kops.FederationList, error)
-}
 
-// RESTClientset is an implementation of clientset that uses a "real" generated REST client
+	// SecretStore builds the secret store for the specified cluster
-type RESTClientset struct {
+	SecretStore(cluster *kops.Cluster) (fi.SecretStore, error)
-	BaseURL    *url.URL
-	KopsClient kopsinternalversion.KopsInterface
-}
 
-// GetCluster implements the GetCluster method of Clientset for a kubernetes-API state store
+	// KeyStore builds the key store for the specified cluster
-func (c *RESTClientset) GetCluster(name string) (*kops.Cluster, error) {
+	KeyStore(cluster *kops.Cluster) (fi.CAStore, error)
-	namespace := restNamespaceForClusterName(name)
-	return c.KopsClient.Clusters(namespace).Get(name, metav1.GetOptions{})
-}
 
-// CreateCluster implements the CreateCluster method of Clientset for a kubernetes-API state store
+	// DeleteCluster deletes all the state for the specified cluster
-func (c *RESTClientset) CreateCluster(cluster *kops.Cluster) (*kops.Cluster, error) {
+	DeleteCluster(cluster *kops.Cluster) error
-	namespace := restNamespaceForClusterName(cluster.Name)
-	return c.KopsClient.Clusters(namespace).Create(cluster)
-}
-
-// UpdateCluster implements the UpdateCluster method of Clientset for a kubernetes-API state store
-func (c *RESTClientset) UpdateCluster(cluster *kops.Cluster, status *kops.ClusterStatus) (*kops.Cluster, error) {
-	glog.Warningf("validating cluster update client side; needs to move to server")
-	old, err := c.GetCluster(cluster.Name)
-	if err != nil {
-		return nil, err
-	}
-	if err := validation.ValidateClusterUpdate(cluster, status, old).ToAggregate(); err != nil {
-		return nil, err
-	}
-
-	namespace := restNamespaceForClusterName(cluster.Name)
-	return c.KopsClient.Clusters(namespace).Update(cluster)
-}
-
-// ConfigBaseFor implements the ConfigBaseFor method of Clientset for a kubernetes-API state store
-func (c *RESTClientset) ConfigBaseFor(cluster *kops.Cluster) (vfs.Path, error) {
-	// URL for clusters looks like  https://<server>/apis/kops/v1alpha2/namespaces/<cluster>/clusters/<cluster>
-	// We probably want to add a subresource for full resources
-	return vfs.Context.BuildVfsPath(c.BaseURL.String())
-}
-
-// ListClusters implements the ListClusters method of Clientset for a kubernetes-API state store
-func (c *RESTClientset) ListClusters(options metav1.ListOptions) (*kops.ClusterList, error) {
-	return c.KopsClient.Clusters(metav1.NamespaceAll).List(options)
-}
-
-// InstanceGroupsFor implements the InstanceGroupsFor method of Clientset for a kubernetes-API state store
-func (c *RESTClientset) InstanceGroupsFor(cluster *kops.Cluster) kopsinternalversion.InstanceGroupInterface {
-	namespace := restNamespaceForClusterName(cluster.Name)
-	return c.KopsClient.InstanceGroups(namespace)
-}
-
-// FederationsFor implements the FederationsFor method of Clientset for a kubernetes-API state store
-func (c *RESTClientset) FederationsFor(federation *kops.Federation) kopsinternalversion.FederationInterface {
-	// Unsure if this should be namespaced or not - probably, so that we can RBAC it...
-	panic("Federations are curently not supported by the server API")
-	//namespace := restNamespaceForFederationName(federation.Name)
-	//return c.KopsClient.Federations(namespace)
-}
-
-// ListFederations implements the ListFederations method of Clientset for a kubernetes-API state store
-func (c *RESTClientset) ListFederations(options metav1.ListOptions) (*kops.FederationList, error) {
-	return c.KopsClient.Federations(metav1.NamespaceAll).List(options)
-}
-
-// GetFederation implements the GetFederation method of Clientset for a kubernetes-API state store
-func (c *RESTClientset) GetFederation(name string) (*kops.Federation, error) {
-	namespace := restNamespaceForFederationName(name)
-	return c.KopsClient.Federations(namespace).Get(name, metav1.GetOptions{})
-}
-
-func restNamespaceForClusterName(clusterName string) string {
-	// We are not allowed dots, so we map them to dashes
-	// This can conflict, but this will simply be a limitation that we pass on to the user
-	// i.e. it will not be possible to create a.b.example.com and a-b.example.com
-	namespace := strings.Replace(clusterName, ".", "-", -1)
-	return namespace
-}
-
-func restNamespaceForFederationName(clusterName string) string {
-	namespace := strings.Replace(clusterName, ".", "-", -1)
-	return namespace
 }

pkg/client/simple/vfsclientset/clientset.go

@@ -17,10 +17,16 @@ limitations under the License.
 package vfsclientset
 
 import (
+	"fmt"
+	"strings"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/apis/kops/registry"
 	kopsinternalversion "k8s.io/kops/pkg/client/clientset_generated/clientset/typed/kops/internalversion"
 	"k8s.io/kops/pkg/client/simple"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/secrets"
 	"k8s.io/kops/util/pkg/vfs"
 )
 
@@ -56,6 +62,9 @@ func (c *VFSClientset) ListClusters(options metav1.ListOptions) (*kops.ClusterLi
 
 // ConfigBaseFor implements the ConfigBaseFor method of simple.Clientset for a VFS-backed state store
 func (c *VFSClientset) ConfigBaseFor(cluster *kops.Cluster) (vfs.Path, error) {
+	if cluster.Spec.ConfigBase != "" {
+		return vfs.Context.BuildVfsPath(cluster.Spec.ConfigBase)
+	}
 	return c.clusters().configBase(cluster.Name)
 }
 
@@ -84,6 +93,73 @@ func (c *VFSClientset) GetFederation(name string) (*kops.Federation, error) {
 	return c.federations().Get(name, metav1.GetOptions{})
 }
 
+func (c *VFSClientset) SecretStore(cluster *kops.Cluster) (fi.SecretStore, error) {
+	configBase, err := registry.ConfigBase(cluster)
+	if err != nil {
+		return nil, err
+	}
+	basedir := configBase.Join("secrets")
+	return secrets.NewVFSSecretStore(basedir), nil
+}
+
+func (c *VFSClientset) KeyStore(cluster *kops.Cluster) (fi.CAStore, error) {
+	configBase, err := registry.ConfigBase(cluster)
+	if err != nil {
+		return nil, err
+	}
+	basedir := configBase.Join("pki")
+	return fi.NewVFSCAStore(basedir), nil
+}
+
+func DeleteAllClusterState(basePath vfs.Path) error {
+	paths, err := basePath.ReadTree()
+	if err != nil {
+		return fmt.Errorf("error listing files in state store: %v", err)
+	}
+
+	for _, path := range paths {
+		relativePath, err := vfs.RelativePath(basePath, path)
+		if err != nil {
+			return err
+		}
+		if relativePath == "config" || relativePath == "cluster.spec" {
+			continue
+		}
+		if strings.HasPrefix(relativePath, "addons/") {
+			continue
+		}
+		if strings.HasPrefix(relativePath, "pki/") {
+			continue
+		}
+		if strings.HasPrefix(relativePath, "secrets/") {
+			continue
+		}
+		if strings.HasPrefix(relativePath, "instancegroup/") {
+			continue
+		}
+
+		return fmt.Errorf("refusing to delete: unknown file found: %s", path)
+	}
+
+	for _, path := range paths {
+		err = path.Remove()
+		if err != nil {
+			return fmt.Errorf("error deleting cluster file %s: %v", path, err)
+		}
+	}
+
+	return nil
+}
+
+func (c *VFSClientset) DeleteCluster(cluster *kops.Cluster) error {
+	configBase, err := registry.ConfigBase(cluster)
+	if err != nil {
+		return err
+	}
+
+	return DeleteAllClusterState(configBase)
+}
+
 func NewVFSClientset(basePath vfs.Path) simple.Clientset {
 	vfsClientset := &VFSClientset{
 		basePath: basePath,

pkg/client/simple/vfsclientset/instancegroup.go

@@ -28,6 +28,7 @@ import (
 	"k8s.io/kops/pkg/apis/kops/v1alpha1"
 	"k8s.io/kops/pkg/apis/kops/validation"
 	kopsinternalversion "k8s.io/kops/pkg/client/clientset_generated/clientset/typed/kops/internalversion"
+	"k8s.io/kops/util/pkg/vfs"
 )
 
 type InstanceGroupVFS struct {
@@ -36,6 +37,27 @@ type InstanceGroupVFS struct {
 	clusterName string
 }
 
+type InstanceGroupMirror interface {
+	WriteMirror(ig *kops.InstanceGroup) error
+}
+
+var _ InstanceGroupMirror = &InstanceGroupVFS{}
+
+func NewInstanceGroupMirror(clusterName string, configBase vfs.Path) InstanceGroupMirror {
+	kind := "InstanceGroup"
+
+	r := &InstanceGroupVFS{
+		clusterName: clusterName,
+	}
+	r.init(kind, configBase.Join("instancegroup"), StoreVersion)
+	defaultReadVersion := v1alpha1.SchemeGroupVersion.WithKind(kind)
+	r.defaultReadVersion = &defaultReadVersion
+	r.validate = func(o runtime.Object) error {
+		return validation.ValidateInstanceGroup(o.(*kops.InstanceGroup))
+	}
+	return r
+}
+
 func newInstanceGroupVFS(c *VFSClientset, clusterName string) *InstanceGroupVFS {
 	if clusterName == "" {
 		glog.Fatalf("clusterName is required")
@@ -112,6 +134,15 @@ func (c *InstanceGroupVFS) Update(g *api.InstanceGroup) (*api.InstanceGroup, err
 	return g, nil
 }
 
+func (c *InstanceGroupVFS) WriteMirror(g *api.InstanceGroup) error {
+	err := c.writeConfig(c.basePath.Join(g.Name), g)
+	if err != nil {
+		return fmt.Errorf("error writing %s: %v", c.kind, err)
+	}
+
+	return nil
+}
+
 func (c *InstanceGroupVFS) Delete(name string, options *metav1.DeleteOptions) error {
 	return c.delete(name, options)
 }

pkg/featureflag/featureflag.go

@@ -55,6 +55,9 @@ var EnableExternalDNS = New("EnableExternalDNS", Bool(false))
 //EnableExternalCloudController toggles the use of cloud-controller-manager introduced in v1.7
 var EnableExternalCloudController = New("EnableExternalCloudController", Bool(false))
 
+// EnableSeparateConfigBase allows a config-base that is different from the state store
+var EnableSeparateConfigBase = New("EnableSeparateConfigBase", Bool(false))
+
 // SpecOverrideFlag allows setting spec values on create
 var SpecOverrideFlag = New("SpecOverrideFlag", Bool(false))
 

pkg/model/pki.go

@@ -22,6 +22,7 @@ import (
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/fitasks"
+	"k8s.io/kops/util/pkg/vfs"
 )
 
 // PKIModelBuilder configures PKI keypairs, as well as tokens
@@ -196,5 +197,32 @@ func (b *PKIModelBuilder) Build(c *fi.ModelBuilderContext) error {
 		c.AddTask(t)
 	}
 
+	{
+		mirrorPath, err := vfs.Context.BuildVfsPath(b.Cluster.Spec.SecretStore)
+		if err != nil {
+			return err
+		}
+
+		t := &fitasks.MirrorSecrets{
+			Name:       fi.String("mirror-secrets"),
+			MirrorPath: mirrorPath,
+		}
+		c.AddTask(t)
+	}
+
+	{
+		mirrorPath, err := vfs.Context.BuildVfsPath(b.Cluster.Spec.KeyStore)
+		if err != nil {
+			return err
+		}
+
+		// Keypair used by the kubelet
+		t := &fitasks.MirrorKeystore{
+			Name:       fi.String("mirror-keystore"),
+			MirrorPath: mirrorPath,
+		}
+		c.AddTask(t)
+	}
+
 	return nil
 }

upup/pkg/fi/ca.go

@@ -51,8 +51,16 @@ type Keystore interface {
 
 	CreateKeypair(name string, template *x509.Certificate, privateKey *pki.PrivateKey) (*pki.Certificate, error)
 
-	// Store the keypair
+	// StoreKeypair writes the keypair to the store
 	StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error
+
+	// MirrorTo will copy secrets to a vfs.Path, which is often easier for a machine to read
+	MirrorTo(basedir vfs.Path) error
+}
+
+// HasVFSPath is implemented by keystore & other stores that use a VFS path as their backing store
+type HasVFSPath interface {
+	VFSPath() vfs.Path
 }
 
 type CAStore interface {
@@ -70,9 +78,6 @@ type CAStore interface {
 	// List will list all the items, but will not fetch the data
 	List() ([]*KeystoreItem, error)
 
-	// VFSPath returns the path where the CAStore is stored
-	VFSPath() vfs.Path
-
 	// AddCert adds an alternative certificate to the pool (primarily useful for CAs)
 	AddCert(name string, cert *pki.Certificate) error
 

upup/pkg/fi/clientset_castore.go

@@ -18,6 +18,7 @@ package fi
 
 import (
 	"bytes"
+	"crypto/md5"
 	crypto_rand "crypto/rand"
 	"crypto/rsa"
 	"crypto/x509"
@@ -610,15 +611,82 @@ func (c *ClientsetCAStore) DeleteSecret(item *KeystoreItem) error {
 	case SecretTypeKeypair:
 		client := c.clientset.Keysets(c.namespace)
 		return DeleteKeysetItem(client, item.Name, kops.SecretTypeKeypair, item.Id)
-
 	default:
 		// Primarily because we need to make sure users can recreate them!
 		return fmt.Errorf("deletion of keystore items of type %v not (yet) supported", item.Type)
 	}
 }
 
-// VFSPath implements CAStore::VFSPath
+func (c *ClientsetCAStore) MirrorTo(basedir vfs.Path) error {
-func (c *ClientsetCAStore) VFSPath() vfs.Path {
+	list, err := c.clientset.Keysets(c.namespace).List(v1.ListOptions{})
-	// We will implement mirroring instead
+	if err != nil {
-	panic("ClientsetCAStore::VFSPath not implemented")
+		return fmt.Errorf("error listing keysets: %v", err)
+	}
+
+	for i := range list.Items {
+		keyset := &list.Items[i]
+
+		if keyset.Spec.Type == kops.SecretTypeSecret {
+			continue
+		}
+
+		primary := FindPrimary(keyset)
+		if primary == nil {
+			return fmt.Errorf("found keyset with no primary data: %s", keyset.Name)
+		}
+
+		switch keyset.Spec.Type {
+		case kops.SecretTypeKeypair:
+			for i := range keyset.Spec.Keys {
+				item := &keyset.Spec.Keys[i]
+				{
+					p := basedir.Join("issued", keyset.Name, item.Id+".crt")
+					err = p.WriteFile(item.PublicMaterial)
+					if err != nil {
+						return fmt.Errorf("error writing %q: %v", p, err)
+					}
+				}
+				{
+					p := basedir.Join("private", keyset.Name, item.Id+".key")
+					err = p.WriteFile(item.PrivateMaterial)
+					if err != nil {
+						return fmt.Errorf("error writing %q: %v", p, err)
+					}
+				}
+			}
+
+		default:
+			return fmt.Errorf("Ignoring unknown secret type: %q", keyset.Spec.Type)
+		}
+	}
+
+	sshCredentials, err := c.clientset.SSHCredentials(c.namespace).List(v1.ListOptions{})
+	if err != nil {
+		return fmt.Errorf("error listing SSHCredentials: %v", err)
+	}
+
+	for i := range sshCredentials.Items {
+		sshCredential := &sshCredentials.Items[i]
+
+		sshPublicKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(sshCredential.Spec.PublicKey))
+		if err != nil {
+			return fmt.Errorf("error parsing SSH public key %q: %v", sshCredential.Name, err)
+		}
+
+		// compute fingerprint to serve as id
+		h := md5.New()
+		_, err = h.Write(sshPublicKey.Marshal())
+		if err != nil {
+			return fmt.Errorf("error fingerprinting SSH public key: %v", err)
+		}
+		id := formatFingerprint(h.Sum(nil))
+
+		p := basedir.Join("ssh", "public", sshCredential.Name, id)
+		err = p.WriteFile([]byte(sshCredential.Spec.PublicKey))
+		if err != nil {
+			return fmt.Errorf("error writing %q: %v", p, err)
+		}
+	}
+
+	return nil
 }

upup/pkg/fi/cloudup/apply_cluster.go

@@ -58,6 +58,7 @@ import (
 
 	"github.com/blang/semver"
 	"github.com/golang/glog"
+	"k8s.io/kops/pkg/client/simple/vfsclientset"
 	"k8s.io/kops/upup/pkg/fi/cloudup/baremetal"
 )
 
@@ -190,12 +191,12 @@ func (c *ApplyClusterCmd) Run() error {
 		return fmt.Errorf("error parsing config base %q: %v", cluster.Spec.ConfigBase, err)
 	}
 
-	keyStore, err := registry.KeyStore(cluster)
+	keyStore, err := c.Clientset.KeyStore(cluster)
 	if err != nil {
 		return err
 	}
 
-	secretStore, err := registry.SecretStore(cluster)
+	secretStore, err := c.Clientset.SecretStore(cluster)
 	if err != nil {
 		return err
 	}
@@ -279,9 +280,11 @@ func (c *ApplyClusterCmd) Run() error {
 	checkExisting := true
 
 	l.AddTypes(map[string]interface{}{
-		"keypair":     &fitasks.Keypair{},
+		"keypair":        &fitasks.Keypair{},
-		"secret":      &fitasks.Secret{},
+		"secret":         &fitasks.Secret{},
-		"managedFile": &fitasks.ManagedFile{},
+		"managedFile":    &fitasks.ManagedFile{},
+		"mirrorKeystore": &fitasks.MirrorKeystore{},
+		"mirrorSecrets":  &fitasks.MirrorSecrets{},
 	})
 
 	cloud, err := BuildCloud(cluster)
@@ -786,11 +789,19 @@ func (c *ApplyClusterCmd) Run() error {
 			return fmt.Errorf("error writing completed cluster spec: %v", err)
 		}
 
+		vfsMirror := vfsclientset.NewInstanceGroupMirror(cluster.Name, configBase)
+
 		for _, g := range c.InstanceGroups {
+			// TODO: We need to update the mirror (below), but do we need to update the primary?
 			_, err := c.Clientset.InstanceGroupsFor(c.Cluster).Update(g)
 			if err != nil {
 				return fmt.Errorf("error writing InstanceGroup %q to registry: %v", g.ObjectMeta.Name, err)
 			}
+
+			// TODO: Don't write if vfsMirror == c.ClientSet
+			if err := vfsMirror.WriteMirror(g); err != nil {
+				return fmt.Errorf("error writing instance group spec to mirror: %v", err)
+			}
 		}
 	}
 
@@ -841,7 +852,7 @@ func findHash(url string) (*hashing.Hash, error) {
 
 // upgradeSpecs ensures that fields are fully populated / defaulted
 func (c *ApplyClusterCmd) upgradeSpecs(assetBuilder *assets.AssetBuilder) error {
-	fullCluster, err := PopulateClusterSpec(c.Cluster, assetBuilder)
+	fullCluster, err := PopulateClusterSpec(c.Clientset, c.Cluster, assetBuilder)
 	if err != nil {
 		return err
 	}

upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go

@@ -64,8 +64,7 @@ func runChannelBuilderTest(t *testing.T, key string) {
 		t.Fatalf("error from PerformAssignments: %v", err)
 	}
 
-	assetBuilder := assets.NewAssetBuilder(nil)
+	fullSpec, err := mockedPopulateClusterSpec(cluster)
-	fullSpec, err := PopulateClusterSpec(cluster, assetBuilder)
 	if err != nil {
 		t.Fatalf("error from PopulateClusterSpec: %v", err)
 	}

upup/pkg/fi/cloudup/populate_cluster_spec.go

@@ -26,10 +26,10 @@ import (
 	"github.com/golang/glog"
 
 	api "k8s.io/kops/pkg/apis/kops"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/pkg/apis/kops/util"
 	"k8s.io/kops/pkg/apis/kops/validation"
 	"k8s.io/kops/pkg/assets"
+	"k8s.io/kops/pkg/client/simple"
 	"k8s.io/kops/pkg/dns"
 	"k8s.io/kops/pkg/model"
 	"k8s.io/kops/pkg/model/components"
@@ -66,7 +66,7 @@ func findModelStore() (vfs.Path, error) {
 
 // PopulateClusterSpec takes a user-specified cluster spec, and computes the full specification that should be set on the cluster.
 // We do this so that we don't need any real "brains" on the node side.
-func PopulateClusterSpec(cluster *api.Cluster, assetBuilder *assets.AssetBuilder) (*api.Cluster, error) {
+func PopulateClusterSpec(clientset simple.Clientset, cluster *api.Cluster, assetBuilder *assets.AssetBuilder) (*api.Cluster, error) {
 	modelStore, err := findModelStore()
 	if err != nil {
 		return nil, err
@@ -78,7 +78,7 @@ func PopulateClusterSpec(cluster *api.Cluster, assetBuilder *assets.AssetBuilder
 		Models:       []string{"config"},
 		assetBuilder: assetBuilder,
 	}
-	err = c.run()
+	err = c.run(clientset)
 	if err != nil {
 		return nil, err
 	}
@@ -95,7 +95,7 @@ func PopulateClusterSpec(cluster *api.Cluster, assetBuilder *assets.AssetBuilder
 // struct is falling through..
 // @kris-nova
 //
-func (c *populateClusterSpec) run() error {
+func (c *populateClusterSpec) run(clientset simple.Clientset) error {
 	if err := validation.ValidateCluster(c.InputCluster, false); err != nil {
 		return err
 	}
@@ -177,32 +177,6 @@ func (c *populateClusterSpec) run() error {
 		}
 	}
 
-	keyStore, err := registry.KeyStore(cluster)
-	if err != nil {
-		return err
-	}
-
-	secretStore, err := registry.SecretStore(cluster)
-	if err != nil {
-		return err
-	}
-
-	if vfs.IsClusterReadable(secretStore.VFSPath()) {
-		vfsPath := secretStore.VFSPath()
-		cluster.Spec.SecretStore = vfsPath.Path()
-	} else {
-		// We could implement this approach, but it seems better to get all clouds using cluster-readable storage
-		return fmt.Errorf("secrets path is not cluster readable: %v", secretStore.VFSPath())
-	}
-
-	if vfs.IsClusterReadable(keyStore.VFSPath()) {
-		vfsPath := keyStore.VFSPath()
-		cluster.Spec.KeyStore = vfsPath.Path()
-	} else {
-		// We could implement this approach, but it seems better to get all clouds using cluster-readable storage
-		return fmt.Errorf("keyStore path is not cluster readable: %v", keyStore.VFSPath())
-	}
-
 	configBase, err := vfs.Context.BuildVfsPath(cluster.Spec.ConfigBase)
 	if err != nil {
 		return fmt.Errorf("error parsing ConfigBase %q: %v", cluster.Spec.ConfigBase, err)
@@ -214,6 +188,46 @@ func (c *populateClusterSpec) run() error {
 		return fmt.Errorf("ConfigBase path is not cluster readable: %v", cluster.Spec.ConfigBase)
 	}
 
+	keyStore, err := clientset.KeyStore(cluster)
+	if err != nil {
+		return err
+	}
+
+	if cluster.Spec.KeyStore == "" {
+		hasVFSPath, ok := keyStore.(fi.HasVFSPath)
+		if !ok {
+			// We will mirror to ConfigBase
+			basedir := configBase.Join("pki")
+			cluster.Spec.KeyStore = basedir.Path()
+		} else if vfs.IsClusterReadable(hasVFSPath.VFSPath()) {
+			vfsPath := hasVFSPath.VFSPath()
+			cluster.Spec.KeyStore = vfsPath.Path()
+		} else {
+			// We could implement this approach, but it seems better to get all clouds using cluster-readable storage
+			return fmt.Errorf("keyStore path is not cluster readable: %v", hasVFSPath.VFSPath())
+		}
+	}
+
+	secretStore, err := clientset.SecretStore(cluster)
+	if err != nil {
+		return err
+	}
+
+	if cluster.Spec.SecretStore == "" {
+		hasVFSPath, ok := secretStore.(fi.HasVFSPath)
+		if !ok {
+			// We will mirror to ConfigBase
+			basedir := configBase.Join("secrets")
+			cluster.Spec.SecretStore = basedir.Path()
+		} else if vfs.IsClusterReadable(hasVFSPath.VFSPath()) {
+			vfsPath := hasVFSPath.VFSPath()
+			cluster.Spec.SecretStore = vfsPath.Path()
+		} else {
+			// We could implement this approach, but it seems better to get all clouds using cluster-readable storage
+			return fmt.Errorf("secrets path is not cluster readable: %v", hasVFSPath.VFSPath())
+		}
+	}
+
 	// Normalize k8s version
 	versionWithoutV := strings.TrimSpace(cluster.Spec.KubernetesVersion)
 	if strings.HasPrefix(versionWithoutV, "v") {

upup/pkg/fi/cloudup/populatecluster_test.go

@@ -24,10 +24,15 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	api "k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/assets"
+	"k8s.io/kops/pkg/client/simple/vfsclientset"
 	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
+	"k8s.io/kops/util/pkg/vfs"
 )
 
 func buildMinimalCluster() *api.Cluster {
+	awsup.InstallMockAWSCloud(MockAWSRegion, "abcd")
+
 	c := &api.Cluster{}
 	c.ObjectMeta.Name = "testcluster.test.com"
 	c.Spec.KubernetesVersion = "1.4.6"
@@ -90,13 +95,24 @@ func TestPopulateCluster_Default_NoError(t *testing.T) {
 
 	addEtcdClusters(c)
 
-	assetBuilder := assets.NewAssetBuilder(nil)
+	_, err = mockedPopulateClusterSpec(c)
-	_, err = PopulateClusterSpec(c, assetBuilder)
 	if err != nil {
 		t.Fatalf("Unexpected error from PopulateCluster: %v", err)
 	}
 }
 
+func mockedPopulateClusterSpec(c *api.Cluster) (*api.Cluster, error) {
+	vfs.Context.ResetMemfsContext(true)
+
+	assetBuilder := assets.NewAssetBuilder(nil)
+	basePath, err := vfs.Context.BuildVfsPath("memfs://tests")
+	if err != nil {
+		return nil, fmt.Errorf("error building vfspath: %v", err)
+	}
+	clientset := vfsclientset.NewVFSClientset(basePath)
+	return PopulateClusterSpec(clientset, c, assetBuilder)
+}
+
 func TestPopulateCluster_Docker_Spec(t *testing.T) {
 	c := buildMinimalCluster()
 	c.Spec.Docker = &api.DockerConfig{
@@ -113,8 +129,7 @@ func TestPopulateCluster_Docker_Spec(t *testing.T) {
 
 	addEtcdClusters(c)
 
-	assetBuilder := assets.NewAssetBuilder(nil)
+	full, err := mockedPopulateClusterSpec(c)
-	full, err := PopulateClusterSpec(c, assetBuilder)
 	if err != nil {
 		t.Fatalf("Unexpected error from PopulateCluster: %v", err)
 	}
@@ -144,8 +159,8 @@ func TestPopulateCluster_StorageDefault(t *testing.T) {
 	}
 
 	addEtcdClusters(c)
-	assetBuilder := assets.NewAssetBuilder(c.Spec.Assets)
+
-	full, err := PopulateClusterSpec(c, assetBuilder)
+	full, err := mockedPopulateClusterSpec(c)
 	if err != nil {
 		t.Fatalf("Unexpected error from PopulateCluster: %v", err)
 	}
@@ -162,8 +177,8 @@ func build(c *api.Cluster) (*api.Cluster, error) {
 	}
 
 	addEtcdClusters(c)
-	assetBuilder := assets.NewAssetBuilder(nil)
+
-	full, err := PopulateClusterSpec(c, assetBuilder)
+	full, err := mockedPopulateClusterSpec(c)
 	if err != nil {
 		return nil, fmt.Errorf("Unexpected error from PopulateCluster: %v", err)
 	}
@@ -239,8 +254,7 @@ func TestPopulateCluster_Custom_CIDR(t *testing.T) {
 
 	addEtcdClusters(c)
 
-	assetBuilder := assets.NewAssetBuilder(nil)
+	full, err := mockedPopulateClusterSpec(c)
-	full, err := PopulateClusterSpec(c, assetBuilder)
 	if err != nil {
 		t.Fatalf("Unexpected error from PopulateCluster: %v", err)
 	}
@@ -260,8 +274,7 @@ func TestPopulateCluster_IsolateMasters(t *testing.T) {
 
 	addEtcdClusters(c)
 
-	assetBuilder := assets.NewAssetBuilder(nil)
+	full, err := mockedPopulateClusterSpec(c)
-	full, err := PopulateClusterSpec(c, assetBuilder)
 	if err != nil {
 		t.Fatalf("Unexpected error from PopulateCluster: %v", err)
 	}
@@ -284,8 +297,7 @@ func TestPopulateCluster_IsolateMastersFalse(t *testing.T) {
 
 	addEtcdClusters(c)
 
-	assetBuilder := assets.NewAssetBuilder(nil)
+	full, err := mockedPopulateClusterSpec(c)
-	full, err := PopulateClusterSpec(c, assetBuilder)
 	if err != nil {
 		t.Fatalf("Unexpected error from PopulateCluster: %v", err)
 	}
@@ -376,8 +388,7 @@ func TestPopulateCluster_BastionIdleTimeoutInvalidNegative_Required(t *testing.T
 }
 
 func expectErrorFromPopulateCluster(t *testing.T, c *api.Cluster, message string) {
-	assetBuilder := assets.NewAssetBuilder(nil)
+	_, err := mockedPopulateClusterSpec(c)
-	_, err := PopulateClusterSpec(c, assetBuilder)
 	if err == nil {
 		t.Fatalf("Expected error from PopulateCluster")
 	}
@@ -411,8 +422,7 @@ func TestPopulateCluster_AnonymousAuth(t *testing.T) {
 
 	addEtcdClusters(c)
 
-	assetBuilder := assets.NewAssetBuilder(nil)
+	full, err := mockedPopulateClusterSpec(c)
-	full, err := PopulateClusterSpec(c, assetBuilder)
 	if err != nil {
 		t.Fatalf("Unexpected error from PopulateCluster: %v", err)
 	}
@@ -437,8 +447,7 @@ func TestPopulateCluster_AnonymousAuth_14(t *testing.T) {
 
 	addEtcdClusters(c)
 
-	assetBuilder := assets.NewAssetBuilder(nil)
+	full, err := mockedPopulateClusterSpec(c)
-	full, err := PopulateClusterSpec(c, assetBuilder)
 	if err != nil {
 		t.Fatalf("Unexpected error from PopulateCluster: %v", err)
 	}
@@ -489,8 +498,7 @@ func TestPopulateCluster_KubeController_High_Enough_Version(t *testing.T) {
 
 	addEtcdClusters(c)
 
-	assetBuilder := assets.NewAssetBuilder(nil)
+	full, err := mockedPopulateClusterSpec(c)
-	full, err := PopulateClusterSpec(c, assetBuilder)
 	if err != nil {
 		t.Fatalf("Unexpected error from PopulateCluster: %v", err)
 	}
@@ -512,8 +520,7 @@ func TestPopulateCluster_KubeController_Fail(t *testing.T) {
 
 	addEtcdClusters(c)
 
-	assetBuilder := assets.NewAssetBuilder(nil)
+	full, err := mockedPopulateClusterSpec(c)
-	full, err := PopulateClusterSpec(c, assetBuilder)
 	if err != nil {
 		t.Fatalf("Unexpected error from PopulateCluster: %v", err)
 	}

upup/pkg/fi/cloudup/validation_test.go

@@ -22,9 +22,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	api "k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/apis/kops/validation"
-	"k8s.io/kops/pkg/assets"
 	"k8s.io/kops/upup/pkg/fi"
-	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	"strings"
 	"testing"
 )
@@ -32,8 +30,6 @@ import (
 const MockAWSRegion = "us-mock-1"
 
 func buildDefaultCluster(t *testing.T) *api.Cluster {
-	awsup.InstallMockAWSCloud(MockAWSRegion, "abcd")
-
 	c := buildMinimalCluster()
 
 	err := PerformAssignments(c)
@@ -61,8 +57,7 @@ func buildDefaultCluster(t *testing.T) *api.Cluster {
 		}
 	}
 
-	assetBuilder := assets.NewAssetBuilder(nil)
+	fullSpec, err := mockedPopulateClusterSpec(c)
-	fullSpec, err := PopulateClusterSpec(c, assetBuilder)
 	if err != nil {
 		t.Fatalf("error from PopulateClusterSpec: %v", err)
 	}

upup/pkg/fi/fitasks/mirrorkeystore.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fitasks
+
+import (
+	"github.com/golang/glog"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/util/pkg/vfs"
+)
+
+//go:generate fitask -type=MirrorKeystore
+type MirrorKeystore struct {
+	Name      *string
+	Lifecycle *fi.Lifecycle
+
+	MirrorPath vfs.Path
+}
+
+var _ fi.HasDependencies = &MirrorKeystore{}
+
+// GetDependencies returns the dependencies for a MirrorKeystore task - it must run after all secrets have been run
+func (e *MirrorKeystore) GetDependencies(tasks map[string]fi.Task) []fi.Task {
+	var deps []fi.Task
+	for _, task := range tasks {
+		if _, ok := task.(*Secret); ok {
+			deps = append(deps, task)
+		}
+	}
+	return deps
+}
+
+// Find implements fi.Task::Find
+func (e *MirrorKeystore) Find(c *fi.Context) (*MirrorKeystore, error) {
+	// TODO: implement Find so that we aren't always mirroring
+	glog.V(2).Infof("MirrorKeystore::Find not implemented; always copying (inefficient)")
+	return nil, nil
+}
+
+// Run implements fi.Task::Run
+func (e *MirrorKeystore) Run(c *fi.Context) error {
+	return fi.DefaultDeltaRunMethod(e, c)
+}
+
+// CheckChanges implements fi.Task::CheckChanges
+func (s *MirrorKeystore) CheckChanges(a, e, changes *MirrorKeystore) error {
+	if a != nil {
+		if changes.Name != nil {
+			return fi.CannotChangeField("Name")
+		}
+	}
+	return nil
+}
+
+// Render implements fi.Task::Render
+func (_ *MirrorKeystore) Render(c *fi.Context, a, e, changes *MirrorKeystore) error {
+	keystore := c.Keystore
+
+	return keystore.MirrorTo(e.MirrorPath)
+}

upup/pkg/fi/fitasks/mirrorkeystore_fitask.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by ""fitask" -type=MirrorKeystore"; DO NOT EDIT
+
+package fitasks
+
+import (
+	"encoding/json"
+
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+// MirrorKeystore
+
+// JSON marshalling boilerplate
+type realMirrorKeystore MirrorKeystore
+
+// UnmarshalJSON implements conversion to JSON, supporitng an alternate specification of the object as a string
+func (o *MirrorKeystore) UnmarshalJSON(data []byte) error {
+	var jsonName string
+	if err := json.Unmarshal(data, &jsonName); err == nil {
+		o.Name = &jsonName
+		return nil
+	}
+
+	var r realMirrorKeystore
+	if err := json.Unmarshal(data, &r); err != nil {
+		return err
+	}
+	*o = MirrorKeystore(r)
+	return nil
+}
+
+var _ fi.HasLifecycle = &MirrorKeystore{}
+
+// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle
+func (o *MirrorKeystore) GetLifecycle() *fi.Lifecycle {
+	return o.Lifecycle
+}
+
+var _ fi.HasName = &MirrorKeystore{}
+
+// GetName returns the Name of the object, implementing fi.HasName
+func (o *MirrorKeystore) GetName() *string {
+	return o.Name
+}
+
+// SetName sets the Name of the object, implementing fi.SetName
+func (o *MirrorKeystore) SetName(name string) {
+	o.Name = &name
+}
+
+// String is the stringer function for the task, producing readable output using fi.TaskAsString
+func (o *MirrorKeystore) String() string {
+	return fi.TaskAsString(o)
+}

upup/pkg/fi/fitasks/mirrorsecrets.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fitasks
+
+import (
+	"github.com/golang/glog"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/util/pkg/vfs"
+)
+
+//go:generate fitask -type=MirrorSecrets
+type MirrorSecrets struct {
+	Name      *string
+	Lifecycle *fi.Lifecycle
+
+	MirrorPath vfs.Path
+}
+
+var _ fi.HasDependencies = &MirrorSecrets{}
+
+// GetDependencies returns the dependencies for a MirrorSecrets task - it must run after all secrets have been run
+func (e *MirrorSecrets) GetDependencies(tasks map[string]fi.Task) []fi.Task {
+	var deps []fi.Task
+	for _, task := range tasks {
+		if _, ok := task.(*Secret); ok {
+			deps = append(deps, task)
+		}
+	}
+	return deps
+}
+
+// Find implements fi.Task::Find
+func (e *MirrorSecrets) Find(c *fi.Context) (*MirrorSecrets, error) {
+	// TODO: implement Find so that we aren't always mirroring
+	glog.V(2).Infof("MirrorSecrets::Find not implemented; always copying (inefficient)")
+	return nil, nil
+}
+
+// Run implemements fi.Task::Run
+func (e *MirrorSecrets) Run(c *fi.Context) error {
+	return fi.DefaultDeltaRunMethod(e, c)
+}
+
+// CheckChanges implements fi.Task::CheckChanges
+func (s *MirrorSecrets) CheckChanges(a, e, changes *MirrorSecrets) error {
+	if a != nil {
+		if changes.Name != nil {
+			return fi.CannotChangeField("Name")
+		}
+	}
+	return nil
+}
+
+// Render implements fi.Task::Render
+func (_ *MirrorSecrets) Render(c *fi.Context, a, e, changes *MirrorSecrets) error {
+	secrets := c.SecretStore
+
+	return secrets.MirrorTo(e.MirrorPath)
+}

upup/pkg/fi/fitasks/mirrorsecrets_fitask.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by ""fitask" -type=MirrorSecrets"; DO NOT EDIT
+
+package fitasks
+
+import (
+	"encoding/json"
+
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+// MirrorSecrets
+
+// JSON marshalling boilerplate
+type realMirrorSecrets MirrorSecrets
+
+// UnmarshalJSON implements conversion to JSON, supporitng an alternate specification of the object as a string
+func (o *MirrorSecrets) UnmarshalJSON(data []byte) error {
+	var jsonName string
+	if err := json.Unmarshal(data, &jsonName); err == nil {
+		o.Name = &jsonName
+		return nil
+	}
+
+	var r realMirrorSecrets
+	if err := json.Unmarshal(data, &r); err != nil {
+		return err
+	}
+	*o = MirrorSecrets(r)
+	return nil
+}
+
+var _ fi.HasLifecycle = &MirrorSecrets{}
+
+// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle
+func (o *MirrorSecrets) GetLifecycle() *fi.Lifecycle {
+	return o.Lifecycle
+}
+
+var _ fi.HasName = &MirrorSecrets{}
+
+// GetName returns the Name of the object, implementing fi.HasName
+func (o *MirrorSecrets) GetName() *string {
+	return o.Name
+}
+
+// SetName sets the Name of the object, implementing fi.SetName
+func (o *MirrorSecrets) SetName(name string) {
+	o.Name = &name
+}
+
+// String is the stringer function for the task, producing readable output using fi.TaskAsString
+func (o *MirrorSecrets) String() string {
+	return fi.TaskAsString(o)
+}

upup/pkg/fi/k8sapi/k8s_keystore.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/client-go/pkg/api/v1"
 	"k8s.io/kops/pkg/pki"
 	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/util/pkg/vfs"
 	"math/big"
 	"time"
 )
@@ -145,3 +146,7 @@ func (c *KubernetesKeystore) StoreKeypair(id string, cert *pki.Certificate, priv
 
 	return err
 }
+
+func (c *KubernetesKeystore) MirrorTo(dest vfs.Path) error {
+	return fmt.Errorf("KubernetesKeystore does not implement MirrorTo")
+}

upup/pkg/fi/secrets.go

@@ -36,8 +36,8 @@ type SecretStore interface {
 	// ListSecrets lists the ids of all known secrets
 	ListSecrets() ([]string, error)
 
-	// VFSPath returns the path where the SecretStore is stored
+	// MirrorTo will copy secrets to a vfs.Path, which is often easier for a machine to read
-	VFSPath() vfs.Path
+	MirrorTo(basedir vfs.Path) error
 }
 
 type Secret struct {

upup/pkg/fi/secrets/clientset_secretstore.go

@@ -17,6 +17,7 @@ limitations under the License.
 package secrets
 
 import (
+	"encoding/json"
 	"fmt"
 	"strings"
 	"time"
@@ -51,6 +52,43 @@ func NewClientsetSecretStore(clientset kopsinternalversion.KopsInterface, namesp
 	return c
 }
 
+func (c *ClientsetSecretStore) MirrorTo(basedir vfs.Path) error {
+	list, err := c.clientset.Keysets(c.namespace).List(v1.ListOptions{})
+	if err != nil {
+		return fmt.Errorf("error listing keysets: %v", err)
+	}
+
+	for i := range list.Items {
+		keyset := &list.Items[i]
+
+		if keyset.Spec.Type != kops.SecretTypeSecret {
+			continue
+		}
+
+		primary := fi.FindPrimary(keyset)
+		if primary == nil {
+			return fmt.Errorf("found secret with no primary data: %s", keyset.Name)
+		}
+
+		name := strings.TrimPrefix(keyset.Name, NamePrefix)
+		p := BuildVfsSecretPath(basedir, name)
+
+		s := &fi.Secret{
+			Data: primary.PrivateMaterial,
+		}
+		data, err := json.Marshal(s)
+		if err != nil {
+			return fmt.Errorf("error serializing secret: %v", err)
+		}
+
+		if err := p.WriteFile(data); err != nil {
+			return fmt.Errorf("error writing secret to %q: %v", p, err)
+		}
+	}
+
+	return nil
+}
+
 // FindSecret implements fi.SecretStore::FindSecret
 func (c *ClientsetSecretStore) FindSecret(name string) (*fi.Secret, error) {
 	s, err := c.loadSecret(name)
@@ -177,10 +215,3 @@ func (c *ClientsetSecretStore) createSecret(s *fi.Secret, name string) (*kops.Ke
 
 	return c.clientset.Keysets(c.namespace).Create(keyset)
 }
-
-// VFSPath implements fi.SecretStore::VFSPath
-func (c *ClientsetSecretStore) VFSPath() vfs.Path {
-	// We will implement mirroring instead
-	glog.Fatalf("ClientsetSecretStore::VFSPath not implemented")
-	return nil
-}

upup/pkg/fi/secrets/vfs_secretstore.go

@@ -42,8 +42,21 @@ func (c *VFSSecretStore) VFSPath() vfs.Path {
 	return c.basedir
 }
 
+func (c *VFSSecretStore) MirrorTo(basedir vfs.Path) error {
+	if basedir.Path() == c.basedir.Path() {
+		return nil
+	}
+	glog.V(2).Infof("Mirroring secret store from %q to %q", c.basedir, basedir)
+
+	return vfs.CopyTree(c.basedir, basedir)
+}
+
+func BuildVfsSecretPath(basedir vfs.Path, name string) vfs.Path {
+	return basedir.Join(name)
+}
+
 func (c *VFSSecretStore) buildSecretPath(name string) vfs.Path {
-	return c.basedir.Join(name)
+	return BuildVfsSecretPath(c.basedir, name)
 }
 
 func (c *VFSSecretStore) FindSecret(id string) (*fi.Secret, error) {

upup/pkg/fi/vfs_castore.go

@@ -404,6 +404,16 @@ func (c *VFSCAStore) List() ([]*KeystoreItem, error) {
 	return items, nil
 }
 
+// MirrorTo will copy keys to a vfs.Path, which is often easier for a machine to read
+func (c *VFSCAStore) MirrorTo(basedir vfs.Path) error {
+	if basedir.Path() == c.basedir.Path() {
+		return nil
+	}
+	glog.V(2).Infof("Mirroring key store from %q to %q", c.basedir, basedir)
+
+	return vfs.CopyTree(c.basedir, basedir)
+}
+
 func (c *VFSCAStore) IssueCert(id string, serial *big.Int, privateKey *pki.PrivateKey, template *x509.Certificate) (*pki.Certificate, error) {
 	glog.Infof("Issuing new certificate: %q", id)
 

upup/pkg/kutil/convert_kubeup_cluster.go

@@ -64,7 +64,7 @@ func (x *ConvertKubeupCluster) Upgrade() error {
 		return fmt.Errorf("OldClusterName must be specified")
 	}
 
-	oldKeyStore, err := registry.KeyStore(cluster)
+	oldKeyStore, err := x.Clientset.KeyStore(cluster)
 	if err != nil {
 		return err
 	}
@@ -83,7 +83,7 @@ func (x *ConvertKubeupCluster) Upgrade() error {
 	}
 	cluster.Spec.ConfigBase = newConfigBase.Path()
 
-	newKeyStore, err := registry.KeyStore(cluster)
+	newKeyStore, err := x.Clientset.KeyStore(cluster)
 	if err != nil {
 		return err
 	}
@@ -107,7 +107,7 @@ func (x *ConvertKubeupCluster) Upgrade() error {
 	}
 
 	assetBuilder := assets.NewAssetBuilder(cluster.Spec.Assets)
-	fullCluster, err := cloudup.PopulateClusterSpec(cluster, assetBuilder)
+	fullCluster, err := cloudup.PopulateClusterSpec(x.Clientset, cluster, assetBuilder)
 	if err != nil {
 		return err
 	}

upup/pkg/kutil/import_cluster.go

@@ -414,7 +414,7 @@ func (x *ImportCluster) ImportAWSCluster() error {
 
 	//b.Context = "aws_" + instancePrefix
 
-	keyStore, err := registry.KeyStore(cluster)
+	keyStore, err := x.Clientset.KeyStore(cluster)
 	if err != nil {
 		return err
 	}

util/pkg/vfs/vfssync.go

@@ -213,3 +213,69 @@ func hashesMatch(src, dest Path) (bool, error) {
 	glog.Infof("No compatible hash: %s and %s", src, dest)
 	return false, nil
 }
+
+// CopyTree copies all files in src to dest.  It copies the whole recursive subtree of files.
+func CopyTree(src Path, dest Path) error {
+	srcFiles, err := src.ReadTree()
+	if err != nil {
+		return fmt.Errorf("error reading source directory %q: %v", src, err)
+	}
+
+	destFiles, err := dest.ReadTree()
+	if err != nil {
+		return fmt.Errorf("error reading source directory %q: %v", src, err)
+	}
+
+	destFileMap := make(map[string]Path)
+	for _, destFile := range destFiles {
+		relativePath, err := RelativePath(dest, destFile)
+		if err != nil {
+			return err
+		}
+
+		destFileMap[relativePath] = destFile
+	}
+
+	for _, srcFile := range srcFiles {
+		relativePath, err := RelativePath(src, srcFile)
+		if err != nil {
+			return err
+		}
+
+		destFile := destFileMap[relativePath]
+		if destFile != nil {
+			match, err := hashesMatch(srcFile, destFile)
+			if err != nil {
+				return err
+			}
+			if match {
+				continue
+			}
+		}
+
+		destFile = dest.Join(relativePath)
+
+		srcData, err := srcFile.ReadFile()
+		if err != nil {
+			return fmt.Errorf("error reading source file %q: %v", srcFile, err)
+		}
+
+		// We do still read the dest file ... unknown if we should if the destFile supported hash
+		destData, err := destFile.ReadFile()
+		if err != nil {
+			if !os.IsNotExist(err) {
+				return fmt.Errorf("error reading dest file %q: %v", destFile, err)
+			}
+		}
+
+		if destData == nil || !bytes.Equal(srcData, destData) {
+			glog.V(2).Infof("Copying data from %s to %s", srcFile, destFile)
+			err = destFile.WriteFile(srcData)
+			if err != nil {
+				return fmt.Errorf("error writing dest file %q: %v", destFile, err)
+			}
+		}
+	}
+
+	return nil
+}