diff --git a/cmd/kops/create.go b/cmd/kops/create.go index f3438c149c..54649320be 100644 --- a/cmd/kops/create.go +++ b/cmd/kops/create.go @@ -27,7 +27,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/kops/cmd/kops/util" kopsapi "k8s.io/kops/pkg/apis/kops" - "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/pkg/apis/kops/v1alpha1" "k8s.io/kops/upup/pkg/fi/cloudup" "k8s.io/kops/util/pkg/vfs" @@ -208,7 +207,7 @@ func RunCreate(f *util.Factory, out io.Writer, c *CreateOptions) error { return err } - keyStore, err := registry.KeyStore(cluster) + keyStore, err := clientset.KeyStore(cluster) if err != nil { return err } diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index 1a162f6317..c65607fe24 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -121,6 +121,9 @@ type CreateClusterOptions struct { // We need VSphereDatastore to support Kubernetes vSphere Cloud Provider (v1.5.3) // We can remove this once we support higher versions. VSphereDatastore string + + // ConfigBase is the location where we will store the configuration, it defaults to the state store + ConfigBase string } func (o *CreateClusterOptions) InitDefaults() { @@ -227,6 +230,11 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command { cmd.Flags().StringVar(&options.Target, "target", options.Target, "Target - direct, terraform, cloudformation") cmd.Flags().StringVar(&options.Models, "model", options.Models, "Models to apply (separate multiple models with commas)") + // Configuration / state location + if featureflag.EnableSeparateConfigBase.Enabled() { + cmd.Flags().StringVar(&options.ConfigBase, "config-base", options.ConfigBase, "A cluster-readable location where we mirror configuration information, separate from the state store. Allows for a state store that is not accessible from the cluster.") + } + cmd.Flags().StringVar(&options.Cloud, "cloud", options.Cloud, "Cloud provider to use - gce, aws, vsphere") cmd.Flags().StringSliceVar(&options.Zones, "zones", options.Zones, "Zones in which to run the cluster") @@ -371,6 +379,7 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e } cluster.Spec.Channel = c.Channel + cluster.Spec.ConfigBase = c.ConfigBase configBase, err := clientset.ConfigBaseFor(cluster) if err != nil { return fmt.Errorf("error building ConfigBase for cluster: %v", err) @@ -933,7 +942,7 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e } assetBuilder := assets.NewAssetBuilder(cluster.Spec.Assets) - fullCluster, err := cloudup.PopulateClusterSpec(cluster, assetBuilder) + fullCluster, err := cloudup.PopulateClusterSpec(clientset, cluster, assetBuilder) if err != nil { return err } @@ -958,7 +967,7 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e return fmt.Errorf("error writing updated configuration: %v", err) } - keyStore, err := registry.KeyStore(cluster) + keyStore, err := clientset.KeyStore(cluster) if err != nil { return err } diff --git a/cmd/kops/create_secret_dockerconfig.go b/cmd/kops/create_secret_dockerconfig.go index c300c3738a..48c3783f8b 100644 --- a/cmd/kops/create_secret_dockerconfig.go +++ b/cmd/kops/create_secret_dockerconfig.go @@ -25,7 +25,6 @@ import ( "github.com/spf13/cobra" "k8s.io/kops/cmd/kops/util" - "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/upup/pkg/fi" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" "k8s.io/kubernetes/pkg/util/i18n" @@ -97,7 +96,12 @@ func RunCreateSecretDockerConfig(f *util.Factory, out io.Writer, options *Create return err } - secretStore, err := registry.SecretStore(cluster) + clientset, err := f.Clientset() + if err != nil { + return err + } + + secretStore, err := clientset.SecretStore(cluster) if err != nil { return err } diff --git a/cmd/kops/create_secret_encryptionconfig.go b/cmd/kops/create_secret_encryptionconfig.go index 893655f477..192e2150c2 100644 --- a/cmd/kops/create_secret_encryptionconfig.go +++ b/cmd/kops/create_secret_encryptionconfig.go @@ -25,7 +25,6 @@ import ( "github.com/spf13/cobra" "k8s.io/kops/cmd/kops/util" "k8s.io/kops/pkg/apis/kops" - "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/upup/pkg/fi" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" "k8s.io/kubernetes/pkg/util/i18n" @@ -98,7 +97,12 @@ func RunCreateSecretEncryptionConfig(f *util.Factory, out io.Writer, options *Cr return err } - secretStore, err := registry.SecretStore(cluster) + clientset, err := f.Clientset() + if err != nil { + return err + } + + secretStore, err := clientset.SecretStore(cluster) if err != nil { return err } diff --git a/cmd/kops/create_secret_sshpublickey.go b/cmd/kops/create_secret_sshpublickey.go index 78bec4addf..8549776f8b 100644 --- a/cmd/kops/create_secret_sshpublickey.go +++ b/cmd/kops/create_secret_sshpublickey.go @@ -24,7 +24,6 @@ import ( "github.com/spf13/cobra" "k8s.io/kops/cmd/kops/util" - "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" "k8s.io/kubernetes/pkg/util/i18n" ) @@ -99,7 +98,12 @@ func RunCreateSecretPublicKey(f *util.Factory, out io.Writer, options *CreateSec return err } - keyStore, err := registry.KeyStore(cluster) + clientset, err := f.Clientset() + if err != nil { + return err + } + + keyStore, err := clientset.KeyStore(cluster) if err != nil { return err } diff --git a/cmd/kops/delete_cluster.go b/cmd/kops/delete_cluster.go index e7078ec4b0..a77ccc7dea 100644 --- a/cmd/kops/delete_cluster.go +++ b/cmd/kops/delete_cluster.go @@ -24,7 +24,6 @@ import ( "github.com/spf13/cobra" "k8s.io/kops/cmd/kops/util" api "k8s.io/kops/pkg/apis/kops" - "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/pkg/kubeconfig" "k8s.io/kops/pkg/resources" "k8s.io/kops/pkg/resources/tracker" @@ -32,7 +31,6 @@ import ( "k8s.io/kops/upup/pkg/fi/cloudup" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/util/pkg/tables" - "k8s.io/kops/util/pkg/vfs" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" "k8s.io/kubernetes/pkg/util/i18n" ) @@ -96,8 +94,6 @@ func NewCmdDeleteCluster(f *util.Factory, out io.Writer) *cobra.Command { type getter func(o interface{}) interface{} func RunDeleteCluster(f *util.Factory, out io.Writer, options *DeleteClusterOptions) error { - var configBase vfs.Path - clusterName := options.ClusterName if clusterName == "" { return fmt.Errorf("--name is required (for safety)") @@ -123,11 +119,6 @@ func RunDeleteCluster(f *util.Factory, out io.Writer, options *DeleteClusterOpti if err != nil { return err } - - configBase, err = registry.ConfigBase(cluster) - if err != nil { - return err - } } wouldDeleteCloudResources := false @@ -205,7 +196,11 @@ func RunDeleteCluster(f *util.Factory, out io.Writer, options *DeleteClusterOpti } return nil } - err := registry.DeleteAllClusterState(configBase) + clientset, err := f.Clientset() + if err != nil { + return err + } + err = clientset.DeleteCluster(cluster) if err != nil { return fmt.Errorf("error removing cluster from state store: %v", err) } diff --git a/cmd/kops/delete_secret.go b/cmd/kops/delete_secret.go index 681c3007dc..5c2a72686c 100644 --- a/cmd/kops/delete_secret.go +++ b/cmd/kops/delete_secret.go @@ -22,7 +22,6 @@ import ( "github.com/spf13/cobra" "k8s.io/kops/cmd/kops/util" - "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/upup/pkg/fi" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" "k8s.io/kubernetes/pkg/util/i18n" @@ -88,17 +87,22 @@ func RunDeleteSecret(f *util.Factory, out io.Writer, options *DeleteSecretOption return fmt.Errorf("SecretName is required") } + clientset, err := f.Clientset() + if err != nil { + return err + } + cluster, err := GetCluster(f, options.ClusterName) if err != nil { return err } - keyStore, err := registry.KeyStore(cluster) + keyStore, err := clientset.KeyStore(cluster) if err != nil { return err } - secretStore, err := registry.SecretStore(cluster) + secretStore, err := clientset.SecretStore(cluster) if err != nil { return err } diff --git a/cmd/kops/describe_secrets.go b/cmd/kops/describe_secrets.go index 55ae4b3a8b..e0a7896a32 100644 --- a/cmd/kops/describe_secrets.go +++ b/cmd/kops/describe_secrets.go @@ -17,17 +17,15 @@ limitations under the License. package main import ( - "fmt" - "bytes" "crypto/rsa" + "fmt" "os" "sort" "strings" "text/tabwriter" "github.com/spf13/cobra" - "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/upup/pkg/fi" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" "k8s.io/kubernetes/pkg/util/i18n" @@ -77,12 +75,17 @@ func (c *DescribeSecretsCommand) Run(args []string) error { return err } - keyStore, err := registry.KeyStore(cluster) + clientset, err := rootCommand.Clientset() if err != nil { return err } - secretStore, err := registry.SecretStore(cluster) + keyStore, err := clientset.KeyStore(cluster) + if err != nil { + return err + } + + secretStore, err := clientset.SecretStore(cluster) if err != nil { return err } diff --git a/cmd/kops/edit_cluster.go b/cmd/kops/edit_cluster.go index 72e5ae79d0..fb7ee1ebf5 100644 --- a/cmd/kops/edit_cluster.go +++ b/cmd/kops/edit_cluster.go @@ -211,7 +211,7 @@ func RunEditCluster(f *util.Factory, cmd *cobra.Command, args []string, out io.W } assetBuilder := assets.NewAssetBuilder(newCluster.Spec.Assets) - fullCluster, err := cloudup.PopulateClusterSpec(newCluster, assetBuilder) + fullCluster, err := cloudup.PopulateClusterSpec(clientset, newCluster, assetBuilder) if err != nil { results = editResults{ file: file, diff --git a/cmd/kops/edit_instancegroup.go b/cmd/kops/edit_instancegroup.go index 5df66554d7..7484f2fd6d 100644 --- a/cmd/kops/edit_instancegroup.go +++ b/cmd/kops/edit_instancegroup.go @@ -168,7 +168,7 @@ func RunEditInstanceGroup(f *util.Factory, cmd *cobra.Command, args []string, ou } assetBuilder := assets.NewAssetBuilder(cluster.Spec.Assets) - fullCluster, err := cloudup.PopulateClusterSpec(cluster, assetBuilder) + fullCluster, err := cloudup.PopulateClusterSpec(clientset, cluster, assetBuilder) if err != nil { return err } diff --git a/cmd/kops/export_kubecfg.go b/cmd/kops/export_kubecfg.go index b7b492956c..aa00e947fe 100644 --- a/cmd/kops/export_kubecfg.go +++ b/cmd/kops/export_kubecfg.go @@ -21,7 +21,6 @@ import ( "github.com/spf13/cobra" "k8s.io/kops/cmd/kops/util" - "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/pkg/kubeconfig" "k8s.io/kops/upup/pkg/fi" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" @@ -73,17 +72,22 @@ func RunExportKubecfg(f *util.Factory, out io.Writer, options *ExportKubecfgOpti return err } + clientset, err := rootCommand.Clientset() + if err != nil { + return err + } + cluster, err := rootCommand.Cluster() if err != nil { return err } - keyStore, err := registry.KeyStore(cluster) + keyStore, err := clientset.KeyStore(cluster) if err != nil { return err } - secretStore, err := registry.SecretStore(cluster) + secretStore, err := clientset.SecretStore(cluster) if err != nil { return err } diff --git a/cmd/kops/get_secrets.go b/cmd/kops/get_secrets.go index 8c36356e10..753cae35fc 100644 --- a/cmd/kops/get_secrets.go +++ b/cmd/kops/get_secrets.go @@ -18,14 +18,12 @@ package main import ( "fmt" + "io" "os" - "strings" "github.com/spf13/cobra" - "io" "k8s.io/kops/cmd/kops/util" - "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/util/pkg/tables" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" @@ -148,12 +146,17 @@ func RunGetSecrets(options *GetSecretsOptions, args []string) error { return err } - keyStore, err := registry.KeyStore(cluster) + clientset, err := rootCommand.Clientset() if err != nil { return err } - secretStore, err := registry.SecretStore(cluster) + keyStore, err := clientset.KeyStore(cluster) + if err != nil { + return err + } + + secretStore, err := clientset.SecretStore(cluster) if err != nil { return err } diff --git a/cmd/kops/update_cluster.go b/cmd/kops/update_cluster.go index d1895b4381..c28bf1f9ca 100644 --- a/cmd/kops/update_cluster.go +++ b/cmd/kops/update_cluster.go @@ -30,7 +30,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kops/cmd/kops/util" "k8s.io/kops/pkg/apis/kops" - "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/pkg/kubeconfig" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup" @@ -144,21 +143,21 @@ func RunUpdateCluster(f *util.Factory, clusterName string, out io.Writer, c *Upd return err } - keyStore, err := registry.KeyStore(cluster) - if err != nil { - return err - } - - secretStore, err := registry.SecretStore(cluster) - if err != nil { - return err - } - clientset, err := f.Clientset() if err != nil { return err } + keyStore, err := clientset.KeyStore(cluster) + if err != nil { + return err + } + + secretStore, err := clientset.SecretStore(cluster) + if err != nil { + return err + } + if c.SSHPublicKey != "" { fmt.Fprintf(out, "--ssh-public-key on update is deprecated - please use `kops create secret --name %s sshpublickey admin -i ~/.ssh/id_rsa.pub` instead\n", cluster.ObjectMeta.Name) diff --git a/cmd/kops/upgrade_cluster.go b/cmd/kops/upgrade_cluster.go index e08654a58d..bc07b580b1 100644 --- a/cmd/kops/upgrade_cluster.go +++ b/cmd/kops/upgrade_cluster.go @@ -290,7 +290,7 @@ func (c *UpgradeClusterCmd) Run(args []string) error { } assetBuilder := assets.NewAssetBuilder(cluster.Spec.Assets) - fullCluster, err := cloudup.PopulateClusterSpec(cluster, assetBuilder) + fullCluster, err := cloudup.PopulateClusterSpec(clientset, cluster, assetBuilder) if err != nil { return err } diff --git a/cmd/kops/util/factory.go b/cmd/kops/util/factory.go index 0cf9a62e6e..22475783b9 100644 --- a/cmd/kops/util/factory.go +++ b/cmd/kops/util/factory.go @@ -29,6 +29,7 @@ import ( // Register our APIs "github.com/golang/glog" _ "k8s.io/kops/pkg/apis/kops/install" + "k8s.io/kops/pkg/client/simple/api" "net/url" "strings" ) @@ -74,6 +75,8 @@ func (f *Factory) Clientset() (simple.Clientset, error) { return nil, fmt.Errorf("Invalid kops server url: %q", registryPath) } + u.Scheme = "https" + config := &rest.Config{ Host: u.Scheme + "://" + u.Host, } @@ -86,7 +89,7 @@ func (f *Factory) Clientset() (simple.Clientset, error) { return nil, fmt.Errorf("error building kops API client: %v", err) } - f.clientset = &simple.RESTClientset{ + f.clientset = &api.RESTClientset{ BaseURL: &url.URL{ Scheme: "k8s", Host: u.Host, diff --git a/examples/kops-api-example/up.go b/examples/kops-api-example/up.go index 4c4393c520..17f5ac170a 100644 --- a/examples/kops-api-example/up.go +++ b/examples/kops-api-example/up.go @@ -20,7 +20,6 @@ import ( "fmt" "io/ioutil" api "k8s.io/kops/pkg/apis/kops" - "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/pkg/client/simple/vfsclientset" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup" @@ -101,7 +100,7 @@ func up() error { } } - keyStore, err := registry.KeyStore(cluster) + keyStore, err := clientset.KeyStore(cluster) if err != nil { return err } diff --git a/federation/apply_federation.go b/federation/apply_federation.go index 3740c61408..fc7f63733f 100644 --- a/federation/apply_federation.go +++ b/federation/apply_federation.go @@ -33,7 +33,6 @@ import ( "k8s.io/kops/federation/targets/kubernetestarget" "k8s.io/kops/federation/tasks" kopsapi "k8s.io/kops/pkg/apis/kops" - "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/pkg/client/simple" "k8s.io/kops/pkg/kubeconfig" "k8s.io/kops/pkg/pki" @@ -177,7 +176,7 @@ func (o *ApplyFederationOperation) Run() error { ClusterName: clusterName, ApiserverHostname: cluster.Spec.MasterPublicName, } - err = a.Run(cluster) + err = a.Run(o.KopsClient, cluster) if err != nil { return err } @@ -195,7 +194,7 @@ func (o *ApplyFederationOperation) Run() error { // Builds a fi.Context applying to the federation namespace in the specified cluster // Note that this operates inside the cluster, for example the KeyStore is backed by secrets in the namespace func (o *ApplyFederationOperation) federationContextForCluster(cluster *kopsapi.Cluster) (*fi.Context, error) { - clusterKeystore, err := registry.KeyStore(cluster) + clusterKeystore, err := o.KopsClient.KeyStore(cluster) if err != nil { return nil, err } diff --git a/federation/federation_cluster.go b/federation/federation_cluster.go index 9775183374..514e5d97e2 100644 --- a/federation/federation_cluster.go +++ b/federation/federation_cluster.go @@ -24,7 +24,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/pkg/api/v1" kopsapi "k8s.io/kops/pkg/apis/kops" - "k8s.io/kops/pkg/apis/kops/registry" + "k8s.io/kops/pkg/client/simple" "k8s.io/kops/pkg/kubeconfig" "k8s.io/kubernetes/federation/apis/federation/v1beta1" "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset" @@ -43,12 +43,12 @@ type FederationCluster struct { ApiserverHostname string } -func (o *FederationCluster) Run(cluster *kopsapi.Cluster) error { - keyStore, err := registry.KeyStore(cluster) +func (o *FederationCluster) Run(clientset simple.Clientset, cluster *kopsapi.Cluster) error { + keyStore, err := clientset.KeyStore(cluster) if err != nil { return err } - secretStore, err := registry.SecretStore(cluster) + secretStore, err := clientset.SecretStore(cluster) if err != nil { return err } diff --git a/hack/.packages b/hack/.packages index 92f050e330..32bd0a2190 100644 --- a/hack/.packages +++ b/hack/.packages @@ -56,6 +56,7 @@ k8s.io/kops/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha1 k8s.io/kops/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2 k8s.io/kops/pkg/client/clientset_generated/internalclientset/typed/kops/v1alpha2/fake k8s.io/kops/pkg/client/simple +k8s.io/kops/pkg/client/simple/api k8s.io/kops/pkg/client/simple/vfsclientset k8s.io/kops/pkg/cloudinstances k8s.io/kops/pkg/diff diff --git a/pkg/apis/kops/registry/registry.go b/pkg/apis/kops/registry/registry.go index cc98123d81..939f6589e4 100644 --- a/pkg/apis/kops/registry/registry.go +++ b/pkg/apis/kops/registry/registry.go @@ -20,10 +20,7 @@ import ( "fmt" "k8s.io/apimachinery/pkg/util/validation/field" api "k8s.io/kops/pkg/apis/kops" - "k8s.io/kops/upup/pkg/fi" - "k8s.io/kops/upup/pkg/fi/secrets" "k8s.io/kops/util/pkg/vfs" - "strings" ) // Path for the user-specified cluster spec @@ -32,46 +29,6 @@ const PathCluster = "config" // Path for completed cluster spec in the state store const PathClusterCompleted = "cluster.spec" -func DeleteAllClusterState(basePath vfs.Path) error { - paths, err := basePath.ReadTree() - if err != nil { - return fmt.Errorf("error listing files in state store: %v", err) - } - - for _, path := range paths { - relativePath, err := vfs.RelativePath(basePath, path) - if err != nil { - return err - } - if relativePath == "config" || relativePath == "cluster.spec" { - continue - } - if strings.HasPrefix(relativePath, "addons/") { - continue - } - if strings.HasPrefix(relativePath, "pki/") { - continue - } - if strings.HasPrefix(relativePath, "secrets/") { - continue - } - if strings.HasPrefix(relativePath, "instancegroup/") { - continue - } - - return fmt.Errorf("refusing to delete: unknown file found: %s", path) - } - - for _, path := range paths { - err = path.Remove() - if err != nil { - return fmt.Errorf("error deleting cluster file %s: %v", path, err) - } - } - - return nil -} - func ConfigBase(c *api.Cluster) (vfs.Path, error) { if c.Spec.ConfigBase == "" { return nil, field.Required(field.NewPath("Spec", "ConfigBase"), "") @@ -82,21 +39,3 @@ func ConfigBase(c *api.Cluster) (vfs.Path, error) { } return configBase, nil } - -func SecretStore(c *api.Cluster) (fi.SecretStore, error) { - configBase, err := ConfigBase(c) - if err != nil { - return nil, err - } - basedir := configBase.Join("secrets") - return secrets.NewVFSSecretStore(basedir), nil -} - -func KeyStore(c *api.Cluster) (fi.CAStore, error) { - configBase, err := ConfigBase(c) - if err != nil { - return nil, err - } - basedir := configBase.Join("pki") - return fi.NewVFSCAStore(basedir), nil -} diff --git a/pkg/client/simple/api/clientset.go b/pkg/client/simple/api/clientset.go new file mode 100644 index 0000000000..4054e55648 --- /dev/null +++ b/pkg/client/simple/api/clientset.go @@ -0,0 +1,198 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "fmt" + "net/url" + "strings" + + "github.com/golang/glog" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/apis/kops/registry" + "k8s.io/kops/pkg/apis/kops/validation" + kopsinternalversion "k8s.io/kops/pkg/client/clientset_generated/clientset/typed/kops/internalversion" + "k8s.io/kops/pkg/client/simple/vfsclientset" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/secrets" + "k8s.io/kops/util/pkg/vfs" + "k8s.io/kubernetes/staging/src/k8s.io/apimachinery/pkg/api/errors" +) + +// RESTClientset is an implementation of clientset that uses a "real" generated REST client +type RESTClientset struct { + BaseURL *url.URL + KopsClient kopsinternalversion.KopsInterface +} + +// GetCluster implements the GetCluster method of Clientset for a kubernetes-API state store +func (c *RESTClientset) GetCluster(name string) (*kops.Cluster, error) { + namespace := restNamespaceForClusterName(name) + return c.KopsClient.Clusters(namespace).Get(name, metav1.GetOptions{}) +} + +// CreateCluster implements the CreateCluster method of Clientset for a kubernetes-API state store +func (c *RESTClientset) CreateCluster(cluster *kops.Cluster) (*kops.Cluster, error) { + namespace := restNamespaceForClusterName(cluster.Name) + return c.KopsClient.Clusters(namespace).Create(cluster) +} + +// UpdateCluster implements the UpdateCluster method of Clientset for a kubernetes-API state store +func (c *RESTClientset) UpdateCluster(cluster *kops.Cluster, status *kops.ClusterStatus) (*kops.Cluster, error) { + glog.Warningf("validating cluster update client side; needs to move to server") + old, err := c.GetCluster(cluster.Name) + if err != nil { + return nil, err + } + if err := validation.ValidateClusterUpdate(cluster, status, old).ToAggregate(); err != nil { + return nil, err + } + + namespace := restNamespaceForClusterName(cluster.Name) + return c.KopsClient.Clusters(namespace).Update(cluster) +} + +// ConfigBaseFor implements the ConfigBaseFor method of Clientset for a kubernetes-API state store +func (c *RESTClientset) ConfigBaseFor(cluster *kops.Cluster) (vfs.Path, error) { + if cluster.Spec.ConfigBase != "" { + return vfs.Context.BuildVfsPath(cluster.Spec.ConfigBase) + } + // URL for clusters looks like https:///apis/kops/v1alpha2/namespaces//clusters/ + // We probably want to add a subresource for full resources + return vfs.Context.BuildVfsPath(c.BaseURL.String()) +} + +// ListClusters implements the ListClusters method of Clientset for a kubernetes-API state store +func (c *RESTClientset) ListClusters(options metav1.ListOptions) (*kops.ClusterList, error) { + return c.KopsClient.Clusters(metav1.NamespaceAll).List(options) +} + +// InstanceGroupsFor implements the InstanceGroupsFor method of Clientset for a kubernetes-API state store +func (c *RESTClientset) InstanceGroupsFor(cluster *kops.Cluster) kopsinternalversion.InstanceGroupInterface { + namespace := restNamespaceForClusterName(cluster.Name) + return c.KopsClient.InstanceGroups(namespace) +} + +// FederationsFor implements the FederationsFor method of Clientset for a kubernetes-API state store +func (c *RESTClientset) FederationsFor(federation *kops.Federation) kopsinternalversion.FederationInterface { + // Unsure if this should be namespaced or not - probably, so that we can RBAC it... + panic("Federations are currently not supported by the server API") + //namespace := restNamespaceForFederationName(federation.Name) + //return c.KopsClient.Federations(namespace) +} + +// ListFederations implements the ListFederations method of Clientset for a kubernetes-API state store +func (c *RESTClientset) ListFederations(options metav1.ListOptions) (*kops.FederationList, error) { + return c.KopsClient.Federations(metav1.NamespaceAll).List(options) +} + +// GetFederation implements the GetFederation method of Clientset for a kubernetes-API state store +func (c *RESTClientset) GetFederation(name string) (*kops.Federation, error) { + namespace := restNamespaceForFederationName(name) + return c.KopsClient.Federations(namespace).Get(name, metav1.GetOptions{}) +} + +func (c *RESTClientset) SecretStore(cluster *kops.Cluster) (fi.SecretStore, error) { + namespace := restNamespaceForClusterName(cluster.Name) + return secrets.NewClientsetSecretStore(c.KopsClient, namespace), nil +} + +func (c *RESTClientset) KeyStore(cluster *kops.Cluster) (fi.CAStore, error) { + namespace := restNamespaceForClusterName(cluster.Name) + return fi.NewClientsetCAStore(c.KopsClient, namespace), nil +} + +func (c *RESTClientset) DeleteCluster(cluster *kops.Cluster) error { + configBase, err := registry.ConfigBase(cluster) + if err != nil { + return err + } + + err = vfsclientset.DeleteAllClusterState(configBase) + if err != nil { + return err + } + + name := cluster.Name + namespace := restNamespaceForClusterName(name) + + { + keysets, err := c.KopsClient.Keysets(namespace).List(metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("error listing Keysets: %v", err) + } + + for i := range keysets.Items { + keyset := &keysets.Items[i] + err = c.KopsClient.Keysets(namespace).Delete(keyset.Name, &metav1.DeleteOptions{}) + if err != nil { + if errors.IsNotFound(err) { + // Unlikely... + glog.Warningf("Keyset was concurrently deleted") + } else { + return fmt.Errorf("error deleting Keyset %q: %v", keyset.Name, err) + } + } + } + } + + { + igs, err := c.KopsClient.InstanceGroups(namespace).List(metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("error listing instance groups: %v", err) + } + + for i := range igs.Items { + ig := &igs.Items[i] + err = c.KopsClient.InstanceGroups(namespace).Delete(ig.Name, &metav1.DeleteOptions{}) + if err != nil { + if errors.IsNotFound(err) { + // Unlikely... + glog.Warningf("instance group was concurrently deleted") + } else { + return fmt.Errorf("error deleting instance group %q: %v", ig.Name, err) + } + } + } + } + + err = c.KopsClient.Clusters(namespace).Delete(name, &metav1.DeleteOptions{}) + if err != nil { + if errors.IsNotFound(err) { + // Unlikely... + glog.Warningf("cluster %q was concurrently deleted", name) + } else { + return fmt.Errorf("error deleting cluster%q: %v", name, err) + } + } + + return nil +} + +func restNamespaceForClusterName(clusterName string) string { + // We are not allowed dots, so we map them to dashes + // This can conflict, but this will simply be a limitation that we pass on to the user + // i.e. it will not be possible to create a.b.example.com and a-b.example.com + namespace := strings.Replace(clusterName, ".", "-", -1) + return namespace +} + +func restNamespaceForFederationName(clusterName string) string { + namespace := strings.Replace(clusterName, ".", "-", -1) + return namespace +} diff --git a/pkg/client/simple/clientset.go b/pkg/client/simple/clientset.go index c097ea692f..79e4772df7 100644 --- a/pkg/client/simple/clientset.go +++ b/pkg/client/simple/clientset.go @@ -17,14 +17,11 @@ limitations under the License. package simple import ( - "github.com/golang/glog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kops/pkg/apis/kops" - "k8s.io/kops/pkg/apis/kops/validation" kopsinternalversion "k8s.io/kops/pkg/client/clientset_generated/clientset/typed/kops/internalversion" + "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/util/pkg/vfs" - "net/url" - "strings" ) type Clientset interface { @@ -54,87 +51,13 @@ type Clientset interface { // ListFederations returns all federations ListFederations(options metav1.ListOptions) (*kops.FederationList, error) -} -// RESTClientset is an implementation of clientset that uses a "real" generated REST client -type RESTClientset struct { - BaseURL *url.URL - KopsClient kopsinternalversion.KopsInterface -} + // SecretStore builds the secret store for the specified cluster + SecretStore(cluster *kops.Cluster) (fi.SecretStore, error) -// GetCluster implements the GetCluster method of Clientset for a kubernetes-API state store -func (c *RESTClientset) GetCluster(name string) (*kops.Cluster, error) { - namespace := restNamespaceForClusterName(name) - return c.KopsClient.Clusters(namespace).Get(name, metav1.GetOptions{}) -} + // KeyStore builds the key store for the specified cluster + KeyStore(cluster *kops.Cluster) (fi.CAStore, error) -// CreateCluster implements the CreateCluster method of Clientset for a kubernetes-API state store -func (c *RESTClientset) CreateCluster(cluster *kops.Cluster) (*kops.Cluster, error) { - namespace := restNamespaceForClusterName(cluster.Name) - return c.KopsClient.Clusters(namespace).Create(cluster) -} - -// UpdateCluster implements the UpdateCluster method of Clientset for a kubernetes-API state store -func (c *RESTClientset) UpdateCluster(cluster *kops.Cluster, status *kops.ClusterStatus) (*kops.Cluster, error) { - glog.Warningf("validating cluster update client side; needs to move to server") - old, err := c.GetCluster(cluster.Name) - if err != nil { - return nil, err - } - if err := validation.ValidateClusterUpdate(cluster, status, old).ToAggregate(); err != nil { - return nil, err - } - - namespace := restNamespaceForClusterName(cluster.Name) - return c.KopsClient.Clusters(namespace).Update(cluster) -} - -// ConfigBaseFor implements the ConfigBaseFor method of Clientset for a kubernetes-API state store -func (c *RESTClientset) ConfigBaseFor(cluster *kops.Cluster) (vfs.Path, error) { - // URL for clusters looks like https:///apis/kops/v1alpha2/namespaces//clusters/ - // We probably want to add a subresource for full resources - return vfs.Context.BuildVfsPath(c.BaseURL.String()) -} - -// ListClusters implements the ListClusters method of Clientset for a kubernetes-API state store -func (c *RESTClientset) ListClusters(options metav1.ListOptions) (*kops.ClusterList, error) { - return c.KopsClient.Clusters(metav1.NamespaceAll).List(options) -} - -// InstanceGroupsFor implements the InstanceGroupsFor method of Clientset for a kubernetes-API state store -func (c *RESTClientset) InstanceGroupsFor(cluster *kops.Cluster) kopsinternalversion.InstanceGroupInterface { - namespace := restNamespaceForClusterName(cluster.Name) - return c.KopsClient.InstanceGroups(namespace) -} - -// FederationsFor implements the FederationsFor method of Clientset for a kubernetes-API state store -func (c *RESTClientset) FederationsFor(federation *kops.Federation) kopsinternalversion.FederationInterface { - // Unsure if this should be namespaced or not - probably, so that we can RBAC it... - panic("Federations are curently not supported by the server API") - //namespace := restNamespaceForFederationName(federation.Name) - //return c.KopsClient.Federations(namespace) -} - -// ListFederations implements the ListFederations method of Clientset for a kubernetes-API state store -func (c *RESTClientset) ListFederations(options metav1.ListOptions) (*kops.FederationList, error) { - return c.KopsClient.Federations(metav1.NamespaceAll).List(options) -} - -// GetFederation implements the GetFederation method of Clientset for a kubernetes-API state store -func (c *RESTClientset) GetFederation(name string) (*kops.Federation, error) { - namespace := restNamespaceForFederationName(name) - return c.KopsClient.Federations(namespace).Get(name, metav1.GetOptions{}) -} - -func restNamespaceForClusterName(clusterName string) string { - // We are not allowed dots, so we map them to dashes - // This can conflict, but this will simply be a limitation that we pass on to the user - // i.e. it will not be possible to create a.b.example.com and a-b.example.com - namespace := strings.Replace(clusterName, ".", "-", -1) - return namespace -} - -func restNamespaceForFederationName(clusterName string) string { - namespace := strings.Replace(clusterName, ".", "-", -1) - return namespace + // DeleteCluster deletes all the state for the specified cluster + DeleteCluster(cluster *kops.Cluster) error } diff --git a/pkg/client/simple/vfsclientset/clientset.go b/pkg/client/simple/vfsclientset/clientset.go index 0512daa5c0..fc963eae6b 100644 --- a/pkg/client/simple/vfsclientset/clientset.go +++ b/pkg/client/simple/vfsclientset/clientset.go @@ -17,10 +17,16 @@ limitations under the License. package vfsclientset import ( + "fmt" + "strings" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/apis/kops/registry" kopsinternalversion "k8s.io/kops/pkg/client/clientset_generated/clientset/typed/kops/internalversion" "k8s.io/kops/pkg/client/simple" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/secrets" "k8s.io/kops/util/pkg/vfs" ) @@ -56,6 +62,9 @@ func (c *VFSClientset) ListClusters(options metav1.ListOptions) (*kops.ClusterLi // ConfigBaseFor implements the ConfigBaseFor method of simple.Clientset for a VFS-backed state store func (c *VFSClientset) ConfigBaseFor(cluster *kops.Cluster) (vfs.Path, error) { + if cluster.Spec.ConfigBase != "" { + return vfs.Context.BuildVfsPath(cluster.Spec.ConfigBase) + } return c.clusters().configBase(cluster.Name) } @@ -84,6 +93,73 @@ func (c *VFSClientset) GetFederation(name string) (*kops.Federation, error) { return c.federations().Get(name, metav1.GetOptions{}) } +func (c *VFSClientset) SecretStore(cluster *kops.Cluster) (fi.SecretStore, error) { + configBase, err := registry.ConfigBase(cluster) + if err != nil { + return nil, err + } + basedir := configBase.Join("secrets") + return secrets.NewVFSSecretStore(basedir), nil +} + +func (c *VFSClientset) KeyStore(cluster *kops.Cluster) (fi.CAStore, error) { + configBase, err := registry.ConfigBase(cluster) + if err != nil { + return nil, err + } + basedir := configBase.Join("pki") + return fi.NewVFSCAStore(basedir), nil +} + +func DeleteAllClusterState(basePath vfs.Path) error { + paths, err := basePath.ReadTree() + if err != nil { + return fmt.Errorf("error listing files in state store: %v", err) + } + + for _, path := range paths { + relativePath, err := vfs.RelativePath(basePath, path) + if err != nil { + return err + } + if relativePath == "config" || relativePath == "cluster.spec" { + continue + } + if strings.HasPrefix(relativePath, "addons/") { + continue + } + if strings.HasPrefix(relativePath, "pki/") { + continue + } + if strings.HasPrefix(relativePath, "secrets/") { + continue + } + if strings.HasPrefix(relativePath, "instancegroup/") { + continue + } + + return fmt.Errorf("refusing to delete: unknown file found: %s", path) + } + + for _, path := range paths { + err = path.Remove() + if err != nil { + return fmt.Errorf("error deleting cluster file %s: %v", path, err) + } + } + + return nil +} + +func (c *VFSClientset) DeleteCluster(cluster *kops.Cluster) error { + configBase, err := registry.ConfigBase(cluster) + if err != nil { + return err + } + + return DeleteAllClusterState(configBase) +} + func NewVFSClientset(basePath vfs.Path) simple.Clientset { vfsClientset := &VFSClientset{ basePath: basePath, diff --git a/pkg/client/simple/vfsclientset/instancegroup.go b/pkg/client/simple/vfsclientset/instancegroup.go index 5a00011a57..457383069a 100644 --- a/pkg/client/simple/vfsclientset/instancegroup.go +++ b/pkg/client/simple/vfsclientset/instancegroup.go @@ -28,6 +28,7 @@ import ( "k8s.io/kops/pkg/apis/kops/v1alpha1" "k8s.io/kops/pkg/apis/kops/validation" kopsinternalversion "k8s.io/kops/pkg/client/clientset_generated/clientset/typed/kops/internalversion" + "k8s.io/kops/util/pkg/vfs" ) type InstanceGroupVFS struct { @@ -36,6 +37,27 @@ type InstanceGroupVFS struct { clusterName string } +type InstanceGroupMirror interface { + WriteMirror(ig *kops.InstanceGroup) error +} + +var _ InstanceGroupMirror = &InstanceGroupVFS{} + +func NewInstanceGroupMirror(clusterName string, configBase vfs.Path) InstanceGroupMirror { + kind := "InstanceGroup" + + r := &InstanceGroupVFS{ + clusterName: clusterName, + } + r.init(kind, configBase.Join("instancegroup"), StoreVersion) + defaultReadVersion := v1alpha1.SchemeGroupVersion.WithKind(kind) + r.defaultReadVersion = &defaultReadVersion + r.validate = func(o runtime.Object) error { + return validation.ValidateInstanceGroup(o.(*kops.InstanceGroup)) + } + return r +} + func newInstanceGroupVFS(c *VFSClientset, clusterName string) *InstanceGroupVFS { if clusterName == "" { glog.Fatalf("clusterName is required") @@ -112,6 +134,15 @@ func (c *InstanceGroupVFS) Update(g *api.InstanceGroup) (*api.InstanceGroup, err return g, nil } +func (c *InstanceGroupVFS) WriteMirror(g *api.InstanceGroup) error { + err := c.writeConfig(c.basePath.Join(g.Name), g) + if err != nil { + return fmt.Errorf("error writing %s: %v", c.kind, err) + } + + return nil +} + func (c *InstanceGroupVFS) Delete(name string, options *metav1.DeleteOptions) error { return c.delete(name, options) } diff --git a/pkg/featureflag/featureflag.go b/pkg/featureflag/featureflag.go index 4c2d78b1ba..f09692b3a3 100644 --- a/pkg/featureflag/featureflag.go +++ b/pkg/featureflag/featureflag.go @@ -55,6 +55,9 @@ var EnableExternalDNS = New("EnableExternalDNS", Bool(false)) //EnableExternalCloudController toggles the use of cloud-controller-manager introduced in v1.7 var EnableExternalCloudController = New("EnableExternalCloudController", Bool(false)) +// EnableSeparateConfigBase allows a config-base that is different from the state store +var EnableSeparateConfigBase = New("EnableSeparateConfigBase", Bool(false)) + // SpecOverrideFlag allows setting spec values on create var SpecOverrideFlag = New("SpecOverrideFlag", Bool(false)) diff --git a/pkg/model/pki.go b/pkg/model/pki.go index 1ffe2a6483..219cad2a5b 100644 --- a/pkg/model/pki.go +++ b/pkg/model/pki.go @@ -22,6 +22,7 @@ import ( "k8s.io/apiserver/pkg/authentication/user" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/fitasks" + "k8s.io/kops/util/pkg/vfs" ) // PKIModelBuilder configures PKI keypairs, as well as tokens @@ -196,5 +197,32 @@ func (b *PKIModelBuilder) Build(c *fi.ModelBuilderContext) error { c.AddTask(t) } + { + mirrorPath, err := vfs.Context.BuildVfsPath(b.Cluster.Spec.SecretStore) + if err != nil { + return err + } + + t := &fitasks.MirrorSecrets{ + Name: fi.String("mirror-secrets"), + MirrorPath: mirrorPath, + } + c.AddTask(t) + } + + { + mirrorPath, err := vfs.Context.BuildVfsPath(b.Cluster.Spec.KeyStore) + if err != nil { + return err + } + + // Keypair used by the kubelet + t := &fitasks.MirrorKeystore{ + Name: fi.String("mirror-keystore"), + MirrorPath: mirrorPath, + } + c.AddTask(t) + } + return nil } diff --git a/upup/pkg/fi/ca.go b/upup/pkg/fi/ca.go index dc20767544..c23b504e8c 100644 --- a/upup/pkg/fi/ca.go +++ b/upup/pkg/fi/ca.go @@ -51,8 +51,16 @@ type Keystore interface { CreateKeypair(name string, template *x509.Certificate, privateKey *pki.PrivateKey) (*pki.Certificate, error) - // Store the keypair + // StoreKeypair writes the keypair to the store StoreKeypair(id string, cert *pki.Certificate, privateKey *pki.PrivateKey) error + + // MirrorTo will copy secrets to a vfs.Path, which is often easier for a machine to read + MirrorTo(basedir vfs.Path) error +} + +// HasVFSPath is implemented by keystore & other stores that use a VFS path as their backing store +type HasVFSPath interface { + VFSPath() vfs.Path } type CAStore interface { @@ -70,9 +78,6 @@ type CAStore interface { // List will list all the items, but will not fetch the data List() ([]*KeystoreItem, error) - // VFSPath returns the path where the CAStore is stored - VFSPath() vfs.Path - // AddCert adds an alternative certificate to the pool (primarily useful for CAs) AddCert(name string, cert *pki.Certificate) error diff --git a/upup/pkg/fi/clientset_castore.go b/upup/pkg/fi/clientset_castore.go index b17e3b9a63..01c4647721 100644 --- a/upup/pkg/fi/clientset_castore.go +++ b/upup/pkg/fi/clientset_castore.go @@ -18,6 +18,7 @@ package fi import ( "bytes" + "crypto/md5" crypto_rand "crypto/rand" "crypto/rsa" "crypto/x509" @@ -610,15 +611,82 @@ func (c *ClientsetCAStore) DeleteSecret(item *KeystoreItem) error { case SecretTypeKeypair: client := c.clientset.Keysets(c.namespace) return DeleteKeysetItem(client, item.Name, kops.SecretTypeKeypair, item.Id) - default: // Primarily because we need to make sure users can recreate them! return fmt.Errorf("deletion of keystore items of type %v not (yet) supported", item.Type) } } -// VFSPath implements CAStore::VFSPath -func (c *ClientsetCAStore) VFSPath() vfs.Path { - // We will implement mirroring instead - panic("ClientsetCAStore::VFSPath not implemented") +func (c *ClientsetCAStore) MirrorTo(basedir vfs.Path) error { + list, err := c.clientset.Keysets(c.namespace).List(v1.ListOptions{}) + if err != nil { + return fmt.Errorf("error listing keysets: %v", err) + } + + for i := range list.Items { + keyset := &list.Items[i] + + if keyset.Spec.Type == kops.SecretTypeSecret { + continue + } + + primary := FindPrimary(keyset) + if primary == nil { + return fmt.Errorf("found keyset with no primary data: %s", keyset.Name) + } + + switch keyset.Spec.Type { + case kops.SecretTypeKeypair: + for i := range keyset.Spec.Keys { + item := &keyset.Spec.Keys[i] + { + p := basedir.Join("issued", keyset.Name, item.Id+".crt") + err = p.WriteFile(item.PublicMaterial) + if err != nil { + return fmt.Errorf("error writing %q: %v", p, err) + } + } + { + p := basedir.Join("private", keyset.Name, item.Id+".key") + err = p.WriteFile(item.PrivateMaterial) + if err != nil { + return fmt.Errorf("error writing %q: %v", p, err) + } + } + } + + default: + return fmt.Errorf("Ignoring unknown secret type: %q", keyset.Spec.Type) + } + } + + sshCredentials, err := c.clientset.SSHCredentials(c.namespace).List(v1.ListOptions{}) + if err != nil { + return fmt.Errorf("error listing SSHCredentials: %v", err) + } + + for i := range sshCredentials.Items { + sshCredential := &sshCredentials.Items[i] + + sshPublicKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(sshCredential.Spec.PublicKey)) + if err != nil { + return fmt.Errorf("error parsing SSH public key %q: %v", sshCredential.Name, err) + } + + // compute fingerprint to serve as id + h := md5.New() + _, err = h.Write(sshPublicKey.Marshal()) + if err != nil { + return fmt.Errorf("error fingerprinting SSH public key: %v", err) + } + id := formatFingerprint(h.Sum(nil)) + + p := basedir.Join("ssh", "public", sshCredential.Name, id) + err = p.WriteFile([]byte(sshCredential.Spec.PublicKey)) + if err != nil { + return fmt.Errorf("error writing %q: %v", p, err) + } + } + + return nil } diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index e4a9c4f11c..3eddfef899 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -58,6 +58,7 @@ import ( "github.com/blang/semver" "github.com/golang/glog" + "k8s.io/kops/pkg/client/simple/vfsclientset" "k8s.io/kops/upup/pkg/fi/cloudup/baremetal" ) @@ -190,12 +191,12 @@ func (c *ApplyClusterCmd) Run() error { return fmt.Errorf("error parsing config base %q: %v", cluster.Spec.ConfigBase, err) } - keyStore, err := registry.KeyStore(cluster) + keyStore, err := c.Clientset.KeyStore(cluster) if err != nil { return err } - secretStore, err := registry.SecretStore(cluster) + secretStore, err := c.Clientset.SecretStore(cluster) if err != nil { return err } @@ -279,9 +280,11 @@ func (c *ApplyClusterCmd) Run() error { checkExisting := true l.AddTypes(map[string]interface{}{ - "keypair": &fitasks.Keypair{}, - "secret": &fitasks.Secret{}, - "managedFile": &fitasks.ManagedFile{}, + "keypair": &fitasks.Keypair{}, + "secret": &fitasks.Secret{}, + "managedFile": &fitasks.ManagedFile{}, + "mirrorKeystore": &fitasks.MirrorKeystore{}, + "mirrorSecrets": &fitasks.MirrorSecrets{}, }) cloud, err := BuildCloud(cluster) @@ -786,11 +789,19 @@ func (c *ApplyClusterCmd) Run() error { return fmt.Errorf("error writing completed cluster spec: %v", err) } + vfsMirror := vfsclientset.NewInstanceGroupMirror(cluster.Name, configBase) + for _, g := range c.InstanceGroups { + // TODO: We need to update the mirror (below), but do we need to update the primary? _, err := c.Clientset.InstanceGroupsFor(c.Cluster).Update(g) if err != nil { return fmt.Errorf("error writing InstanceGroup %q to registry: %v", g.ObjectMeta.Name, err) } + + // TODO: Don't write if vfsMirror == c.ClientSet + if err := vfsMirror.WriteMirror(g); err != nil { + return fmt.Errorf("error writing instance group spec to mirror: %v", err) + } } } @@ -841,7 +852,7 @@ func findHash(url string) (*hashing.Hash, error) { // upgradeSpecs ensures that fields are fully populated / defaulted func (c *ApplyClusterCmd) upgradeSpecs(assetBuilder *assets.AssetBuilder) error { - fullCluster, err := PopulateClusterSpec(c.Cluster, assetBuilder) + fullCluster, err := PopulateClusterSpec(c.Clientset, c.Cluster, assetBuilder) if err != nil { return err } diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go index 5c94f4b095..6d3882108a 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go @@ -64,8 +64,7 @@ func runChannelBuilderTest(t *testing.T, key string) { t.Fatalf("error from PerformAssignments: %v", err) } - assetBuilder := assets.NewAssetBuilder(nil) - fullSpec, err := PopulateClusterSpec(cluster, assetBuilder) + fullSpec, err := mockedPopulateClusterSpec(cluster) if err != nil { t.Fatalf("error from PopulateClusterSpec: %v", err) } diff --git a/upup/pkg/fi/cloudup/populate_cluster_spec.go b/upup/pkg/fi/cloudup/populate_cluster_spec.go index 89d6eaf68f..60858c31f2 100644 --- a/upup/pkg/fi/cloudup/populate_cluster_spec.go +++ b/upup/pkg/fi/cloudup/populate_cluster_spec.go @@ -26,10 +26,10 @@ import ( "github.com/golang/glog" api "k8s.io/kops/pkg/apis/kops" - "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/pkg/apis/kops/util" "k8s.io/kops/pkg/apis/kops/validation" "k8s.io/kops/pkg/assets" + "k8s.io/kops/pkg/client/simple" "k8s.io/kops/pkg/dns" "k8s.io/kops/pkg/model" "k8s.io/kops/pkg/model/components" @@ -66,7 +66,7 @@ func findModelStore() (vfs.Path, error) { // PopulateClusterSpec takes a user-specified cluster spec, and computes the full specification that should be set on the cluster. // We do this so that we don't need any real "brains" on the node side. -func PopulateClusterSpec(cluster *api.Cluster, assetBuilder *assets.AssetBuilder) (*api.Cluster, error) { +func PopulateClusterSpec(clientset simple.Clientset, cluster *api.Cluster, assetBuilder *assets.AssetBuilder) (*api.Cluster, error) { modelStore, err := findModelStore() if err != nil { return nil, err @@ -78,7 +78,7 @@ func PopulateClusterSpec(cluster *api.Cluster, assetBuilder *assets.AssetBuilder Models: []string{"config"}, assetBuilder: assetBuilder, } - err = c.run() + err = c.run(clientset) if err != nil { return nil, err } @@ -95,7 +95,7 @@ func PopulateClusterSpec(cluster *api.Cluster, assetBuilder *assets.AssetBuilder // struct is falling through.. // @kris-nova // -func (c *populateClusterSpec) run() error { +func (c *populateClusterSpec) run(clientset simple.Clientset) error { if err := validation.ValidateCluster(c.InputCluster, false); err != nil { return err } @@ -177,32 +177,6 @@ func (c *populateClusterSpec) run() error { } } - keyStore, err := registry.KeyStore(cluster) - if err != nil { - return err - } - - secretStore, err := registry.SecretStore(cluster) - if err != nil { - return err - } - - if vfs.IsClusterReadable(secretStore.VFSPath()) { - vfsPath := secretStore.VFSPath() - cluster.Spec.SecretStore = vfsPath.Path() - } else { - // We could implement this approach, but it seems better to get all clouds using cluster-readable storage - return fmt.Errorf("secrets path is not cluster readable: %v", secretStore.VFSPath()) - } - - if vfs.IsClusterReadable(keyStore.VFSPath()) { - vfsPath := keyStore.VFSPath() - cluster.Spec.KeyStore = vfsPath.Path() - } else { - // We could implement this approach, but it seems better to get all clouds using cluster-readable storage - return fmt.Errorf("keyStore path is not cluster readable: %v", keyStore.VFSPath()) - } - configBase, err := vfs.Context.BuildVfsPath(cluster.Spec.ConfigBase) if err != nil { return fmt.Errorf("error parsing ConfigBase %q: %v", cluster.Spec.ConfigBase, err) @@ -214,6 +188,46 @@ func (c *populateClusterSpec) run() error { return fmt.Errorf("ConfigBase path is not cluster readable: %v", cluster.Spec.ConfigBase) } + keyStore, err := clientset.KeyStore(cluster) + if err != nil { + return err + } + + if cluster.Spec.KeyStore == "" { + hasVFSPath, ok := keyStore.(fi.HasVFSPath) + if !ok { + // We will mirror to ConfigBase + basedir := configBase.Join("pki") + cluster.Spec.KeyStore = basedir.Path() + } else if vfs.IsClusterReadable(hasVFSPath.VFSPath()) { + vfsPath := hasVFSPath.VFSPath() + cluster.Spec.KeyStore = vfsPath.Path() + } else { + // We could implement this approach, but it seems better to get all clouds using cluster-readable storage + return fmt.Errorf("keyStore path is not cluster readable: %v", hasVFSPath.VFSPath()) + } + } + + secretStore, err := clientset.SecretStore(cluster) + if err != nil { + return err + } + + if cluster.Spec.SecretStore == "" { + hasVFSPath, ok := secretStore.(fi.HasVFSPath) + if !ok { + // We will mirror to ConfigBase + basedir := configBase.Join("secrets") + cluster.Spec.SecretStore = basedir.Path() + } else if vfs.IsClusterReadable(hasVFSPath.VFSPath()) { + vfsPath := hasVFSPath.VFSPath() + cluster.Spec.SecretStore = vfsPath.Path() + } else { + // We could implement this approach, but it seems better to get all clouds using cluster-readable storage + return fmt.Errorf("secrets path is not cluster readable: %v", hasVFSPath.VFSPath()) + } + } + // Normalize k8s version versionWithoutV := strings.TrimSpace(cluster.Spec.KubernetesVersion) if strings.HasPrefix(versionWithoutV, "v") { diff --git a/upup/pkg/fi/cloudup/populatecluster_test.go b/upup/pkg/fi/cloudup/populatecluster_test.go index 97d0b1ebaa..84c1fbe7ab 100644 --- a/upup/pkg/fi/cloudup/populatecluster_test.go +++ b/upup/pkg/fi/cloudup/populatecluster_test.go @@ -24,10 +24,15 @@ import ( "k8s.io/apimachinery/pkg/util/sets" api "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/assets" + "k8s.io/kops/pkg/client/simple/vfsclientset" "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/awsup" + "k8s.io/kops/util/pkg/vfs" ) func buildMinimalCluster() *api.Cluster { + awsup.InstallMockAWSCloud(MockAWSRegion, "abcd") + c := &api.Cluster{} c.ObjectMeta.Name = "testcluster.test.com" c.Spec.KubernetesVersion = "1.4.6" @@ -90,13 +95,24 @@ func TestPopulateCluster_Default_NoError(t *testing.T) { addEtcdClusters(c) - assetBuilder := assets.NewAssetBuilder(nil) - _, err = PopulateClusterSpec(c, assetBuilder) + _, err = mockedPopulateClusterSpec(c) if err != nil { t.Fatalf("Unexpected error from PopulateCluster: %v", err) } } +func mockedPopulateClusterSpec(c *api.Cluster) (*api.Cluster, error) { + vfs.Context.ResetMemfsContext(true) + + assetBuilder := assets.NewAssetBuilder(nil) + basePath, err := vfs.Context.BuildVfsPath("memfs://tests") + if err != nil { + return nil, fmt.Errorf("error building vfspath: %v", err) + } + clientset := vfsclientset.NewVFSClientset(basePath) + return PopulateClusterSpec(clientset, c, assetBuilder) +} + func TestPopulateCluster_Docker_Spec(t *testing.T) { c := buildMinimalCluster() c.Spec.Docker = &api.DockerConfig{ @@ -113,8 +129,7 @@ func TestPopulateCluster_Docker_Spec(t *testing.T) { addEtcdClusters(c) - assetBuilder := assets.NewAssetBuilder(nil) - full, err := PopulateClusterSpec(c, assetBuilder) + full, err := mockedPopulateClusterSpec(c) if err != nil { t.Fatalf("Unexpected error from PopulateCluster: %v", err) } @@ -144,8 +159,8 @@ func TestPopulateCluster_StorageDefault(t *testing.T) { } addEtcdClusters(c) - assetBuilder := assets.NewAssetBuilder(c.Spec.Assets) - full, err := PopulateClusterSpec(c, assetBuilder) + + full, err := mockedPopulateClusterSpec(c) if err != nil { t.Fatalf("Unexpected error from PopulateCluster: %v", err) } @@ -162,8 +177,8 @@ func build(c *api.Cluster) (*api.Cluster, error) { } addEtcdClusters(c) - assetBuilder := assets.NewAssetBuilder(nil) - full, err := PopulateClusterSpec(c, assetBuilder) + + full, err := mockedPopulateClusterSpec(c) if err != nil { return nil, fmt.Errorf("Unexpected error from PopulateCluster: %v", err) } @@ -239,8 +254,7 @@ func TestPopulateCluster_Custom_CIDR(t *testing.T) { addEtcdClusters(c) - assetBuilder := assets.NewAssetBuilder(nil) - full, err := PopulateClusterSpec(c, assetBuilder) + full, err := mockedPopulateClusterSpec(c) if err != nil { t.Fatalf("Unexpected error from PopulateCluster: %v", err) } @@ -260,8 +274,7 @@ func TestPopulateCluster_IsolateMasters(t *testing.T) { addEtcdClusters(c) - assetBuilder := assets.NewAssetBuilder(nil) - full, err := PopulateClusterSpec(c, assetBuilder) + full, err := mockedPopulateClusterSpec(c) if err != nil { t.Fatalf("Unexpected error from PopulateCluster: %v", err) } @@ -284,8 +297,7 @@ func TestPopulateCluster_IsolateMastersFalse(t *testing.T) { addEtcdClusters(c) - assetBuilder := assets.NewAssetBuilder(nil) - full, err := PopulateClusterSpec(c, assetBuilder) + full, err := mockedPopulateClusterSpec(c) if err != nil { t.Fatalf("Unexpected error from PopulateCluster: %v", err) } @@ -376,8 +388,7 @@ func TestPopulateCluster_BastionIdleTimeoutInvalidNegative_Required(t *testing.T } func expectErrorFromPopulateCluster(t *testing.T, c *api.Cluster, message string) { - assetBuilder := assets.NewAssetBuilder(nil) - _, err := PopulateClusterSpec(c, assetBuilder) + _, err := mockedPopulateClusterSpec(c) if err == nil { t.Fatalf("Expected error from PopulateCluster") } @@ -411,8 +422,7 @@ func TestPopulateCluster_AnonymousAuth(t *testing.T) { addEtcdClusters(c) - assetBuilder := assets.NewAssetBuilder(nil) - full, err := PopulateClusterSpec(c, assetBuilder) + full, err := mockedPopulateClusterSpec(c) if err != nil { t.Fatalf("Unexpected error from PopulateCluster: %v", err) } @@ -437,8 +447,7 @@ func TestPopulateCluster_AnonymousAuth_14(t *testing.T) { addEtcdClusters(c) - assetBuilder := assets.NewAssetBuilder(nil) - full, err := PopulateClusterSpec(c, assetBuilder) + full, err := mockedPopulateClusterSpec(c) if err != nil { t.Fatalf("Unexpected error from PopulateCluster: %v", err) } @@ -489,8 +498,7 @@ func TestPopulateCluster_KubeController_High_Enough_Version(t *testing.T) { addEtcdClusters(c) - assetBuilder := assets.NewAssetBuilder(nil) - full, err := PopulateClusterSpec(c, assetBuilder) + full, err := mockedPopulateClusterSpec(c) if err != nil { t.Fatalf("Unexpected error from PopulateCluster: %v", err) } @@ -512,8 +520,7 @@ func TestPopulateCluster_KubeController_Fail(t *testing.T) { addEtcdClusters(c) - assetBuilder := assets.NewAssetBuilder(nil) - full, err := PopulateClusterSpec(c, assetBuilder) + full, err := mockedPopulateClusterSpec(c) if err != nil { t.Fatalf("Unexpected error from PopulateCluster: %v", err) } diff --git a/upup/pkg/fi/cloudup/validation_test.go b/upup/pkg/fi/cloudup/validation_test.go index 921ebee0e9..fbf4ac156a 100644 --- a/upup/pkg/fi/cloudup/validation_test.go +++ b/upup/pkg/fi/cloudup/validation_test.go @@ -22,9 +22,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" api "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/kops/validation" - "k8s.io/kops/pkg/assets" "k8s.io/kops/upup/pkg/fi" - "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "strings" "testing" ) @@ -32,8 +30,6 @@ import ( const MockAWSRegion = "us-mock-1" func buildDefaultCluster(t *testing.T) *api.Cluster { - awsup.InstallMockAWSCloud(MockAWSRegion, "abcd") - c := buildMinimalCluster() err := PerformAssignments(c) @@ -61,8 +57,7 @@ func buildDefaultCluster(t *testing.T) *api.Cluster { } } - assetBuilder := assets.NewAssetBuilder(nil) - fullSpec, err := PopulateClusterSpec(c, assetBuilder) + fullSpec, err := mockedPopulateClusterSpec(c) if err != nil { t.Fatalf("error from PopulateClusterSpec: %v", err) } diff --git a/upup/pkg/fi/fitasks/mirrorkeystore.go b/upup/pkg/fi/fitasks/mirrorkeystore.go new file mode 100644 index 0000000000..0936869ad2 --- /dev/null +++ b/upup/pkg/fi/fitasks/mirrorkeystore.go @@ -0,0 +1,73 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fitasks + +import ( + "github.com/golang/glog" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/util/pkg/vfs" +) + +//go:generate fitask -type=MirrorKeystore +type MirrorKeystore struct { + Name *string + Lifecycle *fi.Lifecycle + + MirrorPath vfs.Path +} + +var _ fi.HasDependencies = &MirrorKeystore{} + +// GetDependencies returns the dependencies for a MirrorKeystore task - it must run after all secrets have been run +func (e *MirrorKeystore) GetDependencies(tasks map[string]fi.Task) []fi.Task { + var deps []fi.Task + for _, task := range tasks { + if _, ok := task.(*Secret); ok { + deps = append(deps, task) + } + } + return deps +} + +// Find implements fi.Task::Find +func (e *MirrorKeystore) Find(c *fi.Context) (*MirrorKeystore, error) { + // TODO: implement Find so that we aren't always mirroring + glog.V(2).Infof("MirrorKeystore::Find not implemented; always copying (inefficient)") + return nil, nil +} + +// Run implements fi.Task::Run +func (e *MirrorKeystore) Run(c *fi.Context) error { + return fi.DefaultDeltaRunMethod(e, c) +} + +// CheckChanges implements fi.Task::CheckChanges +func (s *MirrorKeystore) CheckChanges(a, e, changes *MirrorKeystore) error { + if a != nil { + if changes.Name != nil { + return fi.CannotChangeField("Name") + } + } + return nil +} + +// Render implements fi.Task::Render +func (_ *MirrorKeystore) Render(c *fi.Context, a, e, changes *MirrorKeystore) error { + keystore := c.Keystore + + return keystore.MirrorTo(e.MirrorPath) +} diff --git a/upup/pkg/fi/fitasks/mirrorkeystore_fitask.go b/upup/pkg/fi/fitasks/mirrorkeystore_fitask.go new file mode 100644 index 0000000000..2ce074c1fa --- /dev/null +++ b/upup/pkg/fi/fitasks/mirrorkeystore_fitask.go @@ -0,0 +1,70 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by ""fitask" -type=MirrorKeystore"; DO NOT EDIT + +package fitasks + +import ( + "encoding/json" + + "k8s.io/kops/upup/pkg/fi" +) + +// MirrorKeystore + +// JSON marshalling boilerplate +type realMirrorKeystore MirrorKeystore + +// UnmarshalJSON implements conversion to JSON, supporitng an alternate specification of the object as a string +func (o *MirrorKeystore) UnmarshalJSON(data []byte) error { + var jsonName string + if err := json.Unmarshal(data, &jsonName); err == nil { + o.Name = &jsonName + return nil + } + + var r realMirrorKeystore + if err := json.Unmarshal(data, &r); err != nil { + return err + } + *o = MirrorKeystore(r) + return nil +} + +var _ fi.HasLifecycle = &MirrorKeystore{} + +// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle +func (o *MirrorKeystore) GetLifecycle() *fi.Lifecycle { + return o.Lifecycle +} + +var _ fi.HasName = &MirrorKeystore{} + +// GetName returns the Name of the object, implementing fi.HasName +func (o *MirrorKeystore) GetName() *string { + return o.Name +} + +// SetName sets the Name of the object, implementing fi.SetName +func (o *MirrorKeystore) SetName(name string) { + o.Name = &name +} + +// String is the stringer function for the task, producing readable output using fi.TaskAsString +func (o *MirrorKeystore) String() string { + return fi.TaskAsString(o) +} diff --git a/upup/pkg/fi/fitasks/mirrorsecrets.go b/upup/pkg/fi/fitasks/mirrorsecrets.go new file mode 100644 index 0000000000..e638f83c5c --- /dev/null +++ b/upup/pkg/fi/fitasks/mirrorsecrets.go @@ -0,0 +1,73 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fitasks + +import ( + "github.com/golang/glog" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/util/pkg/vfs" +) + +//go:generate fitask -type=MirrorSecrets +type MirrorSecrets struct { + Name *string + Lifecycle *fi.Lifecycle + + MirrorPath vfs.Path +} + +var _ fi.HasDependencies = &MirrorSecrets{} + +// GetDependencies returns the dependencies for a MirrorSecrets task - it must run after all secrets have been run +func (e *MirrorSecrets) GetDependencies(tasks map[string]fi.Task) []fi.Task { + var deps []fi.Task + for _, task := range tasks { + if _, ok := task.(*Secret); ok { + deps = append(deps, task) + } + } + return deps +} + +// Find implements fi.Task::Find +func (e *MirrorSecrets) Find(c *fi.Context) (*MirrorSecrets, error) { + // TODO: implement Find so that we aren't always mirroring + glog.V(2).Infof("MirrorSecrets::Find not implemented; always copying (inefficient)") + return nil, nil +} + +// Run implemements fi.Task::Run +func (e *MirrorSecrets) Run(c *fi.Context) error { + return fi.DefaultDeltaRunMethod(e, c) +} + +// CheckChanges implements fi.Task::CheckChanges +func (s *MirrorSecrets) CheckChanges(a, e, changes *MirrorSecrets) error { + if a != nil { + if changes.Name != nil { + return fi.CannotChangeField("Name") + } + } + return nil +} + +// Render implements fi.Task::Render +func (_ *MirrorSecrets) Render(c *fi.Context, a, e, changes *MirrorSecrets) error { + secrets := c.SecretStore + + return secrets.MirrorTo(e.MirrorPath) +} diff --git a/upup/pkg/fi/fitasks/mirrorsecrets_fitask.go b/upup/pkg/fi/fitasks/mirrorsecrets_fitask.go new file mode 100644 index 0000000000..0707b3ed8f --- /dev/null +++ b/upup/pkg/fi/fitasks/mirrorsecrets_fitask.go @@ -0,0 +1,70 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by ""fitask" -type=MirrorSecrets"; DO NOT EDIT + +package fitasks + +import ( + "encoding/json" + + "k8s.io/kops/upup/pkg/fi" +) + +// MirrorSecrets + +// JSON marshalling boilerplate +type realMirrorSecrets MirrorSecrets + +// UnmarshalJSON implements conversion to JSON, supporitng an alternate specification of the object as a string +func (o *MirrorSecrets) UnmarshalJSON(data []byte) error { + var jsonName string + if err := json.Unmarshal(data, &jsonName); err == nil { + o.Name = &jsonName + return nil + } + + var r realMirrorSecrets + if err := json.Unmarshal(data, &r); err != nil { + return err + } + *o = MirrorSecrets(r) + return nil +} + +var _ fi.HasLifecycle = &MirrorSecrets{} + +// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle +func (o *MirrorSecrets) GetLifecycle() *fi.Lifecycle { + return o.Lifecycle +} + +var _ fi.HasName = &MirrorSecrets{} + +// GetName returns the Name of the object, implementing fi.HasName +func (o *MirrorSecrets) GetName() *string { + return o.Name +} + +// SetName sets the Name of the object, implementing fi.SetName +func (o *MirrorSecrets) SetName(name string) { + o.Name = &name +} + +// String is the stringer function for the task, producing readable output using fi.TaskAsString +func (o *MirrorSecrets) String() string { + return fi.TaskAsString(o) +} diff --git a/upup/pkg/fi/k8sapi/k8s_keystore.go b/upup/pkg/fi/k8sapi/k8s_keystore.go index 13091deea1..0d70b33b7e 100644 --- a/upup/pkg/fi/k8sapi/k8s_keystore.go +++ b/upup/pkg/fi/k8sapi/k8s_keystore.go @@ -26,6 +26,7 @@ import ( "k8s.io/client-go/pkg/api/v1" "k8s.io/kops/pkg/pki" "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/util/pkg/vfs" "math/big" "time" ) @@ -145,3 +146,7 @@ func (c *KubernetesKeystore) StoreKeypair(id string, cert *pki.Certificate, priv return err } + +func (c *KubernetesKeystore) MirrorTo(dest vfs.Path) error { + return fmt.Errorf("KubernetesKeystore does not implement MirrorTo") +} diff --git a/upup/pkg/fi/secrets.go b/upup/pkg/fi/secrets.go index 9b45f087db..ea81dd2cf4 100644 --- a/upup/pkg/fi/secrets.go +++ b/upup/pkg/fi/secrets.go @@ -36,8 +36,8 @@ type SecretStore interface { // ListSecrets lists the ids of all known secrets ListSecrets() ([]string, error) - // VFSPath returns the path where the SecretStore is stored - VFSPath() vfs.Path + // MirrorTo will copy secrets to a vfs.Path, which is often easier for a machine to read + MirrorTo(basedir vfs.Path) error } type Secret struct { diff --git a/upup/pkg/fi/secrets/clientset_secretstore.go b/upup/pkg/fi/secrets/clientset_secretstore.go index a34c9ed93d..c5ae07b63d 100644 --- a/upup/pkg/fi/secrets/clientset_secretstore.go +++ b/upup/pkg/fi/secrets/clientset_secretstore.go @@ -17,6 +17,7 @@ limitations under the License. package secrets import ( + "encoding/json" "fmt" "strings" "time" @@ -51,6 +52,43 @@ func NewClientsetSecretStore(clientset kopsinternalversion.KopsInterface, namesp return c } +func (c *ClientsetSecretStore) MirrorTo(basedir vfs.Path) error { + list, err := c.clientset.Keysets(c.namespace).List(v1.ListOptions{}) + if err != nil { + return fmt.Errorf("error listing keysets: %v", err) + } + + for i := range list.Items { + keyset := &list.Items[i] + + if keyset.Spec.Type != kops.SecretTypeSecret { + continue + } + + primary := fi.FindPrimary(keyset) + if primary == nil { + return fmt.Errorf("found secret with no primary data: %s", keyset.Name) + } + + name := strings.TrimPrefix(keyset.Name, NamePrefix) + p := BuildVfsSecretPath(basedir, name) + + s := &fi.Secret{ + Data: primary.PrivateMaterial, + } + data, err := json.Marshal(s) + if err != nil { + return fmt.Errorf("error serializing secret: %v", err) + } + + if err := p.WriteFile(data); err != nil { + return fmt.Errorf("error writing secret to %q: %v", p, err) + } + } + + return nil +} + // FindSecret implements fi.SecretStore::FindSecret func (c *ClientsetSecretStore) FindSecret(name string) (*fi.Secret, error) { s, err := c.loadSecret(name) @@ -177,10 +215,3 @@ func (c *ClientsetSecretStore) createSecret(s *fi.Secret, name string) (*kops.Ke return c.clientset.Keysets(c.namespace).Create(keyset) } - -// VFSPath implements fi.SecretStore::VFSPath -func (c *ClientsetSecretStore) VFSPath() vfs.Path { - // We will implement mirroring instead - glog.Fatalf("ClientsetSecretStore::VFSPath not implemented") - return nil -} diff --git a/upup/pkg/fi/secrets/vfs_secretstore.go b/upup/pkg/fi/secrets/vfs_secretstore.go index dc07158dc1..a7eb32e5df 100644 --- a/upup/pkg/fi/secrets/vfs_secretstore.go +++ b/upup/pkg/fi/secrets/vfs_secretstore.go @@ -42,8 +42,21 @@ func (c *VFSSecretStore) VFSPath() vfs.Path { return c.basedir } +func (c *VFSSecretStore) MirrorTo(basedir vfs.Path) error { + if basedir.Path() == c.basedir.Path() { + return nil + } + glog.V(2).Infof("Mirroring secret store from %q to %q", c.basedir, basedir) + + return vfs.CopyTree(c.basedir, basedir) +} + +func BuildVfsSecretPath(basedir vfs.Path, name string) vfs.Path { + return basedir.Join(name) +} + func (c *VFSSecretStore) buildSecretPath(name string) vfs.Path { - return c.basedir.Join(name) + return BuildVfsSecretPath(c.basedir, name) } func (c *VFSSecretStore) FindSecret(id string) (*fi.Secret, error) { diff --git a/upup/pkg/fi/vfs_castore.go b/upup/pkg/fi/vfs_castore.go index 4b4ee35b9f..e6557c34eb 100644 --- a/upup/pkg/fi/vfs_castore.go +++ b/upup/pkg/fi/vfs_castore.go @@ -404,6 +404,16 @@ func (c *VFSCAStore) List() ([]*KeystoreItem, error) { return items, nil } +// MirrorTo will copy keys to a vfs.Path, which is often easier for a machine to read +func (c *VFSCAStore) MirrorTo(basedir vfs.Path) error { + if basedir.Path() == c.basedir.Path() { + return nil + } + glog.V(2).Infof("Mirroring key store from %q to %q", c.basedir, basedir) + + return vfs.CopyTree(c.basedir, basedir) +} + func (c *VFSCAStore) IssueCert(id string, serial *big.Int, privateKey *pki.PrivateKey, template *x509.Certificate) (*pki.Certificate, error) { glog.Infof("Issuing new certificate: %q", id) diff --git a/upup/pkg/kutil/convert_kubeup_cluster.go b/upup/pkg/kutil/convert_kubeup_cluster.go index 2d07522a8a..12cdab8b07 100644 --- a/upup/pkg/kutil/convert_kubeup_cluster.go +++ b/upup/pkg/kutil/convert_kubeup_cluster.go @@ -64,7 +64,7 @@ func (x *ConvertKubeupCluster) Upgrade() error { return fmt.Errorf("OldClusterName must be specified") } - oldKeyStore, err := registry.KeyStore(cluster) + oldKeyStore, err := x.Clientset.KeyStore(cluster) if err != nil { return err } @@ -83,7 +83,7 @@ func (x *ConvertKubeupCluster) Upgrade() error { } cluster.Spec.ConfigBase = newConfigBase.Path() - newKeyStore, err := registry.KeyStore(cluster) + newKeyStore, err := x.Clientset.KeyStore(cluster) if err != nil { return err } @@ -107,7 +107,7 @@ func (x *ConvertKubeupCluster) Upgrade() error { } assetBuilder := assets.NewAssetBuilder(cluster.Spec.Assets) - fullCluster, err := cloudup.PopulateClusterSpec(cluster, assetBuilder) + fullCluster, err := cloudup.PopulateClusterSpec(x.Clientset, cluster, assetBuilder) if err != nil { return err } diff --git a/upup/pkg/kutil/import_cluster.go b/upup/pkg/kutil/import_cluster.go index beb2e89bc7..c430a2566b 100644 --- a/upup/pkg/kutil/import_cluster.go +++ b/upup/pkg/kutil/import_cluster.go @@ -414,7 +414,7 @@ func (x *ImportCluster) ImportAWSCluster() error { //b.Context = "aws_" + instancePrefix - keyStore, err := registry.KeyStore(cluster) + keyStore, err := x.Clientset.KeyStore(cluster) if err != nil { return err } diff --git a/util/pkg/vfs/vfssync.go b/util/pkg/vfs/vfssync.go index 65e7059b44..995c266dd6 100644 --- a/util/pkg/vfs/vfssync.go +++ b/util/pkg/vfs/vfssync.go @@ -213,3 +213,69 @@ func hashesMatch(src, dest Path) (bool, error) { glog.Infof("No compatible hash: %s and %s", src, dest) return false, nil } + +// CopyTree copies all files in src to dest. It copies the whole recursive subtree of files. +func CopyTree(src Path, dest Path) error { + srcFiles, err := src.ReadTree() + if err != nil { + return fmt.Errorf("error reading source directory %q: %v", src, err) + } + + destFiles, err := dest.ReadTree() + if err != nil { + return fmt.Errorf("error reading source directory %q: %v", src, err) + } + + destFileMap := make(map[string]Path) + for _, destFile := range destFiles { + relativePath, err := RelativePath(dest, destFile) + if err != nil { + return err + } + + destFileMap[relativePath] = destFile + } + + for _, srcFile := range srcFiles { + relativePath, err := RelativePath(src, srcFile) + if err != nil { + return err + } + + destFile := destFileMap[relativePath] + if destFile != nil { + match, err := hashesMatch(srcFile, destFile) + if err != nil { + return err + } + if match { + continue + } + } + + destFile = dest.Join(relativePath) + + srcData, err := srcFile.ReadFile() + if err != nil { + return fmt.Errorf("error reading source file %q: %v", srcFile, err) + } + + // We do still read the dest file ... unknown if we should if the destFile supported hash + destData, err := destFile.ReadFile() + if err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("error reading dest file %q: %v", destFile, err) + } + } + + if destData == nil || !bytes.Equal(srcData, destData) { + glog.V(2).Infof("Copying data from %s to %s", srcFile, destFile) + err = destFile.WriteFile(srcData) + if err != nil { + return fmt.Errorf("error writing dest file %q: %v", destFile, err) + } + } + } + + return nil +}