Merge pull request #3368 from georgebuckerfield/enable-data-encryption

Automatic merge from submit-queue. .

Support encryption-at-rest for the kube-apiserver

This PR adds support for enabling encryption-at-rest for data in etcd, via the kube-apiserver (as per https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data).

I've put the functionality behind a feature flag, `+EnableDataEncryption`. It can then be enabled per-cluster by using `--enable-encryption-config` on the command line, or by adding a `kubeEncryptionConfig` section to the cluster spec. This is passed through to the kube-apiserver by the nodeup process. I'm not sure if this is the best way of doing it right now, but it is working.

Fixes #3356.

cmd/kops/create_secret.go

@@ -36,6 +36,9 @@ var (
 
 	kops create secret dockerconfig -f ~/.docker/config.json \
 		--name k8s-cluster.example.com --state s3://example.com
+
+	kops create secret encryptionconfig -f ~/.encryptionconfig.yaml \
+		--name k8s-cluster.example.com --state s3://example.com
 	`))
 
 	create_secret_short = i18n.T(`Create a secret.`)
@@ -52,6 +55,7 @@ func NewCmdCreateSecret(f *util.Factory, out io.Writer) *cobra.Command {
 	// create subcommands
 	cmd.AddCommand(NewCmdCreateSecretPublicKey(f, out))
 	cmd.AddCommand(NewCmdCreateSecretDockerConfig(f, out))
+	cmd.AddCommand(NewCmdCreateSecretEncryptionConfig(f, out))
 
 	return cmd
 }

cmd/kops/create_secret_encryptionconfig.go

@@ -0,0 +1,125 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+
+	"github.com/spf13/cobra"
+	"k8s.io/kops/cmd/kops/util"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/apis/kops/registry"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
+	"k8s.io/kubernetes/pkg/util/i18n"
+)
+
+var (
+	create_secret_encryptionconfig_long = templates.LongDesc(i18n.T(`
+	Create a new encryption config, and store it in the state store.
+	Used to configure encryption-at-rest by the kube-apiserver process
+	on each of the master nodes. The config is not updated by this command.`))
+
+	create_secret_encryptionconfig_example = templates.Examples(i18n.T(`
+	# Create a new encryption config.
+	kops create secret encryptionconfig -f config.yaml \
+		--name k8s-cluster.example.com --state s3://example.com
+	`))
+
+	create_secret_encryptionconfig_short = i18n.T(`Create an encryption config.`)
+)
+
+type CreateSecretEncryptionConfigOptions struct {
+	ClusterName          string
+	EncryptionConfigPath string
+}
+
+func NewCmdCreateSecretEncryptionConfig(f *util.Factory, out io.Writer) *cobra.Command {
+	options := &CreateSecretEncryptionConfigOptions{}
+
+	cmd := &cobra.Command{
+		Use:     "encryptionconfig",
+		Short:   create_secret_encryptionconfig_short,
+		Long:    create_secret_encryptionconfig_long,
+		Example: create_secret_encryptionconfig_example,
+		Run: func(cmd *cobra.Command, args []string) {
+			if len(args) != 0 {
+				exitWithError(fmt.Errorf("syntax: -f <EncryptionConfigPath>"))
+			}
+
+			err := rootCommand.ProcessArgs(args[0:])
+			if err != nil {
+				exitWithError(err)
+			}
+
+			options.ClusterName = rootCommand.ClusterName()
+
+			err = RunCreateSecretEncryptionConfig(f, os.Stdout, options)
+			if err != nil {
+				exitWithError(err)
+			}
+		},
+	}
+
+	cmd.Flags().StringVarP(&options.EncryptionConfigPath, "", "f", "", "Path to encryption config yaml file")
+
+	return cmd
+}
+
+func RunCreateSecretEncryptionConfig(f *util.Factory, out io.Writer, options *CreateSecretEncryptionConfigOptions) error {
+	if options.EncryptionConfigPath == "" {
+		return fmt.Errorf("encryption config path is required (use -f)")
+	}
+
+	secret, err := fi.CreateSecret()
+	if err != nil {
+		return fmt.Errorf("error creating encryption config secret: %v", err)
+	}
+
+	cluster, err := GetCluster(f, options.ClusterName)
+	if err != nil {
+		return err
+	}
+
+	secretStore, err := registry.SecretStore(cluster)
+	if err != nil {
+		return err
+	}
+
+	data, err := ioutil.ReadFile(options.EncryptionConfigPath)
+	if err != nil {
+		return fmt.Errorf("error reading encryption config %v: %v", options.EncryptionConfigPath, err)
+	}
+
+	var parsedData map[string]interface{}
+	err = kops.ParseRawYaml(data, &parsedData)
+	if err != nil {
+		return fmt.Errorf("Unable to parse yaml %v: %v", options.EncryptionConfigPath, err)
+	}
+
+	secret.Data = data
+
+	_, _, err = secretStore.GetOrCreateSecret("encryptionconfig", secret)
+	if err != nil {
+		return fmt.Errorf("error adding encryption config secret: %v", err)
+	}
+
+	return nil
+}

docs/cli/kops_create_secret.md

@@ -19,6 +19,9 @@ Create a secret
   
   kops create secret dockerconfig -f ~/.docker/config.json \
   --name k8s-cluster.example.com --state s3://example.com
+  
+  kops create secret encryptionconfig -f ~/.encryptionconfig.yaml \
+  --name k8s-cluster.example.com --state s3://example.com
 ```
 
 ### Options inherited from parent commands
@@ -39,5 +42,6 @@ Create a secret
 ### SEE ALSO
 * [kops create](kops_create.md)	 - Create a resource by command line, filename or stdin.
 * [kops create secret dockerconfig](kops_create_secret_dockerconfig.md)	 - Create a docker config.
+* [kops create secret encryptionconfig](kops_create_secret_encryptionconfig.md)	 - Create an encryption config.
 * [kops create secret sshpublickey](kops_create_secret_sshpublickey.md)	 - Create a ssh public key.
 

docs/cli/kops_create_secret_encryptionconfig.md

@@ -0,0 +1,48 @@
+
+<!--- This file is automatically generated by make gen-cli-docs; changes should be made in the go CLI command code (under cmd/kops) -->
+
+## kops create secret encryptionconfig
+
+Create an encryption config.
+
+### Synopsis
+
+
+Create a new encryption config, and store it in the state store. Used to configure encryption-at-rest by the kube-apiserver process on each of the master nodes. The config is not updated by this command.
+
+```
+kops create secret encryptionconfig
+```
+
+### Examples
+
+```
+  # Create a new encryption config.
+  kops create secret encryptionconfig -f config.yaml \
+  --name k8s-cluster.example.com --state s3://example.com
+```
+
+### Options
+
+```
+  -f, -- string   Path to encryption config yaml file
+```
+
+### Options inherited from parent commands
+
+```
+      --alsologtostderr                  log to standard error as well as files
+      --config string                    config file (default is $HOME/.kops.yaml)
+      --log_backtrace_at traceLocation   when logging hits line file:N, emit a stack trace (default :0)
+      --log_dir string                   If non-empty, write log files in this directory
+      --logtostderr                      log to standard error instead of files (default false)
+      --name string                      Name of cluster
+      --state string                     Location of state storage
+      --stderrthreshold severity         logs at or above this threshold go to stderr (default 2)
+  -v, --v Level                          log level for V logs
+      --vmodule moduleSpec               comma-separated list of pattern=N settings for file-filtered logging
+```
+
+### SEE ALSO
+* [kops create secret](kops_create_secret.md)	 - Create a secret.
+

nodeup/pkg/model/kubeapiserver.go

@@ -52,6 +52,23 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error {
 		return err
 	}
 
+	if b.Cluster.Spec.EncryptionConfig != nil {
+		if *b.Cluster.Spec.EncryptionConfig && b.IsKubernetesGTE("1.7") {
+			b.Cluster.Spec.KubeAPIServer.ExperimentalEncryptionProviderConfig = fi.String(filepath.Join(b.PathSrvKubernetes(), "encryptionconfig.yaml"))
+			key := "encryptionconfig"
+			encryptioncfg, _ := b.SecretStore.Secret(key)
+			if encryptioncfg != nil {
+				contents := string(encryptioncfg.Data)
+				t := &nodetasks.File{
+					Path:     *b.Cluster.Spec.KubeAPIServer.ExperimentalEncryptionProviderConfig,
+					Contents: fi.NewStringResource(contents),
+					Mode:     fi.String("600"),
+					Type:     nodetasks.FileType_File,
+				}
+				c.AddTask(t)
+			}
+		}
+	}
 	{
 		pod, err := b.buildPod()
 		if err != nil {

nodeup/pkg/model/kubeapiserver_test.go

@@ -17,9 +17,11 @@ limitations under the License.
 package model
 
 import (
+	"testing"
+
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/flagbuilder"
-	"testing"
+	"k8s.io/kops/upup/pkg/fi"
 )
 
 func Test_KubeAPIServer_BuildFlags(t *testing.T) {
@@ -56,6 +58,12 @@ func Test_KubeAPIServer_BuildFlags(t *testing.T) {
 			},
 			"--insecure-port=0 --secure-port=0 --service-node-port-range=30000-33000",
 		},
+		{
+			kops.KubeAPIServerConfig{
+				ExperimentalEncryptionProviderConfig: fi.String("/srv/kubernetes/encryptionconfig.yaml"),
+			},
+			"--experimental-encryption-provider-config=/srv/kubernetes/encryptionconfig.yaml --insecure-port=0 --secure-port=0",
+		},
 	}
 
 	for _, g := range grid {

nodeup/pkg/model/secrets.go

@@ -181,7 +181,7 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error {
 
 		var lines []string
 		for id, token := range allTokens {
-			if id == "dockerconfig" {
+			if id == "dockerconfig" || id == "encryptionconfig" {
 				continue
 			}
 			lines = append(lines, token+","+id+","+id)

pkg/apis/kops/cluster.go

@@ -151,6 +151,8 @@ type ClusterSpec struct {
 
 	// IAM field adds control over the IAM security policies applied to resources
 	IAM *IAMSpec `json:"iam,omitempty"`
+	// EncryptionConfig controls if encryption is enabled
+	EncryptionConfig *bool `json:"encryptionConfig,omitempty"`
 }
 
 // FileAssetSpec defines the structure for a file asset

pkg/apis/kops/componentconfig.go

@@ -256,6 +256,8 @@ type KubeAPIServerConfig struct {
 	AuthorizationMode *string `json:"authorizationMode,omitempty" flag:"authorization-mode"`
 	// AuthorizationRBACSuperUser is the name of the superuser for default rbac
 	AuthorizationRBACSuperUser *string `json:"authorizationRbacSuperUser,omitempty" flag:"authorization-rbac-super-user"`
+	// ExperimentalEncryptionProviderConfig enables encryption at rest for secrets.
+	ExperimentalEncryptionProviderConfig *string `json:"experimentalEncryptionProviderConfig,omitempty" flag:"experimental-encryption-provider-config"`
 }
 
 // KubeControllerManagerConfig is the configuration for the controller

pkg/apis/kops/v1alpha1/cluster.go

@@ -147,6 +147,8 @@ type ClusterSpec struct {
 	Assets *Assets `json:"assets,omitempty"`
 	// IAM field adds control over the IAM security policies applied to resources
 	IAM *IAMSpec `json:"iam,omitempty"`
+	// EncryptionConfig holds the encryption config
+	EncryptionConfig *bool `json:"encryptionConfig,omitempty"`
 }
 
 // FileAssetSpec defines the structure for a file asset

pkg/apis/kops/v1alpha1/componentconfig.go

@@ -256,6 +256,8 @@ type KubeAPIServerConfig struct {
 	AuthorizationMode *string `json:"authorizationMode,omitempty" flag:"authorization-mode"`
 	// AuthorizationRBACSuperUser is the name of the superuser for default rbac
 	AuthorizationRBACSuperUser *string `json:"authorizationRbacSuperUser,omitempty" flag:"authorization-rbac-super-user"`
+	// ExperimentalEncryptionProviderConfig enables encryption at rest for secrets.
+	ExperimentalEncryptionProviderConfig *string `json:"experimentalEncryptionProviderConfig,omitempty" flag:"experimental-encryption-provider-config"`
 }
 
 // KubeControllerManagerConfig is the configuration for the controller

pkg/apis/kops/v1alpha1/zz_generated.conversion.go

@@ -754,6 +754,7 @@ func autoConvert_v1alpha1_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out *
 	} else {
 		out.IAM = nil
 	}
+	out.EncryptionConfig = in.EncryptionConfig
 	return nil
 }
 
@@ -988,6 +989,7 @@ func autoConvert_kops_ClusterSpec_To_v1alpha1_ClusterSpec(in *kops.ClusterSpec,
 	} else {
 		out.IAM = nil
 	}
+	out.EncryptionConfig = in.EncryptionConfig
 	return nil
 }
 
@@ -1752,6 +1754,7 @@ func autoConvert_v1alpha1_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *Ku
 	out.AuthenticationTokenWebhookCacheTTL = in.AuthenticationTokenWebhookCacheTTL
 	out.AuthorizationMode = in.AuthorizationMode
 	out.AuthorizationRBACSuperUser = in.AuthorizationRBACSuperUser
+	out.ExperimentalEncryptionProviderConfig = in.ExperimentalEncryptionProviderConfig
 	return nil
 }
 
@@ -1803,6 +1806,7 @@ func autoConvert_kops_KubeAPIServerConfig_To_v1alpha1_KubeAPIServerConfig(in *ko
 	out.AuthenticationTokenWebhookCacheTTL = in.AuthenticationTokenWebhookCacheTTL
 	out.AuthorizationMode = in.AuthorizationMode
 	out.AuthorizationRBACSuperUser = in.AuthorizationRBACSuperUser
+	out.ExperimentalEncryptionProviderConfig = in.ExperimentalEncryptionProviderConfig
 	return nil
 }
 

pkg/apis/kops/v1alpha2/cluster.go

@@ -177,6 +177,8 @@ type ClusterSpec struct {
 
 	// IAM field adds control over the IAM security policies applied to resources
 	IAM *IAMSpec `json:"iam,omitempty"`
+	// EncryptionConfig holds the encryption config
+	EncryptionConfig *bool `json:"encryptionConfig,omitempty"`
 }
 
 // FileAssetSpec defines the structure for a file asset

pkg/apis/kops/v1alpha2/componentconfig.go

@@ -256,6 +256,8 @@ type KubeAPIServerConfig struct {
 	AuthorizationMode *string `json:"authorizationMode,omitempty" flag:"authorization-mode"`
 	// AuthorizationRBACSuperUser is the name of the superuser for default rbac
 	AuthorizationRBACSuperUser *string `json:"authorizationRbacSuperUser,omitempty" flag:"authorization-rbac-super-user"`
+	// ExperimentalEncryptionProviderConfig enables encryption at rest for secrets.
+	ExperimentalEncryptionProviderConfig *string `json:"experimentalEncryptionProviderConfig,omitempty" flag:"experimental-encryption-provider-config"`
 }
 
 // KubeControllerManagerConfig is the configuration for the controller

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -801,6 +801,7 @@ func autoConvert_v1alpha2_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out *
 	} else {
 		out.IAM = nil
 	}
+	out.EncryptionConfig = in.EncryptionConfig
 	return nil
 }
 
@@ -1050,6 +1051,7 @@ func autoConvert_kops_ClusterSpec_To_v1alpha2_ClusterSpec(in *kops.ClusterSpec,
 	} else {
 		out.IAM = nil
 	}
+	out.EncryptionConfig = in.EncryptionConfig
 	return nil
 }
 
@@ -2011,6 +2013,7 @@ func autoConvert_v1alpha2_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *Ku
 	out.AuthenticationTokenWebhookCacheTTL = in.AuthenticationTokenWebhookCacheTTL
 	out.AuthorizationMode = in.AuthorizationMode
 	out.AuthorizationRBACSuperUser = in.AuthorizationRBACSuperUser
+	out.ExperimentalEncryptionProviderConfig = in.ExperimentalEncryptionProviderConfig
 	return nil
 }
 
@@ -2062,6 +2065,7 @@ func autoConvert_kops_KubeAPIServerConfig_To_v1alpha2_KubeAPIServerConfig(in *ko
 	out.AuthenticationTokenWebhookCacheTTL = in.AuthenticationTokenWebhookCacheTTL
 	out.AuthorizationMode = in.AuthorizationMode
 	out.AuthorizationRBACSuperUser = in.AuthorizationRBACSuperUser
+	out.ExperimentalEncryptionProviderConfig = in.ExperimentalEncryptionProviderConfig
 	return nil
 }