Merge pull request #758 from AbsaOSS/secretsatrest

✨ Add EncryptionConfig support to RKE2ControlPlane

controlplane/api/v1alpha1/conversion.go

@@ -71,6 +71,7 @@ func (src *RKE2ControlPlane) ConvertTo(dstRaw conversion.Hub) error {
 	}
 
 	dst.Spec.ServerConfig.EmbeddedRegistry = restored.Spec.ServerConfig.EmbeddedRegistry
+	dst.Spec.ServerConfig.SecretsEncryptionProvider = restored.Spec.ServerConfig.SecretsEncryptionProvider
 	dst.Spec.MachineTemplate = restored.Spec.MachineTemplate
 	dst.Status = restored.Status
 	dst.Spec.Files = restored.Spec.Files
@@ -149,6 +150,7 @@ func (src *RKE2ControlPlaneTemplate) ConvertTo(dstRaw conversion.Hub) error {
 	}
 
 	dst.Spec.Template.Spec.ServerConfig.EmbeddedRegistry = restored.Spec.Template.Spec.ServerConfig.EmbeddedRegistry
+	dst.Spec.Template.Spec.ServerConfig.SecretsEncryptionProvider = restored.Spec.Template.Spec.ServerConfig.SecretsEncryptionProvider
 	dst.Spec.Template = restored.Spec.Template
 	dst.Status = restored.Status
 	dst.Spec.Template.Spec.MachineTemplate.NodeDrainTimeout = restored.Spec.Template.Spec.MachineTemplate.NodeDrainTimeout

controlplane/api/v1alpha1/zz_generated.conversion.go

@@ -597,6 +597,7 @@ func autoConvert_v1beta1_RKE2ServerConfig_To_v1alpha1_RKE2ServerConfig(in *v1bet
 	if err := Convert_v1beta1_EtcdConfig_To_v1alpha1_EtcdConfig(&in.Etcd, &out.Etcd, s); err != nil {
 		return err
 	}
+	// WARNING: in.SecretsEncryptionProvider requires manual conversion: does not exist in peer-type
 	out.KubeAPIServer = (*apiv1alpha1.ComponentConfig)(unsafe.Pointer(in.KubeAPIServer))
 	out.KubeControllerManager = (*apiv1alpha1.ComponentConfig)(unsafe.Pointer(in.KubeControllerManager))
 	out.KubeScheduler = (*apiv1alpha1.ComponentConfig)(unsafe.Pointer(in.KubeScheduler))

controlplane/api/v1beta1/rke2controlplane_types.go

@@ -211,6 +211,10 @@ type RKE2ServerConfig struct {
 	//+optional
 	Etcd EtcdConfig `json:"etcd,omitempty"`
 
+	// SecretsEncrytion defines encryption at rest configuration
+	//+optional
+	SecretsEncryptionProvider *SecretsEncryption `json:"secretsEncryption,omitempty"`
+
 	// KubeAPIServer defines optional custom configuration of the Kube API Server.
 	//+optional
 	KubeAPIServer *bootstrapv1.ComponentConfig `json:"kubeAPIServer,omitempty"`
@@ -402,6 +406,15 @@ type EtcdS3 struct {
 	Folder string `json:"folder,omitempty"`
 }
 
+// SecretsEncryption defines encryption configuration.
+type SecretsEncryption struct {
+	// EncyptionKey secret reference
+	EncryptionKeySecret *corev1.ObjectReference `json:"encryptionKeySecret,omitempty"`
+	// Encryption provider
+	// +kubebuilder:validation:Enum=aescbc;secretbox
+	Provider string `json:"provider,omitempty"`
+}
+
 // CNI defines the Cni options for deploying RKE2.
 type CNI string
 

controlplane/api/v1beta1/zz_generated.deepcopy.go

@@ -420,6 +420,11 @@ func (in *RKE2ServerConfig) DeepCopyInto(out *RKE2ServerConfig) {
 	}
 	in.DisableComponents.DeepCopyInto(&out.DisableComponents)
 	in.Etcd.DeepCopyInto(&out.Etcd)
+	if in.SecretsEncryptionProvider != nil {
+		in, out := &in.SecretsEncryptionProvider, &out.SecretsEncryptionProvider
+		*out = new(SecretsEncryption)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.KubeAPIServer != nil {
 		in, out := &in.KubeAPIServer, &out.KubeAPIServer
 		*out = new(apiv1beta1.ComponentConfig)
@@ -527,3 +532,23 @@ func (in *RolloutStrategy) DeepCopy() *RolloutStrategy {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SecretsEncryption) DeepCopyInto(out *SecretsEncryption) {
+	*out = *in
+	if in.EncryptionKeySecret != nil {
+		in, out := &in.EncryptionKeySecret, &out.EncryptionKeySecret
+		*out = new(corev1.ObjectReference)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretsEncryption.
+func (in *SecretsEncryption) DeepCopy() *SecretsEncryption {
+	if in == nil {
+		return nil
+	}
+	out := new(SecretsEncryption)
+	in.DeepCopyInto(out)
+	return out
+}

controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_rke2controlplanes.yaml

@@ -2557,6 +2557,59 @@ spec:
                   pauseImage:
                     description: PauseImage Override image to use for pause.
                     type: string
+                  secretsEncryption:
+                    description: SecretsEncrytion defines encryption at rest configuration
+                    properties:
+                      encryptionKeySecret:
+                        description: EncyptionKey secret reference
+                        properties:
+                          apiVersion:
+                            description: API version of the referent.
+                            type: string
+                          fieldPath:
+                            description: |-
+                              If referring to a piece of an object instead of an entire object, this string
+                              should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2].
+                              For example, if the object reference is to a container within a pod, this would take on a value like:
+                              "spec.containers{name}" (where "name" refers to the name of the container that triggered
+                              the event) or if no container name is specified "spec.containers[2]" (container with
+                              index 2 in this pod). This syntax is chosen only to have some well-defined way of
+                              referencing a part of an object.
+                            type: string
+                          kind:
+                            description: |-
+                              Kind of the referent.
+                              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+                            type: string
+                          name:
+                            description: |-
+                              Name of the referent.
+                              More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                            type: string
+                          namespace:
+                            description: |-
+                              Namespace of the referent.
+                              More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/
+                            type: string
+                          resourceVersion:
+                            description: |-
+                              Specific resourceVersion to which this reference is made, if any.
+                              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency
+                            type: string
+                          uid:
+                            description: |-
+                              UID of the referent.
+                              More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids
+                            type: string
+                        type: object
+                        x-kubernetes-map-type: atomic
+                      provider:
+                        description: Encryption provider
+                        enum:
+                        - aescbc
+                        - secretbox
+                        type: string
+                    type: object
                   serviceNodePortRange:
                     description: 'ServiceNodePortRange is the port range to reserve
                       for services with NodePort visibility (default: "30000-32767").'

controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_rke2controlplanetemplates.yaml

@@ -1414,6 +1414,60 @@ spec:
                           pauseImage:
                             description: PauseImage Override image to use for pause.
                             type: string
+                          secretsEncryption:
+                            description: SecretsEncrytion defines encryption at rest
+                              configuration
+                            properties:
+                              encryptionKeySecret:
+                                description: EncyptionKey secret reference
+                                properties:
+                                  apiVersion:
+                                    description: API version of the referent.
+                                    type: string
+                                  fieldPath:
+                                    description: |-
+                                      If referring to a piece of an object instead of an entire object, this string
+                                      should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2].
+                                      For example, if the object reference is to a container within a pod, this would take on a value like:
+                                      "spec.containers{name}" (where "name" refers to the name of the container that triggered
+                                      the event) or if no container name is specified "spec.containers[2]" (container with
+                                      index 2 in this pod). This syntax is chosen only to have some well-defined way of
+                                      referencing a part of an object.
+                                    type: string
+                                  kind:
+                                    description: |-
+                                      Kind of the referent.
+                                      More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+                                    type: string
+                                  name:
+                                    description: |-
+                                      Name of the referent.
+                                      More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                    type: string
+                                  namespace:
+                                    description: |-
+                                      Namespace of the referent.
+                                      More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/
+                                    type: string
+                                  resourceVersion:
+                                    description: |-
+                                      Specific resourceVersion to which this reference is made, if any.
+                                      More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency
+                                    type: string
+                                  uid:
+                                    description: |-
+                                      UID of the referent.
+                                      More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids
+                                    type: string
+                                type: object
+                                x-kubernetes-map-type: atomic
+                              provider:
+                                description: Encryption provider
+                                enum:
+                                - aescbc
+                                - secretbox
+                                type: string
+                            type: object
                           serviceNodePortRange:
                             description: 'ServiceNodePortRange is the port range to
                               reserve for services with NodePort visibility (default:

docs/book/src/02_topics/09_secrets_encryption.md

@@ -0,0 +1,31 @@
+# Configuring Secrets encryption 
+
+## Overview
+
+By default, RKE2 enables Secret encryotion at rest with `aescbc` provider and generate private key automatically. [Refer](https://docs.rke2.io/security/secrets_encryption)
+
+## Customizing Encryption provider
+
+To configure different provider (`aescbc` or `secretbox`) or specify encryption key explicitly configure `spec.serverConfig.secretsEncryption` block
+
+Expample:
+
+```yaml
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+kind: RKE2ControlPlane
+metadata:
+  name: my-cluster-control-plane
+spec:
+  serverConfig:
+    secretsEncryption:
+      provider: "secretbox"
+      encryptionKeySecret:
+        name: encryption-key
+        namespace: exmaple
+```
+
+## Encryption secret format
+
+When configuring the `encryptionKeySecret`, ensure the secret contains the following keys:
+
+- **encryptionKey** - base64 decoded value of the encryption key

pkg/rke2/config.go

@@ -18,6 +18,7 @@ package rke2
 
 import (
 	"context"
+	"encoding/base64"
 	"fmt"
 	"strings"
 
@@ -41,6 +42,9 @@ const (
 	// DefaultRKE2CloudProviderConfigLocation is the default location for the RKE2 cloud provider config file.
 	DefaultRKE2CloudProviderConfigLocation = "/etc/rancher/rke2/cloud-provider-config"
 
+	// EncryptionConfigurationLocation a location where rke2 looks up for encryption config.
+	EncryptionConfigurationLocation = "/var/lib/rancher/rke2/server/cred/encryption-config.json"
+
 	// DefaultRKE2JoinPort is the default port used for joining nodes to the cluster. It is open on the control plane nodes.
 	DefaultRKE2JoinPort = 9345
 
@@ -83,6 +87,8 @@ fi
 # Applying kernel parameters
 sysctl -p /etc/sysctl.d/90-rke2-cis.conf
 `
+	//nolint:lll //intentionally compacted to a single line
+	encptionConfigTemplate = `{"kind":"EncryptionConfiguration","apiVersion":"apiserver.config.k8s.io/v1","resources":[{"resources":["secrets"],"providers":[{"%s":{"keys":[{"name":"enckey","secret":"%s"}]}},{"identity":{}}]}]}`
 )
 
 // ServerConfig is a struct that contains the information needed to generate a RKE2 server config.
@@ -137,6 +143,7 @@ type ServerConfig struct {
 	DatastoreCAFile                   string   `yaml:"datastore-cafile,omitempty"`
 	DatastoreCertFile                 string   `yaml:"datastore-certfile,omitempty"`
 	DatastoreKeyFile                  string   `yaml:"datastore-keyfile,omitempty"`
+	SecretsEncryptionProvider         string   `yaml:"secrets-encryption-provider,omitempty"`
 
 	// We don't expose these fields in the API
 	ClusterCIDR string `yaml:"cluster-cidr,omitempty"`
@@ -359,6 +366,30 @@ func newRKE2ServerConfig(opts ServerConfigOpts) (*ServerConfig, []bootstrapv1.Fi
 		rke2ServerConfig.KubeSchedulerExtraEnv = componentMapToSlice(extraEnv, opts.ServerConfig.KubeScheduler.ExtraEnv)
 	}
 
+	if opts.ServerConfig.SecretsEncryptionProvider != nil {
+		rke2ServerConfig.SecretsEncryptionProvider = opts.ServerConfig.SecretsEncryptionProvider.Provider
+		encryptionSecret := &corev1.Secret{}
+
+		if err := opts.Client.Get(opts.Ctx, types.NamespacedName{
+			Name:      opts.ServerConfig.SecretsEncryptionProvider.EncryptionKeySecret.Name,
+			Namespace: opts.ServerConfig.SecretsEncryptionProvider.EncryptionKeySecret.Namespace,
+		}, encryptionSecret); err != nil {
+			return nil, nil, fmt.Errorf("failed to get encryptionKey secret: %w", err)
+		}
+
+		key, ok := encryptionSecret.Data["encryptionKey"]
+		if !ok {
+			return nil, nil, errors.New("encryptionKey secret missing 'encryptionKey' key")
+		}
+
+		files = append(files, bootstrapv1.File{
+			Path:        EncryptionConfigurationLocation,
+			Content:     generateEncryptionConfig(opts.ServerConfig.SecretsEncryptionProvider.Provider, base64.StdEncoding.EncodeToString(key)),
+			Owner:       consts.DefaultFileOwner,
+			Permissions: "0400",
+		})
+	}
+
 	if opts.ServerConfig.KubeControllerManager != nil {
 		rke2ServerConfig.KubeControllerManagerArgs = opts.ServerConfig.KubeControllerManager.ExtraArgs
 		rke2ServerConfig.KubeControllerManagerImage = opts.ServerConfig.KubeControllerManager.OverrideImage
@@ -727,3 +758,7 @@ func componentMapToSlice(componentType componentType, input map[string]string) [
 
 	return result
 }
+
+func generateEncryptionConfig(provider, key string) string {
+	return fmt.Sprintf(encptionConfigTemplate, provider, key)
+}

pkg/rke2/config_test.go

@@ -31,6 +31,10 @@ import (
 	"github.com/rancher/cluster-api-provider-rke2/pkg/consts"
 )
 
+const (
+	expEncryptionConfig = `{"kind":"EncryptionConfiguration","apiVersion":"apiserver.config.k8s.io/v1","resources":[{"resources":["secrets"],"providers":[{"secretbox":{"keys":[{"name":"enckey","secret":"dGVzdF9lbmNyeXB0aW9uX2tleQ=="}]}},{"identity":{}}]}]}`
+)
+
 var _ = Describe("RKE2ServerConfig", func() {
 	var opts *ServerConfigOpts
 
@@ -274,6 +278,67 @@ var _ = Describe("RKE2ServerConfig", func() {
 	})
 })
 
+var _ = Describe("RKE2 Server Config with secretbox encryption", func() {
+	var opts *ServerConfigOpts
+
+	BeforeEach(func() {
+
+		opts = &ServerConfigOpts{
+			Token: "token",
+			Cluster: v1beta1.Cluster{
+				Spec: v1beta1.ClusterSpec{
+					ClusterNetwork: &v1beta1.ClusterNetwork{
+						Pods: &v1beta1.NetworkRanges{
+							CIDRBlocks: []string{
+								"192.168.0.0/16",
+							},
+						},
+						Services: &v1beta1.NetworkRanges{
+							CIDRBlocks: []string{
+								"192.169.0.0/16",
+							},
+						},
+					},
+				},
+			},
+			ControlPlaneEndpoint: "testendpoint",
+			Ctx:                  context.Background(),
+			Client: fake.NewClientBuilder().WithObjects(
+				&corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "encryption-key",
+						Namespace: "test",
+					},
+					Data: map[string][]byte{
+						"encryptionKey": []byte("test_encryption_key"),
+					},
+				},
+			).Build(),
+			ServerConfig: controlplanev1.RKE2ServerConfig{
+				BindAddress:   "testbindaddress",
+				CNI:           controlplanev1.Cilium,
+				ClusterDNS:    "testdns",
+				ClusterDomain: "testdomain",
+				SecretsEncryptionProvider: &controlplanev1.SecretsEncryption{
+					Provider: "secretbox",
+					EncryptionKeySecret: &corev1.ObjectReference{
+						Name:      "encryption-key",
+						Namespace: "test",
+					},
+				},
+			},
+		}
+	})
+
+	It("should succefully generate a server config with secretbox key provider", func() {
+		rke2ServerConfig, files, err := GenerateInitControlPlaneConfig(*opts)
+		Expect(err).ToNot(HaveOccurred())
+
+		Expect(rke2ServerConfig.SecretsEncryptionProvider).To(Equal("secretbox"))
+		Expect(files[0].Content).To(Equal(expEncryptionConfig))
+	})
+})
+
 var _ = Describe("RKE2 Agent Config", func() {
 	var opts *AgentConfigOpts