Fixed #28 CIS Profile now works and is different for v1.25+ and v1.24- (#110)

Fixes #28 and makes it possible to differenciate CIS profiles between versions
Removed snake case in util.go
Fixed Units tests for the CIS scenario

Signed-off-by: Mohamed Belgaied Hassine <belgaied2@hotmail.com>

bootstrap/api/v1alpha1/rke2config_types.go

@@ -83,7 +83,7 @@ type RKE2AgentConfig struct {
 	Snapshotter string `json:"snapshotter,omitempty"`
 
 	// CISProfile activates CIS compliance of RKE2 for a certain profile
-	// +kubebuilder:validation:Enum=cis-1.23
+	// +kubebuilder:validation:Enum=cis-1.23;cis-1.5;cis-1.6
 	//+optional
 	CISProfile CISProfile `json:"cisProfile,omitempty"`
 
@@ -210,6 +210,12 @@ type CISProfile string
 const (
 	// CIS1_23 references RKE2's CIS Profile "cis-1.23".
 	CIS1_23 CISProfile = "cis-1.23"
+
+	// CIS1_5 references RKE2's CIS Profile "cis-1.5".
+	CIS1_5 CISProfile = "cis-1.5"
+
+	// CIS1_6 references RKE2's CIS Profile "cis-1.6".
+	CIS1_6 CISProfile = "cis-1.6"
 )
 
 // Encoding specifies the cloud-init file encoding.

bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_rke2configs.yaml

@@ -48,6 +48,8 @@ spec:
                       certain profile
                     enum:
                     - cis-1.23
+                    - cis-1.5
+                    - cis-1.6
                     type: string
                   containerRuntimeEndpoint:
                     description: ContainerRuntimeEndpoint Disable embedded containerd

bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_rke2configtemplates.yaml

@@ -61,6 +61,8 @@ spec:
                               for a certain profile
                             enum:
                             - cis-1.23
+                            - cis-1.5
+                            - cis-1.6
                             type: string
                           containerRuntimeEndpoint:
                             description: ContainerRuntimeEndpoint Disable embedded

bootstrap/internal/cloudinit/cloudinit.go

@@ -92,6 +92,7 @@ type BaseUserData struct {
 	SentinelFileCommand string
 	AirGapped           bool
 	NTPServers          []string
+	CISEnabled          bool
 }
 
 func generate(kind string, tpl string, data interface{}) ([]byte, error) {

bootstrap/internal/cloudinit/cloudinit_test.go

@@ -120,3 +120,38 @@ runcmd:
 `))
 	})
 })
+
+var _ = Describe("WorkerCISTest", func() {
+	var input *BaseUserData
+
+	BeforeEach(func() {
+		input = &BaseUserData{
+			AirGapped:   false,
+			CISEnabled:  true,
+			RKE2Version: "v1.25.6+rke2r1",
+		}
+	})
+	It("Should use the RKE2 CIS Node Preparation script", func() {
+		workerCloudInitData, err := NewJoinWorker(input)
+		Expect(err).ToNot(HaveOccurred())
+		workerCloudInitString := string(workerCloudInitData)
+		_, err = GinkgoWriter.Write(workerCloudInitData)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(workerCloudInitString).To(Equal(`## template: jinja
+#cloud-config
+
+write_files:
+-   path: 
+    content: |
+      
+
+runcmd:
+  - 'curl -sfL https://get.rke2.io | INSTALL_RKE2_VERSION=v1.25.6+rke2r1 INSTALL_RKE2_TYPE="agent" sh -s -'
+  - '/opt/rke2-cis-script.sh'
+  - 'systemctl enable rke2-agent.service'
+  - 'systemctl start rke2-agent.service'
+  - 'mkdir /run/cluster-api' 
+  - 'echo success > /run/cluster-api/bootstrap-success.complete'
+`))
+	})
+})

bootstrap/internal/cloudinit/controlplane_init.go

@@ -30,6 +30,8 @@ const (
 runcmd:
 {{- template "commands" .PreRKE2Commands }}
   - {{ if .AirGapped }}INSTALL_RKE2_ARTIFACT_PATH=/opt/rke2-artifacts sh /opt/install.sh{{ else }}'curl -sfL https://get.rke2.io | INSTALL_RKE2_VERSION=%[1]s sh -s - server'{{ end }} 
+{{- if .CISEnabled }}
+  - '/opt/rke2-cis-script.sh'{{ end }}
   - 'systemctl enable rke2-server.service'
   - 'systemctl start rke2-server.service'
   - 'mkdir /run/cluster-api' 

bootstrap/internal/cloudinit/worker_join.go

@@ -28,6 +28,8 @@ const (
 runcmd:
 {{- template "commands" .PreRKE2Commands }}
   - '{{ if .AirGapped }}INSTALL_RKE2_ARTIFACT_PATH=/opt/rke2-artifacts INSTALL_RKE2_TYPE="agent" sh /opt/install.sh{{ else }}curl -sfL https://get.rke2.io | INSTALL_RKE2_VERSION=%[1]s INSTALL_RKE2_TYPE="agent" sh -s -{{end}}'
+{{- if .CISEnabled }}
+  - '/opt/rke2-cis-script.sh'{{ end }}
   - 'systemctl enable rke2-agent.service'
   - 'systemctl start rke2-agent.service'
   - 'mkdir /run/cluster-api' 

bootstrap/internal/controllers/rke2config_controller.go

@@ -401,6 +401,7 @@ func (r *RKE2ConfigReconciler) handleClusterNotInitialized(ctx context.Context,
 	cpinput := &cloudinit.ControlPlaneInput{
 		BaseUserData: cloudinit.BaseUserData{
 			AirGapped:        scope.Config.Spec.AgentConfig.AirGapped,
+			CISEnabled:       scope.Config.Spec.AgentConfig.CISProfile != "",
 			PreRKE2Commands:  scope.Config.Spec.PreRKE2Commands,
 			PostRKE2Commands: scope.Config.Spec.PostRKE2Commands,
 			ConfigFile:       initConfigFile,
@@ -564,6 +565,7 @@ func (r *RKE2ConfigReconciler) joinControlplane(ctx context.Context, scope *Scop
 	cpinput := &cloudinit.ControlPlaneInput{
 		BaseUserData: cloudinit.BaseUserData{
 			AirGapped:        scope.Config.Spec.AgentConfig.AirGapped,
+			CISEnabled:       scope.Config.Spec.AgentConfig.CISProfile != "",
 			PreRKE2Commands:  scope.Config.Spec.PreRKE2Commands,
 			PostRKE2Commands: scope.Config.Spec.PostRKE2Commands,
 			ConfigFile:       initConfigFile,
@@ -655,6 +657,7 @@ func (r *RKE2ConfigReconciler) joinWorker(ctx context.Context, scope *Scope) (re
 	wkInput := &cloudinit.BaseUserData{
 		PreRKE2Commands:  scope.Config.Spec.PreRKE2Commands,
 		AirGapped:        scope.Config.Spec.AgentConfig.AirGapped,
+		CISEnabled:       scope.Config.Spec.AgentConfig.CISProfile != "",
 		PostRKE2Commands: scope.Config.Spec.PostRKE2Commands,
 		ConfigFile:       wkJoinConfigFile,
 		RKE2Version:      scope.Config.Spec.AgentConfig.Version,

controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_rke2controlplanes.yaml

@@ -48,6 +48,8 @@ spec:
                       certain profile
                     enum:
                     - cis-1.23
+                    - cis-1.5
+                    - cis-1.6
                     type: string
                   containerRuntimeEndpoint:
                     description: ContainerRuntimeEndpoint Disable embedded containerd

pkg/consts/global_consts.go

@@ -38,4 +38,13 @@ const (
 
 	// DefaultSyncPeriod is the default resync period for the controller manager's cache.
 	DefaultSyncPeriod = 10 * time.Minute
+
+	// DefaultFileOwner is the default owner of the files created by the controller.
+	DefaultFileOwner = "root:root"
+
+	// DefaultFileMode is the default mode of the files created by the controller.
+	DefaultFileMode = "644"
+
+	// FileModeRootExecutable is the mode of the files created by the controller when the owner is root.
+	FileModeRootExecutable = "700"
 )

pkg/rke2/config.go

@@ -28,6 +28,8 @@ import (
 
 	bootstrapv1 "github.com/rancher-sandbox/cluster-api-provider-rke2/bootstrap/api/v1alpha1"
 	controlplanev1 "github.com/rancher-sandbox/cluster-api-provider-rke2/controlplane/api/v1alpha1"
+	"github.com/rancher-sandbox/cluster-api-provider-rke2/pkg/consts"
+	bsutil "github.com/rancher-sandbox/cluster-api-provider-rke2/pkg/util"
 )
 
 const (
@@ -39,6 +41,40 @@ const (
 
 	// DefaultRKE2JoinPort is the default port used for joining nodes to the cluster. It is open on the control plane nodes.
 	DefaultRKE2JoinPort = 9345
+
+	// CISNodePreparationScript is the script that is used to prepare a node for CIS compliance.
+	CISNodePreparationScript = `#!/bin/bash
+set -e
+
+# Adding etcd user if not present
+if id etcd &>/dev/null; then
+    echo 'user etcd already exists'
+else
+		useradd -r -c "etcd user" -s /sbin/nologin -M etcd -U
+fi
+
+YUM_BASED_PARAM_FILE_FOUND=false
+TAR_BASED_PARAM_FILE_FOUND=false
+
+# Using RKE2 generated kernel parameters
+if [ -f /usr/share/rke2/rke2-cis-sysctl.conf ]; then
+		YUM_BASED_PARAM_FILE_FOUND=true
+		cp -f /usr/share/rke2/rke2-cis-sysctl.conf /etc/sysctl.d/90-rke2-cis.conf
+fi
+
+if [ -f /usr/local/share/rke2/rke2-cis-sysctl.conf ]; then
+		TAR_BASED_PARAM_FILE_FOUND=true
+		cp -f /usr/local/share/rke2/rke2-cis-sysctl.conf /etc/sysctl.d/90-rke2-cis.conf
+fi
+
+if [ "$YUM_BASED_PARAM_FILE_FOUND" = false ] && [ "$TAR_BASED_PARAM_FILE_FOUND" = false ]; then
+		echo "No kernel parameters file found"
+		exit 1
+fi
+
+# Applying kernel parameters
+sysctl -p /etc/sysctl.d/90-rke2-cis.conf
+`
 )
 
 type rke2ServerConfig struct {
@@ -365,6 +401,20 @@ func newRKE2AgentConfig(opts AgentConfigOpts) (*rke2AgentConfig, []bootstrapv1.F
 	files := []bootstrapv1.File{}
 	rke2AgentConfig.ContainerRuntimeEndpoint = opts.AgentConfig.ContainerRuntimeEndpoint
 
+	if opts.AgentConfig.CISProfile != "" {
+		if !bsutil.ProfileCompliant(opts.AgentConfig.CISProfile, opts.AgentConfig.Version) {
+			return nil, nil, fmt.Errorf("profile %q is not supported for version %q", opts.AgentConfig.CISProfile, opts.AgentConfig.Version)
+		}
+
+		files = append(files, bootstrapv1.File{
+			Path:        "/opt/rke2-cis-script.sh",
+			Content:     CISNodePreparationScript,
+			Owner:       consts.DefaultFileOwner,
+			Permissions: consts.FileModeRootExecutable,
+		})
+		rke2AgentConfig.Profile = string(opts.AgentConfig.CISProfile)
+	}
+
 	if opts.CloudProviderConfigMap != nil {
 		cloudProviderConfigMap := &corev1.ConfigMap{}
 		if err := opts.Client.Get(opts.Ctx, types.NamespacedName{
@@ -432,7 +482,6 @@ func newRKE2AgentConfig(opts AgentConfigOpts) (*rke2AgentConfig, []bootstrapv1.F
 	rke2AgentConfig.LbServerPort = opts.AgentConfig.LoadBalancerPort
 	rke2AgentConfig.NodeLabels = opts.AgentConfig.NodeLabels
 	rke2AgentConfig.NodeTaints = opts.AgentConfig.NodeTaints
-	rke2AgentConfig.Profile = string(opts.AgentConfig.CISProfile)
 	rke2AgentConfig.ProtectKernelDefaults = opts.AgentConfig.ProtectKernelDefaults
 
 	if opts.AgentConfig.ResolvConf != nil {

pkg/rke2/config_test.go

@@ -289,6 +289,7 @@ var _ = Describe("RKE2 Agent Config", func() {
 					ExtraEnv:      map[string]string{"testenv": "testenv"},
 					ExtraMounts:   map[string]string{"testmount": "testmount"},
 				},
+				Version: "v1.25.2",
 			},
 		}
 	})
@@ -319,16 +320,16 @@ var _ = Describe("RKE2 Agent Config", func() {
 		Expect(agentConfig.KubeProxyExtraEnv).To(Equal(opts.AgentConfig.KubeProxy.ExtraEnv))
 		Expect(agentConfig.Token).To(Equal(opts.Token))
 
-		Expect(files).To(HaveLen(2))
+		Expect(files).To(HaveLen(3))
 
-		Expect(files[0].Path).To(Equal(agentConfig.ImageCredentialProviderConfig))
-		Expect(files[0].Content).To(Equal("test_credential_config"))
-		Expect(files[0].Owner).To(Equal("root:root"))
-		Expect(files[0].Permissions).To(Equal("0644"))
-
-		Expect(files[1].Path).To(Equal(agentConfig.ResolvConf))
-		Expect(files[1].Content).To(Equal("test_resolv_conf"))
+		Expect(files[1].Path).To(Equal(agentConfig.ImageCredentialProviderConfig))
+		Expect(files[1].Content).To(Equal("test_credential_config"))
 		Expect(files[1].Owner).To(Equal("root:root"))
 		Expect(files[1].Permissions).To(Equal("0644"))
+
+		Expect(files[2].Path).To(Equal(agentConfig.ResolvConf))
+		Expect(files[2].Content).To(Equal("test_resolv_conf"))
+		Expect(files[2].Owner).To(Equal("root:root"))
+		Expect(files[2].Permissions).To(Equal("0644"))
 	})
 })

pkg/util/util.go

@@ -24,14 +24,21 @@ import (
 	"regexp"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/version"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 
+	bootstrapv1 "github.com/rancher-sandbox/cluster-api-provider-rke2/bootstrap/api/v1alpha1"
 	controlplanev1 "github.com/rancher-sandbox/cluster-api-provider-rke2/controlplane/api/v1alpha1"
 )
 
+const (
+	// RKE2_CIS_VERSION_CHANGE is the version where the CIS benchmark changed in RKE2 (because of PSPs).
+	RKE2_CIS_VERSION_CHANGE = "v1.25.0"
+)
+
 // ErrControlPlaneNotFound is returned when a control plane is not found.
 var ErrControlPlaneNotFound = fmt.Errorf("control plane not found")
 
@@ -161,3 +168,37 @@ func GetMapKeysAsString(m map[string][]byte) (keys string) {
 
 	return
 }
+
+// AtLeastv125 returns true if the RKE2 version is at least v1.25.0.
+func AtLeastv125(rke2version string) (bool, error) {
+	kubeVersion, err := Rke2ToKubeVersion(rke2version)
+	if err != nil {
+		return false, err
+	}
+
+	parsedVersion := version.MustParseGeneric(kubeVersion)
+	if parsedVersion.AtLeast(version.MustParseGeneric(RKE2_CIS_VERSION_CHANGE)) {
+		return true, nil
+	}
+
+	return false, nil
+}
+
+// ProfileCompliant returns true if the CIS profile is compliant.
+func ProfileCompliant(profile bootstrapv1.CISProfile, version string) bool {
+	isAtLeastv125, err := AtLeastv125(version)
+	if err != nil {
+		return false
+	}
+
+	switch profile {
+	case bootstrapv1.CIS1_23:
+		return isAtLeastv125
+	case bootstrapv1.CIS1_5:
+		return !isAtLeastv125
+	case bootstrapv1.CIS1_6:
+		return !isAtLeastv125
+	default:
+		return false
+	}
+}

pkg/util/util_test.go

@@ -22,6 +22,7 @@ var _ = Describe(("Testing GetMapKeysAsString"), func() {
 			"foo":   []byte("bar"),
 		}
 		keys := GetMapKeysAsString(testMap)
-		Expect(keys).To(Equal("hello,foo"))
+		keysValid := keys == "hello,foo" || keys == "foo,hello"
+		Expect(keysValid).To(BeTrue())
 	})
 })

samples/docker/cis-profile/rke2-sample.yaml

@@ -0,0 +1,106 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ${CABPR_NAMESPACE}
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster 
+metadata:
+  namespace: ${CABPR_NAMESPACE}
+  name: ${CLUSTER_NAME} 
+spec:
+  clusterNetwork:
+    pods:
+      cidrBlocks:
+      - 10.45.0.0/16
+    services:
+      cidrBlocks:
+      - 10.46.0.0/16
+    serviceDomain: cluster.local
+  controlPlaneRef:
+    apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
+    kind: RKE2ControlPlane
+    name: ${CLUSTER_NAME}-control-plane
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: DockerCluster
+    name: ${CLUSTER_NAME}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerCluster
+metadata:
+  name: ${CLUSTER_NAME}
+  namespace: ${CABPR_NAMESPACE}
+---
+apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
+kind: RKE2ControlPlane
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+  namespace: ${CABPR_NAMESPACE}
+spec: 
+  replicas: ${CABPR_CP_REPLICAS}
+  agentConfig:
+    version: ${KUBERNETES_VERSION}+rke2r1
+    cisProfile: ${CIS_PROFILE}
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: DockerMachineTemplate
+    name: controlplane
+  nodeDrainTimeout: 2m
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachineTemplate
+metadata:
+  name: controlplane
+  namespace: ${CABPR_NAMESPACE}
+spec:
+  template:
+    spec: {} 
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  name: worker-md-0
+  namespace: ${CABPR_NAMESPACE}
+spec:
+  clusterName: ${CLUSTER_NAME}
+  replicas: ${CABPR_WK_REPLICAS}
+  selector:
+    matchLabels:
+      cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
+  template:
+    spec:
+      version: ${KUBERNETES_VERSION}
+      clusterName: ${CLUSTER_NAME}
+      bootstrap:
+        configRef:
+          apiVersion: bootstrap.cluster.x-k8s.io/v1alpha1
+          kind: RKE2ConfigTemplate
+          name: ${CLUSTER_NAME}-agent
+          namespace: ${CABPR_NAMESPACE}
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: DockerMachineTemplate
+        name: worker
+        namespace: ${CABPR_NAMESPACE}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachineTemplate
+metadata:
+  name: worker
+  namespace: ${CABPR_NAMESPACE}
+spec:
+  template:
+    spec: {} 
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1alpha1
+kind: RKE2ConfigTemplate
+metadata:
+  namespace: ${CABPR_NAMESPACE}
+  name: ${CLUSTER_NAME}-agent
+spec: 
+  template:
+    spec:
+      agentConfig:
+        version: ${KUBERNETES_VERSION}+rke2r1
+        cisProfile: ${CIS_PROFILE}