Merge pull request #190 from Danil-Grigorev/arbitrary-cloud-init

Add arbitrary data field

bootstrap/api/v1alpha1/rke2config_types.go

@@ -164,13 +164,21 @@ type RKE2AgentConfig struct {
 	AdditionalUserData AdditionalUserData `json:"additionalUserData,omitempty"`
 }
 
-// AdditionalUserData is a field that allows users to specify additional cloud-init configuration .
+// AdditionalUserData is a field that allows users to specify additional
+// cloud-init configuration .
+// +kubebuilder:validation:XValidation:rule="!has(self.data) || !has(self.config)", message="Only config or data could be populated at once"
 type AdditionalUserData struct {
 	// In case of using ignition, the data format is documented here: https://kinvolk.io/docs/flatcar-container-linux/latest/provisioning/cl-config/
 	// NOTE: All fields of the UserData that are managed by the RKE2Config controller will be ignored, this include "write_files", "runcmd", "ntp".
 	// +optional
+	// Deprecated: Data is reserved for the arbitrary cloud-init data
 	Config string `json:"config,omitempty"`
 
+	// Data allows to pass arbitrary set of key/value pairs consistent with
+	// https://cloudinit.readthedocs.io/en/latest/reference/modules.html
+	// to extend existing cloud-init configuration
+	Data map[string]string `json:"data,omitempty"`
+
 	// Strict controls if Config should be strictly parsed. If so, warnings are treated as errors.
 	// +optional
 	Strict bool `json:"strict,omitempty"`

bootstrap/api/v1alpha1/zz_generated.deepcopy.go

@@ -30,6 +30,13 @@ import (
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *AdditionalUserData) DeepCopyInto(out *AdditionalUserData) {
 	*out = *in
+	if in.Data != nil {
+		in, out := &in.Data, &out.Data
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdditionalUserData.
@@ -209,7 +216,7 @@ func (in *RKE2AgentConfig) DeepCopyInto(out *RKE2AgentConfig) {
 		*out = new(ComponentConfig)
 		(*in).DeepCopyInto(*out)
 	}
-	out.AdditionalUserData = in.AdditionalUserData
+	in.AdditionalUserData.DeepCopyInto(&out.AdditionalUserData)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RKE2AgentConfig.

bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_rke2configs.yaml

@@ -48,13 +48,24 @@ spec:
                           documented here: https://kinvolk.io/docs/flatcar-container-linux/latest/provisioning/cl-config/
                           NOTE: All fields of the UserData that are managed by the
                           RKE2Config controller will be ignored, this include "write_files",
-                          "runcmd", "ntp".'
+                          "runcmd", "ntp". Deprecated: Data is reserved for the arbitrary
+                          cloud-init data'
                         type: string
+                      data:
+                        additionalProperties:
+                          type: string
+                        description: Data allows to pass arbitrary set of key/value
+                          pairs consistent with https://cloudinit.readthedocs.io/en/latest/reference/modules.html
+                          to extend existing cloud-init configuration
+                        type: object
                       strict:
                         description: Strict controls if Config should be strictly
                           parsed. If so, warnings are treated as errors.
                         type: boolean
                     type: object
+                    x-kubernetes-validations:
+                    - message: Only config or data could be populated at once
+                      rule: '!has(self.data) || !has(self.config)'
                   airGapped:
                     description: AirGapped is a boolean value to define if the bootstrapping
                       should be air-gapped, basically supposing that online container

bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_rke2configtemplates.yaml

@@ -61,13 +61,24 @@ spec:
                                   format is documented here: https://kinvolk.io/docs/flatcar-container-linux/latest/provisioning/cl-config/
                                   NOTE: All fields of the UserData that are managed
                                   by the RKE2Config controller will be ignored, this
-                                  include "write_files", "runcmd", "ntp".'
+                                  include "write_files", "runcmd", "ntp". Deprecated:
+                                  Data is reserved for the arbitrary cloud-init data'
                                 type: string
+                              data:
+                                additionalProperties:
+                                  type: string
+                                description: Data allows to pass arbitrary set of
+                                  key/value pairs consistent with https://cloudinit.readthedocs.io/en/latest/reference/modules.html
+                                  to extend existing cloud-init configuration
+                                type: object
                               strict:
                                 description: Strict controls if Config should be strictly
                                   parsed. If so, warnings are treated as errors.
                                 type: boolean
                             type: object
+                            x-kubernetes-validations:
+                            - message: Only config or data could be populated at once
+                              rule: '!has(self.data) || !has(self.config)'
                           airGapped:
                             description: AirGapped is a boolean value to define if
                               the bootstrapping should be air-gapped, basically supposing

bootstrap/internal/cloudinit/cloudinit.go

@@ -86,23 +86,29 @@ ntp:
     {{- end -}}	
 {{- end -}}
 {{- end -}}
+`
+	arbitraryTemplate = `{{- define "arbitrary" -}}{{- range $key, $value := . }}
+{{ $key -}}: {{ $value -}}
+{{- end -}}
+{{- end -}}
 `
 )
 
 // BaseUserData is shared across all the various types of files written to disk.
 type BaseUserData struct {
-	Header              string
+	Header                  string
-	PreRKE2Commands     []string
+	PreRKE2Commands         []string
-	DeployRKE2Commands  []string
+	DeployRKE2Commands      []string
-	PostRKE2Commands    []string
+	PostRKE2Commands        []string
-	WriteFiles          []bootstrapv1.File
+	WriteFiles              []bootstrapv1.File
-	ConfigFile          bootstrapv1.File
+	ConfigFile              bootstrapv1.File
-	RKE2Version         string
+	RKE2Version             string
-	SentinelFileCommand string
+	SentinelFileCommand     string
-	AirGapped           bool
+	AirGapped               bool
-	NTPServers          []string
+	NTPServers              []string
-	CISEnabled          bool
+	CISEnabled              bool
-	AdditionalCloudInit string
+	AdditionalCloudInit     string
+	AdditionalArbitraryData map[string]string
 }
 
 func generate(kind string, tpl string, data interface{}) ([]byte, error) {
@@ -119,6 +125,10 @@ func generate(kind string, tpl string, data interface{}) ([]byte, error) {
 		return nil, errors.Wrap(err, "failed to parse ntp template")
 	}
 
+	if _, err := tm.Parse(arbitraryTemplate); err != nil {
+		return nil, errors.Wrap(err, "failed to parse arbitrary template")
+	}
+
 	t, err := tm.Parse(tpl)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to parse %s template", kind)
@@ -159,3 +169,41 @@ func cleanupAdditionalCloudInit(cloudInitData string) (string, error) {
 
 	return res, nil
 }
+
+func cleanupArbitraryData(arbitraryData map[string]string) error {
+	// Remove ignored fields from the map
+	for _, field := range ignoredCloudInitFields {
+		delete(arbitraryData, field)
+	}
+
+	if len(arbitraryData) == 0 {
+		return nil
+	}
+
+	m := make(map[string]interface{})
+
+	kind := "arbitrary_prepare"
+	tm := template.New(kind).Funcs(defaultTemplateFuncMap)
+
+	if _, err := tm.Parse(arbitraryTemplate); err != nil {
+		return errors.Wrap(err, "failed to parse arbitrary keys template")
+	}
+
+	t, err := tm.Parse(`{{template "arbitrary" .AdditionalArbitraryData}}`)
+	if err != nil {
+		return errors.Wrap(err, "failed to parse arbitrary template")
+	}
+
+	var out bytes.Buffer
+	if err := t.Execute(&out, BaseUserData{
+		AdditionalArbitraryData: arbitraryData,
+	}); err != nil {
+		return errors.Wrapf(err, "failed to generate %s template", kind)
+	}
+
+	if err := yaml.Unmarshal(out.Bytes(), m); err != nil {
+		return fmt.Errorf("failed to unmarshal arbitrary cloud-init data: %w, please check if you put valid yaml data: %s", err, out.String())
+	}
+
+	return nil
+}

bootstrap/internal/cloudinit/cloudinit_test.go

@@ -43,6 +43,7 @@ write_files:
     content: |
       
 
+
 runcmd:
   - 'INSTALL_RKE2_ARTIFACT_PATH=/opt/rke2-artifacts INSTALL_RKE2_TYPE="agent" sh /opt/install.sh'
   - 'systemctl enable rke2-agent.service'
@@ -76,6 +77,7 @@ write_files:
     content: |
       
 
+
 runcmd:
   - 'curl -sfL https://get.rke2.io | INSTALL_RKE2_VERSION=v1.25.6+rke2r1 INSTALL_RKE2_TYPE="agent" sh -s -'
   - 'systemctl enable rke2-agent.service'
@@ -111,6 +113,7 @@ ntp:
   enabled: true
   servers:
   - "test.ntp.org"
+
 runcmd:
   - 'curl -sfL https://get.rke2.io | INSTALL_RKE2_VERSION= INSTALL_RKE2_TYPE="agent" sh -s -'
   - 'systemctl enable rke2-agent.service'
@@ -145,6 +148,7 @@ write_files:
     content: |
       
 
+
 runcmd:
   - 'curl -sfL https://get.rke2.io | INSTALL_RKE2_VERSION=v1.25.6+rke2r1 INSTALL_RKE2_TYPE="agent" sh -s -'
   - '/opt/rke2-cis-script.sh'
@@ -166,6 +170,7 @@ write_files:
 -   path: 
     content: |
 
+
 runcmd:
   - 'curl -sfL https://get.rke2.io | INSTALL_RKE2_VERSION= INSTALL_RKE2_TYPE=\"agent\" sh -s -'
   - 'systemctl enable rke2-agent.service'
@@ -186,8 +191,11 @@ users:
 
 var _ = Describe("CloudInit with custom entries", func() {
 	var input *BaseUserData
+	var cloudInitData string
+	var arbitraryData map[string]string
+
 	BeforeEach(func() {
-		cloudInitData := `## template: jinja
+		cloudInitData = `## template: jinja
 #cloud-config
 device_aliases: {'ephemeral0': '/dev/vdb'}
 disk_setup:
@@ -206,18 +214,29 @@ write_files:
     content: |
       192.168.0.1 test
 
+
 runcmd:
   - 'print hello world' 
 `
-		input = &BaseUserData{
+		arbitraryData = map[string]string{
-			AirGapped:           false,
+			"disk_setup": `
-			CISEnabled:          true,
+  ephemeral0:
-			RKE2Version:         "v1.25.6+rke2r1",
+    table_type: mbr
-			AdditionalCloudInit: cloudInitData,
+    layout: False
+    overwrite: False`,
+			"device_aliases": "{'ephemeral0': '/dev/vdb'}",
+			"runcmd": `
+  - 'print hello world'`,
 		}
 	})
 
-	It("Should remove the runcmd, write_files and ntp lines", func() {
+	It("Should apply the arbitrary data and cleanup the input values", func() {
+		input = &BaseUserData{
+			AirGapped:               false,
+			CISEnabled:              true,
+			RKE2Version:             "v1.25.6+rke2r1",
+			AdditionalArbitraryData: arbitraryData,
+		}
 		workerCloudInitData, err := NewJoinWorker(input)
 		Expect(err).ToNot(HaveOccurred())
 		workerCloudInitString := string(workerCloudInitData)
@@ -232,6 +251,45 @@ write_files:
     content: |
       
 
+
+device_aliases: {'ephemeral0': '/dev/vdb'}
+disk_setup: 
+  ephemeral0:
+    table_type: mbr
+    layout: False
+    overwrite: False
+runcmd:
+  - 'curl -sfL https://get.rke2.io | INSTALL_RKE2_VERSION=v1.25.6+rke2r1 INSTALL_RKE2_TYPE="agent" sh -s -'
+  - '/opt/rke2-cis-script.sh'
+  - 'systemctl enable rke2-agent.service'
+  - 'systemctl start rke2-agent.service'
+  - 'mkdir /run/cluster-api' 
+  - 'echo success > /run/cluster-api/bootstrap-success.complete'
+`))
+	})
+
+	It("Should remove the runcmd, write_files and ntp lines", func() {
+		input = &BaseUserData{
+			AirGapped:           false,
+			CISEnabled:          true,
+			RKE2Version:         "v1.25.6+rke2r1",
+			AdditionalCloudInit: cloudInitData,
+		}
+		workerCloudInitData, err := NewJoinWorker(input)
+		Expect(err).ToNot(HaveOccurred())
+		workerCloudInitString := string(workerCloudInitData)
+		_, err = GinkgoWriter.Write(workerCloudInitData)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(workerCloudInitString).To(Equal(`## template: jinja
+#cloud-config
+
+write_files:
+-   path: 
+    content: |
+      
+
+
 runcmd:
   - 'curl -sfL https://get.rke2.io | INSTALL_RKE2_VERSION=v1.25.6+rke2r1 INSTALL_RKE2_TYPE="agent" sh -s -'
   - '/opt/rke2-cis-script.sh'

bootstrap/internal/cloudinit/controlplane_init.go

@@ -27,6 +27,7 @@ const (
 	controlPlaneCloudInit = `{{.Header}}
 {{template "files" .WriteFiles}}
 {{template "ntp" .NTPServers}}
+{{template "arbitrary" .AdditionalArbitraryData}}
 runcmd:
 {{- template "commands" .PreRKE2Commands }}
   - {{ if .AirGapped }}INSTALL_RKE2_ARTIFACT_PATH=/opt/rke2-artifacts sh /opt/install.sh{{ else }}'curl -sfL https://get.rke2.io | INSTALL_RKE2_VERSION=%[1]s sh -s - server'{{ end }} 
@@ -63,6 +64,10 @@ func NewInitControlPlane(input *ControlPlaneInput) ([]byte, error) {
 		return nil, err
 	}
 
+	if err := cleanupArbitraryData(input.AdditionalArbitraryData); err != nil {
+		return nil, err
+	}
+
 	controlPlaneCloudJoinWithVersion := fmt.Sprintf(controlPlaneCloudInit, input.RKE2Version)
 	userData, err := generate("InitControlplane", controlPlaneCloudJoinWithVersion, input)
 

bootstrap/internal/cloudinit/controlplane_join.go

@@ -35,6 +35,10 @@ func NewJoinControlPlane(input *ControlPlaneInput) ([]byte, error) {
 		return nil, err
 	}
 
+	if err := cleanupArbitraryData(input.AdditionalArbitraryData); err != nil {
+		return nil, err
+	}
+
 	controlPlaneCloudJoinWithVersion := fmt.Sprintf(controlPlaneCloudInit, input.RKE2Version)
 	userData, err := generate("JoinControlplane", controlPlaneCloudJoinWithVersion, input)
 

bootstrap/internal/cloudinit/worker_join.go

@@ -25,6 +25,7 @@ const (
 	workerCloudInit = `{{.Header}}
 {{template "files" .WriteFiles}}
 {{template "ntp" .NTPServers}}
+{{template "arbitrary" .AdditionalArbitraryData}}
 runcmd:
 {{- template "commands" .PreRKE2Commands }}
   - '{{ if .AirGapped }}INSTALL_RKE2_ARTIFACT_PATH=/opt/rke2-artifacts INSTALL_RKE2_TYPE="agent" sh /opt/install.sh{{ else }}curl -sfL https://get.rke2.io | INSTALL_RKE2_VERSION=%[1]s INSTALL_RKE2_TYPE="agent" sh -s -{{end}}'
@@ -54,6 +55,10 @@ func NewJoinWorker(input *BaseUserData) ([]byte, error) {
 		return nil, err
 	}
 
+	if err := cleanupArbitraryData(input.AdditionalArbitraryData); err != nil {
+		return nil, err
+	}
+
 	workerCloudJoinWithVersion := fmt.Sprintf(workerCloudInit, input.RKE2Version)
 	userData, err := generate("JoinWorker", workerCloudJoinWithVersion, input)
 

bootstrap/internal/controllers/rke2config_controller.go

@@ -402,15 +402,16 @@ func (r *RKE2ConfigReconciler) handleClusterNotInitialized(ctx context.Context,
 
 	cpinput := &cloudinit.ControlPlaneInput{
 		BaseUserData: cloudinit.BaseUserData{
-			AirGapped:           scope.Config.Spec.AgentConfig.AirGapped,
+			AirGapped:               scope.Config.Spec.AgentConfig.AirGapped,
-			CISEnabled:          scope.Config.Spec.AgentConfig.CISProfile != "",
+			CISEnabled:              scope.Config.Spec.AgentConfig.CISProfile != "",
-			PreRKE2Commands:     scope.Config.Spec.PreRKE2Commands,
+			PreRKE2Commands:         scope.Config.Spec.PreRKE2Commands,
-			PostRKE2Commands:    scope.Config.Spec.PostRKE2Commands,
+			PostRKE2Commands:        scope.Config.Spec.PostRKE2Commands,
-			ConfigFile:          initConfigFile,
+			ConfigFile:              initConfigFile,
-			RKE2Version:         scope.Config.Spec.AgentConfig.Version,
+			RKE2Version:             scope.Config.Spec.AgentConfig.Version,
-			WriteFiles:          files,
+			WriteFiles:              files,
-			NTPServers:          ntpServers,
+			NTPServers:              ntpServers,
-			AdditionalCloudInit: scope.Config.Spec.AgentConfig.AdditionalUserData.Config,
+			AdditionalCloudInit:     scope.Config.Spec.AgentConfig.AdditionalUserData.Config,
+			AdditionalArbitraryData: scope.Config.Spec.AgentConfig.AdditionalUserData.Data,
 		},
 		Certificates: certificates,
 	}
@@ -694,15 +695,16 @@ func (r *RKE2ConfigReconciler) joinWorker(ctx context.Context, scope *Scope) (re
 	}
 
 	wkInput := &cloudinit.BaseUserData{
-		PreRKE2Commands:     scope.Config.Spec.PreRKE2Commands,
+		PreRKE2Commands:         scope.Config.Spec.PreRKE2Commands,
-		AirGapped:           scope.Config.Spec.AgentConfig.AirGapped,
+		AirGapped:               scope.Config.Spec.AgentConfig.AirGapped,
-		CISEnabled:          scope.Config.Spec.AgentConfig.CISProfile != "",
+		CISEnabled:              scope.Config.Spec.AgentConfig.CISProfile != "",
-		PostRKE2Commands:    scope.Config.Spec.PostRKE2Commands,
+		PostRKE2Commands:        scope.Config.Spec.PostRKE2Commands,
-		ConfigFile:          wkJoinConfigFile,
+		ConfigFile:              wkJoinConfigFile,
-		RKE2Version:         scope.Config.Spec.AgentConfig.Version,
+		RKE2Version:             scope.Config.Spec.AgentConfig.Version,
-		WriteFiles:          files,
+		WriteFiles:              files,
-		NTPServers:          ntpServers,
+		NTPServers:              ntpServers,
-		AdditionalCloudInit: scope.Config.Spec.AgentConfig.AdditionalUserData.Config,
+		AdditionalCloudInit:     scope.Config.Spec.AgentConfig.AdditionalUserData.Config,
+		AdditionalArbitraryData: scope.Config.Spec.AgentConfig.AdditionalUserData.Data,
 	}
 
 	var userData []byte

controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_rke2controlplanes.yaml

@@ -48,13 +48,24 @@ spec:
                           documented here: https://kinvolk.io/docs/flatcar-container-linux/latest/provisioning/cl-config/
                           NOTE: All fields of the UserData that are managed by the
                           RKE2Config controller will be ignored, this include "write_files",
-                          "runcmd", "ntp".'
+                          "runcmd", "ntp". Deprecated: Data is reserved for the arbitrary
+                          cloud-init data'
                         type: string
+                      data:
+                        additionalProperties:
+                          type: string
+                        description: Data allows to pass arbitrary set of key/value
+                          pairs consistent with https://cloudinit.readthedocs.io/en/latest/reference/modules.html
+                          to extend existing cloud-init configuration
+                        type: object
                       strict:
                         description: Strict controls if Config should be strictly
                           parsed. If so, warnings are treated as errors.
                         type: boolean
                     type: object
+                    x-kubernetes-validations:
+                    - message: Only config or data could be populated at once
+                      rule: '!has(self.data) || !has(self.config)'
                   airGapped:
                     description: AirGapped is a boolean value to define if the bootstrapping
                       should be air-gapped, basically supposing that online container

pkg/rke2/workload_cluster_test.go

@@ -158,3 +158,35 @@ var _ = Describe("Node metadata propagation", func() {
 		}))
 	})
 })
+
+var _ = Describe("Cloud-init fields validation", func() {
+	var (
+		err error
+		ns  *corev1.Namespace
+	)
+
+	BeforeEach(func() {
+		ns, err = testEnv.CreateNamespace(ctx, "ns")
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		testEnv.Cleanup(ctx, ns)
+	})
+
+	It("should prevent populating config and data fields", func() {
+		Expect(testEnv.Create(ctx, &bootstrapv1.RKE2Config{ObjectMeta: metav1.ObjectMeta{
+			Name:      "config",
+			Namespace: ns.Name,
+		}, Spec: bootstrapv1.RKE2ConfigSpec{
+			AgentConfig: bootstrapv1.RKE2AgentConfig{
+				AdditionalUserData: bootstrapv1.AdditionalUserData{
+					Config: "some",
+					Data: map[string]string{
+						"no": "way",
+					},
+				},
+			},
+		}})).ToNot(Succeed())
+	})
+})