Add timeZone to ScheduledSparkApplication (#2471)

* Add timeZone to ScheduledSparkApplication

Signed-off-by: Jacob Salway <jacob.salway@gmail.com>

* Update api/v1beta2/scheduledsparkapplication_types.go

Co-authored-by: Yi Chen <github@chenyicn.net>
Signed-off-by: Jacob Salway <jacob.salway@gmail.com>

---------

Signed-off-by: Jacob Salway <jacob.salway@gmail.com>
Co-authored-by: Yi Chen <github@chenyicn.net>

api/v1beta2/scheduledsparkapplication_types.go

@@ -34,6 +34,12 @@ type ScheduledSparkApplicationSpec struct {
 
 	// Schedule is a cron schedule on which the application should run.
 	Schedule string `json:"schedule"`
+	// TimeZone is the time zone in which the cron schedule will be interpreted in.
+	// This value is passed to time.LoadLocation, so it must be either "Local", "UTC",
+	// or a valid IANA location name e.g. "America/New_York".
+	// +optional
+	// Defaults to "Local".
+	TimeZone string `json:"timeZone,omitempty"`
 	// Template is a template from which SparkApplication instances can be created.
 	Template SparkApplicationSpec `json:"template"`
 	// Suspend is a flag telling the controller to suspend subsequent runs of the application if set to true.
@@ -80,6 +86,7 @@ type ScheduledSparkApplicationStatus struct {
 // +kubebuilder:resource:scope=Namespaced,shortName=scheduledsparkapp,singular=scheduledsparkapplication
 // +kubebuilder:subresource:status
 // +kubebuilder:printcolumn:JSONPath=.spec.schedule,name=Schedule,type=string
+// +kubebuilder:printcolumn:JSONPath=.spec.timeZone,name=TimeZone,type=string
 // +kubebuilder:printcolumn:JSONPath=.spec.suspend,name=Suspend,type=string
 // +kubebuilder:printcolumn:JSONPath=.status.lastRun,name=Last Run,type=date
 // +kubebuilder:printcolumn:JSONPath=.status.lastRunName,name=Last Run Name,type=string

charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml

@@ -21,6 +21,9 @@ spec:
     - jsonPath: .spec.schedule
       name: Schedule
       type: string
+    - jsonPath: .spec.timeZone
+      name: TimeZone
+      type: string
     - jsonPath: .spec.suspend
       name: Suspend
       type: string
@@ -12384,6 +12387,13 @@ spec:
                 - sparkVersion
                 - type
                 type: object
+              timeZone:
+                description: |-
+                  TimeZone is the time zone in which the cron schedule will be interpreted in.
+                  This value is passed to time.LoadLocation, so it must be either "Local", "UTC",
+                  or a valid IANA location name e.g. "America/New_York".
+                  Defaults to "Local".
+                type: string
             required:
             - schedule
             - template

config/crd/bases/sparkoperator.k8s.io_scheduledsparkapplications.yaml

@@ -21,6 +21,9 @@ spec:
     - jsonPath: .spec.schedule
       name: Schedule
       type: string
+    - jsonPath: .spec.timeZone
+      name: TimeZone
+      type: string
     - jsonPath: .spec.suspend
       name: Suspend
       type: string
@@ -12384,6 +12387,13 @@ spec:
                 - sparkVersion
                 - type
                 type: object
+              timeZone:
+                description: |-
+                  TimeZone is the time zone in which the cron schedule will be interpreted in.
+                  This value is passed to time.LoadLocation, so it must be either "Local", "UTC",
+                  or a valid IANA location name e.g. "America/New_York".
+                  Defaults to "Local".
+                type: string
             required:
             - schedule
             - template

docs/api-docs.md

@@ -1412,6 +1412,21 @@ string
 </tr>
 <tr>
 <td>
+<code>timeZone</code><br/>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>TimeZone is the time zone in which the cron schedule will be interpreted in.
+This value is passed to time.LoadLocation, so it must be either &ldquo;Local&rdquo;, &ldquo;UTC&rdquo;,
+or a valid IANA location name e.g. &ldquo;America/New_York&rdquo;.
+Defaults to &ldquo;Local&rdquo;.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>template</code><br/>
 <em>
 <a href="#sparkoperator.k8s.io/v1beta2.SparkApplicationSpec">
@@ -1521,6 +1536,21 @@ string
 </tr>
 <tr>
 <td>
+<code>timeZone</code><br/>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>TimeZone is the time zone in which the cron schedule will be interpreted in.
+This value is passed to time.LoadLocation, so it must be either &ldquo;Local&rdquo;, &ldquo;UTC&rdquo;,
+or a valid IANA location name e.g. &ldquo;America/New_York&rdquo;.
+Defaults to &ldquo;Local&rdquo;.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>template</code><br/>
 <em>
 <a href="#sparkoperator.k8s.io/v1beta2.SparkApplicationSpec">

internal/controller/scheduledsparkapplication/controller.go

@@ -21,8 +21,11 @@ import (
 	"fmt"
 	"reflect"
 	"sort"
+	"strings"
 	"time"
 
+	_ "time/tzdata"
+
 	"github.com/robfig/cron/v3"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -107,7 +110,30 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 		return ctrl.Result{}, nil
 	}
 
-	schedule, parseErr := cron.ParseStandard(scheduledApp.Spec.Schedule)
+	timezone := scheduledApp.Spec.TimeZone
+	if timezone == "" {
+		timezone = "Local"
+	} else {
+		// Explicitly validate the timezone for a better user experience, but only if it's explicitly specified
+		_, err = time.LoadLocation(timezone)
+		if err != nil {
+			logger.Error(err, "Failed to load timezone location", "name", scheduledApp.Name, "namespace", scheduledApp.Namespace, "timezone", timezone)
+			scheduledApp.Status.ScheduleState = v1beta2.ScheduleStateFailedValidation
+			scheduledApp.Status.Reason = fmt.Sprintf("Invalid timezone: %v", err)
+			if updateErr := r.updateScheduledSparkApplicationStatus(ctx, scheduledApp); updateErr != nil {
+				return ctrl.Result{Requeue: true}, updateErr
+			}
+			return ctrl.Result{}, nil
+		}
+	}
+
+	// Ensure backwards compatibility if the schedule is relying on internal functionality of robfig/cron
+	cronSchedule := scheduledApp.Spec.Schedule
+	if !strings.HasPrefix(cronSchedule, "CRON_TZ=") && !strings.HasPrefix(cronSchedule, "TZ=") {
+		cronSchedule = fmt.Sprintf("CRON_TZ=%s %s", timezone, cronSchedule)
+	}
+
+	schedule, parseErr := cron.ParseStandard(cronSchedule)
 	if parseErr != nil {
 		logger.Error(err, "Failed to parse schedule of ScheduledSparkApplication", "name", scheduledApp.Name, "namespace", scheduledApp.Namespace, "schedule", scheduledApp.Spec.Schedule)
 		scheduledApp.Status.ScheduleState = v1beta2.ScheduleStateFailedValidation