Upgraded to use the Spark 2.4.0 image

Dockerfile

@@ -14,7 +14,7 @@
 # limitations under the License.
 #
 
-ARG SPARK_IMAGE=gcr.io/spark-operator/spark:v2.3.1
+ARG SPARK_IMAGE=gcr.io/spark-operator/spark:v2.4.0
 
 FROM golang:1.10.2-alpine as builder
 ARG DEP_VERSION="0.4.1"

docs/developer-guide.md

@@ -10,7 +10,7 @@ The easiest way to build without worrying about dependencies is to just build th
 $ docker build -t <image-tag> .
 ```
 
-The operator image is built upon a base Spark image that defaults to `gcr.io/spark-operator/spark:v2.3.1`. If you want to use your own Spark image (e.g., an image with a different version of Spark or some custom dependencies), specify the argument `SPARK_IMAGE` as the following example shows: 
+The operator image is built upon a base Spark image that defaults to `gcr.io/spark-operator/spark:v2.4.0`. If you want to use your own Spark image (e.g., an image with a different version of Spark or some custom dependencies), specify the argument `SPARK_IMAGE` as the following example shows: 
 
 ```bash
 $ docker build --build-arg SPARK_IMAGE=<your Spark image> -t <image-tag> .

docs/quick-start-guide.md

@@ -138,7 +138,7 @@ spec:
     labels:
       version: 2.3.0
     memory: 512m
-  image: gcr.io/ynli-k8s/spark:v2.3.0
+  image: gcr.io/ynli-k8s/spark:v2.4.0
   mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.11-2.3.0.jar
   mainClass: org.apache.spark.examples.SparkPi
   mode: cluster

docs/user-guide.md

@@ -68,7 +68,7 @@ metadata:
 spec:
   type: Scala
   mode: cluster
-  image: gcr.io/spark/spark:v2.3.0
+  image: gcr.io/spark/spark:v2.4.0
   mainClass: org.apache.spark.examples.SparkPi
   mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.11-2.3.0.jar
 ```
@@ -420,7 +420,7 @@ and currently requires building a custom Docker image from the Spark master bran
 
 The operator supports using the Spark metric system to expose metrics to a variety of sinks. Particularly, it is able to automatically configure the metric system to expose metrics to [Prometheus](https://prometheus.io/). Specifically, the field `.spec.monitoring` specifies how application monitoring is handled and particularly how metrics are to be reported. The metric system is configured through the configuration file `metrics.properties`, which gets its content from the field `.spec.monitoring.metricsProperties`. The content of [metrics.properties](../spark-docker/conf/metrics.properties) will be used by default if `.spec.monitoring.metricsProperties` is not specified. You can choose to enable or disable reporting driver and executor metrics using the fields `.spec.monitoring.exposeDriverMetrics` and `.spec.monitoring.exposeExecutorMetrics`, respectively. 
 
-Further, the field `.spec.monitoring.prometheus` specifies how metrics are exposed to Prometheus using the [Prometheus JMX exporter](https://github.com/prometheus/jmx_exporter). When `.spec.monitoring.prometheus` is specified, the operator automatically configures the JMX exporter to run as a Java agent. The only required field of `.spec.monitoring.prometheus` is `jmxExporterJar`, which specified the path to the Prometheus JMX exporter Java agent jar in the container. If you use the image `gcr.io/spark-operator/spark:v2.3.1-gcs-prometheus`, the jar is located at `/prometheus/jmx_prometheus_javaagent-0.3.1.jar`. The field `.spec.monitoring.prometheus.port` specifies the port the JMX exporter Java agent binds to and defaults to `8090` if not specified. The field `.spec.monitoring.prometheus.configuration` specifies the content of the configuration to be used with the JMX exporter. The content of [prometheus.yaml](../spark-docker/conf/prometheus.yaml) will be used by default if `.spec.monitoring.prometheus.configuration` is not specified.    
+Further, the field `.spec.monitoring.prometheus` specifies how metrics are exposed to Prometheus using the [Prometheus JMX exporter](https://github.com/prometheus/jmx_exporter). When `.spec.monitoring.prometheus` is specified, the operator automatically configures the JMX exporter to run as a Java agent. The only required field of `.spec.monitoring.prometheus` is `jmxExporterJar`, which specified the path to the Prometheus JMX exporter Java agent jar in the container. If you use the image `gcr.io/spark-operator/spark:v2.4.0-gcs-prometheus`, the jar is located at `/prometheus/jmx_prometheus_javaagent-0.3.1.jar`. The field `.spec.monitoring.prometheus.port` specifies the port the JMX exporter Java agent binds to and defaults to `8090` if not specified. The field `.spec.monitoring.prometheus.configuration` specifies the content of the configuration to be used with the JMX exporter. The content of [prometheus.yaml](../spark-docker/conf/prometheus.yaml) will be used by default if `.spec.monitoring.prometheus.configuration` is not specified.    
 
 Below is an example that shows how to configure the metric system to expose metrics to Prometheus using the Prometheus JMX exporter. Note that the JMX exporter Java agent jar is listed as a dependency and will be downloaded to where `.spec.dep.jarsDownloadDir` points to in Spark 2.3.x, which is `/var/spark-data/spark-jars` by default. Things will be different in Spark 2.4 as dependencies will be downloaded to the local working directory instead in Spark 2.4. A complete example can be found in [examples/spark-pi-prometheus.yaml](../examples/spark-pi-prometheus.yaml).
 
@@ -512,7 +512,7 @@ spec:
   template:
     type: Scala
     mode: cluster
-    image: gcr.io/spark/spark:v2.3.0
+    image: gcr.io/spark/spark:v2.4.0
     mainClass: org.apache.spark.examples.SparkPi
     mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.11-2.3.0.jar
     driver:

examples/spark-pi-prometheus.yaml

@@ -22,10 +22,10 @@ metadata:
 spec:
   type: Scala
   mode: cluster
-  image: "gcr.io/spark-operator/spark:v2.3.1-gcs-prometheus"
+  image: "gcr.io/spark-operator/spark:v2.4.0-gcs-prometheus"
   imagePullPolicy: Always
   mainClass: org.apache.spark.examples.SparkPi
-  mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.11-2.3.1.jar"
+  mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar"
   deps:
     jars:
     - http://central.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.3.1/jmx_prometheus_javaagent-0.3.1.jar
@@ -34,14 +34,14 @@ spec:
     coreLimit: "200m"
     memory: "512m"
     labels:
-      version: 2.3.1
+      version: 2.4.0
     serviceAccount: spark
   executor:
     cores: 1
     instances: 1
     memory: "512m"
     labels:
-      version: 2.3.1
+      version: 2.4.0
   monitoring:
     exposeDriverMetrics: true
     exposeExecutorMetrics: true

examples/spark-pi-schedule.yaml

@@ -25,21 +25,21 @@ spec:
   template:
     type: Scala
     mode: cluster
-    image: "gcr.io/spark-operator/spark:v2.3.1"
+    image: "gcr.io/spark-operator/spark:v2.4.0"
     imagePullPolicy: Always
     mainClass: org.apache.spark.examples.SparkPi
-    mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.11-2.3.1.jar"
+    mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar"
     driver:
       cores: 0.1
       coreLimit: "200m"
       memory: "512m"
       labels:
-        version: 2.3.1
+        version: 2.4.0
       serviceAccount: spark
     executor:
       cores: 1
       instances: 1
       memory: "512m"
       labels:
-        version: 2.3.1
+        version: 2.4.0
     restartPolicy: Never

examples/spark-pi.yaml

@@ -22,10 +22,10 @@ metadata:
 spec:
   type: Scala
   mode: cluster
-  image: "gcr.io/spark-operator/spark:v2.3.1"
+  image: "gcr.io/spark-operator/spark:v2.4.0"
   imagePullPolicy: Always
   mainClass: org.apache.spark.examples.SparkPi
-  mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.11-2.3.1.jar"
+  mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar"
   volumes:
     - name: "test-volume"
       hostPath:
@@ -36,7 +36,7 @@ spec:
     coreLimit: "200m"
     memory: "512m"
     labels:
-      version: 2.3.1
+      version: 2.4.0
     serviceAccount: spark
     volumeMounts:
       - name: "test-volume"
@@ -46,7 +46,7 @@ spec:
     instances: 1
     memory: "512m"
     labels:
-      version: 2.3.1
+      version: 2.4.0
     volumeMounts:
       - name: "test-volume"
         mountPath: "/tmp"

examples/spark-pyfiles.yaml

@@ -25,7 +25,7 @@ spec:
   type: Python
   pythonVersion: "2"
   mode: cluster
-  image: "gcr.io/spark-operator/spark-py:v2.4.0-SNAPSHOT"
+  image: "gcr.io/spark-operator/spark-py:v2.4.0"
   imagePullPolicy: Always
   mainApplicationFile: local:///opt/spark/examples/src/main/python/pyfiles.py
   arguments:

manifest/spark-operator-with-metrics.yaml

@@ -21,13 +21,13 @@ metadata:
   namespace: spark-operator
   labels:
     app.kubernetes.io/name: sparkoperator
-    app.kubernetes.io/version: v2.3.1-v1alpha1
+    app.kubernetes.io/version: v2.4.0-v1alpha1
 spec:
   replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: sparkoperator
-      app.kubernetes.io/version: v2.3.1-v1alpha1
+      app.kubernetes.io/version: v2.4.0-v1alpha1
   strategy:
     type: Recreate
   template:
@@ -38,14 +38,14 @@ spec:
         prometheus.io/path: "/metrics"
       labels:
         app.kubernetes.io/name: sparkoperator
-        app.kubernetes.io/version: v2.3.1-v1alpha1
+        app.kubernetes.io/version: v2.4.0-v1alpha1
       initializers:
         pending: []
     spec:
       serviceAccountName: sparkoperator
       containers:
       - name: sparkoperator
-        image: gcr.io/spark-operator/spark-operator:v2.3.1-v1alpha1-latest
+        image: gcr.io/spark-operator/spark-operator:v2.4.0-v1alpha1-latest
         imagePullPolicy: Always
         command: ["/usr/bin/spark-operator"]
         ports:

manifest/spark-operator-with-webhook.yaml

@@ -21,20 +21,20 @@ metadata:
   namespace: spark-operator
   labels:
     app.kubernetes.io/name: sparkoperator
-    app.kubernetes.io/version: v2.3.1-v1alpha1
+    app.kubernetes.io/version: v2.4.0-v1alpha1
 spec:
   replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: sparkoperator
-      app.kubernetes.io/version: v2.3.1-v1alpha1
+      app.kubernetes.io/version: v2.4.0-v1alpha1
   strategy:
     type: Recreate
   template:
     metadata:
       labels:
         app.kubernetes.io/name: sparkoperator
-        app.kubernetes.io/version: v2.3.1-v1alpha1
+        app.kubernetes.io/version: v2.4.0-v1alpha1
       initializers:
         pending: []
     spec:
@@ -45,7 +45,7 @@ spec:
           secretName: spark-webhook-certs
       containers:
       - name: sparkoperator
-        image: gcr.io/spark-operator/spark-operator:v2.3.1-v1alpha1-latest
+        image: gcr.io/spark-operator/spark-operator:v2.4.0-v1alpha1-latest
         imagePullPolicy: Always
         volumeMounts:
         - name: webhook-certs
@@ -64,20 +64,20 @@ metadata:
   namespace: spark-operator
   labels:
     app.kubernetes.io/name: sparkoperator
-    app.kubernetes.io/version: v2.3.1-v1alpha1
+    app.kubernetes.io/version: v2.4.0-v1alpha1
 spec:
   backoffLimit: 3
   template:
     metadata:
       labels:
         app.kubernetes.io/name: sparkoperator
-        app.kubernetes.io/version: v2.3.1-v1alpha1
+        app.kubernetes.io/version: v2.4.0-v1alpha1
     spec:
       serviceAccountName: sparkoperator
       restartPolicy: Never
       containers:
       - name: main
-        image: gcr.io/spark-operator/spark-operator:v2.3.1-v1alpha1-latest
+        image: gcr.io/spark-operator/spark-operator:v2.4.0-v1alpha1-latest
         imagePullPolicy: Always
         command: ["/usr/bin/gencerts.sh", "-p"]
 ---
@@ -93,4 +93,4 @@ spec:
       name: webhook
   selector:
     app.kubernetes.io/name: sparkoperator
-    app.kubernetes.io/version: v2.3.1-v1alpha1
+    app.kubernetes.io/version: v2.4.0-v1alpha1

manifest/spark-operator.yaml

@@ -21,27 +21,27 @@ metadata:
   namespace: spark-operator
   labels:
     app.kubernetes.io/name: sparkoperator
-    app.kubernetes.io/version: v2.3.1-v1alpha1
+    app.kubernetes.io/version: v2.4.0-v1alpha1
 spec:
   replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: sparkoperator
-      app.kubernetes.io/version: v2.3.1-v1alpha1
+      app.kubernetes.io/version: v2.4.0-v1alpha1
   strategy:
     type: Recreate
   template:
     metadata:
       labels:
         app.kubernetes.io/name: sparkoperator
-        app.kubernetes.io/version: v2.3.1-v1alpha1
+        app.kubernetes.io/version: v2.4.0-v1alpha1
       initializers:
         pending: []
     spec:
       serviceAccountName: sparkoperator
       containers:
       - name: sparkoperator
-        image: gcr.io/spark-operator/spark-operator:v2.3.1-v1alpha1-latest
+        image: gcr.io/spark-operator/spark-operator:v2.4.0-v1alpha1-latest
         imagePullPolicy: Always
         command: ["/usr/bin/spark-operator"]
         args:

pkg/config/toleration_test.go

@@ -18,6 +18,7 @@ package config
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 	"testing"
 
@@ -57,13 +58,22 @@ func TestFindTolerations(t *testing.T) {
 		t.Fatal(err)
 	}
 	assert.Equal(t, 2, len(tolerations))
-	assert.Equal(t, "Key1", tolerations[0].Key)
+	if tolerations[0].Key == "Key1" {
-	assert.Equal(t, "Value", tolerations[0].Value)
+		assert.Equal(t, "Value", tolerations[0].Value)
-	assert.Equal(t, "Equal", string(tolerations[0].Operator))
+		assert.Equal(t, "Equal", string(tolerations[0].Operator))
-	assert.Equal(t, "NoEffect", string(tolerations[0].Effect))
+		assert.Equal(t, "NoEffect", string(tolerations[0].Effect))
-	assert.Equal(t, "Key2", tolerations[1].Key)
+		assert.Equal(t, "Key2", tolerations[1].Key)
-	assert.Equal(t, "Exists", string(tolerations[1].Operator))
+		assert.Equal(t, "Exists", string(tolerations[1].Operator))
-	assert.Equal(t, "NoSchedule", string(tolerations[1].Effect))
+		assert.Equal(t, "NoSchedule", string(tolerations[1].Effect))
+	} else {
+		assert.Equal(t, "Key1", tolerations[1].Key)
+		assert.Equal(t, "Value", tolerations[1].Value)
+		assert.Equal(t, "Equal", string(tolerations[1].Operator))
+		assert.Equal(t, "NoEffect", string(tolerations[1].Effect))
+		assert.Equal(t, "Key2", tolerations[0].Key)
+		assert.Equal(t, "Exists", string(tolerations[0].Operator))
+		assert.Equal(t, "NoSchedule", string(tolerations[0].Effect))
+	}
 }
 
 func TestGetTolerationAnnotations(t *testing.T) {
@@ -137,6 +147,7 @@ func TestGetDriverTolerationConfOptions(t *testing.T) {
 	}
 
 	assert.Equal(t, 2, len(options))
+	sort.Strings(options)
 	assert.True(t, strings.HasPrefix(options[0], fmt.Sprintf("%s%s%s=", SparkDriverAnnotationKeyPrefix,
 		TolerationsAnnotationPrefix, "toleration1")))
 	assert.True(t, strings.HasPrefix(options[1], fmt.Sprintf("%s%s%s=", SparkDriverAnnotationKeyPrefix,
@@ -172,6 +183,7 @@ func TestGetExecutorTolerationConfOptions(t *testing.T) {
 	}
 
 	assert.Equal(t, 2, len(options))
+	sort.Strings(options)
 	assert.True(t, strings.HasPrefix(options[0], fmt.Sprintf("%s%s%s=", SparkExecutorAnnotationKeyPrefix,
 		TolerationsAnnotationPrefix, "toleration1")))
 	assert.True(t, strings.HasPrefix(options[1], fmt.Sprintf("%s%s%s=", SparkExecutorAnnotationKeyPrefix,

spark-docker/Dockerfile

@@ -14,7 +14,7 @@
 # limitations under the License.
 #
 
-ARG SPARK_IMAGE=gcr.io/spark-operator/spark:v2.3.1
+ARG SPARK_IMAGE=gcr.io/spark-operator/spark:v2.4.0
 FROM ${SPARK_IMAGE}
 
 # Setup dependencies for Google Cloud Storage access.

test/e2e/README.md

@@ -12,5 +12,5 @@ Prerequisites:
 e2e tests are written as Go test. All go test techniques apply (e.g. picking what to run, timeout length). Let's say I want to run all tests in "test/e2e/":
 
 ```bash
-$ go test -v ./test/e2e/ --kubeconfig "$HOME/.kube/config" --operator-image=gcr.io/spark-operator/spark-operator:v2.3.1-v1alpha1-latest
+$ go test -v ./test/e2e/ --kubeconfig "$HOME/.kube/config" --operator-image=gcr.io/spark-operator/spark-operator:v2.4.0-v1alpha1-latest
 ```