From b575f307adb9ffc4ee26d19b8054af602e37a907 Mon Sep 17 00:00:00 2001 From: Yinan Li Date: Sun, 4 Nov 2018 09:39:39 -0800 Subject: [PATCH] Upgraded to use the Spark 2.4.0 image --- Dockerfile | 2 +- docs/developer-guide.md | 2 +- docs/quick-start-guide.md | 2 +- docs/user-guide.md | 6 +++--- examples/spark-pi-prometheus.yaml | 8 +++---- examples/spark-pi-schedule.yaml | 8 +++---- examples/spark-pi.yaml | 8 +++---- examples/spark-pyfiles.yaml | 2 +- manifest/spark-operator-with-metrics.yaml | 8 +++---- manifest/spark-operator-with-webhook.yaml | 16 +++++++------- manifest/spark-operator.yaml | 8 +++---- pkg/config/toleration_test.go | 26 +++++++++++++++++------ spark-docker/Dockerfile | 2 +- test/e2e/README.md | 2 +- 14 files changed, 56 insertions(+), 44 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8ef68e99..3cf20118 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # -ARG SPARK_IMAGE=gcr.io/spark-operator/spark:v2.3.1 +ARG SPARK_IMAGE=gcr.io/spark-operator/spark:v2.4.0 FROM golang:1.10.2-alpine as builder ARG DEP_VERSION="0.4.1" diff --git a/docs/developer-guide.md b/docs/developer-guide.md index 8d979c0a..1cd1b7bd 100644 --- a/docs/developer-guide.md +++ b/docs/developer-guide.md @@ -10,7 +10,7 @@ The easiest way to build without worrying about dependencies is to just build th $ docker build -t . ``` -The operator image is built upon a base Spark image that defaults to `gcr.io/spark-operator/spark:v2.3.1`. If you want to use your own Spark image (e.g., an image with a different version of Spark or some custom dependencies), specify the argument `SPARK_IMAGE` as the following example shows: +The operator image is built upon a base Spark image that defaults to `gcr.io/spark-operator/spark:v2.4.0`. If you want to use your own Spark image (e.g., an image with a different version of Spark or some custom dependencies), specify the argument `SPARK_IMAGE` as the following example shows: ```bash $ docker build --build-arg SPARK_IMAGE= -t . diff --git a/docs/quick-start-guide.md b/docs/quick-start-guide.md index 2dc269f0..98c346ec 100644 --- a/docs/quick-start-guide.md +++ b/docs/quick-start-guide.md @@ -138,7 +138,7 @@ spec: labels: version: 2.3.0 memory: 512m - image: gcr.io/ynli-k8s/spark:v2.3.0 + image: gcr.io/ynli-k8s/spark:v2.4.0 mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.11-2.3.0.jar mainClass: org.apache.spark.examples.SparkPi mode: cluster diff --git a/docs/user-guide.md b/docs/user-guide.md index d051013f..de5cffd4 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -68,7 +68,7 @@ metadata: spec: type: Scala mode: cluster - image: gcr.io/spark/spark:v2.3.0 + image: gcr.io/spark/spark:v2.4.0 mainClass: org.apache.spark.examples.SparkPi mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.11-2.3.0.jar ``` @@ -420,7 +420,7 @@ and currently requires building a custom Docker image from the Spark master bran The operator supports using the Spark metric system to expose metrics to a variety of sinks. Particularly, it is able to automatically configure the metric system to expose metrics to [Prometheus](https://prometheus.io/). Specifically, the field `.spec.monitoring` specifies how application monitoring is handled and particularly how metrics are to be reported. The metric system is configured through the configuration file `metrics.properties`, which gets its content from the field `.spec.monitoring.metricsProperties`. The content of [metrics.properties](../spark-docker/conf/metrics.properties) will be used by default if `.spec.monitoring.metricsProperties` is not specified. You can choose to enable or disable reporting driver and executor metrics using the fields `.spec.monitoring.exposeDriverMetrics` and `.spec.monitoring.exposeExecutorMetrics`, respectively. -Further, the field `.spec.monitoring.prometheus` specifies how metrics are exposed to Prometheus using the [Prometheus JMX exporter](https://github.com/prometheus/jmx_exporter). When `.spec.monitoring.prometheus` is specified, the operator automatically configures the JMX exporter to run as a Java agent. The only required field of `.spec.monitoring.prometheus` is `jmxExporterJar`, which specified the path to the Prometheus JMX exporter Java agent jar in the container. If you use the image `gcr.io/spark-operator/spark:v2.3.1-gcs-prometheus`, the jar is located at `/prometheus/jmx_prometheus_javaagent-0.3.1.jar`. The field `.spec.monitoring.prometheus.port` specifies the port the JMX exporter Java agent binds to and defaults to `8090` if not specified. The field `.spec.monitoring.prometheus.configuration` specifies the content of the configuration to be used with the JMX exporter. The content of [prometheus.yaml](../spark-docker/conf/prometheus.yaml) will be used by default if `.spec.monitoring.prometheus.configuration` is not specified. +Further, the field `.spec.monitoring.prometheus` specifies how metrics are exposed to Prometheus using the [Prometheus JMX exporter](https://github.com/prometheus/jmx_exporter). When `.spec.monitoring.prometheus` is specified, the operator automatically configures the JMX exporter to run as a Java agent. The only required field of `.spec.monitoring.prometheus` is `jmxExporterJar`, which specified the path to the Prometheus JMX exporter Java agent jar in the container. If you use the image `gcr.io/spark-operator/spark:v2.4.0-gcs-prometheus`, the jar is located at `/prometheus/jmx_prometheus_javaagent-0.3.1.jar`. The field `.spec.monitoring.prometheus.port` specifies the port the JMX exporter Java agent binds to and defaults to `8090` if not specified. The field `.spec.monitoring.prometheus.configuration` specifies the content of the configuration to be used with the JMX exporter. The content of [prometheus.yaml](../spark-docker/conf/prometheus.yaml) will be used by default if `.spec.monitoring.prometheus.configuration` is not specified. Below is an example that shows how to configure the metric system to expose metrics to Prometheus using the Prometheus JMX exporter. Note that the JMX exporter Java agent jar is listed as a dependency and will be downloaded to where `.spec.dep.jarsDownloadDir` points to in Spark 2.3.x, which is `/var/spark-data/spark-jars` by default. Things will be different in Spark 2.4 as dependencies will be downloaded to the local working directory instead in Spark 2.4. A complete example can be found in [examples/spark-pi-prometheus.yaml](../examples/spark-pi-prometheus.yaml). @@ -512,7 +512,7 @@ spec: template: type: Scala mode: cluster - image: gcr.io/spark/spark:v2.3.0 + image: gcr.io/spark/spark:v2.4.0 mainClass: org.apache.spark.examples.SparkPi mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.11-2.3.0.jar driver: diff --git a/examples/spark-pi-prometheus.yaml b/examples/spark-pi-prometheus.yaml index d84a3c35..3eda0ab5 100644 --- a/examples/spark-pi-prometheus.yaml +++ b/examples/spark-pi-prometheus.yaml @@ -22,10 +22,10 @@ metadata: spec: type: Scala mode: cluster - image: "gcr.io/spark-operator/spark:v2.3.1-gcs-prometheus" + image: "gcr.io/spark-operator/spark:v2.4.0-gcs-prometheus" imagePullPolicy: Always mainClass: org.apache.spark.examples.SparkPi - mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.11-2.3.1.jar" + mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar" deps: jars: - http://central.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.3.1/jmx_prometheus_javaagent-0.3.1.jar @@ -34,14 +34,14 @@ spec: coreLimit: "200m" memory: "512m" labels: - version: 2.3.1 + version: 2.4.0 serviceAccount: spark executor: cores: 1 instances: 1 memory: "512m" labels: - version: 2.3.1 + version: 2.4.0 monitoring: exposeDriverMetrics: true exposeExecutorMetrics: true diff --git a/examples/spark-pi-schedule.yaml b/examples/spark-pi-schedule.yaml index a9562948..9cb636e4 100644 --- a/examples/spark-pi-schedule.yaml +++ b/examples/spark-pi-schedule.yaml @@ -25,21 +25,21 @@ spec: template: type: Scala mode: cluster - image: "gcr.io/spark-operator/spark:v2.3.1" + image: "gcr.io/spark-operator/spark:v2.4.0" imagePullPolicy: Always mainClass: org.apache.spark.examples.SparkPi - mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.11-2.3.1.jar" + mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar" driver: cores: 0.1 coreLimit: "200m" memory: "512m" labels: - version: 2.3.1 + version: 2.4.0 serviceAccount: spark executor: cores: 1 instances: 1 memory: "512m" labels: - version: 2.3.1 + version: 2.4.0 restartPolicy: Never diff --git a/examples/spark-pi.yaml b/examples/spark-pi.yaml index bcec2356..01d6e101 100644 --- a/examples/spark-pi.yaml +++ b/examples/spark-pi.yaml @@ -22,10 +22,10 @@ metadata: spec: type: Scala mode: cluster - image: "gcr.io/spark-operator/spark:v2.3.1" + image: "gcr.io/spark-operator/spark:v2.4.0" imagePullPolicy: Always mainClass: org.apache.spark.examples.SparkPi - mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.11-2.3.1.jar" + mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar" volumes: - name: "test-volume" hostPath: @@ -36,7 +36,7 @@ spec: coreLimit: "200m" memory: "512m" labels: - version: 2.3.1 + version: 2.4.0 serviceAccount: spark volumeMounts: - name: "test-volume" @@ -46,7 +46,7 @@ spec: instances: 1 memory: "512m" labels: - version: 2.3.1 + version: 2.4.0 volumeMounts: - name: "test-volume" mountPath: "/tmp" diff --git a/examples/spark-pyfiles.yaml b/examples/spark-pyfiles.yaml index d65bd562..8a7a80eb 100644 --- a/examples/spark-pyfiles.yaml +++ b/examples/spark-pyfiles.yaml @@ -25,7 +25,7 @@ spec: type: Python pythonVersion: "2" mode: cluster - image: "gcr.io/spark-operator/spark-py:v2.4.0-SNAPSHOT" + image: "gcr.io/spark-operator/spark-py:v2.4.0" imagePullPolicy: Always mainApplicationFile: local:///opt/spark/examples/src/main/python/pyfiles.py arguments: diff --git a/manifest/spark-operator-with-metrics.yaml b/manifest/spark-operator-with-metrics.yaml index 2cf0b932..7d64eef2 100644 --- a/manifest/spark-operator-with-metrics.yaml +++ b/manifest/spark-operator-with-metrics.yaml @@ -21,13 +21,13 @@ metadata: namespace: spark-operator labels: app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.3.1-v1alpha1 + app.kubernetes.io/version: v2.4.0-v1alpha1 spec: replicas: 1 selector: matchLabels: app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.3.1-v1alpha1 + app.kubernetes.io/version: v2.4.0-v1alpha1 strategy: type: Recreate template: @@ -38,14 +38,14 @@ spec: prometheus.io/path: "/metrics" labels: app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.3.1-v1alpha1 + app.kubernetes.io/version: v2.4.0-v1alpha1 initializers: pending: [] spec: serviceAccountName: sparkoperator containers: - name: sparkoperator - image: gcr.io/spark-operator/spark-operator:v2.3.1-v1alpha1-latest + image: gcr.io/spark-operator/spark-operator:v2.4.0-v1alpha1-latest imagePullPolicy: Always command: ["/usr/bin/spark-operator"] ports: diff --git a/manifest/spark-operator-with-webhook.yaml b/manifest/spark-operator-with-webhook.yaml index 4576a71a..ab6915e5 100644 --- a/manifest/spark-operator-with-webhook.yaml +++ b/manifest/spark-operator-with-webhook.yaml @@ -21,20 +21,20 @@ metadata: namespace: spark-operator labels: app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.3.1-v1alpha1 + app.kubernetes.io/version: v2.4.0-v1alpha1 spec: replicas: 1 selector: matchLabels: app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.3.1-v1alpha1 + app.kubernetes.io/version: v2.4.0-v1alpha1 strategy: type: Recreate template: metadata: labels: app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.3.1-v1alpha1 + app.kubernetes.io/version: v2.4.0-v1alpha1 initializers: pending: [] spec: @@ -45,7 +45,7 @@ spec: secretName: spark-webhook-certs containers: - name: sparkoperator - image: gcr.io/spark-operator/spark-operator:v2.3.1-v1alpha1-latest + image: gcr.io/spark-operator/spark-operator:v2.4.0-v1alpha1-latest imagePullPolicy: Always volumeMounts: - name: webhook-certs @@ -64,20 +64,20 @@ metadata: namespace: spark-operator labels: app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.3.1-v1alpha1 + app.kubernetes.io/version: v2.4.0-v1alpha1 spec: backoffLimit: 3 template: metadata: labels: app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.3.1-v1alpha1 + app.kubernetes.io/version: v2.4.0-v1alpha1 spec: serviceAccountName: sparkoperator restartPolicy: Never containers: - name: main - image: gcr.io/spark-operator/spark-operator:v2.3.1-v1alpha1-latest + image: gcr.io/spark-operator/spark-operator:v2.4.0-v1alpha1-latest imagePullPolicy: Always command: ["/usr/bin/gencerts.sh", "-p"] --- @@ -93,4 +93,4 @@ spec: name: webhook selector: app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.3.1-v1alpha1 + app.kubernetes.io/version: v2.4.0-v1alpha1 diff --git a/manifest/spark-operator.yaml b/manifest/spark-operator.yaml index 559b9533..88390c6b 100644 --- a/manifest/spark-operator.yaml +++ b/manifest/spark-operator.yaml @@ -21,27 +21,27 @@ metadata: namespace: spark-operator labels: app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.3.1-v1alpha1 + app.kubernetes.io/version: v2.4.0-v1alpha1 spec: replicas: 1 selector: matchLabels: app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.3.1-v1alpha1 + app.kubernetes.io/version: v2.4.0-v1alpha1 strategy: type: Recreate template: metadata: labels: app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.3.1-v1alpha1 + app.kubernetes.io/version: v2.4.0-v1alpha1 initializers: pending: [] spec: serviceAccountName: sparkoperator containers: - name: sparkoperator - image: gcr.io/spark-operator/spark-operator:v2.3.1-v1alpha1-latest + image: gcr.io/spark-operator/spark-operator:v2.4.0-v1alpha1-latest imagePullPolicy: Always command: ["/usr/bin/spark-operator"] args: diff --git a/pkg/config/toleration_test.go b/pkg/config/toleration_test.go index be79a17a..e5516214 100644 --- a/pkg/config/toleration_test.go +++ b/pkg/config/toleration_test.go @@ -18,6 +18,7 @@ package config import ( "fmt" + "sort" "strings" "testing" @@ -57,13 +58,22 @@ func TestFindTolerations(t *testing.T) { t.Fatal(err) } assert.Equal(t, 2, len(tolerations)) - assert.Equal(t, "Key1", tolerations[0].Key) - assert.Equal(t, "Value", tolerations[0].Value) - assert.Equal(t, "Equal", string(tolerations[0].Operator)) - assert.Equal(t, "NoEffect", string(tolerations[0].Effect)) - assert.Equal(t, "Key2", tolerations[1].Key) - assert.Equal(t, "Exists", string(tolerations[1].Operator)) - assert.Equal(t, "NoSchedule", string(tolerations[1].Effect)) + if tolerations[0].Key == "Key1" { + assert.Equal(t, "Value", tolerations[0].Value) + assert.Equal(t, "Equal", string(tolerations[0].Operator)) + assert.Equal(t, "NoEffect", string(tolerations[0].Effect)) + assert.Equal(t, "Key2", tolerations[1].Key) + assert.Equal(t, "Exists", string(tolerations[1].Operator)) + assert.Equal(t, "NoSchedule", string(tolerations[1].Effect)) + } else { + assert.Equal(t, "Key1", tolerations[1].Key) + assert.Equal(t, "Value", tolerations[1].Value) + assert.Equal(t, "Equal", string(tolerations[1].Operator)) + assert.Equal(t, "NoEffect", string(tolerations[1].Effect)) + assert.Equal(t, "Key2", tolerations[0].Key) + assert.Equal(t, "Exists", string(tolerations[0].Operator)) + assert.Equal(t, "NoSchedule", string(tolerations[0].Effect)) + } } func TestGetTolerationAnnotations(t *testing.T) { @@ -137,6 +147,7 @@ func TestGetDriverTolerationConfOptions(t *testing.T) { } assert.Equal(t, 2, len(options)) + sort.Strings(options) assert.True(t, strings.HasPrefix(options[0], fmt.Sprintf("%s%s%s=", SparkDriverAnnotationKeyPrefix, TolerationsAnnotationPrefix, "toleration1"))) assert.True(t, strings.HasPrefix(options[1], fmt.Sprintf("%s%s%s=", SparkDriverAnnotationKeyPrefix, @@ -172,6 +183,7 @@ func TestGetExecutorTolerationConfOptions(t *testing.T) { } assert.Equal(t, 2, len(options)) + sort.Strings(options) assert.True(t, strings.HasPrefix(options[0], fmt.Sprintf("%s%s%s=", SparkExecutorAnnotationKeyPrefix, TolerationsAnnotationPrefix, "toleration1"))) assert.True(t, strings.HasPrefix(options[1], fmt.Sprintf("%s%s%s=", SparkExecutorAnnotationKeyPrefix, diff --git a/spark-docker/Dockerfile b/spark-docker/Dockerfile index bc0be80f..74c09273 100644 --- a/spark-docker/Dockerfile +++ b/spark-docker/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # -ARG SPARK_IMAGE=gcr.io/spark-operator/spark:v2.3.1 +ARG SPARK_IMAGE=gcr.io/spark-operator/spark:v2.4.0 FROM ${SPARK_IMAGE} # Setup dependencies for Google Cloud Storage access. diff --git a/test/e2e/README.md b/test/e2e/README.md index b0f1642d..cea3e710 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -12,5 +12,5 @@ Prerequisites: e2e tests are written as Go test. All go test techniques apply (e.g. picking what to run, timeout length). Let's say I want to run all tests in "test/e2e/": ```bash -$ go test -v ./test/e2e/ --kubeconfig "$HOME/.kube/config" --operator-image=gcr.io/spark-operator/spark-operator:v2.3.1-v1alpha1-latest +$ go test -v ./test/e2e/ --kubeconfig "$HOME/.kube/config" --operator-image=gcr.io/spark-operator/spark-operator:v2.4.0-v1alpha1-latest ```