# Copyright 2019 The Kubeflow Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: dataproc_submit_sparksql_job
description: >-
  Submits a Cloud Dataproc job for running Apache Spark SQL queries.
metadata:
  labels:
    add-pod-env: 'true'
inputs:
  - name: project_id
    description: >-
      Required. The ID of the Google Cloud Platform project that the cluster
      belongs to.
    type: GCPProjectID
  - name: region
    description: >-
      Required. The Cloud Dataproc region in which to handle the request.
    type: GCPRegion
  - name: cluster_name
    description: 'Required. The cluster to run the job.'
    type: String
  - name: queries
    default: ''
    description: >-
      Required. The queries to execute. You do not need to
      terminate a query with a semicolon. Multiple queries can be specified
      in one string by separating each with a semicolon. 
    type: List
  - name: query_file_uri
    default: ''
    description: >-
      The HCFS URI of the script that contains SQL queries.
    type: GCSPath
  - name: script_variables
    default: ''
    description: >-
      Optional. Mapping of query variable names to 
      values (equivalent to the Spark SQL command: SET name="value";).
    type: Dict
  - name: sparksql_job
    default: ''
    description: >-
      Optional. The full payload of a 
      [SparkSqlJob](https://cloud.google.com/dataproc/docs/reference/rest/v1/SparkSqlJob).
    type: Dict
  - name: job
    default: ''
    description: >-
      Optional. The full payload of a
      [Dataproc job](https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.jobs).
    type: Dict
  - name: wait_interval
    default: '30'
    description: >-
      Optional. The wait seconds between polling the operation.
      Defaults to 30.
    type: Integer
outputs:
  - name: job_id
    description: 'The ID of the created job.'
    type: String
  - name: MLPipeline UI metadata
    type: UI metadata
implementation:
  container:
    image: gcr.io/ml-pipeline/ml-pipeline-gcp:1.7.0-rc.3
    command: ['python', '-u', '-m', 'kfp_component.launcher']
    args: [
      --ui_metadata_path, {outputPath: MLPipeline UI metadata},
      kfp_component.google.dataproc, submit_sparksql_job,
      --project_id, {inputValue: project_id},
      --region, {inputValue: region},
      --cluster_name, {inputValue: cluster_name},
      --queries, {inputValue: queries},
      --query_file_uri, {inputValue: query_file_uri},
      --script_variables, {inputValue: script_variables},
      --sparksql_job, {inputValue: sparksql_job},
      --job, {inputValue: job},
      --wait_interval, {inputValue: wait_interval},
      --job_id_output_path, {outputPath: job_id},
    ]
    env:
      KFP_POD_NAME: "{{pod.name}}"
