name: model_evaluation
description: |
    TO BE REMOVED.
    Compute evaluation metrics on a trained model's batch prediction results.
    Creates a dataflow job with Apache Beam and TFMA to compute evaluation metrics.

    Args:
        project (str):
            Project to run evaluation container.
        location (Optional[str]):
            Location for running the evaluation.
            If not set, defaulted to `us-central1`.
        root_dir (str):
            The GCS directory for keeping staging files.
            A random subdirectory will be created under the directory to keep job info for resuming
            the job in case of failure.
        problem_type (str):
            The problem type being addressed by this evaluation run.
            `classification` and `regression` are the currently supported problem types.
        predictions_format (Optional[str]):
            The file format for the batch prediction results. `jsonl` is currently the only allowed
            format currently.
            If not set, defaulted to `jsonl`.
        batch_prediction_job (google.VertexBatchPredictionJob):
            The VertexBatchPredictionJob with prediction or explanation results for this evaluation
            run.
            For prediction results, the files should be in format "prediction.results-*".
            For explanation results, the files should be in format "explanation.results-*".
        ground_truth_format(Optional[str]):
            Unstructured data classification. The file format for the ground truth files.
            `jsonl` is currently the only allowed format.
            If not set, defaulted to `jsonl`.
        ground_truth_gcs_source(Optional[Sequence[str]]):
            Unstructured data classification.
            The GCS uris representing where the ground truth is located.
            Used to provide ground truth for each prediction instance when they are not part of the batch prediction jobs prediction instance.
        key_columns(Optional[Sequence[str]]):
            Unstructured data classification.
            The list of fields in the ground truth gcs source to format the joining key.
            Used to merge prediction instances with ground truth data.
        classification_type (Optional[str]):
            Required only for a `classification` problem_type. The type of classification problem.
            Defined as `multiclass` or `multilabel`.
            If not set, defaulted to `multiclass` internally.
        class_names (Optional[Sequence[str]]):
            The list of class names for the ground_truth_column, in the same order they appear in
            the batch predictions jobs predictions output file.
            For instance, if the ground_truth_column could be either `1` or `0`, and the batch
            prediction jobs predictions output contains ["1", "0"] for the prediction_label_column,
            then the class_names input will be ["1", "0"].
            If not set, defaulted to the classes found in the prediction_label_column in the batch prediction jobs predictions file.
        ground_truth_column (str):
            The column name of the feature containing ground truth.
            Formatted to be able to find nested columns, delimeted by `.`.
        prediction_score_column (Optional[str]):
            The column name of the field containing batch prediction scores.
            Formatted to be able to find nested columns, delimeted by `.`.
            If not set, defaulted to `prediction.scores` for a `classification` problem_type,
            `prediction.value` for a `regression` problem_type.
        prediction_label_column (Optional[str]):
            Optional. The column name of the field containing classes the model is scoring.
            Formatted to be able to find nested columns, delimeted by `.`.
            If not set, defaulted to `prediction.classes` for classification.
        prediction_id_column (Optional[str]):
            Optional. The column name of the field containing ids for classes the model is scoring.
            Formatted to be able to find nested columns, delimeted by `.`.
        example_weight_column (Optional[str]):
            Optional. The column name of the field containing example weights.
            Formatted to be able to find nested columns, delimeted by `.`.
        positive_classes (Optional[Sequence[str]]):
            Optional for a `classification` problem_type.
            The list of class names to create binary classification metrics based on one-vs-rest for
            Each value of positive_classes provided.
        generate_feature_attribution (Optional[bool]):
            Optional. If set to True, then the explanations generated by the
            VertexBatchPredictionJob will be used to generate feature attributions. This will only
            pass if the input VertexBatchPredictionJob generated explanations.
            If not set, defaulted to `False`.
        dataflow_service_account (Optional[str]):
            Optional. Service account to run the dataflow job.
            If not set, dataflow will use the default woker service account.

            For more details, see https://cloud.google.com/dataflow/docs/concepts/security-and-permissions#default_worker_service_account
        dataflow_disk_size (Optional[int]):
            Optional. The disk size (in GB) of the machine executing the evaluation run.
            If not set, defaulted to `50`.
        dataflow_machine_type (Optional[str]):
            Optional. The machine type executing the evaluation run.
            If not set, defaulted to `n1-standard-4`.
        dataflow_workers_num (Optional[int]):
            Optional. The number of workers executing the evaluation run.
            If not set, defaulted to `10`.
        dataflow_max_workers_num (Optional[int]):
            Optional. The max number of workers executing the evaluation run.
            If not set, defaulted to `25`.
        dataflow_subnetwork (Optional[str]):
            Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be
            used. More details:
            https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications
        dataflow_use_public_ips (Optional[bool]):
            Specifies whether Dataflow workers use public IP addresses.
        encryption_spec_key_name (Optional[str]):
            Customer-managed encryption key.
    Returns:
        evaluation_metrics (system.Metrics):
            System metrics artifact representing the evaluation metrics in GCS.
            WIP to update to a google.VertexMetrics type with additional functionality.
inputs:
- {name: project, type: String}
- {name: location, type: String, default: "us-central1"}
- {name: root_dir, type: String}
- {name: problem_type, type: String}
- {name: predictions_format, type: String, default: 'jsonl'}
- {name: batch_prediction_job, type: google.VertexBatchPredictionJob}
- {name: ground_truth_format, type: String, default: 'jsonl'}
- {name: ground_truth_gcs_source, type: JsonArray, default: '{}'}
- {name: key_columns, type: JsonArray, default: '{}'}
- {name: classification_type, type: String, default: ''}
- {name: class_names, type: JsonArray, default: '{}'}
- {name: ground_truth_column, type: String}
- {name: prediction_score_column, type: String, default: ''}
- {name: prediction_label_column, type: String, default: ''}
- {name: prediction_id_column, type: String, default: ''}
- {name: example_weight_column, type: String, default: ''}
- {name: positive_classes, type: JsonArray, default: '{}'}
- {name: generate_feature_attribution, type: Boolean, default: False}
- {name: dataflow_service_account, type: String, default: ''}
- {name: dataflow_disk_size, type: Integer, default: 50}
- {name: dataflow_machine_type, type: String, default: 'n1-standard-4'}
- {name: dataflow_workers_num, type: Integer, default: 1}
- {name: dataflow_max_workers_num, type: Integer, default: 5}
- {name: dataflow_subnetwork, type: String, default: ""}
- {name: dataflow_use_public_ips, type: Boolean, default: "true"}
- {name: encryption_spec_key_name, type: String, default: ""}
outputs:
- {name: evaluation_metrics, type: Metrics}
- {name: gcp_resources, type: String}
implementation:
  container:
    image: gcr.io/ml-pipeline/model-evaluation:v0.4
    command:
    - python
    - /main.py
    args:
    - --setup_file
    - /setup.py
    - --json_mode
    - 'true'
    - --project_id
    - {inputValue: project}
    - --location
    - {inputValue: location}
    - --problem_type
    - {inputValue: problem_type}
    - --batch_prediction_format
    - {inputValue: predictions_format}
    - --batch_prediction_gcs_source
    - "{{$.inputs.artifacts['batch_prediction_job'].metadata['gcsOutputDirectory']}}"
    - --ground_truth_format
    - {inputValue: ground_truth_format}
    - --ground_truth_gcs_source
    - {inputValue: ground_truth_gcs_source}
    - --key_prefix_in_prediction_dataset
    - 'instance'
    - --key_columns
    - {inputValue: key_columns}
    - --root_dir
    - "{{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}"
    - --classification_type
    - {inputValue: classification_type}
    - --class_names
    - {inputValue: class_names}
    - --ground_truth_column
    - "instance.{{$.inputs.parameters['ground_truth_column']}}"
    - --prediction_score_column
    - {inputValue: prediction_score_column}
    - --prediction_label_column
    - {inputValue: prediction_label_column}
    - --prediction_id_column
    - {inputValue: prediction_id_column}
    - --example_weight_column
    - {inputValue: example_weight_column}
    - --positive_classes
    - {inputValue: positive_classes}
    - --generate_feature_attribution
    - {inputValue: generate_feature_attribution}
    - --dataflow_job_prefix
    - 'evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}'
    - --dataflow_service_account
    - {inputValue: dataflow_service_account}
    - --dataflow_disk_size
    - {inputValue: dataflow_disk_size}
    - --dataflow_machine_type
    - {inputValue: dataflow_machine_type}
    - --dataflow_workers_num
    - {inputValue: dataflow_workers_num}
    - --dataflow_max_workers_num
    - {inputValue: dataflow_max_workers_num}
    - --dataflow_subnetwork
    - {inputValue: dataflow_subnetwork}
    - --dataflow_use_public_ips
    - {inputValue: dataflow_use_public_ips}
    - --kms_key_name
    - {inputValue: encryption_spec_key_name}
    - --output_metrics_gcs_path
    - {outputUri: evaluation_metrics}
    - --gcp_resources
    - {outputPath: gcp_resources}
    - --executor_input
    - "{{$}}"
