name: dataset_preprocessor_error_analysis
description: |
  Preprocesses test/training datasets for Evaluated Annotation and Vision Error Analysis pipelines.

  Args:
      project (str):
          Required. GCP Project ID.
      location (Optional[str]):
          GCP Region.
          If not set, defaulted to `us-central1`.
      output_dir (str):
          Required. The Google Cloud Storage location where the outputs will be saved to.
      test_dataset (Optional[google.VertexDataset])
          Optional. A google.VertexDataset artifact of the test dataset. If `test_dataset_storage_source_uris` is
          also provided, this Vertex Dataset argument will override the GCS source.
      test_dataset_annotation_set_name (Optional[str])
          Optional. A string of the annotation_set name containing the ground truth of the test datset used for evaluation.
      training_dataset (Optional[google.VertexDataset])
          Optional. A google.VertexDataset artifact of the training dataset. If `training_dataset_storage_source_uris` is
          also provided, this Vertex Dataset argument will override the GCS source.
      training_dataset_annotation_set_name (Optional[str])
          Optional. A string of the annotation_set name containing the ground truth of the test datset used for evaluation.
      test_dataset_storage_source_uris (Optional[Sequence[str]]):
          Optional. Google Cloud Storage URI(-s) to unmanaged test datasets.`jsonl` is currently the
          only allowed format.
          If `test_dataset` is also provided, this field will be overriden by the provided Vertex Dataset.
      training_dataset_storage_source_uris (Optional[Sequence[str]]):
          Optional. Google Cloud Storage URI(-s) to unmanaged test datasets.`jsonl` is currently the
          only allowed format.
          If `training_dataset` is also provided, this field will be overriden by the provided Vertex Dataset.
  Returns:
      batch_prediction_storage_source (str)
        Google Cloud Storage URI to preprocessed dataset instances for Vertex Batch Prediction
        component. Target field name is removed.
        For example,
          {"content":"gs://flowers/roses/001.jpg","mimeType":"image/jpeg"}
      model_evaluation_storage_source (str)
        Google Cloud Storage URI to preprocessed dataset instances for Vertex Model Evaluation
        component. Target field name is defaulted to "ground_truth".
        For example,
          {"content":"gs://flowers/roses/001.jpg","mimeType":"image/jpeg", "ground_truth": "roses"}
      test_data_items_storage_source (str)
        Google Cloud Storage URI to preprocessed test dataset instances for Evaluated
        Annotation or Vision Error Analysis pipelines. It is the input source for Evaluated
        Annotation component and Feature Extraction component.
        For example,
          {"dataItemID":"projects/*/locations/*/datasets/*/dataItems/000",
          "imageGcsUri":"gs://flowers/roses/001.jpg",
          "mimeType":  "image/jpeg",
          "data_split":"test",
          "annotation_set_name":"333",
          "ground_truth_annotations":["roses"],
          "annotation_spec_ids": ["123"],
          "annotation_resource_names":
            ["projects/*/locations/*/datasets/*/dataItems/000/annotations/111"]}
      training_data_items_storage_source (str)
        Google Cloud Storage URI to preprocessed training dataset instances for Vision Error
        Analysis pipelines. It is the input source for Feature Extraction component. It uses the
        same output format as test_data_items_storage_source.
inputs:
  - { name: project, type: String }
  - { name: location, type: String, default: "us-central1" }
  - { name: output_dir, type: String }
  - { name: test_dataset, type: google.VertexDataset, optional: True }
  - { name: test_dataset_annotation_set_name, type: String, optional: True }
  - { name: training_dataset, type: google.VertexDataset, optional: True }
  - { name: training_dataset_annotation_set_name, type: String, optional: True }
  - { name: test_dataset_storage_source_uris, type: JsonArray, default: "[]", optional: True }
  - { name: training_dataset_storage_source_uris, type: JsonArray, default: "[]", optional: True }
outputs:
  - { name: gcp_resources, type: String }
  - { name: batch_prediction_storage_source, type: String }
  - { name: model_evaluation_storage_source, type: String }
  - { name: test_data_items_storage_source, type: String }
  - { name: training_data_items_storage_source, type: String }
implementation:
  container:
    image: gcr.io/ml-pipeline/model-evaluation:v0.8.1
    command:
      - python
      - /main.py
    args:
      - --task
      - "dataset_preprocessor"
      - --project_id
      - { inputValue: project }
      - --location
      - { inputValue: location }
      - --output_dir
      - { inputValue: output_dir }
      - if:
          cond: { isPresent: test_dataset }
          then:
          - --test_dataset
          - "{{$.inputs.artifacts['test_dataset'].metadata['resourceName']}}"
          - --test_dataset_annotation_set_name
          - { inputValue: test_dataset_annotation_set_name }
      - if:
          cond: { isPresent: training_dataset }
          then:
          - --training_dataset
          - "{{$.inputs.artifacts['training_dataset'].metadata['resourceName']}}"
          - --training_dataset_annotation_set_name
          - { inputValue: training_dataset_annotation_set_name }
      - --test_dataset_storage_source_uris
      - { inputValue: test_dataset_storage_source_uris }
      - --training_dataset_storage_source_uris
      - { inputValue: training_dataset_storage_source_uris }
      - --batch_prediction_storage_source
      - { outputPath: batch_prediction_storage_source }
      - --model_evaluation_storage_source
      - { outputPath: model_evaluation_storage_source }
      - --test_data_items_storage_source
      - { outputPath: test_data_items_storage_source }
      - --training_data_items_storage_source
      - { outputPath: training_data_items_storage_source }
      - --gcp_resources
      - { outputPath: gcp_resources }
      - --executor_input
      - "{{$}}"
