name: evaluated_annotation
description: |
  Computes the Evaluated Annotations of a dataset for a model's prediction results. Predicted labels
  are categorized into TruePositive(TP) or FalsePositive(FP) or FalseNegative(FN) based on their
  ground truth annotations.

  Args:
      project (str):
          Required. GCP Project ID.
      location (Optional[str]):
          GCP Region.
          If not set, defaulted to `us-central1`.
      root_dir (str):
          Required. The GCS directory for keeping staging files.
          A random subdirectory will be created under the directory to keep job info for resuming
          the job in case of failure.
      predictions_storage_source (system.Artifact):
          Required. An artifact with its URI pointing toward a GCS directory with prediction files
          to be used for this evaluation. `jsonl` is currently the only allowed format.
          For prediction results, the files should be named "prediction.results-*" or "predictions_*".
      ground_truth_storage_source(str):
          Required. The GCS URI representing where the preprocessed test dataset is located.
          The dataset must contain the ground truth annotations.This field must be the output from
          DatasetPreprocessorErrorAnalysis component. `jsonl` is currently the only allowed format.
      dataflow_service_account (Optional[str]):
          Optional. Service account to run the dataflow job.
          If not set, dataflow will use the default woker service account.

          For more details, see https://cloud.google.com/dataflow/docs/concepts/security-and-permissions#default_worker_service_account
      dataflow_disk_size (Optional[int]):
          Optional. The disk size (in GB) of the machine executing the evaluation run.
          If not set, defaulted to `50`.
      dataflow_machine_type (Optional[str]):
          Optional. The machine type executing the evaluation run.
          If not set, defaulted to `n1-standard-4`.
      dataflow_workers_num (Optional[int]):
          Optional. The number of workers executing the evaluation run.
          If not set, defaulted to `10`.
      dataflow_max_workers_num (Optional[int]):
          Optional. The max number of workers executing the evaluation run.
          If not set, defaulted to `25`.
      dataflow_subnetwork (Optional[str]):
          Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be
          used. More details:
          https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications
      dataflow_use_public_ips (Optional[bool]):
          Specifies whether Dataflow workers use public IP addresses.
      encryption_spec_key_name (Optional[str]):
          Customer-managed encryption key for the Dataflow job. If this is set, then all resources
          created by the Dataflow job will be encrypted with the provided encryption key.
  Returns:
      evaluated_annotation_output_uri(str):
          String representing the GCS URI of the computed evaluated annotations.
inputs:
  - { name: project, type: String }
  - { name: location, type: String, default: "us-central1" }
  - { name: root_dir, type: String }
  - { name: predictions_storage_source, type: Artifact }
  - { name: ground_truth_storage_source, type: String, default: "" }
  - { name: dataflow_service_account, type: String, default: "" }
  - { name: dataflow_disk_size, type: Integer, default: 50 }
  - { name: dataflow_machine_type, type: String, default: "n1-standard-4" }
  - { name: dataflow_workers_num, type: Integer, default: 1 }
  - { name: dataflow_max_workers_num, type: Integer, default: 5 }
  - { name: dataflow_subnetwork, type: String, default: "" }
  - { name: dataflow_use_public_ips, type: Boolean, default: "true" }
  - { name: encryption_spec_key_name, type: String, default: "" }
outputs:
  - { name: evaluated_annotation_output_uri, type: String }
  - { name: gcp_resources, type: String }
implementation:
  container:
    image: gcr.io/ml-pipeline/model-evaluation:v0.9
    command:
      - python
      - /main.py
    args:
      - --task
      - "evaluated_annotation"
      - --display_name
      - "evaluated-annotation-run"
      - --project_id
      - { inputValue: project }
      - --location
      - { inputValue: location }
      - --root_dir
      - "{{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}"
      - --batch_prediction_format
      - "jsonl"
      - if:
          cond: { isPresent: predictions_storage_source }
          then:
            - --batch_prediction_gcs_source
            - "{{$.inputs.artifacts['predictions_storage_source'].uri}}"
      - --ground_truth_format
      - "jsonl"
      - --ground_truth_storage_source
      - { inputValue: ground_truth_storage_source }
      - --dataflow_job_prefix
      - "evaluated-annotation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}"
      - --dataflow_service_account
      - { inputValue: dataflow_service_account }
      - --dataflow_disk_size
      - { inputValue: dataflow_disk_size }
      - --dataflow_machine_type
      - { inputValue: dataflow_machine_type }
      - --dataflow_workers_num
      - { inputValue: dataflow_workers_num }
      - --dataflow_max_workers_num
      - { inputValue: dataflow_max_workers_num }
      - --dataflow_subnetwork
      - { inputValue: dataflow_subnetwork }
      - --dataflow_use_public_ips
      - { inputValue: dataflow_use_public_ips }
      - --kms_key_name
      - { inputValue: encryption_spec_key_name }
      - --evaluated_annotation_output_uri
      - { outputPath: evaluated_annotation_output_uri }
      - --gcp_resources
      - { outputPath: gcp_resources }
      - --executor_input
      - "{{$}}"
