docs(components): Update AutoSxS pipeline to use "question_answering" as task name instead of "question_answer", where "question_answer" is still supported, but deprecated

chore(components): Update RLHF and AutoSxS image tags

PiperOrigin-RevId: 600532640

components/google-cloud/RELEASE.md

@@ -2,6 +2,7 @@
 * Use `large_model_reference` for `model_reference_name` when uploading models from `preview.llm.rlhf_pipeline` instead of hardcoding value as `text-bison@001`.
 * Disable caching when resolving model display names for RLHF-tuned models so a unique name is generated on each `preview.llm.rlhf_pipeline` run.
 * Upload the tuned adapter to Model Registry instead of model checkpoint from `preview.llm.rlhf_pipeline`.
+* Fix the naming of AutoSxS's question answering task. "question_answer" -> "question_answering".
 * Add Vertex model get component (`v1.model.ModelGetOp`).
 * Migrate to Protobuf 4 (`protobuf>=4.21.1,<5`). Require `kfp>=2.6.0`.
 * Support setting version aliases in (`v1.model.ModelUploadOp`).

components/google-cloud/google_cloud_pipeline_components/_implementation/llm/autosxs_arbiter.py

@@ -57,8 +57,8 @@ def autosxs_arbiter(
     human_preference_column: Human preference column included in our inference
       output.
     task: Evaluation task in the form {task}@{version}. task can be one of
-      "summarization", "question_answer". Version is an integer with 3 digits or
+      "summarization", "question_answering". Version is an integer with 3 digits
-      "latest". Ex: summarization@001 or question_answer@latest.
+      or "latest". Ex: summarization@001 or question_answering@latest.
     judgments_format: The format to write judgments to. Can be either 'json' or
       'bigquery'.
     bigquery_destination_prefix: BigQuery table to write judgments to if the

components/google-cloud/google_cloud_pipeline_components/_implementation/llm/env.py

@@ -14,11 +14,16 @@
 """A collection of constants shared across components and pipelines."""
 import os
 
+_DEFAULT_AUTOSXS_IMAGE_TAG = '20240116_0507_RC00'
 
 def get_private_image_tag() -> str:
   return os.getenv('PRIVATE_IMAGE_TAG') or '20231213_0507_RC00'
 
 
+def get_autosxs_image_tag() -> str:
+  return os.getenv('PRIVATE_IMAGE_TAG') or _DEFAULT_AUTOSXS_IMAGE_TAG
+
+
 def get_use_test_machine_spec() -> bool:
   str_value = os.getenv('USE_TEST_MACHINE_SPEC', 'False')
   return str_value.lower() in {'true', '1'}
@@ -45,6 +50,7 @@ PRIVATE_IMAGE_NAME_PREFIX: str = (
     os.getenv('PRIVATE_IMAGE_NAME_PREFIX') or 'rlhf_'
 )
 PRIVATE_IMAGE_TAG: str = get_private_image_tag()
+AUTOSXS_IMAGE_TAG: str = get_autosxs_image_tag()
 
 # Dataset variables:
 TRAIN_SPLIT: str = 'train'

components/google-cloud/google_cloud_pipeline_components/_implementation/llm/utils.py

@@ -123,9 +123,14 @@ def get_default_image_uri(image_name: str) -> str:
   Returns:
     URI of the image.
   """
+  if image_name.find('autosxs') != -1:
+    image_tag = env.get_autosxs_image_tag()
+  else:
+    image_tag = env.get_private_image_tag()
+
   return '/'.join([
       f'{env.PRIVATE_ARTIFACT_REGISTRY_LOCATION}-docker.pkg.dev',
       env.PRIVATE_ARTIFACT_REGISTRY_PROJECT,
       env.PRIVATE_ARTIFACT_REGISTRY,
-      f'{env.PRIVATE_IMAGE_NAME_PREFIX}{image_name}:{env.get_private_image_tag()}',
+      f'{env.PRIVATE_IMAGE_NAME_PREFIX}{image_name}:{image_tag}',
   ])

components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/model_based_llm_evaluation/autosxs/autosxs_pipeline.py

@@ -113,7 +113,7 @@ def autosxs_pipeline(
 
   Args:
     evaluation_dataset: A BigQuery table or comma-separated list of GCS paths to a JSONL dataset containing evaluation examples.
-    task: Evaluation task in the form `{task}@{version}`. task can be one of `[summarization, question_answer]`. Version is an integer with 3 digits or "latest". Ex: `summarization@001` or `question_answer@latest`.
+    task: Evaluation task in the form `{task}@{version}`. task can be one of `[summarization, question_answering]`. Version is an integer with 3 digits or "latest". Ex: `summarization@001` or `question_answering@latest`.
     id_columns: The columns which distinguish unique evaluation examples.
     model_a: A fully-qualified model resource name (`projects/{project}/locations/{location}/models/{model}@{version}`) or publisher model resource name (`publishers/{publisher}/models/{model}`).  This parameter is optional if Model A responses are specified.
     model_b: A fully-qualified model resource name (`projects/{project}/locations/{location}/models/{model}@{version}`) or publisher model resource name (`publishers/{publisher}/models/{model}`).  This parameter is optional if Model B responses are specified.