From 60b66dca0f0229275ae1f015d3c27bb5eea3e6b0 Mon Sep 17 00:00:00 2001 From: Googler Date: Mon, 22 Jan 2024 12:17:15 -0800 Subject: [PATCH] docs(components): Update AutoSxS pipeline to use "question_answering" as task name instead of "question_answer", where "question_answer" is still supported, but deprecated chore(components): Update RLHF and AutoSxS image tags PiperOrigin-RevId: 600532640 --- components/google-cloud/RELEASE.md | 1 + .../_implementation/llm/autosxs_arbiter.py | 4 ++-- .../_implementation/llm/env.py | 6 ++++++ .../_implementation/llm/utils.py | 7 ++++++- .../model_based_llm_evaluation/autosxs/autosxs_pipeline.py | 2 +- 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/components/google-cloud/RELEASE.md b/components/google-cloud/RELEASE.md index 006da0fcaf..a0f5530438 100644 --- a/components/google-cloud/RELEASE.md +++ b/components/google-cloud/RELEASE.md @@ -2,6 +2,7 @@ * Use `large_model_reference` for `model_reference_name` when uploading models from `preview.llm.rlhf_pipeline` instead of hardcoding value as `text-bison@001`. * Disable caching when resolving model display names for RLHF-tuned models so a unique name is generated on each `preview.llm.rlhf_pipeline` run. * Upload the tuned adapter to Model Registry instead of model checkpoint from `preview.llm.rlhf_pipeline`. +* Fix the naming of AutoSxS's question answering task. "question_answer" -> "question_answering". * Add Vertex model get component (`v1.model.ModelGetOp`). * Migrate to Protobuf 4 (`protobuf>=4.21.1,<5`). Require `kfp>=2.6.0`. * Support setting version aliases in (`v1.model.ModelUploadOp`). diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/autosxs_arbiter.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/autosxs_arbiter.py index 6269b41307..8d9c345633 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/autosxs_arbiter.py +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/autosxs_arbiter.py @@ -57,8 +57,8 @@ def autosxs_arbiter( human_preference_column: Human preference column included in our inference output. task: Evaluation task in the form {task}@{version}. task can be one of - "summarization", "question_answer". Version is an integer with 3 digits or - "latest". Ex: summarization@001 or question_answer@latest. + "summarization", "question_answering". Version is an integer with 3 digits + or "latest". Ex: summarization@001 or question_answering@latest. judgments_format: The format to write judgments to. Can be either 'json' or 'bigquery'. bigquery_destination_prefix: BigQuery table to write judgments to if the diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/env.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/env.py index c7511d222c..35d66f6465 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/env.py +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/env.py @@ -14,11 +14,16 @@ """A collection of constants shared across components and pipelines.""" import os +_DEFAULT_AUTOSXS_IMAGE_TAG = '20240116_0507_RC00' def get_private_image_tag() -> str: return os.getenv('PRIVATE_IMAGE_TAG') or '20231213_0507_RC00' +def get_autosxs_image_tag() -> str: + return os.getenv('PRIVATE_IMAGE_TAG') or _DEFAULT_AUTOSXS_IMAGE_TAG + + def get_use_test_machine_spec() -> bool: str_value = os.getenv('USE_TEST_MACHINE_SPEC', 'False') return str_value.lower() in {'true', '1'} @@ -45,6 +50,7 @@ PRIVATE_IMAGE_NAME_PREFIX: str = ( os.getenv('PRIVATE_IMAGE_NAME_PREFIX') or 'rlhf_' ) PRIVATE_IMAGE_TAG: str = get_private_image_tag() +AUTOSXS_IMAGE_TAG: str = get_autosxs_image_tag() # Dataset variables: TRAIN_SPLIT: str = 'train' diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/utils.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/utils.py index fc463f9205..e01bc5d9e6 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/utils.py +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/utils.py @@ -123,9 +123,14 @@ def get_default_image_uri(image_name: str) -> str: Returns: URI of the image. """ + if image_name.find('autosxs') != -1: + image_tag = env.get_autosxs_image_tag() + else: + image_tag = env.get_private_image_tag() + return '/'.join([ f'{env.PRIVATE_ARTIFACT_REGISTRY_LOCATION}-docker.pkg.dev', env.PRIVATE_ARTIFACT_REGISTRY_PROJECT, env.PRIVATE_ARTIFACT_REGISTRY, - f'{env.PRIVATE_IMAGE_NAME_PREFIX}{image_name}:{env.get_private_image_tag()}', + f'{env.PRIVATE_IMAGE_NAME_PREFIX}{image_name}:{image_tag}', ]) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/model_based_llm_evaluation/autosxs/autosxs_pipeline.py b/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/model_based_llm_evaluation/autosxs/autosxs_pipeline.py index a0a9f7b7a1..04bc0eab5e 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/model_based_llm_evaluation/autosxs/autosxs_pipeline.py +++ b/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/model_based_llm_evaluation/autosxs/autosxs_pipeline.py @@ -113,7 +113,7 @@ def autosxs_pipeline( Args: evaluation_dataset: A BigQuery table or comma-separated list of GCS paths to a JSONL dataset containing evaluation examples. - task: Evaluation task in the form `{task}@{version}`. task can be one of `[summarization, question_answer]`. Version is an integer with 3 digits or "latest". Ex: `summarization@001` or `question_answer@latest`. + task: Evaluation task in the form `{task}@{version}`. task can be one of `[summarization, question_answering]`. Version is an integer with 3 digits or "latest". Ex: `summarization@001` or `question_answering@latest`. id_columns: The columns which distinguish unique evaluation examples. model_a: A fully-qualified model resource name (`projects/{project}/locations/{location}/models/{model}@{version}`) or publisher model resource name (`publishers/{publisher}/models/{model}`). This parameter is optional if Model A responses are specified. model_b: A fully-qualified model resource name (`projects/{project}/locations/{location}/models/{model}@{version}`) or publisher model resource name (`publishers/{publisher}/models/{model}`). This parameter is optional if Model B responses are specified.