open-telemetry
/
opentelemetry-python-contrib
mirror of https://github.com/open-telemetry/opentelemetry-python-contrib.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update vertexai instrumentation to be in-line with the latest semantic conventions (#3709) 

* Commit changes

* Fix all tests

* Add changelog

* Fix typecheck

* Fix tests

* get rid of typing.Any subclass

* Update PR to use latest gen ai utils..

* empty commit

* Try to fix typechecker

* Commit latest changes

* Address comments

* Address comments

* Fix lint and spell check

* Fix last typecheck issues..

* add to workspace

---------

Co-authored-by: Aaron Abbott <aaronabbott@google.com>
This commit is contained in:
DylanRussell 2025-09-19 20:54:19 +00:00 committed by GitHub
parent 13fa314cc6
commit 6edb3f8dc7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
29 changed files with 4101 additions and 2812 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dev-requirements.txt
View File

@ -1,6 +1,6 @@
pylint==3.0.2
httpretty==1.1.4
pyright==v1.1.396
pyright==v1.1.404
sphinx==7.1.2
sphinx-rtd-theme==2.0.0rc4
sphinx-autodoc-typehints==1.25.2

3
instrumentation-genai/opentelemetry-instrumentation-vertexai/CHANGELOG.md
View File

@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## Unreleased
- Start making changes to implement the big semantic convention changes made in https://github.com/open-telemetry/semantic-conventions/pull/2179.
Now only a single event (`gen_ai.client.inference.operation.details`) is used to capture Chat History. These changes will be opt-in,
users will need to set the environment variable OTEL_SEMCONV_STABILITY_OPT_IN to `gen_ai_latest_experimental` to see them ([#3386](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3386)).
- Implement uninstrument for `opentelemetry-instrumentation-vertexai`
([#3328](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3328))
- VertexAI support for async calling

5
instrumentation-genai/opentelemetry-instrumentation-vertexai/pyproject.toml
View File

@ -25,8 +25,9 @@ classifiers = [
]
dependencies = [
"opentelemetry-api ~= 1.28",
"opentelemetry-instrumentation ~= 0.49b0",
"opentelemetry-semantic-conventions ~= 0.49b0",
"opentelemetry-instrumentation ~= 0.58b0",
"opentelemetry-util-genai == 0.1b0.dev",
"opentelemetry-semantic-conventions ~= 0.58b0",
]
[project.optional-dependencies]

40
instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/__init__.py
View File

@ -48,6 +48,11 @@ from wrapt import (
)
from opentelemetry._events import get_event_logger
from opentelemetry.instrumentation._semconv import (
_OpenTelemetrySemanticConventionStability,
_OpenTelemetryStabilitySignalType,
_StabilityMode,
)
from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
from opentelemetry.instrumentation.utils import unwrap
from opentelemetry.instrumentation.vertexai.package import _instruments
@ -104,24 +109,49 @@ class VertexAIInstrumentor(BaseInstrumentor):
def _instrument(self, **kwargs: Any):
"""Enable VertexAI instrumentation."""
sem_conv_opt_in_mode = _OpenTelemetrySemanticConventionStability._get_opentelemetry_stability_opt_in_mode(
_OpenTelemetryStabilitySignalType.GEN_AI,
)
tracer_provider = kwargs.get("tracer_provider")
schema = (
Schemas.V1_28_0.value
if sem_conv_opt_in_mode == _StabilityMode.DEFAULT
else Schemas.V1_36_0.value
)
tracer = get_tracer(
__name__,
"",
tracer_provider,
schema_url=Schemas.V1_28_0.value,
schema_url=schema,
)
event_logger_provider = kwargs.get("event_logger_provider")
event_logger = get_event_logger(
__name__,
"",
schema_url=Schemas.V1_28_0.value,
schema_url=schema,
event_logger_provider=event_logger_provider,
)
method_wrappers = MethodWrappers(
tracer, event_logger, is_content_enabled()
sem_conv_opt_in_mode = _OpenTelemetrySemanticConventionStability._get_opentelemetry_stability_opt_in_mode(
_OpenTelemetryStabilitySignalType.GEN_AI,
)
if sem_conv_opt_in_mode == _StabilityMode.DEFAULT:
# Type checker now knows sem_conv_opt_in_mode is a Literal[_StabilityMode.DEFAULT]
method_wrappers = MethodWrappers(
tracer,
event_logger,
is_content_enabled(sem_conv_opt_in_mode),
sem_conv_opt_in_mode,
)
elif sem_conv_opt_in_mode == _StabilityMode.GEN_AI_LATEST_EXPERIMENTAL:
# Type checker now knows it's the other literal
method_wrappers = MethodWrappers(
tracer,
event_logger,
is_content_enabled(sem_conv_opt_in_mode),
sem_conv_opt_in_mode,
)
else:
raise RuntimeError(f"{sem_conv_opt_in_mode} mode not supported")
for client_class, method_name, wrapper in _methods_to_wrap(
method_wrappers
):

148
instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/patch.py
View File

@ -20,12 +20,20 @@ from typing import (
Any,
Awaitable,
Callable,
Literal,
MutableSequence,
Union,
cast,
overload,
)
from opentelemetry._events import EventLogger
from opentelemetry.instrumentation._semconv import (
_StabilityMode,
)
from opentelemetry.instrumentation.vertexai.utils import (
GenerateContentParams,
create_operation_details_event,
get_genai_request_attributes,
get_genai_response_attributes,
get_server_attributes,
@ -34,6 +42,7 @@ from opentelemetry.instrumentation.vertexai.utils import (
response_to_events,
)
from opentelemetry.trace import SpanKind, Tracer
from opentelemetry.util.genai.types import ContentCapturingMode
if TYPE_CHECKING:
from google.cloud.aiplatform_v1.services.prediction_service import client
@ -89,17 +98,48 @@ def _extract_params(
)
# For details about GEN_AI_LATEST_EXPERIMENTAL stability mode see
# https://github.com/open-telemetry/semantic-conventions/blob/v1.37.0/docs/gen-ai/gen-ai-agent-spans.md?plain=1#L18-L37
class MethodWrappers:
@overload
def __init__(
self, tracer: Tracer, event_logger: EventLogger, capture_content: bool
self,
tracer: Tracer,
event_logger: EventLogger,
capture_content: ContentCapturingMode,
sem_conv_opt_in_mode: Literal[
_StabilityMode.GEN_AI_LATEST_EXPERIMENTAL
],
) -> None: ...
@overload
def __init__(
self,
tracer: Tracer,
event_logger: EventLogger,
capture_content: bool,
sem_conv_opt_in_mode: Literal[_StabilityMode.DEFAULT],
) -> None: ...
def __init__(
self,
tracer: Tracer,
event_logger: EventLogger,
capture_content: Union[bool, ContentCapturingMode],
sem_conv_opt_in_mode: Union[
Literal[_StabilityMode.DEFAULT],
Literal[_StabilityMode.GEN_AI_LATEST_EXPERIMENTAL],
],
) -> None:
self.tracer = tracer
self.event_logger = event_logger
self.capture_content = capture_content
self.sem_conv_opt_in_mode = sem_conv_opt_in_mode
@contextmanager
def _with_instrumentation(
def _with_new_instrumentation(
self,
capture_content: ContentCapturingMode,
instance: client.PredictionServiceClient
| client_v1beta1.PredictionServiceClient,
args: Any,
@ -108,7 +148,55 @@ class MethodWrappers:
params = _extract_params(*args, **kwargs)
api_endpoint: str = instance.api_endpoint # type: ignore[reportUnknownMemberType]
span_attributes = {
**get_genai_request_attributes(params),
**get_genai_request_attributes(False, params),
**get_server_attributes(api_endpoint),
}
span_name = get_span_name(span_attributes)
with self.tracer.start_as_current_span(
name=span_name,
kind=SpanKind.CLIENT,
attributes=span_attributes,
) as span:
def handle_response(
response: prediction_service.GenerateContentResponse
| prediction_service_v1beta1.GenerateContentResponse
| None,
) -> None:
if span.is_recording() and response:
# When streaming, this is called multiple times so attributes would be
# overwritten. In practice, it looks the API only returns the interesting
# attributes on the last streamed response. However, I couldn't find
# documentation for this and setting attributes shouldn't be too expensive.
span.set_attributes(
get_genai_response_attributes(response)
)
self.event_logger.emit(
create_operation_details_event(
api_endpoint=api_endpoint,
params=params,
capture_content=capture_content,
response=response,
)
)
yield handle_response
@contextmanager
def _with_default_instrumentation(
self,
capture_content: bool,
instance: client.PredictionServiceClient
| client_v1beta1.PredictionServiceClient,
args: Any,
kwargs: Any,
):
params = _extract_params(*args, **kwargs)
api_endpoint: str = instance.api_endpoint # type: ignore[reportUnknownMemberType]
span_attributes = {
**get_genai_request_attributes(False, params),
**get_server_attributes(api_endpoint),
}
@ -120,7 +208,7 @@ class MethodWrappers:
attributes=span_attributes,
) as span:
for event in request_to_events(
params=params, capture_content=self.capture_content
params=params, capture_content=capture_content
):
self.event_logger.emit(event)
@ -141,7 +229,7 @@ class MethodWrappers:
)
for event in response_to_events(
response=response, capture_content=self.capture_content
response=response, capture_content=capture_content
):
self.event_logger.emit(event)
@ -162,12 +250,25 @@ class MethodWrappers:
prediction_service.GenerateContentResponse
| prediction_service_v1beta1.GenerateContentResponse
):
with self._with_instrumentation(
instance, args, kwargs
) as handle_response:
response = wrapped(*args, **kwargs)
handle_response(response)
return response
if self.sem_conv_opt_in_mode == _StabilityMode.DEFAULT:
capture_content_bool = cast(bool, self.capture_content)
with self._with_default_instrumentation(
capture_content_bool, instance, args, kwargs
) as handle_response:
response = wrapped(*args, **kwargs)
handle_response(response)
return response
else:
capture_content = cast(ContentCapturingMode, self.capture_content)
with self._with_new_instrumentation(
capture_content, instance, args, kwargs
) as handle_response:
response = None
try:
response = wrapped(*args, **kwargs)
return response
finally:
handle_response(response)
async def agenerate_content(
self,
@ -186,9 +287,22 @@ class MethodWrappers:
prediction_service.GenerateContentResponse
| prediction_service_v1beta1.GenerateContentResponse
):
with self._with_instrumentation(
instance, args, kwargs
) as handle_response:
response = await wrapped(*args, **kwargs)
handle_response(response)
return response
if self.sem_conv_opt_in_mode == _StabilityMode.DEFAULT:
capture_content_bool = cast(bool, self.capture_content)
with self._with_default_instrumentation(
capture_content_bool, instance, args, kwargs
) as handle_response:
response = await wrapped(*args, **kwargs)
handle_response(response)
return response
else:
capture_content = cast(ContentCapturingMode, self.capture_content)
with self._with_new_instrumentation(
capture_content, instance, args, kwargs
) as handle_response:
response = None
try:
response = await wrapped(*args, **kwargs)
return response
finally:
handle_response(response)

185
instrumentation-genai/opentelemetry-instrumentation-vertexai/src/opentelemetry/instrumentation/vertexai/utils.py
View File

@ -17,24 +17,29 @@
from __future__ import annotations
import re
from dataclasses import dataclass
from dataclasses import asdict, dataclass
from os import environ
from typing import (
TYPE_CHECKING,
Iterable,
Literal,
Mapping,
Sequence,
Union,
cast,
overload,
)
from urllib.parse import urlparse
from google.protobuf import json_format
from opentelemetry._events import Event
from opentelemetry.instrumentation._semconv import (
_StabilityMode,
)
from opentelemetry.instrumentation.vertexai.events import (
ChoiceMessage,
ChoiceToolCall,
FinishReason,
assistant_event,
choice_event,
system_event,
@ -45,6 +50,17 @@ from opentelemetry.semconv._incubating.attributes import (
gen_ai_attributes as GenAIAttributes,
)
from opentelemetry.semconv.attributes import server_attributes
from opentelemetry.util.genai.types import (
ContentCapturingMode,
FinishReason,
InputMessage,
MessagePart,
OutputMessage,
Text,
ToolCall,
ToolCallResponse,
)
from opentelemetry.util.genai.utils import get_content_capturing_mode
from opentelemetry.util.types import AnyValue, AttributeValue
if TYPE_CHECKING:
@ -105,7 +121,8 @@ def get_server_attributes(
}
def get_genai_request_attributes(
def get_genai_request_attributes( # pylint: disable=too-many-branches
use_latest_semconvs: bool,
params: GenerateContentParams,
operation_name: GenAIAttributes.GenAiOperationNameValues = GenAIAttributes.GenAiOperationNameValues.CHAT,
):
@ -113,9 +130,12 @@ def get_genai_request_attributes(
generation_config = params.generation_config
attributes: dict[str, AttributeValue] = {
GenAIAttributes.GEN_AI_OPERATION_NAME: operation_name.value,
GenAIAttributes.GEN_AI_SYSTEM: GenAIAttributes.GenAiSystemValues.VERTEX_AI.value,
GenAIAttributes.GEN_AI_REQUEST_MODEL: model,
}
if not use_latest_semconvs:
attributes[GenAIAttributes.GEN_AI_SYSTEM] = (
GenAIAttributes.GenAiSystemValues.VERTEX_AI.value
)
if not generation_config:
return attributes
@ -127,6 +147,8 @@ def get_genai_request_attributes(
generation_config.temperature
)
if "top_p" in generation_config:
# There is also a top_k parameter ( The maximum number of tokens to consider when sampling.),
# but no semconv yet exists for it.
attributes[GenAIAttributes.GEN_AI_REQUEST_TOP_P] = (
generation_config.top_p
)
@ -142,16 +164,28 @@ def get_genai_request_attributes(
attributes[GenAIAttributes.GEN_AI_REQUEST_FREQUENCY_PENALTY] = (
generation_config.frequency_penalty
)
# Uncomment once GEN_AI_REQUEST_SEED is released in 1.30
# https://github.com/open-telemetry/semantic-conventions/pull/1710
# if "seed" in generation_config:
# attributes[GenAIAttributes.GEN_AI_REQUEST_SEED] = (
# generation_config.seed
# )
if "stop_sequences" in generation_config:
attributes[GenAIAttributes.GEN_AI_REQUEST_STOP_SEQUENCES] = (
generation_config.stop_sequences
)
if use_latest_semconvs:
if "seed" in generation_config:
attributes[GenAIAttributes.GEN_AI_REQUEST_SEED] = (
generation_config.seed
)
if "candidate_count" in generation_config:
attributes[GenAIAttributes.GEN_AI_REQUEST_CHOICE_COUNT] = (
generation_config.candidate_count
)
if "response_mime_type" in generation_config:
if generation_config.response_mime_type == "text/plain":
attributes[GenAIAttributes.GEN_AI_OUTPUT_TYPE] = "text"
elif generation_config.response_mime_type == "application/json":
attributes[GenAIAttributes.GEN_AI_OUTPUT_TYPE] = "json"
else:
attributes[GenAIAttributes.GEN_AI_OUTPUT_TYPE] = (
generation_config.response_mime_type
)
return attributes
@ -164,8 +198,6 @@ def get_genai_response_attributes(
_map_finish_reason(candidate.finish_reason)
for candidate in response.candidates
]
# TODO: add gen_ai.response.id once available in the python client
# https://github.com/open-telemetry/opentelemetry-python-contrib/issues/3246
return {
GenAIAttributes.GEN_AI_RESPONSE_MODEL: response.model_version,
GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS: finish_reasons,
@ -188,12 +220,29 @@ OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT = (
)
def is_content_enabled() -> bool:
capture_content = environ.get(
OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, "false"
)
@overload
def is_content_enabled(
mode: Literal[_StabilityMode.GEN_AI_LATEST_EXPERIMENTAL],
) -> ContentCapturingMode: ...
return capture_content.lower() == "true"
@overload
def is_content_enabled(mode: Literal[_StabilityMode.DEFAULT]) -> bool: ...
def is_content_enabled(
mode: Union[
Literal[_StabilityMode.DEFAULT],
Literal[_StabilityMode.GEN_AI_LATEST_EXPERIMENTAL],
],
) -> Union[bool, ContentCapturingMode]:
if mode == _StabilityMode.DEFAULT:
capture_content = environ.get(
OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, "false"
)
return capture_content.lower() == "true"
return get_content_capturing_mode()
def get_span_name(span_attributes: Mapping[str, AttributeValue]) -> str:
@ -242,7 +291,7 @@ def request_to_events(
id_=f"{function_response.name}_{idx}",
role=content.role,
content=json_format.MessageToDict(
function_response._pb.response
function_response._pb.response # type: ignore[reportUnknownMemberType]
)
if capture_content
else None,
@ -258,6 +307,100 @@ def request_to_events(
yield user_event(role=content.role, content=request_content)
def create_operation_details_event(
*,
api_endpoint: str,
response: prediction_service.GenerateContentResponse
| prediction_service_v1beta1.GenerateContentResponse
| None,
params: GenerateContentParams,
capture_content: ContentCapturingMode,
) -> Event:
event = Event(name="gen_ai.client.inference.operation.details")
attributes: dict[str, AnyValue] = {
**get_genai_request_attributes(True, params),
**get_server_attributes(api_endpoint),
**(get_genai_response_attributes(response) if response else {}),
}
event.attributes = attributes
if capture_content in {
ContentCapturingMode.NO_CONTENT,
ContentCapturingMode.SPAN_ONLY,
}:
return event
if params.system_instruction:
attributes[GenAIAttributes.GEN_AI_SYSTEM_INSTRUCTIONS] = [
{
"type": "text",
"content": "\n".join(
part.text for part in params.system_instruction.parts
),
}
]
if params.contents:
attributes[GenAIAttributes.GEN_AI_INPUT_MESSAGES] = [
asdict(_convert_content_to_message(content))
for content in params.contents
]
if response and response.candidates:
attributes[GenAIAttributes.GEN_AI_OUTPUT_MESSAGES] = [
asdict(x) for x in _convert_response_to_output_messages(response)
]
return event
def _convert_response_to_output_messages(
response: prediction_service.GenerateContentResponse
| prediction_service_v1beta1.GenerateContentResponse,
) -> list[OutputMessage]:
output_messages: list[OutputMessage] = []
for candidate in response.candidates:
message = _convert_content_to_message(candidate.content)
output_messages.append(
OutputMessage(
finish_reason=_map_finish_reason(candidate.finish_reason),
role=message.role,
parts=message.parts,
)
)
return output_messages
def _convert_content_to_message(
content: content.Content | content_v1beta1.Content,
) -> InputMessage:
parts: MessagePart = []
for idx, part in enumerate(content.parts):
if "function_response" in part:
part = part.function_response
parts.append(
ToolCallResponse(
id=f"{part.name}_{idx}",
response=json_format.MessageToDict(part._pb.response), # type: ignore[reportUnknownMemberType]
)
)
elif "function_call" in part:
part = part.function_call
parts.append(
ToolCall(
id=f"{part.name}_{idx}",
name=part.name,
arguments=json_format.MessageToDict(
part._pb.args, # type: ignore[reportUnknownMemberType]
),
)
)
elif "text" in part:
parts.append(Text(content=part.text))
else:
dict_part = type(part).to_dict( # type: ignore[reportUnknownMemberType]
part, always_print_fields_with_no_presence=False
)
dict_part["type"] = type(part)
parts.append(dict_part)
return InputMessage(role=content.role, parts=parts)
def response_to_events(
*,
response: prediction_service.GenerateContentResponse
@ -302,7 +445,7 @@ def _extract_tool_calls(
function=ChoiceToolCall.Function(
name=part.function_call.name,
arguments=json_format.MessageToDict(
part.function_call._pb.args
part.function_call._pb.args # type: ignore[reportUnknownMemberType]
)
if capture_content
else None,
@ -321,7 +464,9 @@ def _parts_to_any_value(
return [
cast(
"dict[str, AnyValue]",
type(part).to_dict(part, including_default_value_fields=False), # type: ignore[reportUnknownMemberType]
type(part).to_dict( # type: ignore[reportUnknownMemberType]
part, always_print_fields_with_no_presence=False
),
)
for part in parts
]

25
instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_function_call_choice.yaml
View File

@ -49,7 +49,7 @@ interactions:
User-Agent:
- python-requests/2.32.3
method: POST
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-1.5-flash-002:generateContent?%24alt=json%3Benum-encoding%3Dint
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint
response:
body:
string: |-
@ -65,7 +65,8 @@ interactions:
"args": {
"location": "New Delhi"
}
}
},
"thoughtSignature": "CuUGAcu98PA0CgDJVjnZvciC3yQjuC6R/hJiEIzLX88DIJjiqSv2B+3ZhTR62dWOcR8xHdVOUjJB+LgQO/KTzyZ9FZBM5OdkXsXMC+KsrEsjnjDq0wSAtDe8zDhztTD2mmgFT7i0asHN60aoxdkgM3myRHyzmaNjs+F2Ua3wS9MiAqzYdKoPry5kzkL/KQBI3dpsL/JiiGKFgXQ7y+hL3epdjU4ySjVw/x5zKC/MyHHpQIhHwhzKYZR3I1TUR3b0ziBZJknQHFaxNckeDKfY3aXuawPqqqf+RrUd7MEQOE0nnJF/jl7zwhxevPglX9R//MTpzV8bZIBLKu84GYIM1h7x2iHN0t5arER3VDk0kgIG0FsOaMrv+BqRykSsQc/9K2gLdB+ouGSlLJLQ8amnyiuxQC25RHRPlGvJKJCq2wCl2pEAhd7HUoCbiN145pbdoYi+L4vigSIsob7NvGHW/RJiZd1lHXAVJUCmzmhyNMj3iYW3qvO5+H41q5EZjlUT8RNBwN9Lv4V71CRdQnw5oYfH1WncPoVdbtQwPqIzhqJ93JKLJGdSkkNrODdZlt96iOHDchD0PLLz/vGg90La+v+sAvwnKRX/SGXruySlTDKntua4UBFauFD7Cyv/VQvyNpqiP39jnTdjW3J42vKLEtpf/Ldagf1JUcZ/BXQUYamhzuEdRVBg3XB0kbXA/e59vSsqXHPDFurbWzv8q56Jxu/lE8H416iPlWuR25J8SqWahHO039YtOrjGW9+/C69bFxh9HwSJtmUkoX0P0cZmRuYkOl3mpl4sRz/GfebNu7+G4+q9O5SSnpmzxEGUNccRiiTiZK89YPj3vqnODydK5qYTyMaTkYGkInxqCoJbCvXuNnYArwnVuWGKMHHJfmlV9et24zipKdMfb/ZTfdSy9uni4mgMOqd6b4er08Qft1NKktxqmU87izBd5Vt6VSfYsJqW4lrRs4f+28TfFXl2znfKt16Pi+8AuyH+49zS/xkZS5J48rGg0xWvowPkT2vSz/nYGb5xlEDPDABDvfwwib+T32d7cj6dQgZHzQWnPjZgSlSfOYuU2QSZtO5nx8oS2jnF+sb2Ywst0ZMV+ECOsH5OSjQ31Nmd6/PIHGVtFI6veBK78Xon2iuoHNwwqYZj8+hAXRMTkyY="
},
{
"functionCall": {
@ -78,17 +79,18 @@ interactions:
]
},
"finishReason": 1,
"avgLogprobs": -0.00018152029952034354
"avgLogprobs": -1.0146095752716064
}
],
"usageMetadata": {
"promptTokenCount": 72,
"promptTokenCount": 74,
"candidatesTokenCount": 16,
"totalTokenCount": 88,
"totalTokenCount": 290,
"trafficType": 1,
"promptTokensDetails": [
{
"modality": 1,
"tokenCount": 72
"tokenCount": 74
}
],
"candidatesTokensDetails": [
@ -96,11 +98,12 @@ interactions:
"modality": 1,
"tokenCount": 16
}
]
],
"thoughtsTokenCount": 200
},
"modelVersion": "gemini-1.5-flash-002",
"createTime": "2025-02-06T04:26:30.610859Z",
"responseId": "9jmkZ6ukJb382PgPrp7zsQw"
"modelVersion": "gemini-2.5-pro",
"createTime": "2025-08-19T14:55:25.379701Z",
"responseId": "XZCkaLWWF8qm1dkP7rOm6AM"
}
headers:
Content-Type:
@ -112,7 +115,7 @@ interactions:
- X-Origin
- Referer
content-length:
- '1029'
- '2273'
status:
code: 200
message: OK

25
instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_function_call_choice_no_content.yaml
View File

@ -49,7 +49,7 @@ interactions:
User-Agent:
- python-requests/2.32.3
method: POST
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-1.5-flash-002:generateContent?%24alt=json%3Benum-encoding%3Dint
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint
response:
body:
string: |-
@ -65,7 +65,8 @@ interactions:
"args": {
"location": "New Delhi"
}
}
},
"thoughtSignature": "CvUMAcu98PCpw5XB3NsEMaSIsgMKQrbACcz+SUZ/0Yix+Ha8XBPVSElLdPHCDnwIVhxjuEUB5BY7g1JzVH6akIVmr/14Y4YF7K1ed9ds/c9FdqkrEKVrC8sVYO16/jFhxrBDS7rD8XQHhjVAuf1W1+I8AH4mhuhTqaYEzjvq+Ea0NsiysqxqCyLneKW2nY5QNT0xi/rUEri19FScPfxcCdn7uDcMyvEQaXFiWdyA3jUIfgrh7E42BHv+xj0LGdWQ8FXJNBjFed6qcarpul7yAImSlyoXJxAgnJx53XgvacYUN9Eup9Gh5b8bU2c6q8qFMsggiq15kSUN3udbiu0cAilezHalh/OkctHmyXbmLF9qSwr6ZO9ie7Mm6pJxT3lea6VPXeFsUkgqt9TxRvOITDxMPSeoCLBFbPmCbvICrBcRHfms05SzOq9tyfJCPU43X0ge49dnTceHrRvF3IrK7pFzetpK30xFwbsKISARZHZuK49iUUjexHlH2ePpCrWgKkp0YqbGuamJWtj1F6Lj/nw45O+ke+55WhPAtam2NAX4MYkN/AMCVyemq5sNi8nC+Pc5T1GCsGmBDYyy8fftG2pZlwMfS4Mu2ta4kD92cj4Zt29Mvb0jIF1smWWiLSfiqRCAt41oK5CZRUz4cEqTulOfTY5NqBUHODsqWkxG6Zei90KFGb99itmHvy9j/HNPHLbqccGiIdJTWE52EFlTCXNoIgn9wPXMiLs+ScdgVcfcl4Tq9DoPrezddjEgLknz5dwOefayzNBSR0xVk3nQiPvWkCMJaqCQ2rmVou8yRugAltW2jsImPcoOdSdmMIJ4MA6O4GGNus8PAaov4VJJY33BSRZ+qsdMhndpcxCO4c7ygkRv+3QaxdgYTlxPyiPm6Gi3BbxHoU7pAlO/rEgUUqA3CvuCHObeUCC7Re/lPPL7HJS0sn6iXbQ+xZuQ0yucd664MQYTNOxrw0IL6oZUM4u8APJJbB9MixypSWC9av68O6+nANO7e6hvQTsvBcDezLt54cFoykqGDSzxMXvjCY4Vf0HQYzWMQU/hsv8Xe8f2iHdS2xgpUKfBqoQmduPPU13Z1EdKfGA1fQv6psM4fO4fqIi4SqcXuRZqDA2kzVyqkRR811m9oe/Prje9IytviLoMuRTO37WUxeGyPvE74c64SAb2F9/AGO+BhzFAUEHcFqU1i1YilrbURamlt3ckajCfCNCb/93jw64QaeE8lmVhL7wzgjjSme0lXWqjnY8QTH2bUb5nuapyGZcyNrwFnvK5NNLHgS9opAXN+3pKT/LPAAwTRgGH1w0JiwVPa0lfNHnSPCtTBNU35B8SSEWyln416wKOhh+uaJuB8NxuGaDyTwAgvvvH/7rWMAWmyJj3eS3AXUdjdqfOi3cNb/4H5aWXi4ydt/uLv2X+carFVrmq0dFxTqbSkBt+N2DTVWtI+aQG3OQ6gI7vOj3E82CuB/A6/atONMJ8SwNAgDG4iT4Z2sgDV5csuvNx7ssp/PS06lcziZEZyO8YRem8C/Worda9AT2DvWqRdDmuB1vEuhH7RNBsq8s4NZolgNLD3kI6OdvaFK6bvfuFzdsG4P2Ngw7v6AfQYYVPcz9WgovoLSPOy7UA0baxboSbTwxUqnWnX/AgxDbPIzNXKY2UUAA/nnK3RXHUcW/uKdz4C5i3ECttqBGK8gfSPS61fHCSqrlGYmbjlFHmg2trMbyu9pM4DEWBq57AR9BwL6/Smhv7uRWHnOxg/tHNEH7JOOHy4zW/d3Ni4EyLzE/loxQ3wZPzw2gas9ROMq8Z1ht22g/MUd+sL34fx/0hEIvZ9Tu21wyznCyr6R/mfXK9n75kqpehQ87p87W95HK0cvT8sqnhPrCFbghGWwtd9F1pRUOHrYESqQ63PK3Sgt91PcGu8lYTr/CS7fkAXmdRLXoioXVKP0EjwRFbO7pryHY1nOwZEU4RprNYyJYyVOavxpc8pC+blUIHrANKwTyYVnX7/Vfa3VFAfC/1PcObSeUpbuc+7MzYzltjzZnly8yILKDmulagTxoRQOiKvqCM3fq0OMYiivAs2+Zii4ZR8B+bM87NKkHvF69VJDF+iwwWT9MjXh8Ju0rvtrxds64Xl8/E9Uv+gnRp3VOIcttqsNuCssSnV5Yj7LRS1glUXbIwBJeXSjevjydOuQOYdn8S6s5Sd95g91tc8vpqGGP1"
},
{
"functionCall": {
@ -78,17 +79,18 @@ interactions:
]
},
"finishReason": 1,
"avgLogprobs": -0.00018169169197790325
"avgLogprobs": -1.5602271556854248
}
],
"usageMetadata": {
"promptTokenCount": 72,
"promptTokenCount": 74,
"candidatesTokenCount": 16,
"totalTokenCount": 88,
"totalTokenCount": 482,
"trafficType": 1,
"promptTokensDetails": [
{
"modality": 1,
"tokenCount": 72
"tokenCount": 74
}
],
"candidatesTokensDetails": [
@ -96,11 +98,12 @@ interactions:
"modality": 1,
"tokenCount": 16
}
]
],
"thoughtsTokenCount": 392
},
"modelVersion": "gemini-1.5-flash-002",
"createTime": "2025-02-06T04:26:31.346685Z",
"responseId": "9zmkZ72UFZ2nnvgP6p3e-As"
"modelVersion": "gemini-2.5-pro",
"createTime": "2025-08-19T14:55:28.661450Z",
"responseId": "YJCkaMqvKPfQgLUPuoKz8QU"
}
headers:
Content-Type:
@ -112,7 +115,7 @@ interactions:
- X-Origin
- Referer
content-length:
- '1029'
- '3317'
status:
code: 200
message: OK

30
instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content.yaml
View File

@ -27,7 +27,7 @@ interactions:
User-Agent:
- python-requests/2.32.3
method: POST
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-1.5-flash-002:generateContent?%24alt=json%3Benum-encoding%3Dint
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint
response:
body:
string: |-
@ -38,20 +38,36 @@ interactions:
"role": "model",
"parts": [
{
"text": "Okay, I understand. I'm ready for your test. Please proceed.\n"
"text": "This is a test."
}
]
},
"finishReason": 1,
"avgLogprobs": -0.005692833348324424
"avgLogprobs": -12.052484893798828
}
],
"usageMetadata": {
"promptTokenCount": 5,
"candidatesTokenCount": 19,
"totalTokenCount": 24
"candidatesTokenCount": 5,
"totalTokenCount": 304,
"trafficType": 1,
"promptTokensDetails": [
{
"modality": 1,
"tokenCount": 5
}
],
"candidatesTokensDetails": [
{
"modality": 1,
"tokenCount": 5
}
],
"thoughtsTokenCount": 294
},
"modelVersion": "gemini-1.5-flash-002"
"modelVersion": "gemini-2.5-pro",
"createTime": "2025-08-19T15:20:47.464265Z",
"responseId": "T5akaImrHM2Dm9IPlu_Z0QI"
}
headers:
Content-Type:
@ -63,7 +79,7 @@ interactions:
- X-Origin
- Referer
content-length:
- '453'
- '740'
status:
code: 200
message: OK

30
instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_all_events.yaml
View File

@ -51,7 +51,7 @@ interactions:
User-Agent:
- python-requests/2.32.3
method: POST
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-1.5-flash-002:generateContent?%24alt=json%3Benum-encoding%3Dint
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint
response:
body:
string: |-
@ -62,20 +62,36 @@ interactions:
"role": "model",
"parts": [
{
"text": "OpenTelemetry, this is a test.\n"
"text": "OpenTelemetry, this is a test."
}
]
},
"finishReason": 1,
"avgLogprobs": -1.1655389850299496e-06
"avgLogprobs": -8.984109878540039
}
],
"usageMetadata": {
"promptTokenCount": 25,
"candidatesTokenCount": 9,
"totalTokenCount": 34
"candidatesTokenCount": 8,
"totalTokenCount": 439,
"trafficType": 1,
"promptTokensDetails": [
{
"modality": 1,
"tokenCount": 25
}
],
"candidatesTokensDetails": [
{
"modality": 1,
"tokenCount": 8
}
],
"thoughtsTokenCount": 406
},
"modelVersion": "gemini-1.5-flash-002"
"modelVersion": "gemini-2.5-pro",
"createTime": "2025-08-19T14:55:12.290295Z",
"responseId": "UJCkaPfbEdDxgLUPhardyQg"
}
headers:
Content-Type:
@ -87,7 +103,7 @@ interactions:
- X-Origin
- Referer
content-length:
- '422'
- '757'
status:
code: 200
message: OK

29
instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_extra_params.yaml
View File

@ -39,7 +39,7 @@ interactions:
User-Agent:
- python-requests/2.32.3
method: POST
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-1.5-flash-002:generateContent?%24alt=json%3Benum-encoding%3Dint
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint
response:
body:
string: |-
@ -47,23 +47,26 @@ interactions:
"candidates": [
{
"content": {
"role": "model",
"parts": [
{
"text": "Okay, I understand."
}
]
"role": "model"
},
"finishReason": 2,
"avgLogprobs": -0.006721805781126022
"finishReason": 2
}
],
"usageMetadata": {
"promptTokenCount": 5,
"candidatesTokenCount": 5,
"totalTokenCount": 10
"totalTokenCount": 9,
"trafficType": 1,
"promptTokensDetails": [
{
"modality": 1,
"tokenCount": 5
}
],
"thoughtsTokenCount": 4
},
"modelVersion": "gemini-1.5-flash-002"
"modelVersion": "gemini-2.5-pro",
"createTime": "2025-08-19T15:18:43.287437Z",
"responseId": "05WkaM3FEaGSmecPxYzosQM"
}
headers:
Content-Type:
@ -75,7 +78,7 @@ interactions:
- X-Origin
- Referer
content-length:
- '407'
- '468'
status:
code: 200
message: OK

4
instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_invalid_role.yaml
View File

@ -27,7 +27,7 @@ interactions:
User-Agent:
- python-requests/2.32.3
method: POST
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-1.5-flash-002:generateContent?%24alt=json%3Benum-encoding%3Dint
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint
response:
body:
string: |-
@ -49,7 +49,7 @@ interactions:
- X-Origin
- Referer
content-length:
- '416'
- '817'
status:
code: 400
message: Bad Request

4
instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_invalid_temperature.yaml
View File

@ -30,7 +30,7 @@ interactions:
User-Agent:
- python-requests/2.32.3
method: POST
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-1.5-flash-002:generateContent?%24alt=json%3Benum-encoding%3Dint
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint
response:
body:
string: |-
@ -52,7 +52,7 @@ interactions:
- X-Origin
- Referer
content-length:
- '809'
- '1472'
status:
code: 400
message: Bad Request

4
instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_missing_model.yaml
View File

@ -34,7 +34,7 @@ interactions:
{
"error": {
"code": 404,
"message": "Publisher Model `projects/otel-starter-project/locations/us-central1/publishers/google/models/gemini-does-not-exist` not found.",
"message": "Publisher Model `projects/fake-project/locations/us-central1/publishers/google/models/gemini-does-not-exist` not found.",
"status": "NOT_FOUND",
"details": []
}
@ -49,7 +49,7 @@ interactions:
- X-Origin
- Referer
content-length:
- '672'
- '1139'
status:
code: 404
message: Not Found

30
instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_generate_content_without_events.yaml
View File

@ -27,7 +27,7 @@ interactions:
User-Agent:
- python-requests/2.32.3
method: POST
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-1.5-flash-002:generateContent?%24alt=json%3Benum-encoding%3Dint
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint
response:
body:
string: |-
@ -38,20 +38,36 @@ interactions:
"role": "model",
"parts": [
{
"text": "Okay, I understand. I'm ready for your test. Please proceed.\n"
"text": "This is a test."
}
]
},
"finishReason": 1,
"avgLogprobs": -0.005519990466142956
"avgLogprobs": -14.465771484375
}
],
"usageMetadata": {
"promptTokenCount": 5,
"candidatesTokenCount": 19,
"totalTokenCount": 24
"candidatesTokenCount": 5,
"totalTokenCount": 354,
"trafficType": 1,
"promptTokensDetails": [
{
"modality": 1,
"tokenCount": 5
}
],
"candidatesTokensDetails": [
{
"modality": 1,
"tokenCount": 5
}
],
"thoughtsTokenCount": 344
},
"modelVersion": "gemini-1.5-flash-002"
"modelVersion": "gemini-2.5-pro",
"createTime": "2025-08-19T15:18:37.008618Z",
"responseId": "zZWkaKpDipOZ5w__8qRQ"
}
headers:
Content-Type:
@ -63,7 +79,7 @@ interactions:
- X-Origin
- Referer
content-length:
- '453'
- '734'
status:
code: 200
message: OK

24
instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_preview_generate_content_all_input_events.yaml
View File

@ -51,7 +51,7 @@ interactions:
User-Agent:
- python-requests/2.32.3
method: POST
uri: https://us-central1-aiplatform.googleapis.com/v1beta1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-1.5-flash-002:generateContent?%24alt=json%3Benum-encoding%3Dint
uri: https://us-central1-aiplatform.googleapis.com/v1beta1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint
response:
body:
string: |-
@ -62,18 +62,19 @@ interactions:
"role": "model",
"parts": [
{
"text": "OpenTelemetry, this is a test.\n"
"text": "OpenTelemetry, this is a test."
}
]
},
"finishReason": 1,
"avgLogprobs": -1.1655389850299496e-06
"avgLogprobs": -7.633238792419434
}
],
"usageMetadata": {
"promptTokenCount": 25,
"candidatesTokenCount": 9,
"totalTokenCount": 34,
"candidatesTokenCount": 8,
"totalTokenCount": 400,
"trafficType": 1,
"promptTokensDetails": [
{
"modality": 1,
@ -83,13 +84,14 @@ interactions:
"candidatesTokensDetails": [
{
"modality": 1,
"tokenCount": 9
"tokenCount": 8
}
]
],
"thoughtsTokenCount": 367
},
"modelVersion": "gemini-1.5-flash-002",
"createTime": "2025-02-03T22:15:35.089616Z",
"responseId": "B0ChZ5C8BbXInvgP19PTyA4"
"modelVersion": "gemini-2.5-pro",
"createTime": "2025-08-19T14:55:21.164378Z",
"responseId": "WZCkaJqECuavgLUP4fGpsQs"
}
headers:
Content-Type:
@ -101,7 +103,7 @@ interactions:
- X-Origin
- Referer
content-length:
- '715'
- '757'
status:
code: 200
message: OK

29
instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_tool_events.yaml
View File

@ -91,7 +91,7 @@ interactions:
User-Agent:
- python-requests/2.32.3
method: POST
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-1.5-flash-002:generateContent?%24alt=json%3Benum-encoding%3Dint
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint
response:
body:
string: |-
@ -102,34 +102,37 @@ interactions:
"role": "model",
"parts": [
{
"text": "The current temperature in New Delhi is 35 degrees Celsius and in San Francisco is 25 degrees Celsius.\n"
"text": "The current temperature in New Delhi is 35\u00b0C, and in San Francisco, it is 25\u00b0C.",
"thoughtSignature": "CvkTAcu98PDKNriXzBqqOq6Z8+JY3Kh2JIFAPWejBW6qG9oykQy8c/2X1Z9S/rjfGiZWs0O2Wv6audk8a2RAKSNv8+XEOWcqIATzdUdBSNe1In6lX9X7ToamJmbmMc5gA2adQuqq9/m6YPwPro2x9RabqQrLoawGKPPAbJKj5wxX5eFbcM+QRwP+YB/1e/6PmSZ313MDu6GEWH6KvagJwDT0Fm7GLcJIuWRrAVWqJ5ssClSVP7ZNjynsi8cHnJ8hTqZxXFqSpOMQHwl1rZH3Z263Vcjq02f9RPjhWOOkkOCqexG0IHGlvkJWdG1Z62rXNtgFPgvNjE4CLaHNFydD3Inuh03X0X3+/RflG6o2VoAjg9TNw8Fc2/UZoDXX54GQOegXdAk/U2oG+8CEi7Xc3eJj3hkPxzeRGsczlxvI/grNx8b9qHgPJ19mLUV0eO3Jj5rQVM0kMmAJEfFgiE3vsPvTyRx25K8u/jjohSOyIoHqH3YWmCxmYXvC4i0ouMbLoX9gc8caQtFIla9KfApcVEWushi84Or9ZNmKKUhtf9Xg3eRx2940oRNXdAG3susiDujjNAoEZXmz8Z+/ELOWLgFhGHYCjzzbm5ybQ5TYtoiYq66N63Lqq2uCwi+JN3eY87gEmVj80HJVIn+A0iwN0yg3vRptO52ih/kI3Vy3LHqHrE+Vx2ZuAzlwL1S83h2sZu18DCpfRFUkpc+myhUj3DvsZ1Ob7g5urwKCOCCJtSmV6RHYx9UfF01/ZWrpuMOQt2bJWi3M28EHPEC3F21f0HYMSyFnRsN5f4c6bcG54bDaUj0OeHwUxC23vd6HDVyyJOG2Z9qVUn8xOpAv1A2uGS4vYiF5sTT0Fj7+LN0HsIvn7401u3DLoKB3aREuX2iXCFlhH7pXZgqxMDV9ThXJDRoEoFSYpi4OYCS95m9aT38eCzdbqa40Pu2AGrT4XMaFcYwm9OXyaSaEnbbnlSwzJFDNy70Byu/MF4IWmgCzLP1DHEP1CkH5BGDmMd54DizjQm53ngPihK3IFfYmG9aPB3n9RHxCaxYvvSREwHEFG5Ag6zYI2P7k4DK9thHSaf2/s+ziKXeGtvLDHkVLuBKAJvL71D2QocV5V8brFjQDI/wzmhbyJ/SOSfReHVnVq/GYKL1rfliHEwKp+F0i63H1yfzUNef7m+KESgMommnu9D3rg5LalRwrhbKsjBkNdX8CE35qQnX77Be+f6rWsPktA9wDD/QALuJHeqA3QxHmBnOcTjtiCN/aF3rg8VrI8OL+ce7a0Fq+GPSgj3OwlvVmZT4glkMFVvpPwb9Bcj3SSMvtY8enLNM/nqRURHfvwmjAFm4KmStVHwX5CvqtjO0WbgpODjvFiXazBnTsmt8eUmCQvJuhCv14H/HCkdbECEqOEFGLB9hZCAXvVqczctTVqcvad8coPPcQ/Scj0D0QVbK7YMDETiLw9FrEVSGjRpKLnPTbJKyRZZqwy4HRD/GucKt7VS8t6/LUYAy4jKGfWoWUpwTcXSL6EMD4FLV8iPsKHY/aYhOZhQ8ce2MLX7UXfeHKue61yDRUviaexR4ctuM5+WnUEEnBKtVMVjLS6XwgDyZldntlYKkajsvvtFT/6W8UC80lcpMwajs5xH5Eh++ogkICuqcdlmbL3KlkxJeZE4T8kSb32Lna2G5Pn4MK8kQJ1j5p7/3BOvjAGZV1/vywGb0ST4cjCWJDJiVeCUBYY6IZYFrEE4eVKH6s4bwxNU93tluunSlHLDPLjxp4l6M01mDriKqtHOJ7fStaouJwh0vJfsuJ0DKmfGMpbI5Cw1sx6930jvMo0LwtUU6KaH36jcsKOymThM5xj0sWZXmhoQz8zv+WLLM4HhffccnS0BnCeInchRqimX1sMjO4THAzSPmXT5+vonqlE+mCdAy8S706STIS40PL/ZWmNuONUXNCnvODFcKl0YKe3lTSN2uE+hkCX2FcBs48AOmeB62kBpHGQr8UIiKheLcq1+5Dj4AO0TUniOAx4+6h5YmCTZJ2LctJrFWrz1/vC05JLN1D9QKM+KwGNExURHWBpyiBp+VI5NaQrTHaXMqzY9LDrQ2LyTNb/s/lQtTsk2AeDokJE39hRnsHP6o/lNUOEIS2xZVIiVwqRw4PpybolBy0y9zqj+mAYEharImI1cpN7JBQz/NGTPe/JrppxTxRinJPTyMEpj8RaYFN4TRuW+9vp2fl+ezIsj2oelhmWKF86xTrQdfL/W1zXJ/OuC+MpNKoo8CVJ11CGF5Q9e6vrQF6nRiqcfWEEC2IxJoCUATrDyWPBPgTcJBi7xXjJYo7zfKwDEaQWRAKLmVoNH0eyTwPs3Kr6d0aUSwp5pmlncyuqNIbQffbXtMBeI7Sts8mxaae4YFiNhga6dzfqFmDI1C4uDDsyY1DdMbpDT7uBGl3YedYiex/2/J2HohFoOXoubQuX//oN8cjOLSHCF/ZxhnBuwcUBLNcE0uQduCV6AlxP/yX8R0+q5A2Ndst2SOygIptrqScD9uxG4cG9gYJGyRiUdtrZku3+1+w8R/7Vk6GjWyIwOsBLWOC9GU/wdDlu3tbX9PiPHQUEMgf/y7FuY9z3qD2fY51a0qQyQc+pF6DyvUzcwcPz83ArwpkiXd2Ll3CXxtPNCVteS/bmdAOWIqBkvISsLYhxSzN11bef2IY0I8JrJZCNgNr5/nBEllZ5+U2ynLC4WkLhXSm3SO+ToZTslJX9lZRhgAPBxwUY5qy4+UmDCjNl+G7BAJLmA8f61omiDiEK1ZBIfXTdDpLXFUs7V2Dn4iDvJKYBwpPbARkTsJROgxYGWEkjIM0ITecj4XnkcwODbCja7WEUTSgyr6a1vjBzmn+azFUrkUKX8vMnMGTkaF6Ypi68LB2OUjxP6SlFor/AWl8is1AmFWjznWny//nSe9s1dnNoX75UDmPlZLFR3r+2t7N7/Q7DlRBE2LMLXaqZi+SbE++dphRYdamTqTbh2vI0BaeZiZYb6/1ElbpAkip160bfuyOxTrDqIeMo/YtWMBEiPESQrnZmFr4cWbjkDAJzZ75qhRE3nMTDMzdITHCLxdOac2GU+7Waf97XGnT/T3S64Uda1tjIO30qtXmQGA3UqanIrUrE2Myqhlh1b3SsWKFqxfdAqzbvgHOHVIPgGrX5PeDzthNX1kUa9TFNye0roIzxC6wx2zW79rJmy8JBtmatLjtbKkgvWdfMcPNTgop6LPEzD5s67Sv+oGW3I3Q9HS7WDdU7eWqH2ZvvyNJumsSOIwtfCpm114l+jylkhf6FBagmuT8d3yiZSaTogAySsZpr/NRBCALQBMKGENboc3EjUDJIhwZfBjR33RjxecAY2gkWGi8/RbfA9Phbqc2LkiJeWFHWL+B/lrDL3hbdMfJCvPflDZU"
}
]
},
"finishReason": 1,
"avgLogprobs": -0.05196272830168406
"avgLogprobs": -1.1418917729304388
}
],
"usageMetadata": {
"promptTokenCount": 126,
"candidatesTokenCount": 24,
"totalTokenCount": 150,
"promptTokenCount": 128,
"candidatesTokenCount": 26,
"totalTokenCount": 846,
"trafficType": 1,
"promptTokensDetails": [
{
"modality": 1,
"tokenCount": 126
"tokenCount": 128
}
],
"candidatesTokensDetails": [
{
"modality": 1,
"tokenCount": 24
"tokenCount": 26
}
]
],
"thoughtsTokenCount": 692
},
"modelVersion": "gemini-1.5-flash-002",
"createTime": "2025-02-06T04:26:32.376510Z",
"responseId": "-DmkZ779FrGM2PgPtOGmmA8"
"modelVersion": "gemini-2.5-pro",
"createTime": "2025-08-19T14:55:33.172473Z",
"responseId": "ZZCkaLnDCuavgLUP4fGpsQs"
}
headers:
Content-Type:
@ -141,7 +144,7 @@ interactions:
- X-Origin
- Referer
content-length:
- '790'
- '4256'
status:
code: 200
message: OK

29
instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/cassettes/test_tool_events_no_content.yaml
View File

@ -91,7 +91,7 @@ interactions:
User-Agent:
- python-requests/2.32.3
method: POST
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-1.5-flash-002:generateContent?%24alt=json%3Benum-encoding%3Dint
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint
response:
body:
string: |-
@ -102,34 +102,37 @@ interactions:
"role": "model",
"parts": [
{
"text": "The current temperature in New Delhi is 35 degrees Celsius and in San Francisco is 25 degrees Celsius.\n"
"text": "The weather in New Delhi is 35\u00b0C and in San Francisco is 25\u00b0C.",
"thoughtSignature": "CqcOAcu98PDUiUq32HLxu6y5JNxvlcEjIaedPcBi5V86Hbf3vRgAXC4k0aMma0v1gotZVHrinF9edI9bEAdQdFR+2xOsaV1ntNeO4o35ymNNpm1rEv2p047eWxSABiXJ3VANecxNqQuVgxZyhClOn2BNmR/xGb43REabpcMzboVGVT6iKSJ/g3sCIt4bddY1IQ5zTdSV7lvCyZLuu9736VCJjULzAslhxSb9/xBlQu/pvcak6CmFFLZuDamqeJ3RvhalpklF6qz/Eq9nhhpdRPTERmGU7mNe4fiSpql6JxelO56ksKKGzSG+USa9vxcCQRRgAoaLlMHRegwPjfoA4hHgbpLBIUk9BNXrzmW5APOpJO3rqgIvKQ/WsZzIBI3l+slI51WHgVLr4qNvDUe8VKLhbApVj3L+rb5dQ/u6U9V2oSVA5pQFt6Lqubyjg8yCl0q+0dsvIYUqJSaDRONMfS5hwVu+RTU/SOEXLwTNEiDDuj3wO8xQuyitDAHqya797sOH2ThlSoT/c7V3Du00TBCk0eq8XaxyTobPMCaKNbrqjRTfMn5fortZlPwyAJqoTZsaSeaRufCFpgBMy34/QcLiCdVwKN6rLgBpDhtzThnyBzavDP2ltu8sxWEisJXEOel0q8LvBqTqlFY9dWrRScnB/TvitGnwqlV0PVpWU60wH/2Agexw0+aFlMRYxiFRjuHPK9R2UfLbEzVpDC8M+f16QRZOnWQc8L+cxEW/xHT6hCUxcTmSrpldAs2Ss8A21NcPTlr0lNviZxB+4xDq1dxB/VMS/qOnGVbr11nEOOVj+bWJyP/LLTjdUigGs+ISAzmEr2zkUirLdlmz/Mz1YXk9CMIus8WlPVzhQZuXY8dhSKxks0w8bJ5TjJelAwPGFgz6iWSz44IeaH22Fxdvub28/mdmvy3OerI3vfwlePUC9L8n2cg7gZjIE59PXi2x1EUTJfvcBrVTO5qqPuRpJZ9/d3ryEtS9tHsMnzfcEiS2lxNb+EqKaHk74hqvt6vEA+eQaYtMqG5TO79dpGDt0UD1cJEwngfMiL85+5qWHW4g2hSSJYgg55GtdZbxdGs960p8G02FbqUOSVcDGcIBDCkvFSGHJqMATvYQzChiwP2AnDVLN0aWd43UEl3rHESdOGLOqrxetcWmVsDacowhiy6jleS/Xml7d8mqqs2483XCOVMMYPcFh9budRed7PIf7eAYhWzGwt123JLbxqwBOBf1MHUJl2OQBy62HCJLYMlRhr1s2KVRK9Een5cgUa3H07r/s46YS2A0KOP/B0Ub0MflvGBAgo8LABj5cY2Ao1hTKcnBjgJPTNS0awETbtzU3Y74SPMsJ1Ipu2o76TFS3YFq1Fbit1VsceuYxzcqU04NFBtXAU7YojLv9LBOZXk6SukOWI/2cNUOHlzlWsSi34heVFGH4AcDbhc0/QZHhjSTE9FkBY9feTtSnTR16Q1bg3K9+7rxArD51Wj8q1ZNhVtZrzpuWGqoneI0WqJ25U5f1jSzQFW2HyzmIs/B5KiFEk4Do0EiaVcCQ5/1UGReYbgtSKa8PAvUGh2Lqiev1VMl6i4cDs7s3U9swvZdBdK+SauYC7UqEVov1G8UR8Bth2YncqUPSv7W2X7ppA6PwDXSCqCSgp3wnfEW8tQIa6AQW2s4Udp0lbYWr3Vu2DSK3iu8mVa3ybvHs5Go4SR9zJ8xElMn4adprz8jokYpoXK94DVeaUetKqTw6X3FzRb+K+e/W0MHZVEQtAjk2wwcNqAfs8JdXi2Mh/l+0PFp6Hhz0rI1rzZ3B08lVR+QU+0XpN5rerUmq44oE6t+dP6Cv8yl5kVBCjvtHZ9DLUGeSEofOR3JXs34rkWeOLfQ+vRZkYoiIRKkqFhf/RnRoU18vmfsbvK9C3XcO3wmfo58Iw2uYd9L/p6VdB27pPUiBJbTAzXbEA3CWUbedMg2vDZiucS4iBkWlmN91CofwgmNfiImdcauz5ug3TC5qRWUIqX/nNMSscPjpbhp3nmja727fyH1p4ytpXQbRX0do28KpY5XQ9yF9gAWxG2zHxtBGiZVG4iXRlpZlfmFUc4kQLUTrInYUoagh9zF6Cxwrz7smRqgOkWKOYiOTiR9ofTkvpmQpdGpDOsxjcPRd3V+jjkfZwT5yzsM7IOAzzhZKKvPWgfJfzaAI43W+dUWiMWzM5KX7Wliu5J3uEXtldo0lW2WpOiuFK5GIPBh5482IbSAtA14YEplOYVCrCI9UG356TImFYyCLuNKAxbzA3U9TSJCk0tUbssWKq8aIaoEbtvE43P1G8x5teVFR5Wu1nZypgKgYFSS5tRL+niDV82yaCOTIGjdA6yFHq8gRlv9xkmZjyPIB4LfAhAIEz9X0+hNvZBfTofi1UXRh/54GIqr1YIXmJVYKb/srAU5Qne5lQdvFauFIQbnlESVUeb1RA=="
}
]
},
"finishReason": 1,
"avgLogprobs": -0.05223702887694041
"avgLogprobs": -1.18875130740079
}
],
"usageMetadata": {
"promptTokenCount": 126,
"candidatesTokenCount": 24,
"totalTokenCount": 150,
"promptTokenCount": 128,
"candidatesTokenCount": 22,
"totalTokenCount": 575,
"trafficType": 1,
"promptTokensDetails": [
{
"modality": 1,
"tokenCount": 126
"tokenCount": 128
}
],
"candidatesTokensDetails": [
{
"modality": 1,
"tokenCount": 24
"tokenCount": 22
}
]
],
"thoughtsTokenCount": 425
},
"modelVersion": "gemini-1.5-flash-002",
"createTime": "2025-02-06T04:26:33.660225Z",
"responseId": "-TmkZ4GmKIH12PgPof-uQQ"
"modelVersion": "gemini-2.5-pro",
"createTime": "2025-08-19T14:55:39.882212Z",
"responseId": "a5CkaKTsNa3hgLUP59n1oQo"
}
headers:
Content-Type:
@ -141,7 +144,7 @@ interactions:
- X-Origin
- Referer
content-length:
- '789'
- '3277'
status:
code: 200
message: OK

138
instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/conftest.py
View File

@ -1,18 +1,39 @@
"""Unit tests configuration module."""
import asyncio
import json
import os
import re
from typing import Any, Mapping, MutableMapping
from typing import (
Any,
Callable,
Generator,
Mapping,
MutableMapping,
Protocol,
TypeVar,
)
import pytest
import vertexai
import yaml
from google.auth.aio.credentials import (
AnonymousCredentials as AsyncAnonymousCredentials,
)
from google.auth.credentials import AnonymousCredentials
from google.cloud.aiplatform.initializer import _set_async_rest_credentials
from typing_extensions import Concatenate, ParamSpec
from vcr import VCR
from vcr.record_mode import RecordMode
from vcr.request import Request
from vertexai.generative_models import (
GenerativeModel,
)
from opentelemetry.instrumentation._semconv import (
OTEL_SEMCONV_STABILITY_OPT_IN,
_OpenTelemetrySemanticConventionStability,
)
from opentelemetry.instrumentation.vertexai import VertexAIInstrumentor
from opentelemetry.instrumentation.vertexai.utils import (
OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT,
@ -94,10 +115,13 @@ def vertexai_init(vcr: VCR) -> None:
)
@pytest.fixture
@pytest.fixture(scope="function")
def instrument_no_content(
tracer_provider, event_logger_provider, meter_provider
tracer_provider, event_logger_provider, meter_provider, request
):
# Reset global state..
_OpenTelemetrySemanticConventionStability._initialized = False
os.environ.update({OTEL_SEMCONV_STABILITY_OPT_IN: "stable"})
os.environ.update(
{OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "False"}
)
@ -115,10 +139,64 @@ def instrument_no_content(
instrumentor.uninstrument()
@pytest.fixture
def instrument_with_content(
@pytest.fixture(scope="function")
def instrument_no_content_with_experimental_semconvs(
tracer_provider, event_logger_provider, meter_provider, request
):
# Reset global state..
_OpenTelemetrySemanticConventionStability._initialized = False
os.environ.update(
{OTEL_SEMCONV_STABILITY_OPT_IN: "gen_ai_latest_experimental"}
)
os.environ.update(
{OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "NO_CONTENT"}
)
instrumentor = VertexAIInstrumentor()
instrumentor.instrument(
tracer_provider=tracer_provider,
event_logger_provider=event_logger_provider,
meter_provider=meter_provider,
)
yield instrumentor
os.environ.pop(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, None)
if instrumentor.is_instrumented_by_opentelemetry:
instrumentor.uninstrument()
@pytest.fixture(scope="function")
def instrument_with_experimental_semconvs(
tracer_provider, event_logger_provider, meter_provider
):
# Reset global state..
_OpenTelemetrySemanticConventionStability._initialized = False
os.environ.update(
{OTEL_SEMCONV_STABILITY_OPT_IN: "gen_ai_latest_experimental"}
)
os.environ.update(
{OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "SPAN_AND_EVENT"}
)
instrumentor = VertexAIInstrumentor()
instrumentor.instrument(
tracer_provider=tracer_provider,
event_logger_provider=event_logger_provider,
meter_provider=meter_provider,
)
yield instrumentor
os.environ.pop(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, None)
if instrumentor.is_instrumented_by_opentelemetry:
instrumentor.uninstrument()
@pytest.fixture(scope="function")
def instrument_with_content(
tracer_provider, event_logger_provider, meter_provider, request
):
# Reset global state..
_OpenTelemetrySemanticConventionStability._initialized = False
os.environ.update({OTEL_SEMCONV_STABILITY_OPT_IN: "stable"})
os.environ.update(
{OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "True"}
)
@ -239,3 +317,53 @@ class PrettyPrintJSONBody:
def fixture_vcr(vcr):
vcr.register_serializer("yaml", PrettyPrintJSONBody)
return vcr
_P = ParamSpec("_P")
_R = TypeVar("_R")
def _copy_signature(
func_type: Callable[_P, _R],
) -> Callable[
[Callable[..., Any]], Callable[Concatenate[GenerativeModel, _P], _R]
]:
return lambda func: func
# Type annotation for fixture to make LSP work properly
class GenerateContentFixture(Protocol):
@_copy_signature(GenerativeModel.generate_content)
def __call__(self): ...
@pytest.fixture(
name="generate_content",
params=(
pytest.param(False, id="sync"),
pytest.param(True, id="async"),
),
)
def fixture_generate_content(
request: pytest.FixtureRequest,
vcr: VCR,
) -> Generator[GenerateContentFixture, None, None]:
"""This fixture parameterizes tests that use it to test calling both
GenerativeModel.generate_content() and GenerativeModel.generate_content_async().
"""
is_async: bool = request.param
if is_async:
# See
# https://github.com/googleapis/python-aiplatform/blob/cb0e5fedbf45cb0531c0b8611fb7fabdd1f57e56/google/cloud/aiplatform/initializer.py#L717-L729
_set_async_rest_credentials(credentials=AsyncAnonymousCredentials())
def wrapper(model: GenerativeModel, *args, **kwargs) -> None:
if is_async:
return asyncio.run(model.generate_content_async(*args, **kwargs))
return model.generate_content(*args, **kwargs)
with vcr.use_cassette(
request.node.originalname, allow_playback_repeats=True
):
yield wrapper

1
instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/requirements.latest.txt
View File

@ -90,3 +90,4 @@ zipp==3.20.2
-e opentelemetry-instrumentation
-e instrumentation-genai/opentelemetry-instrumentation-vertexai[instruments]
-e util/opentelemetry-util-genai

11
instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/requirements.oldest.txt
View File

@ -64,11 +64,12 @@ vcrpy==6.0.2
wrapt==1.16.0
yarl==1.15.2
zipp==3.20.2
# when updating, also update in pyproject.toml
opentelemetry-api==1.28
opentelemetry-sdk==1.28
opentelemetry-semantic-conventions==0.49b0
opentelemetry-instrumentation==0.49b0
opentelemetry-api==1.37
opentelemetry-sdk==1.37
opentelemetry-semantic-conventions==0.58b0
opentelemetry-instrumentation==0.58b0
-e instrumentation-genai/opentelemetry-instrumentation-vertexai[instruments]
-e util/opentelemetry-util-genai

122
instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/shared_test_utils.py Normal file
View File

@ -0,0 +1,122 @@
# Copyright The OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from vertexai.generative_models import (
Content,
FunctionDeclaration,
GenerativeModel,
Part,
Tool,
)
def weather_tool() -> Tool:
# Adapted from https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/function-calling#parallel-samples
get_current_weather_func = FunctionDeclaration(
name="get_current_weather",
description="Get the current weather in a given location",
parameters={
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The location for which to get the weather. "
"It can be a city name, a city name and state, or a zip code. "
"Examples: 'San Francisco', 'San Francisco, CA', '95616', etc.",
},
},
},
)
return Tool(
function_declarations=[get_current_weather_func],
)
def ask_about_weather(generate_content: callable) -> None:
model = GenerativeModel("gemini-2.5-pro", tools=[weather_tool()])
# Model will respond asking for function calls
generate_content(
model,
[
# User asked about weather
Content(
role="user",
parts=[
Part.from_text(
"Get weather details in New Delhi and San Francisco?"
),
],
),
],
)
def ask_about_weather_function_response(
generate_content: callable,
) -> None:
model = GenerativeModel("gemini-2.5-pro", tools=[weather_tool()])
generate_content(
model,
[
# User asked about weather
Content(
role="user",
parts=[
Part.from_text(
"Get weather details in New Delhi and San Francisco?"
),
],
),
# Model requests two function calls
Content(
role="model",
parts=[
Part.from_dict(
{
"function_call": {
"name": "get_current_weather",
"args": {"location": "New Delhi"},
}
},
),
Part.from_dict(
{
"function_call": {
"name": "get_current_weather",
"args": {"location": "San Francisco"},
}
},
),
],
),
# User responds with function responses
Content(
role="user",
parts=[
Part.from_function_response(
name="get_current_weather",
response={
"content": '{"temperature": 35, "unit": "C"}'
},
),
Part.from_function_response(
name="get_current_weather",
response={
"content": '{"temperature": 25, "unit": "C"}'
},
),
],
),
],
)

173
instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions.py
View File

@ -1,22 +1,7 @@
from __future__ import annotations
import asyncio
from typing import (
Any,
Callable,
Generator,
Protocol,
TypeVar,
)
import pytest
from google.api_core.exceptions import BadRequest, NotFound
from google.auth.aio.credentials import (
AnonymousCredentials as AsyncAnonymousCredentials,
)
from google.cloud.aiplatform.initializer import _set_async_rest_credentials
from typing_extensions import Concatenate, ParamSpec
from vcr import VCR
from vertexai.generative_models import (
Content,
GenerationConfig,
@ -38,39 +23,45 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
from opentelemetry.trace import StatusCode
@pytest.mark.vcr
@pytest.mark.vcr()
def test_generate_content(
span_exporter: InMemorySpanExporter,
log_exporter: InMemoryLogExporter,
generate_content: GenerateContentFixture,
generate_content: callable,
instrument_with_content: VertexAIInstrumentor,
):
model = GenerativeModel("gemini-1.5-flash-002")
model = GenerativeModel("gemini-2.5-pro")
generate_content(
model,
[
Content(role="user", parts=[Part.from_text("Say this is a test")]),
Content(
role="user",
parts=[
Part.from_text("Say this is a test"),
],
),
],
)
# Emits span
spans = span_exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].name == "chat gemini-1.5-flash-002"
assert spans[0].name == "chat gemini-2.5-pro"
assert dict(spans[0].attributes) == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-1.5-flash-002",
"gen_ai.request.model": "gemini-2.5-pro",
"gen_ai.response.finish_reasons": ("stop",),
"gen_ai.response.model": "gemini-1.5-flash-002",
"gen_ai.response.model": "gemini-2.5-pro",
"gen_ai.system": "vertex_ai",
"gen_ai.usage.input_tokens": 5,
"gen_ai.usage.output_tokens": 19,
"gen_ai.usage.output_tokens": 5,
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
}
# Emits user and choice events
logs = log_exporter.get_finished_logs()
# Emits user and choice events
assert len(logs) == 2
user_log, choice_log = [log_data.log_record for log_data in logs]
@ -98,24 +89,20 @@ def test_generate_content(
"finish_reason": "stop",
"index": 0,
"message": {
"content": [
{
"text": "Okay, I understand. I'm ready for your test. Please proceed.\n"
}
],
"content": [{"text": "This is a test."}],
"role": "model",
},
}
@pytest.mark.vcr
@pytest.mark.vcr()
def test_generate_content_without_events(
span_exporter: InMemorySpanExporter,
log_exporter: InMemoryLogExporter,
generate_content: GenerateContentFixture,
generate_content: callable,
instrument_no_content: VertexAIInstrumentor,
):
model = GenerativeModel("gemini-1.5-flash-002")
model = GenerativeModel("gemini-2.5-pro")
generate_content(
model,
[
@ -126,21 +113,21 @@ def test_generate_content_without_events(
# Emits span
spans = span_exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].name == "chat gemini-1.5-flash-002"
assert spans[0].name == "chat gemini-2.5-pro"
assert dict(spans[0].attributes) == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-1.5-flash-002",
"gen_ai.request.model": "gemini-2.5-pro",
"gen_ai.response.finish_reasons": ("stop",),
"gen_ai.response.model": "gemini-1.5-flash-002",
"gen_ai.response.model": "gemini-2.5-pro",
"gen_ai.system": "vertex_ai",
"gen_ai.usage.input_tokens": 5,
"gen_ai.usage.output_tokens": 19,
"gen_ai.usage.output_tokens": 5,
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
}
# Emits user and choice event without body.content
logs = log_exporter.get_finished_logs()
# Emits user and choice event without body.content
assert len(logs) == 2
user_log, choice_log = [log_data.log_record for log_data in logs]
assert user_log.attributes == {
@ -160,10 +147,10 @@ def test_generate_content_without_events(
}
@pytest.mark.vcr
@pytest.mark.vcr()
def test_generate_content_empty_model(
span_exporter: InMemorySpanExporter,
generate_content: GenerateContentFixture,
generate_content: callable,
instrument_with_content: VertexAIInstrumentor,
):
model = GenerativeModel("")
@ -193,10 +180,10 @@ def test_generate_content_empty_model(
assert_span_error(spans[0])
@pytest.mark.vcr
@pytest.mark.vcr()
def test_generate_content_missing_model(
span_exporter: InMemorySpanExporter,
generate_content: GenerateContentFixture,
generate_content: callable,
instrument_with_content: VertexAIInstrumentor,
):
model = GenerativeModel("gemini-does-not-exist")
@ -226,13 +213,13 @@ def test_generate_content_missing_model(
assert_span_error(spans[0])
@pytest.mark.vcr
@pytest.mark.vcr()
def test_generate_content_invalid_temperature(
span_exporter: InMemorySpanExporter,
generate_content: GenerateContentFixture,
generate_content: callable,
instrument_with_content: VertexAIInstrumentor,
):
model = GenerativeModel("gemini-1.5-flash-002")
model = GenerativeModel("gemini-2.5-pro")
try:
# Temperature out of range causes error
generate_content(
@ -249,10 +236,10 @@ def test_generate_content_invalid_temperature(
spans = span_exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].name == "chat gemini-1.5-flash-002"
assert spans[0].name == "chat gemini-2.5-pro"
assert dict(spans[0].attributes) == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-1.5-flash-002",
"gen_ai.request.model": "gemini-2.5-pro",
"gen_ai.request.temperature": 1000.0,
"gen_ai.system": "vertex_ai",
"server.address": "us-central1-aiplatform.googleapis.com",
@ -261,13 +248,13 @@ def test_generate_content_invalid_temperature(
assert_span_error(spans[0])
@pytest.mark.vcr
@pytest.mark.vcr()
def test_generate_content_invalid_role(
log_exporter: InMemoryLogExporter,
generate_content: GenerateContentFixture,
generate_content: callable,
instrument_with_content: VertexAIInstrumentor,
):
model = GenerativeModel("gemini-1.5-flash-002")
model = GenerativeModel("gemini-2.5-pro")
try:
# Fails because role must be "user" or "model"
generate_content(
@ -285,11 +272,12 @@ def test_generate_content_invalid_role(
# Emits the faulty content which caused the request to fail
logs = log_exporter.get_finished_logs()
assert len(logs) == 1
assert logs[0].log_record.attributes == {
log = logs[0].log_record
assert log.attributes == {
"gen_ai.system": "vertex_ai",
"event.name": "gen_ai.user.message",
}
assert logs[0].log_record.body == {
assert log.body == {
"content": [{"text": "Say this is a test"}],
"role": "invalid_role",
}
@ -299,7 +287,7 @@ def test_generate_content_invalid_role(
def test_generate_content_extra_params(
span_exporter,
instrument_no_content,
generate_content: GenerateContentFixture,
generate_content: callable,
):
generation_config = GenerationConfig(
top_k=2,
@ -311,7 +299,7 @@ def test_generate_content_extra_params(
frequency_penalty=1.0,
seed=12345,
)
model = GenerativeModel("gemini-1.5-flash-002")
model = GenerativeModel("gemini-2.5-pro")
generate_content(
model,
[
@ -326,16 +314,16 @@ def test_generate_content_extra_params(
"gen_ai.operation.name": "chat",
"gen_ai.request.frequency_penalty": 1.0,
"gen_ai.request.max_tokens": 5,
"gen_ai.request.model": "gemini-1.5-flash-002",
"gen_ai.request.model": "gemini-2.5-pro",
"gen_ai.request.presence_penalty": -1.5,
"gen_ai.request.stop_sequences": ("\n\n\n",),
"gen_ai.request.temperature": 0.20000000298023224,
"gen_ai.request.top_p": 0.949999988079071,
"gen_ai.response.finish_reasons": ("length",),
"gen_ai.response.model": "gemini-1.5-flash-002",
"gen_ai.response.model": "gemini-2.5-pro",
"gen_ai.system": "vertex_ai",
"gen_ai.usage.input_tokens": 5,
"gen_ai.usage.output_tokens": 5,
"gen_ai.usage.output_tokens": 0,
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
}
@ -353,46 +341,46 @@ def assert_span_error(span: ReadableSpan) -> None:
assert error_events != []
@pytest.mark.vcr
@pytest.mark.vcr()
def test_generate_content_all_events(
log_exporter: InMemoryLogExporter,
generate_content: GenerateContentFixture,
generate_content: callable,
instrument_with_content: VertexAIInstrumentor,
):
generate_content_all_input_events(
GenerativeModel(
"gemini-1.5-flash-002",
"gemini-2.5-pro",
system_instruction=Part.from_text(
"You are a clever language model"
),
),
generate_content,
log_exporter,
instrument_with_content,
)
@pytest.mark.vcr
@pytest.mark.vcr()
def test_preview_generate_content_all_input_events(
log_exporter: InMemoryLogExporter,
generate_content: GenerateContentFixture,
generate_content: callable,
instrument_with_content: VertexAIInstrumentor,
):
generate_content_all_input_events(
PreviewGenerativeModel(
"gemini-1.5-flash-002",
"gemini-2.5-pro",
system_instruction=Part.from_text(
"You are a clever language model"
),
),
generate_content,
log_exporter,
instrument_with_content,
)
def generate_content_all_input_events(
model: GenerativeModel | PreviewGenerativeModel,
generate_content: GenerateContentFixture,
log_exporter: InMemoryLogExporter,
instrument_with_content: VertexAIInstrumentor,
):
model.generate_content(
[
@ -409,6 +397,9 @@ def generate_content_all_input_events(
],
),
],
generation_config=GenerationConfig(
seed=12345, response_mime_type="text/plain"
),
)
# Emits a system event, 2 users events, an assistant event, and the choice (response) event
@ -463,57 +454,7 @@ def generate_content_all_input_events(
"finish_reason": "stop",
"index": 0,
"message": {
"content": [{"text": "OpenTelemetry, this is a test.\n"}],
"content": [{"text": "OpenTelemetry, this is a test."}],
"role": "model",
},
}
_P = ParamSpec("_P")
_R = TypeVar("_R")
def _copy_signature(
func_type: Callable[_P, _R],
) -> Callable[
[Callable[..., Any]], Callable[Concatenate[GenerativeModel, _P], _R]
]:
return lambda func: func
# Type annotation for fixture to make LSP work properly
class GenerateContentFixture(Protocol):
@_copy_signature(GenerativeModel.generate_content)
def __call__(self): ...
@pytest.fixture(
name="generate_content",
params=(
pytest.param(False, id="sync"),
pytest.param(True, id="async"),
),
)
def fixture_generate_content(
request: pytest.FixtureRequest,
vcr: VCR,
) -> Generator[GenerateContentFixture, None, None]:
"""This fixture parameterizes tests that use it to test calling both
GenerativeModel.generate_content() and GenerativeModel.generate_content_async().
"""
is_async: bool = request.param
if is_async:
# See
# https://github.com/googleapis/python-aiplatform/blob/cb0e5fedbf45cb0531c0b8611fb7fabdd1f57e56/google/cloud/aiplatform/initializer.py#L717-L729
_set_async_rest_credentials(credentials=AsyncAnonymousCredentials())
def wrapper(model: GenerativeModel, *args, **kwargs) -> None:
if is_async:
return asyncio.run(model.generate_content_async(*args, **kwargs))
return model.generate_content(*args, **kwargs)
with vcr.use_cassette(
request.node.originalname, allow_playback_repeats=True
):
yield wrapper

464
instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_chat_completions_experimental.py Normal file
View File

@ -0,0 +1,464 @@
from __future__ import annotations
import pytest
from google.api_core.exceptions import BadRequest, NotFound
from vertexai.generative_models import (
Content,
GenerationConfig,
GenerativeModel,
Part,
)
from vertexai.preview.generative_models import (
GenerativeModel as PreviewGenerativeModel,
)
from opentelemetry.instrumentation.vertexai import VertexAIInstrumentor
from opentelemetry.sdk._logs._internal.export.in_memory_log_exporter import (
InMemoryLogExporter,
)
from opentelemetry.sdk.trace import ReadableSpan
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
InMemorySpanExporter,
)
from opentelemetry.trace import StatusCode
@pytest.mark.vcr()
def test_generate_content(
span_exporter: InMemorySpanExporter,
log_exporter: InMemoryLogExporter,
generate_content: callable,
instrument_with_experimental_semconvs: VertexAIInstrumentor,
):
model = GenerativeModel("gemini-2.5-pro")
generate_content(
model,
[
Content(
role="user",
parts=[
Part.from_text("Say this is a test"),
],
),
],
)
# Emits span
spans = span_exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].name == "chat gemini-2.5-pro"
assert dict(spans[0].attributes) == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-2.5-pro",
"gen_ai.response.finish_reasons": ("stop",),
"gen_ai.response.model": "gemini-2.5-pro",
"gen_ai.system": "vertex_ai",
"gen_ai.usage.input_tokens": 5,
"gen_ai.usage.output_tokens": 5,
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
}
logs = log_exporter.get_finished_logs()
assert len(logs) == 1
log = logs[0].log_record
assert log.attributes == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-2.5-pro",
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
"gen_ai.response.model": "gemini-2.5-pro",
"gen_ai.response.finish_reasons": ("stop",),
"gen_ai.usage.input_tokens": 5,
"gen_ai.usage.output_tokens": 5,
"gen_ai.input.messages": (
{
"role": "user",
"parts": ({"type": "text", "content": "Say this is a test"},),
},
),
"gen_ai.output.messages": (
{
"role": "model",
"parts": ({"type": "text", "content": "This is a test."},),
"finish_reason": "stop",
},
),
}
@pytest.mark.vcr()
def test_generate_content_without_events(
span_exporter: InMemorySpanExporter,
log_exporter: InMemoryLogExporter,
generate_content: callable,
instrument_with_experimental_semconvs: VertexAIInstrumentor,
):
model = GenerativeModel("gemini-2.5-pro")
generate_content(
model,
[
Content(role="user", parts=[Part.from_text("Say this is a test")]),
],
)
# Emits span
spans = span_exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].name == "chat gemini-2.5-pro"
assert dict(spans[0].attributes) == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-2.5-pro",
"gen_ai.response.finish_reasons": ("stop",),
"gen_ai.response.model": "gemini-2.5-pro",
"gen_ai.system": "vertex_ai",
"gen_ai.usage.input_tokens": 5,
"gen_ai.usage.output_tokens": 5,
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
}
logs = log_exporter.get_finished_logs()
assert len(logs) == 1
log = logs[0].log_record
print(log.attributes)
assert log.attributes == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-2.5-pro",
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
"gen_ai.response.model": "gemini-2.5-pro",
"gen_ai.response.finish_reasons": ("stop",),
"gen_ai.usage.input_tokens": 5,
"gen_ai.usage.output_tokens": 5,
"gen_ai.input.messages": (
{
"role": "user",
"parts": ({"content": "Say this is a test", "type": "text"},),
},
),
"gen_ai.output.messages": (
{
"role": "model",
"parts": ({"content": "This is a test.", "type": "text"},),
"finish_reason": "stop",
},
),
}
@pytest.mark.vcr()
def test_generate_content_empty_model(
span_exporter: InMemorySpanExporter,
generate_content: callable,
instrument_with_experimental_semconvs: VertexAIInstrumentor,
):
model = GenerativeModel("")
try:
generate_content(
model,
[
Content(
role="user", parts=[Part.from_text("Say this is a test")]
)
],
)
except ValueError:
pass
spans = span_exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].name == "chat"
# Captures invalid params
assert dict(spans[0].attributes) == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "",
"gen_ai.system": "vertex_ai",
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
}
assert_span_error(spans[0])
@pytest.mark.vcr()
def test_generate_content_missing_model(
span_exporter: InMemorySpanExporter,
generate_content: callable,
instrument_with_experimental_semconvs: VertexAIInstrumentor,
):
model = GenerativeModel("gemini-does-not-exist")
try:
generate_content(
model,
[
Content(
role="user", parts=[Part.from_text("Say this is a test")]
)
],
)
except NotFound:
pass
spans = span_exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].name == "chat gemini-does-not-exist"
# Captures invalid params
assert dict(spans[0].attributes) == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-does-not-exist",
"gen_ai.system": "vertex_ai",
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
}
assert_span_error(spans[0])
@pytest.mark.vcr()
def test_generate_content_invalid_temperature(
span_exporter: InMemorySpanExporter,
generate_content: callable,
instrument_with_experimental_semconvs: VertexAIInstrumentor,
):
model = GenerativeModel("gemini-2.5-pro")
try:
# Temperature out of range causes error
generate_content(
model,
[
Content(
role="user", parts=[Part.from_text("Say this is a test")]
)
],
generation_config=GenerationConfig(temperature=1000),
)
except BadRequest:
pass
spans = span_exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].name == "chat gemini-2.5-pro"
assert dict(spans[0].attributes) == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-2.5-pro",
"gen_ai.request.temperature": 1000.0,
"gen_ai.system": "vertex_ai",
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
}
assert_span_error(spans[0])
@pytest.mark.vcr()
def test_generate_content_invalid_role(
log_exporter: InMemoryLogExporter,
generate_content: callable,
instrument_with_experimental_semconvs: VertexAIInstrumentor,
):
model = GenerativeModel("gemini-2.5-pro")
try:
# Fails because role must be "user" or "model"
generate_content(
model,
[
Content(
role="invalid_role",
parts=[Part.from_text("Say this is a test")],
)
],
)
except BadRequest:
pass
# Emits the faulty content which caused the request to fail
logs = log_exporter.get_finished_logs()
assert len(logs) == 1
log = logs[0].log_record
assert log.attributes == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-2.5-pro",
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
"gen_ai.input.messages": (
{
"role": "invalid_role",
"parts": ({"type": "text", "content": "Say this is a test"},),
},
),
}
@pytest.mark.vcr()
def test_generate_content_extra_params(
span_exporter,
instrument_no_content_with_experimental_semconvs,
generate_content: callable,
):
generation_config = GenerationConfig(
top_k=2,
top_p=0.95,
temperature=0.2,
stop_sequences=["\n\n\n"],
max_output_tokens=5,
presence_penalty=-1.5,
frequency_penalty=1.0,
seed=12345,
)
model = GenerativeModel("gemini-2.5-pro")
generate_content(
model,
[
Content(role="user", parts=[Part.from_text("Say this is a test")]),
],
generation_config=generation_config,
)
spans = span_exporter.get_finished_spans()
assert len(spans) == 1
assert dict(spans[0].attributes) == {
"gen_ai.operation.name": "chat",
"gen_ai.request.frequency_penalty": 1.0,
"gen_ai.request.max_tokens": 5,
"gen_ai.request.model": "gemini-2.5-pro",
"gen_ai.request.presence_penalty": -1.5,
"gen_ai.request.stop_sequences": ("\n\n\n",),
"gen_ai.request.temperature": 0.20000000298023224,
"gen_ai.request.top_p": 0.949999988079071,
"gen_ai.response.finish_reasons": ("length",),
"gen_ai.response.model": "gemini-2.5-pro",
"gen_ai.system": "vertex_ai",
"gen_ai.usage.input_tokens": 5,
"gen_ai.usage.output_tokens": 0,
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
}
def assert_span_error(span: ReadableSpan) -> None:
# Sets error status
assert span.status.status_code == StatusCode.ERROR
# TODO: check thate error.type is set
# https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-spans.md
# Records exception event
error_events = [e for e in span.events if e.name == "exception"]
assert error_events != []
@pytest.mark.vcr()
def test_generate_content_all_events(
log_exporter: InMemoryLogExporter,
generate_content: callable,
instrument_with_experimental_semconvs: VertexAIInstrumentor,
):
generate_content_all_input_events(
GenerativeModel(
"gemini-2.5-pro",
system_instruction=Part.from_text(
"You are a clever language model"
),
),
log_exporter,
instrument_with_experimental_semconvs,
)
@pytest.mark.vcr()
def test_preview_generate_content_all_input_events(
log_exporter: InMemoryLogExporter,
generate_content: callable,
instrument_with_experimental_semconvs: VertexAIInstrumentor,
):
generate_content_all_input_events(
PreviewGenerativeModel(
"gemini-2.5-pro",
system_instruction=Part.from_text(
"You are a clever language model"
),
),
log_exporter,
instrument_with_experimental_semconvs,
)
def generate_content_all_input_events(
model: GenerativeModel | PreviewGenerativeModel,
log_exporter: InMemoryLogExporter,
instrument_with_experimental_semconvs: VertexAIInstrumentor,
):
model.generate_content(
[
Content(
role="user", parts=[Part.from_text("My name is OpenTelemetry")]
),
Content(
role="model", parts=[Part.from_text("Hello OpenTelemetry!")]
),
Content(
role="user",
parts=[
Part.from_text("Address me by name and say this is a test")
],
),
],
generation_config=GenerationConfig(
seed=12345, response_mime_type="text/plain"
),
)
# Emits a single log.
logs = log_exporter.get_finished_logs()
assert len(logs) == 1
log = logs[0].log_record
assert log.attributes == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-2.5-pro",
"gen_ai.request.seed": 12345,
"gen_ai.output.type": "text",
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
"gen_ai.response.model": "gemini-2.5-pro",
"gen_ai.response.finish_reasons": ("stop",),
"gen_ai.usage.input_tokens": 25,
"gen_ai.usage.output_tokens": 8,
"gen_ai.system_instructions": (
{"type": "text", "content": "You are a clever language model"},
),
"gen_ai.input.messages": (
{
"role": "user",
"parts": (
{
"type": "text",
"content": "My name is OpenTelemetry",
},
),
},
{
"role": "model",
"parts": (
{"type": "text", "content": "Hello OpenTelemetry!"},
),
},
{
"role": "user",
"parts": (
{
"type": "text",
"content": "Address me by name and say this is a test",
},
),
},
),
"gen_ai.output.messages": (
{
"role": "model",
"parts": (
{
"type": "text",
"content": "OpenTelemetry, this is a test.",
},
),
"finish_reason": "stop",
},
),
}

174
instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling.py
View File

@ -1,11 +1,4 @@
import pytest
from vertexai.generative_models import (
Content,
FunctionDeclaration,
GenerativeModel,
Part,
Tool,
)
from opentelemetry.instrumentation.vertexai import VertexAIInstrumentor
from opentelemetry.sdk._logs._internal.export.in_memory_log_exporter import (
@ -14,27 +7,32 @@ from opentelemetry.sdk._logs._internal.export.in_memory_log_exporter import (
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
InMemorySpanExporter,
)
from tests.shared_test_utils import (
ask_about_weather,
ask_about_weather_function_response,
)
@pytest.mark.vcr
@pytest.mark.vcr()
def test_function_call_choice(
span_exporter: InMemorySpanExporter,
log_exporter: InMemoryLogExporter,
instrument_with_content: VertexAIInstrumentor,
generate_content: callable,
):
ask_about_weather()
ask_about_weather(generate_content)
# Emits span
spans = span_exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].name == "chat gemini-1.5-flash-002"
assert spans[0].name == "chat gemini-2.5-pro"
assert dict(spans[0].attributes) == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-1.5-flash-002",
"gen_ai.request.model": "gemini-2.5-pro",
"gen_ai.response.finish_reasons": ("stop",),
"gen_ai.response.model": "gemini-1.5-flash-002",
"gen_ai.response.model": "gemini-2.5-pro",
"gen_ai.system": "vertex_ai",
"gen_ai.usage.input_tokens": 72,
"gen_ai.usage.input_tokens": 74,
"gen_ai.usage.output_tokens": 16,
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
@ -100,12 +98,13 @@ def test_function_call_choice(
}
@pytest.mark.vcr
@pytest.mark.vcr()
def test_function_call_choice_no_content(
log_exporter: InMemoryLogExporter,
instrument_no_content: VertexAIInstrumentor,
generate_content: callable,
):
ask_about_weather()
ask_about_weather(generate_content)
# Emits user and choice events
logs = log_exporter.get_finished_logs()
@ -142,32 +141,32 @@ def test_function_call_choice_no_content(
}
@pytest.mark.vcr
@pytest.mark.vcr()
def test_tool_events(
span_exporter: InMemorySpanExporter,
log_exporter: InMemoryLogExporter,
instrument_with_content: VertexAIInstrumentor,
generate_content: callable,
):
ask_about_weather_function_response()
ask_about_weather_function_response(generate_content)
# Emits span
spans = span_exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].name == "chat gemini-1.5-flash-002"
assert spans[0].name == "chat gemini-2.5-pro"
assert dict(spans[0].attributes) == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-1.5-flash-002",
"gen_ai.request.model": "gemini-2.5-pro",
"gen_ai.response.finish_reasons": ("stop",),
"gen_ai.response.model": "gemini-1.5-flash-002",
"gen_ai.response.model": "gemini-2.5-pro",
"gen_ai.system": "vertex_ai",
"gen_ai.usage.input_tokens": 126,
"gen_ai.usage.output_tokens": 24,
"gen_ai.usage.input_tokens": 128,
"gen_ai.usage.output_tokens": 26,
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
}
# Emits user, assistant, two tool, and choice events
logs = log_exporter.get_finished_logs()
# Emits user, assistant, two tool, and choice events
assert len(logs) == 5
user_log, assistant_log, tool_log1, tool_log2, choice_log = [
log_data.log_record for log_data in logs
@ -236,7 +235,7 @@ def test_tool_events(
"message": {
"content": [
{
"text": "The current temperature in New Delhi is 35 degrees Celsius and in San Francisco is 25 degrees Celsius.\n"
"text": "The current temperature in New Delhi is 35°C, and in San Francisco, it is 25°C."
}
],
"role": "model",
@ -244,32 +243,32 @@ def test_tool_events(
}
@pytest.mark.vcr
@pytest.mark.vcr()
def test_tool_events_no_content(
span_exporter: InMemorySpanExporter,
log_exporter: InMemoryLogExporter,
instrument_no_content: VertexAIInstrumentor,
generate_content: callable,
):
ask_about_weather_function_response()
ask_about_weather_function_response(generate_content)
# Emits span
spans = span_exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].name == "chat gemini-1.5-flash-002"
assert spans[0].name == "chat gemini-2.5-pro"
assert dict(spans[0].attributes) == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-1.5-flash-002",
"gen_ai.request.model": "gemini-2.5-pro",
"gen_ai.response.finish_reasons": ("stop",),
"gen_ai.response.model": "gemini-1.5-flash-002",
"gen_ai.response.model": "gemini-2.5-pro",
"gen_ai.system": "vertex_ai",
"gen_ai.usage.input_tokens": 126,
"gen_ai.usage.output_tokens": 24,
"gen_ai.usage.input_tokens": 128,
"gen_ai.usage.output_tokens": 22,
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
}
# Emits user, assistant, two tool, and choice events
logs = log_exporter.get_finished_logs()
# Emits user, assistant, two tool, and choice events
assert len(logs) == 5
user_log, assistant_log, tool_log1, tool_log2, choice_log = [
log_data.log_record for log_data in logs
@ -290,13 +289,19 @@ def test_tool_events_no_content(
"gen_ai.system": "vertex_ai",
"event.name": "gen_ai.tool.message",
}
assert tool_log1.body == {"role": "user", "id": "get_current_weather_0"}
assert tool_log1.body == {
"role": "user",
"id": "get_current_weather_0",
}
assert tool_log2.attributes == {
"gen_ai.system": "vertex_ai",
"event.name": "gen_ai.tool.message",
}
assert tool_log2.body == {"role": "user", "id": "get_current_weather_1"}
assert tool_log2.body == {
"role": "user",
"id": "get_current_weather_1",
}
assert choice_log.attributes == {
"gen_ai.system": "vertex_ai",
@ -307,100 +312,3 @@ def test_tool_events_no_content(
"index": 0,
"message": {"role": "model"},
}
def weather_tool() -> Tool:
# Adapted from https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/function-calling#parallel-samples
get_current_weather_func = FunctionDeclaration(
name="get_current_weather",
description="Get the current weather in a given location",
parameters={
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The location for which to get the weather. "
"It can be a city name, a city name and state, or a zip code. "
"Examples: 'San Francisco', 'San Francisco, CA', '95616', etc.",
},
},
},
)
return Tool(
function_declarations=[get_current_weather_func],
)
def ask_about_weather() -> None:
model = GenerativeModel("gemini-1.5-flash-002", tools=[weather_tool()])
# Model will respond asking for function calls
model.generate_content(
[
# User asked about weather
Content(
role="user",
parts=[
Part.from_text(
"Get weather details in New Delhi and San Francisco?"
),
],
),
],
)
def ask_about_weather_function_response() -> None:
model = GenerativeModel("gemini-1.5-flash-002", tools=[weather_tool()])
model.generate_content(
[
# User asked about weather
Content(
role="user",
parts=[
Part.from_text(
"Get weather details in New Delhi and San Francisco?"
),
],
),
# Model requests two function calls
Content(
role="model",
parts=[
Part.from_dict(
{
"function_call": {
"name": "get_current_weather",
"args": {"location": "New Delhi"},
}
},
),
Part.from_dict(
{
"function_call": {
"name": "get_current_weather",
"args": {"location": "San Francisco"},
}
},
),
],
),
# User responds with function responses
Content(
role="user",
parts=[
Part.from_function_response(
name="get_current_weather",
response={
"content": '{"temperature": 35, "unit": "C"}'
},
),
Part.from_function_response(
name="get_current_weather",
response={
"content": '{"temperature": 25, "unit": "C"}'
},
),
],
),
]
)

246
instrumentation-genai/opentelemetry-instrumentation-vertexai/tests/test_function_calling_experimental.py Normal file
View File

@ -0,0 +1,246 @@
import pytest
from opentelemetry.instrumentation.vertexai import VertexAIInstrumentor
from opentelemetry.sdk._logs._internal.export.in_memory_log_exporter import (
InMemoryLogExporter,
)
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
InMemorySpanExporter,
)
from tests.shared_test_utils import (
ask_about_weather,
ask_about_weather_function_response,
)
@pytest.mark.vcr()
def test_function_call_choice(
span_exporter: InMemorySpanExporter,
log_exporter: InMemoryLogExporter,
instrument_with_experimental_semconvs: VertexAIInstrumentor,
generate_content: callable,
):
ask_about_weather(generate_content)
# Emits span
spans = span_exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].name == "chat gemini-2.5-pro"
assert dict(spans[0].attributes) == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-2.5-pro",
"gen_ai.response.finish_reasons": ("stop",),
"gen_ai.response.model": "gemini-2.5-pro",
"gen_ai.system": "vertex_ai",
"gen_ai.usage.input_tokens": 74,
"gen_ai.usage.output_tokens": 16,
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
}
# Emits user and choice events
logs = log_exporter.get_finished_logs()
assert len(logs) == 1
log = logs[0].log_record
assert log.attributes == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-2.5-pro",
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
"gen_ai.response.model": "gemini-2.5-pro",
"gen_ai.response.finish_reasons": ("stop",),
"gen_ai.usage.input_tokens": 74,
"gen_ai.usage.output_tokens": 16,
"gen_ai.input.messages": (
{
"role": "user",
"parts": (
{
"type": "text",
"content": "Get weather details in New Delhi and San Francisco?",
},
),
},
),
"gen_ai.output.messages": (
{
"role": "model",
"parts": (
{
"type": "tool_call",
"arguments": {"location": "New Delhi"},
"name": "get_current_weather",
"id": "get_current_weather_0",
},
{
"type": "tool_call",
"arguments": {"location": "San Francisco"},
"name": "get_current_weather",
"id": "get_current_weather_1",
},
),
"finish_reason": "stop",
},
),
}
@pytest.mark.vcr()
def test_function_call_choice_no_content(
log_exporter: InMemoryLogExporter,
instrument_no_content_with_experimental_semconvs: VertexAIInstrumentor,
generate_content: callable,
):
ask_about_weather(generate_content)
# Emits user and choice events
logs = log_exporter.get_finished_logs()
assert len(logs) == 1
log = logs[0].log_record
assert log.attributes == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-2.5-pro",
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
"gen_ai.response.model": "gemini-2.5-pro",
"gen_ai.response.finish_reasons": ("stop",),
"gen_ai.usage.input_tokens": 74,
"gen_ai.usage.output_tokens": 16,
}
@pytest.mark.vcr()
def test_tool_events(
span_exporter: InMemorySpanExporter,
log_exporter: InMemoryLogExporter,
instrument_with_experimental_semconvs: VertexAIInstrumentor,
generate_content: callable,
):
ask_about_weather_function_response(generate_content)
# Emits span
spans = span_exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].name == "chat gemini-2.5-pro"
assert dict(spans[0].attributes) == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-2.5-pro",
"gen_ai.response.finish_reasons": ("stop",),
"gen_ai.response.model": "gemini-2.5-pro",
"gen_ai.system": "vertex_ai",
"gen_ai.usage.input_tokens": 128,
"gen_ai.usage.output_tokens": 26,
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
}
logs = log_exporter.get_finished_logs()
assert len(logs) == 1
log = logs[0].log_record
assert log.attributes == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-2.5-pro",
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
"gen_ai.response.model": "gemini-2.5-pro",
"gen_ai.response.finish_reasons": ("stop",),
"gen_ai.usage.input_tokens": 128,
"gen_ai.usage.output_tokens": 26,
"gen_ai.input.messages": (
{
"role": "user",
"parts": (
{
"type": "text",
"content": "Get weather details in New Delhi and San Francisco?",
},
),
},
{
"role": "model",
"parts": (
{
"type": "tool_call",
"arguments": {"location": "New Delhi"},
"name": "get_current_weather",
"id": "get_current_weather_0",
},
{
"type": "tool_call",
"arguments": {"location": "San Francisco"},
"name": "get_current_weather",
"id": "get_current_weather_1",
},
),
},
{
"role": "user",
"parts": (
{
"type": "tool_call_response",
"response": {
"content": '{"temperature": 35, "unit": "C"}'
},
"id": "get_current_weather_0",
},
{
"type": "tool_call_response",
"response": {
"content": '{"temperature": 25, "unit": "C"}'
},
"id": "get_current_weather_1",
},
),
},
),
"gen_ai.output.messages": (
{
"role": "model",
"parts": (
{
"type": "text",
"content": "The current temperature in New Delhi is 35°C, and in San Francisco, it is 25°C.",
},
),
"finish_reason": "stop",
},
),
}
@pytest.mark.vcr()
def test_tool_events_no_content(
span_exporter: InMemorySpanExporter,
log_exporter: InMemoryLogExporter,
instrument_no_content_with_experimental_semconvs: VertexAIInstrumentor,
generate_content: callable,
):
ask_about_weather_function_response(generate_content)
# Emits span
spans = span_exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].name == "chat gemini-2.5-pro"
assert dict(spans[0].attributes) == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-2.5-pro",
"gen_ai.response.finish_reasons": ("stop",),
"gen_ai.response.model": "gemini-2.5-pro",
"gen_ai.system": "vertex_ai",
"gen_ai.usage.input_tokens": 128,
"gen_ai.usage.output_tokens": 22,
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
}
logs = log_exporter.get_finished_logs()
assert len(logs) == 1
log = logs[0].log_record
assert log.attributes == {
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gemini-2.5-pro",
"server.address": "us-central1-aiplatform.googleapis.com",
"server.port": 443,
"gen_ai.response.model": "gemini-2.5-pro",
"gen_ai.response.finish_reasons": ("stop",),
"gen_ai.usage.input_tokens": 128,
"gen_ai.usage.output_tokens": 22,
}

4
pyproject.toml
View File

@ -64,6 +64,7 @@ dependencies = [
"opentelemetry-propagator-ot-trace",
"opentelemetry-propagator-aws-xray",
"opentelemetry-util-http",
"opentelemetry-util-genai",
"opentelemetry-instrumentation-vertexai[instruments]",
"opentelemetry-instrumentation-openai-v2[instruments]",
]
@ -134,6 +135,7 @@ opentelemetry-instrumentation-wsgi = { workspace = true }
opentelemetry-propagator-ot-trace = { workspace = true }
opentelemetry-propagator-aws-xray = { workspace = true }
opentelemetry-util-http = { workspace = true }
opentelemetry-util-genai = { workspace = true }
opentelemetry-instrumentation-vertexai = { workspace = true }
opentelemetry-instrumentation-openai-v2 = { workspace = true }
@ -145,7 +147,7 @@ members = [
"exporter/*",
"opentelemetry-instrumentation",
"propagator/*",
"util/opentelemetry-util-http",
"util/*",
]
[tool.ruff]

2
tox.ini
View File

@ -728,6 +728,7 @@ deps =
; FIXME: add coverage testing
allowlist_externals =
sh
pytest
setenv =
; override CORE_REPO_SHA via env variable when testing other branches/commits than main
@ -1023,6 +1024,7 @@ allowlist_externals =
{toxinidir}/scripts/generate_instrumentation_bootstrap.py
{toxinidir}/scripts/generate_instrumentation_readme.py
{toxinidir}/scripts/generate_instrumentation_metapackage.py
pytest
commands =
{toxinidir}/scripts/generate_instrumentation_bootstrap.py

4932
uv.lock
View File

File diff suppressed because it is too large Load Diff