From 8c3cc8539f63f665aaf9c6478a3107399df1e8b9 Mon Sep 17 00:00:00 2001 From: Wiktoria Walczak <98548576+wikaaaaa@users.noreply.github.com> Date: Thu, 19 Feb 2026 21:17:40 +0100 Subject: [PATCH] Align gen_ai.tool_definitions with JSON Schema and add it to completion hook (#4181) * add gen_ai.tool_definitions to completion hook * hash tool defintions * Update CHANGELOG * make tool.defintions optional in on_completion * fix lint errors * add gen_ai.tool_definitions to completion hook * hash tool defintions * fix CompetionHook docstring * fix lint test * add gen_ai.tool_definitions to completion hook * hash tool defintions * Update CHANGELOG * align tool.defintions with JSON schema * fix tests * fix ruff * fix unexpected indent * use local changes from util/opentelemetry-util-genai * fix ruff * add gen_ai.tool_definitions to completion hook * re-add changes lost during merge * put tool.defintions in the telemetry by default and the params behind the flag * fix: Too many local variables * address comments * Add comments to udpate of util-genai version after release --------- Co-authored-by: Aaron Abbott --- .../CHANGELOG.md | 2 + .../pyproject.toml | 2 +- .../google_genai/generate_content.py | 298 +++++++++++++----- .../generate_content/nonstreaming_base.py | 211 ++++++++++--- .../tests/requirements.oldest.txt | 5 +- util/opentelemetry-util-genai/CHANGELOG.md | 1 + .../util/genai/_upload/completion_hook.py | 76 ++++- .../util/genai/completion_hook.py | 3 + .../src/opentelemetry/util/genai/types.py | 20 ++ .../tests/test_upload.py | 74 ++++- 10 files changed, 542 insertions(+), 150 deletions(-) diff --git a/instrumentation-genai/opentelemetry-instrumentation-google-genai/CHANGELOG.md b/instrumentation-genai/opentelemetry-instrumentation-google-genai/CHANGELOG.md index 2643682a2..498dac127 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-google-genai/CHANGELOG.md +++ b/instrumentation-genai/opentelemetry-instrumentation-google-genai/CHANGELOG.md @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased - Fix bug in how tokens are counted when using the streaming `generateContent` method. ([#4152](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4152)). - Add `gen_ai.tool.definitions` attribute to `gen_ai.client.inference.operation.details` log event ([#4142](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4142)). +- Add `gen_ai.tool_definitions` to completion hook ([#4181](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4181)) + ## Version 0.6b0 (2026-01-27) diff --git a/instrumentation-genai/opentelemetry-instrumentation-google-genai/pyproject.toml b/instrumentation-genai/opentelemetry-instrumentation-google-genai/pyproject.toml index 3603cb651..cdbb4b2d6 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-google-genai/pyproject.toml +++ b/instrumentation-genai/opentelemetry-instrumentation-google-genai/pyproject.toml @@ -42,7 +42,7 @@ dependencies = [ "opentelemetry-api ~=1.37", "opentelemetry-instrumentation >=0.58b0, <2", "opentelemetry-semantic-conventions >=0.58b0, <2", - "opentelemetry-util-genai >= 0.2b0, <0.3b0", + "opentelemetry-util-genai >= 0.2b0, <0.3b0", # TODO: update version after release (https://github.com/open-telemetry/opentelemetry-python-contrib/issues/4221) ] [project.optional-dependencies] diff --git a/instrumentation-genai/opentelemetry-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/generate_content.py b/instrumentation-genai/opentelemetry-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/generate_content.py index a7901b3c7..50feffec6 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/generate_content.py +++ b/instrumentation-genai/opentelemetry-instrumentation-google-genai/src/opentelemetry/instrumentation/google_genai/generate_content.py @@ -21,14 +21,7 @@ import logging import os import time import typing -from typing import ( - Any, - AsyncIterator, - Awaitable, - Iterator, - Optional, - Union, -) +from typing import Any, AsyncIterator, Awaitable, Iterator, Optional, Union from google.genai.models import AsyncModels, Models from google.genai.models import t as transformers @@ -65,14 +58,15 @@ from opentelemetry.trace.span import Span from opentelemetry.util.genai.completion_hook import CompletionHook from opentelemetry.util.genai.types import ( ContentCapturingMode, + FunctionToolDefinition, + GenericToolDefinition, InputMessage, MessagePart, OutputMessage, + ToolDefinition, ) from opentelemetry.util.genai.utils import gen_ai_json_dumps -from opentelemetry.util.types import ( - AttributeValue, -) +from opentelemetry.util.types import AttributeValue from .allowlist_util import AllowList from .custom_semconv import GCP_GENAI_OPERATION_CONFIG @@ -208,68 +202,162 @@ def _to_dict(value: object): return json.loads(json.dumps(value)) -def _tool_to_tool_definition(tool: ToolUnionDict) -> MessagePart: +def _model_dump_to_tool_definition(tool: Any) -> ToolDefinition: + model_dump = tool.model_dump(exclude_none=True) + + name = ( + model_dump.get("name") + or getattr(tool, "name", None) + or type(tool).__name__ + ) + description = model_dump.get("description") or getattr( + tool, "description", None + ) + parameters = model_dump.get("parameters") or model_dump.get("inputSchema") + return FunctionToolDefinition( + name=name, + description=description, + parameters=parameters, + ) + + +def _clean_parameters(params: Any) -> Any: + """Converts parameter objects into plain dicts.""" + if params is None: + return None + if isinstance(params, dict): + return params + if hasattr(params, "to_dict"): + return params.to_dict() + if hasattr(params, "model_dump"): + return params.model_dump(exclude_none=True) + + try: + # Check if it's already a standard JSON type. + json.dumps(params) + return params + + except (TypeError, ValueError): + return { + "type": "object", + "properties": { + "serialization_error": { + "type": "string", + "description": f"Failed to serialize parameters: {type(params).__name__}", + } + }, + } + + +def _tool_to_tool_definition(tool: Tool) -> list[ToolDefinition]: + definitions = [] + if tool.function_declarations: + for fd in tool.function_declarations: + definitions.append( + FunctionToolDefinition( + name=getattr(fd, "name", type(fd).__name__), + description=getattr(fd, "description", None), + parameters=_clean_parameters( + getattr(fd, "parameters", None) + ), + ) + ) + + # Generic types if hasattr(tool, "model_dump"): - return tool.model_dump(exclude_none=True) + exclude_fields = {"function_declarations"} + fields = { + k: v + for k, v in tool.model_dump().items() + if v is not None and k not in exclude_fields + } - return str(tool) + for tool_type, _ in fields.items(): + definitions.append( + GenericToolDefinition( + type=tool_type, + name=tool_type, + ) + ) + + return definitions -def _callable_tool_to_tool_definition(tool: Any) -> MessagePart: +def _callable_tool_to_tool_definition(tool: Any) -> ToolDefinition: doc = getattr(tool, "__doc__", "") or "" - return { - "name": getattr(tool, "__name__", type(tool).__name__), - "description": doc.strip(), - } + return FunctionToolDefinition( + name=getattr(tool, "__name__", type(tool).__name__), + description=doc.strip(), + parameters=None, + ) -def _mcp_tool_to_tool_definition(tool: McpTool) -> MessagePart: +def _mcp_tool_to_tool_definition(tool: McpTool) -> ToolDefinition: if hasattr(tool, "model_dump"): - return tool.model_dump(exclude_none=True) + return _model_dump_to_tool_definition(tool) - return { - "name": getattr(tool, "name", type(tool).__name__), - "description": getattr(tool, "description", "") or "", - "input_schema": getattr(tool, "input_schema", {}), - } + return FunctionToolDefinition( + name=getattr(tool, "name", type(tool).__name__), + description=getattr(tool, "description", None), + parameters=getattr(tool, "input_schema", None), + ) -def _to_tool_definition_common(tool: ToolUnionDict) -> MessagePart: - if isinstance(tool, dict): - return tool - +def _to_tool_definition_common(tool: ToolUnionDict) -> list[ToolDefinition]: if isinstance(tool, Tool): return _tool_to_tool_definition(tool) if callable(tool): - return _callable_tool_to_tool_definition(tool) + return [_callable_tool_to_tool_definition(tool)] if _is_mcp_imported and isinstance(tool, McpTool): - return _mcp_tool_to_tool_definition(tool) + return [_mcp_tool_to_tool_definition(tool)] - try: - return {"raw_definition": json.loads(json.dumps(tool))} - except Exception: # pylint: disable=broad-exception-caught - return { - "error": f"failed to serialize tool definition, tool type={type(tool).__name__}" - } + return [ + GenericToolDefinition( + name="UnserializableTool", + type=type(tool).__name__, + ) + ] -def _to_tool_definition(tool: ToolUnionDict) -> MessagePart: +def _to_tool_definition(tool: ToolUnionDict) -> list[ToolDefinition]: if _is_mcp_imported and isinstance(tool, McpClientSession): - return None + return [] return _to_tool_definition_common(tool) -async def _to_tool_definition_async(tool: ToolUnionDict) -> MessagePart: +async def _to_tool_definition_async( + tool: ToolUnionDict, +) -> list[ToolDefinition]: if _is_mcp_imported and isinstance(tool, McpClientSession): result = await tool.list_tools() - return [t.model_dump(exclude_none=True) for t in result.tools] + return [_model_dump_to_tool_definition(t) for t in result.tools] return _to_tool_definition_common(tool) +def _tool_def_without_parameters_attr( + tool_def: list[ToolDefinition], +) -> dict[str, AttributeValue]: + if tool_def == []: + return {} + + return { + GEN_AI_TOOL_DEFINITIONS: [ + dataclasses.asdict( + FunctionToolDefinition( + name=td.name, description=td.description, parameters=None + ) + if isinstance(td, FunctionToolDefinition) + else td + ) + for td in tool_def + ] + } + + def _create_request_attributes( config: Optional[GenerateContentConfigOrDict], allow_list: AllowList, @@ -385,7 +473,7 @@ def _create_completion_details_attributes( input_messages: list[InputMessage], output_messages: list[OutputMessage], system_instructions: list[MessagePart], - tool_definitions: list[MessagePart], + tool_definitions: list[ToolDefinition], as_str: bool = False, ) -> dict[str, AttributeValue]: attributes: dict[str, AttributeValue] = { @@ -404,7 +492,9 @@ def _create_completion_details_attributes( ] if tool_definitions: - attributes[GEN_AI_TOOL_DEFINITIONS] = tool_definitions + attributes[GEN_AI_TOOL_DEFINITIONS] = [ + dataclasses.asdict(tool_def) for tool_def in tool_definitions + ] return attributes @@ -563,44 +653,87 @@ class _GenerateContentInstrumentationHelper: block_reason = response.prompt_feedback.block_reason.name.upper() self._error_type = f"BLOCKED_{block_reason}" - def _maybe_get_tool_definitions(self, config): + def _maybe_get_tool_definitions(self, config) -> list[ToolDefinition]: if ( self.sem_conv_opt_in_mode != _StabilityMode.GEN_AI_LATEST_EXPERIMENTAL ): - return None + return [] - tool_definitions = [] if tools := _config_to_tools(config): - for tool in tools: - definition = _to_tool_definition(tool) - if definition is None: - continue - if isinstance(definition, list): - tool_definitions.extend(definition) - else: - tool_definitions.append(definition) - return tool_definitions + return [ + de for tool in tools for de in _to_tool_definition(tool) if de + ] + return [] - async def _maybe_get_tool_definitions_async(self, config): + async def _maybe_get_tool_definitions_async( + self, config + ) -> list[ToolDefinition]: if ( self.sem_conv_opt_in_mode != _StabilityMode.GEN_AI_LATEST_EXPERIMENTAL ): - return None + return [] tool_definitions = [] if tools := _config_to_tools(config): for tool in tools: - definition = await _to_tool_definition_async(tool) - if definition is None: - continue - if isinstance(definition, list): - tool_definitions.extend(definition) - else: - tool_definitions.append(definition) + definitions = await _to_tool_definition_async(tool) + for de in definitions: + if de: + tool_definitions.append(de) + return tool_definitions + def _maybe_log_completion_details_in_log( + self, + event: LogRecord, + completion_details_attributes: dict[str, AttributeValue], + tool_definitions: Optional[list[ToolDefinition]] = None, + ): + if self._content_recording_enabled in [ + ContentCapturingMode.EVENT_ONLY, + ContentCapturingMode.SPAN_AND_EVENT, + ]: + event.attributes = { + **(event.attributes or {}), + **completion_details_attributes, + } + else: + event.attributes = { + **(event.attributes or {}), + **_tool_def_without_parameters_attr(tool_definitions), + } + + self._otel_wrapper.log_completion_details(event=event) + + def _maybe_log_completion_details_in_span( + self, + span: Span, + completion_details_attributes: dict[str, AttributeValue], + tool_definitions: Optional[list[ToolDefinition]] = None, + ): + if self._content_recording_enabled in [ + ContentCapturingMode.SPAN_ONLY, + ContentCapturingMode.SPAN_AND_EVENT, + ]: + span.set_attributes( + { + k: gen_ai_json_dumps(v) + for k, v in completion_details_attributes.items() + } + ) + # request attributes were already set on the span.. + else: + span.set_attributes( + { + k: gen_ai_json_dumps(v) + for k, v in _tool_def_without_parameters_attr( + tool_definitions + ).items() + } + ) + def _maybe_log_completion_details( self, extra_attributes: dict[str, AttributeValue], @@ -609,7 +742,7 @@ class _GenerateContentInstrumentationHelper: request: Union[ContentListUnion, ContentListUnionDict], candidates: list[Candidate], config: Optional[GenerateContentConfigOrDict] = None, - tool_definitions: list[MessagePart] = None, + tool_definitions: Optional[list[ToolDefinition]] = None, ): if ( self.sem_conv_opt_in_mode @@ -633,10 +766,12 @@ class _GenerateContentInstrumentationHelper: | request_attributes | final_attributes, ) + tool_definitions = tool_definitions or [] self.completion_hook.on_completion( inputs=input_messages, outputs=output_messages, system_instruction=system_instructions, + tool_definitions=tool_definitions, span=span, log_record=event, ) @@ -646,27 +781,16 @@ class _GenerateContentInstrumentationHelper: system_instructions, tool_definitions, ) - if self._content_recording_enabled in [ - ContentCapturingMode.EVENT_ONLY, - ContentCapturingMode.SPAN_AND_EVENT, - ]: - event.attributes = { - **(event.attributes or {}), - **completion_details_attributes, - } - self._otel_wrapper.log_completion_details(event=event) - - if self._content_recording_enabled in [ - ContentCapturingMode.SPAN_ONLY, - ContentCapturingMode.SPAN_AND_EVENT, - ]: - span.set_attributes( - { - k: gen_ai_json_dumps(v) - for k, v in completion_details_attributes.items() - } - ) - # request attributes were already set on the span.. + self._maybe_log_completion_details_in_log( + event=event, + completion_details_attributes=completion_details_attributes, + tool_definitions=tool_definitions, + ) + self._maybe_log_completion_details_in_span( + span=span, + completion_details_attributes=completion_details_attributes, + tool_definitions=tool_definitions, + ) def _maybe_log_system_instruction( self, config: Optional[GenerateContentConfigOrDict] = None diff --git a/instrumentation-genai/opentelemetry-instrumentation-google-genai/tests/generate_content/nonstreaming_base.py b/instrumentation-genai/opentelemetry-instrumentation-google-genai/tests/generate_content/nonstreaming_base.py index ca683c911..4763b8d61 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-google-genai/tests/generate_content/nonstreaming_base.py +++ b/instrumentation-genai/opentelemetry-instrumentation-google-genai/tests/generate_content/nonstreaming_base.py @@ -21,6 +21,7 @@ import pytest from google.genai.types import ( FunctionDeclarationDict, GenerateContentConfig, + GoogleMaps, Part, ToolDict, ) @@ -35,9 +36,7 @@ from opentelemetry.instrumentation._semconv import ( from opentelemetry.instrumentation.google_genai import ( GENERATE_CONTENT_EXTRA_ATTRIBUTES_CONTEXT_KEY, ) -from opentelemetry.semconv._incubating.attributes import ( - gen_ai_attributes, -) +from opentelemetry.semconv._incubating.attributes import gen_ai_attributes from opentelemetry.util.genai.types import ContentCapturingMode from .base import TestCase @@ -80,7 +79,7 @@ def _mock_mcp_client_session() -> McpClientSession: description="Tool from session", inputSchema={ "type": "object", - "properties": {"query": {"type": "string"}}, + "properties": {"id": {"type": "integer"}}, }, ) mock_result = create_autospec(McpListToolsResult, instance=True) @@ -106,9 +105,11 @@ def _mock_tool_dict() -> ToolDict: return ToolDict( function_declarations=[ FunctionDeclarationDict( - name="mock_tool", description="Description of mock tool." + name="mock_tool", + description="Description of mock tool.", ), - ] + ], + google_maps=GoogleMaps(), ) @@ -470,6 +471,81 @@ class NonStreamingTestCase(TestCase): ContentCapturingMode.NO_CONTENT, ContentCapturingMode.SPAN_ONLY, ]: + expected_event_attributes = { + "TOOL_DEFINITIONS_NO_CONTENT": ( + { + "name": "_mock_callable_tool", + "description": "Description of some tool.", + "parameters": None, + "type": "function", + }, + { + "name": "mock_tool", + "description": "Description of mock tool.", + "parameters": None, + "type": "function", + }, + { + "name": "google_maps", + "type": "google_maps", + }, + { + "name": "mcp_tool", + "description": "A standalone mcp tool", + "parameters": None, + "type": "function", + }, + ), + "TOOL_DEFINITIONS_ASYNC_NO_CONTENT": ( + { + "name": "_mock_callable_tool", + "description": "Description of some tool.", + "parameters": None, + "type": "function", + }, + { + "name": "mock_tool", + "description": "Description of mock tool.", + "parameters": None, + "type": "function", + }, + { + "name": "google_maps", + "type": "google_maps", + }, + { + "name": "mcp_tool", + "description": "Tool from session", + "parameters": None, + "type": "function", + }, + { + "name": "mcp_tool", + "description": "A standalone mcp tool", + "parameters": None, + "type": "function", + }, + ), + "TOOL_DEFINITIONS_NO_MCP_NO_CONTENT": ( + { + "name": "_mock_callable_tool", + "description": "Description of some tool.", + "parameters": None, + "type": "function", + }, + { + "name": "mock_tool", + "description": "Description of mock tool.", + "parameters": None, + "type": "function", + }, + { + "name": "google_maps", + "type": "google_maps", + }, + ), + } + self.assertNotIn( gen_ai_attributes.GEN_AI_INPUT_MESSAGES, event.attributes, @@ -482,10 +558,27 @@ class NonStreamingTestCase(TestCase): gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS, event.attributes, ) - self.assertNotIn( - GEN_AI_TOOL_DEFINITIONS, - event.attributes, - ) + if _is_mcp_imported: + self.assertIn( + event.attributes[GEN_AI_TOOL_DEFINITIONS], + [ + expected_event_attributes[ + "TOOL_DEFINITIONS_NO_CONTENT" + ], + expected_event_attributes[ + "TOOL_DEFINITIONS_ASYNC_NO_CONTENT" + ], + ], + ) + else: + self.assertIn( + event.attributes[GEN_AI_TOOL_DEFINITIONS], + [ + expected_event_attributes[ + "TOOL_DEFINITIONS_NO_MCP_NO_CONTENT" + ], + ], + ) else: expected_event_attributes = { @@ -513,72 +606,87 @@ class NonStreamingTestCase(TestCase): { "name": "_mock_callable_tool", "description": "Description of some tool.", + "parameters": None, + "type": "function", }, { - "function_declarations": ( - { - "name": "mock_tool", - "description": "Description of mock tool.", - }, - ) + "name": "mock_tool", + "description": "Description of mock tool.", + "parameters": None, + "type": "function", + }, + { + "name": "google_maps", + "type": "google_maps", }, { "name": "mcp_tool", "description": "A standalone mcp tool", - "inputSchema": { + "parameters": { "type": "object", "properties": { "id": {"type": "integer"} }, }, + "type": "function", }, ), "TOOL_DEFINITIONS_ASYNC": ( { "name": "_mock_callable_tool", "description": "Description of some tool.", + "parameters": None, + "type": "function", }, { - "function_declarations": ( - { - "name": "mock_tool", - "description": "Description of mock tool.", - }, - ) + "name": "mock_tool", + "description": "Description of mock tool.", + "parameters": None, + "type": "function", + }, + { + "name": "google_maps", + "type": "google_maps", }, { "name": "mcp_tool", "description": "Tool from session", - "inputSchema": { - "type": "object", - "properties": { - "query": {"type": "string"} - }, - }, - }, - { - "name": "mcp_tool", - "description": "A standalone mcp tool", - "inputSchema": { + "parameters": { "type": "object", "properties": { "id": {"type": "integer"} }, }, + "type": "function", + }, + { + "name": "mcp_tool", + "description": "A standalone mcp tool", + "parameters": { + "type": "object", + "properties": { + "id": {"type": "integer"} + }, + }, + "type": "function", }, ), "TOOL_DEFINITIONS_NO_MCP": ( { "name": "_mock_callable_tool", "description": "Description of some tool.", + "parameters": None, + "type": "function", }, { - "function_declarations": ( - { - "name": "mock_tool", - "description": "Description of mock tool.", - }, - ) + "name": "mock_tool", + "description": "Description of mock tool.", + "parameters": None, + "type": "function", + }, + { + "name": "google_maps", + "type": "google_maps", }, ), } @@ -696,14 +804,14 @@ class NonStreamingTestCase(TestCase): self.assertIn( span.attributes[GEN_AI_TOOL_DEFINITIONS], [ - '[{"name":"_mock_callable_tool","description":"Description of some tool."},{"function_declarations":[{"description":"Description of mock tool.","name":"mock_tool"}]},{"name":"mcp_tool","description":"Tool from session","inputSchema":{"type":"object","properties":{"query":{"type":"string"}}}},{"name":"mcp_tool","description":"A standalone mcp tool","inputSchema":{"type":"object","properties":{"id":{"type":"integer"}}}}]', - '[{"name":"_mock_callable_tool","description":"Description of some tool."},{"function_declarations":[{"description":"Description of mock tool.","name":"mock_tool"}]},{"name":"mcp_tool","description":"A standalone mcp tool","inputSchema":{"type":"object","properties":{"id":{"type":"integer"}}}}]', + '[{"name":"_mock_callable_tool","description":"Description of some tool.","parameters":null,"type":"function"},{"name":"mock_tool","description":"Description of mock tool.","parameters":null,"type":"function"},{"name":"google_maps","type":"google_maps"},{"name":"mcp_tool","description":"Tool from session","parameters":{"type":"object","properties":{"id":{"type":"integer"}}},"type":"function"},{"name":"mcp_tool","description":"A standalone mcp tool","parameters":{"type":"object","properties":{"id":{"type":"integer"}}},"type":"function"}]', + '[{"name":"_mock_callable_tool","description":"Description of some tool.","parameters":null,"type":"function"},{"name":"mock_tool","description":"Description of mock tool.","parameters":null,"type":"function"},{"name":"google_maps","type":"google_maps"},{"name":"mcp_tool","description":"A standalone mcp tool","parameters":{"type":"object","properties":{"id":{"type":"integer"}}},"type":"function"}]', ], ) else: self.assertEqual( span.attributes[GEN_AI_TOOL_DEFINITIONS], - '[{"name":"_mock_callable_tool","description":"Description of some tool."},{"function_declarations":[{"description":"Description of mock tool.","name":"mock_tool"}]}]', + '[{"name":"_mock_callable_tool","description":"Description of some tool.","parameters":null,"type":"function"},{"name":"mock_tool","description":"Description of mock tool.","parameters":null,"type":"function"},{"name":"google_maps","type":"google_maps"}]', ) else: self.assertNotIn( @@ -718,10 +826,19 @@ class NonStreamingTestCase(TestCase): gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS, span.attributes, ) - self.assertNotIn( - GEN_AI_TOOL_DEFINITIONS, - span.attributes, - ) + if _is_mcp_imported: + self.assertIn( + span.attributes[GEN_AI_TOOL_DEFINITIONS], + [ + '[{"name":"_mock_callable_tool","description":"Description of some tool.","parameters":null,"type":"function"},{"name":"mock_tool","description":"Description of mock tool.","parameters":null,"type":"function"},{"name":"google_maps","type":"google_maps"},{"name":"mcp_tool","description":"Tool from session","parameters":null,"type":"function"},{"name":"mcp_tool","description":"A standalone mcp tool","parameters":null,"type":"function"}]', + '[{"name":"_mock_callable_tool","description":"Description of some tool.","parameters":null,"type":"function"},{"name":"mock_tool","description":"Description of mock tool.","parameters":null,"type":"function"},{"name":"google_maps","type":"google_maps"},{"name":"mcp_tool","description":"A standalone mcp tool","parameters":null,"type":"function"}]', + ], + ) + else: + self.assertEqual( + span.attributes[GEN_AI_TOOL_DEFINITIONS], + '[{"name":"_mock_callable_tool","description":"Description of some tool.","parameters":null,"type":"function"},{"name":"mock_tool","description":"Description of mock tool.","parameters":null,"type":"function"},{"name":"google_maps","type":"google_maps"}]', + ) self.tearDown() diff --git a/instrumentation-genai/opentelemetry-instrumentation-google-genai/tests/requirements.oldest.txt b/instrumentation-genai/opentelemetry-instrumentation-google-genai/tests/requirements.oldest.txt index d7c8b94b2..a4518e67d 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-google-genai/tests/requirements.oldest.txt +++ b/instrumentation-genai/opentelemetry-instrumentation-google-genai/tests/requirements.oldest.txt @@ -25,10 +25,13 @@ opentelemetry-api==1.37.0 opentelemetry-sdk==1.37.0 opentelemetry-semantic-conventions==0.58b0 opentelemetry-instrumentation==0.58b0 -opentelemetry-util-genai[upload]==0.2b0 +# TODO: uncomment and update version after release (https://github.com/open-telemetry/opentelemetry-python-contrib/issues/4221) +# opentelemetry-util-genai[upload]==0.2b0 fsspec==2025.9.0 # Install locally from the folder. This path is relative to the # root directory, given invocation from "tox" at root level. -e instrumentation-genai/opentelemetry-instrumentation-google-genai +# TODO: remove local install after release (https://github.com/open-telemetry/opentelemetry-python-contrib/issues/4221) +-e util/opentelemetry-util-genai[upload] diff --git a/util/opentelemetry-util-genai/CHANGELOG.md b/util/opentelemetry-util-genai/CHANGELOG.md index 16339e79a..0e7c521bf 100644 --- a/util/opentelemetry-util-genai/CHANGELOG.md +++ b/util/opentelemetry-util-genai/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +- Add `gen_ai.tool_definitions` to completion hook ([#4181](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4181)) - Add support for emitting inference events and enrich message types. ([#3994](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3994)) - Add support for `server.address`, `server.port` on all signals and additional metric-only attributes ([#4069](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4069)) diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_upload/completion_hook.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_upload/completion_hook.py index 227e89dfa..4f2286139 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_upload/completion_hook.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/_upload/completion_hook.py @@ -15,7 +15,9 @@ from __future__ import annotations +import dataclasses import hashlib +import json import logging import posixpath import threading @@ -50,6 +52,11 @@ GEN_AI_SYSTEM_INSTRUCTIONS_REF: Final = ( gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS + "_ref" ) +GEN_AI_TOOL_DEFINITIONS = getattr( + gen_ai_attributes, "GEN_AI_TOOL_DEFINITIONS", "gen_ai.tool.definitions" +) +GEN_AI_TOOL_DEFINITIONS_REF: Final = GEN_AI_TOOL_DEFINITIONS + "_ref" + _MESSAGE_INDEX_KEY = "index" _DEFAULT_MAX_QUEUE_SIZE = 20 _DEFAULT_FORMAT = "json" @@ -66,6 +73,7 @@ class Completion: inputs: list[types.InputMessage] | None outputs: list[types.OutputMessage] | None system_instruction: list[types.MessagePart] | None + tool_definitions: list[types.ToolDefinition] | None @dataclass @@ -73,6 +81,7 @@ class CompletionRefs: inputs_ref: str outputs_ref: str system_instruction_ref: str + tool_definitions_ref: str JsonEncodeable = list[dict[str, Any]] @@ -81,14 +90,39 @@ JsonEncodeable = list[dict[str, Any]] UploadData = dict[tuple[str, bool], Callable[[], JsonEncodeable]] -def is_system_instructions_hashable( - system_instruction: list[types.MessagePart] | None, +def is_message_part_list_hashable( + message_parts: list[types.MessagePart] | None, ) -> bool: - return bool(system_instruction) and all( - isinstance(x, types.Text) for x in system_instruction + return bool(message_parts) and all( + isinstance(x, types.Text) for x in message_parts ) +def hash_tool_definitions( + tool_definitions: list[types.ToolDefinition] | None, +) -> str | None: + if not tool_definitions: + return None + try: + tool_dicts = [ + {k: v for k, v in dataclasses.asdict(t).items() if v is not None} + for t in tool_definitions + ] + + encoded_tools = json.dumps( + tool_dicts, + sort_keys=True, + ).encode("utf-8") + + return hashlib.sha256( + encoded_tools, + usedforsecurity=False, + ).hexdigest() + + except (TypeError, AttributeError): + return None + + class UploadCompletionHook(CompletionHook): """An completion hook using ``fsspec`` to upload to external storage @@ -169,12 +203,14 @@ class UploadCompletionHook(CompletionHook): self._semaphore.release() def _calculate_ref_path( - self, system_instruction: list[types.MessagePart] + self, + system_instruction: list[types.MessagePart], + tool_definitions: list[types.ToolDefinition] | None = None, ) -> CompletionRefs: # TODO: experimental with using the trace_id and span_id, or fetching # gen_ai.response.id from the active span. system_instruction_hash = None - if is_system_instructions_hashable(system_instruction): + if is_message_part_list_hashable(system_instruction): # Get a hash of the text. system_instruction_hash = hashlib.sha256( "\n".join(x.content for x in system_instruction).encode( # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue, reportUnknownArgumentType, reportCallIssue, reportArgumentType] @@ -182,6 +218,9 @@ class UploadCompletionHook(CompletionHook): ), usedforsecurity=False, ).hexdigest() + + tool_definitions_hash = hash_tool_definitions(tool_definitions) + uuid_str = str(uuid4()) return CompletionRefs( inputs_ref=posixpath.join( @@ -194,6 +233,10 @@ class UploadCompletionHook(CompletionHook): self._base_path, f"{system_instruction_hash or uuid_str}_system_instruction.{self._format}", ), + tool_definitions_ref=posixpath.join( + self._base_path, + f"{tool_definitions_hash or uuid_str}_tool.definitions.{self._format}", + ), ) def _file_exists(self, path: str) -> bool: @@ -250,25 +293,30 @@ class UploadCompletionHook(CompletionHook): inputs: list[types.InputMessage], outputs: list[types.OutputMessage], system_instruction: list[types.MessagePart], + tool_definitions: list[types.ToolDefinition] | None = None, span: Span | None = None, log_record: LogRecord | None = None, **kwargs: Any, ) -> None: - if not any([inputs, outputs, system_instruction]): + if not any([inputs, outputs, system_instruction, tool_definitions]): return # An empty list will not be uploaded. completion = Completion( inputs=inputs or None, outputs=outputs or None, system_instruction=system_instruction or None, + tool_definitions=tool_definitions or None, ) # generate the paths to upload to - ref_names = self._calculate_ref_path(system_instruction) + ref_names = self._calculate_ref_path( + system_instruction, tool_definitions + ) def to_dict( dataclass_list: list[types.InputMessage] | list[types.OutputMessage] - | list[types.MessagePart], + | list[types.MessagePart] + | list[types.ToolDefinition], ) -> JsonEncodeable: return [asdict(dc) for dc in dataclass_list] @@ -291,12 +339,18 @@ class UploadCompletionHook(CompletionHook): ref_names.system_instruction_ref, completion.system_instruction, GEN_AI_SYSTEM_INSTRUCTIONS_REF, - is_system_instructions_hashable( + is_message_part_list_hashable( completion.system_instruction ), ), + ( + ref_names.tool_definitions_ref, + completion.tool_definitions, + GEN_AI_TOOL_DEFINITIONS_REF, + bool(completion.tool_definitions), + ), ] - if ref # Filter out empty input/output/sys instruction + if ref # Filter out empty input/output/sys instruction/tool defs ] self._submit_all( { diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/completion_hook.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/completion_hook.py index 76d199ce8..b66b1217a 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/completion_hook.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/completion_hook.py @@ -61,6 +61,8 @@ class CompletionHook(Protocol): outputs: The outputs of the GenAI interaction. system_instruction: The system instruction of the GenAI interaction. + tool_definitions: The list of source system tool definitions + available to the GenAI agent or model. span: The span associated with the GenAI interaction. log_record: The event log associated with the GenAI interaction. @@ -72,6 +74,7 @@ class CompletionHook(Protocol): inputs: list[types.InputMessage], outputs: list[types.OutputMessage], system_instruction: list[types.MessagePart], + tool_definitions: list[types.ToolDefinition] | None = None, span: Span | None = None, log_record: LogRecord | None = None, ) -> None: ... diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/types.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/types.py index 0e86885f2..045a65b37 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/types.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/types.py @@ -137,6 +137,26 @@ class Uri: type: Literal["uri"] = "uri" +@dataclass() +class FunctionToolDefinition: + """Represents a function tool definition sent to the model""" + + name: str + description: str | None + parameters: Any + type: Literal["function"] = "function" + + +@dataclass() +class GenericToolDefinition: + """Represents a generic tool definition sent to the model""" + + name: str + type: str + + +ToolDefinition = Union[FunctionToolDefinition, GenericToolDefinition] + MessagePart = Union[ Text, ToolCall, ToolCallResponse, Blob, File, Uri, Reasoning, Any ] diff --git a/util/opentelemetry-util-genai/tests/test_upload.py b/util/opentelemetry-util-genai/tests/test_upload.py index 4b626fb4f..dd87b971e 100644 --- a/util/opentelemetry-util-genai/tests/test_upload.py +++ b/util/opentelemetry-util-genai/tests/test_upload.py @@ -69,6 +69,15 @@ FAKE_OUTPUTS = [ ] FAKE_SYSTEM_INSTRUCTION = [types.Text(content="You are a helpful assistant.")] +FAKE_TOOL_DEFINITIONS: list[types.ToolDefinition] = [ + types.FunctionToolDefinition( + name="test_tool", + description="does something", + parameters=None, + type="function", + ), +] + class ThreadSafeMagicMock(MagicMock): def __init__(self, *args, **kwargs) -> None: @@ -124,6 +133,7 @@ class TestUploadCompletionHook(TestCase): inputs=FAKE_INPUTS, outputs=FAKE_OUTPUTS, system_instruction=FAKE_SYSTEM_INSTRUCTION, + tool_definitions=FAKE_TOOL_DEFINITIONS, ) # all items should be consumed self.hook.shutdown() @@ -131,8 +141,8 @@ class TestUploadCompletionHook(TestCase): time.sleep(0.5) self.assertEqual( self.mock_fs.open.call_count, - 3, - "should have uploaded 3 files", + 4, + "should have uploaded 4 files", ) def test_lru_cache_works(self): @@ -141,6 +151,7 @@ class TestUploadCompletionHook(TestCase): inputs=[], outputs=[], system_instruction=FAKE_SYSTEM_INSTRUCTION, + tool_definitions=[], log_record=record, ) # Wait a bit for file upload to finish.. @@ -161,6 +172,7 @@ class TestUploadCompletionHook(TestCase): inputs=[], outputs=[], system_instruction=[types.Text(content=str(iteration))], + tool_definitions=[], ) self.hook.shutdown() self.assertFalse( @@ -174,6 +186,7 @@ class TestUploadCompletionHook(TestCase): inputs=[], outputs=[], system_instruction=FAKE_SYSTEM_INSTRUCTION, + tool_definitions=[], log_record=record, ) # all items should be consumed @@ -203,6 +216,7 @@ class TestUploadCompletionHook(TestCase): inputs=FAKE_INPUTS, outputs=FAKE_OUTPUTS, system_instruction=FAKE_SYSTEM_INSTRUCTION, + tool_definitions=FAKE_TOOL_DEFINITIONS, ) self.assertLessEqual( @@ -216,6 +230,7 @@ class TestUploadCompletionHook(TestCase): inputs=FAKE_INPUTS, outputs=FAKE_OUTPUTS, system_instruction=FAKE_SYSTEM_INSTRUCTION, + tool_definitions=FAKE_TOOL_DEFINITIONS, ) self.assertIn( @@ -228,6 +243,7 @@ class TestUploadCompletionHook(TestCase): inputs=FAKE_INPUTS, outputs=FAKE_OUTPUTS, system_instruction=FAKE_SYSTEM_INSTRUCTION, + tool_definitions=FAKE_TOOL_DEFINITIONS, ) # shutdown should timeout and return even though there are still items in the queue @@ -241,6 +257,7 @@ class TestUploadCompletionHook(TestCase): inputs=FAKE_INPUTS, outputs=FAKE_OUTPUTS, system_instruction=FAKE_SYSTEM_INSTRUCTION, + tool_definitions=FAKE_TOOL_DEFINITIONS, ) self.hook.shutdown() @@ -264,6 +281,7 @@ class TestUploadCompletionHook(TestCase): inputs=FAKE_INPUTS, outputs=FAKE_OUTPUTS, system_instruction=FAKE_SYSTEM_INSTRUCTION, + tool_definitions=FAKE_TOOL_DEFINITIONS, ) hook.shutdown() @@ -278,8 +296,9 @@ class TestUploadCompletionHook(TestCase): inputs=FAKE_INPUTS, outputs=FAKE_OUTPUTS, system_instruction=FAKE_SYSTEM_INSTRUCTION, + tool_definitions=FAKE_TOOL_DEFINITIONS, ) - self.assertEqual(len(logs.output), 3) + self.assertEqual(len(logs.output), 4) self.assertIn( "attempting to upload file after UploadCompletionHook.shutdown() was already called", logs.output[0], @@ -337,6 +356,7 @@ class TestUploadCompletionHookIntegration(TestBase): inputs=[], outputs=[], system_instruction=system_instructions, + tool_definitions=[], log_record=record, ) self.hook.shutdown() @@ -349,6 +369,42 @@ class TestUploadCompletionHookIntegration(TestBase): # Content should not have been overwritten. self.assert_fsspec_equal(expected_file_name, "asg") + def test_tool_definitions_is_hashed_to_avoid_reupload(self): + expected_hash = ( + "1f559d0102f8c440a667fd5ed587beeed488ec9f3ce0828d39c424bed6546cf5" + ) + # Create the file before upload.. + expected_file_name = f"memory://{expected_hash}_tool.definitions.json" + with fsspec.open(expected_file_name, "wb") as file: + file.write(b"asg") + # FIle should exist. + self.assertTrue(self.hook._file_exists(expected_file_name)) + tool_definitions = [ + types.FunctionToolDefinition( + name="some_tool", + description="does something", + parameters=None, + type="function", + ), + ] + record = LogRecord() + self.hook.on_completion( + inputs=[], + outputs=[], + system_instruction=[], + tool_definitions=tool_definitions, + log_record=record, + ) + self.hook.shutdown() + self.assertIsNotNone(record.attributes) + + self.assertEqual( + record.attributes["gen_ai.tool.definitions_ref"], + expected_file_name, + ) + # Content should not have been overwritten. + self.assert_fsspec_equal(expected_file_name, "asg") + def test_upload_completions(self): tracer = self.tracer_provider.get_tracer(__name__) log_record = LogRecord() @@ -358,6 +414,7 @@ class TestUploadCompletionHookIntegration(TestBase): inputs=FAKE_INPUTS, outputs=FAKE_OUTPUTS, system_instruction=FAKE_SYSTEM_INSTRUCTION, + tool_definitions=FAKE_TOOL_DEFINITIONS, span=span, log_record=log_record, ) @@ -376,6 +433,7 @@ class TestUploadCompletionHookIntegration(TestBase): "gen_ai.input.messages_ref", "gen_ai.output.messages_ref", "gen_ai.system_instructions_ref", + "gen_ai.tool.definitions_ref", ]: self.assertIn(ref_key, attributes) @@ -391,6 +449,10 @@ class TestUploadCompletionHookIntegration(TestBase): span.attributes["gen_ai.system_instructions_ref"], '[{"content":"You are a helpful assistant.","type":"text"}]\n', ) + self.assert_fsspec_equal( + span.attributes["gen_ai.tool.definitions_ref"], + '[{"name":"test_tool","description":"does something","parameters":null,"type":"function"}]\n', + ) def test_stamps_empty_log(self): log_record = LogRecord() @@ -398,6 +460,7 @@ class TestUploadCompletionHookIntegration(TestBase): inputs=FAKE_INPUTS, outputs=FAKE_OUTPUTS, system_instruction=FAKE_SYSTEM_INSTRUCTION, + tool_definitions=FAKE_TOOL_DEFINITIONS, log_record=log_record, ) @@ -405,6 +468,7 @@ class TestUploadCompletionHookIntegration(TestBase): self.assertIn("gen_ai.input.messages_ref", log_record.attributes) self.assertIn("gen_ai.output.messages_ref", log_record.attributes) self.assertIn("gen_ai.system_instructions_ref", log_record.attributes) + self.assertIn("gen_ai.tool.definitions_ref", log_record.attributes) def test_upload_bytes(self) -> None: log_record = LogRecord() @@ -420,6 +484,7 @@ class TestUploadCompletionHookIntegration(TestBase): ], outputs=FAKE_OUTPUTS, system_instruction=FAKE_SYSTEM_INSTRUCTION, + tool_definitions=FAKE_TOOL_DEFINITIONS, log_record=log_record, ) self.hook.shutdown() @@ -438,6 +503,7 @@ class TestUploadCompletionHookIntegration(TestBase): inputs=FAKE_INPUTS, outputs=FAKE_OUTPUTS, system_instruction=FAKE_SYSTEM_INSTRUCTION, + tool_definitions=FAKE_TOOL_DEFINITIONS, log_record=log_record, ) hook.shutdown() @@ -461,6 +527,7 @@ class TestUploadCompletionHookIntegration(TestBase): inputs=FAKE_INPUTS, outputs=FAKE_OUTPUTS, system_instruction=FAKE_SYSTEM_INSTRUCTION, + tool_definitions=FAKE_TOOL_DEFINITIONS, log_record=log_record, ) hook.shutdown() @@ -492,6 +559,7 @@ class TestUploadCompletionHookIntegration(TestBase): inputs=FAKE_INPUTS, outputs=FAKE_OUTPUTS, system_instruction=FAKE_SYSTEM_INSTRUCTION, + tool_definitions=FAKE_TOOL_DEFINITIONS, log_record=log_record, ) hook.shutdown()